?1 # -*- coding:utf-8 -*- ?2 import urlparse ?3 import urllib2 ?4 from bs4 import BeautifulSoup ?5 ??6 url = "http://www.baidu.com" ?7 ??8 urls = [url] # stack of urls to scrape ?9 visited = [url] # historic record of urls 10 ??1 # -*- coding:utf-8 -*- ?2 import urlparse ?3 import urllib2 ?4 from bs4 import BeautifulSoup ?5 ??6 url = "http://www.baidu.com" ?7 ??8 urls = [url] # stack of urls to scrape ?9 visited = [url] # historic record of urls 10 ?11 while len(urls) > 0: 12 ????try: 13 ????????htmltext = urllib2.urlopen(urls[0]).read() 14 ????except: 15 ????????print urls[0] 16 ????soup = BeautifulSoup(htmltext,"html") 17 ?18 ????urls.pop(0) 19 ?20 ????for tag in soup.findAll("a", href=True): 21 ????????tag["href"] = urlparse.urljoin(url, tag["href"]) 22 ????????if url in tag["href"] and tag["href"] not in visited: 23 ????????????urls.append(tag["href"]) 24 ????????????visited.append(tag["href"]) 25 ?26 ????print len(urls)BeautifulSoup ?抓取网站url
原文地址：http://www.cnblogs.com/cuzz/p/BeautifulSoup.html