#Author：Mini
#！/usr/bin/env python
import urllib.request
import urllib.error
import re
data=urllib.request.urlopen("http://news.sina.com.cn/").read()
data1=data.decode("utf-8","ignore")
pat=‘ href="(http://news.sina.com.cn/.*?)">‘
allurl=re.compile(pat).findall(data1)
for i in range(0,len(allurl)):
???try:
???????print(str(i)+"\n\ntime")
???????thisurl=allurl[i]
???????fh="E:/m/"+str(i)+".html"
???????urllib.request.urlretrieve(thisurl,fh)
???????print("success!")
???except urllib.error.URLError as e:
???????if hasattr(e, "code"):
???????????print(e.code)
???????????if hasattr(e, "reason"):
???????????????print(e.reason)
************************************
import urllib.request
import re
import ?urllib.error
def use_proxy(url,proxy_addr):
???proxy=urllib.request.ProxyHandler({"http":proxy_addr})
???opener1=urllib.request.build_opener(proxy,urllib.request.HTTPHandler)
???urllib.request.install_opener(opener1)
proxy_addr="220.161.37.21:8118"
url="http://blog.csdn.net/"
headers=("User-Agent","Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/60.0.3112.90 Safari/537.36Query String Parametersview sourceview URL encoded")
opener=urllib.request.build_opener()
opener.addheaders=[headers]
urllib.request.install_opener(opener)
data=use_proxy(url,proxy_addr)
data=urllib.request.urlopen(url).read().decode("utf-8","ignore")
print(len(data))
pat=‘ <h3 ?class="csdn-tracking-statistics" data-mod="popu_430" data-poputype="feed" ?data-feed-show="false" ?data-dsm="post"><a href="(.*?)"‘
res=re.compile(pat).findall(data)
for i in range(0,len(res)):
try:
???fil="E:/m/"+str(i)+".html"
???urllib.request.urlretrieve(res,filename=fil)
???print(str(i),"\n\ntime")
except urllib.error.URLError as e:
???if hasattr(e, "code"):
???????print(e.code)
???????if hasattr(e, "reason"):
???????????print(e.reason)
web crawling(plus5) news crawling and proxy
原文地址：http://www.cnblogs.com/rabbittail/p/7623041.html