#Author：Mini
#！/usr/bin/env python
import re
import urllib.request
import time
import urllib.error
def use_proxy(proxy_addr,url):
???try:
???????req=urllib.request.Request(url)
???????req.add_header("User-Agent","Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:56.0) Gecko/20100101 Firefox/56.0")
???????proxy=urllib.request.ProxyHandler({‘http‘:proxy_addr})
???????opener=urllib.request.build_opener(proxy,urllib.request.HTTPHandler)
???????urllib.request.install_opener(opener)
???????data=urllib.request.urlopen(req).read()
???????return data
???except urllib.error.URLError as e:
???????if hasattr(e, "code"):
???????????print(e.code)
???????????if hasattr(e, "reason"):
???????????????print(e.reason)
???????????time.sleep(1)
key="Galaxy Macau"
key=key.replace(" ","+")
proxy="127.0.0.1:8888"
for i in range(1,100):
???key=urllib.request.quote(key)
???thispageurl="http://weixin.sogou.com/weixin?oq=&query="+key+"&_sug_type_=1&sut=0&lkt=0%2C0%2C0&s_from=input&ri=0&_sug_=n&type=2&sst0=1507173318682&page="+str(i)+"&ie=utf8&p=40040108&dp=1&w=01015002&dr=1"
???thispagedata=use_proxy(proxy,thispageurl)
???print(len(str(thispagedata)))
???pat1=‘<a target="_blank" href="(.*?)"‘
???res1=re.compile(pat1,re.S).findall(str(thispagedata))
???if(len(res1)==0):
???????print(str(i)+"\t\tfail!")
???????continue
???????for j in range(0,len(res1)):
???????????thisurl=res1[j]
???????????thisurl=thisurl.replace("amp;","")
???????????file="E:/m/"+str(i)+"\tweb\t\t"+str(j)+"\tessay.html"
???????????thisdata=use_proxy(proxy,thisurl)
???????????try:
???????????????fh=open(file,"wb")
???????????????fh.write(thisdata)
???????????????fh.close()
???????????????print(str(i)+"\tweb\t\t"+str(j)+"\tessay\tsuccess!")
???????????except Exception as e:
???????????????print(e)
???????????????print(str(i)+"\tweb\t\t"+str(j)+"\tessay\tfail!")
web crawling(plus5) crawling wechat
原文地址：http://www.cnblogs.com/rabbittail/p/7631854.html