爬虫基础框架 之urllib(一) --- urllib post请求

1 req = request.Request(url=url,headers={‘UserAgent‘:‘Mozilla/5.0 (Windows NT 10.0; Win64;x64)AppleWebKit/537.36 (KHTML, like Gecko)Chrome/71.0.3578.80Safari/537.36‘,‘cookie‘:‘_T_WM=e75d066bb30fae02106ed2a058e3ba08;SUB=_2A25xuVTDeRhGeBN7FIR9izJyTSIHXVTQmHbrDV6PUJbktANLVjZkW1NRFYvPzVjmAtKAY7Kppc7xOninRZqgesm;SUBP=0033WrSXqPxfM725Ws9jqgMF55529P9D9W59rVpGeZ7yg7I7HR0hyYPg5JpX5KzhUgL.Foq0S057Sozfeon2dJLoI05LxKML1heLBBLxKqL1heL1hLxKML12L1hBLxKqLBoeLBKzLxKqLBoeLBKz41K.t;SUHB=04WeHU67Q84JrJ‘})2 用Request对象可以给请求加上请求头，使得请求伪装成浏览器等终端

1 res = request.urlopen(req)2 print(res.status) 3 # 打印状态码

1 with open("wei.html","wb") as fp:2　　fp.write(res.read()) ???

1 from urllib import request,parse2 url = ‘https://fanyi.baidu.com/sug‘

1 headers = {‘User-Agent‘: ‘Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/71.0.3578.80 Safari/537.36‘}

1 data = {"kw":"a"}2 data = parse.urlencode(data).encode("utf-8")3 post提交的数据是二进制，需要用utf-8编码4 5 print(data)

req = request.Request(url=url,headers=headers,data=data)

1 res = request.urlopen(req)2 print(res.read())

1 from urllib import request,parse2 from http import cookiejar

1 cookie = cookiejar.CookieJar() ??　　　　　　　　　 ?# 初始化一个cookie对象2 handler = request.HTTPCookieProcessor(cookie) ????# 创建一个handler对象，携带上cookie3 opener = request.build_opener(handler) ???　　　　 # 创建一个opener对象携带上handler 

1 headers = {‘User-Agent‘: ‘Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/71.0.3578.80 Safari/537.36‘}

1 login_url = ‘http://www.jokeji.cn/user/c.asp?‘

1 dic = {‘u‘:‘bobo666‘,‘p‘:‘a12345678‘,‘sn‘:‘1‘,‘t‘:‘big‘}2 params = parse.urlencode(dic)3 login_url += params

1 login_req = request.Request(url=login_url,headers=headers)

1 res = opener.open(login_req) ???????# ?用opener来发起请求2 print(res.read().decode(‘utf-8‘)) ??# ?此时发起的请求结束以后，相关的cookie信息就会被opener的handler通过cookiejar对象保存

1 page_url = "http://www.jokeji.cn/User/MemberCenter.asp"2 3 page_req = request.Request(url=page_url,headers=headers)

 1 import ssl 2 ssl._create_default_https_context = ssl._create_unverified_context 3 ?4 from urllib import request,parse 5 from time import sleep 6 import re 7 ?8 # 1、【数据的获取】 9 # 封装一个函数，用于将url转化成一个请求对象10 def request_by(url,page):11 ????headers = {12 ????????‘User-Agent‘: ‘Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/71.0.3578.80 Safari/537.36‘}13 ????if page==1:14 ????????page_url = url + ".html"15 ????else:16 ????????page_url = url +"_"+ str(page) + ".html"17 ????print("正在访问：",page_url)18 ????req = request.Request(url=page_url,headers=headers)19 ????return ?req20 ????21 # 封装一个函数，用于对请求对象发起请求并且把响应体返回出去22 def get_html_from(req):23 ????res = request.urlopen(req)24 ????# 每请求一次要休眠一段时间25 ????sleep(1)26 ????return res.read().decode("utf-8")27 28 # 2、【数据的解析】29 def anylasis_data(html):30 ????pat = re.compile(r‘<div class="box picblock.*?<img src2="(.*?)"‘,re.S)31 ????imgs = pat.findall(html)32 ????return imgs33 34 # 3、数据的存储35 def download_imgs(imgs):36 ????for img in imgs:37 ????????# http://pic1.sc.chinaz.com/Files/pic/38 ????????# pic9/201904/zzpic17564_s.jpg39 ????????# 生成图片的名字40 ????????img_name = img.split("/")[-1]41 ????????print("正在下载图片：",img)42 ????????request.urlretrieve(url=img,filename="./meinv/"+img_name)43 ????????sleep(1)44 45 if __name__ == ‘__main__‘:46 ????page_url = "http://sc.chinaz.com/tupian/meinvxiezhen"47 48 ????for i in range(1,2):49 ????????req = request_by(url=page_url,page=i)50 ????????res = get_html_from(req)51 ????????imgs = anylasis_data(res)52 ????????download_imgs(imgs)