爬虫新手学习2-爬虫进阶(urllib和urllib2 的区别、url转码、爬虫GET提交实例、批量爬取贴吧数据、fidder软件安装、有道翻译POST实例、豆瓣ajax数据获取)

#coding:utf-8import urllib ???#负责url编码处理import urllib2url = "https://www.baidu.com/s"word = {"wd": "繁华"}word = urllib.urlencode(word) ??#转换成url编码格式(字符串)newurl = url + "?" + wordheaders = {"User-Agent": "Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/63.0.3239.84 Safari/537.36"}request = urllib2.Request(newurl, headers=headers)response = urllib2.urlopen(request)print response.read()

#coding:utf-8import urllib ???#负责url编码处理import urllib2url = "https://www.baidu.com/s"word = {"wd": "咖菲猫"}word = urllib.urlencode(word) ??#转换成url编码格式(字符串)newurl = url + "?" + wordheaders = {"User-Agent": "Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/63.0.3239.84 Safari/537.36"}request = urllib2.Request(newurl, headers=headers)response = urllib2.urlopen(request)print response.read()

#coding:utf-8import urllib ???#负责url编码处理import urllib2url = "https://www.baidu.com/s"keyword = raw_input("请输入要查询的关键字:")word = {"wd": keyword}word = urllib.urlencode(word) ??#转换成url编码格式(字符串)newurl = url + "?" + wordheaders = {"User-Agent": "Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/63.0.3239.84 Safari/537.36"}request = urllib2.Request(newurl, headers=headers)response = urllib2.urlopen(request)print response.read()

#coding:utf-8import urllibimport urllib2def loadPage(url, filename): ???‘‘‘ ???????作用：根据url发送请求， 获取服务器响应文件 ???????url：需要爬取的url地址 ???????filename：文件名 ???‘‘‘ ???print "正在下载" + filename ???headers = { ???????"User-Agent": "Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/63.0.3239.84 Safari/537.36"} ???request = urllib2.Request(url, headers = headers) ???response = urllib2.urlopen(request) ???return response.read()def writeFile(html, filename): ???‘‘‘ ???????作用：保存服务器相应文件到本地硬盘文件里 ???????html：服务器相应文件 ???????filename：本地磁盘文件名 ???‘‘‘ ???print "正在存储" + filename ???with open(unicode(filename, ‘utf-8‘), ‘w‘) as f: ???????f.write(html) ???print "-" * 20def tiebaSpider(url, beginPage, endPage, name): ???‘‘‘ ???????作用：负责处理url，分配每个url去发送请求 ???????url：需要处理的第一个url ???????beginPage：爬虫执行的起始页面 ???????endPage：爬虫执行的截止页面 ???‘‘‘ ???for page in range(beginPage, endPage + 1): ???????pn = (page - 1) * 50 ???????filename = "第" + name + "-" + str(page) + "页.html" ???????#组合为完整的url，并且pn值每次增加50 ???????fullurl = url + "&pn=" + str(pn) ???????#print fullurl ???????#调用loadPage()发送请求获取HTML页面 ???????html = loadPage(fullurl, filename) ???????#将获取到的HTML页面写入本地磁盘文件 ???????writeFile(html, filename)#模拟main函数"if __name__ == "__main__": ???kw = raw_input("请输入需要爬取的贴吧：") ???beginPage = int(raw_input("请输入起始页：")) ???endPage = int(raw_input("请输入终止页：")) ???url = "http://tieba.baidu.com/f?" ???key = urllib.urlencode({"kw": kw}) ???#组合后的url示例：http://tieba.baidu.com/f?kw=绝地求生 ???newurl = url + key ???tiebaSpider(newurl, beginPage, endPage, kw)

#coding:utf-8import urllibimport urllib2#通过抓包的方式获取的url，并不是浏览器上显示的urlurl = "http://fanyi.youdao.com/translate?smartresult=dict&smartresult=rule&smartresult=ugc&sessionFrom=null"#完整的headersheaders = { ???"Accept": "application/json, text/javascript, */*; q=0.01", ???"Origin": "http://fanyi.youdao.com", ???"X-Requested-With": "XMLHttpRequest", ???"User-Agent": "Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/63.0.3239.84 Safari/537.36", ???"Content-Type": "application/x-www-form-urlencoded; charset=UTF-8"}#用户接口输入key = raw_input("请输入需要翻译的英文单词：")#发送到web服务器的表单数据formdata = { ???"i": key, ???"from": "AUTO", ???"to": "AUTO", ???"smartresult": "dict", ???"client": "fanyideskweb", ???"salt": "1513519318944", ???"sign": "29e3219b8c4e75f76f6e6aba0bb3c4b5", ???"doctype": "json", ???"version": "2.1", ???"keyfrom": "fanyi.web", ???"action": "FY_BY_REALTIME", ???"typoResult": "false"}#经过urlencode转码data = urllib.urlencode(formdata)#如果Request()方法里的data参数有值，那么这个请求就是POST#如果没有，就是Getrequest = urllib2.Request(url, data= data, headers= headers)print urllib2.urlopen(request).read()

#coding:utf-8import urllib2import urlliburl = "https://movie.douban.com/j/chart/top_list?type=11&interval_id=100%3A90&action="headers = {"User-Agent": "Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/63.0.3239.84 Safari/537.36"}formdata = { ???"start": "0", ???"limit": "20"}data = urllib.urlencode(formdata)request = urllib2.Request(url, data = data, headers = headers)print urllib2.urlopen(request).read()