【爬虫】使用urllib.request去爬取小说

import urllib.requestimport re#1获取主页源代码#2获取章节超链接#3获取章节内容#4下载小说#驼峰命名法#注释 获取小说内容def getNovelContent(): ???#获取源代码 HTTP Response对象 ???html = urllib.request.urlopen(‘http://www.quanshuwang.com/book/0/269/‘) ???html = html.read() ???#print(html) ???#设置编码 ???html = html.decode(‘gbk‘) ???#获取超链接 ???#<li><a href="http://www.quanshuwang.com/book/0/269/78850.html" title="第一章 山边小村，共2741字">第一章 山边小村</a></li> ???#正则表达式 通配符 ?.*? 匹配所有 ?（.*?）括号里面是需要的内容 ?分组匹配 ???reg = r‘<li><a href="(.*?)" title=".*?">(.*?)</a></li>‘ ???#目的是增加效率的，可以不写，但写更好 ???reg = re.compile(reg) ???urls = re.findall(reg,html) ???#print(urls) ???for i in urls: ???????#print(i[0]) ???????novel_url = i[0] ???????novel_title = i[1] ???????chapt = urllib.request.urlopen(novel_url).read() ???????chapt_html = chapt.decode(‘gbk‘) ???????# | ||d r‘ |d‘ ????????reg = ‘</script>&nbsp;&nbsp;&nbsp;&nbsp;(.*?)<script type="text/javascript">‘ ???????# S 多行匹配 ???????reg = re.compile(reg,re.S) ???????chapt_content = re.findall(reg,chapt_html) ???????#print(chapt_content[0]) ???????#把没用的替换掉,注意类型进行一次替换之后是列表，br是换行，nbsp是空格 ???????chapt_content = chapt_content[0].replace(‘<br />‘,"") ???????#print(type(chapt_content)) ???????chapt_content = chapt_content.replace(‘&nbsp;&nbsp;&nbsp;&nbsp;‘,"") ???????#从列表变成字符串了，下面不用加索引 ???????#print(chapt_content) ???????#下载,可以加个提示 ???????print("正在保存%s"%novel_title) ???????# w 读写模式 ?wb 二进制读写模式，一般用来读写照片和视频 ?没加具体路径则在py路径下自动新增，也可以保存为doc格式等 ???????f = open(‘{}.txt‘.format(novel_title),‘w‘) ???????f.write(chapt_content) ???????f.closegetNovelContent()

import urllib.requestimport redef getNovelContent(): ???html = urllib.request.urlopen(‘http://www.quanshuwang.com/book/0/269/‘) ???html = html.read() ???html = html.decode(‘gbk‘) ???reg = r‘<li><a href="(.*?)" title=".*?">(.*?)</a></li>‘ ???reg = re.compile(reg) ???urls = re.findall(reg,html) ???for i in urls: ???????novel_url = i[0] ???????novel_title = i[1] ???????chapt = urllib.request.urlopen(novel_url).read() ???????chapt_html = chapt.decode(‘gbk‘) ???????reg = ‘</script>&nbsp;&nbsp;&nbsp;&nbsp;(.*?)<script type="text/javascript">‘ ???????reg = re.compile(reg,re.S) ???????chapt_content = re.findall(reg,chapt_html) ???????chapt_content = chapt_content[0].replace(‘<br />‘,"") ???????chapt_content = chapt_content.replace(‘&nbsp;&nbsp;&nbsp;&nbsp;‘,"") ???????print("正在保存%s"%novel_title) ???????f = open(‘{}.txt‘.format(novel_title),‘w‘) ???????f.write(chapt_content) ???????f.closegetNovelContent()