Scrapy实战：爬取http://quotes.toscrape.com网站数据

import scrapyclass QuoteItem(scrapy.Item): ???# define the fields for your item here like: ???# name = scrapy.Field() ???text = scrapy.Field() ???author = scrapy.Field() ???tags = scrapy.Field()

# -*- coding: utf-8 -*-import scrapyfrom quotetutorial.items import QuoteItemclass QuotesSpider(scrapy.Spider): ???name = ‘quotes‘ ???allowed_domains = [‘quotes.toscrape.com‘] ???start_urls = [‘http://quotes.toscrape.com/‘] ???def parse(self, response): ???????# print(response.status) # 200 ???????quotes = response.css(‘.quote‘) ???????for quote in quotes: ???????????item = QuoteItem() ???????????text = quote.css(‘.text::text‘).extract_first() ???????????author = quote.css(‘.author::text‘).extract_first() ???????????tags = quote.css(‘.tags .tag::text‘).extract() ???????????item[‘text‘] = text ???????????item[‘author‘] = author ???????????item[‘tags‘] = tags ???????????yield item ???????next = response.css(‘.pager .next a::attr(href)‘).extract_first() ?# 获取下一页的相对链接 ???????url = response.urljoin(next) ?# 生成完整的下一页链接 ???????yield scrapy.Request(url=url, callback=self.parse) ?# 把下一页的链接回调给自身再次请求

# -*- coding: utf-8 -*-# Define your item pipelines here## Don‘t forget to add your pipeline to the ITEM_PIPELINES setting# See: https://doc.scrapy.org/en/latest/topics/item-pipeline.html# 使用的话需要在settings文件中设置import pymongo as pymongofrom scrapy.exceptions import DropItemclass TextPipeline(object): ???"""对输出的item进行进一步的处理""" ???def __init__(self): ???????self.limit = 50 ???def process_item(self, item, spider): ???????if item[‘text‘]: ???????????if len(item[‘text‘]) > self.limit: ???????????????item[‘text‘] = item[‘text‘][0:self.limit].rstrip() + ‘......‘ ???????????return item ???????else: ???????????return DropItem(‘Missing Text!‘)class MongoPipeline(object): ???"""把输出的item保存到MongoDB数据库""" ???def __init__(self, mongo_url, mongo_db): ???????self.mongo_uri = mongo_url ???????self.mongo_db = mongo_db ???@classmethod ???def from_crawler(cls, crawler): ???????"""从settings文件获取配置信息""" ???????return cls( ???????????mongo_url=crawler.settings.get(‘MONGO_URI‘), ???????????mongo_db=crawler.settings.get(‘MONGO_DB‘) ???????) ???def open_spider(self, spider): ???????"""初始化mongodb""" ???????self.client = pymongo.MongoClient(self.mongo_uri) ???????self.db = self.client[self.mongo_db] ?# 为啥用[],而不是() ???def process_item(self, item, spider): ???????name = item.__class__.__name__ ?# 获取item的名称用作表名,也就是QuoteItem ???????self.db[name].insert(dict(item)) ?# 为啥要用dict(item) ???????return item ???def close_spider(self, spider): ???????self.client.close()

ITEM_PIPELINES = { ??# ‘quotetutorial.pipelines.TextPipeline‘: 300, ??‘quotetutorial.pipelines.MongoPipeline‘: 400,}MONGO_URI = ‘localhost‘MONGO_DB = ‘quotestutorial‘