scrapy-redis使redis不止保存url

 ?1 class RedisMixin(object): ?2 ????"""Mixin class to implement reading urls from a redis queue.""" ?3 ????redis_key = None ?4 ????redis_batch_size = None ?5 ????redis_encoding = None ?6 ??7 ????# Redis client placeholder. ?8 ????server = None ?9 ?10 ????def start_requests(self): 11 ????????"""Returns a batch of start requests from redis.""" 12 ????????return self.next_requests() 13 ?14 ????def setup_redis(self, crawler=None): 15 ????????"""Setup redis connection and idle signal. 16 ?17 ????????This should be called after the spider has set its crawler object. 18 ????????""" 19 ????????if self.server is not None: 20 ????????????return 21 ?22 ????????if crawler is None: 23 ????????????# We allow optional crawler argument to keep backwards 24 ????????????# compatibility. 25 ????????????# XXX: Raise a deprecation warning. 26 ????????????crawler = getattr(self, ‘crawler‘, None) 27 ?28 ????????if crawler is None: 29 ????????????raise ValueError("crawler is required") 30 ?31 ????????settings = crawler.settings 32 ?33 ????????if self.redis_key is None: 34 ????????????self.redis_key = settings.get( 35 ????????????????‘REDIS_START_URLS_KEY‘, defaults.START_URLS_KEY, 36 ????????????) 37 ?38 ????????self.redis_key = self.redis_key % {‘name‘: self.name} 39 ?40 ????????if not self.redis_key.strip(): 41 ????????????raise ValueError("redis_key must not be empty") 42 ?43 ????????if self.redis_batch_size is None: 44 ????????????# TODO: Deprecate this setting (REDIS_START_URLS_BATCH_SIZE). 45 ????????????self.redis_batch_size = settings.getint( 46 ????????????????‘REDIS_START_URLS_BATCH_SIZE‘, 47 ????????????????settings.getint(‘CONCURRENT_REQUESTS‘), 48 ????????????) 49 ?50 ????????try: 51 ????????????self.redis_batch_size = int(self.redis_batch_size) 52 ????????except (TypeError, ValueError): 53 ????????????raise ValueError("redis_batch_size must be an integer") 54 ?55 ????????if self.redis_encoding is None: 56 ????????????self.redis_encoding = settings.get(‘REDIS_ENCODING‘, defaults.REDIS_ENCODING) 57 ?58 ????????self.logger.info("Reading start URLs from redis key ‘%(redis_key)s‘ " 59 ?????????????????????????"(batch size: %(redis_batch_size)s, encoding: %(redis_encoding)s", 60 ?????????????????????????self.__dict__) 61 ?62 ????????self.server = connection.from_settings(crawler.settings) 63 ????????# The idle signal is called when the spider has no requests left, 64 ????????# that‘s when we will schedule new requests from redis queue 65 ????????crawler.signals.connect(self.spider_idle, signal=signals.spider_idle) 66 ?67 ????def next_requests(self): 68 ????????"""Returns a request to be scheduled or none.""" 69 ????????use_set = self.settings.getbool(‘REDIS_START_URLS_AS_SET‘, defaults.START_URLS_AS_SET) 70 ????????fetch_one = self.server.spop if use_set else self.server.lpop 71 ????????# XXX: Do we need to use a timeout here? 72 ????????found = 0 73 ????????# TODO: Use redis pipeline execution. 74 ????????while found < self.redis_batch_size: 75 ????????????data = fetch_one(self.redis_key) 76 ????????????if not data: 77 ????????????????# Queue empty. 78 ????????????????break 79 ????????????req = self.make_request_from_data(data) 80 ????????????if req: 81 ????????????????yield req 82 ????????????????found += 1 83 ????????????else: 84 ????????????????self.logger.debug("Request not made from data: %r", data) 85 ?86 ????????if found: 87 ????????????self.logger.debug("Read %s requests from ‘%s‘", found, self.redis_key) 88 ?89 ????def make_request_from_data(self, data): 90 ????????"""Returns a Request instance from data coming from Redis. 91 ?92 ????????By default, ``data`` is an encoded URL. You can override this method to 93 ????????provide your own message decoding. 94 ?95 ????????Parameters 96 ????????---------- 97 ????????data : bytes 98 ????????????Message from redis. 99 100 ????????"""101 ????????url = bytes_to_str(data, self.redis_encoding)102 ????????return self.make_requests_from_url(url)103 104 ????def schedule_next_requests(self):105 ????????"""Schedules a request if available"""106 ????????# TODO: While there is capacity, schedule a batch of redis requests.107 ????????for req in self.next_requests():108 ????????????self.crawler.engine.crawl(req, spider=self)109 110 ????def spider_idle(self):111 ????????"""Schedules a request if available, otherwise waits."""112 ????????# XXX: Handle a sentinel to close the spider.113 ????????self.schedule_next_requests()114 ????????raise DontCloseSpider115 116 117 class RedisSpider(RedisMixin, Spider):118 ????"""Spider that reads urls from redis queue when idle.119 120 ????Attributes121 ????----------122 ????redis_key : str (default: REDIS_START_URLS_KEY)123 ????????Redis key where to fetch start URLs from..124 ????redis_batch_size : int (default: CONCURRENT_REQUESTS)125 ????????Number of messages to fetch from redis on each attempt.126 ????redis_encoding : str (default: REDIS_ENCODING)127 ????????Encoding to use when decoding messages from redis queue.128 129 ????Settings130 ????--------131 ????REDIS_START_URLS_KEY : str (default: "<spider.name>:start_urls")132 ????????Default Redis key where to fetch start URLs from..133 ????REDIS_START_URLS_BATCH_SIZE : int (deprecated by CONCURRENT_REQUESTS)134 ????????Default number of messages to fetch from redis on each attempt.135 ????REDIS_START_URLS_AS_SET : bool (default: False)136 ????????Use SET operations to retrieve messages from the redis queue. If False,137 ????????the messages are retrieve using the LPOP command.138 ????REDIS_ENCODING : str (default: "utf-8")139 ????????Default encoding to use when decoding messages from redis queue.140 141 ????"""142 143 ????@classmethod144 ????def from_crawler(self, crawler, *args, **kwargs):145 ????????obj = super(RedisSpider, self).from_crawler(crawler, *args, **kwargs)146 ????????obj.setup_redis(crawler)147 ????????return obj148 149 150 class RedisCrawlSpider(RedisMixin, CrawlSpider):151 ????"""Spider that reads urls from redis queue when idle.152 153 ????Attributes154 ????----------155 ????redis_key : str (default: REDIS_START_URLS_KEY)156 ????????Redis key where to fetch start URLs from..157 ????redis_batch_size : int (default: CONCURRENT_REQUESTS)158 ????????Number of messages to fetch from redis on each attempt.159 ????redis_encoding : str (default: REDIS_ENCODING)160 ????????Encoding to use when decoding messages from redis queue.161 162 ????Settings163 ????--------164 ????REDIS_START_URLS_KEY : str (default: "<spider.name>:start_urls")165 ????????Default Redis key where to fetch start URLs from..166 ????REDIS_START_URLS_BATCH_SIZE : int (deprecated by CONCURRENT_REQUESTS)167 ????????Default number of messages to fetch from redis on each attempt.168 ????REDIS_START_URLS_AS_SET : bool (default: True)169 ????????Use SET operations to retrieve messages from the redis queue.170 ????REDIS_ENCODING : str (default: "utf-8")171 ????????Default encoding to use when decoding messages from redis queue.172 173 ????"""174 175 ????@classmethod176 ????def from_crawler(self, crawler, *args, **kwargs):177 ????????obj = super(RedisCrawlSpider, self).from_crawler(crawler, *args, **kwargs)178 ????????obj.setup_redis(crawler)179 ????????return obj

 1 ????def make_request_from_data(self, data): 2 ????????‘‘‘ 3 ????????:params data bytes, Message from redis 4 ????????‘‘‘ 5 ????????company = bytes_to_str(data, self.redis_encoding) 6 ????????return self.make_requests_from_url(company) 7 ?8 ????def make_requests_from_url(self, company): 9 ????????data = eval(company)10 ????????url = data["url"]11 ????????headers = {12 ????????????"User-Agent": "Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/70.0.3538.67 Safari/537.36",13 ????????????"Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8"14 ????????}15 ????????return Request(url, self.parse, meta={"data": data}, dont_filter=True, headers=headers)