scrapy-redis使redis不止保存url

  1 class RedisMixin(object):
  2     """Mixin class to implement reading urls from a redis queue."""
  3     redis_key = None
  4     redis_batch_size = None
  5     redis_encoding = None
  6 
  7     # Redis client placeholder.
  8     server = None
  9 
 10     def start_requests(self):
 11         """Returns a batch of start requests from redis."""
 12         return self.next_requests()
 13 
 14     def setup_redis(self, crawler=None):
 15         """Setup redis connection and idle signal.
 16 
 17         This should be called after the spider has set its crawler object.
 18         """
 19         if self.server is not None:
 20             return
 21 
 22         if crawler is None:
 23             # We allow optional crawler argument to keep backwards
 24             # compatibility.
 25             # XXX: Raise a deprecation warning.
 26             crawler = getattr(self, ‘crawler‘, None)
 27 
 28         if crawler is None:
 29             raise ValueError("crawler is required")
 30 
 31         settings = crawler.settings
 32 
 33         if self.redis_key is None:
 34             self.redis_key = settings.get(
 35                 ‘REDIS_START_URLS_KEY‘, defaults.START_URLS_KEY,
 36             )
 37 
 38         self.redis_key = self.redis_key % {‘name‘: self.name}
 39 
 40         if not self.redis_key.strip():
 41             raise ValueError("redis_key must not be empty")
 42 
 43         if self.redis_batch_size is None:
 44             # TODO: Deprecate this setting (REDIS_START_URLS_BATCH_SIZE).
 45             self.redis_batch_size = settings.getint(
 46                 ‘REDIS_START_URLS_BATCH_SIZE‘,
 47                 settings.getint(‘CONCURRENT_REQUESTS‘),
 48             )
 49 
 50         try:
 51             self.redis_batch_size = int(self.redis_batch_size)
 52         except (TypeError, ValueError):
 53             raise ValueError("redis_batch_size must be an integer")
 54 
 55         if self.redis_encoding is None:
 56             self.redis_encoding = settings.get(‘REDIS_ENCODING‘, defaults.REDIS_ENCODING)
 57 
 58         self.logger.info("Reading start URLs from redis key ‘%(redis_key)s‘ "
 59                          "(batch size: %(redis_batch_size)s, encoding: %(redis_encoding)s",
 60                          self.__dict__)
 61 
 62         self.server = connection.from_settings(crawler.settings)
 63         # The idle signal is called when the spider has no requests left,
 64         # that‘s when we will schedule new requests from redis queue
 65         crawler.signals.connect(self.spider_idle, signal=signals.spider_idle)
 66 
 67     def next_requests(self):
 68         """Returns a request to be scheduled or none."""
 69         use_set = self.settings.getbool(‘REDIS_START_URLS_AS_SET‘, defaults.START_URLS_AS_SET)
 70         fetch_one = self.server.spop if use_set else self.server.lpop
 71         # XXX: Do we need to use a timeout here?
 72         found = 0
 73         # TODO: Use redis pipeline execution.
 74         while found < self.redis_batch_size:
 75             data = fetch_one(self.redis_key)
 76             if not data:
 77                 # Queue empty.
 78                 break
 79             req = self.make_request_from_data(data)
 80             if req:
 81                 yield req
 82                 found += 1
 83             else:
 84                 self.logger.debug("Request not made from data: %r", data)
 85 
 86         if found:
 87             self.logger.debug("Read %s requests from ‘%s‘", found, self.redis_key)
 88 
 89     def make_request_from_data(self, data):
 90         """Returns a Request instance from data coming from Redis.
 91 
 92         By default, ``data`` is an encoded URL. You can override this method to
 93         provide your own message decoding.
 94 
 95         Parameters
 96         ----------
 97         data : bytes
 98             Message from redis.
 99 
100         """
101         url = bytes_to_str(data, self.redis_encoding)
102         return self.make_requests_from_url(url)
103 
104     def schedule_next_requests(self):
105         """Schedules a request if available"""
106         # TODO: While there is capacity, schedule a batch of redis requests.
107         for req in self.next_requests():
108             self.crawler.engine.crawl(req, spider=self)
109 
110     def spider_idle(self):
111         """Schedules a request if available, otherwise waits."""
112         # XXX: Handle a sentinel to close the spider.
113         self.schedule_next_requests()
114         raise DontCloseSpider
115 
116 
117 class RedisSpider(RedisMixin, Spider):
118     """Spider that reads urls from redis queue when idle.
119 
120     Attributes
121     ----------
122     redis_key : str (default: REDIS_START_URLS_KEY)
123         Redis key where to fetch start URLs from..
124     redis_batch_size : int (default: CONCURRENT_REQUESTS)
125         Number of messages to fetch from redis on each attempt.
126     redis_encoding : str (default: REDIS_ENCODING)
127         Encoding to use when decoding messages from redis queue.
128 
129     Settings
130     --------
131     REDIS_START_URLS_KEY : str (default: "<spider.name>:start_urls")
132         Default Redis key where to fetch start URLs from..
133     REDIS_START_URLS_BATCH_SIZE : int (deprecated by CONCURRENT_REQUESTS)
134         Default number of messages to fetch from redis on each attempt.
135     REDIS_START_URLS_AS_SET : bool (default: False)
136         Use SET operations to retrieve messages from the redis queue. If False,
137         the messages are retrieve using the LPOP command.
138     REDIS_ENCODING : str (default: "utf-8")
139         Default encoding to use when decoding messages from redis queue.
140 
141     """
142 
143     @classmethod
144     def from_crawler(self, crawler, *args, **kwargs):
145         obj = super(RedisSpider, self).from_crawler(crawler, *args, **kwargs)
146         obj.setup_redis(crawler)
147         return obj
148 
149 
150 class RedisCrawlSpider(RedisMixin, CrawlSpider):
151     """Spider that reads urls from redis queue when idle.
152 
153     Attributes
154     ----------
155     redis_key : str (default: REDIS_START_URLS_KEY)
156         Redis key where to fetch start URLs from..
157     redis_batch_size : int (default: CONCURRENT_REQUESTS)
158         Number of messages to fetch from redis on each attempt.
159     redis_encoding : str (default: REDIS_ENCODING)
160         Encoding to use when decoding messages from redis queue.
161 
162     Settings
163     --------
164     REDIS_START_URLS_KEY : str (default: "<spider.name>:start_urls")
165         Default Redis key where to fetch start URLs from..
166     REDIS_START_URLS_BATCH_SIZE : int (deprecated by CONCURRENT_REQUESTS)
167         Default number of messages to fetch from redis on each attempt.
168     REDIS_START_URLS_AS_SET : bool (default: True)
169         Use SET operations to retrieve messages from the redis queue.
170     REDIS_ENCODING : str (default: "utf-8")
171         Default encoding to use when decoding messages from redis queue.
172 
173     """
174 
175     @classmethod
176     def from_crawler(self, crawler, *args, **kwargs):
177         obj = super(RedisCrawlSpider, self).from_crawler(crawler, *args, **kwargs)
178         obj.setup_redis(crawler)
179         return obj

 1     def make_request_from_data(self, data):
 2         ‘‘‘
 3         :params data bytes, Message from redis
 4         ‘‘‘
 5         company = bytes_to_str(data, self.redis_encoding)
 6         return self.make_requests_from_url(company)
 7 
 8     def make_requests_from_url(self, company):
 9         data = eval(company)
10         url = data["url"]
11         headers = {
12             "User-Agent": "Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/70.0.3538.67 Safari/537.36",
13             "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8"
14         }
15         return Request(url, self.parse, meta={"data": data}, dont_filter=True, headers=headers)