scrapy-redis 分布式 案例一

pip install scrapy-redis

git clone git://github.com/rolando/scrapy-redis

from scrapy.linkextractors import LinkExtractor
from scrapy.spiders import CrawlSpider, Rule


class DmozSpider(CrawlSpider):
    """Follow categories and extract links."""
    name = ‘dmoz‘
    allowed_domains = [‘dmoz.org‘]
    start_urls = [‘http://www.dmoz.org/‘]

    rules = [
        Rule(LinkExtractor(
            restrict_css=(‘.top-cat‘, ‘.sub-cat‘, ‘.cat-item‘)
        ), callback=‘parse_directory‘, follow=True),
    ]

    def parse_directory(self, response):
        for div in response.css(‘.title-and-desc‘):
            yield {
                ‘name‘: div.css(‘.site-title::text‘).extract_first(),
                ‘description‘: div.css(‘.site-descr::text‘).extract_first().strip(),
                ‘link‘: div.css(‘a::attr(href)‘).extract_first(),
            }

# 指定schedule队列
# Enables scheduling storing requests queue in redis.
SCHEDULER = "scrapy_redis.scheduler.Scheduler"


# 指定哪个去重方法给request对象去重
# Ensure all spiders share same duplicates filter through redis.
DUPEFILTER_CLASS = "scrapy_redis.dupefilter.RFPDupeFilter"

# Default requests serializer is pickle, but it can be changed to any module
# with loads and dumps functions. Note that pickle is not compatible between
# python versions.
# Caveat: In python 3.x, the serializer must return strings keys and support
# bytes as values. Because of this reason the json or msgpack module will not
# work by default. In python 2.x there is no such issue and you can use
# ‘json‘ or ‘msgpack‘ as serializers.
#SCHEDULER_SERIALIZER = "scrapy_redis.picklecompat"

# 队列中的内容是否持久保存，False：在关闭redis的时候清空redis
# Don‘t cleanup redis queues, allows to pause/resume crawls.
#SCHEDULER_PERSIST = True

# Schedule requests using a priority queue. (default)
#SCHEDULER_QUEUE_CLASS = ‘scrapy_redis.queue.PriorityQueue‘

# Alternative queues.
#SCHEDULER_QUEUE_CLASS = ‘scrapy_redis.queue.FifoQueue‘
#SCHEDULER_QUEUE_CLASS = ‘scrapy_redis.queue.LifoQueue‘

# Max idle time to prevent the spider from being closed when distributed crawling.
# This only works if queue class is SpiderQueue or SpiderStack,
# and may also block the same time when your spider start at the first time (because the queue is empty).
#SCHEDULER_IDLE_BEFORE_CLOSE = 10

# scrapy_redis实现的items保存到redis的pipeline
# Store scraped item in redis for post-processing.
ITEM_PIPELINES = {
    ‘scrapy_redis.pipelines.RedisPipeline‘: 300
}

# The item pipeline serializes and stores the items in this redis key.
#REDIS_ITEMS_KEY = ‘%(spider)s:items‘

# The items serializer is by default ScrapyJSONEncoder. You can use any
# importable path to a callable object.
#REDIS_ITEMS_SERIALIZER = ‘json.dumps‘

# 指定redis的地址
# Specify the host and port to use when connecting to Redis (optional).
#REDIS_HOST = ‘localhost‘
#REDIS_PORT = 6379


# 指定redis的地址
# Specify the full Redis URL for connecting (optional).
# If set, this takes precedence over the REDIS_HOST and REDIS_PORT settings.
#REDIS_URL = ‘redis://user:pass@hostname:9001‘

# Custom redis client parameters (i.e.: socket timeout, etc.)
#REDIS_PARAMS  = {}
# Use custom redis client class.
#REDIS_PARAMS[‘redis_cls‘] = ‘myproject.RedisClient‘

# If True, it uses redis‘ ``spop`` operation. This could be useful if you
# want to avoid duplicates in your start urls list. In this cases, urls must
# be added via ``sadd`` command or you will get a type error from redis.
#REDIS_START_URLS_AS_SET = False

# Default start urls key for RedisSpider and RedisCrawlSpider.
#REDIS_START_URLS_KEY = ‘%(name)s:start_urls‘

# Use other encoding than utf-8 for redis.
#REDIS_ENCODING = ‘latin1‘

# 去重
DUPEFILTER_CLASS = "scrapy_redis.dupefilter.RFPDupeFilter"
# 调度器
SCHEDULER = "scrapy_redis.scheduler.Scheduler"
# 调度器持久化
SCHEDULER_PERSIST = True

# 指定redis地址
REDIS_URL = "redis://192.168.226.150:6379"
ITEM_PIPELINES = {
    ‘example.pipelines.ExamplePipeline‘: 300,
    ‘scrapy_redis.pipelines.RedisPipeline‘: 400, # 保存数据到redis
}

\example>scrapy crawl dmoz

#ITEM_PIPELINES = {
    # ‘example.pipelines.ExamplePipeline‘: 300,
    #‘scrapy_redis.pipelines.RedisPipeline‘: 400, # 保存数据到redis
#}

dmoz:requests 有变化(变多或者变少或者不变)
dmoz:dupefilter 变多
dmoz:items 不变

from scrapy_redis.pipelines import RedisPipeline
    # 调用这个方法，实现数据的保存
    def process_item(self, item, spider):
    # 调用一个异步线程去处理这个item
        return deferToThread(self._process_item, item, spider)

    def _process_item(self, item, spider):
        key = self.item_key(item, spider)
        data = self.serialize(item)
       # 向dmoz:items中添加item
        self.server.rpush(key, data)
        return item

- 使用sha1加密request得到指纹
- 把指纹存在redis的集合中
- 下一次新来一个request，同样的方式生成指纹，判断指纹是否存在reids的集合中

- fp = hashlib.sha1()
- fp.update(request.method)
- fp.update(request.body or b"")
- fp.update(url)
- fp.hexdigest()

added = self.server.sadd(self.key, fp)
return added != 0