scrapy专题（五）：自定义扩展

# extendsions.py
# -*- coding: utf-8-*-
from scrapy import signals
from scrapy.exceptions import NotConfigured

class StatsItemCount(object):
    def __init__(self):
        self.item_count = 0

    @classmethod
    def from_crawler(cls, crawler):
        # instantiate the extension object
        ext = cls()

        # connect the extension object to signals
        crawler.signals.connect(ext.spider_opened,
                                signal=signals.spider_opened)
        crawler.signals.connect(ext.spider_closed,
                                signal=signals.spider_closed)
        crawler.signals.connect(ext.item_scraped, signal=signals.item_scraped)

        # return the extension object
        return ext

    def spider_opened(self, spider):
        spider.logger.info("-----------opened spider %s", spider.name)

    def spider_closed(self, spider):
        spider.logger.info("------------closed spider %s", spider.name)
        spider.logger.info("一共获取到{}条数据".format(self.item_count))

    def item_scraped(self, item, spider):
        self.item_count += 1

# settings.py
EXTENSIONS = {
   ‘ccidcom.extensions.StatsItemCount‘: 999,
}

...
2019-11-21 16:53:23 [ccidcomSpider] INFO: -----------opened spider ccidcomSpider
2019-11-21 16:53:23 [scrapy.extensions.telnet] INFO: Telnet console listening on 127.0.0.1:6023
2019-11-21 16:53:23 [scrapy.core.engine] INFO: Closing spider (finished)
2019-11-21 16:53:23 [ccidcomSpider] INFO: ------------closed spider ccidcomSpider
2019-11-21 16:53:23 [ccidcomSpider] INFO: 一共获取到10条数据
...