Scrapy基础02

def start_requests(self):
    cls = self.__class__
    if method_is_overridden(cls, Spider, ‘make_requests_from_url‘):
        warnings.warn(
            "Spider.make_requests_from_url method is deprecated; it "
            "won‘t be called in future Scrapy releases. Please "
            "override Spider.start_requests method instead (see %s.%s)." % (
                cls.__module__, cls.__name__
            ),
        )
        for url in self.start_urls:
            yield self.make_requests_from_url(url)
    else:
        for url in self.start_urls:
            yield Request(url, dont_filter=True)

import scrapy
from scrapy.http import Request


class ChoutiSpider(scrapy.Spider):
    name = ‘chouti‘
    allowed_domains = [‘chouti.com‘]
    start_urls = [‘http://chouti.com/‘]
    
    def start_requests(self):
        for url in self.start_urls:
            yield Request(url=url,callback=self.parse)

    def parse(self, response):
        pass

    def start_requests(self):
        for url in self.start_urls:
            # yield Request(url=url,callback=self.parse)
            return [Request(url=url,callback=self.parse)]

# from scrapy.crawler import Crawler

def crawl(self, *args, **kwargs):
    assert not self.crawling, "Crawling already taking place"
    self.crawling = True

    try:
        self.spider = self._create_spider(*args, **kwargs)
        self.engine = self._create_engine()
        start_requests = iter(self.spider.start_requests())
        yield self.engine.open_spider(self.spider, start_requests)
        yield defer.maybeDeferred(self.engine.start)
    except Exception:
    	pass
    	...

# settings.py

ITEM_PIPELINES = {
   ‘day25spider.pipelines.Day25SpiderPipeline‘: 300,
   ‘day25spider.pipelines.SecondPipeline‘: 400,
}

...


STORAGE_CONFIG = "chouti.json"
STORAGE_CONFIG2 = "chouti2.json"
DEPTH_LIMIT = 1

ENV = "debug"

# 爬虫文件：chouti.py

import scrapy
from scrapy.http import Request
from scrapy.selector import HtmlXPathSelector
from ..items import Day25SpiderItem


class ChoutiSpider(scrapy.Spider):
    name = ‘chouti‘
    allowed_domains = [‘chouti.com‘]
    start_urls = [‘http://chouti.com/‘]

    def start_requests(self):
        for url in self.start_urls:
            # yield Request(url=url,callback=self.parse)
            return [Request(url=url,callback=self.parse)]

    def parse(self, response):
        # print(response.body)
        # print(response.text)
        hxs = HtmlXPathSelector(response)
        item_list = hxs.xpath(‘//div[@id="content-list"]/div[@class="item"]‘)
        # 找到首页所有消息的连接、标题、作业信息然后yield给pipeline进行持久化
        for item in item_list:
            link = item.xpath(‘./div[@class="news-content"]/div[@class="part1"]/a/@href‘).extract_first()
            title = item.xpath(‘./div[@class="news-content"]/div[@class="part2"]/@share-title‘).extract_first()
            author = item.xpath(
                ‘./div[@class="news-content"]/div[@class="part2"]/a[@class="user-a"]/b/text()‘).extract_first()
            yield Day25SpiderItem(link=link, title=title, author=author)

# items.py

import scrapy


class Day25SpiderItem(scrapy.Item):
    link = scrapy.Field()
    title = scrapy.Field()
    author = scrapy.Field()

# pipelines.py


from scrapy.exceptions import DropItem

class Day25SpiderPipeline(object):
    def __init__(self, file_path,env):
        self.file_path = file_path  # 文件路径
        self.file_obj = None        # 文件对象：用于读写操作
        self.env = env              # 运行环境

    @classmethod
    def from_crawler(cls, crawler):
        """
        初始化时候，用于创建pipeline对象
        :param crawler:
        :return:
        """
        val = crawler.settings.get(‘STORAGE_CONFIG‘)
        env = crawler.settings.get(‘ENV‘)
        return cls(val,env)

    def process_item(self, item, spider):
        print(">>>> ", item)
        if ‘chouti‘ == spider.name:
            self.file_obj.write(item.get(‘link‘) + "\n" + item.get(‘title‘) + "\n" + item.get(‘author‘) + "\n\n")
        if ‘debug‘ == self.env:
            raise DropItem()
        else:
            return item

    def open_spider(self, spider):
        """
        爬虫开始执行时，调用
        :param spider:
        :return:
        """
        if ‘chouti‘ == spider.name:
            # 如果不加：encoding=‘utf-8‘ 会导致文件里中文乱码
            self.file_obj = open(self.file_path, mode=‘a+‘, encoding=‘utf-8‘)

    def close_spider(self, spider):
        """
        爬虫关闭时，被调用
        :param spider:
        :return:
        """
        if ‘chouti‘ == spider.name:
            self.file_obj.close()

class SecondPipeline(object):
    def __init__(self, file_path):
        self.file_path = file_path  # 文件路径
        self.file_obj = None        # 文件对象：用于读写操作

    @classmethod
    def from_crawler(cls, crawler):
        """
        初始化时候，用于创建pipeline对象
        :param crawler:
        :return:
        """
        val = crawler.settings.get(‘STORAGE_CONFIG2‘)
        return cls(val)

    def process_item(self, item, spider):
        print(">>>> ", item)
        if ‘chouti‘ == spider.name:
            self.file_obj.write(item.get(‘link‘) + "\n" + item.get(‘title‘) + "\n" + item.get(‘author‘) + "\n\n")
        return item

    def open_spider(self, spider):
        """
        爬虫开始执行时，调用
        :param spider:
        :return:
        """
        if ‘chouti‘ == spider.name:
            # 如果不加：encoding=‘utf-8‘ 会导致文件里中文乱码
            self.file_obj = open(self.file_path, mode=‘a+‘, encoding=‘utf-8‘)

    def close_spider(self, spider):
        """
        爬虫关闭时，被调用
        :param spider:
        :return:
        """
        if ‘chouti‘ == spider.name:
            self.file_obj.close()

# Scrapy默认的去重方法
# 在 from scrapy.dupefilter import RFPDupeFilter


from __future__ import print_function
import os
import logging

from scrapy.utils.job import job_dir
from scrapy.utils.request import request_fingerprint


class BaseDupeFilter(object):

    @classmethod
    def from_settings(cls, settings):
        return cls()

    def request_seen(self, request):
        return False

    def open(self):  # can return deferred
        pass

    def close(self, reason):  # can return a deferred
        pass

    def log(self, request, spider):  # log that a request has been filtered
        pass


class RFPDupeFilter(BaseDupeFilter):
    """Request Fingerprint duplicates filter"""

    def __init__(self, path=None, debug=False):
        self.file = None
        self.fingerprints = set()
        self.logdupes = True
        self.debug = debug
        self.logger = logging.getLogger(__name__)
        if path:
            self.file = open(os.path.join(path, ‘requests.seen‘), ‘a+‘)
            self.file.seek(0)
            self.fingerprints.update(x.rstrip() for x in self.file)

    @classmethod
    def from_settings(cls, settings):
        debug = settings.getbool(‘DUPEFILTER_DEBUG‘)
        return cls(job_dir(settings), debug)

    def request_seen(self, request):
        fp = self.request_fingerprint(request)
        if fp in self.fingerprints:
            return True
        self.fingerprints.add(fp)
        if self.file:
            self.file.write(fp + os.linesep)

    def request_fingerprint(self, request):
        return request_fingerprint(request)

    def close(self, reason):
        if self.file:
            self.file.close()

    def log(self, request, spider):
        if self.debug:
            msg = "Filtered duplicate request: %(request)s"
            self.logger.debug(msg, {‘request‘: request}, extra={‘spider‘: spider})
        elif self.logdupes:
            msg = ("Filtered duplicate request: %(request)s"
                   " - no more duplicates will be shown"
                   " (see DUPEFILTER_DEBUG to show all duplicates)")
            self.logger.debug(msg, {‘request‘: request}, extra={‘spider‘: spider})
            self.logdupes = False

        spider.crawler.stats.inc_value(‘dupefilter/filtered‘, spider=spider)

# request_fingerprint源码


"""
This module provides some useful functions for working with
scrapy.http.Request objects
"""

from __future__ import print_function
import hashlib
import weakref
from six.moves.urllib.parse import urlunparse

from w3lib.http import basic_auth_header
from scrapy.utils.python import to_bytes, to_native_str

from w3lib.url import canonicalize_url
from scrapy.utils.httpobj import urlparse_cached


_fingerprint_cache = weakref.WeakKeyDictionary()
def request_fingerprint(request, include_headers=None):
    """
    Return the request fingerprint.

    The request fingerprint is a hash that uniquely identifies the resource the
    request points to. For example, take the following two urls:

    http://www.example.com/query?id=111&cat=222
    http://www.example.com/query?cat=222&id=111

    Even though those are two different URLs both point to the same resource
    and are equivalent (ie. they should return the same response).

    Another example are cookies used to store session ids. Suppose the
    following page is only accesible to authenticated users:

    http://www.example.com/members/offers.html

    Lot of sites use a cookie to store the session id, which adds a random
    component to the HTTP Request and thus should be ignored when calculating
    the fingerprint.

    For this reason, request headers are ignored by default when calculating
    the fingeprint. If you want to include specific headers use the
    include_headers argument, which is a list of Request headers to include.

    """
    if include_headers:
        include_headers = tuple(to_bytes(h.lower())
                                 for h in sorted(include_headers))
    cache = _fingerprint_cache.setdefault(request, {})
    if include_headers not in cache:
        fp = hashlib.sha1()
        fp.update(to_bytes(request.method))
        fp.update(to_bytes(canonicalize_url(request.url)))
        fp.update(request.body or b‘‘)
        if include_headers:
            for hdr in include_headers:
                if hdr in request.headers:
                    fp.update(hdr)
                    for v in request.headers.getlist(hdr):
                        fp.update(v)
        cache[include_headers] = fp.hexdigest()
    return cache[include_headers]


def request_authenticate(request, username, password):
    """Autenticate the given request (in place) using the HTTP basic access
    authentication mechanism (RFC 2617) and the given username and password
    """
    request.headers[‘Authorization‘] = basic_auth_header(username, password)


def request_httprepr(request):
    """Return the raw HTTP representation (as bytes) of the given request.
    This is provided only for reference since it‘s not the actual stream of
    bytes that will be send when performing the request (that‘s controlled
    by Twisted).
    """
    parsed = urlparse_cached(request)
    path = urlunparse((‘‘, ‘‘, parsed.path or ‘/‘, parsed.params, parsed.query, ‘‘))
    s = to_bytes(request.method) + b" " + to_bytes(path) + b" HTTP/1.1\r\n"
    s += b"Host: " + to_bytes(parsed.hostname or b‘‘) + b"\r\n"
    if request.headers:
        s += request.headers.to_string() + b"\r\n"
    s += b"\r\n"
    s += request.body
    return s


def referer_str(request):
    """ Return Referer HTTP header suitable for logging. """
    referrer = request.headers.get(‘Referer‘)
    if referrer is None:
        return referrer
    return to_native_str(referrer, errors=‘replace‘)

from scrapy.utils.request import request_fingerprint
from scrapy.http import Request


obj = Request(url=‘http://www.baidu.com‘,callback=lambda x:x**2)
obj2 = Request(url=‘http://www.baidu.com?a=1&b=2‘,callback=lambda x:x**2)
obj3 = Request(url=‘http://www.baidu.com?b=2&a=1‘,callback=lambda x:x**2)
obj4 = Request(url=‘http://www.baidu.com?b=2&a=1‘,headers={‘Content-Type‘:‘application/text‘},callback=lambda x:x**2)
obj5 = Request(url=‘http://www.baidu.com?b=2&a=1‘,headers={‘Content-Type‘:‘application/text‘},callback=lambda x:x**2)
obj6 = Request(url=‘http://www.baidu.com?b=2&a=1‘,headers={‘Content-Type‘:‘application/json‘},callback=lambda x:x**2)
obj7 = Request(url=‘http://www.qq.com‘,callback=lambda x:x**2)

fingerprint = request_fingerprint(obj)
fingerprint2 = request_fingerprint(obj2)
fingerprint3 = request_fingerprint(obj3)
fingerprint4 = request_fingerprint(obj4)
fingerprint5 = request_fingerprint(obj5,include_headers=[‘Content-Type‘])
fingerprint6 = request_fingerprint(obj6,include_headers=[‘Content-Type‘])
fingerprint7 = request_fingerprint(obj7)

print(fingerprint)
print(fingerprint2)
print(fingerprint3)
print(fingerprint4)
print(fingerprint5)
print(fingerprint6)
print(fingerprint7)


###### 结果 ######
75d6587d87b3f4f3aa574b33dbd69ceeb9eafe7b
6b641a2d39886717fd794b75ef37ed73d06a716c
6b641a2d39886717fd794b75ef37ed73d06a716c
6b641a2d39886717fd794b75ef37ed73d06a716c
a00bfc032e3a3202184d6c20a5f5134900458fae
ad6b2ff37f5e73575d09159f5b4bea467d4c13e0
28853c538420894152718dadb9bdf109b333ccb5

# 第一步：创建去重文件


from scrapy.utils.request import request_fingerprint

class MyDupeFilter(object):

    def __init__(self):
        self.visited = set()

    @classmethod
    def from_settings(cls, settings):
        return cls()

    def request_seen(self, request):
        # print(request.url)
        # print(request.callback)

        fp = request_fingerprint(request)
        if fp in self.visited:
            return True  # 返回True 表示已经爬过了
        self.visited.add(fp)


    def open(self):  # can return deferred
        pass

    def close(self, reason):  # can return a deferred
        pass

    def log(self, request, spider):  # log that a request has been filtered
        pass




# 第二步：配置settings.py

DUPEFILTER_CLASS = ‘day25spider.my_filter.MyDupeFilter‘     # 自定义过滤插件

# 自动登录cookie并点赞

import json
import scrapy
from scrapy.http import Request
from scrapy.http.cookies import CookieJar
from scrapy.selector import HtmlXPathSelector
from ..items import Day25SpiderItem



class ChoutiSpider(scrapy.Spider):
    name = ‘chouti‘
    allowed_domains = [‘chouti.com‘]

    cookie_dict = {}

    def start_requests(self):
        url = ‘http://dig.chouti.com/‘
        yield Request(url=url,callback=self.index,dont_filter=True)
        # return [Request(url=url,callback=self.parse)]

    def index(self, response):
        """
        :param response: 会拿到抽屉首页所有相关的内容
        :return: 
        """
        # 第一步：GET请求，获取首页以及未授权的cookie信息
        cookie_jar = CookieJar()
        cookie_jar.extract_cookies(response,response.request)
        # print(cookie_jar)  # <scrapy.http.cookies.CookieJar object at 0x0000000004A32F98>
        # print(cookie_jar._cookies) # cookie的详细内容
        for k, v in cookie_jar._cookies.items():
            for i, j in v.items():
                for m, n in j.items():
                    self.cookie_dict[m] = n.value

        # 第二步：POST请求，发送用户名密码并携带未授权的cookie进行验证和授权
        req = Request(
            url=‘http://dig.chouti.com/login‘,
            method=‘POST‘,
            headers={‘Content-Type‘: ‘application/x-www-form-urlencoded; charset=UTF-8‘},
            body=‘phone=8615066668888&password=xxxxxxxxxx&oneMonth=1‘,
            cookies=self.cookie_dict,
            callback=self.check_login,
            dont_filter=True
        )
        yield req

    def check_login(self,response):
        # 第三步：登录成功后去请求首页
        print(response.text)
        response = json.loads(response.text)
        code = response[‘result‘][‘code‘]
        if ‘9999‘ == code:
            req = Request(
                url=‘http://dig.chouti.com/‘,
                method=‘GET‘,
                cookies=self.cookie_dict,
                callback=self.do_favor,
                dont_filter=True
            )
            yield req
        else:
            print("登陆失败")

    def do_favor(self,response):
        # print(response.url)
        # 第四步：获取首页所有的新闻id 并 执行点赞操作
        hxs = HtmlXPathSelector(response)
        news_list = hxs.select(‘//div[@id="content-list"]/div[@class="item"]‘)
        for new in news_list:
            link_id = new.xpath(‘*/div[@class="part2"]/@share-linkid‘).extract_first()
            req = Request(
                url=‘http://dig.chouti.com/link/vote?linksId=%s‘ % (link_id,),
                method=‘POST‘,
                cookies=self.cookie_dict,
                callback=self.do_favor_result,
                dont_filter=True
            )
            yield req

    def do_favor_result(self,response):
        # 第五步：输出点赞操作的结果
        print(response.text)

# ..commands/crawlall.py  -  即 获取爬虫列表并执行所有的爬虫

from scrapy.commands import ScrapyCommand
from scrapy.crawler import Crawler
from scrapy.crawler import CrawlerProcess
from scrapy.utils.project import get_project_settings

class Command(ScrapyCommand):

    requires_project = True

    def syntax(self):
        return ‘[options]‘

    def short_desc(self):
        return ‘Runs all of the spiders‘

    def run(self, args, opts):
        spider_list = self.crawler_process.spiders.list()
        for name in spider_list:
            self.crawler_process.crawl(name, **opts.__dict__)
        self.crawler_process.start()

# settings.py

COMMANDS_MODULE = ‘day25spider.commands‘

def run(self, args, opts):
    self.crawler_process.crawl(‘chouti‘, **opts.__dict__)
    self.crawler_process.start()
    

在 run() 里面打印下  print(type(self.crawler_process))  
可以得到： <class ‘scrapy.crawler.CrawlerProcess‘>

所以 self.crawler_process 就是 scrapy.crawler.CrawlerProcess 类的一个对象
然后执行顺序就是：
scrapy.crawler.CrawlerProcess.crawl()
    scrapy.crawler.Crawler.crawl()
scrapy.crawler.CrawlerProcess.start()

- 可迭代对象：具有 __iter__() 方法，并且执行后可以返回迭代器
- 迭代器：具有 __next__() 方法 并且逐一向后取值
- 生成器：函数中具有yield关键字
	- 具有 __iter__() 返回的还是自己本身
	- 具有 __next__()

DOWNLOADER_MIDDLEWARES = {
   # ‘day25spider.middlewares.MyCustomDownloaderMiddleware‘: 543,
   ‘day25spider.download_middlewares.DownMiddleware1‘: 543,
}

class DownMiddleware1(object):
    def process_request(self, request, spider):
        """
        请求需要被下载时，经过所有下载器中间件的process_request调用
        :param request: 
        :param spider: 
        :return:  
            None,继续后续中间件去下载；
            Response对象，停止process_request的执行，开始执行process_response
            Request对象，停止中间件的执行，将Request重新调度器
            raise IgnoreRequest异常，停止process_request的执行，开始执行process_exception
        """
        pass
        
    def process_response(self, request, response, spider):
        """
        spider处理完成，返回时调用
        :param response:
        :param result:
        :param spider:
        :return: 
            Response 对象：转交给其他中间件process_response
            Request 对象：停止中间件，request会被重新调度下载
            raise IgnoreRequest 异常：调用Request.errback
        """
        print(‘response1‘)
        return response

    def process_exception(self, request, exception, spider):
        """
        当下载处理器(download handler)或 process_request() (下载中间件)抛出异常
        :param response:
        :param exception:
        :param spider:
        :return: 
            None：继续交给后续中间件处理异常；
            Response对象：停止后续process_exception方法
            Request对象：停止中间件，request将会被重新调用下载
        """
        return None

# Proxy handling
def getproxies_environment():
    """Return a dictionary of scheme -> proxy server URL mappings.

    Scan the environment for variables named <scheme>_proxy;
    this seems to be the standard convention.  If you need a
    different way, you can pass a proxies dictionary to the
    [Fancy]URLopener constructor.

    """
    proxies = {}
    # in order to prefer lowercase variables, process environment in
    # two passes: first matches any, second pass matches lowercase only
    for name, value in os.environ.items():
        name = name.lower()
        if value and name[-6:] == ‘_proxy‘:
            proxies[name[:-6]] = value
    for name, value in os.environ.items():
        if name[-6:] == ‘_proxy‘:
            name = name.lower()
            if value:
                proxies[name[:-6]] = value
            else:
                proxies.pop(name[:-6], None)
    return proxies

# 爬虫文件

def start_requests(self):
    import os
    os.environ[‘http_proxy‘] = "http://root:password@192.168.11.11:9999/"
    os.environ[‘https_proxy‘] = "http://192.168.11.11:9999/"

import six
import base64
import random
# 用来兼容 Python2.x
def to_bytes(text, encoding=None, errors=‘strict‘):
    if isinstance(text, bytes):
        return text
    if not isinstance(text, six.string_types):
        raise TypeError(‘to_bytes must receive a unicode, str or bytes ‘
                        ‘object, got %s‘ % type(text).__name__)
    if encoding is None:
        encoding = ‘utf-8‘
    return text.encode(encoding, errors)

class ProxyMiddleware(object):
    def process_request(self, request, spider):
        PROXIES = [
            {‘ip_port‘: ‘111.11.228.75:80‘, ‘user_pass‘: ‘‘},
            {‘ip_port‘: ‘120.198.243.22:80‘, ‘user_pass‘: ‘‘},
            {‘ip_port‘: ‘111.8.60.9:8123‘, ‘user_pass‘: ‘‘},
            {‘ip_port‘: ‘101.71.27.120:80‘, ‘user_pass‘: ‘‘},
            {‘ip_port‘: ‘122.96.59.104:80‘, ‘user_pass‘: ‘‘},
            {‘ip_port‘: ‘122.224.249.122:8088‘, ‘user_pass‘: ‘‘},
        ]
        proxy = random.choice(PROXIES)
        if proxy[‘user_pass‘] is not None:
            request.meta[‘proxy‘] = to_bytes("http://%s" % proxy[‘ip_port‘])
            encoded_user_pass = base64.encodestring(to_bytes(proxy[‘user_pass‘]))
            request.headers[‘Proxy-Authorization‘] = to_bytes(‘Basic ‘ + encoded_user_pass)
            print("**************ProxyMiddleware have pass************" + proxy[‘ip_port‘])
        else:
            print("**************ProxyMiddleware no pass************" + proxy[‘ip_port‘])
            request.meta[‘proxy‘] = to_bytes("http://%s" % proxy[‘ip_port‘])

# scrapy.signals 这里面给预留了很多的锚点
# 可以按照自己写扩展插件在某个锚点触发一些自定义操作


"""
Scrapy signals

These signals are documented in docs/topics/signals.rst. Please don‘t add new
signals here without documenting them there.
"""

engine_started = object()
engine_stopped = object()
spider_opened = object()
spider_idle = object()
spider_closed = object()
spider_error = object()
request_scheduled = object()
request_dropped = object()
response_received = object()
response_downloaded = object()
item_scraped = object()
item_dropped = object()

# for backwards compatibility
stats_spider_opened = spider_opened
stats_spider_closing = spider_closed
stats_spider_closed = spider_closed

item_passed = item_scraped

request_received = request_scheduled

EXTENSIONS = {
    # ‘scrapy.extensions.telnet.TelnetConsole‘: None,
    ‘day25spider.extends.MyExtension‘:500,
}

from scrapy import signals

class MyExtension(object):
    def __init__(self, value):
        self.value = value

    @classmethod
    def from_crawler(cls, crawler):
        # val = crawler.settings.getint(‘EXTENDS_VAL‘)
        val = crawler.settings.get(‘EXTENDS_VAL‘)
        ext = cls(val)

        crawler.signals.connect(ext.spider_opened, signal=signals.spider_opened)
        crawler.signals.connect(ext.spider_closed, signal=signals.spider_closed)
        crawler.signals.connect(ext.engine_started, signal=signals.engine_started)
        crawler.signals.connect(ext.response_received, signal=signals.response_received)
        crawler.signals.connect(ext.response_downloaded, signal=signals.response_downloaded)

        print(ext.value)
        return ext

    def spider_opened(self, spider):
        print(‘spider_opened‘)

    def spider_closed(self, spider):
        print(‘spider_closed‘)

    def engine_started(self, spider):
        print(‘engine_started‘)

    def response_received(self, spider):
        print(‘response_received‘)

    def response_downloaded(self, spider):
        print(‘response_downloaded‘)