python文件管道 下载图集

# -*- coding: utf-8 -*-
import re
from time import sleep

import scrapy
from scrapy.linkextractors import LinkExtractor
from scrapy.spiders import CrawlSpider, Rule


class AngelSpider(CrawlSpider):
    name = ‘angel‘
    allowed_domains = [‘angelimg.spbeen.com‘]
    start_urls = [‘http://angelimg.spbeen.com/‘]

    base_url = "http://angelimg.spbeen.com"
    rules = (
        Rule(LinkExtractor(allow=r‘^http://angelimg.spbeen.com/ang/\d+$‘), callback=‘parse_item‘, follow=False),
    )

    def parse_item(self, response):
        print(response.url)
        item = response.meta.get(‘item‘,False)
        if item:
            pass
        else:
            item = {}
            item[‘files‘] = []
            item[‘file_urls‘] = []
            dir_name = response.xpath(‘.//div[@class="article"]/h2/text()‘).extract_first()
            item[‘dir_name‘] = dir_name.split(‘【‘)[0]
            item[‘dir_name‘] = re.sub(u"([^\u4e00-\u9fa5\u0030-\u0039\u0041-\u005a\u0061-\u007a])","", item[‘dir_name‘])

        img_url = response.xpath(‘.//div[@id="content"]/a/img/@src‘).extract_first()
        item[‘file_urls‘].append(img_url)
        # 如果有下一页 请求下一页，没有数据丢回管道
        next_url = response.xpath(‘.//div[@class="page"]//a[contains(@class,"next")]/@href‘).extract_first()

        #sleep(1)
        if next_url:
            next_url = self.base_url + next_url
            yield scrapy.Request(next_url,callback=self.parse_item,meta={‘item‘:item})
        else:
            yield item

# -*- coding: utf-8 -*-

# Define your item pipelines here
#
# Don‘t forget to add your pipeline to the ITEM_PIPELINES setting
# See: https://doc.scrapy.org/en/latest/topics/item-pipeline.html
import hashlib
import os

from scrapy.pipelines.files import FilesPipeline

class AngelimgPipeline(object):
    def process_item(self, item, spider):
        return item



from scrapy.http import Request
from scrapy.utils.python import to_bytes

class DealFilePathPipeline(FilesPipeline):
    def get_media_requests(self, item, info):
        return [Request(x,meta={‘item‘:item}) for x in item.get(self.files_urls_field, [])]


    def file_path(self, request, response=None, info=None):
        ## start of deprecation warning block (can be removed in the future)
        def _warn():
            from scrapy.exceptions import ScrapyDeprecationWarning
            import warnings
            warnings.warn(‘FilesPipeline.file_key(url) method is deprecated, please use ‘
                          ‘file_path(request, response=None, info=None) instead‘,
                          category=ScrapyDeprecationWarning, stacklevel=1)

        # check if called from file_key with url as first argument
        if not isinstance(request, Request):
            _warn()
            url = request
        else:
            url = request.url

        # detect if file_key() method has been overridden
        if not hasattr(self.file_key, ‘_base‘):
            _warn()
            return self.file_key(url)
        ## end of deprecation warning block
        item = request.meta.get(‘item‘,{})
        media_guid = hashlib.sha1(to_bytes(url)).hexdigest()  # change to request.url after deprecation
        media_ext = os.path.splitext(url)[1]  # change to request.url after deprecation
        print(item)
        return ‘full2/{}/{}{}‘.format(item[‘dir_name‘],media_guid, media_ext)
        return ‘full/%s%s‘ % (media_guid, media_ext)

    # deprecated
    def file_key(self, url):
        return self.file_path(url)

    file_key._base = True