Scrapy框架(八)--CrawlSpider

# -*- coding: utf-8 -*-
import scrapy
from scrapy.linkextractors import LinkExtractor
from scrapy.spiders import CrawlSpider, Rule
from Sun.items import SunItem,DetailItem


class SunSpider(CrawlSpider):
    name = ‘sun‘
    # allowed_domains = [‘www.xxx.com‘]
    start_urls = [‘http://wz.sun0769.com/political/index/politicsNewest?id=1&type=4‘]
    # 链接提取器：根据指定规则进行指定链接的提取
    link = LinkExtractor(allow=r‘id=1&page=\d+‘)
    link_detail = LinkExtractor(allow=r‘index\?id=\d+‘)
    rules = (
        # 规则解析器：将链接提取器提取到的链接发送请求 并根据callback进行指定的解析操作
        Rule(link, callback=‘parse_item‘, follow=False),
        # follow=True 可以将链接提取器继续作用到 链接提取器提取的链接所对应的页面
        # 通过此设置可对所有的页码进行爬取 调度器有去重过滤功能
        Rule(link_detail, callback=‘parse_detail‘, follow=False),
    )

    # 以下两个方法不可以进行请求传参
    # 两个方法都把数据存储到item中 可采用两个item
    def parse_item(self, response):
        li_list = response.xpath(‘/html/body/div[2]/div[3]/ul[2]/li‘)
        for li in li_list:
            new_num = li.xpath(‘./span[1]/text()‘).extract_first()
            title = li.xpath(‘./span[3]/a/text()‘).extract_first()
            item = SunItem()
            item[‘new_num‘] = new_num
            item[‘title‘] = title
            yield item

    def parse_detail(self, response):
        print(111)
        item = DetailItem()
        new_id = response.xpath(‘/html/body/div[3]/div[2]/div[2]/div[1]/span[4]/text()‘).extract_first()
        content = response.xpath(‘/html/body/div[3]/div[2]/div[2]/div[2]//text()‘).extract_first()
        print(new_id, content)
        item[‘new_id‘] = new_id
        item[‘content‘] = content
        yield item

class SunPipeline:

    def process_item(self, item, spider):

        if item.__class__.__name__ == ‘SunItem‘:
            new_num = item[‘new_num‘]
            title = item[‘title‘]
        else:
            print(22)
            new_id = item[‘new_id‘]
            content = item[‘content‘]
            print(new_id,content)
        return item