import scrapy
from scrapy.linkextractors import LinkExtractor
from scrapy.spiders import CrawlSpider, Rule


class CrawlModelSpider(CrawlSpider):
    name = ‘crawl_model‘
    # allowed_domains = [‘www.xxx.com‘]
    start_urls = [‘https://www.4567kan.com/index.php/vod/show/class/动作/id/1.html‘]

    # LinkExtractor叫做链接提取器
    # 作用：根据指定的规则（allow=‘正则‘）在页面中进行相关链接（url）的提取
    link = LinkExtractor(allow=r‘/id/1/page/\d+\.html‘)
    rules = (
        # Rule叫做规则解析器，链接提取器是被作用在规则解析器
        # 作用：可以接受link提取到的链接，且对链接进行请求发送，根据指定规则（callback）对请求到的数据进行数据解析
        # follow=True:可以将 链接提取器 继续作用到 链接提取器 提取到的页码 所对应的页面源码中
        Rule(link, callback=‘parse_item‘, follow=True),
    )

    def parse_item(self, response):
        print(response)

import scrapy
from scrapy.linkextractors import LinkExtractor
from scrapy.spiders import CrawlSpider, Rule
import scrapy
from moviePro.items import MovieproItem


class MovieSpider(CrawlSpider):
    name = ‘movie‘
    # allowed_domains = [‘www.xxx.com‘]
    start_urls = [‘https://www.4567kan.com/index.php/vod/show/class/动作/id/1/page/1.html‘]

    rules = (
        # 提取页码链接
        Rule(LinkExtractor(allow=r‘/id/1/page/\d+\.html‘), callback=‘parse_item‘, follow=False),
        # 提取详情页链接
        # Rule(LinkExtractor(allow=r‘/movie/index\d+\.html‘),callback=‘parse_detail‘,follow=False)
    )

    def parse_item(self, response):
        # 解析电影的名称
        li_list = response.xpath(‘/html/body/div[1]/div/div/div/div[2]/ul/li‘)
        for li in li_list:
            title = li.xpath(‘./div/a/@title‘).extract_first()
            detail_url = ‘https://www.4567kan.com‘ + li.xpath(‘./div/a/@href‘).extract_first()
            item = MovieproItem()
            item[‘title‘] = title
            yield scrapy.Request(url=detail_url, callback=self.parse_detail, meta={‘item‘: item})
   def parse_detail(self, response):
        desc = response.xpath(‘/html/body/div[1]/div/div/div/div[2]/p[5]/span[2]/text()‘).extract_first()
        item = response.meta[‘item‘]
        item[‘desc‘] = desc
        yield item