python爬虫之Scrapy框架

# -*- coding: utf-8 -*-
import scrapy
from scrapy.http.response.html import HtmlResponse

class BookSpider(scrapy.Spider):
    name = ‘book‘  # 爬虫名
    allowed_domains = [‘douban.com‘]  # 域名。爬虫爬取范围
    start_urls = [‘https://book.douban.com/tag/%E7%BC%96%E7%A8%8B?start=0&type=T‘]  # 起始url，从第一页开始爬取

    # 下载器获取WebServer的response，parse就是解析响应response的内容
    def parse(self, response: HtmlResponse):  # 如何解析html;返回一个可迭代对象：利用yiled
        print(type(response))  # scrapy.http.response.html.HtmlResponse
        print(type(response.text))  # str
        print(type(response.body))  # bytes
        print(response.encoding)  # utf-8
        # 将网页内容写入book.html文件内
        with open(‘/Users/dannihong/Documents/leetcode/scrapy_project/file/book.html‘, ‘w‘, encoding=‘utf-8‘) as f:
            f.write(response.text)
            f.flush()
        except Exception as e:
            print(e)

# -*- coding: utf-8 -*-
from scrapy.http.response.html import HtmlResponse

response = HtmlResponse(‘file:/Users/dannihong/Documents/leetcode/scrapy_project/file/book.html‘, encoding=‘utf-8‘)
with open(‘/Users/dannihong/Documents/leetcode/scrapy_project/file/book.html‘, encoding=‘utf8‘) as f:
    response._set_body(f.read().encode())  # _set_body方法将其放入response对象里；需要传入的参数对象是bytes，所以encode()
subjects = response.css(‘li.subject-item‘)
for subject in subjects:
    # 提取书籍的网页链接
    href = subject.xpath(‘.//h2‘).css(‘a::attr(href)‘).extract()
    print(‘href:‘, href[0])
    # 使用正则表达式，选取评分是9分以上的书籍
    rate = subject.xpath(‘.//span[@class="rating_nums"]/text()‘).re(r‘^9.*‘)
    # rate = subject.css(‘span.rating_nums::text‘).re(r‘^9\..*‘)  # 第二种表达方式
    if rate:
        print(rate[0])

import scrapy

class Test1ProItem(scrapy.Item):
    # define the fields for your item here like:
    # name = scrapy.Field()
    title = scrapy.Field()  # 存放书籍标题的字段
    rate = scrapy.Field()  # 存放书籍评分的字段

# -*- coding: utf-8 -*-
import scrapy
from scrapy.http.response.html import HtmlResponse
from ..items import FirstItem  # 从上一层的items.py文件里导入

class BookSpider(scrapy.Spider):
    name = ‘book‘  # 爬虫名
    allowed_domains = [‘douban.com‘]  # 爬虫爬取范围
    start_urls = [‘https://book.douban.com/tag/%E7%BC%96%E7%A8%8B?start=0&type=T‘]  # 起始url
    custom_settings = {‘file_name‘: ‘/Users/dannihong/Documents/leetcode/scrapy_project/file/books.json‘}  # 一般设置参数
 
    # 下载器获取WebServer的response，parse解析响应的内容；输出items和requests
    def parse(self, response: HtmlResponse):  # 如何解析html;返回一个可迭代对象：利用yiled
        subjects = response.xpath(‘//li[@class="subject-item"]‘)
        items = []  # 如果用items=[]，最后函数要return items
        for subject in subjects:
            item = FirstItem()  # 声明一个item，相当于一个字典，存放要爬取的数据

            title = subject.xpath(‘.//h2/a/text()‘).extract()
            item[‘title‘] = title[0].strip()
            rate = subject.css(‘span.rating_nums::text‘).extract()
            item[‘rate‘] = rate[0].strip()

            items.append(item)

        with open(‘book.json‘, ‘w‘, encoding=‘utf8‘) as f:
            for item in items:
                f.write(‘{} {}\n‘.format(item[‘title‘], item[‘rate‘]))

        return items