scrapy一览

 1 import datetime
 2 import re
 3 
 4 import scrapy
 5 from scrapy.loader import ItemLoader
 6 from scrapy.loader.processors import Join, MapCompose, TakeFirst
 7 
 8 
 9 def add_title(value):
10     """字段之后加上签名"""
11     return value + ‘Pontoon‘
12 
13 
14 class ArticleItemLoader(ItemLoader):
15     """重写ItemLoader，从列表中提取第一个字段"""
16     default_output_processor = TakeFirst()
17 
18 
19 class ChouTiArticleItem(scrapy.Item):
20     """初始化items"""
21     title = scrapy.Field(
22         input_processor=MapCompose(add_title)
23     )
24     article_url = scrapy.Field()
25     article_url_id = scrapy.Field()
26     font_img_url = scrapy.Field()
27     author = scrapy.Field()
28     sign_up = scrapy.Field()
29 
30     def get_insert_sql(self):
31         insert_sql = ‘‘‘
32                         insert into chouti(title, article_url, article_url_id, font_img_url, author, sign_up) 
33                         VALUES (%s, %s, %s, %s, %s, %s)
34         ‘‘‘
35         params = (self["title"], self["article_url"], self["article_url_id"], self["font_img_url"],
36                   self["author"], self["sign_up"],)
37         return insert_sql, params

 1 # -*- coding: utf-8 -*-
 2 import scrapy
 3 from scrapy.http import Request
 4 
 5 from ..items import ChouTiArticleItem, ArticleItemLoader
 6 from ..utils.common import get_md5
 7 
 8 
 9 class ChoutiSpider(scrapy.Spider):
10     name = ‘chouti‘
11     allowed_domains = [‘dig.chouti.com/‘]
12     start_urls = [‘https://dig.chouti.com/‘]
13     header = {
14         "User-Agent": "Mozilla/5.0 (Windows NT 6.3; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) "
15                       "Chrome/71.0.3578.98 Safari/537.36"
16     }
17 
18     def parse(self, response):
19         article_list = response.xpath("//div[@class=‘item‘]")
20         for article in article_list:
21             item_load = ArticleItemLoader(item=ChouTiArticleItem(), selector=article)   # 注意这里的返回值并不是response了
22             article_url = article.xpath(".//a[@class=‘show-content color-chag‘]/@href").extract_first("")
23             item_load.add_xpath("title", ".//a[@class=‘show-content color-chag‘]/text()")
24             item_load.add_value("article_url", article_url)
25             item_load.add_value("article_url_id", get_md5(article_url))
26             item_load.add_xpath("font_img_url", ".//div[@class=‘news-pic‘]/img/@src")
27             item_load.add_xpath("author", ".//a[@class=‘user-a‘]//b/text()")
28             item_load.add_xpath("sign_up", ".//a[@class=‘digg-a‘]//b/text()")
29             article_item = item_load.load_item()  # 解析上述定义的字段
30             yield article_item

 1 import MySQLdb
 2 import MySQLdb.cursors
 3 from twisted.enterprise import adbapi   # 将入库变成异步操作
 4 
 5 
 6 class MysqlTwistedPipleline(object):
 7     """抽屉Pipleline"""
 8     def __init__(self, db_pool):
 9         self.db_pool = db_pool
10 
11     @classmethod
12     def from_settings(cls, settings):
13         """内置的方法自动调用settings"""
14         db_params = dict(
15             host=settings["MYSQL_HOST"],
16             db=settings["MYSQL_DBNAME"],
17             user=settings["MYSQL_USER"],
18             password=settings["MYSQL_PASSWORD"],
19             charset="utf8",
20             cursorclass=MySQLdb.cursors.DictCursor,
21             use_unicode=True,
22         )
23         db_pool = adbapi.ConnectionPool("MySQLdb", **db_params)
24 
25         return cls(db_pool)
26 
27     def process_item(self, item, spider):
28         """使用twisted异步插入数据值数据库"""
29         query = self.db_pool.runInteraction(self.do_insert, item)    # runInteraction() 执行异步操作的函数
30         query.addErrback(self.handle_error, item, spider)     # addErrback() 异步处理异常的函数
31 
32     def handle_error(self, failure, item, spider):
33         """自定义处理异步插入数据的异常"""
34         print(failure)
35 
36     def do_insert(self, cursor, item):
37         """自定义执行具体的插入"""
38         insert_sql, params = item.get_insert_sql()
39         # chouti插入数据
40         cursor.execute(insert_sql, (item["title"], item["article_url"], item["article_url_id"],
41                                     item["font_img_url"], item["author"], item["sign_up"]))

 1 ITEM_PIPELINES = {
 2     ‘ChoutiSpider.pipelines.MysqlTwistedPipleline‘: 3,
 3 
 4 }
 5 ...
 6 
 7 MYSQL_HOST = "127.0.0.1"
 8 MYSQL_DBNAME = "article_spider"    # 数据库名称
 9 MYSQL_USER = "xxxxxx"
10 MYSQL_PASSWORD = "xxxxx"

 1 from scrapy.exceptions import DropItem
 2 
 3 class CustomPipeline(object):
 4     def __init__(self,v):
 5         self.value = v
 6 
 7     def process_item(self, item, spider):
 8         # 操作并进行持久化
 9 
10         # return表示会被后续的pipeline继续处理
11         return item
12         
13         # raise DropItem()  表示将item丢弃，不会被后续pipeline处理
14 
15     @classmethod
16     def from_crawler(cls, crawler):
17         """
18         初始化时候，用于创建pipeline对象
19         :param crawler: 
20         :return: 
21         """
22         val = crawler.settings.getint(‘XXXXX‘)
23         return cls(val)     # 注意这种创建对象的方式，在scrapy源码中会大量遇见
24 
25     def open_spider(self,spider):
26         """
27         爬虫开始执行时，调用
28         :param spider: 
29         :return: 
30         """
31         print(‘spider start‘)
32 
33     def close_spider(self,spider):
34         """
35         爬虫关闭时，被调用
36         :param spider: 
37         :return: 
38         """
39         print(‘spider close‘)