标签：proc dev settings github pipeline enter utf-8 apt-get values
pip install mysqlclient
sudo apt-get install libmysqlclient-devel
sudo apt-get install python-devel mysql-devel
class MysqlPipeline(object): def __init__(self): self.conn = pymysql.connect(‘192.168.1.1‘, ‘root‘, ‘123456‘, ‘titlespider‘, charset=‘utf-8‘, use_unicode=True) self.cursor = self.conn.cursor() def process_item(self, item, spider): insert_sql = """ insert into article(title, cteate_time, url, content) VALUES (%s, %s, %s, %s) """ self.cursor.execute(insert_sql, (item[‘title‘], item[‘cteate_time‘], item[‘url‘], item[‘content‘])) self.conn.commit() return item
# 首先在settings定义数据库关键字变量 MYSQL_HOST = ‘192.168.1.1‘ MYSQL_USER = ‘root‘ MYSQL_PASSWD = ‘123456‘ MYSQL_DB = ‘articlespider‘ # 然后编写Pipeline类 from twisted.enterprise import adbapi import pymysql import pymysql.cursors class MysqlTwistedPipeline(object): def __init__(self, dbpool): self.dbpool = dbpool @classmethod def from_settings(cls, settings): conn_dict = dict( host = settings[‘MYSQL_HOST‘], user = settings[‘MYSQL_USER‘], passwd = settings[‘MYSQL_PASSWD‘], database = settings[‘MYSQL_DB‘], charset = ‘utf-8‘, cursorclass = pymysql.cursors.DictCursor, use_unicode = True ) dbpool = adbapi.ConnectionPool(‘PyMySQL‘, **conn_dict) return cls(dbpool) def process_item(self, item, spider): query = self.dbpool.runInteraction(self.do_insert, item) query.addErrorback(self.handle_error) return item def do_insert(self, item): insert_sql = """ insert into article(title, cteate_time, url, content) VALUES (%s, %s, %s, %s) """ self.cursor.execute(insert_sql, (item[‘title‘], item[‘cteate_time‘], item[‘url‘], item[‘content‘])) self.conn.commit() def handle_error(self, failure): print(failure)
https://github.com/scrapy-plugins/scrapy-djangoitem
标签：proc dev settings github pipeline enter utf-8 apt-get values
原文地址：https://www.cnblogs.com/cq146637/p/9053387.html
