scrapy连接MySQL

# 启动管道组件
ITEM_PIPELINES = {
    ‘QianChengWuYu.mongoDBPiplelines.MongoDBPipline‘: 300,
}

# mysql的连接参数
MYSQL_DB_NAME = ‘qianchengwuyu‘
MYSQL_HOST = ‘localhost‘
MYSQL_USER = ‘‘
MYSQL_PASSWORD = ‘‘

import pymysql

"""
这个类用于scrapy中数据向MYSQL数据库中插入数据
"""
class QianchengwuyuPipeline(object):

    def open_spider(self, spider):
        """
        该方法用于创建数据库连接池对象并连接数据库
        """
        db = spider.settings.get(‘MYSQL_DB_NAME‘, ‘scrapy_default‘)
        host = spider.settings.get(‘MYSQL_HOST‘, ‘localhost‘)
        port  = spider.settings.get(‘MYSQL_PORT‘, 3306)
        user = spider.settings.get(‘MYSQL_USER‘, ‘root‘)
        passwd = spider.settings.get(‘MYSQL_PASSWORD‘, ‘root‘)

        self.db_conn  = pymysql.connect(host=host, port=port, db=db, user=user, passwd=passwd, charset=‘utf8‘)
        self.db_cur = self.db_conn.cursor()
        
    def close_spider(self, spider):
        """
        该方法用于数据插入以及关闭数据库
        """
        self.db_conn.commit()
        self.db_conn.close()

    def process_item(self, item, spider):
        self.insert_db(item)

        return item

    def insert_db(self, item):
        """
        sql语句构造方法
        """
        values =(
            item[‘job‘],
            item[‘company‘],
            item[‘place‘],
            item[‘salary‘]
        )

        sql = ‘INSERT INTO jobs(job,company,place,salary) VALUES(%s,%s,%s,%s)‘
        self.db_cur.execute(sql, values)

"""
这个类用于scrapy中数据向MYSQL数据库中异步插入数据
"""
from twisted.enterprise import adbapi

class MySQLsyncPipeline(object):

    def open_spider(self, spider):
        """
        该方法用于创建数据库连接池对象并连接数据库
        """
        db = spider.settings.get(‘MYSQL_DB_NAME‘, ‘scrapy_default‘)
        host = spider.settings.get(‘MYSQL_HOST‘, ‘localhost‘)
        port = spider.settings.get(‘MYSQL_PORT‘, 3306)
        user = spider.settings.get(‘MYSQL_USER‘, ‘root‘)
        passwd = spider.settings.get(‘MYSQL_PASSWORD‘, ‘root‘)

        self.dbpool = adbapi.ConnectionPool(‘pymysql‘, host=host, db=db, user=user, password=passwd, charset=‘utf8‘)

    def close_spider(self, spider):
        """
        该方法用于数据插入完成后关闭数据库
        """
        self.dbpool.close()

    def process_item(self, item, spider):
        """
        插入数据库方法
        """
        self.dbpool.runInteraction(self.insert_db, item)

        return item

    def insert_db(self, tx, item):
        """
        sql语句构造方法
        """
        values = (
            item[‘job‘],
            item[‘company‘],
            item[‘place‘],
            item[‘salary‘]
        )

        sql = ‘INSERT INTO jobs(job, company, place,salary) VALUES(%s,%s,%s,%s)‘
        tx.execute(sql, values)