使用python scrapy框架抓取cnblog 的文章内容

# -*- coding: utf-8 -*-

from scrapy.spider import Spider
from scrapy.selector import Selector
from myblog.items import MyblogItem
import scrapy
import re
#SITE_URL = ‘http://www.cnblogs.com/threemore/‘


#抓取在cnblog中的文章
class CnblogSpider(Spider):
    #抓取名称 执行命令的时候后面的名称   scrapy crawl cnblog 中的cnblog 就是在这里定义的
    name =‘cnblog‘
    allow_domains = ["cnblogs.com"]
    
    
    #定义抓取的网址
    start_urls = [
        ‘http://www.cnblogs.com/threemore/‘
    ]
    
    #执行函数
    def parse(self,response):
        sel = Selector(response)
        self.log("begins  % s" % response.url)
        article_list = sel.css(‘div.postTitle‘).xpath(‘a‘)
        
        #抓取列表里面的内容也地址后循环抓取列表的内容页面数据
        for article in article_list:
            url = article.xpath(‘@href‘).extract()[0]
            self.log("list article url: % s" % url)
        
            #继续抓取内容页数据
            yield scrapy.Request(url,callback=self.parse_content)
            
        #如果有下一页继续抓取数据
        next_pages = sel.xpath(‘//*[@id="nav_next_page"]/a/@href‘)
        
        if next_pages :
            next_page = next_pages.extract()[0]
            #print next_page
            self.log("next_page: % s" % next_page)
            #自己调用自己  类似php 函数的当中的递归
            yield scrapy.Request(next_page,callback=self.parse)
        
 
            
    #内容页抓取
    def parse_content(self,response):
        self.log("detail views: % s" % response.url)
        
        #定义好的item  只需要在items 文件中定义抓取过来的数据对应的字段
        item = MyblogItem()
        
        #xpath 寻找需要在页面中抓取的数据
        item[‘link‘] =  response.url  
    
        #正则匹配出文章在cnblog中的id
        m = re.search(r"([0-9])+", item[‘link‘])
        if m:
            item[‘aid‘] = m.group(0)
        else:
            item[‘aid‘] = 0;
        item[‘title‘] = response.xpath(‘//*[@id="cb_post_title_url"]/text()‘).extract()[0]
        item[‘content‘] = response.xpath(‘//*[@id="cnblogs_post_body"]‘).extract()[0]
        item[‘date‘] = response.xpath(‘//*[@id="post-date"]‘).extract()
        #print item[‘content‘]
        yield item

# -*- coding: utf-8 -*-

# Define your item pipelines here
#
# Don‘t forget to add your pipeline to the ITEM_PIPELINES setting
# See: http://doc.scrapy.org/en/latest/topics/item-pipeline.html

#需要在setting.py文件中设置ITEM_PIPELINES 将前面的注释打开配置成当前的文件即可
#当前的管道就是这么配置 ‘myblog.pipelines.MyblogPipeline‘: 300,


import MySQLdb,datetime


DEBUG = True

#定义测试环境和正式环境中的mysql  
if DEBUG:
    dbuser = ‘root‘
    dbpass = ‘root‘
    dbname = ‘test‘
    dbhost = ‘127.0.0.1‘
    dbport = ‘3306‘
else:
    dbuser = ‘root‘
    dbpass = ‘root‘
    dbname = ‘test‘
    dbhost = ‘127.0.0.1‘
    dbport = ‘3306‘

class MyblogPipeline(object):
    
    #初始化 链接数据库
    def __init__(self):
        
        self.conn = MySQLdb.connect(user=dbuser, passwd=dbpass, db=dbname, host=dbhost, charset="utf8", use_unicode=True)
    
        self.cursor = self.conn.cursor()
    
        self.cursor.execute(‘truncate table test_cnbog‘)        
    
        self.conn.commit()
    
    
    
    #执行sql语句 
    def process_item(self, item, spider):
        try:
            self.cursor.execute("""INSERT INTO test_cnbog (title, link, aid,content,date)  
                  VALUES (%s,%s,%s,%s,%s)""", 
                  (
                    item[‘title‘].encode(‘utf-8‘), 
                    item[‘link‘].encode(‘utf-8‘),
                    item[‘aid‘],
                    item[‘content‘].encode(‘utf-8‘),
                    datetime.datetime.now(),
                  )
            )
            self.conn.commit()
        except MySQLdb.Error, e:
            
            print u‘Error %d: $s‘ % (e.args[0],e.args[1])
        
        return item

# Configure item pipelines
# See http://scrapy.readthedocs.org/en/latest/topics/item-pipeline.html
ITEM_PIPELINES = {
    ‘myblog.pipelines.MyblogPipeline‘: 300,
}