码迷,mamicode.com
首页 > 其他好文 > 详细

2017-02-08 01:19:09 Scrapy: pk5_mylist

时间:2017-02-25 13:52:40      阅读:193      评论:0      收藏:0      [点我收藏+]

标签:else   box   blog   spider   extra   cti   gen   crawl   href   

[TOC]
###2017-02-08 01:19:09 Scrapy: pk5_mylist.md
> Save the records with MongoDB
#### settings.py
```
# -*- coding: utf-8 -*-
BOT_NAME = ‘bengbeng‘
SPIDER_MODULES = [‘bengbeng.spiders‘]
NEWSPIDER_MODULE = ‘bengbeng.spiders‘
ROBOTSTXT_OBEY = True
ITEM_PIPELINES = {
    ‘bengbeng.pipelines.BengbengPipeline‘: 300,
}
DOWNLOADER_MIDDLEWARES = {  
    ‘scrapy.downloadermiddlewares.useragent.UserAgentMiddleware‘:None,  
    ‘bengbeng.middlewares.RotateUserAgentMiddleware‘:400,  
}  
DOWNLOAD_DELAY = 0.2
COOKIES_ENABLED = True
MONGODB_HOST = ‘127.0.0.1‘   
MONGODB_PORT = 27017
MONGODB_DBNAME = ‘bengbeng‘ 
MONGODB_COLLECTION = ‘scrapy_pk5_mylist‘
```
#### items.py
```
# -*- coding: utf-8 -*-

# Define here the models for your scraped items
#
# See documentation in:
# http://doc.scrapy.org/en/latest/topics/items.html

import scrapy

class BengbengItem(scrapy.Item):
    # define the fields for your item here like:
    # name = scrapy.Field()
    # pm = pk5_mylist
    pm_betid = scrapy.Field()
    pm_result = scrapy.Field()
    pm_result_rate = scrapy.Field()
    pm_rate_bao = scrapy.Field() 
    pm_rate_dui = scrapy.Field()
    pm_rate_shun = scrapy.Field()
    pm_rate_ban = scrapy.Field()
    pm_rate_za = scrapy.Field()

```
#### pipelines.py
```
# -*- coding: utf-8 -*-

# Define your item pipelines here
#
# Don‘t forget to add your pipeline to the ITEM_PIPELINES setting
# See: http://doc.scrapy.org/en/latest/topics/item-pipeline.html


from scrapy.conf import settings
import pymongo

class BengbengPipeline(object):
    def __init__(self):
        bengbengHost = settings[‘MONGODB_HOST‘]
        bengbengPort = settings[‘MONGODB_PORT‘]
        bengbengdbName = settings[‘MONGODB_DBNAME‘]
        bengbengClient = pymongo.MongoClient(host=bengbengHost, port=bengbengPort)
        bengbengtdb = bengbengClient[bengbengdbName]
        self.postbengbeng = bengbengtdb[settings[‘MONGODB_COLLECTION‘]]
    def process_item(self, bengbengitem, spider):        
        # self.postbengbeng.insert({‘bengbeng_xflink‘:bengbengitem[‘bengbeng_xflink‘]})
        if bengbengitem[‘pm_betid‘] != ‘‘:
            bengbeng_pk5_result = dict(bengbengitem)
            # print(item[‘mm6xx_title‘])
            # try to ignore the same URLs before insert into DB .
            if self.postbengbeng.find({‘pm_betid‘:bengbengitem[‘pm_betid‘]}).count() == 0:
                print(‘{0}‘.format(bengbeng_pk5_result))
                self.postbengbeng.insert(bengbeng_pk5_result)
            else:
                # if data was found , then update 
                # bengbeng_pk5_result.pop(‘_id‘)
                print("Record was Found ,Update!".center(40,‘*‘))
                result = self.postbengbeng.find_one({‘pm_betid‘:bengbengitem[‘pm_betid‘]},{‘pm_result‘:1})
                bengbengitem[‘pm_result‘] = result[‘pm_result‘]
                # get the pm_result_rate
                bengbengitem[‘pm_result_rate‘] = bengbengitem[‘pm_rate_{0}‘.format(result[‘pm_result‘])]
                bengbeng_pk5_result = dict(bengbengitem)
                # Update : remove the old records , Insert the new one
                self.postbengbeng.remove({‘pm_betid‘:bengbengitem[‘pm_betid‘]})
                print(‘Update: {0}‘.format(bengbeng_pk5_result))
                self.postbengbeng.insert(bengbeng_pk5_result)
                
            return bengbengitem
```
#### pk5_mylist.py
```
# -*- coding: utf-8 -*-
import scrapy
from scrapy.linkextractors import LinkExtractor
from scrapy.spiders import CrawlSpider, Rule
from scrapy.http.request import Request
from scrapy.http import Request
from scrapy import Spider
from scrapy.selector import Selector
from bengbeng.items import BengbengItem
import re
import time
# try to list xpath
xpath_betid = ‘//*[@id="pkBox"]/ul/*[@class="jl1"]/text()‘
xpath_result = ‘//*[@id="pkBox"]/ul/*[@class="jl3"]/img/@src‘
xpath_all_rate = ‘//*[@id="pkBox"]/ul/*[@class="ck2"]/text()‘
xpath_rate_betid = ‘//*[@id="pkBox"]/div[4]/span[1]/font/text()‘

class Pk5MylistSpider(scrapy.Spider):
    name = "pk5_mylist"
    allowed_domains = ["bengbeng.com"]

    start_urls = [‘http://www.bengbeng.com/pk5mylist.php?p=1‘]
    my_cookies = {‘PHPSESSID‘: ‘0gl0onrnocb149v3jcmhpdk3k0‘}
    pk5_mylist_item = BengbengItem()
    
    for i in range(2,33): # 2,31
        start_urls.append(‘http://www.bengbeng.com/pk5mylist.php?p={0}‘.format(i))
    
    def goForSecondPage(self, response):
        all_rate = Selector(response).xpath(xpath_all_rate).extract()
        rate_betid = Selector(response).xpath(xpath_rate_betid).extract()
        
        #self.pk5_mylist_item = BengbengItem()
        self.pk5_mylist_item[‘pm_betid‘] = rate_betid[0]
        self.pk5_mylist_item[‘pm_rate_bao‘] = all_rate[0]
        self.pk5_mylist_item[‘pm_rate_dui‘] = all_rate[1]
        self.pk5_mylist_item[‘pm_rate_shun‘] = all_rate[2]
        self.pk5_mylist_item[‘pm_rate_ban‘] = all_rate[3]
        self.pk5_mylist_item[‘pm_rate_za‘] = all_rate[4]
        yield self.pk5_mylist_item
        #print(self.pk5_mylist_item)
    # Reload spiders with new cookies
    def start_requests(self):  
        for url in self.start_urls:          
            yield Request(url, cookies=self.my_cookies)  
    def parse(self, response):
        all_betid = Selector(response).xpath(xpath_betid).extract()
        all_result = Selector(response).xpath(xpath_result).extract()

        #self.pk5_mylist_item = BengbengItem()
        dict_result = {
            ‘bz‘:‘bao‘, # Bao
            ‘dz‘:‘dui‘, # Dui
            ‘sz‘:‘shun‘, # Shun
            ‘bs‘:‘ban‘, # Ban
            ‘zl‘:‘za‘, # Za
            }
        result_search = re.compile(‘_(\D+)\.‘)
        if len(all_result) == 29:
            j = 1
            for i in range(29):
                betid = all_betid[j]
                result = dict_result[result_search.findall(all_result[i])[0]]
                self.pk5_mylist_item[‘pm_betid‘] = betid
                self.pk5_mylist_item[‘pm_result‘] = result
                yield self.pk5_mylist_item
                url = ‘http://www.bengbeng.com/pk5mdetail.php?no=‘ + betid
                yield Request(url,cookies=self.my_cookies,callback=self.goForSecondPage)
                j += 1
                #time.sleep(0.6)
        else:
            for i in range(30):
                betid = all_betid[i]
                result = dict_result[result_search.findall(all_result[i])[0]]
                self.pk5_mylist_item[‘pm_betid‘] = betid
                self.pk5_mylist_item[‘pm_result‘] = result
                yield self.pk5_mylist_item
                url = ‘http://www.bengbeng.com/pk5mdetail.php?no=‘ + betid
                yield Request(url,cookies=self.my_cookies,callback=self.goForSecondPage)
                #time.sleep(0.6)
        
```
#### Run:
技术分享
#### Records in DB
```
{
    "_id" : ObjectId("5899fac96f59de2268c28918"),
    "pm_rate_ban" : "2.7767",
    "pm_rate_dui" : "3.7002",
    "pm_rate_bao" : "100.8169",
    "pm_rate_za" : "3.3057",
    "pm_rate_shun" : "16.6743",
    "pm_betid" : "463645",
    "pm_result" : "ban",
    "pm_result_rate" : "2.7767"
}

/* 2 */
{
    "_id" : ObjectId("5899fac86f59de2268c28917"),
    "pm_rate_ban" : "2.7797",
    "pm_rate_dui" : "3.7823",
    "pm_rate_bao" : "113.3838",
    "pm_rate_za" : "3.3606",
    "pm_rate_shun" : "13.7679",
    "pm_betid" : "463644",
    "pm_result" : "shun",
    "pm_result_rate" : "13.7679"
}
```




附件列表

     

    2017-02-08 01:19:09 Scrapy: pk5_mylist

    标签:else   box   blog   spider   extra   cti   gen   crawl   href   

    原文地址:http://www.cnblogs.com/topshooter/p/6441464.html

    (0)
    (0)
       
    举报
    评论 一句话评论(0
    登录后才能评论!
    © 2014 mamicode.com 版权所有  联系我们:gaon5@hotmail.com
    迷上了代码!