python爬虫系列：更上一层楼

demo/
    scrapy.cfg
    tutorial/
        __init__.py
        items.py
        pipelines.py
        settings.py
        spiders/
            __init__.py
            ...

#item.py
 import scrapy
 from scrapy.item import Item, Field
 
 class DoubanItem(scrapy.Item):
      # define the fields for your item here like:
      # name = scrapy.Field(
      title = scrapy.Field()
      url = scrapy.Field()
      rate= scrapy.Field()
      tag = scrapy.Field()

# coding:utf8
import scrapy
from douban.items import DoubanItem
from scrapy.selector import Selector
import re
from douban.pipelines import DoubanPipeline
import json
import urllib

import sys
reload(sys)
sys.setdefaultencoding(‘utf-8‘)
class DmozSpider(scrapy.Spider):
    name = "dmoz"
    allowed_domains = ["douban.com"]
    start_urls = [
        "https://movie.douban.com/j/search_subjects?type=movie&tag=热门&sort=recommend&page_limit=1000&page_start=0",
    ]
    
    def start_requests(self):       
        reqs = []
        tags = [u‘热门‘, u‘最新‘, u‘经典‘, u‘豆瓣高分‘, u‘冷门佳片‘, u‘华语‘, u‘欧美‘,
                u‘韩国‘, u‘日本‘, u‘动作‘, u‘喜剧‘, u‘爱情‘, u‘科幻‘, u‘悬疑‘, u‘恐怖‘, u‘文艺‘]
        
        for i in tags:
            url = ‘https://movie.douban.com/j/search_subjects?type=movie&tag=‘ + str(i) + ‘&sort=recommend&page_limit=1000&page_start=0‘
            req = scrapy.Request(url)
            reqs.append(req)

        return reqs

    def parse(self, response):
        html = response.body
        url = response.url
        # print u‘地址‘,url
        tag = re.findall(u‘tag=(.*?)&‘,url)[0]
        tag=urllib.unquote(tag)
        # print type(tag)

        dictt = json.loads(html)
        dd = dictt[‘subjects‘]
        items = []
        for a in dd:
            # self.get_tag(tag)
            pre_item = TutorialItem()
            pre_item[‘url‘] = a[‘url‘]
            pre_item[‘title‘] = a[‘title‘]
            pre_item[‘rate‘] = a[‘rate‘]
            pre_item[‘tag‘] = tag           
            items.append(pre_item)

        return items

# -*- coding: utf-8 -*-

# Define your item pipelines here
#
# Don‘t forget to add your pipeline to the ITEM_PIPELINES setting
# See: http://doc.scrapy.org/en/latest/topics/item-pipeline.html

import pymongo

class DoubanPipeline(object):

    def process_item(self, item, spider):
        db = spider.settings.get(‘db‘)
        dbb = pymongo.MongoClient(db)
        db = dbb[‘douban‘]
        # lis = (item[‘url‘],item[‘rate‘])
        db.info.insert(dict(item))
        # lis = (item[‘title‘], item[‘PORT‘], item[‘POSITION‘], item[
        #        ‘TYPE‘], item[‘SPEED‘], item[‘last_check_time‘])
        return item

1 MONGO_HOST = "127.0.0.1"  # 主机IP
2 MONGO_PORT = 27017  # 端口号
3 MONGO_DB = "Spider"  # 库名 
4 MONGO_COLL = "douban"  # collection名
5 # MONGO_USER = "Ryana"
6 # MONGO_PSW = "123456"