?scrapy--json(360美图)

# -*- coding: utf-8 -*-
import scrapy
from Tupian360.items import Tupian360Item
import json
import pdb
from Tupian360.settings import USER_AGENT
import random

class ImageSpider(scrapy.Spider):
    name = ‘image‘
    allowed_domains = [‘image.so.com‘]
    pager_count = 0

    headers = {
        ‘Accept‘: ‘application/json, text/javascript, */*; q=0.01‘,
        ‘Accept-Encoding‘: ‘gzip, deflate‘,
        ‘Accept-Language‘: ‘zh-CN,zh;q=0.9‘,
        ‘Connection‘: ‘keep-alive‘,
        ‘Content-Length‘: ‘11‘,
        ‘Content-Type‘: ‘application/x-www-form-urlencoded; charset=UTF-8‘,
        ‘Host‘: ‘image.so.com‘,
        ‘Origin‘: ‘image.so.com‘,
        ‘Referer‘: ‘http://image.so.com/zj?ch=beauty&sn=%s&listtype=new&temp=1‘%pager_count,
        ‘User-Agent‘: USER_AGENT,
        ‘X-Requested-With‘: ‘XMLHttpRequest‘,
    }

    old_urls   = ‘http://image.so.com/zj?ch=beauty&sn=%s&listtype=new&temp=1‘
    start_urls = [old_urls%pager_count]

    def parse(self, response):
        tupian = Tupian360Item()
        sel = json.loads(response.body.decode(‘utf8‘))
        counts = sel[‘count‘]
        self.pager_count += int(counts)
        new_url = self.old_urls%self.pager_count
        for link in sel[‘list‘]:
            tupian[‘image_urls‘] = link[‘qhimg_url‘]
            yield tupian

        yield scrapy.Request(new_url,callback=self.parse)