用scrapy 爬虫框架读取统计局网站的行政区划（备忘记录）

# -*- coding: utf-8 -*-
import scrapy
from scrapy import Request
from ..items import *
from scrapy import  Request
from scrapy.http import  Request
class JgSpider(scrapy.Spider):
    name = ‘jgspider‘
    allowed_domains = [‘stats.gov.cn‘]
    start_urls = [‘http://www.stats.gov.cn/tjsj/tjbz/tjyqhdmhcxhfdm/2019/index.html‘]

    def parse(self, response):

        print(‘begin-----------‘)
        # 省级
        for node in response.css(‘.provincetr a‘):
            item = Class1_Item()
            item[‘name‘]= node.css(‘a::text‘).get()
            next_page=node.css(‘a::attr(href)‘).get()
            item[‘code‘]=next_page.split(‘.‘)[0]
            yield item

            if next_page:

                next_page = response.urljoin(next_page)
                yield scrapy.Request(next_page, callback=self.parse2)


    def parse2(self, response):
        # 市级
        for node in response.css(‘.citytr‘):
            item = Class1_Item()
            item[‘code‘] = node.css(‘a::text‘).getall()[0]
            item[‘name‘] = node.css(‘a::text‘).getall()[1]
            next_page=node.css(‘a::attr(href)‘)[0].get()

            yield item
            if next_page:
                next_page = response.urljoin(next_page)
                yield scrapy.Request(next_page, callback=self.parse3)

    def parse3(self, response):
        # 县级
        for node in response.css(‘tr.countytr‘):
            item = Class1_Item()
            if node.css(‘td::text‘).get() !=None:
                item[‘code‘] = node.css(‘td ::text‘).getall()[0]  # 要提取text，html tag必须探底
                item[‘name‘] = node.css(‘td ::text‘).getall()[1]
            else:
                item[‘code‘] = node.css(‘td a::text‘).getall()[0]#要提取text，html tag必须探底
                item[‘name‘] = node.css(‘td a::text‘).getall()[1]
            yield item

            next_page = node.css(‘a::attr(href)‘).get()
            if next_page:
                pass

import scrapy

def parse(response):
    # do something with the response
    pass

# making a request with a callback that we‘ve defined above
scrapy.Request(‘http://example.com‘, callback=parse)

class ExampleSpider(scrapy.Spider):

    def start_requests(self):
        # invoking initial request
        yield scrapy.Request(‘http://example.com‘, self.parse)

    def parse(self, response):
        # parsing response from the initial request
        # collecting links
        links = response.css(‘a.title::attr(href)‘).extract()
        for link in links:
            # make a request for each link we‘ve collected
            # the handler is different from the one we had
            # in initial request
            yield scrapy.Request(link, self.parse_page)

    def parse_page(self, response):
        # parsing response from each link
        title = response.css(‘h1.title::text‘).extract()
        content = response.css(‘div.content p‘).extract()

        # returning structured data for further processing
        yield {‘title‘: title, ‘content‘: content}

class SimpleSpider(scrapy.Spider):
    # those are the initial urls that you used to write
    # in a start_requests method. A request will be invoked
    # for each url in this list
    start_urls = [‘http://example.com‘]

    # this method will be called automatically for each
    # finished request from the start_urls.
    def parse(self, response):
        # you can either parse the response and return the data
        # or you can collect further urls and create additional
        # handlers for them, like we did with parse_page previously