Spider实例详解

  1 # -*- coding: utf-8 -*-
  2 
  3 import scrapy
  4 from scrapy import Spider, Request, log
  5 from scrapy.selector import Selector
  6 import sys
  7 sys.path.append(‘/home/administrator/Scrapy_Develop/tkdata‘)
  8 from tkspider import TKdataSpider
  9 from tkitems import TotalItem, InvestItem, TenderItem, RepaymentItem
 10 from dict import *
 11 from method import *
 12 
 13 class JJJCSpider(TKdataSpider):
 14     name = ‘jiajiajucai‘
 15     allowed_domains = [‘jiajiajucai.com‘]
 16     start_urls = [‘http://www.jiajiajucai.com‘]
 17 
 18     # 标的列表页数
 19     page = 0
 20     # 标的列表url
 21     list_url = ‘http://www.jiajiajucai.com/invest/index.html?p=%d‘
 22 
 23     # 解析函数
 24     def parse(self, response):
 25         # 解析平台总数据
 26         item = self.parse_total(response)
 27 
 28         # 爬取标的列表
 29         yield Request(self.list_url % self.page, callback=self.parse_list)
 30 
 31 
 32     # 解析平台总数据
 33     def parse_total(self, response):
 34         # 解析数据
 35         sel = Selector(response)
 36         item = TotalItem()
 37         item = self.total(sel, xpaths_total)
 38 
 39         # 处理解析到的数据
 40         item[‘total‘] = total(item[‘total‘])
 41         item[‘dueln‘] = dueln(item[‘dueln‘])
 42         item[‘lncome‘] = lncome(item[‘lncome‘])
 43         item[‘rate‘] = rate(item[‘rate‘])
 44         item[‘person‘] = person(item[‘person‘])
 45         return item
 46     
 47 
 48     # 解析标的列表
 49     def parse_list(self, response):
 50         # 解析标的列表
 51         sel = Selector(response)
 52     xpath_str = ‘//div[@id="con"]/div/table/tr/td/a[@class="BL_name"]/@href‘
 53         sel_list = sel.xpath(xpath_str)
 54         url_list = self.list(sel_list)
 55 
 56         # 请求标的
 57         for url in url_list[0:2]:
 58             self.log(u‘请求标的页：%s‘ % url)
 59             yield Request(url, callback=self.parse_page)
 60             
 61         # 递归获取所有标的列表
 62         """
 63         if len(url_list) > 0:
 64             self.page += 1
 65             yield Request(self.list_url % self.page, callback=self.parse_list)
 66         """
 67 
 68     # 解析标的网页
 69     def parse_page(self, response):
 70         # 标的信息
 71         item = self.parse_invest(response)
 72 
 73         # 投资人列表
 74         item = self.parse_tender(response)
 75         
 76         # 还记录列表
 77         item = self.parse_repayment(response)
 78         self.log(‘‘)
 79         self.log(‘‘)
 80 
 81 
 82     # 解析标的信息
 83     def parse_invest(self, response):
 84         # 解析数据
 85         sel = Selector(response)
 86         item = InvestItem()
 87         item = self.invest(sel, xpaths_invest)
 88 
 89         # 处理解析到的数据
 90         item[‘num‘] = i_num(item[‘num‘])
 91     item[‘title‘] = i_title(item[‘title‘])
 92         item[‘type‘] = i_type(item[‘type‘])
 93         item[‘balance‘] = i_balance(item[‘balance‘])
 94         item[‘rate‘] = i_rate(item[‘rate‘])
 95         item[‘award_rate‘] = i_award_rate(item[‘award_rate‘])
 96         item[‘dead‘] = i_dead(item[‘dead‘])
 97         item[‘r_type‘] = i_r_type(item[‘r_type‘])
 98         item[‘man‘] = i_man(item[‘man‘])
 99         return item
100 
101 
102     # 解析投标记录信息
103     def parse_tender(self, response):
104         # 解析数据
105         sel = Selector(response)
106         sel_tender = sel.xpath(‘//div[@class="wrap"]/table[5]/tr/td/table/tr[3]/td/table/tbody/tr‘)
107         item_list = self.tender(sel_tender, xpaths_tender)
108 
109         # 处理解析到的数据
110         tender_list = []
111         for item in item_list:
112             item[‘man‘] = t_man(item[‘man‘])
113             item[‘balance‘] = t_balance(item[‘balance‘])
114             item[‘type‘] = t_type(item[‘type‘])
115             item[‘time‘] = t_time(item[‘time‘])
116             tender_list.append(item)
117         return tender_list
118 
119 
120     # 解析还款记录
121     def parse_repayment(self, response):
122         # 解析数据
123         sel = Selector(response)
124         sel_repayment = sel.xpath(‘//dl[@class="repay"]/dt‘)[1:]
125         item_list = self.repayment(sel_repayment, xpaths_repayment)
126 
127         # 处理解析到的数据
128         repayment_list = []
129         for item in item_list:
130             item[‘date‘] = r_date(item[‘date‘])
131             item[‘status‘] = r_status(item[‘status‘])
132             item[‘done‘] = r_done(item[‘done‘])
133             item[‘undone‘] = r_undone(item[‘undone‘])
134             item[‘day‘] = r_day(item[‘day‘])
135             repayment_list.append(item)
136         return repayment_list