4444

  1 # -*- coding: utf-8 -*-
  2 from lxml import etree
  3 import requests,re,random
  4 import os,time,json,redis
  5 from myweb.yichewang import user_agent
  6 import urllib.request
  7 from concurrent.futures import ThreadPoolExecutor
  8 import pymongo
  9 import pymysql
 10 from datetime import datetime
 11 from wxpy import *
 12 #bot = Bot(cache_path=True)#微信端监控运行情况
 13 
 14 
 15 
 16 def run_time(func):
 17     def wrap(*arg,**kwargs):
 18         start_time = time.time()
 19         func(*arg,**kwargs)
 20         print(‘获取车型链接类方法运行时间为:‘,time.time() - start_time)
 21         return func
 22         #bot.file_helper.send(‘获取车型链接类方法运行时间为:‘,time.time() - start_time)
 23     return wrap
 24 
 25 def try_expect(funcs):
 26     def getwrap(*arg,**kwargs):
 27         try:
 28             funcs(*arg,**kwargs)
 29             print(‘时间:‘,datetime.now(),‘未发生异常!!!!!!!!!!!!!!‘)
 30             #bot.file_helper.send(‘时间:‘,datetime.now(),‘未发生异常!!!!!!!!!!!!!!‘)
 31         except Exception:
 32             return ‘出错啦!!!!!!!!!!‘
 33         #return funcs
 34     return getwrap
 35 
 36 class Yi_car_data(object):
 37 
 38     headers = {‘Referer‘:‘http://i.yiche.com/authenservice/login.html?returnurl=http%3A%2F%2Fguangzhou.bitauto.com%2F%3Freferrer%3Dhttp%3A%2F%2Fi.yiche.com%2Fauthenservice%2FAboutPassWord%2FResetPasswordResult.aspx%3Freturnurl%3Dhttp%253a%252f%252fi.yiche.com%252fu27686084%252f‘}
 39     login_url = ‘http://i.yiche.com/ajax/Authenservice/login.ashx‘
 40     data ={
 41         ‘txt_LoginName‘:‘15766264244‘,
 42         ‘txt_Password‘:‘123456789aa‘,
 43         ‘txt_Code‘:‘‘,
 44         ‘cbx_keepState‘:‘true‘,
 45         ‘returnurl‘:‘http://guangzhou.bitauto.com/?referrer=http://i.yiche.com/authenservice/AboutPassWord/ResetPasswordResult.aspx?returnurl=http://i.yiche.com/u27686084/‘,
 46         ‘guid‘:‘‘,
 47         ‘Gamut‘:‘true‘
 48 
 49     }
 50     cartype_url = ‘http://api.car.bitauto.com/CarInfo/getlefttreejson.ashx?tagtype=chexing&pagetype=masterbrand&objid=0‘
 51 
 52     def __init__(self):
 53         #client = redis.Redis(host=‘127.0.0.1‘,port=int(6379))
 54         self.pool = redis.ConnectionPool(host=‘127.0.0.1‘, port=6379)
 55         self.r = redis.Redis(connection_pool=self.pool)
 56         #self.bot = Bot(cache_path=True)
 57         self.s_requests = requests.Session()
 58         html = self.s_requests.post(self.login_url,data=self.data,headers = self.headers,timeout=2)
 59         print(html.status_code)
 60 
 61 
 62     def Car_types_all_datas(self):
 63 
 64         self.headers[‘User-Agent‘] = random.choice(user_agent.user_agent_list)
 65         try:
 66             while True:
 67                 time.sleep(1)
 68                 reponse = self.s_requests.get(self.cartype_url,headers=self.headers,timeout=3)
 69                 #print(reponse.text[132:])
 70                 if reponse.status_code ==200 or ‘奥迪‘ in reponse.content:
 71                     break
 72                 else:
 73                     print(‘downing field‘)
 74         except Exception as e:
 75             print(‘抛出异常:‘,e)
 76         cartypeurl_re_patten = ‘url:"(.*?)"‘
 77         cartypename_re_patten = ‘name:"(.*?)"‘
 78         patten_cartypeurl = re.compile(cartypeurl_re_patten)
 79         patten_cartypename = re.compile(cartypename_re_patten)
 80         cartype_url = patten_cartypeurl.findall(str(reponse.text))
 81         cartype_name = patten_cartypename.findall(str(reponse.text))
 82         if len(cartype_url)==0:
 83             if len(cartype_name)==0:
 84                 print(‘解析数据为空!!!!!!!!!!!!!!!!!!!!‘)
 85 
 86         else:
 87             cartype_datas = list(zip(cartype_url,cartype_name))
 88             try:
 89                 self.r.set(‘car_url_name‘,str(cartype_datas))
 90             except Exception as e:
 91                 print(e)
 92                 print(‘插入redis失败!!!!!!!!!!!!!!!‘)
 93             #print(list(zip(cartype_url,cartype_name)))
 94             return cartype_datas
 95 
 96     @try_expect #捕获运行异常
 97     @run_time #计算方法运行时间
 98     def car_type_all_info_time_expect(self):
 99         self.Car_types_all_datas()
100 
101     def car_type_all_info(self):
102         car_type_url = []
103         for url_name in self.Car_types_all_datas():
104             #print(url_name)
105             car_type_url.append(‘http://car.bitauto.com/‘+str(url_name[0]))
106         #print(car_type_url)
107         return car_type_url#返回全部车品牌链接数据
108 
109     def get_car_info(self,url):
110         self.headers[‘User-Agent‘] = random.choice(user_agent.user_agent_list)
111         try:
112             req = self.s_requests.get(url,headers=self.headers,timeout=2)
113             #print(req.text)
114         except:
115             print(‘出错啦!!!!!!!!!!‘)
116         select  = etree.HTML(str(req.text))
117         car_url =[‘http://car.bitauto.com/‘+str(i) for i in select.xpath(‘//li[@class="name"]/a/@href‘)]
118         car_name = select.xpath(‘//li[@class="name"]/a/@title‘)
119 
120         if len(car_url)==0 or len(car_name)==0:
121             print(‘提取数据失败‘)
122         elif len(car_url)!=0 or len(car_name)!=0:
123             #print(list(zip(car_url,car_name)))
124             return list(zip(car_url, car_name))#返回每款车品牌的车系链接和名字
125         else:
126             print(‘#######################################‘)
127 
128     def get_all_cartypes_info(self,infourl):
129         pass
130 
131 
132 
133 def all_run_main():
134     count = 0
135     tn_car = Yi_car_data()
136     # t.Car_types_all_datas()
137     tn_car.car_type_all_info_time_expect()
138     dn_car = tn_car.car_type_all_info()
139     # print(d)
140     for j in dn_car:
141         count += 1
142         print(‘正在解析第‘+str(j)+‘url‘,‘\n‘,count)
143         all_cartype_datas = tn_car.get_car_info(j)
144         #print(f)
145         yield all_cartype_datas
146 def threading_run_main():
147     pass
148 
149 
150 
151 
152 if __name__ =="__main__":
153     print(list(all_run_main()))