爬虫_中国天气网_文字天气预报（xpath）

 1 import requests
 2 from lxml import etree
 3 
 4 headers = ‘Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/65.0.3325.181 Safari/537.36‘
 5 
 6 def get_html(url):
 7     try:
 8         html = requests.get(url, headers={‘User-Agent‘:‘headers‘})
 9         html.encoding = html.apparent_encoding
10         if html.status_code == 200:
11             return html.text
12         return 0
13     
14     except RequestsException:
15         return 0
16 ‘‘‘
17 conMintab:  华北  
18     conMintab2    北京
19     conMintab2    天津
20     conMintab2    河北
21     conMintab2    山西
22     conMintab2    内蒙古
23 
24 
25 //div[@class="conMidtab"][1] 今天
26 //div[@class="conMidtab"][2] 明天 
27                 ...
28 //div[@class="conMidtab"][7] 最后一天
29 ‘‘‘
30 
31 def parse_html(html):
32     wea = []
33     html_element = etree.HTML(html)
34     # !!!!!  trs = html_element.xpath(‘//div[@class="conMidtab"][1]//tr‘)[2:]
35     provinces = html_element.xpath(‘//div[@class="conMidtab"][1]//div[@class="conMidtab2"]‘)
36     for province in provinces:
37         trs = province.xpath(‘.//tr‘)[2:]
38         for tr in trs:
39             weather = {}
40             city = tr.xpath(‘.//td[@width="83"]/a/text()‘)
41             phenomenon = tr.xpath(‘.//td[@width="89"]/text()‘)
42             wind = tr.xpath(‘.//td[@width="162"]//text()‘)
43             hightest = tr.xpath(‘.//td[@width="92"]/text()‘)
44             weather[‘city‘] = city
45             weather[‘phenomenon‘] = phenomenon
46             weather[‘wind‘] = wind
47             weather[‘hightest‘] = hightest
48             while ‘\n‘ in wind:
49                 wind.remove(‘\n‘)
50             wea.append(weather)
51         
52     print(wea)    
53     
54         
55 def main():
56     urls = [‘http://www.weather.com.cn/textFC/hb.shtml‘,
57             ‘http://www.weather.com.cn/textFC/db.shtml‘,
58             ‘http://www.weather.com.cn/textFC/hd.shtml‘,
59             ‘http://www.weather.com.cn/textFC/hz.shtml‘,
60             ‘http://www.weather.com.cn/textFC/hn.shtml‘,
61             ‘http://www.weather.com.cn/textFC/xb.shtml‘,
62             ‘http://www.weather.com.cn/textFC/xn.shtml‘,
63             ‘http://www.weather.com.cn/textFC/gat.shtml‘]
64     for url in urls:
65         html = get_html(url)
66         if html == 0:
67             html = get_html(url)
68         parse_html(html)
69 
70 
71 if __name__ == ‘__main__‘:
72         main()