初学python3-爬取cnnvd漏洞信息

  1 #!/usr/bin/env python3
  2 # -*- coding: utf-8 -*-
  3 # by Kaiho
  4 
  5 import urllib.request
  6 from urllib import parse
  7 from bs4 import BeautifulSoup
  8 import xlwt
  9 import zlib
 10 import re
 11 import time
 12 import xlsxwriter
 13 import sys
 14 import datetime
 15 
 16 ‘‘‘
 17 运行方法：
 18 python holes_crawler 2017-10-01 2017-10-31 178
 19 第一个为开始时间，第二个为结束时间，第三个为总页数。
 20 
 21 ‘‘‘
 22 
 23 
 24 #获得漏洞详情链接列表
 25 def holes_url_list(url,start_time,end_time):
 26     header = {
 27         ‘Host‘: ‘cnnvd.org.cn‘,
 28         ‘User-Agent‘: ‘Mozilla/5.0 (Linux; Android 4.1.2; Nexus 7 Build/JZ054K) AppleWebKit/535.19 (KHTML, like Gecko) Chrome/18.0.1025.166 Safari/535.19‘,
 29         ‘Accept‘: ‘text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8‘,
 30         ‘Accept-Language‘: ‘zh-CN,zh;q=0.8,en-US;q=0.5,en;q=0.3‘,
 31         ‘Accept-Encoding‘: ‘gzip, deflate‘,
 32         ‘Content-Type‘: ‘application/x-www-form-urlencoded‘,
 33         ‘Content-Length‘:‘ 219‘,
 34         ‘Referer‘: ‘http://cnnvd.org.cn/web/vulnerability/queryLds.tag‘,
 35         ‘Cookie‘: ‘SESSION=988d12eb-217f-4163-ba47-5c55e018b388‘,
 36         ‘Connection‘: ‘keep-alive‘,
 37         ‘Upgrade-Insecure-Requests‘: ‘1‘
 38     }    
 39     data = {
 40         ‘CSRFToken‘:‘‘,
 41         ‘cvHazardRating‘:‘‘,
 42         ‘cvVultype‘:‘‘,
 43         ‘qstartdateXq‘:‘‘,
 44         ‘cvUsedStyle‘:‘‘,
 45         ‘cvCnnvdUpdatedateXq‘:‘‘,
 46         ‘cpvendor‘:‘‘,
 47         ‘relLdKey‘:‘‘,
 48         ‘hotLd‘:‘‘,
 49         ‘isArea‘:‘‘,
 50         ‘qcvCname‘:‘‘,
 51         ‘qcvCnnvdid‘:‘CNNVD或CVE编号‘,
 52         ‘qstartdate‘:‘2017-10-30‘,                #---------------》开始日期
 53         ‘qenddate‘:‘2017-10-31‘                    #---------------》结束日期
 54     }
 55     data[‘qstartdate‘] = start_time 
 56     data[‘qenddate‘] = end_time
 57     data = parse.urlencode(data).encode(‘utf-8‘)
 58     holes_url_html = urllib.request.Request(url,headers=header,data=data)
 59     holes_url_html = urllib.request.urlopen(holes_url_html)
 60     holes_url_html = zlib.decompress(holes_url_html.read(), 16+zlib.MAX_WBITS)
 61     holes_url_html = holes_url_html.decode()
 62     
 63     #提取漏洞详情链接
 64     response = r‘href="(.+?)" target="_blank" class="a_title2"‘
 65     holes_link_list = re.compile(response).findall(holes_url_html)
 66     
 67     #添加http前序
 68     i = 0
 69     for link in holes_link_list:
 70         holes_lists.append(‘http://cnnvd.org.cn‘+holes_link_list[i])
 71         i+=1
 72         print("已完成爬行第%d个漏洞链接"%i)
 73         time.sleep(0.2)
 74 
 75 
 76 #调用漏洞列表函数并获得漏洞链接列表
 77 begin = datetime.datetime.now()
 78 holes_lists=[]
 79 j = 1
 80 page_count = sys.argv[3]
 81 page_count = int(page_count) 
 82 start_time = sys.argv[1]
 83 end_time = sys.argv[2]
 84 while j<=page_count :                        #-------------------------------》爬取页数
 85     try:
 86         holes_url = ‘http://cnnvd.org.cn/web/vulnerability/queryLds.tag?pageno=%d&repairLd=‘%j
 87         holes_url_list(holes_url,start_time,end_time)
 88         print("已完成爬行第%d页"%j)
 89         print(‘\n‘)
 90         time.sleep(2.5)
 91         j+=1
 92     except:
 93         print(‘爬取失败，等待5秒后重新爬取。‘)
 94         time.sleep(5)
 95 
 96 #print(holes_lists)    #漏洞链接列表
 97 
 98 
 99 
100 #漏洞信息爬取函数
101 def holes_data(url):
102     header = {
103         ‘Host‘:‘ cnnvd.org.cn‘,
104         ‘Connection‘: ‘keep-alive‘,
105         ‘Accept‘: ‘text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8‘,
106         ‘Upgrade-Insecure-Requests‘:‘ 1‘,
107         ‘User-Agent‘: ‘Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/49.0.2623.221 Safari/537.36 SE 2.X MetaSr 1.0‘,
108         ‘Accept-Encoding‘: ‘gzip, deflate, sdch‘,
109         ‘Accept-Language‘:‘ zh-CN,zh;q=0.8‘,
110         ‘Cookie‘:‘SESSION=988d12eb-217f-4163-ba47-5c55e018b388‘
111     }
112     holes_data_html = urllib.request.Request(url,headers=header)
113     holes_data_html = urllib.request.urlopen(holes_data_html)
114     holes_data_html = zlib.decompress(holes_data_html.read(), 16+zlib.MAX_WBITS)
115     holes_data_html = holes_data_html.decode()
116 
117     global holes_result_list
118     holes_result_list=[]    #抓取信息列表命名
119     
120     #添加漏洞信息详情
121     holes_detainled_soup1 = BeautifulSoup(holes_data_html,‘html.parser‘)
122     holes_detainled_data = holes_detainled_soup1.find(‘div‘,attrs={‘class‘:‘detail_xq w770‘})    ##定义 漏洞信息详情 块的soup
123     holes_detainled_data = holes_detainled_data.encode().decode()
124     holes_detainled_soup = BeautifulSoup(holes_detainled_data,‘html.parser‘)    #二次匹配    
125 
126     holes_detainled_data_list = holes_detainled_soup.find_all(‘li‘)    #标签a信息汇总    
127     
128     try:
129         holes_name = holes_detainled_soup.h2.string    #漏洞名称
130     except:
131         holes_name = ‘‘
132     holes_result_list.append(holes_name)
133     
134     try:
135         holes_cnnvd_num = holes_detainled_soup.span.string    #cnnvd编号
136         holes_cnnvd_num = re.findall(r"\：([\s\S]*)",holes_cnnvd_num)[0]
137     except:
138         holes_cnnvd_num = ‘‘
139     holes_result_list.append(holes_cnnvd_num)
140     
141     try:                            #漏洞等级
142         holes_rank = holes_detainled_soup.a.decode()
143         holes_rank = re.search(u‘([\u4e00-\u9fa5]+)‘,holes_rank).group(0)
144     except:
145         holes_rank = ‘‘
146     holes_result_list.append(holes_rank)
147 
148     holes_cve_html = holes_detainled_data_list[2].encode().decode()    #漏洞cve编号
149     holes_cve_soup = BeautifulSoup(holes_cve_html,‘html.parser‘)
150     try:
151         holes_cve = holes_cve_soup.a.string
152         holes_cve = holes_cve.replace("\r","").replace("\t","").replace("\n","")
153     except:
154         holes_cve = ‘‘
155     holes_result_list.append(holes_cve)
156     
157     holes_type_html = holes_detainled_data_list[3].encode().decode()    #漏洞类型
158     holes_type_soup = BeautifulSoup(holes_type_html,‘html.parser‘)
159     try:
160         holes_type = holes_type_soup.a.string
161         holes_type = holes_type.replace("\r","").replace("\t","").replace("\n","")
162     except:
163         holes_type = ‘‘
164     holes_result_list.append(holes_type)
165     
166     holes_time_html = holes_detainled_data_list[4].encode().decode()    #发布时间
167     holes_time_soup = BeautifulSoup(holes_time_html,‘html.parser‘)
168     try:    
169         holes_time = holes_time_soup.a.string
170         holes_time = holes_time.replace("\r","").replace("\t","").replace("\n","")
171     except:
172         holes_time = ‘‘
173     holes_result_list.append(holes_time)
174 
175     holes_attack_html = holes_detainled_data_list[5].encode().decode()    #威胁类型
176     holes_attack_soup = BeautifulSoup(holes_attack_html,‘html.parser‘)
177     try:    
178         holes_attack = holes_attack_soup.a.string
179         holes_attack = holes_attack.replace("\r","").replace("\t","").replace("\n","")
180     except:
181         holes_attack = ‘‘
182     holes_result_list.append(holes_attack)
183 
184     holes_update_html = holes_detainled_data_list[6].encode().decode()    #更新时间
185     holes_update_soup = BeautifulSoup(holes_update_html,‘html.parser‘)
186     try:
187         holes_update = holes_update_soup.a.string
188         holes_update = holes_update.replace("\r","").replace("\t","").replace("\n","")
189     except:
190         holes_update = ‘‘    
191     holes_result_list.append(holes_update)
192 
193     holes_firm_html = holes_detainled_data_list[7].encode().decode()    #厂商
194     holes_firm_soup = BeautifulSoup(holes_firm_html,‘html.parser‘)
195     try:
196         holes_firm = holes_firm_soup.a.string
197         holes_firm = holes_firm.replace("\r","").replace("\t","").replace("\n","")
198     except:
199         holes_firm = ‘‘
200     holes_result_list.append(holes_firm)
201 
202     holes_source_html = holes_detainled_data_list[8].encode().decode()    #漏洞来源
203     holes_source_soup = BeautifulSoup(holes_source_html,‘html.parser‘)
204     try:
205         holes_source = holes_source_soup.a.string
206         holes_source = holes_source.replace("\r","").replace("\t","").replace("\n","")
207     except:
208         holes_source = ‘‘
209     holes_result_list.append(holes_source)
210     
211 
212     #添加漏洞简介详情
213     holes_title_html = holes_detainled_soup1.find(‘div‘,attrs={‘class‘:‘d_ldjj‘})    #定义“漏洞简介”的soup
214     holes_title_html = holes_title_html.encode().decode()
215     holes_title_soup2 = BeautifulSoup(holes_title_html,‘html.parser‘)
216 
217     try:
218         holes_titles1 = holes_title_soup2.find_all(name=‘p‘)[0].string
219         holes_titles2 = holes_title_soup2.find_all(name=‘p‘)[1].string
220         holes_titles = holes_titles1 + holes_titles2
221         holes_titles = holes_titles.replace(‘ ‘,‘‘).replace(‘\t‘,‘‘).replace(‘\r‘,‘‘).replace(‘\n‘,‘‘)
222     except:
223         holes_titles = ‘‘
224     holes_result_list.append(holes_titles)
225 
226 
227     #漏洞公告
228     holes_notice_html = holes_detainled_soup1.find(‘div‘,attrs={‘class‘:‘d_ldjj m_t_20‘})    #定义“漏洞公告”的soup
229     holes_notice_html = holes_notice_html.encode().decode()
230     holes_notice_soup2 = BeautifulSoup(holes_notice_html,‘html.parser‘)
231     
232     try:
233         holes_notice1 = holes_notice_soup2.find_all(name=‘p‘)[0].string
234         holes_notice2 = holes_notice_soup2.find_all(name=‘p‘)[1].string
235         holes_notice = holes_notice1+holes_notice2
236         holes_notice = holes_notice.replace(‘\n‘,‘‘).replace(‘\r‘,‘‘).replace(‘\t‘,‘‘)
237     except:
238         holes_notice = ‘‘
239     holes_result_list.append(holes_notice)
240 
241 
242     #参考网址
243     holes_reference_html = holes_detainled_soup1.find_all(‘div‘,attrs={‘class‘:‘d_ldjj m_t_20‘})[1]    #定义“参考网址”的soup
244     holes_reference_html = holes_reference_html.encode().decode()
245     holes_reference_soup2 = BeautifulSoup(holes_reference_html,‘html.parser‘)
246 
247     try:
248         holes_reference = holes_reference_soup2.find_all(name=‘p‘)[1].string
249         holes_reference = holes_reference.replace(‘\n‘,‘‘).replace(‘\r‘,‘‘).replace(‘\t‘,‘‘).replace(‘链接:‘,‘‘)
250     except:
251         holes_reference = ‘‘
252     holes_result_list.append(holes_reference)
253     
254 
255     #受影响实体
256     holes_effect_html = holes_detainled_soup1.find_all(‘div‘,attrs={‘class‘:‘d_ldjj m_t_20‘})[2]    #定义“受影响实体”的soup
257     holes_effect_html = holes_effect_html.encode().decode()
258     holes_effect_soup2 = BeautifulSoup(holes_effect_html,‘html.parser‘)
259     try:
260         holes_effect = holes_effect_soup2.find_all(name=‘p‘)[0].string
261         holes_effect = holes_effect.replace(‘\n‘,‘‘).replace(‘\r‘,‘‘).replace(‘\t‘,‘‘).replace(‘ ‘,‘‘)
262     except:
263         try:
264             holes_effect = holes_effect_soup2.find_all(name=‘a‘)[0].string
265             holes_effect = holes_effect.replace(‘\n‘,‘‘).replace(‘\r‘,‘‘).replace(‘\t‘,‘‘).replace(‘ ‘,‘‘)        
266         except:
267             holes_effect = ‘‘
268     holes_result_list.append(holes_effect)
269 
270 
271 
272     #补丁
273     holes_patch_html = holes_detainled_soup1.find_all(‘div‘,attrs={‘class‘:‘d_ldjj m_t_20‘})[3]    #定义“补丁”块soup
274     holes_patch_html = holes_patch_html.encode().decode()
275     holes_patch_soup2 = BeautifulSoup(holes_patch_html,‘html.parser‘)
276     
277 
278     try:
279         holes_patch = holes_patch_soup2.find_all(name=‘p‘)[0].string
280         holes_patch = holes_patch.replace(‘\n‘,‘‘).replace(‘\r‘,‘‘).replace(‘\t‘,‘‘).replace(‘ ‘,‘‘)
281     except:
282         holes_patch = ‘‘
283     holes_result_list.append(holes_patch)
284 
285 
286 #调用漏洞信息函数并爬取漏洞信息
287 holes_result_lists = []
288 print (‘漏洞数量： ‘,len(holes_lists))    
289 a=0
290 while a < len(holes_lists):
291     try:
292         holes_data(holes_lists[a])
293         holes_result_lists.append(holes_result_list)
294         a+=1
295         print("完成爬行第%d个漏洞信息"%a)
296         time.sleep(0.5)
297     except:
298         print(‘爬取失败，等待5秒后重新爬取。‘)
299         time.sleep(5)
300 
301 holes_result_excel = tuple(holes_result_lists)        #列表转换元组
302 
303 #漏洞信息写入excel
304 workbook = xlsxwriter.Workbook(‘holes_data.xlsx‘)
305 worksheet = workbook.add_worksheet()
306 
307 row = 0
308 col = 0
309 worksheet.write(row,0,‘漏洞名称‘)
310 worksheet.write(row,1,‘CNNVD编号‘)
311 worksheet.write(row,2,‘危害等级‘)
312 worksheet.write(row,3,‘CVE编号‘)
313 worksheet.write(row,4,‘漏洞类型‘)
314 worksheet.write(row,5,‘发布时间‘)
315 worksheet.write(row,6,‘攻击途径‘)
316 worksheet.write(row,7,‘更新时间‘)
317 worksheet.write(row,8,‘厂商‘)
318 worksheet.write(row,9,‘漏洞来源‘)
319 worksheet.write(row,10,‘漏洞描述‘)
320 worksheet.write(row,11,‘解决方案‘)
321 worksheet.write(row,12,‘参考链接‘)
322 worksheet.write(row,13,‘受影响实体‘)
323 worksheet.write(row,14,‘补丁‘)
324 
325 row = 1
326 for n1,n2,n3,n4,n5,n6,n7,n8,n9,n10,n11,n12,n13,n14,n15 in holes_result_excel:
327     worksheet.write(row,col,n1)
328     worksheet.write(row,col+1,n2)
329     worksheet.write(row,col+2,n3)
330     worksheet.write(row,col+3,n4)
331     worksheet.write(row,col+4,n5)
332     worksheet.write(row,col+5,n6)
333     worksheet.write(row,col+6,n7)
334     worksheet.write(row,col+7,n8)
335     worksheet.write(row,col+8,n9)
336     worksheet.write(row,col+9,n10)
337     worksheet.write(row,col+10,n11)
338     worksheet.write(row,col+11,n12)
339     worksheet.write(row,col+12,n13)
340     worksheet.write(row,col+13,n14)
341     worksheet.write(row,col+14,n15)
342     row += 1
343 
344 workbook.close()
345 
346 end = datetime.datetime.now()
347 total_time = end - begin
348 print (‘漏洞信息爬取结束‘)
349 print (‘应该爬行漏洞数量： ‘,len(holes_lists))
350 print (‘爬行时间： ‘,total_time)
351 
352 #holes_data(‘http://www.cnnvd.org.cn/web/xxk/ldxqById.tag?CNNVD=CNNVD-201710-987‘)
353 #print (holes_result_list)