Python练习：优酷评论过滤（抓取当前视频全部评论，并过滤不包括所需关键词的留言）

 1 # coding:utf-8
 2 print(‘正在初始化...‘)
 3 import requests
 4 import re
 5 from lxml.html import fromstring
 6 import pyautogui
 7 import sys
 8 import os
 9 from multiprocessing.dummy import Pool
10 from lxml import _elementpath
11 
12 
13 class Youku_comment:
14 
15     """docstring for Youku_comment"""
16 
17     def __init__(self, raw_url):
18 
19         self.pid = re.findall(‘/id_(.*?)\.html‘, raw_url)[0]
20         r1 = requests.get(raw_url)
21         title = re.findall(‘<title>(.*?)</title>‘, r1.text)[0]
22         title = re.sub(‘\W‘, ‘‘, title).replace(‘在线播放优酷网视频高清在线观看‘, ‘‘)
23         totalpn = self.get_totalpn(self.pid)
24         print(‘视频ID：%s‘ % self.pid, ‘\n视频标题：%s‘ %
25               title, ‘\n总页码数：%s\n正在抓取...‘ % totalpn)
26 
27         pp = Pool(30)
28         pagenums = range(1, totalpn + 1)
29         result = pp.map(self.get_comment, pagenums)
30         pp.close()
31         pp.join()
32         result = [i for i in result if i]
33         self.aa = sum(result, [])
34         # print(jieguo)
35         # with open(‘%s.csv‘ % title, ‘w‘, encoding=‘gbk‘) as f:
36         #     f.write(jieguo.encode(‘gbk‘, ‘ignore‘).decode(‘gbk‘))
37 
38     def get_totalpn(self, pid):
39         r = requests.get(
40             ‘http://comments.youku.com/comments/~ajax/vpcommentContent.html?__ap={"videoid":"%s","page":1}‘ % pid)
41         totalpn = (int(r.json()[‘totalSize‘].replace(‘,‘, ‘‘)) // 30) + 1
42         return totalpn
43 
44     def get_comment(self, pagenum):
45         for _ in range(5):
46             try:
47                 r = requests.get(
48                     ‘http://comments.youku.com/comments/~ajax/vpcommentContent.html?__ap={"videoid":"%s","page":%s}‘ % (self.pid, pagenum), timeout=3)
49                 sjson = r.json()
50                 scode = sjson[‘con‘]
51                 ss = re.findall(‘<p id=".*?">.*?</p>‘, scode, flags=re.S)
52                 ss = [re.sub(‘<.*?>‘, ‘‘, i) for i in ss]
53                 if ss:
54                     return ss
55             except:
56                 pass
57 
58 
59 def filt1(str1, kws):
60     kws = kws.split(‘ ‘) if kws else ‘OST 背景 音乐 旋律 歌曲 调子 music 耳熟 BGM 谁唱的 来自 出自 原声‘.split(
61         ‘ ‘)
62     for i in kws:
63         if i in str1:
64             return str1
65 
66 
67 def quchong(ll):
68     ss = ‘‘
69     for i in ll:
70         if i in ss:
71             continue
72         else:
73             ss = ss + ‘\n‘ + i
74     return ss
75 
76 while 1:
77     try:
78         url = pyautogui.prompt(‘请输入网址：‘)
79         if not url:
80             break
81         tt = Youku_comment(url)
82         pinglun = tt.aa
83         while 1:
84             kws = pyautogui.prompt(‘请输入关键词，多个请用空格隔开（直接回车则代表找背景音乐）：‘)
85             kws = kws if kws else 0
86             ss = [filt1(i, kws) for i in pinglun]
87             ss = [i for i in ss if i]
88             ss = quchong(ss)
89             print(‘检索结果：\n‘)
90             print(ss)
91             jixu = pyautogui.confirm(
92                 text=‘是否要继续检索‘, title=‘请确认‘, buttons=[‘是‘, ‘否‘])
93             if jixu == ‘否‘:
94                 break
95 
96     except Exception as e:
97         print(e)
98         print(‘错误，请重试‘)
99 os.system(‘pause‘)