selenium爬取斗鱼所有直播房间信息

 1 import time
 2 from selenium import webdriver
 3 
 4 
 5 class DouyuSpider(object):
 6     def __init__(self):
 7         self.start_rul = ‘https://www.douyu.com/directory/all‘
 8         self.driver = webdriver.Chrome()
 9 
10     def get_content_list(self):
11         time.sleep(10)  # 强制等待10秒，否则可能报错
12         li_list = self.driver.find_elements_by_xpath(‘//ul[@class="layout-Cover-list"]/li‘)
13         content_list = []
14         for li in li_list:
15             item = {}
16             item[‘room_img‘] = li.find_element_by_xpath(‘.//img[@class="DyImg-content is-normal "]‘).get_attribute(‘src‘)
17             item[‘room_title‘] = li.find_element_by_xpath(‘.//h3[@class="DyListCover-intro"]‘).text
18             item[‘root_category‘] = li.find_element_by_xpath(‘.//span[@class="DyListCover-zone"]‘).text
19             item[‘author_name‘] = li.find_element_by_class_name(‘DyListCover-user‘).text
20             item[‘watch_num‘] = li.find_element_by_class_name(‘DyListCover-hot‘).text
21             content_list.append(item)
22             print(item)  # 打印每次获取到的直播房间的信息
23         # 获取下一页的元素,为了防止没有报错，这里用elements，翻到最后一页一定就没有了，返回一个列表
24         next_url = self.driver.find_elements_by_xpath(‘//li[@class=" dy-Pagination-next"]‘)
25         next_url = next_url[0] if len(next_url) > 0 else None
26         return content_list, next_url
27 
28     def save_content_list(self, content_list):
29         pass  # 保存数据这里就不再做演示
30 
31     def run(self):  # 实现主要逻辑
32         # 1.start_url
33         # 2.发送请求，获取响应
34         self.driver.maximize_window()
35         self.driver.get(self.start_rul)
36         # 3.提取数据，提取下一页的元素
37         content_list, next_url = self.get_content_list()
38         # 4.保存数据
39         self.save_content_list(content_list)
40         # 4.点击下一页元素，循环
41         while next_url is not None:
42             next_url.click()
43             content_list, next_url = self.get_content_list()
44             self.save_content_list(content_list)
45 
46 
47 
48 if __name__ == ‘__main__‘:
49     douban = DouyuSpider()
50     douban.run()