(四)基于multiprocessing.dummy线程池爬取梨视频的视频信息

import requests
from lxml import etree
import re
from multiprocessing.dummy import Pool


def get_video_data(video_data):
    """获取视频文件"""
    data = session.get(video_data[‘url‘],headers=headers).content
    dic = {‘name‘:video_data[‘name‘],
           ‘data‘:data}
    return dic


def write(data):
    """持久化存储"""
    with open(data[‘name‘]+‘.mp4‘,‘wb‘) as f:
        f.write(data[‘data‘])




url = ‘https://www.pearvideo.com/category_8‘
headers = {
    ‘User-Agent‘: ‘Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/81.0.4044.92 Safari/537.36‘,
    ‘Connection‘: ‘close‘
}

session = requests.session()
page_text = session.get(url=url, headers=headers).text
tree = etree.HTML(page_text)
li_list = tree.xpath("//ul[@id=‘listvideoListUl‘]/li")
video_urls = []
for li in li_list:
    src = ‘https://www.pearvideo.com/‘ + li.xpath(‘./div/a/@href‘)[0]
    name = li.xpath(‘./div/a/div[2]/text()‘)[0]
    detail_text = session.get(url=src, headers=headers).text
    # print(tree_2.xpath(‘//*[@id= "JprismPlayer"]/video/@src‘)) 结果为空 说明数据是动态加载
    # 在Response中搜索mp4 得到视频对应的链接地址 在script语句中 因为用re
    # srcUrl="https://video.pearvideo.com/mp4/third/20200617/cont-1680618-10008579-104906-hd.mp4",vdoUrl=srcUrl
    ex = r‘srcUrl="(.*?)",vdoUrl=srcUrl‘
    video_src = re.findall(ex, detail_text)[0]
　　# 将视频数据存放在字典中
    dic = {
        ‘name‘: name,
        ‘url‘: video_src
    }
    video_urls.append(dic)

# 用于获取视频信息
pool = Pool(4)
ret = pool.map(get_video_data, video_urls)

# 用于持久化存储
pool_2 = Pool(4)
pool_2.map(write,ret)

# 关闭线程池
pool.close()
pool_2.close()

# 等待主进程结束
pool.join()
pool_2.join()