xmly爬虫案例

 1 #!/usr/bin/python
 2 # -*- coding: utf-8 -*-
 3 # @Time    : 2020/3/25 19:09
 4 # @Author  : Meng
 5 # @File    : paxmly.py
 6 # @Software: PyCharm
 7 
 8 import requests  #模拟浏览器爬取页面 获取页面信息
 9 import parsel #解析网页的包
10 headers = { #伪装成浏览器的请求头  数据格式 是字典
11 ‘user-agent‘: ‘Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/80.0.3987.149 Safari/537.36‘
12 }
13 response = requests.get(url=‘https://www.ximalaya.com/youshengshu/34013148/p2/‘,headers=headers) #爬取指定页的播放列表
14 sel = parsel.Selector(response.text)  #解析网页
15 sound_list = sel.css(‘.sound-list ul li a‘) #获取某一页30条链接
16 list_all=[]  #存所有结果的列表
17 for sound in sound_list[0:30]: #切片
18     # print(sound)
19     media_url = sound.css(‘a::attr(href)‘).extract_first() #取出链接
20     # print(media_url)
21     media_url = media_url.split(‘/‘)[-1]  #取出音频id
22     # print(media_url)
23     media_name = sound.css("a::attr(title)").extract_first()  #取出音频的名字
24     # print(media_name)
25     list_all.append((media_url,media_name)) #追加到列表
26 # print(list_all)
27 #获取歌曲的真实链接  def定义函数  封装好的
28 def media_api(track_id):
29     api_url =f‘https://www.ximalaya.com/revision/play/v1/audio?id={track_id}&ptype=1‘;
30     response = requests.get(api_url,headers = headers)
31     #print(response.json())
32     #json返回字典类型  提取使用[]
33     data_json = response.json()
34     src = data_json[‘data‘][‘src‘]
35     return src
36 #下载器 用来下载mp4
37 def download_meida(media_url,media_name):
38     response = requests.get(media_url,headers = headers);
39     with open(f‘{media_name}.mp4‘,mode=‘wb‘) as f:  #下载文件 存为相应的文件名
40         f.write(response.content)
41 if __name__ == ‘__main__‘:
42     for url in list_all:
43         print(url[1])
44         true_url= media_api(url[0])  #代表我们的音频id  ，唯一标识符
45         name = url[1]
46         download_meida(true_url,name)

 1 #能发送http请求的库
 2 import requests
 3 import parsel
 4 headers = {
 5 ‘User-Agent‘: ‘Mozilla/5.0 (Windows NT 6.1; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/77.0.3865.120 Safari/537.36‘
 6 }
 7 
 8 def download_meida(media_url,media_name):
 9     response = requests.get(media_url,headers = headers);
10     with open(f‘{media_name}.mp4‘,mode=‘wb‘) as f:
11         f.write(response.content)
12 
13 def media_api(track_id):
14     api_url =f‘https://www.ximalaya.com/revision/play/v1/audio?id={track_id}&ptype=1‘;
15     response = requests.get(api_url,headers = headers)
16     #print(response.json())
17     #json返回字典类型  提取使用[]
18     data_json = response.json()
19     src = data_json[‘data‘][‘src‘]
20     return src
21 
22 #print(media_api(98791745))
23 
24 def get_total_page(page_url):
25     #请求页面
26     response = requests.get(page_url,headers = headers)
27     #获取页面html的内容
28     sel = parsel.Selector(response.text)
29     #通过css选择器找到a标签   .sound-list代表 class属性为sound-list 然后下面的ul 下的li 下的a
30     sound_list = sel.css(‘.sound-list ul li a‘)
31     #只有前30个是页面链接 截取前30个
32     for sound in sound_list[:30]:
33         #extract_first()将对象中的文字提取出来
34         #获取a标签的href属性的内容
35         media_url = sound.css(‘a::attr(href)‘).extract_first()
36         #/youshengshu/16411402/98791745 --只去最后面的id
37         media_url = media_url.split(‘/‘)[-1]
38         # 获取a标签的title属性的内容
39         media_name = sound.css(‘a::attr(title)‘).extract_first()
40         #用yield将整个循环的内容返回
41         yield media_url,media_name
42 
43 if __name__ == ‘__main__‘:
44     #循环页数下载 range代表下载的页数范围
45     for page in range(1,3):
46         meidas = get_total_page(f‘https://www.ximalaya.com/yinyue/23841180/p{page}‘)
47         for media_id,media_name in meidas:
48             #print(media_url, media_name)
49             media_url = media_api(media_id)
50             print(media_url)
51             download_meida(media_url, media_name)