XPath之电影天堂数据爬取

  1 # Author:K
  2 import requests
  3 from lxml import etree
  4 import json
  5 import os
  6 # ！！！！！！！！！！！！！！！！！为什么有几个爬不到？？？？Iron Sky 。 I Spit on Your Grave 。 The Lego Movie 2 这三个爬不到！！！
  7 HEADERS = {
  8     ‘User-Agent‘:‘Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/74.0.3729.108 Safari/537.36‘
  9 }
 10 
 11 BASE_URL = ‘https://www.dytt8.net‘
 12 
 13 def get_urls(url):
 14     page_text = requests.get(url = url,headers = HEADERS).text
 15     tree = etree.HTML(page_text)
 16     detail_urls = tree.xpath(‘//div[@class="co_content8"]/ul//a/@href‘)
 17     # 将detail_urls列表里面的每一个值经过lambda匿名函数处理后再组合成detail_urls
 18     detail_urls = map(lambda url:BASE_URL + url,detail_urls)
 19     return detail_urls
 20 
 21 
 22 def parse_html(detail_urls):
 23     movie_infos = {}  # 用字典形式来保存电影信息
 24     for detail_url in detail_urls:
 25         # 这里的编码有点疑问？？？？？
 26         html = requests.get(url = detail_url,headers = HEADERS).content.decode(‘gbk‘)
 27         tree = etree.HTML(html)
 28         movie_poster = tree.xpath(‘//div[@id="Zoom"]//img/@src‘)[0]
 29         movie_clips = tree.xpath(‘//div[@id="Zoom"]//img/@src‘)[1]
 30         movie_infos["movie_poster"] = movie_poster  # 电影海报
 31         movie_infos["movie_clips"] = movie_clips  # 电影片段
 32 
 33         html_text = tree.xpath(‘//div[@id="Zoom"]//text()‘)
 34         actors = [] # 因为主演有很多人，而且是分行的，所以要创建一个列表存储
 35         for index,text in enumerate(html_text):
 36             if text.startswith(‘◎片　　名‘):
 37                 movie_name = text.replace(‘◎片　　名‘,‘‘).strip()
 38                 # 处理特殊字符，不然的话不能下到本地中，现在只知道这种方法。
 39                 if ‘/‘ in movie_name:
 40                     movie_name = movie_name.replace(‘/‘,‘ or ‘)
 41                 if ‘?‘ in movie_name:
 42                     movie_name = movie_name.replace(‘?‘, ‘ ‘)
 43                 movie_infos["movie_name"] = movie_name
 44 
 45 
 46             elif text.startswith(‘◎年　　代‘):
 47                 movie_time = text.replace(‘◎年　　代‘, ‘‘).strip()
 48                 movie_infos["movie_time"] = movie_time
 49 
 50             elif text.startswith(‘◎产　　地‘):
 51                 movie_place = text.replace(‘◎产　　地‘, ‘‘).strip()
 52                 movie_infos["movie_place"] = movie_place
 53 
 54             elif text.startswith(‘◎类　　别‘):
 55                 movie_type = text.replace(‘◎类　　别‘, ‘‘).strip()
 56                 movie_infos["movie_type"] = movie_type
 57 
 58             elif text.startswith(‘◎语　　言‘):
 59                 movie_language = text.replace(‘◎语　　言‘, ‘‘).strip()
 60                 movie_infos["movie_language"] = movie_language
 61 
 62             elif text.startswith(‘◎豆瓣评分‘):
 63                 movie_score = text.replace(‘◎豆瓣评分‘, ‘‘).strip()
 64                 movie_infos["movie_score"] = movie_score
 65 
 66             elif text.startswith(‘◎片　　长‘):
 67                 movie_length = text.replace(‘◎片　　长‘, ‘‘).strip()
 68                 movie_infos["movie_length"] = movie_length
 69 
 70             elif text.startswith(‘◎导　　演‘):
 71                 movie_director = text.replace(‘◎导　　演‘, ‘‘).strip()
 72                 movie_infos["movie_director"] = movie_director
 73 
 74             elif text.startswith(‘◎主　　演‘):
 75                 first_actor = text.replace(‘◎主　　演‘, ‘‘).strip()
 76                 actors.append(first_actor)
 77                 for i in range(index+1,len(html_text)):
 78                     if html_text[i].startswith(‘◎‘):
 79                         break
 80                     else:
 81                         actors.append(html_text[i].strip())
 82                 movie_infos["movie_actors"] = actors
 83 
 84             elif text.startswith(‘◎简　　介‘):
 85                 for i in range(index+1,len(html_text)):
 86                     if html_text[i].startswith(‘【下载地址】‘) or html_text[i].startswith(‘ ‘) or html_text[i].startswith(‘◎‘):
 87                         break
 88                     else:
 89                         movie_introduction = html_text[i].strip()
 90                 movie_infos[‘movie_introduction‘] = movie_introduction
 91 
 92         download_addr = tree.xpath(‘//td[@bgcolor="#fdfddf"]/a/@href‘)[0]
 93         movie_infos[‘download_addr‘] = download_addr
 94         download(movie_infos["movie_name"],movie_infos) # 持久化存储
 95 
 96 def download(name,dict):
 97     if not os.path.exists(‘电影天堂数据爬取‘):
 98         os.mkdir(‘电影天堂数据爬取‘)
 99 
100     json.dumps(dict) # 将字典转成json格式 另外，json格式转成字典的操作是：json.loads(dict)
101     path = ‘电影天堂数据爬取/‘ + name + ‘.json‘
102     fp = open(path,‘w‘,encoding = ‘utf-8‘)
103     json.dump(dict,fp = fp,ensure_ascii = False)
104     print(name,‘保存成功！‘)
105 
106 
107 if __name__ == ‘__main__‘:
108     for page in range(1,2):
109         url = ‘https://www.dytt8.net/html/gndy/dyzz/list_23_{}.html‘.format(page)
110         detail_urls = get_urls(url)  # 得到所有电影的url
111         parse_html(detail_urls)  # 根据urls得到解析页面得到数据