爬虫入门-4-3.爬取豆瓣电影

 1 import requests
 2 
 3 url = "https://movie.douban.com/cinema/nowplaying/changsha/"
 4 proxy = {
 5     ‘HTTP‘: ‘116.209.53.143:9999‘
 6 }
 7 headers = {
 8     ‘User-Agent‘: ‘Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36‘
 9                   ‘ (KHTML, like Gecko) Chrome/72.0.3626.121 Safari/537.36‘,
10     ‘Host‘: ‘movie.douban.com‘
11 }
12 response = requests.get(url, headers=headers, proxies=proxy)
13 with open(‘movie.html‘, ‘w‘, encoding=‘utf-8‘) as f:
14     f.write(response.content.decode(‘utf-8‘))

 1 from lxml import etree
 2 
 3 parser = etree.HTMLParser(encoding=‘utf-8‘)
 4 html = etree.parse(‘movie.html‘, parser=parser)
 5 # 1.获取ul等于lists的第一个ul标签(正在上映的电影)
 6 ul = html.xpath("//ul[@class=‘lists‘]")[0]
 7 # 2.获取ul=lists标签下所有的li标签
 8 lis = ul.xpath("./li")
 9 movies = list()
10 for li in lis:
11     # 3.获取li标签的各个属性,返回的是一个列表,取第一个元素
12     title = li.xpath("@data-title")[0]
13     duration = li.xpath("@data-duration")[0]
14     region = li.xpath("@data-region")[0]
15     director = li.xpath("@data-director")[0]
16     actors = li.xpath("@data-actors")[0]
17     image = li.xpath(".//img/@src")[0]
18     movie = {
19         ‘片名‘: title,
20         ‘时长‘: duration,
21         ‘区域‘: region,
22         ‘导演‘: director,
23         ‘演员‘: actors,
24         ‘海报‘: image
25     }
26     movies.append(movie)
27 for m in movies:
28     print(m)