第三篇 - 爬取豆瓣电影网

 1 import requests
 2 import json
 3 
 4 #请求头
 5 headers = {
 6     ‘Accept‘ : ‘*/*‘,
 7     ‘Accept-Encoding‘ : ‘gzip, deflate, br‘,
 8     ‘Connection‘ : ‘keep-alive‘,
 9     ‘Host‘ : ‘movie.douban.com‘,
10     ‘Referer‘ : ‘https://movie.douban.com/typerank?type_name=%E6%88%98%E4%BA%89&type=22&interval_id=100:90&action=‘,
11     ‘User-Agent‘ : ‘‘,
12     ‘X-Requested-With‘ : ‘XMLHttpRequest‘
13 }
14 
15 def write_films(file_name):
16     ‘‘‘将电影名+评分+url存入movie.txt文件中‘‘‘
17     num_list = [0,20,40,60,76]#动态页面，20为一页
18     index =1#获取电影的个数
19     with open(file_name,‘w‘,encoding=‘utf-8‘) as f:
20         for num in num_list:
21             url_1 = ‘https://movie.douban.com/j/chart/top_list?type=22&interval_id=100%3A90&action=&start=‘
22             url = url_1 + str(num) + ‘&limit=20‘#每一页的url
23             response = requests.get(url=url,headers=headers)
24             r = response.text
25             response_list = json.loads(r)
26             for item in response_list:
27                 rating = eval(item.get(‘rating‘)[0])
28                 url = item.get(‘url‘)
29                 title = item.get(‘title‘)
30                 f.write(str(index)+‘:‘+title+‘,‘+str(rating)+‘,‘+url+‘\n‘)
31                 index += 1
32 
33 if __name__ == ‘__main__‘:
34     write_films(‘movie.txt‘)