标签:def turn rom headers color head __name__ odi one
1 import requests 2 from requests.exceptions import RequestException 3 import re 4 import json 5 from multiprocessing import Pool 6 7 def get_one_page(url): 8 try: 9 headers={‘user-agent‘:‘Mozilla/4.0 (compatible; MSIE 8.0; Windows NT 6.1; Win64; x64; Trident/4.0) 32-bit IE on 64-bit Windows 7‘} 10 response=requests.get(url,headers=headers) 11 if response.status_code==200: 12 return response.text 13 return None 14 except RequestException: 15 return None 16 17 def parse_one_page(html): 18 pattern=re.compile(‘<dd>.*?board-index.*?>(\d+)</i>.*?data-src="(.*?)".*?name"><a.*?>(.*?)</a>.*?star">(.*?)</p>.*?releasetime">(.*?)</p>.*?integer">(.*?)</i>.*?fraction">(.*?)</i>.*?</dd>‘,re.S) 19 items=re.findall(pattern,html) 20 for item in items: 21 yield{ 22 ‘index‘:item[0], 23 ‘image‘:item[1], 24 ‘title‘:item[2], 25 ‘actor‘:item[3].strip()[3:], 26 ‘time‘:item[4].strip()[5:], 27 ‘score‘:item[5]+item[6] 28 } 29 30 def write_to_file(content): 31 with open(‘result.txt‘,‘a‘,encoding=‘utf-8‘) as f: 32 f.write(json.dumps(content,ensure_ascii=False)+‘\n‘) 33 f.close() 34 35 36 def main(offset): 37 url=‘http://maoyan.com/board/4?offset=‘+str(offset) 38 html=get_one_page(url) 39 for item in parse_one_page(html): 40 print(item) 41 write_to_file(item) 42 43 if __name__==‘__main__‘: 44 pool=Pool() 45 pool.map(main,[i*10 for i in range(10)])
标签:def turn rom headers color head __name__ odi one
原文地址:http://www.cnblogs.com/realmonkeykingsun/p/7857642.html