Python3编写网络爬虫04-爬取猫眼电影排行实例

import requests
#爬取第一页 页面信息
def get_one_page(url):
header = {
"User-Agent":"Mozilla/5.0 (Windows; U; Windows NT 6.1; en-US) AppleWebKit/534.16 (KHTML, like Gecko) Chrome/10.0.648.133 Safari/534.16",
}
response = requests.get(url,headers=header)
if response.status_code == 200:#判断是否请求成功
return response.text
return None

def main():
url = ‘http://maoyan.com/board/4‘
html = get_one_page(url)#调用请求函数
print(html)
main()

import requests
import re
def get_one_page(url):
header = {
"User-Agent":"Mozilla/5.0 (Windows; U; Windows NT 6.1; en-US) AppleWebKit/534.16 (KHTML, like Gecko) Chrome/10.0.648.133 Safari/534.16",
}
response = requests.get(url,headers=header)
if response.status_code == 200:
return response.text
return None

def parse_one_page(html):
pattern = re.compile(
‘<dd>.*?board-index.*?>(.*?)</i>.*?data-src="(.*?)".*?name.*?a.*?>(.*?)</a>.*?star.*?>(.*?)</p>.*?releasetime.*?>(.*?)</p>.*?integer.*?>(.*?)</i>.*?fraction.*?>(.*?)</i>.*?</dd>‘,re.S
)
items = re.findall(pattern,html)
print(items)
# 定义一个main函数 调用get_one_page 发送请求 打印结果
def main():
url = ‘http://maoyan.com/board/4‘
html = get_one_page(url)
# print(html)
parse_one_page(html)
main()

import requests
import re


def get_one_page(url):
header = {
"User-Agent":"Mozilla/5.0 (Windows; U; Windows NT 6.1; en-US) AppleWebKit/534.16 (KHTML, like Gecko) Chrome/10.0.648.133 Safari/534.16",
}
response = requests.get(url,headers=header)
if response.status_code == 200:
return response.text
return None

def parse_one_page(html):# html为网页源码
pattern = re.compile(
‘<dd>.*?board-index.*?>(.*?)</i>.*?data-src="(.*?)".*?name.*?a.*?>(.*?)</a>.*?star.*?>(.*?)</p>.*?releasetime.*?>(.*?)</p>.*?integer.*?>(.*?)</i>.*?fraction.*?>(.*?)</i>.*?</dd>‘,re.S
)#定义规则
items = re.findall(pattern,html)#查找整个页面
# print(items)
#遍历结果生成字典
for item in items:
yield {
‘index‘:item[0],
‘image‘: item[1],
‘title‘: item[2].strip(),
‘actor‘: item[3].strip()[3:] if len(item[3]) > 3 else‘‘,
‘time‘: item[4].strip()[5:] if len(item[4]) > 5 else ‘‘,
‘score‘: item[5].strip()+item[6].strip()
}
#返回一个生成器 yield

# 定义一个main函数 调用get_one_page 发送请求 打印结果
def main():
url = ‘http://maoyan.com/board/4‘
html = get_one_page(url)
# print(html)
for item in parse_one_page(html):#遍历生成器
print(item)
main()

写入文件

将提取结果 写入文件 通过json库 的dumps（） 实现字典的序列化 指定ensure_ascii 参数为 False 
#写入文件
def write_to_file(content):
with open(‘result.txt‘,‘a‘,encoding=‘utf-8‘) as f:
print(type(json.dumps(content)))
f.write(json.dumps(content,ensure_ascii=False)+‘\n‘)

import requests
import re
import json
# 请求页面
def get_one_page(url):
header = {
"User-Agent":"Mozilla/5.0 (Windows; U; Windows NT 6.1; en-US) AppleWebKit/534.16 (KHTML, like Gecko) Chrome/10.0.648.133 Safari/534.16",
}
response = requests.get(url,headers=header)
if response.status_code == 200:
return response.text
return None

#解析页面
def parse_one_page(html):# html为网页源码
pattern = re.compile(
‘<dd>.*?board-index.*?>(.*?)</i>.*?data-src="(.*?)".*?name.*?a.*?>(.*?)</a>.*?star.*?>(.*?)</p>.*?releasetime.*?>(.*?)</p>.*?integer.*?>(.*?)</i>.*?fraction.*?>(.*?)</i>.*?</dd>‘,re.S
)#定义规则
items = re.findall(pattern,html)#查找整个页面
# print(items)
#遍历结果生成字典
for item in items:
yield {
‘index‘:item[0],
‘image‘: item[1],
‘title‘: item[2].strip(),
‘actor‘: item[3].strip()[3:] if len(item[3]) > 3 else‘‘,
‘time‘: item[4].strip()[5:] if len(item[4]) > 5 else ‘‘,
‘score‘: item[5].strip()+item[6].strip()
}
#返回一个生成器 yield

#写入文件
def write_to_file(content):
with open(‘result.txt‘,‘a‘,encoding=‘utf-8‘) as f:
print(type(json.dumps(content)))
f.write(json.dumps(content,ensure_ascii=False)+‘\n‘)


# 定义一个main函数 调用get_one_page 发送请求 打印结果
def main():
url = ‘http://maoyan.com/board/4‘
html = get_one_page(url)
# print(html)
for item in parse_one_page(html):#遍历生成器
write_to_file(item)
main()

def main(offset):
url = ‘http://maoyan.com/board/4?offset=‘ + str(offset)
html = get_one_page(url)
# print(html)
for item in parse_one_page(html):#遍历生成器
write_to_file(item)

if __name__ == ‘__main__‘:
for i in range(10):
main(offset = i *10)

#-*-coding:utf-8-*-

import requests #请求库
import re #正则模块
import json #json模块
import time #时间模块
from requests.exceptions import RequestException#捕获异常模块
# 请求页面
def get_one_page(url):
#异常处理
try:
header = {
"User-Agent":"Mozilla/5.0 (Windows; U; Windows NT 6.1; en-US) AppleWebKit/534.16 (KHTML, like Gecko) Chrome/10.0.648.133 Safari/534.16",
}
response = requests.get(url,headers=header)
# 判断状态码是否为200
if response.status_code == 200:
return response.text
return None
except RequestException:
return None

#解析页面
def parse_one_page(html):# html为网页源码
#定义爬取规则
pattern = re.compile(
‘<dd>.*?board-index.*?>(.*?)</i>.*?data-src="(.*?)".*?name.*?a.*?>(.*?)</a>.*?star.*?>(.*?)</p>.*?releasetime.*?>(.*?)</p>.*?integer.*?>(.*?)</i>.*?fraction.*?>(.*?)</i>.*?</dd>‘,re.S
)
# 查找整个页面
items = re.findall(pattern,html)
# print(items)
#遍历结果生成字典
for item in items:
yield {
‘index‘:item[0],
‘image‘: item[1],
‘title‘: item[2].strip(),
‘actor‘: item[3].strip()[3:] if len(item[3]) > 3 else‘‘,
‘time‘: item[4].strip()[5:] if len(item[4]) > 5 else ‘‘,
‘score‘: item[5].strip()+item[6].strip()
}
#返回一个生成器 yield

#写入文件
def write_to_file(content):
with open(‘result.txt‘,‘a‘,encoding=‘utf-8‘) as f:
# print(type(json.dumps(content)))
# ensure_ascii=False 保证输出结果为中文
f.write(json.dumps(content,ensure_ascii=False)+‘\n‘)


# 定义一个main函数 调用get_one_page 发送请求 参数offset 网页偏移量
def main(offset):
#拼接url地址
url = ‘http://maoyan.com/board/4?offset=‘ + str(offset)
# 请求函数
html = get_one_page(url)
# print(html)
# 解析函数 和 文件保存函数
for item in parse_one_page(html):#遍历生成器
write_to_file(item)

if __name__ == ‘__main__‘:
for i in range(10):
main(offset = i *10)
#延时处理
time.sleep(3)