爬取今日头条历史图集将信息保存到MongDB，并且下载图片到本地

def get_response(url,offset):

    header = {‘User-Agent‘: ‘Mozilla/5.0 (X11;Ubuntu;Linux x86_64; rv:52.0) Gecko/20100101 Firefox/52.0‘}

    data = {
        ‘offset‘:offset,
        ‘format‘:‘json‘,
        ‘keyword‘:‘历史‘,
        ‘autoload‘:‘true‘,
        ‘count‘:‘20‘,
        ‘cur_tab‘:‘3‘,
        ‘from‘:‘gallery‘,
    }
    response = requests.get(url,headers=header,params=data)
    response.encoding = ‘utf-8‘
    try:
        if response.status_code == 200:
            html = response.text
            return html
        return None
    except RequestException:
        print(‘请求页面出错！‘)
        return None

url = ‘https://www.toutiao.com/search_content/‘

for offset in range(0,121,20):
    html = get_response(url,str(offset))

def get_data(dic):

    if ‘data‘ in dic.keys():
        for item in dic[‘data‘]:
            yield item[‘article_url‘]

def handle_detail_html(html,url):
    from bs4 import BeautifulSoup
    import re
    from urllib.parse import urljoin

    soup = BeautifulSoup(html,‘lxml‘)

    title_text = soup.find(‘title‘).get_text()

    pattern = re.compile(‘gallery: JSON.parse\((.*?)\),‘)
    result = re.findall(pattern, html)

    pattern = re.compile("(\w{19,23})", re.S)
    result_end = re.findall(pattern, result[0])

    img_url = []
    url_header_1 = ‘http://p3.pstatp.com/origin/pgc-image/‘
    url_header_2 = ‘http://p3.pstatp.com/origin/‘
    for each_url in set(result_end):
        url_1 = Accessable_download(urljoin(url_header_1,each_url))
        url_2 = Accessable_download(urljoin(url_header_2,each_url))
        if url_1:
            img_url.append(urljoin(url_header_1,each_url))
        elif url_2:
            img_url.append(urljoin(url_header_2,each_url))

    data_dic = {
        ‘url‘:url,
        ‘title‘:title_text,
        ‘img_list‘:img_url
    }

    return data_dic

# coding=utf-8

‘‘‘
解析Ajax请求抓取今日头条图集并将信息存储到mongoDB并且下载图片到本地
‘‘‘

import requests
from requests.exceptions import RequestException

# 返回搜索页的json数据
def get_response(url,offset):

    header = {‘User-Agent‘: ‘Mozilla/5.0 (X11;Ubuntu;Linux x86_64; rv:52.0) Gecko/20100101 Firefox/52.0‘}

    data = {
        ‘offset‘:offset,
        ‘format‘:‘json‘,
        ‘keyword‘:‘历史‘,
        ‘autoload‘:‘true‘,
        ‘count‘:‘20‘,
        ‘cur_tab‘:‘3‘,
        ‘from‘:‘gallery‘,
    }
    
    response = requests.get(url,headers=header,params=data)
    response.encoding = ‘utf-8‘
    
    try:
        if response.status_code == 200:
            html = response.text
            return html
        return None
    
    except RequestException:
        print(‘请求页面出错！‘)
        return None

# 将json数据转为python数据。
def loads_python(data):
    import os,json
    
    dic_data = json.loads(data)
    
    return dic_data

# 将json中的每个图集的url找出来，将函数作为迭代器每次输出一个url。
def get_data(dic):

    if ‘data‘ in dic.keys():
        for item in dic[‘data‘]:
            yield item[‘article_url‘]

# 获取每个图集页面的html信息
def get_each_html(url):

    header = {‘User-Agent‘: ‘Mozilla/5.0 (X11;Ubuntu;Linux x86_64; rv:52.0) Gecko/20100101 Firefox/52.0‘}
    try:
        html  = requests.get(url,headers=header)
        html.encoding = ‘utf-8‘
        if html.status_code == 200:
            return html.text
        return None

    except RequestException:
        print(‘图集页面请求错误！‘)
        return None

# 测试图片的url是否可以访问，如果可以就下在图片并保存在本地，然后返回可以访问的图片url。
def Accessable_download(url):
    import os
    from hashlib import md5
    from urllib.request import urlretrieve

    print(‘正在尝试：‘,url)

    filepath = ‘{0}/{1}/{2}.{3}‘.format(os.getcwd(),‘今日头条可爱图集‘,md5(url.encode(‘utf-8‘)).hexdigest(),‘.jpg‘)
    
    if not os.path.exists(os.path.join(os.getcwd(),‘今日头条可爱图集‘)):
        os.mkdir(os.path.join(os.getcwd(),‘今日头条可爱图集‘))
    
    header = {‘User-Agent‘: ‘Mozilla/5.0 (X11;Ubuntu;Linux x86_64; rv:52.0) Gecko/20100101 Firefox/52.0‘}
    
    try:
        response = requests.get(url,headers=header)
        
        if response.status_code == 200:
            urlretrieve(url,filepath)
            print(‘已下载完成：‘,url)
            return True
        
        return False
    
    except:
        return False

# 提取每个图集页面的信息，标题，图片的url，返回字典格式的信息。
def handle_detail_html(html,url):
    from bs4 import BeautifulSoup
    import re
    from urllib.parse import urljoin

    soup = BeautifulSoup(html,‘lxml‘)

    title_text = soup.find(‘title‘).get_text()

    pattern = re.compile(‘gallery: JSON.parse\((.*?)\),‘)
    result = re.findall(pattern, html)

    pattern = re.compile("(\w{19,23})", re.S)
    result_end = re.findall(pattern, result[0])

    img_url = []
    
    url_header_1 = ‘http://p3.pstatp.com/origin/pgc-image/‘
    url_header_2 = ‘http://p3.pstatp.com/origin/‘
    
    for each_url in set(result_end):
        
        url_1 = Accessable_download(urljoin(url_header_1,each_url))
        url_2 = Accessable_download(urljoin(url_header_2,each_url))
        
        if url_1:
            img_url.append(urljoin(url_header_1,each_url))
        elif url_2:
            img_url.append(urljoin(url_header_2,each_url))

    data_dic = {
        ‘url‘:url,
        ‘title‘:title_text,
        ‘img_list‘:img_url
    }

    return data_dic

# 连接数据库，将爬取的信息存储到数据库
def mongo_db_conn(data):
    from pymongo import MongoClient
    
    try:
        conn = MongoClient(‘localhost‘,27017)
        db = conn.maoyan
        collection = db.img_data
        collection.insert_one(data)
        
    except:
        print(‘数据未写入‘)

    conn.close()

# 主函数
def main(url):
    
    i = 0
    for offset in range(0,121,20):

        html = get_response(url,str(offset))

        dic_data = loads_python(html)

        for each_url in get_data(dic_data):
            
            i += 1
            print(i, each_url)
            
            each_html = get_each_html(each_url)
            result = handle_detail_html(each_html,each_url)
            mongo_db_conn(result)
            
            print(i,result)

if __name__ == ‘__main__‘:
    
    # 初始url
    url = ‘https://www.toutiao.com/search_content/‘
    main(url)