爬取今日头条街拍美图

import requests
from urllib.parse import urlencode
import os
from hashlib import md5
from multiprocessing.pool import Pool


def get_page(offset):
    """加载一个ajax请求的结果"""
    params = {
        ‘offset‘:offset,
        ‘format‘:‘json‘,
        ‘keyword‘:‘街拍‘,
        ‘autoload‘:‘true‘,
        ‘count‘:‘20‘,
        ‘cur_tab‘:‘3‘,
        ‘from‘: ‘gallery‘
    }
    url = ‘http://www.toutiao.com/search_content/?‘ + urlencode(params)
    try:
        response = requests.get(url)
        if response.status_code == 200:
            return response.json()
    except requests.ConnectionError:
        return None


def get_images(json):
    """解析方法，提取每条数据的image_list字段中的图片链接，将图片链接和所属标题一同返回"""
    if json.get(‘data‘):
        for item in json.get(‘data‘):
            title = item.get(‘title‘)
            images = item.get(‘image_list‘)
            if images:
                for image in images:
                    yield {
                        ‘image‘:image.get(‘url‘),
                        ‘title‘:title
                    }


def save_image(item):
    """保存图片方法，根据item中的title来创建文件夹，然后请求图片链接，获取图片的二进制数据，并写入文件。图片命名使用了md5"""
    if not os.path.exists(item.get(‘title‘)):
        os.mkdir(item.get(‘title‘))
    try:
        local_image_url = item.get(‘image‘)
        new_image_url = local_image_url.replace(‘list‘,‘large‘)
        response = requests.get(‘http:‘ + new_image_url)
        if response.status_code == 200:
            file_path = ‘{0}/{1}.{2}‘.format(item.get(‘title‘),md5(response.content).hexdigest(),‘jpg‘)
            if not os.path.exists(file_path):
                with open(file_path,‘wb‘) as f:
                    f.write(response.content)
            else:
                print(‘Already Download‘,file_path)
    except requests.ConnectionError:
        print(‘Failed to Save Image‘)


def main(offset):
    """主函数"""
    json = get_page(offset)
    for item in get_images(json):
        print(item)
        save_image(item)


GROUP_START = 1
GROUP_END = 5

if __name__ == ‘__main__‘:
    pool = Pool()
    groups = ([x * 20 for x in range(GROUP_START,GROUP_END + 1)])
    pool.map(main,groups)
    pool.close()
    pool.join()