python利用xpath进行图片爬取（异步高性能测试）

# _*_ coding:utf-8 _*_
"""
@FileName   :6.4k图片解析爬取（异步高性能测试）.py
@CreateTime :2020/8/14 0014 10:01
@Author     : Lurker Zhang
@E-mail     : 289735192@qq.com
@Desc.      :
"""

import requests
from lxml import etree
from setting.config import *
import json
import os
import time
from multiprocessing.dummy import Pool


def main():
    # 图片采集源地址
    # source_url = ‘http://pic.netbian.com/4kmeinv/‘
    # temp_url = ‘http://pic.netbian.com/4kmeinv/index_{}.html‘
    # source_url = ‘http://pic.netbian.com/4kdongman/‘
    # temp_url = ‘http://pic.netbian.com/4kdongman/index_{}.html‘
    source_url = ‘http://pic.netbian.com/4kmingxing/‘
    temp_url = ‘http://pic.netbian.com/4kmingxing/index_{}.html‘
    # 本此采集前多少页,大于1的整数
    page_sum = 136
    all_pic_list_url = []
    if page_sum == 1:
        pic_list_url = source_url
        print(‘开始下载:‘ + pic_list_url)
        all_pic_list_url.append(pic_list_url)
    else:
        # 先采集第一页
        pic_list_url = source_url
        # 调用采集单页图片链接的函数
        all_pic_list_url.append(pic_list_url)
        # 再采集第二页开始后面的页数
        for page_num in range(2, page_sum + 1):
            pic_list_url = temp_url.format(page_num)
            all_pic_list_url.append(pic_list_url)
    # 单页图片多线程解析
    pool1 = Pool(10)
    pool1.map(down_pic, all_pic_list_url)

    print(‘采集完成，本地成功下载{0}张图片,失败{1}张图片。‘.format(total_success, total_fail))
    # 存储已下载文件名列表：
    with open("../depository/mingxing/pic_name_list.json", ‘w‘, encoding=‘utf-8‘) as fp:
        json.dump(pic_name_list, fp)


def down_pic(pic_list_url):
    print("准备解析图片列表页：",pic_list_url)
    # 获取图片列表页的网页数据
    pic_list_page_text = requests.get(url=pic_list_url, headers=headers).text
    tree_1 = etree.HTML(pic_list_page_text)
    # 获取图片地址列表
    pic_show_url_list = tree_1.xpath(‘//div[@class="slist"]/ul//a/@href‘)
    pic_url_list = [get_pic_url(‘http://pic.netbian.com‘ + pic_show_url) for pic_show_url in pic_show_url_list]

    # 开始下载并保存图片(多线程)
    pool2 = Pool(5)
    pool2.map(save_pic, pic_url_list)


def save_pic(pic_url):
    print("准备下载图片：",pic_url)
    global total_success, total_fail, pic_name_list,path
    picname = get_pic_name(pic_url)
    if not picname in pic_name_list:
        # 获取日期作为保存位置文件夹

        pic = requests.get(url=pic_url, headers=headers).content
        try:
            with open(path + picname, ‘wb‘) as fp:
                fp.write(pic)
        except IOError:
            print(picname + "保存失败")
            total_fail += 1
        else:
            pic_name_list.append(picname)
            total_success += 1
            print("成功保存图片:{0},共成功采集{1}张。".format(picname, total_success))

    else:
        print("跳过，已下载过图片：" + picname)
        total_fail += 1


def get_pic_name(pic_url):
    return pic_url.split(‘/‘)[-1]


def get_pic_url(pic_show_url):
    tree = etree.HTML(requests.get(url=pic_show_url, headers=headers).text)
    return ‘http://pic.netbian.com/‘ + tree.xpath(‘//div[@class="photo-pic"]/a/img/@src‘)[0]


if __name__ == ‘__main__‘:
    # 读入已采集图片的名称库，名称存在重复的表示已经采集过将跳过不采集
    if not os.path.exists(‘../depository/mingxing/pic_name_list.json‘):
        with open("../depository/mingxing/pic_name_list.json", ‘w‘, encoding="utf-8") as fp:
            json.dump([], fp)
    with open("../depository/mingxing/pic_name_list.json", "r", encoding="utf-8") as fp:
        pic_name_list = json.load(fp)
    path = ‘../depository/mingxing/‘ + time.strftime(‘%Y%m%d‘, time.localtime()) + ‘/‘
    if not os.path.exists(path):
        os.mkdir(path)
    # 记录本次采集图片的数量
    total_success = 0
    total_fail = 0
    main()