使用requests + beautifulsoup 写一个简单的漫画爬虫

标签：menu   .com   sleep   headers   with open   linux   com   lin   查找   

from bs4 import BeautifulSoup
import requests
import os
from time import sleep

class get_img(object):
    def get_url(self):
        self.new_url = []
        url = "http://www.gumua.com/Manhua/28307.html"          #首页网址
        headers = {‘User-Agent‘: ‘Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:22.0) Gecko/20100101 Firefox/22.0‘}
        res = requests.get(url=url,headers=headers)             #获取网页信息
        soup = BeautifulSoup(res.content,"lxml")                #通过bs4对网页进行解析
        divs =soup.find(class_="d_menu")                        #查找body
        lis = divs.select("li > a")                             #查找li下的a标签
        for i in lis:
            link = i.get("href")
            self.new_url.append("http://www.gumua.com"+link)   #获取漫画的所有网址
            num=i.string                                       #漫画集数
        self.new_url.reverse()                                 #倒叙输出所有地址
        return self.new_url
    def img(self):
        self.get_url()
        imgs = []
        headers={"User-Agent": "Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/55.0.2883.87 Safari/537.36"}
        url = self.new_url
        for link in url:
            res = requests.get(url=link,headers=headers)        #获取所有网址的内容
            soup = BeautifulSoup(res.content,"lxml")            #通过bs4对网页进行解析
            #print(soup)
            body = soup.find("div",class_="r_img ")             #查找属性
            srcs = body.select("img")                           #查找图片
            for link in srcs:
                imgs.append(link.get("src"))                    #查找图片链接地址，追加到list中

        for i in range(len(imgs)):
            root = "/Users/caojialin/work2018/木乃伊新娘/"         #本地目录
            path = root + str(i)+".jpg"                          #路径和名称
            try:
                if not os.path.exists(root):
                    os.mkdir(root)
                if not os.path.exists(path):
                    r = requests.get(imgs[i])
                    r.raise_for_status()
                    sleep(0.1)
                    # 使用with语句可以不用自己手动关闭已经打开的文件流
                    with open(path, "wb") as f:  # 开始写文件，wb代表写二进制文件
                        f.write(r.content)
                    print("爬取完成")
                else:
                    print("文件已存在")
            except Exception as e:
                print("爬取失败:" + str(e))

使用requests + beautifulsoup 写一个简单的漫画爬虫

标签：menu   .com   sleep   headers   with open   linux   com   lin   查找   

原文地址：https://www.cnblogs.com/paoye/p/9767194.html