Python高级应用程序设计任务

#coding:utf-8
from DataOutput import DataOutput
from UrlManager import UrlManager
from HtmlParser import HtmlParser
from SeabornShow import SeabornShow
from HtmlDownloader import HtmlDownloader
import seaborn as sns

class SpiderMan(object):
    def __init__(self):
        self.manager = UrlManager()
        self.downloader = HtmlDownloader()
        self.parser = HtmlParser()
        self.output = DataOutput()
        self.show = SeabornShow()

    def crawl(self,root_url):
        self.manager.add_new_url(root_url)
        while (self.manager.has_new_url() and self.manager.old_url_size() < 100):
            try:
                new_url = self.manager.get_new_url()
                print("》》开始下载页面内容")
                html = self.downloader.download(new_url)
                print("》》开始接解析下载的页面")
                new_urls,data = self.parser.parser(new_url,html)
                self.output.store_data(data)
            except:
                print("crawl failed")
        print("》》对解析的数据进行mysql数据库持久化操作")
        self.output.output_mysql()
        # 数据帧格式数据
        df = self.output.mysql_to_pandas()
        print("》》散点图展示奥迪油耗跟价格的关系")
        self.show.show(df)

if __name__ == "__main__":
    spider_man = SpiderMan()
    aodi = "http://car.bitauto.com/tree_chexing/mb_9/"  # 奥迪列表页地址：
    spider_man.crawl(aodi)

#coding:utf-8

‘‘‘
url管理器
‘‘‘
class UrlManager(object):
    def __init__(self):
        self.new_urls = set()
        self.old_urls = set()

    def has_new_url(self):
        ‘‘‘
        判断是否有url未被爬取
        :return:
        ‘‘‘
        return self.new_url_size() != 0

    def get_new_url(self):
        ‘‘‘
        获取url
        :return:
        ‘‘‘
        if self.has_new_url():
            new_url = self.new_urls.pop()
            self.old_urls.add(new_url)
            return new_url
        else:
            return None

    def add_new_url(self,url):
        ‘‘‘
        增加url
        :param url:
        :return:
        ‘‘‘
        if url is None:
            return
        ‘‘‘
        增加时去重跟判断以处理的url避免重复处理出现死循环
        ‘‘‘
        if url not in self.new_urls and url not in self.old_urls:
            self.new_urls.add(url)

    def add_new_urls(self,urls):
        ‘‘‘
        增加一组url
        :param urls:
        :return:
        ‘‘‘
        if urls is None or len(urls)==0:
            return
        for url in urls:
            self.add_new_url(url)

    def new_url_size(self):
        return len(self.new_urls)

    def old_url_size(self):
        return len(self.old_urls)

import pymysql as ps
import pandas as pd

class MysqlHelper:
    def __init__(self, host, user, password, database, charset):
        self.host = host
        self.user = user
        self.password = password
        self.database = database
        self.charset = charset
        self.db = None
        self.curs = None
    # 数据库连接
    def open(self):
        self.db = ps.connect(host=self.host, user=self.user, password=self.password,database=self.database)
        self.curs = self.db.cursor()
    # 数据库关闭
    def close(self):
        self.curs.close()
        self.db.close()
    # 数据增删改
    def aud(self, sql, params):
        self.open()
        try:
            row = self.curs.execute(sql, params)
            self.db.commit()
            self.close()
            return row
        except :
            print(‘cud出现错误‘)
            self.db.rollback()
            self.close()
            return 0
    # 解析为pandas
    def findPandas(self,sql):
        self.open()
        try:
            df = pd.read_sql(sql=sql,con=self.db)
            return df
        except:
            print(‘解析为pandas出现错误‘)

class AODIItems(object):
    def __init__(self,name,titleImgUrl,referencePrice,guidePrice,displacement,oilConsumption):
        self.name = name
        self.titleImgUrl = titleImgUrl
        self.referencePrice = referencePrice
        self.guidePrice = guidePrice
        self.displacement = displacement
        self.oilConsumption = oilConsumption