Python 爬虫学习3 -简单抓取小说网信息

#coding:utf-8  #为了正常转码 必写
import codecs  #为下面新建excel，转码正确准备得一个包

__author__ = ‘Administrator‘
import requests
from bs4 import BeautifulSoup
"""
get_html函数是为了抓取对应url的html页面，然后返回这个页面。
其实也可以全部写入一个函数，不过这样就会显得函数很臃肿。
将这种公共函数独立来写，进行封装，有助于以后重复利用。
"""
def get_html(url):     
    try:
        r = requests.get(url,timeout = 3000)
        r.raise_for_status
        r.encoding = ‘utf-8‘
        return r.text
    except:
        return
"""
get_content函数是用来提取你所需要的信息的，并把信息写入excel表格。

"""
def get_content(url):
    url_list = []
    html = get_html(url).encode(‘utf-8‘)
    soup = BeautifulSoup(html, "html.parser")
    category_list = soup.find_all(‘div‘, class_=‘index_toplist mright mbottom‘)
    history_list = soup.find_all(‘div‘, class_ = ‘index_toplist mbottom‘)
    for cate in category_list:
        name = cate.find(‘div‘, class_ = ‘toptab‘).span.text
        name = name.encode(‘utf-8‘)
        with codecs.open(‘novel_list.csv‘,‘a+‘,‘utf-8‘) as f:
            f.write(‘\n小说种类:{}\n‘.format(name))

        book_list = cate.find(‘div‘, class_ = ‘topbooks‘).find_all(‘li‘)
        for book in book_list:
            link = ‘http://www.qu.la/‘+book.a[‘href‘]
            title = book.a[‘title‘].encode(‘utf-8‘)
            url_list.append(link)
            with codecs.open(‘novel_list.csv‘,‘a+‘,‘utf-8‘) as f:
                f.write(‘小说名：{} \t 小说地址:{}\n‘.format(title,link))

    for cate in history_list:
        name = cate.find(‘div‘, class_=‘toptab‘).span.string
        with codecs.open(‘novel_list.csv‘,‘a+‘,‘utf-8‘) as f:
            f.write("\n小说种类：{} \n".format(name))

        general_list = cate.find(style=‘display: block;‘) #找到总排行榜
        book_list = general_list.find_all(‘li‘)
        for book in book_list:
            link = ‘http://www.qu.la/‘ + book.a[‘href‘]
            title = book.a[‘title‘]
            url_list.append(link)
            with codecs.open(‘novel_list.csv‘,‘a+‘,‘utf-8‘) as f:
                f.write("小说名：{:<} \t 小说地址：{:<} \n".format(title, link))
    return url_list



def main():
    # 排行榜地址
    base_url = ‘http://www.qu.la/paihangbang/‘
    # 获取排行榜中所有小说链接
    url_list = get_content(base_url)


if __name__==‘__main__‘:
    main()