第一次爬虫实例

import requests
from bs4 import BeautifulSoup
from concurrent.futures import ThreadPoolExecutor


root = ‘http://www.btrenren.com‘

def  bt_video(num):
    url = "http://www.btrenren.com/index.php/Index/index/p/%s.html"%num
    head = {"User-Agent":"Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/57.0.2987.98 Safari/537.36 LBBROWSER"}

    html = requests.get(url=url,headers=head)
    soup = BeautifulSoup(html.content,‘lxml‘)
    data = soup.find(‘div‘,class_="ml")
    data = data.find_all(class_="title")
    with open("video.txt", ‘a‘, encoding=‘utf-8‘) as f:
        for i in data:
            #电影名
            name = i.find_all(‘font‘)[1].get_text().split(‘/‘)[0]
            #日期国家
            vlue = i.find_all(class_=‘des‘)[0].get_text().split(‘\n‘)[0]
            #演员
            vlue1 = i.find_all(class_=‘des‘)[0].get_text().split(‘\n‘)[1]
            #豆瓣评分
            douban = i.find(class_="rt").get_text()
            #网页路径
            href = root + i.find_all(‘a‘)[1].get(‘href‘)
            #print(name,vlue,vlue1,douban,href)
            f.write(‘%s,%s,%s,%s,%s\n‘ % (name, vlue, vlue1, douban,href))
    print(num)


def num_video():
    url = "http://www.btrenren.com/index.php/Index/index/p/1.html"
    head = {"User-Agent": "Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/57.0.2987.98 Safari/537.36 LBBROWSER"}
    html = requests.get(url=url, headers=head)
    soup = BeautifulSoup(html.content, ‘lxml‘)
    number = soup.find(‘ul‘,class_="pagelist")
    number = number.find_all(‘span‘,class_="rows")
    num = number[0].get_text().split(‘ ‘)[1]
    return num





if __name__ == "__main__":
    executor = ThreadPoolExecutor(max_workers=30)
    num = int(num_video())
    for i in range(1,num):
        executor.submit(bt_video,i)
    executor.shutdown()
    exit()