一个完整的大作业--‘’数据观”官方网站数据爬取

import requests
from bs4 import BeautifulSoup


url = ‘http://www.cbdio.com/node_2568.htm‘
res = requests.get(url)
res.encoding = ‘utf-8‘
soup = BeautifulSoup(res.text, ‘html.parser‘)

for items in soup.select(‘li‘):
    if len(items.select(‘.cb-media-title‘))>0:
        title=items.select(‘.cb-media-title‘)[0].text#标题
        url1=items.select(‘a‘)[0][‘href‘]
        url2=‘http://www.cbdio.com/{}‘.format(url1)#链接

        resd=requests.get(url2)
        resd.encoding=‘utf-8‘
        soupd=BeautifulSoup(resd.text,‘html.parser‘)
        source=soupd.select(‘.cb-article-info‘)[0].text.strip()#来源
        content=soupd.select(‘.cb-article‘)[0].text#内容
        print("################################################################################")
        print(‘标题：‘,title,‘\t链接:‘,url2,source)

url=‘http://www.cbdio.com/node_2568.htm‘
res = requests.get(url)
res.encoding = ‘utf-8‘
soup = BeautifulSoup(res.text, ‘html.parser‘)
contentls=[]
for item in soup.select(‘li‘):
    if len(item.select(‘.cb-media-title‘))>0:
        url1=item.select(‘a‘)[0][‘href‘]
        url2=‘http://www.cbdio.com/{}‘.format(url1)
        resd=requests.get(url2)
        resd.encoding=‘utf-8‘
        soupd=BeautifulSoup(resd.text,‘html.parser‘)
        cont=soupd.select(‘.cb-article‘)[0].text#内容
        contentls.append(cont)
print(contentls)
words=jieba.lcut(content)
ls=[]
counts={}
for word in words:
    ls.append(word)
    if len(word)==1:
        continue
    else:
        counts[word]=counts.get(word,0)+1

items = list(counts.items())
items.sort(key = lambda x:x[1], reverse = True)
for i in range(10):
    word , count = items[i]
    print ("{:<5}{:>2}".format(word,count))


#词云制作
from wordcloud import WordCloud
import matplotlib.pyplot as plt

cy = WordCloud(font_path=‘msyh.ttc‘).generate(content)
plt.imshow(cy, interpolation=‘bilinear‘)
plt.axis("off")
plt.show()

import requests
from bs4 import BeautifulSoup


def getTheContent(url1):
    res = requests.get(url1)
    res.encoding = ‘utf-8‘
    soup = BeautifulSoup(res.text, ‘html.parser‘)
    item={}
    item[‘title‘]=soup.select(‘.cb-article-title‘)[0].text#标题
    item[‘url‘]=url1#链接
    resd=requests.get(item[‘url‘])
    resd.encoding=‘utf-8‘
    soupd=BeautifulSoup(resd.text,‘html.parser‘)
    item[‘source‘]=soupd.select(‘.cb-article-info‘)[0].text.strip()#来源
    item[‘content‘]=soupd.select(‘.cb-article‘)[0].text#内容
    return(item)

def getOnePage(pageurl):
    res = requests.get(pageurl)
    res.encoding = ‘utf-8‘
    soup = BeautifulSoup(res.text, ‘html.parser‘)
    itemls=[]
    for item in soup.select(‘li‘):
        if len(item.select(‘.cb-media-title‘))>0:
            url1=item.select(‘a‘)[0][‘href‘]
            url2=‘http://www.cbdio.com/{}‘.format(url1)
            itemls.append(getTheContent(url2))
    return(itemls)



  
#结巴词频统计
import jieba

url=‘http://www.cbdio.com/node_2568.htm‘
res = requests.get(url)
res.encoding = ‘utf-8‘
soup = BeautifulSoup(res.text, ‘html.parser‘)
contentls=[]
for item in soup.select(‘li‘):
    if len(item.select(‘.cb-media-title‘))>0:
        url1=item.select(‘a‘)[0][‘href‘]
        url2=‘http://www.cbdio.com/{}‘.format(url1)
        resd=requests.get(url2)
        resd.encoding=‘utf-8‘
        soupd=BeautifulSoup(resd.text,‘html.parser‘)
        cont=soupd.select(‘.cb-article‘)[0].text#内容
        contentls.append(cont)
print(contentls)


##for each in contentls:
##    f = open("1.txt", ‘r‘, ‘utf-8‘)
##    f.write(each)
####    print(each)
##    f.close()
##    print(‘#‘)
##fo=open(‘1.txt‘,‘r‘)
##content=fo.read()
##
content=str(contentls)

words=jieba.lcut(content)
ls=[]
counts={}
for word in words:
    ls.append(word)
    if len(word)==1:
        continue
    else:
        counts[word]=counts.get(word,0)+1

items = list(counts.items())
items.sort(key = lambda x:x[1], reverse = True)
for i in range(10):
    word , count = items[i]
    print ("{:<5}{:>2}".format(word,count))


#词云制作
from wordcloud import WordCloud
import matplotlib.pyplot as plt

cy = WordCloud(font_path=‘msyh.ttc‘).generate(content)
plt.imshow(cy, interpolation=‘bilinear‘)
plt.axis("off")
plt.show()



#excel导出、数据库存储
import re
import pandas
import sqlite3

itemtotal=[]
for i in range(2,3):
    listurl=‘http://www.cbdio.com/node_2568.htm‘
    itemtotal.extend(getOnePage(listurl))
df =pandas.DataFrame(itemtotal)
df.to_excel(‘BigDataItems.xlsx‘)
with sqlite3.connect(‘BigDataItems.sqlite‘) as db:
    df.to_sql(‘BigDataItems‘,con=db)
    print(‘输出成功！！‘)