标签：col panda data amp tcl tle 综合 pre and
1. 将新闻的正文内容保存到文本文件。
def writeNewsDetail(content): f = open(‘text.txt‘,‘a‘,encoding=‘utf-8‘) f.write(content) f.close() news[‘content‘] = soupd.select(‘.show-content‘)[0].text.strip() writeNewsDetail(news[‘content‘])
2. 将新闻数据结构化为字典的列表:
def getNewsList(pageUrl): res = requests.get(pageUrl) res.encoding = "utf-8" soup = BeautifulSoup(res.text, "html.parser") newsList = [] for news in soup.select(‘li‘): if len(news.select(‘.news-list-title‘)) > 0: newsUrl = news.select(‘a‘)[0].attrs[‘href‘] newsList.append(getNewDetail(newsUrl)) return (newsList) newsTotal = [] url = ‘http://news.gzcc.cn/html/xiaoyuanxinwen/‘ newsTotal.extend(getNewsList(url))
3. 安装pandas，用pandas.DataFrame(newstotal)，创建一个DataFrame对象df.
df = pandas.DataFrame(newsTotal) print(df)
4. 通过df将提取的数据保存到csv或excel 文件。
df.to_excel(‘wxc.xlsx‘)
5. 用pandas提供的函数和方法进行数据分析：
print(df[(df[‘clickCount‘]>3000) & (df[‘source‘] == ‘学校综合办‘)]) print(df[[‘clickCount‘, ‘title‘, ‘source‘]].head(6)) sou = [‘国际学院‘, ‘学生工作处‘] print(df[df[‘source‘].isin(sou)])
import re import requests from bs4 import BeautifulSoup from datetime import datetime import pandas def writeNewsDetail(content): f = open(‘text.txt‘,‘a‘,encoding=‘utf-8‘) f.write(content) f.close() def getClickCount(newsUrl): newId = re.search(‘\_(.*).html‘,newsUrl).group(1).split(‘/‘)[1] clickUrl = "http://oa.gzcc.cn/api.php?op=count&id={}&modelid=80".format(newId) return (int(requests.get(clickUrl).text.split(‘.html‘)[-1].lstrip("(‘").rstrip("‘);"))) def getNewDetail(newsUrl): resd = requests.get(newsUrl) resd.encoding = ‘utf-8‘ soupd = BeautifulSoup(resd.text, ‘html.parser‘) news = {} news[‘title‘] = soupd.select(‘.show-title‘)[0].text info = soupd.select(‘.show-info‘)[0].text news[‘dt‘] = datetime.strptime(info.lstrip(‘发布时间:‘)[0:19], ‘%Y-%m-%d %H:%M:%S‘) if info.find(‘来源：‘)>0: news[‘source‘] = info[info.find(‘来源：‘):].split()[0].lstrip(‘来源：‘) else: news[‘source‘] = ‘none‘ news[‘content‘] = soupd.select(‘.show-content‘)[0].text.strip() writeNewsDetail(news[‘content‘]) news[‘clickCount‘] = getClickCount(newsUrl) return (news) def getNewsList(pageUrl): res = requests.get(pageUrl) res.encoding = "utf-8" soup = BeautifulSoup(res.text, "html.parser") newsList = [] for news in soup.select(‘li‘): if len(news.select(‘.news-list-title‘)) > 0: newsUrl = news.select(‘a‘)[0].attrs[‘href‘] newsList.append(getNewDetail(newsUrl)) return (newsList) def getpageN(): res = requests.get(‘http://news.gzcc.cn/html/xiaoyuanxinwen/‘) res.encoding = "utf-8" soup = BeautifulSoup(res.text, "html.parser") n = int(soup.select(‘.a1‘)[0].text.rstrip(‘条‘)) return (n // 10 + 1) newsTotal = [] url = ‘http://news.gzcc.cn/html/xiaoyuanxinwen/‘ newsTotal.extend(getNewsList(url)) n = getpageN() for i in range(n,n+1): listPageUrl = ‘http://news.gzcc.cn/html/xiaoyuanxinwen/{}.html‘.format(i) newsTotal.extend(getNewsList(listPageUrl)) df = pandas.DataFrame(newsTotal) # print(df) df.to_excel(‘wxc.xlsx‘) # print(df[(df[‘clickCount‘]>3000) & (df[‘source‘] == ‘学校综合办‘)]) #print(df[[‘clickCount‘, ‘title‘, ‘source‘]].head(6)) sou = [‘国际学院‘, ‘学生工作处‘] print(df[df[‘source‘].isin(sou)])
标签：col panda data amp tcl tle 综合 pre and
原文地址：https://www.cnblogs.com/w092/p/8855235.html
