抓取人民日报

1.准备工作：
我这里用的是python3 先pip3 install selenium 安装模块

然后下载浏览器驱动，这下面有两个，google根据对应浏览器版本下载，firefox根据系统类型下载即可
#google 浏览器驱动下载地址 http://npm.taobao.org/mirrors/chromedriver/
#firefox 浏览器驱动下载地址 https://github.com/mozilla/geckodriver/releases/
2.最后贴上脚本
3.from selenium import webdriver
import time
import requests as r
import re
import json

#获取具体的新闻url，并写入文本
def get_news_url(search_time):
    #打开日志文件
    file = open("F:/logs/fiddler.log","r")
    #最新一条url
    file_msg = file.readlines()
    #判断fiddler里面是否有url内容，若读取的内容列表等于0，则表示无内容
    if len(file_msg) == 0:
        return "continue"
    #读取数组内容最后一位，即为最新的url
    url = file_msg[-1].strip("\n")
    print(url.strip("\n"))
    file.close()
    headers = {
    "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3",
    "Accept-Encoding": "gzip, deflate, br",
    "Accept-Language": "zh-CN,zh;q=0.9",
    "Cache-Control": "no-cache",
    "Connection": "keep-alive",
    "Cookie": "acw_tc=2760826c15644797526864019e11da63f32bf5b082a7da38667809b95819f3; SERVERID=f858eac9d6463aa503ba25948984ceb0|1564566072|1564566072",
    "Host": "app.peopleapp.com",
    "Pragma": "no-cache",
    "Upgrade-Insecure-Requests": "1",
    "User-Agent": "Mozilla/5.0 (Windows NT 6.1; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/73.0.3683.103 Safari/537.36"
    }
    #获取抓包的最新url数据
    html = r.get(url,headers=headers).text
    #转换成json
    json_html = json.loads(html)
    #获取json的data数据
    news_data_li = json_html["data"]
    #写入文件
    news_file = open("F:/logs/news.log","a")
    #循环data数据，提取两个id，拼接成url，写入日志文件（这里可以根据实际情况，如对拼接成的url进行判断 分成具体的每个月进行写入日志等）
    for news in news_data_li:
        id = news["id"]
        articleid = news["articleid"]
        news_url = "https://wap.peopleapp.com/article/" + id + "/" + articleid
        print(news_url)
        news_file.write(news_url + "\n")
    news_file.close()
    #这一步只是用来控制的获取数据的时间，若传入的时间在数据内，则返回stop，停止，比如 只搜索 201904到现在的，就传入201904即可，这里根据具体需求改改就行
    if re.search(search_time,html):
        return "continue"
    else:
        return "stop"

#爬取动态页面url
#google 浏览器驱动下载地址 http://npm.taobao.org/mirrors/chromedriver/
#firefox 浏览器驱动下载地址 https://github.com/mozilla/geckodriver/releases/
#browser = webdriver.Chrome(executable_path="D:\python37\chromedriver_win32\chromedriver.exe")
#加载具体的浏览器驱动
browser = webdriver.Firefox(executable_path="D:\python37\geckodriver-v0.24.0-win64\geckodriver.exe")
#设置浏览器需要访问的url
browser.get("https://wap.peopleapp.com/news/1")
#循环操作浏览器往下翻页
while True:
    #翻页脚本
    browser.execute_script(‘window.scrollTo(0, document.body.scrollHeight)‘)
    #翻页后，暂停1秒
    time.sleep(1)
    #调用获取具体详情新闻页的url
    results = get_news_url("201907")
    #判断结果是否停止
    if results == "stop":
        browser.quit()
        break