码迷,mamicode.com
首页 > 其他好文 > 详细

数据爬取

时间:2020-07-05 10:57:17      阅读:64      评论:0      收藏:0      [点我收藏+]

标签:sage   mozilla   tar   dead   value   messages   into   ice   print   

 

import json

 

import pymysql

import requests

from bs4 import BeautifulSoup

import datetime

 

url = ‘https://ncov.dxy.cn/ncovh5/view/pneumonia?from=timeline&isappinstalled=0‘  # 请求地址

headers = {

    ‘user-agent‘: ‘Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/74.0.3729.131 Safari/537.36‘}  # 创建头部信息

response = requests.get(url, headers=headers)  # 发送网络请求

content = response.content.decode(‘utf-8‘)

soup = BeautifulSoup(content, ‘html.parser‘)

 

# 世界确诊getAreaStat

listB = soup.find_all(name=‘script‘, attrs={"id": "getListByCountryTypeService2true"})

 

world_messages = str(listB)[95:-21]

 

print(world_messages)

world_messages_json = json.loads(world_messages)

worldList = []

now_time = datetime.datetime.now().strftime(‘%Y-%m-%d‘)

 

for k in range(len(world_messages_json)):

    worldvalue = (now_time,

                  world_messages_json[k].get(‘countryType‘), world_messages_json[k].get(‘continents‘),

                  world_messages_json[k].get(‘provinceId‘), world_messages_json[k].get(‘provinceName‘),

                  world_messages_json[k].get(‘provinceShortName‘), world_messages_json[k].get(‘cityName‘),

                  world_messages_json[k].get(‘currentConfirmedCount‘), world_messages_json[k].get(‘confirmedCount‘),

                  world_messages_json[k].get(‘suspectedCount‘), world_messages_json[k].get(‘curedCount‘),

                  world_messages_json[k].get(‘deadCount‘), world_messages_json[k].get(‘locationId‘),

                  world_messages_json[k].get(‘countryShortCode‘),)

    worldList.append(worldvalue)

 

db = pymysql.connect("localhost", "root", "0000", "yiqing", charset=‘utf8‘)

cursor = db.cursor()

 

sql_world = "INSERT INTO world_map values (%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s)"

 

worldTuple = tuple(worldList)

print(worldTuple)

 

try:

    cursor.executemany(sql_world, worldTuple)

    db.commit()

except:

    print(‘执行失败,进入回调2‘)

    db.rollback()

 

db.close()

 

 技术图片

 

数据爬取

标签:sage   mozilla   tar   dead   value   messages   into   ice   print   

原文地址:https://www.cnblogs.com/Nyar/p/13237734.html

(0)
(0)
   
举报
评论 一句话评论(0
登录后才能评论!
© 2014 mamicode.com 版权所有  联系我们:gaon5@hotmail.com
迷上了代码!