将爬取的数据导入数据库失败
源代码:
def get_wangyi_data():
url = "https://c.m.163.com/ug/api/wuhan/app/data/list-total"
headers= {
'user-agent':'Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/92.0.4515.131 Safari/537.36 SLBrowser/8.0.0.12022 SLBChan/25'}
r = requests.get(url=url,headers=headers)
d = json.loads(r.text)
data = json.dumps(d['data'])
data_all = data.encode("utf-8").decode("unicode-escape")
data_all = json.loads(data_all) ##提取我们所需要的数据
history = {} #历史数据
for i in data_all["chinaDayList"]:
ds = i["date"]
#ds=json.loads(ds)
tup = time.strptime(ds,"%Y-%m-%d")
ds = time.strftime("%Y-%m-%d",tup)#改变时间格式,不然插入数据库会出错,数据库是datetime类型
confirm = i["total"]["confirm"]
suspect = i["total"]["suspect"]
heal = i["total"]["heal"]
dead = i["total"]["dead"]
history[ds] = {"confirm":confirm,"suspect":suspect,"heal":heal,"dead":dead}
for i in data_all["chinaDayList"]:
ds = i["date"]
#ds=json.loads(ds)
tup = time.strptime(ds,"%Y-%m-%d")
ds = time.strftime("%Y-%m-%d",tup)#改变时间格式,不然插入数据库会出错,数据库是datetime类型
confirm = i["today"]["confirm"]
suspect = i["today"]["suspect"]
heal = i["today"]["heal"]
dead = i["today"]["dead"]
history[ds].update ({"confirm_add":confirm,"suspect_add":suspect,"heal_add":heal,"dead_add":dead})
details = [] #当日详细数据
update_time = data_all["lastUpdateTime"]
data_country = data_all["areaTree"] #25个国家
data_province = data_country[2]["children"] #中国各省
for pro_infos in data_province:
province = pro_infos["name"] #省名
for city_infos in pro_infos["children"]:
city = city_infos["name"]
confirm = city_infos["total"]["confirm"]
confirm_add = city_infos["today"]["confirm"]
heal = city_infos["total"]["heal"]
dead = city_infos["total"]["dead"]
details.append([update_time,province,city,confirm,confirm_add,heal,dead])
return history, details
def get_conn():
"""
:return:连接,游标
"""
#创建连接
conn = pymysql.connect(host = "127.0.0.1",
user = "root",
password = "root",
db = "cov",
charset = "utf8")
#创建游标
cursor = conn.cursor() #执行完毕返回的结果集默认以元组显示
return conn,cursor
def close_conn(conn,cursor):
if cursor:
cursor.close()
if conn:
conn.close()
def update_details():
"""
更新details表
"""
cursor = None
conn = None
try:
li = get_wangyi_data()[1] #0是历史数据字典,1是最新详细数据列表
conn,cursor = get_conn()
sql = "insert into datails(update_time,province,city,confirm,confirm_add,heal,dead) values(%s,%s,%s,%s,%s,%s,%s)"
sql_query = "select %s=(select update_time from details order by id desc limit 1)" #对比当前最大时间戳
cursor.execute(sql_query,li[0][0])
if not cursor.fetchone()[0]:
print(f"{time.asctime()}开始更新最新数据")
for item in li:
cursor.execute(sql,item)
conn.commit() #提交事务 update delete insert操作
print(f"{time.asctime()}更新最新数据完毕")
else:
print(f"{time.asctime()}已是最新数据!")
except:
traceback.print_exc()
finally:
close_conn(conn,cursor)
update_details()
出现如下错误
但我数据库已创建这个表了