from bs4 import BeautifulSoup
import requests
import json
headers={'user-agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/101.0.4951.64 Safari/537.36 Edg/101.0.1210.53'}
commentURL='https://comment.tie.163.com/H8DOPGM90519DTSV.html'
def creatURL(commentURL,offset,limit):
s1='https://gw.m.163.com/commons-user-main/api/v1/commons/user/pc/getUserByCookie?isbtn=false&trace_id=IuKWAHFuVvX1653835134118'
s2='/comments/newList?offset='
name=commentURL.split('/')[-1].split('.')[0]
u=s1+str(name)+s2+str(offset)+'&limit='+str(limit)
return u
res=requests.get(url=creatURL(commentURL,1,40),headers=headers).content
data=json.loads(res.decode())
for key in data['comments'].keys():
print(data['comments'][key]['comment'])
爬取网易新闻评论页的评论,遇到问题for key in data['comments'].keys():
KeyError: 'comments',是不是长度有问题?url=creatURL(commentURL,1,40)是不是说获取40个评论?这个一页上是没有40条评论的,但是总的评论数又超过了40个
网页地址是https://comment.tie.163.com/H8DOPGM90519DTSV.html