import re
import urllib.request
import http.cookiejar
import gzip
from io import StringIO
from bs4 import BeautifulSoup
vid = '1472528692'
comid = '6345478485121181104'
url = 'https://coral.qq.com/article/'+vid+'/comment?commentid='+comid+'&reqnum=20'
headers = {'accept':'*/*',
'accept-language':'zh-CN,zh;q=0.8',
'user-agent':'Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/53.0.2785.104 Safari/537.36 Core/1.53.4033.400 QQBrowser/9.6.12624.400',
'connection':'keep-alive','referer':'qq.com'}
cjar = http.cookiejar.CookieJar()
opener = urllib.request.build_opener(urllib.request.HTTPCookieProcessor(cjar))
headall = []
for key,value in headers.items():
item = (key,value)
headall.append(item)
opener.addheaders = headall
urllib.request.install_opener(opener)
def craw(vid,comid):
url = 'https://coral.qq.com/article/'+vid+'/comment?commentid='+comid+'&reqnum=20'
doc = urllib.request.urlopen(url).read().decode('utf-8')
return doc
idpat = '"id":"(.*?)"'
usepat = '"nick":"(.*?)",'
conpat = '"content":"(.*?)",'
for i in range(1,10):
print('--------------------------------------------')
print("第"+str(i)+"页评论内容")
data = craw(vid,comid)
idlist = re.compile(idpat, re.S).findall(data)
userlist = re.compile(usepat, re.S).findall(data)
conlist = re.compile(conpat, re.S).findall(data)
for j in range(0,20):
print("用户名是："+eval('u"'+userlist[j]+'"'))
print("内容是："+eval('u"'+conlist[j]+'"'))
print("\n")
comid = idlist[19]
我想爬取用户名和评论，但是代码运行之后出现：
'utf-8' codec can't encode characters in position 4-5: surrogates not allowed
请问各位是什么问题吗？
本来这一个是gzip解码，但是我找不到解决解码gzip的方法，看到网上说直接删掉accept encoding那一段就可以了，但是删完之后就出现上面的问题。
求各位大神，已经解决几天这个问题了，新手实在头疼。
感谢！！！
