python爬虫报错，jsonp

Question

描述：爬取动态加载的网站，目标url存在于js中，用jsonp来封装的； 问题：直接爬取可以获得text，但是用json解析就报错，如下 Traceback (most recent call last): File "C:\Users\zhaorl\Desktop\python\0525上午.py", line 20, in res=json.loads(str(res)) File "C:\Python\lib\json_init_.py", line 346, in loads return _default_decoder.decode(s) File "C:\Python\lib\json\decoder.py", line 337, in decode obj, end = self.raw_decode(s, idx=_w(s, 0).end()) File "C:\Python\lib\json\decoder.py", line 355, in raw_decode raise JSONDecodeError("Expecting value", s, err.value) from None json.decoder.JSONDecodeError: Expecting value: line 1 column 1 (char 0)

爬取代码： import requests import json headers={ 'Host': 'xyq.cbg.163.com', 'Referer': 'https://xyq.cbg.163.com/cgi-bin/xyq_overall_search.py?act=show_overall_search_yuanshen', 'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/72.0.3626.121 Safari/537.36' } params={ 'callback': 'Request.JSONP.request_map.request_0', '_': '1624582691785', 'act': 'recommd_by_role', 'page': '1', 'attr_type': 'zongshang', 'count': '15', 'search_type': 'overall_search_yuanshen', 'view_loc': 'overall_search' } url='https://xyq.cbg.163.com/cgi-bin/recommend.py' res=requests.get(url=url,headers=headers,params=params) res=json.loads(str(res)) print(res.text)

Answer 1

参数callback去掉

import requests
import json

headers = {'Host': 'xyq.cbg.163.com',
           'Referer': 'https://xyq.cbg.163.com/cgi-bin/xyq_overall_search.py?act=show_overall_search_yuanshen',
           'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) '
                         'Chrome/72.0.3626.121 Safari/537.36'}
# params = {'callback': 'Request.JSONP.request_map.request_0', '_': '1624582691785', 'act': 'recommd_by_role',
#           'page': '1', 'attr_type': 'zongshang', 'count': '15', 'search_type': 'overall_search_yuanshen',
#           'view_loc': 'overall_search'}

params = {'_': '1624582691785', 'act': 'recommd_by_role',
          'page': '1', 'attr_type': 'zongshang', 'count': '15', 'search_type': 'overall_search_yuanshen',
          'view_loc': 'overall_search'}
url = 'https://xyq.cbg.163.com/cgi-bin/recommend.py'
res = requests.get(url=url, headers=headers, params=params).json()   #  此处添加.json()
# res = json.loads(res)
print(res)

# result：
# {'msg': '请输入验证码！', 'status': 3, 'server_time': 1624614077246}