爬虫抓取XHR文件，获取不到响应的数据

Question

因工作需要，采集目标网站的新闻数据，发现通过关键字搜索，返回的页面数据在常规的html里面是没有的，经过F12分析请求，发现在XHR文件里面的search请求中，响应的内容就是我需要的数据；

根据上图分析请求，编写脚本

试过各种方法，还是换回1000

求朋友指点，万分感谢

Answer 1

你确定你的那一大串数据都是params的吗？应该是data的，或者json的吧，post方法里面有params，但是一般不是这个格式的，你的这些里面是内嵌的，params不是这样的，如果改了以后还不行的话，你就要看看请求头还有什么东西是你没加进去的，或者你把链接发出来，可以帮你分析一下
这是我的

import requests

url = "http://slj.cq.gov.cn/irs/front/search"
headers = {
    "Cookie": "SESSION=YWNiYTEzNjgtZDQ1ZS00Y2JjLWIyMDMtMzFjMzcyZjM0YmYx; _trs_uv=l6ndx3ql_3950_7how; _trs_ua_s_1=l6ndx3ql_3950_bkl1; _trs_user=",
    'Host': "slj.cq.gov.cn",
    'Origin': "http://slj.cq.gov.cn",
    'Referer': "http://slj.cq.gov.cn/cqs/searchResultPC.html?tenantId=59&configTenantId=59&areaCode=500000127&searchWord=%E5%BA%94%E6%80%A5%E5%93%8D%E5%BA%94",
    'User-Agent': "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/103.0.0.0 Safari/537.36",
    'X-Requested-With': "XMLHttpRequest"
}
json = {"id": "59", "tenantId": "59", "searchWord": "应急响应", "dataTypeId": 2063, "dataTypeName": "", "pageNo": 1,
        "pageSize": 10, "orderBy": "related", "searchBy": "title", "appendixType": "", "granularity": "ALL",
        "beginDateTime": "", "endDateTime": "", "isSearchForced": 0, "filters": [],
        "sign": "35762375-cfef-4fcd-9d4e-fb7714f5fc60", "pager": {}, "searchInfo": {}, "dataTypes": [],
        "configTenantId": "59", "historySearchWords": ["应急响应", "国务院"], "operationType": "search", "seniorBox": 0,
        "isDefaultAdvanced": 0, "isAdvancedSearch": 0, "advancedFilters": [], "areaCode": "500000127",
        "customFilter": {"operator": "or", "properties": [
            {"property": "f_202146838317", "operator": "gte", "value": "2022-08-10 17:05:55"},
            {"property": "f_202146235090", "operator": "gte", "value": "2022-08-10 17:05:55"}], "filters": [
            {"operator": "and", "properties": [{"property": "f_202146838317", "operator": "eq", "value": None},
                                               {"property": "f_202146235090", "operator": "eq", "value": None}]}]}}
req = requests.post(url=url, headers=headers, json=json)
print(req.text)
req.close()