python爬虫时Status code是500怎么解

Question

import requests
import json

headers={
    "Accept": "application/json, text/plain, */*",
    "Accept-Encoding": "gzip, deflate, br",
    "Accept-Language": "zh-CN,zh;q=0.9",
    "Connection": "keep-alive",
    "Content-Length": "196",
    "Content-Type": "application/json",
    "Cookie": "ipCheck=111.197.254.190; fp=b7b718dd4be227fad043bc96b7356c66; AMCVS_8E929CC25A1FB2B30A495C97%40AdobeOrg=1; s_ecid=MCMID%7C09581824314115337234201775765768920289; s_cc=true; AMCV_8E929CC25A1FB2B30A495C97%40AdobeOrg=1687686476%7CMCIDTS%7C18829%7CMCMID%7C09581824314115337234201775765768920289%7CMCAAMLH-1627362179%7C11%7CMCAAMB-1627362179%7CRKhpRz8krg2tLO6pguXWp5olkAcUniQYPHaMWWgdJ3xzPWQmdj0y%7CMCOPTOUT-1626764579s%7CNONE%7CMCAID%7CNONE%7CMCSYNCSOP%7C411-18836%7CvVersion%7C3.0.0; __gads=ID=47a664ec4703b87a:T=1626757379:S=ALNI_Mb2dou1xVJM8_OEPFLCfPblhOftug; s_sq=%5B%5BB%5D%5D; JSESSIONID=73rCWMhJHR2erAi55-BVFJOkkCSBPkiTujWZNjP4U57D3s9vNTos!-1146209717; WLSESSION=186802828.20480.0000; utag_main=v_id:017ac24ba8c100152f32bae998e50307300d806b007e8$_sn:1$_se:27$_ss:0$_st:1626760044835$ses_id:1626757376195%3Bexp-session$_pn:5%3Bexp-session$vapi_domain:ieee.org; TS01b03060=012f350623e8a3dd421a470c87d65fd4fedd252488ace2e711a38589d662ff8eff11a278f12f0cbc2c15cdd98299510d055d1d1705",
    "Host": "ieeexplore.ieee.org",
    "Origin": "https://ieeexplore.ieee.org",
    "Referer": "https://ieeexplore.ieee.org/search/searchresult.jsp?queryText=image%20processing&highlight=true&returnType=SEARCH&matchPubs=true&refinements=ContentType:Standards&ranges=2010_2021_Year&returnFacets=ALL&pageNumber=3",
    "sec-ch-ua": '" Not;A Brand";v="99", "Google Chrome";v="91", "Chromium";v="91"',
    "sec-ch-ua-mobile": "?0",
    "Sec-Fetch-Dest": "empty",
    "Sec-Fetch-Mode": "cors",
    "Sec-Fetch-Site": "same-origin",
    "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.164 Safari/537.36"
}

data={
    "highlight": "true",
    "matchPubs": "true",
    "pageNumber": "3",
    "queryText": "image processing",
    "ranges": '["2010_2021_Year"]',
    "refinements": '["ContentType:Standards"]',
    "returnFacets": '["ALL"]',
    "returnType": "SEARCH"
}

url = 'https://ieeexplore.ieee.org/rest/search'
IEEE_response = requests.post(url=url, data=json.dumps(data),headers=headers, verify=False)
print(IEEE_response.status_code)
response_text = IEEE_response.text

爬取ie xplore的论文时，返回的status_code是500，添加了全部请/求头的信息和参数结果还是一样

Answer 1

在传递json格式数据时（content-type: application/json），使用json参数

import requests

headers = {
    "Content-Type": "application/json",
    "Host": "ieeexplore.ieee.org",
    "Origin": "https://ieeexplore.ieee.org",
    "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.164 Safari/537.36"
}

data = {
    "highlight": "true",
    "matchPubs": "true",
    "pageNumber": "3",
    "queryText": "image processing",
    "ranges": ["2010_2021_Year"],           ## 注意这里是数组，而不是字符串
    "refinements": ["ContentType:Standards"],
    "returnFacets": ["ALL"],
    "returnType": "SEARCH"
}

url = 'https://ieeexplore.ieee.org/rest/search'
IEEE_response = requests.post(url=url, json=data, headers=headers)
print(IEEE_response.text)