本人编程小白,经老师要求现在正在利用python对openaq网站的api进行数据抓取,有API key,但是抓取超过一定数量,比如10000左右就会各种报错。出现了Request failed with status code: 408,Request failed with status code: 500错误。下面是代码
import requests
import csv
import json
url = "https://api.openaq.org/v2/measurements?date_from=2000-01-01T00%3A00%3A00Z&date_to=2023-09-13T17%3A43%3A00Z&limit=10000000&page=1&offset=0&sort=desc&radius=1000&country=AR&order_by=datetime"
api_key = "apikey"
headers = {"accept": "application/json","X-API-Key":api_key}
response = requests.get(url, headers=headers, timeout=30)
if response.status_code == 200:
data = response.json()
# 保存为CSV文件
filename = "aaa.txt"
fields = ["locationId", "location", "parameter", "value", "time", "lat", "lon"]
results = data["results"]
rows = []
for result in results:
row = [
result["locationId"],
result["location"],
result["parameter"],
result["value"],
result["date"]["utc"],
result["coordinates"]["latitude"],
result["coordinates"]["longitude"],
]
rows.append(row)
with open(filename, mode="w", newline="", encoding="utf-8") as file:
writer = csv.writer(file)
writer.writerow(fields)
writer.writerows(rows)
print("Data saved to", filename)
else:
print("Request failed with status code:", response.status_code)