爬虫为什么没有文件下载

Question

import requests
import re
import time
from selenium import webdriver

# 设置请求头信息
headers = {
    'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/69.0.3497.100 Safari/537.36'
}

# 使用 Selenium 打开网页
browser = webdriver.Chrome()
url = 'http://www.sse.com.cn/disclosure/credibility/supervision/inquiries/'
browser.get(url)
time.sleep(10)  # 必须要加10秒的延迟，因为有个刷新的动作需要等待一段时间

# 获取页面源码
data = browser.page_source

# 修改正则表达式
p_title = '<td><a href=".*?" target="_blank">(.*?)</a></td>'
p_href = '<td><a href="(.*?)" target="_blank">.*?</a></td>'

# 获取标题和链接
title = re.findall(p_title, data, re.S)
href = re.findall(p_href, data, re.S)

# 循环下载文件
for i in range(len(href)):
    print("正在下载文件：", title[i])
    url = href[i]
    res = requests.get(url, headers=headers)
    file_name = './上交所问询函/' + title[i] + '.pdf'
    with open(file_name, 'wb') as file:
        file.write(res.content)
    print("下载完成：", title[i])

browser.quit()  # 关闭浏览器

Answer 1

使用api接口

import pandas as pd
import requests

headers = {
    'Referer': 'https://www.sse.com.cn/',
    'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/124.0.0.0 Safari/537.36',
}

params = {
    'jsonCallBack': '',
    'isPagination': 'true',
    'pageHelp.pageSize': '100',
    'pageHelp.pageNo': '1',
    'pageHelp.beginPage': '1',
    'pageHelp.cacheSize': '1',
    'pageHelp.endPage': '1',
    'sqlId': 'BS_KCB_GGLL',
    'siteId': '28',
    'channelId': '10743,10744,10012',
    'type': '',
    'stockcode': '',
    'extGGDL': '',
    'createTime': '',
    'createTimeEnd': '',
    'order': 'createTime|desc,stockcode|asc',
    '_': '1715227724607',
}

response = requests.get('https://query.sse.com.cn/commonSoaQuery.do', params=params, headers=headers)
dd=pd.DataFrame(response.json()['result'])
for i in range(len(dd)):
    print(dd.loc[i,'extGSJC'],dd.loc[i,'docURL'])