爬虫运行成功但是没有结果

Question

这个代码运行成功但是没有输出结果

import requests
from bs4 import BeautifulSoup

url = "https://fx.wanfangdata.com.cn/scholartrends/detail?scholar_name=%E5%91%A8%E4%BA%91&org_name=%E8%8B%8F%E5%B7%9E%E5%A4%A7%E5%AD%A6%E9%99%84%E5%B1%9E%E5%84%BF%E7%AB%A5%E5%8C%BB%E9%99%A2&scholar_id=a0040857820"
headers = {
    "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/110.0.0.0 Safari/537.36"
}
pages_to_crawl = 5

for page in range(pages_to_crawl):
    params = {
        "page": page + 1
    }
    response = requests.get(url, headers=headers, params=params)
    soup = BeautifulSoup(response.text, "html.parser")
    article_list = soup.find_all("div", class_="scholar-list-item")
    for article in article_list:
        title_elem = article.find("h4", class_="title")
        title = title_elem.text.strip()
        link = title_elem.a.get("href")
        summary_elem = article.find("div", class_="abstract")
        if summary_elem is not None:
            summary = summary_elem.text.strip()
        else:
            summary = ""
        print(f"Title: {title}\nLink: {link}\nSummary: {summary}\n")

Answer 1

我把你代码运行了一下，抓到的是一些html，里面全是js，没有正文的，你想抓这个站的内容，需要监测xhr信息