Python爬虫爬取微信小说

Question

https://weread.qq.com/web/category/100000
https://weread.qq.com/web/category/200000
Python爬虫爬取微信小说的指定字段，书名、作者、阅读人数、评价、简介，爬取10万条数据。求解

Answer 1

简单的python示例代码就是

import requests
from time import sleep
data = {}

url = 'https://weread.qq.com/web/bookListInCategory/100000?maxIndex={}'
headers = {
    "Accept":"application/json, text/plain, */*",
    "authority":"weread.qq.com",
    "Referer":"https://weread.qq.com/web/category/200000",
    "Sec-Ch-Ua":'"Google Chrome";v="117", "Not;A=Brand";v="8", "Chromium";v="117"',
    "Sec-Ch-Ua-Platform":'"Windows"',
    "Sec-Fetch-Dest":"empty",
    "Sec-Fetch-Mode":"cors",
    "Sec-Fetch-Site":"same-origin",
    "User-Agent":"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/117.0.0.0 Safari/537.36"
}


for i in range(0,201,20):
    # 因为url是动态变化请求，根据滚轮位置
    result = requests.get(url.format(str(i)),headers=headers).json()['books']
    sleep(2)   #主要是防止IP被封掉，时间还可以延长一些
    # 书名、作者、阅读人数、评价、简介
    for item in result:
        info = item.get('bookInfo')
        title = info.get('title','')
        author = info.get('author','')
        readingCount = item.get('readingCount',0)  #阅读人数
        newRating = info.get('newRating',0)   #推荐值，评价还需要进入到详情页里面去，暂时没有做
        intro = info.get('intro','')   #简介
        if title in data.keys():
            continue
        else:
            data[title] = {
                '书名':title,
                '作者':author,
                '阅读人数': readingCount,
                '推荐值':newRating,
                '简介':intro.strip()
            }


print(data)