关于#python#的问题

Question

# coding=utf-8
from bs4 import BeautifulSoup
import re
import urllib.request, urllib.error
import xlwt
findLink = re.compile(r"alt=\"(.*?)\"")
findImgSrc = re.compile(r'<div class="comment-star (?<!\d)(star\d+)"</div>')
findRating = re.compile(r'<div class="comment-time">(.*?)<\/div>')
findJudge = re.compile(r'<p class="comment-con">(.*?)<\/p>/i')
def main():
    baseurl = "https://item.jd.com/100027211987.html#comment"  #要爬取的网页链接
    # 1.爬取网页
    datalist = getData(baseurl)
    savepath = "小米手环7pro.xls"
    saveData(datalist,savepath)
def getData(baseurl, comment=None):
    datalist = []  # 用来存储爬取的网页信息
    url = baseurl + str(10)
    html = askURL(url)  # 保存获取到的网页源码
        # 2.逐一解析数据
    soup = BeautifulSoup(html, "html.parser")
    for item in soup.find_all('div', class_="comment-item"):  # 查找符合要求的字符串
        data = []
        item = str(comment-item)
        link = re.findall(findLink, item)[0]
        data.append(link)
        imgSrc = re.findall(findImgSrc, item)[0]
        data.append(imgSrc)
        rating = re.findall(findRating, item)[0]
        data.append(rating)
        judgeNum = re.findall(findJudge, item)[0]
        data.append(judgeNum)
    return datalist
def askURL(url):
    head = {
        "User-Agent": "Mozilla / 5.0(Windows NT 10.0; Win64; x64) AppleWebKit / 537.36(KHTML, like Gecko) Chrome / 80.0.3987.122  Safari / 537.36"
    }
    request = urllib.request.Request(url, headers=head)
    html = ""
    try:
        response = urllib.request.urlopen(request)
        html = response.read().decode("utf-8")
    except urllib.error.URLError as e:
        if hasattr(e, "code"):
            print(e.code)
        if hasattr(e, "reason"):
            print(e.reason)
    return html
# 保存数据到表格
def saveData(datalist,savepath):
    print("save.......")
    book = xlwt.Workbook(encoding="utf-8",style_compression=0) #创建workbook对象
    sheet = book.add_sheet('小米手环7pro', cell_overwrite_ok=True) #创建工作表
    col = ("用户名","评分","日期","评价")
    for i in range(0,4):
        sheet.write(0,i,col[i])  #列名
    for i in range(0, min(len(datalist), 10)):
        data = datalist[i]
        for j in range(0,4):
            sheet.write(i+1,j,data[j])  #数据
    book.save(savepath) #保存
# 保存数据到数据库
if __name__ == "__main__":  # 当程序执行时
    # 调用函数
     main()
    # init_db("movietest.db")
     print("爬取完毕！")

该代码可以成功运行，但生成的excel文件内无爬取的数据存在，请问应如何解决？

Answer 1

引自免费微信小程序：皆我百晓生

在检查了您的代码之后，我发现有几个地方需要修改以确保数据能正确抓取并保存到Excel文件中。以下是修正后的代码及解释：

1. 正则表达式错误：findImgSrc 和 findRating 的正则表达式写法有误。
2. 循环逻辑错误：在 getData 函数中，您尝试访问一个名为 comment-item 的类，但实际上应该遍历所有评论项，而不需要将 str(comment-item) 转换。
3. 分页处理缺失：原始代码没有处理商品评论的分页逻辑，大多数电商网站的评论是分页显示的。

下面是修正后的代码：

# coding=utf-8
from bs4 import BeautifulSoup
import re
import urllib.request, urllib.error
import xlwt

# 正则表达式调整
findLink = re.compile(r'alt="(.+?)"')
findRating = re.compile(r'<span class="score">(\d+)</span>')  # 修改为匹配评分数字
findDate = re.compile(r'<span class="time">(.+?)</span>')  # 新增匹配评论日期
findComment = re.compile(r'<p class="comment-con">(.+?)</p>')  # 修改为匹配评论内容

def main():
    baseurl = "https://item.jd.com/100027211987.html#comment"  
    datalist = getData(baseurl)
    savepath = "小米手环7pro.xls"
    saveData(datalist, savepath)

def getData(baseurl):
    datalist = []
    page = 0
    while True:  # 添加循环处理分页
        url = f"{baseurl}&page={page}"  # 分页参数
        html = askURL(url)
        soup = BeautifulSoup(html, "html.parser")
        
        # 检查是否有更多评论，如果没有则跳出循环
        if not soup.find('div', class_='comment-item'):
            break
        
        for item in soup.find_all('div', class_="comment-item"):
            data = []
            # 直接从item中提取信息，无需转换为字符串再操作
            link = item.find('img')['alt']
            rating = findRating.search(str(item)).group(1) if findRating.search(str(item)) else ''
            date = findDate.search(str(item)).group(1) if findDate.search(str(item)) else ''
            comment = findComment.search(str(item)).group(1) if findComment.search(str(item)) else ''
            data.append(link)
            data.append(rating)
            data.append(date)
            data.append(comment)
            datalist.append(data)
        page += 1  # 翻页
    return datalist

def askURL(url):
    head = {
        "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/80.0.3987.122 Safari/537.36"
    }
    request = urllib.request.Request(url, headers=head)
    try:
        response = urllib.request.urlopen(request)
        html = response.read().decode("utf-8")
    except urllib.error.URLError as e:
        print("网络连接错误:", e.reason)
        html = ""
    return html

def saveData(datalist, savepath):
    print("Saving data...")
    book = xlwt.Workbook(encoding="utf-8", style_compression=0)
    sheet = book.add_sheet('小米手环7pro', cell_overwrite_ok=True)
    col = ("用户名", "评分", "日期", "评价")
    for i in range(len(col)):
        sheet.write(0, i, col[i])
    for i in range(len(datalist)):
        data = datalist[i]
        for j in range(len(data)):
            sheet.write(i+1, j, data[j])
    book.save(savepath)
    print("数据保存完成！")

if __name__ == "__main__":
    main()
    print("爬取并保存数据完毕！")

这个修订版的代码包括了正确的正则表达式、添加了对评论分页的支持，并修正了数据提取的逻辑。请尝试运行这段代码，它应该能成功抓取并保存小米手环7 Pro的商品评论数据到Excel文件中。