python爬虫只能获得一条数据

Question

问题遇到的现象和发生背景

今天利用python爬虎扑步行街时，想爬取一整面的数据，却发现只能爬取到一条数据不知道问题出在哪里

问题相关代码，请勿粘贴截图

from pymongo import MongoClient
import requests
import datetime
from bs4 import BeautifulSoup

def get_page(link):
headers = {'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) '
'Chrome/104.0.5112.102 Safari/537.36 Edg/104.0.1293.63'}
r = requests.get(link, headers=headers)
print(r.status_code)
html = r.content # 解封装
html = html.decode('UTF_8') # 解码
soup = BeautifulSoup(html, 'lxml')
return soup
def get_data(post_list):
data_list = []
for post in post_list:
url = post.find('div', class_='post-title').a['href']
url = url+'https://bbs.hupu.com/bxj'
title = post.find('div', class_='post-title').a.text.strip()
reply_view = post.find('div', class_='post-datum').text.strip()
reply = reply_view.split('/')[0].strip()
view = reply_view.split('/')[1].strip()
author = post.find('div', class_='post-auth').a.text.strip()
time = post.find('div', class_='post-time').text.strip()
data_list.append([url, title, reply, view, author, time])
return data_list

link = 'https://bbs.hupu.com/bxj'

soup = get_page(link)
post_all = soup.find('ul')
post_list = soup.find_all('li', class_="bbs-sl-web-post-body")
data_list = get_data(post_list)
for each in data_list:
print(each)

运行结果及报错内容

D:\anaconda\envs\pythonProject\python.exe D:/python__demo/pa_chong/data_create.py
200
['/55233916.htmlhttps://bbs.hupu.com/bxj', '现在中国境内最濒危的动物是什么？', '28', '17747', '浮云依旧', '08-23 17:42']

进程已结束,退出代码0

我的解答思路和尝试过的方法

曾百度查过他人的代码
发现在
post_all = soup.find('ul')
post_list = soup.find_all('li', class_="bbs-sl-web-post-body")
引号里的内容有不同但依然不能解决

Answer 1

复制我的代码试试：

import requests
from bs4 import BeautifulSoup

def get_page(link):
    headers = {'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) '
    'Chrome/104.0.5112.102 Safari/537.36 Edg/104.0.1293.63'}
    r = requests.get(link, headers=headers)
    print(r.status_code)
    html = r.content # 解封装
    html = html.decode('UTF_8') # 解码
    soup = BeautifulSoup(html, 'lxml')
    return soup
def get_data(post_list):
    data_list = []
    for post in post_list:
        url = post.find('div', class_='post-title').a['href']
        url = url+'https://bbs.hupu.com/bxj'
        title = post.find('div', class_='post-title').a.text.strip()
        reply_view = post.find('div', class_='post-datum').text.strip()
        reply = reply_view.split('/')[0].strip()
        view = reply_view.split('/')[1].strip()
        author = post.find('div', class_='post-auth').a.text.strip()
        time = post.find('div', class_='post-time').text.strip()
        data_list.append([url, title, reply, view, author, time])
    return data_list

link = 'https://bbs.hupu.com/bxj'
soup = get_page(link)
post_all = soup.find('ul')
post_list = soup.find_all('li', class_="bbs-sl-web-post-body")
data_list = get_data(post_list)
for each in data_list:
    print(each)