爬取网页里表格数据放入列表

Question

爬取网页里表格数据，感觉页面有些复杂，实在爬不出来

import time
import requests
from bs4 import BeautifulSoup
headers = {
    'User-Agent':'Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/63.0.3239.132 Safari/537.36'
}#爬虫[Requests设置请求头Headers],伪造浏览器
# 核心爬取代码
url= 'https://www.basketball-reference.com/leagues/NBA_2014_games-december.html'
params = {"show_ram":1}
response = requests.get(url,params=params, headers=headers)#访问url
listData=[]#定义数组
soup = BeautifulSoup(response.text, 'html.parser')#获取网页源代码
tr = soup.find('tbody').find_all('tr')
#.find定位到所需数据位置  .find_all查找所有的tr（表格）
# 去除标签栏
for j in tr[1:]:        #tr2[1:]遍历第1列到最后一列，表头为第0列
    td = j.find_all('th')#td表格
    Date = td[0].get_text().strip()           
 #   Start(ET) = td[1].get_text().strip()  
#     Visitor/Neutral = td[2].get_text().strip()            
#     Visitor Points = td[3].get_text().strip()       
#     Home/Neutral = td[4].get_text().strip()                    
#     Home Points = td[5].get_text().strip()                   
#     &nbsp=td[6].get_text().strip() 
#     &nbspp=td[7].get_text().strip() 
#     Attend.=td[8].get_text().strip() 
#     Arena=td[9].get_text().strip() 
#     Notes=td[10].get_text().strip()
 

#     listData.append([Date,Start(ET),Visitor/Neutral,Visitor Points,Home/Neutral,Home Points,&nbsp,&nbspp,Attend.,Arena,Notes])
# print (listData)#打印
listData.append([Date])
print (listData)#

https://www.basketball-reference.com/leagues/NBA_2014_games-december.html

Answer 1

TechWhizKid参考GPT回答：

会爬取指定的网页，解析表格数据，并将结果打印出来。它将会获取比赛的日期和时间，访问队伍和它们的得分，主场队伍和它们的得分，比赛成绩链接的URL，加时赛，观众数量，以及比赛地点

import time
import requests
from bs4 import BeautifulSoup

headers = {
    'User-Agent':'Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/63.0.3239.132 Safari/537.36'
}#爬虫[Requests设置请求头Headers],伪造浏览器

url= 'https://www.basketball-reference.com/leagues/NBA_2014_games-december.html'
params = {"show_ram":1}
response = requests.get(url, params=params, headers=headers) #访问url

listData = [] #定义数组
soup = BeautifulSoup(response.text, 'html.parser') #获取网页源代码

tr = soup.find('tbody').find_all('tr') #.find定位到所需数据位置  .find_all查找所有的tr（表格）

for j in tr:
    th = j.find_all('th')  # 查找所有的th（表头）
    td = j.find_all('td')  # 查找所有的td（表格数据）
    
    if th and td:  # 确保每行都有数据
        Date = th[0].get_text().strip()  # 日期和时间
        Visitor_Team = td[0].get_text().strip()  # 访问队伍
        Visitor_Points = td[1].get_text().strip()  # 访问队伍的得分
        Home_Team = td[2].get_text().strip()  # 主场队伍
        Home_Points = td[3].get_text().strip()  # 主场队伍的得分
        Box_Score = 'https://www.basketball-reference.com' + td[4].a['href'] if td[4].a is not None else ''  # 比赛成绩链接URL
        Overtime = td[5].get_text().strip()  # 加时赛
        Attend = td[6].get_text().strip()  # 观众数量
        Arena = td[7].get_text().strip()  # 比赛地点

        listData.append([Date, Visitor_Team, Visitor_Points, Home_Team, Home_Points, Box_Score, Overtime, Attend, Arena])

# 输出结果
for data in listData:
    print(data)