如何用Python+BeautifulSoup批量抓取联赛赛季数据？是否需Selenium？

先解决当前报错「ValueError: No Objects to concatenate」 #

原代码的核心错误点：

• range(2006-2007, 2023-2024) 计算后是range(-1, -1)，循环完全没执行，dfs为空列表，导致pd.concat报错。
• URL构造逻辑错误，重复拼接year生成了无效链接（比如.../stats/-1?page=-11）。
• 硬编码页数范围1-10，实际不同赛季/联赛的页数不固定，会请求不存在的页面返回空数据。

修复后的基础单赛季可运行代码：

from bs4 import BeautifulSoup
import requests
import pandas as pd

dfs = []
season = "2023-2024"
league = "nhl"

page = 1
while True:
    url = f"https://www.eliteprospects.com/league/{league}/stats/{season}?page={page}"
    print(f"Loading {url}")
    response = requests.get(url)
    soup = BeautifulSoup(response.content, 'html.parser')
    
    # 检查是否存在球员数据表格
    stats_table = soup.select_one(".player-stats")
    if not stats_table:
        break  # 无表格说明已到最后一页
    
    df = pd.read_html(str(stats_table))[0].dropna(how="all").reset_index(drop=True)
    # 添加赛季、联赛标识，方便后续数据区分
    df["赛季"] = season
    df["联赛"] = league
    dfs.append(df)
    
    # 检查是否有下一页
    next_btn = soup.select_one("a.page-link[rel='next']")
    if not next_btn:
        break
    page += 1

if dfs:
    df_final = pd.concat(dfs).reset_index(drop=True)
    pd.set_option('display.max_columns', 500)
    pd.set_option('display.width', 150)
    print(df_final)
    df_final.to_csv("data.csv", encoding='utf-8', index=False)
else:
    print("未获取到任何数据")

实现多赛季爬取 #

赛季格式为YYYY-YYYY+1，直接循环生成2006到2023起始年的赛季字符串即可：

import time

dfs = []
# 生成2006-2007到2023-2024的赛季列表
seasons = [f"{year}-{year+1}" for year in range(2006, 2024)]
league = "nhl"

for season in seasons:
    page = 1
    while True:
        url = f"https://www.eliteprospects.com/league/{league}/stats/{season}?page={page}"
        print(f"Loading {url}")
        response = requests.get(url)
        # 添加请求间隔，避免触发反爬
        time.sleep(1)
        soup = BeautifulSoup(response.content, 'html.parser')
        
        stats_table = soup.select_one(".player-stats")
        if not stats_table:
            break
        
        df = pd.read_html(str(stats_table))[0].dropna(how="all").reset_index(drop=True)
        df["赛季"] = season
        df["联赛"] = league
        dfs.append(df)
        
        next_btn = soup.select_one("a.page-link[rel='next']")
        if not next_btn:
            break
        page += 1

if dfs:
    df_final = pd.concat(dfs).reset_index(drop=True)
    df_final.to_csv("nhl_multi_season.csv", encoding='utf-8', index=False)
else:
    print("未获取到任何数据")

实现多联赛爬取（无需Selenium） #

从任意联赛的stats页面抓取下拉菜单中的联赛选项，提取每个联赛的标识（slug）和名称：

def get_leagues():
    url = "https://www.eliteprospects.com/league/nhl/stats/2023-2024"
    response = requests.get(url)
    soup = BeautifulSoup(response.content, 'html.parser')
    # 定位联赛下拉菜单的所有选项
    league_options = soup.select("select#league-select option")
    
    leagues = []
    # 跳过第一个默认占位选项
    for option in league_options[1:]:
        # 从option的value属性提取联赛slug
        league_slug = option["value"].split("/")[-2]
        league_name = option.text.strip()
        leagues.append({"slug": league_slug, "name": league_name})
    return leagues

结合赛季和联赛循环的完整爬取代码：

from bs4 import BeautifulSoup
import requests
import pandas as pd
import time

def get_leagues():
    url = "https://www.eliteprospects.com/league/nhl/stats/2023-2024"
    response = requests.get(url)
    soup = BeautifulSoup(response.content, 'html.parser')
    league_options = soup.select("select#league-select option")
    
    leagues = []
    for option in league_options[1:]:
        league_slug = option["value"].split("/")[-2]
        league_name = option.text.strip()
        leagues.append({"slug": league_slug, "name": league_name})
    return leagues

def scrape_season_league(season, league_slug, league_name):
    dfs = []
    page = 1
    while True:
        url = f"https://www.eliteprospects.com/league/{league_slug}/stats/{season}?page={page}"
        print(f"Loading {league_name} - {season} 第{page}页: {url}")
        try:
            response = requests.get(url, timeout=10)
            response.raise_for_status()  # 检查请求是否成功
        except Exception as e:
            print(f"请求失败: {e}")
            break
        
        time.sleep(1)
        soup = BeautifulSoup(response.content, 'html.parser')
        stats_table = soup.select_one(".player-stats")
        
        if not stats_table:
            break
        
        df = pd.read_html(str(stats_table))[0].dropna(how="all").reset_index(drop=True)
        df["赛季"] = season
        df["联赛Slug"] = league_slug
        df["联赛名称"] = league_name
        dfs.append(df)
        
        next_btn = soup.select_one("a.page-link[rel='next']")
        if not next_btn:
            break
        page += 1
    return dfs

# 主程序
if __name__ == "__main__":
    all_dfs = []
    seasons = [f"{year}-{year+1}" for year in range(2006, 2024)]
    leagues = get_leagues()
    
    for league in leagues:
        for season in seasons:
            season_dfs = scrape_season_league(season, league["slug"], league["name"])
            if season_dfs:
                all_dfs.extend(season_dfs)
    
    if all_dfs:
        df_final = pd.concat(all_dfs).reset_index(drop=True)
        df_final.to_csv("eliteprospects_all_data.csv", encoding='utf-8', index=False)
        print("数据保存完成")
    else:
        print("未获取到任何数据")

球员链接抓取补充 #

在解析数据时，可从表格行的<a>标签提取球员链接：

# 在scrape_season_league函数中，获取stats_table后添加以下代码
rows = stats_table.select("tr")[1:]  # 跳过表头行
player_links = []
for row in rows:
    player_a = row.select_one("td a")
    if player_a:
        player_links.append("https://www.eliteprospects.com" + player_a["href"])
    else:
        player_links.append(None)

# 将链接加入DataFrame
df["球员链接"] = player_links

注意事项 #

• 加入time.sleep(1)控制请求频率，避免触发网站反爬机制。
• 加入异常处理，防止单个请求失败导致整个程序崩溃。
• 部分联赛可能没有对应赛季的数据，程序会自动跳过此类情况。

内容的提问来源于stack exchange，提问作者ijif