ESPN NBA赛事数据Python爬虫故障排查与修复求助

错误原因分析 #

• 反爬拦截：ESPN新增基础反爬机制，未携带浏览器标识的请求会被直接拒绝，导致requests.get(url)返回403 Forbidden错误。
• 冗余请求：原代码同时用urlopen(url)和requests.get(url)发起两次请求，既浪费资源又提升了被检测的概率。
• 日期逻辑缺陷：未处理闰年2月29日的场景，可能生成无效日期URL。
• 页面结构变更：ESPN可能更新了页面元素的CSS类名，原代码依赖的ScoreCell__TeamName等选择器已失效。

修复后的完整代码 #

import requests
import bs4
import pandas as pd
from datetime import date, timedelta

# 配置参数
OUTPUT_PATH = "/Users/Spring 2023/NBA ESPN/OutPut/"
FILE_NAME = "ESPN_NBA.csv"
START_DATE = date(2022, 12, 28)
END_DATE = date(2022, 12, 30)

# 模拟浏览器请求头，绕过反爬拦截
HEADERS = {
    "User-Agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36"
}

def generate_date_range(start, end):
    """生成日期范围内的所有日期字符串（YYYYMMDD格式）"""
    date_list = []
    current_date = start
    while current_date <= end:
        date_str = current_date.strftime("%Y%m%d")
        date_list.append(date_str)
        current_date += timedelta(days=1)
    return date_list

def scrape_espn_scoreboard(date_str):
    """抓取单日期的NBA赛事数据"""
    url = f"https://www.espn.com/nba/scoreboard/_/date/{date_str}"
    try:
        response = requests.get(url, headers=HEADERS, timeout=10)
        response.raise_for_status()  # 捕获HTTP错误
        soup = bs4.BeautifulSoup(response.text, 'lxml')
        
        # 适配当前ESPN页面结构
        game_cards = soup.find_all('div', class_='Scoreboard bg-clr-white flex flex-auto justify-between')
        data = []
        
        for card in game_cards:
            game_date = f"{date_str[4:6]}/{date_str[6:]}/{date_str[:4]}"
            teams = card.find_all('div', class_='ScoreCell__Team')
            if len(teams) != 2:
                continue
            
            # 提取客队信息
            away_team = teams[0].find('div', class_='ScoreCell__TeamName').get_text(strip=True)
            away_score = teams[0].find('div', class_='ScoreCell__Score').get_text(strip=True)
            # 提取主队信息
            home_team = teams[1].find('div', class_='ScoreCell__TeamName').get_text(strip=True)
            home_score = teams[1].find('div', class_='ScoreCell__Score').get_text(strip=True)
            
            data.append([game_date, home_team, home_score, away_team, away_score])
        
        return data
    except Exception as e:
        print(f"抓取日期{date_str}失败: {str(e)}")
        return []

# 主执行流程
if __name__ == "__main__":
    date_str_list = generate_date_range(START_DATE, END_DATE)
    all_data = []
    
    for date_str in date_str_list:
        game_data = scrape_espn_scoreboard(date_str)
        all_data.extend(game_data)
    
    # 保存数据到CSV
    if all_data:
        df = pd.DataFrame(all_data, columns=['Date', 'home_team', 'home_team_score', 'away_team', 'away_team_score'])
        df.to_csv(OUTPUT_PATH + FILE_NAME, index=False)
        
        # 生成球队出场次数统计
        count_df = pd.DataFrame(pd.concat([df['home_team'], df['away_team']]).value_counts(), columns=['Count'])
        count_df.index.rename('Team Name', inplace=True)
        count_df.to_csv(OUTPUT_PATH + f"count_{FILE_NAME}", index=True, header=True)
        
        print("文件已保存")
    else:
        print("未抓取到任何数据")

关键修复点说明 #

1. 请求头优化：添加User-Agent模拟浏览器请求，绕过基础反爬拦截。
2. 日期逻辑重构：用datetime.timedelta自动生成日期范围，避免手动处理闰年、月份天数等复杂逻辑。
3. 请求方式统一：仅使用requests发起一次请求，减少被检测概率。
4. 选择器适配：更新CSS选择器逻辑，优先定位完整比赛卡片再提取数据，提升兼容性。
5. 错误处理增强：添加response.raise_for_status()捕获HTTP错误，输出具体失败原因便于排查。

替代方案：使用ESPN官方API #

直接调用官方API比网页爬虫更稳定，无需担心页面结构变更：

import requests
import pandas as pd
from datetime import date, timedelta

HEADERS = {"User-Agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36"}

def get_api_data(date_str):
    url = f"https://site.api.espn.com/apis/site/v2/sports/basketball/nba/scoreboard?dates={date_str}"
    response = requests.get(url, headers=HEADERS)
    data = response.json()
    game_list = []
    
    for event in data['events']:
        game_date = f"{date_str[4:6]}/{date_str[6:]}/{date_str[:4]}"
        for competition in event['competitions']:
            # 区分主队和客队
            home_idx = 0 if competition['competitors'][0]['homeAway'] == 'home' else 1
            away_idx = 1 - home_idx
            
            home_team = competition['competitors'][home_idx]['team']['displayName']
            home_score = competition['competitors'][home_idx]['score']
            away_team = competition['competitors'][away_idx]['team']['displayName']
            away_score = competition['competitors'][away_idx]['score']
            
            game_list.append([game_date, home_team, home_score, away_team, away_score])
    
    return game_list

# 调用示例
start_date = date(2022,12,28)
end_date = date(2022,12,30)
all_data = []
current_date = start_date
while current_date <= end_date:
    date_str = current_date.strftime("%Y%m%d")
    all_data.extend(get_api_data(date_str))
    current_date += timedelta(days=1)

df = pd.DataFrame(all_data, columns=['Date', 'home_team', 'home_team_score', 'away_team', 'away_team_score'])
df.to_csv("/Users/Spring 2023/NBA ESPN/OutPut/ESPN_NBA_API.csv", index=False)

API优势 #

• 数据结构稳定，不会随页面UI变更失效
• JSON格式解析更高效准确
• 支持批量获取多日期数据，可通过参数调整获取范围

内容的提问来源于stack exchange，提问作者Bhavya