如何修改爬虫代码爬取国际球员数据至Google Sheet

问题分析 #

你之前的G League代码无法直接复用在国际球员统计页面，核心原因是：

• 页面结构不同：球员链接所在的表格元素选择器需要调整
• 目标页面是分页结构（共66页），需要循环遍历所有页码
• 测试代码错误选择了所有<a>标签，导致大量无效请求，且类名拼写错误（profileBox应为profile-box）

修改后的完整代码 #

import requests
from bs4 import BeautifulSoup
import gspread

# 初始化Google Sheet连接
gc = gspread.service_account(filename='creds.json')
sh = gc.open_by_key('1DpasSS8yC1UX6WqAbkQ515BwEEjdDL-x74T0eTW8hLM')
worksheet = sh.get_worksheet(3)

def scrape_player_profile(player_url):
    """爬取单个球员页面的profile-box数据"""
    full_url = f"https://basketball.realgm.com{player_url}"
    try:
        req = requests.get(full_url)
        req.raise_for_status()  # 捕获HTTP请求错误
        soup = BeautifulSoup(req.content, "html.parser")
        profile_box = soup.find("div", class_="profile-box")
        if not profile_box:
            return None
        
        # 解析profile-box内容
        profile_data = {}
        for p in profile_box.find_all("p"):
            text = p.get_text(strip=True)
            if ":" in text:
                key, value = text.split(":", 1)
                profile_data[key.strip()] = value.strip()
        return profile_data
    except Exception as e:
        print(f"爬取球员链接失败 {full_url}: {str(e)}")
        return None

def get_players_from_page(page_url):
    """从单页统计页面提取所有球员链接和姓名"""
    players = []
    try:
        req = requests.get(page_url)
        req.raise_for_status()
        soup = BeautifulSoup(req.content, "html.parser")
        # 精准定位球员列的td（国际统计页面的data-th为"Player"）
        for td in soup.find_all("td", {"data-th": "Player"}):
            a_tag = td.find("a")
            if not a_tag:
                continue
            player_name = a_tag.get_text(strip=True)
            player_href = a_tag.get("href")
            if player_href and "/player/" in player_href:  # 过滤非球员链接
                players.append({"name": player_name, "url": player_href})
        return players
    except Exception as e:
        print(f"获取页面失败 {page_url}: {str(e)}")
        return []

if __name__ == "__main__":
    base_url = "https://basketball.realgm.com/international/stats/2023/Averages/Qualified/All/player/All/desc"
    all_player_data = []
    
    # 遍历1到66页
    for page_num in range(1, 66):
        page_url = f"{base_url}/{page_num}"
        print(f"正在处理第 {page_num} 页: {page_url}")
        players = get_players_from_page(page_url)
        
        for player in players:
            print(f"正在爬取球员: {player['name']}")
            profile = scrape_player_profile(player['url'])
            if profile:
                # 合并姓名、URL和profile数据
                row_data = {"Name": player['name'], "URL": player['url'], **profile}
                all_player_data.append(row_data)
    
    # 将数据写入Google Sheet
    if all_player_data:
        # 获取所有可能的表头（兼容不同球员的profile字段差异）
        all_headers = set()
        for data in all_player_data:
            all_headers.update(data.keys())
        headers = sorted(all_headers)
        
        # 整理成Sheet需要的行格式
        rows = [headers]
        for data in all_player_data:
            row = [data.get(header, "") for header in headers]
            rows.append(row)
        
        # 写入Sheet
        worksheet.append_rows(rows, value_input_option="USER_ENTERED")
        print(f"数据写入完成，共处理 {len(all_player_data)} 名球员")

关键改动说明 #

1. 分页遍历逻辑：复用你CSV代码中的分页循环，遍历1到66页的统计页面
2. 精准链接定位：只提取data-th="Player"的td中的<a>标签，且过滤包含/player/的有效球员链接，避免无效请求
3. 错误处理：添加raise_for_status()捕获HTTP错误，以及异常处理避免单个请求失败导致程序终止
4. 兼容字段差异：收集所有球员的profile字段作为表头，确保不同球员的缺失字段用空值填充
5. 类名修正：将profileBox改为正确的profile-box

内容的提问来源于stack exchange，提问作者Anthony Madle