Selenium爬取NBA球队数据时出现AttributeError崩溃问题求助

开发者阿泽

2026-8-6

NBA数据爬虫崩溃问题：AttributeError: 'NoneType' object has no attribute 'find_all'

我用Selenium开发了一款NBA数据爬虫，目标获取30支球队的进阶数据，但运行时仅完成部分URL爬取就崩溃，抛出AttributeError: 'NoneType' object has no attribute 'find_all'错误。

爬虫代码

#web scraper
from bs4 import BeautifulSoup
import requests
from selenium import webdriver
import pandas as pd
import os


class NBAScraper:
    def __init__(self):
        #part 1
        url = "https://www.nba.com/teams"
        HTML = requests.get(url)
        soup = BeautifulSoup(HTML.text, 'html.parser')
        text = str(soup.find_all("a", "Anchor_anchor__cSc3P TeamFigureLink_teamFigureLink__uqnNO"))
        ids = []
        for i in range(0, 30):
            hr = text.find("stats")
            ids.append(text[(hr+11):(hr+21)])
            text = text[(hr+22):]
        #part 2
        
        names = []
        for j in range(0, 30):    
            url2 = "https://www.nba.com/stats/team/"+str(ids[j])+"/advanced"
            HTML2 = requests.get(url2)
            soup2 = BeautifulSoup(HTML2.text, 'html.parser')
        ##div class="TeamHeader_name__MmHlP
            name = str(soup2.find("div", "TeamHeader_name__MmHlP"))
            ni = name.find("div>")
            ni2 = name.find("<!")
            name1 = name[(ni+4):ni2]
            name = name[ni2:]
            ni3 = name.find("<div>")
            name = name[(ni3+5):]
            ni4 = name.find("</div>")
            name2 = name[:ni4]
            n = name1 + " " + name2
            names.append(n)
        ##tbody class="Crom_body__UYOcU"
        #part 3
        offrtg = []
        defrtg = []
        reb = []
        tov = []
        efg = []
        
        for k in range(0, 30):
            self.driver = webdriver.Chrome()
            url3 = "https://www.nba.com/stats/team/"+str(ids[k])+"/advanced"
            self.driver.get(url3)
            rndrhtml = self.driver.page_source
            self.driver.close()
            #self.driver.quit()
            soup3 = BeautifulSoup(rndrhtml, 'html.parser')
            ovrall = str(soup3.find("tbody", "Crom_body__UYOcU").find_all("td"))
            for d in range(0, 13):
                di = ovrall.find("<td>")
                ovrall = ovrall[(di+4):]
                #conditions
                if d == 2:
                    di2 = ovrall.find("</td>")
                    offrtg.append(float(ovrall[:di2]))
                elif d == 3:
                    di2 = ovrall.find("</td>")
                    defrtg.append(float(ovrall[:di2]))
                elif d == 10:
                    di2 = ovrall.find("</td>")
                    reb.append(float(ovrall[:di2]))
                elif d == 11:
                    di2 = ovrall.find("</td>")
                    tov.append(float(ovrall[:di2]))
                elif d == 12:
                    di2 = ovrall.find("</td>")
                    efg.append(float(ovrall[:di2]))
                    
                   
        #writing to excel
        os.remove(r"C:\Users\jackm\OneDrive\Desktop\NBA\NBASTATS.xlsx")
        d = {'Name': names, 'OFFRTG': offrtg, 'DEFRTG': defrtg, 'REB': reb, 
             'TOV': tov, 'EFG': efg}
        df = pd.DataFrame(data=d)
        df.to_excel(r"C:\Users\jackm\OneDrive\Desktop\NBA\NBASTATS.xlsx", sheet_name="STATS")

                
            
        
NBAScraper()

已尝试的解决手段

调整driver的close/quit调用逻辑
将driver实例放到单独函数中运行
测试单URL：首次运行报错，第二次正常
尝试使用隐式等待，无效果

错误栈信息

Traceback (most recent call last):

  File "C:\Program Files\Spyder\pkgs\spyder_kernels\py3compat.py", line 356, in compat_exec
    exec(code, globals, locals)

  File "c:\users\jackm\spyder\nba.py", line 104, in <module>
    NBAScraper()

  File "c:\users\jackm\spyder\nba.py", line 71, in __init__
    ovrall = str(soup3.find("tbody", "Crom_body__UYOcU").find_all("td"))

AttributeError: 'NoneType' object has no attribute 'find_all'

问题原因与解决方案

核心原因

错误本质是页面动态内容未加载完成就获取了页面源码，导致soup3.find("tbody", "Crom_body__UYOcU")返回None，后续调用find_all触发异常。此外原代码存在多处脆弱的字符串处理逻辑，以及频繁创建销毁driver的资源浪费问题。

具体修复方案

使用显式等待确保元素加载完成
替换直接获取页面源码的逻辑，用WebDriverWait等待目标tbody元素出现后再获取源码，这是解决动态加载问题的核心。
复用ChromeDriver实例
不要在循环中每次新建driver，初始化时创建一次，所有爬取完成后再quit，减少浏览器启动关闭的开销，避免因频繁创建实例导致的加载不稳定。
优化元素解析逻辑
放弃将BeautifulSoup对象转成字符串再截取的方式，直接使用BeautifulSoup的API提取数据，逻辑更稳定且易维护。
添加异常处理
捕获元素未找到的异常，记录错误信息并继续执行，避免单个球队的爬取失败导致整个程序崩溃。
优化球队ID和名称的提取
原代码通过字符串截取获取ID和名称的方式极易受页面结构变化影响，改用BeautifulSoup直接解析属性和标签内容。

修改后的完整代码

from bs4 import BeautifulSoup
import requests
from selenium import webdriver
from selenium.webdriver.common.by import By
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.support import expected_conditions as EC
import pandas as pd
import os
import time


class NBAScraper:
    def __init__(self):
        self.driver = webdriver.Chrome()
        self.wait = WebDriverWait(self.driver, 10)  # 10秒超时等待
        self.ids = self._get_team_ids()
        self.names = self._get_team_names()
        self.stats = self._get_team_stats()
        self._save_to_excel()
        self.driver.quit()

    def _get_team_ids(self):
        """提取30支球队的ID"""
        url = "https://www.nba.com/teams"
        html = requests.get(url).text
        soup = BeautifulSoup(html, 'html.parser')
        team_links = soup.find_all("a", "Anchor_anchor__cSc3P TeamFigureLink_teamFigureLink__uqnNO")
        ids = []
        for link in team_links[:30]:  # 确保只取30支球队
            href = link.get("href")
            # 从href中提取ID，示例href：/stats/team/1610612747/advanced
            team_id = href.split("/")[4]
            ids.append(team_id)
        return ids

    def _get_team_names(self):
        """提取球队名称"""
        names = []
        for team_id in self.ids:
            url = f"https://www.nba.com/stats/team/{team_id}/advanced"
            html = requests.get(url).text
            soup = BeautifulSoup(html, 'html.parser')
            name_div = soup.find("div", "TeamHeader_name__MmHlP")
            if name_div:
                # 直接获取div下的文本内容，合并空格
                full_name = " ".join([text.strip() for text in name_div.stripped_strings])
                names.append(full_name)
            else:
                names.append(f"Unknown_{team_id}")
        return names

    def _get_team_stats(self):
        """爬取球队进阶数据"""
        offrtg = []
        defrtg = []
        reb = []
        tov = []
        efg = []

        for team_id in self.ids:
            url = f"https://www.nba.com/stats/team/{team_id}/advanced"
            self.driver.get(url)
            try:
                # 等待tbody元素加载完成
                tbody = self.wait.until(
                    EC.presence_of_element_located((By.CLASS_NAME, "Crom_body__UYOcU"))
                )
                page_source = self.driver.page_source
                soup = BeautifulSoup(page_source, 'html.parser')
                rows = soup.find("tbody", "Crom_body__UYOcU").find_all("tr")
                if rows:
                    # 取第一行数据（球队总数据）
                    tds = rows[0].find_all("td")
                    # 对应原代码的索引：2=OFFRTG,3=DEFRTG,10=REB,11=TOV,12=EFG
                    offrtg.append(float(tds[2].text.strip()))
                    defrtg.append(float(tds[3].text.strip()))
                    reb.append(float(tds[10].text.strip()))
                    tov.append(float(tds[11].text.strip()))
                    efg.append(float(tds[12].text.strip()))
                else:
                    # 无数据时填充默认值或标记
                    offrtg.append(None)
                    defrtg.append(None)
                    reb.append(None)
                    tov.append(None)
                    efg.append(None)
            except Exception as e:
                print(f"爬取球队ID {team_id} 时出错: {str(e)}")
                # 出错时填充空值，避免数据长度不一致
                offrtg.append(None)
                defrtg.append(None)
                reb.append(None)
                tov.append(None)
                efg.append(None)
            time.sleep(1)  # 添加短暂延迟，避免请求过于频繁被反爬

        return {
            'OFFRTG': offrtg,
            'DEFRTG': defrtg,
            'REB': reb,
            'TOV': tov,
            'EFG': efg
        }

    def _save_to_excel(self):
        """保存数据到Excel"""
        excel_path = r"C:\Users\jackm\OneDrive\Desktop\NBA\NBASTATS.xlsx"
        # 如果文件存在则删除
        if os.path.exists(excel_path):
            os.remove(excel_path)
        # 构造DataFrame
        data = {
            'Name': self.names,
            **self.stats
        }
        df = pd.DataFrame(data)
        df.to_excel(excel_path, sheet_name="STATS", index=False)


if __name__ == "__main__":
    NBAScraper()

额外说明

显式等待的超时时间可根据网络情况调整（当前设为10秒）
添加了time.sleep(1)避免请求过于频繁触发反爬机制
拆分了功能为多个小方法，代码结构更清晰，便于维护和调试
异常处理确保单个球队爬取失败不会中断整个程序
数据保存时添加了文件存在检查，避免os.remove报错

内容的提问来源于stack exchange，提问作者Jack Rodgers

Selenium爬取NBA球队数据时出现AttributeError崩溃问题求助

爬虫代码 #

已尝试的解决手段 #

错误栈信息 #

问题原因与解决方案 #

核心原因 #

具体修复方案 #

修改后的完整代码 #

额外说明 #