Selenium循环采集学校数据出现StaleElementReferenceException错误如何解决

StaleElementReferenceException 触发的核心逻辑是：你点击提交按钮后页面会刷新/局部更新DOM，你提前获取的school下拉框实例、school.options列表对应的原始DOM元素已经被页面销毁，下一轮循环访问k.text时就会因为引用了过期元素抛出错误。

核心思路是不缓存可能会随页面更新的元素，每次操作前都重新从DOM中获取最新的元素，同时用显式等待替代固定时长的time.sleep提升稳定性，修改后的代码如下：

import requests
import scrapy
import selenium
from bs4 import BeautifulSoup
import csv
from selenium import webdriver
from selenium.common.exceptions import TimeoutException, StaleElementReferenceException
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.support import expected_conditions as EC
from selenium.webdriver.common.by import By
import time
from selenium.webdriver.support.ui import Select

driver = webdriver.Chrome()
wait = WebDriverWait(driver, 15) # 全局显式等待，最长等15秒
url = 'https://rajshaladarpan.nic.in/SD1/Home/Public/EmployeeSearch.aspx'
driver.get(url)

# 初始页操作
wait.until(EC.element_to_be_clickable((By.CSS_SELECTOR, '#ContentPlaceHolder1_EmployeeDetail_rbtSchOffSeclection'))).click()
wait.until(EC.element_to_be_clickable((By.CSS_SELECTOR, '#ContentPlaceHolder1_EmployeeDetail_btnSearch'))).click()

# 下拉框路径统一存为变量，避免重复编写
DISTRICT_XPATH = '/html/body/form/div[3]/div[5]/div/div/div[2]/div/div/div/div[1]/div[3]/div/div[3]/div[1]/div[1]/div/select'
BLOCK_XPATH = '/html/body/form/div[3]/div[5]/div/div/div[2]/div/div/div/div[1]/div[3]/div/div[3]/div[1]/div[2]/div/select'
SCHOOL_XPATH = '/html/body/form/div[3]/div[5]/div/div/div[2]/div/div/div/div[1]/div[3]/div/div[3]/div[2]/div[1]/div[2]/select'
SUBMIT_BTN_ID = 'ContentPlaceHolder1_EmployeeDetail_Button6'

dis = [] # 原代码中用到了dis但未声明，此处补充定义

# 先提取所有区县的文本存为普通列表，不直接遍历可能过期的DOM元素
dist = Select(wait.until(EC.presence_of_element_located((By.XPATH, DISTRICT_XPATH))))
dist_text_list = [i.text for i in dist.options if i.text != '--Select District--']

for dist_text in dist_text_list:
    # 每次选择区县前重新获取最新的下拉框元素
    dist = Select(wait.until(EC.presence_of_element_located((By.XPATH, DISTRICT_XPATH))))
    dist.select_by_visible_text(dist_text)
    print(dist_text)
    dis.append(dist_text)
    time.sleep(1)

    # 提取当前区县下所有街区的文本列表
    block = Select(wait.until(EC.presence_of_element_located((By.XPATH, BLOCK_XPATH))))
    block_text_list = [b.text for b in block.options if b.text not in ('Select Block', 'ALL', '')]

    for block_text in block_text_list:
        # 每次选择街区前重新获取最新的下拉框元素
        block = Select(wait.until(EC.presence_of_element_located((By.XPATH, BLOCK_XPATH))))
        block.select_by_visible_text(block_text)
        print(block_text)
        # 等待学校下拉框加载完成
        wait.until(EC.presence_of_element_located((By.XPATH, SCHOOL_XPATH)))
        time.sleep(1)

        # 提取当前街区下所有学校的文本列表
        school = Select(driver.find_element(By.XPATH, SCHOOL_XPATH))
        school_text_list = [k.text for k in school.options if k.text != 'Select School']

        for school_text in school_text_list:
            try:
                # 每次选学校前重新获取最新的学校下拉框元素
                school = Select(wait.until(EC.presence_of_element_located((By.XPATH, SCHOOL_XPATH))))
                school.select_by_visible_text(school_text)
                print(school_text)

                # 提交表单
                wait.until(EC.element_to_be_clickable((By.ID, SUBMIT_BTN_ID))).submit()
                # 等待页面提交完成，可自行补充表格加载判断逻辑
                time.sleep(3)
                # 页面刷新后重新选中之前的区县和街区，恢复选择状态
                dist = Select(wait.until(EC.presence_of_element_located((By.XPATH, DISTRICT_XPATH))))
                dist.select_by_visible_text(dist_text)
                time.sleep(1)
                block = Select(wait.until(EC.presence_of_element_located((By.XPATH, BLOCK_XPATH))))
                block.select_by_visible_text(block_text)
                wait.until(EC.presence_of_element_located((By.XPATH, SCHOOL_XPATH)))
                time.sleep(1)
            except StaleElementReferenceException:
                # 极端情况触发异常时自动重试
                print(f"获取{school_text}失败，重试")
                time.sleep(2)
                continue

• 提前把所有下拉框的选项文本提取出来存为普通字符串列表，遍历字符串而不是遍历随时可能过期的DOM元素对象，从根源上避免访问过期元素属性
• 每次选择下拉框选项前都重新从DOM中获取最新的下拉框元素，保证操作的是当前页面上真实存在的元素
• 提交表单页面刷新后，重新选中之前的区县和街区，保证后续学校选项加载正确
• 增加了StaleElementReferenceException异常捕获，极端情况触发时自动重试

内容的提问来源于stack exchange，提问作者Abhishek Saini