解决Amazon商品详情页Selenium爬取StaleElementReferenceException问题

使用Python Selenium爬取Amazon商品详情页，该商品价格随尺寸不同而变化。需要遍历每个颜色系列对应的尺寸下拉列表，但点击尺寸选项后出现StaleElementReferenceException异常，请求修改代码实现需求。

原代码如下：

from selenium import webdriver
from selenium.webdriver.chrome.service import Service
from selenium.webdriver.common.by import By
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.support import expected_conditions as EC
from selenium.common.exceptions import NoSuchElementException, TimeoutException
from selenium.common.exceptions import StaleElementReferenceException
import time

path = Service('../driver/chromedriver.exe')
driver = webdriver.Chrome(service=path)
driver.maximize_window()

web = 'https://www.amazon.com/Amazon-Essentials-Loose-Fit-Short-Sleeve-Crewneck/dp/B0775CT1J9/ref=sr_1_17?c=ts&keywords=Men%27s+T-Shirts&qid=1688295104&s=apparel&sr=1-17&ts_id=1045624'

try:
    driver.get(web)
    title = WebDriverWait(driver, 10).until(EC.presence_of_element_located((By.CSS_SELECTOR, 'span#productTitle'))).text
    print('Title: ', title)
    try:
        color_boxes = WebDriverWait(driver, 3).until(EC.visibility_of_all_elements_located((By.XPATH, '//ul[@role="radiogroup"]/li')))
        for color_box in color_boxes:
            color_box.click()
            color_family = WebDriverWait(driver, 3).until(EC.presence_of_element_located((By.CSS_SELECTOR, '#variation_color_name span.selection'))).text
            print('color_family: ', color_family)
            time.sleep(1)
            size_box = driver.find_element(By.CSS_SELECTOR, '#dropdown_selected_size_name')
            size_box.click()
            size_available = WebDriverWait(driver, 3).until(EC.presence_of_all_elements_located((By.CSS_SELECTOR, 'li.dropdownAvailable')))
            for size in size_available:
                try:
                    size.click()
                    price = WebDriverWait(driver,6).until(EC.presence_of_element_located((By.CSS_SELECTOR, 'span.reinventPricePriceToPayMargin span.a-price-whole'))).text
                    print(f'price: {price}')
                except StaleElementReferenceException:
                    print("Stale element reference exception occurred.")
                    pass
                size_box = WebDriverWait(driver, 3).until(EC.element_to_be_clickable((By.CSS_SELECTOR, '#dropdown_selected_size_name')))
                size_box.click()
    except (NoSuchElementException, TimeoutException):
        pass
    print('...........................................................................................')
except NoSuchElementException:
    print("Product not found")
    pass

StaleElementReferenceException是因为页面DOM在切换颜色或尺寸时发生了重新渲染，之前缓存的元素（比如size_available列表中的尺寸元素）已经脱离当前DOM树，导致引用失效。原代码提前获取所有尺寸元素并循环操作，后续DOM更新后这些元素就无法再被正常调用。

from selenium import webdriver
from selenium.webdriver.chrome.service import Service
from selenium.webdriver.common.by import By
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.support import expected_conditions as EC
from selenium.common.exceptions import NoSuchElementException, TimeoutException, StaleElementReferenceException

path = Service('../driver/chromedriver.exe')
driver = webdriver.Chrome(service=path)
driver.maximize_window()

web = 'https://www.amazon.com/Amazon-Essentials-Loose-Fit-Short-Sleeve-Crewneck/dp/B0775CT1J9/ref=sr_1_17?c=ts&keywords=Men%27s+T-Shirts&qid=1688295104&s=apparel&sr=1-17&ts_id=1045624'

try:
    driver.get(web)
    title = WebDriverWait(driver, 10).until(EC.presence_of_element_located((By.CSS_SELECTOR, 'span#productTitle'))).text
    print('Title: ', title)
    
    # 获取颜色选项数量，通过索引循环避免持有 stale 元素
    color_count = len(WebDriverWait(driver, 3).until(EC.visibility_of_all_elements_located((By.XPATH, '//ul[@role="radiogroup"]/li'))))
    for color_idx in range(color_count):
        # 每次循环重新定位颜色元素，避免DOM更新后引用失效
        color_boxes = WebDriverWait(driver, 3).until(EC.visibility_of_all_elements_located((By.XPATH, '//ul[@role="radiogroup"]/li')))
        color_box = color_boxes[color_idx]
        color_box.click()
        
        # 获取当前选中的颜色
        color_family = WebDriverWait(driver, 3).until(EC.presence_of_element_located((By.CSS_SELECTOR, '#variation_color_name span.selection'))).text
        print(f'\n当前颜色: {color_family}')
        
        # 处理尺寸下拉
        try:
            # 每次重新定位尺寸下拉按钮
            size_box = WebDriverWait(driver, 3).until(EC.element_to_be_clickable((By.CSS_SELECTOR, '#dropdown_selected_size_name')))
            size_box.click()
            
            # 获取当前可用尺寸的数量，通过索引循环
            size_count = len(WebDriverWait(driver, 3).until(EC.presence_of_all_elements_located((By.CSS_SELECTOR, 'li.dropdownAvailable'))))
            for size_idx in range(size_count):
                try:
                    # 每次循环重新获取可用尺寸列表，避免 stale 引用
                    size_available = WebDriverWait(driver, 3).until(EC.presence_of_all_elements_located((By.CSS_SELECTOR, 'li.dropdownAvailable')))
                    size = size_available[size_idx]
                    size_text = size.text
                    size.click()
                    
                    # 获取对应尺寸的价格
                    price = WebDriverWait(driver, 6).until(
                        EC.presence_of_element_located((By.CSS_SELECTOR, 'span.reinventPricePriceToPayMargin span.a-price-whole'))
                    ).text
                    print(f'尺寸 {size_text}: ${price}')
                    
                    # 重新打开尺寸下拉，准备下一个选项
                    size_box = WebDriverWait(driver, 3).until(EC.element_to_be_clickable((By.CSS_SELECTOR, '#dropdown_selected_size_name')))
                    size_box.click()
                    
                except StaleElementReferenceException:
                    # 遇到 stale 异常时，跳过当前循环，重新获取元素
                    continue
                    
        except (NoSuchElementException, TimeoutException):
            print(f'颜色 {color_family} 无可用尺寸')
            continue
            
except NoSuchElementException:
    print("未找到商品")
finally:
    driver.quit()

• 避免提前缓存元素：不再一次性获取所有颜色/尺寸元素并循环，改为通过索引每次重新定位元素，确保操作的是当前DOM中的有效元素
• 替换固定等待：移除time.sleep()，全部使用显式等待，既提升稳定性又减少不必要的等待时间
• 异常处理优化：遇到StaleElementReferenceException时跳过当前循环，重新获取元素继续执行，而不是直接跳过整个尺寸遍历
• 资源清理：添加finally块确保浏览器正常关闭，避免资源泄漏

内容的提问来源于stack exchange，提问作者Moniruzzaman Monir