遍历导航链接收集网络日志时遇StaleElementReferenceException求助

StaleElementReferenceException 异常的核心原因是：当你点击导航栏中的<a>标签跳转到新页面后，初始从首页DOM中获取的导航元素会失效——因为新页面加载后，原首页的DOM已被销毁，这些元素不再附着于当前DOM树，后续循环访问它们时就会触发异常。

from selenium import webdriver
from selenium.webdriver.common.by import By
from selenium.webdriver.chrome.options import Options
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.support import expected_conditions as EC
from urllib.parse import urljoin
import time
import json


def get_logs():
    capabilities = webdriver.DesiredCapabilities.CHROME.copy()
    capabilities['goog:loggingPrefs'] = {'performance': 'ALL'}
    chrome_options = Options()
    driver = webdriver.Chrome(desired_capabilities=capabilities, options=chrome_options)
    wait = WebDriverWait(driver, 10)
    base_url = "https://www.lambdatest.com/"
    driver.get(base_url)

    # 第一步：收集所有导航链接的URL（避免后续操作依赖失效元素）
    urls = set()
    try:
        # 等待导航栏元素加载完成
        nav_links = wait.until(EC.presence_of_all_elements_located((By.CLASS_NAME, "nav-link")))
        
        for link in nav_links:
            tag_name = link.tag_name
            if tag_name == 'a':
                # 处理直接跳转的导航链接
                href = link.get_attribute('href')
                if href:
                    absolute_url = urljoin(base_url, href)
                    urls.add(absolute_url)
            elif tag_name == 'button':
                # 处理下拉菜单按钮，点击展开并收集子链接
                try:
                    # 使用JS点击避免元素拦截问题
                    driver.execute_script("arguments[0].click();", link)
                    # 等待下拉菜单加载完成
                    wait.until(EC.visibility_of_element_located((By.XPATH, "./following-sibling::ul", link)))
                    # 获取下拉菜单中的所有链接
                    dropdown_links = link.find_elements(By.XPATH, "./following-sibling::ul//a")
                    for a_tag in dropdown_links:
                        href = a_tag.get_attribute('href')
                        if href:
                            absolute_url = urljoin(base_url, href)
                            urls.add(absolute_url)
                except Exception as e:
                    print(f"处理下拉菜单失败: {str(e)}")
                    continue
    except Exception as e:
        print(f"收集导航链接失败: {str(e)}")

    # 第二步：遍历所有URL，收集网络日志
    for url in urls:
        try:
            driver.get(url)
            # 等待页面基本加载完成
            wait.until(EC.presence_of_element_located((By.TAG_NAME, "body")))
            # 额外等待确保网络请求完成
            time.sleep(2)
            # 获取并保存网络日志
            logs = driver.get_log('performance')
            with open('/data/network_logs.json', 'a') as f:
                for log in logs:
                    f.write(json.dumps(log) + '\n')
        except Exception as e:
            print(f"访问URL失败 {url}: {str(e)}")
            continue

    driver.quit()


if __name__ == '__main__':
    get_logs()

• 先收集URL再遍历：提前从首页DOM中提取所有导航链接的URL，避免后续页面跳转后依赖失效元素
• 使用WebDriverWait替代固定sleep：基于元素状态等待，提升脚本稳定性和效率
• 处理相对URL：通过urljoin将相对路径转换为绝对URL，确保导航正确
• 去重处理：使用集合存储URL，避免重复访问同一页面
• 异常捕获：添加局部异常处理，避免单个链接失败导致整个脚本终止
• JS点击下拉按钮：规避元素被遮挡导致的点击失败问题

内容的提问来源于stack exchange，提问作者Roopesh Saravanan