使用Selenium回退页面后如何继续循环遍历find_elements链接

我正在开发一款爬虫工具，从公开电话簿按字母顺序收集姓名、地址和电话号码并导出为CSV文件。目标网站支持按字母搜索，当前处理姓氏以A开头的条目，部分姓氏对应数百条记录，需要循环遍历多页。

现有代码通过find_elements获取元素列表，进入详情页采集信息后回退页面，但频繁触发StaleElementReferenceException异常，尝试过WebDriverWait和延长sleep时间均无效。原代码如下：

from selenium import webdriver
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.support import expected_conditions as EC
from selenium.webdriver.common.by import By

import requests

import time
from selenium.webdriver.common.keys import Keys
from selenium.common.exceptions import NoSuchElementException

from openpyxl import Workbook
from openpyxl.utils import get_column_letter


import csv

PATH = '/Users/j_smith/downloads/chromedriver'

driver = webdriver.Chrome(PATH)

driver.get("https://whitepagescanada.ca/bc/surrey/a-z/A/")    
time.sleep(0.1)


name_elements = driver.find_elements_by_xpath("//ul[@class='links-list-alt']")
num_of_name_elements = len(name_elements)
print("Number of elements in the list:", num_of_name_elements)


for name in name_elements:
    time.sleep(0.1)
    name_link = name.find_element_by_tag_name("a").get_attribute("href")
    driver.get(name_link)

    ind_name_elements = driver.find_elements_by_xpath("//a[@itemprop='url']")
    print("Number of names: ", len(ind_name_elements))
    for inds in ind_name_elements:
        time.sleep(0.1)

        inds_link = inds.get_attribute("href")
        driver.get(inds_link)
    

        phoneNum_inds = driver.find_element_by_xpath("//span[@itemprop='telephone']").text
        name_inds = driver.find_element_by_xpath("//span[@itemprop='name']").text
        address_inds = driver.find_element_by_xpath("//span[@itemprop='streetAddress']").text

        print("The phone number is:", phoneNum_inds)
        print("The name is:", name_inds)
        print("The address is:", address_inds)

        driver.back()
        time.sleep(0.1)

    driver.get("https://whitepagescanada.ca/bc/surrey/A/")



driver.quit()

异常原因分析 #

StaleElementReferenceException的核心问题是：页面导航（如driver.get()或driver.back()）后，之前通过find_elements获取的元素引用会失效。原代码中：

1. 在姓氏列表页获取name_elements元素列表，遍历过程中跳转到子页面，再返回后原元素引用已失效
2. 进入姓氏详情页后获取ind_name_elements元素列表，回退到该页面时，这些元素的DOM节点已被重新渲染，引用无法复用

代码调整方案 #

解决思路是提前提取所有需要的URL并保存到列表中，然后遍历URL列表进行操作，完全避免依赖页面元素的引用。同时替换固定sleep为WebDriverWait，提升稳定性。

调整后的代码：

from selenium import webdriver
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.support import expected_conditions as EC
from selenium.webdriver.common.by import By
from selenium.common.exceptions import NoSuchElementException, TimeoutException
import csv

PATH = '/Users/j_smith/downloads/chromedriver'
driver = webdriver.Chrome(PATH)
wait = WebDriverWait(driver, 10)  # 统一设置等待超时时间

# 存储所有采集到的数据
data = []

try:
    # 1. 进入姓氏索引页面，提取所有姓氏链接
    driver.get("https://whitepagescanada.ca/bc/surrey/a-z/A/")
    # 等待姓氏列表加载完成
    wait.until(EC.presence_of_all_elements_located((By.XPATH, "//ul[@class='links-list-alt']//a")))
    # 提取所有姓氏链接到列表，而非保存元素引用
    name_links = [a.get_attribute("href") for a in driver.find_elements(By.XPATH, "//ul[@class='links-list-alt']//a")]
    print(f"找到{len(name_links)}个姓氏条目")

    for name_link in name_links:
        # 2. 进入单个姓氏的详情页，提取所有个人链接
        driver.get(name_link)
        try:
            # 等待个人列表加载完成
            wait.until(EC.presence_of_all_elements_located((By.XPATH, "//a[@itemprop='url']")))
            # 提取所有个人链接到列表
            ind_links = [a.get_attribute("href") for a in driver.find_elements(By.XPATH, "//a[@itemprop='url']")]
            print(f"当前姓氏下找到{len(ind_links)}条记录")

            for ind_link in ind_links:
                # 3. 进入个人详情页采集数据
                driver.get(ind_link)
                try:
                    # 等待关键元素加载完成
                    wait.until(EC.presence_of_element_located((By.XPATH, "//span[@itemprop='name']")))
                    # 采集数据
                    name_inds = driver.find_element(By.XPATH, "//span[@itemprop='name']").text
                    phoneNum_inds = driver.find_element(By.XPATH, "//span[@itemprop='telephone']").text
                    address_inds = driver.find_element(By.XPATH, "//span[@itemprop='streetAddress']").text

                    print(f"姓名: {name_inds}, 电话: {phoneNum_inds}, 地址: {address_inds}")
                    data.append([name_inds, phoneNum_inds, address_inds])
                except (NoSuchElementException, TimeoutException):
                    print(f"个人页面{ind_link}数据采集失败，跳过")
                    continue

        except (NoSuchElementException, TimeoutException):
            print(f"姓氏页面{name_link}加载失败，跳过")
            continue

finally:
    # 4. 将数据导出为CSV文件
    if data:
        with open('phonebook_data.csv', 'w', newline='', encoding='utf-8') as csvfile:
            writer = csv.writer(csvfile)
            writer.writerow(['姓名', '电话号码', '地址'])
            writer.writerows(data)
        print("数据已成功导出到phonebook_data.csv")
    else:
        print("未采集到任何数据")
    driver.quit()

关键修改点 #

• 提前提取URL列表：不再保存元素引用，而是直接提取所有需要的URL存入列表，遍历列表时直接用driver.get()导航，彻底避免元素引用失效问题
• 替换固定sleep为WebDriverWait：使用显式等待等待元素加载完成，比固定sleep更高效且稳定，同时处理超时和元素不存在的异常
• 异常处理增强：添加try-except捕获页面加载失败、元素不存在等异常，避免爬虫中途崩溃
• CSV导出功能完善：将采集到的数据统一保存后导出，确保数据不丢失

内容的提问来源于stack exchange，提问作者redswoosh12