Selenium WebDriver TimeoutException：登录后无法获取Numbers页面内容

我需要自动获取Numbers页面内容，但必须先完成登录。以下是我的代码，运行超过10次仅成功3~4次，隔天重启电脑也一样，这种情况已持续数日。尝试过多种等待方式、延长超时时间，页面一直加载但问题依旧。

原代码 #

from selenium import webdriver
from selenium.webdriver.common.by import By
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.support import expected_conditions as EC
from selenium.webdriver.support.ui import Select
from bs4 import BeautifulSoup
import time
import random

# List of driver classes
drivers = [webdriver.Firefox, webdriver.Chrome, webdriver.Edge]

# Shuffle the list to choose a driver randomly
random.shuffle(drivers)

# Try to initialize a driver
for driver_class in drivers:
    try:
        driver = driver_class()
        break
    except Exception:
        continue
else:
    raise Exception("No suitable driver found")

# Go to the login page
driver.get('https://www.orangecarrier.com/login')

# Find the email and password fields
email_field = driver.find_element(by=By.NAME, value='email')
password_field = driver.find_element(by=By.NAME, value='password')

# Type your email and password into the fields
email_field.send_keys('xxx@yopmail.com')
password_field.send_keys('xxxx')

# Submit the form
driver.find_element(by=By.ID, value="loginSubmit").click()
driver.get('https://www.orangecarrier.com/numbers')

WebDriverWait(driver, 30).until(EC.visibility_of_element_located((By.TAG_NAME, 'select')))

# Find the select tag
select = Select(driver.find_element(by=By.NAME, value='MyNumbers_length'))
select.select_by_value('500')

WebDriverWait(driver,30).until(EC.visibility_of_element_located((By.TAG_NAME, 'td')))
page_cont = driver.page_source

html_soup = BeautifulSoup(page_cont, "html.parser")
print(html_soup)
driver.close()

报错信息 #

---------------------------------------------------------------------------
TimeoutException                          Traceback (most recent call last)
c:\Users\...\tesst.ipynb Cell 3 line 4
     39 driver.find_element(by=By.ID, value="loginSubmit").click()
     40 driver.get('https://www.orangecarrier.com/numbers')
---> 42 WebDriverWait(driver, 30).until(EC.visibility_of_element_located((By.TAG_NAME, 'select')))
     44 # Find the select tag
     45 select = Select(driver.find_element(by=By.NAME, value='MyNumbers_length'))

File c:\Program Files\Python311\Lib\site-packages\selenium\webdriver\support\wait.py:95, in WebDriverWait.until(self, method, message)
     93     if time.monotonic() > end_time:
     94         break
---> 95 raise TimeoutException(message, screen, stacktrace)

TimeoutException: Message: 
Stacktrace:
    GetHandleVerifier [0x00007FF6DBB08EF2+54786]
(No symbol) [0x00007FF6DBA75612]
(No symbol) [0x00007FF6DB92A64B]
(No symbol) [0x00007FF6DB96B79C]
(No symbol) [0x00007FF6DB96B91C]
(No symbol) [0x00007FF6DB9A6D87]
(No symbol) [0x00007FF6DB98BEAF]
(No symbol) [0x00007FF6DB9A4D02]
(No symbol) [0x00007FF6DB98BC43]
(No symbol) [0x00007FF6DB960941]
(No symbol) [0x00007FF6DB961B84]
GetHandleVerifier [0x00007FF6DBE57F52+3524194]
GetHandleVerifier [0x00007FF6DBEAD800+3874576]
GetHandleVerifier [0x00007FF6DBEA5D7F+3843215]
GetHandleVerifier [0x00007FF6DBBA5086+694166]
(No symbol) [0x00007FF6DBA80A88]
(No symbol) [0x00007FF6DBA7CA94]
(No symbol) [0x00007FF6DBA7CBC2]
(No symbol) [0x00007FF6DBA6CC83]
BaseThreadInitThunk [0x00007FFF8A43257D+29]
RtlUserThreadStart [0x00007FFF8C10AA78+40]

解决方案 #

1. 优化登录跳转逻辑 #

点击登录按钮后，不要直接跳转页面，先等待登录操作完成（比如等待URL变为登录后的地址，或者等待登录成功后才会出现的元素），避免未完成登录就访问Numbers页面导致未授权加载失败：

# 替换原登录后的driver.get逻辑
WebDriverWait(driver, 30).until(EC.url_contains('dashboard'))  # 假设登录后跳转到dashboard页面
driver.get('https://www.orangecarrier.com/numbers')

2. 精准定位元素 #

用更具体的选择器定位目标select，避免用By.TAG_NAME这种宽泛的定位方式：

# 替换原等待select的代码
WebDriverWait(driver, 30).until(EC.visibility_of_element_located((By.NAME, 'MyNumbers_length')))

3. 添加反爬友好的操作间隔 #

在输入账号密码、点击登录后添加随机延迟，模拟真人操作节奏：

# 输入账号密码后添加延迟
time.sleep(random.uniform(1, 2))
# 点击登录后添加延迟
driver.find_element(by=By.ID, value="loginSubmit").click()
time.sleep(random.uniform(2, 3))

4. 配置浏览器优化参数 #

给浏览器添加无头模式、禁用图片加载等配置，减少资源加载时间，同时降低被反爬识别的概率：

# 以Chrome为例添加配置
options = webdriver.ChromeOptions()
options.add_argument('--headless=new')  # 无头模式
options.add_argument('--disable-images')  # 禁用图片加载
driver = webdriver.Chrome(options=options)

5. 添加重试机制 #

给关键步骤添加重试逻辑，超时后重新尝试：

from selenium.common.exceptions import TimeoutException

# 定义重试函数
def wait_with_retry(driver, condition, timeout=30, retries=2):
    for _ in range(retries + 1):
        try:
            return WebDriverWait(driver, timeout).until(condition)
        except TimeoutException:
            driver.refresh()
    raise TimeoutException("多次重试后仍超时")

# 使用重试函数等待元素
wait_with_retry(driver, EC.visibility_of_element_located((By.NAME, 'MyNumbers_length')))

修改后的完整示例代码 #

from selenium import webdriver
from selenium.webdriver.common.by import By
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.support import expected_conditions as EC
from selenium.webdriver.support.ui import Select
from selenium.common.exceptions import TimeoutException
from bs4 import BeautifulSoup
import time
import random

# 配置浏览器选项（以Chrome为例）
options = webdriver.ChromeOptions()
options.add_argument('--disable-images')
options.add_argument('--start-maximized')

# 初始化浏览器
driver = webdriver.Chrome(options=options)

# 重试等待函数
def wait_with_retry(driver, condition, timeout=30, retries=2):
    for _ in range(retries + 1):
        try:
            return WebDriverWait(driver, timeout).until(condition)
        except TimeoutException:
            driver.refresh()
    raise TimeoutException("多次重试后仍超时")

try:
    # 访问登录页
    driver.get('https://www.orangecarrier.com/login')
    
    # 输入账号密码
    email_field = wait_with_retry(driver, EC.visibility_of_element_located((By.NAME, 'email')))
    password_field = wait_with_retry(driver, EC.visibility_of_element_located((By.NAME, 'password')))
    
    email_field.send_keys('xxx@yopmail.com')
    password_field.send_keys('xxxx')
    
    # 模拟真人延迟
    time.sleep(random.uniform(1, 2))
    
    # 提交登录
    login_btn = wait_with_retry(driver, EC.element_to_be_clickable((By.ID, 'loginSubmit')))
    login_btn.click()
    
    # 等待登录完成
    wait_with_retry(driver, EC.url_contains('dashboard'))  # 根据实际登录后跳转URL调整
    time.sleep(random.uniform(2, 3))
    
    # 访问Numbers页面
    driver.get('https://www.orangecarrier.com/numbers')
    
    # 等待目标select加载
    select_element = wait_with_retry(driver, EC.visibility_of_element_located((By.NAME, 'MyNumbers_length')))
    select = Select(select_element)
    select.select_by_value('500')
    
    # 等待表格数据加载
    wait_with_retry(driver, EC.visibility_of_element_located((By.TAG_NAME, 'td')))
    
    # 获取页面内容
    page_cont = driver.page_source
    html_soup = BeautifulSoup(page_cont, "html.parser")
    print(html_soup)
finally:
    driver.quit()  # 确保浏览器关闭

内容的提问来源于stack exchange，提问作者BABYBEAUTY