Selenium代码二次迭代报错修复及优化需求

问题背景 #

现有一段Selenium代码用于爬取Dhan杠杆计算器页面，首次迭代可正常获取杠杆值，但第二次迭代触发TimeoutException，偶尔还会输出错误的杠杆值（如0.00X）。

原代码 #

import sys
import time
from selenium import webdriver
from selenium.webdriver.common.keys import Keys
from selenium.webdriver.common.by import By
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.support import expected_conditions as EC
import chromedriver_autoinstaller

# Setup chrome options
chrome_options = webdriver.ChromeOptions()
chrome_options.add_argument('--headless')  # Ensure GUI is off
chrome_options.add_argument('--no-sandbox')
chrome_options.add_argument('--disable-dev-shm-usage')

# Set path to chromedriver as per your configuration
chromedriver_autoinstaller.install()

# Set the target URL
url = "https://dhan.co/calculators/margin-calculator/"

# Set up the webdriver
driver = webdriver.Chrome(options=chrome_options)

# Navigate to the target URL
driver.get(url)

# List of company names
company_names = ['Aurum Proptech ', 'Bal Pharma ','Entertainment Network','Career Point','Amines & Plasticizers']

def slow_type(element, text):
    for char in text:
        element.send_keys(char)
        time.sleep(0.1)  # Adjust this sleep duration for the desired typing speed

for company_name in company_names:
    # Find the search input field
    search_input = driver.find_element(By.ID, "free-solo-2-demo")
    search_input.clear()

    # Type the company name slowly
    slow_type(search_input, company_name)

    # Wait for the search results to appear
    WebDriverWait(driver, 10).until(
        EC.visibility_of_all_elements_located((By.CLASS_NAME, "MuiAutocomplete-option"))
    )

    # Click on the first suggestion
    suggestion = driver.find_element(By.CLASS_NAME, "MuiAutocomplete-option")
    suggestion.click()

    # Find the quantity input field and enter the quantity
    quantity_input = driver.find_element(By.ID, "qty")
    quantity_input.clear()
    quantity_input.send_keys("1")

    # Wait for a moment to let the page update
    time.sleep(2)

    # Find and extract the leverage information
    leverage_element = driver.find_element(By.ID, "int_leverage")
    leverage_value = leverage_element.text

    # Print the leverage value
    print(f"{company_name} Leverage: {leverage_value}")

# Close the browser window
driver.quit()

错误信息 #

Aurum Proptech  Leverage: 5.00 X
---------------------------------------------------------------------------
TimeoutException                          Traceback (most recent call last)
<ipython-input-78-e4f1f7a6f0fe> in <cell line: 36>()
     43 
     44     # Wait for the search results to appear
---> 45     WebDriverWait(driver, 10).until(
     46         EC.visibility_of_all_elements_located((By.CLASS_NAME, "MuiAutocomplete-option"))
     47     )
/usr/local/lib/python3.10/dist-packages/selenium/webdriver/support/wait.py in until(self, method, message)
    103             if time.monotonic() > end_time:
    104                 break
--> 105         raise TimeoutException(message, screen, stacktrace)
    106 
    107     def until_not(self, method: Callable[[D], T], message: str = "") -> Union[T, Literal[True]]:

TimeoutException: Message:

页面HTML结构 #

<input aria-invalid="false" autocomplete="off" id="free-solo-2-demo" placeholder="Search for scrips you want to invest or trade in.." type="text" class="MuiInputBase-input MuiOutlinedInput-input MuiInputBase-inputAdornedStart MuiAutocomplete-input MuiAutocomplete-inputFocused css-1ixds2g" aria-autocomplete="list" aria-expanded="false" autocapitalize="none" spellcheck="false" role="combobox" value="">
<input type="text" class="w-full border-[0.5px] p-2 border-[#DADADA] outline-none rounded-[6px] font-CircularBook" placeholder="Quantity" id="qty" maxlength="7" autocomplete="off">
<p class="lg:text-lg md:text-base text-sm pt-1 text-[#4B4B4B] font-CircularMedium" id="int_leverage">5.00 X</p>

错误原因分析 #

1. 搜索框未彻底清空：仅用clear()可能无法完全清除输入框内容，导致第二次输入时自动补全逻辑异常。
2. 等待条件不合理：EC.visibility_of_all_elements_located要求所有选项都可见，实际只需要第一个选项出现即可，容易触发超时。
3. 硬编码等待不可靠：time.sleep(2)是固定时长，页面更新速度受网络影响，可能提前读取到未更新的初始值（0.00X）。
4. 元素定位无等待：直接用find_element未确保元素可交互，增加出错概率。

修复后的代码 #

import sys
from selenium import webdriver
from selenium.webdriver.common.keys import Keys
from selenium.webdriver.common.by import By
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.support import expected_conditions as EC
import chromedriver_autoinstaller

# 配置Chrome选项
chrome_options = webdriver.ChromeOptions()
chrome_options.add_argument('--headless=new')  # 新版无头模式，兼容性更好
chrome_options.add_argument('--no-sandbox')
chrome_options.add_argument('--disable-dev-shm-usage')
chrome_options.add_argument('--disable-blink-features=AutomationControlled')  # 规避反爬检测
chrome_options.add_experimental_option("excludeSwitches", ["enable-automation"])
chrome_options.add_experimental_option('useAutomationExtension', False)

# 自动安装ChromeDriver
chromedriver_autoinstaller.install()

# 目标URL
url = "https://dhan.co/calculators/margin-calculator/"

# 初始化浏览器
driver = webdriver.Chrome(options=chrome_options)
driver.get(url)

# 公司名称列表
company_names = ['Aurum Proptech', 'Bal Pharma', 'Entertainment Network', 'Career Point', 'Amines & Plasticizers']

# 全局复用显式等待实例
wait = WebDriverWait(driver, 15)

for company_name in company_names:
    try:
        # 等待搜索框可交互，彻底清空内容
        search_input = wait.until(EC.element_to_be_clickable((By.ID, "free-solo-2-demo")))
        search_input.send_keys(Keys.CONTROL + "a")
        search_input.send_keys(Keys.DELETE)
        search_input.send_keys(company_name)

        # 等待第一个自动补全选项可见并点击
        first_suggestion = wait.until(EC.visibility_of_element_located((By.CSS_SELECTOR, ".MuiAutocomplete-option:first-child")))
        first_suggestion.click()

        # 等待数量输入框可交互，输入1
        quantity_input = wait.until(EC.element_to_be_clickable((By.ID, "qty")))
        quantity_input.send_keys(Keys.CONTROL + "a")
        quantity_input.send_keys(Keys.DELETE)
        quantity_input.send_keys("1")

        # 等待杠杆值更新完成（确保文本包含"X"，避免读取初始值）
        wait.until(EC.text_to_be_present_in_element((By.ID, "int_leverage"), "X"))
        leverage_value = driver.find_element(By.ID, "int_leverage").text

        print(f"{company_name} Leverage: {leverage_value}")
    except Exception as e:
        print(f"处理{company_name}时出错: {str(e)}")
        continue

# 关闭浏览器
driver.quit()

关键优化点 #

• 升级无头模式：使用--headless=new替代旧版，解决部分页面渲染异常问题。
• 彻底清空输入：用Ctrl+A全选后删除，确保输入框无残留内容。
• 精准等待条件：
  • 仅等待第一个自动补全选项可见，减少超时概率。
  • 等待杠杆值元素文本包含"X"，确保页面已完成更新。
• 规避反爬检测：添加参数隐藏自动化特征，降低被网站拦截的风险。
• 异常捕获机制：单个公司处理出错时不终止程序，提升鲁棒性。

内容的提问来源于stack exchange，提问作者Adhithyan Vs