Python Selenium爬取破产公告时TimeoutException问题解决求助

需从指定网站提取当日破产公告：输入当日日期，选择下拉菜单的「Eröffnungen」选项，点击搜索后提取缩放图标对应的公告文本。使用Selenium编写的脚本随机出现TimeoutException，在定位公告文本元素时失败，脚本需每日自动执行，求稳定解决方案。

原代码 #

import...

driver = webdriver.Safari()
driver.maximize_window()
driver.get("https://neu.insolvenzbekanntmachungen.de/ap/suche.jsf")

date_input = WebDriverWait(driver, 10).until(
    ec.visibility_of_element_located((By.ID, "frm_suche:ldi_datumVon:datumHtml5"))
)
curr_date = datetime.today().strftime("%Y-%m-%d")
arg_value = f"arguments[0].value = '{curr_date}';"
driver.execute_script("arguments[0].value = '';", date_input)
driver.execute_script(arg_value, date_input)

# Select the "Eröffnungen" option from the dropdown menu
select_element = WebDriverWait(driver, 20).until(
    ec.visibility_of_element_located((By.ID, "frm_suche:lsom_gegenstand:lsom"))
)
driver.execute_script("arguments[0].scrollIntoView(true);", select_element)
time.sleep(1)
select = Select(select_element)
select.select_by_value("2")

# Execute the search
driver.execute_script("arguments[0].dispatchEvent(new Event('change'));", date_input)
date_input.send_keys(Keys.RETURN)

results_table = WebDriverWait(driver, 20).until(
    ec.visibility_of_element_located((By.ID, "tbl_ergebnis"))
)

rows = results_table.find_elements(By.TAG_NAME, "tr")
data = []

# Iterate over each row of the table
for i, row in enumerate(rows):
    cells = row.find_elements(By.TAG_NAME, "td")
    if len(cells) > 0:
        # Click the zoom icon to open the new window
        zoom_icon = cells[6].find_element(By.TAG_NAME, "input[type='image']")
        WebDriverWait(driver, 20).until(
            ec.element_to_be_clickable((By.TAG_NAME, "input[type='image']"))
        )
        driver.execute_script("arguments[0].scrollIntoView(true);", zoom_icon)
        time.sleep(1)  # Wait for scroll
        driver.execute_script("arguments[0].click();", zoom_icon)

        # Wait for the new window to open
        WebDriverWait(driver, 20).until(ec.new_window_is_opened)
        driver.switch_to.window(driver.window_handles[1])

        # Extract the publication text
        print("here")
        WebDriverWait(driver, 20).until(
            ec.presence_of_element_located((By.XPATH,
"//form[@id='form']//pre[@id='veroefftext']"))
        )
        print("here2")
        pub_text = driver.find_element(By.XPATH, "//form[@id='form']//pre[@id='veroefftext']").text

        # Close the new window and switch to the first window again
        driver.close()
        driver.switch_to.window(driver.window_handles[0])

        # Store the extracted data
        data.append({
            'veroeffentlichungsdatum': cells[0].text,
            'aktenzeichen': cells[1].text,
            'gericht': cells[2].text,
            'name_vorname_bezeichnung': cells[3].text,
            'sitz_wohnsitz': cells[4].text,
            'register': cells[5].text,
            'veroeffentlichungstext': pub_text
        })

df = pd.DataFrame(data)
df.columns = ['veroeffentlichungsdatum', 'aktuelles_aktenzeichen', 'gericht',
              'name_vorname_bezeichnung', 'sitz_wohnsitz', 'register', 'veroeffentlichungstext']
df.to_excel("data/insos.xlsx", index=False)

问题分析 #

随机出现TimeoutException的核心原因：

• 切换新窗口后仅等待窗口打开，未确保页面DOM完全加载
• 使用presence_of_element_located仅判断元素存在，不保证元素内容已渲染完成
• 依赖固定time.sleep，等待逻辑稳定性差
• 全局定位input[type='image']可能匹配到页面其他同类元素，导致点击错误

稳定解决方案 #

1. 优化新窗口等待逻辑 #

切换窗口后，等待页面核心元素可见，而非仅判断窗口存在：

# 等待新窗口打开
original_handles = driver.window_handles
zoom_icon.click()
WebDriverWait(driver, 20).until(lambda d: len(d.window_handles) > len(original_handles))
driver.switch_to.window(driver.window_handles[-1])
# 等待公告文本元素可见
WebDriverWait(driver, 30).until(ec.visibility_of_element_located((By.ID, "veroefftext")))

2. 替换元素定位条件 #

将presence_of_element_located改为visibility_of_element_located，确保元素已渲染完成：

pub_text = WebDriverWait(driver, 30).until(
    ec.visibility_of_element_located((By.ID, "veroefftext"))
).text

3. 移除固定等待，改用显式等待 #

去掉time.sleep(1)，用显式等待替代，避免等待不足或冗余：

# 缩放图标点击前的处理
WebDriverWait(driver, 20).until(ec.element_to_be_clickable(zoom_icon))
driver.execute_script("arguments[0].scrollIntoView({block: 'center'});", zoom_icon)
zoom_icon.click()

4. 精准定位缩放图标 #

避免全局定位，改为当前行内的精准定位：

zoom_icon = WebDriverWait(driver, 20).until(
    ec.element_to_be_clickable((By.XPATH, ".//td[7]/input[@type='image']"))
)

5. 添加重试机制 #

针对自动执行需求，为失败的行添加重试逻辑：

from selenium.common.exceptions import TimeoutException

for i, row in enumerate(rows):
    cells = row.find_elements(By.TAG_NAME, "td")
    if len(cells) > 0:
        success = False
        retry_count = 0
        max_retries = 3
        while not success and retry_count < max_retries:
            try:
                # 点击缩放图标、切换窗口、提取文本的逻辑
                success = True
            except TimeoutException:
                retry_count += 1
                # 重试前清理窗口句柄
                if len(driver.window_handles) > 1:
                    driver.switch_to.window(driver.window_handles[0])
                    driver.close()
                print(f"第{i+1}行重试第{retry_count}次")
        if not success:
            print(f"第{i+1}行提取失败，跳过")
            continue
        # 存储数据逻辑

6. 优化搜索触发逻辑 #

改为直接点击搜索按钮，替代原有的模拟回车操作：

# 替换原搜索执行代码
search_button = WebDriverWait(driver, 20).until(
    ec.element_to_be_clickable((By.ID, "frm_suche:btn_suchen"))
)
search_button.click()

内容的提问来源于stack exchange，提问作者user707