Python Selenium网页爬虫处理元素加载超时及网站报错问题咨询

核心修改点 #

• 新增driver全局超时配置，页面加载、JS执行最长等待时间均设为10秒，超出自动抛出异常
• 移除原代码中隐式等待配置，统一使用显式等待，避免两类等待混用导致的计时逻辑混乱
• 新增网站错误提示检测逻辑，匹配到"Sorry, something went wrong"类报错直接触发刷新
• 修复原代码中WebDriverWait行无效.format()的bug，原选择器无占位符，多余的format会导致元素匹配逻辑异常
• 外层循环新增重试机制，异常时最多重试3次，避免单次刷新失败的情况

修改后完整代码 #

# 需提前导入依赖包
from selenium.common.exceptions import TimeoutException, NoSuchElementException
import sys
import time
from selenium import webdriver
from selenium.webdriver.common.by import By
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.support import expected_conditions as EC

def get_data():
    # 先检测是否出现网站报错提示
    try:
        driver.find_element(By.XPATH, "//*[contains(text(), 'Sorry, something went wrong')]")
        raise RuntimeError("网站返回错误提示页")
    except NoSuchElementException:
        pass
    
    data_list = []
    while len(data_list) < 3:
        try:
            data = driver.find_elements(By.CLASS_NAME, 'text-color-main-secondary.text-sm.font-bold.text-left')
            count = len(data)
            data_list.append(data)
            time.sleep(.05)
            driver.execute_script("window.scrollTo(0, document.body.scrollHeight);")
            # 显式等待设为7秒，加上页面加载超时10秒的限制，总等待时长不会超过10秒
            WebDriverWait(driver, 7).until(EC.visibility_of_element_located((By.CLASS_NAME,
                                                                         'text-color-main-secondary.text-sm.font-bold.text-left')))
        except TimeoutException:
            break

    text = []
    elements = []
    for i in range(len(data_list)):
        for j in range(len(data_list[i])):
            t = data_list[i][j].text
            elements.append(data_list[i][j])
            for word in t.split():
                if '#' in word:
                    text.append(word)
    return text, elements


option = webdriver.ChromeOptions()
option.add_extension('')
path = ''
driver = webdriver.Chrome(executable_path=path, options=option)
# 新增全局超时配置
driver.set_page_load_timeout(10)
driver.set_script_timeout(10)
driver.get('')
login(passphrase)
driver.switch_to.window(driver.window_handles[0])

max_retry = 3
retry_count = 0
while True:
    try:
        if retry_count >= max_retry:
            print("重试次数达上限，等待10分钟后执行下一轮任务")
            retry_count = 0
            time.sleep(600)
            driver.refresh()
            
        infos, elements = get_data()
        data, message = check_data(infos, elements)
        if data:
            send_alert(message)
        # 任务正常执行完成，重置重试计数
        retry_count = 0
        time.sleep(600)
        driver.refresh()
    except (TimeoutException, RuntimeError, NoSuchElementException) as e:
        retry_count += 1
        exception_type, exception_object, exception_traceback = sys.exc_info()
        line_number = exception_traceback.tb_lineno
        print(f"第{retry_count}次重试，异常信息：{e}，行号：{line_number}")
        driver.refresh()
        time.sleep(2)
    except Exception as e:
        exception_type, exception_object, exception_traceback = sys.exc_info()
        line_number = exception_traceback.tb_lineno
        print(f"未知异常：{e}，行号：{line_number}")
        retry_count += 1
        driver.refresh()
        time.sleep(2)

内容的提问来源于stack exchange，提问作者questionmaster69