WebDriver循环获取重定向URL时随机卡死问题求助

核心问题分析 #

你的代码存在几个关键缺陷，导致随机卡死：

1. 等待条件无效：presence_of_element_located((By.TAG_NAME, 'html')) 无法判断重定向是否完成，因为<html>元素在页面启动加载时就已存在，起不到等待重定向结束的作用。
2. 干扰元素处理不彻底：仅处理alert、ESC按键和多窗口，未覆盖现代网站常见的Cookie横幅、全屏广告遮罩、动态加载的广告iframe等干扰项。
3. 无全局超时控制：页面若因广告脚本或反爬机制陷入无限加载，脚本会一直挂起，没有超时退出逻辑。

针对性解决方案 #

1. 初始化浏览器时屏蔽干扰源头 #

添加Chrome参数，从根本上减少广告、弹窗和通知的触发：

from selenium import webdriver
from selenium.webdriver.chrome.options import Options

chrome_options = Options()
# 禁用浏览器通知
chrome_options.add_argument("--disable-notifications")
# 阻止自动弹窗
chrome_options.add_argument("--disable-popup-blocking")
# 设置页面加载超时（30秒）
chrome_options.set_page_load_timeout(30)
# 伪装真实浏览器UA（可选，避免被反爬识别）
chrome_options.add_argument("user-agent=Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/118.0.0.0 Safari/537.36")

browser = webdriver.Chrome(options=chrome_options)

2. 等待重定向完成的正确逻辑 #

替换无效的等待，改为等待URL稳定（连续1秒内URL不再变化），确保重定向彻底完成：

import time

def wait_for_redirect_stable(browser, timeout=30):
    start_time = time.time()
    previous_url = browser.current_url
    while time.time() - start_time < timeout:
        current_url = browser.current_url
        if current_url != previous_url:
            previous_url = current_url
            time.sleep(1)
        else:
            time.sleep(0.5)
            # 二次确认URL是否稳定
            if browser.current_url == previous_url:
                break
    return browser.current_url

3. 增强干扰元素处理 #

针对Cookie横幅、广告遮罩、iframe广告添加定向关闭逻辑：

from selenium.webdriver.common.by import By

def handle_interference(browser):
    # 关闭Cookie同意横幅（适配常见文本）
    try:
        cookie_btn = browser.find_element(By.XPATH, "//button[contains(text(), 'Accept') or contains(text(), '同意') or contains(text(), '确认')]")
        if cookie_btn.is_displayed():
            cookie_btn.click()
            time.sleep(1)
    except:
        pass
    
    # 关闭全屏广告遮罩
    try:
        ad_close_btn = browser.find_element(By.XPATH, "//button[contains(@class, 'close') or contains(@class, 'ad-close') or @aria-label='Close']")
        if ad_close_btn.is_displayed():
            ad_close_btn.click()
            time.sleep(1)
    except:
        pass
    
    # 处理iframe内的广告
    try:
        for iframe in browser.find_elements(By.TAG_NAME, 'iframe'):
            try:
                browser.switch_to.frame(iframe)
                iframe_close_btn = browser.find_element(By.XPATH, "//button[contains(@class, 'close')]")
                iframe_close_btn.click()
                browser.switch_to.default_content()
                time.sleep(1)
            except:
                browser.switch_to.default_content()
                continue
    except:
        pass

4. 完整修改后的代码 #

from selenium import webdriver
from selenium.webdriver.common.by import By
from selenium.webdriver.chrome.options import Options
import time

# 初始化浏览器
chrome_options = Options()
chrome_options.add_argument("--disable-notifications")
chrome_options.add_argument("--disable-popup-blocking")
chrome_options.set_page_load_timeout(30)
chrome_options.add_argument("user-agent=Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/118.0.0.0 Safari/537.36")
browser = webdriver.Chrome(options=chrome_options)

# 待处理链接列表
Link = [
    'https://app.dnbhoovers.com/event/sourceArticleUrlRedirect/1/162418991?eventId=2f331d8e-1bbb-33a0-8174-30aa424a136d',
    'https://app.dnbhoovers.com/event/sourceArticleUrlRedirect/1/162420206?eventId=8be681bb-5739-3349-9aa4-3ae140d47548',
    'https://app.dnbhoovers.com/event/sourceArticleUrlRedirect/1/162418555?eventId=f486fa1c-5d6f-321b-9960-b14c7a15bd49',
    'https://app.dnbhoovers.com/event/sourceArticleUrlRedirect/1/162420946?eventId=58567a43-81a9-3a2e-a5ac-e50a90138ce5',
    'https://app.dnbhoovers.com/event/sourceArticleUrlRedirect/1/162421120?eventId=10aa9569-a69b-3f3a-90e4-013ff42aaf97',
    'https://app.dnbhoovers.com/event/sourceArticleUrlRedirect/1/162418992?eventId=9266999c-6003-37bb-93c3-afd52d775f09',
    'https://app.dnbhoovers.com/event/sourceArticleUrlRedirect/1/162422062?eventId=aa8b2344-e2db-3556-b2d9-dc85d1d83c9a',
    'https://app.dnbhoovers.com/event/sourceArticleUrlRedirect/1/162419497?eventId=cd6764d7-daf3-39a4-8d74-985d95b0bf24',
    'https://app.dnbhoovers.com/event/sourceArticleUrlRedirect/1/162422072?eventId=f2701b2f-0402-39ab-a72e-728b9b216299',
    'https://app.dnbhoovers.com/event/sourceArticleUrlRedirect/1/162418312?eventId=bae82bfa-b30d-3378-a53a-dd4326385af1',
    'https://app.dnbhoovers.com/event/sourceArticleUrlRedirect/1/162419135?eventId=677ed6fe-e4b8-3989-97df-fe6164fc71ed',
    'https://app.dnbhoovers.com/event/sourceArticleUrlRedirect/1/162419403?eventId=e7ae57c3-4f75-330d-a269-07528493edd5',
    'https://app.dnbhoovers.com/event/sourceArticleUrlRedirect/1/162418851?eventId=47a09a61-5beb-350b-a0f7-4e39bd3f8cf4',
    'https://app.dnbhoovers.com/event/sourceArticleUrlRedirect/1/162421562?eventId=efcbc9cf-ee2d-3582-9334-6ba67e335c8d',
    'https://app.dnbhoovers.com/event/sourceArticleUrlRedirect/1/162417353?eventId=c842db45-22b2-3b52-b3b6-7e4a67ac6b52',
    'https://app.dnbhoovers.com/event/sourceArticleUrlRedirect/1/162415817?eventId=8b345b91-7a70-3291-bc32-f1bde38f56a2',
    'https://app.dnbhoovers.com/event/sourceArticleUrlRedirect/1/162417836?eventId=5bc3eb70-304a-3c2d-bcb0-f4c737d1537e',
    'https://app.dnbhoovers.com/event/sourceArticleUrlRedirect/1/162416215?eventId=3ec86460-8913-3b4b-a0ee-ba5429f530ed',
    'https://app.dnbhoovers.com/event/sourceArticleUrlRedirect/1/162412269?eventId=d50ff2ba-3e59-3418-8eee-218264667b72',
    'https://app.dnbhoovers.com/event/sourceArticleUrlRedirect/1/162416690?eventId=17720f16-1233-3c5e-9d4b-aaaade2792c0'
]

redirectedlink = []

def wait_for_redirect_stable(browser, timeout=30):
    start_time = time.time()
    previous_url = browser.current_url
    while time.time() - start_time < timeout:
        current_url = browser.current_url
        if current_url != previous_url:
            previous_url = current_url
            time.sleep(1)
        else:
            time.sleep(0.5)
            if browser.current_url == previous_url:
                break
    return browser.current_url

def handle_interference(browser):
    # 关闭Cookie横幅
    try:
        cookie_btn = browser.find_element(By.XPATH, "//button[contains(text(), 'Accept') or contains(text(), '同意') or contains(text(), '确认')]")
        if cookie_btn.is_displayed():
            cookie_btn.click()
            time.sleep(1)
    except:
        pass
    
    # 关闭广告遮罩
    try:
        ad_close_btn = browser.find_element(By.XPATH, "//button[contains(@class, 'close') or contains(@class, 'ad-close') or @aria-label='Close']")
        if ad_close_btn.is_displayed():
            ad_close_btn.click()
            time.sleep(1)
    except:
        pass
    
    # 处理iframe广告
    try:
        for iframe in browser.find_elements(By.TAG_NAME, 'iframe'):
            try:
                browser.switch_to.frame(iframe)
                iframe_close_btn = browser.find_element(By.XPATH, "//button[contains(@class, 'close')]")
                iframe_close_btn.click()
                browser.switch_to.default_content()
                time.sleep(1)
            except:
                browser.switch_to.default_content()
                continue
    except:
        pass

# 遍历处理链接
for link in Link:
    try:
        browser.get(link)
        # 等待重定向稳定
        final_url = wait_for_redirect_stable(browser)
        # 处理页面干扰元素
        handle_interference(browser)
        # 关闭多余窗口
        parent_window = browser.current_window_handle
        for win_id in browser.window_handles:
            if win_id != parent_window:
                browser.switch_to.window(win_id)
                browser.close()
        browser.switch_to.window(parent_window)
        # 记录最终URL
        redirectedlink.append(final_url)
    except Exception as e:
        print(f"处理链接 {link} 出错: {str(e)}")
        redirectedlink.append(None)
    print(redirectedlink)

browser.quit()

额外优化建议 #

• 针对你提到的卡死网站（如bnnbloomberg.ca、proactiveinvestors.co.uk），可单独编写专属的元素定位规则，这类网站通常有定制化的广告或反爬机制。
• 增加日志文件记录，方便定位具体卡死的链接及当时的页面状态。
• 若仍存在卡死情况，可尝试使用browser.execute_script("window.stop();")强制终止页面加载，避免无限等待。

内容的提问来源于stack exchange，提问作者r_sh_07