使用Selenium循环爬取URL列表时遭拦截，请求排查解决

问题描述 #

使用Selenium爬取一组巴西总统演讲URL列表时，第一次循环成功获取目标内容，但第二次循环触发TimeoutException，无法定位到目标元素。

原实现代码 #

from selenium import webdriver
from selenium.webdriver.support.wait import WebDriverWait
from selenium.webdriver.common.by import By
from selenium.webdriver.support import expected_conditions as EC
import time
from selenium.webdriver.firefox.options import Options
import pandas as pd  # 原代码遗漏导入，需补充

options = Options()
options.add_argument('--disable-blink-features=AutomationControlled')
options.add_argument('--headless') # ensure GUI is off
options.add_argument('--no-sandbox')
options.add_argument('--disable-dev-shm-usage')

expres = pd.DataFrame()

d2 = [
    'http://www.biblioteca.presidencia.gov.br/presidencia/ex-presidentes/michel-temer/discursos-do-presidente-da-republica/discurso-do-presidente-da-republica-michel-temer-durante-cerimonia-de-inauguracao-do-hospital-notre-dame-barretos-sp',
    'http://www.biblioteca.presidencia.gov.br/presidencia/ex-presidentes/michel-temer/discursos-do-presidente-da-republica/discurso-do-presidente-da-republica-michel-temer-durante-a-visita-a-25a-edicao-da-feira-internacional-de-tecnologia-agricola-em-acao-agrishow',
    'http://www.biblioteca.presidencia.gov.br/presidencia/ex-presidentes/michel-temer/discursos-do-presidente-da-republica/brinde-do-presidente-da-republica-michel-temer-no-almoco-oferecido-em-homenagem-ao-senhor-desire-delano-bouterse-presidente-da-republica-do-suriname',
    'http://www.biblioteca.presidencia.gov.br/presidencia/ex-presidentes/michel-temer/discursos-do-presidente-da-republica/declaracao-a-imprensa-do-presidente-da-republica-michel-temer-apos-assinatura-de-atos',
    'http://www.biblioteca.presidencia.gov.br/presidencia/ex-presidentes/michel-temer/discursos-do-presidente-da-republica/discurso-do-presidente-da-republica-michel-temer-durante-a-solenidade-de-abertura-oficial-da-84a-exposicao-internacional-de-gado-zebu-expozebu-2018'
]

for j in d2:
    driver = webdriver.Chrome(options=options)
    driver.maximize_window()
    driver.get(j)
    cont = WebDriverWait(driver, 10).until(EC.visibility_of_element_located((By.XPATH, "//*[@id='content-core']"))).text
    print(cont)

错误信息 #

TimeoutException: Message: 
Stacktrace:
RemoteError@chrome://remote/content/shared/RemoteError.sys.mjs:8:8
WebDriverError@chrome://remote/content/shared/webdriver/Errors.sys.mjs:192:5
NoSuchElementError@chrome://remote/content/shared/webdriver/Errors.sys.mjs:510:5
dom.find/</<@chrome://remote/content/shared/DOM.sys.mjs:136:16

问题原因与修复方案 #

1. 浏览器选项与驱动不匹配 #

原代码导入的是Firefox的Options，但使用的是Chrome驱动，导致配置的反爬选项完全无效，网站容易检测到自动化工具并拦截请求。

修复：改用Chrome专属的ChromeOptions：

from selenium.webdriver.chrome.options import Options

2. 重复创建WebDriver实例 #

每次循环都新建Chrome驱动实例，不仅资源占用高，还会频繁触发网站的反爬检测，且未关闭旧实例可能导致内存泄漏。

修复：将驱动初始化移到循环外，循环结束后统一关闭：

# 初始化驱动（放在循环外）
driver = webdriver.Chrome(options=options)
driver.maximize_window()

for j in d2:
    driver.get(j)
    # 后续操作...

# 循环结束后关闭驱动
driver.quit()

3. 增强反爬配置 #

补充更多模拟正常用户的配置，降低被识别为爬虫的概率：

options.add_argument('--disable-extensions')
options.add_argument('--disable-gpu')
options.add_argument('--start-maximized')
# 添加真实浏览器User-Agent
options.add_argument('user-agent=Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/124.0.0.0 Safari/537.36')
# 禁用自动化提示
options.add_experimental_option("excludeSwitches", ["enable-automation"])
options.add_experimental_option('useAutomationExtension', False)

4. 优化等待策略 #

适当延长等待时间，或改用presence_of_element_located（元素存在即可，无需等待完全可见），提升稳定性：

cont = WebDriverWait(driver, 15).until(EC.presence_of_element_located((By.ID, "content-core"))).text

修改后的完整代码 #

from selenium import webdriver
from selenium.webdriver.support.wait import WebDriverWait
from selenium.webdriver.common.by import By
from selenium.webdriver.support import expected_conditions as EC
from selenium.webdriver.chrome.options import Options
import pandas as pd

options = Options()
# 反爬配置
options.add_argument('--disable-blink-features=AutomationControlled')
options.add_argument('--headless=new')  # 新版无头模式更接近真实浏览器
options.add_argument('--no-sandbox')
options.add_argument('--disable-dev-shm-usage')
options.add_argument('--disable-extensions')
options.add_argument('--disable-gpu')
options.add_argument('user-agent=Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/124.0.0.0 Safari/537.36')
options.add_experimental_option("excludeSwitches", ["enable-automation"])
options.add_experimental_option('useAutomationExtension', False)

expres = pd.DataFrame()

d2 = [
    'http://www.biblioteca.presidencia.gov.br/presidencia/ex-presidentes/michel-temer/discursos-do-presidente-da-republica/discurso-do-presidente-da-republica-michel-temer-durante-cerimonia-de-inauguracao-do-hospital-notre-dame-barretos-sp',
    'http://www.biblioteca.presidencia.gov.br/presidencia/ex-presidentes/michel-temer/discursos-do-presidente-da-republica/discurso-do-presidente-da-republica-michel-temer-durante-a-visita-a-25a-edicao-da-feira-internacional-de-tecnologia-agricola-em-acao-agrishow',
    'http://www.biblioteca.presidencia.gov.br/presidencia/ex-presidentes/michel-temer/discursos-do-presidente-da-republica/brinde-do-presidente-da-republica-michel-temer-no-almoco-oferecido-em-homenagem-ao-senhor-desire-delano-bouterse-presidente-da-republica-do-suriname',
    'http://www.biblioteca.presidencia.gov.br/presidencia/ex-presidentes/michel-temer/discursos-do-presidente-da-republica/declaracao-a-imprensa-do-presidente-da-republica-michel-temer-apos-assinatura-de-atos',
    'http://www.biblioteca.presidencia.gov.br/presidencia/ex-presidentes/michel-temer/discursos-do-presidente-da-republica/discurso-do-presidente-da-republica-michel-temer-durante-a-solenidade-de-abertura-oficial-da-84a-exposicao-internacional-de-gado-zebu-expozebu-2018'
]

# 初始化驱动
driver = webdriver.Chrome(options=options)
driver.maximize_window()

try:
    for j in d2:
        driver.get(j)
        # 等待目标元素加载
        content_element = WebDriverWait(driver, 15).until(
            EC.presence_of_element_located((By.ID, "content-core"))
        )
        cont = content_element.text
        print(cont)
        # 将内容存入DataFrame（可选）
        expres = pd.concat([expres, pd.DataFrame({'content': [cont], 'url': [j]})], ignore_index=True)
finally:
    # 确保驱动关闭
    driver.quit()

# 保存结果（可选）
expres.to_csv('presidential_speeches.csv', index=False)

内容的提问来源于stack exchange，提问作者Marcelo Soares