如何解决Playwright无头浏览器爬取哥伦比亚央行时被反爬拦截

我编写了如下脚本，用于爬取哥伦比亚央行（Banco de la República）的TRM（市场代表性汇率）相关数据：

from playwright.sync_api import sync_playwright
from loguru import logger

# Function that extracts the data from the Bank of Colombia
def main():
    with sync_playwright() as pw:
        # I create the browser to interact with the page:
        browser = pw.chromium.launch(headless=True)

        # Login on the website:
        banco_central_col = browser.new_page()
        banco_central_col.set_default_timeout(240000)
        logger.info(f"'Entering the page -------> Tasa Representativa del Mercado (TRM - Peso por dólar) | Banco de la República'")

        banco_central_col.goto('https://www.banrep.gov.co/es/estadisticas/trm')
        logger.info(f"Successful Entry")
        the_html = banco_central_col.locator("//html").inner_html()
        logger.warning(the_html)
        # logger.info(f"Entering the page -------> '{banco_central_col.title()}'")

        # From here I enter the section of the exchange rate:
        with browser.new_page() as tasa_de_cambio:
            logger.info("My last headless point unfortunately :_(")
            # Taking a screenshot to verify the last connection point:
            banco_central_col.screenshot(path="screenshot.png", full_page=True)

            banco_central_col.locator("//html/body/div[2]/div[1]/div/div/div[3]/div[6]/main/section/div/section[4]/div/div[1]/div[1]/div[2]/div/ul/li[1]/span/a").click(timeout=240000)
            logger.info(f"Clicking on the link 'Serie histórica para un rango de fechas dado (desde 27/11/1991)'")
        new_page = tasa_de_cambio.value
        new_page.wait_for_load_state()
        logger.info(f"Entering the page -------> '{new_page.title()}'")

可视浏览器模式运行一切正常，但切换到headless=True后，网站触发反机器人防护机制。通过调试获取页面HTML并截图发现，无头模式下页面显示反机器人验证界面，无法加载包含目标链接的正常内容；而正常模式下页面能显示包含"Serie histórica para un rango de fechas dado (desde 27/11/1991)"链接的区域，我需要点击该链接继续爬取数据。

依赖版本：

python = "^3.10"
playwright = "^1.27.1"
loguru = "^0.6.0"

针对无头模式被反爬检测的问题，可通过以下方式绕过：

1. 使用Playwright新无头模式 #

Playwright 1.20+支持headless="new"选项，该模式更接近真实浏览器行为，降低被检测概率：

browser = pw.chromium.launch(headless="new")

2. 模拟真实用户环境参数 #

添加匹配目标网站区域的用户代理、视口大小、语言等参数，让无头浏览器更贴近真实用户：

context = browser.new_context(
    user_agent="Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/118.0.0.0 Safari/537.36",
    viewport={"width": 1920, "height": 1080},
    locale="es-CO"
)
banco_central_col = context.new_page()

3. 隐藏自动化特征 #

通过初始化脚本禁用浏览器的自动化检测标志，避免被识别为爬虫：

context.add_init_script("""
    Object.defineProperty(navigator, 'webdriver', {get: () => undefined});
    Object.defineProperty(navigator, 'plugins', {get: () => [1,2,3,4,5]});
    Object.defineProperty(navigator, 'languages', {get: () => ['es-CO', 'es']});
""")

4. 模拟用户操作节奏 #

在页面跳转、点击等操作前添加随机延迟，避免操作过于机械：

import time
import random

# 页面加载后延迟
time.sleep(random.uniform(2, 4))

# 点击链接前延迟
time.sleep(random.uniform(1, 3))
banco_central_col.locator("//html/body/div[2]/div[1]/div/div/div[3]/div[6]/main/section/div/section[4]/div/div[1]/div[1]/div[2]/div/ul/li[1]/span/a").click(timeout=240000)

5. 优化页面跳转逻辑 #

原代码中处理新页面的方式存在问题，改用wait_for_event("popup")等待新页面加载：

# 点击链接后等待新页面弹出
new_page = banco_central_col.wait_for_event("popup")
new_page.wait_for_load_state()
logger.info(f"进入新页面 -------> '{new_page.title()}'")

优化后的完整代码 #

from playwright.sync_api import sync_playwright
from loguru import logger
import time
import random

def main():
    with sync_playwright() as pw:
        # 启用新无头模式
        browser = pw.chromium.launch(headless="new")
        # 创建带真实环境参数的上下文
        context = browser.new_context(
            user_agent="Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/118.0.0.0 Safari/537.36",
            viewport={"width": 1920, "height": 1080},
            locale="es-CO"
        )
        # 添加反检测脚本
        context.add_init_script("""
            Object.defineProperty(navigator, 'webdriver', {get: () => undefined});
            Object.defineProperty(navigator, 'plugins', {get: () => [1,2,3,4,5]});
            Object.defineProperty(navigator, 'languages', {get: () => ['es-CO', 'es']});
        """)
        
        banco_central_col = context.new_page()
        banco_central_col.set_default_timeout(240000)
        logger.info("进入页面 -------> Tasa Representativa del Mercado (TRM - Peso por dólar) | Banco de la República")

        banco_central_col.goto('https://www.banrep.gov.co/es/estadisticas/trm')
        logger.info("成功进入页面")
        # 模拟用户浏览延迟
        time.sleep(random.uniform(2, 4))
        
        banco_central_col.screenshot(path="screenshot.png", full_page=True)
        logger.info("已保存当前页面截图")
        
        # 点击链接前添加延迟
        time.sleep(random.uniform(1, 3))
        banco_central_col.locator("//html/body/div[2]/div[1]/div/div/div[3]/div[6]/main/section/div/section[4]/div/div[1]/div[1]/div[2]/div/ul/li[1]/span/a").click(timeout=240000)
        logger.info("点击链接 'Serie histórica para un rango de fechas dado (desde 27/11/1991)'")
        
        # 等待新页面加载完成
        new_page = banco_central_col.wait_for_event("popup")
        new_page.wait_for_load_state()
        logger.info(f"进入新页面 -------> '{new_page.title()}'")
        
        # 后续爬取逻辑...
        browser.close()

if __name__ == "__main__":
    main()

内容的提问来源于stack exchange，提问作者willyro93