使用httpx构造请求绕过Cloudflare防护的代码突然失效排查

需要获取部署了Cloudflare安全防护机制的目标站点https://betway.com/en/sports的HTML正文内容，已编写Python实现代码如下：

def reqja3():
    """Get request"""
    import ssl, httpx

    ssl_ctx = ssl.SSLContext(protocol=ssl.PROTOCOL_TLSv1_2)
    ssl_ctx.set_alpn_protocols(["h2", "http/1.1"])
    ssl_ctx.set_ecdh_curve("prime256v1")
    ssl_ctx.set_ciphers(
        "TLS_AES_256_GCM_SHA384:TLS_CHACHA20_POLY1305_SHA256:"
        "TLS_AES_128_GCM_SHA256:ECDHE-ECDSA-AES256-GCM-SHA384:"
        "ECDHE-RSA-AES256-GCM-SHA384:DHE-RSA-AES256-GCM-SHA384:"
        "ECDHE-ECDSA-CHACHA20-POLY1305:ECDHE-RSA-CHACHA20-POLY1305:"
        "DHE-RSA-CHACHA20-POLY1305:ECDHE-ECDSA-AES128-GCM-SHA256:"
        "ECDHE-RSA-AES128-GCM-SHA256:DHE-RSA-AES128-GCM-SHA256:"
        "ECDHE-ECDSA-AES256-SHA384:ECDHE-RSA-AES256-SHA384:"
        "DHE-RSA-AES256-SHA256:ECDHE-ECDSA-AES128-SHA256:"
        "ECDHE-RSA-AES128-SHA256:DHE-RSA-AES128-SHA256:"
        "ECDHE-ECDSA-AES256-SHA:ECDHE-RSA-AES256-SHA:"
        "DHE-RSA-AES256-SHA:ECDHE-ECDSA-AES128-SHA:"
        "ECDHE-RSA-AES128-SHA:DHE-RSA-AES128-SHA:"
        "RSA-PSK-AES256-GCM-SHA384:DHE-PSK-AES256-GCM-SHA384:"
        "RSA-PSK-CHACHA20-POLY1305:DHE-PSK-CHACHA20-POLY1305:"
        "ECDHE-PSK-CHACHA20-POLY1305:AES256-GCM-SHA384:"
        "PSK-AES256-GCM-SHA384:PSK-CHACHA20-POLY1305:"
        "RSA-PSK-AES128-GCM-SHA256:DHE-PSK-AES128-GCM-SHA256:"
        "AES128-GCM-SHA256:PSK-AES128-GCM-SHA256:AES256-SHA256:"
        "AES128-SHA256:ECDHE-PSK-AES256-CBC-SHA384:"
        "ECDHE-PSK-AES256-CBC-SHA:SRP-RSA-AES-256-CBC-SHA:"
        "SRP-AES-256-CBC-SHA:RSA-PSK-AES256-CBC-SHA384:"
        "DHE-PSK-AES256-CBC-SHA384:RSA-PSK-AES256-CBC-SHA:"
        "DHE-PSK-AES256-CBC-SHA:AES256-SHA:PSK-AES256-CBC-SHA384:"
        "PSK-AES256-CBC-SHA:ECDHE-PSK-AES128-CBC-SHA256:ECDHE-PSK-AES128-CBC-SHA:"
        "SRP-RSA-AES-128-CBC-SHA:SRP-AES-128-CBC-SHA:RSA-PSK-AES128-CBC-SHA256:"
        "DHE-PSK-AES128-CBC-SHA256:RSA-PSK-AES128-CBC-SHA:"
        "DHE-PSK-AES128-CBC-SHA:AES128-SHA:PSK-AES128-CBC-SHA256:PSK-AES128-CBC-SHA"
    )
    client = httpx.Client(http2=True, verify=ssl_ctx)

    print(
        client.get(
            "https://betway.com/en/sports",
            headers={
                "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,*/*;q=0.8",
                "Accept-Encoding": "gzip, deflate",
                "Accept-Language": "en-US,en;q=0.5",
                "Connection": "keep-alive",
                "Host": "betway.com",
                "Upgrade-Insecure-Requests": "1",
                "Sec-Fetch-Dest": "document",
                "Sec-Fetch-Mode": "navigate",
                "Sec-Fetch-Site": "none",
                "Sec-Fetch-User": "?1",
                "TE": "trailers",
                "User-Agent": "Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:100.0) Gecko/20100101 Firefox/100.0",
                "Cookie": "bw_BrowserId=76501619282281851930349603208014741540; _ga=GA1.2.1003986219.1650841035; _fbp=fb.1.1650841035798.971215073; COOKIE_POLICY_ACCEPTED=true; TrackingVisitId=67e26f62-e357-443d-be0c-83223d7ab902; hash=67e26f62-e357-443d-be0c-83223d7ab902; bw_SessionId=47d27eaa-623f-4434-a03b-9716d4b829a0; StaticResourcesVersion=12.43.0.1; ssc_btag=67e26f62-e357-443d-be0c-83223d7ab902; SpinSportVisitId=d369d188-39c6-41c6-8058-5aa297dd50c0; userLanguage=en; TimezoneOffset=120; _gid=GA1.2.381640013.1652975492; _gat_UA-1515961-1=1; ens_firstPageView=true; _gat=1; AMCVS_74756B615BE2FD4A0A495EB8%40AdobeOrg=1",
            },
        ).text
    )

reqja3()

经此前验证，获取该站点内容无需执行JavaScript，只要构造与真实浏览器特征完全一致的请求即可绕过Cloudflare校验，核心是完整模拟浏览器的请求行为。上述代码已配置SSL参数、指定TLS协议版本、启用HTTP/2支持，同时模拟了Firefox浏览器的标准请求头与有效Cookie信息，这套实现此前一直可以正常获取目标站点内容，但近期突然无法通过Cloudflare校验。

代码存在4处和真实浏览器特征不匹配的配置，会被Cloudflare指纹识别拦截：

• 强制锁定TLS 1.2协议：代码中ssl.PROTOCOL_TLSv1_2直接禁用了TLS 1.3，而当前所有主流浏览器默认优先使用TLS 1.3握手，JA3指纹和真实浏览器完全不符。
• TLS参数硬编码过时：ECDH曲线仅配置prime256v1、cipher套件列表是Firefox 100版本的旧配置，和当前新版Firefox的TLS握手参数顺序、支持套件列表不一致，且硬编码的User-Agent对应Firefox 100属于过时版本，会触发Cloudflare额外校验。
• Cookie硬编码失效：手动拼接的Cookie包含会话ID、临时统计标识等短有效期字段，过期后携带无效Cookie请求会直接被判定为异常流量。
• 请求头特征不匹配：Accept-Encoding缺失br（Brotli）压缩标识、多余携带TE: trailers头、HTTP/2模式下手动传递Connection: keep-alive和Host头属于非浏览器行为，且httpx默认的请求头排序、HTTP/2帧参数和真实Firefox存在差异。

方案1：手动修正httpx配置对齐浏览器特征 #

1. 替换SSL上下文配置，取消TLS 1.2强制锁定，对齐新版Firefox的TLS参数：

ssl_ctx = ssl.create_default_context()
ssl_ctx.set_alpn_protocols(["h2", "http/1.1"])
ssl_ctx.set_ecdh_curve("X25519:prime256v1:secp384r1")
ssl_ctx.set_ciphers(
    "TLS_AES_128_GCM_SHA256:TLS_CHACHA20_POLY1305_SHA256:TLS_AES_256_GCM_SHA384:"
    "ECDHE-ECDSA-AES128-GCM-SHA256:ECDHE-RSA-AES128-GCM-SHA256:ECDHE-ECDSA-CHACHA20-POLY1305:"
    "ECDHE-RSA-CHACHA20-POLY1305:ECDHE-ECDSA-AES256-GCM-SHA384:ECDHE-RSA-AES256-GCM-SHA384:"
    "ECDHE-ECDSA-AES128-SHA256:ECDHE-RSA-AES128-SHA256:ECDHE-ECDSA-AES256-SHA384:"
    "ECDHE-RSA-AES256-SHA384:ECDHE-ECDSA-AES128-SHA:ECDHE-RSA-AES128-SHA:"
    "ECDHE-ECDSA-AES256-SHA:ECDHE-RSA-AES256-SHA:AES128-GCM-SHA256:AES256-GCM-SHA384:"
    "AES128-SHA:AES256-SHA"
)

2. 移除硬编码Cookie、多余请求头，修正请求头配置匹配新版Firefox，使用httpx的自动Cookie持久化能力：

client = httpx.Client(
    http2=True, 
    verify=ssl_ctx,
    follow_redirects=True,
    headers={
        "User-Agent": "Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:128.0) Gecko/20100101 Firefox/128.0",
        "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,*/*;q=0.8",
        "Accept-Language": "en-US,en;q=0.5",
        "Accept-Encoding": "gzip, deflate, br",
        "Upgrade-Insecure-Requests": "1",
        "Sec-Fetch-Dest": "document",
        "Sec-Fetch-Mode": "navigate",
        "Sec-Fetch-Site": "none",
        "Sec-Fetch-User": "?1",
    }
)
resp = client.get("https://betway.com/en/sports")
print(resp.text)

方案2：使用指纹模拟库降低维护成本 #

如果手动调整后仍被拦截，直接使用内置浏览器TLS/HTTP2指纹模拟的库，无需手动维护SSL、请求头参数，示例代码如下：

from curl_cffi import requests

resp = requests.get(
    "https://betway.com/en/sports",
    impersonate="firefox128",
    follow_redirects=True
)
print(resp.text)

注意：模拟浏览器时User-Agent版本需要和TLS、HTTP/2指纹的版本完全对应，不要混用不同版本的参数，否则会被指纹识别判定为异常。

内容的提问来源于stack exchange，提问作者SonyMag