Puppeteer无头模式无法下载octet-stream文件，需绕过Cloudflare

针对你遇到的无头模式下载失败、无法直接获取响应内容的问题，以下是几个实用的解决方案，包含.NET和Node.js示例：

方案1：修复无头模式下载配置，捕获异常完成下载 #

新版Chromium的无头模式（HeadlessMode.New）对下载支持更完善，配合下载事件监听，可捕获ERR_ABORTED异常后等待下载完成。

.NET代码示例 #

var browser = await Puppeteer.LaunchAsync(new LaunchOptions
{
    Headless = HeadlessMode.New,
    Args = new[]
    {
        "--disable-blink-features=AutomationControlled", // 隐藏自动化特征
        "--no-sandbox",
        "--disable-setuid-sandbox"
    }
});

var page = await browser.NewPageAsync();
await page.SetViewportAsync(new ViewPortOptions { Width = 1920, Height = 1080 });
await page.SetUserAgentAsync("Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/118.0.0.0 Safari/537.36");

// 配置下载路径（确保目录存在）
var downloadPath = Path.Combine(Directory.GetCurrentDirectory(), "downloads");
Directory.CreateDirectory(downloadPath);
await page.Client.SendAsync("Page.setDownloadBehavior", new
{
    behavior = "allow",
    downloadPath = downloadPath,
    eventsEnabled = true // 启用下载进度监听
});

// 监听下载完成事件
var downloadCompleteTcs = new TaskCompletionSource<string>();
page.Client.MessageReceived += (sender, e) =>
{
    if (e.MessageID == "Page.downloadProgress")
    {
        var progress = e.MessageData.ToObject<DownloadProgress>();
        if (progress.State == "completed")
        {
            downloadCompleteTcs.SetResult(progress.Guid);
        }
    }
};

// 拦截并修改目标请求
await page.SetRequestInterceptionAsync(true);
page.Request += async (s, e) =>
{
    if (e.Request.Url == "你的目标URL")
    {
        var payloadOverride = new Payload
        {
            Method = "POST",
            PostData = "你的POST数据",
            Headers = new Dictionary<string, string>
            {
                ["Content-Type"] = "application/x-www-form-urlencoded"
            }
        };
        await e.Request.ContinueAsync(payloadOverride);
    }
    else
    {
        await e.Request.ContinueAsync();
    }
};

try
{
    await page.GoToAsync("你的目标URL");
}
catch (NavigationException ex) when (ex.Message.Contains("net::ERR_ABORTED"))
{
    // 忽略异常，等待下载完成
    await downloadCompleteTcs.Task;
    // 获取最新下载文件
    var downloadedFile = new DirectoryInfo(downloadPath)
        .GetFiles()
        .OrderByDescending(f => f.CreationTime)
        .First();
    // 读取内容并处理
    string content = await File.ReadAllTextAsync(downloadedFile.FullName);
    string[] jsonChunks = content.Split('\n', StringSplitOptions.RemoveEmptyEntries);
}

// 辅助类：序列化下载进度数据
public class DownloadProgress
{
    public string Guid { get; set; }
    public string State { get; set; }
}

Node.js代码示例 #

const puppeteer = require('puppeteer');
const fs = require('fs');
const path = require('path');

(async () => {
    const downloadPath = path.join(__dirname, 'downloads');
    !fs.existsSync(downloadPath) && fs.mkdirSync(downloadPath);

    const browser = await puppeteer.launch({
        headless: 'new',
        args: [
            '--disable-blink-features=AutomationControlled',
            '--no-sandbox',
            '--disable-setuid-sandbox'
        ]
    });

    const page = await browser.newPage();
    await page.setViewport({ width: 1920, height: 1080 });
    await page.setUserAgent('Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/118.0.0.0 Safari/537.36');

    // 配置下载行为
    await page._client.send('Page.setDownloadBehavior', {
        behavior: 'allow',
        downloadPath: downloadPath,
        eventsEnabled: true
    });

    // 监听下载完成
    const downloadComplete = new Promise(resolve => {
        page._client.on('Page.downloadProgress', progress => {
            if (progress.state === 'completed') resolve(progress.guid);
        });
    });

    // 拦截请求
    await page.setRequestInterception(true);
    page.on('request', async req => {
        if (req.url() === '你的目标URL') {
            await req.continue({
                method: 'POST',
                postData: '你的POST数据',
                headers: { ...req.headers(), 'Content-Type': 'application/x-www-form-urlencoded' }
            });
        } else await req.continue();
    });

    try {
        await page.goto('你的目标URL');
    } catch (err) {
        if (err.message.includes('net::ERR_ABORTED')) {
            await downloadComplete;
            // 获取最新下载文件
            const latestFile = fs.readdirSync(downloadPath)
                .sort((a, b) => fs.statSync(path.join(downloadPath, b)).mtime - fs.statSync(path.join(downloadPath, a)).mtime)[0];
            const content = fs.readFileSync(path.join(downloadPath, latestFile), 'utf8');
            const jsonChunks = content.split('\n').filter(line => line.trim());
        }
    }

    await browser.close();
})();

方案2：提取Puppeteer会话Cookie，用HttpClient直接请求 #

利用Puppeteer已通过Cloudflare验证的会话Cookie，直接用HttpClient发起请求，绕过下载流程。

.NET代码示例 #

var browser = await Puppeteer.LaunchAsync(new LaunchOptions
{
    Headless = HeadlessMode.New,
    Args = new[] { "--disable-blink-features=AutomationControlled", "--no-sandbox" }
});

var page = await browser.NewPageAsync();
await page.SetViewportAsync(new ViewPortOptions { Width = 1920, Height = 1080 });
// 先访问网站首页，通过Cloudflare验证
await page.GoToAsync("网站基础URL");
await page.WaitForSelectorAsync("验证完成后出现的页面元素");

// 提取Cookie
var cookies = await page.GetCookiesAsync();
var cookieContainer = new CookieContainer();
foreach (var cookie in cookies)
{
    cookieContainer.Add(new Cookie
    {
        Name = cookie.Name,
        Value = cookie.Value,
        Domain = cookie.Domain,
        Path = cookie.Path,
        Expires = cookie.Expires ?? DateTime.Now.AddDays(1)
    });
}

// 用HttpClient发起POST请求
using var httpClient = new HttpClient(new HttpClientHandler { CookieContainer = cookieContainer });
httpClient.DefaultRequestHeaders.UserAgent.ParseAdd("Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/118.0.0.0 Safari/537.36");

var requestContent = new FormUrlEncodedContent(new Dictionary<string, string>
{
    ["key1"] = "value1",
    ["key2"] = "value2"
});

var response = await httpClient.PostAsync("你的目标URL", requestContent);
response.EnsureSuccessStatusCode();
string content = await response.Content.ReadAsStringAsync();
string[] jsonChunks = content.Split('\n', StringSplitOptions.RemoveEmptyEntries);

await browser.CloseAsync();

方案3：用XHR代替fetch，绕过Cloudflare拦截 #

在Puppeteer的Evaluate中使用XMLHttpRequest，比fetch更难被Cloudflare检测。

.NET代码示例 #

var browser = await Puppeteer.LaunchAsync(new LaunchOptions
{
    Headless = HeadlessMode.New,
    Args = new[] { "--disable-blink-features=AutomationControlled", "--no-sandbox" }
});

var page = await browser.NewPageAsync();
await page.SetViewportAsync(new ViewPortOptions { Width = 1920, Height = 1080 });
// 通过Cloudflare验证
await page.GoToAsync("网站基础URL");
await page.WaitForSelectorAsync("验证完成后出现的元素");

// 用XHR发起请求
var content = await page.EvaluateAsync<string>(@"async () => {
    return new Promise((resolve, reject) => {
        const xhr = new XMLHttpRequest();
        xhr.open('POST', '你的目标URL');
        xhr.setRequestHeader('Content-Type', 'application/x-www-form-urlencoded');
        xhr.onload = () => {
            if (xhr.status >= 200 && xhr.status < 300) resolve(xhr.responseText);
            else reject(new Error(`请求失败：${xhr.status}`));
        };
        xhr.onerror = () => reject(new Error('网络错误'));
        xhr.send('key1=value1&key2=value2');
    });
}");

string[] jsonChunks = content.Split('\n', StringSplitOptions.RemoveEmptyEntries);
await browser.CloseAsync();

关键注意事项 #

1. 隐藏自动化特征：必须添加--disable-blink-features=AutomationControlled启动参数，否则Cloudflare极易检测到Puppeteer环境。
2. 模拟真实用户：设置匹配浏览器的User-Agent、Viewport，必要时模拟鼠标移动、滚动等行为。
3. 等待验证完成：先访问网站基础页面，等待Cloudflare的JS挑战或人机验证完成后，再发起目标POST请求。

内容的提问来源于stack exchange，提问作者Arc