求助：解决Node.js Puppeteer的TargetCloseError(Protocol error Network.enable)

我用Node.js的Puppeteer开发了一个网络爬虫，运行时遇到错误：TargetCloseError: Protocol error (Network.enable): Target closed。发现这个错误常出现在快速打开并关闭大量标签页时，推测是浏览器过载导致的。

我的代码 #

import vanillaPuppeteer from 'puppeteer'
import {addExtra} from "puppeteer-extra";
import StealthPlugin from 'puppeteer-extra-plugin-stealth'
import AdblockerPlugin from "puppeteer-extra-plugin-adblocker";
import dotenv from 'dotenv'
import axios from "axios";
import * as cheerio from 'cheerio';
import {Cluster} from "puppeteer-cluster";

...

const fetchData = async (query) => {
    try {
        return await axios.post('https://eintaxid.com/search-ajax.php', `&query=${query}`)
    } catch (error) {
        throw error
    }
}

(async () => {
    dotenv.config()

    const puppeteer = addExtra(vanillaPuppeteer)
    puppeteer.use(StealthPlugin())
    puppeteer.use(AdblockerPlugin())

    const queries = process.env.QUERIES.split(',')
    for (const query of queries) {
        const {data: content} = await fetchData(query)
        const $ = cheerio.load(content)

        const links = []
        $('a').each((i, a) => {
            const href = $(a).attr('href')
            links.push(`https://eintaxid.com${href}`)
        })

        const cluster = await Cluster.launch({
            puppeteer,
            concurrency: Cluster.CONCURRENCY_CONTEXT,
            maxConcurrency: 2,
            puppeteerOptions: {
                defaultViewport: false,
                headless: false,
                args: ['--no-sandbox']
            }
        })

        await cluster.task(async ({page, data: link}) => {
            await page.goto(link, {
                waitUntil: 'networkidle2'
            })
            const data = await parseData(await page.content())
            console.log(data)
        })

        for (const link of links) {
            cluster.queue(link)
        }

        await cluster.idle()
        await cluster.close()
    }
})()

package.json依赖 #

"dependencies": {
    "axios": "^1.5.0",
    "cheerio": "^1.0.0-rc.12",
    "dotenv": "^16.3.1",
    "puppeteer": "^21.2.1",
    "puppeteer-cluster": "^0.23.0",
    "puppeteer-extra": "^3.3.6",
    "puppeteer-extra-plugin-adblocker": "^2.13.6",
    "puppeteer-extra-plugin-stealth": "^2.11.2",
    "uuid": "^9.0.1"
  }

.env文件内容 #

QUERIES=Repair,Rent

错误信息 #

file:///D:/Work/Orders/RG/eintaxid.com-scraper/node_modules/puppeteer-core/lib/esm/puppeteer/common/Connection.js:142
            this._reject(callback, new TargetCloseError('Target closed'));
                                   ^

TargetCloseError: Protocol error (Network.enable): Target closed
    at CallbackRegistry.clear (file:///D:/Work/Orders/RG/eintaxid.com-scraper/node_modules/puppeteer-core/lib/esm/puppeteer/common/Connection.js:142:36)
    at CDPSessionImpl._onClosed (file:///D:/Work/Orders/RG/eintaxid.com-scraper/node_modules/puppeteer-core/lib/esm/puppeteer/common/Connection.js:493:25)       
    at Connection.onMessage (file:///D:/Work/Orders/RG/eintaxid.com-scraper/node_modules/puppeteer-core/lib/esm/puppeteer/common/Connection.js:251:25)
    at WebSocket.<anonymous> (file:///D:/Work/Orders/RG/eintaxid.com-scraper/node_modules/puppeteer-core/lib/esm/puppeteer/common/NodeWebSocketTransport.js:46:32)
    at callListener (D:\Work\Orders\RG\eintaxid.com-scraper\node_modules\ws\lib\event-target.js:290:14)
    at WebSocket.onMessage (D:\Work\Orders\RG\eintaxid.com-scraper\node_modules\ws\lib\event-target.js:209:9)
    at WebSocket.emit (node:events:514:28)
    at Receiver.receiverOnMessage (D:\Work\Orders\RG\eintaxid.com-scraper\node_modules\ws\lib\websocket.js:1192:20)
    at Receiver.emit (node:events:514:28)
    at Receiver.dataMessage (D:\Work\Orders\RG\eintaxid.com-scraper\node_modules\ws\lib\receiver.js:558:14) {
  cause: ProtocolError
      at <instance_members_initializer> (file:///D:/Work/Orders/RG/eintaxid.com-scraper/node_modules/puppeteer-core/lib/esm/puppeteer/common/Connection.js:46:14)
      at new Callback (file:///D:/Work/Orders/RG/eintaxid.com-scraper/node_modules/puppeteer-core/lib/esm/puppeteer/common/Connection.js:50:16)
      at CallbackRegistry.create (file:///D:/Work/Orders/RG/eintaxid.com-scraper/node_modules/puppeteer-core/lib/esm/puppeteer/common/Connection.js:89:26)       
      at Connection._rawSend (file:///D:/Work/Orders/RG/eintaxid.com-scraper/node_modules/puppeteer-core/lib/esm/puppeteer/common/Connection.js:210:26)
      at CDPSessionImpl.send (file:///D:/Work/Orders/RG/eintaxid.com-scraper/node_modules/puppeteer-core/lib/esm/puppeteer/common/Connection.js:458:33)
      at NetworkManager.addClient (file:///D:/Work/Orders/RG/eintaxid.com-scraper/node_modules/puppeteer-core/lib/esm/puppeteer/common/NetworkManager.js:92:20)  
      at FrameManager.initialize (file:///D:/Work/Orders/RG/eintaxid.com-scraper/node_modules/puppeteer-core/lib/esm/puppeteer/common/FrameManager.js:181:54)    
      at FrameManager.onAttachedToTarget (file:///D:/Work/Orders/RG/eintaxid.com-scraper/node_modules/puppeteer-core/lib/esm/puppeteer/common/FrameManager.js:235:19)
      at #onAttachedToTarget (file:///D:/Work/Orders/RG/eintaxid.com-scraper/node_modules/puppeteer-core/lib/esm/puppeteer/common/Page.js:262:28)
      at file:///D:/Work/Orders/RG/eintaxid.com-scraper/node_modules/puppeteer-core/lib/esm/third_party/mitt/index.js:1:243
}

Node.js v18.17.1

我找了很多方案都没解决，求可行的解决办法？

你的核心问题是在循环里重复创建并销毁Cluster实例，每次处理一个query就启动关闭一次浏览器集群，频繁的浏览器启动关闭操作导致了目标连接断开的错误。另外，插件加载和并发配置也可以优化，具体修复步骤如下：

1. 把Cluster实例移到循环外，复用浏览器资源 #

不要在每个query的循环里创建Cluster，而是全局只创建一次，把所有query对应的链接都加入队列，避免频繁启动关闭浏览器：

修改后的核心代码：

(async () => {
    dotenv.config()

    const puppeteer = addExtra(vanillaPuppeteer)
    puppeteer.use(StealthPlugin())
    puppeteer.use(AdblockerPlugin())

    // 全局创建一次Cluster
    const cluster = await Cluster.launch({
        puppeteer,
        concurrency: Cluster.CONCURRENCY_CONTEXT,
        maxConcurrency: 2,
        puppeteerOptions: {
            defaultViewport: false,
            headless: false,
            args: ['--no-sandbox', '--disable-gpu', '--disable-dev-shm-usage'] // 加上额外的优化参数
        }
    })

    await cluster.task(async ({page, data: link}) => {
        try {
            await page.goto(link, {
                waitUntil: 'networkidle2',
                timeout: 30000 // 增加超时时间，避免页面加载卡住
            })
            const data = await parseData(await page.content())
            console.log(data)
        } catch (pageError) {
            console.error(`处理链接失败: ${link}`, pageError)
            // 可以在这里添加重试逻辑
        }
    })

    const queries = process.env.QUERIES.split(',')
    for (const query of queries) {
        const {data: content} = await fetchData(query)
        const $ = cheerio.load(content)

        $('a').each((i, a) => {
            const href = $(a).attr('href')
            if (href) { // 增加空值判断，避免无效链接
                cluster.queue(`https://eintaxid.com${href}`)
            }
        })
    }

    await cluster.idle()
    await cluster.close()
})()

2. 添加浏览器启动优化参数 #

在puppeteerOptions的args里加上以下参数，减少浏览器资源占用：

• --disable-gpu：禁用GPU加速，减少资源消耗
• --disable-dev-shm-usage：避免/dev/shm内存不足的问题
• --single-process（可选）：单进程模式，适合低资源环境

3. 增加错误处理和超时机制 #

在page.goto时添加timeout，并且在task里包裹try-catch，避免单个页面的错误导致整个集群崩溃。

4. 调整并发策略 #

如果还是出现过载问题，可以降低maxConcurrency到1，或者改用Cluster.CONCURRENCY_PAGE模式（复用页面而不是上下文），进一步减少资源消耗：

concurrency: Cluster.CONCURRENCY_PAGE,
maxConcurrency: 1, // 单页面复用

5. 检查插件兼容性 #

部分puppeteer-extra插件可能和高版本Puppeteer（21.x）存在兼容问题，可以尝试降级Puppeteer到20.x版本，或者暂时禁用AdblockerPlugin测试是否是插件导致的连接问题。

内容的提问来源于stack exchange，提问作者Oleksandr