Puppeteer批量下载文档崩溃，抛出net::ERR_ABORTED错误求助

API 探路者

2026-8-1

问题描述

我用NodeJS结合Puppeteer从Scribd批量下载数十万份文档，单文件获取下载链接约需1.5秒。为提升效率，采用主进程（main.js）抓取文档列表，分批次将任务发送给3个子进程（download.js），每个子进程独立打开浏览器处理下载（网站无直接下载链接，需动态生成）。但程序运行1-2轮后崩溃，抛出以下错误：

Error: net::ERR_ABORTED at https://www.scribd.com/document/456861933
    at navigate (D:\Projects\Upwork\Project 6 (Rustom Vazifdar)\code\node_modules\puppeteer-core\lib\cjs\puppeteer\common\Frame.js:238:23)
    at process.processTicksAndRejections (node:internal/process/task_queues:95:5)
    at async Frame.goto (D:\Projects\Upwork\Project 6 (Rustom Vazifdar)\code\node_modules\puppeteer-core\lib\cjs\puppeteer\common\Frame.js:207:21)
    at async CDPPage.goto (D:\Projects\Upwork\Project 6 (Rustom Vazifdar)\code\node_modules\puppeteer-core\lib\cjs\puppeteer\common\Page.js:439:16)
    at async download (D:\Projects\Upwork\Project 6 (Rustom Vazifdar)\code\download.js:59:3)
    at async process.<anonymous> (D:\Projects\Upwork\Project 6 (Rustom Vazifdar)\code\download.js:53:2)

主进程代码（main.js）

const fs = require('fs');
const puppeteer = require('puppeteer');
const path = require('path');
const fork = require('child_process').fork;

var zip = require ('./zip_files');
var fld = require('./create_folders');
var log = require('./login');

var ls = [];

const docType = ['pdf', 'word', 'spreadsheet'];
const docExt = ['.pdf', '.docx', '.xlsx'];

const login_url = 'https://www.scribd.com/login';
const login_email = '';
const login_password = '';

const t_out = 10000;

var pages=[];

const datasetF1 = 'wordset_11.csv';
const datasetF2 = 'wordset_22.csv';

var data_1 = [];
var data_2 = [];

var fileCount = 1000000;

data_1 = fs.readFileSync(datasetF1).toString().split(',');
data_2 = fs.readFileSync(datasetF2).toString().split(',');

(async() => {
    const browser = await puppeteer.launch(({
        headless: false,
        userDataDir: path.join(__dirname,'user_data/main'),
    }));
    console.log(browser) 
    const page = (await browser.pages())[0];
    
    //await log.login(page, login_url, login_email, login_password, t_out);
    
    child();

    for(let i = 0; i < data_1.length; i++){
        for(let j = 0; j < data_2.length; j++){
            let folder = data_1[i].replace(/\n/gm, '').replace(/\r/gm, '') + ' ' + data_2[j].replace(/\n/gm, '').replace(/\r/gm, '');
            let searchTerm = data_1[i].replace(/\n/gm, '').replace(/\r/gm, '') + ' ' + data_2[j].replace(/\n/gm, '').replace(/\r/gm, '');
            for(let pageNum = 1; pageNum < (Math.ceil(fileCount/42) +1) && pageNum < 236; pageNum++){
                let docType = 'pdf';
                let query = 'https://www.scribd.com/search?query='+searchTerm+'&content_type=documents&page='+pageNum+'&filetype='+docType;
                
                console.log(pageNum);
            
                await page.goto(query, { waitUntil: 'networkidle2' });
                
                fs.appendFileSync('progress/query.txt', query + '\n');
                if (pageNum == 1){
                    await page.waitForXPath('//div[@class=&quot;_7a1igU&quot;]', { timeout: t_out }).then(async() => {
                        let fileCountR = await page.$x('//div[@class=&quot;_7a1igU&quot;]');
                        let fileCountS = await (await fileCountR[0].getProperty('textContent')).jsonValue();
                        var fileCount = parseInt(fileCountS.split('of ').pop().split(' results').shift().replace(/,/g, ''));
                        console.log(fileCount);
                    }).catch( e => { 
                        console.log('ERROR1');
                        console.log(e);
                    });
                }
                
                await page.waitForXPath('//section[@data-testid=&quot;search-results&quot;]', { timeout: t_out }).then(async() => {
                    let searchResults = await page.$x('//section[@data-testid=&quot;search-results&quot;]');
                    await searchResults[0].waitForXPath('//div/ul/li');
                    let docPages = await searchResults[0].$x('//div/ul/li');
                    
                    await download(browser, docPages, folder);
                }).catch( e => { 
                    console.log('ERROR2');
                    console.log(e);
                }); 
            
            }
        }
    }
})();


async function child(){
    for(let a = 0; a < 3; a++){
            ls[a] = fork('download.js');
    }
}


async function download(browser, docPages, folder){
    let child_count = 3;

    const paged = await new Promise(async (resolve, reject) => {
        for(let b = 0; b < docPages.length; b+=child_count){
            await send_data(browser, docPages, folder, child_count, b);
            console.log("b : b");
        }
    });
}
async function send_data(browser, docPages, folder, child_count, b){
    let c = 0;
    await new Promise(async (resolve, reject) => {
        for(let a = 0; a < child_count; a++){
            let urlR = await docPages[a+b].$x('//article/a');
            let url = await (await urlR[0].getProperty('href')).jsonValue();
                    
            fs.appendFileSync('progress/page.txt', url + '\n');
            ls[a].on('message', (msg) => {
                if(msg == 'Executed'){

                }if(msg == 'done'){
                    c++
                    if(c == (child_count-1)){
                        resolve({});
                    }
                }
            });
            ls[a].send([url, folder, a]);
            
        }
    });
}

子进程代码（download.js）

const puppeteer = require('puppeteer');

const down_path = 'downloads/';
const error_path = 'errors/';
const fs = require('fs');
const path = require('path');
var log = require('./login');

const login_url = 'https://www.scribd.com/login';
const login_email = '';
const login_password = '';

const t_out = 1000;

const dowload_errorFile = 'errors.txt';

var browser;
var flag = 0;

const t_outforL = 100000;

process.on('message', async (obj)=>{
    console.log("CHILD: data received from parent process", obj);
    console.log("Process Starting : ", process.pid);
    
    url = obj[0];
    loc = obj[1];
    ins = obj[2];
    
    if(!flag){
        browser = await puppeteer.launch(({
            headless: false,
            userDataDir: path.join(__dirname,'user_data/'+ins),
        }));
        const downPage = (await browser.pages())[0];
        await downPage.goto('https://www.scribd.com', { waitUntil: 'networkidle2' });
        flag = 1;
    }
    await download(url, loc);
});
process.send('Executed');

async function download(url, folder){
        const downPage = (await browser.pages())[0];
        await downPage.goto(`${url}`, { waitUntil: 'networkidle2' });
        await downPage.waitForXPath('//button[@data-e2e=&quot;download-button&quot;]', { timeout: 1000 }).then(async ()=>{
            let dwnld = await downPage.$x('//button[@data-e2e=&quot;download-button&quot;]');
            const client = await downPage.target().createCDPSession()
            await client.send('Page.setDownloadBehavior', {
                behavior: 'allow',
                downloadPath: path.join(__dirname, 'downloads/' + folder),
            })
            await dwnld[0].click();
        
            await downPage.waitForXPath('//a[@class=&quot;wrapper__filled-button download_selection_btn&quot;]');
            let clicks = await downPage.$x('//a[@class=&quot;wrapper__filled-button download_selection_btn&quot;]');
            await clicks[0].click();
            await downPage.waitForTimeout(500);
            process.send('done');
        }).catch( e => { 
            console.log('No download exists');
            console.log(e);
        });
        await downPage.waitForTimeout(2500);
}

解决方案

1. 修复`net::ERR_ABORTED`错误

该错误源于页面导航被中断，大概率是网站反爬触发、网络波动或超时导致，可通过以下方式解决：

增加超时+重试机制：给goto和waitForXPath设置更长超时（比如30秒），同时添加2-3次重试逻辑，避免单次失败直接崩溃。
捕获导航异常：在downPage.goto外层包裹try-catch，遇到ERR_ABORTED时延迟重试。

2. 修复子进程通信与任务调度缺陷

当前主进程存在回调堆积、任务并发失控问题：

避免重复绑定事件：子进程创建时一次性绑定message事件，而非每次发送任务都绑定，防止回调堆积导致逻辑混乱。
严格批次控制：确保一批任务全部完成后再发送下一批，用全局计数器跟踪子进程完成状态，触发下一批任务。

3. 解决资源泄漏问题

页面状态清理：子进程每次下载后，要么新开页面，要么通过downPage.setContent('')重置页面状态，避免旧页面资源堆积。
定期重启浏览器：长时间运行后，子进程的浏览器实例会占用大量内存，可每处理N个任务后重启浏览器，释放资源。

关键修复代码示例

子进程（download.js）添加重试逻辑：

async function download(url, folder) {
  const maxRetries = 3;
  let retries = 0;
  while (retries < maxRetries) {
    try {
      const downPage = (await browser.pages())[0];
      await downPage.goto(`${url}`, { waitUntil: 'networkidle2', timeout: 30000 });
      await downPage.waitForXPath('//button[@data-e2e="download-button"]', { timeout: 5000 });
      
      // 原有下载逻辑
      const client = await downPage.target().createCDPSession()
      await client.send('Page.setDownloadBehavior', {
          behavior: 'allow',
          downloadPath: path.join(__dirname, 'downloads/' + folder),
      })
      let dwnld = await downPage.$x('//button[@data-e2e="download-button"]');
      await dwnld[0].click();
      
      await downPage.waitForXPath('//a[@class="wrapper__filled-button download_selection_btn"]');
      let clicks = await downPage.$x('//a[@class="wrapper__filled-button download_selection_btn"]');
      await clicks[0].click();
      await downPage.waitForTimeout(500);
      
      process.send('done');
      return;
    } catch (e) {
      retries++;
      console.log(`下载失败，重试第${retries}次:`, url);
      if (retries === maxRetries) {
        fs.appendFileSync(dowload_errorFile, `${url}\n`);
        process.send('done');
      }
      await new Promise(resolve => setTimeout(resolve, 2000));
    }
  }
}

主进程修复事件绑定与批次控制：

let taskCounter = 0;
let resolveNextBatch = () => {};

async function child() {
  for (let a = 0; a < 3; a++) {
    ls[a] = fork('download.js');
    // 一次性绑定事件
    ls[a].on('message', (msg) => {
      if (msg === 'done') {
        taskCounter++;
        if (taskCounter === 3) {
          taskCounter = 0;
          resolveNextBatch();
        }
      }
    });
  }
}

async function download(browser, docPages, folder) {
  let child_count = 3;
  for (let b = 0; b < docPages.length; b += child_count) {
    await new Promise(resolve => {
      resolveNextBatch = resolve;
      send_data(browser, docPages, folder, child_count, b);
    });
  }
}

内容的提问来源于stack exchange，提问作者DarkZeus

Puppeteer批量下载文档崩溃，抛出net::ERR_ABORTED错误求助

主进程代码（main.js） #

子进程代码（download.js） #

1. 修复net::ERR_ABORTED错误 #

2. 修复子进程通信与任务调度缺陷 #

3. 解决资源泄漏问题 #

关键修复代码示例 #