You need to enable JavaScript to run this app.
优惠活动
大模型
产品
解决方案
定价
更多

Node.js与Puppeteer多实例并行爬取问题求助

问题描述

我正在用Node.js + Puppeteer做网站数据爬取,目标是多Puppeteer实例并行处理任务,但当前代码只能单实例运行,且是批量发送数据。

我的代码流程:

  • 读取本地XLSX文件
  • 处理后将numCpfCnpj与numAcordo列表存入内存
  • 将条目按5条一组拆分,准备用于多进程处理
  • 调用Puppeteer登录页面并搜索numCpfCnpj
  • 搜索完成后继续搜索numAcordo并执行后续逻辑
核心疑问
  1. Puppeteer支持多实例并行运行吗?
  2. 如果直接把脚本部署到AWS Lambda这类云服务,是不是无需额外配置就能运行多实例?
当前代码
const puppeteer = require('puppeteer');
const XLSX = require('xlsx');

const handleLogin = async (page, login, password) => {
    await page.goto('https://www.santandernegocios.com.br/portaldenegocios/#/externo');
    await page.waitForSelector('#userLogin__input');
    await page.waitForSelector('#userPassword__input');
    await page.type('#userLogin__input', login);
    await page.type('#userPassword__input', password);
    await page.focus('#userPassword__input');
    await page.keyboard.press('Enter');
    await page.waitForNavigation();
};

const searchAndNavigate = async (page, cpfCnpj) => {
    await page.waitForSelector('input[name="inputSearch"]');

    // 确保cpfCnpj为字符串类型
    if (typeof cpfCnpj !== 'string') {
        throw new TypeError('cpfCnpj必须是字符串');
    }

    await page.type('input[name="inputSearch"]', cpfCnpj);

  
    await page.waitForFunction(
        (selector, text) => {
            const input = document.querySelector(selector);
            return input && input.value === text;
        },
        {},
        'input[name="inputSearch"]',
        cpfCnpj
    );

    await page.focus('input[name="inputSearch"]');
    await page.keyboard.press('Enter');
    await page.waitForNavigation({ waitUntil: 'networkidle2' });
};

const clickMenuLink = async (page, menuTexts) => {
    await page.evaluate(menuTexts => {
        const sessionMenu = document.querySelector('div.session-menu');
        if (sessionMenu) {
            const ul = sessionMenu.querySelector('ul');
            if (ul) {
                for (let text of menuTexts) {
                    const link = Array.from(ul.querySelectorAll('li a')).find(anchor => anchor.textContent.trim() === text);
                    if (link) {
                        link.click();
                        return;
                    }
                }
            }
        }
    }, menuTexts);
};

const processXlsxFile = async (filePath) => {
    const workbook = XLSX.readFile(filePath);
    const sheet = workbook.Sheets[workbook.SheetNames[0]];
    const data = XLSX.utils.sheet_to_json(sheet, { header: 1 });

    const rows = data.slice(1); // 跳过表头行
    const groupedEntries = [];
    const groupSize = 5;

    for (let i = 0; i < rows.length; i += groupSize) {
        groupedEntries.push(rows.slice(i, i + groupSize).map(row => ({
            numCpfCnpj: String(row[0]),  // 确保为字符串
            numAcordo: String(row[1])    // 确保为字符串
        })));
    }

    return groupedEntries;
};


const processEntries = async (page, entries, menuTexts) => {
    await Promise.all(entries.map(async (entry) => {
        try {
            console.log(entry.numCpfCnpj);
            await searchAndNavigate(page, entry.numCpfCnpj);
            await clickMenuLink(page, menuTexts);
            console.log(`已处理: ${entry.numCpfCnpj} - ${entry.numAcordo}`);
        } catch (error) {
            console.error(`处理${entry.numCpfCnpj} - ${entry.numAcordo}时出错:`, error);
        }
    }));
};

const runPuppeteer = async (filePath, login, password, menuTexts) => {
    const browser = await puppeteer.launch({ headless: false });
    const page = await browser.newPage();

    try {
        await handleLogin(page, login, password);
        const entries = await processXlsxFile(filePath);
        
        // 按组处理条目,限制同时处理5个任务
        for (const group of entries) {
            // 每组最多同时处理5个任务
            const chunkSize = 5;
            for (let i = 0; i < group.length; i += chunkSize) {
                const chunk = group.slice(i, i + chunkSize);
                await processEntries(page, chunk, menuTexts);
            }
        }
        
    } catch (error) {
        console.error('发生错误:', error);
    } finally {
        await browser.close();
    }
};

module.exports = { runPuppeteer };

内容的提问来源于stack exchange,提问作者Pablo Costa

相关产品推荐
方舟 Agent Plan

超全模态模型 × Harness 升级,最新支持 Deepseek-V4.1-Flash、GLM-5.3 系列、Doubao-Seedream-5.0-pro、Kimi-K3 (部分), 限时 9.9 元起

最近更新时间:2026.06.19 19:13:15