Node.js与Puppeteer多实例并行爬取问题求助
问题描述
我正在用Node.js + Puppeteer做网站数据爬取,目标是多Puppeteer实例并行处理任务,但当前代码只能单实例运行,且是批量发送数据。
我的代码流程:
- 读取本地XLSX文件
- 处理后将
numCpfCnpj与numAcordo列表存入内存 - 将条目按5条一组拆分,准备用于多进程处理
- 调用Puppeteer登录页面并搜索
numCpfCnpj - 搜索完成后继续搜索
numAcordo并执行后续逻辑
核心疑问
- Puppeteer支持多实例并行运行吗?
- 如果直接把脚本部署到AWS Lambda这类云服务,是不是无需额外配置就能运行多实例?
当前代码
const puppeteer = require('puppeteer'); const XLSX = require('xlsx'); const handleLogin = async (page, login, password) => { await page.goto('https://www.santandernegocios.com.br/portaldenegocios/#/externo'); await page.waitForSelector('#userLogin__input'); await page.waitForSelector('#userPassword__input'); await page.type('#userLogin__input', login); await page.type('#userPassword__input', password); await page.focus('#userPassword__input'); await page.keyboard.press('Enter'); await page.waitForNavigation(); }; const searchAndNavigate = async (page, cpfCnpj) => { await page.waitForSelector('input[name="inputSearch"]'); // 确保cpfCnpj为字符串类型 if (typeof cpfCnpj !== 'string') { throw new TypeError('cpfCnpj必须是字符串'); } await page.type('input[name="inputSearch"]', cpfCnpj); await page.waitForFunction( (selector, text) => { const input = document.querySelector(selector); return input && input.value === text; }, {}, 'input[name="inputSearch"]', cpfCnpj ); await page.focus('input[name="inputSearch"]'); await page.keyboard.press('Enter'); await page.waitForNavigation({ waitUntil: 'networkidle2' }); }; const clickMenuLink = async (page, menuTexts) => { await page.evaluate(menuTexts => { const sessionMenu = document.querySelector('div.session-menu'); if (sessionMenu) { const ul = sessionMenu.querySelector('ul'); if (ul) { for (let text of menuTexts) { const link = Array.from(ul.querySelectorAll('li a')).find(anchor => anchor.textContent.trim() === text); if (link) { link.click(); return; } } } } }, menuTexts); }; const processXlsxFile = async (filePath) => { const workbook = XLSX.readFile(filePath); const sheet = workbook.Sheets[workbook.SheetNames[0]]; const data = XLSX.utils.sheet_to_json(sheet, { header: 1 }); const rows = data.slice(1); // 跳过表头行 const groupedEntries = []; const groupSize = 5; for (let i = 0; i < rows.length; i += groupSize) { groupedEntries.push(rows.slice(i, i + groupSize).map(row => ({ numCpfCnpj: String(row[0]), // 确保为字符串 numAcordo: String(row[1]) // 确保为字符串 }))); } return groupedEntries; }; const processEntries = async (page, entries, menuTexts) => { await Promise.all(entries.map(async (entry) => { try { console.log(entry.numCpfCnpj); await searchAndNavigate(page, entry.numCpfCnpj); await clickMenuLink(page, menuTexts); console.log(`已处理: ${entry.numCpfCnpj} - ${entry.numAcordo}`); } catch (error) { console.error(`处理${entry.numCpfCnpj} - ${entry.numAcordo}时出错:`, error); } })); }; const runPuppeteer = async (filePath, login, password, menuTexts) => { const browser = await puppeteer.launch({ headless: false }); const page = await browser.newPage(); try { await handleLogin(page, login, password); const entries = await processXlsxFile(filePath); // 按组处理条目,限制同时处理5个任务 for (const group of entries) { // 每组最多同时处理5个任务 const chunkSize = 5; for (let i = 0; i < group.length; i += chunkSize) { const chunk = group.slice(i, i + chunkSize); await processEntries(page, chunk, menuTexts); } } } catch (error) { console.error('发生错误:', error); } finally { await browser.close(); } }; module.exports = { runPuppeteer };
内容的提问来源于stack exchange,提问作者Pablo Costa
相关产品推荐
相关产品推荐

