Puppeteer爬取分页表格遇阻：仅获前25条数据，求解决方案

问题背景 #

使用以下Puppeteer代码爬取Sharkscope指定赛事页面表格第5列奖金内容时，仅能获取前25条数据，无法处理表格分页：

原代码：

const puppeteer = require('puppeteer');

(async () => {
  const browser = await puppeteer.launch();
  const page = await browser.newPage();

  // Navigate to the page
  await page.goto('https://www.sharkscope.com/#Find-Tournament//networks/888Poker(ES-PT)/tournaments/252404158');

  // Wait for the first selector to be available
  let selectorIndex = 1;
  const extractedContent = {};

  while (true) {
    const selector = `#jqg${selectorIndex}`;
    const selectorExists = await page.waitForSelector(`${selector} > td:nth-child(5)`, { timeout: 1000 }).then(() => true).catch(() => false);

    if (!selectorExists) break;

    // Extract information from the selector
    extractedContent[selector] = await page.evaluate((sel) => {
      const element = document.querySelector(`${sel} > td:nth-child(5)`);
      return element ? element.textContent.trim() : null;
    }, selector);

    selectorIndex++;
  }

  // Print the extracted information
  console.log('Extracted content:', extractedContent);

  await browser.close();
})();

原输出仅显示前25条：

Extracted content: {
  '#jqg1': '€323  (Bounties: €64.28)',
  '#jqg2': '€273  (Bounties: €83.56)',
  '#jqg3': '€244  (Bounties: €105)',
  '#jqg4': '€227  (Bounties: €124)',
  '#jqg5': '€189  (Bounties: €121)',
  '#jqg6': '€47.9',
  '#jqg7': '€172  (Bounties: €138)',
  '#jqg8': '€86.77  (Bounties: €62.82)',
  '#jqg9': '€26.89  (Bounties: €8.55)',
  '#jqg10': '€22.62  (Bounties: €4.28)',
  '#jqg11': '€26.9  (Bounties: €8.56)',
  '#jqg12': '€268  (Bounties: €250)',
  '#jqg13': '€89.01  (Bounties: €75.32)',
  '#jqg14': '€22.97  (Bounties: €9.28)',
  '#jqg15': '€13.69',
  '#jqg16': '€57.96  (Bounties: €44.27)',
  '#jqg17': '€15.5  (Bounties: €4.28)',
  '#jqg18': '€11.22',
  '#jqg19': '€36.22  (Bounties: €25)',
  '#jqg20': '€30.5  (Bounties: €19.28)',
  '#jqg21': '€21.22  (Bounties: €10)',
  '#jqg22': '€11.22',
  '#jqg23': '€11.22',
  '#jqg24': '€11.22',
  '#jqg25': '€13.9  (Bounties: €5)'
}

修改后的完整代码 #

const puppeteer = require('puppeteer');

(async () => {
  const browser = await puppeteer.launch({ headless: false }); // 可改为true，显式模式方便调试
  const page = await browser.newPage();

  await page.goto('https://www.sharkscope.com/#Find-Tournament//networks/888Poker(ES-PT)/tournaments/252404158', { waitUntil: 'networkidle2' });
  
  // 等待表格加载完成
  await page.waitForSelector('#grid');
  const extractedContent = [];
  let pageNumber = 1;

  while (true) {
    console.log(`正在爬取第${pageNumber}页数据...`);
    
    // 提取当前页所有行的第5列数据
    const pageData = await page.evaluate(() => {
      const rows = document.querySelectorAll('#grid tr[id^="jqg"]');
      return Array.from(rows).map(row => {
        const col5 = row.querySelector('td:nth-child(5)');
        return col5 ? col5.textContent.trim() : null;
      }).filter(item => item !== null);
    });
    
    extractedContent.push(...pageData);
    
    // 判断下一页按钮是否可用（未被禁用）
    const nextButtonDisabled = await page.$eval('.ui-pg-next', el => el.classList.contains('ui-state-disabled'));
    if (nextButtonDisabled) {
      console.log('已爬取所有页面');
      break;
    }
    
    // 点击下一页并等待表格更新
    await page.click('.ui-pg-next');
    await page.waitForFunction(
      (prevPage) => document.querySelector('.ui-paging-info').textContent.includes(`Page ${prevPage + 1}`),
      {},
      pageNumber
    );
    pageNumber++;
  }

  console.log('完整提取内容:', extractedContent);
  await browser.close();
})();

关键改动说明 #

• 翻页逻辑处理：通过判断下一页按钮的ui-state-disabled类来确定是否还有下一页，避免无限循环
• 高效数据提取：直接选择当前页所有符合条件的行，批量提取第5列数据，替代原逐个id查找的低效方式
• 页面加载等待：点击下一页后，等待分页信息更新，确保新页面数据已加载完成再进行提取
• 调试友好：默认开启显式浏览器模式，方便查看爬取过程中的页面状态

内容的提问来源于stack exchange，提问作者babameopau