如何用Puppeteer提取同class嵌套标签的全部innerText

数据老陈

2026-8-3

问题：Puppeteer仅提取首个新闻实例内容，无法获取后续条目

我正在学习JavaScript和Puppeteer，以《卫报》首页作为练习场景（其HTML标签与类嵌套结构复杂）。我编写了一段脚本，尝试提取指定4个类名（fc-item__kicker、js-headline-text、a标签href、fc-item__standfirst）的内容，但目前仅能提取首个实例的内容，无法获取后续同选择器实例的内容。

HTML片段

<!DOCTYPE html>
<html>
<body>
    <div class="l-side-margins">
        <div class="facia-page">
            <section id="headlines" class="fc-container fc-container--has-toggle">
                <div class="fc-container__inner">
                    <div class="fc-container--rolled-up-hide fc-container__body" id="container-10f21d96-18f6-426f-821b-19df55dfb831">
                        <div class="fc-slice-wrapper">
                            <ul class="u-unstyled l-row l-row--cols-4 fc-slice fc-slice--qqq-q">
                                <li class="fc-slice__item l-row__item l-row__item--span-3 u-faux-block-link">
                                    <div class="fc-item__container">
                                        <div class="fc-item__content">
                                            <div class="fc-item__header">
                                                <h3 class="fc-item__title"><a class="fc-item__link" href="https://www.example.com"><span class="fc-item__kicker">Monterey Park shooting</span> <span class="u-faux-block-link__cta fc-item__headline"><span class="js-headline-text">Beloved dance hall manager named among victims</span></span></a></h3>
                                            </div>
                                            <div class="fc-item__standfirst-wrapper">
                                                <div class="fc-item__standfirst">
                                                    California officials yet to identify eight others who died in Saturday attack, at least 36th mass shooting in US so far this year
                                                </div>
                                            </div>
                                            <div class="fc-item__footer--vertical">
                                                <ul class="fc-sublinks u-unstyled u-faux-block-link__promote">
                                                    <li class="fc-sublink fc-sublink--pillar-news fc-sublink--type-article">
                                                        <h4 class="fc-sublink__title"><a class="fc-sublink__link" href="https://www.example.com"><span class="fc-sublink__kicker">LA mass shooting</span> Man who disarmed California shooter tells of violent struggle for gun</a></h4>
                                                    </li>
                                                </ul>
                                            </div>
                                        </div>
                                    </div>
                                </li>
                            </ul>
                        </div>
                        <div class="fc-slice-wrapper">
                            <ul class="u-unstyled l-row l-row--cols-4 fc-slice fc-slice--q-q-ql-ql">
                                <li class="fc-slice__item l-row__item l-row__item--span-1 u-faux-block-link">
                                    <div class="fc-item fc-item--has-image fc-item--pillar-news fc-item--type-article js-fc-item fc-item--list-media-mobile fc-item--standard-tablet js-snappable">
                                        <div class="fc-item__container">
                                            <div class="fc-item__media-wrapper">
                                                <div class="fc-item__image-container u-responsive-ratio"></div>
                                            </div>
                                            <div class="fc-item__content">
                                                <div class="fc-item__header">
                                                    <h3 class="fc-item__title"><a class="fc-item__link" href="https://www.example.com"><span class="fc-item__kicker">Germany</span> <span class="u-faux-block-link__cta fc-item__headline"><span class="js-headline-text">Five charged over second alleged far-right plot against government</span></span></a></h3>
                                                </div>
                                                <div class="fc-item__standfirst-wrapper">
                                                    <div class="fc-item__standfirst">
                                                        Four men and a woman accused of planning to abduct health minister and overthrow government
                                                    </div>
                                                    <div class="fc-item__meta js-item__meta"></div>
                                                </div>
                                            </div>
                                        </div>
                                    </div>
                                </li>
                            </ul>
                        </div>
                    </div>
                </div>
            </section>
        </div>
    </div>
</body>
</html>

现有脚本

const fs = require('fs');
const puppeteer = require('puppeteer');

async function run() {
    const browser = await puppeteer.launch();
    const page = await browser.newPage();
    await page.goto('https://www.theguardian.com/international/');

    const headlines = await page.evaluate(() => Array.from(document.querySelectorAll('#headlines'), (e) => ({
        kicker: e.querySelector('.fc-item__header .fc-item__kicker').innerText,
        headline: e.querySelector('.fc-item__header .js-headline-text').innerText,
        link: e.querySelector('.fc-item__header  a').href,
        standfirst: e.querySelector('.fc-item__standfirst-wrapper .fc-item__standfirst').textContent.replaceAll("  ", " ").trim(),
    })));

    console.log(headlines);
    
    console.log(headlines);
    // Save data to JSON file
    fs.writeFile('headlines.json', JSON.stringify(headlines), (err) => {
        if (err) throw err;
        console.log('File saved');
    });

    await browser.close();
}
run();

期望结果

[
  {
    kicker: 'Monterey Park shooting',
    headline: 'Beloved dance hall manager named among victims',
    link: 'https://www.example.com',
    standfirst: 'California officials yet to identify eight others who died in Saturday attack, at least 36th mass shooting in US so far this year'
  },
  {
    kicker: 'Germany',
    headline: 'Five charged over second alleged far-right plot against government',
    link: 'https://www.example.com',
    standfirst: 'Four men and a woman accused of planning to abduct health minister and overthrow government'
  }
]

解决方案

问题根源

你的代码中使用document.querySelectorAll('#headlines')，但#headlines是页面中唯一的section元素，遍历后只会得到一个元素。在这个元素内部使用querySelector时，只会返回第一个匹配的子元素，因此只能提取到第一条新闻的数据。

修正后的脚本

const fs = require('fs');
const puppeteer = require('puppeteer');

async function run() {
    const browser = await puppeteer.launch();
    const page = await browser.newPage();
    await page.goto('https://www.theguardian.com/international/');

    // 选择每个新闻内容的容器，遍历所有匹配项
    const headlines = await page.evaluate(() => Array.from(document.querySelectorAll('#headlines .fc-item__content'), (item) => {
        // 为避免元素不存在报错，先检查元素是否存在
        const kickerEl = item.querySelector('.fc-item__header .fc-item__kicker');
        const headlineEl = item.querySelector('.fc-item__header .js-headline-text');
        const linkEl = item.querySelector('.fc-item__header a');
        const standfirstEl = item.querySelector('.fc-item__standfirst-wrapper .fc-item__standfirst');

        return {
            kicker: kickerEl ? kickerEl.innerText : '',
            headline: headlineEl ? headlineEl.innerText : '',
            link: linkEl ? linkEl.href : '',
            standfirst: standfirstEl ? standfirstEl.textContent.replaceAll("  ", " ").trim() : ''
        };
    }));

    console.log(headlines);
    // 使用fs.promises.writeFile替代回调式写法，更符合async/await风格
    await fs.promises.writeFile('headlines.json', JSON.stringify(headlines, null, 2));
    console.log('File saved');

    await browser.close();
}
run();

关键改动说明

选择器调整：将遍历的目标从#headlines改为#headlines .fc-item__content，这个选择器会匹配所有新闻内容的容器，确保能拿到每一条新闻。
空值处理：在获取每个子元素时先检查元素是否存在，避免因页面结构变化导致脚本报错。
文件写入优化：使用fs.promises.writeFile替代传统的回调式写法，与async/await风格保持一致，代码更简洁易读。

内容的提问来源于stack exchange，提问作者Bart Zakkenwasser

如何用Puppeteer提取同class嵌套标签的全部innerText

HTML片段 #

现有脚本 #

期望结果 #

问题根源 #

修正后的脚本 #