如何使用Puppeteer获取img标签的src属性值？

我尝试用Puppeteer提取页面中img标签的src属性值，但运行脚本后得到的结果不符合预期，相关信息如下：

相关HTML片段 #

<img alt="" class="responsive-img" src="https://i.guim.co.uk/img/media/6167380a1330877b8265353f2756b127c2226824/0_81_4256_2554/master/4256.jpg?width=300&amp;quality=85&amp;auto=format&amp;fit=max&amp;s=787aa5ddd44a8a66a06120452e228503">

完整HTML结构 #

<div class="fc-item__container">
    <div class="fc-item__media-wrapper">
        <div class="fc-item__image-container u-responsive-ratio">
            <picture><!--[if IE 9]><video style="display: none;"><![endif]-->
                <source
                    media="(min-width: 980px) and (-webkit-min-device-pixel-ratio: 1.25), (min-width: 980px) and (min-resolution: 120dpi)"
                    srcset="https://i.guim.co.uk/img/media/6167380a1330877b8265353f2756b127c2226824/0_81_4256_2554/master/4256.jpg?width=140&amp;quality=45&amp;auto=format&amp;fit=max&amp;dpr=2&amp;s=35c7a9a7cc4e5ebd8fcdfcb67177a8f4 280w">
                <source media="(min-width: 980px)"
                    srcset="https://i.guim.co.uk/img/media/6167380a1330877b8265353f2756b127c2226824/0_81_4256_2554/master/4256.jpg?width=140&amp;quality=85&amp;auto=format&amp;fit=max&amp;s=f68f029ce1b60ed96581f28a29062e3b 140w">
                <source
                    media="(min-width: 740px) and (-webkit-min-device-pixel-ratio: 1.25), (min-width: 740px) and (min-resolution: 120dpi)"
                    srcset="https://i.guim.co.uk/img/media/6167380a1330877b8265353f2756b127c2226824/0_81_4256_2554/master/4256.jpg?width=140&amp;quality=45&amp;auto=format&amp;fit=max&amp;dpr=2&amp;s=35c7a9a7cc4e5ebd8fcdfcb67177a8f4 280w">
                <source media="(min-width: 740px)"
                    srcset="https://i.guim.co.uk/img/media/6167380a1330877b8265353f2756b127c2226824/0_81_4256_2554/master/4256.jpg?width=140&amp;quality=85&amp;auto=format&amp;fit=max&amp;s=f68f029ce1b60ed96581f28a29062e3b 140w">
                <!--[if IE 9]></video><![endif]-->
                <img alt="" class="responsive-img"
                    src="https://i.guim.co.uk/img/media/6167380a1330877b8265353f2756b127c2226824/0_81_4256_2554/master/4256.jpg?width=300&amp;quality=85&amp;auto=format&amp;fit=max&amp;s=787aa5ddd44a8a66a06120452e228503">
            </picture>
        </div>
    </div>
</div>

我的Puppeteer脚本 #

const fs = require("node:fs/promises");
const puppeteer = require("puppeteer"); // ^19.4.1

const url = "https://www.theguardian.com/international";

let browser;
(async () => {
    browser = await puppeteer.launch();
    const [page] = await browser.pages();
    await page.setJavaScriptEnabled(false);
    await page.setRequestInterception(true);
    page.on("request", req => {
        if (req.url() !== url) {
            req.abort();
        }
        else {
            req.continue();
        }
    });
    await page.goto(url, { waitUntil: "domcontentloaded" });
    const img_src = await page.$$eval(".fc-item__container", els =>
        els.map(e => {
            const text = s => e.querySelector(s)?.textContent.trim();
            return {
                src: e.querySelector(".fc-item__media-wrapper .responsive-img src"),
                image: text(".fc-item__media-wrapper .responsive-img"),
            };
        })
    );
    console.log(img_src);
    await fs.writeFile("img_src.json", JSON.stringify(img_src, null, 2));
})()
    .catch(err => console.error(err))
    .finally(() => browser?.close());

运行结果 #

[
  {
    "src": null,
    "image": ""
  },
  {
    "src": null,
    "image": ""
  }
}
]

问题分析与解决 #

脚本存在几个关键问题，修正后即可正确获取img的src属性：

1. 选择器逻辑错误：.fc-item__media-wrapper .responsive-img src 错误地将src当作子元素选择，实际上src是img标签的属性，需先选中img元素再读取属性。
2. 错误使用textContent：img标签本身无文本内容，textContent必然为空，目标应为src属性而非文本。
3. 请求拦截过于严格：仅允许主URL请求会阻断页面必要资源加载，导致DOM渲染不完整。
4. 禁用JavaScript：The Guardian页面依赖JS渲染内容，禁用JS会导致目标元素无法正常加载。

修正后的脚本 #

const fs = require("node:fs/promises");
const puppeteer = require("puppeteer"); // ^19.4.1

const url = "https://www.theguardian.com/international";

let browser;
(async () => {
    browser = await puppeteer.launch();
    const [page] = await browser.pages();
    
    // 启用JavaScript，保证页面正常渲染动态内容
    // await page.setJavaScriptEnabled(false);
    
    // 优化请求拦截：仅拦截非必要资源，保留核心请求
    await page.setRequestInterception(true);
    page.on("request", req => {
        const resourceType = req.resourceType();
        if (["image", "stylesheet", "font"].includes(resourceType)) {
            req.abort();
        } else {
            req.continue();
        }
    });
    
    // 等待网络空闲，确保DOM完全加载
    await page.goto(url, { waitUntil: "networkidle2" });
    
    const img_src = await page.$$eval(".fc-item__container", els =>
        els.map(e => {
            const img = e.querySelector(".fc-item__media-wrapper .responsive-img");
            return {
                src: img?.getAttribute("src") || null,
            };
        })
    );
    
    console.log(img_src);
    await fs.writeFile("img_src.json", JSON.stringify(img_src, null, 2));
})()
    .catch(err => console.error(err))
    .finally(() => browser?.close());

关键修正点说明 #

• 保留JavaScript：确保页面能渲染动态生成的内容。
• 调整请求拦截：只拦截图片、样式等非必要资源，避免阻断核心DOM渲染请求。
• 正确获取属性：通过getAttribute("src")读取img元素的src属性值。
• 等待页面加载完成：使用networkidle2等待网络空闲，保证所有必要内容加载完毕。

内容的提问来源于stack exchange，提问作者Bart Zakkenwasser