如何在Puppeteer中无需数组处理提取层级导航数据？

需要从给定的HTML导航结构中提取指定格式的层级数据，现有实现通过数组匹配关联父级和子级导航，希望简化代码，直接通过元素选择器关联父级与对应子级，无需事后处理数组。

给定HTML结构 #

<nav class='menu-315AG'>
    <a id='side-nav-1' href='something1' data-id='1'>text1</a>
    <div>
        <a class='side-sub-1 blue' href='something1/somethingA' data-id='1a'>text1a</a>
        <a class='side-sub-1 blue' href='something1/somethingB' data-id='1b'>text1b</a>
        <a class='side-sub-1 blue' href='something1/somethingC' data-id='1c'>text1c</a>
    </div>
    <a id='side-nav-2' href='something2' data-id='2'>text2</a>
    <div>
        <a class='side-sub-1 blue' href='something2/something2A' data-id='2a'>text2a</a>
        <a class='side-sub-1 blue' href='something2/something2B' data-id='2b'>text2b</a>
        <a class='side-sub-1 blue' href='something2/something2C' data-id='2c'>text2c</a>
        <a class='side-sub-1 blue' href='something2/something2D' data-id='2d'>text2d</a>
    </div>
    <a id='side-nav-3' href='something3' data-id='3'>text3</a>
    <div>
        <a class='side-sub-1 blue' href='something3/something3A' data-id='3a'>text3a</a>
        <a class='side-sub-1 blue' href='something3/something3B' data-id='3b'>text3b</a>
    </div>
</nav>

目标数据格式 #

[
    [
        1, 
        'text1', 
        'something1', 
        'side-nav-1', 
        [ 
            ['1a', 'text1a', 'something1/somethingA'],
            ['1b', 'text1b', 'something1/somethingB'],
            ['1c', 'text1c', 'something1/somethingC']
        ] 
    ],
    [
        2, 
        'text2', 
        'something2', 
        'side-nav-2', 
        [ 
            ['2a', 'text2a', 'something2/something2A'],
            ['2b', 'text2b', 'something2/something2B'],
            ['2c', 'text2c', 'something2/something2C'],
            ['2d', 'text2d', 'something2/something2D'] 
        ] 
    ],
    [
        3, 
        'text3', 
        'something3', 
        'side-nav-3', 
        [ 
            ['3a', 'text3a', 'something3/something3A'],
            ['3b', 'text3b', 'something3/something3B']
        ] 
    ]
]

当前实现代码（数组匹配方式） #

const puppeteer = require("puppeteer");

const html = `
    <nav class='menu-315AG'>
        <a id='side-nav-1' href='something1' data-id='1'>text1</a>
        <div>
            <a class='side-sub-1 blue' href='something1/somethingA' data-id='1a'>text1a</a>
            <a class='side-sub-1 blue' href='something1/somethingB' data-id='1b'>text1b</a>
            <a class='side-sub-1 blue' href='something1/somethingC' data-id='1c'>text1c</a>
        </div>
        <a id='side-nav-2' href='something2' data-id='2'>text2</a>
        <div>
            <a class='side-sub-1 blue' href='something2/something2A' data-id='2a'>text2a</a>
            <a class='side-sub-1 blue' href='something2/something2B' data-id='2b'>text2b</a>
            <a class='side-sub-1 blue' href='something2/something2C' data-id='2c'>text2c</a>
            <a class='side-sub-1 blue' href='something2/something2D' data-id='2d'>text2c</a>
        </div>
        <a id='side-nav-3' href='something3' data-id='3'>text3</a>
        <div>
            <a class='side-sub-1 blue' href='something3/something3A' data-id='3a'>text3a</a>
            <a class='side-sub-1 blue' href='something3/something3B' data-id='3b'>text3b</a>
        </div>
    </nav>
`;

let browser;
(async () => {
    browser = await puppeteer.launch();
    const [page] = await browser.pages();
    await page.setContent(html);

    function ppp(str) {
        return console.dir(str, {depth: null,colors: true,maxArrayLength: null});
    }

    let categories = await page.$$eval('nav[class*=menu-]  a[id*=side-nav-]', el => el.map(x => {
        return [
            x.getAttribute('data-id'),              
            x.textContent.trim(), 
            x.getAttribute('href'),
            x.getAttribute('id'),
        ];
    }));

    let subcategories = await page.$$eval('nav[class*=menu-]  div > a[class*=side-sub-]', el => el.map(x => {
        return [
            x.getAttribute('data-id'),              
            x.textContent.trim(), 
            x.getAttribute('href'),
        ];
    }));

    for (let [i,] of categories.entries()) {
        categories[i].push([]); 
        for (let [j,] of subcategories.entries()) {
            if (subcategories[j][2].includes(categories[i][2])) {
                categories[i][4].push(subcategories[j]);
            }
        }
    }

    ppp(categories); 

})()
.catch(err => console.error(err))
.finally(() => browser ?. close());

尝试的错误代码 #

let categoryPages = await this.page.$$('nav[class*=menu-]  a[id*=side-nav-]');

let categories = [];
for (let cat of categoryPages) {

    let text = await cat.evaluate(el => el.textContent.trim());
    let href = await cat.evaluate(el => el.getAttribute('href'));
    let id = await cat.evaluate(el => el.getAttribute('id'));
    let data_id = await cat.evaluate(el => el.getAttribute('data-id'));              

    let subCategories = await cat.page.$$eval('div > a[class*=side-sub-]', el => el.map(x => {
            return [
                x.getAttribute('data-id'),              
                x.textContent.trim(), 
                x.getAttribute('href'),
            ];
    }))

    categories.push([data_id, text, href, id, subCategories]);
}

核心思路 #

每个父级导航<a id*=side-nav->的直接下一个兄弟元素就是包含子导航的<div>，因此可以通过el.nextElementSibling直接获取对应子元素容器，在父元素的上下文中提取子导航数据，彻底避免事后数组匹配的繁琐步骤。

优化后代码（单次$$eval实现） #

const puppeteer = require("puppeteer");

const html = `
    <nav class='menu-315AG'>
        <a id='side-nav-1' href='something1' data-id='1'>text1</a>
        <div>
            <a class='side-sub-1 blue' href='something1/somethingA' data-id='1a'>text1a</a>
            <a class='side-sub-1 blue' href='something1/somethingB' data-id='1b'>text1b</a>
            <a class='side-sub-1 blue' href='something1/somethingC' data-id='1c'>text1c</a>
        </div>
        <a id='side-nav-2' href='something2' data-id='2'>text2</a>
        <div>
            <a class='side-sub-1 blue' href='something2/something2A' data-id='2a'>text2a</a>
            <a class='side-sub-1 blue' href='something2/something2B' data-id='2b'>text2b</a>
            <a class='side-sub-1 blue' href='something2/something2C' data-id='2c'>text2c</a>
            <a class='side-sub-1 blue' href='something2/something2D' data-id='2d'>text2d</a>
        </div>
        <a id='side-nav-3' href='something3' data-id='3'>text3</a>
        <div>
            <a class='side-sub-1 blue' href='something3/something3A' data-id='3a'>text3a</a>
            <a class='side-sub-1 blue' href='something3/something3B' data-id='3b'>text3b</a>
        </div>
    </nav>
`;

let browser;
(async () => {
    browser = await puppeteer.launch();
    const [page] = await browser.pages();
    await page.setContent(html);

    function ppp(str) {
        return console.dir(str, {depth: null, colors: true, maxArrayLength: null});
    }

    // 直接在父元素上下文中关联子导航数据
    const categories = await page.$$eval('nav[class*=menu-] a[id*=side-nav-]', els => 
        els.map(el => {
            // 获取当前父级a对应的子导航容器
            const subContainer = el.nextElementSibling;
            // 提取子导航数据
            const subCategories = Array.from(subContainer.querySelectorAll('a[class*=side-sub-]')).map(subEl => [
                subEl.getAttribute('data-id'),
                subEl.textContent.trim(),
                subEl.getAttribute('href')
            ]);
            // 返回完整的父级+子级数据
            return [
                el.getAttribute('data-id'),
                el.textContent.trim(),
                el.getAttribute('href'),
                el.getAttribute('id'),
                subCategories
            ];
        })
    );

    ppp(categories);

})()
.catch(err => console.error(err))
.finally(() => browser?.close());

错误代码修正说明 #

之前的错误代码中cat.page.$$eval是错误用法：cat是ElementHandle实例，不存在page属性。如果需要用循环ElementHandle的方式实现，可改为在每个元素的evaluate回调中直接获取子元素：

// 循环ElementHandle的修正写法
let categoryPages = await page.$$('nav[class*=menu-] a[id*=side-nav-]');
let categories = [];

for (let cat of categoryPages) {
    const categoryData = await cat.evaluate(el => {
        const subContainer = el.nextElementSibling;
        const subCategories = Array.from(subContainer.querySelectorAll('a[class*=side-sub-]')).map(subEl => [
            subEl.getAttribute('data-id'),
            subEl.textContent.trim(),
            subEl.getAttribute('href')
        ]);
        return [
            el.getAttribute('data-id'),
            el.textContent.trim(),
            el.getAttribute('href'),
            el.getAttribute('id'),
            subCategories
        ];
    });
    categories.push(categoryData);
}

内容的提问来源于stack exchange，提问作者user21473566