Puppeteer无法在酒店网站获取正确CSS选择器致酒店名称提取失败

目标 #

完成网站输入操作后，将所有酒店名称保存至.txt文件中。

问题 #

我的CSS选择器('[class^="js-hotel-location"] > [class^="js-hotel-name"] > span.l-property-name')可能存在错误，导致candidates函数返回空数组；若移除if(!nameEl) return false;代码行，会触发TypeError: Cannot read properties of undefined (reading 'trim')错误。

预期输出（candidates_distance.txt） #

["Aloft Chicago Downtown River North", "The Gwen, a Luxury Collection Hotel, Michigan Avenue Chicago", "Sheraton Grand Chicago Riverwalk"]

实际输出 #

[]

引发问题的代码 #

返回空数组的candidates函数：

// ! STUCK: candidates function returning empty array - likely CSS selector is incorrect 
const candidates = await page.$$eval('[class^="js-hotel-location"] > [class^="js-hotel-name"] > span.l-property-name', elements => {
    return elements
      .filter(el => {
        const nameEl = el.querySelector('[class^="js-hotel-location"] > [class^="js-hotel-name"] > span.l-property-name');
        if(!nameEl) return false;
        const name = nameEl.textContent.trim();
        return name !== "";
      })
      .map(el => el.textContent.trim());
});
fs.writeFile('candidates_name.txt', JSON.stringify(candidates), function (err) {
    if (err) throw err;
});

复现代码 #

constants.js：

module.exports = {
    URL: "https://www.marriott.com/default.mi",
    DESTINATION: "Chicago, IL, USA",
    START_MONTH: "August",
    START_DAY: 7,
    END_MONTH: "August",
    END_DAY: 10,
    PROMO_CODE: "MMP"
}

index.js：

const puppeteer = require('puppeteer-extra');
const constants = require('./marriot_constants.js');
const blockResources = require('puppeteer-extra-plugin-block-resources');
const fs = require('fs/promises');

// ! NOT WORKING: Configure the plugin to block all images and stylesheets
const pluginConfig = blockResources({
    blockedTypes: new Set(['image', 'stylesheet'])
});
puppeteer.use(pluginConfig);

let browser;
(async () => {
    browser = await puppeteer.launch();
    const [page] = await browser.pages();
    const ua = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/69.0.3497.100 Safari/537.36";
    await page.setUserAgent(ua);
    await page.setViewport({width: 800, height: 10700});
    await page.goto(constants.URL, {waitUntil: "domcontentloaded"});

    await page.type('[id$="-search-destination"]', constants.DESTINATION);

    await page.click(".search__calendar-value");
    await page.waitForSelector(".search-dates-popup");

    const findMonth = async targetMonth => {
        for(let i = 0; i < 12; i++){ 
            const month = await page.$eval(".DayPicker-Month", el => el.textContent);
            if(month.includes(targetMonth)) break;
            await page.click('[aria-label="Next Month"]');
        }   
    };

    const chooseDay = day => page.$$eval(".DayPicker-Day-value", (els, day) =>
        els.find(e => e.textContent.trim() === day).click(), String(day)
    );

    await findMonth(constants.START_MONTH);
    await chooseDay(constants.START_DAY);
    await findMonth(constants.END_MONTH);
    await chooseDay(constants.END_DAY);
    await page.click('button.m-button-secondary');

    await page.click('[class^="StyledSpecialRatesDiv"] > [class="t-font-s"]');
    await page.waitForSelector('[class^="StyledPopupMain"]');
    await page.click('[class^="StyledRadioButtonDiv"]:nth-child(5)')
    await page.type('[class^="StyledRadioButtonDiv"]:nth-child(5)', constants.PROMO_CODE);
    await page.waitForSelector('[class="crop-promo-code"] > [class="m-button-secondary m-button-s"]');
    await page.click('[class="crop-promo-code"] > [class="m-button-secondary m-button-s"]');

    await page.click('[id$="find-a-hotel-homePage-form"] > [class^="StyledFindBtn"]');
    await page.waitForNavigation();

    // ! STUCK: candidates function returning null array or cannot pull Hotel Names - likely CSS selector is incorrect 
    const candidates = await page.$$eval('[class^="js-hotel-location"] > [class^="js-hotel-name"] > span.l-property-name', elements => {
        return elements
          .filter(el => {
            const nameEl = el.querySelector('[class^="js-hotel-location"] > [class^="js-hotel-name"] > span.l-property-name');
            if(!nameEl) return false;
            const name = nameEl.textContent.trim();
            return name !== "";
          })
          .map(el => el.textContent.trim());
    });
    fs.writeFile('candidates_name.txt', JSON.stringify(candidates), function (err) {
        if (err) throw err;
    });
      
})()
.catch(err => console.error(err))
.finally(() => browser?.close());

问题根源 #

1. CSS选择器不匹配：万豪搜索结果页的DOM结构中，酒店名称元素的层级并非你指定的[class^="js-hotel-location"] > [class^="js-hotel-name"] > span.l-property-name，导致选择器无法命中目标元素。
2. 冗余DOM查询：在$$eval回调中，el已经是选择器匹配到的元素，你再次用el.querySelector查询相同层级的选择器，必然返回null，直接导致过滤后数组为空。
3. 未等待元素加载：搜索结果是动态渲染的，仅waitForNavigation无法确保所有酒店元素完全加载。

修正代码 #

替换原代码中candidates部分的逻辑：

// 等待酒店名称元素完全加载
await page.waitForSelector('span.l-property-name');

// 获取并处理酒店名称
const candidates = await page.$$eval('span.l-property-name', elements => {
    return elements
        .map(el => el.textContent.trim())
        .filter(name => name !== ""); // 过滤空名称
});

// 使用fs.promises的异步写法（与你引入的fs模块保持一致）
await fs.writeFile('candidates_name.txt', JSON.stringify(candidates));

额外优化建议 #

• 若选择器仍不匹配，可启动Puppeteer时添加headless: false参数，打开浏览器开发者工具直接查看酒店名称元素的真实class和层级，调整选择器。
• 针对动态加载的页面，可增加元素数量校验，确保获取到结果：

// 等待至少有一个酒店名称元素出现
await page.waitForFunction(() => document.querySelectorAll('span.l-property-name').length > 0);

内容的提问来源于stack exchange，提问作者Nelson Mak