Puppeteer无头模式运行遇TimeoutError：等待选择器超时求助

问题场景 #

尝试用Puppeteer抓取嵌套多层div中、class为booking-start-time-label的div文本内容，目标HTML结构如下：

<div class="time time-tile"><div class="js-summary"><div>
            <div class="time-summary">
                <div class="time-summary-left time-summary-grid">
                    <div class="time-summary-left-top">
                        <div class="booking-start-time-label">4:22pm</div>
                        <div class="facility-name-label">
                            <span class="js-booking-side-name"> </span>
                        </div>
                    </div>
                    <div>
                        <div class="booking-slot-details">
                            <span class="booking-slot-holes js-booking-slot-holes">
                                <i class="glyphicon glyphicon-flag"></i>
                                <span>18</span>
                            </span>
                            <span class="booking-slot-players js-booking-slot-players">
                                <i class="glyphicon glyphicon-user"></i>
                                <span>4</span>
                            </span>
                        </div>
                    </div>
                </div>
                <div class="time-summary-right time-summary-grid">
                    <div>
                         <div class="booking-slot-pricing-information js-booking-slot-pricing">
                            <span class="js-booking-green-fee" title="Green Fee">
                                <i class="icon-golf-ball-tee"></i>
                                $34.50
                            </span>
                        </div>
                    </div>
                    <div>
                         <div class="booking-slot-pricing-information js-booking-slot-pricing">
                            <span class="js-booking-cart-fee" title="Cart Fee" style="display: none;">
                                <i class="icon-golf-cart"></i>
                                $0.00
                            </span>
                        </div>
                    </div>
                </div>
            </div>
            
        </div></div></div>

当前使用的代码：

const puppeteer = require('puppeteer');

async function scrape() {
    const browser = await puppeteer.launch()
    const page = await browser.newPage()
    await page.goto("https://foreupsoftware.com/index.php/booking/19671/2149#/teetimes")

    await page.waitForSelector('.booking-start-time-label')
    let element = await page.$('.booking-start-time-label')
    let value = await page.evaluate(el => el.textContent, element)
    console.log(value)

    await browser.close()
}

scrape()

运行时报错：

TimeoutError: Waiting for selector .booking-start-time-label failed: Waiting failed: 30000ms exceeded at Timeout.

解决办法 #

1. 检查元素是否在iframe中 #

很多预订类网站会把核心内容放在iframe内，需先切换到iframe再操作：

const puppeteer = require('puppeteer');

async function scrape() {
    const browser = await puppeteer.launch()
    const page = await browser.newPage()
    await page.goto("https://foreupsoftware.com/index.php/booking/19671/2149#/teetimes", { waitUntil: 'networkidle2' });

    // 等待iframe加载并切换（替换为页面实际的iframe选择器）
    const iframe = await page.waitForSelector('iframe');
    const frame = await iframe.contentFrame();

    await frame.waitForSelector('.booking-start-time-label');
    const element = await frame.$('.booking-start-time-label');
    const value = await frame.evaluate(el => el.textContent.trim(), element);
    console.log(value)

    await browser.close()
}

scrape()

2. 模拟页面交互触发内容加载 #

目标时段可能需要选择日期或其他操作才会渲染，需先完成必要交互：

const puppeteer = require('puppeteer');

async function scrape() {
    const browser = await puppeteer.launch({ headless: false }) // 非无头模式便于观察页面操作
    const page = await browser.newPage()
    await page.goto("https://foreupsoftware.com/index.php/booking/19671/2149#/teetimes", { waitUntil: 'networkidle2' });

    // 示例：选择可预订日期（根据页面实际UI调整选择器）
    await page.waitForSelector('.date-picker');
    await page.click('.date-picker');
    await page.waitForSelector('.available-date');
    await page.click('.available-date');

    // 延长超时时间等待元素加载
    await page.waitForSelector('.booking-start-time-label', { timeout: 60000 });
    const element = await page.$('.booking-start-time-label');
    const value = await page.evaluate(el => el.textContent.trim(), element);
    console.log(value)

    await browser.close()
}

scrape()

3. 优化选择器与等待策略 #

使用父容器选择器缩小查找范围，确保元素已完成渲染：

// 先等待父容器加载完成
await page.waitForSelector('.time-tile');
// 在父容器范围内查找目标元素
const element = await page.$('.time-tile .booking-start-time-label');
if (element) {
    const value = await page.evaluate(el => el.textContent.trim(), element);
    console.log(value);
} else {
    console.log("未找到目标元素");
}

4. 规避反爬检测 #

设置浏览器参数模拟真实用户行为：

const browser = await puppeteer.launch({
    headless: 'new', // 新版无头模式更接近真实浏览器
    args: ['--no-sandbox', '--disable-setuid-sandbox', '--start-maximized'],
    defaultViewport: null // 使用浏览器实际窗口大小
});

// 设置真实的User-Agent
await page.setUserAgent('Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/118.0.0.0 Safari/537.36');

内容的提问来源于stack exchange，提问作者Glix