生产环境Puppeteer爬谷歌地图waitForSelector超时问题求助

问题重现 #

本地运行Puppeteer抓取谷歌地图评论完全正常，但部署到生产环境后出现以下错误：

Error during scraping: ProtocolError: Waiting for selector .wiI7pd failed: Runtime.callFunctionOn timed out. Increase the 'protocolTimeout' setting in launch/connect calls for a higher timeout if needed

有效解决方案 #

1. 调整Puppeteer启动的核心配置 #

错误提示明确指向protocolTimeout，同时生产环境（尤其是Linux服务器）需要禁用沙箱避免浏览器启动异常，还需补充内存、窗口尺寸相关参数：

const browser = await puppeteer.launch({
  headless: "new", // 新版无头模式更贴近真实浏览器行为
  protocolTimeout: 120000, // 延长协议超时至2分钟
  args: [
    "--no-sandbox",
    "--disable-setuid-sandbox",
    "--disable-dev-shm-usage", // 解决Linux下/dev/shm内存不足问题
    "--disable-gpu",
    "--window-size=1920,1080" // 固定窗口尺寸，避免响应式布局影响元素加载
  ]
});

2. 模拟滚动触发懒加载评论 #

谷歌地图评论采用懒加载机制，生产环境网络较慢时，仅等待页面加载完成不会自动加载全部评论，需模拟用户滚动行为：

await page.goto(url, {
  waitUntil: "networkidle2",
  timeout: 60000 // 延长页面跳转超时时间
});

// 滚动页面到底部，触发评论加载
await page.evaluate(async () => {
  await new Promise((resolve) => {
    let totalHeight = 0;
    const distance = 100;
    const timer = setInterval(() => {
      const scrollHeight = document.body.scrollHeight;
      window.scrollBy(0, distance);
      totalHeight += distance;

      if (totalHeight >= scrollHeight - window.innerHeight) {
        clearInterval(timer);
        resolve();
      }
    }, 100);
  });
});

// 等待目标评论元素出现，同时判断元素数量确保加载完成
await page.waitForFunction(
  () => document.querySelectorAll('.wiI7pd').length > 0,
  { timeout: 60000 }
);

3. 完善浏览器指纹规避反爬 #

生产环境IP更容易触发谷歌反爬，补充完整的浏览器指纹信息：

await page.setUserAgent('Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/118.0.0.0 Safari/537.36');
// 设置HTTP请求头模拟真实用户
await page.setExtraHTTPHeaders({
  'Accept-Language': 'zh-CN,zh;q=0.9,en;q=0.8',
  'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8'
});

4. 检测人机验证避免无意义等待 #

生产环境中若触发谷歌人机验证，直接终止流程：

// 检查是否出现验证组件
const hasCaptcha = await page.$('.g-recaptcha') !== null;
if (hasCaptcha) {
  console.error('触发谷歌人机验证，无法继续抓取');
  await browser.close();
  return;
}

优化后的完整代码示例 #

const puppeteer = require('puppeteer');

const getReviews = async (url, output = "json") => {
    url = "https://maps.app.goo.gl/EXgBhBRtwxgW9WZ5A"
    output = output.toLowerCase();
    if (output !== "json" && output !== "object") {
        console.error('INVALID OUTPUT OPTION');
        return;
    }

    url = url.toString();
    let browser;
    try {
        browser = await puppeteer.launch({
            headless: "new",
            protocolTimeout: 120000,
            args: [
                "--no-sandbox",
                "--disable-setuid-sandbox",
                "--disable-dev-shm-usage",
                "--disable-gpu",
                "--window-size=1920,1080"
            ]
        });

        const page = await browser.newPage();
        await page.setUserAgent('Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/118.0.0.0 Safari/537.36');
        await page.setExtraHTTPHeaders({
            'Accept-Language': 'zh-CN,zh;q=0.9,en;q=0.8',
            'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8'
        });

        await page.goto(url, {
            waitUntil: 'networkidle2',
            timeout: 60000
        });

        // 检测人机验证
        const hasCaptcha = await page.$('.g-recaptcha') !== null;
        if (hasCaptcha) {
            console.error('触发谷歌人机验证，终止抓取');
            await browser.close();
            return;
        }

        // 滚动加载评论
        await page.evaluate(async () => {
            await new Promise((resolve) => {
                let totalHeight = 0;
                const distance = 100;
                const timer = setInterval(() => {
                    const scrollHeight = document.body.scrollHeight;
                    window.scrollBy(0, distance);
                    totalHeight += distance;

                    if (totalHeight >= scrollHeight - window.innerHeight) {
                        clearInterval(timer);
                        resolve();
                    }
                }, 100);
            });
        });

        // 等待评论元素加载
        await page.waitForFunction(
            () => document.querySelectorAll('.wiI7pd').length > 0,
            { timeout: 60000 }
        );

        const data = await page.evaluate(() => {
            const reviewAuthorNames = Array.from(document.getElementsByClassName('d4r55')).map(el => el.innerText);
            const dates = Array.from(document.getElementsByClassName('PIpr3c')).map(el => el.innerText);
            const ratings = Array.from(document.getElementsByClassName('kvMYJc')).map(el => el.children.length);
            const reviewsContent = Array.from(document.getElementsByClassName('wiI7pd')).map(el => el.innerText);

            return {
                reviewAuthorNames,
                dates,
                ratings,
                reviewsContent
            }
        });
        console.log(data)
        console.log('done ! closing browser...')
        await browser.close();

        return output === "json" ? JSON.stringify(data) : data;
    } catch (error) {
        console.error('Error during scraping:', error);
        if (browser) {
            await browser.close();
        }
        throw new Error('Error during scraping');
    }
};

getReviews()

内容的提问来源于stack exchange，提问作者Atef Rihane