无头浏览器爬取TikTok数据不全的调试求助

我开发的TikTok爬虫基于Puppeteer，在无头模式（headless: true）下爬取到的视频数量远少于有头模式（headless: false）。需要一套无头浏览器的调试方法来定位原因，最终实现服务器端无头运行的需求。

相关项目信息 #

• 核心代码仓库为tiktok-downloader，运行需TikTok账号登录后保存Cookie以规避弹窗
• 运行步骤：
  • 克隆仓库后执行 node cookieLoader.js，登录TikTok后按回车保存Cookie
  • 默认无头模式下执行 node index.js -m undertimeslopper

若仓库不可用，可使用以下备用代码：

Cookie生成脚本 #

const readline = require('readline');
const { exit } = require("process");
const fs = require('fs');
const puppeteer = require('puppeteer-extra');
const StealthPlugin = require('puppeteer-extra-plugin-stealth');

// 应用反检测插件
puppeteer.use(StealthPlugin());

(async () => {
    const readline = require('node:readline').createInterface({
        input: process.stdin,
        output: process.stdout,
    });
        
 
  const browser = await puppeteer.launch({ headless: false });

  const page = await browser.newPage();
  await page.goto('https://www.tiktok.com');

  readline.question(`Press enter button to save your cookies\n`, async ()=>{
            readline.close();
            const cookies = await page.cookies();
            console.log(cookies)
            await fs.writeFileSync('./cookies.json', JSON.stringify(cookies, null, 2));
            exit()
        });
 
})();

爬取主脚本 #

const chalk = require("chalk");
const fs = require("fs");
const puppeteer = require("puppeteer");
const { exit } = require("process");
const path = require("path");


const loadCookie = async (page) => {
    const cookieJson = await fs.readFileSync(path.join(__dirname,'cookies.json'));
    const cookies = JSON.parse(cookieJson);
    await page.setCookie(...cookies);
}

const generateUrlProfile = (username) => {
    var baseUrl = "https://www.tiktok.com/";
    if (username.includes("@")) {
        baseUrl = `${baseUrl}${username}`;
    } else {
        baseUrl = `${baseUrl}@${username}`;
    }
    return baseUrl;
};

const getListVideoByUsername = async (username) => {

    var baseUrl = await generateUrlProfile(username)
  
    const browser = await puppeteer.launch({
        headless: true,
    })
    
    const page = await browser.newPage()

    await page.setRequestInterception(true)
    page.on('request', (request) => {
    if (request.resourceType() === 'image') request.abort()
    else request.continue()
    })

    await loadCookie(page);
    page.setUserAgent(
        "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/85.0.4182.0 Safari/537.36"
      );
    await page.goto(baseUrl).catch(err =>{
         console.error(err)
         exit();
    });
    await page.keyboard.press('Escape')
    const delay_milliseconds=3000+500
    const delay_after_load=1000
    
    await page.keyboard.press('Escape')

    try {

        await sleep(delay_milliseconds)
    
        const xpathSelector = "//button[contains(text(),'Refresh')]";
        await page.evaluate(xpath => {
            const xpathResult = document.evaluate(xpath, document, null, XPathResult.FIRST_ORDERED_NODE_TYPE, null);
            const element = xpathResult.singleNodeValue;
            if (element) {
                element.click()
            }
        }, xpathSelector);

        await sleep(delay_after_load)
    } 

    catch (error) {
    
    }
    
    await page.keyboard.press('Escape')
    var listVideo = []
    console.log(chalk.green("[*] Getting list video from: " + username))
    var loop = true
    var no_video_found=false
   
    while(loop) {
        listVideo = await page.evaluate(() => {
        const listVideo = document.querySelectorAll('a');
    
        const videoUrls2 = Array.from(listVideo).map(item => item.href)
            .filter(href => href.includes('/video/') || href.includes('/photo/'))
            .filter((value, index, self) => self.indexOf(value) === index).map(item=>item.replace('photo','video'));
            return videoUrls2;
        });
    
        console.log(chalk.green(`[*] ${listVideo.length} video found`))
        previousHeight = await page.evaluate("document.body.scrollHeight").catch(() => {
            
        });
        await page.evaluate("window.scrollTo(0, document.body.scrollHeight)").catch(() => {
        })
                          
        await page.waitForFunction(`document.body.scrollHeight > ${previousHeight}`, {timeout: 10000})
        .catch(() => {
            console.log(chalk.red("[X] No more video found"));
            console.log(chalk.green(`[*] Total video found: ${listVideo.length}`))
            loop = false
            if(listVideo.length===0){
                no_video_found=true
            }
            
            
        });
        await new Promise((resolve) => setTimeout(resolve, 1000));
    } 
    await browser.close()
    return listVideo
}

// 辅助sleep函数
function sleep(ms) {
    return new Promise(resolve => setTimeout(resolve, ms));
}

(async() => {
    getListVideoByUsername('undertimeslopper')
})()

运行输出对比 #

无头模式下 #

[*] Getting list video from: undertimeslopper
[*] 35 video found
[*] 69 video found
[*] 69 video found
[X] No more video found

有头模式下 #

[*] Getting list video from: undertimeslopper
[*] 35 video found
[*] 69 video found
[*] 102 video found
[*] 137 video found
[*] 158 video found
[X] No more video found

无头模式调试方法 #

1. 启用可视化调试界面 #

使用新版无头模式并开启DevTools，直观查看页面渲染状态：

const browser = await puppeteer.launch({
  headless: 'new', // 新版无头模式行为更贴近有头模式
  devtools: true,
  args: ['--window-size=1920,1080'] // 模拟有头模式的窗口尺寸
});

2. 捕获网络请求差异 #

监听并记录两种模式下的网络请求，重点对比视频列表加载的API响应：

page.on('response', async (response) => {
  if (response.url().includes('api.tiktok.com') && response.status() === 200) {
    const data = await response.json().catch(() => null);
    console.log(`请求URL: ${response.url()}, 返回数据长度: ${data ? JSON.stringify(data).length : 0}`);
    // 可将日志写入文件方便后续对比
    fs.appendFileSync('network-log.txt', `${new Date().toISOString()} - ${response.url()}\n`);
  }
});

3. 页面状态截图验证 #

在滚动前后生成页面截图，确认无头模式下新视频是否正确渲染：

// 在滚动循环内添加截图逻辑
await page.screenshot({ path: `screenshot-${Date.now()}.png`, fullPage: true });

4. 统一浏览器环境参数 #

确保无头与有头模式的UA、视口、权限完全一致：

await page.setViewport({ width: 1920, height: 1080 });
const USER_AGENT = "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/85.0.4182.0 Safari/537.36";
page.setUserAgent(USER_AGENT);

5. 检查页面控制台错误 #

捕获页面内的JS错误和警告，排查渲染或请求失败原因：

page.on('pageerror', (err) => {
  console.error('页面JS错误:', err);
});

page.on('console', (msg) => {
  if (msg.type() === 'warning' || msg.type() === 'error') {
    console.log(`页面控制台${msg.type()}:`, msg.text());
  }
});

6. 优化滚动加载逻辑 #

无头模式下页面渲染速度可能不同，调整等待逻辑确保新内容完全加载：

• 替换固定sleep为等待元素出现：await page.waitForSelector('a[href*="/video/"]', { timeout: 15000 });
• 增加滚动后的等待时间，或监听滚动触发的内容加载完成信号

7. 验证Cookie有效性 #

确认无头模式下Cookie是否正确加载，可在页面加载后打印Cookie信息：

const loadedCookies = await page.cookies();
console.log('已加载的Cookie:', loadedCookies.map(c => `${c.name}=${c.value}`));

内容的提问来源于stack exchange，提问作者jgore200377