使用Cheerio在Node.js爬取mydealz.de时无法获取价格等元素的问题

我用Node.js的Cheerio库爬取https://www.mydealz.de/new，能成功获取商品标题，但提取价格、原价和折扣时，返回的都是骨架屏元素，价格与折扣输出为空。

目标HTML结构 #

<div class="threadGrid-title">
    <strong><a href="">title</a></strong>
    <span class=" overflow--fade">
        <span class="overflow--wrap-off flex boxAlign-ai--all-bl">
            <span class="vAlign--all-tt">
                <span
                    class="threadItemCard-price text--b thread-price size--all-l size--fromW3-xl space--mr-0">**price**</span>
            </span>
            <span class="mute--text text--lineThrough space--ml-1 size--all-l size--fromW3-xl ">**old
                price**</span>
            <span class="text--color-charcoal space--ml-1 size--all-l size--fromW3-xl">**Discount%**</span>
        </span>
    </span>
</div>

当前实现代码 #

async function checkDeals() {
    try {
        const response = await axios.get(baseUrl);
        const $ = cheerio.load(response.data);

        const deals = [];

        // Iterate over each deal
        $('.thread--type-list').each((index, element) => {
            const threadTitleElement = $(element).find('.threadGrid-title');
            
            // Print all child elements of .threadGrid-title
            console.log('Child elements of .threadGrid-title:');
            threadTitleElement.each((i, el) => {
                $(el).find('*').each((j, child) => {
                    console.log($.html(child)); // Print HTML of each nested element
                });
            });

            // Attempt to extract deeply nested price and discount
            const priceElement = $(element).find('.threadItemCard-price');
            const discountElement = $(element).find('.text--color-charcoal');

            // Extracting the price and discount with more detailed text processing
            const price = priceElement.map((i, el) => $(el).text().trim()).get().join(' ');
            const discount = discountElement.map((i, el) => $(el).text().trim()).get().join(' ');

            console.log('Price:', price);
            console.log('Discount:', discount);
        });
    } catch (error) {
        console.error('Error fetching the deals:', error);
    }
}

输出结果 #

Child elements of .threadGrid-title:
<strong class="thread-title "><a class="cept-tt thread-link linkPlain thread-title--list js-thread-title" title="(Amazon Prime) Victorinox Universalschäler" href="https://www.mydealz.de/deals/amazon-prime-victorinox-universalschaler-2393896" data-t="threadLink" data-t-click="">(Amazon Prime) Victorinox Universalschäler</a></strong>
<a class="cept-tt thread-link linkPlain thread-title--list js-thread-title" title="(Amazon Prime) Victorinox Universalschäler" href="https://www.mydealz.de/deals/amazon-prime-victorinox-universalschaler-2393896" data-t="threadLink" data-t-click="">(Amazon Prime) Victorinox Universalschäler</a><span class="overflow--fade"><div aria-busy="true" class="js-vue2" data-handler="vue2" data-vue2="{&quot;name&quot;:&quot;ThreadPriceListing&quot;,&quot;props&quot;:{&quot;threadId&quot;:2393896}}"><div class="bRad--a-m space--h-1 bRad--circle skeleton bg--color-greyPanel flex"><img src="/assets/img/skeletons/item-type-F.svg" width="60px" height="28px" class="hide--toW3"><img src="/assets/img/skeletons/item-type-F.svg" width="60px" height="26px" class="hide--fromW3"></div></div><span class="thread-divider"></span><div aria-busy="true" class="js-vue2" data-handler="vue2" data-vue2="{&quot;name&quot;:&quot;MerchantLabelThreadListing&quot;,&quot;props&quot;:{&quot;threadId&quot;:2393896}}"><div class="bRad--a-m space--h-1 bRad--circle skeleton bg--color-greyPanel flex"><img src="/assets/img/skeletons/item-type-F.svg" width="40px" height="20px"></div></div></span>
<div aria-busy="true" class="js-vue2" data-handler="vue2" data-vue2="{&quot;name&quot;:&quot;ThreadPriceListing&quot;,&quot;props&quot;:{&quot;threadId&quot;:2393896}}"><div class="bRad--a-m space--h-1 bRad--circle skeleton bg--color-greyPanel flex"><img src="/assets/img/skeletons/item-type-F.svg" width="60px" height="28px" class="hide--toW3"><img src="/assets/img/skeletons/item-type-F.svg" width="60px" height="26px" class="hide--fromW3"></div></div>
<div class="bRad--a-m space--h-1 bRad--circle skeleton bg--color-greyPanel flex"><img src="/assets/img/skeletons/item-type-F.svg" width="60px" height="28px" class="hide--toW3"><img src="/assets/img/skeletons/item-type-F.svg" width="60px" height="26px" class="hide--fromW3"></div>
<img src="/assets/img/skeletons/item-type-F.svg" width="60px" height="28px" class="hide--toW3">
<img src="/assets/img/skeletons/item-type-F.svg" width="60px" height="26px" class="hide--fromW3">
<span class="thread-divider"></span>
<div aria-busy="true" class="js-vue2" data-handler="vue2" data-vue2="{&quot;name&quot;:&quot;MerchantLabelThreadListing&quot;,&quot;props&quot;:{&quot;threadId&quot;:2393896}}"><div class="bRad--a-m space--h-1 bRad--circle skeleton bg--color-greyPanel flex"><img src="/assets/img/skeletons/item-type-F.svg" width="40px" height="20px"></div></div>
<div class="bRad--a-m space--h-1 bRad--circle skeleton bg--color-greyPanel flex"><img src="/assets/img/skeletons/item-type-F.svg" width="40px" height="20px"></div>
<img src="/assets/img/skeletons/item-type-F.svg" width="40px" height="20px">
Price:
Discount:

原因分析 #

从输出的HTML可以看到，价格相关区域是由Vue组件（js-vue2）动态渲染的，Cheerio只能解析服务器返回的初始静态HTML，而初始HTML里只有骨架屏占位，实际价格数据是前端通过AJAX请求加载后渲染的，所以用Cheerio拿不到动态生成的价格元素。

解决方案 #

方法1：直接调用网站API接口 #

Mydealz的商品数据通过公开API提供，直接请求API能拿到结构化的JSON数据，比解析HTML更高效可靠。

const axios = require('axios');

async function checkDeals() {
    try {
        const apiResponse = await axios.get('https://www.mydealz.de/api/v2/deals/new', {
            headers: {
                'Accept': 'application/json',
                'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/118.0.0.0 Safari/537.36'
            }
        });

        const deals = apiResponse.data.data;
        deals.forEach(deal => {
            console.log('标题:', deal.title);
            console.log('价格:', deal.price);
            console.log('原价:', deal.original_price);
            console.log('折扣:', deal.discount_percent + '%');
            console.log('---');
        });
    } catch (error) {
        console.error('获取数据失败:', error);
    }
}

checkDeals();

方法2：使用Puppeteer渲染动态页面 #

如果需要模拟浏览器渲染流程，可以用Puppeteer启动无头浏览器，等待页面动态渲染完成后再提取元素。

const puppeteer = require('puppeteer');

async function checkDeals() {
    const browser = await puppeteer.launch({ headless: 'new' });
    const page = await browser.newPage();
    await page.setUserAgent('Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/118.0.0.0 Safari/537.36');
    
    try {
        await page.goto('https://www.mydealz.de/new', { waitUntil: 'networkidle2' });
        // 等待价格元素加载完成
        await page.waitForSelector('.threadItemCard-price', { timeout: 5000 });

        const deals = await page.evaluate(() => {
            const dealElements = document.querySelectorAll('.thread--type-list');
            return Array.from(dealElements).map(element => {
                const title = element.querySelector('.threadGrid-title a').textContent.trim();
                const price = element.querySelector('.threadItemCard-price')?.textContent.trim() || '';
                const oldPrice = element.querySelector('.text--lineThrough')?.textContent.trim() || '';
                const discount = element.querySelector('.text--color-charcoal')?.textContent.trim() || '';
                return { title, price, oldPrice, discount };
            });
        });

        deals.forEach(deal => {
            console.log('标题:', deal.title);
            console.log('价格:', deal.price);
            console.log('原价:', deal.oldPrice);
            console.log('折扣:', deal.discount);
            console.log('---');
        });
    } catch (error) {
        console.error('获取数据失败:', error);
    } finally {
        await browser.close();
    }
}

checkDeals();

内容的提问来源于stack exchange，提问作者Muhammad Kazim