如何用Puppeteer与Cheerio获取Script标签中的this.data数组？

问题描述 #

我正尝试获取网页Script标签中this.data = [{"id_penggiat":"6f81520f-1a8b-4d1d-a7d5-defdb630818b"...}]内的数组，目前使用Cheerio与Puppeteer工具，但始终无法成功获取结果。相关Script代码片段如下：

<script>
.......
var PenggiatField = {
    fieldid: '#penggiatField',
    data: true,
    debug: false,
    Init: function(params) {
        var field = this;
                this.data = [{"id_penggiat":"6f81520f-1a8b-4d1d-a7d5-defdb630818b","id_kerma":"9388706e-7156-4d07-bf77-cb9c6d7f30d3","id_mitra":null,"id_unit":"89c82da3-7b28-4305-82c4-347dae042847","id_unit_teknis":null,"no_pihak":"1","nm_pihak":"Institut Teknologi Sepuluh Nopember","alamat":"Kampus ITS Sukolilo Surabaya","ttd_nama":"Tri Joko Wahyu Adi, ST., MT., Ph.D.","ttd_jabatan":"Direktur","pic_nama":"","pic_jabatan":"","pic_email":null,"pelaksana":["c2c72bf4-ec6c-46dd-8be8-d9fb77a90bd6","7e42c5bb-0c44-4b93-9162-ec6fb31cd67c","902bd59e-745d-4903-b1bf-171fe6f0fb8c","c93fb75a-1065-4587-a7af-475d4256f44d","9a6592f0-ff86-401d-8021-0e9f6fc285fe","8f2d4244-d868-45e3-a906-578ce2f9f572","1f8fc06b-4707-4e11-bdc8-312f39017758"],"create_date":"2022-03-16 04:41:14.146436","id_creator":"79ce0204-5c6d-43dd-a310-8c13a557a402","last_update":"2022-03-16 04:54:25.228835","id_updater":"79ce0204-5c6d-43dd-a310-8c13a557a402","soft_delete":"0","a_lembaga":"university","id_institusi":"89c82da3-7b28-4305-82c4-347dae042847","nm_mitra":null,"nm_unit":"Institut Teknologi Sepuluh Nopember","nm_unit_teknis":null,"id_klas_mitra":"12","nm_klas_mitra":"Institusi Pendidikan","id_negara":"ID","nm_negara":"Indonesia"},{"id_penggiat":"c4c3db69-8cbb-4a8e-ae6c-544b532760c6","id_kerma":"9388706e-7156-4d07-bf77-cb9c6d7f30d3","id_mitra":"f4b81538-6442-4a19-b9f6-8581d152725d","id_unit":null,"id_unit_teknis":null,"no_pihak":"2","nm_pihak":"Sekolah Menengah Atas Negeri 1 Karas Magetan","alamat":"Jl. Karas, Punden, Kuwon, Kec. Kendal, Kabupaten Magetan, Jawa Timur 63395","ttd_nama":"Bahtiar Kholili, S.Pd., M.M.Pd.","ttd_jabatan":"Kepala Sekolah","pic_nama":"","pic_jabatan":"","pic_email":null,"pelaksana":null,"create_date":"2022-03-16 04:41:14.146436","id_creator":"79ce0204-5c6d-43dd-a310-8c13a557a402","last_update":"2022-03-16 04:54:25.228835","id_updater":"79ce0204-5c6d-43dd-a310-8c13a557a402","soft_delete":"0","a_lembaga":"mitra","id_institusi":"f4b81538-6442-4a19-b9f6-8581d152725d","nm_mitra":"Sekolah Menengah Atas Negeri 1 Karas Magetan","nm_unit":null,"nm_unit_teknis":null,"id_klas_mitra":"12","nm_klas_mitra":"Institusi Pendidikan","id_negara":"ID","nm_negara":"Indonesia"}];
        
     ........
</script>

解决方案 #

Puppeteer 实现方法 #

Puppeteer可直接在页面上下文执行JS，模拟浏览器环境获取数据：

const puppeteer = require('puppeteer');

(async () => {
  const browser = await puppeteer.launch();
  const page = await browser.newPage();
  await page.goto('目标网页URL');

  // 在页面上下文获取目标数组
  const penggiatData = await page.evaluate(() => {
    // 实例化并初始化PenggiatField
    const fieldInstance = Object.create(PenggiatField);
    fieldInstance.Init();
    return fieldInstance.data;
  });

  console.log('获取到的数据：', penggiatData);
  await browser.close();
})();

若页面已自动执行过PenggiatField.Init()，可简化为直接返回PenggiatField.data。

Cheerio 实现方法 #

Cheerio是静态HTML解析工具，需通过正则匹配脚本内容提取数组：

const cheerio = require('cheerio');
// 替换为你获取到的网页HTML内容
const html = '<网页HTML字符串>';

const $ = cheerio.load(html);

// 遍历所有script标签查找目标内容
$('script').each((index, element) => {
  const scriptContent = $(element).html();
  // 正则匹配`this.data = `后的数组内容
  const dataMatch = scriptContent.match(/this\.data\s*=\s*([\s\S]*?);/);
  
  if (dataMatch && dataMatch[1]) {
    try {
      // 替换HTML实体为双引号后解析JSON
      const jsonString = dataMatch[1].replace(/&quot;/g, '"');
      const penggiatData = JSON.parse(jsonString);
      console.log('获取到的数据：', penggiatData);
    } catch (error) {
      console.error('JSON解析失败：', error);
    }
  }
});

注意事项 #

• Puppeteer适合动态渲染页面，依赖浏览器环境；
• Cheerio是静态解析，需确保正则能准确匹配，脚本格式变化时需调整正则；
• 脚本中的"是HTML转义字符，必须替换为双引号才能正确解析JSON。

内容的提问来源于stack exchange，提问作者ssudo