处理大型JSON数组双循环对比时的JavaScript堆内存溢出问题

问题背景 #

需要处理包含约150万个JSON对象的大型文件，核心操作是将每个对象的slug与其他对象sub_categories数组中的slug匹配，匹配成功则创建新对象存入result数组。

示例数据 #

const data = [
 {
  "slug": "vertical-lift-module-market",
  "id": 68055,
  "related_reports_updated": {
    "sub_categories": [
      {
        "slug": "audience-analytics-market",
        "id": 66684,
        "short_title": "Audience Analytics Market"
      },
      {
        "slug": "mobile-wallet-market",
        "id": 68830,
        "short_title": "Mobile Wallet Market"
      }
    ]
  }
},
{
  "slug": "united-states-real-estate-services---growth-trends-and-forecast-2022---2027",
  "id": 68056,
  "related_reports_updated": {
    "sub_categories": [
      {
        "slug": "canada-real-estate-services-market---growth-trends-and-forecast-2020---2025",
        "id": 68051,
        "short_title": "Canada Real Estate Services Market"
      },
      {
        "slug": "germany-real-estate-services-market--growth-trends-and-forecast-2020---2025",
        "id": 68054,
        "short_title": "Germany Real Estate Services Market"
      }
    ]
  }
 },
 // ... 共150万个对象
]

原实现代码 #

const result = [];

for(var i=0;i<data.length;i++) {
  
   for(var j=0;j<data.length;j++) {
        // 匹配逻辑
  }

} 

console.log(result);

报错信息 #

[41955:0x523ce90]   162238 ms: Mark-sweep (reduce) 4096.9 (4102.7) -> 4096.9 (4104.7) 
MB, 3481.7 / 0.4 ms  (average mu = 0.092, current mu = 0.000) allocation failure scavenge might not succeed

<--- JS stacktrace --->

FATAL ERROR: Ineffective mark-compacts near heap limit Allocation failed - JavaScript heap out of memory
1: 0xa3ac10 node::Abort() [node]
2: 0x970199 node::FatalError(char const*, char const*) [node]
3: 0xbba58e v8::Utils::ReportOOMFailure(v8::internal::Isolate*, char const*, bool) 
[node]
4: 0xbba907 v8::internal::V8::FatalProcessOutOfMemory(v8::internal::Isolate*, char 
const*, bool) [node]
5: 0xd76b25  [node]
6: 0xd776af  [node]
7: 0xd854eb v8::internal::Heap::CollectGarbage(v8::internal::AllocationSpace, 
v8::internal::GarbageCollectionReason, v8::GCCallbackFlags) [node]
8: 0xd890ac v8::internal::Heap::AllocateRawWithRetryOrFailSlowPath(int, 
v8::internal::AllocationType, v8::internal::AllocationOrigin, 
v8::internal::AllocationAlignment) [node]
9: 0xd5778b v8::internal::Factory::NewFillerObject(int, bool, 
v8::internal::AllocationType, v8::internal::AllocationOrigin) [node]
10: 0x109fd4f v8::internal::Runtime_AllocateInYoungGeneration(int, unsigned long*, 
v8::internal::Isolate*) [node]
11: 0x1448f59  [node]

Aborted (core dumped) 

已尝试node --max-old-space-size=4096 index.js扩展内存，但问题依旧。

解决方案 #

1. 优化算法时间复杂度（从O(n²)降到O(n)） #

原双重循环会产生150万×150万=2.25e12次操作，既低效又会堆积大量内存对象。先构建slug到对象的映射表，再遍历匹配：

// 第一步：构建slug到原对象的映射
const slugMap = new Map();
data.forEach(item => {
  slugMap.set(item.slug, item);
});

// 第二步：遍历每个对象的sub_categories，匹配并生成结果
const result = [];
data.forEach(mainItem => {
  const subCats = mainItem.related_reports_updated?.sub_categories || [];
  subCats.forEach(subCat => {
    const matchedItem = slugMap.get(subCat.slug);
    if (matchedItem) {
      // 根据需求创建新对象
      result.push({
        main_slug: mainItem.slug,
        main_id: mainItem.id,
        matched_slug: subCat.slug,
        matched_id: matchedItem.id,
        matched_title: subCat.short_title
      });
    }
  });
});

2. 流式处理（避免一次性加载全部数据） #

如果无法将150万个对象全部存入内存，用Node.js的流式API结合JSONStream模块逐段解析：

首先安装依赖：

npm install jsonstream

然后编写代码：

const fs = require('fs');
const JSONStream = require('JSONStream');
const slugMap = new Map();
const result = [];

// 第一遍流式读取，构建slug映射表
fs.createReadStream('./large-data.json')
  .pipe(JSONStream.parse('*'))
  .on('data', (item) => {
    slugMap.set(item.slug, item);
  })
  .on('end', () => {
    // 第二遍流式读取，匹配并生成结果
    fs.createReadStream('./large-data.json')
      .pipe(JSONStream.parse('*'))
      .on('data', (mainItem) => {
        const subCats = mainItem.related_reports_updated?.sub_categories || [];
        subCats.forEach(subCat => {
          const matchedItem = slugMap.get(subCat.slug);
          if (matchedItem) {
            result.push({
              main_slug: mainItem.slug,
              main_id: mainItem.id,
              matched_slug: subCat.slug,
              matched_id: matchedItem.id,
              matched_title: subCat.short_title
            });
          }
        });
      })
      .on('end', () => {
        // 将结果写入文件，避免内存堆积
        fs.writeFileSync('./result.json', JSON.stringify(result));
        console.log('处理完成');
      });
  });

3. 分批处理+手动触发垃圾回收 #

如果必须一次性加载数据，可将数据分成批次处理，每批处理完后清空变量并触发GC：

const fs = require('fs');
const data = JSON.parse(fs.readFileSync('./large-data.json'));
const slugMap = new Map();
const result = [];
const batchSize = 10000; // 每批处理1万个对象

// 先构建映射表
data.forEach(item => slugMap.set(item.slug, item));

// 分批处理
for (let i = 0; i < data.length; i += batchSize) {
  const batch = data.slice(i, i + batchSize);
  batch.forEach(mainItem => {
    const subCats = mainItem.related_reports_updated?.sub_categories || [];
    subCats.forEach(subCat => {
      const matchedItem = slugMap.get(subCat.slug);
      if (matchedItem) {
        result.push({
          main_slug: mainItem.slug,
          main_id: mainItem.id,
          matched_slug: subCat.slug,
          matched_id: matchedItem.id,
          matched_title: subCat.short_title
        });
      }
    });
  });
  // 清空批次引用，触发垃圾回收
  batch.length = 0;
  if (global.gc) global.gc(); // 需要启动时加 --expose-gc 参数
}

fs.writeFileSync('./result.json', JSON.stringify(result));

启动命令：

node --expose-gc --max-old-space-size=8192 index.js

4. 进一步调整内存参数 #

如果机器内存足够，可尝试更大的内存限制，比如设置为8GB：

node --max-old-space-size=8192 index.js

内容的提问来源于stack exchange，提问作者Digvijay Singh Thakur