Node.js大size请求触发JavaScript堆内存溢出的原因与解决

根本原因 #

你的代码触发内存溢出的核心问题在于频繁调用res.write("-")导致内存缓冲堆积：

• Node.js的HTTP响应对象（res）会将每次res.write()写入的数据缓冲在内存中，直到内部缓冲区满才会向客户端发送数据。
• 当设置超大size（如1亿）时，循环执行1亿次res.write("-")，会创建1亿个独立的小Buffer（每个仅1字节），这些Buffer全部堆积在内存中，迅速耗尽V8的堆内存配额，最终触发内存溢出错误。
• 即使手动扩容V8堆内存，这种方式依然低效，大量小Buffer的创建和管理会带来极高的内存开销和GC压力。

合理解决方案（支持1GB+内容生成） #

要高效生成大体积响应内容，核心思路是减少内存缓冲，采用批量写入或流式输出，以下是两种可行方案：

方案1：批量生成大Buffer块写入 #

一次性生成较大的重复字符块，循环写入该块，大幅减少Buffer创建次数：

#!/usr/bin/node
import fs from "node:fs"
import path from "node:path"
import http from 'node:http'
const port = process.argv[2] || 8080
const baseDir = process.argv[3] || "/tmp/"

function log(msg){
  if(!msg) return
  let now = new Date()
  let ts = now.toLocaleTimeString()
  let tsdate = now.toLocaleDateString().replace(/\//ig,"-")
  fs.appendFile(path.join(baseDir,"fileUpload" + tsdate + ".log"), ts + " " + msg + "\n", (err)=>{
    if(err) console.log(err)
  })
}

// 生成1MB大小的重复字符块
const BLOCK_SIZE = 1024 * 1024;
const block = Buffer.alloc(BLOCK_SIZE, '-');

http.createServer((req, res) => {
  if (req.url != '/generate' || req.method !== 'GET') {
    log(req.headers.host + " " + req.method + " " + req.url + " " + req.headers.size + " usage_fail")
    res.writeHead(200, { 'Content-Type': 'text/plain' });
    return res.end("wrong usage\n")
  }

  const totalSize = parseInt(req.headers.size, 10);
  if (isNaN(totalSize) || totalSize <= 0) {
    log(req.headers.host + " " + req.method + " " + req.url + " invalid_size")
    res.writeHead(400, { 'Content-Type': 'text/plain' });
    return res.end("invalid size\n")
  }

  res.writeHead(200, { 
    'Content-Type': 'text/plain',
    'Content-Length': totalSize.toString() // 提前告知客户端内容长度
  });

  let remaining = totalSize;
  function writeChunk() {
    // 每次写入剩余大小和块大小的较小值
    const writeSize = Math.min(remaining, BLOCK_SIZE);
    const chunk = writeSize === BLOCK_SIZE ? block : block.slice(0, writeSize);
    
    // 检查是否可以继续写入，避免背压
    if (res.write(chunk)) {
      remaining -= writeSize;
      if (remaining > 0) {
        setImmediate(writeChunk); // 非阻塞继续写入
      } else {
        res.end();
      }
    } else {
      // 缓冲区满，等待drain事件再继续
      res.once('drain', writeChunk);
    }
  }

  writeChunk();
}).listen(port, (err) => {
  console.log('Server listening on http://localhost:' + port, err)
})

• 优势：仅创建一个大Buffer块，内存占用稳定（仅1MB左右），即使生成1GB内容，内存开销也几乎不变。
• 额外优化：添加Content-Length头让客户端提前知晓下载大小；处理drain事件避免背压，保证流式输出的稳定性。

方案2：使用Readable流输出 #

利用Node.js的stream.Readable自定义流，按需生成内容，完全避免内存堆积：

#!/usr/bin/node
import fs from "node:fs"
import path from "node:path"
import http from 'node:http'
import { Readable } from 'node:stream';

const port = process.argv[2] || 8080
const baseDir = process.argv[3] || "/tmp/"

function log(msg){
  if(!msg) return
  let now = new Date()
  let ts = now.toLocaleTimeString()
  let tsdate = now.toLocaleDateString().replace(/\//ig,"-")
  fs.appendFile(path.join(baseDir,"fileUpload" + tsdate + ".log"), ts + " " + msg + "\n", (err)=>{
    if(err) console.log(err)
  })
}

http.createServer((req, res) => {
  if (req.url != '/generate' || req.method !== 'GET') {
    log(req.headers.host + " " + req.method + " " + req.url + " " + req.headers.size + " usage_fail")
    res.writeHead(200, { 'Content-Type': 'text/plain' });
    return res.end("wrong usage\n")
  }

  const totalSize = parseInt(req.headers.size, 10);
  if (isNaN(totalSize) || totalSize <= 0) {
    log(req.headers.host + " " + req.method + " " + req.url + " invalid_size")
    res.writeHead(400, { 'Content-Type': 'text/plain' });
    return res.end("invalid size\n")
  }

  res.writeHead(200, { 
    'Content-Type': 'text/plain',
    'Content-Length': totalSize.toString()
  });

  // 自定义可读流
  const contentStream = new Readable({
    read(size) {
      const chunkSize = Math.min(size, totalSize - this.bytesSent);
      if (chunkSize <= 0) {
        this.push(null); // 结束流
        return;
      }
      this.push(Buffer.alloc(chunkSize, '-'));
      this.bytesSent += chunkSize;
    }
  });

  contentStream.pipe(res); // 管道到响应对象，自动处理背压
}).listen(port, (err) => {
  console.log('Server listening on http://localhost:' + port, err)
})

• 优势：完全基于流的方式，按需生成内容块，内存占用极低；pipe()方法自动处理背压，无需手动管理写入逻辑，代码更简洁健壮。

内容的提问来源于stack exchange，提问作者Chan Mo-Che