如何修改Node.js代码处理25GB+图片？解决EIO/ENOTCONN错误

使用Node.js结合imagemin和imagemin-mozjpeg处理图片时，小体积（<500MB）图片处理正常，但处理25GB以上的大体积图片时出现两类错误：

错误类型1 #

node:internal/process/promises:279
triggerUncaughtException(err, true /* fromPromise */);
^

[Error: EIO: i/o error, write] {
errno: -5,
code: 'EIO',
syscall: 'write'
}

错误类型2 #

node:internal/process/promises:279
triggerUncaughtException(err, true /* fromPromise */);
^

Error: read ENOTCONN
at tryReadStart (node:net:614:20)
at Socket._read (node:net:625:5)
at Socket.Readable.read (node:internal/streams/readable:487:10)
at Socket.read (node:net:666:39)
at new Socket (node:net:415:12)
at Object.Socket (node:net:286:41)
at createSocket (node:internal/child_process:328:14)
at ChildProcess.spawn (node:internal/child_process:445:23)
at Object.spawn (node:child_process:700:9)
at execa (file:///mnt/d/Projects/GH_2022/imagemin-mozjpeg/node_modules/execa/index.js:84:26) {
errno: -107,
code: 'ENOTCONN',
syscall: 'read',
originalMessage: 'read ENOTCONN',
shortMessage: 'Command failed with ENOTCONN: /mnt/d/Projects/GH_2022/imagemin-mozjpeg/node_modules/mozjpeg/vendor/cjpeg -quality 75\n' +
'read ENOTCONN',
command: '/mnt/d/Projects/GH_2022/imagemin-mozjpeg/node_modules/mozjpeg/vendor/cjpeg -quality 75',
escapedCommand: '"/mnt/d/Projects/GH_2022/imagemin-mozjpeg/node_modules/mozjpeg/vendor/cjpeg" -quality 75',
exitCode: undefined,
signal: undefined,
signalDescription: undefined,
stdout: Buffer(0) [Uint8Array] [],
stderr: Buffer(0) [Uint8Array] [],
failed: true,
timedOut: false,
isCanceled: false,
killed: false
}

原代码 #

import imagemin from 'imagemin';
import imageminMozjpeg from 'imagemin-mozjpeg';

import { promises as fsPromises } from 'node:fs';
import { promisify } from 'node:util';
import path from 'node:path';
import fs from 'graceful-fs';

const writeFile = promisify(fs.writeFile);

const srcdir = 'images/source';
const distdir = 'images/dist';

imagemin([srcdir + '/**/*.{jpg,jpeg,png}'], {
    plugins: [
        imageminMozjpeg({
            quality: 75
        })
    ]
}).then(files => files
    .forEach(async v => {
        let source = path.parse(v.sourcePath);
        v.destinationPath = `${source.dir.replace(srcdir, distdir)}/${source.name}${source.ext}`;
        await fsPromises.mkdir(path.dirname(v.destinationPath), { recursive: true });
        await writeFile(v.destinationPath, v.data);
    })
);

问题根源在于原代码一次性将大文件加载到内存，触发I/O异常和子进程通信错误。以下是针对性修改：

修改要点 #

• 使用流式处理替代内存缓存，避免大文件占用过多内存
• 逐个处理文件，降低系统并发压力
• 直接调用mozjpeg的CLI工具，通过流管道传输数据，规避imagemin的内存瓶颈

优化后的代码 #

import { spawn } from 'node:child_process';
import { createReadStream, createWriteStream, promises as fsPromises } from 'node:fs';
import path from 'node:path';
import { glob } from 'glob';

const srcdir = 'images/source';
const distdir = 'images/dist';
// 根据实际路径调整cjpeg的位置
const cjpegPath = path.join(__dirname, 'node_modules/mozjpeg/vendor/cjpeg');

// 获取所有目标图片文件
async function getImageFiles() {
    return glob(`${srcdir}/**/*.{jpg,jpeg,png}`);
}

// 单文件流式处理函数
async function processImage(sourcePath) {
    const destPath = sourcePath.replace(srcdir, distdir);
    
    // 创建目标目录
    await fsPromises.mkdir(path.dirname(destPath), { recursive: true });
    
    return new Promise((resolve, reject) => {
        // 启动cjpeg压缩进程
        const cjpeg = spawn(cjpegPath, ['-quality', '75']);
        
        // 源文件流 -> cjpeg标准输入
        const readStream = createReadStream(sourcePath);
        readStream.pipe(cjpeg.stdin);
        
        // cjpeg标准输出 -> 目标文件流
        const writeStream = createWriteStream(destPath);
        cjpeg.stdout.pipe(writeStream);
        
        // 处理各类错误与完成事件
        readStream.on('error', reject);
        writeStream.on('finish', resolve);
        writeStream.on('error', reject);
        cjpeg.on('error', reject);
        cjpeg.on('exit', (code) => {
            if (code !== 0) {
                reject(new Error(`cjpeg进程退出，码值：${code}`));
            }
        });
    });
}

// 主流程：逐个处理文件
async function main() {
    const files = await getImageFiles();
    for (const file of files) {
        console.log(`处理中：${file}`);
        try {
            await processImage(file);
            console.log(`处理完成：${file}`);
        } catch (err) {
            console.error(`处理失败：${file}，错误：${err.message}`);
        }
    }
    console.log('所有文件处理结束');
}

main().catch(err => console.error('主进程错误：', err));

说明 #

1. 流式处理：通过Node.js流模块，将源文件直接通过管道传输给cjpeg工具，处理后的结果直接写入目标文件，全程不加载完整文件到内存，完美适配大体积图片。
2. 逐个处理：使用for...of循环逐个处理文件，避免同时启动过多子进程导致系统资源耗尽。
3. 直接调用CLI：绕过imagemin的内存缓存逻辑，直接调用mozjpeg的cjpeg命令，减少中间层的内存开销。

内容的提问来源于stack exchange，提问作者Howard