Node Crypto无法解密超过stream highWaterMark的文件问题排查

你的代码核心错误在于把每个流chunk当作独立的加密单元处理，但流式传输时无法保证解密阶段读取的chunk刚好对应加密阶段的完整加密包，导致解密时无法正确解析salt、authTag和加密数据的结构。

具体问题拆解 #

• 加密阶段：每个流chunk都会调用app.encrypt()，生成独立的salt、authTag和加密数据，最终加密文件是多个「salt(16B)+authTag(16B)+加密chunk」的拼接。
• 解密阶段：你默认每个读取到的chunk都是一个完整的加密包，但流式读取的chunk是按缓冲区大小拆分的（默认64KB），当加密后的单个包大小超过缓冲区，或者缓冲区拆分刚好切断了salt/authTag/加密数据的边界时，data.subarray(0,16)拿到的就不是正确的salt，后续解密必然失败。
• 调高highWaterMark只是刚好让读取的chunk能容纳完整的加密包，属于侥幸规避问题，不是正确的流式加密方案。

正确的流式加解密实现 #

正确的做法是在整个流生命周期内使用同一个cipher/decipher实例，而不是每个chunk新建实例。修改后的代码如下：

修正后的加解密工具类 #

const crypto = require('crypto');
const { Transform } = require('stream');

class EncryptTransform extends Transform {
  constructor(password) {
    super();
    this.password = password;
    this.salt = crypto.randomBytes(16);
    const key = crypto.scryptSync(password, this.salt, 32, { N: 16384 });
    this.cipher = crypto.createCipheriv('aes-256-gcm', key, this.salt);
    // 先把salt写入流，解密时需要先读取这个salt
    this.push(this.salt);
  }

  _transform(chunk, encoding, callback) {
    try {
      const encryptedChunk = this.cipher.update(chunk, encoding);
      this.push(encryptedChunk);
      callback();
    } catch (err) {
      callback(err);
    }
  }

  _flush(callback) {
    try {
      const finalChunk = this.cipher.final();
      this.push(finalChunk);
      // 最后写入authTag
      const authTag = this.cipher.getAuthTag();
      this.push(authTag);
      callback();
    } catch (err) {
      callback(err);
    }
  }
}

class DecryptTransform extends Transform {
  constructor(password) {
    super();
    this.password = password;
    this.state = 'waitingForSalt';
    this.salt = null;
    this.authTag = null;
    this.decipher = null;
    this.buffer = Buffer.alloc(0);
  }

  _transform(chunk, encoding, callback) {
    try {
      this.buffer = Buffer.concat([this.buffer, Buffer.from(chunk, encoding)]);

      if (this.state === 'waitingForSalt') {
        // 先读取16字节的salt
        if (this.buffer.length >= 16) {
          this.salt = this.buffer.subarray(0, 16);
          this.buffer = this.buffer.subarray(16);
          const key = crypto.scryptSync(this.password, this.salt, 32, { N: 16384 });
          this.decipher = crypto.createDecipheriv('aes-256-gcm', key, this.salt);
          this.state = 'waitingForAuthTag';
        }
      }

      if (this.state === 'waitingForAuthTag') {
        // 最后16字节是authTag，所以先处理中间的加密数据，留到flush阶段处理authTag
        if (this.buffer.length > 16) {
          const dataChunk = this.buffer.subarray(0, this.buffer.length - 16);
          this.buffer = this.buffer.subarray(this.buffer.length - 16);
          const decrypted = this.decipher.update(dataChunk);
          this.push(decrypted);
        }
      } else if (this.state === 'decrypting') {
        const decrypted = this.decipher.update(this.buffer);
        this.push(decrypted);
        this.buffer = Buffer.alloc(0);
      }
      callback();
    } catch (err) {
      callback(err);
    }
  }

  _flush(callback) {
    try {
      if (this.state === 'waitingForAuthTag') {
        // 剩下的16字节是authTag
        this.authTag = this.buffer;
        this.decipher.setAuthTag(this.authTag);
        // 处理可能剩余的加密数据（如果buffer刚好16字节，这里就没有数据）
        const decrypted = this.decipher.update(this.buffer.subarray(0, 0));
        this.push(decrypted);
      }
      const finalChunk = this.decipher.final();
      this.push(finalChunk);
      callback();
    } catch (err) {
      console.error('解密失败：密码错误或文件损坏');
      callback(err);
    }
  }
}

// 导出工具
const streamUtils = {
  createEncryptStream(password) {
    return new EncryptTransform(password);
  },
  createDecryptStream(password) {
    return new DecryptTransform(password);
  }
};

module.exports = streamUtils;

修正后的文件处理代码 #

const fs = require('fs');
const streamUtils = require('./streamUtils');

const file = {
  encrypt(fileLoc, password = 'password') {
    const fileReadStream = fs.createReadStream(fileLoc);
    const newFileLoc = './sample_enc.txt';
    const fileWriteStream = fs.createWriteStream(newFileLoc);
    const encryptStream = streamUtils.createEncryptStream(password);

    fileReadStream.on('error', (err) => {
      console.log('读取文件失败：', err.message);
      fileWriteStream.destroy();
      fs.unlinkSync(newFileLoc, (unlinkErr) => {});
    });

    fileReadStream.pipe(encryptStream).pipe(fileWriteStream);
  },
  decrypt(fileLoc, password = 'password') {
    const fileReadStream = fs.createReadStream(fileLoc);
    const newFileLoc = './sample_dec.txt';
    const fileWriteStream = fs.createWriteStream(newFileLoc);
    const decryptStream = streamUtils.createDecryptStream(password);

    fileReadStream.on('error', (err) => {
      console.log('读取文件失败：', err.message);
      fileWriteStream.destroy();
      fs.unlinkSync(newFileLoc, (unlinkErr) => {});
    });

    decryptStream.on('error', (err) => {
      console.log('解密失败：', err.message);
      fileWriteStream.destroy();
      fs.unlinkSync(newFileLoc, (unlinkErr) => {});
    });

    fileReadStream.pipe(decryptStream).pipe(fileWriteStream);
  }
};

// 使用示例
// file.encrypt('./sample.txt');
// file.decrypt('./sample_enc.txt');

关键改进点 #

• 整个流使用同一个cipher/decipher实例，保证加密上下文的连续性。
• 加密时先写入salt，最后写入authTag，解密时先读取salt初始化decipher，最后读取authTag验证。
• 处理流chunk的边界问题，通过缓冲区暂存不完整的部分，直到凑够所需的结构（salt、authTag）再处理。

内容的提问来源于stack exchange，提问作者Akshay