S3 Glacier取回20KB文件为空问题排查及上传代码求助

核心问题 #

你遇到的空白归档问题，主要由三个关键错误导致：

1. Tree Hash计算错误 #

Glacier分片上传要求的checksum是层级哈希树（Tree Hash），而非整个文件的普通SHA256哈希。你当前直接对整个buffer计算SHA256，和Glacier验证要求不匹配，会导致归档组装失败。

2. 异步循环的闭包陷阱 #

for循环中直接调用异步的uploadMultipartPart，回调执行时循环变量i已经被修改，会导致分片的range参数错误，上传的分片范围混乱，最终归档内容损坏。

3. 原始数据转换错误 #

如果uploadData是CloudWatch日志的事件数组，直接用Buffer.from(uploadData)会将数组转为类似[object Object],[object Object]的无效字符串，这就是你取回的20KB空白内容的来源——实际上传的是这些无效字符串，而非日志数据。

修正后的代码 #

const crypto = require('crypto');
const glacier = require('aws-sdk/clients/glacier');

// 计算Glacier要求的Tree Hash
const calculateTreeHash = (buffer, partSize) => {
  const hashes = [];
  // 按分片大小拆分buffer，计算每个分片的SHA256
  for (let i = 0; i < buffer.length; i += partSize) {
    const chunk = buffer.slice(i, Math.min(i + partSize, buffer.length));
    const hash = crypto.createHash('sha256').update(chunk).digest();
    hashes.push(hash);
  }
  // 递归计算层级哈希树
  while (hashes.length > 1) {
    const newHashes = [];
    for (let i = 0; i < hashes.length; i += 2) {
      if (i + 1 < hashes.length) {
        // 合并两个哈希并计算新哈希
        const combined = Buffer.concat([hashes[i], hashes[i + 1]]);
        newHashes.push(crypto.createHash('sha256').update(combined).digest());
      } else {
        // 奇数个哈希时，直接保留最后一个
        newHashes.push(hashes[i]);
      }
    }
    hashes.splice(0, hashes.length, ...newHashes);
  }
  // 转为十六进制字符串
  return hashes[0].toString('hex');
};

const multiPartUpload = async (uploadData, logGroupName) => {
  let archiveID = null;
  const partSize = 1024 * 1024 * 32;
  // 先将日志数组转为JSON字符串，再转Buffer
  const logContent = JSON.stringify(uploadData);
  const bufferUploadData = Buffer.from(logContent);
  const numPartsTotal = Math.ceil(bufferUploadData.length / partSize);
  console.log("分片数量: ", numPartsTotal);
  
  // 初始化分片上传
  const initiateUpload = () => {
    return new Promise((resolve, reject) => {
      const params = { vaultName: vaultname, partSize: partSize.toString() };
      glacier.initiateMultipartUpload(params, (mpErr, multipart) => {
        if (mpErr) {
          console.log("初始化上传失败:", mpErr.stack);
          reject(mpErr);
        } else {
          console.log("获取上传ID:", multipart.uploadId);
          resolve(multipart.uploadId);
        }
      });
    });
  };

  // 上传单个分片
  const uploadPart = (uploadId, start, end) => {
    return new Promise((resolve, reject) => {
      const range = `bytes ${start}-${end - 1}/*`;
      const partParams = {
        vaultName: vaultname,
        uploadId: uploadId,
        range: range,
        body: bufferUploadData.slice(start, end),
      };
      console.log("上传分片:", range);
      glacier.uploadMultipartPart(partParams, (multiErr, mData) => {
        if (multiErr) {
          console.log("分片上传失败:", multiErr);
          reject(multiErr);
        } else {
          console.log("分片上传完成:", mData);
          resolve();
        }
      });
    });
  };

  // 完成分片上传
  const completeUpload = (uploadId) => {
    return new Promise((resolve, reject) => {
      const treeHash = calculateTreeHash(bufferUploadData, partSize);
      const doneParams = {
        vaultName: vaultname,
        uploadId: uploadId,
        archiveSize: bufferUploadData.length.toString(),
        checksum: treeHash,
      };
      console.log("完成上传...");
      glacier.completeMultipartUpload(doneParams, (err, data) => {
        if (err) {
          console.log("完成上传失败:", err);
          reject(err);
        } else {
          const delta = (new Date() - startTime) / 1000;
          console.log("上传完成耗时:", delta, "秒");
          console.log("归档ID:", data.archiveId);
          console.log("校验和:", data.checksum);
          archiveID = data.archiveId;
          resolve(data.archiveId);
        }
      });
    });
  };

  try {
    const startTime = new Date();
    const uploadId = await initiateUpload();
    // 按顺序上传所有分片，避免异步循环问题
    for (let i = 0; i < bufferUploadData.length; i += partSize) {
      const end = Math.min(i + partSize, bufferUploadData.length);
      await uploadPart(uploadId, i, end);
    }
    const archiveId = await completeUpload(uploadId);
    // 处理DynamoDB存储
    const arr = logGroupName.split("/");
    const functionName = arr[3];
    await dynamodbPutItem(archiveId, functionName);
    console.log("归档ID已存入DynamoDB:", archiveId);
    return archiveId;
  } catch (err) {
    console.log("上传流程出错:", err);
    throw err;
  }
};

关键修改说明 #

1. Tree Hash计算：新增calculateTreeHash函数，严格按照Glacier的层级哈希树规则计算校验和，确保和Glacier的验证逻辑一致。
2. 异步流程改造：将所有Glacier回调转为Promise，用await按顺序上传分片，彻底解决闭包导致的循环变量问题，保证分片上传顺序和范围正确。
3. 数据格式修正：先将日志数组转为JSON字符串，再转为Buffer，确保上传的是可解析的日志内容，而非无效的数组字符串。
4. 错误处理优化：统一用try/catch捕获整个流程的错误，便于排查问题。

内容的提问来源于stack exchange，提问作者Charles Mohapatra