Node.js Lambda从第三方下载文件上传S3时非文本文件损坏

你的Lambda函数处理纯文本文件正常，但图片、PDF等二进制文件上传至S3后损坏，核心原因是将二进制数据当作UTF-8字符串拼接，破坏了原始字节信息。

问题根源 #

在https_get_processor函数中，你用var data = ""初始化数据容器，通过data += chunk拼接响应块时，二进制chunk会被自动转换为UTF-8字符串。而二进制文件的字节序列不符合UTF-8编码规则，转码过程中会丢失或篡改原始数据，最终导致上传到S3的文件损坏。

修复方案 #

1. 改用Buffer存储二进制数据

修改https_get_processor，用Buffer数组收集响应块，最后合并为单个完整的Buffer：

async function https_get_processor(get_params) {
  return await new Promise((resolve, reject) => {
    const chunks = []; // 用数组存储Buffer块

    const req = https.request(get_params, res => {
      res.on('data', chunk => chunks.push(chunk)); // 直接存入原始Buffer

      res.on('end', () => {
        // 合并所有Buffer块为一个完整的二进制Buffer
        const buffer = Buffer.concat(chunks);
        resolve(buffer);
      });
    });

    req.on('error', reject);
    req.end();
  });
}

2. 等待S3上传完成（避免Lambda提前终止）

原代码中s3.upload采用异步回调，但Lambda函数直接返回响应，可能在上传完成前就被强制终止。需要将s3.upload包装为Promise并通过await等待执行完成：

// 替换原有的s3.upload代码
try {
  const uploadResult = await s3.upload(uploadParams).promise();
  console.log("Upload Success", uploadResult.Location);
} catch (err) {
  console.log("Error", err);
}

完整修复后的代码 #

const AWS = require("aws-sdk");
const https = require('https');
const path = require('path');

const s3 = new AWS.S3({apiVersion: '2006-03-01'});

exports.handler = async (event, context, callback) => {
  var payload = event.body;
  const url_host = payload.host;
  const url_path = payload.path;

  const get_params = {
    host: url_host,
    path: url_path,
    port: 443,
    method: 'GET',
    headers: { }
  };

  var resp = await https_get_processor(get_params);

  var uploadParams = {
    Bucket: "bucket_name", 
    Key: path.basename(url_path),
    Body: resp
  };

  // 等待S3上传完成
  try {
    const uploadResult = await s3.upload(uploadParams).promise();
    console.log("Upload Success", uploadResult.Location);
  } catch (err) {
    console.log("Error", err);
  }

  const response = {
    statusCode: 200,
    body: JSON.stringify({message: "Processing completed"})
  };
  return response;
};

async function https_get_processor(get_params) {
  return await new Promise((resolve, reject) => {
    const chunks = [];

    const req = https.request(get_params, res => {
      res.on('data', chunk => chunks.push(chunk));

      res.on('end', () => {
        const buffer = Buffer.concat(chunks);
        resolve(buffer);
      });
    });

    req.on('error', reject);
    req.end();
  });
}

额外注意事项 #

• 确保Lambda执行角色拥有S3上传权限（s3:PutObject）。
• 如果第三方服务器返回Content-Length响应头，可以预先分配Buffer大小优化性能，但Buffer.concat已能处理绝大多数场景。

内容的提问来源于stack exchange，提问作者Dalhousie