如何使用AWS JavaScript SDK复制大于5GB的S3对象？

AWS SDK for JavaScript的copyObject接口仅支持复制≤5GB的对象，要处理更大的对象，需要手动实现分段复制流程，对应boto3自动处理的底层逻辑。核心是通过createMultipartUpload、uploadPartCopy、completeMultipartUpload三个接口完成，下面是完整实现代码：

完整实现代码 #

const AWS = require('aws-sdk');
const s3 = new AWS.S3();

// 配置分段大小（建议设为1GB，最大不超过5GB）
const PART_SIZE = 1024 * 1024 * 1024; // 1GB

async function copyLargeS3Object(sourceBucket, sourceKey, targetBucket, targetKey) {
    let uploadId;
    try {
        // 1. 获取源对象的大小，用于计算分段数
        const headResponse = await s3.headObject({
            Bucket: sourceBucket,
            Key: sourceKey
        }).promise();
        const objectSize = headResponse.ContentLength;
        const partCount = Math.ceil(objectSize / PART_SIZE);

        // 2. 创建分段上传任务
        const createUploadResponse = await s3.createMultipartUpload({
            Bucket: targetBucket,
            Key: targetKey
        }).promise();
        uploadId = createUploadResponse.UploadId;

        // 3. 批量复制每个分段
        const partPromises = [];
        for (let partNumber = 1; partNumber <= partCount; partNumber++) {
            const start = (partNumber - 1) * PART_SIZE;
            const end = Math.min(start + PART_SIZE - 1, objectSize - 1);

            const partPromise = s3.uploadPartCopy({
                Bucket: targetBucket,
                Key: targetKey,
                UploadId: uploadId,
                PartNumber: partNumber,
                CopySource: `${sourceBucket}/${sourceKey}`,
                CopySourceRange: `bytes=${start}-${end}`
            }).promise().then(response => ({
                PartNumber: partNumber,
                ETag: response.CopyPartResult.ETag
            }));
            partPromises.push(partPromise);
        }
        const parts = await Promise.all(partPromises);

        // 4. 完成分段上传
        await s3.completeMultipartUpload({
            Bucket: targetBucket,
            Key: targetKey,
            UploadId: uploadId,
            MultipartUpload: { Parts: parts }
        }).promise();

        console.log(`大对象复制完成：${sourceBucket}/${sourceKey} -> ${targetBucket}/${targetKey}`);
    } catch (error) {
        console.error('复制失败：', error);
        // 若中途出错，中止分段上传以避免存储费用浪费
        if (uploadId) {
            await s3.abortMultipartUpload({
                Bucket: targetBucket,
                Key: targetKey,
                UploadId: uploadId
            }).promise();
        }
        throw error;
    }
}

// 使用示例
(async () => {
    const sourceBucket = 'your-source-bucket';
    const sourceKey = 'large-file.bin';
    const targetBucket = 'your-target-bucket';
    const targetKey = 'copied-large-file.bin';

    await copyLargeS3Object(sourceBucket, sourceKey, targetBucket, targetKey);
})();

关键步骤说明 #

• 获取对象大小：通过headObject获取源对象的总大小，以此确定需要分成多少段。
• 分段大小设置：每个分段最大支持5GB，建议设为1GB（平衡请求数和单段大小）。
• 分段复制：循环调用uploadPartCopy，通过CopySourceRange指定当前分段的字节范围，复制对应部分到目标桶。
• 完成上传：收集所有分段的ETag后，调用completeMultipartUpload合并分段，生成完整的目标对象。
• 异常处理：若复制过程中出错，调用abortMultipartUpload中止任务，避免S3存储未完成的分段产生额外费用。

内容的提问来源于stack exchange，提问作者Sk2415