You need to enable JavaScript to run this app.
导航
对象存储
  • 文档首页
    • /
  • 对象存储

分片上传（Android SDK）

最近更新时间：2024.03.21 15:13:15

首次发布时间：2022.12.01 16:31:39

我的收藏

对于较大的对象，可以对象数据分成多个分片（part）来分别上传，最后将所有上传的分片合并为一个对象。

分片上传步骤

分片上传包括三个基本步骤：

  1. 通过 createMultipartUpload 初始化分片上传任务。
    在上传分片数据之前，需要先通过 createMultipartUpload 接口初始化并获取一个分片任务的 uploadID，后续的上传分片、合并分片、取消分片和列举已上传分片都需要传入 uploadID 参数。初始化分片上传任务不影响已存在的同名对象。
  2. 通过 uploadPart 进行分片上传。
    通过此接口上传分片数据，且需要指定通过 createMultipartUpload 获取的 uploadID，以及分片的编号，编号的范围是 [1, 10000]。同一个对象的同一个分片任务，支持多个分片同时上传，上传顺序不影响最终的合并分片操作。除了最后一个分片，其他分片大小需要大于等于 4MiB。
    SDK 支持通过 uploadPart 进行直接分片上传，也支持 uploadPartFromFile 通过文件进行分片上传。
  3. 通过 completeMultipartUpload 将所有分片合并为一个对象。
    该接口用于合并已经上传的分片数据。在调用该接口时，必须提供有效的分片列表（包含partNumber和etag），TOS 服务端收到请求后，会根据提供的分片列表来合并分片为一个对象。

示例代码

以下代码展示如何通过分片接口分片上传一个对象。

import android.os.Bundle;
import android.util.Log;

import androidx.appcompat.app.AppCompatActivity;

import com.volcengine.tos.TOSV2;
import com.volcengine.tos.TOSV2ClientBuilder;
import com.volcengine.tos.TosException;
import com.volcengine.tos.comm.common.ACLType;
import com.volcengine.tos.comm.common.StorageClassType;
import com.volcengine.tos.comm.io.TosRepeatableBoundedFileInputStream;
import com.volcengine.tos.model.object.CompleteMultipartUploadV2Input;
import com.volcengine.tos.model.object.CompleteMultipartUploadV2Output;
import com.volcengine.tos.model.object.CreateMultipartUploadInput;
import com.volcengine.tos.model.object.CreateMultipartUploadOutput;
import com.volcengine.tos.model.object.ObjectMetaRequestOptions;
import com.volcengine.tos.model.object.UploadPartBasicInput;
import com.volcengine.tos.model.object.UploadPartFromFileInput;
import com.volcengine.tos.model.object.UploadPartFromFileOutput;
import com.volcengine.tos.model.object.UploadPartV2Input;
import com.volcengine.tos.model.object.UploadPartV2Output;
import com.volcengine.tos.model.object.UploadedPartV2;

import java.io.ByteArrayInputStream;
import java.io.FileInputStream;
import java.io.IOException;
import java.io.InputStream;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;

public class MultipartUploadExample extends AppCompatActivity {
    @Override
    protected void onCreate(Bundle savedInstanceState) {
        String endpoint = "your endpoint";
        String region = "your region";
        String accessKey = "your access key";
        String secretKey = "your secret key";
        String securityToken = "your security token";

        String bucketName = "your bucket name";
        String objectKey = "your object key";
        super.onCreate(savedInstanceState);
        setContentView(R.layout.activity_display_message);
        TOSV2 tos = new TOSV2ClientBuilder().build(region, endpoint, accessKey, secretKey, securityToken);

        Thread tosThread = new Thread(new Runnable() {
            @Override
            public void run() {
                String uploadID = null;
                // 1. 初始化分片上传
                try{
                    CreateMultipartUploadInput create = new CreateMultipartUploadInput().setBucket(bucketName).setKey(objectKey);
                    // 如果需要设置对象的元数据，需要在初始化分片上传的时候设置
                    ObjectMetaRequestOptions options = new ObjectMetaRequestOptions();
                    // 设置对象访问权限，此处为私有权限
                    options.setAclType(ACLType.ACL_PRIVATE);
                    // 设置对象存储类型
                    options.setStorageClass(StorageClassType.STORAGE_CLASS_STANDARD);
                    // SDK 默认会根据 objectKey 后缀识别 Content-Type，也可以自定义设置
                    options.setContentType("text/plain");
                    // 设置对象内容语言
                    options.setContentLanguage("en-US");
                    // 设置对象被下载时的名称
                    options.setContentDisposition("attachment;filename=123.txt");
                    // 设置对象的网页缓存行为
                    options.setCacheControl("no-cache, no-store, must-revalidate");
                    // 设置对象的服务端加密方式，当前只支持 AES256
                    options.setServerSideEncryption("AES256");
                    // 自定义对象的元数据，对于自定义的元数据，SDK 会自动对 key 添加
                    // "X-Tos-Meta-" 的前缀，并保存在服务端。
                    Map<String, String> custom = new HashMap<>();
                    custom.put("name", "volc_user");
                    // 在 TOS 服务端存储的元数据为："X-Tos-Meta-name: volc_user"
                    options.setCustomMetadata(custom);
                    create.setOptions(options);

                    CreateMultipartUploadOutput createOutput = tos.createMultipartUpload(create);
                    Log.i("createMultipartUpload", "createMultipartUpload succeed, uploadID is " + createOutput.getUploadID());

                    uploadID = createOutput.getUploadID();
                } catch (TosException e) {
                    Log.e("TosException",  "createMultipartUpload failed");
                    e.printStackTrace();
                }

                // 2. 上传分片，有多种方式可以上传，以下示例分别展示。
                // 假设分片大小统一为 5MB，共上传 5 个分片。
                long partSize = 5 * 1024 * 1024;
                // 已上传分片列表
                List<UploadedPartV2> uploadedParts = new ArrayList<>(5);
                // 第 1-3 个分片通过读取本地文件到 FileInputStream 上传。
                String filePath = "your data file path";
                for (int i = 1; i <= 3; ++i) {
                    try{
                        FileInputStream content = new FileInputStream(filePath);
                        InputStream wrappedContent = new TosRepeatableBoundedFileInputStream(content, partSize);
                        // 每次只上传文件的一部分，需要跳过前面已上传的部分。
                        long skip = (i-1) * partSize;
                        content.skip(skip);
                        UploadPartBasicInput basicInput = new UploadPartBasicInput().setBucket(bucketName)
                                .setKey(objectKey).setUploadID(uploadID).setPartNumber(i);
                        UploadPartV2Input input = new UploadPartV2Input().setUploadPartBasicInput(basicInput)
                                .setContentLength(partSize).setContent(wrappedContent);
                        UploadPartV2Output output = tos.uploadPart(input);
                        // 存储已上传分片信息，必须设置 partNumber 和 etag
                        uploadedParts.add(new UploadedPartV2().setPartNumber(i).setEtag(output.getEtag()));
                        Log.i("uploadPart", "uploadPart succeed, partNumber is " + output.getPartNumber() + ", etag is " +
                                output.getEtag() + ", crc64 value is " + output.getHashCrc64ecma());
                    }catch (IOException | TosException e) {
                        Log.e("TosException", "uploadPart failed");
                        e.printStackTrace();
                    }
                }

                // 第 4 个分片通过 uploadPartFromFile 上传。
                try{
                    UploadPartBasicInput basicInput = new UploadPartBasicInput().setBucket(bucketName)
                            .setKey(objectKey).setUploadID(uploadID).setPartNumber(4);
                    // 每次只上传文件的一部分，需要跳过前面已上传的部分。
                    // 之前已上传 3 个分片，此处跳过前 3 个分片数据，从 offset 的位置开始读取文件数据。
                    long offset = 3 * partSize;
                    UploadPartFromFileInput input = new UploadPartFromFileInput().setUploadPartBasicInput(basicInput)
                            .setFilePath(filePath).setPartSize(partSize).setOffset(offset);
                    UploadPartFromFileOutput output = tos.uploadPartFromFile(input);
                    // 存储已上传分片信息，必须设置 partNumber 和 etag
                    uploadedParts.add(new UploadedPartV2().setPartNumber(4).setEtag(output.getUploadPartV2Output().getEtag()));
                    Log.i("createMultipartUpload", "uploadPart succeed, partNumber is " + output.getUploadPartV2Output().getPartNumber() +
                            ", etag is " + output.getUploadPartV2Output().getEtag() + ", crc64 value is " + output.getUploadPartV2Output().getHashCrc64ecma());
                } catch (TosException e) {
                    Log.e("TosException", "uploadPart failed");
                    e.printStackTrace();
                }

                // 第 5 个分片通过在内存中新建 byte 数组上传。
                try{
                    UploadPartBasicInput basicInput = new UploadPartBasicInput().setBucket(bucketName)
                            .setKey(objectKey).setUploadID(uploadID).setPartNumber(5);
                    // byte 数组数据，对于小 size 的数据可以使用。
                    // 大 size 的数据用 byte 数组内存开销较大，而且 int 长度最长只能支持到约 4GB，不建议使用。
                    byte[] data = new byte[(int)partSize];
                    InputStream content = new ByteArrayInputStream(data);
                    UploadPartV2Input input = new UploadPartV2Input().setUploadPartBasicInput(basicInput)
                            .setContentLength(data.length).setContent(content);
                    UploadPartV2Output output = tos.uploadPart(input);
                    // 存储已上传分片信息，必须设置 partNumber 和 etag
                    uploadedParts.add(new UploadedPartV2().setPartNumber(5).setEtag(output.getEtag()));
                    Log.i("uploadPart", "uploadPart succeed, partNumber is " + output.getPartNumber() + ", etag is " +
                                    output.getEtag() + ", crc64 value is " + output.getHashCrc64ecma());
                } catch (TosException e) {
                    Log.e("TosException", "uploadPart failed");
                    e.printStackTrace();
                }

                // 合并已上传的分片
                try{
                    CompleteMultipartUploadV2Input complete = new CompleteMultipartUploadV2Input().setBucket(bucketName)
                            .setKey(objectKey).setUploadID(uploadID).setUploadedParts(uploadedParts);
                    CompleteMultipartUploadV2Output completedOutput = tos.completeMultipartUpload(complete);
                    Log.i("completeMultipartUpload", "completeMultipartUpload succeed, etag is " +  completedOutput.getEtag() +
                            ", crc64 value is " + completedOutput.getHashCrc64ecma() + ", location is " + completedOutput.getLocation());
                }catch (TosException e) {
                    Log.e("TosException", "completeMultipartUpload failed");
                    e.printStackTrace();
                }
            }
        });

        tosThread.start();
    }
}

取消分片上传任务

您可以通过接口 abortMultipartUpload 来取消分片上传上传，当一个分片上传任务被取消后，对应的 uploadID 不能够再用来进行分片上传，已经上传的片也会被删除。
以下代码展示如何取消分片上传。

mport android.os.Bundle;
import android.util.Log;

import androidx.appcompat.app.AppCompatActivity;

import com.volcengine.tos.TOSV2;
import com.volcengine.tos.TOSV2ClientBuilder;
import com.volcengine.tos.TosException;
import com.volcengine.tos.model.object.AbortMultipartUploadInput;

public class AbortMultipartUploadExample extends AppCompatActivity {
    @Override
    protected void onCreate(Bundle savedInstanceState) {
        String endpoint = "your endpoint";
        String region = "your region";
        String accessKey = "your access key";
        String secretKey = "your secret key";
        String securityToken = "your security token";

        String bucketName = "your bucket name";
        String objectKey = "your object key";
        // 指定的需要取消的分片上传任务的uploadID，
        // 需保证该 uploadID 已通过初始化分片上传接口 createMultipartUpload 调用返回，
        // 否则，对于不存在的 uploadID 会抛出 404 not found exception。
        String uploadID = "the specific uploadID";
        super.onCreate(savedInstanceState);
        setContentView(R.layout.activity_display_message);
        TOSV2 tos = new TOSV2ClientBuilder().build(region, endpoint, accessKey, secretKey, securityToken);

        Thread tosThread = new Thread(new Runnable() {
            @Override
            public void run() {
                try{
                    AbortMultipartUploadInput input = new AbortMultipartUploadInput().setBucket(bucketName)
                            .setKey(objectKey).setUploadID(uploadID);
                    tos.abortMultipartUpload(input);
                } catch (TosException e) {
                    if (e.getStatusCode() == 404) {
                        Log.e("TosException", "abortMultipartUpload failed, the specific upload id is not found");
                    } else {
                        Log.e("TosException", "abortMultipartUpload failed");
                    }
                    e.printStackTrace();
                }
            }
        });

        tosThread.start();
    }
}

列举分片上传任务

您可以通过 listMultipartUploads 列举出已通过 createMultipartUpload 接口初始化，但还未合并或终止的分片上传任务。此接口一次调用只返回最多 1000 个分片上传任务，如果有超过 1000 个任务，可循环调用，直至列举完所有任务。

参数说明

参数说明如下（以下参数均为可选参数）。

参数

示例

含义

用法

delimiter

/

对对象名进行分组的字符。通常使用 / 作为分组字符。

new ListMultipartUploadsV2Input().setDelimiter(String delimiter)

encodingType

url

指定对返回的内容进行编码的编码类型，取值说明如下：

  • url：进行 url 编码。

new ListMultipartUploadsV2Input().setEncodingType(String encodingType)

maxUploads

100

返回分片上传任务的最大数量，最大值为 1000，即一次请求最多返回 1000 个分片上传任务。

new ListMultipartUploadsV2Input().setMaxUploads(int maxUploads)

prefix

aaa

用于指定列举返回对象的前缀名称。可以使用此参数对桶中对象进行分组管理（类似文件夹功能）。

new ListMultipartUploadsV2Input().setPrefix(String prefix)

keyMarker

test.txt

与 uploadIDMarker 配套使用。如果未指定 uploadIDMarker，则返回对象名按字典顺序大于 keyMarker 的分片上传任务列表。如果指定了 uploadIDMarker，返回对象名按字典顺序大于 keyMarker 的分片上传任务列表的同时，还返回对象名等于 keyMarker，分片上传任务 ID 大于 uploadIDMarker 的分片上传任务。

new ListMultipartUploadsV2Input().setKeyMarker(String keyMarker)

uploadIDMarker

f93f6fc9da94371f321e1008

与参数 keyMarker 配套使用。如果未指定 keyMarker，忽略此参数。如果指定了 keyMarker，返回对象名按字典顺序大于 keyMarker 的分片上传任务列表的同时，还返回对象名等于 keyMarker，分片上传任务 ID 大于 uploadIDMarker 的分片上传任务。

new ListMultipartUploadsV2Input().setUploadIDMarker(String uploadIDMarker)

示例代码

以下代码展示如何列举分片任务。

import android.os.Bundle;
import android.util.Log;

import androidx.appcompat.app.AppCompatActivity;

import com.volcengine.tos.TOSV2;
import com.volcengine.tos.TOSV2ClientBuilder;
import com.volcengine.tos.TosException;
import com.volcengine.tos.model.object.ListMultipartUploadsV2Input;
import com.volcengine.tos.model.object.ListMultipartUploadsV2Output;
import com.volcengine.tos.model.object.ListedUpload;

public class ListMultipartUploadsExample extends AppCompatActivity {
    @Override
    protected void onCreate(Bundle savedInstanceState) {
        String endpoint = "your endpoint";
        String region = "your region";
        String accessKey = "your access key";
        String secretKey = "your secret key";
        String securityToken = "your security token";

        String bucketName = "your bucket name";
        super.onCreate(savedInstanceState);
        setContentView(R.layout.activity_display_message);
        TOSV2 tos = new TOSV2ClientBuilder().build(region, endpoint, accessKey, secretKey, securityToken);

        Thread tosThread = new Thread(new Runnable() {
            @Override
            public void run() {
                try{
                    ListMultipartUploadsV2Input input = new ListMultipartUploadsV2Input()
                            // 必须设置 bucket name
                            .setBucket(bucketName).setMaxUploads(10);
                    ListMultipartUploadsV2Output output = tos.listMultipartUploads(input);
                    Log.i("listMultipartUploads", "listMultipartUploads succeed, is truncated? " + output.isTruncated() + 
                            ", next uploadIDMarker is " + output.getNextUploadIdMarker() + ", next keyMarker is " + output.getNextKeyMarker());
                    if (output.getUploads() != null) {
                        for (int i = 0; i < output.getUploads().size(); i++) {
                            ListedUpload upload = output.getUploads().get(i);
                            Log.i("listMultipartUploads", "No." + (i + 1) + " listed upload is " + upload);
                        }
                    }
                } catch (TosException e) {
                    Log.e("TosException", "listMultipartUploads failed");
                    e.printStackTrace();
                }
            }
        });

        tosThread.start();
    }
}

以下代码展示如何分页列举进行中的所有分片任务。

import android.os.Bundle;
import android.util.Log;

import androidx.appcompat.app.AppCompatActivity;

import com.volcengine.tos.TOSV2;
import com.volcengine.tos.TOSV2ClientBuilder;
import com.volcengine.tos.TosException;
import com.volcengine.tos.model.object.ListMultipartUploadsV2Input;
import com.volcengine.tos.model.object.ListMultipartUploadsV2Output;
import com.volcengine.tos.model.object.ListedUpload;

public class ListMultipartUploadsLoopExample extends AppCompatActivity {
    @Override
    protected void onCreate(Bundle savedInstanceState) {
        String endpoint = "your endpoint";
        String region = "your region";
        String accessKey = "your access key";
        String secretKey = "your secret key";
        String securityToken = "your security token";

        String bucketName = "your bucket name";
        super.onCreate(savedInstanceState);
        setContentView(R.layout.activity_display_message);
        TOSV2 tos = new TOSV2ClientBuilder().build(region, endpoint, accessKey, secretKey, securityToken);

        Thread tosThread = new Thread(new Runnable() {
            @Override
            public void run() {
                try{
                    boolean isTruncated = true;
                    String keyMarker = "";
                    String uploadIDMarker = "";
                    int total = 0;
                    while (isTruncated) {
                        ListMultipartUploadsV2Input input = new ListMultipartUploadsV2Input()
                                // 必须设置 bucket name
                                .setBucket(bucketName).setMaxUploads(1000)
                                .setKeyMarker(keyMarker).setUploadIDMarker(uploadIDMarker);
                        ListMultipartUploadsV2Output output = tos.listMultipartUploads(input);
                        Log.i("listMultipartUploads", "listMultipartUploads succeed, is truncated? " + output.isTruncated() +
                                ", next uploadIDMarker is " + output.getNextUploadIdMarker() + ", next keyMarker is " + output.getNextKeyMarker());
                        if (output.getUploads() != null) {
                            for (int i = 0; i < output.getUploads().size(); i++) {
                                ListedUpload upload = output.getUploads().get(i);
                                Log.i("listMultipartUploads", "No." + (i+1+total) + " listed upload is " + upload);
                            }
                            total += output.getUploads().size();
                        }
                        isTruncated = output.isTruncated();
                        keyMarker = output.getNextKeyMarker();
                        uploadIDMarker = output.getNextUploadIdMarker();
                    }

                } catch (TosException e) {
                    Log.e("TosException","listMultipartUploads failed");
                    e.printStackTrace();
                }
            }
        });

        tosThread.start();
    }
}

列举已上传的分片

您可以通过 listParts 接口列举指定 uploadID 下已上传的分片。此接口一次调用只返回最多 1000 个已上传的分片，如果有超过 1000 个分片，可循环调用，直至列举完所有分片。
以下代码展示如何简单列举已上传的分片。

import android.os.Bundle;
import android.util.Log;

import androidx.appcompat.app.AppCompatActivity;

import com.volcengine.tos.TOSV2;
import com.volcengine.tos.TOSV2ClientBuilder;
import com.volcengine.tos.TosException;
import com.volcengine.tos.model.object.ListPartsInput;
import com.volcengine.tos.model.object.ListPartsOutput;
import com.volcengine.tos.model.object.UploadedPartV2;

public class ListPartsExample extends AppCompatActivity {
    @Override
    protected void onCreate(Bundle savedInstanceState) {
        String endpoint = "your endpoint";
        String region = "your region";
        String accessKey = "your access key";
        String secretKey = "your secret key";
        String securityToken = "your security token";

        String bucketName = "your bucket name";
        // 指定的需要列举的分片上传任务的uploadID，
        // 需保证该 uploadID 已通过初始化分片上传接口 createMultipartUpload 调用返回，
        // 否则，对于不存在的 uploadID 会抛出 404 not found exception。
        String uploadID = "the specific uploadID";
        // 与 uploadID 对应的对象 key
        String objectKey = "your object key";
        super.onCreate(savedInstanceState);
        setContentView(R.layout.activity_display_message);
        TOSV2 tos = new TOSV2ClientBuilder().build(region, endpoint, accessKey, secretKey, securityToken);

        Thread tosThread = new Thread(new Runnable() {
            @Override
            public void run() {
                try{
                    ListPartsInput input = new ListPartsInput()
                            // 必须设置 bucket, key, uploadID
                            .setBucket(bucketName).setKey(objectKey).setUploadID(uploadID);
                    ListPartsOutput output = tos.listParts(input);
                    Log.i("listParts", "listParts succeed, is truncated? " + output.isTruncated() +
                            ", next partNumber marker is " + output.getNextPartNumberMarker());
                    if (output.getUploadedParts() != null) {
                        for (int i = 0; i < output.getUploadedParts().size(); i++) {
                            UploadedPartV2 upload = output.getUploadedParts().get(i);
                            Log.i("listParts", "No." + (i + 1) + " uploaded part is " + upload);
                        }
                    }
                } catch (TosException e) {
                    Log.e("TosException", "listParts failed");
                    e.printStackTrace();
                }
            }
        });

        tosThread.start();
    }
}

以下代码展示如何分页列举所有已上传的分片。

import android.os.Bundle;
import android.util.Log;

import androidx.appcompat.app.AppCompatActivity;

import com.volcengine.tos.TOSV2;
import com.volcengine.tos.TOSV2ClientBuilder;
import com.volcengine.tos.TosException;
import com.volcengine.tos.model.object.ListPartsInput;
import com.volcengine.tos.model.object.ListPartsOutput;
import com.volcengine.tos.model.object.UploadedPartV2;

public class ListPartsLoopExample extends AppCompatActivity {
    @Override
    protected void onCreate(Bundle savedInstanceState) {
        String endpoint = "your endpoint";
        String region = "your region";
        String accessKey = "your access key";
        String secretKey = "your secret key";
        String securityToken = "your security token";

        String bucketName = "your bucket name";
        // 指定的需要列举的分片上传任务的uploadID，
        // 需保证该 uploadID 已通过初始化分片上传接口 createMultipartUpload 调用返回，
        // 否则，对于不存在的 uploadID 会抛出 404 not found exception。
        String uploadID = "the specific uploadID";
        // 与 uploadID 对应的对象 key
        String objectKey = "your object key";
        super.onCreate(savedInstanceState);
        setContentView(R.layout.activity_display_message);
        TOSV2 tos = new TOSV2ClientBuilder().build(region, endpoint, accessKey, secretKey, securityToken);

        Thread tosThread = new Thread(new Runnable() {
            @Override
            public void run() {
                try{
                    boolean isTruncated = true;
                    int partNumberMarker = 0;
                    int total = 0;
                    while(isTruncated) {
                        ListPartsInput input = new ListPartsInput().setBucket(bucketName).setKey(objectKey)
                                // 必须设置 bucket, key, uploadID
                                .setUploadID(uploadID).setPartNumberMarker(partNumberMarker);
                        ListPartsOutput output = tos.listParts(input);
                        Log.i("listParts", "listParts succeed, is truncated? " + output.isTruncated() +
                                ", next partNumber marker is " + output.getNextPartNumberMarker());
                        if (output.getUploadedParts() != null) {
                            for (int i = 0; i < output.getUploadedParts().size(); i++) {
                                UploadedPartV2 upload = output.getUploadedParts().get(i);
                                Log.i("listParts", "No." + (i+1+total) + " uploaded part is " + upload);
                            }
                            total += output.getUploadedParts().size();
                        }
                        isTruncated = output.isTruncated();
                        partNumberMarker = output.getNextPartNumberMarker();
                    }
                } catch (TosException e) {
                    Log.e("TosException", "listParts failed");
                    e.printStackTrace();
                }
            }
        });

        tosThread.start();
    }
}

相关文档