文档中心

校验上传对象的一致性

最近更新时间：2024.03.19 14:30:45

首次发布时间：2022.09.08 14:50:56

您可以利用Content-MD5、Content-SHA256或CRC64实现上传对象的一致性校验。本文介绍不同方案的校验流程图及示例代码。

背景信息

不同方案的校验说明，请参见相关概念。
不同方案的差异说明，请参见方案对比。
下载对象的一致性校验方案，请参见校验下载对象的一致性。

通过Content-MD5

调用TOS API接口上传对象或上传分片时，将客户端计算出的待上传数据的Content-MD5值，通过请求头域或表单域传递给TOS，从而保证上传数据的一致性。

流程图

示例代码

Golang

Java

Python

package main

import (
   "bytes"
   "context"
   "crypto/md5"
   "encoding/base64"

   "github.com/volcengine/ve-tos-golang-sdk/v2/tos"
)

func main() {
   var (
      accessKey = "your access key"
      secretKey = "your secret key"
      endpoint  = "your endpoint"
      region    = "your region"
      bucket    = "bucketname"
      key       = "objectname"
   )
   client, err := tos.NewClientV2(endpoint, tos.WithRegion(region), tos.WithCredentials(tos.NewStaticCredentials(accessKey, secretKey)))
   if err != nil {
      panic(err)
   }

   data := []byte("hello world")
   // 计算 MD5
   hash := md5.New()
   hash.Write(data)
   contentMD5 := base64.StdEncoding.EncodeToString(hash.Sum(nil))

   // 指定 Content-MD5 上传对象
   input := &tos.PutObjectV2Input{}
   input.Bucket = bucket
   input.Key = key
   input.Content = bytes.NewReader(data)
   input.ContentMD5 = contentMD5
   _, err = client.PutObjectV2(context.Background(), input)
   if err != nil {
      panic(err)
   }
}

package com.volcengine.tos;

import com.volcengine.tos.auth.Credentials;
import com.volcengine.tos.auth.SignV4;
import com.volcengine.tos.auth.StaticCredentials;

import java.io.ByteArrayInputStream;
import java.io.InputStream;
import java.security.MessageDigest;
import java.util.Base64;

public class Main {
    public static void main(String[] args) {
        String endpoint = "your endpoint";
        String accessKey = "your access key";
        String secretKey = "your secret key";
        String region = "your region";
        String bucket = "bucketname";
        String key = "objectname";

        TOSClient client = new TOSClient(endpoint);
        Credentials c = new StaticCredentials(accessKey, secretKey);
        client.setCredentials(c);
        client.setSigner(new SignV4(c, region));
 
        String data = "hello world";

        try {
            // 计算 MD5
            MessageDigest md5 = MessageDigest.getInstance("MD5");
            md5.update(data.getBytes());
            String contentMD5 = Base64.getEncoder().encodeToString(md5.digest());

            InputStream stream = new ByteArrayInputStream(data.getBytes());
            client.putObject(bucket, key, stream, RequestOptions.withContentMD5(contentMD5));
        } catch (Exception e) {
            e.printStackTrace();
        }
    }
}

import base64
import hashlib

import tos

endpoint = 'your endpoint'
access_key = 'your access key'
secret_key = 'your secret key'
region = 'your region'

bucket = 'bucketname'
key = 'objectname'
data = 'hello world'

# 计算 MD5
hash = hashlib.md5()
hash.update(data.encode())
content_md5 = base64.b64encode(hash.digest()).decode()

client = tos.TosClientV2(access_key, secret_key, endpoint, region)
try:
    resp = client.put_object(bucket, key, content=data, content_md5=content_md5)
    print(resp.status_code)
except Exception as e:
    print(e)

通过Content-SHA256

和利用Content-MD5校验的方式相同，调用TOS API接口上传对象或上传分片时，将客户端计算出的待上传数据的Content-SHA256值，通过请求头域或表单域传递给TOS，从而保证上传数据的一致性。

流程图

示例代码

Golang

Java

Python

package main

import (
   "bytes"
   "context"
   "crypto/sha256"
   "encoding/hex"

   "github.com/volcengine/ve-tos-golang-sdk/v2/tos"
)


func main() {
   var (
      accessKey = "your access key"
      secretKey = "your secret key"
      endpoint  = "your endpoint"
      region    = "your region"
      bucket    = "bucketname"
      key       = "objectname"
   )
   client, err := tos.NewClientV2(endpoint, tos.WithRegion(region), tos.WithCredentials(tos.NewStaticCredentials(accessKey, secretKey)))
   if err != nil {
      panic(err)
   }

   data := []byte("hello world")
   // 计算 SHA256
   hash := sha256.New()
   hash.Write(data)
   contentSHA256 := hex.EncodeToString(hash.Sum(nil))

   // 指定 Content-MD5 上传对象
   input := &tos.PutObjectV2Input{}
   input.Bucket = bucket
   input.Key = key
   input.Content = bytes.NewReader(data)
   input.ContentSHA256 = contentSHA256
   _, err = client.PutObjectV2(context.Background(), input)
   if err != nil {
      panic(err)
   }
}

package com.volcengine.tos;

import com.volcengine.tos.auth.Credentials;
import com.volcengine.tos.auth.SignV4;
import com.volcengine.tos.auth.StaticCredentials;
import com.volcengine.tos.model.object.GetObjectOutput;
import org.apache.commons.codec.binary.Hex;

import java.io.ByteArrayInputStream;
import java.io.InputStream;
import java.security.MessageDigest;

public class Main {

    public static void main(String[] args) {
        String endpoint = "your endpoint";
        String accessKey = "your access key";
        String secretKey = "your secret key";
        String region = "your region";
        String bucket = "bucketname";
        String key = "objectname";

        TOSClient client = new TOSClient(endpoint);
        Credentials c = new StaticCredentials(accessKey, secretKey);
        client.setCredentials(c);
        client.setSigner(new SignV4(c, region));

        String data = "hello world";

        try {
            MessageDigest sha256 = MessageDigest.getInstance("SHA-256");
            sha256.update(data.getBytes());
            String contentSHA256 = Hex.encodeHexString(sha256.digest());

            InputStream stream = new ByteArrayInputStream(data.getBytes());
            client.putObject(bucket, key, stream, RequestOptions.withContentSHA256(contentSHA256));

            GetObjectOutput output = client.getObject(bucket, key);
            output.getContent();

        } catch (Exception e) {
            e.printStackTrace();
        }
    }
}

import hashlib

import tos

endpoint = 'your endpoint'
access_key = 'your access key'
secret_key = 'your secret key'
region = 'your region'

bucket = 'bucketname'
key = 'objectname'
data = 'hello world'

# 计算 SHA256
hash = hashlib.sha256()
hash.update(data.encode())
content_sha256 = hash.digest().hex()

client = tos.TosClientV2(access_key, secret_key, endpoint, region)
try:
    resp = client.put_object(bucket, key, content=data, content_sha256=content_sha256)
    print(resp.status_code)
except Exception as e:
    print(e)

通过CRC64

调用TOS API接口上传对象或上传分片完成后，将客户端计算出的已上传数据的CRC64，与TOS返回的CRC64做比较，从而保证单次API上传数据的一致性。
在分片上传场景中，除了验证单次上传的CRC64外，在合并分片后，通过组合各个分片的CRC64再与服务端返回的整个对象的CRC64做比对，进而保证最终TOS侧生成对象的数据一致性。

流程图

示例代码

Golang

Python

package main

import (
   "bytes"
   "context"
   "hash"
   "io"

   "github.com/volcengine/ve-tos-golang-sdk/v2/tos"
)

type crc64Reader struct {
   r io.Reader
   h hash.Hash64
}

func (cr *crc64Reader) Read(p []byte) (n int, err error) {
   n, err = cr.r.Read(p)
   cr.h.Write(p[:n])
   return n, err
}

func main() {
   var (
      accessKey = "your access key"
      secretKey = "your secret key"
      endpoint  = "your endpoint"
      region    = "your region"
      bucket    = "bucketname"
      key       = "objectname"
   )
   client, err := tos.NewClientV2(endpoint, tos.WithRegion(region), tos.WithCredentials(tos.NewStaticCredentials(accessKey, secretKey)))
   if err != nil {
      panic(err)
   }

   resp, err := client.CreateMultipartUploadV2(context.Background(), &tos.CreateMultipartUploadV2Input{
      Bucket: bucket,
      Key:    key,
   })
   if err != nil {
      panic(err)
   }

   uploadID := resp.UploadID
   parts := make([]tos.UploadedPartV2, 0, 3)

   var crc64Final uint64
   data := make([]byte, 1024*1024*5)
   for i := 1; i <= 2; i++ {
      // 指定 Content-MD5 上传对象
      input := &tos.UploadPartV2Input{}
      input.Bucket = bucket
      input.Key = key
      input.UploadID = uploadID
      input.PartNumber = i

      crc64Hash := tos.NewCRC(tos.DefaultCrcTable(), 0)
      input.Content = &crc64Reader{
         r: bytes.NewReader(data),
         h: crc64Hash,
      }
      output, err := client.UploadPartV2(context.Background(), input)
      if err != nil {
         panic(err)
      }

      crc64Once := crc64Hash.Sum64()
      if output.HashCrc64ecma != crc64Once {
         panic("crc64 not match")
      }

      crc64Final = tos.CRC64Combine(crc64Final, crc64Once, uint64(len(data)))

      parts = append(parts, tos.UploadedPartV2{
         PartNumber: input.PartNumber,
         ETag:       output.ETag,
      })
   }

   output, err := client.CompleteMultipartUploadV2(context.Background(), &tos.CompleteMultipartUploadV2Input{
      Bucket:   bucket,
      Key:      key,
      UploadID: uploadID,
      Parts:    parts,
   })

   if err != nil {
      panic(err)
   }

   if output.HashCrc64ecma != crc64Final {
      panic("crc64 not match")
   }
}

from io import BytesIO
import os
import tos

endpoint = 'your endpoint'
access_key = 'your access key'
secret_key = 'your secret key'
region = 'your region'
bucket_name = 'bucketname'
key = 'objectname'


class Crc64IO(object):
    def __init__(self, data):
        self.data = data
        self.crc64 = tos.utils.Crc64()

    def __iter__(self):
        return self

    def __next__(self):
        return self.next()

    def next(self):
        content = self.data.read(8 * 1024)
        if content:
            self.crc64.update(content)
            return content
        raise StopIteration


# 本地文件完整路径，例如usr/local/testfile.txt
file_name = 'usr/local/testfile.txt'
client = tos.TosClientV2(access_key, secret_key, endpoint, region, enable_crc=False)
try:
    total_size = os.path.getsize(file_name)
    part_size = 1024 * 1024 * 5
    resp = client.create_multipart_upload(bucket_name, key)
    upload_id = resp.upload_id
    parts = []
    crc64 = tos.utils.Crc64()
    crc64_final = 0

    # 上传分片数据
    with open(file_name, 'rb') as f:
        part_number = 1
        offset = 0
        while offset < total_size:
            num_to_upload = min(part_size, total_size - offset)
            crc64_io = Crc64IO(SizeAdapter(f, num_to_upload, init_offset=offset))
            part = client.upload_part(bucket_name, key, upload_id, part_number,
                                      content=crc64_io)

            if crc64_io.crc64.crc != part.hash_crc64_ecma:
                raise Exception('crc64 not match')
            crc64_final = crc64.combine(crc64_final, crc64_io.crc64.crc, num_to_upload)
            parts.append(part)
            offset += num_to_upload
            part_number += 1

    resp = client.complete_multipart_upload(bucket_name, key, upload_id, parts)

    if crc64_final != resp.hash_crc64_ecma:
        raise Exception('crc64 not match')

except tos.exceptions.TosClientError as e:
    # 操作失败，捕获客户端异常，一般情况为非法请求参数或网络异常
    print('fail with client error, message:{}, cause: {}'.format(e.message, e.cause))
except tos.exceptions.TosServerError as e:
    # 操作失败，捕获服务端异常，可从返回信息中获取详细错误信息
    print('fail with server error, code: {}'.format(e.code))
    # request id 可定位具体问题，强烈建议日志中保存
    print('error with request id: {}'.format(e.request_id))
    print('error with message: {}'.format(e.message))
    print('error with http code: {}'.format(e.status_code))
    print('error with ec: {}'.format(e.ec))
    print('error with request url: {}'.format(e.request_url))
except Exception as e:
    print('fail with unknown error: {}'.format(e))

背景信息

通过Content-MD5

流程图

示例代码

通过Content-SHA256

流程图

示例代码

通过CRC64

流程图

示例代码

对象存储

校验上传对象的一致性

背景信息

通过Content-MD5

流程图

示例代码

通过Content-SHA256

流程图

示例代码

通过CRC64

流程图

示例代码