You need to enable JavaScript to run this app.
导航

校验上传对象的一致性

最近更新时间2023.08.04 11:53:35

首次发布时间2022.09.08 14:50:56

您可以利用Content-MD5、Content-SHA256或CRC64实现上传对象的一致性校验。本文介绍不同方案的校验流程图及示例代码。

背景信息

通过Content-MD5

调用TOS API接口上传对象或上传分片时,将客户端计算出的待上传数据的Content-MD5值,通过请求头域或表单域传递给TOS,从而保证上传数据的一致性。

流程图

示例代码

package main

import (
   "bytes"
   "context"
   "crypto/md5"
   "encoding/base64"

   "github.com/volcengine/ve-tos-golang-sdk/v2/tos"
)

func main() {
   var (
      accessKey = "your access key"
      secretKey = "your secret key"
      endpoint  = "your endpoint"
      region    = "your region"
      bucket    = "bucketname"
      key       = "objectname"
   )
   client, err := tos.NewClientV2(endpoint, tos.WithRegion(region), tos.WithCredentials(tos.NewStaticCredentials(accessKey, secretKey)))
   if err != nil {
      panic(err)
   }

   data := []byte("hello world")
   // 计算 MD5
   hash := md5.New()
   hash.Write(data)
   contentMD5 := base64.StdEncoding.EncodeToString(hash.Sum(nil))

   // 指定 Content-MD5 上传对象
   input := &tos.PutObjectV2Input{}
   input.Bucket = bucket
   input.Key = key
   input.Content = bytes.NewReader(data)
   input.ContentMD5 = contentMD5
   _, err = client.PutObjectV2(context.Background(), input)
   if err != nil {
      panic(err)
   }
}

通过Content-SHA256

和利用Content-MD5校验的方式相同,调用TOS API接口上传对象或上传分片时,将客户端计算出的待上传数据的Content-SHA256值,通过请求头域或表单域传递给TOS,从而保证上传数据的一致性。

流程图

示例代码

package main

import (
   "bytes"
   "context"
   "crypto/sha256"
   "encoding/hex"

   "github.com/volcengine/ve-tos-golang-sdk/v2/tos"
)


func main() {
   var (
      accessKey = "your access key"
      secretKey = "your secret key"
      endpoint  = "your endpoint"
      region    = "your region"
      bucket    = "bucketname"
      key       = "objectname"
   )
   client, err := tos.NewClientV2(endpoint, tos.WithRegion(region), tos.WithCredentials(tos.NewStaticCredentials(accessKey, secretKey)))
   if err != nil {
      panic(err)
   }

   data := []byte("hello world")
   // 计算 SHA256
   hash := sha256.New()
   hash.Write(data)
   contentSHA256 := hex.EncodeToString(hash.Sum(nil))

   // 指定 Content-MD5 上传对象
   input := &tos.PutObjectV2Input{}
   input.Bucket = bucket
   input.Key = key
   input.Content = bytes.NewReader(data)
   input.ContentSHA256 = contentSHA256
   _, err = client.PutObjectV2(context.Background(), input)
   if err != nil {
      panic(err)
   }
}

通过CRC64

调用TOS API接口上传对象或上传分片完成后,将客户端计算出的已上传数据的CRC64,与TOS返回的CRC64做比较,从而保证单次API上传数据的一致性。
在分片上传场景中,除了验证单次上传的CRC64外,在合并分片后,通过组合各个分片的CRC64再与服务端返回的整个对象的CRC64做比对,进而保证最终TOS侧生成对象的数据一致性。

流程图

示例代码

package main

import (
   "bytes"
   "context"
   "hash"
   "io"

   "github.com/volcengine/ve-tos-golang-sdk/v2/tos"
)

type crc64Reader struct {
   r io.Reader
   h hash.Hash64
}

func (cr *crc64Reader) Read(p []byte) (n int, err error) {
   n, err = cr.r.Read(p)
   cr.h.Write(p[:n])
   return n, err
}

func main() {
   var (
      accessKey = "your access key"
      secretKey = "your secret key"
      endpoint  = "your endpoint"
      region    = "your region"
      bucket    = "bucketname"
      key       = "objectname"
   )
   client, err := tos.NewClientV2(endpoint, tos.WithRegion(region), tos.WithCredentials(tos.NewStaticCredentials(accessKey, secretKey)))
   if err != nil {
      panic(err)
   }

   resp, err := client.CreateMultipartUploadV2(context.Background(), &tos.CreateMultipartUploadV2Input{
      Bucket: bucket,
      Key:    key,
   })
   if err != nil {
      panic(err)
   }

   uploadID := resp.UploadID
   parts := make([]tos.UploadedPartV2, 0, 3)

   var crc64Final uint64
   data := make([]byte, 1024*1024*5)
   for i := 1; i <= 2; i++ {
      // 指定 Content-MD5 上传对象
      input := &tos.UploadPartV2Input{}
      input.Bucket = bucket
      input.Key = key
      input.UploadID = uploadID
      input.PartNumber = i

      crc64Hash := tos.NewCRC(tos.DefaultCrcTable(), 0)
      input.Content = &crc64Reader{
         r: bytes.NewReader(data),
         h: crc64Hash,
      }
      output, err := client.UploadPartV2(context.Background(), input)
      if err != nil {
         panic(err)
      }

      crc64Once := crc64Hash.Sum64()
      if output.HashCrc64ecma != crc64Once {
         panic("crc64 not match")
      }

      crc64Final = tos.CRC64Combine(crc64Final, crc64Once, uint64(len(data)))

      parts = append(parts, tos.UploadedPartV2{
         PartNumber: input.PartNumber,
         ETag:       output.ETag,
      })
   }

   output, err := client.CompleteMultipartUploadV2(context.Background(), &tos.CompleteMultipartUploadV2Input{
      Bucket:   bucket,
      Key:      key,
      UploadID: uploadID,
      Parts:    parts,
   })

   if err != nil {
      panic(err)
   }

   if output.HashCrc64ecma != crc64Final {
      panic("crc64 not match")
   }
}