您可以利用Content-MD5、Content-SHA256或CRC64实现上传对象的一致性校验。本文介绍不同方案的校验流程图及示例代码。
调用TOS API接口上传对象或上传分片时,将客户端计算出的待上传数据的Content-MD5值,通过请求头域或表单域传递给TOS,从而保证上传数据的一致性。
package main import ( "bytes" "context" "crypto/md5" "encoding/base64" "github.com/volcengine/ve-tos-golang-sdk/v2/tos" ) func main() { var ( accessKey = "your access key" secretKey = "your secret key" endpoint = "your endpoint" region = "your region" bucket = "bucketname" key = "objectname" ) client, err := tos.NewClientV2(endpoint, tos.WithRegion(region), tos.WithCredentials(tos.NewStaticCredentials(accessKey, secretKey))) if err != nil { panic(err) } data := []byte("hello world") // 计算 MD5 hash := md5.New() hash.Write(data) contentMD5 := base64.StdEncoding.EncodeToString(hash.Sum(nil)) // 指定 Content-MD5 上传对象 input := &tos.PutObjectV2Input{} input.Bucket = bucket input.Key = key input.Content = bytes.NewReader(data) input.ContentMD5 = contentMD5 _, err = client.PutObjectV2(context.Background(), input) if err != nil { panic(err) } }
和利用Content-MD5校验的方式相同,调用TOS API接口上传对象或上传分片时,将客户端计算出的待上传数据的Content-SHA256值,通过请求头域或表单域传递给TOS,从而保证上传数据的一致性。
package main import ( "bytes" "context" "crypto/sha256" "encoding/hex" "github.com/volcengine/ve-tos-golang-sdk/v2/tos" ) func main() { var ( accessKey = "your access key" secretKey = "your secret key" endpoint = "your endpoint" region = "your region" bucket = "bucketname" key = "objectname" ) client, err := tos.NewClientV2(endpoint, tos.WithRegion(region), tos.WithCredentials(tos.NewStaticCredentials(accessKey, secretKey))) if err != nil { panic(err) } data := []byte("hello world") // 计算 SHA256 hash := sha256.New() hash.Write(data) contentSHA256 := hex.EncodeToString(hash.Sum(nil)) // 指定 Content-MD5 上传对象 input := &tos.PutObjectV2Input{} input.Bucket = bucket input.Key = key input.Content = bytes.NewReader(data) input.ContentSHA256 = contentSHA256 _, err = client.PutObjectV2(context.Background(), input) if err != nil { panic(err) } }
调用TOS API接口上传对象或上传分片完成后,将客户端计算出的已上传数据的CRC64,与TOS返回的CRC64做比较,从而保证单次API上传数据的一致性。
在分片上传场景中,除了验证单次上传的CRC64外,在合并分片后,通过组合各个分片的CRC64再与服务端返回的整个对象的CRC64做比对,进而保证最终TOS侧生成对象的数据一致性。
package main import ( "bytes" "context" "hash" "io" "github.com/volcengine/ve-tos-golang-sdk/v2/tos" ) type crc64Reader struct { r io.Reader h hash.Hash64 } func (cr *crc64Reader) Read(p []byte) (n int, err error) { n, err = cr.r.Read(p) cr.h.Write(p[:n]) return n, err } func main() { var ( accessKey = "your access key" secretKey = "your secret key" endpoint = "your endpoint" region = "your region" bucket = "bucketname" key = "objectname" ) client, err := tos.NewClientV2(endpoint, tos.WithRegion(region), tos.WithCredentials(tos.NewStaticCredentials(accessKey, secretKey))) if err != nil { panic(err) } resp, err := client.CreateMultipartUploadV2(context.Background(), &tos.CreateMultipartUploadV2Input{ Bucket: bucket, Key: key, }) if err != nil { panic(err) } uploadID := resp.UploadID parts := make([]tos.UploadedPartV2, 0, 3) var crc64Final uint64 data := make([]byte, 1024*1024*5) for i := 1; i <= 2; i++ { // 指定 Content-MD5 上传对象 input := &tos.UploadPartV2Input{} input.Bucket = bucket input.Key = key input.UploadID = uploadID input.PartNumber = i crc64Hash := tos.NewCRC(tos.DefaultCrcTable(), 0) input.Content = &crc64Reader{ r: bytes.NewReader(data), h: crc64Hash, } output, err := client.UploadPartV2(context.Background(), input) if err != nil { panic(err) } crc64Once := crc64Hash.Sum64() if output.HashCrc64ecma != crc64Once { panic("crc64 not match") } crc64Final = tos.CRC64Combine(crc64Final, crc64Once, uint64(len(data))) parts = append(parts, tos.UploadedPartV2{ PartNumber: input.PartNumber, ETag: output.ETag, }) } output, err := client.CompleteMultipartUploadV2(context.Background(), &tos.CompleteMultipartUploadV2Input{ Bucket: bucket, Key: key, UploadID: uploadID, Parts: parts, }) if err != nil { panic(err) } if output.HashCrc64ecma != crc64Final { panic("crc64 not match") } }