对象存储
对象存储
对象存储
文档指南
请输入
产品动态
产品公告
产品计费
快速入门
用户指南
存储桶管理
基础设置
容错与容灾
ObjectSet
文件管理
监控与告警
权限配置
IAM 策略管理
典型配置场景
IAM 子用户授权
使用工具及 SDK 的授权说明
接入点
单区域接入点（SRAP）
内容感知与处理（SenseFlow）
TOS Agent
数据处理
分层命名空间（HNS）
TOS Vectors
API 参考
向量桶
SDK 参考
Python-Vectors
向量桶
向量索引
Java-Vectors
向量桶
向量索引
Node.js-Vectors
向量桶
向量索引
工具指南
tosutil
常用命令
辅助命令
TOS Browser
FSX
TOS PyTorch Connector
tosfsspec
API 参考
签名机制
存储桶
桶基础操作
自定义域名
数据处理
媒体处理
视频转码
音频转码
ObjectSet
生命周期
对象
基础操作
分片上传
SDK 参考
Java
存储桶接口
管理桶配置
ObjectSet
对象接口
下载对象
数据处理
视频处理
日志配置
分层命名空间
Python
数据处理
存储桶接口
对象接口
上传对象
C++
存储桶接口
对象接口
Go
对象接口
存储桶接口
管理桶配置
ObjectSet
数据处理
日志配置
Node.js
存储桶接口
对象接口
上传对象
数据处理
Browser.js
上传对象
.NET
对象接口
PHP
对象接口
异常处理
Android
对象接口
iOS
对象接口
Harmony
存储桶接口
对象接口
上传对象
Rust
存储桶接口
桶基础操作
桶清单
自定义域名
对象接口
基础操作
错误码
01-SERVICE_ERROR
02-AUTH
03-ACCESS_CONTROL
04-QOS
05-REQUEST
06-BUCKET
07-LIFECYCLE
08-ACL
09-CORS
10-POLICY
11-MIRROR
12-REPLICATION
13-WEBSITE
14-NOTIFICATION
15-ENCRYPT
16-CUSTOMDOMAIN
17-OBJECT
18-TAG
19-MULTIPART
20-STORAGE_CLASS
21-CALLBACK
22-DATA_PROCESS
23-BATCH
24-STORAGE_LENS
25-INVENTORY
26-DIRECTORY_BUCKET
32-WORM
常见问题
最佳实践
AIGC-豆包大模型最佳实践
校验数据一致性
- 文档首页
对象存储最佳实践校验数据一致性校验上传对象的一致性
复制全文
下载 pdf校验上传对象的一致性
文档反馈
问问助手通过 Content-MD5 校验
调用 TOS API 上传对象或上传分片时，将客户端计算出的待上传数据的 Content-MD5 值，通过请求头域或表单域传递给 TOS，从而保证上传数据的一致性。
流程图
示例代码
Golang
Java
Python
package main
import (
"bytes"
"context"
"crypto/md5"
"encoding/base64"
"github.com/volcengine/ve-tos-golang-sdk/v2/tos"
)
func main() {
var (
accessKey = "your access key"
secretKey = "your secret key"
endpoint = "your endpoint"
region = "your region"
bucket = "bucketname"
key = "objectname"
)
client, err := tos.NewClientV2(endpoint, tos.WithRegion(region), tos.WithCredentials(tos.NewStaticCredentials(accessKey, secretKey)))
if err != nil {
panic(err)
}
data := []byte("hello world")
// 计算 MD5
hash := md5.New()
hash.Write(data)
contentMD5 := base64.StdEncoding.EncodeToString(hash.Sum(nil))
// 指定 Content-MD5 上传对象
input := &tos.PutObjectV2Input{}
input.Bucket = bucket
input.Key = key
input.Content = bytes.NewReader(data)
input.ContentMD5 = contentMD5
_, err = client.PutObjectV2(context.Background(), input)
if err != nil {
panic(err)
}
}
package com.volcengine.tos;
import com.volcengine.tos.auth.Credentials;
import com.volcengine.tos.auth.SignV4;
import com.volcengine.tos.auth.StaticCredentials;
import java.io.ByteArrayInputStream;
import java.io.InputStream;
import java.security.MessageDigest;
import java.util.Base64;
public class Main {
public static void main(String[] args) {
String endpoint = "your endpoint";
String accessKey = "your access key";
String secretKey = "your secret key";
String region = "your region";
String bucket = "bucketname";
String key = "objectname";
TOSClient client = new TOSClient(endpoint);
Credentials c = new StaticCredentials(accessKey, secretKey);
client.setCredentials(c);
client.setSigner(new SignV4(c, region));
String data = "hello world";
try {
// 计算 MD5
MessageDigest md5 = MessageDigest.getInstance("MD5");
md5.update(data.getBytes());
String contentMD5 = Base64.getEncoder().encodeToString(md5.digest());
InputStream stream = new ByteArrayInputStream(data.getBytes());
client.putObject(bucket, key, stream, RequestOptions.withContentMD5(contentMD5));
} catch (Exception e) {
e.printStackTrace();
}
}
}
import base64
import hashlib
import tos
endpoint = 'your endpoint'
access_key = 'your access key'
secret_key = 'your secret key'
region = 'your region'
bucket = 'bucketname'
key = 'objectname'
data = 'hello world'
# 计算 MD5
hash = hashlib.md5()
hash.update(data.encode())
content_md5 = base64.b64encode(hash.digest()).decode()
client = tos.TosClientV2(access_key, secret_key, endpoint, region)
try:
resp = client.put_object(bucket, key, content=data, content_md5=content_md5)
print(resp.status_code)
except Exception as e:
print(e)
通过 Content-SHA256 校验
和使用 Content-MD5 校验的方式相同，调用 TOS API 上传对象或上传分片时，将客户端计算出的待上传数据的 Content-SHA256 值，通过请求头域或表单域传递给 TOS，从而保证上传数据的一致性。
流程图
示例代码
Golang
Java
Python
package main
import (
"bytes"
"context"
"crypto/sha256"
"encoding/hex"
"github.com/volcengine/ve-tos-golang-sdk/v2/tos"
)
func main() {
var (
accessKey = "your access key"
secretKey = "your secret key"
endpoint = "your endpoint"
region = "your region"
bucket = "bucketname"
key = "objectname"
)
client, err := tos.NewClientV2(endpoint, tos.WithRegion(region), tos.WithCredentials(tos.NewStaticCredentials(accessKey, secretKey)))
if err != nil {
panic(err)
}
data := []byte("hello world")
// 计算 SHA256
hash := sha256.New()
hash.Write(data)
contentSHA256 := hex.EncodeToString(hash.Sum(nil))
// 指定 Content-SHA256 上传对象
input := &tos.PutObjectV2Input{}
input.Bucket = bucket
input.Key = key
input.Content = bytes.NewReader(data)
input.ContentSHA256 = contentSHA256
_, err = client.PutObjectV2(context.Background(), input)
if err != nil {
panic(err)
}
}
package com.volcengine.tos;
import com.volcengine.tos.auth.Credentials;
import com.volcengine.tos.auth.SignV4;
import com.volcengine.tos.auth.StaticCredentials;
import com.volcengine.tos.model.object.GetObjectOutput;
import org.apache.commons.codec.binary.Hex;
import java.io.ByteArrayInputStream;
import java.io.InputStream;
import java.security.MessageDigest;
public class Main {
public static void main(String[] args) {
String endpoint = "your endpoint";
String accessKey = "your access key";
String secretKey = "your secret key";
String region = "your region";
String bucket = "bucketname";
String key = "objectname";
TOSClient client = new TOSClient(endpoint);
Credentials c = new StaticCredentials(accessKey, secretKey);
client.setCredentials(c);
client.setSigner(new SignV4(c, region));
String data = "hello world";
try {
MessageDigest sha256 = MessageDigest.getInstance("SHA-256");
sha256.update(data.getBytes());
String contentSHA256 = Hex.encodeHexString(sha256.digest());
InputStream stream = new ByteArrayInputStream(data.getBytes());
client.putObject(bucket, key, stream, RequestOptions.withContentSHA256(contentSHA256));
GetObjectOutput output = client.getObject(bucket, key);
output.getContent();
} catch (Exception e) {
e.printStackTrace();
}
}
}
import hashlib
import tos
endpoint = 'your endpoint'
access_key = 'your access key'
secret_key = 'your secret key'
region = 'your region'
bucket = 'bucketname'
key = 'objectname'
data = 'hello world'
# 计算 SHA256
hash = hashlib.sha256()
hash.update(data.encode())
content_sha256 = hash.digest().hex()
client = tos.TosClientV2(access_key, secret_key, endpoint, region)
try:
resp = client.put_object(bucket, key, content=data, content_sha256=content_sha256)
print(resp.status_code)
except Exception as e:
print(e)
通过 CRC64 校验
调用 TOS API 上传对象或上传分片完成后，将客户端计算出的已上传数据的 CRC64，与 TOS 返回的 CRC64 做比较，从而保证单次 API 上传数据的一致性。
在分片上传场景中，除了验证单次上传的 CRC64 外，在合并分片后，通过组合各个分片的 CRC64 再与服务端返回的整个对象的 CRC64 做比对，进而保证最终 TOS 侧生成对象的数据一致性。
流程图
示例代码
Golang
Python
package main
import (
"bytes"
"context"
"hash"
"io"
"github.com/volcengine/ve-tos-golang-sdk/v2/tos"
)
type crc64Reader struct {
r io.Reader
h hash.Hash64
}
func (cr *crc64Reader) Read(p []byte) (n int, err error) {
n, err = cr.r.Read(p)
cr.h.Write(p[:n])
return n, err
}
func main() {
var (
accessKey = "your access key"
secretKey = "your secret key"
endpoint = "your endpoint"
region = "your region"
bucket = "bucketname"
key = "objectname"
)
client, err := tos.NewClientV2(endpoint, tos.WithRegion(region), tos.WithCredentials(tos.NewStaticCredentials(accessKey, secretKey)))
if err != nil {
panic(err)
}
resp, err := client.CreateMultipartUploadV2(context.Background(), &tos.CreateMultipartUploadV2Input{
Bucket: bucket,
Key: key,
})
if err != nil {
panic(err)
}
uploadID := resp.UploadID
parts := make([]tos.UploadedPartV2, 0, 3)
var crc64Final uint64
data := make([]byte, 1024*1024*5)
for i := 1; i <= 2; i++ {
input := &tos.UploadPartV2Input{}
input.Bucket = bucket
input.Key = key
input.UploadID = uploadID
input.PartNumber = i
crc64Hash := tos.NewCRC(tos.DefaultCrcTable(), 0)
input.Content = &crc64Reader{
r: bytes.NewReader(data),
h: crc64Hash,
}
output, err := client.UploadPartV2(context.Background(), input)
if err != nil {
panic(err)
}
crc64Once := crc64Hash.Sum64()
if output.HashCrc64ecma != crc64Once {
panic("crc64 not match")
}
crc64Final = tos.CRC64Combine(crc64Final, crc64Once, uint64(len(data)))
parts = append(parts, tos.UploadedPartV2{
PartNumber: input.PartNumber,
ETag: output.ETag,
})
}
output, err := client.CompleteMultipartUploadV2(context.Background(), &tos.CompleteMultipartUploadV2Input{
Bucket: bucket,
Key: key,
UploadID: uploadID,
Parts: parts,
})
if err != nil {
panic(err)
}
if output.HashCrc64ecma != crc64Final {
panic("crc64 not match")
}
}
from io import BytesIO
import os
import tos
endpoint = 'your endpoint'
access_key = 'your access key'
secret_key = 'your secret key'
region = 'your region'
bucket_name = 'bucketname'
key = 'objectname'
class Crc64IO(object):
def __init__(self, data):
self.data = data
self.crc64 = tos.utils.Crc64()
def __iter__(self):
return self
def __next__(self):
return self.next()
def next(self):
content = self.data.read(8 * 1024)
if content:
self.crc64.update(content)
return content
raise StopIteration
# 本地文件完整路径，例如usr/local/testfile.txt
file_name = 'usr/local/testfile.txt'
client = tos.TosClientV2(access_key, secret_key, endpoint, region, enable_crc=False)
try:
total_size = os.path.getsize(file_name)
part_size = 1024 * 1024 * 5
resp = client.create_multipart_upload(bucket_name, key)
upload_id = resp.upload_id
parts = []
crc64 = tos.utils.Crc64()
crc64_final = 0
# 上传分片数据
with open(file_name, 'rb') as f:
part_number = 1
offset = 0
while offset < total_size:
num_to_upload = min(part_size, total_size - offset)
crc64_io = Crc64IO(SizeAdapter(f, num_to_upload, init_offset=offset))
part = client.upload_part(bucket_name, key, upload_id, part_number,
content=crc64_io)
if crc64_io.crc64.crc != part.hash_crc64_ecma:
raise Exception('crc64 not match')
crc64_final = crc64.combine(crc64_final, crc64_io.crc64.crc, num_to_upload)
parts.append(part)
offset += num_to_upload
part_number += 1
resp = client.complete_multipart_upload(bucket_name, key, upload_id, parts)
if crc64_final != resp.hash_crc64_ecma:
raise Exception('crc64 not match')
except tos.exceptions.TosClientError as e:
# 操作失败，捕获客户端异常，一般情况为非法请求参数或网络异常
print('fail with client error, message:{}, cause: {}'.format(e.message, e.cause))
except tos.exceptions.TosServerError as e:
# 操作失败，捕获服务端异常，可从返回信息中获取详细错误信息
print('fail with server error, code: {}'.format(e.code))
# request id 可定位具体问题，强烈建议日志中保存
print('error with request id: {}'.format(e.request_id))
print('error with message: {}'.format(e.message))
print('error with http code: {}'.format(e.status_code))
print('error with ec: {}'.format(e.ec))
print('error with request url: {}'.format(e.request_url))
except Exception as e:
print('fail with unknown error: {}'.format(e))
方案对比
校验方式 | 通过Content-MD5 | 通过Content-SHA256 | 通过CRC64 |
校验方式 | 通过Content-MD5 | 通过Content-SHA256 | 通过CRC64 |
适用场景 |
|
|
|
缺点 |
|
| 校验过程在客户端完成，此时 TOS 侧已生成实际上传的对象，需要额外的做覆盖或删除等清理工作。 |
支持的 API |
| ||
推荐说明 |
| | |
最近更新时间：2026.10.10 16:36:50
这个页面对您有帮助吗？
有用
有用
无用
无用