Torch Object Storage
Torch Object Storage
Document
- Documentation
Torch Object StorageSDK ReferenceGoObject InterfaceUpload ObjectMultipart Upload (Go SDK)
Copy page
Download PDFUpload Object
Multipart Upload (Go SDK)
Copy pageDownload PDFMultipart Upload (Go SDK)
When uploading a large object, it can be divided into multiple data blocks (parts) for separate upload. Finally, the merge operation is invoked to combine the uploaded parts into a single object.
Precautions
- Before initiating multipart upload, you must have the tos:PutObject permission. For specific instructions, please refer to the Permission configuration guide.
- Before canceling a multipart upload, you must have the tos:AbortMultipartUpload permission. For detailed instructions, please refer to the Permission configuration guide.
- Part numbers start from 1 and have a maximum value of 10,000. Except for the last part, the minimum size of all other parts is 4 MiB.
- When uploading an object, the object name must comply with specific requirements. For more information, see Object naming rules.
- TOS is a distributed object storage product designed for large-scale storage. Its internal partitions store object index data. To maximize the horizontal scalability of throughput when uploading and downloading objects, and to reduce the likelihood of hotspot partitions, please avoid naming objects in lexicographically increasing order. For more information, see Performance optimization.
- If an object with the same name already exists in the bucket, the new object will overwrite the existing one. If versioning is enabled for your bucket, the original object will be retained, and a new version number will be generated to identify the newly uploaded object.
Steps
Multipart upload generally consists of the following three steps:
- Initialize a multipart upload: Call the CreateMultipartUploadV2 method to obtain the globally unique UploadID generated by TOS.
- Upload the parts: Call the UploadPartV2 method to upload part data.
tip
- For the same multipart upload (identified by UploadID), the part number (PartNumber) indicates the relative position of the part within the entire object. If data is uploaded multiple times using the same part number, TOS will overwrite the original data and retain only the most recently uploaded version.
- The response header contains the MD5 value of the data, which can be retrieved via the Etag. When merging parts, you must specify all part information (part number and ETag value) from the current multipart upload.
- Complete the multipart upload: After all parts have been uploaded, call the CompleteMultipartUploadV2 method to merge all parts into a single object.
Sample code
Complete multipart upload process
The following code demonstrates the complete process of uploading a local file using multipart upload, with the ACL set to Private, the storage type specified as Infrequent Access, and custom metadata added — all specified during the upload process.
package main
import (
"context"
"fmt"
"io"
"os"
"github.com/volcengine/ve-tos-golang-sdk/v2/tos"
"github.com/volcengine/ve-tos-golang-sdk/v2/tos/enum"
)
func checkErr(err error) {
if err != nil {
if serverErr, ok := err.(*tos.TosServerError); ok {
fmt.Println("Error:", serverErr.Error())
fmt.Println("Request ID:", serverErr.RequestID)
fmt.Println("Response Status Code:", serverErr.StatusCode)
fmt.Println("Response Header:", serverErr.Header)
fmt.Println("Response Err Code:", serverErr.Code)
fmt.Println("Response Err Msg:", serverErr.Message)
} else if clientErr, ok := err.(*tos.TosClientError); ok {
fmt.Println("Error:", clientErr.Error())
fmt.Println("Client Cause Err:", clientErr.Cause.Error())
} else {
fmt.Println("Error:", err)
}
panic(err)
}
}
func main() {
var (
accessKey = os.Getenv("TOS_ACCESS_KEY")
secretKey = os.Getenv("TOS_SECRET_KEY")
endpoint = "your endpoint"
region = "your region"
bucketName = "*** Provide your bucket name ***"
objectKey = "*** Provide your object name ***"
ctx = context.Background()
)
client, err := tos.NewClientV2(endpoint, tos.WithRegion(region), tos.WithCredentials(tos.NewStaticCredentials(accessKey, secretKey)))
checkErr(err)
createMultipartOutput, err := client.CreateMultipartUploadV2(ctx, &tos.CreateMultipartUploadV2Input{
Bucket: bucketName,
Key: objectKey,
ACL: enum.ACLPrivate,
StorageClass: enum.StorageClassIa,
Meta: map[string]string{"key": "value"},
})
checkErr(err)
fmt.Println("CreateMultipartUploadV2 Request ID:", createMultipartOutput.RequestID)
fmt.Println("CreateMultipartUploadV2 Upload ID:", createMultipartOutput.UploadID)
localFile := "/root/example.txt"
fd, err := os.Open(localFile)
checkErr(err)
defer fd.Close()
stat, err := os.Stat(localFile)
checkErr(err)
fileSize := stat.Size()
partNumber := 1
partSize := int64(20 * 1024 * 1024)
offset := int64(0)
var parts []tos.UploadedPartV2
for offset < fileSize {
uploadSize := partSize
if fileSize-offset < partSize {
uploadSize = fileSize - offset
}
fd.Seek(offset, io.SeekStart)
partOutput, err := client.UploadPartV2(ctx, &tos.UploadPartV2Input{
UploadPartBasicInput: tos.UploadPartBasicInput{
Bucket: bucketName,
Key: objectKey,
UploadID: createMultipartOutput.UploadID,
PartNumber: partNumber,
},
Content: io.LimitReader(fd, uploadSize),
ContentLength: uploadSize,
})
checkErr(err)
fmt.Println("upload Request ID:", partOutput.RequestID)
parts = append(parts, tos.UploadedPartV2{PartNumber: partNumber, ETag: partOutput.ETag})
offset += uploadSize
partNumber++
}
completeOutput, err := client.CompleteMultipartUploadV2(ctx, &tos.CompleteMultipartUploadV2Input{
Bucket: bucketName,
Key: objectKey,
UploadID: createMultipartOutput.UploadID,
Parts: parts,
})
checkErr(err)
fmt.Println("CompleteMultipartUploadV2 Request ID:", completeOutput.RequestID)
}
List uploaded parts
The following code is used to list information about the uploaded parts of a specified object within a specified bucket.
package main
import (
"context"
"fmt"
"github.com/volcengine/ve-tos-golang-sdk/v2/tos"
)
func checkErr(err error) {
if err != nil {
if serverErr, ok := err.(*tos.TosServerError); ok {
fmt.Println("Error:", serverErr.Error())
fmt.Println("Request ID:", serverErr.RequestID)
fmt.Println("Response Status Code:", serverErr.StatusCode)
fmt.Println("Response Header:", serverErr.Header)
fmt.Println("Response Err Code:", serverErr.Code)
fmt.Println("Response Err Msg:", serverErr.Message)
} else if clientErr, ok := err.(*tos.TosClientError); ok {
fmt.Println("Error:", clientErr.Error())
fmt.Println("Client Cause Err:", clientErr.Cause.Error())
} else {
fmt.Println("Error:", err)
}
panic(err)
}
}
func main() {
var (
accessKey = os.Getenv("TOS_ACCESS_KEY")
secretKey = os.Getenv("TOS_SECRET_KEY")
endpoint = "your endpoint"
region = "your region"
bucketName = "*** Provide your bucket name ***"
objectKey = "*** Provide your object name ***"
uploadID = "*** Provide upload ID ***"
ctx = context.Background()
)
client, err := tos.NewClientV2(endpoint, tos.WithRegion(region), tos.WithCredentials(tos.NewStaticCredentials(accessKey, secretKey)))
checkErr(err)
truncated := true
marker := 0
for truncated {
output, err := client.ListParts(ctx, &tos.ListPartsInput{
Bucket: bucketName,
Key: objectKey,
UploadID: uploadID,
PartNumberMarker: marker,
})
checkErr(err)
truncated = output.IsTruncated
marker = output.NextPartNumberMarker
for _, part := range output.Parts {
fmt.Println("Part Number:", part.PartNumber)
fmt.Println("ETag:", part.ETag)
fmt.Println("Size:", part.Size)
}
}
}
Abort multipart uploads
You can abort a multipart upload using the AbortMultipartUpload method. Once a multipart upload is aborted, TOS will delete all previously uploaded multipart data. You will also be unable to perform any further operations on this multipart upload.
package main
import (
"context"
"fmt"
"github.com/volcengine/ve-tos-golang-sdk/v2/tos"
)
func checkErr(err error) {
if err != nil {
if serverErr, ok := err.(*tos.TosServerError); ok {
fmt.Println("Error:", serverErr.Error())
fmt.Println("Request ID:", serverErr.RequestID)
fmt.Println("Response Status Code:", serverErr.StatusCode)
fmt.Println("Response Header:", serverErr.Header)
fmt.Println("Response Err Code:", serverErr.Code)
fmt.Println("Response Err Msg:", serverErr.Message)
} else if clientErr, ok := err.(*tos.TosClientError); ok {
fmt.Println("Error:", clientErr.Error())
fmt.Println("Client Cause Err:", clientErr.Cause.Error())
} else {
fmt.Println("Error:", err)
}
panic(err)
}
}
func main() {
var (
accessKey = os.Getenv("TOS_ACCESS_KEY")
secretKey = os.Getenv("TOS_SECRET_KEY")
endpoint = "your endpoint"
region = "your region"
bucketName = "*** Provide your bucket name ***"
objectKey = "*** Provide your object name ***"
uploadID = "*** Provide upload ID ***"
ctx = context.Background()
)
client, err := tos.NewClientV2(endpoint, tos.WithRegion(region), tos.WithCredentials(tos.NewStaticCredentials(accessKey, secretKey)))
checkErr(err)
output, err := client.AbortMultipartUpload(ctx, &tos.AbortMultipartUploadInput{
Bucket: bucketName,
Key: objectKey,
UploadID: uploadID,
})
checkErr(err)
fmt.Println("AbortMultipartUpload Request ID:", output.RequestID)
}
Relevant documentation
- For API documentation on initiating a multipart upload, refer to CreateMultipartUpload.
- For API documentation on uploading parts, refer to UploadPart.
- For API documentation on completing a multipart upload, refer to CompleteMultipartUpload.
- For API documentation on aborting a multipart upload, refer to AbortMultipartUpload.
Last updated: 2026.08.07 17:45:20
All rights reserved © Beijing Volcano Engine Technology Co., Ltd2024
