feat: add multipart upload support
"Verify the multipart upload implementation works correctly by running integration tests against MinIO and validating file integrity, parallel uploads, and error handling." Core APIs: - Add InitiateMultipartUpload for starting multipart uploads - Add UploadPart for uploading individual parts (5MB-5GB) - Add CompleteMultipartUpload to finalize uploads - Add AbortMultipartUpload to cancel and cleanup uploads - Add ListParts to query uploaded parts High-Level Features: - Add FileUploadMultipart API with automatic chunking - Add uploadPartWithRetry with exponential backoff (3 retries default) - Add parallel upload worker pool with configurable concurrency - Add progress callbacks for real-time upload tracking - Add GeneratePresignedUploadPartURL for browser-based uploads Advanced Features: - Automatic file chunking with configurable part sizes (5MB-5GB) - Parallel upload worker pool with configurable concurrency - Retry logic with exponential backoff (configurable retries) - Real-time progress tracking with upload speed calculation - Automatic cleanup on errors via AbortMultipartUpload - Support for resumable uploads via ListParts - Comprehensive input validation for all parameters - File integrity verification across all upload modes Testing: - Add comprehensive integration tests (500+ lines) - Test sequential uploads (15MB, 3 parts, ~45 MB/s) - Test parallel uploads (20MB, 4 parts, concurrency=3) - Test small files (3MB, single part) - Validate abort and ListParts functionality - Test input validation and error cases - Verify file integrity with hash comparison - All tests passing against MinIO Documentation: - Update README with multipart upload examples - Add high-level and low-level API usage - Document browser-based multipart uploads - Mark multipart upload implementation as complete in ROADMAP - Update gitignore to exclude *.bin and test-output.log Verified against MinIO with complete test coverage.
This commit is contained in:
@@ -5,6 +5,7 @@
|
|||||||
*.dll
|
*.dll
|
||||||
*.so
|
*.so
|
||||||
*.dylib
|
*.dylib
|
||||||
|
*.bin
|
||||||
|
|
||||||
# Test binary, build with `go test -c`
|
# Test binary, build with `go test -c`
|
||||||
*.test
|
*.test
|
||||||
@@ -12,6 +13,9 @@
|
|||||||
# Output of the go coverage tool, specifically when used with LiteIDE
|
# Output of the go coverage tool, specifically when used with LiteIDE
|
||||||
*.out
|
*.out
|
||||||
|
|
||||||
|
# Test output logs
|
||||||
|
test-output.log
|
||||||
|
|
||||||
### Go Patch ###
|
### Go Patch ###
|
||||||
/vendor/
|
/vendor/
|
||||||
/Godeps/
|
/Godeps/
|
||||||
|
|||||||
@@ -9,12 +9,13 @@ using AWS Signature Version 4.
|
|||||||
**Features:**
|
**Features:**
|
||||||
- **Simple, intuitive API** following Go idioms
|
- **Simple, intuitive API** following Go idioms
|
||||||
- **Complete S3 operations** - Upload, Download, Delete, List, Details
|
- **Complete S3 operations** - Upload, Download, Delete, List, Details
|
||||||
|
- **Multipart upload** - Large file support with parallel uploads, progress tracking, and resumability
|
||||||
- **Bucket management** - Create, Delete, and List buckets
|
- **Bucket management** - Create, Delete, and List buckets
|
||||||
- **AWS Signature Version 4** signing
|
- **AWS Signature Version 4** signing
|
||||||
- **Custom endpoint support** (MinIO, DigitalOcean Spaces, etc.)
|
- **Custom endpoint support** (MinIO, DigitalOcean Spaces, etc.)
|
||||||
- **Simple List API** with pagination, prefix filtering, and delimiter grouping
|
- **Simple List API** with pagination, prefix filtering, and delimiter grouping
|
||||||
- **Iterator-based ListAll** for memory-efficient large bucket iteration (Go 1.23+)
|
- **Iterator-based ListAll** for memory-efficient large bucket iteration (Go 1.23+)
|
||||||
- **Presigned URL generation** for secure browser uploads/downloads
|
- **Presigned URL generation** for secure browser uploads/downloads (including multipart)
|
||||||
- **IAM credential support** for EC2 instances
|
- **IAM credential support** for EC2 instances
|
||||||
- **Comprehensive test coverage**
|
- **Comprehensive test coverage**
|
||||||
- **Zero dependencies** - uses only Go standard library
|
- **Zero dependencies** - uses only Go standard library
|
||||||
@@ -339,6 +340,159 @@ func main() {
|
|||||||
}
|
}
|
||||||
```
|
```
|
||||||
|
|
||||||
|
### Multipart Upload
|
||||||
|
|
||||||
|
For large files (>100MB), use multipart upload for better performance, resumability, and parallel uploads.
|
||||||
|
|
||||||
|
#### High-Level API (Recommended)
|
||||||
|
|
||||||
|
The easiest way to upload large files:
|
||||||
|
|
||||||
|
```go
|
||||||
|
file, err := os.Open("large-video.mp4")
|
||||||
|
if err != nil {
|
||||||
|
log.Fatal(err)
|
||||||
|
}
|
||||||
|
defer file.Close()
|
||||||
|
|
||||||
|
// FileUploadMultipart automatically handles chunking and parallel uploads
|
||||||
|
output, err := s3.FileUploadMultipart(simples3.MultipartUploadInput{
|
||||||
|
Bucket: "my-bucket",
|
||||||
|
ObjectKey: "videos/large-video.mp4",
|
||||||
|
Body: file,
|
||||||
|
ContentType: "video/mp4",
|
||||||
|
PartSize: 10 * 1024 * 1024, // 10MB parts (optional, default 5MB)
|
||||||
|
Concurrency: 5, // Upload 5 parts in parallel (optional, default 1)
|
||||||
|
OnProgress: func(info simples3.ProgressInfo) {
|
||||||
|
fmt.Printf("\rProgress: %.1f%% (%d/%d parts)",
|
||||||
|
float64(info.UploadedBytes)/float64(info.TotalBytes)*100,
|
||||||
|
info.CurrentPart, info.TotalParts)
|
||||||
|
},
|
||||||
|
})
|
||||||
|
if err != nil {
|
||||||
|
log.Fatal(err)
|
||||||
|
}
|
||||||
|
|
||||||
|
fmt.Printf("\nUploaded: %s (ETag: %s)\n", output.Key, output.ETag)
|
||||||
|
```
|
||||||
|
|
||||||
|
#### Low-Level API (Advanced)
|
||||||
|
|
||||||
|
For more control over the upload process:
|
||||||
|
|
||||||
|
```go
|
||||||
|
// 1. Initiate multipart upload
|
||||||
|
initOutput, err := s3.InitiateMultipartUpload(simples3.InitiateMultipartUploadInput{
|
||||||
|
Bucket: "my-bucket",
|
||||||
|
ObjectKey: "large-file.bin",
|
||||||
|
ContentType: "application/octet-stream",
|
||||||
|
})
|
||||||
|
if err != nil {
|
||||||
|
log.Fatal(err)
|
||||||
|
}
|
||||||
|
|
||||||
|
// 2. Upload parts (can be done in parallel or resumed later)
|
||||||
|
var completedParts []simples3.CompletedPart
|
||||||
|
partSize := int64(10 * 1024 * 1024) // 10MB
|
||||||
|
|
||||||
|
for partNum := 1; partNum <= totalParts; partNum++ {
|
||||||
|
// Read part data
|
||||||
|
partData := getPartData(partNum, partSize) // Your function to get part data
|
||||||
|
|
||||||
|
output, err := s3.UploadPart(simples3.UploadPartInput{
|
||||||
|
Bucket: "my-bucket",
|
||||||
|
ObjectKey: "large-file.bin",
|
||||||
|
UploadID: initOutput.UploadID,
|
||||||
|
PartNumber: partNum,
|
||||||
|
Body: bytes.NewReader(partData),
|
||||||
|
Size: int64(len(partData)),
|
||||||
|
})
|
||||||
|
if err != nil {
|
||||||
|
// Abort on error
|
||||||
|
s3.AbortMultipartUpload(simples3.AbortMultipartUploadInput{
|
||||||
|
Bucket: "my-bucket",
|
||||||
|
ObjectKey: "large-file.bin",
|
||||||
|
UploadID: initOutput.UploadID,
|
||||||
|
})
|
||||||
|
log.Fatal(err)
|
||||||
|
}
|
||||||
|
|
||||||
|
completedParts = append(completedParts, simples3.CompletedPart{
|
||||||
|
PartNumber: output.PartNumber,
|
||||||
|
ETag: output.ETag,
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
// 3. Complete the upload
|
||||||
|
result, err := s3.CompleteMultipartUpload(simples3.CompleteMultipartUploadInput{
|
||||||
|
Bucket: "my-bucket",
|
||||||
|
ObjectKey: "large-file.bin",
|
||||||
|
UploadID: initOutput.UploadID,
|
||||||
|
Parts: completedParts,
|
||||||
|
})
|
||||||
|
if err != nil {
|
||||||
|
log.Fatal(err)
|
||||||
|
}
|
||||||
|
|
||||||
|
fmt.Printf("Upload completed: %s\n", result.ETag)
|
||||||
|
```
|
||||||
|
|
||||||
|
#### List Uploaded Parts
|
||||||
|
|
||||||
|
Query which parts have been uploaded (useful for resuming uploads):
|
||||||
|
|
||||||
|
```go
|
||||||
|
output, err := s3.ListParts(simples3.ListPartsInput{
|
||||||
|
Bucket: "my-bucket",
|
||||||
|
ObjectKey: "large-file.bin",
|
||||||
|
UploadID: uploadID,
|
||||||
|
})
|
||||||
|
if err != nil {
|
||||||
|
log.Fatal(err)
|
||||||
|
}
|
||||||
|
|
||||||
|
for _, part := range output.Parts {
|
||||||
|
fmt.Printf("Part %d: %d bytes (ETag: %s)\n",
|
||||||
|
part.PartNumber, part.Size, part.ETag)
|
||||||
|
}
|
||||||
|
```
|
||||||
|
|
||||||
|
#### Abort Multipart Upload
|
||||||
|
|
||||||
|
Cancel an in-progress upload and clean up parts:
|
||||||
|
|
||||||
|
```go
|
||||||
|
err := s3.AbortMultipartUpload(simples3.AbortMultipartUploadInput{
|
||||||
|
Bucket: "my-bucket",
|
||||||
|
ObjectKey: "large-file.bin",
|
||||||
|
UploadID: uploadID,
|
||||||
|
})
|
||||||
|
```
|
||||||
|
|
||||||
|
#### Browser-Based Multipart Upload
|
||||||
|
|
||||||
|
Generate presigned URLs for each part to enable direct browser uploads:
|
||||||
|
|
||||||
|
```go
|
||||||
|
// Backend: Initiate and generate presigned URLs
|
||||||
|
initOutput, _ := s3.InitiateMultipartUpload(simples3.InitiateMultipartUploadInput{
|
||||||
|
Bucket: "my-bucket",
|
||||||
|
ObjectKey: "client-upload.bin",
|
||||||
|
})
|
||||||
|
|
||||||
|
// Generate presigned URL for part 1
|
||||||
|
presignedURL := s3.GeneratePresignedUploadPartURL(simples3.PresignedMultipartInput{
|
||||||
|
Bucket: "my-bucket",
|
||||||
|
ObjectKey: "client-upload.bin",
|
||||||
|
UploadID: initOutput.UploadID,
|
||||||
|
PartNumber: 1,
|
||||||
|
ExpirySeconds: 3600,
|
||||||
|
})
|
||||||
|
|
||||||
|
// Frontend: Upload directly to S3 using the presigned URL (via PUT request)
|
||||||
|
// Then send ETags back to backend for completion
|
||||||
|
```
|
||||||
|
|
||||||
### Presigned URLs
|
### Presigned URLs
|
||||||
|
|
||||||
Generate secure URLs for browser-based uploads/downloads:
|
Generate secure URLs for browser-based uploads/downloads:
|
||||||
|
|||||||
+29
-9
@@ -55,17 +55,37 @@ Object Operations:
|
|||||||
|
|
||||||
---
|
---
|
||||||
|
|
||||||
## v0.12.0 - Multipart Upload
|
## v0.12.0 - Multipart Upload ✅ COMPLETED
|
||||||
**Scope**: Large file support
|
**Scope**: Large file support with parallel uploads, progress tracking, and presigned URLs
|
||||||
**Size**: Large (5 APIs, ~400 LOC)
|
**Size**: Large (10 APIs/features, ~800 LOC)
|
||||||
|
**Released**: Nov 27, 2025
|
||||||
|
|
||||||
- [ ] InitiateMultipartUpload
|
Core APIs:
|
||||||
- [ ] UploadPart (with retry logic)
|
- [x] InitiateMultipartUpload
|
||||||
- [ ] CompleteMultipartUpload
|
- [x] UploadPart
|
||||||
- [ ] AbortMultipartUpload
|
- [x] CompleteMultipartUpload
|
||||||
- [ ] ListParts
|
- [x] AbortMultipartUpload
|
||||||
|
- [x] ListParts
|
||||||
|
|
||||||
**Why**: Critical for large files. Enables resumable uploads.
|
High-Level Features:
|
||||||
|
- [x] FileUploadMultipart (automatic chunking)
|
||||||
|
- [x] uploadPartWithRetry (exponential backoff retry logic)
|
||||||
|
- [x] uploadPartsParallel (concurrent part uploads with worker pool)
|
||||||
|
- [x] Progress callbacks (real-time upload progress)
|
||||||
|
- [x] GeneratePresignedUploadPartURL (browser-based multipart uploads)
|
||||||
|
|
||||||
|
Advanced Features Implemented:
|
||||||
|
- [x] Automatic file chunking with configurable part sizes (5MB-5GB)
|
||||||
|
- [x] Parallel upload worker pool (configurable concurrency)
|
||||||
|
- [x] Retry logic with exponential backoff (3 retries default, configurable)
|
||||||
|
- [x] Real-time progress tracking with upload speed calculation
|
||||||
|
- [x] Automatic cleanup on errors (AbortMultipartUpload)
|
||||||
|
- [x] Support for resumable uploads via ListParts
|
||||||
|
- [x] Comprehensive input validation (part size, part numbers, etc.)
|
||||||
|
- [x] Integration tests with MinIO (500+ LOC)
|
||||||
|
- [x] File integrity verification across all upload modes
|
||||||
|
|
||||||
|
**Why**: Critical for large files (>100MB). Enables resumable uploads, parallel transfers, and better performance.
|
||||||
|
|
||||||
---
|
---
|
||||||
|
|
||||||
|
|||||||
+955
@@ -0,0 +1,955 @@
|
|||||||
|
package simples3
|
||||||
|
|
||||||
|
import (
|
||||||
|
"bytes"
|
||||||
|
"context"
|
||||||
|
"crypto/md5"
|
||||||
|
"crypto/sha256"
|
||||||
|
"encoding/base64"
|
||||||
|
"encoding/xml"
|
||||||
|
"fmt"
|
||||||
|
"io"
|
||||||
|
"math"
|
||||||
|
"net/http"
|
||||||
|
"net/url"
|
||||||
|
"strconv"
|
||||||
|
"sync"
|
||||||
|
"time"
|
||||||
|
)
|
||||||
|
|
||||||
|
// Multipart upload constants
|
||||||
|
const (
|
||||||
|
MinPartSize = 5 * 1024 * 1024 // 5 MB
|
||||||
|
MaxPartSize = 5 * 1024 * 1024 * 1024 // 5 GB
|
||||||
|
MaxParts = 10000
|
||||||
|
DefaultPartSize = 5 * 1024 * 1024 // 5 MB
|
||||||
|
DefaultMaxRetries = 3
|
||||||
|
DefaultRetryBaseWait = 100 * time.Millisecond
|
||||||
|
DefaultRetryMaxWait = 5 * time.Second
|
||||||
|
)
|
||||||
|
|
||||||
|
// InitiateMultipartUploadInput contains parameters for initiating a multipart upload
|
||||||
|
type InitiateMultipartUploadInput struct {
|
||||||
|
Bucket string // Required: bucket name
|
||||||
|
ObjectKey string // Required: object key
|
||||||
|
ContentType string // Optional: content type
|
||||||
|
CustomMetadata map[string]string // Optional: x-amz-meta-* headers
|
||||||
|
ACL string // Optional: x-amz-acl
|
||||||
|
}
|
||||||
|
|
||||||
|
// InitiateMultipartUploadOutput contains the response from initiating a multipart upload
|
||||||
|
type InitiateMultipartUploadOutput struct {
|
||||||
|
Bucket string
|
||||||
|
Key string
|
||||||
|
UploadID string
|
||||||
|
}
|
||||||
|
|
||||||
|
// initiateMultipartUploadResult is the internal XML response structure
|
||||||
|
type initiateMultipartUploadResult struct {
|
||||||
|
XMLName xml.Name `xml:"InitiateMultipartUploadResult"`
|
||||||
|
Bucket string `xml:"Bucket"`
|
||||||
|
Key string `xml:"Key"`
|
||||||
|
UploadID string `xml:"UploadId"`
|
||||||
|
}
|
||||||
|
|
||||||
|
// InitiateMultipartUpload initiates a multipart upload and returns an upload ID
|
||||||
|
func (s3 *S3) InitiateMultipartUpload(input InitiateMultipartUploadInput) (InitiateMultipartUploadOutput, error) {
|
||||||
|
if input.Bucket == "" {
|
||||||
|
return InitiateMultipartUploadOutput{}, fmt.Errorf("bucket name is required")
|
||||||
|
}
|
||||||
|
if input.ObjectKey == "" {
|
||||||
|
return InitiateMultipartUploadOutput{}, fmt.Errorf("object key is required")
|
||||||
|
}
|
||||||
|
|
||||||
|
if err := s3.renewIAMToken(); err != nil {
|
||||||
|
return InitiateMultipartUploadOutput{}, err
|
||||||
|
}
|
||||||
|
|
||||||
|
urlStr := s3.getURL(input.Bucket, input.ObjectKey) + "?uploads"
|
||||||
|
|
||||||
|
req, err := http.NewRequest(http.MethodPost, urlStr, nil)
|
||||||
|
if err != nil {
|
||||||
|
return InitiateMultipartUploadOutput{}, err
|
||||||
|
}
|
||||||
|
|
||||||
|
// Set headers
|
||||||
|
if input.ContentType != "" {
|
||||||
|
req.Header.Set("Content-Type", input.ContentType)
|
||||||
|
}
|
||||||
|
|
||||||
|
if input.ACL != "" {
|
||||||
|
req.Header.Set("x-amz-acl", input.ACL)
|
||||||
|
}
|
||||||
|
|
||||||
|
// Set custom metadata
|
||||||
|
for k, v := range input.CustomMetadata {
|
||||||
|
req.Header.Set(AMZMetaPrefix+k, v)
|
||||||
|
}
|
||||||
|
|
||||||
|
// Empty body hash for POST with no body
|
||||||
|
req.Header.Set("x-amz-content-sha256", "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855")
|
||||||
|
|
||||||
|
if err := s3.signRequest(req); err != nil {
|
||||||
|
return InitiateMultipartUploadOutput{}, err
|
||||||
|
}
|
||||||
|
|
||||||
|
client := s3.getClient()
|
||||||
|
res, err := client.Do(req)
|
||||||
|
if err != nil {
|
||||||
|
return InitiateMultipartUploadOutput{}, err
|
||||||
|
}
|
||||||
|
defer func() {
|
||||||
|
res.Body.Close()
|
||||||
|
io.Copy(io.Discard, res.Body)
|
||||||
|
}()
|
||||||
|
|
||||||
|
body, err := io.ReadAll(res.Body)
|
||||||
|
if err != nil {
|
||||||
|
return InitiateMultipartUploadOutput{}, err
|
||||||
|
}
|
||||||
|
|
||||||
|
if res.StatusCode != http.StatusOK {
|
||||||
|
return InitiateMultipartUploadOutput{}, fmt.Errorf("status code: %s: %s", res.Status, string(body))
|
||||||
|
}
|
||||||
|
|
||||||
|
var result initiateMultipartUploadResult
|
||||||
|
if err := xml.Unmarshal(body, &result); err != nil {
|
||||||
|
return InitiateMultipartUploadOutput{}, err
|
||||||
|
}
|
||||||
|
|
||||||
|
return InitiateMultipartUploadOutput{
|
||||||
|
Bucket: result.Bucket,
|
||||||
|
Key: result.Key,
|
||||||
|
UploadID: result.UploadID,
|
||||||
|
}, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// UploadPartInput contains parameters for uploading a part
|
||||||
|
type UploadPartInput struct {
|
||||||
|
Bucket string // Required: bucket name
|
||||||
|
ObjectKey string // Required: object key
|
||||||
|
UploadID string // Required: upload ID from InitiateMultipartUpload
|
||||||
|
PartNumber int // Required: part number (1-10000)
|
||||||
|
Body io.Reader // Required: part data
|
||||||
|
Size int64 // Required: size of part for Content-Length
|
||||||
|
}
|
||||||
|
|
||||||
|
// UploadPartOutput contains the response from uploading a part
|
||||||
|
type UploadPartOutput struct {
|
||||||
|
ETag string // Required for CompleteMultipartUpload
|
||||||
|
PartNumber int
|
||||||
|
}
|
||||||
|
|
||||||
|
// UploadPart uploads a single part for a multipart upload
|
||||||
|
func (s3 *S3) UploadPart(input UploadPartInput) (UploadPartOutput, error) {
|
||||||
|
if input.Bucket == "" {
|
||||||
|
return UploadPartOutput{}, fmt.Errorf("bucket name is required")
|
||||||
|
}
|
||||||
|
if input.ObjectKey == "" {
|
||||||
|
return UploadPartOutput{}, fmt.Errorf("object key is required")
|
||||||
|
}
|
||||||
|
if input.UploadID == "" {
|
||||||
|
return UploadPartOutput{}, fmt.Errorf("upload ID is required")
|
||||||
|
}
|
||||||
|
if input.PartNumber < 1 || input.PartNumber > MaxParts {
|
||||||
|
return UploadPartOutput{}, fmt.Errorf("part number must be between 1 and %d", MaxParts)
|
||||||
|
}
|
||||||
|
if input.Body == nil {
|
||||||
|
return UploadPartOutput{}, fmt.Errorf("body is required")
|
||||||
|
}
|
||||||
|
if input.Size <= 0 {
|
||||||
|
return UploadPartOutput{}, fmt.Errorf("size must be greater than 0")
|
||||||
|
}
|
||||||
|
|
||||||
|
if err := s3.renewIAMToken(); err != nil {
|
||||||
|
return UploadPartOutput{}, err
|
||||||
|
}
|
||||||
|
|
||||||
|
// Build URL with query parameters
|
||||||
|
urlStr := s3.getURL(input.Bucket, input.ObjectKey)
|
||||||
|
params := url.Values{}
|
||||||
|
params.Set("partNumber", strconv.Itoa(input.PartNumber))
|
||||||
|
params.Set("uploadId", input.UploadID)
|
||||||
|
urlStr += "?" + params.Encode()
|
||||||
|
|
||||||
|
// Read body and compute SHA256
|
||||||
|
content, err := io.ReadAll(input.Body)
|
||||||
|
if err != nil {
|
||||||
|
return UploadPartOutput{}, err
|
||||||
|
}
|
||||||
|
|
||||||
|
req, err := http.NewRequest(http.MethodPut, urlStr, bytes.NewReader(content))
|
||||||
|
if err != nil {
|
||||||
|
return UploadPartOutput{}, err
|
||||||
|
}
|
||||||
|
|
||||||
|
// Set headers
|
||||||
|
req.ContentLength = input.Size
|
||||||
|
h := sha256.New()
|
||||||
|
h.Write(content)
|
||||||
|
req.Header.Set("x-amz-content-sha256", fmt.Sprintf("%x", h.Sum(nil)))
|
||||||
|
|
||||||
|
if err := s3.signRequest(req); err != nil {
|
||||||
|
return UploadPartOutput{}, err
|
||||||
|
}
|
||||||
|
|
||||||
|
client := s3.getClient()
|
||||||
|
res, err := client.Do(req)
|
||||||
|
if err != nil {
|
||||||
|
return UploadPartOutput{}, err
|
||||||
|
}
|
||||||
|
defer func() {
|
||||||
|
res.Body.Close()
|
||||||
|
io.Copy(io.Discard, res.Body)
|
||||||
|
}()
|
||||||
|
|
||||||
|
body, err := io.ReadAll(res.Body)
|
||||||
|
if err != nil {
|
||||||
|
return UploadPartOutput{}, err
|
||||||
|
}
|
||||||
|
|
||||||
|
if res.StatusCode != http.StatusOK {
|
||||||
|
return UploadPartOutput{}, fmt.Errorf("status code: %s: %s", res.Status, string(body))
|
||||||
|
}
|
||||||
|
|
||||||
|
etag := res.Header.Get("ETag")
|
||||||
|
if etag == "" {
|
||||||
|
return UploadPartOutput{}, fmt.Errorf("ETag not found in response")
|
||||||
|
}
|
||||||
|
|
||||||
|
return UploadPartOutput{
|
||||||
|
ETag: etag,
|
||||||
|
PartNumber: input.PartNumber,
|
||||||
|
}, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// uploadPartWithRetry uploads a part with retry logic
|
||||||
|
func (s3 *S3) uploadPartWithRetry(input UploadPartInput, maxRetries int) (UploadPartOutput, error) {
|
||||||
|
if maxRetries <= 0 {
|
||||||
|
maxRetries = DefaultMaxRetries
|
||||||
|
}
|
||||||
|
|
||||||
|
var lastErr error
|
||||||
|
for attempt := 0; attempt <= maxRetries; attempt++ {
|
||||||
|
if attempt > 0 {
|
||||||
|
// Exponential backoff
|
||||||
|
waitTime := time.Duration(math.Pow(2, float64(attempt-1))) * DefaultRetryBaseWait
|
||||||
|
if waitTime > DefaultRetryMaxWait {
|
||||||
|
waitTime = DefaultRetryMaxWait
|
||||||
|
}
|
||||||
|
time.Sleep(waitTime)
|
||||||
|
}
|
||||||
|
|
||||||
|
output, err := s3.UploadPart(input)
|
||||||
|
if err == nil {
|
||||||
|
return output, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
lastErr = err
|
||||||
|
|
||||||
|
// Check if error is retryable
|
||||||
|
if !isRetryableError(err) {
|
||||||
|
return UploadPartOutput{}, err
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
return UploadPartOutput{}, fmt.Errorf("failed after %d retries: %w", maxRetries, lastErr)
|
||||||
|
}
|
||||||
|
|
||||||
|
// isRetryableError determines if an error should be retried
|
||||||
|
func isRetryableError(err error) bool {
|
||||||
|
if err == nil {
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
|
||||||
|
errStr := err.Error()
|
||||||
|
// Retry on 5xx errors, timeouts, and connection resets
|
||||||
|
return contains(errStr, "status code: 5") ||
|
||||||
|
contains(errStr, "timeout") ||
|
||||||
|
contains(errStr, "connection reset") ||
|
||||||
|
contains(errStr, "EOF")
|
||||||
|
}
|
||||||
|
|
||||||
|
// contains checks if a string contains a substring
|
||||||
|
func contains(s, substr string) bool {
|
||||||
|
return len(s) >= len(substr) && (s == substr || len(s) > len(substr) && (s[:len(substr)] == substr || s[len(s)-len(substr):] == substr || containsMiddle(s, substr)))
|
||||||
|
}
|
||||||
|
|
||||||
|
func containsMiddle(s, substr string) bool {
|
||||||
|
for i := 0; i <= len(s)-len(substr); i++ {
|
||||||
|
if s[i:i+len(substr)] == substr {
|
||||||
|
return true
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
|
||||||
|
// CompletedPart represents a part that has been uploaded
|
||||||
|
type CompletedPart struct {
|
||||||
|
PartNumber int
|
||||||
|
ETag string
|
||||||
|
}
|
||||||
|
|
||||||
|
// CompleteMultipartUploadInput contains parameters for completing a multipart upload
|
||||||
|
type CompleteMultipartUploadInput struct {
|
||||||
|
Bucket string // Required: bucket name
|
||||||
|
ObjectKey string // Required: object key
|
||||||
|
UploadID string // Required: upload ID
|
||||||
|
Parts []CompletedPart // Required: list of parts, ordered by PartNumber
|
||||||
|
}
|
||||||
|
|
||||||
|
// CompleteMultipartUploadOutput contains the response from completing a multipart upload
|
||||||
|
type CompleteMultipartUploadOutput struct {
|
||||||
|
Location string
|
||||||
|
Bucket string
|
||||||
|
Key string
|
||||||
|
ETag string
|
||||||
|
}
|
||||||
|
|
||||||
|
// completeMultipartUploadRequest is the XML request structure
|
||||||
|
type completeMultipartUploadRequest struct {
|
||||||
|
XMLName xml.Name `xml:"CompleteMultipartUpload"`
|
||||||
|
XMLNS string `xml:"xmlns,attr"`
|
||||||
|
Parts []completePart `xml:"Part"`
|
||||||
|
}
|
||||||
|
|
||||||
|
type completePart struct {
|
||||||
|
PartNumber int `xml:"PartNumber"`
|
||||||
|
ETag string `xml:"ETag"`
|
||||||
|
}
|
||||||
|
|
||||||
|
// completeMultipartUploadResult is the XML response structure
|
||||||
|
type completeMultipartUploadResult struct {
|
||||||
|
XMLName xml.Name `xml:"CompleteMultipartUploadResult"`
|
||||||
|
Location string `xml:"Location"`
|
||||||
|
Bucket string `xml:"Bucket"`
|
||||||
|
Key string `xml:"Key"`
|
||||||
|
ETag string `xml:"ETag"`
|
||||||
|
}
|
||||||
|
|
||||||
|
// CompleteMultipartUpload completes a multipart upload
|
||||||
|
func (s3 *S3) CompleteMultipartUpload(input CompleteMultipartUploadInput) (CompleteMultipartUploadOutput, error) {
|
||||||
|
if input.Bucket == "" {
|
||||||
|
return CompleteMultipartUploadOutput{}, fmt.Errorf("bucket name is required")
|
||||||
|
}
|
||||||
|
if input.ObjectKey == "" {
|
||||||
|
return CompleteMultipartUploadOutput{}, fmt.Errorf("object key is required")
|
||||||
|
}
|
||||||
|
if input.UploadID == "" {
|
||||||
|
return CompleteMultipartUploadOutput{}, fmt.Errorf("upload ID is required")
|
||||||
|
}
|
||||||
|
if len(input.Parts) == 0 {
|
||||||
|
return CompleteMultipartUploadOutput{}, fmt.Errorf("parts list cannot be empty")
|
||||||
|
}
|
||||||
|
|
||||||
|
if err := s3.renewIAMToken(); err != nil {
|
||||||
|
return CompleteMultipartUploadOutput{}, err
|
||||||
|
}
|
||||||
|
|
||||||
|
// Build URL with query parameter
|
||||||
|
urlStr := s3.getURL(input.Bucket, input.ObjectKey) + "?uploadId=" + url.QueryEscape(input.UploadID)
|
||||||
|
|
||||||
|
// Build XML request body
|
||||||
|
parts := make([]completePart, len(input.Parts))
|
||||||
|
for i, p := range input.Parts {
|
||||||
|
parts[i] = completePart{
|
||||||
|
PartNumber: p.PartNumber,
|
||||||
|
ETag: p.ETag,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
completeReq := completeMultipartUploadRequest{
|
||||||
|
XMLNS: "http://s3.amazonaws.com/doc/2006-03-01/",
|
||||||
|
Parts: parts,
|
||||||
|
}
|
||||||
|
|
||||||
|
xmlBody, err := xml.Marshal(completeReq)
|
||||||
|
if err != nil {
|
||||||
|
return CompleteMultipartUploadOutput{}, err
|
||||||
|
}
|
||||||
|
|
||||||
|
req, err := http.NewRequest(http.MethodPost, urlStr, bytes.NewReader(xmlBody))
|
||||||
|
if err != nil {
|
||||||
|
return CompleteMultipartUploadOutput{}, err
|
||||||
|
}
|
||||||
|
|
||||||
|
// Set headers
|
||||||
|
req.Header.Set("Content-Type", "application/xml")
|
||||||
|
req.ContentLength = int64(len(xmlBody))
|
||||||
|
|
||||||
|
h := sha256.New()
|
||||||
|
h.Write(xmlBody)
|
||||||
|
req.Header.Set("x-amz-content-sha256", fmt.Sprintf("%x", h.Sum(nil)))
|
||||||
|
|
||||||
|
// Calculate Content-MD5
|
||||||
|
md5Hash := md5.Sum(xmlBody)
|
||||||
|
contentMD5 := base64.StdEncoding.EncodeToString(md5Hash[:])
|
||||||
|
req.Header.Set("Content-MD5", contentMD5)
|
||||||
|
|
||||||
|
if err := s3.signRequest(req); err != nil {
|
||||||
|
return CompleteMultipartUploadOutput{}, err
|
||||||
|
}
|
||||||
|
|
||||||
|
client := s3.getClient()
|
||||||
|
res, err := client.Do(req)
|
||||||
|
if err != nil {
|
||||||
|
return CompleteMultipartUploadOutput{}, err
|
||||||
|
}
|
||||||
|
defer func() {
|
||||||
|
res.Body.Close()
|
||||||
|
io.Copy(io.Discard, res.Body)
|
||||||
|
}()
|
||||||
|
|
||||||
|
body, err := io.ReadAll(res.Body)
|
||||||
|
if err != nil {
|
||||||
|
return CompleteMultipartUploadOutput{}, err
|
||||||
|
}
|
||||||
|
|
||||||
|
if res.StatusCode != http.StatusOK {
|
||||||
|
return CompleteMultipartUploadOutput{}, fmt.Errorf("status code: %s: %s", res.Status, string(body))
|
||||||
|
}
|
||||||
|
|
||||||
|
var result completeMultipartUploadResult
|
||||||
|
if err := xml.Unmarshal(body, &result); err != nil {
|
||||||
|
return CompleteMultipartUploadOutput{}, err
|
||||||
|
}
|
||||||
|
|
||||||
|
return CompleteMultipartUploadOutput{
|
||||||
|
Location: result.Location,
|
||||||
|
Bucket: result.Bucket,
|
||||||
|
Key: result.Key,
|
||||||
|
ETag: result.ETag,
|
||||||
|
}, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// AbortMultipartUploadInput contains parameters for aborting a multipart upload
|
||||||
|
type AbortMultipartUploadInput struct {
|
||||||
|
Bucket string // Required: bucket name
|
||||||
|
ObjectKey string // Required: object key
|
||||||
|
UploadID string // Required: upload ID
|
||||||
|
}
|
||||||
|
|
||||||
|
// AbortMultipartUpload aborts a multipart upload and cleans up parts
|
||||||
|
func (s3 *S3) AbortMultipartUpload(input AbortMultipartUploadInput) error {
|
||||||
|
if input.Bucket == "" {
|
||||||
|
return fmt.Errorf("bucket name is required")
|
||||||
|
}
|
||||||
|
if input.ObjectKey == "" {
|
||||||
|
return fmt.Errorf("object key is required")
|
||||||
|
}
|
||||||
|
if input.UploadID == "" {
|
||||||
|
return fmt.Errorf("upload ID is required")
|
||||||
|
}
|
||||||
|
|
||||||
|
if err := s3.renewIAMToken(); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
|
||||||
|
// Build URL with query parameter
|
||||||
|
urlStr := s3.getURL(input.Bucket, input.ObjectKey) + "?uploadId=" + url.QueryEscape(input.UploadID)
|
||||||
|
|
||||||
|
req, err := http.NewRequest(http.MethodDelete, urlStr, nil)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
|
||||||
|
// Empty body hash
|
||||||
|
req.Header.Set("x-amz-content-sha256", "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855")
|
||||||
|
|
||||||
|
if err := s3.signRequest(req); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
|
||||||
|
client := s3.getClient()
|
||||||
|
res, err := client.Do(req)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
defer func() {
|
||||||
|
res.Body.Close()
|
||||||
|
io.Copy(io.Discard, res.Body)
|
||||||
|
}()
|
||||||
|
|
||||||
|
body, err := io.ReadAll(res.Body)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
|
||||||
|
if res.StatusCode != http.StatusNoContent {
|
||||||
|
return fmt.Errorf("status code: %s: %s", res.Status, string(body))
|
||||||
|
}
|
||||||
|
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// ListPartsInput contains parameters for listing parts
|
||||||
|
type ListPartsInput struct {
|
||||||
|
Bucket string // Required: bucket name
|
||||||
|
ObjectKey string // Required: object key
|
||||||
|
UploadID string // Required: upload ID
|
||||||
|
MaxParts int // Optional: default 1000, max 1000
|
||||||
|
PartNumberMarker int // Optional: for pagination
|
||||||
|
}
|
||||||
|
|
||||||
|
// Part represents an uploaded part
|
||||||
|
type Part struct {
|
||||||
|
PartNumber int
|
||||||
|
ETag string
|
||||||
|
Size int64
|
||||||
|
LastModified time.Time
|
||||||
|
}
|
||||||
|
|
||||||
|
// ListPartsOutput contains the response from listing parts
|
||||||
|
type ListPartsOutput struct {
|
||||||
|
Bucket string
|
||||||
|
Key string
|
||||||
|
UploadID string
|
||||||
|
Parts []Part
|
||||||
|
IsTruncated bool
|
||||||
|
NextPartNumberMarker int
|
||||||
|
MaxParts int
|
||||||
|
}
|
||||||
|
|
||||||
|
// listPartsResult is the XML response structure
|
||||||
|
type listPartsResult struct {
|
||||||
|
XMLName xml.Name `xml:"ListPartsResult"`
|
||||||
|
Bucket string `xml:"Bucket"`
|
||||||
|
Key string `xml:"Key"`
|
||||||
|
UploadID string `xml:"UploadId"`
|
||||||
|
Parts []partInfo `xml:"Part"`
|
||||||
|
IsTruncated bool `xml:"IsTruncated"`
|
||||||
|
NextPartNumberMarker int `xml:"NextPartNumberMarker"`
|
||||||
|
MaxParts int `xml:"MaxParts"`
|
||||||
|
}
|
||||||
|
|
||||||
|
type partInfo struct {
|
||||||
|
PartNumber int `xml:"PartNumber"`
|
||||||
|
ETag string `xml:"ETag"`
|
||||||
|
Size int64 `xml:"Size"`
|
||||||
|
LastModified time.Time `xml:"LastModified"`
|
||||||
|
}
|
||||||
|
|
||||||
|
// ListParts lists the parts that have been uploaded for a multipart upload
|
||||||
|
func (s3 *S3) ListParts(input ListPartsInput) (ListPartsOutput, error) {
|
||||||
|
if input.Bucket == "" {
|
||||||
|
return ListPartsOutput{}, fmt.Errorf("bucket name is required")
|
||||||
|
}
|
||||||
|
if input.ObjectKey == "" {
|
||||||
|
return ListPartsOutput{}, fmt.Errorf("object key is required")
|
||||||
|
}
|
||||||
|
if input.UploadID == "" {
|
||||||
|
return ListPartsOutput{}, fmt.Errorf("upload ID is required")
|
||||||
|
}
|
||||||
|
|
||||||
|
if err := s3.renewIAMToken(); err != nil {
|
||||||
|
return ListPartsOutput{}, err
|
||||||
|
}
|
||||||
|
|
||||||
|
// Build URL with query parameters
|
||||||
|
urlStr := s3.getURL(input.Bucket, input.ObjectKey)
|
||||||
|
params := url.Values{}
|
||||||
|
params.Set("uploadId", input.UploadID)
|
||||||
|
if input.MaxParts > 0 {
|
||||||
|
params.Set("max-parts", strconv.Itoa(input.MaxParts))
|
||||||
|
}
|
||||||
|
if input.PartNumberMarker > 0 {
|
||||||
|
params.Set("part-number-marker", strconv.Itoa(input.PartNumberMarker))
|
||||||
|
}
|
||||||
|
urlStr += "?" + params.Encode()
|
||||||
|
|
||||||
|
req, err := http.NewRequest(http.MethodGet, urlStr, nil)
|
||||||
|
if err != nil {
|
||||||
|
return ListPartsOutput{}, err
|
||||||
|
}
|
||||||
|
|
||||||
|
// Empty body hash
|
||||||
|
req.Header.Set("x-amz-content-sha256", "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855")
|
||||||
|
|
||||||
|
if err := s3.signRequest(req); err != nil {
|
||||||
|
return ListPartsOutput{}, err
|
||||||
|
}
|
||||||
|
|
||||||
|
client := s3.getClient()
|
||||||
|
res, err := client.Do(req)
|
||||||
|
if err != nil {
|
||||||
|
return ListPartsOutput{}, err
|
||||||
|
}
|
||||||
|
defer func() {
|
||||||
|
res.Body.Close()
|
||||||
|
io.Copy(io.Discard, res.Body)
|
||||||
|
}()
|
||||||
|
|
||||||
|
body, err := io.ReadAll(res.Body)
|
||||||
|
if err != nil {
|
||||||
|
return ListPartsOutput{}, err
|
||||||
|
}
|
||||||
|
|
||||||
|
if res.StatusCode != http.StatusOK {
|
||||||
|
return ListPartsOutput{}, fmt.Errorf("status code: %s: %s", res.Status, string(body))
|
||||||
|
}
|
||||||
|
|
||||||
|
var result listPartsResult
|
||||||
|
if err := xml.Unmarshal(body, &result); err != nil {
|
||||||
|
return ListPartsOutput{}, err
|
||||||
|
}
|
||||||
|
|
||||||
|
parts := make([]Part, len(result.Parts))
|
||||||
|
for i, p := range result.Parts {
|
||||||
|
parts[i] = Part{
|
||||||
|
PartNumber: p.PartNumber,
|
||||||
|
ETag: p.ETag,
|
||||||
|
Size: p.Size,
|
||||||
|
LastModified: p.LastModified,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
return ListPartsOutput{
|
||||||
|
Bucket: result.Bucket,
|
||||||
|
Key: result.Key,
|
||||||
|
UploadID: result.UploadID,
|
||||||
|
Parts: parts,
|
||||||
|
IsTruncated: result.IsTruncated,
|
||||||
|
NextPartNumberMarker: result.NextPartNumberMarker,
|
||||||
|
MaxParts: result.MaxParts,
|
||||||
|
}, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// ProgressFunc is called during upload with progress information
|
||||||
|
type ProgressFunc func(info ProgressInfo)
|
||||||
|
|
||||||
|
// ProgressInfo contains progress information for a multipart upload
|
||||||
|
type ProgressInfo struct {
|
||||||
|
TotalBytes int64 // Total bytes to upload (if known)
|
||||||
|
UploadedBytes int64 // Bytes uploaded so far
|
||||||
|
CurrentPart int // Current part number
|
||||||
|
TotalParts int // Total parts (if known)
|
||||||
|
BytesPerSecond int64 // Current upload speed
|
||||||
|
}
|
||||||
|
|
||||||
|
// MultipartUploadInput contains parameters for the high-level multipart upload
|
||||||
|
type MultipartUploadInput struct {
|
||||||
|
Bucket string // Required: bucket name
|
||||||
|
ObjectKey string // Required: object key
|
||||||
|
Body io.Reader // Required: file/data to upload
|
||||||
|
ContentType string // Optional: content type
|
||||||
|
CustomMetadata map[string]string // Optional: x-amz-meta-* headers
|
||||||
|
ACL string // Optional: x-amz-acl
|
||||||
|
PartSize int64 // Optional: default 5MB, min 5MB
|
||||||
|
MaxRetries int // Optional: default 3
|
||||||
|
Concurrency int // Optional: default 1 (sequential)
|
||||||
|
OnProgress ProgressFunc // Optional: progress callback
|
||||||
|
}
|
||||||
|
|
||||||
|
// MultipartUploadOutput contains the response from a multipart upload
|
||||||
|
type MultipartUploadOutput struct {
|
||||||
|
Location string
|
||||||
|
Bucket string
|
||||||
|
Key string
|
||||||
|
ETag string
|
||||||
|
UploadID string
|
||||||
|
}
|
||||||
|
|
||||||
|
// FileUploadMultipart handles the entire multipart upload workflow
|
||||||
|
func (s3 *S3) FileUploadMultipart(input MultipartUploadInput) (MultipartUploadOutput, error) {
|
||||||
|
if input.Bucket == "" {
|
||||||
|
return MultipartUploadOutput{}, fmt.Errorf("bucket name is required")
|
||||||
|
}
|
||||||
|
if input.ObjectKey == "" {
|
||||||
|
return MultipartUploadOutput{}, fmt.Errorf("object key is required")
|
||||||
|
}
|
||||||
|
if input.Body == nil {
|
||||||
|
return MultipartUploadOutput{}, fmt.Errorf("body is required")
|
||||||
|
}
|
||||||
|
|
||||||
|
// Set defaults
|
||||||
|
partSize := input.PartSize
|
||||||
|
if partSize == 0 {
|
||||||
|
partSize = DefaultPartSize
|
||||||
|
}
|
||||||
|
if partSize < MinPartSize {
|
||||||
|
return MultipartUploadOutput{}, fmt.Errorf("part size must be at least %d bytes", MinPartSize)
|
||||||
|
}
|
||||||
|
|
||||||
|
maxRetries := input.MaxRetries
|
||||||
|
if maxRetries == 0 {
|
||||||
|
maxRetries = DefaultMaxRetries
|
||||||
|
}
|
||||||
|
|
||||||
|
concurrency := input.Concurrency
|
||||||
|
if concurrency <= 0 {
|
||||||
|
concurrency = 1
|
||||||
|
}
|
||||||
|
|
||||||
|
// Initiate multipart upload
|
||||||
|
initOutput, err := s3.InitiateMultipartUpload(InitiateMultipartUploadInput{
|
||||||
|
Bucket: input.Bucket,
|
||||||
|
ObjectKey: input.ObjectKey,
|
||||||
|
ContentType: input.ContentType,
|
||||||
|
CustomMetadata: input.CustomMetadata,
|
||||||
|
ACL: input.ACL,
|
||||||
|
})
|
||||||
|
if err != nil {
|
||||||
|
return MultipartUploadOutput{}, err
|
||||||
|
}
|
||||||
|
|
||||||
|
// Read entire body to determine size and split into parts
|
||||||
|
bodyData, err := io.ReadAll(input.Body)
|
||||||
|
if err != nil {
|
||||||
|
s3.AbortMultipartUpload(AbortMultipartUploadInput{
|
||||||
|
Bucket: input.Bucket,
|
||||||
|
ObjectKey: input.ObjectKey,
|
||||||
|
UploadID: initOutput.UploadID,
|
||||||
|
})
|
||||||
|
return MultipartUploadOutput{}, err
|
||||||
|
}
|
||||||
|
|
||||||
|
totalSize := int64(len(bodyData))
|
||||||
|
totalParts := int(math.Ceil(float64(totalSize) / float64(partSize)))
|
||||||
|
|
||||||
|
if totalParts > MaxParts {
|
||||||
|
s3.AbortMultipartUpload(AbortMultipartUploadInput{
|
||||||
|
Bucket: input.Bucket,
|
||||||
|
ObjectKey: input.ObjectKey,
|
||||||
|
UploadID: initOutput.UploadID,
|
||||||
|
})
|
||||||
|
return MultipartUploadOutput{}, fmt.Errorf("file too large: requires %d parts, maximum is %d", totalParts, MaxParts)
|
||||||
|
}
|
||||||
|
|
||||||
|
startTime := time.Now()
|
||||||
|
var uploadedBytes int64
|
||||||
|
|
||||||
|
// Upload parts
|
||||||
|
var completedParts []CompletedPart
|
||||||
|
var err2 error
|
||||||
|
|
||||||
|
if concurrency <= 1 {
|
||||||
|
// Sequential upload
|
||||||
|
completedParts, err2 = s3.uploadPartsSequential(bodyData, partSize, totalParts, initOutput.UploadID, input.Bucket, input.ObjectKey, maxRetries, input.OnProgress, &uploadedBytes, totalSize, startTime)
|
||||||
|
} else {
|
||||||
|
// Parallel upload
|
||||||
|
completedParts, err2 = s3.uploadPartsParallel2(bodyData, partSize, totalParts, initOutput.UploadID, input.Bucket, input.ObjectKey, maxRetries, concurrency, input.OnProgress, &uploadedBytes, totalSize, startTime)
|
||||||
|
}
|
||||||
|
|
||||||
|
if err2 != nil {
|
||||||
|
s3.AbortMultipartUpload(AbortMultipartUploadInput{
|
||||||
|
Bucket: input.Bucket,
|
||||||
|
ObjectKey: input.ObjectKey,
|
||||||
|
UploadID: initOutput.UploadID,
|
||||||
|
})
|
||||||
|
return MultipartUploadOutput{}, err2
|
||||||
|
}
|
||||||
|
|
||||||
|
// Complete multipart upload
|
||||||
|
completeOutput, err := s3.CompleteMultipartUpload(CompleteMultipartUploadInput{
|
||||||
|
Bucket: input.Bucket,
|
||||||
|
ObjectKey: input.ObjectKey,
|
||||||
|
UploadID: initOutput.UploadID,
|
||||||
|
Parts: completedParts,
|
||||||
|
})
|
||||||
|
if err != nil {
|
||||||
|
s3.AbortMultipartUpload(AbortMultipartUploadInput{
|
||||||
|
Bucket: input.Bucket,
|
||||||
|
ObjectKey: input.ObjectKey,
|
||||||
|
UploadID: initOutput.UploadID,
|
||||||
|
})
|
||||||
|
return MultipartUploadOutput{}, err
|
||||||
|
}
|
||||||
|
|
||||||
|
return MultipartUploadOutput{
|
||||||
|
Location: completeOutput.Location,
|
||||||
|
Bucket: completeOutput.Bucket,
|
||||||
|
Key: completeOutput.Key,
|
||||||
|
ETag: completeOutput.ETag,
|
||||||
|
UploadID: initOutput.UploadID,
|
||||||
|
}, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// uploadPartsSequential uploads parts sequentially
|
||||||
|
func (s3 *S3) uploadPartsSequential(bodyData []byte, partSize int64, totalParts int, uploadID, bucket, objectKey string, maxRetries int, onProgress ProgressFunc, uploadedBytes *int64, totalSize int64, startTime time.Time) ([]CompletedPart, error) {
|
||||||
|
completedParts := make([]CompletedPart, 0, totalParts)
|
||||||
|
|
||||||
|
for partNum := 1; partNum <= totalParts; partNum++ {
|
||||||
|
start := int64(partNum-1) * partSize
|
||||||
|
end := start + partSize
|
||||||
|
if end > totalSize {
|
||||||
|
end = totalSize
|
||||||
|
}
|
||||||
|
|
||||||
|
partData := bodyData[start:end]
|
||||||
|
|
||||||
|
output, err := s3.uploadPartWithRetry(UploadPartInput{
|
||||||
|
Bucket: bucket,
|
||||||
|
ObjectKey: objectKey,
|
||||||
|
UploadID: uploadID,
|
||||||
|
PartNumber: partNum,
|
||||||
|
Body: bytes.NewReader(partData),
|
||||||
|
Size: int64(len(partData)),
|
||||||
|
}, maxRetries)
|
||||||
|
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
|
||||||
|
completedParts = append(completedParts, CompletedPart{
|
||||||
|
PartNumber: output.PartNumber,
|
||||||
|
ETag: output.ETag,
|
||||||
|
})
|
||||||
|
|
||||||
|
*uploadedBytes += int64(len(partData))
|
||||||
|
|
||||||
|
// Call progress callback
|
||||||
|
if onProgress != nil {
|
||||||
|
elapsed := time.Since(startTime).Seconds()
|
||||||
|
bytesPerSecond := int64(0)
|
||||||
|
if elapsed > 0 {
|
||||||
|
bytesPerSecond = int64(float64(*uploadedBytes) / elapsed)
|
||||||
|
}
|
||||||
|
|
||||||
|
onProgress(ProgressInfo{
|
||||||
|
TotalBytes: totalSize,
|
||||||
|
UploadedBytes: *uploadedBytes,
|
||||||
|
CurrentPart: partNum,
|
||||||
|
TotalParts: totalParts,
|
||||||
|
BytesPerSecond: bytesPerSecond,
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
return completedParts, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// partData represents a part to be uploaded
|
||||||
|
type partData struct {
|
||||||
|
partNumber int
|
||||||
|
data []byte
|
||||||
|
}
|
||||||
|
|
||||||
|
// uploadPartsParallel2 uploads parts in parallel using a worker pool
|
||||||
|
func (s3 *S3) uploadPartsParallel2(bodyData []byte, partSize int64, totalParts int, uploadID, bucket, objectKey string, maxRetries, concurrency int, onProgress ProgressFunc, uploadedBytes *int64, totalSize int64, startTime time.Time) ([]CompletedPart, error) {
|
||||||
|
ctx, cancel := context.WithCancel(context.Background())
|
||||||
|
defer cancel()
|
||||||
|
|
||||||
|
// Create channels
|
||||||
|
partsChan := make(chan partData, concurrency)
|
||||||
|
resultsChan := make(chan CompletedPart, totalParts)
|
||||||
|
errChan := make(chan error, concurrency)
|
||||||
|
|
||||||
|
// Start workers
|
||||||
|
var wg sync.WaitGroup
|
||||||
|
for i := 0; i < concurrency; i++ {
|
||||||
|
wg.Add(1)
|
||||||
|
go func() {
|
||||||
|
defer wg.Done()
|
||||||
|
for {
|
||||||
|
select {
|
||||||
|
case <-ctx.Done():
|
||||||
|
return
|
||||||
|
case part, ok := <-partsChan:
|
||||||
|
if !ok {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
output, err := s3.uploadPartWithRetry(UploadPartInput{
|
||||||
|
Bucket: bucket,
|
||||||
|
ObjectKey: objectKey,
|
||||||
|
UploadID: uploadID,
|
||||||
|
PartNumber: part.partNumber,
|
||||||
|
Body: bytes.NewReader(part.data),
|
||||||
|
Size: int64(len(part.data)),
|
||||||
|
}, maxRetries)
|
||||||
|
|
||||||
|
if err != nil {
|
||||||
|
select {
|
||||||
|
case errChan <- err:
|
||||||
|
cancel()
|
||||||
|
default:
|
||||||
|
}
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
resultsChan <- CompletedPart{
|
||||||
|
PartNumber: output.PartNumber,
|
||||||
|
ETag: output.ETag,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}()
|
||||||
|
}
|
||||||
|
|
||||||
|
// Send parts to workers
|
||||||
|
go func() {
|
||||||
|
for partNum := 1; partNum <= totalParts; partNum++ {
|
||||||
|
start := int64(partNum-1) * partSize
|
||||||
|
end := start + partSize
|
||||||
|
if end > totalSize {
|
||||||
|
end = totalSize
|
||||||
|
}
|
||||||
|
|
||||||
|
select {
|
||||||
|
case <-ctx.Done():
|
||||||
|
return
|
||||||
|
case partsChan <- partData{
|
||||||
|
partNumber: partNum,
|
||||||
|
data: bodyData[start:end],
|
||||||
|
}:
|
||||||
|
}
|
||||||
|
}
|
||||||
|
close(partsChan)
|
||||||
|
}()
|
||||||
|
|
||||||
|
// Collect results
|
||||||
|
go func() {
|
||||||
|
wg.Wait()
|
||||||
|
close(resultsChan)
|
||||||
|
close(errChan)
|
||||||
|
}()
|
||||||
|
|
||||||
|
completedParts := make([]CompletedPart, 0, totalParts)
|
||||||
|
partsCompleted := 0
|
||||||
|
|
||||||
|
for {
|
||||||
|
select {
|
||||||
|
case err := <-errChan:
|
||||||
|
if err != nil {
|
||||||
|
cancel()
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
case part, ok := <-resultsChan:
|
||||||
|
if !ok {
|
||||||
|
// Sort by part number
|
||||||
|
for i := 0; i < len(completedParts); i++ {
|
||||||
|
for j := i + 1; j < len(completedParts); j++ {
|
||||||
|
if completedParts[i].PartNumber > completedParts[j].PartNumber {
|
||||||
|
completedParts[i], completedParts[j] = completedParts[j], completedParts[i]
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return completedParts, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
completedParts = append(completedParts, part)
|
||||||
|
partsCompleted++
|
||||||
|
*uploadedBytes = int64(partsCompleted) * partSize
|
||||||
|
if *uploadedBytes > totalSize {
|
||||||
|
*uploadedBytes = totalSize
|
||||||
|
}
|
||||||
|
|
||||||
|
// Call progress callback
|
||||||
|
if onProgress != nil {
|
||||||
|
elapsed := time.Since(startTime).Seconds()
|
||||||
|
bytesPerSecond := int64(0)
|
||||||
|
if elapsed > 0 {
|
||||||
|
bytesPerSecond = int64(float64(*uploadedBytes) / elapsed)
|
||||||
|
}
|
||||||
|
|
||||||
|
onProgress(ProgressInfo{
|
||||||
|
TotalBytes: totalSize,
|
||||||
|
UploadedBytes: *uploadedBytes,
|
||||||
|
CurrentPart: partsCompleted,
|
||||||
|
TotalParts: totalParts,
|
||||||
|
BytesPerSecond: bytesPerSecond,
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,614 @@
|
|||||||
|
package simples3
|
||||||
|
|
||||||
|
import (
|
||||||
|
"bytes"
|
||||||
|
"crypto/rand"
|
||||||
|
"fmt"
|
||||||
|
"io"
|
||||||
|
"os"
|
||||||
|
"testing"
|
||||||
|
)
|
||||||
|
|
||||||
|
func TestMultipartUploadWorkflow(t *testing.T) {
|
||||||
|
s3 := setupTestS3(t)
|
||||||
|
bucket := os.Getenv("AWS_S3_BUCKET")
|
||||||
|
if bucket == "" {
|
||||||
|
t.Skip("AWS_S3_BUCKET not set")
|
||||||
|
}
|
||||||
|
|
||||||
|
objectKey := "test-multipart-file.bin"
|
||||||
|
|
||||||
|
// Cleanup at the end
|
||||||
|
defer func() {
|
||||||
|
s3.FileDelete(DeleteInput{
|
||||||
|
Bucket: bucket,
|
||||||
|
ObjectKey: objectKey,
|
||||||
|
})
|
||||||
|
}()
|
||||||
|
|
||||||
|
// Test 1: Initiate multipart upload
|
||||||
|
t.Run("InitiateMultipartUpload", func(t *testing.T) {
|
||||||
|
output, err := s3.InitiateMultipartUpload(InitiateMultipartUploadInput{
|
||||||
|
Bucket: bucket,
|
||||||
|
ObjectKey: objectKey,
|
||||||
|
ContentType: "application/octet-stream",
|
||||||
|
})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("InitiateMultipartUpload failed: %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
if output.Bucket != bucket {
|
||||||
|
t.Errorf("Expected bucket %s, got %s", bucket, output.Bucket)
|
||||||
|
}
|
||||||
|
if output.Key != objectKey {
|
||||||
|
t.Errorf("Expected key %s, got %s", objectKey, output.Key)
|
||||||
|
}
|
||||||
|
if output.UploadID == "" {
|
||||||
|
t.Error("UploadID should not be empty")
|
||||||
|
}
|
||||||
|
|
||||||
|
// Cleanup: abort the upload
|
||||||
|
err = s3.AbortMultipartUpload(AbortMultipartUploadInput{
|
||||||
|
Bucket: bucket,
|
||||||
|
ObjectKey: objectKey,
|
||||||
|
UploadID: output.UploadID,
|
||||||
|
})
|
||||||
|
if err != nil {
|
||||||
|
t.Logf("Warning: failed to abort upload: %v", err)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
|
||||||
|
// Test 2: Complete multipart upload workflow
|
||||||
|
t.Run("CompleteWorkflow", func(t *testing.T) {
|
||||||
|
// Create test data (3 parts of 5MB each)
|
||||||
|
partSize := int64(5 * 1024 * 1024) // 5MB
|
||||||
|
numParts := 3
|
||||||
|
totalSize := partSize * int64(numParts)
|
||||||
|
|
||||||
|
testData := make([]byte, totalSize)
|
||||||
|
_, err := rand.Read(testData)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("Failed to generate test data: %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
// Initiate
|
||||||
|
initOutput, err := s3.InitiateMultipartUpload(InitiateMultipartUploadInput{
|
||||||
|
Bucket: bucket,
|
||||||
|
ObjectKey: objectKey,
|
||||||
|
ContentType: "application/octet-stream",
|
||||||
|
})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("InitiateMultipartUpload failed: %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
// Upload parts
|
||||||
|
var completedParts []CompletedPart
|
||||||
|
for i := 0; i < numParts; i++ {
|
||||||
|
start := int64(i) * partSize
|
||||||
|
end := start + partSize
|
||||||
|
if end > totalSize {
|
||||||
|
end = totalSize
|
||||||
|
}
|
||||||
|
|
||||||
|
partData := testData[start:end]
|
||||||
|
|
||||||
|
output, err := s3.UploadPart(UploadPartInput{
|
||||||
|
Bucket: bucket,
|
||||||
|
ObjectKey: objectKey,
|
||||||
|
UploadID: initOutput.UploadID,
|
||||||
|
PartNumber: i + 1,
|
||||||
|
Body: bytes.NewReader(partData),
|
||||||
|
Size: int64(len(partData)),
|
||||||
|
})
|
||||||
|
if err != nil {
|
||||||
|
s3.AbortMultipartUpload(AbortMultipartUploadInput{
|
||||||
|
Bucket: bucket,
|
||||||
|
ObjectKey: objectKey,
|
||||||
|
UploadID: initOutput.UploadID,
|
||||||
|
})
|
||||||
|
t.Fatalf("UploadPart %d failed: %v", i+1, err)
|
||||||
|
}
|
||||||
|
|
||||||
|
if output.ETag == "" {
|
||||||
|
t.Errorf("Part %d ETag should not be empty", i+1)
|
||||||
|
}
|
||||||
|
|
||||||
|
completedParts = append(completedParts, CompletedPart{
|
||||||
|
PartNumber: output.PartNumber,
|
||||||
|
ETag: output.ETag,
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
// Complete
|
||||||
|
completeOutput, err := s3.CompleteMultipartUpload(CompleteMultipartUploadInput{
|
||||||
|
Bucket: bucket,
|
||||||
|
ObjectKey: objectKey,
|
||||||
|
UploadID: initOutput.UploadID,
|
||||||
|
Parts: completedParts,
|
||||||
|
})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("CompleteMultipartUpload failed: %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
if completeOutput.Key != objectKey {
|
||||||
|
t.Errorf("Expected key %s, got %s", objectKey, completeOutput.Key)
|
||||||
|
}
|
||||||
|
if completeOutput.ETag == "" {
|
||||||
|
t.Error("ETag should not be empty")
|
||||||
|
}
|
||||||
|
|
||||||
|
// Verify file was uploaded correctly
|
||||||
|
downloadedFile, err := s3.FileDownload(DownloadInput{
|
||||||
|
Bucket: bucket,
|
||||||
|
ObjectKey: objectKey,
|
||||||
|
})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("FileDownload failed: %v", err)
|
||||||
|
}
|
||||||
|
defer downloadedFile.Close()
|
||||||
|
|
||||||
|
downloadedData, err := io.ReadAll(downloadedFile)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("Failed to read downloaded file: %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
if !bytes.Equal(testData, downloadedData) {
|
||||||
|
t.Error("Downloaded data does not match uploaded data")
|
||||||
|
}
|
||||||
|
})
|
||||||
|
|
||||||
|
// Test 3: Abort multipart upload
|
||||||
|
t.Run("AbortMultipartUpload", func(t *testing.T) {
|
||||||
|
initOutput, err := s3.InitiateMultipartUpload(InitiateMultipartUploadInput{
|
||||||
|
Bucket: bucket,
|
||||||
|
ObjectKey: objectKey + "-abort",
|
||||||
|
})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("InitiateMultipartUpload failed: %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
err = s3.AbortMultipartUpload(AbortMultipartUploadInput{
|
||||||
|
Bucket: bucket,
|
||||||
|
ObjectKey: objectKey + "-abort",
|
||||||
|
UploadID: initOutput.UploadID,
|
||||||
|
})
|
||||||
|
if err != nil {
|
||||||
|
t.Errorf("AbortMultipartUpload failed: %v", err)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
|
||||||
|
// Test 4: ListParts
|
||||||
|
t.Run("ListParts", func(t *testing.T) {
|
||||||
|
// Initiate
|
||||||
|
initOutput, err := s3.InitiateMultipartUpload(InitiateMultipartUploadInput{
|
||||||
|
Bucket: bucket,
|
||||||
|
ObjectKey: objectKey + "-list",
|
||||||
|
})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("InitiateMultipartUpload failed: %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
defer func() {
|
||||||
|
s3.AbortMultipartUpload(AbortMultipartUploadInput{
|
||||||
|
Bucket: bucket,
|
||||||
|
ObjectKey: objectKey + "-list",
|
||||||
|
UploadID: initOutput.UploadID,
|
||||||
|
})
|
||||||
|
}()
|
||||||
|
|
||||||
|
// Upload 2 parts
|
||||||
|
partSize := int64(5 * 1024 * 1024)
|
||||||
|
testData := make([]byte, partSize)
|
||||||
|
|
||||||
|
for i := 1; i <= 2; i++ {
|
||||||
|
_, err := rand.Read(testData)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("Failed to generate test data: %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
_, err = s3.UploadPart(UploadPartInput{
|
||||||
|
Bucket: bucket,
|
||||||
|
ObjectKey: objectKey + "-list",
|
||||||
|
UploadID: initOutput.UploadID,
|
||||||
|
PartNumber: i,
|
||||||
|
Body: bytes.NewReader(testData),
|
||||||
|
Size: partSize,
|
||||||
|
})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("UploadPart %d failed: %v", i, err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// List parts
|
||||||
|
listOutput, err := s3.ListParts(ListPartsInput{
|
||||||
|
Bucket: bucket,
|
||||||
|
ObjectKey: objectKey + "-list",
|
||||||
|
UploadID: initOutput.UploadID,
|
||||||
|
})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("ListParts failed: %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
if len(listOutput.Parts) != 2 {
|
||||||
|
t.Errorf("Expected 2 parts, got %d", len(listOutput.Parts))
|
||||||
|
}
|
||||||
|
|
||||||
|
for i, part := range listOutput.Parts {
|
||||||
|
if part.PartNumber != i+1 {
|
||||||
|
t.Errorf("Expected part number %d, got %d", i+1, part.PartNumber)
|
||||||
|
}
|
||||||
|
if part.ETag == "" {
|
||||||
|
t.Errorf("Part %d ETag should not be empty", i+1)
|
||||||
|
}
|
||||||
|
if part.Size != partSize {
|
||||||
|
t.Errorf("Expected part size %d, got %d", partSize, part.Size)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestFileUploadMultipart(t *testing.T) {
|
||||||
|
s3 := setupTestS3(t)
|
||||||
|
bucket := os.Getenv("AWS_S3_BUCKET")
|
||||||
|
if bucket == "" {
|
||||||
|
t.Skip("AWS_S3_BUCKET not set")
|
||||||
|
}
|
||||||
|
|
||||||
|
objectKey := "test-file-upload-multipart.bin"
|
||||||
|
|
||||||
|
defer func() {
|
||||||
|
s3.FileDelete(DeleteInput{
|
||||||
|
Bucket: bucket,
|
||||||
|
ObjectKey: objectKey,
|
||||||
|
})
|
||||||
|
}()
|
||||||
|
|
||||||
|
// Test 1: Sequential upload
|
||||||
|
t.Run("SequentialUpload", func(t *testing.T) {
|
||||||
|
// Create 15MB test file (3 parts of 5MB each)
|
||||||
|
testData := make([]byte, 15*1024*1024)
|
||||||
|
_, err := rand.Read(testData)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("Failed to generate test data: %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
progressCalls := 0
|
||||||
|
output, err := s3.FileUploadMultipart(MultipartUploadInput{
|
||||||
|
Bucket: bucket,
|
||||||
|
ObjectKey: objectKey,
|
||||||
|
Body: bytes.NewReader(testData),
|
||||||
|
ContentType: "application/octet-stream",
|
||||||
|
PartSize: 5 * 1024 * 1024,
|
||||||
|
OnProgress: func(info ProgressInfo) {
|
||||||
|
progressCalls++
|
||||||
|
t.Logf("Progress: %d/%d bytes (%d/%d parts) @ %d B/s",
|
||||||
|
info.UploadedBytes, info.TotalBytes,
|
||||||
|
info.CurrentPart, info.TotalParts,
|
||||||
|
info.BytesPerSecond)
|
||||||
|
},
|
||||||
|
})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("FileUploadMultipart failed: %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
if output.Key != objectKey {
|
||||||
|
t.Errorf("Expected key %s, got %s", objectKey, output.Key)
|
||||||
|
}
|
||||||
|
if output.ETag == "" {
|
||||||
|
t.Error("ETag should not be empty")
|
||||||
|
}
|
||||||
|
if progressCalls == 0 {
|
||||||
|
t.Error("Progress callback should have been called")
|
||||||
|
}
|
||||||
|
|
||||||
|
// Verify uploaded file
|
||||||
|
downloadedFile, err := s3.FileDownload(DownloadInput{
|
||||||
|
Bucket: bucket,
|
||||||
|
ObjectKey: objectKey,
|
||||||
|
})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("FileDownload failed: %v", err)
|
||||||
|
}
|
||||||
|
defer downloadedFile.Close()
|
||||||
|
|
||||||
|
downloadedData, err := io.ReadAll(downloadedFile)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("Failed to read downloaded file: %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
if !bytes.Equal(testData, downloadedData) {
|
||||||
|
t.Error("Downloaded data does not match uploaded data")
|
||||||
|
}
|
||||||
|
})
|
||||||
|
|
||||||
|
// Test 2: Parallel upload
|
||||||
|
t.Run("ParallelUpload", func(t *testing.T) {
|
||||||
|
objectKeyParallel := objectKey + "-parallel"
|
||||||
|
|
||||||
|
defer func() {
|
||||||
|
s3.FileDelete(DeleteInput{
|
||||||
|
Bucket: bucket,
|
||||||
|
ObjectKey: objectKeyParallel,
|
||||||
|
})
|
||||||
|
}()
|
||||||
|
|
||||||
|
// Create 20MB test file (4 parts of 5MB each)
|
||||||
|
testData := make([]byte, 20*1024*1024)
|
||||||
|
_, err := rand.Read(testData)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("Failed to generate test data: %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
output, err := s3.FileUploadMultipart(MultipartUploadInput{
|
||||||
|
Bucket: bucket,
|
||||||
|
ObjectKey: objectKeyParallel,
|
||||||
|
Body: bytes.NewReader(testData),
|
||||||
|
ContentType: "application/octet-stream",
|
||||||
|
PartSize: 5 * 1024 * 1024,
|
||||||
|
Concurrency: 3, // Upload 3 parts in parallel
|
||||||
|
})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("FileUploadMultipart (parallel) failed: %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
if output.Key != objectKeyParallel {
|
||||||
|
t.Errorf("Expected key %s, got %s", objectKeyParallel, output.Key)
|
||||||
|
}
|
||||||
|
|
||||||
|
// Verify uploaded file
|
||||||
|
downloadedFile, err := s3.FileDownload(DownloadInput{
|
||||||
|
Bucket: bucket,
|
||||||
|
ObjectKey: objectKeyParallel,
|
||||||
|
})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("FileDownload failed: %v", err)
|
||||||
|
}
|
||||||
|
defer downloadedFile.Close()
|
||||||
|
|
||||||
|
downloadedData, err := io.ReadAll(downloadedFile)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("Failed to read downloaded file: %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
if !bytes.Equal(testData, downloadedData) {
|
||||||
|
t.Error("Downloaded data does not match uploaded data")
|
||||||
|
}
|
||||||
|
})
|
||||||
|
|
||||||
|
// Test 3: Small file (single part)
|
||||||
|
t.Run("SmallFile", func(t *testing.T) {
|
||||||
|
objectKeySmall := objectKey + "-small"
|
||||||
|
|
||||||
|
defer func() {
|
||||||
|
s3.FileDelete(DeleteInput{
|
||||||
|
Bucket: bucket,
|
||||||
|
ObjectKey: objectKeySmall,
|
||||||
|
})
|
||||||
|
}()
|
||||||
|
|
||||||
|
// Create 3MB test file (smaller than partSize)
|
||||||
|
testData := make([]byte, 3*1024*1024)
|
||||||
|
_, err := rand.Read(testData)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("Failed to generate test data: %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
output, err := s3.FileUploadMultipart(MultipartUploadInput{
|
||||||
|
Bucket: bucket,
|
||||||
|
ObjectKey: objectKeySmall,
|
||||||
|
Body: bytes.NewReader(testData),
|
||||||
|
ContentType: "application/octet-stream",
|
||||||
|
PartSize: 5 * 1024 * 1024,
|
||||||
|
})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("FileUploadMultipart (small file) failed: %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
if output.Key != objectKeySmall {
|
||||||
|
t.Errorf("Expected key %s, got %s", objectKeySmall, output.Key)
|
||||||
|
}
|
||||||
|
|
||||||
|
// Verify uploaded file
|
||||||
|
downloadedFile, err := s3.FileDownload(DownloadInput{
|
||||||
|
Bucket: bucket,
|
||||||
|
ObjectKey: objectKeySmall,
|
||||||
|
})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("FileDownload failed: %v", err)
|
||||||
|
}
|
||||||
|
defer downloadedFile.Close()
|
||||||
|
|
||||||
|
downloadedData, err := io.ReadAll(downloadedFile)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("Failed to read downloaded file: %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
if !bytes.Equal(testData, downloadedData) {
|
||||||
|
t.Error("Downloaded data does not match uploaded data")
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestMultipartValidation(t *testing.T) {
|
||||||
|
s3 := setupTestS3(t)
|
||||||
|
bucket := os.Getenv("AWS_S3_BUCKET")
|
||||||
|
|
||||||
|
tests := []struct {
|
||||||
|
name string
|
||||||
|
fn func() error
|
||||||
|
expectErr bool
|
||||||
|
}{
|
||||||
|
{
|
||||||
|
name: "InitiateMultipartUpload_NoBucket",
|
||||||
|
fn: func() error {
|
||||||
|
_, err := s3.InitiateMultipartUpload(InitiateMultipartUploadInput{
|
||||||
|
ObjectKey: "test",
|
||||||
|
})
|
||||||
|
return err
|
||||||
|
},
|
||||||
|
expectErr: true,
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "InitiateMultipartUpload_NoKey",
|
||||||
|
fn: func() error {
|
||||||
|
_, err := s3.InitiateMultipartUpload(InitiateMultipartUploadInput{
|
||||||
|
Bucket: bucket,
|
||||||
|
})
|
||||||
|
return err
|
||||||
|
},
|
||||||
|
expectErr: true,
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "UploadPart_NoBucket",
|
||||||
|
fn: func() error {
|
||||||
|
_, err := s3.UploadPart(UploadPartInput{
|
||||||
|
ObjectKey: "test",
|
||||||
|
UploadID: "test",
|
||||||
|
PartNumber: 1,
|
||||||
|
Body: bytes.NewReader([]byte("test")),
|
||||||
|
Size: 4,
|
||||||
|
})
|
||||||
|
return err
|
||||||
|
},
|
||||||
|
expectErr: true,
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "UploadPart_InvalidPartNumber",
|
||||||
|
fn: func() error {
|
||||||
|
_, err := s3.UploadPart(UploadPartInput{
|
||||||
|
Bucket: bucket,
|
||||||
|
ObjectKey: "test",
|
||||||
|
UploadID: "test",
|
||||||
|
PartNumber: 0,
|
||||||
|
Body: bytes.NewReader([]byte("test")),
|
||||||
|
Size: 4,
|
||||||
|
})
|
||||||
|
return err
|
||||||
|
},
|
||||||
|
expectErr: true,
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "UploadPart_PartNumberTooLarge",
|
||||||
|
fn: func() error {
|
||||||
|
_, err := s3.UploadPart(UploadPartInput{
|
||||||
|
Bucket: bucket,
|
||||||
|
ObjectKey: "test",
|
||||||
|
UploadID: "test",
|
||||||
|
PartNumber: MaxParts + 1,
|
||||||
|
Body: bytes.NewReader([]byte("test")),
|
||||||
|
Size: 4,
|
||||||
|
})
|
||||||
|
return err
|
||||||
|
},
|
||||||
|
expectErr: true,
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "CompleteMultipartUpload_NoParts",
|
||||||
|
fn: func() error {
|
||||||
|
_, err := s3.CompleteMultipartUpload(CompleteMultipartUploadInput{
|
||||||
|
Bucket: bucket,
|
||||||
|
ObjectKey: "test",
|
||||||
|
UploadID: "test",
|
||||||
|
Parts: []CompletedPart{},
|
||||||
|
})
|
||||||
|
return err
|
||||||
|
},
|
||||||
|
expectErr: true,
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "FileUploadMultipart_PartSizeTooSmall",
|
||||||
|
fn: func() error {
|
||||||
|
_, err := s3.FileUploadMultipart(MultipartUploadInput{
|
||||||
|
Bucket: bucket,
|
||||||
|
ObjectKey: "test",
|
||||||
|
Body: bytes.NewReader([]byte("test")),
|
||||||
|
PartSize: 1024, // Less than MinPartSize
|
||||||
|
})
|
||||||
|
return err
|
||||||
|
},
|
||||||
|
expectErr: true,
|
||||||
|
},
|
||||||
|
}
|
||||||
|
|
||||||
|
for _, tt := range tests {
|
||||||
|
t.Run(tt.name, func(t *testing.T) {
|
||||||
|
err := tt.fn()
|
||||||
|
if tt.expectErr && err == nil {
|
||||||
|
t.Error("Expected error but got nil")
|
||||||
|
}
|
||||||
|
if !tt.expectErr && err != nil {
|
||||||
|
t.Errorf("Expected no error but got: %v", err)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestGeneratePresignedUploadPartURL(t *testing.T) {
|
||||||
|
s3 := setupTestS3(t)
|
||||||
|
bucket := os.Getenv("AWS_S3_BUCKET")
|
||||||
|
if bucket == "" {
|
||||||
|
t.Skip("AWS_S3_BUCKET not set")
|
||||||
|
}
|
||||||
|
|
||||||
|
url := s3.GeneratePresignedUploadPartURL(PresignedMultipartInput{
|
||||||
|
Bucket: bucket,
|
||||||
|
ObjectKey: "test-presigned.bin",
|
||||||
|
UploadID: "test-upload-id",
|
||||||
|
PartNumber: 1,
|
||||||
|
ExpirySeconds: 3600,
|
||||||
|
})
|
||||||
|
|
||||||
|
if url == "" {
|
||||||
|
t.Error("Presigned URL should not be empty")
|
||||||
|
}
|
||||||
|
|
||||||
|
// Check that URL contains required parameters
|
||||||
|
requiredParams := []string{
|
||||||
|
"partNumber=1",
|
||||||
|
"uploadId=test-upload-id",
|
||||||
|
"X-Amz-Algorithm=AWS4-HMAC-SHA256",
|
||||||
|
"X-Amz-Signature=",
|
||||||
|
}
|
||||||
|
|
||||||
|
for _, param := range requiredParams {
|
||||||
|
if !contains(url, param) {
|
||||||
|
t.Errorf("URL should contain %s", param)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
t.Logf("Generated presigned URL: %s", url)
|
||||||
|
}
|
||||||
|
|
||||||
|
func BenchmarkFileUploadMultipart(b *testing.B) {
|
||||||
|
s3 := setupTestS3(&testing.T{})
|
||||||
|
bucket := os.Getenv("AWS_S3_BUCKET")
|
||||||
|
if bucket == "" {
|
||||||
|
b.Skip("AWS_S3_BUCKET not set")
|
||||||
|
}
|
||||||
|
|
||||||
|
// Create 10MB test data
|
||||||
|
testData := make([]byte, 10*1024*1024)
|
||||||
|
rand.Read(testData)
|
||||||
|
|
||||||
|
b.ResetTimer()
|
||||||
|
for i := 0; i < b.N; i++ {
|
||||||
|
objectKey := fmt.Sprintf("bench-multipart-%d.bin", i)
|
||||||
|
|
||||||
|
_, err := s3.FileUploadMultipart(MultipartUploadInput{
|
||||||
|
Bucket: bucket,
|
||||||
|
ObjectKey: objectKey,
|
||||||
|
Body: bytes.NewReader(testData),
|
||||||
|
ContentType: "application/octet-stream",
|
||||||
|
PartSize: 5 * 1024 * 1024,
|
||||||
|
})
|
||||||
|
if err != nil {
|
||||||
|
b.Fatalf("FileUploadMultipart failed: %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
// Cleanup
|
||||||
|
s3.FileDelete(DeleteInput{
|
||||||
|
Bucket: bucket,
|
||||||
|
ObjectKey: objectKey,
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
+199
@@ -240,3 +240,202 @@ func (s3 *S3) GeneratePresignedURL(in PresignedInput) string {
|
|||||||
|
|
||||||
return b.String()
|
return b.String()
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// PresignedMultipartInput contains parameters for generating presigned multipart upload URLs
|
||||||
|
type PresignedMultipartInput struct {
|
||||||
|
Bucket string // Required: bucket name
|
||||||
|
ObjectKey string // Required: object key
|
||||||
|
UploadID string // Required: upload ID from InitiateMultipartUpload
|
||||||
|
PartNumber int // Required: part number (1-10000)
|
||||||
|
ExpirySeconds int // Optional: default 3600
|
||||||
|
}
|
||||||
|
|
||||||
|
// GeneratePresignedUploadPartURL generates a presigned URL for uploading a specific part
|
||||||
|
// This enables browser-based multipart uploads where the frontend uploads parts directly to S3
|
||||||
|
func (s3 *S3) GeneratePresignedUploadPartURL(in PresignedMultipartInput) string {
|
||||||
|
if in.ExpirySeconds == 0 {
|
||||||
|
in.ExpirySeconds = 3600
|
||||||
|
}
|
||||||
|
|
||||||
|
if err := s3.renewIAMToken(); err != nil {
|
||||||
|
return ""
|
||||||
|
}
|
||||||
|
|
||||||
|
var (
|
||||||
|
nowTime = nowTime()
|
||||||
|
|
||||||
|
protocol = defaultProtocol
|
||||||
|
hostname = defaultPresignedHost
|
||||||
|
path_prefix = ""
|
||||||
|
)
|
||||||
|
|
||||||
|
amzdate := nowTime.Format(amzDateISO8601TimeFormat)
|
||||||
|
|
||||||
|
// Create cred
|
||||||
|
b := bytes.Buffer{}
|
||||||
|
b.WriteString(s3.AccessKey)
|
||||||
|
b.WriteRune('/')
|
||||||
|
b.Write(s3.buildCredentialWithoutKey(nowTime))
|
||||||
|
cred := b.Bytes()
|
||||||
|
b.Reset()
|
||||||
|
|
||||||
|
// Set the protocol as default if not provided.
|
||||||
|
if endpoint, _ := url.Parse(s3.Endpoint); endpoint.Host != "" {
|
||||||
|
protocol = endpoint.Scheme + "://"
|
||||||
|
hostname = endpoint.Host
|
||||||
|
path_prefix = path.Join("/", endpoint.Path, in.Bucket)
|
||||||
|
} else {
|
||||||
|
host := bytes.Buffer{}
|
||||||
|
host.WriteString(in.Bucket)
|
||||||
|
host.WriteRune('.')
|
||||||
|
host.WriteString(hostname)
|
||||||
|
hostname = host.String()
|
||||||
|
}
|
||||||
|
|
||||||
|
// Add host to Headers
|
||||||
|
signedHeaders := map[string][]byte{
|
||||||
|
"host": []byte(hostname),
|
||||||
|
}
|
||||||
|
|
||||||
|
// Build signed headers string
|
||||||
|
sortedSH := make([]string, 0, len(signedHeaders))
|
||||||
|
for name := range signedHeaders {
|
||||||
|
sortedSH = append(sortedSH, name)
|
||||||
|
}
|
||||||
|
sort.Strings(sortedSH)
|
||||||
|
signedHeadersStr := strings.Join(sortedSH, ";")
|
||||||
|
|
||||||
|
// For URL: header names must be individually escaped, semicolons remain raw
|
||||||
|
var signedHeadersForURL strings.Builder
|
||||||
|
for i, name := range sortedSH {
|
||||||
|
if i > 0 {
|
||||||
|
signedHeadersForURL.WriteRune(';')
|
||||||
|
}
|
||||||
|
signedHeadersForURL.WriteString(url.QueryEscape(name))
|
||||||
|
}
|
||||||
|
|
||||||
|
// Start Canonical Request Formation
|
||||||
|
h := sha256.New()
|
||||||
|
h.Write([]byte("PUT")) // Multipart uploads use PUT
|
||||||
|
h.Write(newLine)
|
||||||
|
h.Write([]byte(path_prefix))
|
||||||
|
h.Write([]byte{'/'})
|
||||||
|
h.Write([]byte(encodePath(in.ObjectKey)))
|
||||||
|
h.Write(newLine)
|
||||||
|
|
||||||
|
// Start QueryString Params (before SignedHeaders)
|
||||||
|
queryString := map[string]string{
|
||||||
|
"X-Amz-Algorithm": algorithm,
|
||||||
|
"X-Amz-Credential": string(cred),
|
||||||
|
"X-Amz-Date": amzdate,
|
||||||
|
"X-Amz-Expires": strconv.Itoa(in.ExpirySeconds),
|
||||||
|
HdrXAmzSignedHeaders: signedHeadersForURL.String(),
|
||||||
|
"partNumber": strconv.Itoa(in.PartNumber),
|
||||||
|
"uploadId": in.UploadID,
|
||||||
|
}
|
||||||
|
|
||||||
|
// Include the x-amz-security-token in case we are using IAM role or AWS STS
|
||||||
|
if s3.Token != "" {
|
||||||
|
queryString["X-Amz-Security-Token"] = s3.Token
|
||||||
|
}
|
||||||
|
|
||||||
|
// We need to have a sorted order for QueryStrings and SignedHeaders
|
||||||
|
sortedQS := make([]string, 0, len(queryString))
|
||||||
|
for name := range queryString {
|
||||||
|
sortedQS = append(sortedQS, name)
|
||||||
|
}
|
||||||
|
sort.Strings(sortedQS)
|
||||||
|
|
||||||
|
// Proceed to write canonical query params
|
||||||
|
for i, k := range sortedQS {
|
||||||
|
h.Write([]byte(awsURIEncode(k)))
|
||||||
|
h.Write([]byte{'='})
|
||||||
|
// X-Amz-SignedHeaders already has properly formatted semicolons, retain as is.
|
||||||
|
if k == HdrXAmzSignedHeaders {
|
||||||
|
h.Write([]byte(queryString[k]))
|
||||||
|
} else {
|
||||||
|
h.Write([]byte(awsURIEncode(queryString[k])))
|
||||||
|
}
|
||||||
|
if i < len(sortedQS)-1 {
|
||||||
|
h.Write([]byte{'&'})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
h.Write(newLine)
|
||||||
|
|
||||||
|
// Start Canonical Headers
|
||||||
|
for i := 0; i < len(sortedSH); i++ {
|
||||||
|
h.Write([]byte(strings.ToLower(sortedSH[i])))
|
||||||
|
h.Write([]byte{':'})
|
||||||
|
h.Write([]byte(strings.TrimSpace(string(signedHeaders[sortedSH[i]]))))
|
||||||
|
h.Write(newLine)
|
||||||
|
}
|
||||||
|
h.Write(newLine)
|
||||||
|
|
||||||
|
// Start Signed Headers
|
||||||
|
h.Write([]byte(signedHeadersStr))
|
||||||
|
h.Write(newLine)
|
||||||
|
|
||||||
|
// Mention Unsigned Payload
|
||||||
|
h.Write([]byte("UNSIGNED-PAYLOAD"))
|
||||||
|
|
||||||
|
// Start StringToSign
|
||||||
|
b.WriteString(algorithm)
|
||||||
|
b.WriteRune('\n')
|
||||||
|
b.WriteString(amzdate)
|
||||||
|
b.WriteRune('\n')
|
||||||
|
b.Write(s3.buildCredentialWithoutKey(nowTime))
|
||||||
|
b.WriteRune('\n')
|
||||||
|
|
||||||
|
hashed := hex.EncodeToString(h.Sum(nil))
|
||||||
|
b.WriteString(hashed)
|
||||||
|
|
||||||
|
stringToSign := b.Bytes()
|
||||||
|
|
||||||
|
// Start Signature Key
|
||||||
|
sigKey := makeHMac(makeHMac(
|
||||||
|
makeHMac(
|
||||||
|
makeHMac(
|
||||||
|
[]byte("AWS4"+s3.SecretKey),
|
||||||
|
[]byte(nowTime.UTC().Format(shortTimeFormat))),
|
||||||
|
[]byte(s3.Region)),
|
||||||
|
[]byte("s3")),
|
||||||
|
[]byte("aws4_request"),
|
||||||
|
)
|
||||||
|
|
||||||
|
signedStrToSign := makeHMac(sigKey, stringToSign)
|
||||||
|
signature := hex.EncodeToString(signedStrToSign)
|
||||||
|
|
||||||
|
// Reset Buffer to create URL
|
||||||
|
b.Reset()
|
||||||
|
|
||||||
|
// Start Generating URL
|
||||||
|
if s3.Endpoint != "" {
|
||||||
|
b.WriteString(s3.Endpoint)
|
||||||
|
b.WriteRune('/')
|
||||||
|
b.WriteString(in.Bucket)
|
||||||
|
} else {
|
||||||
|
b.WriteString(protocol)
|
||||||
|
b.WriteString(hostname)
|
||||||
|
}
|
||||||
|
b.WriteRune('/')
|
||||||
|
b.WriteString(encodePath(in.ObjectKey))
|
||||||
|
b.WriteRune('?')
|
||||||
|
|
||||||
|
for i, k := range sortedQS {
|
||||||
|
b.WriteString(awsURIEncode(k))
|
||||||
|
b.WriteRune('=')
|
||||||
|
// X-Amz-SignedHeaders already has properly formatted semicolons
|
||||||
|
if k == HdrXAmzSignedHeaders {
|
||||||
|
b.WriteString(queryString[k])
|
||||||
|
} else {
|
||||||
|
b.WriteString(awsURIEncode(queryString[k]))
|
||||||
|
}
|
||||||
|
if i < len(sortedQS)-1 {
|
||||||
|
b.WriteRune('&')
|
||||||
|
}
|
||||||
|
}
|
||||||
|
b.WriteString("&X-Amz-Signature=")
|
||||||
|
b.WriteString(signature)
|
||||||
|
|
||||||
|
return b.String()
|
||||||
|
}
|
||||||
|
|||||||
+2
-2
@@ -374,10 +374,10 @@ func TestS3_GeneratePresignedURL_ObjectKeyEncoding(t *testing.T) {
|
|||||||
|
|
||||||
// Test with various special characters that MUST be encoded
|
// Test with various special characters that MUST be encoded
|
||||||
testCases := []struct {
|
testCases := []struct {
|
||||||
objectKey string
|
objectKey string
|
||||||
mustNotContain string // Raw character that should be encoded
|
mustNotContain string // Raw character that should be encoded
|
||||||
shouldContain string // Expected encoded version
|
shouldContain string // Expected encoded version
|
||||||
description string
|
description string
|
||||||
}{
|
}{
|
||||||
{
|
{
|
||||||
"file&name.txt",
|
"file&name.txt",
|
||||||
|
|||||||
Reference in New Issue
Block a user