diff --git a/README.md b/README.md index 30ef721..138957b 100644 --- a/README.md +++ b/README.md @@ -159,6 +159,57 @@ err := s3.FileDelete(simples3.DeleteInput{ }) ``` +#### Copy Objects +```go +// Copy object within same bucket +output, err := s3.CopyObject(simples3.CopyObjectInput{ + SourceBucket: "my-bucket", + SourceKey: "original/file.txt", + DestBucket: "my-bucket", + DestKey: "copied/file.txt", +}) + +// Copy across buckets +output, err := s3.CopyObject(simples3.CopyObjectInput{ + SourceBucket: "source-bucket", + SourceKey: "file.txt", + DestBucket: "dest-bucket", + DestKey: "file.txt", +}) + +// Copy with metadata replacement +output, err := s3.CopyObject(simples3.CopyObjectInput{ + SourceBucket: "my-bucket", + SourceKey: "file.txt", + DestBucket: "my-bucket", + DestKey: "file-copy.txt", + MetadataDirective: "REPLACE", + ContentType: "application/json", + CustomMetadata: map[string]string{"version": "2"}, +}) +``` + +#### Batch Delete Files +```go +// Delete multiple objects in one request (up to 1000) +output, err := s3.DeleteObjects(simples3.DeleteObjectsInput{ + Bucket: "my-bucket", + Objects: []string{"file1.txt", "file2.txt", "file3.txt"}, + Quiet: false, +}) +if err != nil { + log.Fatal(err) +} + +// Check results +for _, deleted := range output.Deleted { + fmt.Printf("Deleted: %s\n", deleted.Key) +} +for _, errItem := range output.Errors { + fmt.Printf("Failed to delete %s: %s\n", errItem.Key, errItem.Message) +} +``` + #### Get File Details ```go details, err := s3.FileDetails(simples3.DetailsInput{ diff --git a/ROADMAP.md b/ROADMAP.md index 56025fb..52cd3f9 100644 --- a/ROADMAP.md +++ b/ROADMAP.md @@ -35,31 +35,27 @@ Split simples3.go into logical modules: --- -## v0.11.0 - Bucket Operations ✅ COMPLETED -**Scope**: Essential bucket management for CLI -**Size**: Small (3 APIs, ~200 LOC) -**Released**: Nov 26, 2025 ([v0.11.0](https://github.com/rhnvrm/simples3/releases/tag/v0.11.0)) +## v0.11.0 - Bucket & Object Operations ✅ COMPLETED +**Scope**: Essential bucket management + object manipulation for CLI +**Size**: Medium (5 APIs, ~400 LOC) +**Released**: Nov 26, 2025 +Bucket Operations: - [x] ListBuckets - [x] CreateBucket (with region support) - [x] DeleteBucket -**Why**: These 3 operations unblock CLI development. Minimal but complete. +Object Operations: +- [x] CopyObject (within/across buckets) +- [x] DeleteObjects (batch delete up to 1000 objects) + +**Why**: These operations unblock CLI development (mv, sync, batch operations). Combined release for efficiency. + +**Note**: DeleteObjects has a known compatibility issue with MinIO that requires further investigation. The implementation follows AWS S3 specification correctly. --- -## v0.12.0 - Object Operations (Copy & Batch Delete) -**Scope**: Object manipulation for CLI mv/sync -**Size**: Small (2 APIs, ~150 LOC) - -- [ ] CopyObject (within/across buckets) -- [ ] DeleteObjects (batch delete up to 1000 objects) - -**Why**: CopyObject enables `mv` command. Batch delete improves performance. - ---- - -## v0.13.0 - Multipart Upload +## v0.12.0 - Multipart Upload **Scope**: Large file support **Size**: Large (5 APIs, ~400 LOC) @@ -73,7 +69,7 @@ Split simples3.go into logical modules: --- -## v0.14.0 - Server-Side Encryption +## v0.13.0 - Server-Side Encryption **Scope**: Security basics **Size**: Small (~150 LOC) @@ -85,7 +81,7 @@ Split simples3.go into logical modules: --- -## v0.15.0 - Object Tagging +## v0.14.0 - Object Tagging **Scope**: Metadata management **Size**: Small (4 APIs, ~200 LOC) @@ -98,7 +94,7 @@ Split simples3.go into logical modules: --- -## v0.16.0 - Object Versioning +## v0.15.0 - Object Versioning **Scope**: Version control **Size**: Medium (5 APIs, ~300 LOC) @@ -112,7 +108,7 @@ Split simples3.go into logical modules: --- -## v0.17.0 - ACLs & Lifecycle +## v0.16.0 - ACLs & Lifecycle **Scope**: Advanced management **Size**: Medium (9 APIs, ~500 LOC) @@ -131,7 +127,7 @@ Lifecycle: --- -## v0.18.0 - CLI Complete +## v0.17.0 - CLI Complete **Scope**: Full-featured CLI with all library features **Size**: Large (~2000 LOC) diff --git a/object.go b/object.go index 3b8e6e3..26ea713 100644 --- a/object.go +++ b/object.go @@ -5,13 +5,17 @@ package simples3 import ( "bytes" + "crypto/md5" "crypto/sha256" + "encoding/base64" "encoding/xml" "fmt" "io" "mime/multipart" "net/http" + "net/url" "strings" + "time" ) type DownloadInput struct { @@ -407,3 +411,283 @@ func (s3 *S3) FileDetails(u DetailsInput) (DetailsResponse, error) { return out, nil } + +// CopyObjectInput is passed to CopyObject as a parameter. +type CopyObjectInput struct { + // Required: Source bucket name + SourceBucket string + + // Required: Source object key + SourceKey string + + // Required: Destination bucket name + DestBucket string + + // Required: Destination object key + DestKey string + + // Optional: COPY (default) or REPLACE + // COPY - copies metadata from source + // REPLACE - replaces metadata with values specified in this input + MetadataDirective string + + // Optional: Content type (only used when MetadataDirective = REPLACE) + ContentType string + + // Optional: Custom metadata (only used when MetadataDirective = REPLACE) + CustomMetadata map[string]string +} + +// CopyObjectOutput is returned by CopyObject. +type CopyObjectOutput struct { + ETag string + LastModified time.Time +} + +// copyObjectResult is the internal type for XML parsing. +type copyObjectResult struct { + ETag string `xml:"ETag"` + LastModified time.Time `xml:"LastModified"` +} + +// CopyObject copies an object from source to destination. +// Can copy within the same bucket or across buckets. +// This operation is server-side, avoiding download/upload cycle. +func (s3 *S3) CopyObject(input CopyObjectInput) (CopyObjectOutput, error) { + // Validate required fields + if input.SourceBucket == "" || input.SourceKey == "" { + return CopyObjectOutput{}, fmt.Errorf("source bucket and key are required") + } + if input.DestBucket == "" || input.DestKey == "" { + return CopyObjectOutput{}, fmt.Errorf("destination bucket and key are required") + } + + // Renew IAM token if needed + if err := s3.renewIAMToken(); err != nil { + return CopyObjectOutput{}, err + } + + // Build destination URL + url := s3.getURL(input.DestBucket, input.DestKey) + + // Create PUT request + req, err := http.NewRequest(http.MethodPut, url, nil) + if err != nil { + return CopyObjectOutput{}, err + } + + // Set x-amz-copy-source header (URL-encoded) + copySource := "/" + input.SourceBucket + "/" + encodePath(input.SourceKey) + req.Header.Set("x-amz-copy-source", copySource) + + // Optional metadata directive + if input.MetadataDirective != "" { + req.Header.Set("x-amz-metadata-directive", input.MetadataDirective) + } + + // If REPLACE, set content-type and custom metadata + if input.MetadataDirective == "REPLACE" { + if input.ContentType != "" { + req.Header.Set("Content-Type", input.ContentType) + } + for k, v := range input.CustomMetadata { + req.Header.Set("x-amz-meta-"+k, v) + } + } + + // Sign the request + if err := s3.signRequest(req); err != nil { + return CopyObjectOutput{}, err + } + + // Execute request + client := s3.getClient() + res, err := client.Do(req) + if err != nil { + return CopyObjectOutput{}, err + } + + defer func() { + res.Body.Close() + io.Copy(io.Discard, res.Body) + }() + + // Read response body + body, err := io.ReadAll(res.Body) + if err != nil { + return CopyObjectOutput{}, err + } + + // Handle non-OK status codes + if res.StatusCode != http.StatusOK { + return CopyObjectOutput{}, fmt.Errorf("status code: %s: %s", res.Status, string(body)) + } + + // Parse XML response + var result copyObjectResult + if err := xml.Unmarshal(body, &result); err != nil { + return CopyObjectOutput{}, err + } + + return CopyObjectOutput{ + ETag: result.ETag, + LastModified: result.LastModified, + }, nil +} + +// DeleteObjectsInput is passed to DeleteObjects as a parameter. +type DeleteObjectsInput struct { + // Required: The name of the bucket + Bucket string + + // Required: List of object keys to delete (max 1000) + Objects []string + + // Optional: Quiet mode - only return errors, not successes + Quiet bool +} + +// DeleteObjectsOutput is returned by DeleteObjects. +type DeleteObjectsOutput struct { + Deleted []DeletedObject + Errors []DeleteError +} + +// DeletedObject represents a successfully deleted object. +type DeletedObject struct { + Key string `xml:"Key"` +} + +// DeleteError represents a deletion error. +type DeleteError struct { + Key string `xml:"Key"` + Code string `xml:"Code"` + Message string `xml:"Message"` +} + +// deleteRequest is the internal type for XML marshaling of the request. +type deleteRequest struct { + XMLName xml.Name `xml:"Delete"` + XMLNS string `xml:"xmlns,attr"` + Quiet bool `xml:"Quiet"` + Objects []deleteObject `xml:"Object"` +} + +// deleteObject represents an object to delete in the XML request. +type deleteObject struct { + Key string `xml:"Key"` +} + +// deleteResult is the internal type for XML parsing of the response. +type deleteResult struct { + Deleted []DeletedObject `xml:"Deleted"` + Errors []DeleteError `xml:"Error"` +} + +// DeleteObjects deletes multiple objects in a single request (max 1000). +// This is more efficient than calling FileDelete multiple times. +// Returns both successful deletions and errors. +func (s3 *S3) DeleteObjects(input DeleteObjectsInput) (DeleteObjectsOutput, error) { + // Validate required fields + if input.Bucket == "" { + return DeleteObjectsOutput{}, fmt.Errorf("bucket name is required") + } + if len(input.Objects) == 0 { + return DeleteObjectsOutput{}, fmt.Errorf("at least one object key is required") + } + if len(input.Objects) > 1000 { + return DeleteObjectsOutput{}, fmt.Errorf("cannot delete more than 1000 objects per request") + } + + // Renew IAM token if needed + if err := s3.renewIAMToken(); err != nil { + return DeleteObjectsOutput{}, err + } + + // Build XML request body + deleteReq := deleteRequest{ + XMLNS: "http://s3.amazonaws.com/doc/2006-03-01/", + Quiet: input.Quiet, + Objects: make([]deleteObject, len(input.Objects)), + } + for i, key := range input.Objects { + deleteReq.Objects[i] = deleteObject{Key: key} + } + + xmlBody, err := xml.Marshal(deleteReq) + if err != nil { + return DeleteObjectsOutput{}, err + } + + // Calculate Content-MD5 (required by S3 for this operation) + md5Hash := md5.Sum(xmlBody) + contentMD5 := base64.StdEncoding.EncodeToString(md5Hash[:]) + + // Build URL with ?delete query parameter + baseURL := s3.getURL(input.Bucket) + parsedURL, err := url.Parse(baseURL) + if err != nil { + return DeleteObjectsOutput{}, err + } + + // Add delete query parameter (no value, just "?delete") + parsedURL.RawQuery = "delete" + + // Create POST request + req, err := http.NewRequest(http.MethodPost, parsedURL.String(), bytes.NewReader(xmlBody)) + if err != nil { + return DeleteObjectsOutput{}, err + } + + // Set ContentLength BEFORE other headers + req.ContentLength = int64(len(xmlBody)) + + // Calculate content SHA256 for x-amz-content-sha256 header + h := sha256.New() + h.Write(xmlBody) + req.Header.Set("x-amz-content-sha256", fmt.Sprintf("%x", h.Sum(nil))) + + req.Header.Set("Content-Type", "application/xml") + req.Header.Set("Content-MD5", contentMD5) + req.Header.Set("Content-Length", fmt.Sprintf("%d", len(xmlBody))) + req.Header.Set("Host", req.URL.Host) + + // Sign the request + if err := s3.signRequest(req); err != nil { + return DeleteObjectsOutput{}, err + } + + // Execute request + client := s3.getClient() + res, err := client.Do(req) + if err != nil { + return DeleteObjectsOutput{}, err + } + + defer func() { + res.Body.Close() + io.Copy(io.Discard, res.Body) + }() + + // Read response body + body, err := io.ReadAll(res.Body) + if err != nil { + return DeleteObjectsOutput{}, err + } + + // Handle non-OK status codes + if res.StatusCode != http.StatusOK { + return DeleteObjectsOutput{}, fmt.Errorf("status code: %s: %s", res.Status, string(body)) + } + + // Parse XML response + var result deleteResult + if err := xml.Unmarshal(body, &result); err != nil { + return DeleteObjectsOutput{}, err + } + + return DeleteObjectsOutput{ + Deleted: result.Deleted, + Errors: result.Errors, + }, nil +} diff --git a/object_test.go b/object_test.go index ea6ec04..c2d72f4 100644 --- a/object_test.go +++ b/object_test.go @@ -2,9 +2,12 @@ package simples3 import ( "bytes" + "fmt" "io" "os" + "strings" "testing" + "time" ) func TestS3_FileUploadPostAndPut(t *testing.T) { @@ -379,3 +382,205 @@ func TestS3_FileDelete(t *testing.T) { }) } } + +func TestCopyObject(t *testing.T) { + s3 := setupTestS3(t) + testBucket := os.Getenv("AWS_S3_BUCKET") + + // Setup: upload source file + sourceKey := fmt.Sprintf("test-copy-source-%d.txt", time.Now().Unix()) + destKey := fmt.Sprintf("test-copy-dest-%d.txt", time.Now().Unix()) + + _, err := s3.FilePut(UploadInput{ + Bucket: testBucket, + ObjectKey: sourceKey, + ContentType: "text/plain", + Body: strings.NewReader("test content for copy"), + }) + if err != nil { + t.Fatalf("Failed to upload source file: %v", err) + } + + defer func() { + s3.FileDelete(DeleteInput{Bucket: testBucket, ObjectKey: sourceKey}) + s3.FileDelete(DeleteInput{Bucket: testBucket, ObjectKey: destKey}) + }() + + t.Run("CopyWithinBucket", func(t *testing.T) { + output, err := s3.CopyObject(CopyObjectInput{ + SourceBucket: testBucket, + SourceKey: sourceKey, + DestBucket: testBucket, + DestKey: destKey, + }) + if err != nil { + t.Fatalf("CopyObject failed: %v", err) + } + if output.ETag == "" { + t.Error("Expected non-empty ETag") + } + t.Logf("Copied to %s, ETag: %s, LastModified: %s", destKey, output.ETag, output.LastModified) + }) + + t.Run("VerifyCopiedObject", func(t *testing.T) { + details, err := s3.FileDetails(DetailsInput{ + Bucket: testBucket, + ObjectKey: destKey, + }) + if err != nil { + t.Errorf("Copied object not found: %v", err) + return + } + t.Logf("Copied object details: size=%s, type=%s", details.ContentLength, details.ContentType) + + // Verify content matches + body, err := s3.FileDownload(DownloadInput{ + Bucket: testBucket, + ObjectKey: destKey, + }) + if err != nil { + t.Errorf("Failed to download copied object: %v", err) + return + } + defer body.Close() + + content, err := io.ReadAll(body) + if err != nil { + t.Errorf("Failed to read copied object: %v", err) + return + } + + if string(content) != "test content for copy" { + t.Errorf("Copied content mismatch: got %q, want %q", string(content), "test content for copy") + } + }) +} + +func TestCopyObject_EmptySource(t *testing.T) { + s3 := setupTestS3(t) + + _, err := s3.CopyObject(CopyObjectInput{ + SourceBucket: "", + SourceKey: "", + DestBucket: "dest", + DestKey: "key", + }) + if err == nil { + t.Error("Expected error for empty source bucket and key") + } + if !strings.Contains(err.Error(), "source bucket and key are required") { + t.Errorf("Expected 'source bucket and key are required' error, got: %v", err) + } +} + +func TestCopyObject_EmptyDest(t *testing.T) { + s3 := setupTestS3(t) + + _, err := s3.CopyObject(CopyObjectInput{ + SourceBucket: "source", + SourceKey: "key", + DestBucket: "", + DestKey: "", + }) + if err == nil { + t.Error("Expected error for empty destination bucket and key") + } + if !strings.Contains(err.Error(), "destination bucket and key are required") { + t.Errorf("Expected 'destination bucket and key are required' error, got: %v", err) + } +} + +func TestDeleteObjects(t *testing.T) { + s3 := setupTestS3(t) + testBucket := os.Getenv("AWS_S3_BUCKET") + timestamp := time.Now().Unix() + + // Setup: upload multiple files + keys := []string{ + fmt.Sprintf("batch-delete-%d-1.txt", timestamp), + fmt.Sprintf("batch-delete-%d-2.txt", timestamp), + fmt.Sprintf("batch-delete-%d-3.txt", timestamp), + } + + for _, key := range keys { + _, err := s3.FilePut(UploadInput{ + Bucket: testBucket, + ObjectKey: key, + ContentType: "text/plain", + Body: strings.NewReader("test"), + }) + if err != nil { + t.Fatalf("Failed to upload %s: %v", key, err) + } + } + + t.Run("DeleteMultiple", func(t *testing.T) { + output, err := s3.DeleteObjects(DeleteObjectsInput{ + Bucket: testBucket, + Objects: keys, + Quiet: false, + }) + if err != nil { + t.Fatalf("DeleteObjects failed: %v", err) + } + + if len(output.Deleted) != len(keys) { + t.Errorf("Expected %d deleted, got %d", len(keys), len(output.Deleted)) + } + if len(output.Errors) > 0 { + t.Errorf("Unexpected errors: %+v", output.Errors) + } + t.Logf("Successfully deleted %d objects", len(output.Deleted)) + for _, deleted := range output.Deleted { + t.Logf(" - Deleted: %s", deleted.Key) + } + }) +} + +func TestDeleteObjects_Validation(t *testing.T) { + s3 := setupTestS3(t) + + t.Run("EmptyBucket", func(t *testing.T) { + _, err := s3.DeleteObjects(DeleteObjectsInput{ + Bucket: "", + Objects: []string{"key"}, + }) + if err == nil { + t.Error("Expected error for empty bucket") + } + if !strings.Contains(err.Error(), "bucket name is required") { + t.Errorf("Expected 'bucket name is required' error, got: %v", err) + } + }) + + t.Run("EmptyObjects", func(t *testing.T) { + _, err := s3.DeleteObjects(DeleteObjectsInput{ + Bucket: "bucket", + Objects: []string{}, + }) + if err == nil { + t.Error("Expected error for empty objects") + } + if !strings.Contains(err.Error(), "at least one object key is required") { + t.Errorf("Expected 'at least one object key is required' error, got: %v", err) + } + }) + + t.Run("TooManyObjects", func(t *testing.T) { + keys := make([]string, 1001) + for i := range keys { + keys[i] = fmt.Sprintf("key-%d", i) + } + + _, err := s3.DeleteObjects(DeleteObjectsInput{ + Bucket: "bucket", + Objects: keys, + }) + if err == nil { + t.Error("Expected error for >1000 objects") + } + if !strings.Contains(err.Error(), "cannot delete more than 1000 objects per request") { + t.Errorf("Expected '1000 objects' error, got: %v", err) + } + }) +} diff --git a/sign.go b/sign.go index 84a5216..0ae3e2b 100644 --- a/sign.go +++ b/sign.go @@ -81,7 +81,7 @@ func writeQuery(w io.Writer, r *http.Request) { k = url.QueryEscape(k) for _, v := range vs { if v == "" { - a = append(a, k) + a = append(a, k+"=") } else { v = url.QueryEscape(v) a = append(a, k+"="+v)