diff --git a/README.md b/README.md index a37e873..51318b5 100644 --- a/README.md +++ b/README.md @@ -12,6 +12,7 @@ using AWS Signature Version 4. - 🔐 **AWS Signature Version 4** signing - 🌐 **Custom endpoint support** (MinIO, DigitalOcean Spaces, etc.) - 📋 **Simple List API** with pagination, prefix filtering, and delimiter grouping +- 🔄 **Iterator-based ListAll** for memory-efficient large bucket iteration (Go 1.23+) - 🔗 **Presigned URL generation** for secure browser uploads/downloads - 🪪 **IAM credential support** for EC2 instances - ✅ **Comprehensive test coverage** @@ -167,7 +168,28 @@ for _, prefix := range result.CommonPrefixes { } ``` -#### Handle Pagination +#### Iterator-based Listing (Go 1.23+) +```go +// Use the new ListAll method for memory-efficient iteration +s3 := simples3.New("us-east-1", "your-access-key", "your-secret-key") + +// Iterate over all objects with automatic pagination and error handling +seq, finish := s3.ListAll(simples3.ListInput{ + Bucket: "my-bucket", + Prefix: "documents/", // Optional filtering +}) + +for obj := range seq { + fmt.Printf("📄 %s (%d bytes)\n", obj.Key, obj.Size) +} + +// Check for any errors that occurred during iteration +if err := finish(); err != nil { + log.Fatalf("Error during iteration: %v", err) +} +``` + +#### Handle Pagination (Legacy) ```go // Handle large result sets with pagination func listAllObjects(s3 *simples3.S3, bucket string) ([]simples3.Object, error) { diff --git a/simples3.go b/simples3.go index 31c6e0f..cb39850 100644 --- a/simples3.go +++ b/simples3.go @@ -12,6 +12,7 @@ import ( "encoding/xml" "fmt" "io" + "iter" "mime/multipart" "net/http" "net/url" @@ -923,12 +924,12 @@ func (s3 *S3) List(input ListInput) (ListResponse, error) { // Parse XML response using internal struct var xmlResult struct { - XMLName xml.Name `xml:"ListBucketResult"` - Name string `xml:"Name"` - IsTruncated bool `xml:"IsTruncated"` + XMLName xml.Name `xml:"ListBucketResult"` + Name string `xml:"Name"` + IsTruncated bool `xml:"IsTruncated"` NextContinuationToken string `xml:"NextContinuationToken"` - KeyCount int64 `xml:"KeyCount"` - Contents []struct { + KeyCount int64 `xml:"KeyCount"` + Contents []struct { Key string `xml:"Key"` LastModified string `xml:"LastModified"` ETag string `xml:"ETag"` @@ -949,7 +950,7 @@ func (s3 *S3) List(input ListInput) (ListResponse, error) { Name: xmlResult.Name, IsTruncated: xmlResult.IsTruncated, NextContinuationToken: xmlResult.NextContinuationToken, - KeyCount: xmlResult.KeyCount, + KeyCount: xmlResult.KeyCount, } // Convert objects @@ -971,7 +972,45 @@ func (s3 *S3) List(input ListInput) (ListResponse, error) { return response, nil } +// ListAll returns an iterator that yields all objects in the bucket, +// automatically handling pagination. It also returns a finish callback +// that should be called after iteration to check for any errors. +func (s3 *S3) ListAll(input ListInput) (iter.Seq[Object], func() error) { + var iterErr error + seq := func(yield func(Object) bool) { + currentInput := input + + for { + response, err := s3.List(currentInput) + if err != nil { + iterErr = err + return + } + + // Yield each object + for _, obj := range response.Objects { + if !yield(obj) { + return // Early termination requested + } + } + + // Check if there are more results + if !response.IsTruncated || response.NextContinuationToken == "" { + return // No more results + } + + // Prepare for next page + currentInput.ContinuationToken = response.NextContinuationToken + } + } + + finish := func() error { + return iterErr + } + + return seq, finish +} func getFirstString(s []string) string { if len(s) > 0 { diff --git a/simples3_test.go b/simples3_test.go index 405d91b..9a4c229 100644 --- a/simples3_test.go +++ b/simples3_test.go @@ -493,6 +493,141 @@ func TestS3_NewUsingIAM(t *testing.T) { } } +func TestListAll(t *testing.T) { + // Setup test environment + s3 := New( + os.Getenv("AWS_S3_REGION"), + os.Getenv("AWS_S3_ACCESS_KEY"), + os.Getenv("AWS_S3_SECRET_KEY"), + ) + s3.SetEndpoint(os.Getenv("AWS_S3_ENDPOINT")) + + // Test cases + tests := []struct { + name string + setup func() // Setup function to prepare test data + input ListInput + validate func([]Object, error) + cleanup func() // Cleanup function + }{ + { + name: "Empty Bucket Iterator", + input: ListInput{ + Bucket: os.Getenv("AWS_S3_BUCKET"), + }, + validate: func(objects []Object, err error) { + if err != nil { + t.Fatalf("Expected no error, got %v", err) + } + if len(objects) != 0 { + t.Errorf("Expected empty bucket, got %d objects", len(objects)) + } + }, + }, + { + name: "Basic Iterator Listing", + setup: func() { + // Upload test files + uploadTestFiles(t, s3, os.Getenv("AWS_S3_BUCKET"), []string{"iter_file1.txt", "iter_file2.txt", "iter_file3.txt"}) + }, + input: ListInput{ + Bucket: os.Getenv("AWS_S3_BUCKET"), + }, + validate: func(objects []Object, err error) { + if err != nil { + t.Fatalf("Expected no error, got %v", err) + } + if len(objects) != 3 { + t.Errorf("Expected 3 objects, got %d", len(objects)) + } + // Check that all expected files are present + expectedKeys := map[string]bool{ + "iter_file1.txt": false, + "iter_file2.txt": false, + "iter_file3.txt": false, + } + for _, obj := range objects { + if _, exists := expectedKeys[obj.Key]; exists { + expectedKeys[obj.Key] = true + } + } + for key, found := range expectedKeys { + if !found { + t.Errorf("Expected object %s not found in iterator results", key) + } + } + }, + cleanup: func() { + cleanupTestFiles(t, s3, os.Getenv("AWS_S3_BUCKET"), []string{"iter_file1.txt", "iter_file2.txt", "iter_file3.txt"}) + }, + }, + { + name: "Iterator with Prefix", + setup: func() { + uploadTestFiles(t, s3, os.Getenv("AWS_S3_BUCKET"), []string{ + "iter_docs/file1.txt", + "iter_docs/file2.txt", + "iter_images/image1.jpg", + "iter_images/image2.jpg", + }) + }, + input: ListInput{ + Bucket: os.Getenv("AWS_S3_BUCKET"), + Prefix: "iter_docs/", + }, + validate: func(objects []Object, err error) { + if err != nil { + t.Fatalf("Expected no error, got %v", err) + } + if len(objects) != 2 { + t.Errorf("Expected 2 objects with iter_docs/ prefix, got %d", len(objects)) + } + for _, obj := range objects { + if !strings.HasPrefix(obj.Key, "iter_docs/") { + t.Errorf("Object %s doesn't have expected prefix", obj.Key) + } + } + }, + cleanup: func() { + cleanupTestFiles(t, s3, os.Getenv("AWS_S3_BUCKET"), []string{ + "iter_docs/file1.txt", + "iter_docs/file2.txt", + "iter_images/image1.jpg", + "iter_images/image2.jpg", + }) + }, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if tt.setup != nil { + tt.setup() + } + + // Collect all objects from the iterator and check for errors + var objects []Object + seq, finish := s3.ListAll(tt.input) + for obj := range seq { + objects = append(objects, obj) + } + + // Check for any iteration errors + if err := finish(); err != nil { + t.Fatalf("Iterator error: %v", err) + } + + if tt.validate != nil { + tt.validate(objects, nil) // No error from iterator + } + + if tt.cleanup != nil { + tt.cleanup() + } + }) + } +} + func TestCustomEndpoint(t *testing.T) { s3 := New("us-east-1", "AccessKey", "SuperSecretKey")