From e410cc846e50579def3e7c10cb17a9d6b19d9f95 Mon Sep 17 00:00:00 2001 From: Rohan Verma Date: Wed, 26 Nov 2025 21:11:00 +0530 Subject: [PATCH] feat: add bucket operations (ListBuckets, CreateBucket, DeleteBucket) Adds bucket management capabilities: - ListBuckets: list all buckets for AWS account with owner info - CreateBucket: create bucket with optional region support - DeleteBucket: delete empty buckets Includes comprehensive tests and README documentation. Prompt: "Please review the changes we have made for this bucket api" --- README.md | 47 +++++++++ ROADMAP.md | 24 ++--- bucket.go | 246 +++++++++++++++++++++++++++++++++++++++++++++++ bucket_test.go | 254 +++++++++++++++++++++++++++++++++++++++++++++++++ 4 files changed, 560 insertions(+), 11 deletions(-) create mode 100644 bucket.go create mode 100644 bucket_test.go diff --git a/README.md b/README.md index 1f790f7..30ef721 100644 --- a/README.md +++ b/README.md @@ -9,6 +9,7 @@ using AWS Signature Version 4. **Features:** - **Simple, intuitive API** following Go idioms - **Complete S3 operations** - Upload, Download, Delete, List, Details +- **Bucket management** - Create, Delete, and List buckets - **AWS Signature Version 4** signing - **Custom endpoint support** (MinIO, DigitalOcean Spaces, etc.) - **Simple List API** with pagination, prefix filtering, and delimiter grouping @@ -63,6 +64,52 @@ func main() { ## API Reference +### Bucket Operations + +#### List All Buckets +```go +// List all buckets for the AWS account +result, err := s3.ListBuckets(simples3.ListBucketsInput{}) +if err != nil { + log.Fatal(err) +} + +fmt.Printf("Found %d buckets:\n", len(result.Buckets)) +for _, bucket := range result.Buckets { + fmt.Printf("- %s (created: %s)\n", bucket.Name, bucket.CreationDate) +} + +// Owner information is also available +fmt.Printf("Owner: %s (%s)\n", result.Owner.DisplayName, result.Owner.ID) +``` + +#### Create a Bucket +```go +// Create a new bucket +output, err := s3.CreateBucket(simples3.CreateBucketInput{ + Bucket: "my-new-bucket", + Region: "us-east-1", // Optional, defaults to client region +}) +if err != nil { + log.Fatal(err) +} + +fmt.Printf("Bucket created at: %s\n", output.Location) +``` + +#### Delete a Bucket +```go +// Delete an empty bucket +err := s3.DeleteBucket(simples3.DeleteBucketInput{ + Bucket: "my-old-bucket", +}) +if err != nil { + log.Fatal(err) +} + +// Note: The bucket must be empty before deletion +``` + ### File Operations #### Upload Files diff --git a/ROADMAP.md b/ROADMAP.md index 8e2b136..56025fb 100644 --- a/ROADMAP.md +++ b/ROADMAP.md @@ -18,29 +18,31 @@ - Content-Disposition and ACL support - Zero dependencies (stdlib only) -## v0.10.2 - Code Organization (Refactor) +## v0.10.2 - Code Organization (Refactor) ✅ COMPLETED **Scope**: Internal restructuring, no API changes **Size**: Medium (refactor ~1064 LOC) +**Released**: Nov 26, 2025 ([v0.10.2](https://github.com/rhnvrm/simples3/releases/tag/v0.10.2)) Split simples3.go into logical modules: -- [ ] `simples3.go` - Core (S3 struct, New(), config methods: SetEndpoint/SetToken/SetClient) -- [ ] `iam.go` - IAM (fetchIMDSToken, fetchIAMData, NewUsingIAM, renewIAMToken, SetIAMData) -- [ ] `object.go` - Object ops (FileUpload, FilePut, FileDownload, FileDelete, FileDetails) -- [ ] `list.go` - List ops (List, ListAll) -- [ ] `helpers.go` - Utils (encodePath, detectFileSize, getFirstString, getURL) -- [ ] Ensure all tests pass after refactor +- [x] `simples3.go` - Core (S3 struct, New(), config methods: SetEndpoint/SetToken/SetClient) +- [x] `iam.go` - IAM (fetchIMDSToken, fetchIAMData, NewUsingIAM, renewIAMToken, SetIAMData) +- [x] `object.go` - Object ops (FileUpload, FilePut, FileDownload, FileDelete, FileDetails) +- [x] `list.go` - List ops (List, ListAll) +- [x] `helpers.go` - Utils (encodePath, detectFileSize, getFirstString, getURL) +- [x] Ensure all tests pass after refactor **Why**: 1064-line file hard to navigate. Clean foundation before adding features. --- -## v0.11.0 - Bucket Operations +## v0.11.0 - Bucket Operations ✅ COMPLETED **Scope**: Essential bucket management for CLI **Size**: Small (3 APIs, ~200 LOC) +**Released**: Nov 26, 2025 ([v0.11.0](https://github.com/rhnvrm/simples3/releases/tag/v0.11.0)) -- [ ] ListBuckets -- [ ] CreateBucket (with region support) -- [ ] DeleteBucket +- [x] ListBuckets +- [x] CreateBucket (with region support) +- [x] DeleteBucket **Why**: These 3 operations unblock CLI development. Minimal but complete. diff --git a/bucket.go b/bucket.go new file mode 100644 index 0000000..e24d01d --- /dev/null +++ b/bucket.go @@ -0,0 +1,246 @@ +// LICENSE BSD-2-Clause-FreeBSD +// Copyright (c) 2018, Rohan Verma + +package simples3 + +import ( + "encoding/xml" + "fmt" + "io" + "net/http" + "strings" + "time" +) + +// Bucket represents a single S3 bucket. +type Bucket struct { + Name string `xml:"Name"` + CreationDate time.Time `xml:"CreationDate"` +} + +// ListBucketsInput is passed to ListBuckets as a parameter. +type ListBucketsInput struct { + // Empty - ListBuckets lists all buckets for the account +} + +// ListBucketsOutput is returned by ListBuckets. +type ListBucketsOutput struct { + Buckets []Bucket `xml:"Buckets>Bucket"` + Owner struct { + ID string `xml:"ID"` + DisplayName string `xml:"DisplayName"` + } `xml:"Owner"` +} + +// CreateBucketInput is passed to CreateBucket as a parameter. +type CreateBucketInput struct { + // Required: The name of the bucket to create + Bucket string + + // Optional: AWS region for the bucket. + // If empty, uses the region from S3 struct. + // Note: For us-east-1, no LocationConstraint is needed. + Region string +} + +// CreateBucketOutput is returned by CreateBucket. +type CreateBucketOutput struct { + // Location header from the response + Location string +} + +// DeleteBucketInput is passed to DeleteBucket as a parameter. +type DeleteBucketInput struct { + // Required: The name of the bucket to delete + // Note: The bucket must be empty before deletion + Bucket string +} + +// ListBuckets lists all S3 buckets for the AWS account. +// It makes a GET request to the S3 service endpoint (not a specific bucket). +func (s3 *S3) ListBuckets(input ListBucketsInput) (ListBucketsOutput, error) { + // Renew IAM token if needed + if err := s3.renewIAMToken(); err != nil { + return ListBucketsOutput{}, err + } + + // Build endpoint URL - ListBuckets uses the service endpoint (no bucket name) + var endpoint string + if len(s3.Endpoint) > 0 { + endpoint = s3.Endpoint + "/" + } else { + // For AWS, use the service endpoint + endpoint = fmt.Sprintf("https://s3.%s.amazonaws.com/", s3.Region) + } + + // Create HTTP request + req, err := http.NewRequest(http.MethodGet, endpoint, nil) + if err != nil { + return ListBucketsOutput{}, err + } + + // Sign the request + if err := s3.signRequest(req); err != nil { + return ListBucketsOutput{}, err + } + + // Execute request + client := s3.getClient() + res, err := client.Do(req) + if err != nil { + return ListBucketsOutput{}, err + } + + defer func() { + res.Body.Close() + io.Copy(io.Discard, res.Body) + }() + + // Read response body + body, err := io.ReadAll(res.Body) + if err != nil { + return ListBucketsOutput{}, err + } + + // Handle non-OK status codes + if res.StatusCode != http.StatusOK { + return ListBucketsOutput{}, fmt.Errorf("status code: %s: %s", res.Status, string(body)) + } + + // Parse XML response + var result ListBucketsOutput + if err := xml.Unmarshal(body, &result); err != nil { + return ListBucketsOutput{}, err + } + + return result, nil +} + +// CreateBucket creates a new S3 bucket. +// For regions other than us-east-1, it sends a LocationConstraint in the request body. +func (s3 *S3) CreateBucket(input CreateBucketInput) (CreateBucketOutput, error) { + // Validate input + if input.Bucket == "" { + return CreateBucketOutput{}, fmt.Errorf("bucket name is required") + } + + // Renew IAM token if needed + if err := s3.renewIAMToken(); err != nil { + return CreateBucketOutput{}, err + } + + // Determine region to use + region := input.Region + if region == "" { + region = s3.Region + } + + // Build endpoint URL + url := s3.getURL(input.Bucket) + + // Prepare request body for non-us-east-1 regions + // AWS S3 requires LocationConstraint for regions other than us-east-1 + var body io.Reader + if region != "us-east-1" && region != "" { + xmlBody := fmt.Sprintf( + "%s", + region, + ) + body = strings.NewReader(xmlBody) + } + + // Create HTTP request + req, err := http.NewRequest(http.MethodPut, url, body) + if err != nil { + return CreateBucketOutput{}, err + } + + // Sign the request + if err := s3.signRequest(req); err != nil { + return CreateBucketOutput{}, err + } + + // Execute request + client := s3.getClient() + res, err := client.Do(req) + if err != nil { + return CreateBucketOutput{}, err + } + + defer func() { + res.Body.Close() + io.Copy(io.Discard, res.Body) + }() + + // Read response body for error messages + bodyBytes, err := io.ReadAll(res.Body) + if err != nil { + return CreateBucketOutput{}, err + } + + // Handle non-success status codes + // AWS returns 200 OK or 201 Created on success + if res.StatusCode != http.StatusOK && res.StatusCode != http.StatusCreated { + return CreateBucketOutput{}, fmt.Errorf("status code: %s: %s", res.Status, string(bodyBytes)) + } + + // Extract location from response header + location := res.Header.Get("Location") + + return CreateBucketOutput{Location: location}, nil +} + +// DeleteBucket deletes an empty S3 bucket. +// The bucket must be empty (no objects) before it can be deleted. +// Returns an error if the bucket is not empty or does not exist. +func (s3 *S3) DeleteBucket(input DeleteBucketInput) error { + // Validate input + if input.Bucket == "" { + return fmt.Errorf("bucket name is required") + } + + // Renew IAM token if needed + if err := s3.renewIAMToken(); err != nil { + return err + } + + // Build endpoint URL + url := s3.getURL(input.Bucket) + + // Create HTTP request + req, err := http.NewRequest(http.MethodDelete, url, nil) + if err != nil { + return err + } + + // Sign the request + if err := s3.signRequest(req); err != nil { + return err + } + + // Execute request + client := s3.getClient() + res, err := client.Do(req) + if err != nil { + return err + } + + defer func() { + res.Body.Close() + io.Copy(io.Discard, res.Body) + }() + + // Read response body for error messages + bodyBytes, err := io.ReadAll(res.Body) + if err != nil { + return err + } + + // Handle non-success status codes + // AWS returns 204 No Content on successful deletion + if res.StatusCode != http.StatusNoContent { + return fmt.Errorf("status code: %s: %s", res.Status, string(bodyBytes)) + } + + return nil +} diff --git a/bucket_test.go b/bucket_test.go new file mode 100644 index 0000000..ac64f76 --- /dev/null +++ b/bucket_test.go @@ -0,0 +1,254 @@ +package simples3 + +import ( + "fmt" + "os" + "testing" + "time" +) + +func setupTestS3(t *testing.T) *S3 { + t.Helper() + + s3 := New( + os.Getenv("AWS_S3_REGION"), + os.Getenv("AWS_S3_ACCESS_KEY"), + os.Getenv("AWS_S3_SECRET_KEY"), + ) + + if endpoint := os.Getenv("AWS_S3_ENDPOINT"); endpoint != "" { + s3.SetEndpoint(endpoint) + } + + return s3 +} + +func TestListBuckets(t *testing.T) { + s3 := setupTestS3(t) + + result, err := s3.ListBuckets(ListBucketsInput{}) + if err != nil { + t.Fatalf("ListBuckets failed: %v", err) + } + + // ListBuckets should succeed even if there are no buckets + t.Logf("Found %d bucket(s)", len(result.Buckets)) + for _, bucket := range result.Buckets { + t.Logf(" - %s (created: %s)", bucket.Name, bucket.CreationDate) + } + + // Verify Owner information is present + if result.Owner.ID != "" { + t.Logf("Owner ID: %s", result.Owner.ID) + } + if result.Owner.DisplayName != "" { + t.Logf("Owner DisplayName: %s", result.Owner.DisplayName) + } +} + +func TestCreateAndDeleteBucket(t *testing.T) { + s3 := setupTestS3(t) + + // Generate a unique bucket name using timestamp + testBucket := fmt.Sprintf("simples3-test-%d", time.Now().Unix()) + + t.Run("CreateBucket", func(t *testing.T) { + output, err := s3.CreateBucket(CreateBucketInput{ + Bucket: testBucket, + }) + if err != nil { + t.Fatalf("CreateBucket failed: %v", err) + } + t.Logf("Created bucket: %s", testBucket) + if output.Location != "" { + t.Logf("Location: %s", output.Location) + } + }) + + // Give S3/MinIO a moment to propagate the bucket creation + time.Sleep(100 * time.Millisecond) + + t.Run("VerifyBucketExists", func(t *testing.T) { + result, err := s3.ListBuckets(ListBucketsInput{}) + if err != nil { + t.Fatalf("ListBuckets failed: %v", err) + } + + found := false + for _, bucket := range result.Buckets { + if bucket.Name == testBucket { + found = true + t.Logf("Verified bucket exists: %s (created: %s)", bucket.Name, bucket.CreationDate) + break + } + } + + if !found { + t.Errorf("Created bucket %s not found in ListBuckets", testBucket) + } + }) + + t.Run("DeleteBucket", func(t *testing.T) { + err := s3.DeleteBucket(DeleteBucketInput{ + Bucket: testBucket, + }) + if err != nil { + t.Fatalf("DeleteBucket failed: %v", err) + } + t.Logf("Deleted bucket: %s", testBucket) + }) + + // Give S3/MinIO a moment to propagate the bucket deletion + time.Sleep(100 * time.Millisecond) + + t.Run("VerifyBucketDeleted", func(t *testing.T) { + result, err := s3.ListBuckets(ListBucketsInput{}) + if err != nil { + t.Fatalf("ListBuckets failed: %v", err) + } + + found := false + for _, bucket := range result.Buckets { + if bucket.Name == testBucket { + found = true + break + } + } + + if found { + t.Errorf("Deleted bucket %s still appears in ListBuckets", testBucket) + } else { + t.Logf("Verified bucket deleted: %s", testBucket) + } + }) +} + +func TestCreateBucket_WithRegion(t *testing.T) { + s3 := setupTestS3(t) + + // Generate a unique bucket name + testBucket := fmt.Sprintf("simples3-test-region-%d", time.Now().Unix()) + + // Create bucket with explicit region + region := os.Getenv("AWS_S3_REGION") + if region == "" { + region = "us-east-1" + } + + output, err := s3.CreateBucket(CreateBucketInput{ + Bucket: testBucket, + Region: region, + }) + if err != nil { + t.Fatalf("CreateBucket with region failed: %v", err) + } + t.Logf("Created bucket with region %s: %s", region, testBucket) + if output.Location != "" { + t.Logf("Location: %s", output.Location) + } + + // Cleanup + defer func() { + time.Sleep(100 * time.Millisecond) + err := s3.DeleteBucket(DeleteBucketInput{ + Bucket: testBucket, + }) + if err != nil { + t.Logf("Failed to cleanup bucket %s: %v", testBucket, err) + } + }() + + // Verify bucket exists + result, err := s3.ListBuckets(ListBucketsInput{}) + if err != nil { + t.Fatalf("ListBuckets failed: %v", err) + } + + found := false + for _, bucket := range result.Buckets { + if bucket.Name == testBucket { + found = true + break + } + } + + if !found { + t.Errorf("Created bucket %s not found in ListBuckets", testBucket) + } +} + +func TestCreateBucket_EmptyName(t *testing.T) { + s3 := setupTestS3(t) + + _, err := s3.CreateBucket(CreateBucketInput{ + Bucket: "", + }) + if err == nil { + t.Error("Expected error when creating bucket with empty name, got nil") + } + if err != nil && err.Error() != "bucket name is required" { + t.Errorf("Expected 'bucket name is required' error, got: %v", err) + } +} + +func TestDeleteBucket_EmptyName(t *testing.T) { + s3 := setupTestS3(t) + + err := s3.DeleteBucket(DeleteBucketInput{ + Bucket: "", + }) + if err == nil { + t.Error("Expected error when deleting bucket with empty name, got nil") + } + if err != nil && err.Error() != "bucket name is required" { + t.Errorf("Expected 'bucket name is required' error, got: %v", err) + } +} + +func TestDeleteBucket_NonExistent(t *testing.T) { + s3 := setupTestS3(t) + + // Try to delete a bucket that doesn't exist + nonExistentBucket := fmt.Sprintf("nonexistent-bucket-%d", time.Now().Unix()) + err := s3.DeleteBucket(DeleteBucketInput{ + Bucket: nonExistentBucket, + }) + if err == nil { + t.Error("Expected error when deleting non-existent bucket, got nil") + } else { + t.Logf("Got expected error for non-existent bucket: %v", err) + } +} + +func TestCreateBucket_Duplicate(t *testing.T) { + s3 := setupTestS3(t) + + testBucket := fmt.Sprintf("simples3-test-dup-%d", time.Now().Unix()) + + // Create bucket first time + _, err := s3.CreateBucket(CreateBucketInput{ + Bucket: testBucket, + }) + if err != nil { + t.Fatalf("First CreateBucket failed: %v", err) + } + + defer func() { + time.Sleep(100 * time.Millisecond) + s3.DeleteBucket(DeleteBucketInput{ + Bucket: testBucket, + }) + }() + + time.Sleep(100 * time.Millisecond) + + // Try to create the same bucket again + _, err = s3.CreateBucket(CreateBucketInput{ + Bucket: testBucket, + }) + if err == nil { + t.Error("Expected error when creating duplicate bucket, got nil") + } else { + t.Logf("Got expected error for duplicate bucket: %v", err) + } +}