diff --git a/helpers.go b/helpers.go new file mode 100644 index 0000000..712ad3d --- /dev/null +++ b/helpers.go @@ -0,0 +1,97 @@ +// LICENSE BSD-2-Clause-FreeBSD +// Copyright (c) 2018, Rohan Verma + +package simples3 + +import ( + "encoding/hex" + "fmt" + "io" + "regexp" + "strings" + "unicode/utf8" +) + +// getURL constructs a URL for a given path, with multiple optional +// arguments as individual subfolders, based on the endpoint +// specified in s3 struct. +func (s3 *S3) getURL(path string, args ...string) (uri string) { + if len(args) > 0 { + path += "/" + strings.Join(args, "/") + } + // need to encode special characters in the path part of the URL + encodedPath := encodePath(path) + + if len(s3.Endpoint) > 0 { + uri = s3.Endpoint + "/" + encodedPath + } else { + uri = fmt.Sprintf(s3.URIFormat, s3.Region, encodedPath) + } + + return uri +} + +func detectFileSize(body io.Seeker) (int64, error) { + pos, err := body.Seek(0, 1) + if err != nil { + return -1, err + } + defer body.Seek(pos, 0) + + n, err := body.Seek(0, 2) //nolint:gomnd + if err != nil { + return -1, err + } + return n, nil +} + +func getFirstString(s []string) string { + if len(s) > 0 { + return s[0] + } + + return "" +} + +// if object matches reserved string, no need to encode them. +var reservedObjectNames = regexp.MustCompile("^[a-zA-Z0-9-_.~/]+$") + +// encodePath encode the strings from UTF-8 byte representations to HTML hex escape sequences +// +// This is necessary since regular url.Parse() and url.Encode() functions do not support UTF-8 +// non english characters cannot be parsed due to the nature in which url.Encode() is written +// +// This function on the other hand is a direct replacement for url.Encode() technique to support +// pretty much every UTF-8 character. +// adapted from +// https://github.com/minio/minio-go/blob/fe1f3855b146c1b6ce4199740d317e44cf9e85c2/pkg/s3utils/utils.go#L285 +func encodePath(pathName string) string { + if reservedObjectNames.MatchString(pathName) { + return pathName + } + var encodedPathname strings.Builder + for _, s := range pathName { + if 'A' <= s && s <= 'Z' || 'a' <= s && s <= 'z' || '0' <= s && s <= '9' { // §2.3 Unreserved characters (mark) + encodedPathname.WriteRune(s) + continue + } + switch s { + case '-', '_', '.', '~', '/': // §2.3 Unreserved characters (mark) + encodedPathname.WriteRune(s) + continue + default: + lenR := utf8.RuneLen(s) + if lenR < 0 { + // if utf8 cannot convert, return the same string as is + return pathName + } + u := make([]byte, lenR) + utf8.EncodeRune(u, s) + for _, r := range u { + hex := hex.EncodeToString([]byte{r}) + encodedPathname.WriteString("%" + strings.ToUpper(hex)) + } + } + } + return encodedPathname.String() +} diff --git a/iam.go b/iam.go new file mode 100644 index 0000000..4f657ad --- /dev/null +++ b/iam.go @@ -0,0 +1,202 @@ +// LICENSE BSD-2-Clause-FreeBSD +// Copyright (c) 2018, Rohan Verma + +package simples3 + +import ( + "encoding/json" + "fmt" + "io" + "net/http" + "time" +) + +const ( + imdsTokenHeader = "X-aws-ec2-metadata-token" + imdsTokenTtlHeader = "X-aws-ec2-metadata-token-ttl-seconds" + metadataBaseURL = "http://169.254.169.254/latest" + securityCredentialsURI = "/meta-data/iam/security-credentials/" + imdsTokenURI = "/api/token" + defaultIMDSTokenTTL = "60" +) + +// IAMResponse is used by NewUsingIAM to auto +// detect the credentials. +type IAMResponse struct { + Code string `json:"Code"` + LastUpdated string `json:"LastUpdated"` + Type string `json:"Type"` + AccessKeyID string `json:"AccessKeyId"` + SecretAccessKey string `json:"SecretAccessKey"` + Token string `json:"Token"` + Expiration time.Time `json:"Expiration"` +} + +// NewUsingIAM automatically generates an Instance of S3 +// using instance metatdata. +func NewUsingIAM(region string) (*S3, error) { + return newUsingIAM( + &http.Client{ + // Set a timeout of 3 seconds for AWS IAM Calls. + Timeout: time.Second * 3, //nolint:gomnd + }, metadataBaseURL, region) +} + +// fetchIMDSToken retrieves an IMDSv2 token from the +// EC2 instance metadata service. It returns a token and boolean, +// only if IMDSv2 is enabled in the EC2 instance metadata +// configuration, otherwise returns an error. +func fetchIMDSToken(cl *http.Client, baseURL string) (string, bool, error) { + req, err := http.NewRequest(http.MethodPut, baseURL+imdsTokenURI, nil) + if err != nil { + return "", false, err + } + + // Set the token TTL to 60 seconds. + req.Header.Set(imdsTokenTtlHeader, defaultIMDSTokenTTL) + + resp, err := cl.Do(req) + if err != nil { + return "", false, err + } + + defer func() { + resp.Body.Close() + io.Copy(io.Discard, resp.Body) + }() + + if resp.StatusCode != http.StatusOK { + return "", false, fmt.Errorf("failed to request IMDSv2 token: %s", resp.Status) + } + + token, err := io.ReadAll(resp.Body) + if err != nil { + return "", false, err + } + + return string(token), true, nil +} + +// fetchIAMData fetches the IAM data from the given URL. +// In case of a normal AWS setup, baseURL would be metadataBaseURL. +// You can use this method, to manually fetch IAM data from a custom +// endpoint and pass it to SetIAMData. +func fetchIAMData(cl *http.Client, baseURL string) (IAMResponse, error) { + token, useIMDSv2, err := fetchIMDSToken(cl, baseURL) + if err != nil { + return IAMResponse{}, fmt.Errorf("error fetching IMDSv2 token: %w", err) + } + + url := baseURL + securityCredentialsURI + + req, err := http.NewRequest(http.MethodGet, url, nil) + if err != nil { + return IAMResponse{}, fmt.Errorf("error creating imdsv2 token request: %w", err) + } + + if useIMDSv2 { + req.Header.Set(imdsTokenHeader, token) + } + + resp, err := cl.Do(req) + if err != nil { + return IAMResponse{}, err + } + + if resp.StatusCode != http.StatusOK { + return IAMResponse{}, fmt.Errorf("error fetching IAM data: %s", resp.Status) + } + + role, err := io.ReadAll(resp.Body) + if err != nil { + return IAMResponse{}, err + } + + _, _ = io.Copy(io.Discard, resp.Body) + _ = resp.Body.Close() + + req, err = http.NewRequest(http.MethodGet, url+string(role), nil) + if err != nil { + return IAMResponse{}, fmt.Errorf("error creating role request: %w", err) + } + if useIMDSv2 { + req.Header.Set(imdsTokenHeader, token) + } + + resp, err = cl.Do(req) + if err != nil { + return IAMResponse{}, fmt.Errorf("error fetching role data: %w", err) + } + + defer func() { + // Drain and close the body to let the Transport reuse the connection + io.Copy(io.Discard, resp.Body) + resp.Body.Close() + }() + + if resp.StatusCode != http.StatusOK { + return IAMResponse{}, fmt.Errorf("error fetching role data, got non 200 code: %s", resp.Status) + } + + var jResp IAMResponse + jsonString, err := io.ReadAll(resp.Body) + if err != nil { + return IAMResponse{}, fmt.Errorf("error reading role data: %w", err) + } + + if err := json.Unmarshal(jsonString, &jResp); err != nil { + return IAMResponse{}, fmt.Errorf("error unmarshalling role data: %w (%s)", err, jsonString) + } + + return jResp, nil +} + +func newUsingIAM(cl *http.Client, baseURL, region string) (*S3, error) { + // Get the IAM role + iamResp, err := fetchIAMData(cl, baseURL) + if err != nil { + return nil, fmt.Errorf("error fetching IAM data: %w", err) + } + + return &S3{ + Region: region, + AccessKey: iamResp.AccessKeyID, + SecretKey: iamResp.SecretAccessKey, + Token: iamResp.Token, + + URIFormat: "https://s3.%s.amazonaws.com/%s", + initMode: "iam", + expiry: iamResp.Expiration, + }, nil +} + +// setIAMData sets the IAM data on the S3 instance. +func (s3 *S3) SetIAMData(iamResp IAMResponse) { + s3.AccessKey = iamResp.AccessKeyID + s3.SecretKey = iamResp.SecretAccessKey + s3.Token = iamResp.Token +} + +func (s3 *S3) renewIAMToken() error { + if s3.initMode != "iam" { + return nil + } + + if time.Since(s3.expiry) < 0 { + return nil + } + + s3.mu.Lock() + defer s3.mu.Unlock() + iamResp, err := fetchIAMData(s3.getClient(), metadataBaseURL) + if err != nil { + return fmt.Errorf("error fetching IAM data: %w", err) + } + + s3.expiry = iamResp.Expiration + s3.Token = iamResp.Token + s3.AccessKey = iamResp.AccessKeyID + s3.SecretKey = iamResp.SecretAccessKey + + return nil +} diff --git a/iam_test.go b/iam_test.go new file mode 100644 index 0000000..2752b04 --- /dev/null +++ b/iam_test.go @@ -0,0 +1,110 @@ +package simples3 + +import ( + "errors" + "io" + "net" + "net/http" + "net/http/httptest" + "testing" + "time" +) + +func TestS3_NewUsingIAM(t *testing.T) { + var ( + iam = `test-new-s3-using-iam` + resp = `{"Code" : "Success","LastUpdated" : "2018-12-24T10:18:01Z", + "Type" : "AWS-HMAC","AccessKeyId" : "abc", + "SecretAccessKey" : "abc","Token" : "abc", + "Expiration" : "2018-12-24T16:24:59Z"}` + respIMDSToken = `AQAEAJWopi8yvjKYXyWJbzESE0cms-OoTnptJzS3M9g5iNcl06UEkQ==` + ) + + tsFail := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + time.Sleep(2 * time.Second) + })) + defer tsFail.Close() + + genServerHandlerFunc := func(failIMDS bool) http.HandlerFunc { + return func(w http.ResponseWriter, r *http.Request) { + switch r.Method { + case "GET": + if !failIMDS { + // check if token is present + if r.Header.Get(imdsTokenHeader) == "" { + w.WriteHeader(http.StatusUnauthorized) + return + } + } + + url := securityCredentialsURI + if r.URL.EscapedPath() == url { + w.WriteHeader(http.StatusOK) + io.WriteString(w, iam) + } + if r.URL.EscapedPath() == url+iam { + w.WriteHeader(http.StatusOK) + w.Header().Set("Content-Type", "application/json") + io.WriteString(w, resp) + } + case "PUT": + if failIMDS { + w.WriteHeader(http.StatusNotFound) + return + } + + if r.URL.EscapedPath() == imdsTokenURI { + if r.Header.Get(imdsTokenTtlHeader) != "60" { + w.WriteHeader(http.StatusBadRequest) + return + } + + w.WriteHeader(http.StatusOK) + io.WriteString(w, respIMDSToken) + } + default: + t.Errorf("Expected 'GET' or 'PUT' request, got '%s'", r.Method) + } + } + } + + ts := httptest.NewServer(http.HandlerFunc(genServerHandlerFunc(false))) + defer ts.Close() + + tsFailIMDS := httptest.NewServer(http.HandlerFunc(genServerHandlerFunc(true))) + defer tsFailIMDS.Close() + + cl := &http.Client{Timeout: 1 * time.Second} + + // Test for timeout. + _, err := newUsingIAM(cl, tsFail.URL, "abc") + if err == nil { + t.Errorf("Expected error, got nil") + } else { + var timeoutError net.Error + + if errors.As(err, &timeoutError) && !timeoutError.Timeout() { + t.Errorf("newUsingIAM() timeout check. got error = %v", err) + } + } + + // Test for successful IAM fetch. + s3, err := newUsingIAM(cl, ts.URL, "abc") + if err != nil { + t.Errorf("newUsingIAM() error = %v", err) + } + + if s3 == nil { + t.Errorf("newUsingIAM() got = %v", s3) + } + + if s3.AccessKey != "abc" || s3.SecretKey != "abc" || s3.Region != "abc" { + t.Errorf("S3.FileDelete() got = %v", s3) + } + + // Test for failed IMDS token fetch. + _, err = newUsingIAM(cl, tsFailIMDS.URL, "abc") + if err == nil { + t.Errorf("Expected error, got nil") + } +} diff --git a/list.go b/list.go new file mode 100644 index 0000000..0e691da --- /dev/null +++ b/list.go @@ -0,0 +1,260 @@ +// LICENSE BSD-2-Clause-FreeBSD +// Copyright (c) 2018, Rohan Verma + +package simples3 + +import ( + "encoding/xml" + "fmt" + "io" + "iter" + "net/http" + "net/url" +) + +// ListInput is passed to List as a parameter. +type ListInput struct { + // Required: The name of the bucket to list objects from + Bucket string + + // Optional: A delimiter to group objects by (commonly "/") + Delimiter string + + // Optional: Only list objects starting with this prefix + Prefix string + + // Optional: Maximum number of objects to return + MaxKeys int64 + + // Optional: Token for pagination from a previous request + ContinuationToken string + + // Optional: Object key to start listing after + StartAfter string +} + +// ListResponse is returned by List. +type ListResponse struct { + // Name of the bucket + Name string + + // Whether the results were truncated (more results available) + IsTruncated bool + + // Token to get the next page of results (if truncated) + NextContinuationToken string + + // List of objects in the bucket + Objects []Object + + // Common prefixes when using delimiter (like directories) + CommonPrefixes []string + + // Total number of keys returned + KeyCount int64 +} + +// Object represents an S3 object in a bucket. +type Object struct { + // Name of the object + Key string + + // Size in bytes + Size int64 + + // When the object was last modified + LastModified string + + // Entity tag of the object + ETag string + + // Storage class (e.g., "STANDARD") + StorageClass string +} + +// S3Error represents an S3 API error response +type S3Error struct { + XMLName xml.Name `xml:"Error"` + Code string `xml:"Code"` + Message string `xml:"Message"` + RequestID string `xml:"RequestId"` + HostID string `xml:"HostId"` +} + +// Error returns a string representation of the S3Error +func (e S3Error) Error() string { + return fmt.Sprintf("S3 Error: %s - %s", e.Code, e.Message) +} + +// List implements a simple S3 object listing API +func (s3 *S3) List(input ListInput) (ListResponse, error) { + // Input validation + if input.Bucket == "" { + return ListResponse{}, fmt.Errorf("bucket name cannot be empty") + } + if input.MaxKeys < 0 { + return ListResponse{}, fmt.Errorf("MaxKeys cannot be negative") + } + if input.MaxKeys > 1000 { + return ListResponse{}, fmt.Errorf("MaxKeys cannot exceed 1000") + } + + // Build query parameters - ListObjectsV2 uses query params, not path + query := url.Values{} + query.Set("list-type", "2") // Required parameter + + // Add optional parameters if they exist + if input.ContinuationToken != "" { + query.Set("continuation-token", input.ContinuationToken) + } + if input.Delimiter != "" { + query.Set("delimiter", input.Delimiter) + } + if input.MaxKeys > 0 { + query.Set("max-keys", fmt.Sprintf("%d", input.MaxKeys)) + } + if input.Prefix != "" { + query.Set("prefix", input.Prefix) + } + if input.StartAfter != "" { + query.Set("start-after", input.StartAfter) + } + + // Build base URL + baseURL := s3.getURL(input.Bucket) + + // Parse URL and add query parameters + parsedURL, err := url.Parse(baseURL) + if err != nil { + return ListResponse{}, err + } + parsedURL.RawQuery = query.Encode() + + // Create HTTP request + req, err := http.NewRequest(http.MethodGet, parsedURL.String(), nil) + if err != nil { + return ListResponse{}, err + } + + // Apply AWS V4 signing + if err := s3.renewIAMToken(); err != nil { + return ListResponse{}, err + } + if err := s3.signRequest(req); err != nil { + return ListResponse{}, err + } + + // Execute request + client := s3.getClient() + res, err := client.Do(req) + if err != nil { + return ListResponse{}, err + } + + // Close response body when done + defer res.Body.Close() + + // Read response body once + body, err := io.ReadAll(res.Body) + if err != nil { + return ListResponse{}, err + } + + // Handle response status codes + if res.StatusCode != http.StatusOK { + var s3Err S3Error + if xmlErr := xml.Unmarshal(body, &s3Err); xmlErr == nil { + return ListResponse{}, fmt.Errorf("S3 Error: %s - %s", s3Err.Code, s3Err.Message) + } + return ListResponse{}, fmt.Errorf("status code: %s: %s", res.Status, string(body)) + } + + // Parse XML response using internal struct + var xmlResult struct { + XMLName xml.Name `xml:"ListBucketResult"` + Name string `xml:"Name"` + IsTruncated bool `xml:"IsTruncated"` + NextContinuationToken string `xml:"NextContinuationToken"` + KeyCount int64 `xml:"KeyCount"` + Contents []struct { + Key string `xml:"Key"` + LastModified string `xml:"LastModified"` + ETag string `xml:"ETag"` + Size int64 `xml:"Size"` + StorageClass string `xml:"StorageClass"` + } `xml:"Contents"` + CommonPrefixes []struct { + Prefix string `xml:"Prefix"` + } `xml:"CommonPrefixes"` + } + + if err := xml.Unmarshal(body, &xmlResult); err != nil { + return ListResponse{}, err + } + + // Convert to simple response format + response := ListResponse{ + Name: xmlResult.Name, + IsTruncated: xmlResult.IsTruncated, + NextContinuationToken: xmlResult.NextContinuationToken, + KeyCount: xmlResult.KeyCount, + } + + // Convert objects + for _, obj := range xmlResult.Contents { + response.Objects = append(response.Objects, Object{ + Key: obj.Key, + Size: obj.Size, + LastModified: obj.LastModified, + ETag: obj.ETag, + StorageClass: obj.StorageClass, + }) + } + + // Convert common prefixes + for _, prefix := range xmlResult.CommonPrefixes { + response.CommonPrefixes = append(response.CommonPrefixes, prefix.Prefix) + } + + return response, nil +} + +// ListAll returns an iterator that yields all objects in the bucket, +// automatically handling pagination. It also returns a finish callback +// that should be called after iteration to check for any errors. +func (s3 *S3) ListAll(input ListInput) (iter.Seq[Object], func() error) { + var iterErr error + + seq := func(yield func(Object) bool) { + currentInput := input + + for { + response, err := s3.List(currentInput) + if err != nil { + iterErr = err + return + } + + // Yield each object + for _, obj := range response.Objects { + if !yield(obj) { + return // Early termination requested + } + } + + // Check if there are more results + if !response.IsTruncated || response.NextContinuationToken == "" { + return // No more results + } + + // Prepare for next page + currentInput.ContinuationToken = response.NextContinuationToken + } + } + + finish := func() error { + return iterErr + } + + return seq, finish +} diff --git a/list_test.go b/list_test.go new file mode 100644 index 0000000..fb82e56 --- /dev/null +++ b/list_test.go @@ -0,0 +1,321 @@ +package simples3 + +import ( + "fmt" + "os" + "strings" + "testing" +) + +func TestListAll(t *testing.T) { + // Setup test environment + s3 := New( + os.Getenv("AWS_S3_REGION"), + os.Getenv("AWS_S3_ACCESS_KEY"), + os.Getenv("AWS_S3_SECRET_KEY"), + ) + s3.SetEndpoint(os.Getenv("AWS_S3_ENDPOINT")) + + // Test cases + tests := []struct { + name string + setup func() // Setup function to prepare test data + input ListInput + validate func([]Object, error) + cleanup func() // Cleanup function + }{ + { + name: "Empty Bucket Iterator", + input: ListInput{ + Bucket: os.Getenv("AWS_S3_BUCKET"), + }, + validate: func(objects []Object, err error) { + if err != nil { + t.Fatalf("Expected no error, got %v", err) + } + if len(objects) != 0 { + t.Errorf("Expected empty bucket, got %d objects", len(objects)) + } + }, + }, + { + name: "Basic Iterator Listing", + setup: func() { + // Upload test files + uploadTestFiles(t, s3, os.Getenv("AWS_S3_BUCKET"), []string{"iter_file1.txt", "iter_file2.txt", "iter_file3.txt"}) + }, + input: ListInput{ + Bucket: os.Getenv("AWS_S3_BUCKET"), + }, + validate: func(objects []Object, err error) { + if err != nil { + t.Fatalf("Expected no error, got %v", err) + } + if len(objects) != 3 { + t.Errorf("Expected 3 objects, got %d", len(objects)) + } + // Check that all expected files are present + expectedKeys := map[string]bool{ + "iter_file1.txt": false, + "iter_file2.txt": false, + "iter_file3.txt": false, + } + for _, obj := range objects { + if _, exists := expectedKeys[obj.Key]; exists { + expectedKeys[obj.Key] = true + } + } + for key, found := range expectedKeys { + if !found { + t.Errorf("Expected object %s not found in iterator results", key) + } + } + }, + cleanup: func() { + cleanupTestFiles(t, s3, os.Getenv("AWS_S3_BUCKET"), []string{"iter_file1.txt", "iter_file2.txt", "iter_file3.txt"}) + }, + }, + { + name: "Iterator with Prefix", + setup: func() { + uploadTestFiles(t, s3, os.Getenv("AWS_S3_BUCKET"), []string{ + "iter_docs/file1.txt", + "iter_docs/file2.txt", + "iter_images/image1.jpg", + "iter_images/image2.jpg", + }) + }, + input: ListInput{ + Bucket: os.Getenv("AWS_S3_BUCKET"), + Prefix: "iter_docs/", + }, + validate: func(objects []Object, err error) { + if err != nil { + t.Fatalf("Expected no error, got %v", err) + } + if len(objects) != 2 { + t.Errorf("Expected 2 objects with iter_docs/ prefix, got %d", len(objects)) + } + for _, obj := range objects { + if !strings.HasPrefix(obj.Key, "iter_docs/") { + t.Errorf("Object %s doesn't have expected prefix", obj.Key) + } + } + }, + cleanup: func() { + cleanupTestFiles(t, s3, os.Getenv("AWS_S3_BUCKET"), []string{ + "iter_docs/file1.txt", + "iter_docs/file2.txt", + "iter_images/image1.jpg", + "iter_images/image2.jpg", + }) + }, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if tt.setup != nil { + tt.setup() + } + + // Collect all objects from the iterator and check for errors + var objects []Object + seq, finish := s3.ListAll(tt.input) + for obj := range seq { + objects = append(objects, obj) + } + + // Check for any iteration errors + if err := finish(); err != nil { + t.Fatalf("Iterator error: %v", err) + } + + if tt.validate != nil { + tt.validate(objects, nil) // No error from iterator + } + + if tt.cleanup != nil { + tt.cleanup() + } + }) + } +} + +func TestList(t *testing.T) { + // Setup test environment + s3 := New( + os.Getenv("AWS_S3_REGION"), + os.Getenv("AWS_S3_ACCESS_KEY"), + os.Getenv("AWS_S3_SECRET_KEY"), + ) + s3.SetEndpoint(os.Getenv("AWS_S3_ENDPOINT")) + + // Test cases + tests := []struct { + name string + setup func() // Setup function to prepare test data + input ListInput + validate func(ListResponse, error) + cleanup func() // Cleanup function + }{ + { + name: "Empty Bucket", + input: ListInput{ + Bucket: os.Getenv("AWS_S3_BUCKET"), + }, + validate: func(result ListResponse, err error) { + if err != nil { + t.Fatalf("Expected no error, got %v", err) + } + if len(result.Objects) != 0 { + t.Errorf("Expected empty bucket, got %d objects", len(result.Objects)) + } + }, + }, + { + name: "Basic Listing", + setup: func() { + // Upload test files + uploadTestFiles(t, s3, os.Getenv("AWS_S3_BUCKET"), []string{"file1.txt", "file2.txt", "file3.txt"}) + }, + input: ListInput{ + Bucket: os.Getenv("AWS_S3_BUCKET"), + }, + validate: func(result ListResponse, err error) { + if err != nil { + t.Fatalf("Expected no error, got %v", err) + } + if len(result.Objects) != 3 { + t.Errorf("Expected 3 objects, got %d", len(result.Objects)) + } + }, + cleanup: func() { + cleanupTestFiles(t, s3, os.Getenv("AWS_S3_BUCKET"), []string{"file1.txt", "file2.txt", "file3.txt"}) + }, + }, + { + name: "Pagination Test", + setup: func() { + // Upload many files for pagination testing + var filenames []string + for i := 1; i <= 25; i++ { + filenames = append(filenames, fmt.Sprintf("file%03d.txt", i)) + } + uploadTestFiles(t, s3, os.Getenv("AWS_S3_BUCKET"), filenames) + }, + input: ListInput{ + Bucket: os.Getenv("AWS_S3_BUCKET"), + MaxKeys: 10, + }, + validate: func(result ListResponse, err error) { + if err != nil { + t.Fatalf("Expected no error, got %v", err) + } + if len(result.Objects) != 10 { + t.Errorf("Expected 10 objects, got %d", len(result.Objects)) + } + if !result.IsTruncated { + t.Errorf("Expected IsTruncated=true for pagination") + } + if result.NextContinuationToken == "" { + t.Errorf("Expected NextContinuationToken for pagination: %v", result.NextContinuationToken) + } + }, + cleanup: func() { + var filenames []string + for i := 1; i <= 25; i++ { + filenames = append(filenames, fmt.Sprintf("file%03d.txt", i)) + } + cleanupTestFiles(t, s3, os.Getenv("AWS_S3_BUCKET"), filenames) + }, + }, + { + name: "Prefix Filtering", + setup: func() { + uploadTestFiles(t, s3, os.Getenv("AWS_S3_BUCKET"), []string{ + "documents/file1.txt", + "documents/file2.txt", + "images/image1.jpg", + "images/image2.jpg", + }) + }, + input: ListInput{ + Bucket: os.Getenv("AWS_S3_BUCKET"), + Prefix: "documents/", + }, + validate: func(result ListResponse, err error) { + if err != nil { + t.Fatalf("Expected no error, got %v", err) + } + if len(result.Objects) != 2 { + t.Errorf("Expected 2 objects with documents/ prefix, got %d", len(result.Objects)) + } + for _, obj := range result.Objects { + if !strings.HasPrefix(obj.Key, "documents/") { + t.Errorf("Object %s doesn't have expected prefix", obj.Key) + } + } + }, + cleanup: func() { + cleanupTestFiles(t, s3, os.Getenv("AWS_S3_BUCKET"), []string{ + "documents/file1.txt", + "documents/file2.txt", + "images/image1.jpg", + "images/image2.jpg", + }) + }, + }, + { + name: "Delimiter Grouping", + setup: func() { + uploadTestFiles(t, s3, os.Getenv("AWS_S3_BUCKET"), []string{ + "documents/important/file1.txt", + "documents/backup/file2.txt", + "images/landscape/image1.jpg", + "images/portrait/image2.jpg", + }) + }, + input: ListInput{ + Bucket: os.Getenv("AWS_S3_BUCKET"), + Delimiter: "/", + Prefix: "documents/", + }, + validate: func(result ListResponse, err error) { + if err != nil { + t.Fatalf("Expected no error, got %v", err) + } + // Should return common prefixes instead of individual files + if len(result.CommonPrefixes) != 2 { + t.Errorf("Expected 2 common prefixes, got %d", len(result.CommonPrefixes)) + } + }, + cleanup: func() { + cleanupTestFiles(t, s3, os.Getenv("AWS_S3_BUCKET"), []string{ + "documents/important/file1.txt", + "documents/backup/file2.txt", + "images/landscape/image1.jpg", + "images/portrait/image2.jpg", + }) + }, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if tt.setup != nil { + tt.setup() + } + + result, err := s3.List(tt.input) + + if tt.validate != nil { + tt.validate(result, err) + } + + if tt.cleanup != nil { + tt.cleanup() + } + }) + } +} diff --git a/object.go b/object.go new file mode 100644 index 0000000..3b8e6e3 --- /dev/null +++ b/object.go @@ -0,0 +1,409 @@ +// LICENSE BSD-2-Clause-FreeBSD +// Copyright (c) 2018, Rohan Verma + +package simples3 + +import ( + "bytes" + "crypto/sha256" + "encoding/xml" + "fmt" + "io" + "mime/multipart" + "net/http" + "strings" +) + +type DownloadInput struct { + Bucket string + ObjectKey string +} + +// DetailsInput is passed to FileDetails as a parameter. +type DetailsInput struct { + Bucket string + ObjectKey string +} + +// DetailsResponse is returned by FileDetails. +type DetailsResponse struct { + ContentType string + ContentLength string + AcceptRanges string + Date string + Etag string + LastModified string + Server string + AmzID2 string + AmzRequestID string + AmzMeta map[string]string + ExtraHeaders map[string]string +} + +// UploadInput is passed to FileUpload as a parameter. +type UploadInput struct { + // essential fields + Bucket string + ObjectKey string + FileName string + ContentType string + + // optional fields + ContentDisposition string + ACL string + // Setting key/value pairs adds user-defined metadata + // keys to the object, prefixed with AMZMetaPrefix. + CustomMetadata map[string]string + + Body io.ReadSeeker +} + +// UploadResponse receives the following XML +// in case of success, since we set a 201 response from S3. +// Sample response: +// +// +// https://s3.amazonaws.com/link-to-the-file +// s3-bucket +// development/8614bd40-691b-4668-9241-3b342c6cf429/image.jpg +// "32-bit-tag" +// +type UploadResponse struct { + Location string `xml:"Location"` + Bucket string `xml:"Bucket"` + Key string `xml:"Key"` + ETag string `xml:"ETag"` +} + +// PutResponse is returned when the action is successful, +// and the service sends back an HTTP 200 response. The response +// returns ETag along with HTTP headers. +type PutResponse struct { + ETag string + Headers http.Header +} + +// DeleteInput is passed to FileDelete as a parameter. +type DeleteInput struct { + Bucket string + ObjectKey string +} + +// FileDownload makes a GET call and returns a io.ReadCloser. +// After reading the response body, ensure closing the response. +func (s3 *S3) FileDownload(u DownloadInput) (io.ReadCloser, error) { + req, err := http.NewRequest( + http.MethodGet, s3.getURL(u.Bucket, u.ObjectKey), nil, + ) + if err != nil { + return nil, err + } + + if err := s3.renewIAMToken(); err != nil { + return nil, err + } + if err := s3.signRequest(req); err != nil { + return nil, err + } + + res, err := s3.getClient().Do(req) + if err != nil { + return nil, err + } + + if res.StatusCode != http.StatusOK { + return nil, fmt.Errorf("status code: %s", res.Status) + } + + return res.Body, nil +} + +// FilePut makes a PUT call to S3. +func (s3 *S3) FilePut(u UploadInput) (PutResponse, error) { + fSize, err := detectFileSize(u.Body) + if err != nil { + return PutResponse{}, err + } + + content := make([]byte, fSize) + _, err = u.Body.Read(content) + if err != nil { + return PutResponse{}, err + } + u.Body.Seek(0, 0) + + req, er := http.NewRequest(http.MethodPut, s3.getURL(u.Bucket, u.ObjectKey), u.Body) + if er != nil { + return PutResponse{}, err + } + + if u.ContentType == "" { + u.ContentType = "application/octet-stream" + } + + h := sha256.New() + h.Write(content) + req.Header.Set("x-amz-content-sha256", fmt.Sprintf("%x", h.Sum(nil))) + + req.Header.Set("Content-Type", u.ContentType) + req.Header.Set("Content-Length", fmt.Sprintf("%d", fSize)) + req.Header.Set("Host", req.URL.Host) + + for k, v := range u.CustomMetadata { + req.Header.Set("x-amz-meta-"+k, v) + } + + if u.ContentDisposition != "" { + req.Header.Set("Content-Disposition", u.ContentDisposition) + } + + if u.ACL != "" { + req.Header.Set("x-amz-acl", u.ACL) + } + + req.ContentLength = fSize + + if err := s3.renewIAMToken(); err != nil { + return PutResponse{}, err + } + if err := s3.signRequest(req); err != nil { + return PutResponse{}, err + } + + // debug(httputil.DumpRequest(req, true)) + // Submit the request + client := s3.getClient() + res, err := client.Do(req) + if err != nil { + return PutResponse{}, err + } + + defer func() { + res.Body.Close() + io.Copy(io.Discard, res.Body) + }() + + data, err := io.ReadAll(res.Body) + if err != nil { + return PutResponse{}, err + } + + // Check the response + if res.StatusCode != http.StatusOK { + return PutResponse{}, fmt.Errorf("status code: %s: %q", res.Status, data) + } + + return PutResponse{ + ETag: res.Header.Get("ETag"), + Headers: res.Header.Clone(), + }, nil +} + +// FileUpload makes a POST call with the file written as multipart +// and on successful upload, checks for 200 OK. +func (s3 *S3) FileUpload(u UploadInput) (UploadResponse, error) { + fSize, err := detectFileSize(u.Body) + if err != nil { + return UploadResponse{}, err + } + + uc := UploadConfig{ + UploadURL: s3.getURL(u.Bucket), + BucketName: u.Bucket, + ObjectKey: u.ObjectKey, + ContentType: u.ContentType, + ContentDisposition: u.ContentDisposition, + ACL: u.ACL, + FileSize: fSize, + MetaData: map[string]string{ + "success_action_status": "201", // returns XML doc on success + }, + } + + // Set custom metadata. + for k, v := range u.CustomMetadata { + if !strings.HasPrefix(k, AMZMetaPrefix) { + k = AMZMetaPrefix + k + } + + uc.MetaData[k] = v + } + + if err := s3.renewIAMToken(); err != nil { + return UploadResponse{}, err + } + + policies, err := s3.CreateUploadPolicies(uc) + if err != nil { + return UploadResponse{}, err + } + + var b bytes.Buffer + w := multipart.NewWriter(&b) + + for k, v := range policies.Form { + if err = w.WriteField(k, v); err != nil { + return UploadResponse{}, err + } + } + + fw, err := w.CreateFormFile("file", u.FileName) + if err != nil { + return UploadResponse{}, err + } + + if _, err = io.Copy(fw, u.Body); err != nil { + return UploadResponse{}, err + } + + // Don't forget to close the multipart writer. + // If you don't close it, your request will be missing the terminating boundary. + if err := w.Close(); err != nil { + return UploadResponse{}, err + } + + // Now that you have a form, you can submit it to your handler. + req, err := http.NewRequest(http.MethodPost, policies.URL, &b) + if err != nil { + return UploadResponse{}, err + } + // Don't forget to set the content type, this will contain the boundary. + req.Header.Set("Content-Type", w.FormDataContentType()) + + // Submit the request + client := s3.getClient() + res, err := client.Do(req) + if err != nil { + return UploadResponse{}, err + } + + defer func() { + res.Body.Close() + io.Copy(io.Discard, res.Body) + }() + + data, err := io.ReadAll(res.Body) + if err != nil { + return UploadResponse{}, err + } + + // Check the response + if res.StatusCode != http.StatusCreated { + return UploadResponse{}, fmt.Errorf("status code: %s: %q", res.Status, data) + } + + var ur UploadResponse + _ = xml.Unmarshal(data, &ur) + return ur, nil +} + +// FileDelete makes a DELETE call with the file written as multipart +// and on successful upload, checks for 204 No Content. +func (s3 *S3) FileDelete(u DeleteInput) error { + req, err := http.NewRequest( + http.MethodDelete, s3.getURL(u.Bucket, u.ObjectKey), nil, + ) + if err != nil { + return err + } + + if err := s3.renewIAMToken(); err != nil { + return err + } + + if err := s3.signRequest(req); err != nil { + return err + } + + // Submit the request + client := s3.getClient() + res, err := client.Do(req) + if err != nil { + return err + } + + defer func() { + res.Body.Close() + io.Copy(io.Discard, res.Body) + }() + + // Check the response + if res.StatusCode != http.StatusNoContent { + return fmt.Errorf("status code: %s", res.Status) + } + + return nil +} + +func (s3 *S3) FileDetails(u DetailsInput) (DetailsResponse, error) { + req, err := http.NewRequest( + http.MethodHead, s3.getURL(u.Bucket, u.ObjectKey), nil, + ) + if err != nil { + return DetailsResponse{}, err + } + + if err := s3.renewIAMToken(); err != nil { + return DetailsResponse{}, err + } + + if err := s3.signRequest(req); err != nil { + return DetailsResponse{}, err + } + + res, err := s3.getClient().Do(req) + if err != nil { + return DetailsResponse{}, err + } + + defer func() { + res.Body.Close() + io.Copy(io.Discard, res.Body) + }() + + if res.StatusCode != http.StatusOK { + return DetailsResponse{}, fmt.Errorf("status code: %s", res.Status) + } + + var out DetailsResponse + for k, v := range res.Header { + lk := strings.ToLower(k) + + switch lk { + case "content-type": + out.ContentType = getFirstString(v) + case "content-length": + out.ContentLength = getFirstString(v) + case "accept-ranges": + out.AcceptRanges = getFirstString(v) + case "date": + out.Date = getFirstString(v) + case "etag": + out.Etag = getFirstString(v) + case "last-modified": + out.LastModified = getFirstString(v) + case "server": + out.Server = getFirstString(v) + case "x-amz-id-2": + out.AmzID2 = getFirstString(v) + case "x-amz-request-id": + out.AmzRequestID = getFirstString(v) + default: + if strings.HasPrefix(lk, AMZMetaPrefix) { + if out.AmzMeta == nil { + out.AmzMeta = map[string]string{} + } + + out.AmzMeta[k] = getFirstString(v) + } else { + if out.ExtraHeaders == nil { + out.ExtraHeaders = map[string]string{} + } + + out.ExtraHeaders[k] = getFirstString(v) + } + } + } + + return out, nil +} diff --git a/object_test.go b/object_test.go new file mode 100644 index 0000000..ea6ec04 --- /dev/null +++ b/object_test.go @@ -0,0 +1,381 @@ +package simples3 + +import ( + "bytes" + "io" + "os" + "testing" +) + +func TestS3_FileUploadPostAndPut(t *testing.T) { + testTxt, err := os.Open("testdata/test.txt") + if err != nil { + t.Fatalf("Failed to open test.txt: %v", err) + } + defer testTxt.Close() + // Note: cannot re-use the same file descriptor due to seeking! + testTxtSpecialChars, err := os.Open("testdata/test.txt") + if err != nil { + t.Fatalf("Failed to open test.txt for special chars: %v", err) + } + defer testTxtSpecialChars.Close() + testPng, err := os.Open("testdata/avatar.png") + if err != nil { + t.Fatalf("Failed to open avatar.png: %v", err) + } + defer testPng.Close() + + type args struct { + u UploadInput + } + tests := []struct { + name string + fields tConfig + args args + testDetails bool + wantErr bool + }{ + { + name: "Upload test.txt", + fields: tConfig{ + AccessKey: os.Getenv("AWS_S3_ACCESS_KEY"), + SecretKey: os.Getenv("AWS_S3_SECRET_KEY"), + Endpoint: os.Getenv("AWS_S3_ENDPOINT"), + Region: os.Getenv("AWS_S3_REGION"), + }, + args: args{ + UploadInput{ + Bucket: os.Getenv("AWS_S3_BUCKET"), + ObjectKey: "test.txt", + ContentType: "text/plain", + FileName: "test.txt", + Body: testTxt, + }, + }, + wantErr: false, + }, + { + name: "Upload test.txt with custom metadata", + fields: tConfig{ + AccessKey: os.Getenv("AWS_S3_ACCESS_KEY"), + SecretKey: os.Getenv("AWS_S3_SECRET_KEY"), + Endpoint: os.Getenv("AWS_S3_ENDPOINT"), + Region: os.Getenv("AWS_S3_REGION"), + }, + args: args{ + UploadInput{ + Bucket: os.Getenv("AWS_S3_BUCKET"), + ObjectKey: "test_metadata.txt", + ContentType: "text/plain", + FileName: "test.txt", + Body: testTxt, + CustomMetadata: map[string]string{ + "test-metadata": "foo-bar", + }, + }, + }, + wantErr: false, + testDetails: true, + }, + { + name: "Upload avatar.png", + fields: tConfig{ + AccessKey: os.Getenv("AWS_S3_ACCESS_KEY"), + SecretKey: os.Getenv("AWS_S3_SECRET_KEY"), + Endpoint: os.Getenv("AWS_S3_ENDPOINT"), + Region: os.Getenv("AWS_S3_REGION"), + }, + args: args{ + UploadInput{ + Bucket: os.Getenv("AWS_S3_BUCKET"), + ObjectKey: "xyz/image.png", + ContentType: "image/png", + FileName: "avatar.png", + Body: testPng, + }, + }, + wantErr: false, + }, + { + name: "Upload special filename txt", + fields: tConfig{ + AccessKey: os.Getenv("AWS_S3_ACCESS_KEY"), + SecretKey: os.Getenv("AWS_S3_SECRET_KEY"), + Endpoint: os.Getenv("AWS_S3_ENDPOINT"), + Region: os.Getenv("AWS_S3_REGION"), + }, + args: args{ + UploadInput{ + Bucket: os.Getenv("AWS_S3_BUCKET"), + ObjectKey: "xyz/example file%with$special&chars(1)?.txt", + ContentType: "text/plain", + FileName: "example file%with$special&chars(1)?.txt", + Body: testTxtSpecialChars, + }, + }, + wantErr: false, + }, + } + for _, testcase := range tests { + tt := testcase + t.Run(tt.name+"_post", func(t *testing.T) { + s3 := New(tt.fields.Region, tt.fields.AccessKey, tt.fields.SecretKey) + s3.SetEndpoint(tt.fields.Endpoint) + + resp, err := s3.FileUpload(tt.args.u) + if (err != nil) != tt.wantErr { + t.Errorf("S3.FileUpload() error = %v, wantErr %v", err, tt.wantErr) + } + + // reset file, to reuse in further tests. + tt.args.u.Body.Seek(0, 0) + + // check for empty response + if (resp == UploadResponse{}) { + t.Errorf("S3.FileUpload() returned empty response, %v", resp) + } + + if tt.testDetails { + dResp, err := s3.FileDetails(DetailsInput{ + Bucket: tt.args.u.Bucket, + ObjectKey: tt.args.u.ObjectKey, + }) + + if (err != nil) != tt.wantErr { + t.Errorf("S3.FileDetails() error = %v, wantErr %v", err, tt.wantErr) + } + + if len(dResp.AmzMeta) != len(tt.args.u.CustomMetadata) { + t.Errorf("S3.FileDetails() returned incorrect metadata, got: %#v", dResp) + } + } + }) + t.Run(tt.name+"_put", func(t *testing.T) { + s3 := New(tt.fields.Region, tt.fields.AccessKey, tt.fields.SecretKey) + s3.SetEndpoint(tt.fields.Endpoint) + + resp, err := s3.FilePut(tt.args.u) + if (err != nil) != tt.wantErr { + t.Errorf("S3.FileUpload() error = %v, wantErr %v", err, tt.wantErr) + } + + // reset file, to reuse in further tests. + tt.args.u.Body.Seek(0, 0) + + // check for empty response + if resp.ETag == "" { + t.Errorf("S3.FileUpload() returned empty response, %v", resp) + } + + if tt.testDetails { + dResp, err := s3.FileDetails(DetailsInput{ + Bucket: tt.args.u.Bucket, + ObjectKey: tt.args.u.ObjectKey, + }) + + if (err != nil) != tt.wantErr { + t.Errorf("S3.FileUpload() error = %v, wantErr %v", err, tt.wantErr) + } + + if len(dResp.AmzMeta) != len(tt.args.u.CustomMetadata) { + t.Errorf("S3.FileDetails() returned incorrect metadata, got: %#v", dResp) + } + } + }) + } +} + +func TestS3_FileDownload(t *testing.T) { + testTxt, err := os.Open("testdata/test.txt") + if err != nil { + t.Fatal(err) + } + defer testTxt.Close() + testTxtData, err := io.ReadAll(testTxt) + if err != nil { + t.Fatal(err) + } + + testPng, err := os.Open("testdata/avatar.png") + if err != nil { + t.Fatal(err) + } + defer testPng.Close() + testPngData, err := io.ReadAll(testPng) + if err != nil { + t.Fatal(err) + } + + type args struct { + u DownloadInput + } + tests := []struct { + name string + fields tConfig + args args + wantErr bool + wantResponse []byte + }{ + { + name: "txt", + fields: tConfig{ + AccessKey: os.Getenv("AWS_S3_ACCESS_KEY"), + SecretKey: os.Getenv("AWS_S3_SECRET_KEY"), + Endpoint: os.Getenv("AWS_S3_ENDPOINT"), + Region: os.Getenv("AWS_S3_REGION"), + }, + args: args{ + u: DownloadInput{ + Bucket: os.Getenv("AWS_S3_BUCKET"), + ObjectKey: "test.txt", + }, + }, + wantErr: false, + wantResponse: testTxtData, + }, + { + name: "png", + fields: tConfig{ + AccessKey: os.Getenv("AWS_S3_ACCESS_KEY"), + SecretKey: os.Getenv("AWS_S3_SECRET_KEY"), + Endpoint: os.Getenv("AWS_S3_ENDPOINT"), + Region: os.Getenv("AWS_S3_REGION"), + }, + args: args{ + u: DownloadInput{ + Bucket: os.Getenv("AWS_S3_BUCKET"), + ObjectKey: "xyz/image.png", + }, + }, + wantErr: false, + wantResponse: testPngData, + }, + { + name: "txt-special-filename", + fields: tConfig{ + AccessKey: os.Getenv("AWS_S3_ACCESS_KEY"), + SecretKey: os.Getenv("AWS_S3_SECRET_KEY"), + Endpoint: os.Getenv("AWS_S3_ENDPOINT"), + Region: os.Getenv("AWS_S3_REGION"), + }, + args: args{ + u: DownloadInput{ + Bucket: os.Getenv("AWS_S3_BUCKET"), + ObjectKey: "xyz/example file%with$special&chars(1)?.txt", + }, + }, + wantErr: false, + wantResponse: testTxtData, + }, + } + + for _, testcase := range tests { + tt := testcase + t.Run(tt.name, func(t *testing.T) { + s3 := New(tt.fields.Region, tt.fields.AccessKey, tt.fields.SecretKey) + s3.SetEndpoint(tt.fields.Endpoint) + resp, err := s3.FileDownload(tt.args.u) + if (err != nil) != tt.wantErr { + t.Fatalf("S3.FileDownload() error = %v, wantErr %v", err, tt.wantErr) + } + + got, err := io.ReadAll(resp) + if err != nil { + t.Fatalf("error = %v", err) + } + + resp.Close() + + if !bytes.Equal(got, tt.wantResponse) { + t.Fatalf("S3.FileDownload() = %v, want %v", got, tt.wantResponse) + } + }) + } +} + +func TestS3_FileDelete(t *testing.T) { + type args struct { + u DeleteInput + } + tests := []struct { + name string + fields tConfig + args args + wantErr bool + }{ + { + name: "Delete test.txt", + fields: tConfig{ + AccessKey: os.Getenv("AWS_S3_ACCESS_KEY"), + SecretKey: os.Getenv("AWS_S3_SECRET_KEY"), + Endpoint: os.Getenv("AWS_S3_ENDPOINT"), + Region: os.Getenv("AWS_S3_REGION"), + }, + args: args{ + DeleteInput{ + Bucket: os.Getenv("AWS_S3_BUCKET"), + ObjectKey: "test.txt", + }, + }, + wantErr: false, + }, + { + name: "Delete test_metadata.txt", + fields: tConfig{ + AccessKey: os.Getenv("AWS_S3_ACCESS_KEY"), + SecretKey: os.Getenv("AWS_S3_SECRET_KEY"), + Endpoint: os.Getenv("AWS_S3_ENDPOINT"), + Region: os.Getenv("AWS_S3_REGION"), + }, + args: args{ + DeleteInput{ + Bucket: os.Getenv("AWS_S3_BUCKET"), + ObjectKey: "test_metadata.txt", + }, + }, + wantErr: false, + }, + { + name: "Delete avatar.png", + fields: tConfig{ + AccessKey: os.Getenv("AWS_S3_ACCESS_KEY"), + SecretKey: os.Getenv("AWS_S3_SECRET_KEY"), + Endpoint: os.Getenv("AWS_S3_ENDPOINT"), + Region: os.Getenv("AWS_S3_REGION"), + }, + args: args{ + DeleteInput{ + Bucket: os.Getenv("AWS_S3_BUCKET"), + ObjectKey: "xyz/image.png", + }, + }, + wantErr: false, + }, + { + name: "Delete special filename txt", + fields: tConfig{ + AccessKey: os.Getenv("AWS_S3_ACCESS_KEY"), + SecretKey: os.Getenv("AWS_S3_SECRET_KEY"), + Endpoint: os.Getenv("AWS_S3_ENDPOINT"), + Region: os.Getenv("AWS_S3_REGION"), + }, + args: args{ + DeleteInput{ + Bucket: os.Getenv("AWS_S3_BUCKET"), + ObjectKey: "xyz/example file%with$special&chars(1)?.txt", + }, + }, + wantErr: false, + }, + } + for _, testcase := range tests { + tt := testcase + t.Run(tt.name, func(t *testing.T) { + s3 := New(tt.fields.Region, tt.fields.AccessKey, tt.fields.SecretKey) + s3.SetEndpoint(tt.fields.Endpoint) + if err := s3.FileDelete(tt.args.u); (err != nil) != tt.wantErr { + t.Errorf("S3.FileDelete() error = %v, wantErr %v", err, tt.wantErr) + } + }) + } +} diff --git a/simples3.go b/simples3.go index cb39850..192e4ee 100644 --- a/simples3.go +++ b/simples3.go @@ -7,30 +7,14 @@ import ( "bytes" "crypto/hmac" "crypto/sha256" - "encoding/hex" - "encoding/json" - "encoding/xml" "fmt" - "io" - "iter" - "mime/multipart" "net/http" - "net/url" - "regexp" "strings" "sync" "time" - "unicode/utf8" ) const ( - imdsTokenHeader = "X-aws-ec2-metadata-token" - imdsTokenTtlHeader = "X-aws-ec2-metadata-token-ttl-seconds" - metadataBaseURL = "http://169.254.169.254/latest" - securityCredentialsURI = "/meta-data/iam/security-credentials/" - imdsTokenURI = "/api/token" - defaultIMDSTokenTTL = "60" - // AMZMetaPrefix to prefix metadata key. AMZMetaPrefix = "x-amz-meta-" ) @@ -51,168 +35,6 @@ type S3 struct { mu sync.Mutex } -// DownloadInput is passed to FileDownload as a parameter. -type DownloadInput struct { - Bucket string - ObjectKey string -} - -// DetailsInput is passed to FileDetails as a parameter. -type DetailsInput struct { - Bucket string - ObjectKey string -} - -// DetailsResponse is returned by FileDetails. -type DetailsResponse struct { - ContentType string - ContentLength string - AcceptRanges string - Date string - Etag string - LastModified string - Server string - AmzID2 string - AmzRequestID string - AmzMeta map[string]string - ExtraHeaders map[string]string -} - -// UploadInput is passed to FileUpload as a parameter. -type UploadInput struct { - // essential fields - Bucket string - ObjectKey string - FileName string - ContentType string - - // optional fields - ContentDisposition string - ACL string - // Setting key/value pairs adds user-defined metadata - // keys to the object, prefixed with AMZMetaPrefix. - CustomMetadata map[string]string - - Body io.ReadSeeker -} - -// UploadResponse receives the following XML -// in case of success, since we set a 201 response from S3. -// Sample response: -// -// -// https://s3.amazonaws.com/link-to-the-file -// s3-bucket -// development/8614bd40-691b-4668-9241-3b342c6cf429/image.jpg -// "32-bit-tag" -// -type UploadResponse struct { - Location string `xml:"Location"` - Bucket string `xml:"Bucket"` - Key string `xml:"Key"` - ETag string `xml:"ETag"` -} - -// PutResponse is returned when the action is successful, -// and the service sends back an HTTP 200 response. The response -// returns ETag along with HTTP headers. -type PutResponse struct { - ETag string - Headers http.Header -} - -// DeleteInput is passed to FileDelete as a parameter. -type DeleteInput struct { - Bucket string - ObjectKey string -} - -// ListInput is passed to List as a parameter. -type ListInput struct { - // Required: The name of the bucket to list objects from - Bucket string - - // Optional: A delimiter to group objects by (commonly "/") - Delimiter string - - // Optional: Only list objects starting with this prefix - Prefix string - - // Optional: Maximum number of objects to return - MaxKeys int64 - - // Optional: Token for pagination from a previous request - ContinuationToken string - - // Optional: Object key to start listing after - StartAfter string -} - -// ListResponse is returned by List. -type ListResponse struct { - // Name of the bucket - Name string - - // Whether the results were truncated (more results available) - IsTruncated bool - - // Token to get the next page of results (if truncated) - NextContinuationToken string - - // List of objects in the bucket - Objects []Object - - // Common prefixes when using delimiter (like directories) - CommonPrefixes []string - - // Total number of keys returned - KeyCount int64 -} - -// Object represents an S3 object in a bucket. -type Object struct { - // Name of the object - Key string - - // Size in bytes - Size int64 - - // When the object was last modified - LastModified string - - // Entity tag of the object - ETag string - - // Storage class (e.g., "STANDARD") - StorageClass string -} - -// S3Error represents an S3 API error response -type S3Error struct { - XMLName xml.Name `xml:"Error"` - Code string `xml:"Code"` - Message string `xml:"Message"` - RequestID string `xml:"RequestId"` - HostID string `xml:"HostId"` -} - -// Error returns a string representation of the S3Error -func (e S3Error) Error() string { - return fmt.Sprintf("S3 Error: %s - %s", e.Code, e.Message) -} - -// IAMResponse is used by NewUsingIAM to auto -// detect the credentials. -type IAMResponse struct { - Code string `json:"Code"` - LastUpdated string `json:"LastUpdated"` - Type string `json:"Type"` - AccessKeyID string `json:"AccessKeyId"` - SecretAccessKey string `json:"SecretAccessKey"` - Token string `json:"Token"` - Expiration time.Time `json:"Expiration"` -} - // New returns an instance of S3. func New(region, accessKey, secretKey string) *S3 { return &S3{ @@ -224,151 +46,6 @@ func New(region, accessKey, secretKey string) *S3 { } } -// NewUsingIAM automatically generates an Instance of S3 -// using instance metatdata. -func NewUsingIAM(region string) (*S3, error) { - return newUsingIAM( - &http.Client{ - // Set a timeout of 3 seconds for AWS IAM Calls. - Timeout: time.Second * 3, //nolint:gomnd - }, metadataBaseURL, region) -} - -// fetchIMDSToken retrieves an IMDSv2 token from the -// EC2 instance metadata service. It returns a token and boolean, -// only if IMDSv2 is enabled in the EC2 instance metadata -// configuration, otherwise returns an error. -func fetchIMDSToken(cl *http.Client, baseURL string) (string, bool, error) { - req, err := http.NewRequest(http.MethodPut, baseURL+imdsTokenURI, nil) - if err != nil { - return "", false, err - } - - // Set the token TTL to 60 seconds. - req.Header.Set(imdsTokenTtlHeader, defaultIMDSTokenTTL) - - resp, err := cl.Do(req) - if err != nil { - return "", false, err - } - - defer func() { - resp.Body.Close() - io.Copy(io.Discard, resp.Body) - }() - - if resp.StatusCode != http.StatusOK { - return "", false, fmt.Errorf("failed to request IMDSv2 token: %s", resp.Status) - } - - token, err := io.ReadAll(resp.Body) - if err != nil { - return "", false, err - } - - return string(token), true, nil -} - -// fetchIAMData fetches the IAM data from the given URL. -// In case of a normal AWS setup, baseURL would be metadataBaseURL. -// You can use this method, to manually fetch IAM data from a custom -// endpoint and pass it to SetIAMData. -func fetchIAMData(cl *http.Client, baseURL string) (IAMResponse, error) { - token, useIMDSv2, err := fetchIMDSToken(cl, baseURL) - if err != nil { - return IAMResponse{}, fmt.Errorf("error fetching IMDSv2 token: %w", err) - } - - url := baseURL + securityCredentialsURI - - req, err := http.NewRequest(http.MethodGet, url, nil) - if err != nil { - return IAMResponse{}, fmt.Errorf("error creating imdsv2 token request: %w", err) - } - - if useIMDSv2 { - req.Header.Set(imdsTokenHeader, token) - } - - resp, err := cl.Do(req) - if err != nil { - return IAMResponse{}, err - } - - if resp.StatusCode != http.StatusOK { - return IAMResponse{}, fmt.Errorf("error fetching IAM data: %s", resp.Status) - } - - role, err := io.ReadAll(resp.Body) - if err != nil { - return IAMResponse{}, err - } - - _, _ = io.Copy(io.Discard, resp.Body) - _ = resp.Body.Close() - - req, err = http.NewRequest(http.MethodGet, url+string(role), nil) - if err != nil { - return IAMResponse{}, fmt.Errorf("error creating role request: %w", err) - } - if useIMDSv2 { - req.Header.Set(imdsTokenHeader, token) - } - - resp, err = cl.Do(req) - if err != nil { - return IAMResponse{}, fmt.Errorf("error fetching role data: %w", err) - } - - defer func() { - // Drain and close the body to let the Transport reuse the connection - io.Copy(io.Discard, resp.Body) - resp.Body.Close() - }() - - if resp.StatusCode != http.StatusOK { - return IAMResponse{}, fmt.Errorf("error fetching role data, got non 200 code: %s", resp.Status) - } - - var jResp IAMResponse - jsonString, err := io.ReadAll(resp.Body) - if err != nil { - return IAMResponse{}, fmt.Errorf("error reading role data: %w", err) - } - - if err := json.Unmarshal(jsonString, &jResp); err != nil { - return IAMResponse{}, fmt.Errorf("error unmarshalling role data: %w (%s)", err, jsonString) - } - - return jResp, nil -} - -func newUsingIAM(cl *http.Client, baseURL, region string) (*S3, error) { - // Get the IAM role - iamResp, err := fetchIAMData(cl, baseURL) - if err != nil { - return nil, fmt.Errorf("error fetching IAM data: %w", err) - } - - return &S3{ - Region: region, - AccessKey: iamResp.AccessKeyID, - SecretKey: iamResp.SecretAccessKey, - Token: iamResp.Token, - - URIFormat: "https://s3.%s.amazonaws.com/%s", - initMode: "iam", - expiry: iamResp.Expiration, - }, nil -} - -// setIAMData sets the IAM data on the S3 instance. -func (s3 *S3) SetIAMData(iamResp IAMResponse) { - s3.AccessKey = iamResp.AccessKeyID - s3.SecretKey = iamResp.SecretAccessKey - s3.Token = iamResp.Token -} - func (s3 *S3) getClient() *http.Client { if s3.Client == nil { return http.DefaultClient @@ -376,25 +53,6 @@ func (s3 *S3) getClient() *http.Client { return s3.Client } -// getURL constructs a URL for a given path, with multiple optional -// arguments as individual subfolders, based on the endpoint -// specified in s3 struct. -func (s3 *S3) getURL(path string, args ...string) (uri string) { - if len(args) > 0 { - path += "/" + strings.Join(args, "/") - } - // need to encode special characters in the path part of the URL - encodedPath := encodePath(path) - - if len(s3.Endpoint) > 0 { - uri = s3.Endpoint + "/" + encodedPath - } else { - uri = fmt.Sprintf(s3.URIFormat, s3.Region, encodedPath) - } - - return uri -} - // SetEndpoint can be used to the set a custom endpoint for // using an alternate instance compatible with the s3 API. // If no protocol is included in the URI, defaults to HTTPS. @@ -422,20 +80,6 @@ func (s3 *S3) SetToken(token string) *S3 { return s3 } -func detectFileSize(body io.Seeker) (int64, error) { - pos, err := body.Seek(0, 1) - if err != nil { - return -1, err - } - defer body.Seek(pos, 0) - - n, err := body.Seek(0, 2) //nolint:gomnd - if err != nil { - return -1, err - } - return n, nil -} - // SetClient can be used to set the http client to be // used by the package. If client passed is nil, // http.DefaultClient is used. @@ -495,570 +139,3 @@ func (s3 *S3) signRequest(req *http.Request) error { req.Header.Set("Authorization", auth.String()) return nil } - -func (s3 *S3) renewIAMToken() error { - if s3.initMode != "iam" { - return nil - } - - if time.Since(s3.expiry) < 0 { - return nil - } - - s3.mu.Lock() - defer s3.mu.Unlock() - iamResp, err := fetchIAMData(s3.getClient(), metadataBaseURL) - if err != nil { - return fmt.Errorf("error fetching IAM data: %w", err) - } - - s3.expiry = iamResp.Expiration - s3.Token = iamResp.Token - s3.AccessKey = iamResp.AccessKeyID - s3.SecretKey = iamResp.SecretAccessKey - - return nil -} - -// FileDownload makes a GET call and returns a io.ReadCloser. -// After reading the response body, ensure closing the response. -func (s3 *S3) FileDownload(u DownloadInput) (io.ReadCloser, error) { - req, err := http.NewRequest( - http.MethodGet, s3.getURL(u.Bucket, u.ObjectKey), nil, - ) - if err != nil { - return nil, err - } - - if err := s3.renewIAMToken(); err != nil { - return nil, err - } - if err := s3.signRequest(req); err != nil { - return nil, err - } - - res, err := s3.getClient().Do(req) - if err != nil { - return nil, err - } - - if res.StatusCode != http.StatusOK { - return nil, fmt.Errorf("status code: %s", res.Status) - } - - return res.Body, nil -} - -// FilePut makes a PUT call to S3. -func (s3 *S3) FilePut(u UploadInput) (PutResponse, error) { - fSize, err := detectFileSize(u.Body) - if err != nil { - return PutResponse{}, err - } - - content := make([]byte, fSize) - _, err = u.Body.Read(content) - if err != nil { - return PutResponse{}, err - } - u.Body.Seek(0, 0) - - req, er := http.NewRequest(http.MethodPut, s3.getURL(u.Bucket, u.ObjectKey), u.Body) - if er != nil { - return PutResponse{}, err - } - - if u.ContentType == "" { - u.ContentType = "application/octet-stream" - } - - h := sha256.New() - h.Write(content) - req.Header.Set("x-amz-content-sha256", fmt.Sprintf("%x", h.Sum(nil))) - - req.Header.Set("Content-Type", u.ContentType) - req.Header.Set("Content-Length", fmt.Sprintf("%d", fSize)) - req.Header.Set("Host", req.URL.Host) - - for k, v := range u.CustomMetadata { - req.Header.Set("x-amz-meta-"+k, v) - } - - if u.ContentDisposition != "" { - req.Header.Set("Content-Disposition", u.ContentDisposition) - } - - if u.ACL != "" { - req.Header.Set("x-amz-acl", u.ACL) - } - - req.ContentLength = fSize - - if err := s3.renewIAMToken(); err != nil { - return PutResponse{}, err - } - if err := s3.signRequest(req); err != nil { - return PutResponse{}, err - } - - // debug(httputil.DumpRequest(req, true)) - // Submit the request - client := s3.getClient() - res, err := client.Do(req) - if err != nil { - return PutResponse{}, err - } - - defer func() { - res.Body.Close() - io.Copy(io.Discard, res.Body) - }() - - data, err := io.ReadAll(res.Body) - if err != nil { - return PutResponse{}, err - } - - // Check the response - if res.StatusCode != http.StatusOK { - return PutResponse{}, fmt.Errorf("status code: %s: %q", res.Status, data) - } - - return PutResponse{ - ETag: res.Header.Get("ETag"), - Headers: res.Header.Clone(), - }, nil -} - -// FileUpload makes a POST call with the file written as multipart -// and on successful upload, checks for 200 OK. -func (s3 *S3) FileUpload(u UploadInput) (UploadResponse, error) { - fSize, err := detectFileSize(u.Body) - if err != nil { - return UploadResponse{}, err - } - - uc := UploadConfig{ - UploadURL: s3.getURL(u.Bucket), - BucketName: u.Bucket, - ObjectKey: u.ObjectKey, - ContentType: u.ContentType, - ContentDisposition: u.ContentDisposition, - ACL: u.ACL, - FileSize: fSize, - MetaData: map[string]string{ - "success_action_status": "201", // returns XML doc on success - }, - } - - // Set custom metadata. - for k, v := range u.CustomMetadata { - if !strings.HasPrefix(k, AMZMetaPrefix) { - k = AMZMetaPrefix + k - } - - uc.MetaData[k] = v - } - - if err := s3.renewIAMToken(); err != nil { - return UploadResponse{}, err - } - - policies, err := s3.CreateUploadPolicies(uc) - if err != nil { - return UploadResponse{}, err - } - - var b bytes.Buffer - w := multipart.NewWriter(&b) - - for k, v := range policies.Form { - if err = w.WriteField(k, v); err != nil { - return UploadResponse{}, err - } - } - - fw, err := w.CreateFormFile("file", u.FileName) - if err != nil { - return UploadResponse{}, err - } - - if _, err = io.Copy(fw, u.Body); err != nil { - return UploadResponse{}, err - } - - // Don't forget to close the multipart writer. - // If you don't close it, your request will be missing the terminating boundary. - if err := w.Close(); err != nil { - return UploadResponse{}, err - } - - // Now that you have a form, you can submit it to your handler. - req, err := http.NewRequest(http.MethodPost, policies.URL, &b) - if err != nil { - return UploadResponse{}, err - } - // Don't forget to set the content type, this will contain the boundary. - req.Header.Set("Content-Type", w.FormDataContentType()) - - // Submit the request - client := s3.getClient() - res, err := client.Do(req) - if err != nil { - return UploadResponse{}, err - } - - defer func() { - res.Body.Close() - io.Copy(io.Discard, res.Body) - }() - - data, err := io.ReadAll(res.Body) - if err != nil { - return UploadResponse{}, err - } - - // Check the response - if res.StatusCode != http.StatusCreated { - return UploadResponse{}, fmt.Errorf("status code: %s: %q", res.Status, data) - } - - var ur UploadResponse - _ = xml.Unmarshal(data, &ur) - return ur, nil -} - -// FileDelete makes a DELETE call with the file written as multipart -// and on successful upload, checks for 204 No Content. -func (s3 *S3) FileDelete(u DeleteInput) error { - req, err := http.NewRequest( - http.MethodDelete, s3.getURL(u.Bucket, u.ObjectKey), nil, - ) - if err != nil { - return err - } - - if err := s3.renewIAMToken(); err != nil { - return err - } - - if err := s3.signRequest(req); err != nil { - return err - } - - // Submit the request - client := s3.getClient() - res, err := client.Do(req) - if err != nil { - return err - } - - defer func() { - res.Body.Close() - io.Copy(io.Discard, res.Body) - }() - - // Check the response - if res.StatusCode != http.StatusNoContent { - return fmt.Errorf("status code: %s", res.Status) - } - - return nil -} - -func (s3 *S3) FileDetails(u DetailsInput) (DetailsResponse, error) { - req, err := http.NewRequest( - http.MethodHead, s3.getURL(u.Bucket, u.ObjectKey), nil, - ) - if err != nil { - return DetailsResponse{}, err - } - - if err := s3.renewIAMToken(); err != nil { - return DetailsResponse{}, err - } - - if err := s3.signRequest(req); err != nil { - return DetailsResponse{}, err - } - - res, err := s3.getClient().Do(req) - if err != nil { - return DetailsResponse{}, err - } - - defer func() { - res.Body.Close() - io.Copy(io.Discard, res.Body) - }() - - if res.StatusCode != http.StatusOK { - return DetailsResponse{}, fmt.Errorf("status code: %s", res.Status) - } - - var out DetailsResponse - for k, v := range res.Header { - lk := strings.ToLower(k) - - switch lk { - case "content-type": - out.ContentType = getFirstString(v) - case "content-length": - out.ContentLength = getFirstString(v) - case "accept-ranges": - out.AcceptRanges = getFirstString(v) - case "date": - out.Date = getFirstString(v) - case "etag": - out.Etag = getFirstString(v) - case "last-modified": - out.LastModified = getFirstString(v) - case "server": - out.Server = getFirstString(v) - case "x-amz-id-2": - out.AmzID2 = getFirstString(v) - case "x-amz-request-id": - out.AmzRequestID = getFirstString(v) - default: - if strings.HasPrefix(lk, AMZMetaPrefix) { - if out.AmzMeta == nil { - out.AmzMeta = map[string]string{} - } - - out.AmzMeta[k] = getFirstString(v) - } else { - if out.ExtraHeaders == nil { - out.ExtraHeaders = map[string]string{} - } - - out.ExtraHeaders[k] = getFirstString(v) - } - } - } - - return out, nil -} - -// List implements a simple S3 object listing API -func (s3 *S3) List(input ListInput) (ListResponse, error) { - // Input validation - if input.Bucket == "" { - return ListResponse{}, fmt.Errorf("bucket name cannot be empty") - } - if input.MaxKeys < 0 { - return ListResponse{}, fmt.Errorf("MaxKeys cannot be negative") - } - if input.MaxKeys > 1000 { - return ListResponse{}, fmt.Errorf("MaxKeys cannot exceed 1000") - } - - // Build query parameters - ListObjectsV2 uses query params, not path - query := url.Values{} - query.Set("list-type", "2") // Required parameter - - // Add optional parameters if they exist - if input.ContinuationToken != "" { - query.Set("continuation-token", input.ContinuationToken) - } - if input.Delimiter != "" { - query.Set("delimiter", input.Delimiter) - } - if input.MaxKeys > 0 { - query.Set("max-keys", fmt.Sprintf("%d", input.MaxKeys)) - } - if input.Prefix != "" { - query.Set("prefix", input.Prefix) - } - if input.StartAfter != "" { - query.Set("start-after", input.StartAfter) - } - - // Build base URL - baseURL := s3.getURL(input.Bucket) - - // Parse URL and add query parameters - parsedURL, err := url.Parse(baseURL) - if err != nil { - return ListResponse{}, err - } - parsedURL.RawQuery = query.Encode() - - // Create HTTP request - req, err := http.NewRequest(http.MethodGet, parsedURL.String(), nil) - if err != nil { - return ListResponse{}, err - } - - // Apply AWS V4 signing - if err := s3.renewIAMToken(); err != nil { - return ListResponse{}, err - } - if err := s3.signRequest(req); err != nil { - return ListResponse{}, err - } - - // Execute request - client := s3.getClient() - res, err := client.Do(req) - if err != nil { - return ListResponse{}, err - } - - // Close response body when done - defer res.Body.Close() - - // Read response body once - body, err := io.ReadAll(res.Body) - if err != nil { - return ListResponse{}, err - } - - // Handle response status codes - if res.StatusCode != http.StatusOK { - var s3Err S3Error - if xmlErr := xml.Unmarshal(body, &s3Err); xmlErr == nil { - return ListResponse{}, fmt.Errorf("S3 Error: %s - %s", s3Err.Code, s3Err.Message) - } - return ListResponse{}, fmt.Errorf("status code: %s: %s", res.Status, string(body)) - } - - // Parse XML response using internal struct - var xmlResult struct { - XMLName xml.Name `xml:"ListBucketResult"` - Name string `xml:"Name"` - IsTruncated bool `xml:"IsTruncated"` - NextContinuationToken string `xml:"NextContinuationToken"` - KeyCount int64 `xml:"KeyCount"` - Contents []struct { - Key string `xml:"Key"` - LastModified string `xml:"LastModified"` - ETag string `xml:"ETag"` - Size int64 `xml:"Size"` - StorageClass string `xml:"StorageClass"` - } `xml:"Contents"` - CommonPrefixes []struct { - Prefix string `xml:"Prefix"` - } `xml:"CommonPrefixes"` - } - - if err := xml.Unmarshal(body, &xmlResult); err != nil { - return ListResponse{}, err - } - - // Convert to simple response format - response := ListResponse{ - Name: xmlResult.Name, - IsTruncated: xmlResult.IsTruncated, - NextContinuationToken: xmlResult.NextContinuationToken, - KeyCount: xmlResult.KeyCount, - } - - // Convert objects - for _, obj := range xmlResult.Contents { - response.Objects = append(response.Objects, Object{ - Key: obj.Key, - Size: obj.Size, - LastModified: obj.LastModified, - ETag: obj.ETag, - StorageClass: obj.StorageClass, - }) - } - - // Convert common prefixes - for _, prefix := range xmlResult.CommonPrefixes { - response.CommonPrefixes = append(response.CommonPrefixes, prefix.Prefix) - } - - return response, nil -} - -// ListAll returns an iterator that yields all objects in the bucket, -// automatically handling pagination. It also returns a finish callback -// that should be called after iteration to check for any errors. -func (s3 *S3) ListAll(input ListInput) (iter.Seq[Object], func() error) { - var iterErr error - - seq := func(yield func(Object) bool) { - currentInput := input - - for { - response, err := s3.List(currentInput) - if err != nil { - iterErr = err - return - } - - // Yield each object - for _, obj := range response.Objects { - if !yield(obj) { - return // Early termination requested - } - } - - // Check if there are more results - if !response.IsTruncated || response.NextContinuationToken == "" { - return // No more results - } - - // Prepare for next page - currentInput.ContinuationToken = response.NextContinuationToken - } - } - - finish := func() error { - return iterErr - } - - return seq, finish -} - -func getFirstString(s []string) string { - if len(s) > 0 { - return s[0] - } - - return "" -} - -// if object matches reserved string, no need to encode them. -var reservedObjectNames = regexp.MustCompile("^[a-zA-Z0-9-_.~/]+$") - -// encodePath encode the strings from UTF-8 byte representations to HTML hex escape sequences -// -// This is necessary since regular url.Parse() and url.Encode() functions do not support UTF-8 -// non english characters cannot be parsed due to the nature in which url.Encode() is written -// -// This function on the other hand is a direct replacement for url.Encode() technique to support -// pretty much every UTF-8 character. -// adapted from -// https://github.com/minio/minio-go/blob/fe1f3855b146c1b6ce4199740d317e44cf9e85c2/pkg/s3utils/utils.go#L285 -func encodePath(pathName string) string { - if reservedObjectNames.MatchString(pathName) { - return pathName - } - var encodedPathname strings.Builder - for _, s := range pathName { - if 'A' <= s && s <= 'Z' || 'a' <= s && s <= 'z' || '0' <= s && s <= '9' { // §2.3 Unreserved characters (mark) - encodedPathname.WriteRune(s) - continue - } - switch s { - case '-', '_', '.', '~', '/': // §2.3 Unreserved characters (mark) - encodedPathname.WriteRune(s) - continue - default: - lenR := utf8.RuneLen(s) - if lenR < 0 { - // if utf8 cannot convert, return the same string as is - return pathName - } - u := make([]byte, lenR) - utf8.EncodeRune(u, s) - for _, r := range u { - hex := hex.EncodeToString([]byte{r}) - encodedPathname.WriteString("%" + strings.ToUpper(hex)) - } - } - } - return encodedPathname.String() -} diff --git a/simples3_test.go b/simples3_test.go index 9a4c229..5c64c97 100644 --- a/simples3_test.go +++ b/simples3_test.go @@ -1,17 +1,8 @@ package simples3 import ( - "bytes" - "errors" - "fmt" - "io" - "net" - "net/http" - "net/http/httptest" - "os" "strings" "testing" - "time" ) type tConfig struct { @@ -21,613 +12,6 @@ type tConfig struct { Region string } -func TestS3_FileUploadPostAndPut(t *testing.T) { - testTxt, err := os.Open("testdata/test.txt") - if err != nil { - t.Fatalf("Failed to open test.txt: %v", err) - } - defer testTxt.Close() - // Note: cannot re-use the same file descriptor due to seeking! - testTxtSpecialChars, err := os.Open("testdata/test.txt") - if err != nil { - t.Fatalf("Failed to open test.txt for special chars: %v", err) - } - defer testTxtSpecialChars.Close() - testPng, err := os.Open("testdata/avatar.png") - if err != nil { - t.Fatalf("Failed to open avatar.png: %v", err) - } - defer testPng.Close() - - type args struct { - u UploadInput - } - tests := []struct { - name string - fields tConfig - args args - testDetails bool - wantErr bool - }{ - { - name: "Upload test.txt", - fields: tConfig{ - AccessKey: os.Getenv("AWS_S3_ACCESS_KEY"), - SecretKey: os.Getenv("AWS_S3_SECRET_KEY"), - Endpoint: os.Getenv("AWS_S3_ENDPOINT"), - Region: os.Getenv("AWS_S3_REGION"), - }, - args: args{ - UploadInput{ - Bucket: os.Getenv("AWS_S3_BUCKET"), - ObjectKey: "test.txt", - ContentType: "text/plain", - FileName: "test.txt", - Body: testTxt, - }, - }, - wantErr: false, - }, - { - name: "Upload test.txt with custom metadata", - fields: tConfig{ - AccessKey: os.Getenv("AWS_S3_ACCESS_KEY"), - SecretKey: os.Getenv("AWS_S3_SECRET_KEY"), - Endpoint: os.Getenv("AWS_S3_ENDPOINT"), - Region: os.Getenv("AWS_S3_REGION"), - }, - args: args{ - UploadInput{ - Bucket: os.Getenv("AWS_S3_BUCKET"), - ObjectKey: "test_metadata.txt", - ContentType: "text/plain", - FileName: "test.txt", - Body: testTxt, - CustomMetadata: map[string]string{ - "test-metadata": "foo-bar", - }, - }, - }, - wantErr: false, - testDetails: true, - }, - { - name: "Upload avatar.png", - fields: tConfig{ - AccessKey: os.Getenv("AWS_S3_ACCESS_KEY"), - SecretKey: os.Getenv("AWS_S3_SECRET_KEY"), - Endpoint: os.Getenv("AWS_S3_ENDPOINT"), - Region: os.Getenv("AWS_S3_REGION"), - }, - args: args{ - UploadInput{ - Bucket: os.Getenv("AWS_S3_BUCKET"), - ObjectKey: "xyz/image.png", - ContentType: "image/png", - FileName: "avatar.png", - Body: testPng, - }, - }, - wantErr: false, - }, - { - name: "Upload special filename txt", - fields: tConfig{ - AccessKey: os.Getenv("AWS_S3_ACCESS_KEY"), - SecretKey: os.Getenv("AWS_S3_SECRET_KEY"), - Endpoint: os.Getenv("AWS_S3_ENDPOINT"), - Region: os.Getenv("AWS_S3_REGION"), - }, - args: args{ - UploadInput{ - Bucket: os.Getenv("AWS_S3_BUCKET"), - ObjectKey: "xyz/example file%with$special&chars(1)?.txt", - ContentType: "text/plain", - FileName: "example file%with$special&chars(1)?.txt", - Body: testTxtSpecialChars, - }, - }, - wantErr: false, - }, - } - for _, testcase := range tests { - tt := testcase - t.Run(tt.name+"_post", func(t *testing.T) { - s3 := New(tt.fields.Region, tt.fields.AccessKey, tt.fields.SecretKey) - s3.SetEndpoint(tt.fields.Endpoint) - - resp, err := s3.FileUpload(tt.args.u) - if (err != nil) != tt.wantErr { - t.Errorf("S3.FileUpload() error = %v, wantErr %v", err, tt.wantErr) - } - - // reset file, to reuse in further tests. - tt.args.u.Body.Seek(0, 0) - - // check for empty response - if (resp == UploadResponse{}) { - t.Errorf("S3.FileUpload() returned empty response, %v", resp) - } - - if tt.testDetails { - dResp, err := s3.FileDetails(DetailsInput{ - Bucket: tt.args.u.Bucket, - ObjectKey: tt.args.u.ObjectKey, - }) - - if (err != nil) != tt.wantErr { - t.Errorf("S3.FileDetails() error = %v, wantErr %v", err, tt.wantErr) - } - - if len(dResp.AmzMeta) != len(tt.args.u.CustomMetadata) { - t.Errorf("S3.FileDetails() returned incorrect metadata, got: %#v", dResp) - } - } - }) - t.Run(tt.name+"_put", func(t *testing.T) { - s3 := New(tt.fields.Region, tt.fields.AccessKey, tt.fields.SecretKey) - s3.SetEndpoint(tt.fields.Endpoint) - - resp, err := s3.FilePut(tt.args.u) - if (err != nil) != tt.wantErr { - t.Errorf("S3.FileUpload() error = %v, wantErr %v", err, tt.wantErr) - } - - // reset file, to reuse in further tests. - tt.args.u.Body.Seek(0, 0) - - // check for empty response - if resp.ETag == "" { - t.Errorf("S3.FileUpload() returned empty response, %v", resp) - } - - if tt.testDetails { - dResp, err := s3.FileDetails(DetailsInput{ - Bucket: tt.args.u.Bucket, - ObjectKey: tt.args.u.ObjectKey, - }) - - if (err != nil) != tt.wantErr { - t.Errorf("S3.FileUpload() error = %v, wantErr %v", err, tt.wantErr) - } - - if len(dResp.AmzMeta) != len(tt.args.u.CustomMetadata) { - t.Errorf("S3.FileDetails() returned incorrect metadata, got: %#v", dResp) - } - } - }) - } -} - -func TestS3_FileDownload(t *testing.T) { - testTxt, err := os.Open("testdata/test.txt") - if err != nil { - t.Fatal(err) - } - defer testTxt.Close() - testTxtData, err := io.ReadAll(testTxt) - if err != nil { - t.Fatal(err) - } - - testPng, err := os.Open("testdata/avatar.png") - if err != nil { - t.Fatal(err) - } - defer testPng.Close() - testPngData, err := io.ReadAll(testPng) - if err != nil { - t.Fatal(err) - } - - type args struct { - u DownloadInput - } - tests := []struct { - name string - fields tConfig - args args - wantErr bool - wantResponse []byte - }{ - { - name: "txt", - fields: tConfig{ - AccessKey: os.Getenv("AWS_S3_ACCESS_KEY"), - SecretKey: os.Getenv("AWS_S3_SECRET_KEY"), - Endpoint: os.Getenv("AWS_S3_ENDPOINT"), - Region: os.Getenv("AWS_S3_REGION"), - }, - args: args{ - u: DownloadInput{ - Bucket: os.Getenv("AWS_S3_BUCKET"), - ObjectKey: "test.txt", - }, - }, - wantErr: false, - wantResponse: testTxtData, - }, - { - name: "png", - fields: tConfig{ - AccessKey: os.Getenv("AWS_S3_ACCESS_KEY"), - SecretKey: os.Getenv("AWS_S3_SECRET_KEY"), - Endpoint: os.Getenv("AWS_S3_ENDPOINT"), - Region: os.Getenv("AWS_S3_REGION"), - }, - args: args{ - u: DownloadInput{ - Bucket: os.Getenv("AWS_S3_BUCKET"), - ObjectKey: "xyz/image.png", - }, - }, - wantErr: false, - wantResponse: testPngData, - }, - { - name: "txt-special-filename", - fields: tConfig{ - AccessKey: os.Getenv("AWS_S3_ACCESS_KEY"), - SecretKey: os.Getenv("AWS_S3_SECRET_KEY"), - Endpoint: os.Getenv("AWS_S3_ENDPOINT"), - Region: os.Getenv("AWS_S3_REGION"), - }, - args: args{ - u: DownloadInput{ - Bucket: os.Getenv("AWS_S3_BUCKET"), - ObjectKey: "xyz/example file%with$special&chars(1)?.txt", - }, - }, - wantErr: false, - wantResponse: testTxtData, - }, - } - - for _, testcase := range tests { - tt := testcase - t.Run(tt.name, func(t *testing.T) { - s3 := New(tt.fields.Region, tt.fields.AccessKey, tt.fields.SecretKey) - s3.SetEndpoint(tt.fields.Endpoint) - resp, err := s3.FileDownload(tt.args.u) - if (err != nil) != tt.wantErr { - t.Fatalf("S3.FileDownload() error = %v, wantErr %v", err, tt.wantErr) - } - - got, err := io.ReadAll(resp) - if err != nil { - t.Fatalf("error = %v", err) - } - - resp.Close() - - if !bytes.Equal(got, tt.wantResponse) { - t.Fatalf("S3.FileDownload() = %v, want %v", got, tt.wantResponse) - } - }) - } -} - -func TestS3_FileDelete(t *testing.T) { - type args struct { - u DeleteInput - } - tests := []struct { - name string - fields tConfig - args args - wantErr bool - }{ - { - name: "Delete test.txt", - fields: tConfig{ - AccessKey: os.Getenv("AWS_S3_ACCESS_KEY"), - SecretKey: os.Getenv("AWS_S3_SECRET_KEY"), - Endpoint: os.Getenv("AWS_S3_ENDPOINT"), - Region: os.Getenv("AWS_S3_REGION"), - }, - args: args{ - DeleteInput{ - Bucket: os.Getenv("AWS_S3_BUCKET"), - ObjectKey: "test.txt", - }, - }, - wantErr: false, - }, - { - name: "Delete test_metadata.txt", - fields: tConfig{ - AccessKey: os.Getenv("AWS_S3_ACCESS_KEY"), - SecretKey: os.Getenv("AWS_S3_SECRET_KEY"), - Endpoint: os.Getenv("AWS_S3_ENDPOINT"), - Region: os.Getenv("AWS_S3_REGION"), - }, - args: args{ - DeleteInput{ - Bucket: os.Getenv("AWS_S3_BUCKET"), - ObjectKey: "test_metadata.txt", - }, - }, - wantErr: false, - }, - { - name: "Delete avatar.png", - fields: tConfig{ - AccessKey: os.Getenv("AWS_S3_ACCESS_KEY"), - SecretKey: os.Getenv("AWS_S3_SECRET_KEY"), - Endpoint: os.Getenv("AWS_S3_ENDPOINT"), - Region: os.Getenv("AWS_S3_REGION"), - }, - args: args{ - DeleteInput{ - Bucket: os.Getenv("AWS_S3_BUCKET"), - ObjectKey: "xyz/image.png", - }, - }, - wantErr: false, - }, - { - name: "Delete special filename txt", - fields: tConfig{ - AccessKey: os.Getenv("AWS_S3_ACCESS_KEY"), - SecretKey: os.Getenv("AWS_S3_SECRET_KEY"), - Endpoint: os.Getenv("AWS_S3_ENDPOINT"), - Region: os.Getenv("AWS_S3_REGION"), - }, - args: args{ - DeleteInput{ - Bucket: os.Getenv("AWS_S3_BUCKET"), - ObjectKey: "xyz/example file%with$special&chars(1)?.txt", - }, - }, - wantErr: false, - }, - } - for _, testcase := range tests { - tt := testcase - t.Run(tt.name, func(t *testing.T) { - s3 := New(tt.fields.Region, tt.fields.AccessKey, tt.fields.SecretKey) - s3.SetEndpoint(tt.fields.Endpoint) - if err := s3.FileDelete(tt.args.u); (err != nil) != tt.wantErr { - t.Errorf("S3.FileDelete() error = %v, wantErr %v", err, tt.wantErr) - } - }) - } -} - -func TestS3_NewUsingIAM(t *testing.T) { - var ( - iam = `test-new-s3-using-iam` - resp = `{"Code" : "Success","LastUpdated" : "2018-12-24T10:18:01Z", - "Type" : "AWS-HMAC","AccessKeyId" : "abc", - "SecretAccessKey" : "abc","Token" : "abc", - "Expiration" : "2018-12-24T16:24:59Z"}` - respIMDSToken = `AQAEAJWopi8yvjKYXyWJbzESE0cms-OoTnptJzS3M9g5iNcl06UEkQ==` - ) - - tsFail := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - time.Sleep(2 * time.Second) - })) - defer tsFail.Close() - - genServerHandlerFunc := func(failIMDS bool) http.HandlerFunc { - return func(w http.ResponseWriter, r *http.Request) { - switch r.Method { - case "GET": - if !failIMDS { - // check if token is present - if r.Header.Get(imdsTokenHeader) == "" { - w.WriteHeader(http.StatusUnauthorized) - return - } - } - - url := securityCredentialsURI - if r.URL.EscapedPath() == url { - w.WriteHeader(http.StatusOK) - io.WriteString(w, iam) - } - if r.URL.EscapedPath() == url+iam { - w.WriteHeader(http.StatusOK) - w.Header().Set("Content-Type", "application/json") - io.WriteString(w, resp) - } - case "PUT": - if failIMDS { - w.WriteHeader(http.StatusNotFound) - return - } - - if r.URL.EscapedPath() == imdsTokenURI { - if r.Header.Get(imdsTokenTtlHeader) != "60" { - w.WriteHeader(http.StatusBadRequest) - return - } - - w.WriteHeader(http.StatusOK) - io.WriteString(w, respIMDSToken) - } - default: - t.Errorf("Expected 'GET' or 'PUT' request, got '%s'", r.Method) - } - } - } - - ts := httptest.NewServer(http.HandlerFunc(genServerHandlerFunc(false))) - defer ts.Close() - - tsFailIMDS := httptest.NewServer(http.HandlerFunc(genServerHandlerFunc(true))) - defer tsFailIMDS.Close() - - cl := &http.Client{Timeout: 1 * time.Second} - - // Test for timeout. - _, err := newUsingIAM(cl, tsFail.URL, "abc") - if err == nil { - t.Errorf("Expected error, got nil") - } else { - var timeoutError net.Error - - if errors.As(err, &timeoutError) && !timeoutError.Timeout() { - t.Errorf("newUsingIAM() timeout check. got error = %v", err) - } - } - - // Test for successful IAM fetch. - s3, err := newUsingIAM(cl, ts.URL, "abc") - if err != nil { - t.Errorf("newUsingIAM() error = %v", err) - } - - if s3 == nil { - t.Errorf("newUsingIAM() got = %v", s3) - } - - if s3.AccessKey != "abc" || s3.SecretKey != "abc" || s3.Region != "abc" { - t.Errorf("S3.FileDelete() got = %v", s3) - } - - // Test for failed IMDS token fetch. - _, err = newUsingIAM(cl, tsFailIMDS.URL, "abc") - if err == nil { - t.Errorf("Expected error, got nil") - } -} - -func TestListAll(t *testing.T) { - // Setup test environment - s3 := New( - os.Getenv("AWS_S3_REGION"), - os.Getenv("AWS_S3_ACCESS_KEY"), - os.Getenv("AWS_S3_SECRET_KEY"), - ) - s3.SetEndpoint(os.Getenv("AWS_S3_ENDPOINT")) - - // Test cases - tests := []struct { - name string - setup func() // Setup function to prepare test data - input ListInput - validate func([]Object, error) - cleanup func() // Cleanup function - }{ - { - name: "Empty Bucket Iterator", - input: ListInput{ - Bucket: os.Getenv("AWS_S3_BUCKET"), - }, - validate: func(objects []Object, err error) { - if err != nil { - t.Fatalf("Expected no error, got %v", err) - } - if len(objects) != 0 { - t.Errorf("Expected empty bucket, got %d objects", len(objects)) - } - }, - }, - { - name: "Basic Iterator Listing", - setup: func() { - // Upload test files - uploadTestFiles(t, s3, os.Getenv("AWS_S3_BUCKET"), []string{"iter_file1.txt", "iter_file2.txt", "iter_file3.txt"}) - }, - input: ListInput{ - Bucket: os.Getenv("AWS_S3_BUCKET"), - }, - validate: func(objects []Object, err error) { - if err != nil { - t.Fatalf("Expected no error, got %v", err) - } - if len(objects) != 3 { - t.Errorf("Expected 3 objects, got %d", len(objects)) - } - // Check that all expected files are present - expectedKeys := map[string]bool{ - "iter_file1.txt": false, - "iter_file2.txt": false, - "iter_file3.txt": false, - } - for _, obj := range objects { - if _, exists := expectedKeys[obj.Key]; exists { - expectedKeys[obj.Key] = true - } - } - for key, found := range expectedKeys { - if !found { - t.Errorf("Expected object %s not found in iterator results", key) - } - } - }, - cleanup: func() { - cleanupTestFiles(t, s3, os.Getenv("AWS_S3_BUCKET"), []string{"iter_file1.txt", "iter_file2.txt", "iter_file3.txt"}) - }, - }, - { - name: "Iterator with Prefix", - setup: func() { - uploadTestFiles(t, s3, os.Getenv("AWS_S3_BUCKET"), []string{ - "iter_docs/file1.txt", - "iter_docs/file2.txt", - "iter_images/image1.jpg", - "iter_images/image2.jpg", - }) - }, - input: ListInput{ - Bucket: os.Getenv("AWS_S3_BUCKET"), - Prefix: "iter_docs/", - }, - validate: func(objects []Object, err error) { - if err != nil { - t.Fatalf("Expected no error, got %v", err) - } - if len(objects) != 2 { - t.Errorf("Expected 2 objects with iter_docs/ prefix, got %d", len(objects)) - } - for _, obj := range objects { - if !strings.HasPrefix(obj.Key, "iter_docs/") { - t.Errorf("Object %s doesn't have expected prefix", obj.Key) - } - } - }, - cleanup: func() { - cleanupTestFiles(t, s3, os.Getenv("AWS_S3_BUCKET"), []string{ - "iter_docs/file1.txt", - "iter_docs/file2.txt", - "iter_images/image1.jpg", - "iter_images/image2.jpg", - }) - }, - }, - } - - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - if tt.setup != nil { - tt.setup() - } - - // Collect all objects from the iterator and check for errors - var objects []Object - seq, finish := s3.ListAll(tt.input) - for obj := range seq { - objects = append(objects, obj) - } - - // Check for any iteration errors - if err := finish(); err != nil { - t.Fatalf("Iterator error: %v", err) - } - - if tt.validate != nil { - tt.validate(objects, nil) // No error from iterator - } - - if tt.cleanup != nil { - tt.cleanup() - } - }) - } -} - func TestCustomEndpoint(t *testing.T) { s3 := New("us-east-1", "AccessKey", "SuperSecretKey") @@ -705,184 +89,6 @@ func TestGetURL(t *testing.T) { } } -func TestList(t *testing.T) { - // Setup test environment - s3 := New( - os.Getenv("AWS_S3_REGION"), - os.Getenv("AWS_S3_ACCESS_KEY"), - os.Getenv("AWS_S3_SECRET_KEY"), - ) - s3.SetEndpoint(os.Getenv("AWS_S3_ENDPOINT")) - - // Test cases - tests := []struct { - name string - setup func() // Setup function to prepare test data - input ListInput - validate func(ListResponse, error) - cleanup func() // Cleanup function - }{ - { - name: "Empty Bucket", - input: ListInput{ - Bucket: os.Getenv("AWS_S3_BUCKET"), - }, - validate: func(result ListResponse, err error) { - if err != nil { - t.Fatalf("Expected no error, got %v", err) - } - if len(result.Objects) != 0 { - t.Errorf("Expected empty bucket, got %d objects", len(result.Objects)) - } - }, - }, - { - name: "Basic Listing", - setup: func() { - // Upload test files - uploadTestFiles(t, s3, os.Getenv("AWS_S3_BUCKET"), []string{"file1.txt", "file2.txt", "file3.txt"}) - }, - input: ListInput{ - Bucket: os.Getenv("AWS_S3_BUCKET"), - }, - validate: func(result ListResponse, err error) { - if err != nil { - t.Fatalf("Expected no error, got %v", err) - } - if len(result.Objects) != 3 { - t.Errorf("Expected 3 objects, got %d", len(result.Objects)) - } - }, - cleanup: func() { - cleanupTestFiles(t, s3, os.Getenv("AWS_S3_BUCKET"), []string{"file1.txt", "file2.txt", "file3.txt"}) - }, - }, - { - name: "Pagination Test", - setup: func() { - // Upload many files for pagination testing - var filenames []string - for i := 1; i <= 25; i++ { - filenames = append(filenames, fmt.Sprintf("file%03d.txt", i)) - } - uploadTestFiles(t, s3, os.Getenv("AWS_S3_BUCKET"), filenames) - }, - input: ListInput{ - Bucket: os.Getenv("AWS_S3_BUCKET"), - MaxKeys: 10, - }, - validate: func(result ListResponse, err error) { - if err != nil { - t.Fatalf("Expected no error, got %v", err) - } - if len(result.Objects) != 10 { - t.Errorf("Expected 10 objects, got %d", len(result.Objects)) - } - if !result.IsTruncated { - t.Errorf("Expected IsTruncated=true for pagination") - } - if result.NextContinuationToken == "" { - t.Errorf("Expected NextContinuationToken for pagination: %v", result.NextContinuationToken) - } - }, - cleanup: func() { - var filenames []string - for i := 1; i <= 25; i++ { - filenames = append(filenames, fmt.Sprintf("file%03d.txt", i)) - } - cleanupTestFiles(t, s3, os.Getenv("AWS_S3_BUCKET"), filenames) - }, - }, - { - name: "Prefix Filtering", - setup: func() { - uploadTestFiles(t, s3, os.Getenv("AWS_S3_BUCKET"), []string{ - "documents/file1.txt", - "documents/file2.txt", - "images/image1.jpg", - "images/image2.jpg", - }) - }, - input: ListInput{ - Bucket: os.Getenv("AWS_S3_BUCKET"), - Prefix: "documents/", - }, - validate: func(result ListResponse, err error) { - if err != nil { - t.Fatalf("Expected no error, got %v", err) - } - if len(result.Objects) != 2 { - t.Errorf("Expected 2 objects with documents/ prefix, got %d", len(result.Objects)) - } - for _, obj := range result.Objects { - if !strings.HasPrefix(obj.Key, "documents/") { - t.Errorf("Object %s doesn't have expected prefix", obj.Key) - } - } - }, - cleanup: func() { - cleanupTestFiles(t, s3, os.Getenv("AWS_S3_BUCKET"), []string{ - "documents/file1.txt", - "documents/file2.txt", - "images/image1.jpg", - "images/image2.jpg", - }) - }, - }, - { - name: "Delimiter Grouping", - setup: func() { - uploadTestFiles(t, s3, os.Getenv("AWS_S3_BUCKET"), []string{ - "documents/important/file1.txt", - "documents/backup/file2.txt", - "images/landscape/image1.jpg", - "images/portrait/image2.jpg", - }) - }, - input: ListInput{ - Bucket: os.Getenv("AWS_S3_BUCKET"), - Delimiter: "/", - Prefix: "documents/", - }, - validate: func(result ListResponse, err error) { - if err != nil { - t.Fatalf("Expected no error, got %v", err) - } - // Should return common prefixes instead of individual files - if len(result.CommonPrefixes) != 2 { - t.Errorf("Expected 2 common prefixes, got %d", len(result.CommonPrefixes)) - } - }, - cleanup: func() { - cleanupTestFiles(t, s3, os.Getenv("AWS_S3_BUCKET"), []string{ - "documents/important/file1.txt", - "documents/backup/file2.txt", - "images/landscape/image1.jpg", - "images/portrait/image2.jpg", - }) - }, - }, - } - - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - if tt.setup != nil { - tt.setup() - } - - result, err := s3.List(tt.input) - - if tt.validate != nil { - tt.validate(result, err) - } - - if tt.cleanup != nil { - tt.cleanup() - } - }) - } -} - // Helper functions func uploadTestFiles(t *testing.T, s3 *S3, bucket string, filenames []string) { for _, filename := range filenames {