eiac/simples3
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

refactor: split simples3.go into logical modules

Browse Source 
Split monolithic simples3.go (1064 LOC) into 5 focused modules:
- simples3.go (141 LOC) - Core S3 struct and configuration
- helpers.go (97 LOC) - Utility functions
- iam.go (202 LOC) - IAM credential handling
- list.go (260 LOC) - List operations
- object.go (409 LOC) - File operations

Split simples3_test.go (912 LOC) into 4 test modules:
- simples3_test.go (118 LOC) - Shared test helpers
- iam_test.go (110 LOC) - IAM tests
- list_test.go (321 LOC) - List operation tests
- object_test.go (381 LOC) - File operation tests

Zero functional changes. All tests pass identically. 72.5% test coverage maintained.

"Review v0.10.2 code refactor"
This commit is contained in:
Rohan Verma
2025-11-26 14:57:15 +05:30
parent 61124858fb
commit 2334cb5429
9 changed files with 1780 additions and 1717 deletions
Download Patch File Download Diff File Expand all files Collapse all files
helpers.go
+97
View File
@@ -0,0 +1,97 @@
// LICENSE BSD-2-Clause-FreeBSD
// Copyright (c) 2018, Rohan Verma <hello@rohanverma.net>
package simples3
import (
"encoding/hex"
"fmt"
"io"
"regexp"
"strings"
"unicode/utf8"
)
// getURL constructs a URL for a given path, with multiple optional
// arguments as individual subfolders, based on the endpoint
// specified in s3 struct.
func (s3 *S3) getURL(path string, args ...string) (uri string) {
if len(args) > 0 {
path += "/" + strings.Join(args, "/")
}
// need to encode special characters in the path part of the URL
encodedPath := encodePath(path)
if len(s3.Endpoint) > 0 {
uri = s3.Endpoint + "/" + encodedPath
} else {
uri = fmt.Sprintf(s3.URIFormat, s3.Region, encodedPath)
}
return uri
}
func detectFileSize(body io.Seeker) (int64, error) {
pos, err := body.Seek(0, 1)
if err != nil {
return -1, err
}
defer body.Seek(pos, 0)
n, err := body.Seek(0, 2) //nolint:gomnd
if err != nil {
return -1, err
}
return n, nil
}
func getFirstString(s []string) string {
if len(s) > 0 {
return s[0]
}
return ""
}
// if object matches reserved string, no need to encode them.
var reservedObjectNames = regexp.MustCompile("^[a-zA-Z0-9-_.~/]+$")
// encodePath encode the strings from UTF-8 byte representations to HTML hex escape sequences
//
// This is necessary since regular url.Parse() and url.Encode() functions do not support UTF-8
// non english characters cannot be parsed due to the nature in which url.Encode() is written
//
// This function on the other hand is a direct replacement for url.Encode() technique to support
// pretty much every UTF-8 character.
// adapted from
// https://github.com/minio/minio-go/blob/fe1f3855b146c1b6ce4199740d317e44cf9e85c2/pkg/s3utils/utils.go#L285
func encodePath(pathName string) string {
if reservedObjectNames.MatchString(pathName) {
return pathName
}
var encodedPathname strings.Builder
for _, s := range pathName {
if 'A' <= s && s <= 'Z' || 'a' <= s && s <= 'z' || '0' <= s && s <= '9' { // §2.3 Unreserved characters (mark)
encodedPathname.WriteRune(s)
continue
}
switch s {
case '-', '_', '.', '~', '/': // §2.3 Unreserved characters (mark)
encodedPathname.WriteRune(s)
continue
default:
lenR := utf8.RuneLen(s)
if lenR < 0 {
// if utf8 cannot convert, return the same string as is
return pathName
}
u := make([]byte, lenR)
utf8.EncodeRune(u, s)
for _, r := range u {
hex := hex.EncodeToString([]byte{r})
encodedPathname.WriteString("%" + strings.ToUpper(hex))
}
}
}
return encodedPathname.String()
}
iam.go
+202
View File
@@ -0,0 +1,202 @@
// LICENSE BSD-2-Clause-FreeBSD
// Copyright (c) 2018, Rohan Verma <hello@rohanverma.net>
package simples3
import (
"encoding/json"
"fmt"
"io"
"net/http"
"time"
)
const (
imdsTokenHeader = "X-aws-ec2-metadata-token"
imdsTokenTtlHeader = "X-aws-ec2-metadata-token-ttl-seconds"
metadataBaseURL = "http://169.254.169.254/latest"
securityCredentialsURI = "/meta-data/iam/security-credentials/"
imdsTokenURI = "/api/token"
defaultIMDSTokenTTL = "60"
)
// IAMResponse is used by NewUsingIAM to auto
// detect the credentials.
type IAMResponse struct {
Code string `json:"Code"`
LastUpdated string `json:"LastUpdated"`
Type string `json:"Type"`
AccessKeyID string `json:"AccessKeyId"`
SecretAccessKey string `json:"SecretAccessKey"`
Token string `json:"Token"`
Expiration time.Time `json:"Expiration"`
}
// NewUsingIAM automatically generates an Instance of S3
// using instance metatdata.
func NewUsingIAM(region string) (*S3, error) {
return newUsingIAM(
&http.Client{
// Set a timeout of 3 seconds for AWS IAM Calls.
Timeout: time.Second * 3, //nolint:gomnd
}, metadataBaseURL, region)
}
// fetchIMDSToken retrieves an IMDSv2 token from the
// EC2 instance metadata service. It returns a token and boolean,
// only if IMDSv2 is enabled in the EC2 instance metadata
// configuration, otherwise returns an error.
func fetchIMDSToken(cl *http.Client, baseURL string) (string, bool, error) {
req, err := http.NewRequest(http.MethodPut, baseURL+imdsTokenURI, nil)
if err != nil {
return "", false, err
}
// Set the token TTL to 60 seconds.
req.Header.Set(imdsTokenTtlHeader, defaultIMDSTokenTTL)
resp, err := cl.Do(req)
if err != nil {
return "", false, err
}
defer func() {
resp.Body.Close()
io.Copy(io.Discard, resp.Body)
}()
if resp.StatusCode != http.StatusOK {
return "", false, fmt.Errorf("failed to request IMDSv2 token: %s", resp.Status)
}
token, err := io.ReadAll(resp.Body)
if err != nil {
return "", false, err
}
return string(token), true, nil
}
// fetchIAMData fetches the IAM data from the given URL.
// In case of a normal AWS setup, baseURL would be metadataBaseURL.
// You can use this method, to manually fetch IAM data from a custom
// endpoint and pass it to SetIAMData.
func fetchIAMData(cl *http.Client, baseURL string) (IAMResponse, error) {
token, useIMDSv2, err := fetchIMDSToken(cl, baseURL)
if err != nil {
return IAMResponse{}, fmt.Errorf("error fetching IMDSv2 token: %w", err)
}
url := baseURL + securityCredentialsURI
req, err := http.NewRequest(http.MethodGet, url, nil)
if err != nil {
return IAMResponse{}, fmt.Errorf("error creating imdsv2 token request: %w", err)
}
if useIMDSv2 {
req.Header.Set(imdsTokenHeader, token)
}
resp, err := cl.Do(req)
if err != nil {
return IAMResponse{}, err
}
if resp.StatusCode != http.StatusOK {
return IAMResponse{}, fmt.Errorf("error fetching IAM data: %s", resp.Status)
}
role, err := io.ReadAll(resp.Body)
if err != nil {
return IAMResponse{}, err
}
_, _ = io.Copy(io.Discard, resp.Body)
_ = resp.Body.Close()
req, err = http.NewRequest(http.MethodGet, url+string(role), nil)
if err != nil {
return IAMResponse{}, fmt.Errorf("error creating role request: %w", err)
}
if useIMDSv2 {
req.Header.Set(imdsTokenHeader, token)
}
resp, err = cl.Do(req)
if err != nil {
return IAMResponse{}, fmt.Errorf("error fetching role data: %w", err)
}
defer func() {
// Drain and close the body to let the Transport reuse the connection
io.Copy(io.Discard, resp.Body)
resp.Body.Close()
}()
if resp.StatusCode != http.StatusOK {
return IAMResponse{}, fmt.Errorf("error fetching role data, got non 200 code: %s", resp.Status)
}
var jResp IAMResponse
jsonString, err := io.ReadAll(resp.Body)
if err != nil {
return IAMResponse{}, fmt.Errorf("error reading role data: %w", err)
}
if err := json.Unmarshal(jsonString, &jResp); err != nil {
return IAMResponse{}, fmt.Errorf("error unmarshalling role data: %w (%s)", err, jsonString)
}
return jResp, nil
}
func newUsingIAM(cl *http.Client, baseURL, region string) (*S3, error) {
// Get the IAM role
iamResp, err := fetchIAMData(cl, baseURL)
if err != nil {
return nil, fmt.Errorf("error fetching IAM data: %w", err)
}
return &S3{
Region: region,
AccessKey: iamResp.AccessKeyID,
SecretKey: iamResp.SecretAccessKey,
Token: iamResp.Token,
URIFormat: "https://s3.%s.amazonaws.com/%s",
initMode: "iam",
expiry: iamResp.Expiration,
}, nil
}
// setIAMData sets the IAM data on the S3 instance.
func (s3 *S3) SetIAMData(iamResp IAMResponse) {
s3.AccessKey = iamResp.AccessKeyID
s3.SecretKey = iamResp.SecretAccessKey
s3.Token = iamResp.Token
}
func (s3 *S3) renewIAMToken() error {
if s3.initMode != "iam" {
return nil
}
if time.Since(s3.expiry) < 0 {
return nil
}
s3.mu.Lock()
defer s3.mu.Unlock()
iamResp, err := fetchIAMData(s3.getClient(), metadataBaseURL)
if err != nil {
return fmt.Errorf("error fetching IAM data: %w", err)
}
s3.expiry = iamResp.Expiration
s3.Token = iamResp.Token
s3.AccessKey = iamResp.AccessKeyID
s3.SecretKey = iamResp.SecretAccessKey
return nil
}
iam_test.go
+110
View File
@@ -0,0 +1,110 @@
package simples3
import (
"errors"
"io"
"net"
"net/http"
"net/http/httptest"
"testing"
"time"
)
func TestS3_NewUsingIAM(t *testing.T) {
var (
iam = `test-new-s3-using-iam`
resp = `{"Code" : "Success","LastUpdated" : "2018-12-24T10:18:01Z",
"Type" : "AWS-HMAC","AccessKeyId" : "abc",
"SecretAccessKey" : "abc","Token" : "abc",
"Expiration" : "2018-12-24T16:24:59Z"}`
respIMDSToken = `AQAEAJWopi8yvjKYXyWJbzESE0cms-OoTnptJzS3M9g5iNcl06UEkQ==`
)
tsFail := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
time.Sleep(2 * time.Second)
}))
defer tsFail.Close()
genServerHandlerFunc := func(failIMDS bool) http.HandlerFunc {
return func(w http.ResponseWriter, r *http.Request) {
switch r.Method {
case "GET":
if !failIMDS {
// check if token is present
if r.Header.Get(imdsTokenHeader) == "" {
w.WriteHeader(http.StatusUnauthorized)
return
}
}
url := securityCredentialsURI
if r.URL.EscapedPath() == url {
w.WriteHeader(http.StatusOK)
io.WriteString(w, iam)
}
if r.URL.EscapedPath() == url+iam {
w.WriteHeader(http.StatusOK)
w.Header().Set("Content-Type", "application/json")
io.WriteString(w, resp)
}
case "PUT":
if failIMDS {
w.WriteHeader(http.StatusNotFound)
return
}
if r.URL.EscapedPath() == imdsTokenURI {
if r.Header.Get(imdsTokenTtlHeader) != "60" {
w.WriteHeader(http.StatusBadRequest)
return
}
w.WriteHeader(http.StatusOK)
io.WriteString(w, respIMDSToken)
}
default:
t.Errorf("Expected 'GET' or 'PUT' request, got '%s'", r.Method)
}
}
}
ts := httptest.NewServer(http.HandlerFunc(genServerHandlerFunc(false)))
defer ts.Close()
tsFailIMDS := httptest.NewServer(http.HandlerFunc(genServerHandlerFunc(true)))
defer tsFailIMDS.Close()
cl := &http.Client{Timeout: 1 * time.Second}
// Test for timeout.
_, err := newUsingIAM(cl, tsFail.URL, "abc")
if err == nil {
t.Errorf("Expected error, got nil")
} else {
var timeoutError net.Error
if errors.As(err, &timeoutError) && !timeoutError.Timeout() {
t.Errorf("newUsingIAM() timeout check. got error = %v", err)
}
}
// Test for successful IAM fetch.
s3, err := newUsingIAM(cl, ts.URL, "abc")
if err != nil {
t.Errorf("newUsingIAM() error = %v", err)
}
if s3 == nil {
t.Errorf("newUsingIAM() got = %v", s3)
}
if s3.AccessKey != "abc" || s3.SecretKey != "abc" || s3.Region != "abc" {
t.Errorf("S3.FileDelete() got = %v", s3)
}
// Test for failed IMDS token fetch.
_, err = newUsingIAM(cl, tsFailIMDS.URL, "abc")
if err == nil {
t.Errorf("Expected error, got nil")
}
}
list.go
+260
View File
@@ -0,0 +1,260 @@
// LICENSE BSD-2-Clause-FreeBSD
// Copyright (c) 2018, Rohan Verma <hello@rohanverma.net>
package simples3
import (
"encoding/xml"
"fmt"
"io"
"iter"
"net/http"
"net/url"
)
// ListInput is passed to List as a parameter.
type ListInput struct {
// Required: The name of the bucket to list objects from
Bucket string
// Optional: A delimiter to group objects by (commonly "/")
Delimiter string
// Optional: Only list objects starting with this prefix
Prefix string
// Optional: Maximum number of objects to return
MaxKeys int64
// Optional: Token for pagination from a previous request
ContinuationToken string
// Optional: Object key to start listing after
StartAfter string
}
// ListResponse is returned by List.
type ListResponse struct {
// Name of the bucket
Name string
// Whether the results were truncated (more results available)
IsTruncated bool
// Token to get the next page of results (if truncated)
NextContinuationToken string
// List of objects in the bucket
Objects []Object
// Common prefixes when using delimiter (like directories)
CommonPrefixes []string
// Total number of keys returned
KeyCount int64
}
// Object represents an S3 object in a bucket.
type Object struct {
// Name of the object
Key string
// Size in bytes
Size int64
// When the object was last modified
LastModified string
// Entity tag of the object
ETag string
// Storage class (e.g., "STANDARD")
StorageClass string
}
// S3Error represents an S3 API error response
type S3Error struct {
XMLName xml.Name `xml:"Error"`
Code string `xml:"Code"`
Message string `xml:"Message"`
RequestID string `xml:"RequestId"`
HostID string `xml:"HostId"`
}
// Error returns a string representation of the S3Error
func (e S3Error) Error() string {
return fmt.Sprintf("S3 Error: %s - %s", e.Code, e.Message)
}
// List implements a simple S3 object listing API
func (s3 *S3) List(input ListInput) (ListResponse, error) {
// Input validation
if input.Bucket == "" {
return ListResponse{}, fmt.Errorf("bucket name cannot be empty")
}
if input.MaxKeys < 0 {
return ListResponse{}, fmt.Errorf("MaxKeys cannot be negative")
}
if input.MaxKeys > 1000 {
return ListResponse{}, fmt.Errorf("MaxKeys cannot exceed 1000")
}
// Build query parameters - ListObjectsV2 uses query params, not path
query := url.Values{}
query.Set("list-type", "2") // Required parameter
// Add optional parameters if they exist
if input.ContinuationToken != "" {
query.Set("continuation-token", input.ContinuationToken)
}
if input.Delimiter != "" {
query.Set("delimiter", input.Delimiter)
}
if input.MaxKeys > 0 {
query.Set("max-keys", fmt.Sprintf("%d", input.MaxKeys))
}
if input.Prefix != "" {
query.Set("prefix", input.Prefix)
}
if input.StartAfter != "" {
query.Set("start-after", input.StartAfter)
}
// Build base URL
baseURL := s3.getURL(input.Bucket)
// Parse URL and add query parameters
parsedURL, err := url.Parse(baseURL)
if err != nil {
return ListResponse{}, err
}
parsedURL.RawQuery = query.Encode()
// Create HTTP request
req, err := http.NewRequest(http.MethodGet, parsedURL.String(), nil)
if err != nil {
return ListResponse{}, err
}
// Apply AWS V4 signing
if err := s3.renewIAMToken(); err != nil {
return ListResponse{}, err
}
if err := s3.signRequest(req); err != nil {
return ListResponse{}, err
}
// Execute request
client := s3.getClient()
res, err := client.Do(req)
if err != nil {
return ListResponse{}, err
}
// Close response body when done
defer res.Body.Close()
// Read response body once
body, err := io.ReadAll(res.Body)
if err != nil {
return ListResponse{}, err
}
// Handle response status codes
if res.StatusCode != http.StatusOK {
var s3Err S3Error
if xmlErr := xml.Unmarshal(body, &s3Err); xmlErr == nil {
return ListResponse{}, fmt.Errorf("S3 Error: %s - %s", s3Err.Code, s3Err.Message)
}
return ListResponse{}, fmt.Errorf("status code: %s: %s", res.Status, string(body))
}
// Parse XML response using internal struct
var xmlResult struct {
XMLName xml.Name `xml:"ListBucketResult"`
Name string `xml:"Name"`
IsTruncated bool `xml:"IsTruncated"`
NextContinuationToken string `xml:"NextContinuationToken"`
KeyCount int64 `xml:"KeyCount"`
Contents []struct {
Key string `xml:"Key"`
LastModified string `xml:"LastModified"`
ETag string `xml:"ETag"`
Size int64 `xml:"Size"`
StorageClass string `xml:"StorageClass"`
} `xml:"Contents"`
CommonPrefixes []struct {
Prefix string `xml:"Prefix"`
} `xml:"CommonPrefixes"`
}
if err := xml.Unmarshal(body, &xmlResult); err != nil {
return ListResponse{}, err
}
// Convert to simple response format
response := ListResponse{
Name: xmlResult.Name,
IsTruncated: xmlResult.IsTruncated,
NextContinuationToken: xmlResult.NextContinuationToken,
KeyCount: xmlResult.KeyCount,
}
// Convert objects
for _, obj := range xmlResult.Contents {
response.Objects = append(response.Objects, Object{
Key: obj.Key,
Size: obj.Size,
LastModified: obj.LastModified,
ETag: obj.ETag,
StorageClass: obj.StorageClass,
})
}
// Convert common prefixes
for _, prefix := range xmlResult.CommonPrefixes {
response.CommonPrefixes = append(response.CommonPrefixes, prefix.Prefix)
}
return response, nil
}
// ListAll returns an iterator that yields all objects in the bucket,
// automatically handling pagination. It also returns a finish callback
// that should be called after iteration to check for any errors.
func (s3 *S3) ListAll(input ListInput) (iter.Seq[Object], func() error) {
var iterErr error
seq := func(yield func(Object) bool) {
currentInput := input
for {
response, err := s3.List(currentInput)
if err != nil {
iterErr = err
return
}
// Yield each object
for _, obj := range response.Objects {
if !yield(obj) {
return // Early termination requested
}
}
// Check if there are more results
if !response.IsTruncated || response.NextContinuationToken == "" {
return // No more results
}
// Prepare for next page
currentInput.ContinuationToken = response.NextContinuationToken
}
}
finish := func() error {
return iterErr
}
return seq, finish
}
list_test.go
+321
View File
@@ -0,0 +1,321 @@
package simples3
import (
"fmt"
"os"
"strings"
"testing"
)
func TestListAll(t *testing.T) {
// Setup test environment
s3 := New(
os.Getenv("AWS_S3_REGION"),
os.Getenv("AWS_S3_ACCESS_KEY"),
os.Getenv("AWS_S3_SECRET_KEY"),
)
s3.SetEndpoint(os.Getenv("AWS_S3_ENDPOINT"))
// Test cases
tests := []struct {
name string
setup func() // Setup function to prepare test data
input ListInput
validate func([]Object, error)
cleanup func() // Cleanup function
}{
{
name: "Empty Bucket Iterator",
input: ListInput{
Bucket: os.Getenv("AWS_S3_BUCKET"),
},
validate: func(objects []Object, err error) {
if err != nil {
t.Fatalf("Expected no error, got %v", err)
}
if len(objects) != 0 {
t.Errorf("Expected empty bucket, got %d objects", len(objects))
}
},
},
{
name: "Basic Iterator Listing",
setup: func() {
// Upload test files
uploadTestFiles(t, s3, os.Getenv("AWS_S3_BUCKET"), []string{"iter_file1.txt", "iter_file2.txt", "iter_file3.txt"})
},
input: ListInput{
Bucket: os.Getenv("AWS_S3_BUCKET"),
},
validate: func(objects []Object, err error) {
if err != nil {
t.Fatalf("Expected no error, got %v", err)
}
if len(objects) != 3 {
t.Errorf("Expected 3 objects, got %d", len(objects))
}
// Check that all expected files are present
expectedKeys := map[string]bool{
"iter_file1.txt": false,
"iter_file2.txt": false,
"iter_file3.txt": false,
}
for _, obj := range objects {
if _, exists := expectedKeys[obj.Key]; exists {
expectedKeys[obj.Key] = true
}
}
for key, found := range expectedKeys {
if !found {
t.Errorf("Expected object %s not found in iterator results", key)
}
}
},
cleanup: func() {
cleanupTestFiles(t, s3, os.Getenv("AWS_S3_BUCKET"), []string{"iter_file1.txt", "iter_file2.txt", "iter_file3.txt"})
},
},
{
name: "Iterator with Prefix",
setup: func() {
uploadTestFiles(t, s3, os.Getenv("AWS_S3_BUCKET"), []string{
"iter_docs/file1.txt",
"iter_docs/file2.txt",
"iter_images/image1.jpg",
"iter_images/image2.jpg",
})
},
input: ListInput{
Bucket: os.Getenv("AWS_S3_BUCKET"),
Prefix: "iter_docs/",
},
validate: func(objects []Object, err error) {
if err != nil {
t.Fatalf("Expected no error, got %v", err)
}
if len(objects) != 2 {
t.Errorf("Expected 2 objects with iter_docs/ prefix, got %d", len(objects))
}
for _, obj := range objects {
if !strings.HasPrefix(obj.Key, "iter_docs/") {
t.Errorf("Object %s doesn't have expected prefix", obj.Key)
}
}
},
cleanup: func() {
cleanupTestFiles(t, s3, os.Getenv("AWS_S3_BUCKET"), []string{
"iter_docs/file1.txt",
"iter_docs/file2.txt",
"iter_images/image1.jpg",
"iter_images/image2.jpg",
})
},
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
if tt.setup != nil {
tt.setup()
}
// Collect all objects from the iterator and check for errors
var objects []Object
seq, finish := s3.ListAll(tt.input)
for obj := range seq {
objects = append(objects, obj)
}
// Check for any iteration errors
if err := finish(); err != nil {
t.Fatalf("Iterator error: %v", err)
}
if tt.validate != nil {
tt.validate(objects, nil) // No error from iterator
}
if tt.cleanup != nil {
tt.cleanup()
}
})
}
}
func TestList(t *testing.T) {
// Setup test environment
s3 := New(
os.Getenv("AWS_S3_REGION"),
os.Getenv("AWS_S3_ACCESS_KEY"),
os.Getenv("AWS_S3_SECRET_KEY"),
)
s3.SetEndpoint(os.Getenv("AWS_S3_ENDPOINT"))
// Test cases
tests := []struct {
name string
setup func() // Setup function to prepare test data
input ListInput
validate func(ListResponse, error)
cleanup func() // Cleanup function
}{
{
name: "Empty Bucket",
input: ListInput{
Bucket: os.Getenv("AWS_S3_BUCKET"),
},
validate: func(result ListResponse, err error) {
if err != nil {
t.Fatalf("Expected no error, got %v", err)
}
if len(result.Objects) != 0 {
t.Errorf("Expected empty bucket, got %d objects", len(result.Objects))
}
},
},
{
name: "Basic Listing",
setup: func() {
// Upload test files
uploadTestFiles(t, s3, os.Getenv("AWS_S3_BUCKET"), []string{"file1.txt", "file2.txt", "file3.txt"})
},
input: ListInput{
Bucket: os.Getenv("AWS_S3_BUCKET"),
},
validate: func(result ListResponse, err error) {
if err != nil {
t.Fatalf("Expected no error, got %v", err)
}
if len(result.Objects) != 3 {
t.Errorf("Expected 3 objects, got %d", len(result.Objects))
}
},
cleanup: func() {
cleanupTestFiles(t, s3, os.Getenv("AWS_S3_BUCKET"), []string{"file1.txt", "file2.txt", "file3.txt"})
},
},
{
name: "Pagination Test",
setup: func() {
// Upload many files for pagination testing
var filenames []string
for i := 1; i <= 25; i++ {
filenames = append(filenames, fmt.Sprintf("file%03d.txt", i))
}
uploadTestFiles(t, s3, os.Getenv("AWS_S3_BUCKET"), filenames)
},
input: ListInput{
Bucket: os.Getenv("AWS_S3_BUCKET"),
MaxKeys: 10,
},
validate: func(result ListResponse, err error) {
if err != nil {
t.Fatalf("Expected no error, got %v", err)
}
if len(result.Objects) != 10 {
t.Errorf("Expected 10 objects, got %d", len(result.Objects))
}
if !result.IsTruncated {
t.Errorf("Expected IsTruncated=true for pagination")
}
if result.NextContinuationToken == "" {
t.Errorf("Expected NextContinuationToken for pagination: %v", result.NextContinuationToken)
}
},
cleanup: func() {
var filenames []string
for i := 1; i <= 25; i++ {
filenames = append(filenames, fmt.Sprintf("file%03d.txt", i))
}
cleanupTestFiles(t, s3, os.Getenv("AWS_S3_BUCKET"), filenames)
},
},
{
name: "Prefix Filtering",
setup: func() {
uploadTestFiles(t, s3, os.Getenv("AWS_S3_BUCKET"), []string{
"documents/file1.txt",
"documents/file2.txt",
"images/image1.jpg",
"images/image2.jpg",
})
},
input: ListInput{
Bucket: os.Getenv("AWS_S3_BUCKET"),
Prefix: "documents/",
},
validate: func(result ListResponse, err error) {
if err != nil {
t.Fatalf("Expected no error, got %v", err)
}
if len(result.Objects) != 2 {
t.Errorf("Expected 2 objects with documents/ prefix, got %d", len(result.Objects))
}
for _, obj := range result.Objects {
if !strings.HasPrefix(obj.Key, "documents/") {
t.Errorf("Object %s doesn't have expected prefix", obj.Key)
}
}
},
cleanup: func() {
cleanupTestFiles(t, s3, os.Getenv("AWS_S3_BUCKET"), []string{
"documents/file1.txt",
"documents/file2.txt",
"images/image1.jpg",
"images/image2.jpg",
})
},
},
{
name: "Delimiter Grouping",
setup: func() {
uploadTestFiles(t, s3, os.Getenv("AWS_S3_BUCKET"), []string{
"documents/important/file1.txt",
"documents/backup/file2.txt",
"images/landscape/image1.jpg",
"images/portrait/image2.jpg",
})
},
input: ListInput{
Bucket: os.Getenv("AWS_S3_BUCKET"),
Delimiter: "/",
Prefix: "documents/",
},
validate: func(result ListResponse, err error) {
if err != nil {
t.Fatalf("Expected no error, got %v", err)
}
// Should return common prefixes instead of individual files
if len(result.CommonPrefixes) != 2 {
t.Errorf("Expected 2 common prefixes, got %d", len(result.CommonPrefixes))
}
},
cleanup: func() {
cleanupTestFiles(t, s3, os.Getenv("AWS_S3_BUCKET"), []string{
"documents/important/file1.txt",
"documents/backup/file2.txt",
"images/landscape/image1.jpg",
"images/portrait/image2.jpg",
})
},
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
if tt.setup != nil {
tt.setup()
}
result, err := s3.List(tt.input)
if tt.validate != nil {
tt.validate(result, err)
}
if tt.cleanup != nil {
tt.cleanup()
}
})
}
}
object.go
+409
View File
@@ -0,0 +1,409 @@
// LICENSE BSD-2-Clause-FreeBSD
// Copyright (c) 2018, Rohan Verma <hello@rohanverma.net>
package simples3
import (
"bytes"
"crypto/sha256"
"encoding/xml"
"fmt"
"io"
"mime/multipart"
"net/http"
"strings"
)
type DownloadInput struct {
Bucket string
ObjectKey string
}
// DetailsInput is passed to FileDetails as a parameter.
type DetailsInput struct {
Bucket string
ObjectKey string
}
// DetailsResponse is returned by FileDetails.
type DetailsResponse struct {
ContentType string
ContentLength string
AcceptRanges string
Date string
Etag string
LastModified string
Server string
AmzID2 string
AmzRequestID string
AmzMeta map[string]string
ExtraHeaders map[string]string
}
// UploadInput is passed to FileUpload as a parameter.
type UploadInput struct {
// essential fields
Bucket string
ObjectKey string
FileName string
ContentType string
// optional fields
ContentDisposition string
ACL string
// Setting key/value pairs adds user-defined metadata
// keys to the object, prefixed with AMZMetaPrefix.
CustomMetadata map[string]string
Body io.ReadSeeker
}
// UploadResponse receives the following XML
// in case of success, since we set a 201 response from S3.
// Sample response:
//
// <PostResponse>
// <Location>https://s3.amazonaws.com/link-to-the-file</Location>
// <Bucket>s3-bucket</Bucket>
// <Key>development/8614bd40-691b-4668-9241-3b342c6cf429/image.jpg</Key>
// <ETag>"32-bit-tag"</ETag>
// </PostResponse>
type UploadResponse struct {
Location string `xml:"Location"`
Bucket string `xml:"Bucket"`
Key string `xml:"Key"`
ETag string `xml:"ETag"`
}
// PutResponse is returned when the action is successful,
// and the service sends back an HTTP 200 response. The response
// returns ETag along with HTTP headers.
type PutResponse struct {
ETag string
Headers http.Header
}
// DeleteInput is passed to FileDelete as a parameter.
type DeleteInput struct {
Bucket string
ObjectKey string
}
// FileDownload makes a GET call and returns a io.ReadCloser.
// After reading the response body, ensure closing the response.
func (s3 *S3) FileDownload(u DownloadInput) (io.ReadCloser, error) {
req, err := http.NewRequest(
http.MethodGet, s3.getURL(u.Bucket, u.ObjectKey), nil,
)
if err != nil {
return nil, err
}
if err := s3.renewIAMToken(); err != nil {
return nil, err
}
if err := s3.signRequest(req); err != nil {
return nil, err
}
res, err := s3.getClient().Do(req)
if err != nil {
return nil, err
}
if res.StatusCode != http.StatusOK {
return nil, fmt.Errorf("status code: %s", res.Status)
}
return res.Body, nil
}
// FilePut makes a PUT call to S3.
func (s3 *S3) FilePut(u UploadInput) (PutResponse, error) {
fSize, err := detectFileSize(u.Body)
if err != nil {
return PutResponse{}, err
}
content := make([]byte, fSize)
_, err = u.Body.Read(content)
if err != nil {
return PutResponse{}, err
}
u.Body.Seek(0, 0)
req, er := http.NewRequest(http.MethodPut, s3.getURL(u.Bucket, u.ObjectKey), u.Body)
if er != nil {
return PutResponse{}, err
}
if u.ContentType == "" {
u.ContentType = "application/octet-stream"
}
h := sha256.New()
h.Write(content)
req.Header.Set("x-amz-content-sha256", fmt.Sprintf("%x", h.Sum(nil)))
req.Header.Set("Content-Type", u.ContentType)
req.Header.Set("Content-Length", fmt.Sprintf("%d", fSize))
req.Header.Set("Host", req.URL.Host)
for k, v := range u.CustomMetadata {
req.Header.Set("x-amz-meta-"+k, v)
}
if u.ContentDisposition != "" {
req.Header.Set("Content-Disposition", u.ContentDisposition)
}
if u.ACL != "" {
req.Header.Set("x-amz-acl", u.ACL)
}
req.ContentLength = fSize
if err := s3.renewIAMToken(); err != nil {
return PutResponse{}, err
}
if err := s3.signRequest(req); err != nil {
return PutResponse{}, err
}
// debug(httputil.DumpRequest(req, true))
// Submit the request
client := s3.getClient()
res, err := client.Do(req)
if err != nil {
return PutResponse{}, err
}
defer func() {
res.Body.Close()
io.Copy(io.Discard, res.Body)
}()
data, err := io.ReadAll(res.Body)
if err != nil {
return PutResponse{}, err
}
// Check the response
if res.StatusCode != http.StatusOK {
return PutResponse{}, fmt.Errorf("status code: %s: %q", res.Status, data)
}
return PutResponse{
ETag: res.Header.Get("ETag"),
Headers: res.Header.Clone(),
}, nil
}
// FileUpload makes a POST call with the file written as multipart
// and on successful upload, checks for 200 OK.
func (s3 *S3) FileUpload(u UploadInput) (UploadResponse, error) {
fSize, err := detectFileSize(u.Body)
if err != nil {
return UploadResponse{}, err
}
uc := UploadConfig{
UploadURL: s3.getURL(u.Bucket),
BucketName: u.Bucket,
ObjectKey: u.ObjectKey,
ContentType: u.ContentType,
ContentDisposition: u.ContentDisposition,
ACL: u.ACL,
FileSize: fSize,
MetaData: map[string]string{
"success_action_status": "201", // returns XML doc on success
},
}
// Set custom metadata.
for k, v := range u.CustomMetadata {
if !strings.HasPrefix(k, AMZMetaPrefix) {
k = AMZMetaPrefix + k
}
uc.MetaData[k] = v
}
if err := s3.renewIAMToken(); err != nil {
return UploadResponse{}, err
}
policies, err := s3.CreateUploadPolicies(uc)
if err != nil {
return UploadResponse{}, err
}
var b bytes.Buffer
w := multipart.NewWriter(&b)
for k, v := range policies.Form {
if err = w.WriteField(k, v); err != nil {
return UploadResponse{}, err
}
}
fw, err := w.CreateFormFile("file", u.FileName)
if err != nil {
return UploadResponse{}, err
}
if _, err = io.Copy(fw, u.Body); err != nil {
return UploadResponse{}, err
}
// Don't forget to close the multipart writer.
// If you don't close it, your request will be missing the terminating boundary.
if err := w.Close(); err != nil {
return UploadResponse{}, err
}
// Now that you have a form, you can submit it to your handler.
req, err := http.NewRequest(http.MethodPost, policies.URL, &b)
if err != nil {
return UploadResponse{}, err
}
// Don't forget to set the content type, this will contain the boundary.
req.Header.Set("Content-Type", w.FormDataContentType())
// Submit the request
client := s3.getClient()
res, err := client.Do(req)
if err != nil {
return UploadResponse{}, err
}
defer func() {
res.Body.Close()
io.Copy(io.Discard, res.Body)
}()
data, err := io.ReadAll(res.Body)
if err != nil {
return UploadResponse{}, err
}
// Check the response
if res.StatusCode != http.StatusCreated {
return UploadResponse{}, fmt.Errorf("status code: %s: %q", res.Status, data)
}
var ur UploadResponse
_ = xml.Unmarshal(data, &ur)
return ur, nil
}
// FileDelete makes a DELETE call with the file written as multipart
// and on successful upload, checks for 204 No Content.
func (s3 *S3) FileDelete(u DeleteInput) error {
req, err := http.NewRequest(
http.MethodDelete, s3.getURL(u.Bucket, u.ObjectKey), nil,
)
if err != nil {
return err
}
if err := s3.renewIAMToken(); err != nil {
return err
}
if err := s3.signRequest(req); err != nil {
return err
}
// Submit the request
client := s3.getClient()
res, err := client.Do(req)
if err != nil {
return err
}
defer func() {
res.Body.Close()
io.Copy(io.Discard, res.Body)
}()
// Check the response
if res.StatusCode != http.StatusNoContent {
return fmt.Errorf("status code: %s", res.Status)
}
return nil
}
func (s3 *S3) FileDetails(u DetailsInput) (DetailsResponse, error) {
req, err := http.NewRequest(
http.MethodHead, s3.getURL(u.Bucket, u.ObjectKey), nil,
)
if err != nil {
return DetailsResponse{}, err
}
if err := s3.renewIAMToken(); err != nil {
return DetailsResponse{}, err
}
if err := s3.signRequest(req); err != nil {
return DetailsResponse{}, err
}
res, err := s3.getClient().Do(req)
if err != nil {
return DetailsResponse{}, err
}
defer func() {
res.Body.Close()
io.Copy(io.Discard, res.Body)
}()
if res.StatusCode != http.StatusOK {
return DetailsResponse{}, fmt.Errorf("status code: %s", res.Status)
}
var out DetailsResponse
for k, v := range res.Header {
lk := strings.ToLower(k)
switch lk {
case "content-type":
out.ContentType = getFirstString(v)
case "content-length":
out.ContentLength = getFirstString(v)
case "accept-ranges":
out.AcceptRanges = getFirstString(v)
case "date":
out.Date = getFirstString(v)
case "etag":
out.Etag = getFirstString(v)
case "last-modified":
out.LastModified = getFirstString(v)
case "server":
out.Server = getFirstString(v)
case "x-amz-id-2":
out.AmzID2 = getFirstString(v)
case "x-amz-request-id":
out.AmzRequestID = getFirstString(v)
default:
if strings.HasPrefix(lk, AMZMetaPrefix) {
if out.AmzMeta == nil {
out.AmzMeta = map[string]string{}
}
out.AmzMeta[k] = getFirstString(v)
} else {
if out.ExtraHeaders == nil {
out.ExtraHeaders = map[string]string{}
}
out.ExtraHeaders[k] = getFirstString(v)
}
}
}
return out, nil
}
object_test.go
+381
View File
@@ -0,0 +1,381 @@
package simples3
import (
"bytes"
"io"
"os"
"testing"
)
func TestS3_FileUploadPostAndPut(t *testing.T) {
testTxt, err := os.Open("testdata/test.txt")
if err != nil {
t.Fatalf("Failed to open test.txt: %v", err)
}
defer testTxt.Close()
// Note: cannot re-use the same file descriptor due to seeking!
testTxtSpecialChars, err := os.Open("testdata/test.txt")
if err != nil {
t.Fatalf("Failed to open test.txt for special chars: %v", err)
}
defer testTxtSpecialChars.Close()
testPng, err := os.Open("testdata/avatar.png")
if err != nil {
t.Fatalf("Failed to open avatar.png: %v", err)
}
defer testPng.Close()
type args struct {
u UploadInput
}
tests := []struct {
name string
fields tConfig
args args
testDetails bool
wantErr bool
}{
{
name: "Upload test.txt",
fields: tConfig{
AccessKey: os.Getenv("AWS_S3_ACCESS_KEY"),
SecretKey: os.Getenv("AWS_S3_SECRET_KEY"),
Endpoint: os.Getenv("AWS_S3_ENDPOINT"),
Region: os.Getenv("AWS_S3_REGION"),
},
args: args{
UploadInput{
Bucket: os.Getenv("AWS_S3_BUCKET"),
ObjectKey: "test.txt",
ContentType: "text/plain",
FileName: "test.txt",
Body: testTxt,
},
},
wantErr: false,
},
{
name: "Upload test.txt with custom metadata",
fields: tConfig{
AccessKey: os.Getenv("AWS_S3_ACCESS_KEY"),
SecretKey: os.Getenv("AWS_S3_SECRET_KEY"),
Endpoint: os.Getenv("AWS_S3_ENDPOINT"),
Region: os.Getenv("AWS_S3_REGION"),
},
args: args{
UploadInput{
Bucket: os.Getenv("AWS_S3_BUCKET"),
ObjectKey: "test_metadata.txt",
ContentType: "text/plain",
FileName: "test.txt",
Body: testTxt,
CustomMetadata: map[string]string{
"test-metadata": "foo-bar",
},
},
},
wantErr: false,
testDetails: true,
},
{
name: "Upload avatar.png",
fields: tConfig{
AccessKey: os.Getenv("AWS_S3_ACCESS_KEY"),
SecretKey: os.Getenv("AWS_S3_SECRET_KEY"),
Endpoint: os.Getenv("AWS_S3_ENDPOINT"),
Region: os.Getenv("AWS_S3_REGION"),
},
args: args{
UploadInput{
Bucket: os.Getenv("AWS_S3_BUCKET"),
ObjectKey: "xyz/image.png",
ContentType: "image/png",
FileName: "avatar.png",
Body: testPng,
},
},
wantErr: false,
},
{
name: "Upload special filename txt",
fields: tConfig{
AccessKey: os.Getenv("AWS_S3_ACCESS_KEY"),
SecretKey: os.Getenv("AWS_S3_SECRET_KEY"),
Endpoint: os.Getenv("AWS_S3_ENDPOINT"),
Region: os.Getenv("AWS_S3_REGION"),
},
args: args{
UploadInput{
Bucket: os.Getenv("AWS_S3_BUCKET"),
ObjectKey: "xyz/example file%with$special&chars(1)?.txt",
ContentType: "text/plain",
FileName: "example file%with$special&chars(1)?.txt",
Body: testTxtSpecialChars,
},
},
wantErr: false,
},
}
for _, testcase := range tests {
tt := testcase
t.Run(tt.name+"_post", func(t *testing.T) {
s3 := New(tt.fields.Region, tt.fields.AccessKey, tt.fields.SecretKey)
s3.SetEndpoint(tt.fields.Endpoint)
resp, err := s3.FileUpload(tt.args.u)
if (err != nil) != tt.wantErr {
t.Errorf("S3.FileUpload() error = %v, wantErr %v", err, tt.wantErr)
}
// reset file, to reuse in further tests.
tt.args.u.Body.Seek(0, 0)
// check for empty response
if (resp == UploadResponse{}) {
t.Errorf("S3.FileUpload() returned empty response, %v", resp)
}
if tt.testDetails {
dResp, err := s3.FileDetails(DetailsInput{
Bucket: tt.args.u.Bucket,
ObjectKey: tt.args.u.ObjectKey,
})
if (err != nil) != tt.wantErr {
t.Errorf("S3.FileDetails() error = %v, wantErr %v", err, tt.wantErr)
}
if len(dResp.AmzMeta) != len(tt.args.u.CustomMetadata) {
t.Errorf("S3.FileDetails() returned incorrect metadata, got: %#v", dResp)
}
}
})
t.Run(tt.name+"_put", func(t *testing.T) {
s3 := New(tt.fields.Region, tt.fields.AccessKey, tt.fields.SecretKey)
s3.SetEndpoint(tt.fields.Endpoint)
resp, err := s3.FilePut(tt.args.u)
if (err != nil) != tt.wantErr {
t.Errorf("S3.FileUpload() error = %v, wantErr %v", err, tt.wantErr)
}
// reset file, to reuse in further tests.
tt.args.u.Body.Seek(0, 0)
// check for empty response
if resp.ETag == "" {
t.Errorf("S3.FileUpload() returned empty response, %v", resp)
}
if tt.testDetails {
dResp, err := s3.FileDetails(DetailsInput{
Bucket: tt.args.u.Bucket,
ObjectKey: tt.args.u.ObjectKey,
})
if (err != nil) != tt.wantErr {
t.Errorf("S3.FileUpload() error = %v, wantErr %v", err, tt.wantErr)
}
if len(dResp.AmzMeta) != len(tt.args.u.CustomMetadata) {
t.Errorf("S3.FileDetails() returned incorrect metadata, got: %#v", dResp)
}
}
})
}
}
func TestS3_FileDownload(t *testing.T) {
testTxt, err := os.Open("testdata/test.txt")
if err != nil {
t.Fatal(err)
}
defer testTxt.Close()
testTxtData, err := io.ReadAll(testTxt)
if err != nil {
t.Fatal(err)
}
testPng, err := os.Open("testdata/avatar.png")
if err != nil {
t.Fatal(err)
}
defer testPng.Close()
testPngData, err := io.ReadAll(testPng)
if err != nil {
t.Fatal(err)
}
type args struct {
u DownloadInput
}
tests := []struct {
name string
fields tConfig
args args
wantErr bool
wantResponse []byte
}{
{
name: "txt",
fields: tConfig{
AccessKey: os.Getenv("AWS_S3_ACCESS_KEY"),
SecretKey: os.Getenv("AWS_S3_SECRET_KEY"),
Endpoint: os.Getenv("AWS_S3_ENDPOINT"),
Region: os.Getenv("AWS_S3_REGION"),
},
args: args{
u: DownloadInput{
Bucket: os.Getenv("AWS_S3_BUCKET"),
ObjectKey: "test.txt",
},
},
wantErr: false,
wantResponse: testTxtData,
},
{
name: "png",
fields: tConfig{
AccessKey: os.Getenv("AWS_S3_ACCESS_KEY"),
SecretKey: os.Getenv("AWS_S3_SECRET_KEY"),
Endpoint: os.Getenv("AWS_S3_ENDPOINT"),
Region: os.Getenv("AWS_S3_REGION"),
},
args: args{
u: DownloadInput{
Bucket: os.Getenv("AWS_S3_BUCKET"),
ObjectKey: "xyz/image.png",
},
},
wantErr: false,
wantResponse: testPngData,
},
{
name: "txt-special-filename",
fields: tConfig{
AccessKey: os.Getenv("AWS_S3_ACCESS_KEY"),
SecretKey: os.Getenv("AWS_S3_SECRET_KEY"),
Endpoint: os.Getenv("AWS_S3_ENDPOINT"),
Region: os.Getenv("AWS_S3_REGION"),
},
args: args{
u: DownloadInput{
Bucket: os.Getenv("AWS_S3_BUCKET"),
ObjectKey: "xyz/example file%with$special&chars(1)?.txt",
},
},
wantErr: false,
wantResponse: testTxtData,
},
}
for _, testcase := range tests {
tt := testcase
t.Run(tt.name, func(t *testing.T) {
s3 := New(tt.fields.Region, tt.fields.AccessKey, tt.fields.SecretKey)
s3.SetEndpoint(tt.fields.Endpoint)
resp, err := s3.FileDownload(tt.args.u)
if (err != nil) != tt.wantErr {
t.Fatalf("S3.FileDownload() error = %v, wantErr %v", err, tt.wantErr)
}
got, err := io.ReadAll(resp)
if err != nil {
t.Fatalf("error = %v", err)
}
resp.Close()
if !bytes.Equal(got, tt.wantResponse) {
t.Fatalf("S3.FileDownload() = %v, want %v", got, tt.wantResponse)
}
})
}
}
func TestS3_FileDelete(t *testing.T) {
type args struct {
u DeleteInput
}
tests := []struct {
name string
fields tConfig
args args
wantErr bool
}{
{
name: "Delete test.txt",
fields: tConfig{
AccessKey: os.Getenv("AWS_S3_ACCESS_KEY"),
SecretKey: os.Getenv("AWS_S3_SECRET_KEY"),
Endpoint: os.Getenv("AWS_S3_ENDPOINT"),
Region: os.Getenv("AWS_S3_REGION"),
},
args: args{
DeleteInput{
Bucket: os.Getenv("AWS_S3_BUCKET"),
ObjectKey: "test.txt",
},
},
wantErr: false,
},
{
name: "Delete test_metadata.txt",
fields: tConfig{
AccessKey: os.Getenv("AWS_S3_ACCESS_KEY"),
SecretKey: os.Getenv("AWS_S3_SECRET_KEY"),
Endpoint: os.Getenv("AWS_S3_ENDPOINT"),
Region: os.Getenv("AWS_S3_REGION"),
},
args: args{
DeleteInput{
Bucket: os.Getenv("AWS_S3_BUCKET"),
ObjectKey: "test_metadata.txt",
},
},
wantErr: false,
},
{
name: "Delete avatar.png",
fields: tConfig{
AccessKey: os.Getenv("AWS_S3_ACCESS_KEY"),
SecretKey: os.Getenv("AWS_S3_SECRET_KEY"),
Endpoint: os.Getenv("AWS_S3_ENDPOINT"),
Region: os.Getenv("AWS_S3_REGION"),
},
args: args{
DeleteInput{
Bucket: os.Getenv("AWS_S3_BUCKET"),
ObjectKey: "xyz/image.png",
},
},
wantErr: false,
},
{
name: "Delete special filename txt",
fields: tConfig{
AccessKey: os.Getenv("AWS_S3_ACCESS_KEY"),
SecretKey: os.Getenv("AWS_S3_SECRET_KEY"),
Endpoint: os.Getenv("AWS_S3_ENDPOINT"),
Region: os.Getenv("AWS_S3_REGION"),
},
args: args{
DeleteInput{
Bucket: os.Getenv("AWS_S3_BUCKET"),
ObjectKey: "xyz/example file%with$special&chars(1)?.txt",
},
},
wantErr: false,
},
}
for _, testcase := range tests {
tt := testcase
t.Run(tt.name, func(t *testing.T) {
s3 := New(tt.fields.Region, tt.fields.AccessKey, tt.fields.SecretKey)
s3.SetEndpoint(tt.fields.Endpoint)
if err := s3.FileDelete(tt.args.u); (err != nil) != tt.wantErr {
t.Errorf("S3.FileDelete() error = %v, wantErr %v", err, tt.wantErr)
}
})
}
}
simples3.go
-923
View File
@@ -7,30 +7,14 @@ import (
"bytes"
"crypto/hmac"
"crypto/sha256"
"encoding/hex"
"encoding/json"
"encoding/xml"
"fmt"
"io"
"iter"
"mime/multipart"
"net/http"
"net/url"
"regexp"
"strings"
"sync"
"time"
"unicode/utf8"
)
const (
imdsTokenHeader = "X-aws-ec2-metadata-token"
imdsTokenTtlHeader = "X-aws-ec2-metadata-token-ttl-seconds"
metadataBaseURL = "http://169.254.169.254/latest"
securityCredentialsURI = "/meta-data/iam/security-credentials/"
imdsTokenURI = "/api/token"
defaultIMDSTokenTTL = "60"
// AMZMetaPrefix to prefix metadata key.
AMZMetaPrefix = "x-amz-meta-"
)
@@ -51,168 +35,6 @@ type S3 struct {
mu sync.Mutex
}
// DownloadInput is passed to FileDownload as a parameter.
type DownloadInput struct {
Bucket string
ObjectKey string
}
// DetailsInput is passed to FileDetails as a parameter.
type DetailsInput struct {
Bucket string
ObjectKey string
}
// DetailsResponse is returned by FileDetails.
type DetailsResponse struct {
ContentType string
ContentLength string
AcceptRanges string
Date string
Etag string
LastModified string
Server string
AmzID2 string
AmzRequestID string
AmzMeta map[string]string
ExtraHeaders map[string]string
}
// UploadInput is passed to FileUpload as a parameter.
type UploadInput struct {
// essential fields
Bucket string
ObjectKey string
FileName string
ContentType string
// optional fields
ContentDisposition string
ACL string
// Setting key/value pairs adds user-defined metadata
// keys to the object, prefixed with AMZMetaPrefix.
CustomMetadata map[string]string
Body io.ReadSeeker
}
// UploadResponse receives the following XML
// in case of success, since we set a 201 response from S3.
// Sample response:
//
// <PostResponse>
// <Location>https://s3.amazonaws.com/link-to-the-file</Location>
// <Bucket>s3-bucket</Bucket>
// <Key>development/8614bd40-691b-4668-9241-3b342c6cf429/image.jpg</Key>
// <ETag>"32-bit-tag"</ETag>
// </PostResponse>
type UploadResponse struct {
Location string `xml:"Location"`
Bucket string `xml:"Bucket"`
Key string `xml:"Key"`
ETag string `xml:"ETag"`
}
// PutResponse is returned when the action is successful,
// and the service sends back an HTTP 200 response. The response
// returns ETag along with HTTP headers.
type PutResponse struct {
ETag string
Headers http.Header
}
// DeleteInput is passed to FileDelete as a parameter.
type DeleteInput struct {
Bucket string
ObjectKey string
}
// ListInput is passed to List as a parameter.
type ListInput struct {
// Required: The name of the bucket to list objects from
Bucket string
// Optional: A delimiter to group objects by (commonly "/")
Delimiter string
// Optional: Only list objects starting with this prefix
Prefix string
// Optional: Maximum number of objects to return
MaxKeys int64
// Optional: Token for pagination from a previous request
ContinuationToken string
// Optional: Object key to start listing after
StartAfter string
}
// ListResponse is returned by List.
type ListResponse struct {
// Name of the bucket
Name string
// Whether the results were truncated (more results available)
IsTruncated bool
// Token to get the next page of results (if truncated)
NextContinuationToken string
// List of objects in the bucket
Objects []Object
// Common prefixes when using delimiter (like directories)
CommonPrefixes []string
// Total number of keys returned
KeyCount int64
}
// Object represents an S3 object in a bucket.
type Object struct {
// Name of the object
Key string
// Size in bytes
Size int64
// When the object was last modified
LastModified string
// Entity tag of the object
ETag string
// Storage class (e.g., "STANDARD")
StorageClass string
}
// S3Error represents an S3 API error response
type S3Error struct {
XMLName xml.Name `xml:"Error"`
Code string `xml:"Code"`
Message string `xml:"Message"`
RequestID string `xml:"RequestId"`
HostID string `xml:"HostId"`
}
// Error returns a string representation of the S3Error
func (e S3Error) Error() string {
return fmt.Sprintf("S3 Error: %s - %s", e.Code, e.Message)
}
// IAMResponse is used by NewUsingIAM to auto
// detect the credentials.
type IAMResponse struct {
Code string `json:"Code"`
LastUpdated string `json:"LastUpdated"`
Type string `json:"Type"`
AccessKeyID string `json:"AccessKeyId"`
SecretAccessKey string `json:"SecretAccessKey"`
Token string `json:"Token"`
Expiration time.Time `json:"Expiration"`
}
// New returns an instance of S3.
func New(region, accessKey, secretKey string) *S3 {
return &S3{
@@ -224,151 +46,6 @@ func New(region, accessKey, secretKey string) *S3 {
}
}
// NewUsingIAM automatically generates an Instance of S3
// using instance metatdata.
func NewUsingIAM(region string) (*S3, error) {
return newUsingIAM(
&http.Client{
// Set a timeout of 3 seconds for AWS IAM Calls.
Timeout: time.Second * 3, //nolint:gomnd
}, metadataBaseURL, region)
}
// fetchIMDSToken retrieves an IMDSv2 token from the
// EC2 instance metadata service. It returns a token and boolean,
// only if IMDSv2 is enabled in the EC2 instance metadata
// configuration, otherwise returns an error.
func fetchIMDSToken(cl *http.Client, baseURL string) (string, bool, error) {
req, err := http.NewRequest(http.MethodPut, baseURL+imdsTokenURI, nil)
if err != nil {
return "", false, err
}
// Set the token TTL to 60 seconds.
req.Header.Set(imdsTokenTtlHeader, defaultIMDSTokenTTL)
resp, err := cl.Do(req)
if err != nil {
return "", false, err
}
defer func() {
resp.Body.Close()
io.Copy(io.Discard, resp.Body)
}()
if resp.StatusCode != http.StatusOK {
return "", false, fmt.Errorf("failed to request IMDSv2 token: %s", resp.Status)
}
token, err := io.ReadAll(resp.Body)
if err != nil {
return "", false, err
}
return string(token), true, nil
}
// fetchIAMData fetches the IAM data from the given URL.
// In case of a normal AWS setup, baseURL would be metadataBaseURL.
// You can use this method, to manually fetch IAM data from a custom
// endpoint and pass it to SetIAMData.
func fetchIAMData(cl *http.Client, baseURL string) (IAMResponse, error) {
token, useIMDSv2, err := fetchIMDSToken(cl, baseURL)
if err != nil {
return IAMResponse{}, fmt.Errorf("error fetching IMDSv2 token: %w", err)
}
url := baseURL + securityCredentialsURI
req, err := http.NewRequest(http.MethodGet, url, nil)
if err != nil {
return IAMResponse{}, fmt.Errorf("error creating imdsv2 token request: %w", err)
}
if useIMDSv2 {
req.Header.Set(imdsTokenHeader, token)
}
resp, err := cl.Do(req)
if err != nil {
return IAMResponse{}, err
}
if resp.StatusCode != http.StatusOK {
return IAMResponse{}, fmt.Errorf("error fetching IAM data: %s", resp.Status)
}
role, err := io.ReadAll(resp.Body)
if err != nil {
return IAMResponse{}, err
}
_, _ = io.Copy(io.Discard, resp.Body)
_ = resp.Body.Close()
req, err = http.NewRequest(http.MethodGet, url+string(role), nil)
if err != nil {
return IAMResponse{}, fmt.Errorf("error creating role request: %w", err)
}
if useIMDSv2 {
req.Header.Set(imdsTokenHeader, token)
}
resp, err = cl.Do(req)
if err != nil {
return IAMResponse{}, fmt.Errorf("error fetching role data: %w", err)
}
defer func() {
// Drain and close the body to let the Transport reuse the connection
io.Copy(io.Discard, resp.Body)
resp.Body.Close()
}()
if resp.StatusCode != http.StatusOK {
return IAMResponse{}, fmt.Errorf("error fetching role data, got non 200 code: %s", resp.Status)
}
var jResp IAMResponse
jsonString, err := io.ReadAll(resp.Body)
if err != nil {
return IAMResponse{}, fmt.Errorf("error reading role data: %w", err)
}
if err := json.Unmarshal(jsonString, &jResp); err != nil {
return IAMResponse{}, fmt.Errorf("error unmarshalling role data: %w (%s)", err, jsonString)
}
return jResp, nil
}
func newUsingIAM(cl *http.Client, baseURL, region string) (*S3, error) {
// Get the IAM role
iamResp, err := fetchIAMData(cl, baseURL)
if err != nil {
return nil, fmt.Errorf("error fetching IAM data: %w", err)
}
return &S3{
Region: region,
AccessKey: iamResp.AccessKeyID,
SecretKey: iamResp.SecretAccessKey,
Token: iamResp.Token,
URIFormat: "https://s3.%s.amazonaws.com/%s",
initMode: "iam",
expiry: iamResp.Expiration,
}, nil
}
// setIAMData sets the IAM data on the S3 instance.
func (s3 *S3) SetIAMData(iamResp IAMResponse) {
s3.AccessKey = iamResp.AccessKeyID
s3.SecretKey = iamResp.SecretAccessKey
s3.Token = iamResp.Token
}
func (s3 *S3) getClient() *http.Client {
if s3.Client == nil {
return http.DefaultClient
@@ -376,25 +53,6 @@ func (s3 *S3) getClient() *http.Client {
return s3.Client
}
// getURL constructs a URL for a given path, with multiple optional
// arguments as individual subfolders, based on the endpoint
// specified in s3 struct.
func (s3 *S3) getURL(path string, args ...string) (uri string) {
if len(args) > 0 {
path += "/" + strings.Join(args, "/")
}
// need to encode special characters in the path part of the URL
encodedPath := encodePath(path)
if len(s3.Endpoint) > 0 {
uri = s3.Endpoint + "/" + encodedPath
} else {
uri = fmt.Sprintf(s3.URIFormat, s3.Region, encodedPath)
}
return uri
}
// SetEndpoint can be used to the set a custom endpoint for
// using an alternate instance compatible with the s3 API.
// If no protocol is included in the URI, defaults to HTTPS.
@@ -422,20 +80,6 @@ func (s3 *S3) SetToken(token string) *S3 {
return s3
}
func detectFileSize(body io.Seeker) (int64, error) {
pos, err := body.Seek(0, 1)
if err != nil {
return -1, err
}
defer body.Seek(pos, 0)
n, err := body.Seek(0, 2) //nolint:gomnd
if err != nil {
return -1, err
}
return n, nil
}
// SetClient can be used to set the http client to be
// used by the package. If client passed is nil,
// http.DefaultClient is used.
@@ -495,570 +139,3 @@ func (s3 *S3) signRequest(req *http.Request) error {
req.Header.Set("Authorization", auth.String())
return nil
}
func (s3 *S3) renewIAMToken() error {
if s3.initMode != "iam" {
return nil
}
if time.Since(s3.expiry) < 0 {
return nil
}
s3.mu.Lock()
defer s3.mu.Unlock()
iamResp, err := fetchIAMData(s3.getClient(), metadataBaseURL)
if err != nil {
return fmt.Errorf("error fetching IAM data: %w", err)
}
s3.expiry = iamResp.Expiration
s3.Token = iamResp.Token
s3.AccessKey = iamResp.AccessKeyID
s3.SecretKey = iamResp.SecretAccessKey
return nil
}
// FileDownload makes a GET call and returns a io.ReadCloser.
// After reading the response body, ensure closing the response.
func (s3 *S3) FileDownload(u DownloadInput) (io.ReadCloser, error) {
req, err := http.NewRequest(
http.MethodGet, s3.getURL(u.Bucket, u.ObjectKey), nil,
)
if err != nil {
return nil, err
}
if err := s3.renewIAMToken(); err != nil {
return nil, err
}
if err := s3.signRequest(req); err != nil {
return nil, err
}
res, err := s3.getClient().Do(req)
if err != nil {
return nil, err
}
if res.StatusCode != http.StatusOK {
return nil, fmt.Errorf("status code: %s", res.Status)
}
return res.Body, nil
}
// FilePut makes a PUT call to S3.
func (s3 *S3) FilePut(u UploadInput) (PutResponse, error) {
fSize, err := detectFileSize(u.Body)
if err != nil {
return PutResponse{}, err
}
content := make([]byte, fSize)
_, err = u.Body.Read(content)
if err != nil {
return PutResponse{}, err
}
u.Body.Seek(0, 0)
req, er := http.NewRequest(http.MethodPut, s3.getURL(u.Bucket, u.ObjectKey), u.Body)
if er != nil {
return PutResponse{}, err
}
if u.ContentType == "" {
u.ContentType = "application/octet-stream"
}
h := sha256.New()
h.Write(content)
req.Header.Set("x-amz-content-sha256", fmt.Sprintf("%x", h.Sum(nil)))
req.Header.Set("Content-Type", u.ContentType)
req.Header.Set("Content-Length", fmt.Sprintf("%d", fSize))
req.Header.Set("Host", req.URL.Host)
for k, v := range u.CustomMetadata {
req.Header.Set("x-amz-meta-"+k, v)
}
if u.ContentDisposition != "" {
req.Header.Set("Content-Disposition", u.ContentDisposition)
}
if u.ACL != "" {
req.Header.Set("x-amz-acl", u.ACL)
}
req.ContentLength = fSize
if err := s3.renewIAMToken(); err != nil {
return PutResponse{}, err
}
if err := s3.signRequest(req); err != nil {
return PutResponse{}, err
}
// debug(httputil.DumpRequest(req, true))
// Submit the request
client := s3.getClient()
res, err := client.Do(req)
if err != nil {
return PutResponse{}, err
}
defer func() {
res.Body.Close()
io.Copy(io.Discard, res.Body)
}()
data, err := io.ReadAll(res.Body)
if err != nil {
return PutResponse{}, err
}
// Check the response
if res.StatusCode != http.StatusOK {
return PutResponse{}, fmt.Errorf("status code: %s: %q", res.Status, data)
}
return PutResponse{
ETag: res.Header.Get("ETag"),
Headers: res.Header.Clone(),
}, nil
}
// FileUpload makes a POST call with the file written as multipart
// and on successful upload, checks for 200 OK.
func (s3 *S3) FileUpload(u UploadInput) (UploadResponse, error) {
fSize, err := detectFileSize(u.Body)
if err != nil {
return UploadResponse{}, err
}
uc := UploadConfig{
UploadURL: s3.getURL(u.Bucket),
BucketName: u.Bucket,
ObjectKey: u.ObjectKey,
ContentType: u.ContentType,
ContentDisposition: u.ContentDisposition,
ACL: u.ACL,
FileSize: fSize,
MetaData: map[string]string{
"success_action_status": "201", // returns XML doc on success
},
}
// Set custom metadata.
for k, v := range u.CustomMetadata {
if !strings.HasPrefix(k, AMZMetaPrefix) {
k = AMZMetaPrefix + k
}
uc.MetaData[k] = v
}
if err := s3.renewIAMToken(); err != nil {
return UploadResponse{}, err
}
policies, err := s3.CreateUploadPolicies(uc)
if err != nil {
return UploadResponse{}, err
}
var b bytes.Buffer
w := multipart.NewWriter(&b)
for k, v := range policies.Form {
if err = w.WriteField(k, v); err != nil {
return UploadResponse{}, err
}
}
fw, err := w.CreateFormFile("file", u.FileName)
if err != nil {
return UploadResponse{}, err
}
if _, err = io.Copy(fw, u.Body); err != nil {
return UploadResponse{}, err
}
// Don't forget to close the multipart writer.
// If you don't close it, your request will be missing the terminating boundary.
if err := w.Close(); err != nil {
return UploadResponse{}, err
}
// Now that you have a form, you can submit it to your handler.
req, err := http.NewRequest(http.MethodPost, policies.URL, &b)
if err != nil {
return UploadResponse{}, err
}
// Don't forget to set the content type, this will contain the boundary.
req.Header.Set("Content-Type", w.FormDataContentType())
// Submit the request
client := s3.getClient()
res, err := client.Do(req)
if err != nil {
return UploadResponse{}, err
}
defer func() {
res.Body.Close()
io.Copy(io.Discard, res.Body)
}()
data, err := io.ReadAll(res.Body)
if err != nil {
return UploadResponse{}, err
}
// Check the response
if res.StatusCode != http.StatusCreated {
return UploadResponse{}, fmt.Errorf("status code: %s: %q", res.Status, data)
}
var ur UploadResponse
_ = xml.Unmarshal(data, &ur)
return ur, nil
}
// FileDelete makes a DELETE call with the file written as multipart
// and on successful upload, checks for 204 No Content.
func (s3 *S3) FileDelete(u DeleteInput) error {
req, err := http.NewRequest(
http.MethodDelete, s3.getURL(u.Bucket, u.ObjectKey), nil,
)
if err != nil {
return err
}
if err := s3.renewIAMToken(); err != nil {
return err
}
if err := s3.signRequest(req); err != nil {
return err
}
// Submit the request
client := s3.getClient()
res, err := client.Do(req)
if err != nil {
return err
}
defer func() {
res.Body.Close()
io.Copy(io.Discard, res.Body)
}()
// Check the response
if res.StatusCode != http.StatusNoContent {
return fmt.Errorf("status code: %s", res.Status)
}
return nil
}
func (s3 *S3) FileDetails(u DetailsInput) (DetailsResponse, error) {
req, err := http.NewRequest(
http.MethodHead, s3.getURL(u.Bucket, u.ObjectKey), nil,
)
if err != nil {
return DetailsResponse{}, err
}
if err := s3.renewIAMToken(); err != nil {
return DetailsResponse{}, err
}
if err := s3.signRequest(req); err != nil {
return DetailsResponse{}, err
}
res, err := s3.getClient().Do(req)
if err != nil {
return DetailsResponse{}, err
}
defer func() {
res.Body.Close()
io.Copy(io.Discard, res.Body)
}()
if res.StatusCode != http.StatusOK {
return DetailsResponse{}, fmt.Errorf("status code: %s", res.Status)
}
var out DetailsResponse
for k, v := range res.Header {
lk := strings.ToLower(k)
switch lk {
case "content-type":
out.ContentType = getFirstString(v)
case "content-length":
out.ContentLength = getFirstString(v)
case "accept-ranges":
out.AcceptRanges = getFirstString(v)
case "date":
out.Date = getFirstString(v)
case "etag":
out.Etag = getFirstString(v)
case "last-modified":
out.LastModified = getFirstString(v)
case "server":
out.Server = getFirstString(v)
case "x-amz-id-2":
out.AmzID2 = getFirstString(v)
case "x-amz-request-id":
out.AmzRequestID = getFirstString(v)
default:
if strings.HasPrefix(lk, AMZMetaPrefix) {
if out.AmzMeta == nil {
out.AmzMeta = map[string]string{}
}
out.AmzMeta[k] = getFirstString(v)
} else {
if out.ExtraHeaders == nil {
out.ExtraHeaders = map[string]string{}
}
out.ExtraHeaders[k] = getFirstString(v)
}
}
}
return out, nil
}
// List implements a simple S3 object listing API
func (s3 *S3) List(input ListInput) (ListResponse, error) {
// Input validation
if input.Bucket == "" {
return ListResponse{}, fmt.Errorf("bucket name cannot be empty")
}
if input.MaxKeys < 0 {
return ListResponse{}, fmt.Errorf("MaxKeys cannot be negative")
}
if input.MaxKeys > 1000 {
return ListResponse{}, fmt.Errorf("MaxKeys cannot exceed 1000")
}
// Build query parameters - ListObjectsV2 uses query params, not path
query := url.Values{}
query.Set("list-type", "2") // Required parameter
// Add optional parameters if they exist
if input.ContinuationToken != "" {
query.Set("continuation-token", input.ContinuationToken)
}
if input.Delimiter != "" {
query.Set("delimiter", input.Delimiter)
}
if input.MaxKeys > 0 {
query.Set("max-keys", fmt.Sprintf("%d", input.MaxKeys))
}
if input.Prefix != "" {
query.Set("prefix", input.Prefix)
}
if input.StartAfter != "" {
query.Set("start-after", input.StartAfter)
}
// Build base URL
baseURL := s3.getURL(input.Bucket)
// Parse URL and add query parameters
parsedURL, err := url.Parse(baseURL)
if err != nil {
return ListResponse{}, err
}
parsedURL.RawQuery = query.Encode()
// Create HTTP request
req, err := http.NewRequest(http.MethodGet, parsedURL.String(), nil)
if err != nil {
return ListResponse{}, err
}
// Apply AWS V4 signing
if err := s3.renewIAMToken(); err != nil {
return ListResponse{}, err
}
if err := s3.signRequest(req); err != nil {
return ListResponse{}, err
}
// Execute request
client := s3.getClient()
res, err := client.Do(req)
if err != nil {
return ListResponse{}, err
}
// Close response body when done
defer res.Body.Close()
// Read response body once
body, err := io.ReadAll(res.Body)
if err != nil {
return ListResponse{}, err
}
// Handle response status codes
if res.StatusCode != http.StatusOK {
var s3Err S3Error
if xmlErr := xml.Unmarshal(body, &s3Err); xmlErr == nil {
return ListResponse{}, fmt.Errorf("S3 Error: %s - %s", s3Err.Code, s3Err.Message)
}
return ListResponse{}, fmt.Errorf("status code: %s: %s", res.Status, string(body))
}
// Parse XML response using internal struct
var xmlResult struct {
XMLName xml.Name `xml:"ListBucketResult"`
Name string `xml:"Name"`
IsTruncated bool `xml:"IsTruncated"`
NextContinuationToken string `xml:"NextContinuationToken"`
KeyCount int64 `xml:"KeyCount"`
Contents []struct {
Key string `xml:"Key"`
LastModified string `xml:"LastModified"`
ETag string `xml:"ETag"`
Size int64 `xml:"Size"`
StorageClass string `xml:"StorageClass"`
} `xml:"Contents"`
CommonPrefixes []struct {
Prefix string `xml:"Prefix"`
} `xml:"CommonPrefixes"`
}
if err := xml.Unmarshal(body, &xmlResult); err != nil {
return ListResponse{}, err
}
// Convert to simple response format
response := ListResponse{
Name: xmlResult.Name,
IsTruncated: xmlResult.IsTruncated,
NextContinuationToken: xmlResult.NextContinuationToken,
KeyCount: xmlResult.KeyCount,
}
// Convert objects
for _, obj := range xmlResult.Contents {
response.Objects = append(response.Objects, Object{
Key: obj.Key,
Size: obj.Size,
LastModified: obj.LastModified,
ETag: obj.ETag,
StorageClass: obj.StorageClass,
})
}
// Convert common prefixes
for _, prefix := range xmlResult.CommonPrefixes {
response.CommonPrefixes = append(response.CommonPrefixes, prefix.Prefix)
}
return response, nil
}
// ListAll returns an iterator that yields all objects in the bucket,
// automatically handling pagination. It also returns a finish callback
// that should be called after iteration to check for any errors.
func (s3 *S3) ListAll(input ListInput) (iter.Seq[Object], func() error) {
var iterErr error
seq := func(yield func(Object) bool) {
currentInput := input
for {
response, err := s3.List(currentInput)
if err != nil {
iterErr = err
return
}
// Yield each object
for _, obj := range response.Objects {
if !yield(obj) {
return // Early termination requested
}
}
// Check if there are more results
if !response.IsTruncated || response.NextContinuationToken == "" {
return // No more results
}
// Prepare for next page
currentInput.ContinuationToken = response.NextContinuationToken
}
}
finish := func() error {
return iterErr
}
return seq, finish
}
func getFirstString(s []string) string {
if len(s) > 0 {
return s[0]
}
return ""
}
// if object matches reserved string, no need to encode them.
var reservedObjectNames = regexp.MustCompile("^[a-zA-Z0-9-_.~/]+$")
// encodePath encode the strings from UTF-8 byte representations to HTML hex escape sequences
//
// This is necessary since regular url.Parse() and url.Encode() functions do not support UTF-8
// non english characters cannot be parsed due to the nature in which url.Encode() is written
//
// This function on the other hand is a direct replacement for url.Encode() technique to support
// pretty much every UTF-8 character.
// adapted from
// https://github.com/minio/minio-go/blob/fe1f3855b146c1b6ce4199740d317e44cf9e85c2/pkg/s3utils/utils.go#L285
func encodePath(pathName string) string {
if reservedObjectNames.MatchString(pathName) {
return pathName
}
var encodedPathname strings.Builder
for _, s := range pathName {
if 'A' <= s && s <= 'Z' || 'a' <= s && s <= 'z' || '0' <= s && s <= '9' { // §2.3 Unreserved characters (mark)
encodedPathname.WriteRune(s)
continue
}
switch s {
case '-', '_', '.', '~', '/': // §2.3 Unreserved characters (mark)
encodedPathname.WriteRune(s)
continue
default:
lenR := utf8.RuneLen(s)
if lenR < 0 {
// if utf8 cannot convert, return the same string as is
return pathName
}
u := make([]byte, lenR)
utf8.EncodeRune(u, s)
for _, r := range u {
hex := hex.EncodeToString([]byte{r})
encodedPathname.WriteString("%" + strings.ToUpper(hex))
}
}
}
return encodedPathname.String()
}
simples3_test.go
-794
View File
@@ -1,17 +1,8 @@
package simples3
import (
"bytes"
"errors"
"fmt"
"io"
"net"
"net/http"
"net/http/httptest"
"os"
"strings"
"testing"
"time"
)
type tConfig struct {
@@ -21,613 +12,6 @@ type tConfig struct {
Region string
}
func TestS3_FileUploadPostAndPut(t *testing.T) {
testTxt, err := os.Open("testdata/test.txt")
if err != nil {
t.Fatalf("Failed to open test.txt: %v", err)
}
defer testTxt.Close()
// Note: cannot re-use the same file descriptor due to seeking!
testTxtSpecialChars, err := os.Open("testdata/test.txt")
if err != nil {
t.Fatalf("Failed to open test.txt for special chars: %v", err)
}
defer testTxtSpecialChars.Close()
testPng, err := os.Open("testdata/avatar.png")
if err != nil {
t.Fatalf("Failed to open avatar.png: %v", err)
}
defer testPng.Close()
type args struct {
u UploadInput
}
tests := []struct {
name string
fields tConfig
args args
testDetails bool
wantErr bool
}{
{
name: "Upload test.txt",
fields: tConfig{
AccessKey: os.Getenv("AWS_S3_ACCESS_KEY"),
SecretKey: os.Getenv("AWS_S3_SECRET_KEY"),
Endpoint: os.Getenv("AWS_S3_ENDPOINT"),
Region: os.Getenv("AWS_S3_REGION"),
},
args: args{
UploadInput{
Bucket: os.Getenv("AWS_S3_BUCKET"),
ObjectKey: "test.txt",
ContentType: "text/plain",
FileName: "test.txt",
Body: testTxt,
},
},
wantErr: false,
},
{
name: "Upload test.txt with custom metadata",
fields: tConfig{
AccessKey: os.Getenv("AWS_S3_ACCESS_KEY"),
SecretKey: os.Getenv("AWS_S3_SECRET_KEY"),
Endpoint: os.Getenv("AWS_S3_ENDPOINT"),
Region: os.Getenv("AWS_S3_REGION"),
},
args: args{
UploadInput{
Bucket: os.Getenv("AWS_S3_BUCKET"),
ObjectKey: "test_metadata.txt",
ContentType: "text/plain",
FileName: "test.txt",
Body: testTxt,
CustomMetadata: map[string]string{
"test-metadata": "foo-bar",
},
},
},
wantErr: false,
testDetails: true,
},
{
name: "Upload avatar.png",
fields: tConfig{
AccessKey: os.Getenv("AWS_S3_ACCESS_KEY"),
SecretKey: os.Getenv("AWS_S3_SECRET_KEY"),
Endpoint: os.Getenv("AWS_S3_ENDPOINT"),
Region: os.Getenv("AWS_S3_REGION"),
},
args: args{
UploadInput{
Bucket: os.Getenv("AWS_S3_BUCKET"),
ObjectKey: "xyz/image.png",
ContentType: "image/png",
FileName: "avatar.png",
Body: testPng,
},
},
wantErr: false,
},
{
name: "Upload special filename txt",
fields: tConfig{
AccessKey: os.Getenv("AWS_S3_ACCESS_KEY"),
SecretKey: os.Getenv("AWS_S3_SECRET_KEY"),
Endpoint: os.Getenv("AWS_S3_ENDPOINT"),
Region: os.Getenv("AWS_S3_REGION"),
},
args: args{
UploadInput{
Bucket: os.Getenv("AWS_S3_BUCKET"),
ObjectKey: "xyz/example file%with$special&chars(1)?.txt",
ContentType: "text/plain",
FileName: "example file%with$special&chars(1)?.txt",
Body: testTxtSpecialChars,
},
},
wantErr: false,
},
}
for _, testcase := range tests {
tt := testcase
t.Run(tt.name+"_post", func(t *testing.T) {
s3 := New(tt.fields.Region, tt.fields.AccessKey, tt.fields.SecretKey)
s3.SetEndpoint(tt.fields.Endpoint)
resp, err := s3.FileUpload(tt.args.u)
if (err != nil) != tt.wantErr {
t.Errorf("S3.FileUpload() error = %v, wantErr %v", err, tt.wantErr)
}
// reset file, to reuse in further tests.
tt.args.u.Body.Seek(0, 0)
// check for empty response
if (resp == UploadResponse{}) {
t.Errorf("S3.FileUpload() returned empty response, %v", resp)
}
if tt.testDetails {
dResp, err := s3.FileDetails(DetailsInput{
Bucket: tt.args.u.Bucket,
ObjectKey: tt.args.u.ObjectKey,
})
if (err != nil) != tt.wantErr {
t.Errorf("S3.FileDetails() error = %v, wantErr %v", err, tt.wantErr)
}
if len(dResp.AmzMeta) != len(tt.args.u.CustomMetadata) {
t.Errorf("S3.FileDetails() returned incorrect metadata, got: %#v", dResp)
}
}
})
t.Run(tt.name+"_put", func(t *testing.T) {
s3 := New(tt.fields.Region, tt.fields.AccessKey, tt.fields.SecretKey)
s3.SetEndpoint(tt.fields.Endpoint)
resp, err := s3.FilePut(tt.args.u)
if (err != nil) != tt.wantErr {
t.Errorf("S3.FileUpload() error = %v, wantErr %v", err, tt.wantErr)
}
// reset file, to reuse in further tests.
tt.args.u.Body.Seek(0, 0)
// check for empty response
if resp.ETag == "" {
t.Errorf("S3.FileUpload() returned empty response, %v", resp)
}
if tt.testDetails {
dResp, err := s3.FileDetails(DetailsInput{
Bucket: tt.args.u.Bucket,
ObjectKey: tt.args.u.ObjectKey,
})
if (err != nil) != tt.wantErr {
t.Errorf("S3.FileUpload() error = %v, wantErr %v", err, tt.wantErr)
}
if len(dResp.AmzMeta) != len(tt.args.u.CustomMetadata) {
t.Errorf("S3.FileDetails() returned incorrect metadata, got: %#v", dResp)
}
}
})
}
}
func TestS3_FileDownload(t *testing.T) {
testTxt, err := os.Open("testdata/test.txt")
if err != nil {
t.Fatal(err)
}
defer testTxt.Close()
testTxtData, err := io.ReadAll(testTxt)
if err != nil {
t.Fatal(err)
}
testPng, err := os.Open("testdata/avatar.png")
if err != nil {
t.Fatal(err)
}
defer testPng.Close()
testPngData, err := io.ReadAll(testPng)
if err != nil {
t.Fatal(err)
}
type args struct {
u DownloadInput
}
tests := []struct {
name string
fields tConfig
args args
wantErr bool
wantResponse []byte
}{
{
name: "txt",
fields: tConfig{
AccessKey: os.Getenv("AWS_S3_ACCESS_KEY"),
SecretKey: os.Getenv("AWS_S3_SECRET_KEY"),
Endpoint: os.Getenv("AWS_S3_ENDPOINT"),
Region: os.Getenv("AWS_S3_REGION"),
},
args: args{
u: DownloadInput{
Bucket: os.Getenv("AWS_S3_BUCKET"),
ObjectKey: "test.txt",
},
},
wantErr: false,
wantResponse: testTxtData,
},
{
name: "png",
fields: tConfig{
AccessKey: os.Getenv("AWS_S3_ACCESS_KEY"),
SecretKey: os.Getenv("AWS_S3_SECRET_KEY"),
Endpoint: os.Getenv("AWS_S3_ENDPOINT"),
Region: os.Getenv("AWS_S3_REGION"),
},
args: args{
u: DownloadInput{
Bucket: os.Getenv("AWS_S3_BUCKET"),
ObjectKey: "xyz/image.png",
},
},
wantErr: false,
wantResponse: testPngData,
},
{
name: "txt-special-filename",
fields: tConfig{
AccessKey: os.Getenv("AWS_S3_ACCESS_KEY"),
SecretKey: os.Getenv("AWS_S3_SECRET_KEY"),
Endpoint: os.Getenv("AWS_S3_ENDPOINT"),
Region: os.Getenv("AWS_S3_REGION"),
},
args: args{
u: DownloadInput{
Bucket: os.Getenv("AWS_S3_BUCKET"),
ObjectKey: "xyz/example file%with$special&chars(1)?.txt",
},
},
wantErr: false,
wantResponse: testTxtData,
},
}
for _, testcase := range tests {
tt := testcase
t.Run(tt.name, func(t *testing.T) {
s3 := New(tt.fields.Region, tt.fields.AccessKey, tt.fields.SecretKey)
s3.SetEndpoint(tt.fields.Endpoint)
resp, err := s3.FileDownload(tt.args.u)
if (err != nil) != tt.wantErr {
t.Fatalf("S3.FileDownload() error = %v, wantErr %v", err, tt.wantErr)
}
got, err := io.ReadAll(resp)
if err != nil {
t.Fatalf("error = %v", err)
}
resp.Close()
if !bytes.Equal(got, tt.wantResponse) {
t.Fatalf("S3.FileDownload() = %v, want %v", got, tt.wantResponse)
}
})
}
}
func TestS3_FileDelete(t *testing.T) {
type args struct {
u DeleteInput
}
tests := []struct {
name string
fields tConfig
args args
wantErr bool
}{
{
name: "Delete test.txt",
fields: tConfig{
AccessKey: os.Getenv("AWS_S3_ACCESS_KEY"),
SecretKey: os.Getenv("AWS_S3_SECRET_KEY"),
Endpoint: os.Getenv("AWS_S3_ENDPOINT"),
Region: os.Getenv("AWS_S3_REGION"),
},
args: args{
DeleteInput{
Bucket: os.Getenv("AWS_S3_BUCKET"),
ObjectKey: "test.txt",
},
},
wantErr: false,
},
{
name: "Delete test_metadata.txt",
fields: tConfig{
AccessKey: os.Getenv("AWS_S3_ACCESS_KEY"),
SecretKey: os.Getenv("AWS_S3_SECRET_KEY"),
Endpoint: os.Getenv("AWS_S3_ENDPOINT"),
Region: os.Getenv("AWS_S3_REGION"),
},
args: args{
DeleteInput{
Bucket: os.Getenv("AWS_S3_BUCKET"),
ObjectKey: "test_metadata.txt",
},
},
wantErr: false,
},
{
name: "Delete avatar.png",
fields: tConfig{
AccessKey: os.Getenv("AWS_S3_ACCESS_KEY"),
SecretKey: os.Getenv("AWS_S3_SECRET_KEY"),
Endpoint: os.Getenv("AWS_S3_ENDPOINT"),
Region: os.Getenv("AWS_S3_REGION"),
},
args: args{
DeleteInput{
Bucket: os.Getenv("AWS_S3_BUCKET"),
ObjectKey: "xyz/image.png",
},
},
wantErr: false,
},
{
name: "Delete special filename txt",
fields: tConfig{
AccessKey: os.Getenv("AWS_S3_ACCESS_KEY"),
SecretKey: os.Getenv("AWS_S3_SECRET_KEY"),
Endpoint: os.Getenv("AWS_S3_ENDPOINT"),
Region: os.Getenv("AWS_S3_REGION"),
},
args: args{
DeleteInput{
Bucket: os.Getenv("AWS_S3_BUCKET"),
ObjectKey: "xyz/example file%with$special&chars(1)?.txt",
},
},
wantErr: false,
},
}
for _, testcase := range tests {
tt := testcase
t.Run(tt.name, func(t *testing.T) {
s3 := New(tt.fields.Region, tt.fields.AccessKey, tt.fields.SecretKey)
s3.SetEndpoint(tt.fields.Endpoint)
if err := s3.FileDelete(tt.args.u); (err != nil) != tt.wantErr {
t.Errorf("S3.FileDelete() error = %v, wantErr %v", err, tt.wantErr)
}
})
}
}
func TestS3_NewUsingIAM(t *testing.T) {
var (
iam = `test-new-s3-using-iam`
resp = `{"Code" : "Success","LastUpdated" : "2018-12-24T10:18:01Z",
"Type" : "AWS-HMAC","AccessKeyId" : "abc",
"SecretAccessKey" : "abc","Token" : "abc",
"Expiration" : "2018-12-24T16:24:59Z"}`
respIMDSToken = `AQAEAJWopi8yvjKYXyWJbzESE0cms-OoTnptJzS3M9g5iNcl06UEkQ==`
)
tsFail := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
time.Sleep(2 * time.Second)
}))
defer tsFail.Close()
genServerHandlerFunc := func(failIMDS bool) http.HandlerFunc {
return func(w http.ResponseWriter, r *http.Request) {
switch r.Method {
case "GET":
if !failIMDS {
// check if token is present
if r.Header.Get(imdsTokenHeader) == "" {
w.WriteHeader(http.StatusUnauthorized)
return
}
}
url := securityCredentialsURI
if r.URL.EscapedPath() == url {
w.WriteHeader(http.StatusOK)
io.WriteString(w, iam)
}
if r.URL.EscapedPath() == url+iam {
w.WriteHeader(http.StatusOK)
w.Header().Set("Content-Type", "application/json")
io.WriteString(w, resp)
}
case "PUT":
if failIMDS {
w.WriteHeader(http.StatusNotFound)
return
}
if r.URL.EscapedPath() == imdsTokenURI {
if r.Header.Get(imdsTokenTtlHeader) != "60" {
w.WriteHeader(http.StatusBadRequest)
return
}
w.WriteHeader(http.StatusOK)
io.WriteString(w, respIMDSToken)
}
default:
t.Errorf("Expected 'GET' or 'PUT' request, got '%s'", r.Method)
}
}
}
ts := httptest.NewServer(http.HandlerFunc(genServerHandlerFunc(false)))
defer ts.Close()
tsFailIMDS := httptest.NewServer(http.HandlerFunc(genServerHandlerFunc(true)))
defer tsFailIMDS.Close()
cl := &http.Client{Timeout: 1 * time.Second}
// Test for timeout.
_, err := newUsingIAM(cl, tsFail.URL, "abc")
if err == nil {
t.Errorf("Expected error, got nil")
} else {
var timeoutError net.Error
if errors.As(err, &timeoutError) && !timeoutError.Timeout() {
t.Errorf("newUsingIAM() timeout check. got error = %v", err)
}
}
// Test for successful IAM fetch.
s3, err := newUsingIAM(cl, ts.URL, "abc")
if err != nil {
t.Errorf("newUsingIAM() error = %v", err)
}
if s3 == nil {
t.Errorf("newUsingIAM() got = %v", s3)
}
if s3.AccessKey != "abc" || s3.SecretKey != "abc" || s3.Region != "abc" {
t.Errorf("S3.FileDelete() got = %v", s3)
}
// Test for failed IMDS token fetch.
_, err = newUsingIAM(cl, tsFailIMDS.URL, "abc")
if err == nil {
t.Errorf("Expected error, got nil")
}
}
func TestListAll(t *testing.T) {
// Setup test environment
s3 := New(
os.Getenv("AWS_S3_REGION"),
os.Getenv("AWS_S3_ACCESS_KEY"),
os.Getenv("AWS_S3_SECRET_KEY"),
)
s3.SetEndpoint(os.Getenv("AWS_S3_ENDPOINT"))
// Test cases
tests := []struct {
name string
setup func() // Setup function to prepare test data
input ListInput
validate func([]Object, error)
cleanup func() // Cleanup function
}{
{
name: "Empty Bucket Iterator",
input: ListInput{
Bucket: os.Getenv("AWS_S3_BUCKET"),
},
validate: func(objects []Object, err error) {
if err != nil {
t.Fatalf("Expected no error, got %v", err)
}
if len(objects) != 0 {
t.Errorf("Expected empty bucket, got %d objects", len(objects))
}
},
},
{
name: "Basic Iterator Listing",
setup: func() {
// Upload test files
uploadTestFiles(t, s3, os.Getenv("AWS_S3_BUCKET"), []string{"iter_file1.txt", "iter_file2.txt", "iter_file3.txt"})
},
input: ListInput{
Bucket: os.Getenv("AWS_S3_BUCKET"),
},
validate: func(objects []Object, err error) {
if err != nil {
t.Fatalf("Expected no error, got %v", err)
}
if len(objects) != 3 {
t.Errorf("Expected 3 objects, got %d", len(objects))
}
// Check that all expected files are present
expectedKeys := map[string]bool{
"iter_file1.txt": false,
"iter_file2.txt": false,
"iter_file3.txt": false,
}
for _, obj := range objects {
if _, exists := expectedKeys[obj.Key]; exists {
expectedKeys[obj.Key] = true
}
}
for key, found := range expectedKeys {
if !found {
t.Errorf("Expected object %s not found in iterator results", key)
}
}
},
cleanup: func() {
cleanupTestFiles(t, s3, os.Getenv("AWS_S3_BUCKET"), []string{"iter_file1.txt", "iter_file2.txt", "iter_file3.txt"})
},
},
{
name: "Iterator with Prefix",
setup: func() {
uploadTestFiles(t, s3, os.Getenv("AWS_S3_BUCKET"), []string{
"iter_docs/file1.txt",
"iter_docs/file2.txt",
"iter_images/image1.jpg",
"iter_images/image2.jpg",
})
},
input: ListInput{
Bucket: os.Getenv("AWS_S3_BUCKET"),
Prefix: "iter_docs/",
},
validate: func(objects []Object, err error) {
if err != nil {
t.Fatalf("Expected no error, got %v", err)
}
if len(objects) != 2 {
t.Errorf("Expected 2 objects with iter_docs/ prefix, got %d", len(objects))
}
for _, obj := range objects {
if !strings.HasPrefix(obj.Key, "iter_docs/") {
t.Errorf("Object %s doesn't have expected prefix", obj.Key)
}
}
},
cleanup: func() {
cleanupTestFiles(t, s3, os.Getenv("AWS_S3_BUCKET"), []string{
"iter_docs/file1.txt",
"iter_docs/file2.txt",
"iter_images/image1.jpg",
"iter_images/image2.jpg",
})
},
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
if tt.setup != nil {
tt.setup()
}
// Collect all objects from the iterator and check for errors
var objects []Object
seq, finish := s3.ListAll(tt.input)
for obj := range seq {
objects = append(objects, obj)
}
// Check for any iteration errors
if err := finish(); err != nil {
t.Fatalf("Iterator error: %v", err)
}
if tt.validate != nil {
tt.validate(objects, nil) // No error from iterator
}
if tt.cleanup != nil {
tt.cleanup()
}
})
}
}
func TestCustomEndpoint(t *testing.T) {
s3 := New("us-east-1", "AccessKey", "SuperSecretKey")
@@ -705,184 +89,6 @@ func TestGetURL(t *testing.T) {
}
}
func TestList(t *testing.T) {
// Setup test environment
s3 := New(
os.Getenv("AWS_S3_REGION"),
os.Getenv("AWS_S3_ACCESS_KEY"),
os.Getenv("AWS_S3_SECRET_KEY"),
)
s3.SetEndpoint(os.Getenv("AWS_S3_ENDPOINT"))
// Test cases
tests := []struct {
name string
setup func() // Setup function to prepare test data
input ListInput
validate func(ListResponse, error)
cleanup func() // Cleanup function
}{
{
name: "Empty Bucket",
input: ListInput{
Bucket: os.Getenv("AWS_S3_BUCKET"),
},
validate: func(result ListResponse, err error) {
if err != nil {
t.Fatalf("Expected no error, got %v", err)
}
if len(result.Objects) != 0 {
t.Errorf("Expected empty bucket, got %d objects", len(result.Objects))
}
},
},
{
name: "Basic Listing",
setup: func() {
// Upload test files
uploadTestFiles(t, s3, os.Getenv("AWS_S3_BUCKET"), []string{"file1.txt", "file2.txt", "file3.txt"})
},
input: ListInput{
Bucket: os.Getenv("AWS_S3_BUCKET"),
},
validate: func(result ListResponse, err error) {
if err != nil {
t.Fatalf("Expected no error, got %v", err)
}
if len(result.Objects) != 3 {
t.Errorf("Expected 3 objects, got %d", len(result.Objects))
}
},
cleanup: func() {
cleanupTestFiles(t, s3, os.Getenv("AWS_S3_BUCKET"), []string{"file1.txt", "file2.txt", "file3.txt"})
},
},
{
name: "Pagination Test",
setup: func() {
// Upload many files for pagination testing
var filenames []string
for i := 1; i <= 25; i++ {
filenames = append(filenames, fmt.Sprintf("file%03d.txt", i))
}
uploadTestFiles(t, s3, os.Getenv("AWS_S3_BUCKET"), filenames)
},
input: ListInput{
Bucket: os.Getenv("AWS_S3_BUCKET"),
MaxKeys: 10,
},
validate: func(result ListResponse, err error) {
if err != nil {
t.Fatalf("Expected no error, got %v", err)
}
if len(result.Objects) != 10 {
t.Errorf("Expected 10 objects, got %d", len(result.Objects))
}
if !result.IsTruncated {
t.Errorf("Expected IsTruncated=true for pagination")
}
if result.NextContinuationToken == "" {
t.Errorf("Expected NextContinuationToken for pagination: %v", result.NextContinuationToken)
}
},
cleanup: func() {
var filenames []string
for i := 1; i <= 25; i++ {
filenames = append(filenames, fmt.Sprintf("file%03d.txt", i))
}
cleanupTestFiles(t, s3, os.Getenv("AWS_S3_BUCKET"), filenames)
},
},
{
name: "Prefix Filtering",
setup: func() {
uploadTestFiles(t, s3, os.Getenv("AWS_S3_BUCKET"), []string{
"documents/file1.txt",
"documents/file2.txt",
"images/image1.jpg",
"images/image2.jpg",
})
},
input: ListInput{
Bucket: os.Getenv("AWS_S3_BUCKET"),
Prefix: "documents/",
},
validate: func(result ListResponse, err error) {
if err != nil {
t.Fatalf("Expected no error, got %v", err)
}
if len(result.Objects) != 2 {
t.Errorf("Expected 2 objects with documents/ prefix, got %d", len(result.Objects))
}
for _, obj := range result.Objects {
if !strings.HasPrefix(obj.Key, "documents/") {
t.Errorf("Object %s doesn't have expected prefix", obj.Key)
}
}
},
cleanup: func() {
cleanupTestFiles(t, s3, os.Getenv("AWS_S3_BUCKET"), []string{
"documents/file1.txt",
"documents/file2.txt",
"images/image1.jpg",
"images/image2.jpg",
})
},
},
{
name: "Delimiter Grouping",
setup: func() {
uploadTestFiles(t, s3, os.Getenv("AWS_S3_BUCKET"), []string{
"documents/important/file1.txt",
"documents/backup/file2.txt",
"images/landscape/image1.jpg",
"images/portrait/image2.jpg",
})
},
input: ListInput{
Bucket: os.Getenv("AWS_S3_BUCKET"),
Delimiter: "/",
Prefix: "documents/",
},
validate: func(result ListResponse, err error) {
if err != nil {
t.Fatalf("Expected no error, got %v", err)
}
// Should return common prefixes instead of individual files
if len(result.CommonPrefixes) != 2 {
t.Errorf("Expected 2 common prefixes, got %d", len(result.CommonPrefixes))
}
},
cleanup: func() {
cleanupTestFiles(t, s3, os.Getenv("AWS_S3_BUCKET"), []string{
"documents/important/file1.txt",
"documents/backup/file2.txt",
"images/landscape/image1.jpg",
"images/portrait/image2.jpg",
})
},
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
if tt.setup != nil {
tt.setup()
}
result, err := s3.List(tt.input)
if tt.validate != nil {
tt.validate(result, err)
}
if tt.cleanup != nil {
tt.cleanup()
}
})
}
}
// Helper functions
func uploadTestFiles(t *testing.T, s3 *S3, bucket string, filenames []string) {
for _, filename := range filenames {