From 66870855b73d9082ae6e1bc7c8e415528064fd75 Mon Sep 17 00:00:00 2001 From: Rohan Date: Wed, 5 Jan 2022 18:24:24 +0530 Subject: [PATCH] feat: add support for timeout and customization in IAM setup. This commit adds a defaultHTTPClient to have a timeout of 10s instead of the one provided by Go. It introduces a few new methods such as - FetchAndSetIAMData which is a helper method to be able to use the custom http client set by the user. Closes #19 --- README.md | 5 ++-- simples3.go | 60 +++++++++++++++++++++++++++++++++--------------- simples3_test.go | 28 +++++++++++++++++++--- 3 files changed, 70 insertions(+), 23 deletions(-) diff --git a/README.md b/README.md index 69c1e4c..9b8bed0 100644 --- a/README.md +++ b/README.md @@ -2,8 +2,8 @@ ## Overview [![GoDoc](https://godoc.org/github.com/rhnvrm/simples3?status.svg)](https://godoc.org/github.com/rhnvrm/simples3) [![Go Report Card](https://goreportcard.com/badge/github.com/rhnvrm/simples3)](https://goreportcard.com/report/github.com/rhnvrm/simples3) [![GoCover](https://gocover.io/_badge/github.com/rhnvrm/simples3)](https://gocover.io/_badge/github.com/rhnvrm/simples3) [![Zerodha Tech](https://zerodha.tech/static/images/github-badge.svg)](https://zerodha.tech) -SimpleS3 is a golang library for uploading and deleting objects -on S3 buckets using REST API calls or Presigned URLs signed +SimpleS3 is a Go library for manipulating objects +in S3 buckets using REST API calls or Presigned URLs signed using AWS Signature Version 4. ## Install @@ -54,6 +54,7 @@ file, _ := s3.FileDownload(simples3.DownloadInput{ Bucket: AWSBucket, ObjectKey: "test.txt", }) + data, _ := ioutil.ReadAll(file) file.Close() diff --git a/simples3.go b/simples3.go index 11e1b2f..479f242 100644 --- a/simples3.go +++ b/simples3.go @@ -24,8 +24,7 @@ import ( const ( securityCredentialsURL = "http://169.254.169.254/latest/meta-data/iam/security-credentials/" -) -const ( + // AMZMetaPrefix to prefix metadata key. AMZMetaPrefix = "x-amz-meta-" ) @@ -143,54 +142,79 @@ func New(region, accessKey, secretKey string) *S3 { // NewUsingIAM automatically generates an Instance of S3 // using instance metatdata. func NewUsingIAM(region string) (*S3, error) { - return newUsingIAMImpl(securityCredentialsURL, region) + return newUsingIAM( + &http.Client{ + // Set a timeout of 3 seconds for AWS IAM Calls. + Timeout: time.Second * 3, //nolint:gomnd + }, securityCredentialsURL, region) } -func newUsingIAMImpl(baseURL, region string) (*S3, error) { - // Get the IAM role - resp, err := http.Get(baseURL) +// fetchIAMData fetches the IAM data from the given URL. +// In case of a normal AWS setup, baseURL would be securityCredentialsURL. +// You can use this method, to manually fetch IAM data from a custom +// endpoint and pass it to SetIAMData. +func fetchIAMData(cl *http.Client, baseURL string) (IAMResponse, error) { + resp, err := cl.Get(baseURL) if err != nil { - return nil, err + return IAMResponse{}, err } defer resp.Body.Close() + if resp.StatusCode != http.StatusOK { - return nil, errors.New(http.StatusText(resp.StatusCode)) + return IAMResponse{}, fmt.Errorf("Error fetching IAM data: %s", resp.Status) } role, err := ioutil.ReadAll(resp.Body) if err != nil { - return nil, err + return IAMResponse{}, err } resp, err = http.Get(baseURL + "/" + string(role)) if err != nil { - return nil, err + return IAMResponse{}, err } defer resp.Body.Close() if resp.StatusCode != http.StatusOK { - return nil, errors.New(http.StatusText(resp.StatusCode)) + return IAMResponse{}, errors.New(http.StatusText(resp.StatusCode)) } - var jsonResp IAMResponse + var jResp IAMResponse jsonString, err := ioutil.ReadAll(resp.Body) if err != nil { - return nil, err + return IAMResponse{}, err } - if err := json.Unmarshal(jsonString, &jsonResp); err != nil { - return nil, err + if err := json.Unmarshal(jsonString, &jResp); err != nil { + return IAMResponse{}, err + } + + return jResp, nil +} + +func newUsingIAM(cl *http.Client, credUrl, region string) (*S3, error) { + // Get the IAM role + iamResp, err := fetchIAMData(cl, credUrl) + if err != nil { + return nil, fmt.Errorf("Error fetching IAM data: %w", err) } return &S3{ Region: region, - AccessKey: jsonResp.AccessKeyID, - SecretKey: jsonResp.SecretAccessKey, - Token: jsonResp.Token, + AccessKey: iamResp.AccessKeyID, + SecretKey: iamResp.SecretAccessKey, + Token: iamResp.Token, URIFormat: "https://s3.%s.amazonaws.com/%s", }, nil } +// setIAMData sets the IAM data on the S3 instance. +func (s3 *S3) setIAMData(iamResp IAMResponse) { + s3.AccessKey = iamResp.AccessKeyID + s3.SecretKey = iamResp.SecretAccessKey + s3.Token = iamResp.Token +} + func (s3 *S3) getClient() *http.Client { if s3.Client == nil { return http.DefaultClient diff --git a/simples3_test.go b/simples3_test.go index aaeed64..15e73d8 100644 --- a/simples3_test.go +++ b/simples3_test.go @@ -2,12 +2,15 @@ package simples3 import ( "bytes" + "errors" "io" "io/ioutil" + "net" "net/http" "net/http/httptest" "os" "testing" + "time" ) type tConfig struct { @@ -398,6 +401,12 @@ func TestS3_NewUsingIAM(t *testing.T) { "SecretAccessKey" : "abc","Token" : "abc", "Expiration" : "2018-12-24T16:24:59Z"}` ) + + tsFail := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + time.Sleep(2 * time.Second) + })) + defer tsFail.Close() + ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { if r.Method != "GET" { t.Errorf("Expected 'GET' request, got '%s'", r.Method) @@ -414,10 +423,23 @@ func TestS3_NewUsingIAM(t *testing.T) { })) defer ts.Close() - s3, err := newUsingIAMImpl(ts.URL, "abc") - if err != nil { - t.Errorf("S3.FileDelete() error = %v", err) + // Test for timeout. + _, err := newUsingIAM(&http.Client{Timeout: 1 * time.Second}, tsFail.URL, "abc") + if err == nil { + t.Errorf("Expected error, got nil") + } else { + var timeoutError net.Error + if errors.As(err, &timeoutError); !timeoutError.Timeout() { + t.Errorf("newUsingIAM() timeout check. got error = %v", err) + } } + + // Test for successful IAM fetch. + s3, err := newUsingIAM(http.DefaultClient, ts.URL, "abc") + if err != nil { + t.Errorf("newUsingIAM() error = %v", err) + } + if s3.AccessKey != "abc" && s3.SecretKey != "abc" && s3.Region != "abc" { t.Errorf("S3.FileDelete() got = %v", s3) }