diff --git a/README.md b/README.md index d6c96de..0422cf7 100644 --- a/README.md +++ b/README.md @@ -2,7 +2,7 @@ ## Overview [![GoDoc](https://godoc.org/github.com/rhnvrm/simples3?status.svg)](https://godoc.org/github.com/rhnvrm/simples3) [![Go Report Card](https://goreportcard.com/badge/github.com/rhnvrm/simples3)](https://goreportcard.com/report/github.com/rhnvrm/simples3) -SimpleS3 is a golang library for uploading and deleting objects on S3 buckets using the REST API calls signed using AWS Signature Version 4 +SimpleS3 is a golang library for uploading and deleting objects on S3 buckets using REST API calls or Presigned URLs signed using AWS Signature Version 4. ## Install @@ -16,7 +16,16 @@ go get github.com/rhnvrm/simples3 testTxt, _ := os.Open("test.txt") defer testTxt.Close() +// Create an instance of the package +// You can either create by manually supplying credentials +// (preferably using Environment vars) s3 := simples3.New(Region, AWSAccessKey, AWSSecretKey) +// or you can use this on an EC2 instance to +// obtain credentials from IAM attached to the instance. +s3 := simples3.NewUsingIAM(Region) + +// File Upload is as simple as providing the following +// details. err := s3.FileUpload(simples3.UploadInput{ Bucket: AWSBucket, ObjectKey: "test.txt", @@ -24,19 +33,34 @@ err := s3.FileUpload(simples3.UploadInput{ FileName: "test.txt", Body: testTxt, } + +// Similarly, Files can be deleted. +err := s3.FileDelete(simples3.DeleteInput{ + Bucket: os.Getenv("AWS_S3_BUCKET"), + ObjectKey: "test.txt", +} + +// You can also use this library to generate +// Presigned URLs that can for eg. be used to +// GET/PUT files on S3 through the browser. +var time, _ = time.Parse(time.RFC1123, "Fri, 24 May 2013 00:00:00 GMT") + +url := s.GeneratePresignedURL(PresignedInput{ + Bucket: "examplebucket", + ObjectKey: "test.txt", + Method: "GET", + Timestamp: time, + ExpirySeconds: 86400, +}) ``` ## Contributing -ToDo. - -## Contributors - -ToDo. +You are more than welcome to contribute to this project. Fork and make a Pull Request, or create an Issue if you see any problem or want to propose a feature. ## Author -ToDo. +Rohan Verma ## License diff --git a/presigned.go b/presigned.go index 0406864..937a59b 100644 --- a/presigned.go +++ b/presigned.go @@ -16,6 +16,7 @@ const ( defaultPresignedURLFormat = "%s.s3.amazonaws.com" // ) +// PresignedInput is passed to GeneratePresignedURL as a parameter. type PresignedInput struct { Bucket string ObjectKey string @@ -25,10 +26,13 @@ type PresignedInput struct { ExpirySeconds int } +// GeneratePresignedURL creates a Presigned URL that can be used +// for Authentication using Query Parameters. +// (https://docs.aws.amazon.com/AmazonS3/latest/API/sigv4-query-string-auth.html) func (s3 *S3) GeneratePresignedURL(in PresignedInput) string { var ( nowTime = in.Timestamp.UTC() - cred = s3.buildCredential(nowTime) + cred = fmt.Sprintf("%s/"+s3.buildCredentialWithoutKey(nowTime), s3.AccessKey) amzdate = nowTime.Format(amzDateISO8601TimeFormat) expiry = strconv.Itoa(in.ExpirySeconds) ) @@ -41,7 +45,7 @@ func (s3 *S3) GeneratePresignedURL(in PresignedInput) string { signedHeaders["host"] = fmt.Sprintf(defaultPresignedURLFormat, in.Bucket) // Start Canonical Request Formation - h := sha256.New() + h := sha256.New() // We write the canonical request directly to the SHA256 hash. fmt.Fprintf(h, "%s\n", in.Method) // HTTP Verb fmt.Fprintf(h, "/%s\n", in.ObjectKey) // CanonicalURL @@ -54,44 +58,53 @@ func (s3 *S3) GeneratePresignedURL(in PresignedInput) string { } // We need to have a sorted order, - keysQS := make([]string, 0, len(queryString)) + // for QueryStrings and SignedHeaders + sortedQS := make([]string, 0, len(queryString)) for name := range queryString { - keysQS = append(keysQS, name) + sortedQS = append(sortedQS, name) } - sort.Strings(keysQS) //sort by key - for _, k := range keysQS { + sort.Strings(sortedQS) //sort by key + + sortedSH := make([]string, 0, len(signedHeaders)) + for name := range signedHeaders { + sortedSH = append(sortedSH, name) + } + sort.Strings(sortedSH) //sort by key + + // Proceed to write canonical query params + for _, k := range sortedQS { fmt.Fprintf(h, "%s=%s&", url.QueryEscape(k), url.QueryEscape(queryString[k])) // HTTP Verb } fmt.Fprintf(h, "X-Amz-SignedHeaders=") // Add Signed Headers to Query String first := true - for k := range signedHeaders { + for k := range sortedSH { if first { - fmt.Fprintf(h, "%s", url.QueryEscape(k)) + fmt.Fprintf(h, "%s", url.QueryEscape(sortedSH[k])) first = false } else { - fmt.Fprintf(h, "%s", url.QueryEscape(";"+k)) + fmt.Fprintf(h, "%s", url.QueryEscape(";"+sortedSH[k])) } } fmt.Fprintf(h, "\n") // End QueryString Params // Start Canonical Headers - for k, v := range signedHeaders { - fmt.Fprintf(h, "%s:%s\n", strings.ToLower(k), strings.TrimSpace(v)) + for _, k := range sortedSH { + fmt.Fprintf(h, "%s:%s\n", strings.ToLower(k), strings.TrimSpace(signedHeaders[k])) } fmt.Fprintf(h, "\n") // End Canonical Headers // Start Signed Headers first = true - for k := range signedHeaders { + for k := range sortedSH { if first { - fmt.Fprintf(h, "%s", url.QueryEscape(k)) + fmt.Fprintf(h, "%s", url.QueryEscape(sortedSH[k])) first = false } else { - fmt.Fprintf(h, "%s", url.QueryEscape(";"+k)) + fmt.Fprintf(h, "%s", url.QueryEscape(";"+sortedSH[k])) } } fmt.Fprintf(h, "\n") @@ -126,7 +139,8 @@ func (s3 *S3) GeneratePresignedURL(in PresignedInput) string { []byte("s3")), []byte("aws4_request"), ) - // sigKey gen verified using https://docs.aws.amazon.com/general/latest/gr/signature-v4-examples.html#signature-v4-examples-other + // sigKey gen verified using + // https://docs.aws.amazon.com/general/latest/gr/signature-v4-examples.html#signature-v4-examples-other // (TODO: add a test using the same, consolidate with signKeys()) signedStrToSign := makeHMac(sigKey, stringToSign) @@ -142,21 +156,20 @@ func (s3 *S3) GeneratePresignedURL(in PresignedInput) string { // We don't need to have a sorted order here, // but just to preserve tests. - for _, k := range keysQS { + for _, k := range sortedQS { fmt.Fprintf(&b, "%s=%s&", url.QueryEscape(k), url.QueryEscape(queryString[k])) // HTTP Verb } fmt.Fprintf(&b, "%s=", "X-Amz-SignedHeaders") first = true - for k := range signedHeaders { + for k := range sortedSH { if first { - fmt.Fprintf(&b, "%s", url.QueryEscape(k)) + fmt.Fprintf(&b, "%s", url.QueryEscape(sortedSH[k])) first = false } else { - fmt.Fprintf(&b, "%s", url.QueryEscape(";"+k)) + fmt.Fprintf(&b, "%s", url.QueryEscape(";"+sortedSH[k])) } } fmt.Fprintf(&b, "&X-Amz-Signature=%s", signature) - resultURL := b.String() - return resultURL + return b.String() } diff --git a/presigned_test.go b/presigned_test.go index dd04c2c..be7a31e 100644 --- a/presigned_test.go +++ b/presigned_test.go @@ -49,7 +49,7 @@ func TestS3_GeneratePresignedURL_Personal(t *testing.T) { }) } -func TestS3_GeneratePresignedURL_POST(t *testing.T) { +func TestS3_GeneratePresignedURL_PUT(t *testing.T) { t.Run("Test", func(t *testing.T) { s := New( os.Getenv("AWS_S3_REGION"), @@ -68,3 +68,21 @@ func TestS3_GeneratePresignedURL_POST(t *testing.T) { } }) } + +func BenchmarkS3_GeneratePresigned(b *testing.B) { + // run the Fib function b.N times + s := New( + os.Getenv("AWS_S3_REGION"), + os.Getenv("AWS_S3_ACCESS_KEY"), + os.Getenv("AWS_S3_SECRET_KEY"), + ) + for n := 0; n < b.N; n++ { + s.GeneratePresignedURL(PresignedInput{ + Bucket: os.Getenv("AWS_S3_BUCKET"), + ObjectKey: "test.txt", + Method: "GET", + Timestamp: NowTime(), + ExpirySeconds: 3600, + }) + } +} diff --git a/simples3.go b/simples3.go index a19e7c9..c090632 100644 --- a/simples3.go +++ b/simples3.go @@ -7,6 +7,8 @@ import ( "bytes" "crypto/hmac" "crypto/sha256" + "encoding/json" + "errors" "fmt" "io" "io/ioutil" @@ -16,6 +18,10 @@ import ( "time" ) +const ( + securityCredentialsURL = "http://169.254.169.254/latest/meta-data/iam/security-credentials/" +) + // S3 provides a wrapper around your S3 credentials. type S3 struct { AccessKey string @@ -40,6 +46,18 @@ type DeleteInput struct { ObjectKey string } +// IAMResponse is used by NewUsingIAM to auto +// detect the credentials +type IAMResponse struct { + Code string `json:"Code"` + LastUpdated string `json:"LastUpdated"` + Type string `json:"Type"` + AccessKeyID string `json:"AccessKeyId"` + SecretAccessKey string `json:"SecretAccessKey"` + Token string `json:"Token"` + Expiration string `json:"Expiration"` +} + // New returns an instance of S3. func New(region, accessKey, secretKey string) *S3 { return &S3{ @@ -51,6 +69,53 @@ func New(region, accessKey, secretKey string) *S3 { } } +// NewUsingIAM automatically generates an Instance of S3 +// using instance metatdata. +func NewUsingIAM(region string) (*S3, error) { + + // Get the IAM role + resp, err := http.Get(securityCredentialsURL) + if err != nil { + return nil, err + } + defer resp.Body.Close() + if resp.StatusCode != 200 { + return nil, errors.New(http.StatusText(resp.StatusCode)) + } + + role, err := ioutil.ReadAll(resp.Body) + if err != nil { + return nil, err + } + + resp, err = http.Get(securityCredentialsURL + "/" + string(role)) + if err != nil { + return nil, err + } + defer resp.Body.Close() + if resp.StatusCode != 200 { + return nil, errors.New(http.StatusText(resp.StatusCode)) + } + + var jsonResp IAMResponse + jsonString, err := ioutil.ReadAll(resp.Body) + if err != nil { + return nil, err + } + + if err := json.Unmarshal(jsonString, &jsonResp); err != nil { + return nil, err + } + + return &S3{ + Region: region, + AccessKey: jsonResp.AccessKeyID, + SecretKey: jsonResp.SecretAccessKey, + + URIFormat: "https://s3.%s.amazonaws.com/%s", + }, nil +} + func detectFileSize(body multipart.File) int64 { switch r := body.(type) { case io.Seeker: @@ -133,7 +198,7 @@ func (s3 *S3) FileUpload(u UploadInput) error { return nil } -// FileDelete makes a POST call with the file written as multipart +// FileDelete makes a DELETE call with the file written as multipart // and on successful upload, checks for 204 No Content. func (s3 *S3) FileDelete(u DeleteInput) error { url := fmt.Sprintf(