From f30c5f90526da8c2fe33319375067f291176f6ca Mon Sep 17 00:00:00 2001 From: Rohan Verma Date: Thu, 6 Dec 2018 15:54:47 +0530 Subject: [PATCH] init --- LICENSE | 26 +++++++ README.md | 44 +++++++++++ go.mod | 1 + go.sum | 0 policy.go | 154 ++++++++++++++++++++++++++++++++++++ sign.go | 156 +++++++++++++++++++++++++++++++++++++ simples3.go | 198 +++++++++++++++++++++++++++++++++++++++++++++++ simples3_test.go | 97 +++++++++++++++++++++++ test.txt | 1 + 9 files changed, 677 insertions(+) create mode 100644 LICENSE create mode 100644 README.md create mode 100644 go.mod create mode 100644 go.sum create mode 100644 policy.go create mode 100644 sign.go create mode 100644 simples3.go create mode 100644 simples3_test.go create mode 100644 test.txt diff --git a/LICENSE b/LICENSE new file mode 100644 index 0000000..f3263e6 --- /dev/null +++ b/LICENSE @@ -0,0 +1,26 @@ +Copyright (c) 2018, Rohan Verma +All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met: + +1. Redistributions of source code must retain the above copyright notice, this + list of conditions and the following disclaimer. +2. Redistributions in binary form must reproduce the above copyright notice, + this list of conditions and the following disclaimer in the documentation + and/or other materials provided with the distribution. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND +ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR +ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES +(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; +LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND +ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +The views and conclusions contained in the software and documentation are those +of the authors and should not be interpreted as representing official policies, +either expressed or implied. \ No newline at end of file diff --git a/README.md b/README.md new file mode 100644 index 0000000..06b7ac0 --- /dev/null +++ b/README.md @@ -0,0 +1,44 @@ +# simples3 : Simple no frills AWS S3 Library using REST with V4 Signing + +## Overview [![GoDoc](https://godoc.org/github.com/rhnvrm/simples3?status.svg)](https://godoc.org/github.com/rhnvrm/simples3) [![Go Report Card](https://goreportcard.com/badge/github.com/rhnvrm/simples3)](https://goreportcard.com/report/github.com/rhnvrm/simples3) + +SimpleS3 is a golang library for uploading and deleting objects on S3 buckets using the REST API calls signed using AWS Signature Version 4 + +## Install + +``` +go get github.com/rhnvrm/simples3 +``` + +## Example + +``` +testTxt, _ := os.Open("test.txt") +defer testTxt.Close() + +s3 := simples3.New(Region, AWSAccessKey, AWSSecretKey) +err := s3.FileUpload(simples3.UploadInput{ + Bucket: "zerodha-testbucket", + ObjectKey: "test.txt", + ContentType: "text/plain", + FileName: "test.txt", + Body: testTxt, +} +``` + +## Contributing + +ToDo. + +## Contributors + +ToDo. + +## Author + +ToDo. + +## License + +MIT. + diff --git a/go.mod b/go.mod new file mode 100644 index 0000000..dda828e --- /dev/null +++ b/go.mod @@ -0,0 +1 @@ +module github.com/rhnvrm/simples3 diff --git a/go.sum b/go.sum new file mode 100644 index 0000000..e69de29 diff --git a/policy.go b/policy.go new file mode 100644 index 0000000..cdca498 --- /dev/null +++ b/policy.go @@ -0,0 +1,154 @@ +// LICENSE MIT +// Copyright (c) 2018, Rohan Verma +// Copyright (c) 2017, L Campbell +// forked from: https://github.com/lye/s3/ +// For previous license information visit +// https://github.com/lye/s3/blob/master/LICENSE + +package simples3 + +import ( + "crypto/hmac" + "crypto/sha256" + "encoding/base64" + "encoding/hex" + "encoding/json" + "fmt" + "time" +) + +// UploadConfig generate policies from config +type UploadConfig struct { + // Required + BucketName string + ObjectKey string + ContentType string + FileSize int64 + // Optional + UploadURL string + Expiration time.Duration + MetaData map[string]string +} + +type DeleteConfig struct { + // Required + BucketName string + ObjectKey string + ContentType string + // Optional + Expiration time.Duration +} + +// UploadPolicies Amazon s3 upload policies +type UploadPolicies struct { + URL string + Form map[string]string +} + +// PolicyJSON is policy rule +type PolicyJSON struct { + Expiration string `json:"expiration"` + Conditions []interface{} `json:"conditions"` +} + +const ( + expirationTimeFormat = "2006-01-02T15:04:05ZZ07:00" + amzDateISO8601TimeFormat = "20060102T150405Z" + shortTimeFormat = "20060102" + algorithm = "AWS4-HMAC-SHA256" + serviceName = "s3" + + defaultUploadURLFormat = "http://%s.s3.amazonaws.com/" // + defaultExpirationHour = 1 * time.Hour +) + +// NowTime mockable time.Now() +var NowTime = func() time.Time { + return time.Now() +} + +var lf = []byte{'\n'} + +// CreatePolicies create amazon s3 to upload policies return +// https://docs.aws.amazon.com/ja_jp/AmazonS3/latest/API/sigv4-authentication-HTTPPOST.html +func (s3 *S3) CreateUploadPolicies(uploadConfig UploadConfig) (UploadPolicies, error) { + nowTime := NowTime() + credential := s3.buildCredential(nowTime) + data, err := buildUploadSign(nowTime, credential, uploadConfig) + if err != nil { + return UploadPolicies{}, err + } + // 1. StringToSign + policy := base64.StdEncoding.EncodeToString(data) + // 2. Signing Key + hash := hmac.New(sha256.New, buildSignature(nowTime, s3.SecretKey, s3.Region, serviceName)) + hash.Write([]byte(policy)) + // 3. Signature + signature := hex.EncodeToString(hash.Sum(nil)) + + uploadURL := uploadConfig.UploadURL + if uploadURL == "" { + uploadURL = fmt.Sprintf(defaultUploadURLFormat, uploadConfig.BucketName) + } + form := map[string]string{ + "key": uploadConfig.ObjectKey, + "Content-Type": uploadConfig.ContentType, + "X-Amz-Credential": credential, + "X-Amz-Algorithm": algorithm, + "X-Amz-Date": nowTime.Format(amzDateISO8601TimeFormat), + "Policy": policy, + "X-Amz-Signature": signature, + } + for k, v := range uploadConfig.MetaData { + form[k] = v + } + return UploadPolicies{ + URL: uploadURL, + Form: form, + }, nil +} + +func buildUploadSign(nowTime time.Time, credential string, uploadConfig UploadConfig) ([]byte, error) { + conditions := []interface{}{ + map[string]string{"bucket": uploadConfig.BucketName}, + map[string]string{"key": uploadConfig.ObjectKey}, + map[string]string{"Content-Type": uploadConfig.ContentType}, + []interface{}{"content-length-range", uploadConfig.FileSize, uploadConfig.FileSize}, + map[string]string{"x-amz-credential": credential}, + map[string]string{"x-amz-algorithm": algorithm}, + map[string]string{"x-amz-date": nowTime.Format(amzDateISO8601TimeFormat)}, + } + for k, v := range uploadConfig.MetaData { + conditions = append(conditions, map[string]string{k: v}) + } + + expiration := defaultExpirationHour + if uploadConfig.Expiration > 0 { + expiration = uploadConfig.Expiration + } + + return json.Marshal(&PolicyJSON{ + Expiration: nowTime.Add(expiration).Format(expirationTimeFormat), + Conditions: conditions, + }) +} + +func (c S3) buildCredential(nowTime time.Time) string { + return fmt.Sprintf("%s/%s/%s/%s/%s", c.AccessKey, nowTime.UTC().Format(shortTimeFormat), c.Region, serviceName, "aws4_request") +} + +func buildSignature(nowTime time.Time, secretAccessKey string, regionName string, serviceName string) []byte { + shortTime := nowTime.UTC().Format(shortTimeFormat) + + date := makeHMac([]byte("AWS4"+secretAccessKey), []byte(shortTime)) + region := makeHMac(date, []byte(regionName)) + service := makeHMac(region, []byte(serviceName)) + credentials := makeHMac(service, []byte("aws4_request")) + return credentials +} + +func makeHMac(key []byte, data []byte) []byte { + hash := hmac.New(sha256.New, key) + hash.Write(data) + return hash.Sum(nil) +} diff --git a/sign.go b/sign.go new file mode 100644 index 0000000..aa858d1 --- /dev/null +++ b/sign.go @@ -0,0 +1,156 @@ +// LICENSE MIT +// Copyright (c) 2018, Rohan Verma +// Copyright (C) 2012 Blake Mizerany +// contains code from: github.com/bmizerany/aws4 + +package simples3 + +import ( + "bytes" + "crypto/hmac" + "crypto/sha256" + "fmt" + "io" + "io/ioutil" + "net/http" + "net/url" + "path/filepath" + "sort" + "strings" + "time" +) + +func (s3 *S3) signKeys(t time.Time) []byte { + h := ghmac([]byte("AWS4"+s3.SecretKey), []byte(t.Format(shortTimeFormat))) + h = ghmac(h, []byte(s3.Region)) + h = ghmac(h, []byte(serviceName)) + h = ghmac(h, []byte("aws4_request")) + return h +} + +func (s3 *S3) writeRequest(w io.Writer, r *http.Request) { + r.Header.Set("host", r.Host) + + w.Write([]byte(r.Method)) + w.Write(lf) + writeURI(w, r) + w.Write(lf) + writeQuery(w, r) + w.Write(lf) + writeHeader(w, r) + w.Write(lf) + w.Write(lf) + writeHeaderList(w, r) + w.Write(lf) + writeBody(w, r) +} + +func (s3 *S3) writeStringToSign(w io.Writer, t time.Time, r *http.Request) { + w.Write([]byte("AWS4-HMAC-SHA256")) + w.Write(lf) + w.Write([]byte(t.Format(amzDateISO8601TimeFormat))) + w.Write(lf) + + w.Write([]byte(s3.creds(t))) + w.Write(lf) + + h := sha256.New() + s3.writeRequest(h, r) + fmt.Fprintf(w, "%x", h.Sum(nil)) +} + +func (s3 *S3) creds(t time.Time) string { + return t.Format(shortTimeFormat) + "/" + s3.Region + "/" + serviceName + "/aws4_request" +} + +func writeURI(w io.Writer, r *http.Request) { + path := r.URL.RequestURI() + if r.URL.RawQuery != "" { + path = path[:len(path)-len(r.URL.RawQuery)-1] + } + slash := strings.HasSuffix(path, "/") + path = filepath.Clean(path) + if path != "/" && slash { + path += "/" + } + w.Write([]byte(path)) +} + +func writeQuery(w io.Writer, r *http.Request) { + var a []string + for k, vs := range r.URL.Query() { + k = url.QueryEscape(k) + for _, v := range vs { + if v == "" { + a = append(a, k) + } else { + v = url.QueryEscape(v) + a = append(a, k+"="+v) + } + } + } + sort.Strings(a) + for i, s := range a { + if i > 0 { + w.Write([]byte{'&'}) + } + w.Write([]byte(s)) + } +} + +func writeHeader(w io.Writer, r *http.Request) { + i, a := 0, make([]string, len(r.Header)) + for k, v := range r.Header { + sort.Strings(v) + a[i] = strings.ToLower(k) + ":" + strings.Join(v, ",") + i++ + } + sort.Strings(a) + for i, s := range a { + if i > 0 { + w.Write(lf) + } + io.WriteString(w, s) + } +} + +func writeHeaderList(w io.Writer, r *http.Request) { + i, a := 0, make([]string, len(r.Header)) + for k, _ := range r.Header { + a[i] = strings.ToLower(k) + i++ + } + sort.Strings(a) + for i, s := range a { + if i > 0 { + w.Write([]byte{';'}) + } + w.Write([]byte(s)) + } +} + +func writeBody(w io.Writer, r *http.Request) { + var b []byte + // If the payload is empty, use the empty string as the input to the SHA256 function + // http://docs.amazonwebservices.com/general/latest/gr/sigv4-create-canonical-request.html + if r.Body == nil { + b = []byte("") + } else { + var err error + b, err = ioutil.ReadAll(r.Body) + if err != nil { + panic(err) + } + r.Body = ioutil.NopCloser(bytes.NewBuffer(b)) + } + + h := sha256.New() + h.Write(b) + fmt.Fprintf(w, "%x", h.Sum(nil)) +} + +func ghmac(key, data []byte) []byte { + h := hmac.New(sha256.New, key) + h.Write(data) + return h.Sum(nil) +} diff --git a/simples3.go b/simples3.go new file mode 100644 index 0000000..59f2bee --- /dev/null +++ b/simples3.go @@ -0,0 +1,198 @@ +// LICENSE MIT +// Copyright (c) 2018, Rohan Verma + +package simples3 + +import ( + "bytes" + "crypto/hmac" + "crypto/sha256" + "fmt" + "io" + "io/ioutil" + "log" + "mime/multipart" + "net/http" + "net/http/httputil" + "time" +) + +// S3 provides a wrapper around your S3 credentials. +type S3 struct { + AccessKey string + SecretKey string + Region string + + URIFormat string +} + +type UploadInput struct { + Bucket string + ObjectKey string + FileName string + ContentType string + Body multipart.File +} + +type DeleteInput struct { + Bucket string + ObjectKey string +} + +func New(region, accessKey, secretKey string) *S3 { + return &S3{ + Region: region, + AccessKey: accessKey, + SecretKey: secretKey, + + URIFormat: "https://s3.%s.amazonaws.com/%s", + } +} + +func detectFileSize(body multipart.File) int64 { + switch r := body.(type) { + case io.Seeker: + pos, _ := r.Seek(0, 1) + defer r.Seek(pos, 0) + + n, err := r.Seek(0, 2) + if err != nil { + return -1 + } + return n + } + return -1 +} + +func (s3 *S3) FileUpload(u UploadInput) error { + policies, err := s3.CreateUploadPolicies(UploadConfig{ + UploadURL: fmt.Sprintf(s3.URIFormat, s3.Region, u.Bucket), + BucketName: u.Bucket, + ObjectKey: u.ObjectKey, + ContentType: u.ContentType, + FileSize: detectFileSize(u.Body), + MetaData: map[string]string{ + "success_action_status": "200", + }, + }) + + if err != nil { + return err + } + + var b bytes.Buffer + w := multipart.NewWriter(&b) + + for k, v := range policies.Form { + log.Println(k, v) + if err := w.WriteField(k, v); err != nil { + return err + } + } + + fw, err := w.CreateFormFile("file", u.FileName) + if err != nil { + return err + } + if _, err = io.Copy(fw, u.Body); err != nil { + return err + } + + // Don't forget to close the multipart writer. + // If you don't close it, your request will be missing the terminating boundary. + if err := w.Close(); err != nil { + return err + } + + // Now that you have a form, you can submit it to your handler. + req, err := http.NewRequest("POST", policies.URL, &b) + if err != nil { + return err + } + // Don't forget to set the content type, this will contain the boundary. + req.Header.Set("Content-Type", w.FormDataContentType()) + + // Submit the request + client := &http.Client{} + res, err := client.Do(req) + if err != nil { + return err + } + + // Check the response + if res.Status != "200 OK" { + err = fmt.Errorf("status code: %s", res.Status) + data, _ := ioutil.ReadAll(res.Body) + log.Printf("response:%s\n", string(data)) + return err + } + + return nil +} + +func (s3 *S3) FileDelete(u DeleteInput) error { + url := fmt.Sprintf( + s3.URIFormat+"/%s", + s3.Region, u.Bucket, u.ObjectKey, + ) + log.Println(url) + req, err := http.NewRequest( + "DELETE", url, nil, + ) + if err != nil { + return err + } + + date := req.Header.Get("Date") + t := time.Now().UTC() + if date != "" { + var err error + t, err = time.Parse(http.TimeFormat, date) + if err != nil { + return err + } + } + req.Header.Set("Date", t.Format(amzDateISO8601TimeFormat)) + + // The x-amz-content-sha256 header is required for all AWS + // Signature Version 4 requests. It provides a hash of the + // request payload. If there is no payload, you must provide + // the hash of an empty string. + emptyhash := "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855" + req.Header.Set("x-amz-content-sha256", emptyhash) + + k := s3.signKeys(t) + h := hmac.New(sha256.New, k) + + s3.writeStringToSign(h, t, req) + + auth := bytes.NewBufferString("AWS4-HMAC-SHA256 ") + auth.Write([]byte("Credential=" + s3.AccessKey + "/" + s3.creds(t))) + auth.Write([]byte{',', ' '}) + auth.Write([]byte("SignedHeaders=")) + writeHeaderList(auth, req) + auth.Write([]byte{',', ' '}) + auth.Write([]byte("Signature=" + fmt.Sprintf("%x", h.Sum(nil)))) + + req.Header.Set("Authorization", auth.String()) + + dump, err := httputil.DumpRequest(req, true) + log.Printf("%q", dump) + + // Submit the request + client := &http.Client{} + res, err := client.Do(req) + if err != nil { + return err + } + + // // Check the response + if res.Status != "204 No Content" { + err = fmt.Errorf("status code: %s", res.Status) + data, _ := ioutil.ReadAll(res.Body) + log.Printf("response:%s\n", string(data)) + return err + } + + return nil +} diff --git a/simples3_test.go b/simples3_test.go new file mode 100644 index 0000000..40f10ae --- /dev/null +++ b/simples3_test.go @@ -0,0 +1,97 @@ +package simples3 + +import ( + "os" + "testing" +) + +func TestS3_FileUpload(t *testing.T) { + testTxt, err := os.Open("test.txt") + if err != nil { + return + } + defer testTxt.Close() + + type fields struct { + AccessKey string + SecretKey string + Region string + } + type args struct { + u UploadInput + } + tests := []struct { + name string + fields fields + args args + wantErr bool + }{ + { + name: "Upload test.txt", + fields: fields{ + AccessKey: os.Getenv("AWS_S3_ACCESS_KEY"), + SecretKey: os.Getenv("AWS_S3_SECRET_KEY"), + Region: os.Getenv("AWS_S3_REGION"), + }, + args: args{ + UploadInput{ + Bucket: "zerodha-testbucket", + ObjectKey: "test.txt", + ContentType: "text/plain", + FileName: "test.txt", + Body: testTxt, + }, + }, + wantErr: false, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + s3 := New(tt.fields.Region, tt.fields.AccessKey, tt.fields.SecretKey) + if err := s3.FileUpload(tt.args.u); (err != nil) != tt.wantErr { + t.Errorf("S3.FileUpload() error = %v, wantErr %v", err, tt.wantErr) + } + }) + } +} + +func TestS3_FileDelete(t *testing.T) { + type fields struct { + AccessKey string + SecretKey string + Region string + } + type args struct { + u DeleteInput + } + tests := []struct { + name string + fields fields + args args + wantErr bool + }{ + { + name: "Delete test.txt", + fields: fields{ + AccessKey: os.Getenv("MONGO_PASS"), + SecretKey: os.Getenv("AWS_S3_ACCESS_KEY"), + Region: os.Getenv("AWS_S3_ACCESS_KEY"), + }, + args: args{ + DeleteInput{ + Bucket: "zerodha-testbucket", + ObjectKey: "test.txt", + }, + }, + wantErr: false, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + s3 := New(tt.fields.Region, tt.fields.AccessKey, tt.fields.SecretKey) + if err := s3.FileDelete(tt.args.u); (err != nil) != tt.wantErr { + t.Errorf("S3.FileDelete() error = %v, wantErr %v", err, tt.wantErr) + } + }) + } +} diff --git a/test.txt b/test.txt new file mode 100644 index 0000000..95d09f2 --- /dev/null +++ b/test.txt @@ -0,0 +1 @@ +hello world \ No newline at end of file