eiac/simples3
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

init

Browse Source
This commit is contained in:
Rohan Verma
2018-12-06 15:54:47 +05:30
commit f30c5f9052
9 changed files with 677 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files
LICENSE
+26
View File
@@ -0,0 +1,26 @@
Copyright (c) 2018, Rohan Verma <hello@rohanverma.net>
All rights reserved.
Redistribution and use in source and binary forms, with or without
modification, are permitted provided that the following conditions are met:
1. Redistributions of source code must retain the above copyright notice, this
list of conditions and the following disclaimer.
2. Redistributions in binary form must reproduce the above copyright notice,
this list of conditions and the following disclaimer in the documentation
and/or other materials provided with the distribution.
THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR
ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
The views and conclusions contained in the software and documentation are those
of the authors and should not be interpreted as representing official policies,
either expressed or implied.
README.md
+44
View File
@@ -0,0 +1,44 @@
# simples3 : Simple no frills AWS S3 Library using REST with V4 Signing
## Overview [![GoDoc](https://godoc.org/github.com/rhnvrm/simples3?status.svg)](https://godoc.org/github.com/rhnvrm/simples3) [![Go Report Card](https://goreportcard.com/badge/github.com/rhnvrm/simples3)](https://goreportcard.com/report/github.com/rhnvrm/simples3)
SimpleS3 is a golang library for uploading and deleting objects on S3 buckets using the REST API calls signed using AWS Signature Version 4
## Install
```
go get github.com/rhnvrm/simples3
```
## Example
```
testTxt, _ := os.Open("test.txt")
defer testTxt.Close()
s3 := simples3.New(Region, AWSAccessKey, AWSSecretKey)
err := s3.FileUpload(simples3.UploadInput{
Bucket: "zerodha-testbucket",
ObjectKey: "test.txt",
ContentType: "text/plain",
FileName: "test.txt",
Body: testTxt,
}
```
## Contributing
ToDo.
## Contributors
ToDo.
## Author
ToDo.
## License
MIT.
go.mod
+1
View File
@@ -0,0 +1 @@
module github.com/rhnvrm/simples3
go.sum
View File
policy.go
+154
View File
@@ -0,0 +1,154 @@
// LICENSE MIT
// Copyright (c) 2018, Rohan Verma <hello@rohanverma.net>
// Copyright (c) 2017, L Campbell
// forked from: https://github.com/lye/s3/
// For previous license information visit
// https://github.com/lye/s3/blob/master/LICENSE
package simples3
import (
"crypto/hmac"
"crypto/sha256"
"encoding/base64"
"encoding/hex"
"encoding/json"
"fmt"
"time"
)
// UploadConfig generate policies from config
type UploadConfig struct {
// Required
BucketName string
ObjectKey string
ContentType string
FileSize int64
// Optional
UploadURL string
Expiration time.Duration
MetaData map[string]string
}
type DeleteConfig struct {
// Required
BucketName string
ObjectKey string
ContentType string
// Optional
Expiration time.Duration
}
// UploadPolicies Amazon s3 upload policies
type UploadPolicies struct {
URL string
Form map[string]string
}
// PolicyJSON is policy rule
type PolicyJSON struct {
Expiration string `json:"expiration"`
Conditions []interface{} `json:"conditions"`
}
const (
expirationTimeFormat = "2006-01-02T15:04:05ZZ07:00"
amzDateISO8601TimeFormat = "20060102T150405Z"
shortTimeFormat = "20060102"
algorithm = "AWS4-HMAC-SHA256"
serviceName = "s3"
defaultUploadURLFormat = "http://%s.s3.amazonaws.com/" // <bucketName>
defaultExpirationHour = 1 * time.Hour
)
// NowTime mockable time.Now()
var NowTime = func() time.Time {
return time.Now()
}
var lf = []byte{'\n'}
// CreatePolicies create amazon s3 to upload policies return
// https://docs.aws.amazon.com/ja_jp/AmazonS3/latest/API/sigv4-authentication-HTTPPOST.html
func (s3 *S3) CreateUploadPolicies(uploadConfig UploadConfig) (UploadPolicies, error) {
nowTime := NowTime()
credential := s3.buildCredential(nowTime)
data, err := buildUploadSign(nowTime, credential, uploadConfig)
if err != nil {
return UploadPolicies{}, err
}
// 1. StringToSign
policy := base64.StdEncoding.EncodeToString(data)
// 2. Signing Key
hash := hmac.New(sha256.New, buildSignature(nowTime, s3.SecretKey, s3.Region, serviceName))
hash.Write([]byte(policy))
// 3. Signature
signature := hex.EncodeToString(hash.Sum(nil))
uploadURL := uploadConfig.UploadURL
if uploadURL == "" {
uploadURL = fmt.Sprintf(defaultUploadURLFormat, uploadConfig.BucketName)
}
form := map[string]string{
"key": uploadConfig.ObjectKey,
"Content-Type": uploadConfig.ContentType,
"X-Amz-Credential": credential,
"X-Amz-Algorithm": algorithm,
"X-Amz-Date": nowTime.Format(amzDateISO8601TimeFormat),
"Policy": policy,
"X-Amz-Signature": signature,
}
for k, v := range uploadConfig.MetaData {
form[k] = v
}
return UploadPolicies{
URL: uploadURL,
Form: form,
}, nil
}
func buildUploadSign(nowTime time.Time, credential string, uploadConfig UploadConfig) ([]byte, error) {
conditions := []interface{}{
map[string]string{"bucket": uploadConfig.BucketName},
map[string]string{"key": uploadConfig.ObjectKey},
map[string]string{"Content-Type": uploadConfig.ContentType},
[]interface{}{"content-length-range", uploadConfig.FileSize, uploadConfig.FileSize},
map[string]string{"x-amz-credential": credential},
map[string]string{"x-amz-algorithm": algorithm},
map[string]string{"x-amz-date": nowTime.Format(amzDateISO8601TimeFormat)},
}
for k, v := range uploadConfig.MetaData {
conditions = append(conditions, map[string]string{k: v})
}
expiration := defaultExpirationHour
if uploadConfig.Expiration > 0 {
expiration = uploadConfig.Expiration
}
return json.Marshal(&PolicyJSON{
Expiration: nowTime.Add(expiration).Format(expirationTimeFormat),
Conditions: conditions,
})
}
func (c S3) buildCredential(nowTime time.Time) string {
return fmt.Sprintf("%s/%s/%s/%s/%s", c.AccessKey, nowTime.UTC().Format(shortTimeFormat), c.Region, serviceName, "aws4_request")
}
func buildSignature(nowTime time.Time, secretAccessKey string, regionName string, serviceName string) []byte {
shortTime := nowTime.UTC().Format(shortTimeFormat)
date := makeHMac([]byte("AWS4"+secretAccessKey), []byte(shortTime))
region := makeHMac(date, []byte(regionName))
service := makeHMac(region, []byte(serviceName))
credentials := makeHMac(service, []byte("aws4_request"))
return credentials
}
func makeHMac(key []byte, data []byte) []byte {
hash := hmac.New(sha256.New, key)
hash.Write(data)
return hash.Sum(nil)
}
sign.go
+156
View File
@@ -0,0 +1,156 @@
// LICENSE MIT
// Copyright (c) 2018, Rohan Verma <hello@rohanverma.net>
// Copyright (C) 2012 Blake Mizerany
// contains code from: github.com/bmizerany/aws4
package simples3
import (
"bytes"
"crypto/hmac"
"crypto/sha256"
"fmt"
"io"
"io/ioutil"
"net/http"
"net/url"
"path/filepath"
"sort"
"strings"
"time"
)
func (s3 *S3) signKeys(t time.Time) []byte {
h := ghmac([]byte("AWS4"+s3.SecretKey), []byte(t.Format(shortTimeFormat)))
h = ghmac(h, []byte(s3.Region))
h = ghmac(h, []byte(serviceName))
h = ghmac(h, []byte("aws4_request"))
return h
}
func (s3 *S3) writeRequest(w io.Writer, r *http.Request) {
r.Header.Set("host", r.Host)
w.Write([]byte(r.Method))
w.Write(lf)
writeURI(w, r)
w.Write(lf)
writeQuery(w, r)
w.Write(lf)
writeHeader(w, r)
w.Write(lf)
w.Write(lf)
writeHeaderList(w, r)
w.Write(lf)
writeBody(w, r)
}
func (s3 *S3) writeStringToSign(w io.Writer, t time.Time, r *http.Request) {
w.Write([]byte("AWS4-HMAC-SHA256"))
w.Write(lf)
w.Write([]byte(t.Format(amzDateISO8601TimeFormat)))
w.Write(lf)
w.Write([]byte(s3.creds(t)))
w.Write(lf)
h := sha256.New()
s3.writeRequest(h, r)
fmt.Fprintf(w, "%x", h.Sum(nil))
}
func (s3 *S3) creds(t time.Time) string {
return t.Format(shortTimeFormat) + "/" + s3.Region + "/" + serviceName + "/aws4_request"
}
func writeURI(w io.Writer, r *http.Request) {
path := r.URL.RequestURI()
if r.URL.RawQuery != "" {
path = path[:len(path)-len(r.URL.RawQuery)-1]
}
slash := strings.HasSuffix(path, "/")
path = filepath.Clean(path)
if path != "/" && slash {
path += "/"
}
w.Write([]byte(path))
}
func writeQuery(w io.Writer, r *http.Request) {
var a []string
for k, vs := range r.URL.Query() {
k = url.QueryEscape(k)
for _, v := range vs {
if v == "" {
a = append(a, k)
} else {
v = url.QueryEscape(v)
a = append(a, k+"="+v)
}
}
}
sort.Strings(a)
for i, s := range a {
if i > 0 {
w.Write([]byte{'&'})
}
w.Write([]byte(s))
}
}
func writeHeader(w io.Writer, r *http.Request) {
i, a := 0, make([]string, len(r.Header))
for k, v := range r.Header {
sort.Strings(v)
a[i] = strings.ToLower(k) + ":" + strings.Join(v, ",")
i++
}
sort.Strings(a)
for i, s := range a {
if i > 0 {
w.Write(lf)
}
io.WriteString(w, s)
}
}
func writeHeaderList(w io.Writer, r *http.Request) {
i, a := 0, make([]string, len(r.Header))
for k, _ := range r.Header {
a[i] = strings.ToLower(k)
i++
}
sort.Strings(a)
for i, s := range a {
if i > 0 {
w.Write([]byte{';'})
}
w.Write([]byte(s))
}
}
func writeBody(w io.Writer, r *http.Request) {
var b []byte
// If the payload is empty, use the empty string as the input to the SHA256 function
// http://docs.amazonwebservices.com/general/latest/gr/sigv4-create-canonical-request.html
if r.Body == nil {
b = []byte("")
} else {
var err error
b, err = ioutil.ReadAll(r.Body)
if err != nil {
panic(err)
}
r.Body = ioutil.NopCloser(bytes.NewBuffer(b))
}
h := sha256.New()
h.Write(b)
fmt.Fprintf(w, "%x", h.Sum(nil))
}
func ghmac(key, data []byte) []byte {
h := hmac.New(sha256.New, key)
h.Write(data)
return h.Sum(nil)
}
simples3.go
+198
View File
@@ -0,0 +1,198 @@
// LICENSE MIT
// Copyright (c) 2018, Rohan Verma <hello@rohanverma.net>
package simples3
import (
"bytes"
"crypto/hmac"
"crypto/sha256"
"fmt"
"io"
"io/ioutil"
"log"
"mime/multipart"
"net/http"
"net/http/httputil"
"time"
)
// S3 provides a wrapper around your S3 credentials.
type S3 struct {
AccessKey string
SecretKey string
Region string
URIFormat string
}
type UploadInput struct {
Bucket string
ObjectKey string
FileName string
ContentType string
Body multipart.File
}
type DeleteInput struct {
Bucket string
ObjectKey string
}
func New(region, accessKey, secretKey string) *S3 {
return &S3{
Region: region,
AccessKey: accessKey,
SecretKey: secretKey,
URIFormat: "https://s3.%s.amazonaws.com/%s",
}
}
func detectFileSize(body multipart.File) int64 {
switch r := body.(type) {
case io.Seeker:
pos, _ := r.Seek(0, 1)
defer r.Seek(pos, 0)
n, err := r.Seek(0, 2)
if err != nil {
return -1
}
return n
}
return -1
}
func (s3 *S3) FileUpload(u UploadInput) error {
policies, err := s3.CreateUploadPolicies(UploadConfig{
UploadURL: fmt.Sprintf(s3.URIFormat, s3.Region, u.Bucket),
BucketName: u.Bucket,
ObjectKey: u.ObjectKey,
ContentType: u.ContentType,
FileSize: detectFileSize(u.Body),
MetaData: map[string]string{
"success_action_status": "200",
},
})
if err != nil {
return err
}
var b bytes.Buffer
w := multipart.NewWriter(&b)
for k, v := range policies.Form {
log.Println(k, v)
if err := w.WriteField(k, v); err != nil {
return err
}
}
fw, err := w.CreateFormFile("file", u.FileName)
if err != nil {
return err
}
if _, err = io.Copy(fw, u.Body); err != nil {
return err
}
// Don't forget to close the multipart writer.
// If you don't close it, your request will be missing the terminating boundary.
if err := w.Close(); err != nil {
return err
}
// Now that you have a form, you can submit it to your handler.
req, err := http.NewRequest("POST", policies.URL, &b)
if err != nil {
return err
}
// Don't forget to set the content type, this will contain the boundary.
req.Header.Set("Content-Type", w.FormDataContentType())
// Submit the request
client := &http.Client{}
res, err := client.Do(req)
if err != nil {
return err
}
// Check the response
if res.Status != "200 OK" {
err = fmt.Errorf("status code: %s", res.Status)
data, _ := ioutil.ReadAll(res.Body)
log.Printf("response:%s\n", string(data))
return err
}
return nil
}
func (s3 *S3) FileDelete(u DeleteInput) error {
url := fmt.Sprintf(
s3.URIFormat+"/%s",
s3.Region, u.Bucket, u.ObjectKey,
)
log.Println(url)
req, err := http.NewRequest(
"DELETE", url, nil,
)
if err != nil {
return err
}
date := req.Header.Get("Date")
t := time.Now().UTC()
if date != "" {
var err error
t, err = time.Parse(http.TimeFormat, date)
if err != nil {
return err
}
}
req.Header.Set("Date", t.Format(amzDateISO8601TimeFormat))
// The x-amz-content-sha256 header is required for all AWS
// Signature Version 4 requests. It provides a hash of the
// request payload. If there is no payload, you must provide
// the hash of an empty string.
emptyhash := "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855"
req.Header.Set("x-amz-content-sha256", emptyhash)
k := s3.signKeys(t)
h := hmac.New(sha256.New, k)
s3.writeStringToSign(h, t, req)
auth := bytes.NewBufferString("AWS4-HMAC-SHA256 ")
auth.Write([]byte("Credential=" + s3.AccessKey + "/" + s3.creds(t)))
auth.Write([]byte{',', ' '})
auth.Write([]byte("SignedHeaders="))
writeHeaderList(auth, req)
auth.Write([]byte{',', ' '})
auth.Write([]byte("Signature=" + fmt.Sprintf("%x", h.Sum(nil))))
req.Header.Set("Authorization", auth.String())
dump, err := httputil.DumpRequest(req, true)
log.Printf("%q", dump)
// Submit the request
client := &http.Client{}
res, err := client.Do(req)
if err != nil {
return err
}
// // Check the response
if res.Status != "204 No Content" {
err = fmt.Errorf("status code: %s", res.Status)
data, _ := ioutil.ReadAll(res.Body)
log.Printf("response:%s\n", string(data))
return err
}
return nil
}
simples3_test.go
+97
View File
@@ -0,0 +1,97 @@
package simples3
import (
"os"
"testing"
)
func TestS3_FileUpload(t *testing.T) {
testTxt, err := os.Open("test.txt")
if err != nil {
return
}
defer testTxt.Close()
type fields struct {
AccessKey string
SecretKey string
Region string
}
type args struct {
u UploadInput
}
tests := []struct {
name string
fields fields
args args
wantErr bool
}{
{
name: "Upload test.txt",
fields: fields{
AccessKey: os.Getenv("AWS_S3_ACCESS_KEY"),
SecretKey: os.Getenv("AWS_S3_SECRET_KEY"),
Region: os.Getenv("AWS_S3_REGION"),
},
args: args{
UploadInput{
Bucket: "zerodha-testbucket",
ObjectKey: "test.txt",
ContentType: "text/plain",
FileName: "test.txt",
Body: testTxt,
},
},
wantErr: false,
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
s3 := New(tt.fields.Region, tt.fields.AccessKey, tt.fields.SecretKey)
if err := s3.FileUpload(tt.args.u); (err != nil) != tt.wantErr {
t.Errorf("S3.FileUpload() error = %v, wantErr %v", err, tt.wantErr)
}
})
}
}
func TestS3_FileDelete(t *testing.T) {
type fields struct {
AccessKey string
SecretKey string
Region string
}
type args struct {
u DeleteInput
}
tests := []struct {
name string
fields fields
args args
wantErr bool
}{
{
name: "Delete test.txt",
fields: fields{
AccessKey: os.Getenv("MONGO_PASS"),
SecretKey: os.Getenv("AWS_S3_ACCESS_KEY"),
Region: os.Getenv("AWS_S3_ACCESS_KEY"),
},
args: args{
DeleteInput{
Bucket: "zerodha-testbucket",
ObjectKey: "test.txt",
},
},
wantErr: false,
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
s3 := New(tt.fields.Region, tt.fields.AccessKey, tt.fields.SecretKey)
if err := s3.FileDelete(tt.args.u); (err != nil) != tt.wantErr {
t.Errorf("S3.FileDelete() error = %v, wantErr %v", err, tt.wantErr)
}
})
}
}
test.txt
+1
View File
@@ -0,0 +1 @@
hello world