eiac/simples3
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Update readme and docstrings, minor refactors.

Browse Source
This commit is contained in:
Rohan Verma
2018-12-24 17:31:36 +05:30
parent d218b23f05
commit e4c7d93da1
4 changed files with 150 additions and 30 deletions
Download Patch File Download Diff File Expand all files Collapse all files
README.md
+31 -7
View File
@@ -2,7 +2,7 @@
## Overview [![GoDoc](https://godoc.org/github.com/rhnvrm/simples3?status.svg)](https://godoc.org/github.com/rhnvrm/simples3) [![Go Report Card](https://goreportcard.com/badge/github.com/rhnvrm/simples3)](https://goreportcard.com/report/github.com/rhnvrm/simples3)
SimpleS3 is a golang library for uploading and deleting objects on S3 buckets using the REST API calls signed using AWS Signature Version 4
SimpleS3 is a golang library for uploading and deleting objects on S3 buckets using REST API calls or Presigned URLs signed using AWS Signature Version 4.
## Install
@@ -16,7 +16,16 @@ go get github.com/rhnvrm/simples3
testTxt, _ := os.Open("test.txt")
defer testTxt.Close()
// Create an instance of the package
// You can either create by manually supplying credentials
// (preferably using Environment vars)
s3 := simples3.New(Region, AWSAccessKey, AWSSecretKey)
// or you can use this on an EC2 instance to
// obtain credentials from IAM attached to the instance.
s3 := simples3.NewUsingIAM(Region)
// File Upload is as simple as providing the following
// details.
err := s3.FileUpload(simples3.UploadInput{
Bucket: AWSBucket,
ObjectKey: "test.txt",
@@ -24,19 +33,34 @@ err := s3.FileUpload(simples3.UploadInput{
FileName: "test.txt",
Body: testTxt,
}
// Similarly, Files can be deleted.
err := s3.FileDelete(simples3.DeleteInput{
Bucket: os.Getenv("AWS_S3_BUCKET"),
ObjectKey: "test.txt",
}
// You can also use this library to generate
// Presigned URLs that can for eg. be used to
// GET/PUT files on S3 through the browser.
var time, _ = time.Parse(time.RFC1123, "Fri, 24 May 2013 00:00:00 GMT")
url := s.GeneratePresignedURL(PresignedInput{
Bucket: "examplebucket",
ObjectKey: "test.txt",
Method: "GET",
Timestamp: time,
ExpirySeconds: 86400,
})
```
## Contributing
ToDo.
## Contributors
ToDo.
You are more than welcome to contribute to this project. Fork and make a Pull Request, or create an Issue if you see any problem or want to propose a feature.
## Author
ToDo.
Rohan Verma <hello@rohanverma.net>
## License
presigned.go
+34 -21
View File
@@ -16,6 +16,7 @@ const (
defaultPresignedURLFormat = "%s.s3.amazonaws.com" // <bucket>
)
// PresignedInput is passed to GeneratePresignedURL as a parameter.
type PresignedInput struct {
Bucket string
ObjectKey string
@@ -25,10 +26,13 @@ type PresignedInput struct {
ExpirySeconds int
}
// GeneratePresignedURL creates a Presigned URL that can be used
// for Authentication using Query Parameters.
// (https://docs.aws.amazon.com/AmazonS3/latest/API/sigv4-query-string-auth.html)
func (s3 *S3) GeneratePresignedURL(in PresignedInput) string {
var (
nowTime = in.Timestamp.UTC()
cred = s3.buildCredential(nowTime)
cred = fmt.Sprintf("%s/"+s3.buildCredentialWithoutKey(nowTime), s3.AccessKey)
amzdate = nowTime.Format(amzDateISO8601TimeFormat)
expiry = strconv.Itoa(in.ExpirySeconds)
)
@@ -41,7 +45,7 @@ func (s3 *S3) GeneratePresignedURL(in PresignedInput) string {
signedHeaders["host"] = fmt.Sprintf(defaultPresignedURLFormat, in.Bucket)
// Start Canonical Request Formation
h := sha256.New()
h := sha256.New() // We write the canonical request directly to the SHA256 hash.
fmt.Fprintf(h, "%s\n", in.Method) // HTTP Verb
fmt.Fprintf(h, "/%s\n", in.ObjectKey) // CanonicalURL
@@ -54,44 +58,53 @@ func (s3 *S3) GeneratePresignedURL(in PresignedInput) string {
}
// We need to have a sorted order,
keysQS := make([]string, 0, len(queryString))
// for QueryStrings and SignedHeaders
sortedQS := make([]string, 0, len(queryString))
for name := range queryString {
keysQS = append(keysQS, name)
sortedQS = append(sortedQS, name)
}
sort.Strings(keysQS) //sort by key
for _, k := range keysQS {
sort.Strings(sortedQS) //sort by key
sortedSH := make([]string, 0, len(signedHeaders))
for name := range signedHeaders {
sortedSH = append(sortedSH, name)
}
sort.Strings(sortedSH) //sort by key
// Proceed to write canonical query params
for _, k := range sortedQS {
fmt.Fprintf(h, "%s=%s&", url.QueryEscape(k), url.QueryEscape(queryString[k])) // HTTP Verb
}
fmt.Fprintf(h, "X-Amz-SignedHeaders=")
// Add Signed Headers to Query String
first := true
for k := range signedHeaders {
for k := range sortedSH {
if first {
fmt.Fprintf(h, "%s", url.QueryEscape(k))
fmt.Fprintf(h, "%s", url.QueryEscape(sortedSH[k]))
first = false
} else {
fmt.Fprintf(h, "%s", url.QueryEscape(";"+k))
fmt.Fprintf(h, "%s", url.QueryEscape(";"+sortedSH[k]))
}
}
fmt.Fprintf(h, "\n")
// End QueryString Params
// Start Canonical Headers
for k, v := range signedHeaders {
fmt.Fprintf(h, "%s:%s\n", strings.ToLower(k), strings.TrimSpace(v))
for _, k := range sortedSH {
fmt.Fprintf(h, "%s:%s\n", strings.ToLower(k), strings.TrimSpace(signedHeaders[k]))
}
fmt.Fprintf(h, "\n")
// End Canonical Headers
// Start Signed Headers
first = true
for k := range signedHeaders {
for k := range sortedSH {
if first {
fmt.Fprintf(h, "%s", url.QueryEscape(k))
fmt.Fprintf(h, "%s", url.QueryEscape(sortedSH[k]))
first = false
} else {
fmt.Fprintf(h, "%s", url.QueryEscape(";"+k))
fmt.Fprintf(h, "%s", url.QueryEscape(";"+sortedSH[k]))
}
}
fmt.Fprintf(h, "\n")
@@ -126,7 +139,8 @@ func (s3 *S3) GeneratePresignedURL(in PresignedInput) string {
[]byte("s3")),
[]byte("aws4_request"),
)
// sigKey gen verified using https://docs.aws.amazon.com/general/latest/gr/signature-v4-examples.html#signature-v4-examples-other
// sigKey gen verified using
// https://docs.aws.amazon.com/general/latest/gr/signature-v4-examples.html#signature-v4-examples-other
// (TODO: add a test using the same, consolidate with signKeys())
signedStrToSign := makeHMac(sigKey, stringToSign)
@@ -142,21 +156,20 @@ func (s3 *S3) GeneratePresignedURL(in PresignedInput) string {
// We don't need to have a sorted order here,
// but just to preserve tests.
for _, k := range keysQS {
for _, k := range sortedQS {
fmt.Fprintf(&b, "%s=%s&", url.QueryEscape(k), url.QueryEscape(queryString[k])) // HTTP Verb
}
fmt.Fprintf(&b, "%s=", "X-Amz-SignedHeaders")
first = true
for k := range signedHeaders {
for k := range sortedSH {
if first {
fmt.Fprintf(&b, "%s", url.QueryEscape(k))
fmt.Fprintf(&b, "%s", url.QueryEscape(sortedSH[k]))
first = false
} else {
fmt.Fprintf(&b, "%s", url.QueryEscape(";"+k))
fmt.Fprintf(&b, "%s", url.QueryEscape(";"+sortedSH[k]))
}
}
fmt.Fprintf(&b, "&X-Amz-Signature=%s", signature)
resultURL := b.String()
return resultURL
return b.String()
}
presigned_test.go
+19 -1
View File
@@ -49,7 +49,7 @@ func TestS3_GeneratePresignedURL_Personal(t *testing.T) {
})
}
func TestS3_GeneratePresignedURL_POST(t *testing.T) {
func TestS3_GeneratePresignedURL_PUT(t *testing.T) {
t.Run("Test", func(t *testing.T) {
s := New(
os.Getenv("AWS_S3_REGION"),
@@ -68,3 +68,21 @@ func TestS3_GeneratePresignedURL_POST(t *testing.T) {
}
})
}
func BenchmarkS3_GeneratePresigned(b *testing.B) {
// run the Fib function b.N times
s := New(
os.Getenv("AWS_S3_REGION"),
os.Getenv("AWS_S3_ACCESS_KEY"),
os.Getenv("AWS_S3_SECRET_KEY"),
)
for n := 0; n < b.N; n++ {
s.GeneratePresignedURL(PresignedInput{
Bucket: os.Getenv("AWS_S3_BUCKET"),
ObjectKey: "test.txt",
Method: "GET",
Timestamp: NowTime(),
ExpirySeconds: 3600,
})
}
}
simples3.go
+66 -1
View File
@@ -7,6 +7,8 @@ import (
"bytes"
"crypto/hmac"
"crypto/sha256"
"encoding/json"
"errors"
"fmt"
"io"
"io/ioutil"
@@ -16,6 +18,10 @@ import (
"time"
)
const (
securityCredentialsURL = "http://169.254.169.254/latest/meta-data/iam/security-credentials/"
)
// S3 provides a wrapper around your S3 credentials.
type S3 struct {
AccessKey string
@@ -40,6 +46,18 @@ type DeleteInput struct {
ObjectKey string
}
// IAMResponse is used by NewUsingIAM to auto
// detect the credentials
type IAMResponse struct {
Code string `json:"Code"`
LastUpdated string `json:"LastUpdated"`
Type string `json:"Type"`
AccessKeyID string `json:"AccessKeyId"`
SecretAccessKey string `json:"SecretAccessKey"`
Token string `json:"Token"`
Expiration string `json:"Expiration"`
}
// New returns an instance of S3.
func New(region, accessKey, secretKey string) *S3 {
return &S3{
@@ -51,6 +69,53 @@ func New(region, accessKey, secretKey string) *S3 {
}
}
// NewUsingIAM automatically generates an Instance of S3
// using instance metatdata.
func NewUsingIAM(region string) (*S3, error) {
// Get the IAM role
resp, err := http.Get(securityCredentialsURL)
if err != nil {
return nil, err
}
defer resp.Body.Close()
if resp.StatusCode != 200 {
return nil, errors.New(http.StatusText(resp.StatusCode))
}
role, err := ioutil.ReadAll(resp.Body)
if err != nil {
return nil, err
}
resp, err = http.Get(securityCredentialsURL + "/" + string(role))
if err != nil {
return nil, err
}
defer resp.Body.Close()
if resp.StatusCode != 200 {
return nil, errors.New(http.StatusText(resp.StatusCode))
}
var jsonResp IAMResponse
jsonString, err := ioutil.ReadAll(resp.Body)
if err != nil {
return nil, err
}
if err := json.Unmarshal(jsonString, &jsonResp); err != nil {
return nil, err
}
return &S3{
Region: region,
AccessKey: jsonResp.AccessKeyID,
SecretKey: jsonResp.SecretAccessKey,
URIFormat: "https://s3.%s.amazonaws.com/%s",
}, nil
}
func detectFileSize(body multipart.File) int64 {
switch r := body.(type) {
case io.Seeker:
@@ -133,7 +198,7 @@ func (s3 *S3) FileUpload(u UploadInput) error {
return nil
}
// FileDelete makes a POST call with the file written as multipart
// FileDelete makes a DELETE call with the file written as multipart
// and on successful upload, checks for 204 No Content.
func (s3 *S3) FileDelete(u DeleteInput) error {
url := fmt.Sprintf(