eiac/simples3
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Add Endpoint for some compatible instance and fix time format (#3)

Browse Source 
* Add Endpoint for some compatible instance.

* Fix: InvalidExpiration
* Fix: RequestTimeTooSkewed

Compatible s3: https://developer.qiniu.com/kodo/manual/4088/s3-access-domainname

* - clean code

* Fix: InvalidExpiration
This commit is contained in:
Liut
2019-12-11 07:53:05 +00:00
committed by Rohan Verma
parent 3d5b071ef7
commit 9a5e82a977
6 changed files with 72 additions and 37 deletions
Download Patch File Download Diff File Expand all files Collapse all files
.gitignore
+3 -1
View File
@@ -21,4 +21,6 @@
.settings/
.vscode/
tsconfig.json
jsconfig.json
jsconfig.json
.env
policy.go
+5 -5
View File
@@ -45,7 +45,7 @@ type PolicyJSON struct {
}
const (
expirationTimeFormat = "2006-01-02T15:04:05ZZ07:00"
expirationTimeFormat = "2006-01-02T15:04:05Z07:00"
amzDateISO8601TimeFormat = "20060102T150405Z"
shortTimeFormat = "20060102"
algorithm = "AWS4-HMAC-SHA256"
@@ -57,7 +57,7 @@ const (
// NowTime mockable time.Now()
var NowTime = func() time.Time {
return time.Now()
return time.Now().UTC()
}
var newLine = []byte{'\n'}
@@ -131,7 +131,7 @@ func (s3 S3) buildCredential(nowTime time.Time) []byte {
var b bytes.Buffer
b.WriteString(s3.AccessKey)
b.WriteRune('/')
b.WriteString(nowTime.UTC().Format(shortTimeFormat))
b.WriteString(nowTime.Format(shortTimeFormat))
b.WriteRune('/')
b.WriteString(s3.Region)
b.WriteRune('/')
@@ -143,7 +143,7 @@ func (s3 S3) buildCredential(nowTime time.Time) []byte {
func (s3 S3) buildCredentialWithoutKey(nowTime time.Time) []byte {
var b bytes.Buffer
b.WriteString(nowTime.UTC().Format(shortTimeFormat))
b.WriteString(nowTime.Format(shortTimeFormat))
b.WriteRune('/')
b.WriteString(s3.Region)
b.WriteRune('/')
@@ -154,7 +154,7 @@ func (s3 S3) buildCredentialWithoutKey(nowTime time.Time) []byte {
}
func buildSignature(nowTime time.Time, secretAccessKey string, regionName string, serviceName string) []byte {
shortTime := nowTime.UTC().Format(shortTimeFormat)
shortTime := nowTime.Format(shortTimeFormat)
date := makeHMac([]byte("AWS4"+secretAccessKey), []byte(shortTime))
region := makeHMac(date, []byte(regionName))
presigned.go
+12 -4
View File
@@ -25,6 +25,7 @@ type PresignedInput struct {
ExtraHeaders map[string]string
ExpirySeconds int
Protocol string
Endpoint string
}
// GeneratePresignedURL creates a Presigned URL that can be used
@@ -32,11 +33,15 @@ type PresignedInput struct {
// (https://docs.aws.amazon.com/AmazonS3/latest/API/sigv4-query-string-auth.html)
func (s3 *S3) GeneratePresignedURL(in PresignedInput) string {
var (
nowTime = in.Timestamp.UTC()
nowTime = NowTime()
amzdate = nowTime.Format(amzDateISO8601TimeFormat)
protocol = defaultProtocol
endpoint = defaultPresignedHost
)
if !in.Timestamp.IsZero() {
nowTime = in.Timestamp.UTC()
}
amzdate := nowTime.Format(amzDateISO8601TimeFormat)
// Create cred
b := bytes.Buffer{}
@@ -50,6 +55,9 @@ func (s3 *S3) GeneratePresignedURL(in PresignedInput) string {
if in.Protocol != "" {
protocol = in.Protocol
}
if in.Endpoint != "" {
endpoint = in.Endpoint
}
// Add host to Headers
signedHeaders := map[string][]byte{}
@@ -59,7 +67,7 @@ func (s3 *S3) GeneratePresignedURL(in PresignedInput) string {
host := bytes.Buffer{}
host.WriteString(in.Bucket)
host.WriteRune('.')
host.WriteString(defaultPresignedHost)
host.WriteString(endpoint)
signedHeaders["host"] = host.Bytes()
// Start Canonical Request Formation
@@ -184,7 +192,7 @@ func (s3 *S3) GeneratePresignedURL(in PresignedInput) string {
b.WriteString(protocol)
b.WriteString(in.Bucket)
b.WriteRune('.')
b.WriteString(defaultPresignedHost)
b.WriteString(endpoint)
b.WriteRune('/')
b.WriteString(in.ObjectKey)
b.WriteRune('?')
presigned_test.go
+4
View File
@@ -39,6 +39,7 @@ func TestS3_GeneratePresignedURL_Personal(t *testing.T) {
dontwant := ""
if got := s.GeneratePresignedURL(PresignedInput{
Bucket: os.Getenv("AWS_S3_BUCKET"),
Endpoint: os.Getenv("AWS_S3_ENDPOINT"),
ObjectKey: "test1.txt",
Method: "GET",
Timestamp: NowTime(),
@@ -59,6 +60,7 @@ func TestS3_GeneratePresignedURL_ExtraHeader(t *testing.T) {
dontwant := ""
if got := s.GeneratePresignedURL(PresignedInput{
Bucket: os.Getenv("AWS_S3_BUCKET"),
Endpoint: os.Getenv("AWS_S3_ENDPOINT"),
ObjectKey: "test2.txt",
Method: "GET",
Timestamp: NowTime(),
@@ -82,6 +84,7 @@ func TestS3_GeneratePresignedURL_PUT(t *testing.T) {
dontwant := ""
if got := s.GeneratePresignedURL(PresignedInput{
Bucket: os.Getenv("AWS_S3_BUCKET"),
Endpoint: os.Getenv("AWS_S3_ENDPOINT"),
ObjectKey: "test2.txt",
Method: "PUT",
Timestamp: NowTime(),
@@ -105,6 +108,7 @@ func BenchmarkS3_GeneratePresigned(b *testing.B) {
for n := 0; n < b.N; n++ {
s.GeneratePresignedURL(PresignedInput{
Bucket: os.Getenv("AWS_S3_BUCKET"),
Endpoint: os.Getenv("AWS_S3_ENDPOINT"),
ObjectKey: "test.txt",
Method: "GET",
Timestamp: NowTime(),
simples3.go
+31 -12
View File
@@ -15,6 +15,7 @@ import (
"io/ioutil"
"mime/multipart"
"net/http"
"strings"
"time"
)
@@ -29,6 +30,7 @@ type S3 struct {
Region string
Client *http.Client
Endpoint string
URIFormat string
}
@@ -143,6 +145,27 @@ func (s3 *S3) getClient() *http.Client {
return s3.Client
}
func (s3 *S3) getURL(bucket string, args ...string) (uri string) {
if len(s3.Endpoint) > 0 {
uri = defaultProtocol + s3.Endpoint + "/" + bucket
} else {
uri = fmt.Sprintf(s3.URIFormat, s3.Region, bucket)
}
if len(args) > 0 {
uri = uri + "/" + strings.Join(args, "/")
}
return
}
// SetEndpoint ...
func (s3 *S3) SetEndpoint(uri string) *S3 {
if len(uri) > 0 {
s3.Endpoint = uri
}
return s3
}
func detectFileSize(body io.Seeker) (int64, error) {
pos, err := body.Seek(0, 1)
if err != nil {
@@ -177,7 +200,7 @@ func (s3 *S3) FileUpload(u UploadInput) (UploadResponse, error) {
return UploadResponse{}, err
}
policies, err := s3.CreateUploadPolicies(UploadConfig{
UploadURL: fmt.Sprintf(s3.URIFormat, s3.Region, u.Bucket),
UploadURL: s3.getURL(u.Bucket),
BucketName: u.Bucket,
ObjectKey: u.ObjectKey,
ContentType: u.ContentType,
@@ -230,15 +253,15 @@ func (s3 *S3) FileUpload(u UploadInput) (UploadResponse, error) {
}
defer res.Body.Close()
// Check the response
if res.Status != "201 Created" {
return UploadResponse{}, fmt.Errorf("status code: %s", res.Status)
}
data, err := ioutil.ReadAll(res.Body)
if err != nil {
return UploadResponse{}, err
}
// Check the response
if res.StatusCode != 201 {
return UploadResponse{}, fmt.Errorf("status code: %s: %q", res.Status, data)
}
var ur UploadResponse
xml.Unmarshal(data, &ur)
return ur, nil
@@ -247,12 +270,8 @@ func (s3 *S3) FileUpload(u UploadInput) (UploadResponse, error) {
// FileDelete makes a DELETE call with the file written as multipart
// and on successful upload, checks for 204 No Content.
func (s3 *S3) FileDelete(u DeleteInput) error {
url := fmt.Sprintf(
s3.URIFormat+"/%s",
s3.Region, u.Bucket, u.ObjectKey,
)
req, err := http.NewRequest(
"DELETE", url, nil,
"DELETE", s3.getURL(u.Bucket, u.ObjectKey), nil,
)
if err != nil {
return err
@@ -298,7 +317,7 @@ func (s3 *S3) FileDelete(u DeleteInput) error {
}
// Check the response
if res.Status != "204 No Content" {
if res.StatusCode != 204 {
return fmt.Errorf("status code: %s", res.Status)
}
simples3_test.go
+17 -15
View File
@@ -8,6 +8,13 @@ import (
"testing"
)
type tConfig struct {
AccessKey string
SecretKey string
Endpoint string
Region string
}
func TestS3_FileUpload(t *testing.T) {
testTxt, err := os.Open("test.txt")
if err != nil {
@@ -20,25 +27,21 @@ func TestS3_FileUpload(t *testing.T) {
}
defer testPng.Close()
type fields struct {
AccessKey string
SecretKey string
Region string
}
type args struct {
u UploadInput
}
tests := []struct {
name string
fields fields
fields tConfig
args args
wantErr bool
}{
{
name: "Upload test.txt",
fields: fields{
fields: tConfig{
AccessKey: os.Getenv("AWS_S3_ACCESS_KEY"),
SecretKey: os.Getenv("AWS_S3_SECRET_KEY"),
Endpoint: os.Getenv("AWS_S3_ENDPOINT"),
Region: os.Getenv("AWS_S3_REGION"),
},
args: args{
@@ -54,9 +57,10 @@ func TestS3_FileUpload(t *testing.T) {
},
{
name: "Upload avatar.png",
fields: fields{
fields: tConfig{
AccessKey: os.Getenv("AWS_S3_ACCESS_KEY"),
SecretKey: os.Getenv("AWS_S3_SECRET_KEY"),
Endpoint: os.Getenv("AWS_S3_ENDPOINT"),
Region: os.Getenv("AWS_S3_REGION"),
},
args: args{
@@ -74,6 +78,7 @@ func TestS3_FileUpload(t *testing.T) {
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
s3 := New(tt.fields.Region, tt.fields.AccessKey, tt.fields.SecretKey)
s3.SetEndpoint(tt.fields.Endpoint)
resp, err := s3.FileUpload(tt.args.u)
if (err != nil) != tt.wantErr {
t.Errorf("S3.FileUpload() error = %v, wantErr %v", err, tt.wantErr)
@@ -87,25 +92,21 @@ func TestS3_FileUpload(t *testing.T) {
}
func TestS3_FileDelete(t *testing.T) {
type fields struct {
AccessKey string
SecretKey string
Region string
}
type args struct {
u DeleteInput
}
tests := []struct {
name string
fields fields
fields tConfig
args args
wantErr bool
}{
{
name: "Delete test.txt",
fields: fields{
fields: tConfig{
AccessKey: os.Getenv("AWS_S3_ACCESS_KEY"),
SecretKey: os.Getenv("AWS_S3_SECRET_KEY"),
Endpoint: os.Getenv("AWS_S3_ENDPOINT"),
Region: os.Getenv("AWS_S3_REGION"),
},
args: args{
@@ -120,6 +121,7 @@ func TestS3_FileDelete(t *testing.T) {
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
s3 := New(tt.fields.Region, tt.fields.AccessKey, tt.fields.SecretKey)
s3.SetEndpoint(tt.fields.Endpoint)
if err := s3.FileDelete(tt.args.u); (err != nil) != tt.wantErr {
t.Errorf("S3.FileDelete() error = %v, wantErr %v", err, tt.wantErr)
}