Add Endpoint for some compatible instance and fix time format (#3)
* Add Endpoint for some compatible instance. * Fix: InvalidExpiration * Fix: RequestTimeTooSkewed Compatible s3: https://developer.qiniu.com/kodo/manual/4088/s3-access-domainname * - clean code * Fix: InvalidExpiration
This commit is contained in:
+3
-1
@@ -21,4 +21,6 @@
|
|||||||
.settings/
|
.settings/
|
||||||
.vscode/
|
.vscode/
|
||||||
tsconfig.json
|
tsconfig.json
|
||||||
jsconfig.json
|
jsconfig.json
|
||||||
|
|
||||||
|
.env
|
||||||
|
|||||||
@@ -45,7 +45,7 @@ type PolicyJSON struct {
|
|||||||
}
|
}
|
||||||
|
|
||||||
const (
|
const (
|
||||||
expirationTimeFormat = "2006-01-02T15:04:05ZZ07:00"
|
expirationTimeFormat = "2006-01-02T15:04:05Z07:00"
|
||||||
amzDateISO8601TimeFormat = "20060102T150405Z"
|
amzDateISO8601TimeFormat = "20060102T150405Z"
|
||||||
shortTimeFormat = "20060102"
|
shortTimeFormat = "20060102"
|
||||||
algorithm = "AWS4-HMAC-SHA256"
|
algorithm = "AWS4-HMAC-SHA256"
|
||||||
@@ -57,7 +57,7 @@ const (
|
|||||||
|
|
||||||
// NowTime mockable time.Now()
|
// NowTime mockable time.Now()
|
||||||
var NowTime = func() time.Time {
|
var NowTime = func() time.Time {
|
||||||
return time.Now()
|
return time.Now().UTC()
|
||||||
}
|
}
|
||||||
|
|
||||||
var newLine = []byte{'\n'}
|
var newLine = []byte{'\n'}
|
||||||
@@ -131,7 +131,7 @@ func (s3 S3) buildCredential(nowTime time.Time) []byte {
|
|||||||
var b bytes.Buffer
|
var b bytes.Buffer
|
||||||
b.WriteString(s3.AccessKey)
|
b.WriteString(s3.AccessKey)
|
||||||
b.WriteRune('/')
|
b.WriteRune('/')
|
||||||
b.WriteString(nowTime.UTC().Format(shortTimeFormat))
|
b.WriteString(nowTime.Format(shortTimeFormat))
|
||||||
b.WriteRune('/')
|
b.WriteRune('/')
|
||||||
b.WriteString(s3.Region)
|
b.WriteString(s3.Region)
|
||||||
b.WriteRune('/')
|
b.WriteRune('/')
|
||||||
@@ -143,7 +143,7 @@ func (s3 S3) buildCredential(nowTime time.Time) []byte {
|
|||||||
|
|
||||||
func (s3 S3) buildCredentialWithoutKey(nowTime time.Time) []byte {
|
func (s3 S3) buildCredentialWithoutKey(nowTime time.Time) []byte {
|
||||||
var b bytes.Buffer
|
var b bytes.Buffer
|
||||||
b.WriteString(nowTime.UTC().Format(shortTimeFormat))
|
b.WriteString(nowTime.Format(shortTimeFormat))
|
||||||
b.WriteRune('/')
|
b.WriteRune('/')
|
||||||
b.WriteString(s3.Region)
|
b.WriteString(s3.Region)
|
||||||
b.WriteRune('/')
|
b.WriteRune('/')
|
||||||
@@ -154,7 +154,7 @@ func (s3 S3) buildCredentialWithoutKey(nowTime time.Time) []byte {
|
|||||||
}
|
}
|
||||||
|
|
||||||
func buildSignature(nowTime time.Time, secretAccessKey string, regionName string, serviceName string) []byte {
|
func buildSignature(nowTime time.Time, secretAccessKey string, regionName string, serviceName string) []byte {
|
||||||
shortTime := nowTime.UTC().Format(shortTimeFormat)
|
shortTime := nowTime.Format(shortTimeFormat)
|
||||||
|
|
||||||
date := makeHMac([]byte("AWS4"+secretAccessKey), []byte(shortTime))
|
date := makeHMac([]byte("AWS4"+secretAccessKey), []byte(shortTime))
|
||||||
region := makeHMac(date, []byte(regionName))
|
region := makeHMac(date, []byte(regionName))
|
||||||
|
|||||||
+12
-4
@@ -25,6 +25,7 @@ type PresignedInput struct {
|
|||||||
ExtraHeaders map[string]string
|
ExtraHeaders map[string]string
|
||||||
ExpirySeconds int
|
ExpirySeconds int
|
||||||
Protocol string
|
Protocol string
|
||||||
|
Endpoint string
|
||||||
}
|
}
|
||||||
|
|
||||||
// GeneratePresignedURL creates a Presigned URL that can be used
|
// GeneratePresignedURL creates a Presigned URL that can be used
|
||||||
@@ -32,11 +33,15 @@ type PresignedInput struct {
|
|||||||
// (https://docs.aws.amazon.com/AmazonS3/latest/API/sigv4-query-string-auth.html)
|
// (https://docs.aws.amazon.com/AmazonS3/latest/API/sigv4-query-string-auth.html)
|
||||||
func (s3 *S3) GeneratePresignedURL(in PresignedInput) string {
|
func (s3 *S3) GeneratePresignedURL(in PresignedInput) string {
|
||||||
var (
|
var (
|
||||||
nowTime = in.Timestamp.UTC()
|
nowTime = NowTime()
|
||||||
|
|
||||||
amzdate = nowTime.Format(amzDateISO8601TimeFormat)
|
|
||||||
protocol = defaultProtocol
|
protocol = defaultProtocol
|
||||||
|
endpoint = defaultPresignedHost
|
||||||
)
|
)
|
||||||
|
if !in.Timestamp.IsZero() {
|
||||||
|
nowTime = in.Timestamp.UTC()
|
||||||
|
}
|
||||||
|
amzdate := nowTime.Format(amzDateISO8601TimeFormat)
|
||||||
|
|
||||||
// Create cred
|
// Create cred
|
||||||
b := bytes.Buffer{}
|
b := bytes.Buffer{}
|
||||||
@@ -50,6 +55,9 @@ func (s3 *S3) GeneratePresignedURL(in PresignedInput) string {
|
|||||||
if in.Protocol != "" {
|
if in.Protocol != "" {
|
||||||
protocol = in.Protocol
|
protocol = in.Protocol
|
||||||
}
|
}
|
||||||
|
if in.Endpoint != "" {
|
||||||
|
endpoint = in.Endpoint
|
||||||
|
}
|
||||||
|
|
||||||
// Add host to Headers
|
// Add host to Headers
|
||||||
signedHeaders := map[string][]byte{}
|
signedHeaders := map[string][]byte{}
|
||||||
@@ -59,7 +67,7 @@ func (s3 *S3) GeneratePresignedURL(in PresignedInput) string {
|
|||||||
host := bytes.Buffer{}
|
host := bytes.Buffer{}
|
||||||
host.WriteString(in.Bucket)
|
host.WriteString(in.Bucket)
|
||||||
host.WriteRune('.')
|
host.WriteRune('.')
|
||||||
host.WriteString(defaultPresignedHost)
|
host.WriteString(endpoint)
|
||||||
signedHeaders["host"] = host.Bytes()
|
signedHeaders["host"] = host.Bytes()
|
||||||
|
|
||||||
// Start Canonical Request Formation
|
// Start Canonical Request Formation
|
||||||
@@ -184,7 +192,7 @@ func (s3 *S3) GeneratePresignedURL(in PresignedInput) string {
|
|||||||
b.WriteString(protocol)
|
b.WriteString(protocol)
|
||||||
b.WriteString(in.Bucket)
|
b.WriteString(in.Bucket)
|
||||||
b.WriteRune('.')
|
b.WriteRune('.')
|
||||||
b.WriteString(defaultPresignedHost)
|
b.WriteString(endpoint)
|
||||||
b.WriteRune('/')
|
b.WriteRune('/')
|
||||||
b.WriteString(in.ObjectKey)
|
b.WriteString(in.ObjectKey)
|
||||||
b.WriteRune('?')
|
b.WriteRune('?')
|
||||||
|
|||||||
@@ -39,6 +39,7 @@ func TestS3_GeneratePresignedURL_Personal(t *testing.T) {
|
|||||||
dontwant := ""
|
dontwant := ""
|
||||||
if got := s.GeneratePresignedURL(PresignedInput{
|
if got := s.GeneratePresignedURL(PresignedInput{
|
||||||
Bucket: os.Getenv("AWS_S3_BUCKET"),
|
Bucket: os.Getenv("AWS_S3_BUCKET"),
|
||||||
|
Endpoint: os.Getenv("AWS_S3_ENDPOINT"),
|
||||||
ObjectKey: "test1.txt",
|
ObjectKey: "test1.txt",
|
||||||
Method: "GET",
|
Method: "GET",
|
||||||
Timestamp: NowTime(),
|
Timestamp: NowTime(),
|
||||||
@@ -59,6 +60,7 @@ func TestS3_GeneratePresignedURL_ExtraHeader(t *testing.T) {
|
|||||||
dontwant := ""
|
dontwant := ""
|
||||||
if got := s.GeneratePresignedURL(PresignedInput{
|
if got := s.GeneratePresignedURL(PresignedInput{
|
||||||
Bucket: os.Getenv("AWS_S3_BUCKET"),
|
Bucket: os.Getenv("AWS_S3_BUCKET"),
|
||||||
|
Endpoint: os.Getenv("AWS_S3_ENDPOINT"),
|
||||||
ObjectKey: "test2.txt",
|
ObjectKey: "test2.txt",
|
||||||
Method: "GET",
|
Method: "GET",
|
||||||
Timestamp: NowTime(),
|
Timestamp: NowTime(),
|
||||||
@@ -82,6 +84,7 @@ func TestS3_GeneratePresignedURL_PUT(t *testing.T) {
|
|||||||
dontwant := ""
|
dontwant := ""
|
||||||
if got := s.GeneratePresignedURL(PresignedInput{
|
if got := s.GeneratePresignedURL(PresignedInput{
|
||||||
Bucket: os.Getenv("AWS_S3_BUCKET"),
|
Bucket: os.Getenv("AWS_S3_BUCKET"),
|
||||||
|
Endpoint: os.Getenv("AWS_S3_ENDPOINT"),
|
||||||
ObjectKey: "test2.txt",
|
ObjectKey: "test2.txt",
|
||||||
Method: "PUT",
|
Method: "PUT",
|
||||||
Timestamp: NowTime(),
|
Timestamp: NowTime(),
|
||||||
@@ -105,6 +108,7 @@ func BenchmarkS3_GeneratePresigned(b *testing.B) {
|
|||||||
for n := 0; n < b.N; n++ {
|
for n := 0; n < b.N; n++ {
|
||||||
s.GeneratePresignedURL(PresignedInput{
|
s.GeneratePresignedURL(PresignedInput{
|
||||||
Bucket: os.Getenv("AWS_S3_BUCKET"),
|
Bucket: os.Getenv("AWS_S3_BUCKET"),
|
||||||
|
Endpoint: os.Getenv("AWS_S3_ENDPOINT"),
|
||||||
ObjectKey: "test.txt",
|
ObjectKey: "test.txt",
|
||||||
Method: "GET",
|
Method: "GET",
|
||||||
Timestamp: NowTime(),
|
Timestamp: NowTime(),
|
||||||
|
|||||||
+31
-12
@@ -15,6 +15,7 @@ import (
|
|||||||
"io/ioutil"
|
"io/ioutil"
|
||||||
"mime/multipart"
|
"mime/multipart"
|
||||||
"net/http"
|
"net/http"
|
||||||
|
"strings"
|
||||||
"time"
|
"time"
|
||||||
)
|
)
|
||||||
|
|
||||||
@@ -29,6 +30,7 @@ type S3 struct {
|
|||||||
Region string
|
Region string
|
||||||
Client *http.Client
|
Client *http.Client
|
||||||
|
|
||||||
|
Endpoint string
|
||||||
URIFormat string
|
URIFormat string
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -143,6 +145,27 @@ func (s3 *S3) getClient() *http.Client {
|
|||||||
return s3.Client
|
return s3.Client
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func (s3 *S3) getURL(bucket string, args ...string) (uri string) {
|
||||||
|
if len(s3.Endpoint) > 0 {
|
||||||
|
uri = defaultProtocol + s3.Endpoint + "/" + bucket
|
||||||
|
} else {
|
||||||
|
uri = fmt.Sprintf(s3.URIFormat, s3.Region, bucket)
|
||||||
|
}
|
||||||
|
|
||||||
|
if len(args) > 0 {
|
||||||
|
uri = uri + "/" + strings.Join(args, "/")
|
||||||
|
}
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
// SetEndpoint ...
|
||||||
|
func (s3 *S3) SetEndpoint(uri string) *S3 {
|
||||||
|
if len(uri) > 0 {
|
||||||
|
s3.Endpoint = uri
|
||||||
|
}
|
||||||
|
return s3
|
||||||
|
}
|
||||||
|
|
||||||
func detectFileSize(body io.Seeker) (int64, error) {
|
func detectFileSize(body io.Seeker) (int64, error) {
|
||||||
pos, err := body.Seek(0, 1)
|
pos, err := body.Seek(0, 1)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
@@ -177,7 +200,7 @@ func (s3 *S3) FileUpload(u UploadInput) (UploadResponse, error) {
|
|||||||
return UploadResponse{}, err
|
return UploadResponse{}, err
|
||||||
}
|
}
|
||||||
policies, err := s3.CreateUploadPolicies(UploadConfig{
|
policies, err := s3.CreateUploadPolicies(UploadConfig{
|
||||||
UploadURL: fmt.Sprintf(s3.URIFormat, s3.Region, u.Bucket),
|
UploadURL: s3.getURL(u.Bucket),
|
||||||
BucketName: u.Bucket,
|
BucketName: u.Bucket,
|
||||||
ObjectKey: u.ObjectKey,
|
ObjectKey: u.ObjectKey,
|
||||||
ContentType: u.ContentType,
|
ContentType: u.ContentType,
|
||||||
@@ -230,15 +253,15 @@ func (s3 *S3) FileUpload(u UploadInput) (UploadResponse, error) {
|
|||||||
}
|
}
|
||||||
defer res.Body.Close()
|
defer res.Body.Close()
|
||||||
|
|
||||||
// Check the response
|
|
||||||
if res.Status != "201 Created" {
|
|
||||||
return UploadResponse{}, fmt.Errorf("status code: %s", res.Status)
|
|
||||||
}
|
|
||||||
|
|
||||||
data, err := ioutil.ReadAll(res.Body)
|
data, err := ioutil.ReadAll(res.Body)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return UploadResponse{}, err
|
return UploadResponse{}, err
|
||||||
}
|
}
|
||||||
|
// Check the response
|
||||||
|
if res.StatusCode != 201 {
|
||||||
|
return UploadResponse{}, fmt.Errorf("status code: %s: %q", res.Status, data)
|
||||||
|
}
|
||||||
|
|
||||||
var ur UploadResponse
|
var ur UploadResponse
|
||||||
xml.Unmarshal(data, &ur)
|
xml.Unmarshal(data, &ur)
|
||||||
return ur, nil
|
return ur, nil
|
||||||
@@ -247,12 +270,8 @@ func (s3 *S3) FileUpload(u UploadInput) (UploadResponse, error) {
|
|||||||
// FileDelete makes a DELETE call with the file written as multipart
|
// FileDelete makes a DELETE call with the file written as multipart
|
||||||
// and on successful upload, checks for 204 No Content.
|
// and on successful upload, checks for 204 No Content.
|
||||||
func (s3 *S3) FileDelete(u DeleteInput) error {
|
func (s3 *S3) FileDelete(u DeleteInput) error {
|
||||||
url := fmt.Sprintf(
|
|
||||||
s3.URIFormat+"/%s",
|
|
||||||
s3.Region, u.Bucket, u.ObjectKey,
|
|
||||||
)
|
|
||||||
req, err := http.NewRequest(
|
req, err := http.NewRequest(
|
||||||
"DELETE", url, nil,
|
"DELETE", s3.getURL(u.Bucket, u.ObjectKey), nil,
|
||||||
)
|
)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return err
|
return err
|
||||||
@@ -298,7 +317,7 @@ func (s3 *S3) FileDelete(u DeleteInput) error {
|
|||||||
}
|
}
|
||||||
|
|
||||||
// Check the response
|
// Check the response
|
||||||
if res.Status != "204 No Content" {
|
if res.StatusCode != 204 {
|
||||||
return fmt.Errorf("status code: %s", res.Status)
|
return fmt.Errorf("status code: %s", res.Status)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
+17
-15
@@ -8,6 +8,13 @@ import (
|
|||||||
"testing"
|
"testing"
|
||||||
)
|
)
|
||||||
|
|
||||||
|
type tConfig struct {
|
||||||
|
AccessKey string
|
||||||
|
SecretKey string
|
||||||
|
Endpoint string
|
||||||
|
Region string
|
||||||
|
}
|
||||||
|
|
||||||
func TestS3_FileUpload(t *testing.T) {
|
func TestS3_FileUpload(t *testing.T) {
|
||||||
testTxt, err := os.Open("test.txt")
|
testTxt, err := os.Open("test.txt")
|
||||||
if err != nil {
|
if err != nil {
|
||||||
@@ -20,25 +27,21 @@ func TestS3_FileUpload(t *testing.T) {
|
|||||||
}
|
}
|
||||||
defer testPng.Close()
|
defer testPng.Close()
|
||||||
|
|
||||||
type fields struct {
|
|
||||||
AccessKey string
|
|
||||||
SecretKey string
|
|
||||||
Region string
|
|
||||||
}
|
|
||||||
type args struct {
|
type args struct {
|
||||||
u UploadInput
|
u UploadInput
|
||||||
}
|
}
|
||||||
tests := []struct {
|
tests := []struct {
|
||||||
name string
|
name string
|
||||||
fields fields
|
fields tConfig
|
||||||
args args
|
args args
|
||||||
wantErr bool
|
wantErr bool
|
||||||
}{
|
}{
|
||||||
{
|
{
|
||||||
name: "Upload test.txt",
|
name: "Upload test.txt",
|
||||||
fields: fields{
|
fields: tConfig{
|
||||||
AccessKey: os.Getenv("AWS_S3_ACCESS_KEY"),
|
AccessKey: os.Getenv("AWS_S3_ACCESS_KEY"),
|
||||||
SecretKey: os.Getenv("AWS_S3_SECRET_KEY"),
|
SecretKey: os.Getenv("AWS_S3_SECRET_KEY"),
|
||||||
|
Endpoint: os.Getenv("AWS_S3_ENDPOINT"),
|
||||||
Region: os.Getenv("AWS_S3_REGION"),
|
Region: os.Getenv("AWS_S3_REGION"),
|
||||||
},
|
},
|
||||||
args: args{
|
args: args{
|
||||||
@@ -54,9 +57,10 @@ func TestS3_FileUpload(t *testing.T) {
|
|||||||
},
|
},
|
||||||
{
|
{
|
||||||
name: "Upload avatar.png",
|
name: "Upload avatar.png",
|
||||||
fields: fields{
|
fields: tConfig{
|
||||||
AccessKey: os.Getenv("AWS_S3_ACCESS_KEY"),
|
AccessKey: os.Getenv("AWS_S3_ACCESS_KEY"),
|
||||||
SecretKey: os.Getenv("AWS_S3_SECRET_KEY"),
|
SecretKey: os.Getenv("AWS_S3_SECRET_KEY"),
|
||||||
|
Endpoint: os.Getenv("AWS_S3_ENDPOINT"),
|
||||||
Region: os.Getenv("AWS_S3_REGION"),
|
Region: os.Getenv("AWS_S3_REGION"),
|
||||||
},
|
},
|
||||||
args: args{
|
args: args{
|
||||||
@@ -74,6 +78,7 @@ func TestS3_FileUpload(t *testing.T) {
|
|||||||
for _, tt := range tests {
|
for _, tt := range tests {
|
||||||
t.Run(tt.name, func(t *testing.T) {
|
t.Run(tt.name, func(t *testing.T) {
|
||||||
s3 := New(tt.fields.Region, tt.fields.AccessKey, tt.fields.SecretKey)
|
s3 := New(tt.fields.Region, tt.fields.AccessKey, tt.fields.SecretKey)
|
||||||
|
s3.SetEndpoint(tt.fields.Endpoint)
|
||||||
resp, err := s3.FileUpload(tt.args.u)
|
resp, err := s3.FileUpload(tt.args.u)
|
||||||
if (err != nil) != tt.wantErr {
|
if (err != nil) != tt.wantErr {
|
||||||
t.Errorf("S3.FileUpload() error = %v, wantErr %v", err, tt.wantErr)
|
t.Errorf("S3.FileUpload() error = %v, wantErr %v", err, tt.wantErr)
|
||||||
@@ -87,25 +92,21 @@ func TestS3_FileUpload(t *testing.T) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
func TestS3_FileDelete(t *testing.T) {
|
func TestS3_FileDelete(t *testing.T) {
|
||||||
type fields struct {
|
|
||||||
AccessKey string
|
|
||||||
SecretKey string
|
|
||||||
Region string
|
|
||||||
}
|
|
||||||
type args struct {
|
type args struct {
|
||||||
u DeleteInput
|
u DeleteInput
|
||||||
}
|
}
|
||||||
tests := []struct {
|
tests := []struct {
|
||||||
name string
|
name string
|
||||||
fields fields
|
fields tConfig
|
||||||
args args
|
args args
|
||||||
wantErr bool
|
wantErr bool
|
||||||
}{
|
}{
|
||||||
{
|
{
|
||||||
name: "Delete test.txt",
|
name: "Delete test.txt",
|
||||||
fields: fields{
|
fields: tConfig{
|
||||||
AccessKey: os.Getenv("AWS_S3_ACCESS_KEY"),
|
AccessKey: os.Getenv("AWS_S3_ACCESS_KEY"),
|
||||||
SecretKey: os.Getenv("AWS_S3_SECRET_KEY"),
|
SecretKey: os.Getenv("AWS_S3_SECRET_KEY"),
|
||||||
|
Endpoint: os.Getenv("AWS_S3_ENDPOINT"),
|
||||||
Region: os.Getenv("AWS_S3_REGION"),
|
Region: os.Getenv("AWS_S3_REGION"),
|
||||||
},
|
},
|
||||||
args: args{
|
args: args{
|
||||||
@@ -120,6 +121,7 @@ func TestS3_FileDelete(t *testing.T) {
|
|||||||
for _, tt := range tests {
|
for _, tt := range tests {
|
||||||
t.Run(tt.name, func(t *testing.T) {
|
t.Run(tt.name, func(t *testing.T) {
|
||||||
s3 := New(tt.fields.Region, tt.fields.AccessKey, tt.fields.SecretKey)
|
s3 := New(tt.fields.Region, tt.fields.AccessKey, tt.fields.SecretKey)
|
||||||
|
s3.SetEndpoint(tt.fields.Endpoint)
|
||||||
if err := s3.FileDelete(tt.args.u); (err != nil) != tt.wantErr {
|
if err := s3.FileDelete(tt.args.u); (err != nil) != tt.wantErr {
|
||||||
t.Errorf("S3.FileDelete() error = %v, wantErr %v", err, tt.wantErr)
|
t.Errorf("S3.FileDelete() error = %v, wantErr %v", err, tt.wantErr)
|
||||||
}
|
}
|
||||||
|
|||||||
Reference in New Issue
Block a user