eiac/simples3
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Merge pull request #18 from rhnvrm/feat-file-put

Browse Source 
feat: add support for FilePut
This commit is contained in:
Rohan Verma
2021-12-29 15:02:42 +05:30
committed by GitHub
parent b23aec1a12 a9d0bbf9a8
commit ad0419ef77
3 changed files with 163 additions and 25 deletions
Download Patch File Download Diff File Expand all files Collapse all files
policy.go
+5 -5
View File
@@ -34,13 +34,13 @@ type UploadConfig struct {
MetaData map[string]string
}
// UploadPolicies Amazon s3 upload policies
// UploadPolicies Amazon s3 upload policies.
type UploadPolicies struct {
URL string
Form map[string]string
}
// PolicyJSON is policy rule
// PolicyJSON is policy rule.
type PolicyJSON struct {
Expiration string `json:"expiration"`
Conditions []interface{} `json:"conditions"`
@@ -57,12 +57,12 @@ const (
defaultExpirationHour = 1 * time.Hour
)
// nowTime mockable time.Now()
var nowTime = func() time.Time {
// nowTime is returns UTC time.
func nowTime() time.Time {
return time.Now().UTC()
}
var newLine = []byte{'\n'}
var newLine = []byte{'\n'} //nolint
// CreateUploadPolicies creates amazon s3 sigv4 compatible
// policy and signing keys with the signature returns the upload policy.
simples3.go
+105 -16
View File
@@ -80,7 +80,9 @@ type UploadInput struct {
// optional fields
ContentDisposition string
ACL string
CustomMetadata map[string]string // Setting key/value pairs adds user-defined metadata keys to the object, prefixed with AMZMetaPrefix.
// Setting key/value pairs adds user-defined metadata
// keys to the object, prefixed with AMZMetaPrefix.
CustomMetadata map[string]string
Body io.ReadSeeker
}
@@ -101,6 +103,14 @@ type UploadResponse struct {
ETag string `xml:"ETag"`
}
// PutResponse is returned when the action is successful,
// and the service sends back an HTTP 200 response. The response
// returns ETag along with HTTP headers.
type PutResponse struct {
ETag string
Headers http.Header
}
// DeleteInput is passed to FileDelete as a parameter.
type DeleteInput struct {
Bucket string
@@ -108,7 +118,7 @@ type DeleteInput struct {
}
// IAMResponse is used by NewUsingIAM to auto
// detect the credentials
// detect the credentials.
type IAMResponse struct {
Code string `json:"Code"`
LastUpdated string `json:"LastUpdated"`
@@ -143,7 +153,7 @@ func newUsingIAMImpl(baseURL, region string) (*S3, error) {
return nil, err
}
defer resp.Body.Close()
if resp.StatusCode != 200 {
if resp.StatusCode != http.StatusOK {
return nil, errors.New(http.StatusText(resp.StatusCode))
}
@@ -157,7 +167,7 @@ func newUsingIAMImpl(baseURL, region string) (*S3, error) {
return nil, err
}
defer resp.Body.Close()
if resp.StatusCode != 200 {
if resp.StatusCode != http.StatusOK {
return nil, errors.New(http.StatusText(resp.StatusCode))
}
@@ -241,7 +251,7 @@ func detectFileSize(body io.Seeker) (int64, error) {
}
defer body.Seek(pos, 0)
n, err := body.Seek(0, 2)
n, err := body.Seek(0, 2) //nolint:gomnd
if err != nil {
return -1, err
}
@@ -280,8 +290,11 @@ func (s3 *S3) signRequest(req *http.Request) error {
// Signature Version 4 requests. It provides a hash of the
// request payload. If there is no payload, you must provide
// the hash of an empty string.
emptyhash := "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855"
req.Header.Set("x-amz-content-sha256", emptyhash)
if req.Header.Get("x-amz-content-sha256") == "" {
emptyhash := "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855"
req.Header.Set("x-amz-content-sha256", emptyhash)
}
k := s3.signKeys(t)
h := hmac.New(sha256.New, k)
@@ -319,13 +332,87 @@ func (s3 *S3) FileDownload(u DownloadInput) (io.ReadCloser, error) {
return nil, err
}
if res.StatusCode != 200 {
if res.StatusCode != http.StatusOK {
return nil, fmt.Errorf("status code: %s", res.Status)
}
return res.Body, nil
}
// FilePut makes a PUT call to S3.
func (s3 *S3) FilePut(u UploadInput) (PutResponse, error) {
fSize, err := detectFileSize(u.Body)
if err != nil {
return PutResponse{}, err
}
content := make([]byte, fSize)
_, err = u.Body.Read(content)
if err != nil {
return PutResponse{}, err
}
u.Body.Seek(0, 0)
req, er := http.NewRequest(http.MethodPut, s3.getURL(u.Bucket, u.ObjectKey), u.Body)
if er != nil {
return PutResponse{}, err
}
if u.ContentType == "" {
u.ContentType = "application/octet-stream"
}
h := sha256.New()
h.Write(content)
req.Header.Set("x-amz-content-sha256", fmt.Sprintf("%x", h.Sum(nil)))
req.Header.Set("Content-Type", u.ContentType)
req.Header.Set("Content-Length", fmt.Sprintf("%d", fSize))
req.Header.Set("Host", req.URL.Host)
for k, v := range u.CustomMetadata {
req.Header.Set("x-amz-meta-"+k, v)
}
if u.ContentDisposition != "" {
req.Header.Set("Content-Disposition", u.ContentDisposition)
}
if u.ACL != "" {
req.Header.Set("x-amz-acl", u.ACL)
}
req.ContentLength = fSize
if err := s3.signRequest(req); err != nil {
return PutResponse{}, err
}
// debug(httputil.DumpRequest(req, true))
// Submit the request
client := s3.getClient()
res, err := client.Do(req)
if err != nil {
return PutResponse{}, err
}
defer res.Body.Close()
data, err := ioutil.ReadAll(res.Body)
if err != nil {
return PutResponse{}, err
}
// Check the response
if res.StatusCode != http.StatusOK {
return PutResponse{}, fmt.Errorf("status code: %s: %q", res.Status, data)
}
return PutResponse{
ETag: res.Header.Get("ETag"),
Headers: res.Header.Clone(),
}, nil
}
// FileUpload makes a POST call with the file written as multipart
// and on successful upload, checks for 200 OK.
func (s3 *S3) FileUpload(u UploadInput) (UploadResponse, error) {
@@ -404,8 +491,9 @@ func (s3 *S3) FileUpload(u UploadInput) (UploadResponse, error) {
if err != nil {
return UploadResponse{}, err
}
// Check the response
if res.StatusCode != 201 {
if res.StatusCode != http.StatusCreated {
return UploadResponse{}, fmt.Errorf("status code: %s: %q", res.Status, data)
}
@@ -436,7 +524,7 @@ func (s3 *S3) FileDelete(u DeleteInput) error {
}
// Check the response
if res.StatusCode != 204 {
if res.StatusCode != http.StatusNoContent {
return fmt.Errorf("status code: %s", res.Status)
}
@@ -460,7 +548,7 @@ func (s3 *S3) FileDetails(u DetailsInput) (DetailsResponse, error) {
return DetailsResponse{}, err
}
if res.StatusCode != 200 {
if res.StatusCode != http.StatusOK {
return DetailsResponse{}, fmt.Errorf("status code: %s", res.Status)
}
@@ -515,7 +603,7 @@ func getFirstString(s []string) string {
return ""
}
// if object matches reserved string, no need to encode them
// if object matches reserved string, no need to encode them.
var reservedObjectNames = regexp.MustCompile("^[a-zA-Z0-9-_.~/]+$")
// encodePath encode the strings from UTF-8 byte representations to HTML hex escape sequences
@@ -525,7 +613,8 @@ var reservedObjectNames = regexp.MustCompile("^[a-zA-Z0-9-_.~/]+$")
//
// This function on the other hand is a direct replacement for url.Encode() technique to support
// pretty much every UTF-8 character.
// adapted from https://github.com/minio/minio-go/blob/fe1f3855b146c1b6ce4199740d317e44cf9e85c2/pkg/s3utils/utils.go#L285
// adapted from
// https://github.com/minio/minio-go/blob/fe1f3855b146c1b6ce4199740d317e44cf9e85c2/pkg/s3utils/utils.go#L285
func encodePath(pathName string) string {
if reservedObjectNames.MatchString(pathName) {
return pathName
@@ -541,12 +630,12 @@ func encodePath(pathName string) string {
encodedPathname.WriteRune(s)
continue
default:
len := utf8.RuneLen(s)
if len < 0 {
lenR := utf8.RuneLen(s)
if lenR < 0 {
// if utf8 cannot convert, return the same string as is
return pathName
}
u := make([]byte, len)
u := make([]byte, lenR)
utf8.EncodeRune(u, s)
for _, r := range u {
hex := hex.EncodeToString([]byte{r})
simples3_test.go
+53 -4
View File
@@ -17,7 +17,7 @@ type tConfig struct {
Region string
}
func TestS3_FileUpload(t *testing.T) {
func TestS3_FileUploadPostAndPut(t *testing.T) {
testTxt, err := os.Open("testdata/test.txt")
if err != nil {
return
@@ -128,9 +128,10 @@ func TestS3_FileUpload(t *testing.T) {
}
for _, testcase := range tests {
tt := testcase
t.Run(tt.name, func(t *testing.T) {
t.Run(tt.name+"_post", func(t *testing.T) {
s3 := New(tt.fields.Region, tt.fields.AccessKey, tt.fields.SecretKey)
s3.SetEndpoint(tt.fields.Endpoint)
resp, err := s3.FileUpload(tt.args.u)
if (err != nil) != tt.wantErr {
t.Errorf("S3.FileUpload() error = %v, wantErr %v", err, tt.wantErr)
@@ -154,8 +155,40 @@ func TestS3_FileUpload(t *testing.T) {
t.Errorf("S3.FileUpload() error = %v, wantErr %v", err, tt.wantErr)
}
if len(dResp.AmzMeta) != 1 {
t.Errorf("S3.FileDetails() returned incorrect metadata, got: %v", dResp.AmzMeta)
if len(dResp.AmzMeta) != len(tt.args.u.CustomMetadata) {
t.Errorf("S3.FileDetails() returned incorrect metadata, got: %#v", dResp)
}
}
})
t.Run(tt.name+"_put", func(t *testing.T) {
s3 := New(tt.fields.Region, tt.fields.AccessKey, tt.fields.SecretKey)
s3.SetEndpoint(tt.fields.Endpoint)
resp, err := s3.FilePut(tt.args.u)
if (err != nil) != tt.wantErr {
t.Errorf("S3.FileUpload() error = %v, wantErr %v", err, tt.wantErr)
}
// reset file, to reuse in further tests.
tt.args.u.Body.Seek(0, 0)
// check for empty response
if resp.ETag == "" {
t.Errorf("S3.FileUpload() returned empty response, %v", resp)
}
if tt.testDetails {
dResp, err := s3.FileDetails(DetailsInput{
Bucket: tt.args.u.Bucket,
ObjectKey: tt.args.u.ObjectKey,
})
if (err != nil) != tt.wantErr {
t.Errorf("S3.FileUpload() error = %v, wantErr %v", err, tt.wantErr)
}
if len(dResp.AmzMeta) != len(tt.args.u.CustomMetadata) {
t.Errorf("S3.FileDetails() returned incorrect metadata, got: %#v", dResp)
}
}
})
@@ -296,6 +329,22 @@ func TestS3_FileDelete(t *testing.T) {
},
wantErr: false,
},
{
name: "Delete test_metadata.txt",
fields: tConfig{
AccessKey: os.Getenv("AWS_S3_ACCESS_KEY"),
SecretKey: os.Getenv("AWS_S3_SECRET_KEY"),
Endpoint: os.Getenv("AWS_S3_ENDPOINT"),
Region: os.Getenv("AWS_S3_REGION"),
},
args: args{
DeleteInput{
Bucket: os.Getenv("AWS_S3_BUCKET"),
ObjectKey: "test_metadata.txt",
},
},
wantErr: false,
},
{
name: "Delete avatar.png",
fields: tConfig{