// LICENSE BSD-2-Clause-FreeBSD // Copyright (c) 2018, Rohan Verma package simples3 import ( "bytes" "crypto/hmac" "crypto/sha256" "encoding/hex" "encoding/json" "encoding/xml" "errors" "fmt" "io" "io/ioutil" "mime/multipart" "net/http" "regexp" "strings" "time" "unicode/utf8" ) const ( securityCredentialsURL = "http://169.254.169.254/latest/meta-data/iam/security-credentials/" ) const ( // AMZMetaPrefix to prefix metadata key. AMZMetaPrefix = "x-amz-meta-" ) // S3 provides a wrapper around your S3 credentials. type S3 struct { AccessKey string SecretKey string Region string Client *http.Client Token string Endpoint string URIFormat string } // DownloadInput is passed to FileDownload as a parameter. type DownloadInput struct { Bucket string ObjectKey string } // DetailsInput is passed to FileDetails as a parameter. type DetailsInput struct { Bucket string ObjectKey string } // DetailsResponse is returned by FileDetails. type DetailsResponse struct { ContentType string ContentLength string AcceptRanges string Date string Etag string LastModified string Server string AmzID2 string AmzRequestID string AmzMeta map[string]string ExtraHeaders map[string]string } // UploadInput is passed to FileUpload as a parameter. type UploadInput struct { // essential fields Bucket string ObjectKey string FileName string ContentType string // optional fields ContentDisposition string ACL string // Setting key/value pairs adds user-defined metadata // keys to the object, prefixed with AMZMetaPrefix. CustomMetadata map[string]string Body io.ReadSeeker } // UploadResponse receives the following XML // in case of success, since we set a 201 response from S3. // Sample response: // // https://s3.amazonaws.com/link-to-the-file // s3-bucket // development/8614bd40-691b-4668-9241-3b342c6cf429/image.jpg // "32-bit-tag" // type UploadResponse struct { Location string `xml:"Location"` Bucket string `xml:"Bucket"` Key string `xml:"Key"` ETag string `xml:"ETag"` } // PutResponse is returned when the action is successful, // and the service sends back an HTTP 200 response. The response // returns ETag along with HTTP headers. type PutResponse struct { ETag string Headers http.Header } // DeleteInput is passed to FileDelete as a parameter. type DeleteInput struct { Bucket string ObjectKey string } // IAMResponse is used by NewUsingIAM to auto // detect the credentials. type IAMResponse struct { Code string `json:"Code"` LastUpdated string `json:"LastUpdated"` Type string `json:"Type"` AccessKeyID string `json:"AccessKeyId"` SecretAccessKey string `json:"SecretAccessKey"` Token string `json:"Token"` Expiration string `json:"Expiration"` } // New returns an instance of S3. func New(region, accessKey, secretKey string) *S3 { return &S3{ Region: region, AccessKey: accessKey, SecretKey: secretKey, URIFormat: "https://s3.%s.amazonaws.com/%s", } } // NewUsingIAM automatically generates an Instance of S3 // using instance metatdata. func NewUsingIAM(region string) (*S3, error) { return newUsingIAMImpl(securityCredentialsURL, region) } func newUsingIAMImpl(baseURL, region string) (*S3, error) { // Get the IAM role resp, err := http.Get(baseURL) if err != nil { return nil, err } defer resp.Body.Close() if resp.StatusCode != http.StatusOK { return nil, errors.New(http.StatusText(resp.StatusCode)) } role, err := ioutil.ReadAll(resp.Body) if err != nil { return nil, err } resp, err = http.Get(baseURL + "/" + string(role)) if err != nil { return nil, err } defer resp.Body.Close() if resp.StatusCode != http.StatusOK { return nil, errors.New(http.StatusText(resp.StatusCode)) } var jsonResp IAMResponse jsonString, err := ioutil.ReadAll(resp.Body) if err != nil { return nil, err } if err := json.Unmarshal(jsonString, &jsonResp); err != nil { return nil, err } return &S3{ Region: region, AccessKey: jsonResp.AccessKeyID, SecretKey: jsonResp.SecretAccessKey, Token: jsonResp.Token, URIFormat: "https://s3.%s.amazonaws.com/%s", }, nil } func (s3 *S3) getClient() *http.Client { if s3.Client == nil { return http.DefaultClient } return s3.Client } // getURL constructs a URL for a given path, with multiple optional // arguments as individual subfolders, based on the endpoint // specified in s3 struct. func (s3 *S3) getURL(path string, args ...string) (uri string) { if len(args) > 0 { path += "/" + strings.Join(args, "/") } // need to encode special characters in the path part of the URL encodedPath := encodePath(path) if len(s3.Endpoint) > 0 { uri = s3.Endpoint + "/" + encodedPath } else { uri = fmt.Sprintf(s3.URIFormat, s3.Region, encodedPath) } return uri } // SetEndpoint can be used to the set a custom endpoint for // using an alternate instance compatible with the s3 API. // If no protocol is included in the URI, defaults to HTTPS. func (s3 *S3) SetEndpoint(uri string) *S3 { if len(uri) > 0 { if !strings.HasPrefix(uri, "http") { uri = "https://" + uri } // make sure there is no trailing slash if uri[len(uri)-1] == '/' { uri = uri[:len(uri)-1] } s3.Endpoint = uri } return s3 } // SetToken can be used to set a Temporary Security Credential token obtained from // using an IAM role or AWS STS. func (s3 *S3) SetToken(token string) *S3 { if token != "" { s3.Token = token } return s3 } func detectFileSize(body io.Seeker) (int64, error) { pos, err := body.Seek(0, 1) if err != nil { return -1, err } defer body.Seek(pos, 0) n, err := body.Seek(0, 2) //nolint:gomnd if err != nil { return -1, err } return n, nil } // SetClient can be used to set the http client to be // used by the package. If client passed is nil, // http.DefaultClient is used. func (s3 *S3) SetClient(client *http.Client) *S3 { if client != nil { s3.Client = client } else { s3.Client = http.DefaultClient } return s3 } func (s3 *S3) signRequest(req *http.Request) error { var ( err error date = req.Header.Get("Date") t = time.Now().UTC() ) if date != "" { t, err = time.Parse(http.TimeFormat, date) if err != nil { return err } } req.Header.Set("Date", t.Format(amzDateISO8601TimeFormat)) // The x-amz-content-sha256 header is required for all AWS // Signature Version 4 requests. It provides a hash of the // request payload. If there is no payload, you must provide // the hash of an empty string. if req.Header.Get("x-amz-content-sha256") == "" { emptyhash := "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855" req.Header.Set("x-amz-content-sha256", emptyhash) } k := s3.signKeys(t) h := hmac.New(sha256.New, k) s3.writeStringToSign(h, t, req) auth := bytes.NewBufferString(algorithm) auth.Write([]byte(" Credential=" + s3.AccessKey + "/" + s3.creds(t))) auth.Write([]byte{',', ' '}) auth.Write([]byte("SignedHeaders=")) writeHeaderList(auth, req) auth.Write([]byte{',', ' '}) auth.Write([]byte("Signature=" + fmt.Sprintf("%x", h.Sum(nil)))) req.Header.Set("Authorization", auth.String()) return nil } // FileDownload makes a GET call and returns a io.ReadCloser. // After reading the response body, ensure closing the response. func (s3 *S3) FileDownload(u DownloadInput) (io.ReadCloser, error) { req, err := http.NewRequest( http.MethodGet, s3.getURL(u.Bucket, u.ObjectKey), nil, ) if err != nil { return nil, err } if err := s3.signRequest(req); err != nil { return nil, err } res, err := s3.getClient().Do(req) if err != nil { return nil, err } if res.StatusCode != http.StatusOK { return nil, fmt.Errorf("status code: %s", res.Status) } return res.Body, nil } // FilePut makes a PUT call to S3. func (s3 *S3) FilePut(u UploadInput) (PutResponse, error) { fSize, err := detectFileSize(u.Body) if err != nil { return PutResponse{}, err } content := make([]byte, fSize) _, err = u.Body.Read(content) if err != nil { return PutResponse{}, err } u.Body.Seek(0, 0) req, er := http.NewRequest(http.MethodPut, s3.getURL(u.Bucket, u.ObjectKey), u.Body) if er != nil { return PutResponse{}, err } if u.ContentType == "" { u.ContentType = "application/octet-stream" } h := sha256.New() h.Write(content) req.Header.Set("x-amz-content-sha256", fmt.Sprintf("%x", h.Sum(nil))) req.Header.Set("Content-Type", u.ContentType) req.Header.Set("Content-Length", fmt.Sprintf("%d", fSize)) req.Header.Set("Host", req.URL.Host) for k, v := range u.CustomMetadata { req.Header.Set("x-amz-meta-"+k, v) } if u.ContentDisposition != "" { req.Header.Set("Content-Disposition", u.ContentDisposition) } if u.ACL != "" { req.Header.Set("x-amz-acl", u.ACL) } req.ContentLength = fSize if err := s3.signRequest(req); err != nil { return PutResponse{}, err } // debug(httputil.DumpRequest(req, true)) // Submit the request client := s3.getClient() res, err := client.Do(req) if err != nil { return PutResponse{}, err } defer res.Body.Close() data, err := ioutil.ReadAll(res.Body) if err != nil { return PutResponse{}, err } // Check the response if res.StatusCode != http.StatusOK { return PutResponse{}, fmt.Errorf("status code: %s: %q", res.Status, data) } return PutResponse{ ETag: res.Header.Get("ETag"), Headers: res.Header.Clone(), }, nil } // FileUpload makes a POST call with the file written as multipart // and on successful upload, checks for 200 OK. func (s3 *S3) FileUpload(u UploadInput) (UploadResponse, error) { fSize, err := detectFileSize(u.Body) if err != nil { return UploadResponse{}, err } uc := UploadConfig{ UploadURL: s3.getURL(u.Bucket), BucketName: u.Bucket, ObjectKey: u.ObjectKey, ContentType: u.ContentType, ContentDisposition: u.ContentDisposition, ACL: u.ACL, FileSize: fSize, MetaData: map[string]string{ "success_action_status": "201", // returns XML doc on success }, } // Set custom metadata. for k, v := range u.CustomMetadata { if !strings.HasPrefix(k, AMZMetaPrefix) { k = AMZMetaPrefix + k } uc.MetaData[k] = v } policies, err := s3.CreateUploadPolicies(uc) if err != nil { return UploadResponse{}, err } var b bytes.Buffer w := multipart.NewWriter(&b) for k, v := range policies.Form { if err = w.WriteField(k, v); err != nil { return UploadResponse{}, err } } fw, err := w.CreateFormFile("file", u.FileName) if err != nil { return UploadResponse{}, err } if _, err = io.Copy(fw, u.Body); err != nil { return UploadResponse{}, err } // Don't forget to close the multipart writer. // If you don't close it, your request will be missing the terminating boundary. if err := w.Close(); err != nil { return UploadResponse{}, err } // Now that you have a form, you can submit it to your handler. req, err := http.NewRequest(http.MethodPost, policies.URL, &b) if err != nil { return UploadResponse{}, err } // Don't forget to set the content type, this will contain the boundary. req.Header.Set("Content-Type", w.FormDataContentType()) // Submit the request client := s3.getClient() res, err := client.Do(req) if err != nil { return UploadResponse{}, err } defer res.Body.Close() data, err := ioutil.ReadAll(res.Body) if err != nil { return UploadResponse{}, err } // Check the response if res.StatusCode != http.StatusCreated { return UploadResponse{}, fmt.Errorf("status code: %s: %q", res.Status, data) } var ur UploadResponse xml.Unmarshal(data, &ur) return ur, nil } // FileDelete makes a DELETE call with the file written as multipart // and on successful upload, checks for 204 No Content. func (s3 *S3) FileDelete(u DeleteInput) error { req, err := http.NewRequest( http.MethodDelete, s3.getURL(u.Bucket, u.ObjectKey), nil, ) if err != nil { return err } if err := s3.signRequest(req); err != nil { return err } // Submit the request client := s3.getClient() res, err := client.Do(req) if err != nil { return err } // Check the response if res.StatusCode != http.StatusNoContent { return fmt.Errorf("status code: %s", res.Status) } return nil } func (s3 *S3) FileDetails(u DetailsInput) (DetailsResponse, error) { req, err := http.NewRequest( http.MethodHead, s3.getURL(u.Bucket, u.ObjectKey), nil, ) if err != nil { return DetailsResponse{}, err } if err := s3.signRequest(req); err != nil { return DetailsResponse{}, err } res, err := s3.getClient().Do(req) if err != nil { return DetailsResponse{}, err } if res.StatusCode != http.StatusOK { return DetailsResponse{}, fmt.Errorf("status code: %s", res.Status) } var out DetailsResponse for k, v := range res.Header { lk := strings.ToLower(k) switch lk { case "content-type": out.ContentType = getFirstString(v) case "content-length": out.ContentLength = getFirstString(v) case "accept-ranges": out.AcceptRanges = getFirstString(v) case "date": out.Date = getFirstString(v) case "etag": out.Etag = getFirstString(v) case "last-modified": out.LastModified = getFirstString(v) case "server": out.Server = getFirstString(v) case "x-amz-id-2": out.AmzID2 = getFirstString(v) case "x-amz-request-id": out.AmzRequestID = getFirstString(v) default: if strings.HasPrefix(lk, AMZMetaPrefix) { if out.AmzMeta == nil { out.AmzMeta = map[string]string{} } out.AmzMeta[k] = getFirstString(v) } else { if out.ExtraHeaders == nil { out.ExtraHeaders = map[string]string{} } out.ExtraHeaders[k] = getFirstString(v) } } } return out, nil } func getFirstString(s []string) string { if len(s) > 0 { return s[0] } return "" } // if object matches reserved string, no need to encode them. var reservedObjectNames = regexp.MustCompile("^[a-zA-Z0-9-_.~/]+$") // encodePath encode the strings from UTF-8 byte representations to HTML hex escape sequences // // This is necessary since regular url.Parse() and url.Encode() functions do not support UTF-8 // non english characters cannot be parsed due to the nature in which url.Encode() is written // // This function on the other hand is a direct replacement for url.Encode() technique to support // pretty much every UTF-8 character. // adapted from // https://github.com/minio/minio-go/blob/fe1f3855b146c1b6ce4199740d317e44cf9e85c2/pkg/s3utils/utils.go#L285 func encodePath(pathName string) string { if reservedObjectNames.MatchString(pathName) { return pathName } var encodedPathname strings.Builder for _, s := range pathName { if 'A' <= s && s <= 'Z' || 'a' <= s && s <= 'z' || '0' <= s && s <= '9' { // §2.3 Unreserved characters (mark) encodedPathname.WriteRune(s) continue } switch s { case '-', '_', '.', '~', '/': // §2.3 Unreserved characters (mark) encodedPathname.WriteRune(s) continue default: lenR := utf8.RuneLen(s) if lenR < 0 { // if utf8 cannot convert, return the same string as is return pathName } u := make([]byte, lenR) utf8.EncodeRune(u, s) for _, r := range u { hex := hex.EncodeToString([]byte{r}) encodedPathname.WriteString("%" + strings.ToUpper(hex)) } } } return encodedPathname.String() }