From 9210db974cd6e28703b076b59d298905caf86264 Mon Sep 17 00:00:00 2001 From: ys-ll Date: Sat, 1 Aug 2026 23:42:15 +0800 Subject: [PATCH] feat: add UseVirtualHostedStyle flag for S3-compatible services S3-compatible services such as Alibaba Cloud OSS, Tencent COS and Huawei OBS reject path-style URLs with 'SecondLevelDomainForbidden' and only accept virtual-hosted style (https://bucket.endpoint/key). This adds a UseVirtualHostedStyle field and SetVirtualHostedStyle() setter. When enabled and a custom Endpoint is set, the first path segment is treated as the bucket and prepended to the host as a subdomain. AWS S3 and MinIO accept both styles, so this is a no-op for them. Includes tests for OSS-style endpoint, http/https scheme preservation and the empty-path fallback used by ListBuckets-style operations. --- helpers.go | 47 ++++++++++++++++++++++++++++++ simples3.go | 25 ++++++++++++++-- simples3_test.go | 75 ++++++++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 145 insertions(+), 2 deletions(-) diff --git a/helpers.go b/helpers.go index 701aa92..fb3f568 100644 --- a/helpers.go +++ b/helpers.go @@ -17,6 +17,12 @@ import ( // getURL constructs a URL for a given path, with multiple optional // arguments as individual subfolders, based on the endpoint // specified in s3 struct. +// +// When s3.UseVirtualHostedStyle is true and a custom Endpoint is configured, +// the first path segment is treated as the bucket and is prepended to the +// host as a subdomain (https://./). The remaining +// path segments become the URL path. S3-compatible services that reject +// path-style URLs (e.g. Alibaba Cloud OSS) require this form. func (s3 *S3) getURL(path string, args ...string) (uri string) { if len(args) > 0 { path += "/" + strings.Join(args, "/") @@ -25,6 +31,11 @@ func (s3 *S3) getURL(path string, args ...string) (uri string) { encodedPath := encodePath(path) if len(s3.Endpoint) > 0 { + if s3.UseVirtualHostedStyle && encodedPath != "" { + if vhURI, ok := s3.virtualHostedURL(encodedPath); ok { + return vhURI + } + } uri = s3.Endpoint + "/" + encodedPath } else { uri = fmt.Sprintf(s3.URIFormat, s3.Region, encodedPath) @@ -33,6 +44,42 @@ func (s3 *S3) getURL(path string, args ...string) (uri string) { return uri } +// virtualHostedURL rewrites an "endpoint/bucket/key..." path into a +// virtual-hosted style URL of the form "scheme://bucket.endpoint/key...". +// Returns false when the path doesn't carry a bucket (no leading segment) +// so the caller can fall back to its default URL construction. +func (s3 *S3) virtualHostedURL(encodedPath string) (string, bool) { + parts := strings.SplitN(encodedPath, "/", 2) + bucket := parts[0] + if bucket == "" { + return "", false + } + + scheme := "https" + host := s3.Endpoint + if i := strings.Index(host, "://"); i >= 0 { + scheme = host[:i] + host = host[i+3:] + } + // strip userinfo, port-less normalization isn't needed for AWS SigV4 host + // signing — the AWS SigV4 canonical host matches the request Host header. + if i := strings.Index(host, "/"); i >= 0 { + host = host[:i] + } + + var b strings.Builder + b.WriteString(scheme) + b.WriteString("://") + b.WriteString(bucket) + b.WriteByte('.') + b.WriteString(host) + if len(parts) > 1 && parts[1] != "" { + b.WriteByte('/') + b.WriteString(parts[1]) + } + return b.String(), true +} + func detectFileSize(body io.Seeker) (int64, error) { pos, err := body.Seek(0, 1) if err != nil { diff --git a/simples3.go b/simples3.go index 192e4ee..ec22cc4 100644 --- a/simples3.go +++ b/simples3.go @@ -29,8 +29,17 @@ type S3 struct { Token string Endpoint string URIFormat string - initMode string - expiry time.Time + // UseVirtualHostedStyle enables virtual-hosted style addressing when a + // custom Endpoint is set. In virtual-hosted style the bucket name is + // embedded as a subdomain of the endpoint + // (e.g. https://mybucket.oss-cn-hangzhou.aliyuncs.com/key) instead of + // appearing in the URL path (https://oss-cn-hangzhou.aliyuncs.com/mybucket/key). + // S3-compatible services such as Alibaba Cloud OSS, Tencent COS and + // some private clouds reject path-style requests with + // "SecondLevelDomainForbidden", so this flag is required for them. + UseVirtualHostedStyle bool + initMode string + expiry time.Time mu sync.Mutex } @@ -92,6 +101,18 @@ func (s3 *S3) SetClient(client *http.Client) *S3 { return s3 } +// SetVirtualHostedStyle toggles virtual-hosted style URL addressing for S3 +// operations against a custom Endpoint. When enabled, the bucket name is +// embedded as a subdomain of the endpoint +// (https://mybucket./). S3-compatible services that reject +// path-style URLs (Alibaba Cloud OSS, Tencent COS, Huawei OBS, etc.) need +// this enabled. AWS S3 supports both styles; this is a no-op when no custom +// Endpoint is set. +func (s3 *S3) SetVirtualHostedStyle(enabled bool) *S3 { + s3.UseVirtualHostedStyle = enabled + return s3 +} + func (s3 *S3) signRequest(req *http.Request) error { var ( err error diff --git a/simples3_test.go b/simples3_test.go index 5c64c97..41cb063 100644 --- a/simples3_test.go +++ b/simples3_test.go @@ -89,6 +89,81 @@ func TestGetURL(t *testing.T) { } } +func TestVirtualHostedStyle(t *testing.T) { + tests := []struct { + name string + endpoint string + path string + args []string + want string + }{ + { + name: "OSS-style: bucket + key with custom endpoint, no scheme", + endpoint: "oss-cn-hangzhou.aliyuncs.com", + path: "mybucket", + args: []string{"path/to/key.txt"}, + want: "https://mybucket.oss-cn-hangzhou.aliyuncs.com/path/to/key.txt", + }, + { + name: "OSS-style: bucket only (e.g. ListObjects)", + endpoint: "oss-cn-hangzhou.aliyuncs.com", + path: "mybucket", + want: "https://mybucket.oss-cn-hangzhou.aliyuncs.com", + }, + { + name: "explicit https scheme preserved", + endpoint: "https://oss-cn-hangzhou.aliyuncs.com", + path: "mybucket", + want: "https://mybucket.oss-cn-hangzhou.aliyuncs.com", + }, + { + name: "explicit http scheme preserved (for local MinIO)", + endpoint: "http://localhost:9000", + path: "mybucket", + want: "http://mybucket.localhost:9000", + }, + { + name: "trailing path on endpoint stripped before subdomain", + endpoint: "https://oss-cn-hangzhou.aliyuncs.com/extra/", + path: "mybucket", + want: "https://mybucket.oss-cn-hangzhou.aliyuncs.com", + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + s3 := New("cn-hangzhou", "AccessKey", "SecretKey"). + SetEndpoint(tt.endpoint). + SetVirtualHostedStyle(true) + got := s3.getURL(tt.path, tt.args...) + if got != tt.want { + t.Errorf("virtual-hosted getURL() got = %q, want %q", got, tt.want) + } + }) + } +} + +func TestVirtualHostedStyleDefaultOff(t *testing.T) { + // Without SetVirtualHostedStyle(true) we must keep the original + // path-style behaviour, so the S3-SDK upgrade stays non-breaking. + s3 := New("us-east-1", "AccessKey", "SecretKey"). + SetEndpoint("https://oss-cn-hangzhou.aliyuncs.com") + if got := s3.getURL("mybucket", "k.txt"); got != "https://oss-cn-hangzhou.aliyuncs.com/mybucket/k.txt" { + t.Errorf("default path-style broken: got %q", got) + } +} + +func TestVirtualHostedStyleEmptyPath(t *testing.T) { + // When path is empty, virtual-hosted style can't be applied + // (no bucket to use as subdomain). Caller still gets the original + // path-style URL so ListBuckets-style operations still work. + s3 := New("us-east-1", "AccessKey", "SecretKey"). + SetEndpoint("https://oss-cn-hangzhou.aliyuncs.com"). + SetVirtualHostedStyle(true) + if got := s3.getURL(""); got != "https://oss-cn-hangzhou.aliyuncs.com/" { + t.Errorf("empty path should fall back to path-style: got %q", got) + } +} + // Helper functions func uploadTestFiles(t *testing.T, s3 *S3, bucket string, filenames []string) { for _, filename := range filenames {