feat: add UseVirtualHostedStyle flag for S3-compatible services

S3-compatible services such as Alibaba Cloud OSS, Tencent COS and Huawei OBS reject path-style URLs with 'SecondLevelDomainForbidden' and only accept virtual-hosted style (https://bucket.endpoint/key). This adds a UseVirtualHostedStyle field and SetVirtualHostedStyle() setter. When enabled and a custom Endpoint is set, the first path segment is treated as the bucket and prepended to the host as a subdomain. AWS S3 and MinIO accept both styles, so this is a no-op for them. Includes tests for OSS-style endpoint, http/https scheme preservation and the empty-path fallback used by ListBuckets-style operations.
2026-08-01 23:42:15 +08:00
parent 1d8c6e314b
commit 9210db974c
3 changed files with 145 additions and 2 deletions
@@ -17,6 +17,12 @@ import (
 // getURL constructs a URL for a given path, with multiple optional
 // arguments as individual subfolders, based on the endpoint
 // specified in s3 struct.
+//
+// When s3.UseVirtualHostedStyle is true and a custom Endpoint is configured,
+// the first path segment is treated as the bucket and is prepended to the
+// host as a subdomain (https://<bucket>.<endpoint>/<rest>). The remaining
+// path segments become the URL path. S3-compatible services that reject
+// path-style URLs (e.g. Alibaba Cloud OSS) require this form.
 func (s3 *S3) getURL(path string, args ...string) (uri string) {
 	if len(args) > 0 {
 		path += "/" + strings.Join(args, "/")
@@ -25,6 +31,11 @@ func (s3 *S3) getURL(path string, args ...string) (uri string) {
 	encodedPath := encodePath(path)

 	if len(s3.Endpoint) > 0 {
+		if s3.UseVirtualHostedStyle && encodedPath != "" {
+			if vhURI, ok := s3.virtualHostedURL(encodedPath); ok {
+				return vhURI
+			}
+		}
 		uri = s3.Endpoint + "/" + encodedPath
 	} else {
 		uri = fmt.Sprintf(s3.URIFormat, s3.Region, encodedPath)
@@ -33,6 +44,42 @@ func (s3 *S3) getURL(path string, args ...string) (uri string) {
 	return uri
 }

+// virtualHostedURL rewrites an "endpoint/bucket/key..." path into a
+// virtual-hosted style URL of the form "scheme://bucket.endpoint/key...".
+// Returns false when the path doesn't carry a bucket (no leading segment)
+// so the caller can fall back to its default URL construction.
+func (s3 *S3) virtualHostedURL(encodedPath string) (string, bool) {
+	parts := strings.SplitN(encodedPath, "/", 2)
+	bucket := parts[0]
+	if bucket == "" {
+		return "", false
+	}
+
+	scheme := "https"
+	host := s3.Endpoint
+	if i := strings.Index(host, "://"); i >= 0 {
+		scheme = host[:i]
+		host = host[i+3:]
+	}
+	// strip userinfo, port-less normalization isn't needed for AWS SigV4 host
+	// signing — the AWS SigV4 canonical host matches the request Host header.
+	if i := strings.Index(host, "/"); i >= 0 {
+		host = host[:i]
+	}
+
+	var b strings.Builder
+	b.WriteString(scheme)
+	b.WriteString("://")
+	b.WriteString(bucket)
+	b.WriteByte('.')
+	b.WriteString(host)
+	if len(parts) > 1 && parts[1] != "" {
+		b.WriteByte('/')
+		b.WriteString(parts[1])
+	}
+	return b.String(), true
+}
+
 func detectFileSize(body io.Seeker) (int64, error) {
 	pos, err := body.Seek(0, 1)
 	if err != nil {
@@ -29,8 +29,17 @@ type S3 struct {
 	Token     string
 	Endpoint  string
 	URIFormat string
-	initMode  string
-	expiry    time.Time
+	// UseVirtualHostedStyle enables virtual-hosted style addressing when a
+	// custom Endpoint is set. In virtual-hosted style the bucket name is
+	// embedded as a subdomain of the endpoint
+	// (e.g. https://mybucket.oss-cn-hangzhou.aliyuncs.com/key) instead of
+	// appearing in the URL path (https://oss-cn-hangzhou.aliyuncs.com/mybucket/key).
+	// S3-compatible services such as Alibaba Cloud OSS, Tencent COS and
+	// some private clouds reject path-style requests with
+	// "SecondLevelDomainForbidden", so this flag is required for them.
+	UseVirtualHostedStyle bool
+	initMode              string
+	expiry                time.Time

 	mu sync.Mutex
 }
@@ -92,6 +101,18 @@ func (s3 *S3) SetClient(client *http.Client) *S3 {
 	return s3
 }

+// SetVirtualHostedStyle toggles virtual-hosted style URL addressing for S3
+// operations against a custom Endpoint. When enabled, the bucket name is
+// embedded as a subdomain of the endpoint
+// (https://mybucket.<endpoint>/<key>). S3-compatible services that reject
+// path-style URLs (Alibaba Cloud OSS, Tencent COS, Huawei OBS, etc.) need
+// this enabled. AWS S3 supports both styles; this is a no-op when no custom
+// Endpoint is set.
+func (s3 *S3) SetVirtualHostedStyle(enabled bool) *S3 {
+	s3.UseVirtualHostedStyle = enabled
+	return s3
+}
+
 func (s3 *S3) signRequest(req *http.Request) error {
 	var (
 		err error
@@ -89,6 +89,81 @@ func TestGetURL(t *testing.T) {
 	}
 }

+func TestVirtualHostedStyle(t *testing.T) {
+	tests := []struct {
+		name     string
+		endpoint string
+		path     string
+		args     []string
+		want     string
+	}{
+		{
+			name:     "OSS-style: bucket + key with custom endpoint, no scheme",
+			endpoint: "oss-cn-hangzhou.aliyuncs.com",
+			path:     "mybucket",
+			args:     []string{"path/to/key.txt"},
+			want:     "https://mybucket.oss-cn-hangzhou.aliyuncs.com/path/to/key.txt",
+		},
+		{
+			name:     "OSS-style: bucket only (e.g. ListObjects)",
+			endpoint: "oss-cn-hangzhou.aliyuncs.com",
+			path:     "mybucket",
+			want:     "https://mybucket.oss-cn-hangzhou.aliyuncs.com",
+		},
+		{
+			name:     "explicit https scheme preserved",
+			endpoint: "https://oss-cn-hangzhou.aliyuncs.com",
+			path:     "mybucket",
+			want:     "https://mybucket.oss-cn-hangzhou.aliyuncs.com",
+		},
+		{
+			name:     "explicit http scheme preserved (for local MinIO)",
+			endpoint: "http://localhost:9000",
+			path:     "mybucket",
+			want:     "http://mybucket.localhost:9000",
+		},
+		{
+			name:     "trailing path on endpoint stripped before subdomain",
+			endpoint: "https://oss-cn-hangzhou.aliyuncs.com/extra/",
+			path:     "mybucket",
+			want:     "https://mybucket.oss-cn-hangzhou.aliyuncs.com",
+		},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			s3 := New("cn-hangzhou", "AccessKey", "SecretKey").
+				SetEndpoint(tt.endpoint).
+				SetVirtualHostedStyle(true)
+			got := s3.getURL(tt.path, tt.args...)
+			if got != tt.want {
+				t.Errorf("virtual-hosted getURL() got = %q, want %q", got, tt.want)
+			}
+		})
+	}
+}
+
+func TestVirtualHostedStyleDefaultOff(t *testing.T) {
+	// Without SetVirtualHostedStyle(true) we must keep the original
+	// path-style behaviour, so the S3-SDK upgrade stays non-breaking.
+	s3 := New("us-east-1", "AccessKey", "SecretKey").
+		SetEndpoint("https://oss-cn-hangzhou.aliyuncs.com")
+	if got := s3.getURL("mybucket", "k.txt"); got != "https://oss-cn-hangzhou.aliyuncs.com/mybucket/k.txt" {
+		t.Errorf("default path-style broken: got %q", got)
+	}
+}
+
+func TestVirtualHostedStyleEmptyPath(t *testing.T) {
+	// When path is empty, virtual-hosted style can't be applied
+	// (no bucket to use as subdomain). Caller still gets the original
+	// path-style URL so ListBuckets-style operations still work.
+	s3 := New("us-east-1", "AccessKey", "SecretKey").
+		SetEndpoint("https://oss-cn-hangzhou.aliyuncs.com").
+		SetVirtualHostedStyle(true)
+	if got := s3.getURL(""); got != "https://oss-cn-hangzhou.aliyuncs.com/" {
+		t.Errorf("empty path should fall back to path-style: got %q", got)
+	}
+}
+
 // Helper functions
 func uploadTestFiles(t *testing.T, s3 *S3, bucket string, filenames []string) {
 	for _, filename := range filenames {