feat: implement Simple List API for S3 objects

- Add List method with ListInput and ListResponse structs following simples3 patterns - Implement pagination, prefix filtering, and delimiter grouping - Add comprehensive test suite for List functionality - Update tests to use environment variables instead of hardcoded values - Fix deprecated io/ioutil usage throughout codebase - Add comprehensive README with API documentation and examples - Add Nix development environment with flake.nix - Add Justfile for modern command runner - Add Docker Compose setup for local MinIO testing - Update GitHub Actions to use Nix and Justfile for CI consistency - Fix test naming and setup flow issues New API: s3.List(ListInput{ Bucket: "my-bucket", Prefix: "documents/", MaxKeys: 100, }) All tests pass with new simplified API design.
2025-09-18 16:17:08 +05:30
parent e5f20476ea
commit f661a85ff3
12 changed files with 960 additions and 95 deletions
@@ -12,9 +12,9 @@ import (
 	"encoding/xml"
 	"fmt"
 	"io"
-	"io/ioutil"
 	"mime/multipart"
 	"net/http"
+	"net/url"
 	"regexp"
 	"strings"
 	"sync"
@@ -126,6 +126,80 @@ type DeleteInput struct {
 	ObjectKey string
 }

+// ListInput is passed to List as a parameter.
+type ListInput struct {
+	// Required: The name of the bucket to list objects from
+	Bucket string
+
+	// Optional: A delimiter to group objects by (commonly "/")
+	Delimiter string
+
+	// Optional: Only list objects starting with this prefix
+	Prefix string
+
+	// Optional: Maximum number of objects to return
+	MaxKeys int64
+
+	// Optional: Token for pagination from a previous request
+	ContinuationToken string
+
+	// Optional: Object key to start listing after
+	StartAfter string
+}
+
+// ListResponse is returned by List.
+type ListResponse struct {
+	// Name of the bucket
+	Name string
+
+	// Whether the results were truncated (more results available)
+	IsTruncated bool
+
+	// Token to get the next page of results (if truncated)
+	NextContinuationToken string
+
+	// List of objects in the bucket
+	Objects []Object
+
+	// Common prefixes when using delimiter (like directories)
+	CommonPrefixes []string
+
+	// Total number of keys returned
+	KeyCount int64
+}
+
+// Object represents an S3 object in a bucket.
+type Object struct {
+	// Name of the object
+	Key string
+
+	// Size in bytes
+	Size int64
+
+	// When the object was last modified
+	LastModified string
+
+	// Entity tag of the object
+	ETag string
+
+	// Storage class (e.g., "STANDARD")
+	StorageClass string
+}
+
+// S3Error represents an S3 API error response
+type S3Error struct {
+	XMLName   xml.Name `xml:"Error"`
+	Code      string   `xml:"Code"`
+	Message   string   `xml:"Message"`
+	RequestID string   `xml:"RequestId"`
+	HostID    string   `xml:"HostId"`
+}
+
+// Error returns a string representation of the S3Error
+func (e S3Error) Error() string {
+	return fmt.Sprintf("S3 Error: %s - %s", e.Code, e.Message)
+}
+
 // IAMResponse is used by NewUsingIAM to auto
 // detect the credentials.
 type IAMResponse struct {
@@ -179,14 +253,14 @@ func fetchIMDSToken(cl *http.Client, baseURL string) (string, bool, error) {

 	defer func() {
 		resp.Body.Close()
-		io.Copy(ioutil.Discard, resp.Body)
+		io.Copy(io.Discard, resp.Body)
 	}()

 	if resp.StatusCode != http.StatusOK {
 		return "", false, fmt.Errorf("failed to request IMDSv2 token: %s", resp.Status)
 	}

-	token, err := ioutil.ReadAll(resp.Body)
+	token, err := io.ReadAll(resp.Body)
 	if err != nil {
 		return "", false, err
 	}
@@ -224,12 +298,12 @@ func fetchIAMData(cl *http.Client, baseURL string) (IAMResponse, error) {
 		return IAMResponse{}, fmt.Errorf("error fetching IAM data: %s", resp.Status)
 	}

-	role, err := ioutil.ReadAll(resp.Body)
+	role, err := io.ReadAll(resp.Body)
 	if err != nil {
 		return IAMResponse{}, err
 	}

-	_, _ = io.Copy(ioutil.Discard, resp.Body)
+	_, _ = io.Copy(io.Discard, resp.Body)
 	_ = resp.Body.Close()

 	req, err = http.NewRequest(http.MethodGet, url+string(role), nil)
@@ -247,7 +321,7 @@ func fetchIAMData(cl *http.Client, baseURL string) (IAMResponse, error) {

 	defer func() {
 		// Drain and close the body to let the Transport reuse the connection
-		io.Copy(ioutil.Discard, resp.Body)
+		io.Copy(io.Discard, resp.Body)
 		resp.Body.Close()
 	}()

@@ -256,7 +330,7 @@ func fetchIAMData(cl *http.Client, baseURL string) (IAMResponse, error) {
 	}

 	var jResp IAMResponse
-	jsonString, err := ioutil.ReadAll(resp.Body)
+	jsonString, err := io.ReadAll(resp.Body)
 	if err != nil {
 		return IAMResponse{}, fmt.Errorf("error reading role data: %w", err)
 	}
@@ -536,10 +610,10 @@ func (s3 *S3) FilePut(u UploadInput) (PutResponse, error) {

 	defer func() {
 		res.Body.Close()
-		io.Copy(ioutil.Discard, res.Body)
+		io.Copy(io.Discard, res.Body)
 	}()

-	data, err := ioutil.ReadAll(res.Body)
+	data, err := io.ReadAll(res.Body)
 	if err != nil {
 		return PutResponse{}, err
 	}
@@ -635,10 +709,10 @@ func (s3 *S3) FileUpload(u UploadInput) (UploadResponse, error) {

 	defer func() {
 		res.Body.Close()
-		io.Copy(ioutil.Discard, res.Body)
+		io.Copy(io.Discard, res.Body)
 	}()

-	data, err := ioutil.ReadAll(res.Body)
+	data, err := io.ReadAll(res.Body)
 	if err != nil {
 		return UploadResponse{}, err
 	}
@@ -680,7 +754,7 @@ func (s3 *S3) FileDelete(u DeleteInput) error {

 	defer func() {
 		res.Body.Close()
-		io.Copy(ioutil.Discard, res.Body)
+		io.Copy(io.Discard, res.Body)
 	}()

 	// Check the response
@@ -714,7 +788,7 @@ func (s3 *S3) FileDetails(u DetailsInput) (DetailsResponse, error) {

 	defer func() {
 		res.Body.Close()
-		io.Copy(ioutil.Discard, res.Body)
+		io.Copy(io.Discard, res.Body)
 	}()

 	if res.StatusCode != http.StatusOK {
@@ -764,6 +838,141 @@ func (s3 *S3) FileDetails(u DetailsInput) (DetailsResponse, error) {
 	return out, nil
 }

+// List implements a simple S3 object listing API
+func (s3 *S3) List(input ListInput) (ListResponse, error) {
+	// Input validation
+	if input.Bucket == "" {
+		return ListResponse{}, fmt.Errorf("bucket name cannot be empty")
+	}
+	if input.MaxKeys < 0 {
+		return ListResponse{}, fmt.Errorf("MaxKeys cannot be negative")
+	}
+	if input.MaxKeys > 1000 {
+		return ListResponse{}, fmt.Errorf("MaxKeys cannot exceed 1000")
+	}
+
+	// Build query parameters - ListObjectsV2 uses query params, not path
+	query := url.Values{}
+	query.Set("list-type", "2") // Required parameter
+
+	// Add optional parameters if they exist
+	if input.ContinuationToken != "" {
+		query.Set("continuation-token", input.ContinuationToken)
+	}
+	if input.Delimiter != "" {
+		query.Set("delimiter", input.Delimiter)
+	}
+	if input.MaxKeys > 0 {
+		query.Set("max-keys", fmt.Sprintf("%d", input.MaxKeys))
+	}
+	if input.Prefix != "" {
+		query.Set("prefix", input.Prefix)
+	}
+	if input.StartAfter != "" {
+		query.Set("start-after", input.StartAfter)
+	}
+
+	// Build base URL
+	baseURL := s3.getURL(input.Bucket)
+
+	// Parse URL and add query parameters
+	parsedURL, err := url.Parse(baseURL)
+	if err != nil {
+		return ListResponse{}, err
+	}
+	parsedURL.RawQuery = query.Encode()
+
+	// Create HTTP request
+	req, err := http.NewRequest(http.MethodGet, parsedURL.String(), nil)
+	if err != nil {
+		return ListResponse{}, err
+	}
+
+	// Apply AWS V4 signing
+	if err := s3.renewIAMToken(); err != nil {
+		return ListResponse{}, err
+	}
+	if err := s3.signRequest(req); err != nil {
+		return ListResponse{}, err
+	}
+
+	// Execute request
+	client := s3.getClient()
+	res, err := client.Do(req)
+	if err != nil {
+		return ListResponse{}, err
+	}
+
+	// Close response body when done
+	defer res.Body.Close()
+
+	// Read response body once
+	body, err := io.ReadAll(res.Body)
+	if err != nil {
+		return ListResponse{}, err
+	}
+
+	// Handle response status codes
+	if res.StatusCode != http.StatusOK {
+		var s3Err S3Error
+		if xmlErr := xml.Unmarshal(body, &s3Err); xmlErr == nil {
+			return ListResponse{}, fmt.Errorf("S3 Error: %s - %s", s3Err.Code, s3Err.Message)
+		}
+		return ListResponse{}, fmt.Errorf("status code: %s: %s", res.Status, string(body))
+	}
+
+	// Parse XML response using internal struct
+	var xmlResult struct {
+		XMLName              xml.Name `xml:"ListBucketResult"`
+		Name                 string   `xml:"Name"`
+		IsTruncated          bool     `xml:"IsTruncated"`
+		NextContinuationToken string   `xml:"NextContinuationToken"`
+		KeyCount             int64    `xml:"KeyCount"`
+		Contents             []struct {
+			Key          string `xml:"Key"`
+			LastModified string `xml:"LastModified"`
+			ETag         string `xml:"ETag"`
+			Size         int64  `xml:"Size"`
+			StorageClass string `xml:"StorageClass"`
+		} `xml:"Contents"`
+		CommonPrefixes []struct {
+			Prefix string `xml:"Prefix"`
+		} `xml:"CommonPrefixes"`
+	}
+
+	if err := xml.Unmarshal(body, &xmlResult); err != nil {
+		return ListResponse{}, err
+	}
+
+	// Convert to simple response format
+	response := ListResponse{
+		Name:                  xmlResult.Name,
+		IsTruncated:           xmlResult.IsTruncated,
+		NextContinuationToken: xmlResult.NextContinuationToken,
+		KeyCount:             xmlResult.KeyCount,
+	}
+
+	// Convert objects
+	for _, obj := range xmlResult.Contents {
+		response.Objects = append(response.Objects, Object{
+			Key:          obj.Key,
+			Size:         obj.Size,
+			LastModified: obj.LastModified,
+			ETag:         obj.ETag,
+			StorageClass: obj.StorageClass,
+		})
+	}
+
+	// Convert common prefixes
+	for _, prefix := range xmlResult.CommonPrefixes {
+		response.CommonPrefixes = append(response.CommonPrefixes, prefix.Prefix)
+	}
+
+	return response, nil
+}
+
+
+
 func getFirstString(s []string) string {
 	if len(s) > 0 {
 		return s[0]