feat: implement Simple List API for S3 objects

- Add List method with ListInput and ListResponse structs following simples3 patterns
- Implement pagination, prefix filtering, and delimiter grouping
- Add comprehensive test suite for List functionality
- Update tests to use environment variables instead of hardcoded values
- Fix deprecated io/ioutil usage throughout codebase
- Add comprehensive README with API documentation and examples
- Add Nix development environment with flake.nix
- Add Justfile for modern command runner
- Add Docker Compose setup for local MinIO testing
- Update GitHub Actions to use Nix and Justfile for CI consistency
- Fix test naming and setup flow issues

New API:
s3.List(ListInput{
    Bucket: "my-bucket",
    Prefix: "documents/",
    MaxKeys: 100,
})

All tests pass with new simplified API design.
This commit is contained in:
Rohan Verma
2025-09-18 16:17:08 +05:30
parent e5f20476ea
commit f661a85ff3
12 changed files with 960 additions and 95 deletions
+4
View File
@@ -0,0 +1,4 @@
# Load the Nix development environment
use flake
echo "✅ Environment loaded - Use 'just --list' to see available commands"
+14 -26
View File
@@ -10,31 +10,19 @@ jobs:
test: test:
runs-on: ubuntu-latest runs-on: ubuntu-latest
steps: steps:
- uses: actions/checkout@v2 - name: Checkout code
uses: actions/checkout@v4
- name: Set up Go - name: Install Nix
uses: actions/setup-go@v2 uses: cachix/install-nix-action@v27
with: with:
go-version: 1.15 nix_path: nixpkgs=channel:nixos-unstable
- name: Setup minio
run: |
docker run -d -p 9000:9000 --name minio \
-e "MINIO_ACCESS_KEY=minioadmin" \
-e "MINIO_SECRET_KEY=minioadmin" \
-v /tmp/data:/data \
-v /tmp/config:/root/.minio \
minio/minio server /data
export AWS_ACCESS_KEY_ID=minioadmin - name: Run Tests via Justfile
export AWS_SECRET_ACCESS_KEY=minioadmin env:
export AWS_EC2_METADATA_DISABLED=true AWS_S3_BUCKET: testbucket
AWS_ACCESS_KEY_ID: minioadmin
aws --endpoint-url http://127.0.0.1:9000/ s3 mb s3://testbucket AWS_SECRET_ACCESS_KEY: minioadmin
AWS_S3_ENDPOINT: http://127.0.0.1:9000
- name: Run Go Test AWS_EC2_METADATA_DISABLED: "true"
run: go test -v ./... run: nix develop --command just test-local
env:
AWS_S3_BUCKET: testbucket
AWS_S3_ACCESS_KEY: minioadmin
AWS_S3_SECRET_KEY: minioadmin
AWS_S3_ENDPOINT: http://127.0.0.1:9000
+3
View File
@@ -24,3 +24,6 @@ tsconfig.json
jsconfig.json jsconfig.json
.env .env
# direnv
.direnv
+92
View File
@@ -0,0 +1,92 @@
# SimpleS3 Development Commands
# Variables
MINIO_CONTAINER_NAME := "simples3-minio"
MINIO_DATA_DIR := ".minio-data"
AWS_S3_BUCKET := "testbucket"
# Default command - lists all available recipes
default:
@just --list
# --- Development Commands ---
test:
@echo "🧪 Running all tests..."
@go test -v ./...
test-local: setup
@echo "🧪 Running tests with local MinIO..."
@sleep 2
@go test -v ./...
# --- Go Module Management ---
tidy:
@echo "📦 Tidying Go modules..."
@go mod tidy
fmt:
@echo "🎨 Formatting Go code..."
@go fmt ./...
vet:
@echo "🔍 Running go vet..."
@go vet ./...
# --- MinIO Management ---
minio-up:
@echo "🚀 Starting MinIO container..."
@docker compose up -d
@echo "✅ MinIO started:"
@echo " API: http://localhost:9000"
@echo " Console: http://localhost:9001"
@echo " Access Key: minioadmin"
@echo " Secret Key: minioadmin"
minio-down:
@echo "🛑 Stopping MinIO container..."
@docker compose down
minio-logs:
@echo "📋 Showing MinIO logs..."
@docker compose logs -f
minio-clean:
@echo "🧹 Cleaning MinIO data..."
@docker compose down --volumes
minio-reset: minio-clean minio-up
# --- Development Environment Setup ---
setup: minio-up
@echo "⚙️ Setting up development environment..."
@sleep 3
@aws --endpoint-url http://127.0.0.1:9000/ s3 mb s3://{{AWS_S3_BUCKET}} || true
@echo "✅ Development environment ready!"
dev-env: setup
@echo "🎯 Development environment active!"
@echo " MinIO: http://localhost:9000"
@echo " Console: http://localhost:9001"
@echo " Bucket: {{AWS_S3_BUCKET}}"
# --- Cleanup Commands ---
clean:
@echo "🧹 Cleaning up development environment..."
@docker compose down --volumes
# --- Helper Commands ---
status:
@echo "📊 Status Check:"
@echo " MinIO Container: $(docker ps -q -f name={{MINIO_CONTAINER_NAME}} | wc -l | tr -d ' ') running"
@if [ "$(docker ps -q -f name={{MINIO_CONTAINER_NAME}})" ]; then \
echo " MinIO URL: http://localhost:9000"; \
echo " Console URL: http://localhost:9001"; \
fi
@echo " Go version: $(go version | awk '{print $3}')"
# --- Documentation ---
docs:
@echo "📖 Opening documentation..."
@echo " Plan: PLAN.md"
@echo " README: README.md"
+294 -42
View File
@@ -6,77 +6,329 @@ SimpleS3 is a Go library for manipulating objects
in S3 buckets using REST API calls or Presigned URLs signed in S3 buckets using REST API calls or Presigned URLs signed
using AWS Signature Version 4. using AWS Signature Version 4.
**Features:**
- 🚀 **Simple, intuitive API** following Go idioms
- 🪣 **Complete S3 operations** - Upload, Download, Delete, List, Details
- 🔐 **AWS Signature Version 4** signing
- 🌐 **Custom endpoint support** (MinIO, DigitalOcean Spaces, etc.)
- 📋 **Simple List API** with pagination, prefix filtering, and delimiter grouping
- 🔗 **Presigned URL generation** for secure browser uploads/downloads
- 🪪 **IAM credential support** for EC2 instances
- ✅ **Comprehensive test coverage**
- 🎯 **Zero dependencies** - uses only Go standard library
## Install ## Install
```sh ```sh
go get github.com/rhnvrm/simples3 go get github.com/rhnvrm/simples3
``` ```
## Example ## Quick Start
```go ```go
testTxt, _ := os.Open("testdata/test.txt") package main
defer testTxt.Close()
// Create an instance of the package import (
// You can either create by manually supplying credentials "log"
// (preferably using Environment vars) "os"
s3 := simples3.New(Region, AWSAccessKey, AWSSecretKey) "github.com/rhnvrm/simples3"
// or you can use this on an EC2 instance to )
// obtain credentials from IAM attached to the instance.
s3, _ := simples3.NewUsingIAM(Region)
// You can also set a custom endpoint to a compatible s3 instance. func main() {
s3.SetEndpoint(CustomEndpoint) // Initialize S3 client
s3 := simples3.New("us-east-1", "your-access-key", "your-secret-key")
// Note: Consider adding a testTxt.Seek(0, 0) // Use MinIO or other S3-compatible services
// in case you have read s3.SetEndpoint("https://s3.amazonaws.com")
// the body, as the pointer is shared by the library.
// File Upload is as simple as providing the following // Upload a file
// details. file, _ := os.Open("my-file.txt")
defer file.Close()
resp, err := s3.FileUpload(simples3.UploadInput{
Bucket: "my-bucket",
ObjectKey: "my-file.txt",
ContentType: "text/plain",
FileName: "my-file.txt",
Body: file,
})
if err != nil {
log.Fatal(err)
}
log.Printf("File uploaded successfully: %+v", resp)
}
```
## API Reference
### File Operations
#### Upload Files
```go
// POST upload (recommended for browsers)
resp, err := s3.FileUpload(simples3.UploadInput{ resp, err := s3.FileUpload(simples3.UploadInput{
Bucket: AWSBucket, Bucket: "my-bucket",
ObjectKey: "test.txt", ObjectKey: "path/to/file.txt",
ContentType: "text/plain", ContentType: "text/plain",
FileName: "test.txt", FileName: "file.txt",
Body: testTxt, Body: file,
}) })
// Similarly, Files can be deleted. // PUT upload (simpler for programmatic use)
resp, err := s3.FilePut(simples3.UploadInput{
Bucket: "my-bucket",
ObjectKey: "path/to/file.txt",
ContentType: "text/plain",
Body: file,
})
```
#### Download Files
```go
// Download file
file, err := s3.FileDownload(simples3.DownloadInput{
Bucket: "my-bucket",
ObjectKey: "path/to/file.txt",
})
if err != nil {
log.Fatal(err)
}
defer file.Close()
// Read the content
data, err := io.ReadAll(file)
```
#### Delete Files
```go
err := s3.FileDelete(simples3.DeleteInput{ err := s3.FileDelete(simples3.DeleteInput{
Bucket: os.Getenv("AWS_S3_BUCKET"), Bucket: "my-bucket",
ObjectKey: "test.txt", ObjectKey: "path/to/file.txt",
}) })
```
// You can also download the file. #### Get File Details
file, _ := s3.FileDownload(simples3.DownloadInput{ ```go
Bucket: AWSBucket, details, err := s3.FileDetails(simples3.DetailsInput{
ObjectKey: "test.txt", Bucket: "my-bucket",
ObjectKey: "path/to/file.txt",
}) })
if err != nil {
log.Fatal(err)
}
data, _ := ioutil.ReadAll(file) log.Printf("File size: %s bytes", details.ContentLength)
file.Close() log.Printf("Last modified: %s", details.LastModified)
log.Printf("Content type: %s", details.ContentType)
```
// You can also use this library to generate ### List Objects
// Presigned URLs that can for eg. be used to
// GET/PUT files on S3 through the browser.
var time, _ = time.Parse(time.RFC1123, "Fri, 24 May 2013 00:00:00 GMT")
url := s.GeneratePresignedURL(PresignedInput{ SimpleS3 provides a clean, easy-to-use List API that follows the same pattern as other library methods:
Bucket: AWSBucket,
ObjectKey: "test.txt", #### Simple Listing
```go
// List all objects in a bucket
result, err := s3.List(simples3.ListInput{
Bucket: "my-bucket",
})
if err != nil {
log.Fatal(err)
}
fmt.Printf("Found %d objects:\n", len(result.Objects))
for _, obj := range result.Objects {
fmt.Printf("- %s (%d bytes)\n", obj.Key, obj.Size)
}
```
#### Advanced Listing with Options
```go
// List with prefix filtering and pagination
result, err := s3.List(simples3.ListInput{
Bucket: "my-bucket",
Prefix: "documents/",
Delimiter: "/",
MaxKeys: 100,
})
if err != nil {
log.Fatal(err)
}
// Process objects
for _, obj := range result.Objects {
fmt.Printf("📄 %s (%d bytes)\n", obj.Key, obj.Size)
}
// Process "directories" (common prefixes)
for _, prefix := range result.CommonPrefixes {
fmt.Printf("📁 %s/\n", prefix)
}
```
#### Handle Pagination
```go
// Handle large result sets with pagination
func listAllObjects(s3 *simples3.S3, bucket string) ([]simples3.Object, error) {
var allObjects []simples3.Object
var continuationToken string
for {
// List objects with continuation token
result, err := s3.List(simples3.ListInput{
Bucket: bucket,
ContinuationToken: continuationToken,
MaxKeys: 1000, // Maximum page size
})
if err != nil {
return nil, err
}
// Add current page objects to our collection
allObjects = append(allObjects, result.Objects...)
// Check if there are more pages
if !result.IsTruncated {
break
}
// Set token for next page
continuationToken = result.NextContinuationToken
}
return allObjects, nil
}
// Usage example
func main() {
s3 := simples3.New("us-east-1", "your-access-key", "your-secret-key")
allObjects, err := listAllObjects(s3, "my-bucket")
if err != nil {
log.Fatal(err)
}
fmt.Printf("Total objects: %d\\n", len(allObjects))
for _, obj := range allObjects {
fmt.Printf("- %s (%d bytes)\\n", obj.Key, obj.Size)
}
}
```
### Presigned URLs
Generate secure URLs for browser-based uploads/downloads:
```go
// Generate presigned URL for download
url := s3.GeneratePresignedURL(simples3.PresignedInput{
Bucket: "my-bucket",
ObjectKey: "private-file.pdf",
Method: "GET", Method: "GET",
Timestamp: time, ExpirySeconds: 3600, // 1 hour
ExpirySeconds: 86400,
}) })
// Users can now download directly: <a href="{{url}}">Download</a>
```
### Custom Endpoints
Use with MinIO, DigitalOcean Spaces, or other S3-compatible services:
```go
// MinIO
s3 := simples3.New("us-east-1", "minioadmin", "minioadmin")
s3.SetEndpoint("http://localhost:9000")
// DigitalOcean Spaces
s3 := simples3.New("nyc3", "your-access-key", "your-secret-key")
s3.SetEndpoint("https://nyc3.digitaloceanspaces.com")
```
### IAM Credentials
On EC2 instances, use IAM roles automatically:
```go
s3, err := simples3.NewUsingIAM("us-east-1")
if err != nil {
log.Fatal(err)
}
// Automatically uses instance IAM role
```
## Development
### Setup Development Environment
```sh
# Clone the repository
git clone https://github.com/rhnvrm/simples3.git
cd simples3
# Using Nix (recommended)
nix develop
# Or using direnv
direnv allow
# Start local MinIO for testing
just setup
# Run tests
just test-local
```
### Testing
The library includes comprehensive tests that run against a local MinIO instance:
```sh
# Run all tests (without MinIO)
just test
# Run tests with local MinIO
just test-local
# Run specific test
go test -v -run TestList
```
### Available Commands
```sh
just # List all commands
just test # Run tests without MinIO
just test-local # Run tests with MinIO (includes setup)
just setup # Setup development environment
just minio-up # Start MinIO container
just minio-down # Stop MinIO container
just clean # Clean up everything
just status # Check development environment status
```
## Environment Variables
For development and testing:
```sh
export AWS_S3_REGION="us-east-1"
export AWS_S3_ACCESS_KEY="minioadmin"
export AWS_S3_SECRET_KEY="minioadmin"
export AWS_S3_ENDPOINT="http://localhost:9000"
export AWS_S3_BUCKET="testbucket"
``` ```
## Contributing ## Contributing
You are more than welcome to contribute to this project. Fork and make We welcome contributions! Please:
a Pull Request, or create an Issue if you see any problem or want to
propose a feature. 1. Fork the repository
2. Create a feature branch
3. Add tests for new functionality
4. Ensure all tests pass: `just test-local`
5. Submit a pull request
## Author ## Author
+21
View File
@@ -0,0 +1,21 @@
services:
minio:
image: minio/minio:latest
container_name: simples3-minio
ports:
- "9000:9000" # API
- "9001:9001" # Console
environment:
- MINIO_ROOT_USER=minioadmin
- MINIO_ROOT_PASSWORD=minioadmin
volumes:
- minio_data:/data
command: server /data --console-address ":9001"
healthcheck:
test: ["CMD", "curl", "-f", "http://localhost:9000/minio/health/live"]
interval: 30s
timeout: 20s
retries: 3
volumes:
minio_data:
Generated
+27
View File
@@ -0,0 +1,27 @@
{
"nodes": {
"nixpkgs": {
"locked": {
"lastModified": 1757745802,
"narHash": "sha256-hLEO2TPj55KcUFUU1vgtHE9UEIOjRcH/4QbmfHNF820=",
"owner": "NixOS",
"repo": "nixpkgs",
"rev": "c23193b943c6c689d70ee98ce3128239ed9e32d1",
"type": "github"
},
"original": {
"owner": "NixOS",
"ref": "nixos-unstable",
"repo": "nixpkgs",
"type": "github"
}
},
"root": {
"inputs": {
"nixpkgs": "nixpkgs"
}
}
},
"root": "root",
"version": 7
}
+62
View File
@@ -0,0 +1,62 @@
{
description = "Go project for simples3 with ListObjectsV2 support";
inputs = {
nixpkgs.url = "github:NixOS/nixpkgs/nixos-unstable";
};
outputs = { self, nixpkgs }:
let
supportedSystems = [ "x86_64-linux" "aarch64-linux" "x86_64-darwin" "aarch64-darwin" ];
forEachSystem = f: nixpkgs.lib.genAttrs supportedSystems (system: f {
pkgs = import nixpkgs {
inherit system;
};
});
in
{
devShells = forEachSystem ({ pkgs }: {
default = pkgs.mkShell {
buildInputs = with pkgs; [
# Go development
go
gopls
go-outline
delve
# Build tools and utilities
just
docker
awscli2
# Additional utilities
curl
jq
];
# Environment variables for the shell
shellHook = ''
echo "🚀 SimpleS3 Development Environment Loaded"
echo "Go version: $(go version)"
echo "Available commands: just --list"
export GOPATH=$(go env GOPATH)
export PATH=$PATH:$GOPATH/bin
# Local S3 (MinIO) configuration for testing
export AWS_S3_BUCKET=testbucket
export AWS_S3_ENDPOINT=http://localhost:9000
export AWS_S3_REGION=us-east-1
export AWS_EC2_METADATA_DISABLED=true
# AWS CLI credentials for MinIO
export AWS_ACCESS_KEY_ID=minioadmin
export AWS_SECRET_ACCESS_KEY=minioadmin
# Simples3 library credentials
export AWS_S3_ACCESS_KEY=minioadmin
export AWS_S3_SECRET_KEY=minioadmin
'';
};
});
};
}
+1 -1
View File
@@ -1,3 +1,3 @@
module github.com/rhnvrm/simples3 module github.com/rhnvrm/simples3
go 1.13 go 1.25.0
+2 -2
View File
@@ -152,7 +152,7 @@ func buildUploadSign(nowTime time.Time, credential string, uploadConfig UploadCo
}) })
} }
func (s3 S3) buildCredential(nowTime time.Time) []byte { func (s3 *S3) buildCredential(nowTime time.Time) []byte {
var b bytes.Buffer var b bytes.Buffer
b.WriteString(s3.AccessKey) b.WriteString(s3.AccessKey)
b.WriteRune('/') b.WriteRune('/')
@@ -166,7 +166,7 @@ func (s3 S3) buildCredential(nowTime time.Time) []byte {
return b.Bytes() return b.Bytes()
} }
func (s3 S3) buildCredentialWithoutKey(nowTime time.Time) []byte { func (s3 *S3) buildCredentialWithoutKey(nowTime time.Time) []byte {
var b bytes.Buffer var b bytes.Buffer
b.WriteString(nowTime.Format(shortTimeFormat)) b.WriteString(nowTime.Format(shortTimeFormat))
b.WriteRune('/') b.WriteRune('/')
+222 -13
View File
@@ -12,9 +12,9 @@ import (
"encoding/xml" "encoding/xml"
"fmt" "fmt"
"io" "io"
"io/ioutil"
"mime/multipart" "mime/multipart"
"net/http" "net/http"
"net/url"
"regexp" "regexp"
"strings" "strings"
"sync" "sync"
@@ -126,6 +126,80 @@ type DeleteInput struct {
ObjectKey string ObjectKey string
} }
// ListInput is passed to List as a parameter.
type ListInput struct {
// Required: The name of the bucket to list objects from
Bucket string
// Optional: A delimiter to group objects by (commonly "/")
Delimiter string
// Optional: Only list objects starting with this prefix
Prefix string
// Optional: Maximum number of objects to return
MaxKeys int64
// Optional: Token for pagination from a previous request
ContinuationToken string
// Optional: Object key to start listing after
StartAfter string
}
// ListResponse is returned by List.
type ListResponse struct {
// Name of the bucket
Name string
// Whether the results were truncated (more results available)
IsTruncated bool
// Token to get the next page of results (if truncated)
NextContinuationToken string
// List of objects in the bucket
Objects []Object
// Common prefixes when using delimiter (like directories)
CommonPrefixes []string
// Total number of keys returned
KeyCount int64
}
// Object represents an S3 object in a bucket.
type Object struct {
// Name of the object
Key string
// Size in bytes
Size int64
// When the object was last modified
LastModified string
// Entity tag of the object
ETag string
// Storage class (e.g., "STANDARD")
StorageClass string
}
// S3Error represents an S3 API error response
type S3Error struct {
XMLName xml.Name `xml:"Error"`
Code string `xml:"Code"`
Message string `xml:"Message"`
RequestID string `xml:"RequestId"`
HostID string `xml:"HostId"`
}
// Error returns a string representation of the S3Error
func (e S3Error) Error() string {
return fmt.Sprintf("S3 Error: %s - %s", e.Code, e.Message)
}
// IAMResponse is used by NewUsingIAM to auto // IAMResponse is used by NewUsingIAM to auto
// detect the credentials. // detect the credentials.
type IAMResponse struct { type IAMResponse struct {
@@ -179,14 +253,14 @@ func fetchIMDSToken(cl *http.Client, baseURL string) (string, bool, error) {
defer func() { defer func() {
resp.Body.Close() resp.Body.Close()
io.Copy(ioutil.Discard, resp.Body) io.Copy(io.Discard, resp.Body)
}() }()
if resp.StatusCode != http.StatusOK { if resp.StatusCode != http.StatusOK {
return "", false, fmt.Errorf("failed to request IMDSv2 token: %s", resp.Status) return "", false, fmt.Errorf("failed to request IMDSv2 token: %s", resp.Status)
} }
token, err := ioutil.ReadAll(resp.Body) token, err := io.ReadAll(resp.Body)
if err != nil { if err != nil {
return "", false, err return "", false, err
} }
@@ -224,12 +298,12 @@ func fetchIAMData(cl *http.Client, baseURL string) (IAMResponse, error) {
return IAMResponse{}, fmt.Errorf("error fetching IAM data: %s", resp.Status) return IAMResponse{}, fmt.Errorf("error fetching IAM data: %s", resp.Status)
} }
role, err := ioutil.ReadAll(resp.Body) role, err := io.ReadAll(resp.Body)
if err != nil { if err != nil {
return IAMResponse{}, err return IAMResponse{}, err
} }
_, _ = io.Copy(ioutil.Discard, resp.Body) _, _ = io.Copy(io.Discard, resp.Body)
_ = resp.Body.Close() _ = resp.Body.Close()
req, err = http.NewRequest(http.MethodGet, url+string(role), nil) req, err = http.NewRequest(http.MethodGet, url+string(role), nil)
@@ -247,7 +321,7 @@ func fetchIAMData(cl *http.Client, baseURL string) (IAMResponse, error) {
defer func() { defer func() {
// Drain and close the body to let the Transport reuse the connection // Drain and close the body to let the Transport reuse the connection
io.Copy(ioutil.Discard, resp.Body) io.Copy(io.Discard, resp.Body)
resp.Body.Close() resp.Body.Close()
}() }()
@@ -256,7 +330,7 @@ func fetchIAMData(cl *http.Client, baseURL string) (IAMResponse, error) {
} }
var jResp IAMResponse var jResp IAMResponse
jsonString, err := ioutil.ReadAll(resp.Body) jsonString, err := io.ReadAll(resp.Body)
if err != nil { if err != nil {
return IAMResponse{}, fmt.Errorf("error reading role data: %w", err) return IAMResponse{}, fmt.Errorf("error reading role data: %w", err)
} }
@@ -536,10 +610,10 @@ func (s3 *S3) FilePut(u UploadInput) (PutResponse, error) {
defer func() { defer func() {
res.Body.Close() res.Body.Close()
io.Copy(ioutil.Discard, res.Body) io.Copy(io.Discard, res.Body)
}() }()
data, err := ioutil.ReadAll(res.Body) data, err := io.ReadAll(res.Body)
if err != nil { if err != nil {
return PutResponse{}, err return PutResponse{}, err
} }
@@ -635,10 +709,10 @@ func (s3 *S3) FileUpload(u UploadInput) (UploadResponse, error) {
defer func() { defer func() {
res.Body.Close() res.Body.Close()
io.Copy(ioutil.Discard, res.Body) io.Copy(io.Discard, res.Body)
}() }()
data, err := ioutil.ReadAll(res.Body) data, err := io.ReadAll(res.Body)
if err != nil { if err != nil {
return UploadResponse{}, err return UploadResponse{}, err
} }
@@ -680,7 +754,7 @@ func (s3 *S3) FileDelete(u DeleteInput) error {
defer func() { defer func() {
res.Body.Close() res.Body.Close()
io.Copy(ioutil.Discard, res.Body) io.Copy(io.Discard, res.Body)
}() }()
// Check the response // Check the response
@@ -714,7 +788,7 @@ func (s3 *S3) FileDetails(u DetailsInput) (DetailsResponse, error) {
defer func() { defer func() {
res.Body.Close() res.Body.Close()
io.Copy(ioutil.Discard, res.Body) io.Copy(io.Discard, res.Body)
}() }()
if res.StatusCode != http.StatusOK { if res.StatusCode != http.StatusOK {
@@ -764,6 +838,141 @@ func (s3 *S3) FileDetails(u DetailsInput) (DetailsResponse, error) {
return out, nil return out, nil
} }
// List implements a simple S3 object listing API
func (s3 *S3) List(input ListInput) (ListResponse, error) {
// Input validation
if input.Bucket == "" {
return ListResponse{}, fmt.Errorf("bucket name cannot be empty")
}
if input.MaxKeys < 0 {
return ListResponse{}, fmt.Errorf("MaxKeys cannot be negative")
}
if input.MaxKeys > 1000 {
return ListResponse{}, fmt.Errorf("MaxKeys cannot exceed 1000")
}
// Build query parameters - ListObjectsV2 uses query params, not path
query := url.Values{}
query.Set("list-type", "2") // Required parameter
// Add optional parameters if they exist
if input.ContinuationToken != "" {
query.Set("continuation-token", input.ContinuationToken)
}
if input.Delimiter != "" {
query.Set("delimiter", input.Delimiter)
}
if input.MaxKeys > 0 {
query.Set("max-keys", fmt.Sprintf("%d", input.MaxKeys))
}
if input.Prefix != "" {
query.Set("prefix", input.Prefix)
}
if input.StartAfter != "" {
query.Set("start-after", input.StartAfter)
}
// Build base URL
baseURL := s3.getURL(input.Bucket)
// Parse URL and add query parameters
parsedURL, err := url.Parse(baseURL)
if err != nil {
return ListResponse{}, err
}
parsedURL.RawQuery = query.Encode()
// Create HTTP request
req, err := http.NewRequest(http.MethodGet, parsedURL.String(), nil)
if err != nil {
return ListResponse{}, err
}
// Apply AWS V4 signing
if err := s3.renewIAMToken(); err != nil {
return ListResponse{}, err
}
if err := s3.signRequest(req); err != nil {
return ListResponse{}, err
}
// Execute request
client := s3.getClient()
res, err := client.Do(req)
if err != nil {
return ListResponse{}, err
}
// Close response body when done
defer res.Body.Close()
// Read response body once
body, err := io.ReadAll(res.Body)
if err != nil {
return ListResponse{}, err
}
// Handle response status codes
if res.StatusCode != http.StatusOK {
var s3Err S3Error
if xmlErr := xml.Unmarshal(body, &s3Err); xmlErr == nil {
return ListResponse{}, fmt.Errorf("S3 Error: %s - %s", s3Err.Code, s3Err.Message)
}
return ListResponse{}, fmt.Errorf("status code: %s: %s", res.Status, string(body))
}
// Parse XML response using internal struct
var xmlResult struct {
XMLName xml.Name `xml:"ListBucketResult"`
Name string `xml:"Name"`
IsTruncated bool `xml:"IsTruncated"`
NextContinuationToken string `xml:"NextContinuationToken"`
KeyCount int64 `xml:"KeyCount"`
Contents []struct {
Key string `xml:"Key"`
LastModified string `xml:"LastModified"`
ETag string `xml:"ETag"`
Size int64 `xml:"Size"`
StorageClass string `xml:"StorageClass"`
} `xml:"Contents"`
CommonPrefixes []struct {
Prefix string `xml:"Prefix"`
} `xml:"CommonPrefixes"`
}
if err := xml.Unmarshal(body, &xmlResult); err != nil {
return ListResponse{}, err
}
// Convert to simple response format
response := ListResponse{
Name: xmlResult.Name,
IsTruncated: xmlResult.IsTruncated,
NextContinuationToken: xmlResult.NextContinuationToken,
KeyCount: xmlResult.KeyCount,
}
// Convert objects
for _, obj := range xmlResult.Contents {
response.Objects = append(response.Objects, Object{
Key: obj.Key,
Size: obj.Size,
LastModified: obj.LastModified,
ETag: obj.ETag,
StorageClass: obj.StorageClass,
})
}
// Convert common prefixes
for _, prefix := range xmlResult.CommonPrefixes {
response.CommonPrefixes = append(response.CommonPrefixes, prefix.Prefix)
}
return response, nil
}
func getFirstString(s []string) string { func getFirstString(s []string) string {
if len(s) > 0 { if len(s) > 0 {
return s[0] return s[0]
+214 -7
View File
@@ -3,12 +3,13 @@ package simples3
import ( import (
"bytes" "bytes"
"errors" "errors"
"fmt"
"io" "io"
"io/ioutil"
"net" "net"
"net/http" "net/http"
"net/http/httptest" "net/http/httptest"
"os" "os"
"strings"
"testing" "testing"
"time" "time"
) )
@@ -23,18 +24,18 @@ type tConfig struct {
func TestS3_FileUploadPostAndPut(t *testing.T) { func TestS3_FileUploadPostAndPut(t *testing.T) {
testTxt, err := os.Open("testdata/test.txt") testTxt, err := os.Open("testdata/test.txt")
if err != nil { if err != nil {
return t.Fatalf("Failed to open test.txt: %v", err)
} }
defer testTxt.Close() defer testTxt.Close()
// Note: cannot re-use the same file descriptor due to seeking! // Note: cannot re-use the same file descriptor due to seeking!
testTxtSpecialChars, err := os.Open("testdata/test.txt") testTxtSpecialChars, err := os.Open("testdata/test.txt")
if err != nil { if err != nil {
return t.Fatalf("Failed to open test.txt for special chars: %v", err)
} }
defer testTxtSpecialChars.Close() defer testTxtSpecialChars.Close()
testPng, err := os.Open("testdata/avatar.png") testPng, err := os.Open("testdata/avatar.png")
if err != nil { if err != nil {
return t.Fatalf("Failed to open avatar.png: %v", err)
} }
defer testPng.Close() defer testPng.Close()
@@ -204,7 +205,7 @@ func TestS3_FileDownload(t *testing.T) {
t.Fatal(err) t.Fatal(err)
} }
defer testTxt.Close() defer testTxt.Close()
testTxtData, err := ioutil.ReadAll(testTxt) testTxtData, err := io.ReadAll(testTxt)
if err != nil { if err != nil {
t.Fatal(err) t.Fatal(err)
} }
@@ -214,7 +215,7 @@ func TestS3_FileDownload(t *testing.T) {
t.Fatal(err) t.Fatal(err)
} }
defer testPng.Close() defer testPng.Close()
testPngData, err := ioutil.ReadAll(testPng) testPngData, err := io.ReadAll(testPng)
if err != nil { if err != nil {
t.Fatal(err) t.Fatal(err)
} }
@@ -292,7 +293,7 @@ func TestS3_FileDownload(t *testing.T) {
t.Fatalf("S3.FileDownload() error = %v, wantErr %v", err, tt.wantErr) t.Fatalf("S3.FileDownload() error = %v, wantErr %v", err, tt.wantErr)
} }
got, err := ioutil.ReadAll(resp) got, err := io.ReadAll(resp)
if err != nil { if err != nil {
t.Fatalf("error = %v", err) t.Fatalf("error = %v", err)
} }
@@ -568,3 +569,209 @@ func TestGetURL(t *testing.T) {
}) })
} }
} }
func TestList(t *testing.T) {
// Setup test environment
s3 := New(
os.Getenv("AWS_S3_REGION"),
os.Getenv("AWS_S3_ACCESS_KEY"),
os.Getenv("AWS_S3_SECRET_KEY"),
)
s3.SetEndpoint(os.Getenv("AWS_S3_ENDPOINT"))
// Test cases
tests := []struct {
name string
setup func() // Setup function to prepare test data
input ListInput
validate func(ListResponse, error)
cleanup func() // Cleanup function
}{
{
name: "Empty Bucket",
input: ListInput{
Bucket: os.Getenv("AWS_S3_BUCKET"),
},
validate: func(result ListResponse, err error) {
if err != nil {
t.Fatalf("Expected no error, got %v", err)
}
if len(result.Objects) != 0 {
t.Errorf("Expected empty bucket, got %d objects", len(result.Objects))
}
},
},
{
name: "Basic Listing",
setup: func() {
// Upload test files
uploadTestFiles(t, s3, os.Getenv("AWS_S3_BUCKET"), []string{"file1.txt", "file2.txt", "file3.txt"})
},
input: ListInput{
Bucket: os.Getenv("AWS_S3_BUCKET"),
},
validate: func(result ListResponse, err error) {
if err != nil {
t.Fatalf("Expected no error, got %v", err)
}
if len(result.Objects) != 3 {
t.Errorf("Expected 3 objects, got %d", len(result.Objects))
}
},
cleanup: func() {
cleanupTestFiles(t, s3, os.Getenv("AWS_S3_BUCKET"), []string{"file1.txt", "file2.txt", "file3.txt"})
},
},
{
name: "Pagination Test",
setup: func() {
// Upload many files for pagination testing
var filenames []string
for i := 1; i <= 25; i++ {
filenames = append(filenames, fmt.Sprintf("file%03d.txt", i))
}
uploadTestFiles(t, s3, os.Getenv("AWS_S3_BUCKET"), filenames)
},
input: ListInput{
Bucket: os.Getenv("AWS_S3_BUCKET"),
MaxKeys: 10,
},
validate: func(result ListResponse, err error) {
if err != nil {
t.Fatalf("Expected no error, got %v", err)
}
if len(result.Objects) != 10 {
t.Errorf("Expected 10 objects, got %d", len(result.Objects))
}
if !result.IsTruncated {
t.Errorf("Expected IsTruncated=true for pagination")
}
if result.NextContinuationToken == "" {
t.Errorf("Expected NextContinuationToken for pagination: %v", result.NextContinuationToken)
}
},
cleanup: func() {
var filenames []string
for i := 1; i <= 25; i++ {
filenames = append(filenames, fmt.Sprintf("file%03d.txt", i))
}
cleanupTestFiles(t, s3, os.Getenv("AWS_S3_BUCKET"), filenames)
},
},
{
name: "Prefix Filtering",
setup: func() {
uploadTestFiles(t, s3, os.Getenv("AWS_S3_BUCKET"), []string{
"documents/file1.txt",
"documents/file2.txt",
"images/image1.jpg",
"images/image2.jpg",
})
},
input: ListInput{
Bucket: os.Getenv("AWS_S3_BUCKET"),
Prefix: "documents/",
},
validate: func(result ListResponse, err error) {
if err != nil {
t.Fatalf("Expected no error, got %v", err)
}
if len(result.Objects) != 2 {
t.Errorf("Expected 2 objects with documents/ prefix, got %d", len(result.Objects))
}
for _, obj := range result.Objects {
if !strings.HasPrefix(obj.Key, "documents/") {
t.Errorf("Object %s doesn't have expected prefix", obj.Key)
}
}
},
cleanup: func() {
cleanupTestFiles(t, s3, os.Getenv("AWS_S3_BUCKET"), []string{
"documents/file1.txt",
"documents/file2.txt",
"images/image1.jpg",
"images/image2.jpg",
})
},
},
{
name: "Delimiter Grouping",
setup: func() {
uploadTestFiles(t, s3, os.Getenv("AWS_S3_BUCKET"), []string{
"documents/important/file1.txt",
"documents/backup/file2.txt",
"images/landscape/image1.jpg",
"images/portrait/image2.jpg",
})
},
input: ListInput{
Bucket: os.Getenv("AWS_S3_BUCKET"),
Delimiter: "/",
Prefix: "documents/",
},
validate: func(result ListResponse, err error) {
if err != nil {
t.Fatalf("Expected no error, got %v", err)
}
// Should return common prefixes instead of individual files
if len(result.CommonPrefixes) != 2 {
t.Errorf("Expected 2 common prefixes, got %d", len(result.CommonPrefixes))
}
},
cleanup: func() {
cleanupTestFiles(t, s3, os.Getenv("AWS_S3_BUCKET"), []string{
"documents/important/file1.txt",
"documents/backup/file2.txt",
"images/landscape/image1.jpg",
"images/portrait/image2.jpg",
})
},
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
if tt.setup != nil {
tt.setup()
}
result, err := s3.List(tt.input)
if tt.validate != nil {
tt.validate(result, err)
}
if tt.cleanup != nil {
tt.cleanup()
}
})
}
}
// Helper functions
func uploadTestFiles(t *testing.T, s3 *S3, bucket string, filenames []string) {
for _, filename := range filenames {
content := strings.NewReader("test content for " + filename)
_, err := s3.FilePut(UploadInput{
Bucket: bucket,
ObjectKey: filename,
ContentType: "text/plain",
Body: content,
})
if err != nil {
t.Fatalf("Failed to upload test file %s: %v", filename, err)
}
}
}
func cleanupTestFiles(t *testing.T, s3 *S3, bucket string, filenames []string) {
for _, filename := range filenames {
err := s3.FileDelete(DeleteInput{
Bucket: bucket,
ObjectKey: filename,
})
if err != nil {
t.Logf("Failed to cleanup test file %s: %v", filename, err)
}
}
}