feat: implement Simple List API for S3 objects
- Add List method with ListInput and ListResponse structs following simples3 patterns
- Implement pagination, prefix filtering, and delimiter grouping
- Add comprehensive test suite for List functionality
- Update tests to use environment variables instead of hardcoded values
- Fix deprecated io/ioutil usage throughout codebase
- Add comprehensive README with API documentation and examples
- Add Nix development environment with flake.nix
- Add Justfile for modern command runner
- Add Docker Compose setup for local MinIO testing
- Update GitHub Actions to use Nix and Justfile for CI consistency
- Fix test naming and setup flow issues
New API:
s3.List(ListInput{
Bucket: "my-bucket",
Prefix: "documents/",
MaxKeys: 100,
})
All tests pass with new simplified API design.
This commit is contained in:
@@ -0,0 +1,4 @@
|
|||||||
|
# Load the Nix development environment
|
||||||
|
use flake
|
||||||
|
|
||||||
|
echo "✅ Environment loaded - Use 'just --list' to see available commands"
|
||||||
@@ -10,31 +10,19 @@ jobs:
|
|||||||
test:
|
test:
|
||||||
runs-on: ubuntu-latest
|
runs-on: ubuntu-latest
|
||||||
steps:
|
steps:
|
||||||
- uses: actions/checkout@v2
|
- name: Checkout code
|
||||||
|
uses: actions/checkout@v4
|
||||||
|
|
||||||
- name: Set up Go
|
- name: Install Nix
|
||||||
uses: actions/setup-go@v2
|
uses: cachix/install-nix-action@v27
|
||||||
with:
|
with:
|
||||||
go-version: 1.15
|
nix_path: nixpkgs=channel:nixos-unstable
|
||||||
- name: Setup minio
|
|
||||||
run: |
|
- name: Run Tests via Justfile
|
||||||
docker run -d -p 9000:9000 --name minio \
|
env:
|
||||||
-e "MINIO_ACCESS_KEY=minioadmin" \
|
AWS_S3_BUCKET: testbucket
|
||||||
-e "MINIO_SECRET_KEY=minioadmin" \
|
AWS_ACCESS_KEY_ID: minioadmin
|
||||||
-v /tmp/data:/data \
|
AWS_SECRET_ACCESS_KEY: minioadmin
|
||||||
-v /tmp/config:/root/.minio \
|
AWS_S3_ENDPOINT: http://127.0.0.1:9000
|
||||||
minio/minio server /data
|
AWS_EC2_METADATA_DISABLED: "true"
|
||||||
|
run: nix develop --command just test-local
|
||||||
export AWS_ACCESS_KEY_ID=minioadmin
|
|
||||||
export AWS_SECRET_ACCESS_KEY=minioadmin
|
|
||||||
export AWS_EC2_METADATA_DISABLED=true
|
|
||||||
|
|
||||||
aws --endpoint-url http://127.0.0.1:9000/ s3 mb s3://testbucket
|
|
||||||
|
|
||||||
- name: Run Go Test
|
|
||||||
run: go test -v ./...
|
|
||||||
env:
|
|
||||||
AWS_S3_BUCKET: testbucket
|
|
||||||
AWS_S3_ACCESS_KEY: minioadmin
|
|
||||||
AWS_S3_SECRET_KEY: minioadmin
|
|
||||||
AWS_S3_ENDPOINT: http://127.0.0.1:9000
|
|
||||||
@@ -24,3 +24,6 @@ tsconfig.json
|
|||||||
jsconfig.json
|
jsconfig.json
|
||||||
|
|
||||||
.env
|
.env
|
||||||
|
|
||||||
|
# direnv
|
||||||
|
.direnv
|
||||||
@@ -0,0 +1,92 @@
|
|||||||
|
# SimpleS3 Development Commands
|
||||||
|
|
||||||
|
# Variables
|
||||||
|
MINIO_CONTAINER_NAME := "simples3-minio"
|
||||||
|
MINIO_DATA_DIR := ".minio-data"
|
||||||
|
AWS_S3_BUCKET := "testbucket"
|
||||||
|
|
||||||
|
# Default command - lists all available recipes
|
||||||
|
default:
|
||||||
|
@just --list
|
||||||
|
|
||||||
|
# --- Development Commands ---
|
||||||
|
test:
|
||||||
|
@echo "🧪 Running all tests..."
|
||||||
|
@go test -v ./...
|
||||||
|
|
||||||
|
test-local: setup
|
||||||
|
@echo "🧪 Running tests with local MinIO..."
|
||||||
|
@sleep 2
|
||||||
|
@go test -v ./...
|
||||||
|
|
||||||
|
|
||||||
|
# --- Go Module Management ---
|
||||||
|
tidy:
|
||||||
|
@echo "📦 Tidying Go modules..."
|
||||||
|
@go mod tidy
|
||||||
|
|
||||||
|
fmt:
|
||||||
|
@echo "🎨 Formatting Go code..."
|
||||||
|
@go fmt ./...
|
||||||
|
|
||||||
|
vet:
|
||||||
|
@echo "🔍 Running go vet..."
|
||||||
|
@go vet ./...
|
||||||
|
|
||||||
|
# --- MinIO Management ---
|
||||||
|
minio-up:
|
||||||
|
@echo "🚀 Starting MinIO container..."
|
||||||
|
@docker compose up -d
|
||||||
|
@echo "✅ MinIO started:"
|
||||||
|
@echo " API: http://localhost:9000"
|
||||||
|
@echo " Console: http://localhost:9001"
|
||||||
|
@echo " Access Key: minioadmin"
|
||||||
|
@echo " Secret Key: minioadmin"
|
||||||
|
|
||||||
|
minio-down:
|
||||||
|
@echo "🛑 Stopping MinIO container..."
|
||||||
|
@docker compose down
|
||||||
|
|
||||||
|
minio-logs:
|
||||||
|
@echo "📋 Showing MinIO logs..."
|
||||||
|
@docker compose logs -f
|
||||||
|
|
||||||
|
minio-clean:
|
||||||
|
@echo "🧹 Cleaning MinIO data..."
|
||||||
|
@docker compose down --volumes
|
||||||
|
|
||||||
|
minio-reset: minio-clean minio-up
|
||||||
|
|
||||||
|
# --- Development Environment Setup ---
|
||||||
|
setup: minio-up
|
||||||
|
@echo "⚙️ Setting up development environment..."
|
||||||
|
@sleep 3
|
||||||
|
@aws --endpoint-url http://127.0.0.1:9000/ s3 mb s3://{{AWS_S3_BUCKET}} || true
|
||||||
|
@echo "✅ Development environment ready!"
|
||||||
|
|
||||||
|
dev-env: setup
|
||||||
|
@echo "🎯 Development environment active!"
|
||||||
|
@echo " MinIO: http://localhost:9000"
|
||||||
|
@echo " Console: http://localhost:9001"
|
||||||
|
@echo " Bucket: {{AWS_S3_BUCKET}}"
|
||||||
|
|
||||||
|
# --- Cleanup Commands ---
|
||||||
|
clean:
|
||||||
|
@echo "🧹 Cleaning up development environment..."
|
||||||
|
@docker compose down --volumes
|
||||||
|
|
||||||
|
# --- Helper Commands ---
|
||||||
|
status:
|
||||||
|
@echo "📊 Status Check:"
|
||||||
|
@echo " MinIO Container: $(docker ps -q -f name={{MINIO_CONTAINER_NAME}} | wc -l | tr -d ' ') running"
|
||||||
|
@if [ "$(docker ps -q -f name={{MINIO_CONTAINER_NAME}})" ]; then \
|
||||||
|
echo " MinIO URL: http://localhost:9000"; \
|
||||||
|
echo " Console URL: http://localhost:9001"; \
|
||||||
|
fi
|
||||||
|
@echo " Go version: $(go version | awk '{print $3}')"
|
||||||
|
|
||||||
|
# --- Documentation ---
|
||||||
|
docs:
|
||||||
|
@echo "📖 Opening documentation..."
|
||||||
|
@echo " Plan: PLAN.md"
|
||||||
|
@echo " README: README.md"
|
||||||
@@ -1,82 +1,334 @@
|
|||||||
# simples3 : Simple no frills AWS S3 Library using REST with V4 Signing
|
# simples3 : Simple no frills AWS S3 Library using REST with V4 Signing
|
||||||
|
|
||||||
## Overview [](https://godoc.org/github.com/rhnvrm/simples3) [](https://goreportcard.com/report/github.com/rhnvrm/simples3) [](https://gocover.io/_badge/github.com/rhnvrm/simples3) [](https://zerodha.tech)
|
## Overview [](https://godoc.org/github.com/rhnvrm/simples3) [](https://goreportcard.com/report/github.com/rhnvrm/simples3) [](https://gocover.io/_badge/github.com/rhnvrm/simples3) [](https://zerodha.tech)
|
||||||
|
|
||||||
SimpleS3 is a Go library for manipulating objects
|
SimpleS3 is a Go library for manipulating objects
|
||||||
in S3 buckets using REST API calls or Presigned URLs signed
|
in S3 buckets using REST API calls or Presigned URLs signed
|
||||||
using AWS Signature Version 4.
|
using AWS Signature Version 4.
|
||||||
|
|
||||||
|
**Features:**
|
||||||
|
- 🚀 **Simple, intuitive API** following Go idioms
|
||||||
|
- 🪣 **Complete S3 operations** - Upload, Download, Delete, List, Details
|
||||||
|
- 🔐 **AWS Signature Version 4** signing
|
||||||
|
- 🌐 **Custom endpoint support** (MinIO, DigitalOcean Spaces, etc.)
|
||||||
|
- 📋 **Simple List API** with pagination, prefix filtering, and delimiter grouping
|
||||||
|
- 🔗 **Presigned URL generation** for secure browser uploads/downloads
|
||||||
|
- 🪪 **IAM credential support** for EC2 instances
|
||||||
|
- ✅ **Comprehensive test coverage**
|
||||||
|
- 🎯 **Zero dependencies** - uses only Go standard library
|
||||||
|
|
||||||
## Install
|
## Install
|
||||||
|
|
||||||
```sh
|
```sh
|
||||||
go get github.com/rhnvrm/simples3
|
go get github.com/rhnvrm/simples3
|
||||||
```
|
```
|
||||||
|
|
||||||
## Example
|
## Quick Start
|
||||||
|
|
||||||
```go
|
```go
|
||||||
testTxt, _ := os.Open("testdata/test.txt")
|
package main
|
||||||
defer testTxt.Close()
|
|
||||||
|
|
||||||
// Create an instance of the package
|
import (
|
||||||
// You can either create by manually supplying credentials
|
"log"
|
||||||
// (preferably using Environment vars)
|
"os"
|
||||||
s3 := simples3.New(Region, AWSAccessKey, AWSSecretKey)
|
"github.com/rhnvrm/simples3"
|
||||||
// or you can use this on an EC2 instance to
|
)
|
||||||
// obtain credentials from IAM attached to the instance.
|
|
||||||
s3, _ := simples3.NewUsingIAM(Region)
|
|
||||||
|
|
||||||
// You can also set a custom endpoint to a compatible s3 instance.
|
func main() {
|
||||||
s3.SetEndpoint(CustomEndpoint)
|
// Initialize S3 client
|
||||||
|
s3 := simples3.New("us-east-1", "your-access-key", "your-secret-key")
|
||||||
|
|
||||||
// Note: Consider adding a testTxt.Seek(0, 0)
|
// Use MinIO or other S3-compatible services
|
||||||
// in case you have read
|
s3.SetEndpoint("https://s3.amazonaws.com")
|
||||||
// the body, as the pointer is shared by the library.
|
|
||||||
|
|
||||||
// File Upload is as simple as providing the following
|
// Upload a file
|
||||||
// details.
|
file, _ := os.Open("my-file.txt")
|
||||||
|
defer file.Close()
|
||||||
|
|
||||||
|
resp, err := s3.FileUpload(simples3.UploadInput{
|
||||||
|
Bucket: "my-bucket",
|
||||||
|
ObjectKey: "my-file.txt",
|
||||||
|
ContentType: "text/plain",
|
||||||
|
FileName: "my-file.txt",
|
||||||
|
Body: file,
|
||||||
|
})
|
||||||
|
if err != nil {
|
||||||
|
log.Fatal(err)
|
||||||
|
}
|
||||||
|
|
||||||
|
log.Printf("File uploaded successfully: %+v", resp)
|
||||||
|
}
|
||||||
|
```
|
||||||
|
|
||||||
|
## API Reference
|
||||||
|
|
||||||
|
### File Operations
|
||||||
|
|
||||||
|
#### Upload Files
|
||||||
|
```go
|
||||||
|
// POST upload (recommended for browsers)
|
||||||
resp, err := s3.FileUpload(simples3.UploadInput{
|
resp, err := s3.FileUpload(simples3.UploadInput{
|
||||||
Bucket: AWSBucket,
|
Bucket: "my-bucket",
|
||||||
ObjectKey: "test.txt",
|
ObjectKey: "path/to/file.txt",
|
||||||
ContentType: "text/plain",
|
ContentType: "text/plain",
|
||||||
FileName: "test.txt",
|
FileName: "file.txt",
|
||||||
Body: testTxt,
|
Body: file,
|
||||||
})
|
})
|
||||||
|
|
||||||
// Similarly, Files can be deleted.
|
// PUT upload (simpler for programmatic use)
|
||||||
|
resp, err := s3.FilePut(simples3.UploadInput{
|
||||||
|
Bucket: "my-bucket",
|
||||||
|
ObjectKey: "path/to/file.txt",
|
||||||
|
ContentType: "text/plain",
|
||||||
|
Body: file,
|
||||||
|
})
|
||||||
|
```
|
||||||
|
|
||||||
|
#### Download Files
|
||||||
|
```go
|
||||||
|
// Download file
|
||||||
|
file, err := s3.FileDownload(simples3.DownloadInput{
|
||||||
|
Bucket: "my-bucket",
|
||||||
|
ObjectKey: "path/to/file.txt",
|
||||||
|
})
|
||||||
|
if err != nil {
|
||||||
|
log.Fatal(err)
|
||||||
|
}
|
||||||
|
defer file.Close()
|
||||||
|
|
||||||
|
// Read the content
|
||||||
|
data, err := io.ReadAll(file)
|
||||||
|
```
|
||||||
|
|
||||||
|
#### Delete Files
|
||||||
|
```go
|
||||||
err := s3.FileDelete(simples3.DeleteInput{
|
err := s3.FileDelete(simples3.DeleteInput{
|
||||||
Bucket: os.Getenv("AWS_S3_BUCKET"),
|
Bucket: "my-bucket",
|
||||||
ObjectKey: "test.txt",
|
ObjectKey: "path/to/file.txt",
|
||||||
})
|
})
|
||||||
|
```
|
||||||
|
|
||||||
// You can also download the file.
|
#### Get File Details
|
||||||
file, _ := s3.FileDownload(simples3.DownloadInput{
|
```go
|
||||||
Bucket: AWSBucket,
|
details, err := s3.FileDetails(simples3.DetailsInput{
|
||||||
ObjectKey: "test.txt",
|
Bucket: "my-bucket",
|
||||||
|
ObjectKey: "path/to/file.txt",
|
||||||
})
|
})
|
||||||
|
if err != nil {
|
||||||
|
log.Fatal(err)
|
||||||
|
}
|
||||||
|
|
||||||
data, _ := ioutil.ReadAll(file)
|
log.Printf("File size: %s bytes", details.ContentLength)
|
||||||
file.Close()
|
log.Printf("Last modified: %s", details.LastModified)
|
||||||
|
log.Printf("Content type: %s", details.ContentType)
|
||||||
|
```
|
||||||
|
|
||||||
// You can also use this library to generate
|
### List Objects
|
||||||
// Presigned URLs that can for eg. be used to
|
|
||||||
// GET/PUT files on S3 through the browser.
|
|
||||||
var time, _ = time.Parse(time.RFC1123, "Fri, 24 May 2013 00:00:00 GMT")
|
|
||||||
|
|
||||||
url := s.GeneratePresignedURL(PresignedInput{
|
SimpleS3 provides a clean, easy-to-use List API that follows the same pattern as other library methods:
|
||||||
Bucket: AWSBucket,
|
|
||||||
ObjectKey: "test.txt",
|
#### Simple Listing
|
||||||
|
```go
|
||||||
|
// List all objects in a bucket
|
||||||
|
result, err := s3.List(simples3.ListInput{
|
||||||
|
Bucket: "my-bucket",
|
||||||
|
})
|
||||||
|
if err != nil {
|
||||||
|
log.Fatal(err)
|
||||||
|
}
|
||||||
|
|
||||||
|
fmt.Printf("Found %d objects:\n", len(result.Objects))
|
||||||
|
for _, obj := range result.Objects {
|
||||||
|
fmt.Printf("- %s (%d bytes)\n", obj.Key, obj.Size)
|
||||||
|
}
|
||||||
|
```
|
||||||
|
|
||||||
|
#### Advanced Listing with Options
|
||||||
|
```go
|
||||||
|
// List with prefix filtering and pagination
|
||||||
|
result, err := s3.List(simples3.ListInput{
|
||||||
|
Bucket: "my-bucket",
|
||||||
|
Prefix: "documents/",
|
||||||
|
Delimiter: "/",
|
||||||
|
MaxKeys: 100,
|
||||||
|
})
|
||||||
|
if err != nil {
|
||||||
|
log.Fatal(err)
|
||||||
|
}
|
||||||
|
|
||||||
|
// Process objects
|
||||||
|
for _, obj := range result.Objects {
|
||||||
|
fmt.Printf("📄 %s (%d bytes)\n", obj.Key, obj.Size)
|
||||||
|
}
|
||||||
|
|
||||||
|
// Process "directories" (common prefixes)
|
||||||
|
for _, prefix := range result.CommonPrefixes {
|
||||||
|
fmt.Printf("📁 %s/\n", prefix)
|
||||||
|
}
|
||||||
|
```
|
||||||
|
|
||||||
|
#### Handle Pagination
|
||||||
|
```go
|
||||||
|
// Handle large result sets with pagination
|
||||||
|
func listAllObjects(s3 *simples3.S3, bucket string) ([]simples3.Object, error) {
|
||||||
|
var allObjects []simples3.Object
|
||||||
|
var continuationToken string
|
||||||
|
|
||||||
|
for {
|
||||||
|
// List objects with continuation token
|
||||||
|
result, err := s3.List(simples3.ListInput{
|
||||||
|
Bucket: bucket,
|
||||||
|
ContinuationToken: continuationToken,
|
||||||
|
MaxKeys: 1000, // Maximum page size
|
||||||
|
})
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
|
||||||
|
// Add current page objects to our collection
|
||||||
|
allObjects = append(allObjects, result.Objects...)
|
||||||
|
|
||||||
|
// Check if there are more pages
|
||||||
|
if !result.IsTruncated {
|
||||||
|
break
|
||||||
|
}
|
||||||
|
|
||||||
|
// Set token for next page
|
||||||
|
continuationToken = result.NextContinuationToken
|
||||||
|
}
|
||||||
|
|
||||||
|
return allObjects, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// Usage example
|
||||||
|
func main() {
|
||||||
|
s3 := simples3.New("us-east-1", "your-access-key", "your-secret-key")
|
||||||
|
|
||||||
|
allObjects, err := listAllObjects(s3, "my-bucket")
|
||||||
|
if err != nil {
|
||||||
|
log.Fatal(err)
|
||||||
|
}
|
||||||
|
|
||||||
|
fmt.Printf("Total objects: %d\\n", len(allObjects))
|
||||||
|
for _, obj := range allObjects {
|
||||||
|
fmt.Printf("- %s (%d bytes)\\n", obj.Key, obj.Size)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
```
|
||||||
|
|
||||||
|
### Presigned URLs
|
||||||
|
|
||||||
|
Generate secure URLs for browser-based uploads/downloads:
|
||||||
|
|
||||||
|
```go
|
||||||
|
// Generate presigned URL for download
|
||||||
|
url := s3.GeneratePresignedURL(simples3.PresignedInput{
|
||||||
|
Bucket: "my-bucket",
|
||||||
|
ObjectKey: "private-file.pdf",
|
||||||
Method: "GET",
|
Method: "GET",
|
||||||
Timestamp: time,
|
ExpirySeconds: 3600, // 1 hour
|
||||||
ExpirySeconds: 86400,
|
|
||||||
})
|
})
|
||||||
|
|
||||||
|
// Users can now download directly: <a href="{{url}}">Download</a>
|
||||||
|
```
|
||||||
|
|
||||||
|
### Custom Endpoints
|
||||||
|
|
||||||
|
Use with MinIO, DigitalOcean Spaces, or other S3-compatible services:
|
||||||
|
|
||||||
|
```go
|
||||||
|
// MinIO
|
||||||
|
s3 := simples3.New("us-east-1", "minioadmin", "minioadmin")
|
||||||
|
s3.SetEndpoint("http://localhost:9000")
|
||||||
|
|
||||||
|
// DigitalOcean Spaces
|
||||||
|
s3 := simples3.New("nyc3", "your-access-key", "your-secret-key")
|
||||||
|
s3.SetEndpoint("https://nyc3.digitaloceanspaces.com")
|
||||||
|
```
|
||||||
|
|
||||||
|
### IAM Credentials
|
||||||
|
|
||||||
|
On EC2 instances, use IAM roles automatically:
|
||||||
|
|
||||||
|
```go
|
||||||
|
s3, err := simples3.NewUsingIAM("us-east-1")
|
||||||
|
if err != nil {
|
||||||
|
log.Fatal(err)
|
||||||
|
}
|
||||||
|
// Automatically uses instance IAM role
|
||||||
|
```
|
||||||
|
|
||||||
|
## Development
|
||||||
|
|
||||||
|
### Setup Development Environment
|
||||||
|
|
||||||
|
```sh
|
||||||
|
# Clone the repository
|
||||||
|
git clone https://github.com/rhnvrm/simples3.git
|
||||||
|
cd simples3
|
||||||
|
|
||||||
|
# Using Nix (recommended)
|
||||||
|
nix develop
|
||||||
|
|
||||||
|
# Or using direnv
|
||||||
|
direnv allow
|
||||||
|
|
||||||
|
# Start local MinIO for testing
|
||||||
|
just setup
|
||||||
|
|
||||||
|
# Run tests
|
||||||
|
just test-local
|
||||||
|
```
|
||||||
|
|
||||||
|
### Testing
|
||||||
|
|
||||||
|
The library includes comprehensive tests that run against a local MinIO instance:
|
||||||
|
|
||||||
|
```sh
|
||||||
|
# Run all tests (without MinIO)
|
||||||
|
just test
|
||||||
|
|
||||||
|
# Run tests with local MinIO
|
||||||
|
just test-local
|
||||||
|
|
||||||
|
# Run specific test
|
||||||
|
go test -v -run TestList
|
||||||
|
```
|
||||||
|
|
||||||
|
### Available Commands
|
||||||
|
|
||||||
|
```sh
|
||||||
|
just # List all commands
|
||||||
|
just test # Run tests without MinIO
|
||||||
|
just test-local # Run tests with MinIO (includes setup)
|
||||||
|
just setup # Setup development environment
|
||||||
|
just minio-up # Start MinIO container
|
||||||
|
just minio-down # Stop MinIO container
|
||||||
|
just clean # Clean up everything
|
||||||
|
just status # Check development environment status
|
||||||
|
```
|
||||||
|
|
||||||
|
## Environment Variables
|
||||||
|
|
||||||
|
For development and testing:
|
||||||
|
|
||||||
|
```sh
|
||||||
|
export AWS_S3_REGION="us-east-1"
|
||||||
|
export AWS_S3_ACCESS_KEY="minioadmin"
|
||||||
|
export AWS_S3_SECRET_KEY="minioadmin"
|
||||||
|
export AWS_S3_ENDPOINT="http://localhost:9000"
|
||||||
|
export AWS_S3_BUCKET="testbucket"
|
||||||
```
|
```
|
||||||
|
|
||||||
## Contributing
|
## Contributing
|
||||||
|
|
||||||
You are more than welcome to contribute to this project. Fork and make
|
We welcome contributions! Please:
|
||||||
a Pull Request, or create an Issue if you see any problem or want to
|
|
||||||
propose a feature.
|
1. Fork the repository
|
||||||
|
2. Create a feature branch
|
||||||
|
3. Add tests for new functionality
|
||||||
|
4. Ensure all tests pass: `just test-local`
|
||||||
|
5. Submit a pull request
|
||||||
|
|
||||||
## Author
|
## Author
|
||||||
|
|
||||||
|
|||||||
@@ -0,0 +1,21 @@
|
|||||||
|
services:
|
||||||
|
minio:
|
||||||
|
image: minio/minio:latest
|
||||||
|
container_name: simples3-minio
|
||||||
|
ports:
|
||||||
|
- "9000:9000" # API
|
||||||
|
- "9001:9001" # Console
|
||||||
|
environment:
|
||||||
|
- MINIO_ROOT_USER=minioadmin
|
||||||
|
- MINIO_ROOT_PASSWORD=minioadmin
|
||||||
|
volumes:
|
||||||
|
- minio_data:/data
|
||||||
|
command: server /data --console-address ":9001"
|
||||||
|
healthcheck:
|
||||||
|
test: ["CMD", "curl", "-f", "http://localhost:9000/minio/health/live"]
|
||||||
|
interval: 30s
|
||||||
|
timeout: 20s
|
||||||
|
retries: 3
|
||||||
|
|
||||||
|
volumes:
|
||||||
|
minio_data:
|
||||||
Generated
+27
@@ -0,0 +1,27 @@
|
|||||||
|
{
|
||||||
|
"nodes": {
|
||||||
|
"nixpkgs": {
|
||||||
|
"locked": {
|
||||||
|
"lastModified": 1757745802,
|
||||||
|
"narHash": "sha256-hLEO2TPj55KcUFUU1vgtHE9UEIOjRcH/4QbmfHNF820=",
|
||||||
|
"owner": "NixOS",
|
||||||
|
"repo": "nixpkgs",
|
||||||
|
"rev": "c23193b943c6c689d70ee98ce3128239ed9e32d1",
|
||||||
|
"type": "github"
|
||||||
|
},
|
||||||
|
"original": {
|
||||||
|
"owner": "NixOS",
|
||||||
|
"ref": "nixos-unstable",
|
||||||
|
"repo": "nixpkgs",
|
||||||
|
"type": "github"
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"root": {
|
||||||
|
"inputs": {
|
||||||
|
"nixpkgs": "nixpkgs"
|
||||||
|
}
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"root": "root",
|
||||||
|
"version": 7
|
||||||
|
}
|
||||||
@@ -0,0 +1,62 @@
|
|||||||
|
{
|
||||||
|
description = "Go project for simples3 with ListObjectsV2 support";
|
||||||
|
|
||||||
|
inputs = {
|
||||||
|
nixpkgs.url = "github:NixOS/nixpkgs/nixos-unstable";
|
||||||
|
};
|
||||||
|
|
||||||
|
outputs = { self, nixpkgs }:
|
||||||
|
let
|
||||||
|
supportedSystems = [ "x86_64-linux" "aarch64-linux" "x86_64-darwin" "aarch64-darwin" ];
|
||||||
|
forEachSystem = f: nixpkgs.lib.genAttrs supportedSystems (system: f {
|
||||||
|
pkgs = import nixpkgs {
|
||||||
|
inherit system;
|
||||||
|
};
|
||||||
|
});
|
||||||
|
in
|
||||||
|
{
|
||||||
|
devShells = forEachSystem ({ pkgs }: {
|
||||||
|
default = pkgs.mkShell {
|
||||||
|
buildInputs = with pkgs; [
|
||||||
|
# Go development
|
||||||
|
go
|
||||||
|
gopls
|
||||||
|
go-outline
|
||||||
|
delve
|
||||||
|
|
||||||
|
# Build tools and utilities
|
||||||
|
just
|
||||||
|
docker
|
||||||
|
awscli2
|
||||||
|
|
||||||
|
# Additional utilities
|
||||||
|
curl
|
||||||
|
jq
|
||||||
|
];
|
||||||
|
|
||||||
|
# Environment variables for the shell
|
||||||
|
shellHook = ''
|
||||||
|
echo "🚀 SimpleS3 Development Environment Loaded"
|
||||||
|
echo "Go version: $(go version)"
|
||||||
|
echo "Available commands: just --list"
|
||||||
|
export GOPATH=$(go env GOPATH)
|
||||||
|
export PATH=$PATH:$GOPATH/bin
|
||||||
|
|
||||||
|
# Local S3 (MinIO) configuration for testing
|
||||||
|
export AWS_S3_BUCKET=testbucket
|
||||||
|
export AWS_S3_ENDPOINT=http://localhost:9000
|
||||||
|
export AWS_S3_REGION=us-east-1
|
||||||
|
export AWS_EC2_METADATA_DISABLED=true
|
||||||
|
|
||||||
|
# AWS CLI credentials for MinIO
|
||||||
|
export AWS_ACCESS_KEY_ID=minioadmin
|
||||||
|
export AWS_SECRET_ACCESS_KEY=minioadmin
|
||||||
|
|
||||||
|
# Simples3 library credentials
|
||||||
|
export AWS_S3_ACCESS_KEY=minioadmin
|
||||||
|
export AWS_S3_SECRET_KEY=minioadmin
|
||||||
|
'';
|
||||||
|
};
|
||||||
|
});
|
||||||
|
};
|
||||||
|
}
|
||||||
@@ -1,3 +1,3 @@
|
|||||||
module github.com/rhnvrm/simples3
|
module github.com/rhnvrm/simples3
|
||||||
|
|
||||||
go 1.13
|
go 1.25.0
|
||||||
|
|||||||
@@ -152,7 +152,7 @@ func buildUploadSign(nowTime time.Time, credential string, uploadConfig UploadCo
|
|||||||
})
|
})
|
||||||
}
|
}
|
||||||
|
|
||||||
func (s3 S3) buildCredential(nowTime time.Time) []byte {
|
func (s3 *S3) buildCredential(nowTime time.Time) []byte {
|
||||||
var b bytes.Buffer
|
var b bytes.Buffer
|
||||||
b.WriteString(s3.AccessKey)
|
b.WriteString(s3.AccessKey)
|
||||||
b.WriteRune('/')
|
b.WriteRune('/')
|
||||||
@@ -166,7 +166,7 @@ func (s3 S3) buildCredential(nowTime time.Time) []byte {
|
|||||||
return b.Bytes()
|
return b.Bytes()
|
||||||
}
|
}
|
||||||
|
|
||||||
func (s3 S3) buildCredentialWithoutKey(nowTime time.Time) []byte {
|
func (s3 *S3) buildCredentialWithoutKey(nowTime time.Time) []byte {
|
||||||
var b bytes.Buffer
|
var b bytes.Buffer
|
||||||
b.WriteString(nowTime.Format(shortTimeFormat))
|
b.WriteString(nowTime.Format(shortTimeFormat))
|
||||||
b.WriteRune('/')
|
b.WriteRune('/')
|
||||||
|
|||||||
+222
-13
@@ -12,9 +12,9 @@ import (
|
|||||||
"encoding/xml"
|
"encoding/xml"
|
||||||
"fmt"
|
"fmt"
|
||||||
"io"
|
"io"
|
||||||
"io/ioutil"
|
|
||||||
"mime/multipart"
|
"mime/multipart"
|
||||||
"net/http"
|
"net/http"
|
||||||
|
"net/url"
|
||||||
"regexp"
|
"regexp"
|
||||||
"strings"
|
"strings"
|
||||||
"sync"
|
"sync"
|
||||||
@@ -126,6 +126,80 @@ type DeleteInput struct {
|
|||||||
ObjectKey string
|
ObjectKey string
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// ListInput is passed to List as a parameter.
|
||||||
|
type ListInput struct {
|
||||||
|
// Required: The name of the bucket to list objects from
|
||||||
|
Bucket string
|
||||||
|
|
||||||
|
// Optional: A delimiter to group objects by (commonly "/")
|
||||||
|
Delimiter string
|
||||||
|
|
||||||
|
// Optional: Only list objects starting with this prefix
|
||||||
|
Prefix string
|
||||||
|
|
||||||
|
// Optional: Maximum number of objects to return
|
||||||
|
MaxKeys int64
|
||||||
|
|
||||||
|
// Optional: Token for pagination from a previous request
|
||||||
|
ContinuationToken string
|
||||||
|
|
||||||
|
// Optional: Object key to start listing after
|
||||||
|
StartAfter string
|
||||||
|
}
|
||||||
|
|
||||||
|
// ListResponse is returned by List.
|
||||||
|
type ListResponse struct {
|
||||||
|
// Name of the bucket
|
||||||
|
Name string
|
||||||
|
|
||||||
|
// Whether the results were truncated (more results available)
|
||||||
|
IsTruncated bool
|
||||||
|
|
||||||
|
// Token to get the next page of results (if truncated)
|
||||||
|
NextContinuationToken string
|
||||||
|
|
||||||
|
// List of objects in the bucket
|
||||||
|
Objects []Object
|
||||||
|
|
||||||
|
// Common prefixes when using delimiter (like directories)
|
||||||
|
CommonPrefixes []string
|
||||||
|
|
||||||
|
// Total number of keys returned
|
||||||
|
KeyCount int64
|
||||||
|
}
|
||||||
|
|
||||||
|
// Object represents an S3 object in a bucket.
|
||||||
|
type Object struct {
|
||||||
|
// Name of the object
|
||||||
|
Key string
|
||||||
|
|
||||||
|
// Size in bytes
|
||||||
|
Size int64
|
||||||
|
|
||||||
|
// When the object was last modified
|
||||||
|
LastModified string
|
||||||
|
|
||||||
|
// Entity tag of the object
|
||||||
|
ETag string
|
||||||
|
|
||||||
|
// Storage class (e.g., "STANDARD")
|
||||||
|
StorageClass string
|
||||||
|
}
|
||||||
|
|
||||||
|
// S3Error represents an S3 API error response
|
||||||
|
type S3Error struct {
|
||||||
|
XMLName xml.Name `xml:"Error"`
|
||||||
|
Code string `xml:"Code"`
|
||||||
|
Message string `xml:"Message"`
|
||||||
|
RequestID string `xml:"RequestId"`
|
||||||
|
HostID string `xml:"HostId"`
|
||||||
|
}
|
||||||
|
|
||||||
|
// Error returns a string representation of the S3Error
|
||||||
|
func (e S3Error) Error() string {
|
||||||
|
return fmt.Sprintf("S3 Error: %s - %s", e.Code, e.Message)
|
||||||
|
}
|
||||||
|
|
||||||
// IAMResponse is used by NewUsingIAM to auto
|
// IAMResponse is used by NewUsingIAM to auto
|
||||||
// detect the credentials.
|
// detect the credentials.
|
||||||
type IAMResponse struct {
|
type IAMResponse struct {
|
||||||
@@ -179,14 +253,14 @@ func fetchIMDSToken(cl *http.Client, baseURL string) (string, bool, error) {
|
|||||||
|
|
||||||
defer func() {
|
defer func() {
|
||||||
resp.Body.Close()
|
resp.Body.Close()
|
||||||
io.Copy(ioutil.Discard, resp.Body)
|
io.Copy(io.Discard, resp.Body)
|
||||||
}()
|
}()
|
||||||
|
|
||||||
if resp.StatusCode != http.StatusOK {
|
if resp.StatusCode != http.StatusOK {
|
||||||
return "", false, fmt.Errorf("failed to request IMDSv2 token: %s", resp.Status)
|
return "", false, fmt.Errorf("failed to request IMDSv2 token: %s", resp.Status)
|
||||||
}
|
}
|
||||||
|
|
||||||
token, err := ioutil.ReadAll(resp.Body)
|
token, err := io.ReadAll(resp.Body)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return "", false, err
|
return "", false, err
|
||||||
}
|
}
|
||||||
@@ -224,12 +298,12 @@ func fetchIAMData(cl *http.Client, baseURL string) (IAMResponse, error) {
|
|||||||
return IAMResponse{}, fmt.Errorf("error fetching IAM data: %s", resp.Status)
|
return IAMResponse{}, fmt.Errorf("error fetching IAM data: %s", resp.Status)
|
||||||
}
|
}
|
||||||
|
|
||||||
role, err := ioutil.ReadAll(resp.Body)
|
role, err := io.ReadAll(resp.Body)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return IAMResponse{}, err
|
return IAMResponse{}, err
|
||||||
}
|
}
|
||||||
|
|
||||||
_, _ = io.Copy(ioutil.Discard, resp.Body)
|
_, _ = io.Copy(io.Discard, resp.Body)
|
||||||
_ = resp.Body.Close()
|
_ = resp.Body.Close()
|
||||||
|
|
||||||
req, err = http.NewRequest(http.MethodGet, url+string(role), nil)
|
req, err = http.NewRequest(http.MethodGet, url+string(role), nil)
|
||||||
@@ -247,7 +321,7 @@ func fetchIAMData(cl *http.Client, baseURL string) (IAMResponse, error) {
|
|||||||
|
|
||||||
defer func() {
|
defer func() {
|
||||||
// Drain and close the body to let the Transport reuse the connection
|
// Drain and close the body to let the Transport reuse the connection
|
||||||
io.Copy(ioutil.Discard, resp.Body)
|
io.Copy(io.Discard, resp.Body)
|
||||||
resp.Body.Close()
|
resp.Body.Close()
|
||||||
}()
|
}()
|
||||||
|
|
||||||
@@ -256,7 +330,7 @@ func fetchIAMData(cl *http.Client, baseURL string) (IAMResponse, error) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
var jResp IAMResponse
|
var jResp IAMResponse
|
||||||
jsonString, err := ioutil.ReadAll(resp.Body)
|
jsonString, err := io.ReadAll(resp.Body)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return IAMResponse{}, fmt.Errorf("error reading role data: %w", err)
|
return IAMResponse{}, fmt.Errorf("error reading role data: %w", err)
|
||||||
}
|
}
|
||||||
@@ -536,10 +610,10 @@ func (s3 *S3) FilePut(u UploadInput) (PutResponse, error) {
|
|||||||
|
|
||||||
defer func() {
|
defer func() {
|
||||||
res.Body.Close()
|
res.Body.Close()
|
||||||
io.Copy(ioutil.Discard, res.Body)
|
io.Copy(io.Discard, res.Body)
|
||||||
}()
|
}()
|
||||||
|
|
||||||
data, err := ioutil.ReadAll(res.Body)
|
data, err := io.ReadAll(res.Body)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return PutResponse{}, err
|
return PutResponse{}, err
|
||||||
}
|
}
|
||||||
@@ -635,10 +709,10 @@ func (s3 *S3) FileUpload(u UploadInput) (UploadResponse, error) {
|
|||||||
|
|
||||||
defer func() {
|
defer func() {
|
||||||
res.Body.Close()
|
res.Body.Close()
|
||||||
io.Copy(ioutil.Discard, res.Body)
|
io.Copy(io.Discard, res.Body)
|
||||||
}()
|
}()
|
||||||
|
|
||||||
data, err := ioutil.ReadAll(res.Body)
|
data, err := io.ReadAll(res.Body)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return UploadResponse{}, err
|
return UploadResponse{}, err
|
||||||
}
|
}
|
||||||
@@ -680,7 +754,7 @@ func (s3 *S3) FileDelete(u DeleteInput) error {
|
|||||||
|
|
||||||
defer func() {
|
defer func() {
|
||||||
res.Body.Close()
|
res.Body.Close()
|
||||||
io.Copy(ioutil.Discard, res.Body)
|
io.Copy(io.Discard, res.Body)
|
||||||
}()
|
}()
|
||||||
|
|
||||||
// Check the response
|
// Check the response
|
||||||
@@ -714,7 +788,7 @@ func (s3 *S3) FileDetails(u DetailsInput) (DetailsResponse, error) {
|
|||||||
|
|
||||||
defer func() {
|
defer func() {
|
||||||
res.Body.Close()
|
res.Body.Close()
|
||||||
io.Copy(ioutil.Discard, res.Body)
|
io.Copy(io.Discard, res.Body)
|
||||||
}()
|
}()
|
||||||
|
|
||||||
if res.StatusCode != http.StatusOK {
|
if res.StatusCode != http.StatusOK {
|
||||||
@@ -764,6 +838,141 @@ func (s3 *S3) FileDetails(u DetailsInput) (DetailsResponse, error) {
|
|||||||
return out, nil
|
return out, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// List implements a simple S3 object listing API
|
||||||
|
func (s3 *S3) List(input ListInput) (ListResponse, error) {
|
||||||
|
// Input validation
|
||||||
|
if input.Bucket == "" {
|
||||||
|
return ListResponse{}, fmt.Errorf("bucket name cannot be empty")
|
||||||
|
}
|
||||||
|
if input.MaxKeys < 0 {
|
||||||
|
return ListResponse{}, fmt.Errorf("MaxKeys cannot be negative")
|
||||||
|
}
|
||||||
|
if input.MaxKeys > 1000 {
|
||||||
|
return ListResponse{}, fmt.Errorf("MaxKeys cannot exceed 1000")
|
||||||
|
}
|
||||||
|
|
||||||
|
// Build query parameters - ListObjectsV2 uses query params, not path
|
||||||
|
query := url.Values{}
|
||||||
|
query.Set("list-type", "2") // Required parameter
|
||||||
|
|
||||||
|
// Add optional parameters if they exist
|
||||||
|
if input.ContinuationToken != "" {
|
||||||
|
query.Set("continuation-token", input.ContinuationToken)
|
||||||
|
}
|
||||||
|
if input.Delimiter != "" {
|
||||||
|
query.Set("delimiter", input.Delimiter)
|
||||||
|
}
|
||||||
|
if input.MaxKeys > 0 {
|
||||||
|
query.Set("max-keys", fmt.Sprintf("%d", input.MaxKeys))
|
||||||
|
}
|
||||||
|
if input.Prefix != "" {
|
||||||
|
query.Set("prefix", input.Prefix)
|
||||||
|
}
|
||||||
|
if input.StartAfter != "" {
|
||||||
|
query.Set("start-after", input.StartAfter)
|
||||||
|
}
|
||||||
|
|
||||||
|
// Build base URL
|
||||||
|
baseURL := s3.getURL(input.Bucket)
|
||||||
|
|
||||||
|
// Parse URL and add query parameters
|
||||||
|
parsedURL, err := url.Parse(baseURL)
|
||||||
|
if err != nil {
|
||||||
|
return ListResponse{}, err
|
||||||
|
}
|
||||||
|
parsedURL.RawQuery = query.Encode()
|
||||||
|
|
||||||
|
// Create HTTP request
|
||||||
|
req, err := http.NewRequest(http.MethodGet, parsedURL.String(), nil)
|
||||||
|
if err != nil {
|
||||||
|
return ListResponse{}, err
|
||||||
|
}
|
||||||
|
|
||||||
|
// Apply AWS V4 signing
|
||||||
|
if err := s3.renewIAMToken(); err != nil {
|
||||||
|
return ListResponse{}, err
|
||||||
|
}
|
||||||
|
if err := s3.signRequest(req); err != nil {
|
||||||
|
return ListResponse{}, err
|
||||||
|
}
|
||||||
|
|
||||||
|
// Execute request
|
||||||
|
client := s3.getClient()
|
||||||
|
res, err := client.Do(req)
|
||||||
|
if err != nil {
|
||||||
|
return ListResponse{}, err
|
||||||
|
}
|
||||||
|
|
||||||
|
// Close response body when done
|
||||||
|
defer res.Body.Close()
|
||||||
|
|
||||||
|
// Read response body once
|
||||||
|
body, err := io.ReadAll(res.Body)
|
||||||
|
if err != nil {
|
||||||
|
return ListResponse{}, err
|
||||||
|
}
|
||||||
|
|
||||||
|
// Handle response status codes
|
||||||
|
if res.StatusCode != http.StatusOK {
|
||||||
|
var s3Err S3Error
|
||||||
|
if xmlErr := xml.Unmarshal(body, &s3Err); xmlErr == nil {
|
||||||
|
return ListResponse{}, fmt.Errorf("S3 Error: %s - %s", s3Err.Code, s3Err.Message)
|
||||||
|
}
|
||||||
|
return ListResponse{}, fmt.Errorf("status code: %s: %s", res.Status, string(body))
|
||||||
|
}
|
||||||
|
|
||||||
|
// Parse XML response using internal struct
|
||||||
|
var xmlResult struct {
|
||||||
|
XMLName xml.Name `xml:"ListBucketResult"`
|
||||||
|
Name string `xml:"Name"`
|
||||||
|
IsTruncated bool `xml:"IsTruncated"`
|
||||||
|
NextContinuationToken string `xml:"NextContinuationToken"`
|
||||||
|
KeyCount int64 `xml:"KeyCount"`
|
||||||
|
Contents []struct {
|
||||||
|
Key string `xml:"Key"`
|
||||||
|
LastModified string `xml:"LastModified"`
|
||||||
|
ETag string `xml:"ETag"`
|
||||||
|
Size int64 `xml:"Size"`
|
||||||
|
StorageClass string `xml:"StorageClass"`
|
||||||
|
} `xml:"Contents"`
|
||||||
|
CommonPrefixes []struct {
|
||||||
|
Prefix string `xml:"Prefix"`
|
||||||
|
} `xml:"CommonPrefixes"`
|
||||||
|
}
|
||||||
|
|
||||||
|
if err := xml.Unmarshal(body, &xmlResult); err != nil {
|
||||||
|
return ListResponse{}, err
|
||||||
|
}
|
||||||
|
|
||||||
|
// Convert to simple response format
|
||||||
|
response := ListResponse{
|
||||||
|
Name: xmlResult.Name,
|
||||||
|
IsTruncated: xmlResult.IsTruncated,
|
||||||
|
NextContinuationToken: xmlResult.NextContinuationToken,
|
||||||
|
KeyCount: xmlResult.KeyCount,
|
||||||
|
}
|
||||||
|
|
||||||
|
// Convert objects
|
||||||
|
for _, obj := range xmlResult.Contents {
|
||||||
|
response.Objects = append(response.Objects, Object{
|
||||||
|
Key: obj.Key,
|
||||||
|
Size: obj.Size,
|
||||||
|
LastModified: obj.LastModified,
|
||||||
|
ETag: obj.ETag,
|
||||||
|
StorageClass: obj.StorageClass,
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
// Convert common prefixes
|
||||||
|
for _, prefix := range xmlResult.CommonPrefixes {
|
||||||
|
response.CommonPrefixes = append(response.CommonPrefixes, prefix.Prefix)
|
||||||
|
}
|
||||||
|
|
||||||
|
return response, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
func getFirstString(s []string) string {
|
func getFirstString(s []string) string {
|
||||||
if len(s) > 0 {
|
if len(s) > 0 {
|
||||||
return s[0]
|
return s[0]
|
||||||
|
|||||||
+214
-7
@@ -3,12 +3,13 @@ package simples3
|
|||||||
import (
|
import (
|
||||||
"bytes"
|
"bytes"
|
||||||
"errors"
|
"errors"
|
||||||
|
"fmt"
|
||||||
"io"
|
"io"
|
||||||
"io/ioutil"
|
|
||||||
"net"
|
"net"
|
||||||
"net/http"
|
"net/http"
|
||||||
"net/http/httptest"
|
"net/http/httptest"
|
||||||
"os"
|
"os"
|
||||||
|
"strings"
|
||||||
"testing"
|
"testing"
|
||||||
"time"
|
"time"
|
||||||
)
|
)
|
||||||
@@ -23,18 +24,18 @@ type tConfig struct {
|
|||||||
func TestS3_FileUploadPostAndPut(t *testing.T) {
|
func TestS3_FileUploadPostAndPut(t *testing.T) {
|
||||||
testTxt, err := os.Open("testdata/test.txt")
|
testTxt, err := os.Open("testdata/test.txt")
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return
|
t.Fatalf("Failed to open test.txt: %v", err)
|
||||||
}
|
}
|
||||||
defer testTxt.Close()
|
defer testTxt.Close()
|
||||||
// Note: cannot re-use the same file descriptor due to seeking!
|
// Note: cannot re-use the same file descriptor due to seeking!
|
||||||
testTxtSpecialChars, err := os.Open("testdata/test.txt")
|
testTxtSpecialChars, err := os.Open("testdata/test.txt")
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return
|
t.Fatalf("Failed to open test.txt for special chars: %v", err)
|
||||||
}
|
}
|
||||||
defer testTxtSpecialChars.Close()
|
defer testTxtSpecialChars.Close()
|
||||||
testPng, err := os.Open("testdata/avatar.png")
|
testPng, err := os.Open("testdata/avatar.png")
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return
|
t.Fatalf("Failed to open avatar.png: %v", err)
|
||||||
}
|
}
|
||||||
defer testPng.Close()
|
defer testPng.Close()
|
||||||
|
|
||||||
@@ -204,7 +205,7 @@ func TestS3_FileDownload(t *testing.T) {
|
|||||||
t.Fatal(err)
|
t.Fatal(err)
|
||||||
}
|
}
|
||||||
defer testTxt.Close()
|
defer testTxt.Close()
|
||||||
testTxtData, err := ioutil.ReadAll(testTxt)
|
testTxtData, err := io.ReadAll(testTxt)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatal(err)
|
t.Fatal(err)
|
||||||
}
|
}
|
||||||
@@ -214,7 +215,7 @@ func TestS3_FileDownload(t *testing.T) {
|
|||||||
t.Fatal(err)
|
t.Fatal(err)
|
||||||
}
|
}
|
||||||
defer testPng.Close()
|
defer testPng.Close()
|
||||||
testPngData, err := ioutil.ReadAll(testPng)
|
testPngData, err := io.ReadAll(testPng)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatal(err)
|
t.Fatal(err)
|
||||||
}
|
}
|
||||||
@@ -292,7 +293,7 @@ func TestS3_FileDownload(t *testing.T) {
|
|||||||
t.Fatalf("S3.FileDownload() error = %v, wantErr %v", err, tt.wantErr)
|
t.Fatalf("S3.FileDownload() error = %v, wantErr %v", err, tt.wantErr)
|
||||||
}
|
}
|
||||||
|
|
||||||
got, err := ioutil.ReadAll(resp)
|
got, err := io.ReadAll(resp)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatalf("error = %v", err)
|
t.Fatalf("error = %v", err)
|
||||||
}
|
}
|
||||||
@@ -568,3 +569,209 @@ func TestGetURL(t *testing.T) {
|
|||||||
})
|
})
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func TestList(t *testing.T) {
|
||||||
|
// Setup test environment
|
||||||
|
s3 := New(
|
||||||
|
os.Getenv("AWS_S3_REGION"),
|
||||||
|
os.Getenv("AWS_S3_ACCESS_KEY"),
|
||||||
|
os.Getenv("AWS_S3_SECRET_KEY"),
|
||||||
|
)
|
||||||
|
s3.SetEndpoint(os.Getenv("AWS_S3_ENDPOINT"))
|
||||||
|
|
||||||
|
// Test cases
|
||||||
|
tests := []struct {
|
||||||
|
name string
|
||||||
|
setup func() // Setup function to prepare test data
|
||||||
|
input ListInput
|
||||||
|
validate func(ListResponse, error)
|
||||||
|
cleanup func() // Cleanup function
|
||||||
|
}{
|
||||||
|
{
|
||||||
|
name: "Empty Bucket",
|
||||||
|
input: ListInput{
|
||||||
|
Bucket: os.Getenv("AWS_S3_BUCKET"),
|
||||||
|
},
|
||||||
|
validate: func(result ListResponse, err error) {
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("Expected no error, got %v", err)
|
||||||
|
}
|
||||||
|
if len(result.Objects) != 0 {
|
||||||
|
t.Errorf("Expected empty bucket, got %d objects", len(result.Objects))
|
||||||
|
}
|
||||||
|
},
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "Basic Listing",
|
||||||
|
setup: func() {
|
||||||
|
// Upload test files
|
||||||
|
uploadTestFiles(t, s3, os.Getenv("AWS_S3_BUCKET"), []string{"file1.txt", "file2.txt", "file3.txt"})
|
||||||
|
},
|
||||||
|
input: ListInput{
|
||||||
|
Bucket: os.Getenv("AWS_S3_BUCKET"),
|
||||||
|
},
|
||||||
|
validate: func(result ListResponse, err error) {
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("Expected no error, got %v", err)
|
||||||
|
}
|
||||||
|
if len(result.Objects) != 3 {
|
||||||
|
t.Errorf("Expected 3 objects, got %d", len(result.Objects))
|
||||||
|
}
|
||||||
|
},
|
||||||
|
cleanup: func() {
|
||||||
|
cleanupTestFiles(t, s3, os.Getenv("AWS_S3_BUCKET"), []string{"file1.txt", "file2.txt", "file3.txt"})
|
||||||
|
},
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "Pagination Test",
|
||||||
|
setup: func() {
|
||||||
|
// Upload many files for pagination testing
|
||||||
|
var filenames []string
|
||||||
|
for i := 1; i <= 25; i++ {
|
||||||
|
filenames = append(filenames, fmt.Sprintf("file%03d.txt", i))
|
||||||
|
}
|
||||||
|
uploadTestFiles(t, s3, os.Getenv("AWS_S3_BUCKET"), filenames)
|
||||||
|
},
|
||||||
|
input: ListInput{
|
||||||
|
Bucket: os.Getenv("AWS_S3_BUCKET"),
|
||||||
|
MaxKeys: 10,
|
||||||
|
},
|
||||||
|
validate: func(result ListResponse, err error) {
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("Expected no error, got %v", err)
|
||||||
|
}
|
||||||
|
if len(result.Objects) != 10 {
|
||||||
|
t.Errorf("Expected 10 objects, got %d", len(result.Objects))
|
||||||
|
}
|
||||||
|
if !result.IsTruncated {
|
||||||
|
t.Errorf("Expected IsTruncated=true for pagination")
|
||||||
|
}
|
||||||
|
if result.NextContinuationToken == "" {
|
||||||
|
t.Errorf("Expected NextContinuationToken for pagination: %v", result.NextContinuationToken)
|
||||||
|
}
|
||||||
|
},
|
||||||
|
cleanup: func() {
|
||||||
|
var filenames []string
|
||||||
|
for i := 1; i <= 25; i++ {
|
||||||
|
filenames = append(filenames, fmt.Sprintf("file%03d.txt", i))
|
||||||
|
}
|
||||||
|
cleanupTestFiles(t, s3, os.Getenv("AWS_S3_BUCKET"), filenames)
|
||||||
|
},
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "Prefix Filtering",
|
||||||
|
setup: func() {
|
||||||
|
uploadTestFiles(t, s3, os.Getenv("AWS_S3_BUCKET"), []string{
|
||||||
|
"documents/file1.txt",
|
||||||
|
"documents/file2.txt",
|
||||||
|
"images/image1.jpg",
|
||||||
|
"images/image2.jpg",
|
||||||
|
})
|
||||||
|
},
|
||||||
|
input: ListInput{
|
||||||
|
Bucket: os.Getenv("AWS_S3_BUCKET"),
|
||||||
|
Prefix: "documents/",
|
||||||
|
},
|
||||||
|
validate: func(result ListResponse, err error) {
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("Expected no error, got %v", err)
|
||||||
|
}
|
||||||
|
if len(result.Objects) != 2 {
|
||||||
|
t.Errorf("Expected 2 objects with documents/ prefix, got %d", len(result.Objects))
|
||||||
|
}
|
||||||
|
for _, obj := range result.Objects {
|
||||||
|
if !strings.HasPrefix(obj.Key, "documents/") {
|
||||||
|
t.Errorf("Object %s doesn't have expected prefix", obj.Key)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
},
|
||||||
|
cleanup: func() {
|
||||||
|
cleanupTestFiles(t, s3, os.Getenv("AWS_S3_BUCKET"), []string{
|
||||||
|
"documents/file1.txt",
|
||||||
|
"documents/file2.txt",
|
||||||
|
"images/image1.jpg",
|
||||||
|
"images/image2.jpg",
|
||||||
|
})
|
||||||
|
},
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "Delimiter Grouping",
|
||||||
|
setup: func() {
|
||||||
|
uploadTestFiles(t, s3, os.Getenv("AWS_S3_BUCKET"), []string{
|
||||||
|
"documents/important/file1.txt",
|
||||||
|
"documents/backup/file2.txt",
|
||||||
|
"images/landscape/image1.jpg",
|
||||||
|
"images/portrait/image2.jpg",
|
||||||
|
})
|
||||||
|
},
|
||||||
|
input: ListInput{
|
||||||
|
Bucket: os.Getenv("AWS_S3_BUCKET"),
|
||||||
|
Delimiter: "/",
|
||||||
|
Prefix: "documents/",
|
||||||
|
},
|
||||||
|
validate: func(result ListResponse, err error) {
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("Expected no error, got %v", err)
|
||||||
|
}
|
||||||
|
// Should return common prefixes instead of individual files
|
||||||
|
if len(result.CommonPrefixes) != 2 {
|
||||||
|
t.Errorf("Expected 2 common prefixes, got %d", len(result.CommonPrefixes))
|
||||||
|
}
|
||||||
|
},
|
||||||
|
cleanup: func() {
|
||||||
|
cleanupTestFiles(t, s3, os.Getenv("AWS_S3_BUCKET"), []string{
|
||||||
|
"documents/important/file1.txt",
|
||||||
|
"documents/backup/file2.txt",
|
||||||
|
"images/landscape/image1.jpg",
|
||||||
|
"images/portrait/image2.jpg",
|
||||||
|
})
|
||||||
|
},
|
||||||
|
},
|
||||||
|
}
|
||||||
|
|
||||||
|
for _, tt := range tests {
|
||||||
|
t.Run(tt.name, func(t *testing.T) {
|
||||||
|
if tt.setup != nil {
|
||||||
|
tt.setup()
|
||||||
|
}
|
||||||
|
|
||||||
|
result, err := s3.List(tt.input)
|
||||||
|
|
||||||
|
if tt.validate != nil {
|
||||||
|
tt.validate(result, err)
|
||||||
|
}
|
||||||
|
|
||||||
|
if tt.cleanup != nil {
|
||||||
|
tt.cleanup()
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Helper functions
|
||||||
|
func uploadTestFiles(t *testing.T, s3 *S3, bucket string, filenames []string) {
|
||||||
|
for _, filename := range filenames {
|
||||||
|
content := strings.NewReader("test content for " + filename)
|
||||||
|
_, err := s3.FilePut(UploadInput{
|
||||||
|
Bucket: bucket,
|
||||||
|
ObjectKey: filename,
|
||||||
|
ContentType: "text/plain",
|
||||||
|
Body: content,
|
||||||
|
})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("Failed to upload test file %s: %v", filename, err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func cleanupTestFiles(t *testing.T, s3 *S3, bucket string, filenames []string) {
|
||||||
|
for _, filename := range filenames {
|
||||||
|
err := s3.FileDelete(DeleteInput{
|
||||||
|
Bucket: bucket,
|
||||||
|
ObjectKey: filename,
|
||||||
|
})
|
||||||
|
if err != nil {
|
||||||
|
t.Logf("Failed to cleanup test file %s: %v", filename, err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|||||||
Reference in New Issue
Block a user