S3 tests: use testcontainers (#79)

* S3 tests: use testcontainers

Remove internal s3testing package in favour of testcontainers, as suggested in #61.

* s3 tests: skip tests based on testcontainers if no provider

backends/s3/credentials_test.go

@@ -9,13 +9,15 @@ import (
 	"time"
 
 	"github.com/PowerDNS/simpleblob/backends/s3"
-	"github.com/PowerDNS/simpleblob/backends/s3/s3testing"
 	"github.com/PowerDNS/simpleblob/tester"
 	"github.com/minio/minio-go/v7"
 	"github.com/minio/minio-go/v7/pkg/credentials"
+	"github.com/testcontainers/testcontainers-go"
+	testcontainersminio "github.com/testcontainers/testcontainers-go/modules/minio"
 )
 
 func TestFileSecretsCredentials(t *testing.T) {
+	testcontainers.SkipIfProviderIsNotHealthy(t)
 	tempDir := t.TempDir()
 
 	access, secret := secretsPaths(tempDir)
@@ -30,15 +32,20 @@ func TestFileSecretsCredentials(t *testing.T) {
 	ctx, cancel := context.WithTimeout(ctx, 10*time.Second)
 	defer cancel()
 
-	// Create server
-	addr, stop, err := s3testing.ServeMinio(ctx, tempDir)
-	if errors.Is(err, s3testing.ErrMinioNotFound) {
-		t.Skip("minio binary not found locally, make sure it is in PATH")
+	container, err := testcontainersminio.Run(ctx, "quay.io/minio/minio")
+	if err != nil {
+		t.Fatal(err)
 	}
+	defer func() {
+		if err := container.Terminate(ctx); err != nil {
+			t.Log(err)
+		}
+	}()
+
+	addr, err := container.ConnectionString(ctx)
 	if err != nil {
 		t.Fatal(err)
 	}
-	defer func() { _ = stop() }()
 
 	// Create minio client, using our provider.
 	creds := credentials.New(provider)
@@ -65,7 +72,7 @@ func TestFileSecretsCredentials(t *testing.T) {
 	// First credential files creation.
 	// Keep them empty for now,
 	// so that calls to the server will fail.
-	writeSecrets(t, tempDir, "")
+	writeSecrets(t, tempDir, "", "")
 
 	// The files do not hold the right values,
 	// so a call to the server should fail.
@@ -74,31 +81,38 @@ func TestFileSecretsCredentials(t *testing.T) {
 	// Write the right keys to the files.
 	// We're not testing expiry here,
 	// and forcing credentials cache to update.
-	writeSecrets(t, tempDir, s3testing.AdminUserOrPassword)
+	writeSecrets(t, tempDir, container.Username, container.Password)
 	creds.Expire()
 	assertClientSuccess(true, "after changing files content")
 
 	// Change content of the files.
-	writeSecrets(t, tempDir, "badcredentials")
+	writeSecrets(t, tempDir, "bad-user", "bad-password")
 	creds.Expire()
 	assertClientSuccess(false, "after changing again, to bad credentials")
 }
 
 func TestBackendWithSecrets(t *testing.T) {
+	testcontainers.SkipIfProviderIsNotHealthy(t)
 	tempDir := t.TempDir()
 
 	ctx := context.Background()
 	ctx, cancel := context.WithTimeout(ctx, 10*time.Second)
 	defer cancel()
 
-	addr, stop, err := s3testing.ServeMinio(ctx, tempDir)
-	if errors.Is(err, s3testing.ErrMinioNotFound) {
-		t.Skip("minio binary not found locally, make sure it is in PATH")
+	container, err := testcontainersminio.Run(ctx, "quay.io/minio/minio")
+	if err != nil {
+		t.Fatal(err)
 	}
+	defer func() {
+		if err := container.Terminate(ctx); err != nil {
+			t.Log(err)
+		}
+	}()
+
+	addr, err := container.ConnectionString(ctx)
 	if err != nil {
 		t.Fatal(err)
 	}
-	defer func() { _ = stop() }()
 
 	// Prepare backend options to reuse.
 	// These will not change.
@@ -119,15 +133,15 @@ func TestBackendWithSecrets(t *testing.T) {
 	}
 
 	// Now write files, but with bad content.
-	writeSecrets(t, tempDir, "")
+	writeSecrets(t, tempDir, "", "")
 	_, err = s3.New(ctx, opt)
 	if err == nil || err.Error() != "Access Denied." {
 		t.Fatal("backend should not start with bad credentials")
 	}
 
 	// Write the good content.
 	// Now the backend should start and be able to perform a request.
-	writeSecrets(t, tempDir, s3testing.AdminUserOrPassword)
+	writeSecrets(t, tempDir, container.Username, container.Password)
 
 	backend, err := s3.New(ctx, opt)
 	if err != nil {
@@ -153,14 +167,13 @@ func secretsPaths(dir string) (access, secret string) {
 
 // writeSecrets writes content to files called "access-key" and "secret-key"
 // in dir.
-// It returns
-func writeSecrets(t testing.TB, dir, content string) {
+func writeSecrets(t testing.TB, dir, adminUser, password string) {
 	access, secret := secretsPaths(dir)
-	err := os.WriteFile(access, []byte(content), 0666)
+	err := os.WriteFile(access, []byte(adminUser), 0666)
 	if err != nil {
 		t.Fatal(err)
 	}
-	err = os.WriteFile(secret, []byte(content), 0666)
+	err = os.WriteFile(secret, []byte(password), 0666)
 	if err != nil {
 		t.Fatal(err)
 	}

backends/s3/s3_test.go

@@ -2,58 +2,40 @@ package s3
 
 import (
 	"context"
-	"os"
 	"testing"
 	"time"
 
 	"github.com/minio/minio-go/v7"
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
-	"gopkg.in/yaml.v2"
+	"github.com/testcontainers/testcontainers-go"
+	testcontainersminio "github.com/testcontainers/testcontainers-go/modules/minio"
 
 	"github.com/PowerDNS/simpleblob/tester"
 )
 
-// TestConfigPathEnv is the path to a YAML file with the Options
-// with an S3 bucket configuration that can be used for testing.
-// The bucket will be emptied before every run!!!
-//
-// To run a Minio for this :
-//
-//	env MINIO_ROOT_USER=test MINIO_ROOT_PASSWORD=secret minio server /tmp/test-data/
-//
-// Example test config:
-//
-//	{
-//	  "access_key": "test",
-//	  "secret_key": "verysecret",
-//	  "region": "us-east-1",
-//	  "bucket": "test-bucket",
-//	  "endpoint_url": "http://127.0.0.1:9000"
-//	}
-const TestConfigPathEnv = "SIMPLEBLOB_TEST_S3_CONFIG"
-
 func getBackend(ctx context.Context, t *testing.T) (b *Backend) {
-	cfgPath := os.Getenv(TestConfigPathEnv)
-	if cfgPath == "" {
-		t.Skipf("S3 tests skipped, set the %s env var to run these", TestConfigPathEnv)
-		return
+	testcontainers.SkipIfProviderIsNotHealthy(t)
+	container, err := testcontainersminio.Run(ctx, "quay.io/minio/minio")
+	if err != nil {
+		t.Fatal(err)
 	}
 
-	cfgContents, err := os.ReadFile(cfgPath)
-	require.NoError(t, err)
-
-	var opt Options
-	err = yaml.Unmarshal(cfgContents, &opt)
-	require.NoError(t, err)
+	url, err := container.ConnectionString(ctx)
+	if err != nil {
+		t.Fatal(err)
+	}
 
-	b, err = New(ctx, opt)
+	b, err = New(ctx, Options{
+		EndpointURL:  "http://" + url,
+		AccessKey:    container.Username,
+		SecretKey:    container.Password,
+		Bucket:       "test-bucket",
+		CreateBucket: true,
+	})
 	require.NoError(t, err)
 
-	cleanup := func() {
-		ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
-		defer cancel()
-
+	cleanStorage := func(ctx context.Context) {
 		blobs, err := b.List(ctx, "")
 		if err != nil {
 			t.Logf("Blobs list error: %s", err)
@@ -69,8 +51,15 @@ func getBackend(ctx context.Context, t *testing.T) (b *Backend) {
 		err = b.client.RemoveObject(ctx, b.opt.Bucket, b.markerName, minio.RemoveObjectOptions{})
 		require.NoError(t, err)
 	}
-	t.Cleanup(cleanup)
-	cleanup()
+	t.Cleanup(func() {
+		ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
+		defer cancel()
+		cleanStorage(ctx)
+		if err := container.Terminate(ctx); err != nil {
+			t.Log(err)
+		}
+	})
+	cleanStorage(ctx)
 
 	return b
 }

go.mod

@@ -10,27 +10,67 @@ require (
 	github.com/minio/minio-go/v7 v7.0.78
 	github.com/prometheus/client_golang v1.20.4
 	github.com/stretchr/testify v1.9.0
+	github.com/testcontainers/testcontainers-go v0.33.0
+	github.com/testcontainers/testcontainers-go/modules/minio v0.33.0
 	gopkg.in/yaml.v2 v2.4.0
 )
 
 require (
+	dario.cat/mergo v1.0.0 // indirect
+	github.com/Azure/go-ansiterm v0.0.0-20210617225240-d185dfc1b5a1 // indirect
+	github.com/Microsoft/go-winio v0.6.2 // indirect
 	github.com/beorn7/perks v1.0.1 // indirect
+	github.com/cenkalti/backoff/v4 v4.2.1 // indirect
 	github.com/cespare/xxhash/v2 v2.3.0 // indirect
+	github.com/containerd/containerd v1.7.18 // indirect
+	github.com/containerd/log v0.1.0 // indirect
+	github.com/containerd/platforms v0.2.1 // indirect
+	github.com/cpuguy83/dockercfg v0.3.1 // indirect
 	github.com/davecgh/go-spew v1.1.1 // indirect
+	github.com/distribution/reference v0.6.0 // indirect
+	github.com/docker/docker v27.1.1+incompatible // indirect
+	github.com/docker/go-connections v0.5.0 // indirect
+	github.com/docker/go-units v0.5.0 // indirect
 	github.com/dustin/go-humanize v1.0.1 // indirect
+	github.com/felixge/httpsnoop v1.0.4 // indirect
 	github.com/go-ini/ini v1.67.0 // indirect
+	github.com/go-logr/stdr v1.2.2 // indirect
+	github.com/go-ole/go-ole v1.2.6 // indirect
 	github.com/goccy/go-json v0.10.3 // indirect
+	github.com/gogo/protobuf v1.3.2 // indirect
 	github.com/google/uuid v1.6.0 // indirect
 	github.com/klauspost/compress v1.17.11 // indirect
 	github.com/klauspost/cpuid/v2 v2.2.8 // indirect
 	github.com/kr/text v0.2.0 // indirect
+	github.com/lufia/plan9stats v0.0.0-20211012122336-39d0f177ccd0 // indirect
+	github.com/magiconair/properties v1.8.7 // indirect
 	github.com/minio/md5-simd v1.1.2 // indirect
+	github.com/moby/docker-image-spec v1.3.1 // indirect
+	github.com/moby/patternmatcher v0.6.0 // indirect
+	github.com/moby/sys/sequential v0.5.0 // indirect
+	github.com/moby/sys/user v0.1.0 // indirect
+	github.com/moby/term v0.5.0 // indirect
+	github.com/morikuni/aec v1.0.0 // indirect
 	github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect
+	github.com/opencontainers/go-digest v1.0.0 // indirect
+	github.com/opencontainers/image-spec v1.1.0 // indirect
+	github.com/pkg/errors v0.9.1 // indirect
 	github.com/pmezard/go-difflib v1.0.0 // indirect
+	github.com/power-devops/perfstat v0.0.0-20210106213030-5aafc221ea8c // indirect
 	github.com/prometheus/client_model v0.6.1 // indirect
 	github.com/prometheus/common v0.55.0 // indirect
 	github.com/prometheus/procfs v0.15.1 // indirect
 	github.com/rs/xid v1.6.0 // indirect
+	github.com/shirou/gopsutil/v3 v3.23.12 // indirect
+	github.com/shoenig/go-m1cpu v0.1.6 // indirect
+	github.com/sirupsen/logrus v1.9.3 // indirect
+	github.com/tklauser/go-sysconf v0.3.12 // indirect
+	github.com/tklauser/numcpus v0.6.1 // indirect
+	github.com/yusufpapurcu/wmi v1.2.3 // indirect
+	go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.49.0 // indirect
+	go.opentelemetry.io/otel v1.24.0 // indirect
+	go.opentelemetry.io/otel/metric v1.24.0 // indirect
+	go.opentelemetry.io/otel/trace v1.24.0 // indirect
 	golang.org/x/crypto v0.28.0 // indirect
 	golang.org/x/net v0.30.0 // indirect
 	golang.org/x/sys v0.26.0 // indirect