kubetest2-kops: automatically create buckets

When working with boskos, this means we don't need to have a shared
bucket with dynamic permissions.

tests/e2e/go.mod

@@ -8,9 +8,12 @@ replace k8s.io/kops => ../../.
 replace k8s.io/client-go => k8s.io/client-go v0.24.2
 
 require (
+	github.com/aws/aws-sdk-go v1.44.283
 	github.com/blang/semver/v4 v4.0.0
 	github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510
+	github.com/google/uuid v1.3.0
 	github.com/octago/sflags v0.2.0
+	github.com/spf13/cobra v1.7.0
 	github.com/spf13/pflag v1.0.5
 	k8s.io/api v0.27.3
 	k8s.io/apimachinery v0.27.3
@@ -35,7 +38,6 @@ require (
 	github.com/StackExchange/wmi v1.2.1 // indirect
 	github.com/acomagu/bufpipe v1.0.3 // indirect
 	github.com/apparentlymart/go-cidr v1.1.0 // indirect
-	github.com/aws/aws-sdk-go v1.44.283 // indirect
 	github.com/beorn7/perks v1.0.1 // indirect
 	github.com/blang/semver v3.5.1+incompatible // indirect
 	github.com/cespare/xxhash/v2 v2.2.0 // indirect
@@ -70,7 +72,6 @@ require (
 	github.com/google/go-querystring v1.1.0 // indirect
 	github.com/google/gofuzz v1.2.1-0.20210504230335-f78f29fc09ea // indirect
 	github.com/google/s2a-go v0.1.4 // indirect
-	github.com/google/uuid v1.3.0 // indirect
 	github.com/googleapis/enterprise-certificate-proxy v0.2.4 // indirect
 	github.com/googleapis/gax-go/v2 v2.10.0 // indirect
 	github.com/gophercloud/gophercloud v1.4.0 // indirect
@@ -106,7 +107,6 @@ require (
 	github.com/sergi/go-diff v1.3.1 // indirect
 	github.com/shirou/gopsutil/v3 v3.21.10 // indirect
 	github.com/sirupsen/logrus v1.9.0 // indirect
-	github.com/spf13/cobra v1.7.0 // indirect
 	github.com/ulikunitz/xz v0.5.10 // indirect
 	github.com/vbatts/tar-split v0.11.3 // indirect
 	github.com/xanzy/ssh-agent v0.3.1 // indirect

tests/e2e/kubetest2-kops/deployer/common.go

@@ -25,7 +25,9 @@ import (
 	"path/filepath"
 	"strings"
 
+	"github.com/aws/aws-sdk-go/aws/credentials"
 	"k8s.io/klog/v2"
+	"k8s.io/kops/tests/e2e/kubetest2-kops/aws"
 	"k8s.io/kops/tests/e2e/kubetest2-kops/gce"
 	"k8s.io/kops/tests/e2e/pkg/kops"
 	"k8s.io/kops/tests/e2e/pkg/target"
@@ -83,10 +85,8 @@ func (d *deployer) initialize(ctx context.Context) error {
 			if !ok {
 				return fmt.Errorf("secret-access-key not found in boskos resource %q", resource.Name)
 			}
-			d.awsStaticCredentials = &awsStaticCredentials{
-				AccessKeyID:     accessKeyIDObj.(string),
-				SecretAccessKey: secretAccessKeyObj.(string),
-			}
+			d.awsCredentials = credentials.NewStaticCredentials(accessKeyIDObj.(string), secretAccessKeyObj.(string), "")
+			d.createBucket = true
 		}
 
 		if d.SSHPrivateKeyPath == "" || d.SSHPublicKeyPath == "" {
@@ -213,9 +213,13 @@ func (d *deployer) env() []string {
 		// https://github.com/kubernetes/kubernetes/blob/a750d8054a6cb3167f495829ce3e77ab0ccca48e/test/e2e/framework/ssh/ssh.go#L59-L62
 		vars = append(vars, fmt.Sprintf("KUBE_SSH_KEY_PATH=%v", d.SSHPrivateKeyPath))
 
-		if d.awsStaticCredentials != nil {
-			vars = append(vars, fmt.Sprintf("AWS_ACCESS_KEY_ID=%v", d.awsStaticCredentials.AccessKeyID))
-			vars = append(vars, fmt.Sprintf("AWS_SECRET_ACCESS_KEY=%v", d.awsStaticCredentials.SecretAccessKey))
+		if d.awsCredentials != nil {
+			credentials, err := d.awsCredentials.Get()
+			if err != nil {
+				klog.Fatalf("error getting aws credentials: %v", err)
+			}
+			vars = append(vars, fmt.Sprintf("AWS_ACCESS_KEY_ID=%v", credentials.AccessKeyID))
+			vars = append(vars, fmt.Sprintf("AWS_SECRET_ACCESS_KEY=%v", credentials.SecretAccessKey))
 		}
 	} else if d.CloudProvider == "digitalocean" {
 		// Pass through some env vars if set
@@ -277,11 +281,18 @@ func defaultClusterName(cloudProvider string) (string, error) {
 // stateStore returns the kops state store to use
 // defaulting to values used in prow jobs
 func (d *deployer) stateStore() string {
+	ctx := context.TODO()
+
 	ss := os.Getenv("KOPS_STATE_STORE")
 	if ss == "" {
 		switch d.CloudProvider {
 		case "aws":
-			ss = "s3://k8s-kops-prow"
+			bucketName, err := aws.AWSBucketName(ctx, d.awsCredentials)
+			if err != nil {
+				klog.Fatalf("error building aws bucket name: %v", err)
+			}
+			ss = "s3://" + bucketName
+			d.createBucket = true
 		case "gce":
 			d.createBucket = true
 			ss = "gs://" + gce.GCSBucketName(d.GCPProject)

tests/e2e/kubetest2-kops/deployer/deployer.go

@@ -22,6 +22,7 @@ import (
 	"sync"
 	"time"
 
+	"github.com/aws/aws-sdk-go/aws/credentials"
 	"github.com/octago/sflags/gen/gpflag"
 	"github.com/spf13/pflag"
 	"k8s.io/klog/v2"
@@ -85,13 +86,8 @@ type deployer struct {
 
 	boskos boskosHelper
 
-	// awsStaticCredentials holds credentials for AWS loaded from boskos
-	awsStaticCredentials *awsStaticCredentials
-}
-
-type awsStaticCredentials struct {
-	AccessKeyID     string
-	SecretAccessKey string
+	// awsCredentials holds credentials for AWS loaded from boskos
+	awsCredentials *credentials.Credentials
 }
 
 // assert that New implements types.NewDeployer

tests/e2e/kubetest2-kops/deployer/down.go

@@ -18,9 +18,11 @@ package deployer
 
 import (
 	"context"
+	"fmt"
 	"strings"
 
 	"k8s.io/klog/v2"
+	"k8s.io/kops/tests/e2e/kubetest2-kops/aws"
 	"k8s.io/kops/tests/e2e/kubetest2-kops/gce"
 	"sigs.k8s.io/kubetest2/pkg/exec"
 )
@@ -55,8 +57,22 @@ func (d *deployer) Down() error {
 		return err
 	}
 
-	if d.CloudProvider == "gce" && d.createBucket {
-		gce.DeleteGCSBucket(d.stateStore(), d.GCPProject)
+	if d.createBucket {
+		switch d.CloudProvider {
+		case "gce":
+			gce.DeleteGCSBucket(d.stateStore(), d.GCPProject)
+		case "aws":
+			bucketName, err := aws.AWSBucketName(ctx, d.awsCredentials)
+			if err != nil {
+				return fmt.Errorf("error building aws bucket name: %w", err)
+			}
+
+			if err := aws.DeleteAWSBucket(ctx, d.awsCredentials, bucketName); err != nil {
+				klog.Warningf("error deleting AWS bucket: %w", err)
+			}
+		default:
+			return fmt.Errorf("bucket cleanup not implemented for cloud %q", d.CloudProvider)
+		}
 	}
 
 	if err := d.boskos.Cleanup(ctx); err != nil {

tests/e2e/kubetest2-kops/deployer/up.go

@@ -17,6 +17,7 @@ limitations under the License.
 package deployer
 
 import (
+	"context"
 	"errors"
 	"fmt"
 	osexec "os/exec"
@@ -36,6 +37,8 @@ import (
 )
 
 func (d *deployer) Up() error {
+	ctx := context.TODO()
+
 	if err := d.init(); err != nil {
 		return err
 	}
@@ -47,9 +50,18 @@ func (d *deployer) Up() error {
 		_ = d.Down()
 	}
 
-	if d.CloudProvider == "gce" && d.createBucket {
-		if err := gce.EnsureGCSBucket(d.stateStore(), d.GCPProject); err != nil {
-			return err
+	if d.createBucket {
+		switch d.CloudProvider {
+		case "gce":
+			if err := gce.EnsureGCSBucket(d.stateStore(), d.GCPProject); err != nil {
+				return err
+			}
+		case "aws":
+			if err := aws.EnsureAWSBucket(ctx, d.awsCredentials, d.stateStore()); err != nil {
+				return err
+			}
+		default:
+			return fmt.Errorf("bucket creation not implemented for cloud %q", d.CloudProvider)
 		}
 	}