kubetest2-kops: add support for boskos-resource-type flag

This will cause AWS tests to acquire an aws-account from boskos (when
set to aws-account), and run the tests with those credentials.

tests/e2e/kubetest2-kops/deployer/boskos.go

@@ -0,0 +1,92 @@
+/*
+Copyright 2023 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package deployer
+
+import (
+	"context"
+	"fmt"
+	"os"
+	"time"
+
+	"k8s.io/klog/v2"
+	"sigs.k8s.io/boskos/client"
+	"sigs.k8s.io/boskos/common"
+	"sigs.k8s.io/kubetest2/pkg/boskos"
+)
+
+type boskosHelper struct {
+	// boskos holds the client we use for boskos communication.
+	boskos *client.Client
+
+	// this channel serves as a signal channel for the boskos heartbeat goroutine
+	// so that it can be explicitly closed
+	boskosHeartbeatClose chan struct{}
+
+	// resources tracks acquired resources so they can be freed in Cleanup.
+	resources []*common.Resource
+}
+
+func (h *boskosHelper) Acquire(ctx context.Context, resourceType string) (*common.Resource, error) {
+	if h.boskos == nil {
+		h.boskosHeartbeatClose = make(chan struct{})
+
+		boskosURL := os.Getenv("BOSKOS_HOST")
+		if boskosURL == "" {
+			boskosURL = "http://boskos.test-pods.svc.cluster.local."
+		}
+		boskosClient, err := boskos.NewClient(boskosURL)
+		if err != nil {
+			return nil, fmt.Errorf("failed to make boskos client for %q: %w", boskosURL, err)
+		}
+		h.boskos = boskosClient
+	}
+
+	resource, err := boskos.Acquire(
+		h.boskos,
+		resourceType,
+		5*time.Minute,
+		5*time.Minute,
+		h.boskosHeartbeatClose,
+	)
+	if err != nil {
+		return nil, fmt.Errorf("failed to get %q resource from boskos: %w", resourceType, err)
+	}
+	h.resources = append(h.resources, resource)
+
+	return resource, nil
+}
+
+// Cleanup releases any resources acquired from boskos
+func (h *boskosHelper) Cleanup(ctx context.Context) error {
+	if h.boskos != nil {
+		var resourceNames []string
+		for _, resource := range h.resources {
+			klog.V(2).Info("releasing boskos resource %v %q", resource.Type, resource.Name)
+			resourceNames = append(resourceNames, resource.Name)
+		}
+		err := boskos.Release(
+			h.boskos,
+			resourceNames,
+			h.boskosHeartbeatClose,
+		)
+		if err != nil {
+			return fmt.Errorf("failed to release boskos resources %v: %w", resourceNames, err)
+		}
+	}
+
+	return nil
+}

tests/e2e/kubetest2-kops/deployer/common.go

@@ -17,30 +17,29 @@ limitations under the License.
 package deployer
 
 import (
+	"context"
 	"errors"
 	"fmt"
 	"os"
 	"path"
 	"path/filepath"
 	"strings"
-	"time"
 
 	"k8s.io/klog/v2"
 	"k8s.io/kops/tests/e2e/kubetest2-kops/gce"
 	"k8s.io/kops/tests/e2e/pkg/kops"
 	"k8s.io/kops/tests/e2e/pkg/target"
 	"k8s.io/kops/tests/e2e/pkg/util"
-	"sigs.k8s.io/kubetest2/pkg/boskos"
 )
 
 func (d *deployer) init() error {
 	var err error
-	d.doInit.Do(func() { err = d.initialize() })
+	d.doInit.Do(func() { err = d.initialize(context.TODO()) })
 	return err
 }
 
 // initialize should only be called by init(), behind a sync.Once
-func (d *deployer) initialize() error {
+func (d *deployer) initialize(ctx context.Context) error {
 	if d.commonOptions.ShouldBuild() {
 		if err := d.verifyBuildFlags(); err != nil {
 			return fmt.Errorf("init failed to check build flags: %v", err)
@@ -67,6 +66,29 @@ func (d *deployer) initialize() error {
 
 	switch d.CloudProvider {
 	case "aws":
+		if d.BoskosResourceType != "" {
+			klog.V(1).Info("acquiring AWS credentials from Boskos")
+
+			resource, err := d.boskos.Acquire(ctx, d.BoskosResourceType)
+			if err != nil {
+				return fmt.Errorf("init failed to get resource %q from boskos: %w", d.BoskosResourceType, err)
+			}
+			klog.V(1).Infof("Got AWS account %s from boskos", resource.Name)
+
+			accessKeyIDObj, ok := resource.UserData.Load("access-key-id")
+			if !ok {
+				return fmt.Errorf("access-key-id not found in boskos resource %q", resource.Name)
+			}
+			secretAccessKeyObj, ok := resource.UserData.Load("secret-access-key")
+			if !ok {
+				return fmt.Errorf("secret-access-key not found in boskos resource %q", resource.Name)
+			}
+			d.awsStaticCredentials = &awsStaticCredentials{
+				AccessKeyID:     accessKeyIDObj.(string),
+				SecretAccessKey: secretAccessKeyObj.(string),
+			}
+		}
+
 		if d.SSHPrivateKeyPath == "" || d.SSHPublicKeyPath == "" {
 			publicKeyPath, privateKeyPath, err := util.CreateSSHKeyPair(d.ClusterName)
 			if err != nil {
@@ -87,21 +109,10 @@ func (d *deployer) initialize() error {
 		if d.GCPProject == "" {
 			klog.V(1).Info("No GCP project provided, acquiring from Boskos")
 
-			boskosClient, err := boskos.NewClient("http://boskos.test-pods.svc.cluster.local.")
+			resourceType := "gce-project"
+			resource, err := d.boskos.Acquire(ctx, resourceType)
 			if err != nil {
-				return fmt.Errorf("failed to make boskos client: %s", err)
-			}
-			d.boskos = boskosClient
-
-			resource, err := boskos.Acquire(
-				d.boskos,
-				"gce-project",
-				5*time.Minute,
-				5*time.Minute,
-				d.boskosHeartbeatClose,
-			)
-			if err != nil {
-				return fmt.Errorf("init failed to get project from boskos: %s", err)
+				return fmt.Errorf("init failed to get %q resource from boskos: %w", resourceType, err)
 			}
 			d.GCPProject = resource.Name
 			klog.V(1).Infof("Got project %s from boskos", d.GCPProject)
@@ -201,6 +212,11 @@ func (d *deployer) env() []string {
 		// Recognized by the e2e framework
 		// https://github.com/kubernetes/kubernetes/blob/a750d8054a6cb3167f495829ce3e77ab0ccca48e/test/e2e/framework/ssh/ssh.go#L59-L62
 		vars = append(vars, fmt.Sprintf("KUBE_SSH_KEY_PATH=%v", d.SSHPrivateKeyPath))
+
+		if d.awsStaticCredentials != nil {
+			vars = append(vars, fmt.Sprintf("AWS_ACCESS_KEY_ID=%v", d.awsStaticCredentials.AccessKeyID))
+			vars = append(vars, fmt.Sprintf("AWS_SECRET_ACCESS_KEY=%v", d.awsStaticCredentials.SecretAccessKey))
+		}
 	} else if d.CloudProvider == "digitalocean" {
 		// Pass through some env vars if set
 		for _, k := range []string{"DIGITALOCEAN_ACCESS_TOKEN", "S3_ACCESS_KEY_ID", "S3_SECRET_ACCESS_KEY"} {

tests/e2e/kubetest2-kops/deployer/deployer.go

@@ -28,7 +28,6 @@ import (
 	"k8s.io/kops/tests/e2e/kubetest2-kops/builder"
 	"k8s.io/kops/tests/e2e/pkg/target"
 
-	"sigs.k8s.io/boskos/client"
 	"sigs.k8s.io/kubetest2/pkg/types"
 )
 
@@ -82,13 +81,17 @@ type deployer struct {
 	manifestPath string
 	terraform    *target.Terraform
 
-	// boskos struct field will be non-nil when the deployer is
-	// using boskos to acquire a GCP project
-	boskos *client.Client
+	BoskosResourceType string `flag:"boskos-resource-type" desc:"Resource type to acquire from boskos, for credentials"`
 
-	// this channel serves as a signal channel for the hearbeat goroutine
-	// so that it can be explicitly closed
-	boskosHeartbeatClose chan struct{}
+	boskos boskosHelper
+
+	// awsStaticCredentials holds credentials for AWS loaded from boskos
+	awsStaticCredentials *awsStaticCredentials
+}
+
+type awsStaticCredentials struct {
+	AccessKeyID     string
+	SecretAccessKey string
 }
 
 // assert that New implements types.NewDeployer
@@ -106,9 +109,8 @@ func (d *deployer) Provider() string {
 func New(opts types.Options) (types.Deployer, *pflag.FlagSet) {
 	// create a deployer object and set fields that are not flag controlled
 	d := &deployer{
-		commonOptions:        opts,
-		BuildOptions:         &builder.BuildOptions{},
-		boskosHeartbeatClose: make(chan struct{}),
+		commonOptions: opts,
+		BuildOptions:  &builder.BuildOptions{},
 	}
 
 	dir, err := defaultArtifactsDir()

tests/e2e/kubetest2-kops/deployer/down.go

@@ -17,16 +17,17 @@ limitations under the License.
 package deployer
 
 import (
-	"fmt"
+	"context"
 	"strings"
 
 	"k8s.io/klog/v2"
 	"k8s.io/kops/tests/e2e/kubetest2-kops/gce"
-	"sigs.k8s.io/kubetest2/pkg/boskos"
 	"sigs.k8s.io/kubetest2/pkg/exec"
 )
 
 func (d *deployer) Down() error {
+	ctx := context.TODO()
+
 	if err := d.init(); err != nil {
 		return err
 	}
@@ -58,16 +59,8 @@ func (d *deployer) Down() error {
 		gce.DeleteGCSBucket(d.stateStore(), d.GCPProject)
 	}
 
-	if d.boskos != nil {
-		klog.V(2).Info("releasing boskos project")
-		err := boskos.Release(
-			d.boskos,
-			[]string{d.GCPProject},
-			d.boskosHeartbeatClose,
-		)
-		if err != nil {
-			return fmt.Errorf("down failed to release boskos project: %s", err)
-		}
+	if err := d.boskos.Cleanup(ctx); err != nil {
+		return err
 	}
 	return nil
 }