From 0e5d19808ea045a6f1294df7b8f4bbefd4b55166 Mon Sep 17 00:00:00 2001 From: justinsb Date: Thu, 22 Jun 2023 10:46:50 -0400 Subject: [PATCH] kubetest2-kops: add support for boskos-resource-type flag This will cause AWS tests to acquire an aws-account from boskos (when set to aws-account), and run the tests with those credentials. --- tests/e2e/kubetest2-kops/deployer/boskos.go | 92 +++++++++++++++++++ tests/e2e/kubetest2-kops/deployer/common.go | 52 +++++++---- tests/e2e/kubetest2-kops/deployer/deployer.go | 22 +++-- tests/e2e/kubetest2-kops/deployer/down.go | 17 +--- 4 files changed, 143 insertions(+), 40 deletions(-) create mode 100644 tests/e2e/kubetest2-kops/deployer/boskos.go diff --git a/tests/e2e/kubetest2-kops/deployer/boskos.go b/tests/e2e/kubetest2-kops/deployer/boskos.go new file mode 100644 index 0000000000000..c944feb0d8cc6 --- /dev/null +++ b/tests/e2e/kubetest2-kops/deployer/boskos.go @@ -0,0 +1,92 @@ +/* +Copyright 2023 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package deployer + +import ( + "context" + "fmt" + "os" + "time" + + "k8s.io/klog/v2" + "sigs.k8s.io/boskos/client" + "sigs.k8s.io/boskos/common" + "sigs.k8s.io/kubetest2/pkg/boskos" +) + +type boskosHelper struct { + // boskos holds the client we use for boskos communication. + boskos *client.Client + + // this channel serves as a signal channel for the boskos heartbeat goroutine + // so that it can be explicitly closed + boskosHeartbeatClose chan struct{} + + // resources tracks acquired resources so they can be freed in Cleanup. + resources []*common.Resource +} + +func (h *boskosHelper) Acquire(ctx context.Context, resourceType string) (*common.Resource, error) { + if h.boskos == nil { + h.boskosHeartbeatClose = make(chan struct{}) + + boskosURL := os.Getenv("BOSKOS_HOST") + if boskosURL == "" { + boskosURL = "http://boskos.test-pods.svc.cluster.local." + } + boskosClient, err := boskos.NewClient(boskosURL) + if err != nil { + return nil, fmt.Errorf("failed to make boskos client for %q: %w", boskosURL, err) + } + h.boskos = boskosClient + } + + resource, err := boskos.Acquire( + h.boskos, + resourceType, + 5*time.Minute, + 5*time.Minute, + h.boskosHeartbeatClose, + ) + if err != nil { + return nil, fmt.Errorf("failed to get %q resource from boskos: %w", resourceType, err) + } + h.resources = append(h.resources, resource) + + return resource, nil +} + +// Cleanup releases any resources acquired from boskos +func (h *boskosHelper) Cleanup(ctx context.Context) error { + if h.boskos != nil { + var resourceNames []string + for _, resource := range h.resources { + klog.V(2).Info("releasing boskos resource %v %q", resource.Type, resource.Name) + resourceNames = append(resourceNames, resource.Name) + } + err := boskos.Release( + h.boskos, + resourceNames, + h.boskosHeartbeatClose, + ) + if err != nil { + return fmt.Errorf("failed to release boskos resources %v: %w", resourceNames, err) + } + } + + return nil +} diff --git a/tests/e2e/kubetest2-kops/deployer/common.go b/tests/e2e/kubetest2-kops/deployer/common.go index 5d42d4c568879..e27e77131a57f 100644 --- a/tests/e2e/kubetest2-kops/deployer/common.go +++ b/tests/e2e/kubetest2-kops/deployer/common.go @@ -17,30 +17,29 @@ limitations under the License. package deployer import ( + "context" "errors" "fmt" "os" "path" "path/filepath" "strings" - "time" "k8s.io/klog/v2" "k8s.io/kops/tests/e2e/kubetest2-kops/gce" "k8s.io/kops/tests/e2e/pkg/kops" "k8s.io/kops/tests/e2e/pkg/target" "k8s.io/kops/tests/e2e/pkg/util" - "sigs.k8s.io/kubetest2/pkg/boskos" ) func (d *deployer) init() error { var err error - d.doInit.Do(func() { err = d.initialize() }) + d.doInit.Do(func() { err = d.initialize(context.TODO()) }) return err } // initialize should only be called by init(), behind a sync.Once -func (d *deployer) initialize() error { +func (d *deployer) initialize(ctx context.Context) error { if d.commonOptions.ShouldBuild() { if err := d.verifyBuildFlags(); err != nil { return fmt.Errorf("init failed to check build flags: %v", err) @@ -67,6 +66,29 @@ func (d *deployer) initialize() error { switch d.CloudProvider { case "aws": + if d.BoskosResourceType != "" { + klog.V(1).Info("acquiring AWS credentials from Boskos") + + resource, err := d.boskos.Acquire(ctx, d.BoskosResourceType) + if err != nil { + return fmt.Errorf("init failed to get resource %q from boskos: %w", d.BoskosResourceType, err) + } + klog.V(1).Infof("Got AWS account %s from boskos", resource.Name) + + accessKeyIDObj, ok := resource.UserData.Load("access-key-id") + if !ok { + return fmt.Errorf("access-key-id not found in boskos resource %q", resource.Name) + } + secretAccessKeyObj, ok := resource.UserData.Load("secret-access-key") + if !ok { + return fmt.Errorf("secret-access-key not found in boskos resource %q", resource.Name) + } + d.awsStaticCredentials = &awsStaticCredentials{ + AccessKeyID: accessKeyIDObj.(string), + SecretAccessKey: secretAccessKeyObj.(string), + } + } + if d.SSHPrivateKeyPath == "" || d.SSHPublicKeyPath == "" { publicKeyPath, privateKeyPath, err := util.CreateSSHKeyPair(d.ClusterName) if err != nil { @@ -87,21 +109,10 @@ func (d *deployer) initialize() error { if d.GCPProject == "" { klog.V(1).Info("No GCP project provided, acquiring from Boskos") - boskosClient, err := boskos.NewClient("http://boskos.test-pods.svc.cluster.local.") + resourceType := "gce-project" + resource, err := d.boskos.Acquire(ctx, resourceType) if err != nil { - return fmt.Errorf("failed to make boskos client: %s", err) - } - d.boskos = boskosClient - - resource, err := boskos.Acquire( - d.boskos, - "gce-project", - 5*time.Minute, - 5*time.Minute, - d.boskosHeartbeatClose, - ) - if err != nil { - return fmt.Errorf("init failed to get project from boskos: %s", err) + return fmt.Errorf("init failed to get %q resource from boskos: %w", resourceType, err) } d.GCPProject = resource.Name klog.V(1).Infof("Got project %s from boskos", d.GCPProject) @@ -201,6 +212,11 @@ func (d *deployer) env() []string { // Recognized by the e2e framework // https://github.com/kubernetes/kubernetes/blob/a750d8054a6cb3167f495829ce3e77ab0ccca48e/test/e2e/framework/ssh/ssh.go#L59-L62 vars = append(vars, fmt.Sprintf("KUBE_SSH_KEY_PATH=%v", d.SSHPrivateKeyPath)) + + if d.awsStaticCredentials != nil { + vars = append(vars, fmt.Sprintf("AWS_ACCESS_KEY_ID=%v", d.awsStaticCredentials.AccessKeyID)) + vars = append(vars, fmt.Sprintf("AWS_SECRET_ACCESS_KEY=%v", d.awsStaticCredentials.SecretAccessKey)) + } } else if d.CloudProvider == "digitalocean" { // Pass through some env vars if set for _, k := range []string{"DIGITALOCEAN_ACCESS_TOKEN", "S3_ACCESS_KEY_ID", "S3_SECRET_ACCESS_KEY"} { diff --git a/tests/e2e/kubetest2-kops/deployer/deployer.go b/tests/e2e/kubetest2-kops/deployer/deployer.go index 5ad0289636228..b5d7e5844aa93 100644 --- a/tests/e2e/kubetest2-kops/deployer/deployer.go +++ b/tests/e2e/kubetest2-kops/deployer/deployer.go @@ -28,7 +28,6 @@ import ( "k8s.io/kops/tests/e2e/kubetest2-kops/builder" "k8s.io/kops/tests/e2e/pkg/target" - "sigs.k8s.io/boskos/client" "sigs.k8s.io/kubetest2/pkg/types" ) @@ -82,13 +81,17 @@ type deployer struct { manifestPath string terraform *target.Terraform - // boskos struct field will be non-nil when the deployer is - // using boskos to acquire a GCP project - boskos *client.Client + BoskosResourceType string `flag:"boskos-resource-type" desc:"Resource type to acquire from boskos, for credentials"` - // this channel serves as a signal channel for the hearbeat goroutine - // so that it can be explicitly closed - boskosHeartbeatClose chan struct{} + boskos boskosHelper + + // awsStaticCredentials holds credentials for AWS loaded from boskos + awsStaticCredentials *awsStaticCredentials +} + +type awsStaticCredentials struct { + AccessKeyID string + SecretAccessKey string } // assert that New implements types.NewDeployer @@ -106,9 +109,8 @@ func (d *deployer) Provider() string { func New(opts types.Options) (types.Deployer, *pflag.FlagSet) { // create a deployer object and set fields that are not flag controlled d := &deployer{ - commonOptions: opts, - BuildOptions: &builder.BuildOptions{}, - boskosHeartbeatClose: make(chan struct{}), + commonOptions: opts, + BuildOptions: &builder.BuildOptions{}, } dir, err := defaultArtifactsDir() diff --git a/tests/e2e/kubetest2-kops/deployer/down.go b/tests/e2e/kubetest2-kops/deployer/down.go index d35d05d21c4aa..bb4b60969ea48 100644 --- a/tests/e2e/kubetest2-kops/deployer/down.go +++ b/tests/e2e/kubetest2-kops/deployer/down.go @@ -17,16 +17,17 @@ limitations under the License. package deployer import ( - "fmt" + "context" "strings" "k8s.io/klog/v2" "k8s.io/kops/tests/e2e/kubetest2-kops/gce" - "sigs.k8s.io/kubetest2/pkg/boskos" "sigs.k8s.io/kubetest2/pkg/exec" ) func (d *deployer) Down() error { + ctx := context.TODO() + if err := d.init(); err != nil { return err } @@ -58,16 +59,8 @@ func (d *deployer) Down() error { gce.DeleteGCSBucket(d.stateStore(), d.GCPProject) } - if d.boskos != nil { - klog.V(2).Info("releasing boskos project") - err := boskos.Release( - d.boskos, - []string{d.GCPProject}, - d.boskosHeartbeatClose, - ) - if err != nil { - return fmt.Errorf("down failed to release boskos project: %s", err) - } + if err := d.boskos.Cleanup(ctx); err != nil { + return err } return nil }