Add Vault e2e tests

e2e/Dockerfile

@@ -17,17 +17,15 @@ RUN apk add -U --no-cache \
     curl \
     tzdata \
     libc6-compat \
-    openssl
+    openssl \
+    jq
 
 COPY --from=builder /go/bin/ginkgo /usr/local/bin/
 COPY --from=builder /usr/local/bin/kubectl /usr/local/bin/
 COPY --from=builder /usr/local/bin/helm /usr/local/bin/
 
 COPY entrypoint.sh                  /entrypoint.sh
 COPY e2e.test                       /e2e.test
-COPY wait-for-secret-manager.sh     /wait-for-secret-manager.sh
-COPY wait-for-localstack.sh         /wait-for-localstack.sh
 COPY k8s                            /k8s
-COPY localstack.deployment.yaml     /localstack.deployment.yaml
 
 CMD [ "/entrypoint.sh" ]

e2e/addon/vault/util.go

@@ -0,0 +1,156 @@
+/*
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package vault
+
+import (
+	"context"
+	"errors"
+	"fmt"
+	"time"
+
+	vault "github.com/hashicorp/vault/api"
+
+	smmeta "github.com/itscontained/secret-manager/pkg/apis/meta/v1"
+	smv1alpha1 "github.com/itscontained/secret-manager/pkg/apis/secretmanager/v1alpha1"
+
+	corev1 "k8s.io/api/core/v1"
+
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/types"
+	"k8s.io/apimachinery/pkg/util/uuid"
+)
+
+func (v *Vault) CreateSecretStoreVaultKubeAuth() (*smv1alpha1.SecretStoreSpec, error) {
+	randomSuffix := uuid.NewUUID()
+	v.kubePath = fmt.Sprintf("kubernetes-%s", randomSuffix)
+	v.kubeRole = fmt.Sprintf("role-%s", randomSuffix)
+
+	// Create Kubernetes auth backend
+	err := v.vaultClient.Sys().EnableAuthWithOptions(v.kubePath, &vault.EnableAuthOptions{
+		Type: "kubernetes",
+	})
+	if err != nil {
+		return nil, err
+	}
+
+	// Get token reviewer jwt for vault
+	tokenReviewerAccount := &corev1.ServiceAccount{}
+	v.KubeClient.Get(context.Background(), types.NamespacedName{
+		Namespace: v.Namespace,
+		Name:      serviceAccountName,
+	}, tokenReviewerAccount)
+
+	if len(tokenReviewerAccount.Secrets) == 0 {
+		return nil, errors.New("vault serviceaccount has no associated secret")
+	}
+	tokenSecret := &corev1.Secret{}
+	v.KubeClient.Get(context.Background(), types.NamespacedName{
+		Namespace: v.Namespace,
+		Name:      tokenReviewerAccount.Secrets[0].Name,
+	}, tokenSecret)
+
+	kubeReq := v.vaultClient.NewRequest("POST", fmt.Sprintf("/v1/auth/%s/config", v.kubePath))
+	kubeData := map[string]interface{}{
+		"kubernetes_host":    "https://kubernetes.default",
+		"token_reviewer_jwt": string(tokenSecret.Data["token"]),
+	}
+	kubeReq.SetJSONBody(kubeData)
+	_, err = v.vaultClient.RawRequest(kubeReq)
+	if err != nil {
+		return nil, err
+	}
+
+	// Create vault role for Kubernetes auth
+	svcAccountName := fmt.Sprintf("kv-reader-%s", randomSuffix)
+	req := v.vaultClient.NewRequest("POST", fmt.Sprintf("/v1/auth/%s/role/%s", v.kubePath, v.kubeRole))
+	roleData := map[string]interface{}{
+		"bound_service_account_names":      []string{svcAccountName},
+		"bound_service_account_namespaces": []string{v.Namespace},
+		"token_policies":                   []string{v.secretRole},
+	}
+	req.SetJSONBody(roleData)
+	_, err = v.vaultClient.RawRequest(req)
+	if err != nil {
+		return nil, err
+	}
+
+	err = v.KubeClient.Create(context.Background(), &corev1.ServiceAccount{
+		ObjectMeta: metav1.ObjectMeta{
+			Name:      svcAccountName,
+			Namespace: v.Namespace,
+		},
+	})
+	if err != nil {
+		return nil, err
+	}
+
+	svcAccountCreated := &corev1.ServiceAccount{}
+	svcAccountKey := types.NamespacedName{
+		Name:      svcAccountName,
+		Namespace: v.Namespace,
+	}
+
+	err = try(3, time.Second*2, func() error {
+		err = v.KubeClient.Get(context.Background(), svcAccountKey, svcAccountCreated)
+		if err != nil {
+			return err
+		}
+
+		if len(svcAccountCreated.Secrets) < 1 {
+			return errors.New("created service account has no token secret")
+		}
+		return nil
+	})
+	if err != nil {
+		return nil, err
+	}
+
+	return &smv1alpha1.SecretStoreSpec{
+		Vault: &smv1alpha1.VaultStore{
+			Server: v.Host,
+			Path:   v.kvPath,
+			Auth: smv1alpha1.VaultAuth{
+				Kubernetes: &smv1alpha1.VaultKubernetesAuth{
+					Path: v.kubePath,
+					Role: v.kubeRole,
+					SecretRef: &smmeta.SecretKeySelector{
+						LocalObjectReference: smmeta.LocalObjectReference{
+							Name: svcAccountCreated.Secrets[0].Name,
+						},
+						Key: "token",
+					},
+				},
+			},
+		},
+	}, nil
+}
+
+// try attempts a function for n retries, with pollPeriod waiting in-between
+// when the function returns no error, nil is return. Error is returned
+// after retries with error wrapped.
+func try(retries int, pollPeriod time.Duration, f func() error) error {
+	attempt := 0
+	var err error
+	for attempt < retries {
+		err = f()
+		if err == nil {
+			return nil
+		}
+		time.Sleep(pollPeriod)
+		attempt++
+	}
+
+	return fmt.Errorf("retry attempts failed: %w", err)
+}

e2e/addon/vault/vault.go

@@ -0,0 +1,155 @@
+/*
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package vault
+
+import (
+	"context"
+	"errors"
+	"fmt"
+	"os/exec"
+	"path/filepath"
+	"strings"
+	"time"
+
+	vault "github.com/hashicorp/vault/api"
+
+	rbacv1 "k8s.io/api/rbac/v1"
+
+	apierrors "k8s.io/apimachinery/pkg/api/errors"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+
+	ctrlclient "sigs.k8s.io/controller-runtime/pkg/client"
+)
+
+const (
+	selector           string = "app=vault"
+	serviceAddress     string = "http://vault.%s:8200"
+	serviceAccountName string = "vault"
+	defaultToken       string = "root"
+	secretPolicy       string = `
+path "%s/*" {
+  capabilities = ["read"]
+}
+`
+)
+
+// Vault describes the configuration details for an instance of Vault
+// deployed to the test cluster
+type Vault struct {
+	// Kubectl is the path to kubectl
+	Kubectl string
+
+	// Namespace is the namespace to deploy Vault into
+	Namespace string
+
+	// KubeClient is a configured Kubernetes clientset for addons to use.
+	KubeClient ctrlclient.Client
+
+	// Host is the hostname that can be used to connect to Vault
+	Host string
+
+	// BasePath is root of deployment manifests
+	BasePath string
+
+	vaultClient *vault.Client
+	kvPath      string
+	secretRole  string
+	kubePath    string
+	kubeRole    string
+}
+
+func (v *Vault) Setup() error {
+	if v.Kubectl == "" {
+		return errors.New("kubectl must be set")
+	}
+
+	if v.Namespace == "" {
+		return errors.New("namespace must be set")
+	}
+	v.kvPath = "secret"
+
+	err := Run(fmt.Sprintf("%s apply --timeout 2m -f %s -n %s", v.Kubectl, filepath.Join(v.BasePath, "/k8s/vault/vault.yaml"), v.Namespace))
+	if err != nil {
+		return err
+	}
+	// create cluster resources because namespace is not known beforehand
+	err = v.createClusterResources()
+	if err != nil {
+		return err
+	}
+
+	time.Sleep(10 * time.Second)
+	err = Run(fmt.Sprintf("%s wait pod --for=condition=Ready --timeout 2m -n %s -l %s", v.Kubectl, v.Namespace, selector))
+	if err != nil {
+		return err
+	}
+	v.Host = fmt.Sprintf(serviceAddress, v.Namespace)
+	vaultConfig := vault.DefaultConfig()
+	vaultConfig.Address = v.Host
+	v.vaultClient, err = vault.NewClient(vaultConfig)
+	if err != nil {
+		return err
+	}
+	v.vaultClient.SetToken(defaultToken)
+
+	v.secretRole = "kv-reader"
+	err = v.vaultClient.Sys().PutPolicy(v.secretRole, fmt.Sprintf(secretPolicy, v.kvPath))
+	return err
+}
+
+func Run(cmd string) error {
+	splitCmd := strings.Split(cmd, " ")
+	c := exec.Command(splitCmd[0], splitCmd[1:]...)
+	out, err := c.CombinedOutput()
+	if err != nil {
+		return fmt.Errorf("error executing command: %s\nerr: %w", string(out), err)
+	}
+	return nil
+}
+
+func (v *Vault) CreateSecret(name string, secret map[string]string) error {
+	vaultPath := fmt.Sprintf("%s/data/%s", v.kvPath, name)
+	vaultData := make(map[string]interface{}, 1)
+	vaultData["data"] = secret
+	_, err := v.vaultClient.Logical().Write(vaultPath, vaultData)
+	return err
+}
+
+func (v *Vault) createClusterResources() error {
+	vaultCRB := &rbacv1.ClusterRoleBinding{
+		ObjectMeta: metav1.ObjectMeta{
+			Name: fmt.Sprintf("vault-auth-%s", v.Namespace),
+		},
+		RoleRef: rbacv1.RoleRef{
+			Kind: "ClusterRole",
+			Name: "system:auth-delegator",
+		},
+		Subjects: []rbacv1.Subject{
+			{
+				Kind:      rbacv1.ServiceAccountKind,
+				Name:      serviceAccountName,
+				Namespace: v.Namespace,
+			},
+		},
+	}
+
+	err := v.KubeClient.Create(context.Background(), vaultCRB)
+	if err != nil {
+		if !apierrors.IsAlreadyExists(err) {
+			return err
+		}
+	}
+	return nil
+}

e2e/entrypoint.sh

@@ -39,7 +39,7 @@ ginkgo_args=(
   "-slowSpecThreshold=${SLOW_E2E_THRESHOLD}"
   "-r"
   "-v"
-  "-timeout=45m"
+  "-timeout=5m"
 )
 
 echo -e "${BGREEN}Running e2e test suite (FOCUS=${FOCUS})...${NC}"

e2e/framework/aws.go

@@ -17,11 +17,18 @@ package framework
 import (
 	"context"
 	"fmt"
+	"os/exec"
 
 	"github.com/aws/aws-sdk-go-v2/aws"
 	"github.com/aws/aws-sdk-go-v2/aws/endpoints"
 	"github.com/aws/aws-sdk-go-v2/aws/external"
 	"github.com/aws/aws-sdk-go-v2/service/secretsmanager"
+
+	"github.com/onsi/ginkgo"
+)
+
+const (
+	localstackDeploy = "/k8s/aws/deploy.sh"
 )
 
 // CreateAWSSecretsManagerSecret creates a sm secret with the given value
@@ -42,7 +49,7 @@ func CreateAWSSecretsManagerSecret(namespace, name, secret string) error {
 	return err
 }
 
-// localResolver resolves endpoints to
+// localResolver resolves endpoints to e2e localstack
 type localResolver struct {
 	endpoints.Resolver
 	namespace string
@@ -54,3 +61,14 @@ func (r *localResolver) ResolveEndpoint(service, region string) (aws.Endpoint, e
 		URL: fmt.Sprintf("http://localstack.%s", r.namespace),
 	}, nil
 }
+
+// NewLocalstack deploys a fresh localstack instance into the specified namespace
+func (f *Framework) NewLocalstack(namespace string) error {
+	ginkgo.By("launching localstack")
+	cmd := exec.Command(localstackDeploy, namespace, f.HelmValues)
+	out, err := cmd.CombinedOutput()
+	if err != nil {
+		return fmt.Errorf("unexpected error creating localstack: %v.\nLogs:\n%v", err, string(out))
+	}
+	return nil
+}