From fad9f1c3495935b8b60913f957357c3491e5d073 Mon Sep 17 00:00:00 2001 From: Malte Poll <1780588+malt3@users.noreply.github.com> Date: Thu, 18 Apr 2024 16:20:22 +0200 Subject: [PATCH] e2e: use ContrastTest for servicemesh test --- .github/workflows/e2e_servicemesh.yml | 12 +- e2e/servicemesh/servicemesh_test.go | 178 ++++++++------------------ 2 files changed, 52 insertions(+), 138 deletions(-) diff --git a/.github/workflows/e2e_servicemesh.yml b/.github/workflows/e2e_servicemesh.yml index d3fd7171f8..79bccf2fbb 100644 --- a/.github/workflows/e2e_servicemesh.yml +++ b/.github/workflows/e2e_servicemesh.yml @@ -38,33 +38,25 @@ jobs: with: creds: ${{ secrets.CONTRAST_CI_INFRA_AZURE }} - uses: nicknovitski/nix-develop@a2060d116a50b36dfab02280af558e73ab52427d # v1.1.0 - - name: Generate namespace suffix - id: ns - run: | - uuid=$(cat /proc/sys/kernel/random/uuid) - uid=${uuid##*-} - echo "namespace_suffix=$uid" >> "$GITHUB_OUTPUT" - name: Create justfile.env run: | cat < justfile.env container_registry=${{ env.container_registry }} azure_resource_group=${{ env.azure_resource_group }} - namespace_suffix=-${{ steps.ns.outputs.namespace_suffix }} EOF - name: Get credentials for CI cluster run: | just get-credentials - name: Build and prepare deployments run: | - just coordinator initializer service-mesh-proxy - just populate emojivoto-sm-ingress + just coordinator initializer port-forwarder service-mesh-proxy - name: Setup Summary run: | cat ./workspace/just.namespace | tee -a "${GITHUB_STEP_SUMMARY}" cat ./workspace/just.perf | tee -a "${GITHUB_STEP_SUMMARY}" - name: E2E Test run: | - env K8S_NAMESPACE=$(cat ./workspace/just.namespace) nix shell .#contrast.e2e --command servicemesh.test -test.v + nix shell .#contrast.e2e --command servicemesh.test -test.v workspace/just.containerlookup - name: Undeploy if: always() && inputs.skip-undeploy != 'true' run: | diff --git a/e2e/servicemesh/servicemesh_test.go b/e2e/servicemesh/servicemesh_test.go index 84dcfa01ee..7bb2ccb66a 100644 --- a/e2e/servicemesh/servicemesh_test.go +++ b/e2e/servicemesh/servicemesh_test.go @@ -4,150 +4,55 @@ package servicemesh import ( - "bytes" "context" "crypto/tls" "crypto/x509" + "flag" "fmt" - "io" + "log" "net/http" "os" - "path" - "path/filepath" "testing" "time" - "github.com/edgelesssys/contrast/cli/cmd" + "github.com/edgelesssys/contrast/e2e/internal/contrasttest" "github.com/edgelesssys/contrast/e2e/internal/kubeclient" - "github.com/edgelesssys/contrast/internal/kubeapi" - "github.com/stretchr/testify/assert" + "github.com/edgelesssys/contrast/e2e/internal/kuberesource" "github.com/stretchr/testify/require" "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" ) -// namespace the tests are executed in. -const namespaceEnv = "K8S_NAMESPACE" +var imageReplacements map[string]string // TestIngress tests that the ingress proxies work as configured. func TestIngress(t *testing.T) { - c := kubeclient.NewForTest(t) + ct := contrasttest.New(t) - namespace := os.Getenv(namespaceEnv) - require.NotEmpty(t, namespace, "environment variable %q must be set", namespaceEnv) - - resources, err := filepath.Glob("./workspace/deployment/*.yml") + resources, err := kuberesource.EmojivotoIngress() require.NoError(t, err) - require.True(t, t.Run("generate", func(t *testing.T) { - require := require.New(t) - - args := []string{ - "--workspace-dir", "./workspace", - } - args = append(args, resources...) - - generate := cmd.NewGenerateCmd() - generate.Flags().String("workspace-dir", "", "") // Make generate aware of root flags - generate.SetArgs(args) - generate.SetOut(io.Discard) - errBuf := &bytes.Buffer{} - generate.SetErr(errBuf) - - require.NoError(generate.Execute(), "could not generate manifest: %s", errBuf) - })) + resources = kuberesource.PatchImages(resources, imageReplacements) - // TODO(burgerdev): policy hash should come from contrast generate output. - coordinatorPolicyHashBytes, err := os.ReadFile("workspace/coordinator-policy.sha256") + unstructuredResources, err := kuberesource.ResourcesToUnstructured(resources) require.NoError(t, err) - coordinatorPolicyHash := string(coordinatorPolicyHashBytes) - require.NotEmpty(t, coordinatorPolicyHash, "expected apply to fill coordinator policy hash") - require.True(t, t.Run("apply", func(t *testing.T) { - require := require.New(t) - - var objects []*unstructured.Unstructured - for _, file := range resources { - yaml, err := os.ReadFile(file) - require.NoError(err) - fileObjects, err := kubeapi.UnmarshalUnstructuredK8SResource(yaml) - require.NoError(err) - objects = append(objects, fileObjects...) + var objects []*unstructured.Unstructured + for _, obj := range unstructuredResources { + // TODO(burgerdev): remove once demo deployments don't contain namespaces anymore. + if obj.GetKind() == "Namespace" { + continue } + objects = append(objects, obj) + } - ctx, cancel := context.WithTimeout(context.Background(), 1*time.Minute) - defer cancel() - - c := kubeclient.NewForTest(t) - require.NoError(c.Apply(ctx, objects...)) - }), "Kubernetes resources need to be applied for subsequent tests") - - require.True(t, t.Run("set", func(t *testing.T) { - require := require.New(t) - - ctx, cancel := context.WithTimeout(context.Background(), 1*time.Minute) - defer cancel() - - require.NoError(c.WaitForDeployment(ctx, namespace, "coordinator")) - - coordinator, cancelPortForward, err := c.PortForwardPod(ctx, namespace, "port-forwarder-coordinator", "1313") - require.NoError(err) - defer cancelPortForward() - - args := []string{ - "--coordinator-policy-hash", coordinatorPolicyHash, - "--coordinator", coordinator, - "--workspace-dir", "./workspace", - } - args = append(args, resources...) - - set := cmd.NewSetCmd() - set.Flags().String("workspace-dir", "", "") // Make set aware of root flags - set.SetArgs(args) - set.SetOut(io.Discard) - errBuf := &bytes.Buffer{} - set.SetErr(errBuf) - - require.NoError(set.Execute(), "could not set manifest at coordinator: %s", errBuf) - }), "contrast set needs to succeed for subsequent tests") - - certs := make(map[string][]byte) - - require.True(t, t.Run("contrast verify", func(t *testing.T) { - require := require.New(t) - - ctx, cancel := context.WithTimeout(context.Background(), 1*time.Minute) - defer cancel() - - require.NoError(c.WaitForDeployment(ctx, namespace, "coordinator")) + ct.Init(t, objects) - coordinator, cancelPortForward, err := c.PortForwardPod(ctx, namespace, "port-forwarder-coordinator", "1313") - require.NoError(err) - defer cancelPortForward() + require.True(t, t.Run("generate", ct.Generate), "contrast generate needs to succeed for subsequent tests") - workspaceDir, err := os.MkdirTemp("", "contrast-verify.*") - require.NoError(err) + require.True(t, t.Run("apply", ct.Apply), "Kubernetes resources need to be applied for subsequent tests") - verify := cmd.NewVerifyCmd() - verify.SetArgs([]string{ - "--workspace-dir", workspaceDir, - "--coordinator-policy-hash", coordinatorPolicyHash, - "--coordinator", coordinator, - }) - verify.SetOut(io.Discard) - errBuf := &bytes.Buffer{} - verify.SetErr(errBuf) - - require.NoError(verify.Execute(), "could not verify coordinator: %s", errBuf) - - for _, certFile := range []string{ - "coordinator-root.pem", - "mesh-root.pem", - } { - pem, err := os.ReadFile(path.Join(workspaceDir, certFile)) - assert.NoError(t, err) - certs[certFile] = pem - } - }), "contrast verify needs to succeed for subsequent tests") + require.True(t, t.Run("set", ct.Set), "contrast set needs to succeed for subsequent tests") + require.True(t, t.Run("contrast verify", ct.Verify), "contrast verify needs to succeed for subsequent tests") require.True(t, t.Run("deployments become available", func(t *testing.T) { require := require.New(t) @@ -155,25 +60,27 @@ func TestIngress(t *testing.T) { ctx, cancel := context.WithTimeout(context.Background(), 1*time.Minute) defer cancel() - require.NoError(c.WaitForDeployment(ctx, namespace, "vote-bot")) - require.NoError(c.WaitForDeployment(ctx, namespace, "emoji")) - require.NoError(c.WaitForDeployment(ctx, namespace, "voting")) - require.NoError(c.WaitForDeployment(ctx, namespace, "web")) + require.NoError(ct.Kubeclient.WaitForDeployment(ctx, ct.Namespace, "vote-bot")) + require.NoError(ct.Kubeclient.WaitForDeployment(ctx, ct.Namespace, "emoji")) + require.NoError(ct.Kubeclient.WaitForDeployment(ctx, ct.Namespace, "voting")) + require.NoError(ct.Kubeclient.WaitForDeployment(ctx, ct.Namespace, "web")) }), "deployments need to be ready for subsequent tests") - for certFile, pem := range certs { + certs := map[string]*x509.CertPool{ + "coordinator-root.pem": ct.RootCACert(), + "mesh-ca.pem": ct.MeshCACert(), + } + for certFile, pool := range certs { t.Run("go dial web with ca "+certFile, func(t *testing.T) { require := require.New(t) ctx, cancel := context.WithTimeout(context.Background(), 1*time.Minute) defer cancel() - web, cancelPortForward, err := c.PortForwardPod(ctx, namespace, "port-forwarder-emojivoto-web", "8080") + web, cancelPortForward, err := ct.Kubeclient.PortForwardPod(ctx, ct.Namespace, "port-forwarder-emojivoto-web", "8080") require.NoError(err) t.Cleanup(cancelPortForward) - pool := x509.NewCertPool() - require.True(pool.AppendCertsFromPEM(pem)) tlsConf := &tls.Config{RootCAs: pool} hc := &http.Client{Transport: &http.Transport{TLSClientConfig: tlsConf}} req, err := http.NewRequest("GET", fmt.Sprintf("https://%s/", web), nil) @@ -193,9 +100,9 @@ func TestIngress(t *testing.T) { c := kubeclient.NewForTest(t) - frontendPods, err := c.PodsFromDeployment(ctx, namespace, "web") + frontendPods, err := c.PodsFromDeployment(ctx, ct.Namespace, "web") require.NoError(err) - require.Len(frontendPods, 1, "pod not found: %s/%s", namespace, "web") + require.Len(frontendPods, 1, "pod not found: %s/%s", ct.Namespace, "web") // The emoji service does not have an ingress proxy configuration, so we expect all ingress // traffic to be proxied with mandatory mutual TLS. @@ -207,11 +114,26 @@ func TestIngress(t *testing.T) { // name (i.e., the CN), so we tell curl to connect to expect the deployment name but // resolve the service name. argv = append(argv, "--connect-to", "emoji:8801:emoji-svc:8801") - stdout, stderr, err := c.Exec(ctx, namespace, frontendPods[0].Name, argv) + stdout, stderr, err := c.Exec(ctx, ct.Namespace, frontendPods[0].Name, argv) require.Error(err, "Expected call without client certificate to fail.\nstdout: %s\nstderr: %q", stdout, stderr) argv = append(argv, "--cert", "/tls-config/certChain.pem", "--key", "/tls-config/key.pem") - stdout, stderr, err = c.Exec(ctx, namespace, frontendPods[0].Name, argv) + stdout, stderr, err = c.Exec(ctx, ct.Namespace, frontendPods[0].Name, argv) require.NoError(err, "Expected call with client certificate to succeed.\nstdout: %s\nstderr: %q", stdout, stderr) }) } + +func TestMain(m *testing.M) { + flag.Parse() + + f, err := os.Open(flag.Arg(0)) + if err != nil { + log.Fatalf("could not open image definition file %q: %v", flag.Arg(0), err) + } + imageReplacements, err = kuberesource.ImageReplacementsFromFile(f) + if err != nil { + log.Fatalf("could not parse image definition file %q: %v", flag.Arg(0), err) + } + + os.Exit(m.Run()) +}