diff --git a/.github/workflows/e2e_servicemesh.yml b/.github/workflows/e2e_servicemesh.yml new file mode 100644 index 0000000000..0eda90d712 --- /dev/null +++ b/.github/workflows/e2e_servicemesh.yml @@ -0,0 +1,69 @@ +name: e2e test service-mesh + +on: + workflow_dispatch: + inputs: + skip-undeploy: + description: "Skip undeploy" + required: false + default: "false" + pull_request: + +env: + container_registry: ghcr.io/edgelesssys + azure_resource_group: contrast-ci + +jobs: + test: + runs-on: ubuntu-22.04 + permissions: + contents: read + packages: write + steps: + - uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1 + - uses: ./.github/actions/setup_nix + with: + githubToken: ${{ secrets.GITHUB_TOKEN }} + cachixToken: ${{ secrets.CACHIX_AUTH_TOKEN }} + - name: Log in to ghcr.io Container registry + uses: docker/login-action@e92390c5fb421da1463c202d546fed0ec5c39f20 # v3.1.0 + with: + registry: ghcr.io + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + - name: Login to Azure + uses: azure/login@8c334a195cbb38e46038007b304988d888bf676a # v2.0.0 + with: + creds: ${{ secrets.CONTRAST_CI_INFRA_AZURE }} + - uses: nicknovitski/nix-develop@a2060d116a50b36dfab02280af558e73ab52427d # v1.1.0 + - name: Generate namespace suffix + id: ns + run: | + uuid=$(cat /proc/sys/kernel/random/uuid) + uid=${uuid##*-} + echo "namespace_suffix=$uid" >> "$GITHUB_OUTPUT" + - name: Create justfile.env + run: | + cat < justfile.env + container_registry=${{ env.container_registry }} + azure_resource_group=${{ env.azure_resource_group }} + namespace_suffix=-${{ steps.ns.outputs.namespace_suffix }} + EOF + - name: Get credentials for CI cluster + run: | + just get-credentials + - name: Build and prepare deployments + run: | + just coordinator initializer service-mesh-proxy + just populate emojivoto-sm-ingress + - name: Setup Summary + run: | + cat ./workspace/just.namespace | tee -a "${GITHUB_STEP_SUMMARY}" + cat ./workspace/just.perf | tee -a "${GITHUB_STEP_SUMMARY}" + - name: E2E Test + run: | + env K8S_NAMESPACE=$(cat ./workspace/just.namespace) nix shell .#contrast.e2e --command servicemesh.test -test.v + - name: Undeploy + if: always() && inputs.skip-undeploy != 'true' + run: | + just undeploy diff --git a/e2e/servicemesh/servicemesh_test.go b/e2e/servicemesh/servicemesh_test.go new file mode 100644 index 0000000000..84dcfa01ee --- /dev/null +++ b/e2e/servicemesh/servicemesh_test.go @@ -0,0 +1,217 @@ +//go:build e2e +// +build e2e + +package servicemesh + +import ( + "bytes" + "context" + "crypto/tls" + "crypto/x509" + "fmt" + "io" + "net/http" + "os" + "path" + "path/filepath" + "testing" + "time" + + "github.com/edgelesssys/contrast/cli/cmd" + "github.com/edgelesssys/contrast/e2e/internal/kubeclient" + "github.com/edgelesssys/contrast/internal/kubeapi" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" +) + +// namespace the tests are executed in. +const namespaceEnv = "K8S_NAMESPACE" + +// TestIngress tests that the ingress proxies work as configured. +func TestIngress(t *testing.T) { + c := kubeclient.NewForTest(t) + + namespace := os.Getenv(namespaceEnv) + require.NotEmpty(t, namespace, "environment variable %q must be set", namespaceEnv) + + resources, err := filepath.Glob("./workspace/deployment/*.yml") + require.NoError(t, err) + + require.True(t, t.Run("generate", func(t *testing.T) { + require := require.New(t) + + args := []string{ + "--workspace-dir", "./workspace", + } + args = append(args, resources...) + + generate := cmd.NewGenerateCmd() + generate.Flags().String("workspace-dir", "", "") // Make generate aware of root flags + generate.SetArgs(args) + generate.SetOut(io.Discard) + errBuf := &bytes.Buffer{} + generate.SetErr(errBuf) + + require.NoError(generate.Execute(), "could not generate manifest: %s", errBuf) + })) + + // TODO(burgerdev): policy hash should come from contrast generate output. + coordinatorPolicyHashBytes, err := os.ReadFile("workspace/coordinator-policy.sha256") + require.NoError(t, err) + coordinatorPolicyHash := string(coordinatorPolicyHashBytes) + require.NotEmpty(t, coordinatorPolicyHash, "expected apply to fill coordinator policy hash") + + require.True(t, t.Run("apply", func(t *testing.T) { + require := require.New(t) + + var objects []*unstructured.Unstructured + for _, file := range resources { + yaml, err := os.ReadFile(file) + require.NoError(err) + fileObjects, err := kubeapi.UnmarshalUnstructuredK8SResource(yaml) + require.NoError(err) + objects = append(objects, fileObjects...) + } + + ctx, cancel := context.WithTimeout(context.Background(), 1*time.Minute) + defer cancel() + + c := kubeclient.NewForTest(t) + require.NoError(c.Apply(ctx, objects...)) + }), "Kubernetes resources need to be applied for subsequent tests") + + require.True(t, t.Run("set", func(t *testing.T) { + require := require.New(t) + + ctx, cancel := context.WithTimeout(context.Background(), 1*time.Minute) + defer cancel() + + require.NoError(c.WaitForDeployment(ctx, namespace, "coordinator")) + + coordinator, cancelPortForward, err := c.PortForwardPod(ctx, namespace, "port-forwarder-coordinator", "1313") + require.NoError(err) + defer cancelPortForward() + + args := []string{ + "--coordinator-policy-hash", coordinatorPolicyHash, + "--coordinator", coordinator, + "--workspace-dir", "./workspace", + } + args = append(args, resources...) + + set := cmd.NewSetCmd() + set.Flags().String("workspace-dir", "", "") // Make set aware of root flags + set.SetArgs(args) + set.SetOut(io.Discard) + errBuf := &bytes.Buffer{} + set.SetErr(errBuf) + + require.NoError(set.Execute(), "could not set manifest at coordinator: %s", errBuf) + }), "contrast set needs to succeed for subsequent tests") + + certs := make(map[string][]byte) + + require.True(t, t.Run("contrast verify", func(t *testing.T) { + require := require.New(t) + + ctx, cancel := context.WithTimeout(context.Background(), 1*time.Minute) + defer cancel() + + require.NoError(c.WaitForDeployment(ctx, namespace, "coordinator")) + + coordinator, cancelPortForward, err := c.PortForwardPod(ctx, namespace, "port-forwarder-coordinator", "1313") + require.NoError(err) + defer cancelPortForward() + + workspaceDir, err := os.MkdirTemp("", "contrast-verify.*") + require.NoError(err) + + verify := cmd.NewVerifyCmd() + verify.SetArgs([]string{ + "--workspace-dir", workspaceDir, + "--coordinator-policy-hash", coordinatorPolicyHash, + "--coordinator", coordinator, + }) + verify.SetOut(io.Discard) + errBuf := &bytes.Buffer{} + verify.SetErr(errBuf) + + require.NoError(verify.Execute(), "could not verify coordinator: %s", errBuf) + + for _, certFile := range []string{ + "coordinator-root.pem", + "mesh-root.pem", + } { + pem, err := os.ReadFile(path.Join(workspaceDir, certFile)) + assert.NoError(t, err) + certs[certFile] = pem + } + }), "contrast verify needs to succeed for subsequent tests") + + require.True(t, t.Run("deployments become available", func(t *testing.T) { + require := require.New(t) + + ctx, cancel := context.WithTimeout(context.Background(), 1*time.Minute) + defer cancel() + + require.NoError(c.WaitForDeployment(ctx, namespace, "vote-bot")) + require.NoError(c.WaitForDeployment(ctx, namespace, "emoji")) + require.NoError(c.WaitForDeployment(ctx, namespace, "voting")) + require.NoError(c.WaitForDeployment(ctx, namespace, "web")) + }), "deployments need to be ready for subsequent tests") + + for certFile, pem := range certs { + t.Run("go dial web with ca "+certFile, func(t *testing.T) { + require := require.New(t) + + ctx, cancel := context.WithTimeout(context.Background(), 1*time.Minute) + defer cancel() + + web, cancelPortForward, err := c.PortForwardPod(ctx, namespace, "port-forwarder-emojivoto-web", "8080") + require.NoError(err) + t.Cleanup(cancelPortForward) + + pool := x509.NewCertPool() + require.True(pool.AppendCertsFromPEM(pem)) + tlsConf := &tls.Config{RootCAs: pool} + hc := &http.Client{Transport: &http.Transport{TLSClientConfig: tlsConf}} + req, err := http.NewRequest("GET", fmt.Sprintf("https://%s/", web), nil) + require.NoError(err) + resp, err := hc.Do(req) + require.NoError(err) + defer resp.Body.Close() + require.Equal(http.StatusOK, resp.StatusCode) + }) + } + + t.Run("client certificates are required if not explicitly disabled", func(t *testing.T) { + require := require.New(t) + + ctx, cancel := context.WithTimeout(context.Background(), 1*time.Minute) + defer cancel() + + c := kubeclient.NewForTest(t) + + frontendPods, err := c.PodsFromDeployment(ctx, namespace, "web") + require.NoError(err) + require.Len(frontendPods, 1, "pod not found: %s/%s", namespace, "web") + + // The emoji service does not have an ingress proxy configuration, so we expect all ingress + // traffic to be proxied with mandatory mutual TLS. + // This test also verifies that client connections are not affected by the ingress proxy, + // because we're running the commands on a pod with enabled proxy. + + argv := []string{"curl", "-sS", "--cacert", "/tls-config/MeshCACert.pem", "https://emoji:8801/metrics"} + // curl does not like the wildcard cert and the service name does not match the deployment + // name (i.e., the CN), so we tell curl to connect to expect the deployment name but + // resolve the service name. + argv = append(argv, "--connect-to", "emoji:8801:emoji-svc:8801") + stdout, stderr, err := c.Exec(ctx, namespace, frontendPods[0].Name, argv) + require.Error(err, "Expected call without client certificate to fail.\nstdout: %s\nstderr: %q", stdout, stderr) + + argv = append(argv, "--cert", "/tls-config/certChain.pem", "--key", "/tls-config/key.pem") + stdout, stderr, err = c.Exec(ctx, namespace, frontendPods[0].Name, argv) + require.NoError(err, "Expected call with client certificate to succeed.\nstdout: %s\nstderr: %q", stdout, stderr) + }) +} diff --git a/packages/by-name/contrast/package.nix b/packages/by-name/contrast/package.nix index 003bfc781f..f5b7235641 100644 --- a/packages/by-name/contrast/package.nix +++ b/packages/by-name/contrast/package.nix @@ -14,7 +14,7 @@ let ldflags = [ "-s" ]; - subPackages = [ "e2e/openssl" ]; + subPackages = [ "e2e/openssl" "e2e/servicemesh" ]; }; packageOutputs = [ "coordinator" "initializer" "cli" ];