diff --git a/.github/workflows/e2e_openssl.yml b/.github/workflows/e2e_openssl.yml index 42efac78ea..543e2b981a 100644 --- a/.github/workflows/e2e_openssl.yml +++ b/.github/workflows/e2e_openssl.yml @@ -45,7 +45,7 @@ jobs: - name: Get credentials for CI cluster run: | just get-credentials - - name: Build, deploy, contrast generate, contrast set, contrast verify + - name: Build and prepare deployments run: | just coordinator initializer openssl just populate openssl diff --git a/.github/workflows/e2e_servicemesh.yml b/.github/workflows/e2e_servicemesh.yml new file mode 100644 index 0000000000..0eda90d712 --- /dev/null +++ b/.github/workflows/e2e_servicemesh.yml @@ -0,0 +1,69 @@ +name: e2e test service-mesh + +on: + workflow_dispatch: + inputs: + skip-undeploy: + description: "Skip undeploy" + required: false + default: "false" + pull_request: + +env: + container_registry: ghcr.io/edgelesssys + azure_resource_group: contrast-ci + +jobs: + test: + runs-on: ubuntu-22.04 + permissions: + contents: read + packages: write + steps: + - uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1 + - uses: ./.github/actions/setup_nix + with: + githubToken: ${{ secrets.GITHUB_TOKEN }} + cachixToken: ${{ secrets.CACHIX_AUTH_TOKEN }} + - name: Log in to ghcr.io Container registry + uses: docker/login-action@e92390c5fb421da1463c202d546fed0ec5c39f20 # v3.1.0 + with: + registry: ghcr.io + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + - name: Login to Azure + uses: azure/login@8c334a195cbb38e46038007b304988d888bf676a # v2.0.0 + with: + creds: ${{ secrets.CONTRAST_CI_INFRA_AZURE }} + - uses: nicknovitski/nix-develop@a2060d116a50b36dfab02280af558e73ab52427d # v1.1.0 + - name: Generate namespace suffix + id: ns + run: | + uuid=$(cat /proc/sys/kernel/random/uuid) + uid=${uuid##*-} + echo "namespace_suffix=$uid" >> "$GITHUB_OUTPUT" + - name: Create justfile.env + run: | + cat < justfile.env + container_registry=${{ env.container_registry }} + azure_resource_group=${{ env.azure_resource_group }} + namespace_suffix=-${{ steps.ns.outputs.namespace_suffix }} + EOF + - name: Get credentials for CI cluster + run: | + just get-credentials + - name: Build and prepare deployments + run: | + just coordinator initializer service-mesh-proxy + just populate emojivoto-sm-ingress + - name: Setup Summary + run: | + cat ./workspace/just.namespace | tee -a "${GITHUB_STEP_SUMMARY}" + cat ./workspace/just.perf | tee -a "${GITHUB_STEP_SUMMARY}" + - name: E2E Test + run: | + env K8S_NAMESPACE=$(cat ./workspace/just.namespace) nix shell .#contrast.e2e --command servicemesh.test -test.v + - name: Undeploy + if: always() && inputs.skip-undeploy != 'true' + run: | + just undeploy diff --git a/deployments/emojivoto-sm-ingress/coordinator.yml b/deployments/emojivoto-sm-ingress/coordinator.yml new file mode 100644 index 0000000000..300c35db76 --- /dev/null +++ b/deployments/emojivoto-sm-ingress/coordinator.yml @@ -0,0 +1,48 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: coordinator + namespace: edg-default +spec: + selector: + matchLabels: + app.kubernetes.io/name: coordinator + replicas: 1 + template: + metadata: + labels: + app.kubernetes.io/name: coordinator + annotations: + contrast.edgeless.systems/pod-role: coordinator + spec: + runtimeClassName: kata-cc-isolation + containers: + - name: coordinator + image: "ghcr.io/edgelesssys/contrast/coordinator:latest" + ports: + - containerPort: 7777 + - containerPort: 1313 + env: + - name: contrast_LOG_LEVEL + value: "debug" + resources: + requests: + memory: 100Mi + limits: + memory: 100Mi +--- +apiVersion: v1 +kind: Service +metadata: + name: coordinator + namespace: edg-default +spec: + ports: + - name: intercom + port: 7777 + protocol: TCP + - name: coordapi + port: 1313 + protocol: TCP + selector: + app.kubernetes.io/name: coordinator diff --git a/deployments/emojivoto-sm-ingress/emoji.yml b/deployments/emojivoto-sm-ingress/emoji.yml new file mode 100644 index 0000000000..eb13b95dbf --- /dev/null +++ b/deployments/emojivoto-sm-ingress/emoji.yml @@ -0,0 +1,101 @@ +kind: ServiceAccount +apiVersion: v1 +metadata: + name: emoji + namespace: edg-default +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: emoji + namespace: edg-default + labels: + app.kubernetes.io/name: emoji + app.kubernetes.io/part-of: emojivoto + app.kubernetes.io/version: v11 +spec: + replicas: 1 + selector: + matchLabels: + app.kubernetes.io/name: emoji-svc + version: v11 + template: + metadata: + labels: + app.kubernetes.io/name: emoji-svc + version: v11 + spec: + runtimeClassName: kata-cc-isolation + initContainers: + - name: initializer + image: "ghcr.io/edgelesssys/contrast/initializer:latest" + env: + - name: COORDINATOR_HOST + value: coordinator + volumeMounts: + - name: tls-certs + mountPath: /tls-config + resources: + requests: + memory: 50Mi + limits: + memory: 50Mi + - name: sidecar + image: "ghcr.io/edgelesssys/contrast/service-mesh-proxy:latest" + restartPolicy: Always + volumeMounts: + - name: tls-certs + mountPath: /tls-config + securityContext: + privileged: true + capabilities: + add: + - NET_ADMIN + serviceAccountName: emoji + containers: + - env: + - name: GRPC_PORT + value: "8080" + - name: PROM_PORT + value: "8801" + - name: EDG_CERT_PATH + value: /tls-config/certChain.pem + - name: EDG_CA_PATH + value: /tls-config/MeshCACert.pem + - name: EDG_KEY_PATH + value: /tls-config/key.pem + image: docker.l5d.io/buoyantio/emojivoto-emoji-svc:v11 + name: emoji-svc + ports: + - containerPort: 8080 + name: grpc + - containerPort: 8801 + name: prom + resources: + requests: + cpu: 100m + memory: 50Mi + limits: + memory: 50Mi + volumeMounts: + - name: tls-certs + mountPath: /tls-config + volumes: + - name: tls-certs + emptyDir: {} +--- +apiVersion: v1 +kind: Service +metadata: + name: emoji-svc + namespace: edg-default +spec: + selector: + app.kubernetes.io/name: emoji-svc + ports: + - name: grpc + port: 8080 + targetPort: 8080 + - name: prom + port: 8801 + targetPort: 8801 diff --git a/deployments/emojivoto-sm-ingress/ns.yml b/deployments/emojivoto-sm-ingress/ns.yml new file mode 100644 index 0000000000..ed2712cc89 --- /dev/null +++ b/deployments/emojivoto-sm-ingress/ns.yml @@ -0,0 +1,4 @@ +apiVersion: v1 +kind: Namespace +metadata: + name: edg-default diff --git a/deployments/emojivoto-sm-ingress/portforwarder.yml b/deployments/emojivoto-sm-ingress/portforwarder.yml new file mode 100644 index 0000000000..b145e4e779 --- /dev/null +++ b/deployments/emojivoto-sm-ingress/portforwarder.yml @@ -0,0 +1,59 @@ +apiVersion: v1 +kind: Pod +metadata: + name: port-forwarder-coordinator + namespace: edg-default + labels: + app.kubernetes.io/name: port-forwarder-coordinator +spec: + containers: + - name: port-forwarder + image: "ghcr.io/edgelesssys/contrast/port-forwarder:latest" + env: + - name: LISTEN_PORT + value: "1313" + - name: FORWARD_HOST + value: coordinator + - name: FORWARD_PORT + value: "1313" + command: + - /bin/bash + - "-c" + - echo Starting port-forward with socat; exec socat -d -d TCP-LISTEN:${LISTEN_PORT},fork TCP:${FORWARD_HOST}:${FORWARD_PORT} + ports: + - containerPort: 1313 + resources: + requests: + memory: 50Mi + limits: + memory: 50Mi +--- +apiVersion: v1 +kind: Pod +metadata: + name: port-forwarder-emojivoto-web + namespace: edg-default + labels: + app.kubernetes.io/name: port-forwarder-emojivoto-web +spec: + containers: + - name: port-forwarder + image: "ghcr.io/edgelesssys/contrast/port-forwarder:latest" + env: + - name: LISTEN_PORT + value: "8080" + - name: FORWARD_HOST + value: web-svc + - name: FORWARD_PORT + value: "443" + command: + - /bin/bash + - "-c" + - echo Starting port-forward with socat; exec socat -d -d TCP-LISTEN:${LISTEN_PORT},fork TCP:${FORWARD_HOST}:${FORWARD_PORT} + ports: + - containerPort: 8080 + resources: + requests: + memory: 50Mi + limits: + memory: 50Mi diff --git a/deployments/emojivoto-sm-ingress/vote-bot.yml b/deployments/emojivoto-sm-ingress/vote-bot.yml new file mode 100644 index 0000000000..be4802217a --- /dev/null +++ b/deployments/emojivoto-sm-ingress/vote-bot.yml @@ -0,0 +1,35 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: vote-bot + namespace: edg-default + labels: + app.kubernetes.io/name: vote-bot + app.kubernetes.io/part-of: emojivoto + app.kubernetes.io/version: v11 +spec: + replicas: 1 + selector: + matchLabels: + app.kubernetes.io/name: vote-bot + version: v11 + template: + metadata: + labels: + app.kubernetes.io/name: vote-bot + version: v11 + spec: + containers: + - command: + - emojivoto-vote-bot + env: + - name: WEB_HOST + value: web-svc:443 + image: ghcr.io/3u13r/emojivoto-web:coco-1 + name: vote-bot + resources: + requests: + cpu: 10m + memory: 25Mi + limits: + memory: 25Mi diff --git a/deployments/emojivoto-sm-ingress/voting.yml b/deployments/emojivoto-sm-ingress/voting.yml new file mode 100644 index 0000000000..2e0a62dd8b --- /dev/null +++ b/deployments/emojivoto-sm-ingress/voting.yml @@ -0,0 +1,101 @@ +kind: ServiceAccount +apiVersion: v1 +metadata: + name: voting + namespace: edg-default +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: voting + namespace: edg-default + labels: + app.kubernetes.io/name: voting + app.kubernetes.io/part-of: emojivoto + app.kubernetes.io/version: v11 +spec: + replicas: 1 + selector: + matchLabels: + app.kubernetes.io/name: voting-svc + version: v11 + template: + metadata: + labels: + app.kubernetes.io/name: voting-svc + version: v11 + spec: + runtimeClassName: kata-cc-isolation + initContainers: + - name: initializer + image: "ghcr.io/edgelesssys/contrast/initializer:latest" + env: + - name: COORDINATOR_HOST + value: coordinator + volumeMounts: + - name: tls-certs + mountPath: /tls-config + resources: + requests: + memory: 50Mi + limits: + memory: 50Mi + - name: sidecar + image: "ghcr.io/edgelesssys/contrast/service-mesh-proxy:latest" + restartPolicy: Always + volumeMounts: + - name: tls-certs + mountPath: /tls-config + securityContext: + privileged: true + capabilities: + add: + - NET_ADMIN + serviceAccountName: voting + containers: + - env: + - name: GRPC_PORT + value: "8080" + - name: PROM_PORT + value: "8801" + - name: EDG_CERT_PATH + value: /tls-config/certChain.pem + - name: EDG_CA_PATH + value: /tls-config/MeshCACert.pem + - name: EDG_KEY_PATH + value: /tls-config/key.pem + image: docker.l5d.io/buoyantio/emojivoto-voting-svc:v11 + name: voting-svc + ports: + - containerPort: 8080 + name: grpc + - containerPort: 8801 + name: prom + resources: + requests: + cpu: 100m + memory: 50Mi + limits: + memory: 50Mi + volumeMounts: + - name: tls-certs + mountPath: /tls-config + volumes: + - name: tls-certs + emptyDir: {} +--- +apiVersion: v1 +kind: Service +metadata: + name: voting-svc + namespace: edg-default +spec: + selector: + app.kubernetes.io/name: voting-svc + ports: + - name: grpc + port: 8080 + targetPort: 8080 + - name: prom + port: 8801 + targetPort: 8801 diff --git a/deployments/emojivoto-sm-ingress/web.yml b/deployments/emojivoto-sm-ingress/web.yml new file mode 100644 index 0000000000..aae8ed1225 --- /dev/null +++ b/deployments/emojivoto-sm-ingress/web.yml @@ -0,0 +1,103 @@ +kind: ServiceAccount +apiVersion: v1 +metadata: + name: web + namespace: edg-default +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: web + namespace: edg-default + labels: + app.kubernetes.io/name: web + app.kubernetes.io/part-of: emojivoto + app.kubernetes.io/version: v11 +spec: + replicas: 1 + selector: + matchLabels: + app.kubernetes.io/name: web-svc + version: v11 + template: + metadata: + labels: + app.kubernetes.io/name: web-svc + version: v11 + spec: + runtimeClassName: kata-cc-isolation + initContainers: + - name: initializer + image: "ghcr.io/edgelesssys/contrast/initializer:latest" + env: + - name: COORDINATOR_HOST + value: coordinator + volumeMounts: + - name: tls-certs + mountPath: /tls-config + - name: sidecar + image: "ghcr.io/edgelesssys/contrast/service-mesh-proxy:latest" + restartPolicy: Always + volumeMounts: + - name: tls-certs + mountPath: /tls-config + env: + - name: EDG_INGRESS_PROXY_CONFIG + value: "web#8080#false" + - name: EDG_EGRESS_PROXY_CONFIG + value: "emoji#127.137.0.1:8081#emoji-svc:8080##voting#127.137.0.2:8081#voting-svc:8080" + securityContext: + privileged: true + capabilities: + add: + - NET_ADMIN + serviceAccountName: web + containers: + - env: + - name: WEB_PORT + value: "8080" + - name: EMOJISVC_HOST + value: 127.137.0.1:8081 + - name: VOTINGSVC_HOST + value: 127.137.0.2:8081 + - name: INDEX_BUNDLE + value: dist/index_bundle.js + - name: EDG_CERT_PATH + value: /tls-config/certChain.pem + - name: EDG_CA_PATH + value: /tls-config/MeshCACert.pem + - name: EDG_KEY_PATH + value: /tls-config/key.pem + - name: EDG_DISABLE_CLIENT_AUTH + value: "true" + image: docker.l5d.io/buoyantio/emojivoto-web:v11 + name: web-svc + ports: + - containerPort: 8080 + name: https + resources: + requests: + cpu: 100m + memory: 50Mi + limits: + memory: 50Mi + volumeMounts: + - name: tls-certs + mountPath: /tls-config + volumes: + - name: tls-certs + emptyDir: {} +--- +apiVersion: v1 +kind: Service +metadata: + name: web-svc + namespace: edg-default +spec: + type: ClusterIP + selector: + app.kubernetes.io/name: web-svc + ports: + - name: https + port: 443 + targetPort: 8080 diff --git a/e2e/servicemesh/servicemesh_test.go b/e2e/servicemesh/servicemesh_test.go new file mode 100644 index 0000000000..84dcfa01ee --- /dev/null +++ b/e2e/servicemesh/servicemesh_test.go @@ -0,0 +1,217 @@ +//go:build e2e +// +build e2e + +package servicemesh + +import ( + "bytes" + "context" + "crypto/tls" + "crypto/x509" + "fmt" + "io" + "net/http" + "os" + "path" + "path/filepath" + "testing" + "time" + + "github.com/edgelesssys/contrast/cli/cmd" + "github.com/edgelesssys/contrast/e2e/internal/kubeclient" + "github.com/edgelesssys/contrast/internal/kubeapi" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" +) + +// namespace the tests are executed in. +const namespaceEnv = "K8S_NAMESPACE" + +// TestIngress tests that the ingress proxies work as configured. +func TestIngress(t *testing.T) { + c := kubeclient.NewForTest(t) + + namespace := os.Getenv(namespaceEnv) + require.NotEmpty(t, namespace, "environment variable %q must be set", namespaceEnv) + + resources, err := filepath.Glob("./workspace/deployment/*.yml") + require.NoError(t, err) + + require.True(t, t.Run("generate", func(t *testing.T) { + require := require.New(t) + + args := []string{ + "--workspace-dir", "./workspace", + } + args = append(args, resources...) + + generate := cmd.NewGenerateCmd() + generate.Flags().String("workspace-dir", "", "") // Make generate aware of root flags + generate.SetArgs(args) + generate.SetOut(io.Discard) + errBuf := &bytes.Buffer{} + generate.SetErr(errBuf) + + require.NoError(generate.Execute(), "could not generate manifest: %s", errBuf) + })) + + // TODO(burgerdev): policy hash should come from contrast generate output. + coordinatorPolicyHashBytes, err := os.ReadFile("workspace/coordinator-policy.sha256") + require.NoError(t, err) + coordinatorPolicyHash := string(coordinatorPolicyHashBytes) + require.NotEmpty(t, coordinatorPolicyHash, "expected apply to fill coordinator policy hash") + + require.True(t, t.Run("apply", func(t *testing.T) { + require := require.New(t) + + var objects []*unstructured.Unstructured + for _, file := range resources { + yaml, err := os.ReadFile(file) + require.NoError(err) + fileObjects, err := kubeapi.UnmarshalUnstructuredK8SResource(yaml) + require.NoError(err) + objects = append(objects, fileObjects...) + } + + ctx, cancel := context.WithTimeout(context.Background(), 1*time.Minute) + defer cancel() + + c := kubeclient.NewForTest(t) + require.NoError(c.Apply(ctx, objects...)) + }), "Kubernetes resources need to be applied for subsequent tests") + + require.True(t, t.Run("set", func(t *testing.T) { + require := require.New(t) + + ctx, cancel := context.WithTimeout(context.Background(), 1*time.Minute) + defer cancel() + + require.NoError(c.WaitForDeployment(ctx, namespace, "coordinator")) + + coordinator, cancelPortForward, err := c.PortForwardPod(ctx, namespace, "port-forwarder-coordinator", "1313") + require.NoError(err) + defer cancelPortForward() + + args := []string{ + "--coordinator-policy-hash", coordinatorPolicyHash, + "--coordinator", coordinator, + "--workspace-dir", "./workspace", + } + args = append(args, resources...) + + set := cmd.NewSetCmd() + set.Flags().String("workspace-dir", "", "") // Make set aware of root flags + set.SetArgs(args) + set.SetOut(io.Discard) + errBuf := &bytes.Buffer{} + set.SetErr(errBuf) + + require.NoError(set.Execute(), "could not set manifest at coordinator: %s", errBuf) + }), "contrast set needs to succeed for subsequent tests") + + certs := make(map[string][]byte) + + require.True(t, t.Run("contrast verify", func(t *testing.T) { + require := require.New(t) + + ctx, cancel := context.WithTimeout(context.Background(), 1*time.Minute) + defer cancel() + + require.NoError(c.WaitForDeployment(ctx, namespace, "coordinator")) + + coordinator, cancelPortForward, err := c.PortForwardPod(ctx, namespace, "port-forwarder-coordinator", "1313") + require.NoError(err) + defer cancelPortForward() + + workspaceDir, err := os.MkdirTemp("", "contrast-verify.*") + require.NoError(err) + + verify := cmd.NewVerifyCmd() + verify.SetArgs([]string{ + "--workspace-dir", workspaceDir, + "--coordinator-policy-hash", coordinatorPolicyHash, + "--coordinator", coordinator, + }) + verify.SetOut(io.Discard) + errBuf := &bytes.Buffer{} + verify.SetErr(errBuf) + + require.NoError(verify.Execute(), "could not verify coordinator: %s", errBuf) + + for _, certFile := range []string{ + "coordinator-root.pem", + "mesh-root.pem", + } { + pem, err := os.ReadFile(path.Join(workspaceDir, certFile)) + assert.NoError(t, err) + certs[certFile] = pem + } + }), "contrast verify needs to succeed for subsequent tests") + + require.True(t, t.Run("deployments become available", func(t *testing.T) { + require := require.New(t) + + ctx, cancel := context.WithTimeout(context.Background(), 1*time.Minute) + defer cancel() + + require.NoError(c.WaitForDeployment(ctx, namespace, "vote-bot")) + require.NoError(c.WaitForDeployment(ctx, namespace, "emoji")) + require.NoError(c.WaitForDeployment(ctx, namespace, "voting")) + require.NoError(c.WaitForDeployment(ctx, namespace, "web")) + }), "deployments need to be ready for subsequent tests") + + for certFile, pem := range certs { + t.Run("go dial web with ca "+certFile, func(t *testing.T) { + require := require.New(t) + + ctx, cancel := context.WithTimeout(context.Background(), 1*time.Minute) + defer cancel() + + web, cancelPortForward, err := c.PortForwardPod(ctx, namespace, "port-forwarder-emojivoto-web", "8080") + require.NoError(err) + t.Cleanup(cancelPortForward) + + pool := x509.NewCertPool() + require.True(pool.AppendCertsFromPEM(pem)) + tlsConf := &tls.Config{RootCAs: pool} + hc := &http.Client{Transport: &http.Transport{TLSClientConfig: tlsConf}} + req, err := http.NewRequest("GET", fmt.Sprintf("https://%s/", web), nil) + require.NoError(err) + resp, err := hc.Do(req) + require.NoError(err) + defer resp.Body.Close() + require.Equal(http.StatusOK, resp.StatusCode) + }) + } + + t.Run("client certificates are required if not explicitly disabled", func(t *testing.T) { + require := require.New(t) + + ctx, cancel := context.WithTimeout(context.Background(), 1*time.Minute) + defer cancel() + + c := kubeclient.NewForTest(t) + + frontendPods, err := c.PodsFromDeployment(ctx, namespace, "web") + require.NoError(err) + require.Len(frontendPods, 1, "pod not found: %s/%s", namespace, "web") + + // The emoji service does not have an ingress proxy configuration, so we expect all ingress + // traffic to be proxied with mandatory mutual TLS. + // This test also verifies that client connections are not affected by the ingress proxy, + // because we're running the commands on a pod with enabled proxy. + + argv := []string{"curl", "-sS", "--cacert", "/tls-config/MeshCACert.pem", "https://emoji:8801/metrics"} + // curl does not like the wildcard cert and the service name does not match the deployment + // name (i.e., the CN), so we tell curl to connect to expect the deployment name but + // resolve the service name. + argv = append(argv, "--connect-to", "emoji:8801:emoji-svc:8801") + stdout, stderr, err := c.Exec(ctx, namespace, frontendPods[0].Name, argv) + require.Error(err, "Expected call without client certificate to fail.\nstdout: %s\nstderr: %q", stdout, stderr) + + argv = append(argv, "--cert", "/tls-config/certChain.pem", "--key", "/tls-config/key.pem") + stdout, stderr, err = c.Exec(ctx, namespace, frontendPods[0].Name, argv) + require.NoError(err, "Expected call with client certificate to succeed.\nstdout: %s\nstderr: %q", stdout, stderr) + }) +} diff --git a/justfile b/justfile index 73715c88ff..54e6db7f61 100644 --- a/justfile +++ b/justfile @@ -157,7 +157,7 @@ wait-for-workload target=default_deploy_target: nix run .#scripts.kubectl-wait-ready -- $ns openssl-client nix run .#scripts.kubectl-wait-ready -- $ns openssl-frontend ;; - "emojivoto" | "emojivoto-sm-egress") + "emojivoto" | "emojivoto-sm-egress" | "emojivoto-sm-ingress") nix run .#scripts.kubectl-wait-ready -- $ns emoji-svc nix run .#scripts.kubectl-wait-ready -- $ns vote-bot nix run .#scripts.kubectl-wait-ready -- $ns voting-svc diff --git a/packages/by-name/contrast/package.nix b/packages/by-name/contrast/package.nix index 003bfc781f..f5b7235641 100644 --- a/packages/by-name/contrast/package.nix +++ b/packages/by-name/contrast/package.nix @@ -14,7 +14,7 @@ let ldflags = [ "-s" ]; - subPackages = [ "e2e/openssl" ]; + subPackages = [ "e2e/openssl" "e2e/servicemesh" ]; }; packageOutputs = [ "coordinator" "initializer" "cli" ]; diff --git a/packages/by-name/genpolicy-msft/package.nix b/packages/by-name/genpolicy-msft/package.nix index 1c1b0a5823..6b03699230 100644 --- a/packages/by-name/genpolicy-msft/package.nix +++ b/packages/by-name/genpolicy-msft/package.nix @@ -29,6 +29,12 @@ rustPlatform.buildRustPackage rec { url = "https://github.com/kata-containers/kata-containers/commit/befef119ff4df2868cdc88d4273c8be965387793.patch"; sha256 = "sha256-4pfYrP9KaPVcrFbm6DkiZUNckUq0fKWZPfCONW8/kso="; }) + # TODO(3u13r): drop this patch when msft fork adopted this from upstream + (fetchpatch { + name = "genpolicy_msft_settings_dev.patch"; + url = "https://github.com/kata-containers/kata-containers/commit/5398b6466c58676db7f73370e1a56f4fbb35d8cf.patch"; + sha256 = "sha256-cJ/uUF2F//QAP79AXu3tgfcByrWy2bvUjPfOAIZrtD8="; + }) ]; patchFlags = [ "-p4" ]; diff --git a/packages/by-name/service-mesh/package.nix b/packages/by-name/service-mesh/package.nix index d532316f8a..36ff82e131 100644 --- a/packages/by-name/service-mesh/package.nix +++ b/packages/by-name/service-mesh/package.nix @@ -23,7 +23,7 @@ buildGoModule rec { }; proxyVendor = true; - vendorHash = "sha256-PyqMDkayG7yMNN5X+RqutqTX/yBh209V3vz7qw7dzS8="; + vendorHash = "sha256-p0vkQqe6Q11N0pSP28faXwnMszJyLCUhjeMBTabZWCI="; subPackages = [ "." ]; diff --git a/packages/containers.nix b/packages/containers.nix index 6e5f954894..a82ab312dc 100644 --- a/packages/containers.nix +++ b/packages/containers.nix @@ -63,7 +63,7 @@ let service-mesh-proxy = dockerTools.buildImage { name = "service-mesh-proxy"; tag = "v${service-mesh.version}"; - copyToRoot = [ envoy ]; + copyToRoot = [ envoy iptables-legacy ]; config = { Cmd = [ "${service-mesh}/bin/service-mesh" ]; Env = [ "PATH=/bin" ]; # This is only here for policy generation. diff --git a/service-mesh/config.go b/service-mesh/config.go index 60d7f3d00a..7d65c2f282 100644 --- a/service-mesh/config.go +++ b/service-mesh/config.go @@ -12,24 +12,34 @@ import ( envoyCoreV3 "github.com/envoyproxy/go-control-plane/envoy/config/core/v3" endpointV3 "github.com/envoyproxy/go-control-plane/envoy/config/endpoint/v3" envoyConfigListenerV3 "github.com/envoyproxy/go-control-plane/envoy/config/listener/v3" + envoyOrigDstV3 "github.com/envoyproxy/go-control-plane/envoy/extensions/filters/listener/original_dst/v3" envoyConfigTCPProxyV3 "github.com/envoyproxy/go-control-plane/envoy/extensions/filters/network/tcp_proxy/v3" envoyTLSV3 "github.com/envoyproxy/go-control-plane/envoy/extensions/transport_sockets/tls/v3" "google.golang.org/protobuf/encoding/protojson" "google.golang.org/protobuf/types/known/anypb" + "google.golang.org/protobuf/types/known/wrapperspb" ) var loopbackCIDR = netip.MustParsePrefix("127.0.0.1/8") // ProxyConfig represents the configuration for the proxy. -type ProxyConfig []configEntry - -type configEntry struct { +type ProxyConfig struct { + egress []egressConfigEntry + ingress []ingressConfigEntry +} +type egressConfigEntry struct { name string + clusterName string listenAddr netip.Addr listenPort uint16 remoteDomain string remotePort uint16 } +type ingressConfigEntry struct { + name string + listenPort uint16 + disableTLS bool +} // ParseProxyConfig parses the proxy configuration from the given string. // The configuration is expected to be in the following format: @@ -39,39 +49,72 @@ type configEntry struct { // Example: // // emoji#127.137.0.1:8081#emoji-svc:8080##voting#127.137.0.2:8081#voting-svc:8080 -func ParseProxyConfig(data string) (ProxyConfig, error) { - entries := strings.Split(data, "##") +func ParseProxyConfig(ingressConfig, egressConfig string) (ProxyConfig, error) { + if ingressConfig == "" && egressConfig == "" { + return ProxyConfig{}, nil + } + + entries := strings.Split(egressConfig, "##") var cfg ProxyConfig for _, entry := range entries { + if entry == "" { + continue + } parts := strings.Split(entry, "#") if len(parts) != 3 { - return nil, fmt.Errorf("invalid entry: %s", entry) + return ProxyConfig{}, fmt.Errorf("invalid entry: %s", entry) } listenAddrPort, err := netip.ParseAddrPort(parts[1]) if err != nil { - return nil, fmt.Errorf("invalid listen address: %s", parts[1]) + return ProxyConfig{}, fmt.Errorf("invalid listen address: %s", parts[1]) } if !loopbackCIDR.Contains(listenAddrPort.Addr()) { - return nil, fmt.Errorf("listen address %s is not in local CIDR %s", listenAddrPort.Addr(), loopbackCIDR) + return ProxyConfig{}, fmt.Errorf("listen address %s is not in local CIDR %s", listenAddrPort.Addr(), loopbackCIDR) } remoteDomain := parts[2] remoteDomain, remotePort, err := net.SplitHostPort(remoteDomain) if err != nil { - return nil, fmt.Errorf("invalid remote domain: %s", remoteDomain) + return ProxyConfig{}, fmt.Errorf("invalid remote domain: %s", remoteDomain) } remotePortInt, err := strconv.Atoi(remotePort) if err != nil { - return nil, fmt.Errorf("invalid remote port: %s", remotePort) + return ProxyConfig{}, fmt.Errorf("invalid remote port: %s", remotePort) } - cfg = append(cfg, configEntry{ + cfg.egress = append(cfg.egress, egressConfigEntry{ name: parts[0], + clusterName: parts[0], listenAddr: listenAddrPort.Addr(), listenPort: listenAddrPort.Port(), remotePort: uint16(remotePortInt), remoteDomain: remoteDomain, }) } + + for _, entry := range strings.Split(ingressConfig, "##") { + if entry == "" { + continue + } + parts := strings.Split(entry, "#") + if len(parts) != 3 { + return ProxyConfig{}, fmt.Errorf("invalid entry: %s", entry) + } + listenPort, err := strconv.Atoi(parts[1]) + if err != nil { + return ProxyConfig{}, fmt.Errorf("invalid listen port: %s", parts[1]) + } + disableTLS, err := strconv.ParseBool(parts[2]) + if err != nil { + return ProxyConfig{}, fmt.Errorf("invalid disable TLS: %s", parts[2]) + } + cfg.ingress = append(cfg.ingress, ingressConfigEntry{ + name: parts[0], + listenPort: uint16(listenPort), + disableTLS: disableTLS, + }) + + } + return cfg, nil } @@ -81,9 +124,11 @@ func (c ProxyConfig) ToEnvoyConfig() ([]byte, error) { config := &envoyConfigBootstrapV3.Bootstrap{ StaticResources: &envoyConfigBootstrapV3.Bootstrap_StaticResources{}, } - listeners := make([]*envoyConfigListenerV3.Listener, 0, len(c)) - clusters := make([]*envoyConfigClusterV3.Cluster, 0, len(c)) - for _, entry := range c { + listeners := make([]*envoyConfigListenerV3.Listener, 0) + clusters := make([]*envoyConfigClusterV3.Cluster, 0) + + // Create listeners and clusters for egress traffic. + for _, entry := range c.egress { listener, err := listener(entry) if err != nil { return nil, err @@ -95,6 +140,28 @@ func (c ProxyConfig) ToEnvoyConfig() ([]byte, error) { } clusters = append(clusters, cluster) } + + // Create listeners and clusters for ingress traffic. + ingrListenerClientAuth, err := ingressListener("ingress", 15006, true) + if err != nil { + return nil, err + } + ingrListenerNoClientAuth, err := ingressListener("ingressWithoutClientAuth", 15007, false) + if err != nil { + return nil, err + } + + ingressCluster := &envoyConfigClusterV3.Cluster{ + Name: "ingress", + ClusterDiscoveryType: &envoyConfigClusterV3.Cluster_Type{Type: envoyConfigClusterV3.Cluster_ORIGINAL_DST}, + DnsLookupFamily: envoyConfigClusterV3.Cluster_V4_ONLY, + LbPolicy: envoyConfigClusterV3.Cluster_CLUSTER_PROVIDED, + } + + listeners = append(listeners, ingrListenerClientAuth) + listeners = append(listeners, ingrListenerNoClientAuth) + clusters = append(clusters, ingressCluster) + config.StaticResources.Listeners = listeners config.StaticResources.Clusters = clusters @@ -110,11 +177,11 @@ func (c ProxyConfig) ToEnvoyConfig() ([]byte, error) { return configBytes, nil } -func listener(entry configEntry) (*envoyConfigListenerV3.Listener, error) { +func listener(entry egressConfigEntry) (*envoyConfigListenerV3.Listener, error) { proxy := &envoyConfigTCPProxyV3.TcpProxy{ StatPrefix: entry.name, ClusterSpecifier: &envoyConfigTCPProxyV3.TcpProxy_Cluster{ - Cluster: entry.name, + Cluster: entry.clusterName, }, } @@ -150,8 +217,8 @@ func listener(entry configEntry) (*envoyConfigListenerV3.Listener, error) { }, nil } -func cluster(entry configEntry) (*envoyConfigClusterV3.Cluster, error) { - socket, err := tlsTransportSocket() +func cluster(entry egressConfigEntry) (*envoyConfigClusterV3.Cluster, error) { + socket, err := upstreamTLSTransportSocket() if err != nil { return nil, err } @@ -191,7 +258,37 @@ func cluster(entry configEntry) (*envoyConfigClusterV3.Cluster, error) { }, nil } -func tlsTransportSocket() (*envoyCoreV3.TransportSocket, error) { +func ingressListener(name string, listenPort uint16, requireClientCertificate bool) (*envoyConfigListenerV3.Listener, error) { + ingressListener, err := listener(egressConfigEntry{ + name: name, + clusterName: "ingress", + listenAddr: netip.MustParseAddr("0.0.0.0"), + listenPort: listenPort, + }) + if err != nil { + return nil, err + } + ingressListener.Transparent = &wrapperspb.BoolValue{Value: true} + originalDstConfig := &envoyOrigDstV3.OriginalDst{} + originalDstAny, err := anypb.New(originalDstConfig) + if err != nil { + return nil, err + } + ingressListener.ListenerFilters = []*envoyConfigListenerV3.ListenerFilter{ + { + Name: "envoy.filters.listener.original_dst", + ConfigType: &envoyConfigListenerV3.ListenerFilter_TypedConfig{TypedConfig: originalDstAny}, + }, + } + tlsSock, err := downstreamTLSTransportSocket(requireClientCertificate) + if err != nil { + return nil, err + } + ingressListener.FilterChains[0].TransportSocket = tlsSock + return ingressListener, nil +} + +func upstreamTLSTransportSocket() (*envoyCoreV3.TransportSocket, error) { tls := &envoyTLSV3.UpstreamTlsContext{ CommonTlsContext: &envoyTLSV3.CommonTlsContext{ TlsCertificates: []*envoyTLSV3.TlsCertificate{ @@ -231,3 +328,45 @@ func tlsTransportSocket() (*envoyCoreV3.TransportSocket, error) { }, }, nil } + +func downstreamTLSTransportSocket(requireClientCertificate bool) (*envoyCoreV3.TransportSocket, error) { + tls := &envoyTLSV3.DownstreamTlsContext{ + CommonTlsContext: &envoyTLSV3.CommonTlsContext{ + TlsCertificates: []*envoyTLSV3.TlsCertificate{ + { + PrivateKey: &envoyCoreV3.DataSource{ + Specifier: &envoyCoreV3.DataSource_Filename{ + Filename: "/tls-config/key.pem", + }, + }, + CertificateChain: &envoyCoreV3.DataSource{ + Specifier: &envoyCoreV3.DataSource_Filename{ + Filename: "/tls-config/certChain.pem", + }, + }, + }, + }, + ValidationContextType: &envoyTLSV3.CommonTlsContext_ValidationContext{ + ValidationContext: &envoyTLSV3.CertificateValidationContext{ + TrustedCa: &envoyCoreV3.DataSource{ + Specifier: &envoyCoreV3.DataSource_Filename{ + Filename: "/tls-config/MeshCACert.pem", + }, + }, + }, + }, + }, + RequireClientCertificate: &wrapperspb.BoolValue{Value: requireClientCertificate}, + } + tlsAny, err := anypb.New(tls) + if err != nil { + return nil, err + } + + return &envoyCoreV3.TransportSocket{ + Name: "envoy.transport_sockets.tls", + ConfigType: &envoyCoreV3.TransportSocket_TypedConfig{ + TypedConfig: tlsAny, + }, + }, nil +} diff --git a/service-mesh/go.mod b/service-mesh/go.mod index 3a0a6a8261..5b70330606 100644 --- a/service-mesh/go.mod +++ b/service-mesh/go.mod @@ -5,6 +5,7 @@ go 1.21 require ( github.com/envoyproxy/go-control-plane v0.12.0 google.golang.org/protobuf v1.33.0 + github.com/coreos/go-iptables v0.7.0 ) require ( diff --git a/service-mesh/go.sum b/service-mesh/go.sum index 1943febdfa..596627b2e3 100644 --- a/service-mesh/go.sum +++ b/service-mesh/go.sum @@ -6,6 +6,8 @@ github.com/census-instrumentation/opencensus-proto v0.4.1/go.mod h1:4T9NM4+4Vw91 github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= github.com/cncf/xds/go v0.0.0-20230607035331-e9ce68804cb4 h1:/inchEIKaYC1Akx+H+gqO04wryn5h75LSazbRlnya1k= github.com/cncf/xds/go v0.0.0-20230607035331-e9ce68804cb4/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= +github.com/coreos/go-iptables v0.7.0 h1:XWM3V+MPRr5/q51NuWSgU0fqMad64Zyxs8ZUoMsamr8= +github.com/coreos/go-iptables v0.7.0/go.mod h1:Qe8Bv2Xik5FyTXwgIbLAnv2sWSBmvWdFETJConOQ//Q= github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= github.com/envoyproxy/go-control-plane v0.12.0 h1:4X+VP1GHd1Mhj6IB5mMeGbLCleqxjletLK6K0rbxyZI= github.com/envoyproxy/go-control-plane v0.12.0/go.mod h1:ZBTaoJ23lqITozF0M6G4/IragXCQKCnYbmlmtHvwRG0= diff --git a/service-mesh/iptables.go b/service-mesh/iptables.go new file mode 100644 index 0000000000..0077b642ab --- /dev/null +++ b/service-mesh/iptables.go @@ -0,0 +1,88 @@ +package main + +import ( + "fmt" + "os" + + "github.com/coreos/go-iptables/iptables" +) + +// EnvoyIngressPort is the port that the envoy proxy listens on for incoming traffic. +const EnvoyIngressPort = 15006 + +// EnvoyIngressPortNoClientCert is the port that the envoy proxy listens on for +// incoming traffic without requiring a client certificate. +const EnvoyIngressPortNoClientCert = 15007 + +// IngressIPTableRules sets up the iptables rules for the ingress proxy. +func IngressIPTableRules(ingressEntries []ingressConfigEntry) error { + // Create missing `/run/xtables.lock` file. + if err := os.Mkdir("/run", 0o755); err != nil { + if !os.IsExist(err) { + return fmt.Errorf("failed to create /run directory: %w", err) + } + } + file, err := os.Create("/run/xtables.lock") + if err != nil { + return fmt.Errorf("failed to create /run/xtables.lock: %w", err) + } + _ = file.Close() + + iptablesExec, err := iptables.New() + if err != nil { + return fmt.Errorf("failed to create iptables client: %w", err) + } + + // Reconcile to clean iptables chains. + // Similar to `ClearChain`, all errors are treated as "chain already exists" + _ = iptablesExec.NewChain("mangle", "EDG_INBOUND") + _ = iptablesExec.NewChain("mangle", "EDG_IN_REDIRECT") + + // Route all TCP traffic to the EDG_INBOUND chain. + if err := iptablesExec.AppendUnique("mangle", "PREROUTING", "-p", "tcp", "-j", "EDG_INBOUND"); err != nil { + return fmt.Errorf("failed to append EDG_INBOUND chain to PREROUTING chain: %w", err) + } + + // RETURN all local traffic from the EDG_INBOUND chain back to the PREROUTING chain. + if err := iptablesExec.AppendUnique("mangle", "EDG_INBOUND", "-p", "tcp", "-i", "lo", "-j", "RETURN"); err != nil { + return fmt.Errorf("failed to append dport exception to EDG_INBOUND chain: %w", err) + } + // RETURN all related and established traffic. + // Since the mangle table executes on every packet and not just before the + // connection is established, as the nat table does, we need to explicitly + // return established traffic. Then using tproxy is similar to a REDIRECT + // rule in the nat table but without the nat overhead. + // This rule is likely needed to exempt outbound TCP connections. + // We use "conntrack" instead of "-m socket" as stated in the official + // documentation because we might not have a kernel with the "xt_socket" + // module (see: https://github.com/istio/istio/pull/22527). + // In our own Contrast image the module is available, but we cannot + // guarantee that it is available in all environments. + if err := iptablesExec.AppendUnique("mangle", "EDG_INBOUND", "-p", "tcp", "-m", "conntrack", "--ctstate", "RELATED,ESTABLISHED", "-j", "RETURN"); err != nil { + return fmt.Errorf("failed to append dport exception to EDG_INBOUND chain: %w", err) + } + // Route all other traffic to the EDG_IN_REDIRECT chain. + if err := iptablesExec.AppendUnique("mangle", "EDG_INBOUND", "-p", "tcp", "-j", "EDG_IN_REDIRECT"); err != nil { + return fmt.Errorf("failed to append EDG_IN_REDIRECT chain to EDG_INBOUND chain: %w", err) + } + + for _, entry := range ingressEntries { + if entry.disableTLS { + if err := iptablesExec.AppendUnique("mangle", "EDG_IN_REDIRECT", "-p", "tcp", "--dport", fmt.Sprintf("%d", entry.listenPort), "-j", "RETURN"); err != nil { + return fmt.Errorf("failed to append dport exception to EDG_IN_REDIRECT chain to disable TLS: %w", err) + } + } else { + if err := iptablesExec.AppendUnique("mangle", "EDG_IN_REDIRECT", "-p", "tcp", "--dport", fmt.Sprintf("%d", entry.listenPort), "-j", "TPROXY", "--on-port", fmt.Sprintf("%d", EnvoyIngressPortNoClientCert)); err != nil { + return fmt.Errorf("failed to append dport exception to EDG_IN_REDIRECT chain to disable client auth: %w", err) + } + } + } + + // Route all remaining traffic (TCP SYN packets that do not have a TLS exemption) + // to the Envoy proxy port that requires client authentication. + if err := iptablesExec.AppendUnique("mangle", "EDG_IN_REDIRECT", "-p", "tcp", "-j", "TPROXY", "--on-port", fmt.Sprintf("%d", EnvoyIngressPort)); err != nil { + return fmt.Errorf("failed to append default TPROXY rule to EDG_IN_REDIRECT chain: %w", err) + } + + return nil +} diff --git a/service-mesh/main.go b/service-mesh/main.go index 4086cd42cd..8c0f02d888 100644 --- a/service-mesh/main.go +++ b/service-mesh/main.go @@ -9,8 +9,9 @@ import ( ) const ( - proxyConfigEnvVar = "EDG_PROXY_CONFIG" - envoyConfigFile = "/envoy-config.yml" + egressProxyConfigEnvVar = "EDG_EGRESS_PROXY_CONFIG" + ingressProxyConfigEnvVar = "EDG_INGRESS_PROXY_CONFIG" + envoyConfigFile = "/envoy-config.yml" ) var version = "0.0.0-dev" @@ -25,12 +26,13 @@ func main() { func run() (retErr error) { log.Printf("service-mesh version %s\n", version) - proxyConfig := os.Getenv(proxyConfigEnvVar) - if proxyConfig == "" { - return fmt.Errorf("no proxy configuration found in environment") - } + egressProxyConfig := os.Getenv(egressProxyConfigEnvVar) + log.Println("Ingress Proxy configuration:", egressProxyConfig) + + ingressProxyConfig := os.Getenv(ingressProxyConfigEnvVar) + log.Println("Egress Proxy configuration:", ingressProxyConfig) - pconfig, err := ParseProxyConfig(proxyConfig) + pconfig, err := ParseProxyConfig(ingressProxyConfig, egressProxyConfig) if err != nil { return err } @@ -46,6 +48,10 @@ func run() (retErr error) { return err } + if err := IngressIPTableRules(pconfig.ingress); err != nil { + return fmt.Errorf("failed to set up iptables rules: %w", err) + } + // execute the envoy binary envoyBin, err := exec.LookPath("envoy") if err != nil {