Skip to content

Commit 91add55

Browse files
burgerdev3u13r
burgerdev
authored and
3u13r
committed
e2e: test service mesh ingress on emojivoto
1 parent 82ff0f4 commit 91add55

File tree

3 files changed

+278
-1
lines changed

3 files changed

+278
-1
lines changed

‎.github/workflows/e2e_servicemesh.yml

+69
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,69 @@
1+
name: e2e test service-mesh
2+
3+
on:
4+
workflow_dispatch:
5+
inputs:
6+
skip-undeploy:
7+
description: "Skip undeploy"
8+
required: false
9+
default: "false"
10+
pull_request:
11+
12+
env:
13+
container_registry: ghcr.io/edgelesssys
14+
azure_resource_group: contrast-ci
15+
16+
jobs:
17+
test:
18+
runs-on: ubuntu-22.04
19+
permissions:
20+
contents: read
21+
packages: write
22+
steps:
23+
- uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1
24+
- uses: ./.github/actions/setup_nix
25+
with:
26+
githubToken: ${{ secrets.GITHUB_TOKEN }}
27+
cachixToken: ${{ secrets.CACHIX_AUTH_TOKEN }}
28+
- name: Log in to ghcr.io Container registry
29+
uses: docker/login-action@e92390c5fb421da1463c202d546fed0ec5c39f20 # v3.1.0
30+
with:
31+
registry: ghcr.io
32+
username: ${{ github.actor }}
33+
password: ${{ secrets.GITHUB_TOKEN }}
34+
- name: Login to Azure
35+
uses: azure/login@8c334a195cbb38e46038007b304988d888bf676a # v2.0.0
36+
with:
37+
creds: ${{ secrets.CONTRAST_CI_INFRA_AZURE }}
38+
- uses: nicknovitski/nix-develop@a2060d116a50b36dfab02280af558e73ab52427d # v1.1.0
39+
- name: Generate namespace suffix
40+
id: ns
41+
run: |
42+
uuid=$(cat /proc/sys/kernel/random/uuid)
43+
uid=${uuid##*-}
44+
echo "namespace_suffix=$uid" >> "$GITHUB_OUTPUT"
45+
- name: Create justfile.env
46+
run: |
47+
cat <<EOF > justfile.env
48+
container_registry=${{ env.container_registry }}
49+
azure_resource_group=${{ env.azure_resource_group }}
50+
namespace_suffix=-${{ steps.ns.outputs.namespace_suffix }}
51+
EOF
52+
- name: Get credentials for CI cluster
53+
run: |
54+
just get-credentials
55+
- name: Build and prepare deployments
56+
run: |
57+
just coordinator initializer service-mesh-proxy
58+
just populate emojivoto-sm-ingress
59+
- name: Setup Summary
60+
run: |
61+
cat ./workspace/just.namespace | tee -a "${GITHUB_STEP_SUMMARY}"
62+
cat ./workspace/just.perf | tee -a "${GITHUB_STEP_SUMMARY}"
63+
- name: E2E Test
64+
run: |
65+
env K8S_NAMESPACE=$(cat ./workspace/just.namespace) nix shell .#contrast.e2e --command servicemesh.test -test.v
66+
- name: Undeploy
67+
if: always() && inputs.skip-undeploy != 'true'
68+
run: |
69+
just undeploy

‎e2e/servicemesh/servicemesh_test.go

+208
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,208 @@
1+
//go:build e2e
2+
// +build e2e
3+
4+
package servicemesh
5+
6+
import (
7+
"bytes"
8+
"context"
9+
"crypto/tls"
10+
"crypto/x509"
11+
"fmt"
12+
"io"
13+
"net/http"
14+
"os"
15+
"path"
16+
"path/filepath"
17+
"testing"
18+
"time"
19+
20+
"github.com/edgelesssys/contrast/cli/cmd"
21+
"github.com/edgelesssys/contrast/e2e/internal/kubeclient"
22+
"github.com/edgelesssys/contrast/internal/kubeapi"
23+
"github.com/stretchr/testify/assert"
24+
"github.com/stretchr/testify/require"
25+
"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
26+
)
27+
28+
// namespace the tests are executed in.
29+
const namespaceEnv = "K8S_NAMESPACE"
30+
31+
// TestIngress tests that the ingress proxies work as configured.
32+
func TestIngress(t *testing.T) {
33+
c := kubeclient.NewForTest(t)
34+
35+
namespace := os.Getenv(namespaceEnv)
36+
require.NotEmpty(t, namespace, "environment variable %q must be set", namespaceEnv)
37+
38+
resources, err := filepath.Glob("./workspace/deployment/*.yml")
39+
require.NoError(t, err)
40+
41+
require.True(t, t.Run("generate", func(t *testing.T) {
42+
require := require.New(t)
43+
44+
args := []string{
45+
"--workspace-dir", "./workspace",
46+
}
47+
args = append(args, resources...)
48+
49+
generate := cmd.NewGenerateCmd()
50+
generate.Flags().String("workspace-dir", "", "") // Make generate aware of root flags
51+
generate.SetArgs(args)
52+
generate.SetOut(io.Discard)
53+
errBuf := &bytes.Buffer{}
54+
generate.SetErr(errBuf)
55+
56+
require.NoError(generate.Execute(), "could not generate manifest: %s", errBuf)
57+
}))
58+
59+
// TODO(burgerdev): policy hash should come from contrast generate output.
60+
coordinatorPolicyHashBytes, err := os.ReadFile("workspace/coordinator-policy.sha256")
61+
require.NoError(t, err)
62+
coordinatorPolicyHash := string(coordinatorPolicyHashBytes)
63+
require.NotEmpty(t, coordinatorPolicyHash, "expected apply to fill coordinator policy hash")
64+
65+
require.True(t, t.Run("apply", func(t *testing.T) {
66+
require := require.New(t)
67+
68+
var objects []*unstructured.Unstructured
69+
for _, file := range resources {
70+
yaml, err := os.ReadFile(file)
71+
require.NoError(err)
72+
fileObjects, err := kubeapi.UnmarshalUnstructuredK8SResource(yaml)
73+
require.NoError(err)
74+
objects = append(objects, fileObjects...)
75+
}
76+
77+
ctx, cancel := context.WithTimeout(context.Background(), 1*time.Minute)
78+
defer cancel()
79+
80+
c := kubeclient.NewForTest(t)
81+
require.NoError(c.Apply(ctx, objects...))
82+
}), "Kubernetes resources need to be applied for subsequent tests")
83+
84+
require.True(t, t.Run("set", func(t *testing.T) {
85+
require := require.New(t)
86+
87+
ctx, cancel := context.WithTimeout(context.Background(), 1*time.Minute)
88+
defer cancel()
89+
90+
require.NoError(c.WaitForDeployment(ctx, namespace, "coordinator"))
91+
92+
coordinator, cancelPortForward, err := c.PortForwardPod(ctx, namespace, "port-forwarder-coordinator", "1313")
93+
require.NoError(err)
94+
defer cancelPortForward()
95+
96+
args := []string{
97+
"--coordinator-policy-hash", coordinatorPolicyHash,
98+
"--coordinator", coordinator,
99+
"--workspace-dir", "./workspace",
100+
}
101+
args = append(args, resources...)
102+
103+
set := cmd.NewSetCmd()
104+
set.Flags().String("workspace-dir", "", "") // Make set aware of root flags
105+
set.SetArgs(args)
106+
set.SetOut(io.Discard)
107+
errBuf := &bytes.Buffer{}
108+
set.SetErr(errBuf)
109+
110+
require.NoError(set.Execute(), "could not set manifest at coordinator: %s", errBuf)
111+
}), "contrast set needs to succeed for subsequent tests")
112+
113+
certs := make(map[string][]byte)
114+
115+
require.True(t, t.Run("contrast verify", func(t *testing.T) {
116+
require := require.New(t)
117+
118+
ctx, cancel := context.WithTimeout(context.Background(), 1*time.Minute)
119+
defer cancel()
120+
121+
require.NoError(c.WaitForDeployment(ctx, namespace, "coordinator"))
122+
123+
coordinator, cancelPortForward, err := c.PortForwardPod(ctx, namespace, "port-forwarder-coordinator", "1313")
124+
require.NoError(err)
125+
defer cancelPortForward()
126+
127+
workspaceDir, err := os.MkdirTemp("", "contrast-verify.*")
128+
require.NoError(err)
129+
130+
verify := cmd.NewVerifyCmd()
131+
verify.SetArgs([]string{
132+
"--workspace-dir", workspaceDir,
133+
"--coordinator-policy-hash", coordinatorPolicyHash,
134+
"--coordinator", coordinator,
135+
})
136+
verify.SetOut(io.Discard)
137+
errBuf := &bytes.Buffer{}
138+
verify.SetErr(errBuf)
139+
140+
require.NoError(verify.Execute(), "could not verify coordinator: %s", errBuf)
141+
142+
for _, certFile := range []string{
143+
"coordinator-root.pem",
144+
"mesh-root.pem",
145+
} {
146+
pem, err := os.ReadFile(path.Join(workspaceDir, certFile))
147+
assert.NoError(t, err)
148+
certs[certFile] = pem
149+
}
150+
}), "contrast verify needs to succeed for subsequent tests")
151+
152+
// TODO(@3u13r): This is a workaround to wait for the deployment to be ready.
153+
time.Sleep(20 * time.Second)
154+
155+
for certFile, pem := range certs {
156+
t.Run("go dial web with ca "+certFile, func(t *testing.T) {
157+
require := require.New(t)
158+
159+
ctx, cancel := context.WithTimeout(context.Background(), 1*time.Minute)
160+
defer cancel()
161+
162+
web, cancelPortForward, err := c.PortForwardPod(ctx, namespace, "port-forwarder-emojivoto-web", "8080")
163+
require.NoError(err)
164+
t.Cleanup(cancelPortForward)
165+
166+
pool := x509.NewCertPool()
167+
require.True(pool.AppendCertsFromPEM(pem))
168+
tlsConf := &tls.Config{RootCAs: pool}
169+
hc := &http.Client{Transport: &http.Transport{TLSClientConfig: tlsConf}}
170+
req, err := http.NewRequest("GET", fmt.Sprintf("https://%s/", web), nil)
171+
require.NoError(err)
172+
resp, err := hc.Do(req)
173+
require.NoError(err)
174+
defer resp.Body.Close()
175+
require.Equal(http.StatusOK, resp.StatusCode)
176+
})
177+
}
178+
179+
t.Run("client certificates are required if not explicitly disabled", func(t *testing.T) {
180+
require := require.New(t)
181+
182+
ctx, cancel := context.WithTimeout(context.Background(), 1*time.Minute)
183+
defer cancel()
184+
185+
c := kubeclient.NewForTest(t)
186+
187+
frontendPods, err := c.PodsFromDeployment(ctx, namespace, "web")
188+
require.NoError(err)
189+
require.Len(frontendPods, 1, "pod not found: %s/%s", namespace, "web")
190+
191+
// The emoji service does not have an ingress proxy configuration, so we expect all ingress
192+
// traffic to be proxied with mandatory mutual TLS.
193+
// This test also verifies that client connections are not affected by the ingress proxy,
194+
// because we're running the commands on a pod with enabled proxy.
195+
196+
argv := []string{"curl", "-sS", "--cacert", "/tls-config/MeshCACert.pem", "https://emoji:8801/metrics"}
197+
// curl does not like the wildcard cert and the service name does not match the deployment
198+
// name (i.e., the CN), so we tell curl to connect to expect the deployment name but
199+
// resolve the service name.
200+
argv = append(argv, "--connect-to", "emoji:8801:emoji-svc:8801")
201+
stdout, stderr, err := c.Exec(ctx, namespace, frontendPods[0].Name, argv)
202+
require.Error(err, "Expected call without client certificate to fail.\nstdout: %s\nstderr: %q", stdout, stderr)
203+
204+
argv = append(argv, "--cert", "/tls-config/certChain.pem", "--key", "/tls-config/key.pem")
205+
stdout, stderr, err = c.Exec(ctx, namespace, frontendPods[0].Name, argv)
206+
require.NoError(err, "Expected call with client certificate to succeed.\nstdout: %s\nstderr: %q", stdout, stderr)
207+
})
208+
}

‎packages/by-name/contrast/package.nix

+1-1
Original file line numberDiff line numberDiff line change
@@ -14,7 +14,7 @@ let
1414

1515
ldflags = [ "-s" ];
1616

17-
subPackages = [ "e2e/openssl" ];
17+
subPackages = [ "e2e/openssl" "e2e/servicemesh" ];
1818
};
1919

2020
packageOutputs = [ "coordinator" "initializer" "cli" ];

0 commit comments

Comments
 (0)