Skip to content

Commit

Permalink
e2e: use ContrastTest for servicemesh test
Browse files Browse the repository at this point in the history
  • Loading branch information
@malt3
malt3 committed Apr 18, 2024
1 parent 04e4cd3 commit fad9f1c
Show file tree
Hide file tree
Showing 2 changed files with 52 additions and 138 deletions.
12 changes: 2 additions & 10 deletions .github/workflows/e2e_servicemesh.yml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -38,33 +38,25 @@ jobs:
with:
creds: ${{ secrets.CONTRAST_CI_INFRA_AZURE }}
- uses: nicknovitski/nix-develop@a2060d116a50b36dfab02280af558e73ab52427d # v1.1.0
- name: Generate namespace suffix
id: ns
run: |
uuid=$(cat /proc/sys/kernel/random/uuid)
uid=${uuid##*-}
echo "namespace_suffix=$uid" >> "$GITHUB_OUTPUT"
- name: Create justfile.env
run: |
cat <<EOF > justfile.env
container_registry=${{ env.container_registry }}
azure_resource_group=${{ env.azure_resource_group }}
namespace_suffix=-${{ steps.ns.outputs.namespace_suffix }}
EOF
- name: Get credentials for CI cluster
run: |
just get-credentials
- name: Build and prepare deployments
run: |
just coordinator initializer service-mesh-proxy
just populate emojivoto-sm-ingress
just coordinator initializer port-forwarder service-mesh-proxy
- name: Setup Summary
run: |
cat ./workspace/just.namespace | tee -a "${GITHUB_STEP_SUMMARY}"
cat ./workspace/just.perf | tee -a "${GITHUB_STEP_SUMMARY}"
- name: E2E Test
run: |
env K8S_NAMESPACE=$(cat ./workspace/just.namespace) nix shell .#contrast.e2e --command servicemesh.test -test.v
nix shell .#contrast.e2e --command servicemesh.test -test.v workspace/just.containerlookup
- name: Undeploy
if: always() && inputs.skip-undeploy != 'true'
run: |
Expand Down
178 changes: 50 additions & 128 deletions e2e/servicemesh/servicemesh_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -4,176 +4,83 @@
package servicemesh

import (
"bytes"
"context"
"crypto/tls"
"crypto/x509"
"flag"
"fmt"
"io"
"log"
"net/http"
"os"
"path"
"path/filepath"
"testing"
"time"

"github.com/edgelesssys/contrast/cli/cmd"
"github.com/edgelesssys/contrast/e2e/internal/contrasttest"
"github.com/edgelesssys/contrast/e2e/internal/kubeclient"
"github.com/edgelesssys/contrast/internal/kubeapi"
"github.com/stretchr/testify/assert"
"github.com/edgelesssys/contrast/e2e/internal/kuberesource"
"github.com/stretchr/testify/require"
"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
)

// namespace the tests are executed in.
const namespaceEnv = "K8S_NAMESPACE"
var imageReplacements map[string]string

// TestIngress tests that the ingress proxies work as configured.
func TestIngress(t *testing.T) {
c := kubeclient.NewForTest(t)
ct := contrasttest.New(t)

namespace := os.Getenv(namespaceEnv)
require.NotEmpty(t, namespace, "environment variable %q must be set", namespaceEnv)

resources, err := filepath.Glob("./workspace/deployment/*.yml")
resources, err := kuberesource.EmojivotoIngress()
require.NoError(t, err)

require.True(t, t.Run("generate", func(t *testing.T) {
require := require.New(t)

args := []string{
"--workspace-dir", "./workspace",
}
args = append(args, resources...)

generate := cmd.NewGenerateCmd()
generate.Flags().String("workspace-dir", "", "") // Make generate aware of root flags
generate.SetArgs(args)
generate.SetOut(io.Discard)
errBuf := &bytes.Buffer{}
generate.SetErr(errBuf)

require.NoError(generate.Execute(), "could not generate manifest: %s", errBuf)
}))
resources = kuberesource.PatchImages(resources, imageReplacements)

// TODO(burgerdev): policy hash should come from contrast generate output.
coordinatorPolicyHashBytes, err := os.ReadFile("workspace/coordinator-policy.sha256")
unstructuredResources, err := kuberesource.ResourcesToUnstructured(resources)
require.NoError(t, err)
coordinatorPolicyHash := string(coordinatorPolicyHashBytes)
require.NotEmpty(t, coordinatorPolicyHash, "expected apply to fill coordinator policy hash")

require.True(t, t.Run("apply", func(t *testing.T) {
require := require.New(t)

var objects []*unstructured.Unstructured
for _, file := range resources {
yaml, err := os.ReadFile(file)
require.NoError(err)
fileObjects, err := kubeapi.UnmarshalUnstructuredK8SResource(yaml)
require.NoError(err)
objects = append(objects, fileObjects...)
var objects []*unstructured.Unstructured
for _, obj := range unstructuredResources {
// TODO(burgerdev): remove once demo deployments don't contain namespaces anymore.
if obj.GetKind() == "Namespace" {
continue
}
objects = append(objects, obj)
}

ctx, cancel := context.WithTimeout(context.Background(), 1*time.Minute)
defer cancel()

c := kubeclient.NewForTest(t)
require.NoError(c.Apply(ctx, objects...))
}), "Kubernetes resources need to be applied for subsequent tests")

require.True(t, t.Run("set", func(t *testing.T) {
require := require.New(t)

ctx, cancel := context.WithTimeout(context.Background(), 1*time.Minute)
defer cancel()

require.NoError(c.WaitForDeployment(ctx, namespace, "coordinator"))

coordinator, cancelPortForward, err := c.PortForwardPod(ctx, namespace, "port-forwarder-coordinator", "1313")
require.NoError(err)
defer cancelPortForward()

args := []string{
"--coordinator-policy-hash", coordinatorPolicyHash,
"--coordinator", coordinator,
"--workspace-dir", "./workspace",
}
args = append(args, resources...)

set := cmd.NewSetCmd()
set.Flags().String("workspace-dir", "", "") // Make set aware of root flags
set.SetArgs(args)
set.SetOut(io.Discard)
errBuf := &bytes.Buffer{}
set.SetErr(errBuf)

require.NoError(set.Execute(), "could not set manifest at coordinator: %s", errBuf)
}), "contrast set needs to succeed for subsequent tests")

certs := make(map[string][]byte)

require.True(t, t.Run("contrast verify", func(t *testing.T) {
require := require.New(t)

ctx, cancel := context.WithTimeout(context.Background(), 1*time.Minute)
defer cancel()

require.NoError(c.WaitForDeployment(ctx, namespace, "coordinator"))
ct.Init(t, objects)

coordinator, cancelPortForward, err := c.PortForwardPod(ctx, namespace, "port-forwarder-coordinator", "1313")
require.NoError(err)
defer cancelPortForward()
require.True(t, t.Run("generate", ct.Generate), "contrast generate needs to succeed for subsequent tests")

workspaceDir, err := os.MkdirTemp("", "contrast-verify.*")
require.NoError(err)
require.True(t, t.Run("apply", ct.Apply), "Kubernetes resources need to be applied for subsequent tests")

verify := cmd.NewVerifyCmd()
verify.SetArgs([]string{
"--workspace-dir", workspaceDir,
"--coordinator-policy-hash", coordinatorPolicyHash,
"--coordinator", coordinator,
})
verify.SetOut(io.Discard)
errBuf := &bytes.Buffer{}
verify.SetErr(errBuf)

require.NoError(verify.Execute(), "could not verify coordinator: %s", errBuf)

for _, certFile := range []string{
"coordinator-root.pem",
"mesh-root.pem",
} {
pem, err := os.ReadFile(path.Join(workspaceDir, certFile))
assert.NoError(t, err)
certs[certFile] = pem
}
}), "contrast verify needs to succeed for subsequent tests")
require.True(t, t.Run("set", ct.Set), "contrast set needs to succeed for subsequent tests")
require.True(t, t.Run("contrast verify", ct.Verify), "contrast verify needs to succeed for subsequent tests")

require.True(t, t.Run("deployments become available", func(t *testing.T) {
require := require.New(t)

ctx, cancel := context.WithTimeout(context.Background(), 1*time.Minute)
defer cancel()

require.NoError(c.WaitForDeployment(ctx, namespace, "vote-bot"))
require.NoError(c.WaitForDeployment(ctx, namespace, "emoji"))
require.NoError(c.WaitForDeployment(ctx, namespace, "voting"))
require.NoError(c.WaitForDeployment(ctx, namespace, "web"))
require.NoError(ct.Kubeclient.WaitForDeployment(ctx, ct.Namespace, "vote-bot"))
require.NoError(ct.Kubeclient.WaitForDeployment(ctx, ct.Namespace, "emoji"))
require.NoError(ct.Kubeclient.WaitForDeployment(ctx, ct.Namespace, "voting"))
require.NoError(ct.Kubeclient.WaitForDeployment(ctx, ct.Namespace, "web"))
}), "deployments need to be ready for subsequent tests")

for certFile, pem := range certs {
certs := map[string]*x509.CertPool{
"coordinator-root.pem": ct.RootCACert(),
"mesh-ca.pem": ct.MeshCACert(),
}
for certFile, pool := range certs {
t.Run("go dial web with ca "+certFile, func(t *testing.T) {
require := require.New(t)

ctx, cancel := context.WithTimeout(context.Background(), 1*time.Minute)
defer cancel()

web, cancelPortForward, err := c.PortForwardPod(ctx, namespace, "port-forwarder-emojivoto-web", "8080")
web, cancelPortForward, err := ct.Kubeclient.PortForwardPod(ctx, ct.Namespace, "port-forwarder-emojivoto-web", "8080")
require.NoError(err)
t.Cleanup(cancelPortForward)

pool := x509.NewCertPool()
require.True(pool.AppendCertsFromPEM(pem))
tlsConf := &tls.Config{RootCAs: pool}
hc := &http.Client{Transport: &http.Transport{TLSClientConfig: tlsConf}}
req, err := http.NewRequest("GET", fmt.Sprintf("https://%s/", web), nil)
Expand All @@ -193,9 +100,9 @@ func TestIngress(t *testing.T) {

c := kubeclient.NewForTest(t)

frontendPods, err := c.PodsFromDeployment(ctx, namespace, "web")
frontendPods, err := c.PodsFromDeployment(ctx, ct.Namespace, "web")
require.NoError(err)
require.Len(frontendPods, 1, "pod not found: %s/%s", namespace, "web")
require.Len(frontendPods, 1, "pod not found: %s/%s", ct.Namespace, "web")

// The emoji service does not have an ingress proxy configuration, so we expect all ingress
// traffic to be proxied with mandatory mutual TLS.
Expand All @@ -207,11 +114,26 @@ func TestIngress(t *testing.T) {
// name (i.e., the CN), so we tell curl to connect to expect the deployment name but
// resolve the service name.
argv = append(argv, "--connect-to", "emoji:8801:emoji-svc:8801")
stdout, stderr, err := c.Exec(ctx, namespace, frontendPods[0].Name, argv)
stdout, stderr, err := c.Exec(ctx, ct.Namespace, frontendPods[0].Name, argv)
require.Error(err, "Expected call without client certificate to fail.\nstdout: %s\nstderr: %q", stdout, stderr)

argv = append(argv, "--cert", "/tls-config/certChain.pem", "--key", "/tls-config/key.pem")
stdout, stderr, err = c.Exec(ctx, namespace, frontendPods[0].Name, argv)
stdout, stderr, err = c.Exec(ctx, ct.Namespace, frontendPods[0].Name, argv)
require.NoError(err, "Expected call with client certificate to succeed.\nstdout: %s\nstderr: %q", stdout, stderr)
})
}

func TestMain(m *testing.M) {
flag.Parse()

f, err := os.Open(flag.Arg(0))
if err != nil {
log.Fatalf("could not open image definition file %q: %v", flag.Arg(0), err)
}
imageReplacements, err = kuberesource.ImageReplacementsFromFile(f)
if err != nil {
log.Fatalf("could not parse image definition file %q: %v", flag.Arg(0), err)
}

os.Exit(m.Run())
}

0 comments on commit fad9f1c

Please sign in to comment.