diff --git a/.github/workflows/e2e_openssl.yml b/.github/workflows/e2e_openssl.yml index 0e54315f0f..42efac78ea 100644 --- a/.github/workflows/e2e_openssl.yml +++ b/.github/workflows/e2e_openssl.yml @@ -36,18 +36,11 @@ jobs: with: creds: ${{ secrets.CONTRAST_CI_INFRA_AZURE }} - uses: nicknovitski/nix-develop@a2060d116a50b36dfab02280af558e73ab52427d # v1.1.0 - - name: Generate namespace suffix - id: ns - run: | - uuid=$(cat /proc/sys/kernel/random/uuid) - uid=${uuid##*-} - echo "namespace_suffix=$uid" >> "$GITHUB_OUTPUT" - name: Create justfile.env run: | cat < justfile.env container_registry=${{ env.container_registry }} azure_resource_group=${{ env.azure_resource_group }} - namespace_suffix=-${{ steps.ns.outputs.namespace_suffix }} EOF - name: Get credentials for CI cluster run: | @@ -58,12 +51,7 @@ jobs: just populate openssl - name: Setup Summary run: | - cat ./workspace/just.namespace | tee -a "${GITHUB_STEP_SUMMARY}" cat ./workspace/just.perf | tee -a "${GITHUB_STEP_SUMMARY}" - name: E2E Test run: | - env K8S_NAMESPACE=$(cat ./workspace/just.namespace) nix shell .#contrast.e2e --command openssl.test -test.v - - name: Undeploy - if: always() && inputs.skip-undeploy != 'true' - run: | - just undeploy + nix shell .#contrast.e2e --command openssl.test -test.v diff --git a/e2e/internal/contrasttest/contrasttest.go b/e2e/internal/contrasttest/contrasttest.go new file mode 100644 index 0000000000..da44ce5e86 --- /dev/null +++ b/e2e/internal/contrasttest/contrasttest.go @@ -0,0 +1,203 @@ +package contrasttest + +import ( + "bytes" + "context" + "crypto/rand" + "crypto/x509" + "encoding/hex" + "io" + "os" + "path" + "regexp" + "strings" + "testing" + "time" + + "github.com/edgelesssys/contrast/cli/cmd" + "github.com/edgelesssys/contrast/e2e/internal/kubeclient" + "github.com/edgelesssys/contrast/e2e/internal/kuberesource" + "github.com/edgelesssys/contrast/internal/kubeapi" + "github.com/stretchr/testify/require" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" +) + +// ContrastTest is the Contrast test helper struct. +type ContrastTest struct { + // inputs, usually filled by New() + Namespace string + WorkDir string + Kubeclient *kubeclient.Kubeclient + + // outputs of contrast subcommands + coordinatorPolicyHash string + meshCACertPEM []byte + rootCACertPEM []byte +} + +// New creates a new contrasttest.T object bound to the given test. +func New(t *testing.T) *ContrastTest { + return &ContrastTest{ + Namespace: makeNamespace(t), + WorkDir: t.TempDir(), + Kubeclient: kubeclient.NewForTest(t), + } +} + +// Init patches the given resources for the test environment and makes them available to Generate and Set. +func (ct *ContrastTest) Init(t *testing.T, objs []*unstructured.Unstructured) { + require := require.New(t) + + // Create namespace + namespace, err := kuberesource.ResourcesToUnstructured([]any{kuberesource.Namespace(ct.Namespace)}) + require.NoError(err) + // Creating a namespace should not take too long. + ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) + err = ct.Kubeclient.Apply(ctx, namespace...) + cancel() + require.NoError(err) + t.Cleanup(func() { + // Deleting the namespace may take some time due to pod cleanup, but we don't want to wait until the test times out. + ctx, cancel := context.WithTimeout(context.Background(), 2*time.Minute) + defer cancel() + if err := ct.Kubeclient.Delete(ctx, namespace...); err != nil { + t.Logf("Could not delete namespace %q: %v", ct.Namespace, err) + } + }) + + // Add the namespace for this test. + for _, obj := range objs { + require.NoError(ct.Kubeclient.PatchNamespace(ct.Namespace, obj)) + } + + // TODO(burgerdev): patch images + + // Write resources to this test's tempdir. + buf, err := kuberesource.EncodeUnstructured(objs) + require.NoError(err) + require.NoError(os.WriteFile(path.Join(ct.WorkDir, "resources.yaml"), buf, 0o644)) +} + +// Generate runs the contrast generate command. +func (ct *ContrastTest) Generate(t *testing.T) { + require := require.New(t) + + args := append(ct.commonArgs(), path.Join(ct.WorkDir, "resources.yaml")) + + generate := cmd.NewGenerateCmd() + generate.Flags().String("workspace-dir", "", "") // Make generate aware of root flags + generate.SetArgs(args) + generate.SetOut(io.Discard) + errBuf := &bytes.Buffer{} + generate.SetErr(errBuf) + + require.NoError(generate.Execute(), "could not generate manifest: %s", errBuf) + hash, err := os.ReadFile(path.Join(ct.WorkDir, "coordinator-policy.sha256")) + require.NoError(err) + require.NotEmpty(hash, "expected apply to fill coordinator policy hash") + ct.coordinatorPolicyHash = string(hash) +} + +// Apply the generated resources to the Kubernetes test environment. +func (ct *ContrastTest) Apply(t *testing.T) { + require := require.New(t) + + yaml, err := os.ReadFile(path.Join(ct.WorkDir, "resources.yaml")) + require.NoError(err) + objects, err := kubeapi.UnmarshalUnstructuredK8SResource(yaml) + require.NoError(err) + + ctx, cancel := context.WithTimeout(context.Background(), 1*time.Minute) + defer cancel() + + require.NoError(ct.Kubeclient.Apply(ctx, objects...)) +} + +// Set runs the contrast set subcommand. +func (ct *ContrastTest) Set(t *testing.T) { + require := require.New(t) + + ctx, cancel := context.WithTimeout(context.Background(), 1*time.Minute) + defer cancel() + + require.NoError(ct.Kubeclient.WaitForDeployment(ctx, ct.Namespace, "coordinator")) + + coordinator, cancelPortForward, err := ct.Kubeclient.PortForwardPod(ctx, ct.Namespace, "port-forwarder-coordinator", "1313") + require.NoError(err) + defer cancelPortForward() + + args := append(ct.commonArgs(), + "--coordinator-policy-hash", ct.coordinatorPolicyHash, + "--coordinator", coordinator, + path.Join(ct.WorkDir, "resources.yaml")) + + set := cmd.NewSetCmd() + set.Flags().String("workspace-dir", "", "") // Make set aware of root flags + set.SetArgs(args) + set.SetOut(io.Discard) + errBuf := &bytes.Buffer{} + set.SetErr(errBuf) + + require.NoError(set.Execute(), "could not set manifest at coordinator: %s", errBuf) +} + +// Verify runs the contrast verify subcommand. +func (ct *ContrastTest) Verify(t *testing.T) { + require := require.New(t) + + ctx, cancel := context.WithTimeout(context.Background(), 1*time.Minute) + defer cancel() + + require.NoError(ct.Kubeclient.WaitForDeployment(ctx, ct.Namespace, "coordinator")) + + coordinator, cancelPortForward, err := ct.Kubeclient.PortForwardPod(ctx, ct.Namespace, "port-forwarder-coordinator", "1313") + require.NoError(err) + defer cancelPortForward() + + verify := cmd.NewVerifyCmd() + verify.SetArgs(append( + ct.commonArgs(), + "--coordinator-policy-hash", ct.coordinatorPolicyHash, + "--coordinator", coordinator, + )) + verify.SetOut(io.Discard) + errBuf := &bytes.Buffer{} + verify.SetErr(errBuf) + + require.NoError(verify.Execute(), "could not verify coordinator: %s", errBuf) + + ct.meshCACertPEM, err = os.ReadFile(path.Join(ct.WorkDir, "mesh-root.pem")) + require.NoError(err) + ct.rootCACertPEM, err = os.ReadFile(path.Join(ct.WorkDir, "coordinator-root.pem")) + require.NoError(err) +} + +// MeshCACert returns a CertPool that contains the coordinator mesh CA cert. +func (ct *ContrastTest) MeshCACert() *x509.CertPool { + pool := x509.NewCertPool() + pool.AppendCertsFromPEM(ct.meshCACertPEM) + return pool +} + +// RootCACert returns a CertPool that contains the coordinator root CA cert. +func (ct *ContrastTest) RootCACert() *x509.CertPool { + pool := x509.NewCertPool() + pool.AppendCertsFromPEM(ct.rootCACertPEM) + return pool +} + +func (ct *ContrastTest) commonArgs() []string { + return []string{ + "--workspace-dir", ct.WorkDir, + } +} + +func makeNamespace(t *testing.T) string { + buf := make([]byte, 4) + re := regexp.MustCompile("[a-z0-9-]+") + n, err := rand.Reader.Read(buf) + require.NoError(t, err) + require.Equal(t, 4, n) + + return strings.Join(append(re.FindAllString(strings.ToLower(t.Name()), -1), hex.EncodeToString(buf)), "-") +} diff --git a/e2e/openssl/openssl_test.go b/e2e/openssl/openssl_test.go index baa773e9df..2e9463e59e 100644 --- a/e2e/openssl/openssl_test.go +++ b/e2e/openssl/openssl_test.go @@ -4,29 +4,23 @@ package openssl import ( - "bytes" "context" "crypto/tls" "crypto/x509" - "io" "os" - "path" "path/filepath" "testing" "time" - "github.com/edgelesssys/contrast/cli/cmd" + "github.com/edgelesssys/contrast/e2e/internal/contrasttest" "github.com/edgelesssys/contrast/e2e/internal/kubeclient" "github.com/edgelesssys/contrast/internal/kubeapi" - "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" ) // namespace the tests are executed in. const ( - namespaceEnv = "K8S_NAMESPACE" - opensslFrontend = "openssl-frontend" opensslBackend = "openssl-backend" ) @@ -34,139 +28,52 @@ const ( // TestOpenSSL runs e2e tests on the example OpenSSL deployment. func TestOpenSSL(t *testing.T) { c := kubeclient.NewForTest(t) + ct := contrasttest.New(t) - namespace := os.Getenv(namespaceEnv) - require.NotEmpty(t, namespace, "environment variable %q must be set", namespaceEnv) - + // TODO(burgerdev): The deployment should be generated programmatically. resources, err := filepath.Glob("./workspace/deployment/*.yml") require.NoError(t, err) - - require.True(t, t.Run("generate", func(t *testing.T) { - require := require.New(t) - - args := []string{ - "--workspace-dir", "./workspace", - } - args = append(args, resources...) - - generate := cmd.NewGenerateCmd() - generate.Flags().String("workspace-dir", "", "") // Make generate aware of root flags - generate.SetArgs(args) - generate.SetOut(io.Discard) - errBuf := &bytes.Buffer{} - generate.SetErr(errBuf) - - require.NoError(generate.Execute(), "could not generate manifest: %s", errBuf) - })) - - // TODO(burgerdev): policy hash should come from contrast generate output. - coordinatorPolicyHashBytes, err := os.ReadFile("workspace/coordinator-policy.sha256") - require.NoError(t, err) - coordinatorPolicyHash := string(coordinatorPolicyHashBytes) - require.NotEmpty(t, coordinatorPolicyHash, "expected apply to fill coordinator policy hash") - - require.True(t, t.Run("apply", func(t *testing.T) { - require := require.New(t) - - var objects []*unstructured.Unstructured - for _, file := range resources { - yaml, err := os.ReadFile(file) - require.NoError(err) - fileObjects, err := kubeapi.UnmarshalUnstructuredK8SResource(yaml) - require.NoError(err) - objects = append(objects, fileObjects...) + var objects []*unstructured.Unstructured + for _, file := range resources { + yaml, err := os.ReadFile(file) + require.NoError(t, err) + fileObjects, err := kubeapi.UnmarshalUnstructuredK8SResource(yaml) + require.NoError(t, err) + + // TODO(burgerdev): remove once demo deployments don't contain namespaces anymore. + for _, obj := range fileObjects { + if obj.GetKind() == "Namespace" { + continue + } + objects = append(objects, obj) } + } - ctx, cancel := context.WithTimeout(context.Background(), 1*time.Minute) - defer cancel() - - c := kubeclient.NewForTest(t) - require.NoError(c.Apply(ctx, objects...)) - }), "Kubernetes resources need to be applied for subsequent tests") - - require.True(t, t.Run("set", func(t *testing.T) { - require := require.New(t) - - ctx, cancel := context.WithTimeout(context.Background(), 1*time.Minute) - defer cancel() - - require.NoError(c.WaitForDeployment(ctx, namespace, "coordinator")) - - coordinator, cancelPortForward, err := c.PortForwardPod(ctx, namespace, "port-forwarder-coordinator", "1313") - require.NoError(err) - defer cancelPortForward() - - args := []string{ - "--coordinator-policy-hash", coordinatorPolicyHash, - "--coordinator", coordinator, - "--workspace-dir", "./workspace", - } - args = append(args, resources...) - - set := cmd.NewSetCmd() - set.Flags().String("workspace-dir", "", "") // Make set aware of root flags - set.SetArgs(args) - set.SetOut(io.Discard) - errBuf := &bytes.Buffer{} - set.SetErr(errBuf) - - require.NoError(set.Execute(), "could not set manifest at coordinator: %s", errBuf) - }), "contrast set needs to succeed for subsequent tests") - - certs := make(map[string][]byte) - - require.True(t, t.Run("contrast verify", func(t *testing.T) { - require := require.New(t) - - ctx, cancel := context.WithTimeout(context.Background(), 1*time.Minute) - defer cancel() - - require.NoError(c.WaitForDeployment(ctx, namespace, "coordinator")) + ct.Init(t, objects) + require.True(t, t.Run("generate", ct.Generate), "contrast generate needs to succeed for subsequent tests") - coordinator, cancelPortForward, err := c.PortForwardPod(ctx, namespace, "port-forwarder-coordinator", "1313") - require.NoError(err) - defer cancelPortForward() + require.True(t, t.Run("apply", ct.Apply), "Kubernetes resources need to be applied for subsequent tests") - workspaceDir, err := os.MkdirTemp("", "contrast-verify.*") - require.NoError(err) + require.True(t, t.Run("set", ct.Set), "contrast set needs to succeed for subsequent tests") - verify := cmd.NewVerifyCmd() - verify.SetArgs([]string{ - "--workspace-dir", workspaceDir, - "--coordinator-policy-hash", coordinatorPolicyHash, - "--coordinator", coordinator, - }) - verify.SetOut(io.Discard) - errBuf := &bytes.Buffer{} - verify.SetErr(errBuf) - - require.NoError(verify.Execute(), "could not verify coordinator: %s", errBuf) - - for _, certFile := range []string{ - "coordinator-root.pem", - "mesh-root.pem", - } { - pem, err := os.ReadFile(path.Join(workspaceDir, certFile)) - assert.NoError(t, err) - certs[certFile] = pem - } - }), "contrast verify needs to succeed for subsequent tests") + require.True(t, t.Run("contrast verify", ct.Verify), "contrast verify needs to succeed for subsequent tests") - for certFile, pem := range certs { - t.Run("go dial frontend with ca "+certFile, func(t *testing.T) { + for cert, pool := range map[string]*x509.CertPool{ + "mesh CA cert": ct.MeshCACert(), + "root CA cert": ct.RootCACert(), + } { + t.Run("go dial frontend with "+cert, func(t *testing.T) { ctx, cancel := context.WithTimeout(context.Background(), 1*time.Minute) defer cancel() require := require.New(t) - require.NoError(c.WaitForDeployment(ctx, namespace, opensslFrontend)) + require.NoError(c.WaitForDeployment(ctx, ct.Namespace, opensslFrontend)) - addr, cancelPortForward, err := c.PortForwardPod(ctx, namespace, "port-forwarder-openssl-frontend", "443") + addr, cancelPortForward, err := c.PortForwardPod(ctx, ct.Namespace, "port-forwarder-openssl-frontend", "443") require.NoError(err) defer cancelPortForward() - pool := x509.NewCertPool() - require.True(pool.AppendCertsFromPEM(pem)) dialer := &tls.Dialer{Config: &tls.Config{RootCAs: pool}} conn, err := dialer.DialContext(ctx, "tcp", addr) require.NoError(err) @@ -184,19 +91,19 @@ func TestOpenSSL(t *testing.T) { c := kubeclient.NewForTest(t) - require.NoError(c.WaitForDeployment(ctx, namespace, opensslFrontend)) - require.NoError(c.WaitForDeployment(ctx, namespace, opensslBackend)) + require.NoError(c.WaitForDeployment(ctx, ct.Namespace, opensslFrontend)) + require.NoError(c.WaitForDeployment(ctx, ct.Namespace, opensslBackend)) - frontendPods, err := c.PodsFromDeployment(ctx, namespace, opensslFrontend) + frontendPods, err := c.PodsFromDeployment(ctx, ct.Namespace, opensslFrontend) require.NoError(err) - require.Len(frontendPods, 1, "pod not found: %s/%s", namespace, opensslFrontend) + require.Len(frontendPods, 1, "pod not found: %s/%s", ct.Namespace, opensslFrontend) // Call the backend server from the frontend. If this command produces no TLS error, we verified that // - the certificate in the frontend pod can be used as a client certificate // - the certificate in the backend pod can be used as a server certificate // - the backend's CA configuration accepted the frontend certificate // - the frontend's CA configuration accepted the backend certificate - stdout, stderr, err := c.Exec(ctx, namespace, frontendPods[0].Name, + stdout, stderr, err := c.Exec(ctx, ct.Namespace, frontendPods[0].Name, []string{"/bin/bash", "-c", `printf "GET / HTTP/1.0\nHost: openssl-backend\n" | openssl s_client -connect openssl-backend:443 -verify_return_error -CAfile /tls-config/MeshCACert.pem -cert /tls-config/certChain.pem -key /tls-config/key.pem`}, ) t.Log(stdout)