diff --git a/e2e/internal/contrasttest/contrasttest.go b/e2e/internal/contrasttest/contrasttest.go index c2c500a673..151a7b9a32 100644 --- a/e2e/internal/contrasttest/contrasttest.go +++ b/e2e/internal/contrasttest/contrasttest.go @@ -199,17 +199,19 @@ func (ct *ContrastTest) Set(t *testing.T) { require.NoError(set.Execute(), "could not set manifest at coordinator: %s", errBuf) } -// Verify runs the contrast verify subcommand. -func (ct *ContrastTest) Verify(t *testing.T) { - require := require.New(t) - +// RunVerify runs the contrast verify subcommand. +func (ct *ContrastTest) RunVerify() error { ctx, cancel := context.WithTimeout(context.Background(), 1*time.Minute) defer cancel() - require.NoError(ct.Kubeclient.WaitFor(ctx, kubeclient.StatefulSet{}, ct.Namespace, "coordinator")) + if err := ct.Kubeclient.WaitFor(ctx, kubeclient.StatefulSet{}, ct.Namespace, "coordinator"); err != nil { + return fmt.Errorf("waiting for coordinator: %w", err) + } coordinator, cancelPortForward, err := ct.Kubeclient.PortForwardPod(ctx, ct.Namespace, "port-forwarder-coordinator", "1313") - require.NoError(err) + if err != nil { + return err + } defer cancelPortForward() verify := cmd.NewVerifyCmd() @@ -223,12 +225,51 @@ func (ct *ContrastTest) Verify(t *testing.T) { errBuf := &bytes.Buffer{} verify.SetErr(errBuf) - require.NoError(verify.Execute(), "could not verify coordinator: %s", errBuf) + if err := verify.Execute(); err != nil { + return fmt.Errorf("running verify failed: %w\n%s", err, errBuf) + } ct.meshCACertPEM, err = os.ReadFile(path.Join(ct.WorkDir, "mesh-ca.pem")) - require.NoError(err) + if err != nil { + return fmt.Errorf("no mesh ca cert: %w", err) + } ct.rootCACertPEM, err = os.ReadFile(path.Join(ct.WorkDir, "coordinator-root-ca.pem")) + if err != nil { + return fmt.Errorf("no root ca cert: %w", err) + } + return nil +} + +// Verify runs the contrast verify subcommand and fails the test if it is not successful. +func (ct *ContrastTest) Verify(t *testing.T) { + require.NoError(t, ct.RunVerify()) +} + +// Recover runs the contrast recover subcommand. +func (ct *ContrastTest) Recover(t *testing.T) { + require := require.New(t) + + ctx, cancel := context.WithTimeout(context.Background(), 1*time.Minute) + defer cancel() + + require.NoError(ct.Kubeclient.WaitFor(ctx, kubeclient.StatefulSet{}, ct.Namespace, "coordinator")) + + coordinator, cancelPortForward, err := ct.Kubeclient.PortForwardPod(ctx, ct.Namespace, "port-forwarder-coordinator", "1313") require.NoError(err) + defer cancelPortForward() + + args := append(ct.commonArgs(), + "--coordinator-policy-hash", ct.coordinatorPolicyHash, + "--coordinator", coordinator) + + set := cmd.NewRecoverCmd() + set.Flags().String("workspace-dir", "", "") // Make set aware of root flags + set.SetArgs(args) + set.SetOut(io.Discard) + errBuf := &bytes.Buffer{} + set.SetErr(errBuf) + + require.NoError(set.Execute(), "could not recover coordinator: %s", errBuf) } // MeshCACert returns a CertPool that contains the coordinator mesh CA cert. diff --git a/e2e/openssl/openssl_test.go b/e2e/openssl/openssl_test.go index f4acbe0fea..a1b047efdc 100644 --- a/e2e/openssl/openssl_test.go +++ b/e2e/openssl/openssl_test.go @@ -21,6 +21,7 @@ import ( "github.com/edgelesssys/contrast/e2e/internal/kubeclient" "github.com/edgelesssys/contrast/internal/kuberesource" "github.com/edgelesssys/contrast/internal/manifest" + "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" ) @@ -28,6 +29,9 @@ import ( const ( opensslFrontend = "openssl-frontend" opensslBackend = "openssl-backend" + + meshCAFile = "mesh-ca.pem" + rootCAFile = "coordinator-root-ca.pem" ) var ( @@ -115,7 +119,7 @@ func TestOpenSSL(t *testing.T) { // - the certificate in the backend pod can be used as a server certificate // - the backend's CA configuration accepted the frontend certificate // - the frontend's CA configuration accepted the backend certificate - stdout, stderr, err := c.ExecDeployment(ctx, ct.Namespace, opensslFrontend, []string{"/bin/bash", "-c", opensslConnectCmd("openssl-backend:443", "mesh-ca.pem")}) + stdout, stderr, err := c.ExecDeployment(ctx, ct.Namespace, opensslFrontend, []string{"/bin/bash", "-c", opensslConnectCmd("openssl-backend:443", meshCAFile)}) t.Log(stdout) require.NoError(err, "stderr: %q", stderr) }) @@ -152,14 +156,14 @@ func TestOpenSSL(t *testing.T) { require.NoError(c.WaitFor(ctx, kubeclient.Deployment{}, ct.Namespace, deploymentToRestart)) // This should not succeed because the certificates have changed. - stdout, stderr, err := c.ExecDeployment(ctx, ct.Namespace, opensslFrontend, []string{"/bin/bash", "-c", opensslConnectCmd("openssl-backend:443", "mesh-ca.pem")}) + stdout, stderr, err := c.ExecDeployment(ctx, ct.Namespace, opensslFrontend, []string{"/bin/bash", "-c", opensslConnectCmd("openssl-backend:443", meshCAFile)}) t.Log("openssl with wrong certificates:", stdout) require.Error(err) require.Contains(stderr, "certificate signature failure") // Connect from backend to fronted, because the frontend does not require client certs. // This should succeed because the root cert did not change. - stdout, stderr, err = c.ExecDeployment(ctx, ct.Namespace, opensslBackend, []string{"/bin/bash", "-c", opensslConnectCmd("openssl-frontend:443", "coordinator-root-ca.pem")}) + stdout, stderr, err = c.ExecDeployment(ctx, ct.Namespace, opensslBackend, []string{"/bin/bash", "-c", opensslConnectCmd("openssl-frontend:443", rootCAFile)}) t.Log("openssl with root certificate:", stdout) require.NoError(err, "stderr: %q", stderr) @@ -172,11 +176,41 @@ func TestOpenSSL(t *testing.T) { require.NoError(c.WaitFor(ctx, kubeclient.Deployment{}, ct.Namespace, d)) // This should succeed since both workloads now have updated certificates. - stdout, stderr, err = c.ExecDeployment(ctx, ct.Namespace, opensslFrontend, []string{"/bin/bash", "-c", opensslConnectCmd("openssl-backend:443", "mesh-ca.pem")}) + stdout, stderr, err = c.ExecDeployment(ctx, ct.Namespace, opensslFrontend, []string{"/bin/bash", "-c", opensslConnectCmd("openssl-backend:443", meshCAFile)}) t.Log("openssl with correct certificates:", stdout) require.NoError(err, "stderr: %q", stderr) }) } + + t.Run("coordinator recovery", func(t *testing.T) { + require := require.New(t) + + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Minute) + defer cancel() + + c := kubeclient.NewForTest(t) + + require.NoError(c.Restart(ctx, kubeclient.StatefulSet{}, ct.Namespace, "coordinator")) + + require.ErrorContains(ct.RunVerify(), "recovery") + + require.True(t.Run("contrast recover", ct.Recover)) + + require.True(t.Run("contrast verify", ct.Verify)) + + require.NoError(c.Restart(ctx, kubeclient.Deployment{}, ct.Namespace, opensslFrontend)) + require.NoError(c.WaitFor(ctx, kubeclient.Deployment{}, ct.Namespace, opensslFrontend)) + + for _, cert := range []string{rootCAFile, meshCAFile} { + t.Run(cert, func(t *testing.T) { + stdout, stderr, err := c.ExecDeployment(ctx, ct.Namespace, opensslBackend, []string{"/bin/bash", "-c", opensslConnectCmd("openssl-frontend:443", cert)}) + if err != nil { + t.Logf("openssl with %q after recovery:\n%s", cert, stdout) + } + assert.NoError(t, err, "stderr: %q", stderr) + }) + } + }) } func TestMain(m *testing.M) {