Skip to content

Commit

Permalink
e2e: add recovery test
Browse files Browse the repository at this point in the history
  • Loading branch information
@burgerdev
burgerdev committed Jul 1, 2024
1 parent 7dcd2ef commit a7c953d
Show file tree
Hide file tree
Showing 2 changed files with 87 additions and 12 deletions.
57 changes: 49 additions & 8 deletions e2e/internal/contrasttest/contrasttest.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -199,17 +199,19 @@ func (ct *ContrastTest) Set(t *testing.T) {
require.NoError(set.Execute(), "could not set manifest at coordinator: %s", errBuf)
}

// Verify runs the contrast verify subcommand.
func (ct *ContrastTest) Verify(t *testing.T) {
require := require.New(t)

// RunVerify runs the contrast verify subcommand.
func (ct *ContrastTest) RunVerify() error {
ctx, cancel := context.WithTimeout(context.Background(), 1*time.Minute)
defer cancel()

require.NoError(ct.Kubeclient.WaitFor(ctx, kubeclient.StatefulSet{}, ct.Namespace, "coordinator"))
if err := ct.Kubeclient.WaitFor(ctx, kubeclient.StatefulSet{}, ct.Namespace, "coordinator"); err != nil {
return fmt.Errorf("waiting for coordinator: %w", err)
}

coordinator, cancelPortForward, err := ct.Kubeclient.PortForwardPod(ctx, ct.Namespace, "port-forwarder-coordinator", "1313")
require.NoError(err)
if err != nil {
return err
}
defer cancelPortForward()

verify := cmd.NewVerifyCmd()
Expand All @@ -223,12 +225,51 @@ func (ct *ContrastTest) Verify(t *testing.T) {
errBuf := &bytes.Buffer{}
verify.SetErr(errBuf)

require.NoError(verify.Execute(), "could not verify coordinator: %s", errBuf)
if err := verify.Execute(); err != nil {
return fmt.Errorf("running verify failed: %w\n%s", err, errBuf)
}

ct.meshCACertPEM, err = os.ReadFile(path.Join(ct.WorkDir, "mesh-ca.pem"))
require.NoError(err)
if err != nil {
return fmt.Errorf("no mesh ca cert: %w", err)
}
ct.rootCACertPEM, err = os.ReadFile(path.Join(ct.WorkDir, "coordinator-root-ca.pem"))
if err != nil {
return fmt.Errorf("no root ca cert: %w", err)
}
return nil
}

// Verify runs the contrast verify subcommand and fails the test if it is not successful.
func (ct *ContrastTest) Verify(t *testing.T) {
require.NoError(t, ct.RunVerify())
}

// Recover runs the contrast recover subcommand.
func (ct *ContrastTest) Recover(t *testing.T) {
require := require.New(t)

ctx, cancel := context.WithTimeout(context.Background(), 1*time.Minute)
defer cancel()

require.NoError(ct.Kubeclient.WaitFor(ctx, kubeclient.StatefulSet{}, ct.Namespace, "coordinator"))

coordinator, cancelPortForward, err := ct.Kubeclient.PortForwardPod(ctx, ct.Namespace, "port-forwarder-coordinator", "1313")
require.NoError(err)
defer cancelPortForward()

args := append(ct.commonArgs(),
"--coordinator-policy-hash", ct.coordinatorPolicyHash,
"--coordinator", coordinator)

set := cmd.NewRecoverCmd()
set.Flags().String("workspace-dir", "", "") // Make set aware of root flags
set.SetArgs(args)
set.SetOut(io.Discard)
errBuf := &bytes.Buffer{}
set.SetErr(errBuf)

require.NoError(set.Execute(), "could not recover coordinator: %s", errBuf)
}

// MeshCACert returns a CertPool that contains the coordinator mesh CA cert.
Expand Down
42 changes: 38 additions & 4 deletions e2e/openssl/openssl_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -21,13 +21,17 @@ import (
"github.com/edgelesssys/contrast/e2e/internal/kubeclient"
"github.com/edgelesssys/contrast/internal/kuberesource"
"github.com/edgelesssys/contrast/internal/manifest"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)

// namespace the tests are executed in.
const (
opensslFrontend = "openssl-frontend"
opensslBackend = "openssl-backend"

meshCAFile = "mesh-ca.pem"
rootCAFile = "coordinator-root-ca.pem"
)

var (
Expand Down Expand Up @@ -115,7 +119,7 @@ func TestOpenSSL(t *testing.T) {
// - the certificate in the backend pod can be used as a server certificate
// - the backend's CA configuration accepted the frontend certificate
// - the frontend's CA configuration accepted the backend certificate
stdout, stderr, err := c.ExecDeployment(ctx, ct.Namespace, opensslFrontend, []string{"/bin/bash", "-c", opensslConnectCmd("openssl-backend:443", "mesh-ca.pem")})
stdout, stderr, err := c.ExecDeployment(ctx, ct.Namespace, opensslFrontend, []string{"/bin/bash", "-c", opensslConnectCmd("openssl-backend:443", meshCAFile)})
t.Log(stdout)
require.NoError(err, "stderr: %q", stderr)
})
Expand Down Expand Up @@ -152,14 +156,14 @@ func TestOpenSSL(t *testing.T) {
require.NoError(c.WaitFor(ctx, kubeclient.Deployment{}, ct.Namespace, deploymentToRestart))

// This should not succeed because the certificates have changed.
stdout, stderr, err := c.ExecDeployment(ctx, ct.Namespace, opensslFrontend, []string{"/bin/bash", "-c", opensslConnectCmd("openssl-backend:443", "mesh-ca.pem")})
stdout, stderr, err := c.ExecDeployment(ctx, ct.Namespace, opensslFrontend, []string{"/bin/bash", "-c", opensslConnectCmd("openssl-backend:443", meshCAFile)})
t.Log("openssl with wrong certificates:", stdout)
require.Error(err)
require.Contains(stderr, "certificate signature failure")

// Connect from backend to fronted, because the frontend does not require client certs.
// This should succeed because the root cert did not change.
stdout, stderr, err = c.ExecDeployment(ctx, ct.Namespace, opensslBackend, []string{"/bin/bash", "-c", opensslConnectCmd("openssl-frontend:443", "coordinator-root-ca.pem")})
stdout, stderr, err = c.ExecDeployment(ctx, ct.Namespace, opensslBackend, []string{"/bin/bash", "-c", opensslConnectCmd("openssl-frontend:443", rootCAFile)})
t.Log("openssl with root certificate:", stdout)
require.NoError(err, "stderr: %q", stderr)

Expand All @@ -172,11 +176,41 @@ func TestOpenSSL(t *testing.T) {
require.NoError(c.WaitFor(ctx, kubeclient.Deployment{}, ct.Namespace, d))

// This should succeed since both workloads now have updated certificates.
stdout, stderr, err = c.ExecDeployment(ctx, ct.Namespace, opensslFrontend, []string{"/bin/bash", "-c", opensslConnectCmd("openssl-backend:443", "mesh-ca.pem")})
stdout, stderr, err = c.ExecDeployment(ctx, ct.Namespace, opensslFrontend, []string{"/bin/bash", "-c", opensslConnectCmd("openssl-backend:443", meshCAFile)})
t.Log("openssl with correct certificates:", stdout)
require.NoError(err, "stderr: %q", stderr)
})
}

t.Run("coordinator recovery", func(t *testing.T) {
require := require.New(t)

ctx, cancel := context.WithTimeout(context.Background(), 5*time.Minute)
defer cancel()

c := kubeclient.NewForTest(t)

require.NoError(c.Restart(ctx, kubeclient.StatefulSet{}, ct.Namespace, "coordinator"))

require.ErrorContains(ct.RunVerify(), "recovery")

require.True(t.Run("contrast recover", ct.Recover))

require.True(t.Run("contrast verify", ct.Verify))

require.NoError(c.Restart(ctx, kubeclient.Deployment{}, ct.Namespace, opensslFrontend))
require.NoError(c.WaitFor(ctx, kubeclient.Deployment{}, ct.Namespace, opensslFrontend))

for _, cert := range []string{rootCAFile, meshCAFile} {
t.Run(cert, func(t *testing.T) {
stdout, stderr, err := c.ExecDeployment(ctx, ct.Namespace, opensslBackend, []string{"/bin/bash", "-c", opensslConnectCmd("openssl-frontend:443", cert)})
if err != nil {
t.Logf("openssl with %q after recovery:\n%s", cert, stdout)
}
assert.NoError(t, err, "stderr: %q", stderr)
})
}
})
}

func TestMain(m *testing.M) {
Expand Down

0 comments on commit a7c953d

Please sign in to comment.