From df9762c7ab0b94467d7ce03e7c5dbcb18cfd704b Mon Sep 17 00:00:00 2001 From: Markus Rudy Date: Fri, 19 Apr 2024 09:35:46 +0200 Subject: [PATCH] e2e: add test for release candidates --- e2e/internal/kubeclient/deploy.go | 33 ++++ e2e/release/release_test.go | 258 ++++++++++++++++++++++++++ go.mod | 2 + go.sum | 5 + packages/by-name/contrast/package.nix | 4 +- 5 files changed, 300 insertions(+), 2 deletions(-) create mode 100644 e2e/release/release_test.go diff --git a/e2e/internal/kubeclient/deploy.go b/e2e/internal/kubeclient/deploy.go index 69aa14e060..246c334f8d 100644 --- a/e2e/internal/kubeclient/deploy.go +++ b/e2e/internal/kubeclient/deploy.go @@ -99,6 +99,39 @@ func (c *Kubeclient) WaitForDeployment(ctx context.Context, namespace, name stri } } +// WaitForLoadBalancer waits until the given service is configured with an external IP and returns it. +func (c *Kubeclient) WaitForLoadBalancer(ctx context.Context, namespace, name string) (string, error) { + watcher, err := c.client.CoreV1().Services(namespace).Watch(ctx, metav1.ListOptions{FieldSelector: "metadata.name=" + name}) + if err != nil { + return "", err + } + for { + select { + case evt := <-watcher.ResultChan(): + switch evt.Type { + case watch.Added: + fallthrough + case watch.Modified: + svc, ok := evt.Object.(*corev1.Service) + if !ok { + return "", fmt.Errorf("watcher received unexpected type %T", evt.Object) + } + for _, ingress := range svc.Status.LoadBalancer.Ingress { + if ingress.IP != "" { + return ingress.IP, nil + } + } + case watch.Deleted: + return "", fmt.Errorf("service %s/%s was deleted while waiting for it", namespace, name) + default: + c.log.Warn("ignoring unexpected watch event", "type", evt.Type, "object", evt.Object) + } + case <-ctx.Done(): + return "", ctx.Err() + } + } +} + func (c *Kubeclient) toJSON(a any) string { s, err := json.Marshal(a) if err != nil { diff --git a/e2e/release/release_test.go b/e2e/release/release_test.go new file mode 100644 index 0000000000..e643654b86 --- /dev/null +++ b/e2e/release/release_test.go @@ -0,0 +1,258 @@ +//go:build e2e +// +build e2e + +package release + +import ( + "context" + "crypto/rand" + "crypto/tls" + "crypto/x509" + "encoding/hex" + "flag" + "io" + "net" + "net/http" + "os" + "os/exec" + "path" + "path/filepath" + "testing" + "time" + + "github.com/edgelesssys/contrast/e2e/internal/kubeclient" + "github.com/edgelesssys/contrast/e2e/internal/kuberesource" + "github.com/edgelesssys/contrast/internal/kubeapi" + "github.com/google/go-github/v61/github" + "github.com/stretchr/testify/require" +) + +const ( + tokenEnvVar = "GH_TOKEN" +) + +var ( + owner = flag.String("owner", "edgelesssys", "Github repository owner") + repo = flag.String("repo", "contrast", "Github repository") + tag = flag.String("tag", "", "tag name of the release to download") + namespace = flag.String("namespace", "", "k8s namespace to install resources to (will be deleted unless --keep is set)") + keep = flag.Bool("keep", false, "don't delete test resources and deployment") +) + +// TestRelease downloads a release from Github, sets up the coordinator, installs the demo +// deployment and runs some simple smoke tests. +func TestRelease(t *testing.T) { + ctx := context.Background() + k := kubeclient.NewForTest(t) + + if *namespace == "" { + *namespace = randomNamespace(t) + t.Logf("Created test namespace %s", *namespace) + } + + require.True(t, t.Run("create-namespace", func(t *testing.T) { + require := require.New(t) + ctx, cancel := context.WithTimeout(ctx, 1*time.Minute) + defer cancel() + + res, err := kuberesource.ResourcesToUnstructured([]any{kuberesource.Namespace(*namespace)}) + require.NoError(err) + require.NoError(k.Apply(ctx, res...)) + }), "the namespace is required for subsequent tests to run") + + t.Cleanup(func() { + if *keep { + return + } + ctx, cancel := context.WithTimeout(ctx, 1*time.Minute) + defer cancel() + + res, err := kuberesource.ResourcesToUnstructured([]any{kuberesource.Namespace(*namespace)}) + if err != nil { + return + } + k.Delete(ctx, res...) + }) + + dir := fetchRelease(ctx, t) + + contrast := &contrast{dir} + + for _, sub := range []string{"help"} { + contrast.Run(t, ctx, 2*time.Second, sub) + } + + var coordinatorIP string + require.True(t, t.Run("apply-coordinator", func(t *testing.T) { + require := require.New(t) + ctx, cancel := context.WithTimeout(ctx, 5*time.Minute) + defer cancel() + + yaml, err := os.ReadFile(path.Join(dir, "coordinator.yml")) + require.NoError(err) + resources, err := kubeapi.UnmarshalUnstructuredK8SResource(yaml) + require.NoError(err) + + for _, r := range resources { + require.NoError(k.PatchNamespace(*namespace, r)) + } + + require.NoError(k.Apply(ctx, resources...)) + require.NoError(k.WaitForDeployment(ctx, *namespace, "coordinator")) + coordinatorIP, err = k.WaitForLoadBalancer(ctx, *namespace, "coordinator") + require.NoError(err) + }), "the coordinator is required for subsequent tests to run") + + require.True(t, t.Run("unpack-deployment", func(t *testing.T) { + require := require.New(t) + ctx, cancel := context.WithTimeout(ctx, 10*time.Second) + defer cancel() + + cmd := exec.CommandContext(ctx, "unzip", "emojivoto-demo.zip") + cmd.Dir = dir + out, err := cmd.CombinedOutput() + require.NoError(err, "output:\n%s", string(out)) + + infos, err := os.ReadDir(path.Join(dir, "deployment")) + require.NoError(err) + for _, info := range infos { + name := path.Join(path.Join(dir, "deployment"), info.Name()) + yaml, err := os.ReadFile(name) + resources, err := kubeapi.UnmarshalUnstructuredK8SResource(yaml) + require.NoError(err) + + for _, r := range resources { + require.NoError(k.PatchNamespace(*namespace, r)) + } + newYAML, err := kuberesource.EncodeUnstructured(resources) + require.NoError(err) + require.NoError(os.WriteFile(name, newYAML, 0o644)) + + } + }), "unpacking needs to succeed for subsequent tests to run") + + contrast.Run(t, ctx, 2*time.Minute, "generate", "deployment/") + contrast.Run(t, ctx, 1*time.Minute, "set", "-c", coordinatorIP+":1313", "deployment/") + contrast.Run(t, ctx, 1*time.Minute, "verify", "-c", coordinatorIP+":1313") + + require.True(t, t.Run("apply-demo", func(t *testing.T) { + require := require.New(t) + ctx, cancel := context.WithTimeout(ctx, 5*time.Minute) + defer cancel() + + files, err := filepath.Glob(path.Join(dir, "deployment", "*.yml")) + require.NoError(err) + for _, file := range files { + yaml, err := os.ReadFile(file) + require.NoError(err) + resources, err := kubeapi.UnmarshalUnstructuredK8SResource(yaml) + require.NoError(err) + require.NoError(k.Apply(ctx, resources...)) + } + + require.NoError(k.WaitForDeployment(ctx, *namespace, "vote-bot")) + require.NoError(k.WaitForDeployment(ctx, *namespace, "voting")) + require.NoError(k.WaitForDeployment(ctx, *namespace, "emoji")) + require.NoError(k.WaitForDeployment(ctx, *namespace, "web")) + }), "applying the demo is required for subsequent tests to run") + + t.Run("test-demo", func(t *testing.T) { + require := require.New(t) + ctx, cancel := context.WithTimeout(ctx, 1*time.Minute) + defer cancel() + + emojiwebIP, err := k.WaitForLoadBalancer(ctx, *namespace, "web-svc") + require.NoError(err) + + cfg := &tls.Config{RootCAs: x509.NewCertPool()} + pem, err := os.ReadFile(path.Join(dir, "verify", "mesh-root.pem")) + require.NoError(err) + require.True(cfg.RootCAs.AppendCertsFromPEM(pem)) + + c := http.Client{ + Transport: &http.Transport{ + DialContext: func(ctx context.Context, network, addr string) (net.Conn, error) { + return (&net.Dialer{}).DialContext(ctx, "tcp", net.JoinHostPort(emojiwebIP, "443")) + }, + TLSClientConfig: cfg, + }, + } + req, err := http.NewRequestWithContext(ctx, http.MethodGet, "https://web", nil) + require.NoError(err) + resp, err := c.Do(req) + require.NoError(err) + require.Equal(http.StatusOK, resp.StatusCode) + }) +} + +type contrast struct { + dir string +} + +func (c *contrast) Run(t *testing.T, ctx context.Context, timeout time.Duration, args ...string) { + require.True(t, t.Run(args[0], func(t *testing.T) { + ctx, cancel := context.WithTimeout(ctx, timeout) + defer cancel() + cmd := exec.CommandContext(ctx, "./contrast", args...) + cmd.Dir = c.dir + out, err := cmd.CombinedOutput() + require.NoError(t, err, "output:\n%s", string(out)) + }), args[0]+" needs to succeed for subsequent tests to run") +} + +func randomNamespace(t *testing.T) string { + buf := make([]byte, 4) + n, err := rand.Read(buf) + require.NoError(t, err) + require.Equal(t, 4, n) + return "releasetest-" + hex.EncodeToString(buf) +} + +func fetchRelease(ctx context.Context, t *testing.T) string { + require := require.New(t) + ctx, cancel := context.WithTimeout(ctx, 2*time.Minute) + defer cancel() + + token := os.Getenv(tokenEnvVar) + require.NotEmpty(token, "environment variable %q must contain a Github access token", tokenEnvVar) + gh := github.NewClient(nil).WithAuthToken(token) + + var dir string + if *keep { + var err error + dir, err = os.MkdirTemp("", "releasetest-") + require.NoError(err) + t.Logf("Created test directory %s", dir) + } else { + dir = t.TempDir() + } + + // Find our target release. There is GetReleaseByTag, but we may be looking for a draft release. + rels, _, err := gh.Repositories.ListReleases(ctx, *owner, *repo, nil) + require.NoError(err) + var release *github.RepositoryRelease + for _, rel := range rels { + if *rel.TagName == *tag { + release = rel + break + } + } + require.NotNil(release) + + for _, asset := range release.Assets { + f, err := os.OpenFile(path.Join(dir, *asset.Name), os.O_CREATE|os.O_RDWR, 0o777) + require.NoError(err) + body, _, err := gh.Repositories.DownloadReleaseAsset(ctx, *owner, *repo, *asset.ID, http.DefaultClient) + require.NoError(err) + _, err = io.Copy(f, body) + require.NoError(err) + f.Close() + } + + return dir +} + +func TestMain(m *testing.M) { + flag.Parse() + os.Exit(m.Run()) +} diff --git a/go.mod b/go.mod index 0b90bea650..931cf7ed98 100644 --- a/go.mod +++ b/go.mod @@ -3,6 +3,7 @@ module github.com/edgelesssys/contrast go 1.21 require ( + github.com/google/go-github/v61 v61.0.0 github.com/google/go-sev-guest v0.11.1 github.com/spf13/afero v1.11.0 github.com/spf13/cobra v1.8.0 @@ -32,6 +33,7 @@ require ( github.com/golang/protobuf v1.5.4 // indirect github.com/google/gnostic-models v0.6.8 // indirect github.com/google/go-configfs-tsm v0.2.2 // indirect + github.com/google/go-querystring v1.1.0 // indirect github.com/google/gofuzz v1.2.0 // indirect github.com/google/logger v1.1.1 // indirect github.com/google/uuid v1.6.0 // indirect diff --git a/go.sum b/go.sum index 73c598726a..9a9bfe95b1 100644 --- a/go.sum +++ b/go.sum @@ -25,12 +25,17 @@ github.com/golang/protobuf v1.5.4 h1:i7eJL8qZTpSEXOPTxNKhASYpMn+8e5Q6AdndVa1dWek github.com/golang/protobuf v1.5.4/go.mod h1:lnTiLA8Wa4RWRcIUkrtSVa5nRhsEGBg48fD6rSs7xps= github.com/google/gnostic-models v0.6.8 h1:yo/ABAfM5IMRsS1VnXjTBvUb61tFIHozhlYvRgGre9I= github.com/google/gnostic-models v0.6.8/go.mod h1:5n7qKqH0f5wFt+aWF8CW6pZLLNOfYuF5OpfBSENuI8U= +github.com/google/go-cmp v0.5.2/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI= github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= github.com/google/go-configfs-tsm v0.2.2 h1:YnJ9rXIOj5BYD7/0DNnzs8AOp7UcvjfTvt215EWcs98= github.com/google/go-configfs-tsm v0.2.2/go.mod h1:EL1GTDFMb5PZQWDviGfZV9n87WeGTR/JUg13RfwkgRo= +github.com/google/go-github/v61 v61.0.0 h1:VwQCBwhyE9JclCI+22/7mLB1PuU9eowCXKY5pNlu1go= +github.com/google/go-github/v61 v61.0.0/go.mod h1:0WR+KmsWX75G2EbpyGsGmradjo3IiciuI4BmdVCobQY= +github.com/google/go-querystring v1.1.0 h1:AnCroh3fv4ZBgVIf1Iwtovgjaw/GiKJo8M8yD/fhyJ8= +github.com/google/go-querystring v1.1.0/go.mod h1:Kcdr2DB4koayq7X8pmAG4sNG59So17icRSOU623lUBU= github.com/google/go-sev-guest v0.11.1 h1:gnww4U8fHV5DCPz4gykr1s8SEX1fFNcxCBy+vvXN24k= github.com/google/go-sev-guest v0.11.1/go.mod h1:qBOfb+JmgsUI3aUyzQoGC13Kpp9zwLeWvuyXmA9q77w= github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= diff --git a/packages/by-name/contrast/package.nix b/packages/by-name/contrast/package.nix index fa7bd0f60e..8ec13da90f 100644 --- a/packages/by-name/contrast/package.nix +++ b/packages/by-name/contrast/package.nix @@ -15,7 +15,7 @@ let ldflags = [ "-s" ]; - subPackages = [ "e2e/openssl" "e2e/servicemesh" ]; + subPackages = [ "e2e/openssl" "e2e/servicemesh" "e2e/release" ]; }; runtimeHandler = lib.removeSuffix "\n" (builtins.readFile "${runtime-class-files}/runtime-handler"); @@ -51,7 +51,7 @@ buildGoModule rec { }; proxyVendor = true; - vendorHash = "sha256-m67gNUGvb4z7OyHvJdOX7SZKgBWn11OEA28oJiQjpXI="; + vendorHash = "sha256-tM+z5RoZ2ClB88OvenYMu3DVUXWnjFEF7xK9p6/06jc="; subPackages = packageOutputs ++ [ "e2e/internal/kuberesource/resourcegen" ];