From 555bdd17bdee247692bce1f0644a0cca52b6f010 Mon Sep 17 00:00:00 2001 From: Markus Rudy Date: Fri, 19 Apr 2024 09:35:46 +0200 Subject: [PATCH 1/4] e2e: add test for release candidates --- e2e/internal/kubeclient/deploy.go | 59 ++++++ e2e/release/release_test.go | 285 ++++++++++++++++++++++++++ go.mod | 2 + go.sum | 5 + packages/by-name/contrast/package.nix | 4 +- 5 files changed, 353 insertions(+), 2 deletions(-) create mode 100644 e2e/release/release_test.go diff --git a/e2e/internal/kubeclient/deploy.go b/e2e/internal/kubeclient/deploy.go index 101869c3e..52b8b6a1e 100644 --- a/e2e/internal/kubeclient/deploy.go +++ b/e2e/internal/kubeclient/deploy.go @@ -7,7 +7,9 @@ import ( "context" "encoding/json" "fmt" + "net" "sort" + "strconv" "time" appsv1 "k8s.io/api/apps/v1" @@ -150,6 +152,63 @@ func (c *Kubeclient) WaitForDaemonset(ctx context.Context, namespace, name strin } } +// WaitForLoadBalancer waits until the given service is configured with an external IP and returns it. +func (c *Kubeclient) WaitForLoadBalancer(ctx context.Context, namespace, name string) (string, error) { + watcher, err := c.client.CoreV1().Services(namespace).Watch(ctx, metav1.ListOptions{FieldSelector: "metadata.name=" + name}) + if err != nil { + return "", err + } + var ip string + var port int +loop: + for { + select { + case evt := <-watcher.ResultChan(): + switch evt.Type { + case watch.Added: + fallthrough + case watch.Modified: + svc, ok := evt.Object.(*corev1.Service) + if !ok { + return "", fmt.Errorf("watcher received unexpected type %T", evt.Object) + } + for _, ingress := range svc.Status.LoadBalancer.Ingress { + if ingress.IP != "" { + ip = ingress.IP + // TODO(burgerdev): deal with more than one port, and protocols other than TCP + port = int(svc.Spec.Ports[0].Port) + break loop + } + } + case watch.Deleted: + return "", fmt.Errorf("service %s/%s was deleted while waiting for it", namespace, name) + default: + c.log.Warn("ignoring unexpected watch event", "type", evt.Type, "object", evt.Object) + } + case <-ctx.Done(): + return "", fmt.Errorf("LoadBalancer %s/%s did not get a public IP before %w", namespace, name, ctx.Err()) + } + } + + ticker := time.NewTicker(time.Second) + defer ticker.Stop() + + dialer := &net.Dialer{} + for { + select { + case <-ticker.C: + conn, err := dialer.DialContext(ctx, "tcp", net.JoinHostPort(ip, strconv.Itoa(port))) + if err == nil { + conn.Close() + return ip, nil + } + c.log.Info("probe failed", "namespace", namespace, "name", name, "error", err) + case <-ctx.Done(): + return "", fmt.Errorf("LoadBalancer %s/%s never responded to probing before %w", namespace, name, ctx.Err()) + } + } +} + func (c *Kubeclient) toJSON(a any) string { s, err := json.Marshal(a) if err != nil { diff --git a/e2e/release/release_test.go b/e2e/release/release_test.go new file mode 100644 index 000000000..71b176977 --- /dev/null +++ b/e2e/release/release_test.go @@ -0,0 +1,285 @@ +// Copyright 2024 Edgeless Systems GmbH +// SPDX-License-Identifier: AGPL-3.0-only + +//go:build e2e +// +build e2e + +package release + +import ( + "context" + "crypto/rand" + "crypto/tls" + "crypto/x509" + "encoding/hex" + "flag" + "io" + "net" + "net/http" + "os" + "os/exec" + "path" + "path/filepath" + "testing" + "time" + + "github.com/edgelesssys/contrast/e2e/internal/kubeclient" + "github.com/edgelesssys/contrast/internal/kubeapi" + "github.com/edgelesssys/contrast/internal/kuberesource" + "github.com/google/go-github/v61/github" + "github.com/stretchr/testify/require" +) + +const ( + tokenEnvVar = "GH_TOKEN" +) + +var ( + owner = flag.String("owner", "edgelesssys", "Github repository owner") + repo = flag.String("repo", "contrast", "Github repository") + tag = flag.String("tag", "", "tag name of the release to download") + namespace = flag.String("namespace", "", "k8s namespace to install resources to (will be deleted unless --keep is set)") + keep = flag.Bool("keep", false, "don't delete test resources and deployment") +) + +// TestRelease downloads a release from Github, sets up the coordinator, installs the demo +// deployment and runs some simple smoke tests. +func TestRelease(t *testing.T) { + ctx := context.Background() + k := kubeclient.NewForTest(t) + + if *namespace == "" { + *namespace = randomNamespace(t) + t.Logf("Created test namespace %s", *namespace) + } + + require.True(t, t.Run("create-namespace", func(t *testing.T) { + require := require.New(t) + ctx, cancel := context.WithTimeout(ctx, 1*time.Minute) + defer cancel() + + res, err := kuberesource.ResourcesToUnstructured([]any{kuberesource.Namespace(*namespace)}) + require.NoError(err) + require.NoError(k.Apply(ctx, res...)) + }), "the namespace is required for subsequent tests to run") + + t.Cleanup(func() { + if *keep { + return + } + ctx, cancel := context.WithTimeout(ctx, 1*time.Minute) + defer cancel() + + res, err := kuberesource.ResourcesToUnstructured([]any{kuberesource.Namespace(*namespace)}) + if err != nil { + return + } + k.Delete(ctx, res...) + }) + + dir := fetchRelease(ctx, t) + + contrast := &contrast{dir} + + for _, sub := range []string{"help"} { + contrast.Run(t, ctx, 2*time.Second, sub) + } + require.True(t, t.Run("apply-runtime", func(t *testing.T) { + require := require.New(t) + ctx, cancel := context.WithTimeout(ctx, 5*time.Minute) + defer cancel() + + yaml, err := os.ReadFile(path.Join(dir, "runtime.yml")) + require.NoError(err) + resources, err := kubeapi.UnmarshalUnstructuredK8SResource(yaml) + require.NoError(err) + + for _, r := range resources { + if r.GetKind() != "RuntimeClass" { + r.SetNamespace(*namespace) + } + } + + require.NoError(k.Apply(ctx, resources...)) + require.NoError(k.WaitForDaemonset(ctx, *namespace, "contrast-node-installer")) + }), "the runtime is required for subsequent tests to run") + + var coordinatorIP string + require.True(t, t.Run("apply-coordinator", func(t *testing.T) { + require := require.New(t) + ctx, cancel := context.WithTimeout(ctx, 5*time.Minute) + defer cancel() + + yaml, err := os.ReadFile(path.Join(dir, "coordinator.yml")) + require.NoError(err) + resources, err := kubeapi.UnmarshalUnstructuredK8SResource(yaml) + require.NoError(err) + + for _, r := range resources { + r.SetNamespace(*namespace) + } + + require.NoError(k.Apply(ctx, resources...)) + require.NoError(k.WaitForDeployment(ctx, *namespace, "coordinator")) + coordinatorIP, err = k.WaitForLoadBalancer(ctx, *namespace, "coordinator") + require.NoError(err) + }), "the coordinator is required for subsequent tests to run") + + require.True(t, t.Run("unpack-deployment", func(t *testing.T) { + require := require.New(t) + ctx, cancel := context.WithTimeout(ctx, 10*time.Second) + defer cancel() + + cmd := exec.CommandContext(ctx, "unzip", "emojivoto-demo.zip") + cmd.Dir = dir + out, err := cmd.CombinedOutput() + require.NoError(err, "output:\n%s", string(out)) + + infos, err := os.ReadDir(path.Join(dir, "deployment")) + require.NoError(err) + for _, info := range infos { + name := path.Join(path.Join(dir, "deployment"), info.Name()) + yaml, err := os.ReadFile(name) + require.NoError(err) + resources, err := kubeapi.UnmarshalUnstructuredK8SResource(yaml) + require.NoError(err) + + for _, r := range resources { + r.SetNamespace(*namespace) + } + newYAML, err := kuberesource.EncodeUnstructured(resources) + require.NoError(err) + require.NoError(os.WriteFile(name, newYAML, 0o644)) + + } + }), "unpacking needs to succeed for subsequent tests to run") + + contrast.Run(t, ctx, 2*time.Minute, "generate", "deployment/") + contrast.Run(t, ctx, 1*time.Minute, "set", "-c", coordinatorIP+":1313", "deployment/") + contrast.Run(t, ctx, 1*time.Minute, "verify", "-c", coordinatorIP+":1313") + + require.True(t, t.Run("apply-demo", func(t *testing.T) { + require := require.New(t) + ctx, cancel := context.WithTimeout(ctx, 5*time.Minute) + defer cancel() + + files, err := filepath.Glob(path.Join(dir, "deployment", "*.yml")) + require.NoError(err) + for _, file := range files { + yaml, err := os.ReadFile(file) + require.NoError(err) + resources, err := kubeapi.UnmarshalUnstructuredK8SResource(yaml) + require.NoError(err) + require.NoError(k.Apply(ctx, resources...)) + } + + require.NoError(k.WaitForDeployment(ctx, *namespace, "vote-bot")) + require.NoError(k.WaitForDeployment(ctx, *namespace, "voting")) + require.NoError(k.WaitForDeployment(ctx, *namespace, "emoji")) + require.NoError(k.WaitForDeployment(ctx, *namespace, "web")) + }), "applying the demo is required for subsequent tests to run") + + t.Run("test-demo", func(t *testing.T) { + require := require.New(t) + ctx, cancel := context.WithTimeout(ctx, 2*time.Minute) + defer cancel() + + emojiwebIP, err := k.WaitForLoadBalancer(ctx, *namespace, "web-svc") + require.NoError(err) + + cfg := &tls.Config{RootCAs: x509.NewCertPool()} + pem, err := os.ReadFile(path.Join(dir, "verify", "mesh-ca.pem")) + require.NoError(err) + require.True(cfg.RootCAs.AppendCertsFromPEM(pem)) + + c := http.Client{ + Transport: &http.Transport{ + DialContext: func(ctx context.Context, network, addr string) (net.Conn, error) { + return (&net.Dialer{}).DialContext(ctx, "tcp", net.JoinHostPort(emojiwebIP, "443")) + }, + TLSClientConfig: cfg, + }, + } + req, err := http.NewRequestWithContext(ctx, http.MethodGet, "https://web", nil) + require.NoError(err) + resp, err := c.Do(req) + require.NoError(err) + require.Equal(http.StatusOK, resp.StatusCode) + }) +} + +type contrast struct { + dir string +} + +func (c *contrast) Run(t *testing.T, ctx context.Context, timeout time.Duration, args ...string) { + require.True(t, t.Run(args[0], func(t *testing.T) { + ctx, cancel := context.WithTimeout(ctx, timeout) + defer cancel() + + args = append([]string{"--log-level", "debug"}, args...) + cmd := exec.CommandContext(ctx, "./contrast", args...) + cmd.Dir = c.dir + out, err := cmd.CombinedOutput() + require.NoError(t, err, "output:\n%s", string(out)) + }), args[0]+" needs to succeed for subsequent tests to run") +} + +func randomNamespace(t *testing.T) string { + buf := make([]byte, 4) + n, err := rand.Read(buf) + require.NoError(t, err) + require.Equal(t, 4, n) + return "releasetest-" + hex.EncodeToString(buf) +} + +// fetchRelease downloads the release corresponding to the global tag variable and returns the directory. +func fetchRelease(ctx context.Context, t *testing.T) string { + require := require.New(t) + ctx, cancel := context.WithTimeout(ctx, 2*time.Minute) + defer cancel() + + token := os.Getenv(tokenEnvVar) + require.NotEmpty(token, "environment variable %q must contain a Github access token", tokenEnvVar) + gh := github.NewClient(nil).WithAuthToken(token) + + var dir string + if *keep { + var err error + dir, err = os.MkdirTemp("", "releasetest-") + require.NoError(err) + t.Logf("Created test directory %s", dir) + } else { + dir = t.TempDir() + } + + // Find our target release. There is GetReleaseByTag, but we may be looking for a draft release. + rels, resp, err := gh.Repositories.ListReleases(ctx, *owner, *repo, nil) + require.NoError(err) + var release *github.RepositoryRelease + for _, rel := range rels { + t.Logf("Checking release %q", *rel.TagName) + if *rel.TagName == *tag { + release = rel + break + } + } + require.NotNil(release, "release %q not found among %d releases\nGithub response:\n%#v", *tag, len(rels), resp) + + for _, asset := range release.Assets { + f, err := os.OpenFile(path.Join(dir, *asset.Name), os.O_CREATE|os.O_RDWR, 0o777) + require.NoError(err) + body, _, err := gh.Repositories.DownloadReleaseAsset(ctx, *owner, *repo, *asset.ID, http.DefaultClient) + require.NoError(err, "could not fetch release asset %q (id: %d)", asset.Name, asset.ID) + _, err = io.Copy(f, body) + require.NoError(err) + f.Close() + } + + return dir +} + +func TestMain(m *testing.M) { + flag.Parse() + os.Exit(m.Run()) +} diff --git a/go.mod b/go.mod index 0b9ed8fda..fb241a680 100644 --- a/go.mod +++ b/go.mod @@ -5,6 +5,7 @@ go 1.22.0 toolchain go1.22.2 require ( + github.com/google/go-github/v61 v61.0.0 github.com/google/go-sev-guest v0.11.1 github.com/spf13/afero v1.11.0 github.com/spf13/cobra v1.8.0 @@ -35,6 +36,7 @@ require ( github.com/golang/protobuf v1.5.4 // indirect github.com/google/gnostic-models v0.6.8 // indirect github.com/google/go-configfs-tsm v0.2.2 // indirect + github.com/google/go-querystring v1.1.0 // indirect github.com/google/gofuzz v1.2.0 // indirect github.com/google/logger v1.1.1 // indirect github.com/google/uuid v1.6.0 // indirect diff --git a/go.sum b/go.sum index 9f28cd563..04871532e 100644 --- a/go.sum +++ b/go.sum @@ -25,12 +25,17 @@ github.com/golang/protobuf v1.5.4 h1:i7eJL8qZTpSEXOPTxNKhASYpMn+8e5Q6AdndVa1dWek github.com/golang/protobuf v1.5.4/go.mod h1:lnTiLA8Wa4RWRcIUkrtSVa5nRhsEGBg48fD6rSs7xps= github.com/google/gnostic-models v0.6.8 h1:yo/ABAfM5IMRsS1VnXjTBvUb61tFIHozhlYvRgGre9I= github.com/google/gnostic-models v0.6.8/go.mod h1:5n7qKqH0f5wFt+aWF8CW6pZLLNOfYuF5OpfBSENuI8U= +github.com/google/go-cmp v0.5.2/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI= github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= github.com/google/go-configfs-tsm v0.2.2 h1:YnJ9rXIOj5BYD7/0DNnzs8AOp7UcvjfTvt215EWcs98= github.com/google/go-configfs-tsm v0.2.2/go.mod h1:EL1GTDFMb5PZQWDviGfZV9n87WeGTR/JUg13RfwkgRo= +github.com/google/go-github/v61 v61.0.0 h1:VwQCBwhyE9JclCI+22/7mLB1PuU9eowCXKY5pNlu1go= +github.com/google/go-github/v61 v61.0.0/go.mod h1:0WR+KmsWX75G2EbpyGsGmradjo3IiciuI4BmdVCobQY= +github.com/google/go-querystring v1.1.0 h1:AnCroh3fv4ZBgVIf1Iwtovgjaw/GiKJo8M8yD/fhyJ8= +github.com/google/go-querystring v1.1.0/go.mod h1:Kcdr2DB4koayq7X8pmAG4sNG59So17icRSOU623lUBU= github.com/google/go-sev-guest v0.11.1 h1:gnww4U8fHV5DCPz4gykr1s8SEX1fFNcxCBy+vvXN24k= github.com/google/go-sev-guest v0.11.1/go.mod h1:qBOfb+JmgsUI3aUyzQoGC13Kpp9zwLeWvuyXmA9q77w= github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= diff --git a/packages/by-name/contrast/package.nix b/packages/by-name/contrast/package.nix index 2a8b9e4c4..4855ccee2 100644 --- a/packages/by-name/contrast/package.nix +++ b/packages/by-name/contrast/package.nix @@ -23,7 +23,7 @@ let "-X github.com/edgelesssys/contrast/internal/kuberesource.runtimeHandler=${runtimeHandler}" ]; - subPackages = [ "e2e/openssl" "e2e/servicemesh" ]; + subPackages = [ "e2e/openssl" "e2e/servicemesh" "e2e/release" ]; }; launchDigest = builtins.readFile "${runtime-class-files}/launch-digest.hex"; @@ -61,7 +61,7 @@ buildGoModule rec { }; proxyVendor = true; - vendorHash = "sha256-i+7DhygotCNhczpaZlI9O7enKVOW7smauOKcGQhOtzI="; + vendorHash = "sha256-9Jn6xN/tRiFhSoNhyCACLLeqBBYNfNW3EhIhIEwoYPY="; subPackages = packageOutputs ++ [ "internal/kuberesource/resourcegen" ]; From fbae4b2a95cafe88af1befdc9d139b6fe2a21c0d Mon Sep 17 00:00:00 2001 From: Markus Rudy Date: Wed, 24 Apr 2024 16:04:42 +0200 Subject: [PATCH 2/4] release: add test to release workflow --- .github/workflows/release.yml | 40 +++++++++++++++++++++++++++++++++++ 1 file changed, 40 insertions(+) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index f9e715a21..942f8bc9a 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -20,6 +20,7 @@ on: env: container_registry: ghcr.io/edgelesssys + azure_resource_group: contrast-ci concurrency: group: ${{ github.ref }} @@ -266,6 +267,45 @@ jobs: version: ${{ needs.process-inputs.outputs.NEXT_PATCH_PRE_WITHOUT_V }} commit: true + test: + runs-on: ubuntu-22.04 + permissions: + # Job needs content:write to see draft releases. + contents: write + packages: read + needs: release + steps: + - uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1 + - uses: ./.github/actions/setup_nix + with: + githubToken: ${{ secrets.GITHUB_TOKEN }} + cachixToken: ${{ secrets.CACHIX_AUTH_TOKEN }} + - name: Log in to ghcr.io Container registry + uses: docker/login-action@e92390c5fb421da1463c202d546fed0ec5c39f20 # v3.1.0 + with: + registry: ghcr.io + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + - name: Login to Azure + uses: azure/login@6b2456866fc08b011acb422a92a4aa20e2c4de32 # v2.1.0 + with: + creds: ${{ secrets.CONTRAST_CI_INFRA_AZURE }} + - uses: nicknovitski/nix-develop@a2060d116a50b36dfab02280af558e73ab52427d # v1.1.0 + - name: Create justfile.env + run: | + cat < justfile.env + container_registry=${{ env.container_registry }} + azure_resource_group=${{ env.azure_resource_group }} + EOF + - name: Get credentials for CI cluster + run: | + just get-credentials + - name: E2E Test + env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + run: | + nix shell .#contrast.e2e --command release.test -test.v --tag ${{ inputs.version }} + create-github-stuff: name: Create backport label and milestone if: ${{ inputs.kind == 'minor' }} From 19908f14e2799a526f61a9edbd32730a91b0aef5 Mon Sep 17 00:00:00 2001 From: Markus Rudy Date: Fri, 3 May 2024 16:09:05 +0200 Subject: [PATCH 3/4] release: expose coordinator and demo --- .github/workflows/release.yml | 2 +- packages/scripts.nix | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 942f8bc9a..8a6c8c106 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -231,7 +231,7 @@ jobs: mkdir -p workspace nix run .#scripts.write-coordinator-yaml -- "${coordinatorImgTagged}" > workspace/coordinator.yml nix shell .#contrast --command resourcegen --namespace kube-system --image-replacements ./image-replacements.txt runtime > workspace/runtime.yml - nix shell .#contrast --command resourcegen --image-replacements ./image-replacements.txt emojivoto > deployment/emojivoto-demo.yml + nix shell .#contrast --command resourcegen --image-replacements ./image-replacements.txt --add-load-balancers emojivoto > deployment/emojivoto-demo.yml zip -r deployment/emojivoto-demo.zip deployment/emojivoto-demo.yml - name: Update coordinator policy hash run: | diff --git a/packages/scripts.nix b/packages/scripts.nix index 048230662..6016396f0 100644 --- a/packages/scripts.nix +++ b/packages/scripts.nix @@ -200,12 +200,12 @@ with pkgs; trap 'rm -rf $tmpdir' EXIT echo "ghcr.io/edgelesssys/contrast/coordinator:latest=$imageRef" > "$tmpdir/image-replacements.txt" - resourcegen --image-replacements "$tmpdir/image-replacements.txt" coordinator-release > "$tmpdir/coordinator_base.yml" + resourcegen --image-replacements "$tmpdir/image-replacements.txt" --add-load-balancers coordinator-release > "$tmpdir/coordinator_base.yml" pushd "$tmpdir" >/dev/null cp ${genpolicy-msft.rules-coordinator}/genpolicy-rules.rego rules.rego cp ${genpolicy-msft.settings}/genpolicy-settings.json . - genpolicy < "$tmpdir/coordinator.yml" + genpolicy < "$tmpdir/coordinator_base.yml" popd >/dev/null ''; }; From e4e841190014dfd906b0f441a0796bfd1d9fc23f Mon Sep 17 00:00:00 2001 From: Markus Rudy Date: Fri, 3 May 2024 16:37:33 +0200 Subject: [PATCH 4/4] release: fix paths --- .github/workflows/release.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 8a6c8c106..6d253760b 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -228,11 +228,11 @@ jobs: echo "ghcr.io/edgelesssys/contrast/node-installer:latest=$nodeInstallerImgTagged" >> image-replacements.txt - name: Create portable coordinator resource definitions run: | - mkdir -p workspace + mkdir -p workspace deployment nix run .#scripts.write-coordinator-yaml -- "${coordinatorImgTagged}" > workspace/coordinator.yml nix shell .#contrast --command resourcegen --namespace kube-system --image-replacements ./image-replacements.txt runtime > workspace/runtime.yml nix shell .#contrast --command resourcegen --image-replacements ./image-replacements.txt --add-load-balancers emojivoto > deployment/emojivoto-demo.yml - zip -r deployment/emojivoto-demo.zip deployment/emojivoto-demo.yml + zip -r workspace/emojivoto-demo.zip deployment/emojivoto-demo.yml - name: Update coordinator policy hash run: | yq < workspace/coordinator.yml \ @@ -257,7 +257,7 @@ jobs: result-cli/bin/contrast workspace/coordinator.yml workspace/runtime.yml - deployment/emojivoto-demo.zip + workspace/emojivoto-demo.zip - name: Reset temporary changes run: | git reset --hard ${{ needs.process-inputs.outputs.WORKING_BRANCH }}