e2e: add test for release candidates

.github/workflows/release.yml

@@ -20,6 +20,7 @@ on:
 
 env:
   container_registry: ghcr.io/edgelesssys
+  azure_resource_group: contrast-ci
 
 concurrency:
   group: ${{ github.ref }}
@@ -227,11 +228,11 @@ jobs:
           echo "ghcr.io/edgelesssys/contrast/node-installer:latest=$nodeInstallerImgTagged" >> image-replacements.txt
       - name: Create portable coordinator resource definitions
         run: |
-          mkdir -p workspace
+          mkdir -p workspace deployment
           nix run .#scripts.write-coordinator-yaml -- "${coordinatorImgTagged}" > workspace/coordinator.yml
           nix shell .#contrast --command resourcegen --namespace kube-system --image-replacements ./image-replacements.txt runtime > workspace/runtime.yml
-          nix shell .#contrast --command resourcegen --image-replacements ./image-replacements.txt emojivoto > deployment/emojivoto-demo.yml
-          zip -r deployment/emojivoto-demo.zip deployment/emojivoto-demo.yml
+          nix shell .#contrast --command resourcegen --image-replacements ./image-replacements.txt --add-load-balancers emojivoto > deployment/emojivoto-demo.yml
+          zip -r workspace/emojivoto-demo.zip deployment/emojivoto-demo.yml
       - name: Update coordinator policy hash
         run: |
           yq < workspace/coordinator.yml \
@@ -256,7 +257,7 @@ jobs:
             result-cli/bin/contrast
             workspace/coordinator.yml
             workspace/runtime.yml
-            deployment/emojivoto-demo.zip
+            workspace/emojivoto-demo.zip
       - name: Reset temporary changes
         run: |
           git reset --hard ${{ needs.process-inputs.outputs.WORKING_BRANCH }}
@@ -266,6 +267,45 @@ jobs:
           version: ${{ needs.process-inputs.outputs.NEXT_PATCH_PRE_WITHOUT_V }}
           commit: true
 
+  test:
+    runs-on: ubuntu-22.04
+    permissions:
+      # Job needs content:write to see draft releases.
+      contents: write
+      packages: read
+    needs: release
+    steps:
+      - uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1
+      - uses: ./.github/actions/setup_nix
+        with:
+          githubToken: ${{ secrets.GITHUB_TOKEN }}
+          cachixToken: ${{ secrets.CACHIX_AUTH_TOKEN }}
+      - name: Log in to ghcr.io Container registry
+        uses: docker/login-action@e92390c5fb421da1463c202d546fed0ec5c39f20 # v3.1.0
+        with:
+          registry: ghcr.io
+          username: ${{ github.actor }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+      - name: Login to Azure
+        uses: azure/login@6b2456866fc08b011acb422a92a4aa20e2c4de32 # v2.1.0
+        with:
+          creds: ${{ secrets.CONTRAST_CI_INFRA_AZURE }}
+      - uses: nicknovitski/nix-develop@a2060d116a50b36dfab02280af558e73ab52427d # v1.1.0
+      - name: Create justfile.env
+        run: |
+          cat <<EOF > justfile.env
+          container_registry=${{ env.container_registry }}
+          azure_resource_group=${{ env.azure_resource_group }}
+          EOF
+      - name: Get credentials for CI cluster
+        run: |
+          just get-credentials
+      - name: E2E Test
+        env:
+          GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+        run: |
+          nix shell .#contrast.e2e --command release.test -test.v --tag ${{ inputs.version }}
+
   create-github-stuff:
     name: Create backport label and milestone
     if: ${{ inputs.kind == 'minor' }}

e2e/internal/kubeclient/deploy.go

@@ -7,7 +7,9 @@ import (
 	"context"
 	"encoding/json"
 	"fmt"
+	"net"
 	"sort"
+	"strconv"
 	"time"
 
 	appsv1 "k8s.io/api/apps/v1"
@@ -150,6 +152,63 @@ func (c *Kubeclient) WaitForDaemonset(ctx context.Context, namespace, name strin
 	}
 }
 
+// WaitForLoadBalancer waits until the given service is configured with an external IP and returns it.
+func (c *Kubeclient) WaitForLoadBalancer(ctx context.Context, namespace, name string) (string, error) {
+	watcher, err := c.client.CoreV1().Services(namespace).Watch(ctx, metav1.ListOptions{FieldSelector: "metadata.name=" + name})
+	if err != nil {
+		return "", err
+	}
+	var ip string
+	var port int
+loop:
+	for {
+		select {
+		case evt := <-watcher.ResultChan():
+			switch evt.Type {
+			case watch.Added:
+				fallthrough
+			case watch.Modified:
+				svc, ok := evt.Object.(*corev1.Service)
+				if !ok {
+					return "", fmt.Errorf("watcher received unexpected type %T", evt.Object)
+				}
+				for _, ingress := range svc.Status.LoadBalancer.Ingress {
+					if ingress.IP != "" {
+						ip = ingress.IP
+						// TODO(burgerdev): deal with more than one port, and protocols other than TCP
+						port = int(svc.Spec.Ports[0].Port)
+						break loop
+					}
+				}
+			case watch.Deleted:
+				return "", fmt.Errorf("service %s/%s was deleted while waiting for it", namespace, name)
+			default:
+				c.log.Warn("ignoring unexpected watch event", "type", evt.Type, "object", evt.Object)
+			}
+		case <-ctx.Done():
+			return "", fmt.Errorf("LoadBalancer %s/%s did not get a public IP before %w", namespace, name, ctx.Err())
+		}
+	}
+
+	ticker := time.NewTicker(time.Second)
+	defer ticker.Stop()
+
+	dialer := &net.Dialer{}
+	for {
+		select {
+		case <-ticker.C:
+			conn, err := dialer.DialContext(ctx, "tcp", net.JoinHostPort(ip, strconv.Itoa(port)))
+			if err == nil {
+				conn.Close()
+				return ip, nil
+			}
+			c.log.Info("probe failed", "namespace", namespace, "name", name, "error", err)
+		case <-ctx.Done():
+			return "", fmt.Errorf("LoadBalancer %s/%s never responded to probing before %w", namespace, name, ctx.Err())
+		}
+	}
+}
+
 func (c *Kubeclient) toJSON(a any) string {
 	s, err := json.Marshal(a)
 	if err != nil {