test upgrade

.github/workflows/e2e-test-provider-example.yml

@@ -242,15 +242,6 @@ jobs:
         run: |
           cp ${{ github.workspace }}/terraform-provider-constellation/examples/full/${{ inputs.cloudProvider }}/main.tf ${{ github.workspace }}/cluster/main.tf
 
-      - name: Apply Terraform Cluster
-        id: apply_terraform
-        if: inputs.cloudProvider != 'azure'
-        working-directory: ${{ github.workspace }}/cluster
-        shell: bash
-        run: |
-          terraform init
-          TF_LOG=INFO terraform apply -auto-approve
-
       - name: Download CLI
         if: inputs.cloudProvider == 'azure'
         shell: bash
@@ -260,6 +251,15 @@ jobs:
           ./constellation version
           cp ./constellation ${{ github.workspace }}/cluster
 
+      - name: Apply Terraform Cluster
+        id: apply_terraform
+        if: inputs.cloudProvider != 'azure'
+        working-directory: ${{ github.workspace }}/cluster
+        shell: bash
+        run: |
+          terraform init
+          TF_LOG=INFO terraform apply -auto-approve
+
       - name: Apply Azure Terraform Cluster
         id: apply_azure_terraform
         if: inputs.cloudProvider == 'azure'
@@ -272,6 +272,45 @@ jobs:
           ./constellation maa-patch $(terraform output -raw maa_url)
           TF_LOG=INFO terraform apply -target constellation_cluster.azure_example -auto-approve
 
+      - name: Upgrade cluster
+        working-directory: ${{ github.workspace }}/cluster
+        shell: bash
+        run: |
+          cat >> _override.tf <<EOF
+          locals {
+            version = "v2.14.0"
+          }
+          EOF
+          # TODO change provider version (once registry is available)
+          # TODO rename version to image_version
+
+      - name: Apply Terraform Cluster
+        working-directory: ${{ github.workspace }}/cluster
+        shell: bash
+        run: |
+          terraform init
+          TF_LOG=INFO terraform apply -auto-approve
+
+      - name: Assert upgrade successful
+        # ${{ inputs.toImage && inputs.toImage || steps.find-image.outputs.output }}
+        env:
+          IMAGE: v2.14.0
+          #KUBERNETES: ${{ inputs.toKubernetes }}
+          #MICROSERVICES: ${{ inputs.toMicroservices }}
+          #WORKERNODES:  ${{ needs.split-nodeCount.outputs.workerNodes }}
+          #CONTROLNODES: ${{ needs.split-nodeCount.outputs.controlPlaneNodes }}
+        run: |
+          terraform output -raw kubeconfig > constellation-admin.conf
+          export KUBECONFIG=$(realpath constellation-admin.conf)
+
+          if [[ -n ${MICROSERVICES} ]]; then
+          MICROSERVICES_FLAG="--target-microservices=$MICROSERVICES"
+          fi
+          if [[ -n ${KUBERNETES} ]]; then
+            KUBERNETES_FLAG="--target-kubernetes=$KUBERNETES"
+          fi
+          bazel run //e2e/provider-upgrade:provider-upgrade_test -- --want-worker 2 --want-control 2 --target-image "$IMAGE" "$KUBERNETES_FLAG" "$MICROSERVICES_FLAG"
+
       - name: Destroy Terraform Cluster
       # outcome is part of the steps context (https://docs.github.com/en/actions/learn-github-actions/contexts#steps-context)
         if: always() && (steps.apply_terraform.outcome != 'skipped' || steps.apply_azure_terraform.outcome != 'skipped')

e2e/internal/upgrade/BUILD.bazel

@@ -13,6 +13,10 @@ go_library(
         "//internal/constants",
         "//internal/logger",
         "//internal/semver",
+        "//internal/versions",
+        "@com_github_stretchr_testify//require",
+        "@io_k8s_apimachinery//pkg/apis/meta/v1:meta",
+        "@io_k8s_client_go//kubernetes",
         "@sh_helm_helm_v3//pkg/action",
         "@sh_helm_helm_v3//pkg/cli",
     ],

e2e/internal/upgrade/upgrade.go

@@ -17,3 +17,168 @@ SPDX-License-Identifier: AGPL-3.0-only
 //
 //   - set or fetch measurements depending on target image
 package upgrade
+
+import (
+	"bufio"
+	"context"
+	"fmt"
+	"io"
+	"log"
+	"os/exec"
+	"strings"
+	"sync"
+	"testing"
+	"time"
+
+	"github.com/edgelesssys/constellation/v2/internal/semver"
+	"github.com/edgelesssys/constellation/v2/internal/versions"
+	"github.com/stretchr/testify/require"
+	metaV1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/client-go/kubernetes"
+)
+
+type VersionContainer struct {
+	ImageRef      string
+	Kubernetes    versions.ValidK8sVersion
+	Microservices semver.Semver
+}
+
+func AssertUpgradeSuccessful(t *testing.T, cli string, targetVersions VersionContainer, k *kubernetes.Clientset, wantControl, wantWorker int, timeout time.Duration) {
+	wg := queryStatusAsync(t, cli)
+	require.NotNil(t, k)
+	testMicroservicesEventuallyHaveVersion(t, targetVersions.Microservices, timeout)
+	testNodesEventuallyHaveVersion(t, k, targetVersions, wantControl+wantWorker, timeout)
+
+	wg.Wait()
+}
+
+func queryStatusAsync(t *testing.T, cli string) *sync.WaitGroup {
+	var wg sync.WaitGroup
+	wg.Add(1)
+	go func() {
+		defer wg.Done()
+		// The first control plane node should finish upgrading after 20 minutes. If it does not, something is fishy.
+		// Nodes can upgrade in <5mins.
+		testStatusEventuallyWorks(t, cli, 20*time.Minute)
+	}()
+
+	return &wg
+}
+
+func testStatusEventuallyWorks(t *testing.T, cli string, timeout time.Duration) {
+	require.Eventually(t, func() bool {
+		// Show versions set in cluster.
+		// The string after "Cluster status:" in the output might not be updated yet.
+		// This is only updated after the operator finishes one reconcile loop.
+		cmd := exec.CommandContext(context.Background(), cli, "status")
+		stdout, stderr, err := runCommandWithSeparateOutputs(cmd)
+		if err != nil {
+			log.Printf("Stdout: %s\nStderr: %s", string(stdout), string(stderr))
+			return false
+		}
+
+		log.Println(string(stdout))
+		return true
+	}, timeout, time.Minute)
+}
+
+func testMicroservicesEventuallyHaveVersion(t *testing.T, wantMicroserviceVersion semver.Semver, timeout time.Duration) {
+	require.Eventually(t, func() bool {
+		version, err := servicesVersion(t)
+		if err != nil {
+			log.Printf("Unable to fetch microservice version: %v\n", err)
+			return false
+		}
+
+		if version != wantMicroserviceVersion {
+			log.Printf("Microservices still at version %v, want %v\n", version, wantMicroserviceVersion)
+			return false
+		}
+
+		return true
+	}, timeout, time.Minute)
+}
+
+func testNodesEventuallyHaveVersion(t *testing.T, k *kubernetes.Clientset, targetVersions VersionContainer, totalNodeCount int, timeout time.Duration) {
+	require.Eventually(t, func() bool {
+		nodes, err := k.CoreV1().Nodes().List(context.Background(), metaV1.ListOptions{})
+		if err != nil {
+			log.Println(err)
+			return false
+		}
+		require.False(t, len(nodes.Items) < totalNodeCount, "expected at least %v nodes, got %v", totalNodeCount, len(nodes.Items))
+
+		allUpdated := true
+		log.Printf("Node status (%v):", time.Now())
+		for _, node := range nodes.Items {
+			for key, value := range node.Annotations {
+				if key == "constellation.edgeless.systems/node-image" {
+					if !strings.EqualFold(value, targetVersions.ImageRef) {
+						log.Printf("\t%s: Image %s, want %s\n", node.Name, value, targetVersions.ImageRef)
+						allUpdated = false
+					}
+				}
+			}
+
+			kubeletVersion := node.Status.NodeInfo.KubeletVersion
+			if kubeletVersion != string(targetVersions.Kubernetes) {
+				log.Printf("\t%s: K8s (Kubelet) %s, want %s\n", node.Name, kubeletVersion, targetVersions.Kubernetes)
+				allUpdated = false
+			}
+			kubeProxyVersion := node.Status.NodeInfo.KubeProxyVersion
+			if kubeProxyVersion != string(targetVersions.Kubernetes) {
+				log.Printf("\t%s: K8s (Proxy) %s, want %s\n", node.Name, kubeProxyVersion, targetVersions.Kubernetes)
+				allUpdated = false
+			}
+		}
+
+		return allUpdated
+	}, timeout, time.Minute)
+}
+
+// runCommandWithSeparateOutputs runs the given command while separating buffers for
+// stdout and stderr.
+func runCommandWithSeparateOutputs(cmd *exec.Cmd) (stdout, stderr []byte, err error) {
+	stdout = []byte{}
+	stderr = []byte{}
+
+	stdoutIn, err := cmd.StdoutPipe()
+	if err != nil {
+		err = fmt.Errorf("create stdout pipe: %w", err)
+		return
+	}
+	stderrIn, err := cmd.StderrPipe()
+	if err != nil {
+		err = fmt.Errorf("create stderr pipe: %w", err)
+		return
+	}
+
+	err = cmd.Start()
+	if err != nil {
+		err = fmt.Errorf("start command: %w", err)
+		return
+	}
+
+	continuouslyPrintOutput := func(r io.Reader, prefix string) {
+		scanner := bufio.NewScanner(r)
+		for scanner.Scan() {
+			output := scanner.Text()
+			fmt.Printf("%s: %s\n", prefix, output)
+			switch prefix {
+			case "stdout":
+				stdout = append(stdout, output...)
+			case "stderr":
+				stderr = append(stderr, output...)
+			}
+		}
+	}
+
+	go continuouslyPrintOutput(stdoutIn, "stdout")
+	go continuouslyPrintOutput(stderrIn, "stderr")
+
+	if err = cmd.Wait(); err != nil {
+		err = fmt.Errorf("wait for command to finish: %w", err)
+	}
+
+	return stdout, stderr, err
+}