diff --git a/.github/workflows/e2e-test-provider-example.yml b/.github/workflows/e2e-test-provider-example.yml index 47bacc3b79b..d3a30a2bdca 100644 --- a/.github/workflows/e2e-test-provider-example.yml +++ b/.github/workflows/e2e-test-provider-example.yml @@ -23,6 +23,18 @@ on: providerVersion: description: "Constellation Terraform provider version to use (without v prefix). Empty value means build from source." type: string + toImage: + description: Image (shortpath) the cluster is upgraded to, or empty for main/nightly. + type: string + required: false + toKubernetes: + description: Kubernetes version to target for the upgrade, empty for no upgrade. + type: string + required: false + toProviderVersion: + description: Provider version and microservice version to target for the upgrade, empty for no upgrade. + type: string + required: false workflow_call: inputs: ref: @@ -41,6 +53,18 @@ on: providerVersion: description: "Constellation Terraform provider version to use. Empty value means build from source." type: string + toImage: + description: Image (shortpath) the cluster is upgraded to, or empty for main/nightly. + type: string + required: false + toKubernetes: + description: Kubernetes version to target for the upgrade, empty for target's default version. + type: string + required: false + toProviderVersion: + description: Provider version and microservice version to target for the upgrade, empty for no upgrade. + type: string + required: false jobs: provider-example-test: @@ -237,7 +261,6 @@ jobs: cp ${{ github.workspace }}/terraform-provider-constellation/examples/full/${{ inputs.cloudProvider }}/main.tf ${{ github.workspace }}/cluster/main.tf - name: Download CLI - if: inputs.cloudProvider == 'azure' shell: bash run: | curl -fsSL -o constellation https://github.com/edgelesssys/constellation/releases/download/v2.13.0/constellation-linux-amd64 @@ -260,6 +283,70 @@ jobs: TF_LOG=INFO terraform apply -auto-approve fi + - name: Update cluster configuration + working-directory: ${{ github.workspace }}/cluster + shell: bash + run: | + if [[ "${{ inputs.toImage }}" != "" ]]; then + cat >> _override.tf <> _override.tf < constellation-admin.conf + + if [[ -n ${MICROSERVICES} ]]; then + MICROSERVICES_FLAG="--target-microservices=$MICROSERVICES" + fi + if [[ -n ${KUBERNETES} ]]; then + KUBERNETES_FLAG="--target-kubernetes=$KUBERNETES" + fi + if [[ -n ${IMAGE} ]]; then + IMAGE_FLAG="--target-image=$IMAGE" + fi + + # cfg must be in same dir as KUBECONFIG + ./constellation config generate ${{ inputs.cloudProvider }} + # make cfg valid + yq e '.provider.azure.subscription = "123e4567-e89b-12d3-a456-426614174000"' -i constellation-conf.yaml + yq e '.provider.azure.tenant = "123e4567-e89b-12d3-a456-426614174001"' -i constellation-conf.yaml + yq e '.provider.azure.location = "eastus"' -i constellation-conf.yaml + yq e '.provider.azure.resourceGroup = "myResourceGroup"' -i constellation-conf.yaml + yq e '.provider.azure.userAssignedIdentity = "myIdentity"' -i constellation-conf.yaml + + KUBECONFIG=${{ github.workspace }}/cluster/constellation-admin.conf bazel run //e2e/provider-upgrade:provider-upgrade_test -- --want-worker $WORKERNODES --want-control $CONTROLNODES --cli ${{ github.workspace }}/cluster/constellation $IMAGE_FLAG $KUBERNETES_FLAG $MICROSERVICES_FLAG + - name: Destroy Terraform Cluster # outcome is part of the steps context (https://docs.github.com/en/actions/learn-github-actions/contexts#steps-context) if: always() && steps.apply_terraform.outcome != 'skipped' diff --git a/e2e/internal/upgrade/BUILD.bazel b/e2e/internal/upgrade/BUILD.bazel index 2d16064bdb8..86e934e4c88 100644 --- a/e2e/internal/upgrade/BUILD.bazel +++ b/e2e/internal/upgrade/BUILD.bazel @@ -13,6 +13,11 @@ go_library( "//internal/constants", "//internal/logger", "//internal/semver", + "//internal/versions", + "@com_github_stretchr_testify//require", + "@io_bazel_rules_go//go/runfiles:go_default_library", + "@io_k8s_apimachinery//pkg/apis/meta/v1:meta", + "@io_k8s_client_go//kubernetes", "@sh_helm_helm_v3//pkg/action", "@sh_helm_helm_v3//pkg/cli", ], @@ -44,7 +49,6 @@ go_test( "//internal/versions", "@com_github_spf13_afero//:afero", "@com_github_stretchr_testify//require", - "@io_bazel_rules_go//go/runfiles:go_default_library", "@io_k8s_api//core/v1:core", "@io_k8s_apimachinery//pkg/apis/meta/v1:meta", "@io_k8s_client_go//kubernetes", diff --git a/e2e/internal/upgrade/upgrade.go b/e2e/internal/upgrade/upgrade.go index 6fa50365502..819ec28e928 100644 --- a/e2e/internal/upgrade/upgrade.go +++ b/e2e/internal/upgrade/upgrade.go @@ -1,3 +1,5 @@ +//go:build e2e + /* Copyright (c) Edgeless Systems GmbH @@ -17,3 +19,239 @@ SPDX-License-Identifier: AGPL-3.0-only // // - set or fetch measurements depending on target image package upgrade + +import ( + "bufio" + "context" + "errors" + "fmt" + "io" + "log" + "os" + "os/exec" + "path/filepath" + "strings" + "sync" + "testing" + "time" + + "github.com/bazelbuild/rules_go/go/runfiles" + "github.com/edgelesssys/constellation/v2/internal/semver" + "github.com/edgelesssys/constellation/v2/internal/versions" + "github.com/stretchr/testify/require" + metaV1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/client-go/kubernetes" +) + +// VersionContainer contains the versions that the cluster should be upgraded to. +type VersionContainer struct { + ImageRef string + Kubernetes versions.ValidK8sVersion + Microservices semver.Semver +} + +// AssertUpgradeSuccessful tests that the upgrade to the target version is successful. +func AssertUpgradeSuccessful(t *testing.T, cli string, targetVersions VersionContainer, k *kubernetes.Clientset, wantControl, wantWorker int, timeout time.Duration) { + wg := queryStatusAsync(t, cli) + require.NotNil(t, k) + testMicroservicesEventuallyHaveVersion(t, targetVersions.Microservices, timeout) + testNodesEventuallyHaveVersion(t, k, targetVersions, wantControl+wantWorker, timeout) + + wg.Wait() +} + +func queryStatusAsync(t *testing.T, cli string) *sync.WaitGroup { + var wg sync.WaitGroup + wg.Add(1) + go func() { + defer wg.Done() + // The first control plane node should finish upgrading after 20 minutes. If it does not, something is fishy. + // Nodes can upgrade in <5mins. + testStatusEventuallyWorks(t, cli, 20*time.Minute) + }() + + return &wg +} + +func testStatusEventuallyWorks(t *testing.T, cli string, timeout time.Duration) { + require.Eventually(t, func() bool { + // Show versions set in cluster. + // The string after "Cluster status:" in the output might not be updated yet. + // This is only updated after the operator finishes one reconcile loop. + cmd := exec.CommandContext(context.Background(), cli, "status") + stdout, stderr, err := runCommandWithSeparateOutputs(cmd) + if err != nil { + log.Printf("Stdout: %s\nStderr: %s", string(stdout), string(stderr)) + return false + } + + log.Println(string(stdout)) + return true + }, timeout, time.Minute) +} + +func testMicroservicesEventuallyHaveVersion(t *testing.T, wantMicroserviceVersion semver.Semver, timeout time.Duration) { + require.Eventually(t, func() bool { + version, err := servicesVersion(t) + if err != nil { + log.Printf("Unable to fetch microservice version: %v\n", err) + return false + } + + if version != wantMicroserviceVersion { + log.Printf("Microservices still at version %v, want %v\n", version, wantMicroserviceVersion) + return false + } + + return true + }, timeout, time.Minute) +} + +func testNodesEventuallyHaveVersion(t *testing.T, k *kubernetes.Clientset, targetVersions VersionContainer, totalNodeCount int, timeout time.Duration) { + require.Eventually(t, func() bool { + nodes, err := k.CoreV1().Nodes().List(context.Background(), metaV1.ListOptions{}) + if err != nil { + log.Println(err) + return false + } + require.False(t, len(nodes.Items) < totalNodeCount, "expected at least %v nodes, got %v", totalNodeCount, len(nodes.Items)) + + allUpdated := true + log.Printf("Node status (%v):", time.Now()) + for _, node := range nodes.Items { + for key, value := range node.Annotations { + if key == "constellation.edgeless.systems/node-image" { + if !strings.EqualFold(value, targetVersions.ImageRef) { + log.Printf("\t%s: Image %s, want %s\n", node.Name, value, targetVersions.ImageRef) + allUpdated = false + } + } + } + + kubeletVersion := node.Status.NodeInfo.KubeletVersion + if kubeletVersion != string(targetVersions.Kubernetes) { + log.Printf("\t%s: K8s (Kubelet) %s, want %s\n", node.Name, kubeletVersion, targetVersions.Kubernetes) + allUpdated = false + } + kubeProxyVersion := node.Status.NodeInfo.KubeProxyVersion + if kubeProxyVersion != string(targetVersions.Kubernetes) { + log.Printf("\t%s: K8s (Proxy) %s, want %s\n", node.Name, kubeProxyVersion, targetVersions.Kubernetes) + allUpdated = false + } + } + + return allUpdated + }, timeout, time.Minute) +} + +// runCommandWithSeparateOutputs runs the given command while separating buffers for +// stdout and stderr. +func runCommandWithSeparateOutputs(cmd *exec.Cmd) (stdout, stderr []byte, err error) { + stdout = []byte{} + stderr = []byte{} + + stdoutIn, err := cmd.StdoutPipe() + if err != nil { + err = fmt.Errorf("create stdout pipe: %w", err) + return + } + stderrIn, err := cmd.StderrPipe() + if err != nil { + err = fmt.Errorf("create stderr pipe: %w", err) + return + } + + err = cmd.Start() + if err != nil { + err = fmt.Errorf("start command: %w", err) + return + } + + continuouslyPrintOutput := func(r io.Reader, prefix string) { + scanner := bufio.NewScanner(r) + for scanner.Scan() { + output := scanner.Text() + fmt.Printf("%s: %s\n", prefix, output) + switch prefix { + case "stdout": + stdout = append(stdout, output...) + case "stderr": + stderr = append(stderr, output...) + } + } + } + + go continuouslyPrintOutput(stdoutIn, "stdout") + go continuouslyPrintOutput(stderrIn, "stderr") + + if err = cmd.Wait(); err != nil { + err = fmt.Errorf("wait for command to finish: %w", err) + } + + return stdout, stderr, err +} + +// Setup checks that the prerequisites for the test are met: +// - a workspace is set +// - a CLI path is set +// - the constellation-upgrade folder does not exist. +func Setup(workspace, cliPath string) error { + workingDir, err := workingDir(workspace) + if err != nil { + return fmt.Errorf("getting working directory: %w", err) + } + + if err := os.Chdir(workingDir); err != nil { + return fmt.Errorf("changing working directory: %w", err) + } + + if _, err := getCLIPath(cliPath); err != nil { + return fmt.Errorf("getting CLI path: %w", err) + } + return nil +} + +// workingDir returns the path to the workspace. +func workingDir(workspace string) (string, error) { + workingDir := os.Getenv("BUILD_WORKING_DIRECTORY") + switch { + case workingDir != "": + return workingDir, nil + case workspace != "": + return workspace, nil + default: + return "", errors.New("neither 'BUILD_WORKING_DIRECTORY' nor 'workspace' flag set") + } +} + +// getCLIPath returns the path to the CLI. +func getCLIPath(cliPathFlag string) (string, error) { + pathCLI := os.Getenv("PATH_CLI") + var relCLIPath string + switch { + case pathCLI != "": + relCLIPath = pathCLI + case cliPathFlag != "": + relCLIPath = cliPathFlag + default: + return "", errors.New("neither 'PATH_CLI' nor 'cli' flag set") + } + + // try to find the CLI in the working directory + // (e.g. when running via `go test` or when specifying a path manually) + workdir, err := os.Getwd() + if err != nil { + return "", fmt.Errorf("getting working directory: %w", err) + } + + absCLIPath := relCLIPath + if !filepath.IsAbs(relCLIPath) { + absCLIPath = filepath.Join(workdir, relCLIPath) + } + if _, err := os.Stat(absCLIPath); err == nil { + return absCLIPath, nil + } + + // fall back to runfiles (e.g. when running via bazel) + return runfiles.Rlocation(pathCLI) +} diff --git a/e2e/internal/upgrade/upgrade_test.go b/e2e/internal/upgrade/upgrade_test.go index bf9da20e2b0..7334c7a8734 100644 --- a/e2e/internal/upgrade/upgrade_test.go +++ b/e2e/internal/upgrade/upgrade_test.go @@ -9,22 +9,17 @@ SPDX-License-Identifier: AGPL-3.0-only package upgrade import ( - "bufio" "context" "errors" "flag" "fmt" - "io" "log" "os" "os/exec" - "path/filepath" "strings" - "sync" "testing" "time" - "github.com/bazelbuild/rules_go/go/runfiles" "github.com/edgelesssys/constellation/v2/e2e/internal/kubectl" "github.com/edgelesssys/constellation/v2/internal/api/attestationconfigapi" "github.com/edgelesssys/constellation/v2/internal/config" @@ -63,7 +58,7 @@ var ( func TestUpgrade(t *testing.T) { require := require.New(t) - err := setup() + err := Setup(*workspace, *cliPath) require.NoError(err) k, err := kubectl.New() @@ -97,77 +92,7 @@ func TestUpgrade(t *testing.T) { log.Println("Triggering upgrade.") runUpgradeApply(require, cli) - wg := queryStatusAsync(t, cli) - - testMicroservicesEventuallyHaveVersion(t, targetVersions.microservices, *timeout) - testNodesEventuallyHaveVersion(t, k, targetVersions, *wantControl+*wantWorker, *timeout) - - wg.Wait() -} - -// setup checks that the prerequisites for the test are met: -// - a workspace is set -// - a CLI path is set -// - the constellation-upgrade folder does not exist. -func setup() error { - workingDir, err := workingDir(*workspace) - if err != nil { - return fmt.Errorf("getting working directory: %w", err) - } - - if err := os.Chdir(workingDir); err != nil { - return fmt.Errorf("changing working directory: %w", err) - } - - if _, err := getCLIPath(*cliPath); err != nil { - return fmt.Errorf("getting CLI path: %w", err) - } - return nil -} - -// workingDir returns the path to the workspace. -func workingDir(workspace string) (string, error) { - workingDir := os.Getenv("BUILD_WORKING_DIRECTORY") - switch { - case workingDir != "": - return workingDir, nil - case workspace != "": - return workspace, nil - default: - return "", errors.New("neither 'BUILD_WORKING_DIRECTORY' nor 'workspace' flag set") - } -} - -// getCLIPath returns the path to the CLI. -func getCLIPath(cliPathFlag string) (string, error) { - pathCLI := os.Getenv("PATH_CLI") - var relCLIPath string - switch { - case pathCLI != "": - relCLIPath = pathCLI - case cliPathFlag != "": - relCLIPath = cliPathFlag - default: - return "", errors.New("neither 'PATH_CLI' nor 'cli' flag set") - } - - // try to find the CLI in the working directory - // (e.g. when running via `go test` or when specifying a path manually) - workdir, err := os.Getwd() - if err != nil { - return "", fmt.Errorf("getting working directory: %w", err) - } - - absCLIPath := relCLIPath - if !filepath.IsAbs(relCLIPath) { - absCLIPath = filepath.Join(workdir, relCLIPath) - } - if _, err := os.Stat(absCLIPath); err == nil { - return absCLIPath, nil - } - - // fall back to runfiles (e.g. when running via bazel) - return runfiles.Rlocation(pathCLI) + AssertUpgradeSuccessful(t, cli, targetVersions, k, *wantControl, *wantWorker, *timeout) } // testPodsEventuallyReady checks that: @@ -249,7 +174,7 @@ func testNodesEventuallyAvailable(t *testing.T, k *kubernetes.Clientset, wantCon }, time.Minute*30, time.Minute) } -func writeUpgradeConfig(require *require.Assertions, image string, kubernetes string, microservices string) versionContainer { +func writeUpgradeConfig(require *require.Assertions, image string, kubernetes string, microservices string) VersionContainer { fileHandler := file.NewHandler(afero.NewOsFs()) attestationFetcher := attestationconfigapi.NewFetcher() cfg, err := config.New(fileHandler, constants.ConfigFilename, attestationFetcher, true) @@ -298,7 +223,7 @@ func writeUpgradeConfig(require *require.Assertions, image string, kubernetes st err = fileHandler.WriteYAML(constants.ConfigFilename, cfg, file.OptOverwrite) require.NoError(err) - return versionContainer{imageRef: imageRef, kubernetes: kubernetesVersion, microservices: microserviceVersion} + return VersionContainer{ImageRef: imageRef, Kubernetes: kubernetesVersion, Microservices: microserviceVersion} } // runUpgradeCheck executes 'upgrade check' and does basic checks on the output. @@ -361,140 +286,3 @@ func containsUnexepectedMsg(input string) error { } return nil } - -func queryStatusAsync(t *testing.T, cli string) *sync.WaitGroup { - var wg sync.WaitGroup - wg.Add(1) - go func() { - defer wg.Done() - // The first control plane node should finish upgrading after 20 minutes. If it does not, something is fishy. - // Nodes can upgrade in <5mins. - testStatusEventuallyWorks(t, cli, 20*time.Minute) - }() - - return &wg -} - -func testStatusEventuallyWorks(t *testing.T, cli string, timeout time.Duration) { - require.Eventually(t, func() bool { - // Show versions set in cluster. - // The string after "Cluster status:" in the output might not be updated yet. - // This is only updated after the operator finishes one reconcile loop. - cmd := exec.CommandContext(context.Background(), cli, "status") - stdout, stderr, err := runCommandWithSeparateOutputs(cmd) - if err != nil { - log.Printf("Stdout: %s\nStderr: %s", string(stdout), string(stderr)) - return false - } - - log.Println(string(stdout)) - return true - }, timeout, time.Minute) -} - -func testMicroservicesEventuallyHaveVersion(t *testing.T, wantMicroserviceVersion semver.Semver, timeout time.Duration) { - require.Eventually(t, func() bool { - version, err := servicesVersion(t) - if err != nil { - log.Printf("Unable to fetch microservice version: %v\n", err) - return false - } - - if version != wantMicroserviceVersion { - log.Printf("Microservices still at version %v, want %v\n", version, wantMicroserviceVersion) - return false - } - - return true - }, timeout, time.Minute) -} - -func testNodesEventuallyHaveVersion(t *testing.T, k *kubernetes.Clientset, targetVersions versionContainer, totalNodeCount int, timeout time.Duration) { - require.Eventually(t, func() bool { - nodes, err := k.CoreV1().Nodes().List(context.Background(), metaV1.ListOptions{}) - if err != nil { - log.Println(err) - return false - } - require.False(t, len(nodes.Items) < totalNodeCount, "expected at least %v nodes, got %v", totalNodeCount, len(nodes.Items)) - - allUpdated := true - log.Printf("Node status (%v):", time.Now()) - for _, node := range nodes.Items { - for key, value := range node.Annotations { - if key == "constellation.edgeless.systems/node-image" { - if !strings.EqualFold(value, targetVersions.imageRef) { - log.Printf("\t%s: Image %s, want %s\n", node.Name, value, targetVersions.imageRef) - allUpdated = false - } - } - } - - kubeletVersion := node.Status.NodeInfo.KubeletVersion - if kubeletVersion != string(targetVersions.kubernetes) { - log.Printf("\t%s: K8s (Kubelet) %s, want %s\n", node.Name, kubeletVersion, targetVersions.kubernetes) - allUpdated = false - } - kubeProxyVersion := node.Status.NodeInfo.KubeProxyVersion - if kubeProxyVersion != string(targetVersions.kubernetes) { - log.Printf("\t%s: K8s (Proxy) %s, want %s\n", node.Name, kubeProxyVersion, targetVersions.kubernetes) - allUpdated = false - } - } - - return allUpdated - }, timeout, time.Minute) -} - -type versionContainer struct { - imageRef string - kubernetes versions.ValidK8sVersion - microservices semver.Semver -} - -// runCommandWithSeparateOutputs runs the given command while separating buffers for -// stdout and stderr. -func runCommandWithSeparateOutputs(cmd *exec.Cmd) (stdout, stderr []byte, err error) { - stdout = []byte{} - stderr = []byte{} - - stdoutIn, err := cmd.StdoutPipe() - if err != nil { - err = fmt.Errorf("create stdout pipe: %w", err) - return - } - stderrIn, err := cmd.StderrPipe() - if err != nil { - err = fmt.Errorf("create stderr pipe: %w", err) - return - } - - err = cmd.Start() - if err != nil { - err = fmt.Errorf("start command: %w", err) - return - } - - continuouslyPrintOutput := func(r io.Reader, prefix string) { - scanner := bufio.NewScanner(r) - for scanner.Scan() { - output := scanner.Text() - fmt.Printf("%s: %s\n", prefix, output) - switch prefix { - case "stdout": - stdout = append(stdout, output...) - case "stderr": - stderr = append(stderr, output...) - } - } - } - - go continuouslyPrintOutput(stdoutIn, "stdout") - go continuouslyPrintOutput(stderrIn, "stderr") - - if err = cmd.Wait(); err != nil { - err = fmt.Errorf("wait for command to finish: %w", err) - } - - return stdout, stderr, err -} diff --git a/e2e/provider-upgrade/BUILD.bazel b/e2e/provider-upgrade/BUILD.bazel new file mode 100644 index 00000000000..43dea50ce87 --- /dev/null +++ b/e2e/provider-upgrade/BUILD.bazel @@ -0,0 +1,16 @@ +load("//bazel/go:go_test.bzl", "go_test") + +go_test( + name = "provider-upgrade_test", + srcs = ["upgrade_test.go"], + # keep + gotags = ["e2e"], + deps = [ + "//e2e/internal/kubectl", + "//e2e/internal/upgrade", + "//internal/config", + "//internal/semver", + "//internal/versions", + "@com_github_stretchr_testify//require", + ], +) diff --git a/e2e/provider-upgrade/upgrade_test.go b/e2e/provider-upgrade/upgrade_test.go new file mode 100644 index 00000000000..1ad908bbb67 --- /dev/null +++ b/e2e/provider-upgrade/upgrade_test.go @@ -0,0 +1,65 @@ +/* +Copyright (c) Edgeless Systems GmbH + +SPDX-License-Identifier: AGPL-3.0-only +*/ + +// End-to-end test that is used by the e2e Terraform provider test. +package main + +import ( + "flag" + "testing" + "time" + + "github.com/edgelesssys/constellation/v2/e2e/internal/kubectl" + "github.com/edgelesssys/constellation/v2/e2e/internal/upgrade" + "github.com/edgelesssys/constellation/v2/internal/config" + "github.com/edgelesssys/constellation/v2/internal/semver" + "github.com/edgelesssys/constellation/v2/internal/versions" + "github.com/stretchr/testify/require" +) + +var ( + targetImage = flag.String("target-image", "", "Image (shortversion) to upgrade to.") + targetKubernetes = flag.String("target-kubernetes", "", "Kubernetes version (MAJOR.MINOR.PATCH) to upgrade to. Defaults to default version of target CLI.") + targetMicroservices = flag.String("target-microservices", "", "Microservice version (MAJOR.MINOR.PATCH) to upgrade to. Defaults to default version of target CLI.") + // When executing the test as a bazel target the CLI path is supplied through an env variable that bazel sets. + // When executing via `go test` extra care should be taken that the supplied CLI is built on the same commit as this test. + // When executing the test as a bazel target the workspace path is supplied through an env variable that bazel sets. + workspace = flag.String("workspace", "", "Constellation workspace in which to run the tests.") + cliPath = flag.String("cli", "", "Constellation CLI to run the tests.") + wantWorker = flag.Int("want-worker", 0, "Number of wanted worker nodes.") + wantControl = flag.Int("want-control", 0, "Number of wanted control nodes.") + timeout = flag.Duration("timeout", 3*time.Hour, "Timeout after which the cluster should have converged to the target version.") +) + +// TestUpgradeSuccessful tests that the upgrade to the target version is successful. +func TestUpgradeSuccessful(t *testing.T) { + defaultConfig := config.Default() + var k8sV versions.ValidK8sVersion + if *targetKubernetes == "" { + k8sV = defaultConfig.KubernetesVersion + } else { + k8sV = versions.ValidK8sVersion(*targetKubernetes) + } + var microV semver.Semver + if *targetMicroservices == "" { + microV = defaultConfig.MicroserviceVersion + } else { + var err error + microV, err = semver.New(*targetMicroservices) + require.NoError(t, err) + } + v := upgrade.VersionContainer{ + ImageRef: *targetImage, + Kubernetes: k8sV, + Microservices: microV, + } + k, err := kubectl.New() + require.NoError(t, err) + + err = upgrade.Setup(*workspace, *cliPath) + require.NoError(t, err) + upgrade.AssertUpgradeSuccessful(t, *cliPath, v, k, *wantControl, *wantWorker, *timeout) +}