Skip to content

Commit

Permalink
add upgrade to test
Browse files Browse the repository at this point in the history
  • Loading branch information
elchead committed Dec 27, 2023
1 parent 92b2e9e commit 166d4d5
Show file tree
Hide file tree
Showing 6 changed files with 416 additions and 218 deletions.
89 changes: 88 additions & 1 deletion .github/workflows/e2e-test-provider-example.yml
Original file line number Diff line number Diff line change
Expand Up @@ -23,6 +23,18 @@ on:
providerVersion:
description: "Constellation Terraform provider version to use (without v prefix). Empty value means build from source."
type: string
toImage:
description: Image (shortpath) the cluster is upgraded to, or empty for main/nightly.
type: string
required: false
toKubernetes:
description: Kubernetes version to target for the upgrade, empty for no upgrade.
type: string
required: false
toProviderVersion:
description: Provider version and microservice version to target for the upgrade, empty for no upgrade.
type: string
required: false
workflow_call:
inputs:
ref:
Expand All @@ -41,6 +53,18 @@ on:
providerVersion:
description: "Constellation Terraform provider version to use. Empty value means build from source."
type: string
toImage:
description: Image (shortpath) the cluster is upgraded to, or empty for main/nightly.
type: string
required: false
toKubernetes:
description: Kubernetes version to target for the upgrade, empty for target's default version.
type: string
required: false
toProviderVersion:
description: Provider version and microservice version to target for the upgrade, empty for no upgrade.
type: string
required: false

jobs:
provider-example-test:
Expand Down Expand Up @@ -237,7 +261,6 @@ jobs:
cp ${{ github.workspace }}/terraform-provider-constellation/examples/full/${{ inputs.cloudProvider }}/main.tf ${{ github.workspace }}/cluster/main.tf
- name: Download CLI
if: inputs.cloudProvider == 'azure'
shell: bash
run: |
curl -fsSL -o constellation https://github.com/edgelesssys/constellation/releases/download/v2.13.0/constellation-linux-amd64
Expand All @@ -260,6 +283,70 @@ jobs:
TF_LOG=INFO terraform apply -auto-approve
fi
- name: Update cluster configuration
working-directory: ${{ github.workspace }}/cluster
shell: bash
run: |
if [[ "${{ inputs.toImage }}" != "" ]]; then
cat >> _override.tf <<EOF
locals {
version = "${{ inputs.toImage }}"
}
EOF
fi
if [[ "${{ inputs.toProviderVersion }}" != "" ]]; then
cat >> _override.tf <<EOF
terraform {
required_providers {
constellation = {
source = "edgelesssys/constellation"
version = "${{ inputs.toProviderVersion }}"
}
}
}
EOF
fi
# TODO rename version to image_version
- name: Upgrade Terraform Cluster
working-directory: ${{ github.workspace }}/cluster
shell: bash
run: |
terraform init --upgrade
TF_LOG=INFO terraform apply -auto-approve
- name: Assert upgrade successful
working-directory: ${{ github.workspace }}/cluster
env:
IMAGE: ${{ inputs.toImage }}
KUBERNETES: ${{ inputs.toKubernetes }}
MICROSERVICES: ${{ inputs.toProviderVersion }}
WORKERNODES: 2
CONTROLNODES: 3
run: |
terraform output -raw kubeconfig > constellation-admin.conf
if [[ -n ${MICROSERVICES} ]]; then
MICROSERVICES_FLAG="--target-microservices=$MICROSERVICES"
fi
if [[ -n ${KUBERNETES} ]]; then
KUBERNETES_FLAG="--target-kubernetes=$KUBERNETES"
fi
if [[ -n ${IMAGE} ]]; then
IMAGE_FLAG="--target-image=$IMAGE"
fi
# cfg must be in same dir as KUBECONFIG
./constellation config generate ${{ inputs.cloudProvider }}
# make cfg valid
yq e '.provider.azure.subscription = "123e4567-e89b-12d3-a456-426614174000"' -i constellation-conf.yaml
yq e '.provider.azure.tenant = "123e4567-e89b-12d3-a456-426614174001"' -i constellation-conf.yaml
yq e '.provider.azure.location = "eastus"' -i constellation-conf.yaml
yq e '.provider.azure.resourceGroup = "myResourceGroup"' -i constellation-conf.yaml
yq e '.provider.azure.userAssignedIdentity = "myIdentity"' -i constellation-conf.yaml
KUBECONFIG=${{ github.workspace }}/cluster/constellation-admin.conf bazel run //e2e/provider-upgrade:provider-upgrade_test -- --want-worker $WORKERNODES --want-control $CONTROLNODES --cli ${{ github.workspace }}/cluster/constellation $IMAGE_FLAG $KUBERNETES_FLAG $MICROSERVICES_FLAG
- name: Destroy Terraform Cluster
# outcome is part of the steps context (https://docs.github.com/en/actions/learn-github-actions/contexts#steps-context)
if: always() && steps.apply_terraform.outcome != 'skipped'
Expand Down
6 changes: 5 additions & 1 deletion e2e/internal/upgrade/BUILD.bazel
Original file line number Diff line number Diff line change
Expand Up @@ -13,6 +13,11 @@ go_library(
"//internal/constants",
"//internal/logger",
"//internal/semver",
"//internal/versions",
"@com_github_stretchr_testify//require",
"@io_bazel_rules_go//go/runfiles:go_default_library",
"@io_k8s_apimachinery//pkg/apis/meta/v1:meta",
"@io_k8s_client_go//kubernetes",
"@sh_helm_helm_v3//pkg/action",
"@sh_helm_helm_v3//pkg/cli",
],
Expand Down Expand Up @@ -44,7 +49,6 @@ go_test(
"//internal/versions",
"@com_github_spf13_afero//:afero",
"@com_github_stretchr_testify//require",
"@io_bazel_rules_go//go/runfiles:go_default_library",
"@io_k8s_api//core/v1:core",
"@io_k8s_apimachinery//pkg/apis/meta/v1:meta",
"@io_k8s_client_go//kubernetes",
Expand Down
238 changes: 238 additions & 0 deletions e2e/internal/upgrade/upgrade.go
Original file line number Diff line number Diff line change
@@ -1,3 +1,5 @@
//go:build e2e

/*
Copyright (c) Edgeless Systems GmbH
Expand All @@ -17,3 +19,239 @@ SPDX-License-Identifier: AGPL-3.0-only
//
// - set or fetch measurements depending on target image
package upgrade

import (
"bufio"
"context"
"errors"
"fmt"
"io"
"log"
"os"
"os/exec"
"path/filepath"
"strings"
"sync"
"testing"
"time"

"github.com/bazelbuild/rules_go/go/runfiles"
"github.com/edgelesssys/constellation/v2/internal/semver"
"github.com/edgelesssys/constellation/v2/internal/versions"
"github.com/stretchr/testify/require"
metaV1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/client-go/kubernetes"
)

// VersionContainer contains the versions that the cluster should be upgraded to.
type VersionContainer struct {
ImageRef string
Kubernetes versions.ValidK8sVersion
Microservices semver.Semver
}

// AssertUpgradeSuccessful tests that the upgrade to the target version is successful.
func AssertUpgradeSuccessful(t *testing.T, cli string, targetVersions VersionContainer, k *kubernetes.Clientset, wantControl, wantWorker int, timeout time.Duration) {
wg := queryStatusAsync(t, cli)
require.NotNil(t, k)
testMicroservicesEventuallyHaveVersion(t, targetVersions.Microservices, timeout)
testNodesEventuallyHaveVersion(t, k, targetVersions, wantControl+wantWorker, timeout)

wg.Wait()
}

func queryStatusAsync(t *testing.T, cli string) *sync.WaitGroup {
var wg sync.WaitGroup
wg.Add(1)
go func() {
defer wg.Done()
// The first control plane node should finish upgrading after 20 minutes. If it does not, something is fishy.
// Nodes can upgrade in <5mins.
testStatusEventuallyWorks(t, cli, 20*time.Minute)
}()

return &wg
}

func testStatusEventuallyWorks(t *testing.T, cli string, timeout time.Duration) {
require.Eventually(t, func() bool {
// Show versions set in cluster.
// The string after "Cluster status:" in the output might not be updated yet.
// This is only updated after the operator finishes one reconcile loop.
cmd := exec.CommandContext(context.Background(), cli, "status")
stdout, stderr, err := runCommandWithSeparateOutputs(cmd)
if err != nil {
log.Printf("Stdout: %s\nStderr: %s", string(stdout), string(stderr))
return false
}

log.Println(string(stdout))
return true
}, timeout, time.Minute)
}

func testMicroservicesEventuallyHaveVersion(t *testing.T, wantMicroserviceVersion semver.Semver, timeout time.Duration) {
require.Eventually(t, func() bool {
version, err := servicesVersion(t)
if err != nil {
log.Printf("Unable to fetch microservice version: %v\n", err)
return false
}

if version != wantMicroserviceVersion {
log.Printf("Microservices still at version %v, want %v\n", version, wantMicroserviceVersion)
return false
}

return true
}, timeout, time.Minute)
}

func testNodesEventuallyHaveVersion(t *testing.T, k *kubernetes.Clientset, targetVersions VersionContainer, totalNodeCount int, timeout time.Duration) {
require.Eventually(t, func() bool {
nodes, err := k.CoreV1().Nodes().List(context.Background(), metaV1.ListOptions{})
if err != nil {
log.Println(err)
return false
}
require.False(t, len(nodes.Items) < totalNodeCount, "expected at least %v nodes, got %v", totalNodeCount, len(nodes.Items))

allUpdated := true
log.Printf("Node status (%v):", time.Now())
for _, node := range nodes.Items {
for key, value := range node.Annotations {
if key == "constellation.edgeless.systems/node-image" {
if !strings.EqualFold(value, targetVersions.ImageRef) {
log.Printf("\t%s: Image %s, want %s\n", node.Name, value, targetVersions.ImageRef)
allUpdated = false
}
}
}

kubeletVersion := node.Status.NodeInfo.KubeletVersion
if kubeletVersion != string(targetVersions.Kubernetes) {
log.Printf("\t%s: K8s (Kubelet) %s, want %s\n", node.Name, kubeletVersion, targetVersions.Kubernetes)
allUpdated = false
}
kubeProxyVersion := node.Status.NodeInfo.KubeProxyVersion
if kubeProxyVersion != string(targetVersions.Kubernetes) {
log.Printf("\t%s: K8s (Proxy) %s, want %s\n", node.Name, kubeProxyVersion, targetVersions.Kubernetes)
allUpdated = false
}
}

return allUpdated
}, timeout, time.Minute)
}

// runCommandWithSeparateOutputs runs the given command while separating buffers for
// stdout and stderr.
func runCommandWithSeparateOutputs(cmd *exec.Cmd) (stdout, stderr []byte, err error) {
stdout = []byte{}
stderr = []byte{}

stdoutIn, err := cmd.StdoutPipe()
if err != nil {
err = fmt.Errorf("create stdout pipe: %w", err)
return
}
stderrIn, err := cmd.StderrPipe()
if err != nil {
err = fmt.Errorf("create stderr pipe: %w", err)
return
}

err = cmd.Start()
if err != nil {
err = fmt.Errorf("start command: %w", err)
return
}

continuouslyPrintOutput := func(r io.Reader, prefix string) {
scanner := bufio.NewScanner(r)
for scanner.Scan() {
output := scanner.Text()
fmt.Printf("%s: %s\n", prefix, output)
switch prefix {
case "stdout":
stdout = append(stdout, output...)
case "stderr":
stderr = append(stderr, output...)
}
}
}

go continuouslyPrintOutput(stdoutIn, "stdout")
go continuouslyPrintOutput(stderrIn, "stderr")

if err = cmd.Wait(); err != nil {
err = fmt.Errorf("wait for command to finish: %w", err)
}

return stdout, stderr, err
}

// Setup checks that the prerequisites for the test are met:
// - a workspace is set
// - a CLI path is set
// - the constellation-upgrade folder does not exist.
func Setup(workspace, cliPath string) error {
workingDir, err := workingDir(workspace)
if err != nil {
return fmt.Errorf("getting working directory: %w", err)
}

if err := os.Chdir(workingDir); err != nil {
return fmt.Errorf("changing working directory: %w", err)
}

if _, err := getCLIPath(cliPath); err != nil {
return fmt.Errorf("getting CLI path: %w", err)
}
return nil
}

// workingDir returns the path to the workspace.
func workingDir(workspace string) (string, error) {
workingDir := os.Getenv("BUILD_WORKING_DIRECTORY")
switch {
case workingDir != "":
return workingDir, nil
case workspace != "":
return workspace, nil
default:
return "", errors.New("neither 'BUILD_WORKING_DIRECTORY' nor 'workspace' flag set")
}
}

// getCLIPath returns the path to the CLI.
func getCLIPath(cliPathFlag string) (string, error) {
pathCLI := os.Getenv("PATH_CLI")
var relCLIPath string
switch {
case pathCLI != "":
relCLIPath = pathCLI
case cliPathFlag != "":
relCLIPath = cliPathFlag
default:
return "", errors.New("neither 'PATH_CLI' nor 'cli' flag set")
}

// try to find the CLI in the working directory
// (e.g. when running via `go test` or when specifying a path manually)
workdir, err := os.Getwd()
if err != nil {
return "", fmt.Errorf("getting working directory: %w", err)
}

absCLIPath := relCLIPath
if !filepath.IsAbs(relCLIPath) {
absCLIPath = filepath.Join(workdir, relCLIPath)
}
if _, err := os.Stat(absCLIPath); err == nil {
return absCLIPath, nil
}

// fall back to runfiles (e.g. when running via bazel)
return runfiles.Rlocation(pathCLI)
}
Loading

0 comments on commit 166d4d5

Please sign in to comment.