From 04c25bb6087194274f252983b2ff6ed7847bde7f Mon Sep 17 00:00:00 2001 From: Patrick Seidensal Date: Fri, 8 Nov 2024 09:39:17 +0100 Subject: [PATCH] Spawn arbitrary amount of downstream clusters Using the environment variable FLEET_E2E_DS_CLUSTER_COUNT, an arbitrary amount of downstream clusters can be spawned, e.g.: ``` FLEET_E2E_DS_CLUSTER_COUNT=4 ./dev/setup-multi-cluster ``` This environment variable affects - dev/setup-k3ds - dev/import-images-k3d - dev/setup-multi-cluster --- .github/scripts/label-downstream-cluster.sh | 12 ++++++-- dev/import-images-k3d | 15 ++++++++- dev/setup-fleet-managed-downstream | 34 +++++++++++---------- dev/setup-k3ds | 25 ++++++++++++--- dev/setup-multi-cluster | 2 ++ 5 files changed, 63 insertions(+), 25 deletions(-) diff --git a/.github/scripts/label-downstream-cluster.sh b/.github/scripts/label-downstream-cluster.sh index 20549a78cd..d065362ac3 100755 --- a/.github/scripts/label-downstream-cluster.sh +++ b/.github/scripts/label-downstream-cluster.sh @@ -4,6 +4,12 @@ set -euxo pipefail ns=${FLEET_E2E_NS_DOWNSTREAM-fleet-default} -{ grep -q -m 1 -e "1/1"; kill $!; } < <(kubectl get clusters.fleet.cattle.io -n "$ns" -w) -name=$(kubectl get clusters.fleet.cattle.io -o=jsonpath='{.items[0].metadata.name}' -n "$ns") -kubectl patch clusters.fleet.cattle.io -n "$ns" "$name" --type=json -p '[{"op": "add", "path": "/metadata/labels/env", "value": "test" }]' +# Wait for clusters to become "ready" by waiting for bundles to be become ready. +num_clusters=$(k3d cluster list -o json | jq -r '.[].name | select( . | contains("downstream") )' | wc -l) +while [[ $(kubectl get clusters.fleet.cattle.io -n "$ns" | grep '1/1' -c) -ne $num_clusters ]]; do + sleep 1 +done + +for cluster in $(kubectl get clusters.fleet.cattle.io -n "$ns" -o=jsonpath='{.items[*].metadata.name}'); do + kubectl patch clusters.fleet.cattle.io -n "$ns" "$cluster" --type=json -p '[{"op": "add", "path": "/metadata/labels/env", "value": "test" }]' +done diff --git a/dev/import-images-k3d b/dev/import-images-k3d index eed770aca5..1f6af298d1 100755 --- a/dev/import-images-k3d +++ b/dev/import-images-k3d @@ -2,13 +2,26 @@ set -euxo pipefail +# The upstream cluster to import all the images to. upstream_ctx="${FLEET_E2E_CLUSTER-k3d-upstream}" + +# The single downstream cluster to import the agent image to. downstream_ctx="${FLEET_E2E_CLUSTER_DOWNSTREAM-k3d-downstream}" +# If multi-cluster is enabled, import the agent image to all downstream clusters. +FLEET_E2E_DS_CLUSTER_COUNT="${FLEET_E2E_DS_CLUSTER_COUNT:-1}" + k3d image import rancher/fleet:dev rancher/fleet-agent:dev -m direct -c "${upstream_ctx#k3d-}" if [ "$upstream_ctx" != "$downstream_ctx" ]; then - k3d image import rancher/fleet-agent:dev -m direct -c "${downstream_ctx#k3d-}" + if [ "$FLEET_E2E_DS_CLUSTER_COUNT" -gt 1 ]; then + for cluster in $(k3d cluster list -o json | \ + jq -r ".[].name | select(. | contains(\"${downstream_ctx#k3d-}\"))"); do + k3d image import rancher/fleet-agent:dev -m direct -c "${cluster}" + done + else + k3d image import rancher/fleet-agent:dev -m direct -c "${downstream_ctx#k3d-}" + fi else echo "not importing agent to any downstream clusters. Set FLEET_E2E_CLUSTER_DOWNSTREAM" fi diff --git a/dev/setup-fleet-managed-downstream b/dev/setup-fleet-managed-downstream index 0d8c29a3a8..f24756515c 100755 --- a/dev/setup-fleet-managed-downstream +++ b/dev/setup-fleet-managed-downstream @@ -8,14 +8,15 @@ if [ ! -d ./charts/fleet ]; then exit 1 fi -# fetching from local kubeconfig -host=$( docker inspect -f '{{range.NetworkSettings.Networks}}{{.IPAddress}}{{end}}' k3d-downstream-server-0 ) -ca=$( kubectl config view --flatten -o jsonpath='{.clusters[?(@.name == "k3d-downstream")].cluster.certificate-authority-data}' ) -client_cert=$( kubectl config view --flatten -o jsonpath='{.users[?(@.name == "admin@k3d-downstream")].user.client-certificate-data}' ) -token=$( kubectl config view --flatten -o jsonpath='{.users[?(@.name == "admin@k3d-downstream")].user.client-key-data}' ) -server="https://$host:6443" +for cluster in $(k3d cluster list -o json | jq -r '.[].name | select(. | contains("downstream"))'); do + # fetching from local kubeconfig + host=$( docker inspect -f '{{range.NetworkSettings.Networks}}{{.IPAddress}}{{end}}' k3d-$cluster-server-0 ) + ca=$( kubectl config view --flatten -o jsonpath="{.clusters[?(@.name == \"k3d-$cluster\")].cluster.certificate-authority-data}" ) + client_cert=$( kubectl config view --flatten -o jsonpath="{.users[?(@.name == \"admin@k3d-$cluster\")].user.client-certificate-data}" ) + token=$( kubectl config view --flatten -o jsonpath="{.users[?(@.name == \"admin@k3d-$cluster\")].user.client-key-data}" ) + server="https://$host:6443" -value=$(cat < "$TMP_CONFIG" + cat <"$TMP_CONFIG" mirrors: "docker.io": endpoint: @@ -22,8 +23,22 @@ fi # https://hub.docker.com/r/rancher/k3s/tags #args="$args -i docker.io/rancher/k3s:v1.22.15-k3s1" -k3d cluster create upstream --servers 3 --api-port 36443 -p '80:80@server:0' -p '443:443@server:0' --k3s-arg '--tls-san=k3d-upstream-server-0@server:0' $args -k3d cluster create downstream --servers 1 --api-port 36444 -p '5080:80@server:0' -p '3444:443@server:0' $args -#k3d cluster create downstream2 --servers 1 --api-port 36445 -p '6080:80@server:0' -p '3445:443@server:0' $args -#k3d cluster create downstream3 --servers 1 --api-port 36446 -p '7080:80@server:0' -p '3446:443@server:0' $args +k3d cluster create upstream \ + --servers 3 \ + --api-port 36443 \ + -p '80:80@server:0' \ + -p '443:443@server:0' \ + --k3s-arg '--tls-san=k3d-upstream-server-0@server:0' \ + $args + +for i in $(seq 1 "$FLEET_E2E_DS_CLUSTER_COUNT"); do + k3d cluster create "downstream$i" \ + --servers 1 \ + --api-port $((36443 + i)) \ + -p "$((4080 + (1000 * i))):80@server:0" \ + -p "$((3443 + i)):443@server:0" \ + --k3s-arg "--tls-san=k3d-downstream$i-server-0@server:0" \ + $args +done + kubectl config use-context k3d-upstream diff --git a/dev/setup-multi-cluster b/dev/setup-multi-cluster index 209089b04d..5f5728da2a 100755 --- a/dev/setup-multi-cluster +++ b/dev/setup-multi-cluster @@ -6,6 +6,8 @@ export CUSTOM_CONFIG_FILE="env.multi-cluster" # shellcheck source=dev/setup-cluster-config source dev/setup-cluster-config +FLEET_E2E_DS_CLUSTER_COUNT=${FLEET_E2E_DS_CLUSTER_COUNT:-1} + # Cleans with settings sourced, so it should be rather selective. ./dev/k3d-act-clean