Spawn arbitrary amount of downstream clusters

Using the environment variable FLEET_E2E_DS_CLUSTER_COUNT, an arbitrary
amount of downstream clusters can be spawned, e.g.:

```
FLEET_E2E_DS_CLUSTER_COUNT=4 ./dev/setup-multi-cluster
```

This environment variable affects

- dev/setup-k3ds
- dev/import-images-k3d
- dev/setup-multi-cluster

.github/scripts/label-downstream-cluster.sh

@@ -4,6 +4,12 @@ set -euxo pipefail
 
 ns=${FLEET_E2E_NS_DOWNSTREAM-fleet-default}
 
-{ grep -q -m 1 -e "1/1"; kill $!; } < <(kubectl get clusters.fleet.cattle.io -n "$ns" -w)
-name=$(kubectl get clusters.fleet.cattle.io -o=jsonpath='{.items[0].metadata.name}' -n "$ns")
-kubectl patch clusters.fleet.cattle.io -n "$ns" "$name" --type=json -p '[{"op": "add", "path": "/metadata/labels/env", "value": "test" }]'
+# Wait for clusters to become "ready" by waiting for bundles to be become ready.
+num_clusters=$(k3d cluster list -o json | jq -r '.[].name | select( . | contains("downstream") )' | wc -l)
+while [[ $(kubectl get clusters.fleet.cattle.io -n "$ns" | grep '1/1' -c) -ne $num_clusters ]]; do
+    sleep 1
+done
+
+for cluster in $(kubectl get clusters.fleet.cattle.io -n "$ns" -o=jsonpath='{.items[*].metadata.name}'); do
+  kubectl patch clusters.fleet.cattle.io -n "$ns" "$cluster" --type=json -p '[{"op": "add", "path": "/metadata/labels/env", "value": "test" }]'
+done

dev/import-images-k3d

@@ -2,13 +2,26 @@
 
 set -euxo pipefail
 
+# The upstream cluster to import all the images to.
 upstream_ctx="${FLEET_E2E_CLUSTER-k3d-upstream}"
+
+# The single downstream cluster to import the agent image to.
 downstream_ctx="${FLEET_E2E_CLUSTER_DOWNSTREAM-k3d-downstream}"
 
+# If multi-cluster is enabled, import the agent image to all downstream clusters.
+FLEET_E2E_DS_CLUSTER_COUNT="${FLEET_E2E_DS_CLUSTER_COUNT:-1}"
+
 k3d image import rancher/fleet:dev rancher/fleet-agent:dev -m direct -c "${upstream_ctx#k3d-}"
 
 if [ "$upstream_ctx" != "$downstream_ctx" ]; then
-  k3d image import rancher/fleet-agent:dev -m direct -c "${downstream_ctx#k3d-}"
+  if [ "$FLEET_E2E_DS_CLUSTER_COUNT" -gt 1 ]; then
+    for cluster in $(k3d cluster list -o json | \
+        jq -r ".[].name | select(. | contains(\"${downstream_ctx#k3d-}\"))"); do
+      k3d image import rancher/fleet-agent:dev -m direct -c "${cluster}"
+    done
+  else
+    k3d image import rancher/fleet-agent:dev -m direct -c "${downstream_ctx#k3d-}"
+  fi
 else
   echo "not importing agent to any downstream clusters. Set FLEET_E2E_CLUSTER_DOWNSTREAM"
 fi

dev/setup-fleet-managed-downstream

@@ -8,14 +8,15 @@ if [ ! -d ./charts/fleet ]; then
   exit 1
 fi
 
-# fetching from local kubeconfig
-host=$( docker inspect -f '{{range.NetworkSettings.Networks}}{{.IPAddress}}{{end}}' k3d-downstream-server-0 )
-ca=$( kubectl config view --flatten -o jsonpath='{.clusters[?(@.name == "k3d-downstream")].cluster.certificate-authority-data}' )
-client_cert=$( kubectl config view --flatten -o jsonpath='{.users[?(@.name == "admin@k3d-downstream")].user.client-certificate-data}' )
-token=$( kubectl config view --flatten -o jsonpath='{.users[?(@.name == "admin@k3d-downstream")].user.client-key-data}' )
-server="https://$host:6443"
+for cluster in $(k3d cluster list -o json | jq -r '.[].name | select(. | contains("downstream"))'); do
+    # fetching from local kubeconfig
+    host=$( docker inspect -f '{{range.NetworkSettings.Networks}}{{.IPAddress}}{{end}}' k3d-$cluster-server-0 )
+    ca=$( kubectl config view --flatten -o jsonpath="{.clusters[?(@.name == \"k3d-$cluster\")].cluster.certificate-authority-data}" )
+    client_cert=$( kubectl config view --flatten -o jsonpath="{.users[?(@.name == \"admin@k3d-$cluster\")].user.client-certificate-data}" )
+    token=$( kubectl config view --flatten -o jsonpath="{.users[?(@.name == \"admin@k3d-$cluster\")].user.client-key-data}" )
+    server="https://$host:6443"
 
-value=$(cat <<EOF
+    value=$(cat <<EOF
 apiVersion: v1
 kind: Config
 current-context: default
@@ -36,20 +37,21 @@ users:
     client-certificate-data: $client_cert
     client-key-data: $token
 EOF
-)
+    )
+    kubectl create ns fleet-default || true
+    kubectl delete secret -n fleet-default kbcfg-$cluster || true
+    kubectl create secret generic -n fleet-default kbcfg-$cluster --from-literal=value="$value"
 
-kubectl create ns fleet-default || true
-kubectl delete secret -n fleet-default kbcfg-second || true
-kubectl create secret generic -n fleet-default kbcfg-second --from-literal=value="$value"
-
-kubectl apply -n fleet-default -f - <<EOF
+    kubectl apply -n fleet-default -f - <<EOF
 apiVersion: "fleet.cattle.io/v1alpha1"
 kind: Cluster
 metadata:
-  name: second
+  name: $cluster
   namespace: fleet-default
   labels:
-    name: second
+    name: $cluster
 spec:
-  kubeConfigSecret: kbcfg-second
+  kubeConfigSecret: kbcfg-$cluster
 EOF
+
+done

dev/setup-k3ds

@@ -4,12 +4,13 @@ set -euxo pipefail
 
 args=${k3d_args---network fleet}
 docker_mirror=${docker_mirror-}
+FLEET_E2E_DS_CLUSTER_COUNT=${FLEET_E2E_DS_CLUSTER_COUNT:-1}
 
 if [ -n "$docker_mirror" ]; then
   TMP_CONFIG="$(mktemp)"
   trap "rm -f $TMP_CONFIG" EXIT
 
-  cat << EOF > "$TMP_CONFIG"
+  cat <<EOF >"$TMP_CONFIG"
 mirrors:
   "docker.io":
       endpoint:
@@ -22,8 +23,22 @@ fi
 # https://hub.docker.com/r/rancher/k3s/tags
 #args="$args -i docker.io/rancher/k3s:v1.22.15-k3s1"
 
-k3d cluster create upstream --servers 3 --api-port 36443 -p '80:80@server:0' -p '443:443@server:0' --k3s-arg '--tls-san=k3d-upstream-server-0@server:0' $args
-k3d cluster create downstream --servers 1 --api-port 36444 -p '5080:80@server:0' -p '3444:443@server:0' $args
-#k3d cluster create downstream2 --servers 1 --api-port 36445 -p '6080:80@server:0' -p '3445:443@server:0' $args
-#k3d cluster create downstream3 --servers 1 --api-port 36446 -p '7080:80@server:0' -p '3446:443@server:0' $args
+k3d cluster create upstream \
+  --servers 3 \
+  --api-port 36443 \
+  -p '80:80@server:0' \
+  -p '443:443@server:0' \
+  --k3s-arg '--tls-san=k3d-upstream-server-0@server:0' \
+  $args
+
+for i in $(seq 1 "$FLEET_E2E_DS_CLUSTER_COUNT"); do
+  k3d cluster create "downstream$i" \
+    --servers 1 \
+    --api-port $((36443 + i)) \
+    -p "$((4080 + (1000 * i))):80@server:0" \
+    -p "$((3443 + i)):443@server:0" \
+    --k3s-arg "--tls-san=k3d-downstream$i-server-0@server:0" \
+    $args
+done
+
 kubectl config use-context k3d-upstream

dev/setup-multi-cluster

@@ -6,6 +6,8 @@ export CUSTOM_CONFIG_FILE="env.multi-cluster"
 # shellcheck source=dev/setup-cluster-config
 source dev/setup-cluster-config
 
+FLEET_E2E_DS_CLUSTER_COUNT=${FLEET_E2E_DS_CLUSTER_COUNT:-1}
+
 # Cleans with settings sourced, so it should be rather selective.
 ./dev/k3d-act-clean