Addind Nodeselector for shards

rancher · Jun 26, 2024 · c301686 · c301686
1 parent de3f46a
commit c301686
Show file tree

Hide file tree

Showing 14 changed files with 150 additions and 67 deletions.
diff --git a/.github/scripts/deploy-fleet.sh b/.github/scripts/deploy-fleet.sh
@@ -2,7 +2,7 @@
 
 set -euxo pipefail
 
-shards=${SHARDS-""}
+shards_json=${SHARDS-""}
 
 function eventually {
   for _ in $(seq 1 3); do
@@ -28,6 +28,25 @@ host=$(kubectl get node k3d-upstream-server-0 -o jsonpath='{.status.addresses[?(
 ca=$( kubectl config view --flatten -o jsonpath='{.clusters[?(@.name == "k3d-upstream")].cluster.certificate-authority-data}' | base64 -d )
 server="https://$host:6443"
 
+# Constructing the shards settings dynamically
+shards_settings=""
+if [ -n "$shards_json" ]; then
+  index=0
+  for shard in $(echo "${shards_json}" | jq -c '.[]'); do
+    shard_id=$(echo "$shard" | jq -r '.id')
+    shards_settings="$shards_settings --set shards[$index].id=$shard_id"
+    node_selector=$(echo "$shard" | jq -r '.nodeSelector // empty')
+    if [ -n "$node_selector" ]; then
+      for key in $(echo "$node_selector" | jq -r 'keys[]'); do
+        value=$(echo "$node_selector" | jq -r --arg key "$key" '.[$key]')
+        escaped_key=$(echo "$key" | sed 's/\./\\./g')
+        shards_settings="$shards_settings --set shards[$index].nodeSelector.\"$escaped_key\"=$value"
+      done
+    fi
+    index=$((index + 1))
+  done
+fi
+
 eventually helm upgrade --install fleet-crd charts/fleet-crd \
   --atomic \
   -n cattle-fleet-system \
@@ -43,7 +62,7 @@ eventually helm upgrade --install fleet charts/fleet \
   --set agentImage.imagePullPolicy=IfNotPresent \
   --set apiServerCA="$ca" \
   --set apiServerURL="$server" \
-  --set shards="{$shards}" \
+  $shards_settings \
   --set debug=true --set debugLevel=1
 
 # wait for controller and agent rollout

diff --git a/.github/workflows/aks.yml b/.github/workflows/aks.yml
@@ -126,7 +126,7 @@ jobs:
         run: |
           export KUBECONFIG="$GITHUB_WORKSPACE/kubeconfig-fleet-ci"
           echo "${{ steps.meta-fleet.outputs.tags }} ${{ steps.meta-fleet-agent.outputs.tags }}"
-          SHARDS="shard1,shard2,shard3" ./.github/scripts/deploy-fleet.sh \
+          SHARDS='[{"id":"shard0"},{"id":"shard1"},{"id":"shard2"}]' ./.github/scripts/deploy-fleet.sh \
                                        ${{ steps.meta-fleet.outputs.tags }} \
                                        ${{ steps.meta-fleet-agent.outputs.tags }}
       -

diff --git a/.github/workflows/e2e-ci.yml b/.github/workflows/e2e-ci.yml
@@ -82,7 +82,7 @@ jobs:
       -
         name: Deploy Fleet
         run: |
-          SHARDS="shard1,shard2,shard3" ./.github/scripts/deploy-fleet.sh
+          SHARDS='[{"id":"shard0"},{"id":"shard1"},{"id":"shard2"}]' ./.github/scripts/deploy-fleet.sh
       -
         name: Create Zot certificates for OCI tests
         env:

diff --git a/.github/workflows/gke.yml b/.github/workflows/gke.yml
@@ -130,7 +130,7 @@ jobs:
         name: Deploy Fleet
         run: |
           echo "${{ steps.meta-fleet.outputs.tags }} ${{ steps.meta-fleet-agent.outputs.tags }}"
-          SHARDS="shard1,shard2,shard3" ./.github/scripts/deploy-fleet.sh \
+          SHARDS='[{"id":"shard0"},{"id":"shard1"},{"id":"shard2"}]' ./.github/scripts/deploy-fleet.sh \
                                        ${{ steps.meta-fleet.outputs.tags }} \
                                        ${{ steps.meta-fleet-agent.outputs.tags }}
       -

diff --git a/.github/workflows/nightly-ci.yml b/.github/workflows/nightly-ci.yml
@@ -83,7 +83,7 @@ jobs:
       -
         name: Deploy Fleet
         run: |
-          SHARDS="shard1,shard2,shard3" ./.github/scripts/deploy-fleet.sh
+          SHARDS='[{"id":"shard0"},{"id":"shard1"},{"id":"shard2"}]' ./.github/scripts/deploy-fleet.sh
       -
         name: Create Zot certificates for OCI tests
         if: ${{ matrix.test_type == 'e2e-nosecrets' }}

diff --git a/charts/fleet/templates/deployment.yaml b/charts/fleet/templates/deployment.yaml
@@ -1,12 +1,19 @@
-{{ $shards := list "" }}
-{{ if .Values.shards }}
-{{ $shards = concat $shards .Values.shards | uniq }}
-{{ end }}
-{{ range $shards }}
+{{- $shards := list (dict "id" "" "nodeSelector" dict) -}}
+{{- $uniqueShards := list -}}
+{{- if .Values.shards -}}
+  {{- range .Values.shards -}}
+    {{- if not (has .id $uniqueShards) -}}
+      {{- $shards = append $shards . -}}
+      {{- $uniqueShards = append $uniqueShards .id -}}
+    {{- end -}}
+  {{- end -}}
+{{- end -}}
+
+{{ range $shard := $shards }}
 apiVersion: apps/v1
 kind: Deployment
 metadata:
-  name: "fleet-controller{{if . }}-shard-{{ . }}{{end}}"
+  name: "fleet-controller{{if $shard.id }}-shard-{{ $shard.id }}{{end}}"
 spec:
   selector:
     matchLabels:
@@ -15,8 +22,8 @@ spec:
     metadata:
       labels:
         app: fleet-controller
-        fleet.cattle.io/shard-id: "{{ . }}"
-        {{- if empty . }}
+        fleet.cattle.io/shard-id: "{{ $shard.id }}"
+        {{- if empty $shard.id }}
         fleet.cattle.io/shard-default: "true"
         {{- end }}
     spec:
@@ -89,9 +96,9 @@ spec:
         {{- end }}
         command:
         - fleetcontroller
-        {{- if . }}
+        {{- if $shard.id }}
         - --shard-id
-        - {{ quote . }}
+        - {{ quote $shard.id }}
         {{- end }}
         {{- if not $.Values.metrics.enabled }}
         - --disable-metrics
@@ -116,7 +123,7 @@ spec:
           - mountPath: /tmp/pprof
             name: pprof
         {{- end }}
-      {{- if not . }} # Only deploy cleanup and agent management through sharding-less deployment
+      {{- if not $shard.id }} # Only deploy cleanup and agent management through sharding-less deployment
       - env:
         - name: NAMESPACE
           valueFrom:
@@ -221,11 +228,16 @@ spec:
       {{- end }}
 
       serviceAccountName: fleet-controller
-      nodeSelector: {{ include "linux-node-selector" . | nindent 8 }}
+      nodeSelector: {{ include "linux-node-selector" $shard.id | nindent 8 }}
 {{- if $.Values.nodeSelector }}
 {{ toYaml $.Values.nodeSelector | indent 8 }}
 {{- end }}
-      tolerations: {{ include "linux-node-tolerations" . | nindent 8 }}
+{{- if $shard.nodeSelector -}}
+{{- range $key, $value := $shard.nodeSelector }}
+{{ $key | indent 8}}: {{ $value }}
+{{- end }}
+{{- end }}
+      tolerations: {{ include "linux-node-tolerations" $shard.id | nindent 8 }}
 {{- if $.Values.tolerations }}
 {{ toYaml $.Values.tolerations | indent 8 }}
 {{- end }}

diff --git a/charts/fleet/templates/deployment_gitjob.yaml b/charts/fleet/templates/deployment_gitjob.yaml
@@ -1,13 +1,20 @@
-{{ $shards := list "" }}
-{{ if .Values.shards }}
-{{ $shards = concat $shards .Values.shards | uniq }}
-{{ end }}
-{{ range $shards }}
+{{- $shards := list (dict "id" "" "nodeSelector" dict) -}}
+{{- $uniqueShards := list -}}
+{{- if .Values.shards -}}
+  {{- range .Values.shards -}}
+    {{- if not (has .id $uniqueShards) -}}
+      {{- $shards = append $shards . -}}
+      {{- $uniqueShards = append $uniqueShards .id -}}
+    {{- end -}}
+  {{- end -}}
+{{- end -}}
+
+{{ range $shard := $shards }}
 {{- if $.Values.gitops.enabled }}
 apiVersion: apps/v1
 kind: Deployment
 metadata:
-  name: "gitjob{{if . }}-shard-{{ . }}{{end}}"
+  name: "gitjob{{if $shard.id }}-shard-{{ $shard.id }}{{end}}"
 spec:
   selector:
     matchLabels:
@@ -16,8 +23,8 @@ spec:
     metadata:
       labels:
         app: "gitjob"
-        fleet.cattle.io/shard-id: "{{ . }}"
-        {{- if empty . }}
+        fleet.cattle.io/shard-id: "{{ $shard.id }}"
+        {{- if empty $shard.id }}
         fleet.cattle.io/shard-default: "true"
         {{- end }}
     spec:
@@ -38,9 +45,9 @@ spec:
           {{- if $.Values.debug }}
           - --debug
           {{- end }}
-          {{- if . }}
+          {{- if $shard.id }}
           - --shard-id
-          - {{ quote . }}
+          - {{ quote $shard.id }}
           {{- end }}
           {{- if not $.Values.metrics.enabled }}
           - --disable-metrics
@@ -77,11 +84,11 @@ spec:
 {{- if $.Values.extraEnv }}
 {{ toYaml $.Values.extraEnv | indent 12}}
 {{- end }}
-      nodeSelector: {{ include "linux-node-selector" . | nindent 8 }}
+      nodeSelector: {{ include "linux-node-selector" $shard.id | nindent 8 }}
 {{- if $.Values.nodeSelector }}
 {{ toYaml $.Values.nodeSelector | indent 8 }}
 {{- end }}
-      tolerations: {{ include "linux-node-tolerations" . | nindent 8 }}
+      tolerations: {{ include "linux-node-tolerations" $shard.id | nindent 8 }}
 {{- if $.Values.tolerations }}
 {{ toYaml $.Values.tolerations | indent 8 }}
 {{- end }}

diff --git a/charts/fleet/templates/service.yaml b/charts/fleet/templates/service.yaml
@@ -1,13 +1,20 @@
 {{- if .Values.metrics.enabled }}
-{{ $shards := list "" }}
-{{ if .Values.shards }}
-{{ $shards = concat $shards .Values.shards | uniq }}
-{{ end }}
-{{ range $shards }}
+{{- $shards := list (dict "id" "" "nodeSelector" dict) -}}
+{{- $uniqueShards := list -}}
+{{- if .Values.shards -}}
+  {{- range .Values.shards -}}
+    {{- if not (has .id $uniqueShards) -}}
+      {{- $shards = append $shards . -}}
+      {{- $uniqueShards = append $uniqueShards .id -}}
+    {{- end -}}
+  {{- end -}}
+{{- end -}}
+
+{{ range $shard := $shards }}
 apiVersion: v1
 kind: Service
 metadata:
-  name: "monitoring-fleet-controller{{if . }}-shard-{{ . }}{{end}}"
+  name: "monitoring-fleet-controller{{if  $shard.id  }}-shard-{{  $shard.id   }}{{end}}"
   labels:
     app: fleet-controller
 spec:
@@ -19,10 +26,10 @@ spec:
     name: metrics
   selector:
     app: fleet-controller
-    {{- if empty . }}
+    {{- if empty  $shard.id   }}
     fleet.cattle.io/shard-default: "true"
     {{- else }}
-    fleet.cattle.io/shard-id: "{{ . }}"
+    fleet.cattle.io/shard-id: "{{  $shard.id   }}"
     {{- end }}
 ---
 {{- end }}

diff --git a/charts/fleet/templates/service_gitjob.yaml b/charts/fleet/templates/service_gitjob.yaml
@@ -13,15 +13,22 @@ spec:
     app: "gitjob"
 ---
 {{- if .Values.metrics.enabled }}
-{{ $shards := list "" }}
-{{ if .Values.shards }}
-{{ $shards = concat $shards .Values.shards | uniq }}
-{{ end }}
-{{ range $shards }}
+{{- $shards := list (dict "id" "" "nodeSelector" dict) -}}
+{{- $uniqueShards := list -}}
+{{- if .Values.shards -}}
+  {{- range .Values.shards -}}
+    {{- if not (has .id $uniqueShards) -}}
+      {{- $shards = append $shards . -}}
+      {{- $uniqueShards = append $uniqueShards .id -}}
+    {{- end -}}
+  {{- end -}}
+{{- end -}}
+
+{{ range $shard := $shards }}
 apiVersion: v1
 kind: Service
 metadata:
-  name: "monitoring-gitjob{{if . }}-shard-{{ . }}{{end}}"
+  name: "monitoring-gitjob{{if $shard.id }}-shard-{{ $shard.id }}{{end}}"
   labels:
     app: gitjob
 spec:
@@ -33,10 +40,10 @@ spec:
     name: metrics
   selector:
     app: gitjob
-    {{- if empty . }}
+    {{- if empty $shard.id }}
     fleet.cattle.io/shard-default: "true"
     {{- else }}
-    fleet.cattle.io/shard-id: "{{ . }}"
+    fleet.cattle.io/shard-id: "{{ $shard.id }}"
     {{- end }}
 ---
 {{- end }}

diff --git a/charts/fleet/templates/service_gitops_metrics.yaml b/charts/fleet/templates/service_gitops_metrics.yaml
diff --git a/charts/fleet/values.yaml b/charts/fleet/values.yaml
@@ -52,7 +52,6 @@ bootstrap:
   branch: master
   paths: ""
 
-
 global:
   cattle:
     systemDefaultRegistry: ""
@@ -108,3 +107,14 @@ controller:
 # extraEnv:
 # - name: EXPERIMENTAL_OCI_STORAGE
 #   value: "true"
+
+# shards:
+#   - id: shard0
+#     nodeSelector:
+#       kubernetes.io/hostname: k3d-upstream-server-0
+#   - id: shard1
+#     nodeSelector:
+#       kubernetes.io/hostname: k3d-upstream-server-1
+#   - id: shard2
+#     nodeSelector:
+#       kubernetes.io/hostname: k3d-upstream-server-2
diff --git a/dev/setup-fleet b/dev/setup-fleet
@@ -4,7 +4,7 @@
 set -euxo pipefail
 
 cluster_name=${1-upstream}
-shards=${2-""}
+shards_json=${2:-}
 
 if [ ! -d ./charts/fleet ]; then
   echo "please change the current directory to the fleet repo checkout"
@@ -16,10 +16,30 @@ host=$( docker inspect -f '{{range.NetworkSettings.Networks}}{{.IPAddress}}{{end
 ca=$( kubectl config view --flatten -o jsonpath='{.clusters[?(@.name == "k3d-'"$cluster_name"'")].cluster.certificate-authority-data}' | base64 -d )
 server="https://$host:6443"
 helm -n cattle-fleet-system upgrade --install --create-namespace --wait fleet-crd charts/fleet-crd
+
+# Constructing the shards settings dynamically
+shards_settings=""
+if [ -n "$shards_json" ]; then
+  index=0
+  for shard in $(echo "${shards_json}" | jq -c '.[]'); do
+    shard_id=$(echo "$shard" | jq -r '.id')
+    shards_settings="$shards_settings --set shards[$index].id=$shard_id"
+    node_selector=$(echo "$shard" | jq -r '.nodeSelector // empty')
+    if [ -n "$node_selector" ]; then
+      for key in $(echo "$node_selector" | jq -r 'keys[]'); do
+        value=$(echo "$node_selector" | jq -r --arg key "$key" '.[$key]')
+        escaped_key=$(echo "$key" | sed 's/\./\\./g')
+        shards_settings="$shards_settings --set shards[$index].nodeSelector.\"$escaped_key\"=$value"
+      done
+    fi
+    index=$((index + 1))
+  done
+fi
+
 helm -n cattle-fleet-system upgrade --install --create-namespace --wait --reset-values \
   --set apiServerCA="$ca" \
   --set apiServerURL="$server" \
-  --set shards="{$shards}" \
+  $shards_settings \
   --set debug=true --set debugLevel=1 fleet charts/fleet
 
 # wait for controller and agent rollout