From c301686120142d0037ed9f476c06733ba0778305 Mon Sep 17 00:00:00 2001 From: GIRARDI Tommy Date: Wed, 19 Jun 2024 14:08:51 +0200 Subject: [PATCH] Addind Nodeselector for shards --- .github/scripts/deploy-fleet.sh | 23 ++++++++++- .github/workflows/aks.yml | 2 +- .github/workflows/e2e-ci.yml | 2 +- .github/workflows/gke.yml | 2 +- .github/workflows/nightly-ci.yml | 2 +- charts/fleet/templates/deployment.yaml | 38 ++++++++++++------- charts/fleet/templates/deployment_gitjob.yaml | 31 +++++++++------ charts/fleet/templates/service.yaml | 23 +++++++---- charts/fleet/templates/service_gitjob.yaml | 23 +++++++---- .../templates/service_gitops_metrics.yaml | 15 -------- charts/fleet/values.yaml | 12 +++++- dev/setup-fleet | 24 +++++++++++- dev/setup-single-cluster | 18 ++++++++- e2e/single-cluster/sharding_test.go | 2 +- 14 files changed, 150 insertions(+), 67 deletions(-) delete mode 100644 charts/fleet/templates/service_gitops_metrics.yaml diff --git a/.github/scripts/deploy-fleet.sh b/.github/scripts/deploy-fleet.sh index 7e8b77052a..d58943694b 100755 --- a/.github/scripts/deploy-fleet.sh +++ b/.github/scripts/deploy-fleet.sh @@ -2,7 +2,7 @@ set -euxo pipefail -shards=${SHARDS-""} +shards_json=${SHARDS-""} function eventually { for _ in $(seq 1 3); do @@ -28,6 +28,25 @@ host=$(kubectl get node k3d-upstream-server-0 -o jsonpath='{.status.addresses[?( ca=$( kubectl config view --flatten -o jsonpath='{.clusters[?(@.name == "k3d-upstream")].cluster.certificate-authority-data}' | base64 -d ) server="https://$host:6443" +# Constructing the shards settings dynamically +shards_settings="" +if [ -n "$shards_json" ]; then + index=0 + for shard in $(echo "${shards_json}" | jq -c '.[]'); do + shard_id=$(echo "$shard" | jq -r '.id') + shards_settings="$shards_settings --set shards[$index].id=$shard_id" + node_selector=$(echo "$shard" | jq -r '.nodeSelector // empty') + if [ -n "$node_selector" ]; then + for key in $(echo "$node_selector" | jq -r 'keys[]'); do + value=$(echo "$node_selector" | jq -r --arg key "$key" '.[$key]') + escaped_key=$(echo "$key" | sed 's/\./\\./g') + shards_settings="$shards_settings --set shards[$index].nodeSelector.\"$escaped_key\"=$value" + done + fi + index=$((index + 1)) + done +fi + eventually helm upgrade --install fleet-crd charts/fleet-crd \ --atomic \ -n cattle-fleet-system \ @@ -43,7 +62,7 @@ eventually helm upgrade --install fleet charts/fleet \ --set agentImage.imagePullPolicy=IfNotPresent \ --set apiServerCA="$ca" \ --set apiServerURL="$server" \ - --set shards="{$shards}" \ + $shards_settings \ --set debug=true --set debugLevel=1 # wait for controller and agent rollout diff --git a/.github/workflows/aks.yml b/.github/workflows/aks.yml index e1b66fffeb..4a17ec6d39 100644 --- a/.github/workflows/aks.yml +++ b/.github/workflows/aks.yml @@ -126,7 +126,7 @@ jobs: run: | export KUBECONFIG="$GITHUB_WORKSPACE/kubeconfig-fleet-ci" echo "${{ steps.meta-fleet.outputs.tags }} ${{ steps.meta-fleet-agent.outputs.tags }}" - SHARDS="shard1,shard2,shard3" ./.github/scripts/deploy-fleet.sh \ + SHARDS='[{"id":"shard0"},{"id":"shard1"},{"id":"shard2"}]' ./.github/scripts/deploy-fleet.sh \ ${{ steps.meta-fleet.outputs.tags }} \ ${{ steps.meta-fleet-agent.outputs.tags }} - diff --git a/.github/workflows/e2e-ci.yml b/.github/workflows/e2e-ci.yml index 9c9b992f0b..06e3ecda50 100644 --- a/.github/workflows/e2e-ci.yml +++ b/.github/workflows/e2e-ci.yml @@ -82,7 +82,7 @@ jobs: - name: Deploy Fleet run: | - SHARDS="shard1,shard2,shard3" ./.github/scripts/deploy-fleet.sh + SHARDS='[{"id":"shard0"},{"id":"shard1"},{"id":"shard2"}]' ./.github/scripts/deploy-fleet.sh - name: Create Zot certificates for OCI tests env: diff --git a/.github/workflows/gke.yml b/.github/workflows/gke.yml index d523505f20..d49a08e308 100644 --- a/.github/workflows/gke.yml +++ b/.github/workflows/gke.yml @@ -130,7 +130,7 @@ jobs: name: Deploy Fleet run: | echo "${{ steps.meta-fleet.outputs.tags }} ${{ steps.meta-fleet-agent.outputs.tags }}" - SHARDS="shard1,shard2,shard3" ./.github/scripts/deploy-fleet.sh \ + SHARDS='[{"id":"shard0"},{"id":"shard1"},{"id":"shard2"}]' ./.github/scripts/deploy-fleet.sh \ ${{ steps.meta-fleet.outputs.tags }} \ ${{ steps.meta-fleet-agent.outputs.tags }} - diff --git a/.github/workflows/nightly-ci.yml b/.github/workflows/nightly-ci.yml index 26c84c452e..45e0f52b49 100644 --- a/.github/workflows/nightly-ci.yml +++ b/.github/workflows/nightly-ci.yml @@ -83,7 +83,7 @@ jobs: - name: Deploy Fleet run: | - SHARDS="shard1,shard2,shard3" ./.github/scripts/deploy-fleet.sh + SHARDS='[{"id":"shard0"},{"id":"shard1"},{"id":"shard2"}]' ./.github/scripts/deploy-fleet.sh - name: Create Zot certificates for OCI tests if: ${{ matrix.test_type == 'e2e-nosecrets' }} diff --git a/charts/fleet/templates/deployment.yaml b/charts/fleet/templates/deployment.yaml index 2b6b0f0160..5b872262a4 100644 --- a/charts/fleet/templates/deployment.yaml +++ b/charts/fleet/templates/deployment.yaml @@ -1,12 +1,19 @@ -{{ $shards := list "" }} -{{ if .Values.shards }} -{{ $shards = concat $shards .Values.shards | uniq }} -{{ end }} -{{ range $shards }} +{{- $shards := list (dict "id" "" "nodeSelector" dict) -}} +{{- $uniqueShards := list -}} +{{- if .Values.shards -}} + {{- range .Values.shards -}} + {{- if not (has .id $uniqueShards) -}} + {{- $shards = append $shards . -}} + {{- $uniqueShards = append $uniqueShards .id -}} + {{- end -}} + {{- end -}} +{{- end -}} + +{{ range $shard := $shards }} apiVersion: apps/v1 kind: Deployment metadata: - name: "fleet-controller{{if . }}-shard-{{ . }}{{end}}" + name: "fleet-controller{{if $shard.id }}-shard-{{ $shard.id }}{{end}}" spec: selector: matchLabels: @@ -15,8 +22,8 @@ spec: metadata: labels: app: fleet-controller - fleet.cattle.io/shard-id: "{{ . }}" - {{- if empty . }} + fleet.cattle.io/shard-id: "{{ $shard.id }}" + {{- if empty $shard.id }} fleet.cattle.io/shard-default: "true" {{- end }} spec: @@ -89,9 +96,9 @@ spec: {{- end }} command: - fleetcontroller - {{- if . }} + {{- if $shard.id }} - --shard-id - - {{ quote . }} + - {{ quote $shard.id }} {{- end }} {{- if not $.Values.metrics.enabled }} - --disable-metrics @@ -116,7 +123,7 @@ spec: - mountPath: /tmp/pprof name: pprof {{- end }} - {{- if not . }} # Only deploy cleanup and agent management through sharding-less deployment + {{- if not $shard.id }} # Only deploy cleanup and agent management through sharding-less deployment - env: - name: NAMESPACE valueFrom: @@ -221,11 +228,16 @@ spec: {{- end }} serviceAccountName: fleet-controller - nodeSelector: {{ include "linux-node-selector" . | nindent 8 }} + nodeSelector: {{ include "linux-node-selector" $shard.id | nindent 8 }} {{- if $.Values.nodeSelector }} {{ toYaml $.Values.nodeSelector | indent 8 }} {{- end }} - tolerations: {{ include "linux-node-tolerations" . | nindent 8 }} +{{- if $shard.nodeSelector -}} +{{- range $key, $value := $shard.nodeSelector }} +{{ $key | indent 8}}: {{ $value }} +{{- end }} +{{- end }} + tolerations: {{ include "linux-node-tolerations" $shard.id | nindent 8 }} {{- if $.Values.tolerations }} {{ toYaml $.Values.tolerations | indent 8 }} {{- end }} diff --git a/charts/fleet/templates/deployment_gitjob.yaml b/charts/fleet/templates/deployment_gitjob.yaml index 519b37f89b..0f9a1bf909 100644 --- a/charts/fleet/templates/deployment_gitjob.yaml +++ b/charts/fleet/templates/deployment_gitjob.yaml @@ -1,13 +1,20 @@ -{{ $shards := list "" }} -{{ if .Values.shards }} -{{ $shards = concat $shards .Values.shards | uniq }} -{{ end }} -{{ range $shards }} +{{- $shards := list (dict "id" "" "nodeSelector" dict) -}} +{{- $uniqueShards := list -}} +{{- if .Values.shards -}} + {{- range .Values.shards -}} + {{- if not (has .id $uniqueShards) -}} + {{- $shards = append $shards . -}} + {{- $uniqueShards = append $uniqueShards .id -}} + {{- end -}} + {{- end -}} +{{- end -}} + +{{ range $shard := $shards }} {{- if $.Values.gitops.enabled }} apiVersion: apps/v1 kind: Deployment metadata: - name: "gitjob{{if . }}-shard-{{ . }}{{end}}" + name: "gitjob{{if $shard.id }}-shard-{{ $shard.id }}{{end}}" spec: selector: matchLabels: @@ -16,8 +23,8 @@ spec: metadata: labels: app: "gitjob" - fleet.cattle.io/shard-id: "{{ . }}" - {{- if empty . }} + fleet.cattle.io/shard-id: "{{ $shard.id }}" + {{- if empty $shard.id }} fleet.cattle.io/shard-default: "true" {{- end }} spec: @@ -38,9 +45,9 @@ spec: {{- if $.Values.debug }} - --debug {{- end }} - {{- if . }} + {{- if $shard.id }} - --shard-id - - {{ quote . }} + - {{ quote $shard.id }} {{- end }} {{- if not $.Values.metrics.enabled }} - --disable-metrics @@ -77,11 +84,11 @@ spec: {{- if $.Values.extraEnv }} {{ toYaml $.Values.extraEnv | indent 12}} {{- end }} - nodeSelector: {{ include "linux-node-selector" . | nindent 8 }} + nodeSelector: {{ include "linux-node-selector" $shard.id | nindent 8 }} {{- if $.Values.nodeSelector }} {{ toYaml $.Values.nodeSelector | indent 8 }} {{- end }} - tolerations: {{ include "linux-node-tolerations" . | nindent 8 }} + tolerations: {{ include "linux-node-tolerations" $shard.id | nindent 8 }} {{- if $.Values.tolerations }} {{ toYaml $.Values.tolerations | indent 8 }} {{- end }} diff --git a/charts/fleet/templates/service.yaml b/charts/fleet/templates/service.yaml index 2d6ba88717..49aefb490a 100644 --- a/charts/fleet/templates/service.yaml +++ b/charts/fleet/templates/service.yaml @@ -1,13 +1,20 @@ {{- if .Values.metrics.enabled }} -{{ $shards := list "" }} -{{ if .Values.shards }} -{{ $shards = concat $shards .Values.shards | uniq }} -{{ end }} -{{ range $shards }} +{{- $shards := list (dict "id" "" "nodeSelector" dict) -}} +{{- $uniqueShards := list -}} +{{- if .Values.shards -}} + {{- range .Values.shards -}} + {{- if not (has .id $uniqueShards) -}} + {{- $shards = append $shards . -}} + {{- $uniqueShards = append $uniqueShards .id -}} + {{- end -}} + {{- end -}} +{{- end -}} + +{{ range $shard := $shards }} apiVersion: v1 kind: Service metadata: - name: "monitoring-fleet-controller{{if . }}-shard-{{ . }}{{end}}" + name: "monitoring-fleet-controller{{if $shard.id }}-shard-{{ $shard.id }}{{end}}" labels: app: fleet-controller spec: @@ -19,10 +26,10 @@ spec: name: metrics selector: app: fleet-controller - {{- if empty . }} + {{- if empty $shard.id }} fleet.cattle.io/shard-default: "true" {{- else }} - fleet.cattle.io/shard-id: "{{ . }}" + fleet.cattle.io/shard-id: "{{ $shard.id }}" {{- end }} --- {{- end }} diff --git a/charts/fleet/templates/service_gitjob.yaml b/charts/fleet/templates/service_gitjob.yaml index 98a8fe7746..f33c9a5457 100644 --- a/charts/fleet/templates/service_gitjob.yaml +++ b/charts/fleet/templates/service_gitjob.yaml @@ -13,15 +13,22 @@ spec: app: "gitjob" --- {{- if .Values.metrics.enabled }} -{{ $shards := list "" }} -{{ if .Values.shards }} -{{ $shards = concat $shards .Values.shards | uniq }} -{{ end }} -{{ range $shards }} +{{- $shards := list (dict "id" "" "nodeSelector" dict) -}} +{{- $uniqueShards := list -}} +{{- if .Values.shards -}} + {{- range .Values.shards -}} + {{- if not (has .id $uniqueShards) -}} + {{- $shards = append $shards . -}} + {{- $uniqueShards = append $uniqueShards .id -}} + {{- end -}} + {{- end -}} +{{- end -}} + +{{ range $shard := $shards }} apiVersion: v1 kind: Service metadata: - name: "monitoring-gitjob{{if . }}-shard-{{ . }}{{end}}" + name: "monitoring-gitjob{{if $shard.id }}-shard-{{ $shard.id }}{{end}}" labels: app: gitjob spec: @@ -33,10 +40,10 @@ spec: name: metrics selector: app: gitjob - {{- if empty . }} + {{- if empty $shard.id }} fleet.cattle.io/shard-default: "true" {{- else }} - fleet.cattle.io/shard-id: "{{ . }}" + fleet.cattle.io/shard-id: "{{ $shard.id }}" {{- end }} --- {{- end }} diff --git a/charts/fleet/templates/service_gitops_metrics.yaml b/charts/fleet/templates/service_gitops_metrics.yaml deleted file mode 100644 index 428df7c410..0000000000 --- a/charts/fleet/templates/service_gitops_metrics.yaml +++ /dev/null @@ -1,15 +0,0 @@ -apiVersion: v1 -kind: Service -metadata: - name: "monitoring-gitops-controller" - labels: - app: gitjob -spec: - type: ClusterIP - ports: - - port: 8081 - targetPort: 8081 - protocol: TCP - name: metrics - selector: - app: gitjob diff --git a/charts/fleet/values.yaml b/charts/fleet/values.yaml index 9bedc76091..8ce206f4ce 100644 --- a/charts/fleet/values.yaml +++ b/charts/fleet/values.yaml @@ -52,7 +52,6 @@ bootstrap: branch: master paths: "" - global: cattle: systemDefaultRegistry: "" @@ -108,3 +107,14 @@ controller: # extraEnv: # - name: EXPERIMENTAL_OCI_STORAGE # value: "true" + +# shards: +# - id: shard0 +# nodeSelector: +# kubernetes.io/hostname: k3d-upstream-server-0 +# - id: shard1 +# nodeSelector: +# kubernetes.io/hostname: k3d-upstream-server-1 +# - id: shard2 +# nodeSelector: +# kubernetes.io/hostname: k3d-upstream-server-2 diff --git a/dev/setup-fleet b/dev/setup-fleet index 81625e8df7..ee912ebe23 100755 --- a/dev/setup-fleet +++ b/dev/setup-fleet @@ -4,7 +4,7 @@ set -euxo pipefail cluster_name=${1-upstream} -shards=${2-""} +shards_json=${2:-} if [ ! -d ./charts/fleet ]; then echo "please change the current directory to the fleet repo checkout" @@ -16,10 +16,30 @@ host=$( docker inspect -f '{{range.NetworkSettings.Networks}}{{.IPAddress}}{{end ca=$( kubectl config view --flatten -o jsonpath='{.clusters[?(@.name == "k3d-'"$cluster_name"'")].cluster.certificate-authority-data}' | base64 -d ) server="https://$host:6443" helm -n cattle-fleet-system upgrade --install --create-namespace --wait fleet-crd charts/fleet-crd + +# Constructing the shards settings dynamically +shards_settings="" +if [ -n "$shards_json" ]; then + index=0 + for shard in $(echo "${shards_json}" | jq -c '.[]'); do + shard_id=$(echo "$shard" | jq -r '.id') + shards_settings="$shards_settings --set shards[$index].id=$shard_id" + node_selector=$(echo "$shard" | jq -r '.nodeSelector // empty') + if [ -n "$node_selector" ]; then + for key in $(echo "$node_selector" | jq -r 'keys[]'); do + value=$(echo "$node_selector" | jq -r --arg key "$key" '.[$key]') + escaped_key=$(echo "$key" | sed 's/\./\\./g') + shards_settings="$shards_settings --set shards[$index].nodeSelector.\"$escaped_key\"=$value" + done + fi + index=$((index + 1)) + done +fi + helm -n cattle-fleet-system upgrade --install --create-namespace --wait --reset-values \ --set apiServerCA="$ca" \ --set apiServerURL="$server" \ - --set shards="{$shards}" \ + $shards_settings \ --set debug=true --set debugLevel=1 fleet charts/fleet # wait for controller and agent rollout diff --git a/dev/setup-single-cluster b/dev/setup-single-cluster index 1af5fea1af..f948053ee6 100755 --- a/dev/setup-single-cluster +++ b/dev/setup-single-cluster @@ -12,7 +12,23 @@ source ./dev/setup-cluster-config ./dev/setup-k3d ./dev/build-fleet ./dev/import-images-k3d -./dev/setup-fleet upstream shard1,shard2,shard3 +./dev/setup-fleet upstream '[ + { + "id": "shard0", + "nodeSelector": { + "kubernetes.io/hostname": "k3d-upstream-server-0" + } + }, + { + "id": "shard1" + }, + { + "id": "shard2", + "nodeSelector": { + "kubernetes.io/hostname": "k3d-upstream-server-2" + } + } +]' # needed for gitrepo tests ./dev/import-images-tests-k3d diff --git a/e2e/single-cluster/sharding_test.go b/e2e/single-cluster/sharding_test.go index d609cc5fb0..9b4cf05bbc 100644 --- a/e2e/single-cluster/sharding_test.go +++ b/e2e/single-cluster/sharding_test.go @@ -11,7 +11,7 @@ import ( "github.com/rancher/fleet/e2e/testenv/kubectl" ) -var shards = []string{"shard1", "shard2", "shard3"} +var shards = []string{"shard0", "shard1", "shard2"} var _ = Describe("Filtering events by shard", Label("sharding"), Ordered, func() { var (