From 898ad0f5cb716028b759a183720a13ed6a33be23 Mon Sep 17 00:00:00 2001 From: Y-Rookie Date: Fri, 18 Oct 2024 15:42:33 +0800 Subject: [PATCH 1/4] refactor minio addon --- addons-cluster/minio/templates/cluster.yaml | 1 - addons-cluster/minio/templates/rbac.yaml | 1 - .../scripts/initialize-patch-configmap.sh | 58 +++++++ addons/minio/scripts/startup.sh | 71 ++++++++ .../minio/templates/_helper_create_bucket.txt | 4 +- addons/minio/templates/_helpers.tpl | 162 +++--------------- addons/minio/templates/cmpd.yaml | 149 ++++++---------- addons/minio/templates/cpmv.yaml | 4 +- ...figmap.yaml => minio-config-template.yaml} | 2 +- .../templates/minio-scripts-template.yaml | 10 ++ addons/minio/values.yaml | 9 +- 11 files changed, 226 insertions(+), 245 deletions(-) delete mode 100644 addons-cluster/minio/templates/rbac.yaml create mode 100644 addons/minio/scripts/initialize-patch-configmap.sh create mode 100644 addons/minio/scripts/startup.sh rename addons/minio/templates/{configmap.yaml => minio-config-template.yaml} (79%) create mode 100644 addons/minio/templates/minio-scripts-template.yaml diff --git a/addons-cluster/minio/templates/cluster.yaml b/addons-cluster/minio/templates/cluster.yaml index 5c15777b1..450260a6b 100644 --- a/addons-cluster/minio/templates/cluster.yaml +++ b/addons-cluster/minio/templates/cluster.yaml @@ -6,7 +6,6 @@ metadata: labels: {{ include "kblib.clusterLabels" . | nindent 4 }} spec: terminationPolicy: {{ .Values.extra.terminationPolicy }} - {{ include "kblib.affinity" . | indent 2 }} componentSpecs: - componentDef: minio name: minio diff --git a/addons-cluster/minio/templates/rbac.yaml b/addons-cluster/minio/templates/rbac.yaml deleted file mode 100644 index 08875e8bf..000000000 --- a/addons-cluster/minio/templates/rbac.yaml +++ /dev/null @@ -1 +0,0 @@ -{{- include "kblib.rbac" . }} \ No newline at end of file diff --git a/addons/minio/scripts/initialize-patch-configmap.sh b/addons/minio/scripts/initialize-patch-configmap.sh new file mode 100644 index 000000000..f648cef3e --- /dev/null +++ b/addons/minio/scripts/initialize-patch-configmap.sh @@ -0,0 +1,58 @@ +#!/bin/bash + +get_current_cm_key_value() { + local name=$1 + local namespace=$2 + local key=$3 + + kubectl get configmaps "$name" -n "$namespace" -o jsonpath="{.data.$key}" | tr -d '[]' +} + +update_cm_key_value() { + local name=$1 + local namespace=$2 + local key=$3 + local new_value=$4 + + kubectl patch configmap "$name" -n "$namespace" --type strategic -p "{\"data\":{\"$key\":\"$new_value\"}}" +} + +get_cm_key_new_value() { + local cur=$1 + local replicas=$2 + + if [[ -z "$cur" ]]; then + echo "[$replicas]" + else + IFS=',' read -ra array <<< "$cur" + last=${array[-1]} + if [[ "$last" == "$replicas" ]]; then + echo "[$cur]" + else + echo "[$cur,$replicas]" + fi + fi +} + +update_configmap() { + local name="$MINIO_COMPONENT_NAME-minio-configuration" + local namespace="$CLUSTER_NAMESPACE" + local key="MINIO_REPLICAS_HISTORY" + local replicas="$MINIO_COMP_REPLICAS" + + cur=$(get_current_cm_key_value "$name" "$namespace" "$key") + new=$(get_cm_key_new_value "$cur" "$replicas") + + update_cm_key_value "$name" "$namespace" "$key" "$new" + echo "ConfigMap $name updated successfully with $key=$new" +} + +# This is magic for shellspec ut framework. +# Sometime, functions are defined in a single shell script. +# You will want to test it. but you do not want to run the script. +# When included from shellspec, __SOURCED__ variable defined and script +# end here. The script path is assigned to the __SOURCED__ variable. +${__SOURCED__:+false} : || return 0 + +# main +update_configmap \ No newline at end of file diff --git a/addons/minio/scripts/startup.sh b/addons/minio/scripts/startup.sh new file mode 100644 index 000000000..d1769e44b --- /dev/null +++ b/addons/minio/scripts/startup.sh @@ -0,0 +1,71 @@ +#!/bin/bash + +replicas_history_file="/minio-config/MINIO_REPLICAS_HISTORY" +bucket_dir="/data" + +init_buckets() { + local buckets=$1 + IFS=',' read -ra BUCKET_ARRAY <<< "$buckets" + for bucket in "${BUCKET_ARRAY[@]}"; do + directory="$bucket_dir/$bucket" + if mkdir -p "$directory"; then + echo "Successfully init bucket: $directory" + else + echo "Failed to init bucket: $directory" + fi + done +} + +read_replicas_history() { + local file=$1 + content=$(cat "$file") + content=$(echo "$content" | tr -d '[]') + IFS=',' read -r -a replicas <<< "$content" + echo "${replicas[@]}" +} + +generate_server_pool() { + local replicas=("$@") + local server="" + for ((i=0; i < ${#replicas[@]}; i++)); do + if [ $i -eq 0 ]; then + cur=${replicas[i]} + server+=" $HTTP_PROTOCOL://$MINIO_COMPONENT_NAME-{0...$((cur-1))}.$MINIO_COMPONENT_NAME-headless.$CLUSTER_NAMESPACE.svc.$CLUSTER_DOMAIN/data" + else + prev=${replicas[i-1]} + cur=${replicas[i]} + server+=" $HTTP_PROTOCOL://$MINIO_COMPONENT_NAME-{$((prev))...$((cur-1))}.$MINIO_COMPONENT_NAME-headless.$CLUSTER_NAMESPACE.svc.$CLUSTER_DOMAIN/data" + fi + done + echo "$server" +} + +startup() { + if [ ! -f "$replicas_history_file" ]; then + echo "minio config don't existed" + exit 1 + fi + + buckets="$MINIO_BUCKETS" + if [ -n "$buckets" ]; then + init_buckets "$buckets" + fi + + mapfile -t replicas < <(read_replicas_history "$replicas_history_file") + server=$(generate_server_pool "${replicas[@]}") + echo "the minio server pool is $server" + + cmd="/usr/bin/docker-entrypoint.sh minio server $server -S $CERTS_PATH --address :$MINIO_API_PORT --console-address :$MINIO_CONSOLE_PORT" + echo "Starting minio server with command: $cmd" + eval "$cmd" +} + +# This is magic for shellspec ut framework. +# Sometime, functions are defined in a single shell script. +# You will want to test it. but you do not want to run the script. +# When included from shellspec, __SOURCED__ variable defined and script +# end here. The script path is assigned to the __SOURCED__ variable. +${__SOURCED__:+false} : || return 0 + +# main +startup \ No newline at end of file diff --git a/addons/minio/templates/_helper_create_bucket.txt b/addons/minio/templates/_helper_create_bucket.txt index ad2f546b7..fde29f8e5 100644 --- a/addons/minio/templates/_helper_create_bucket.txt +++ b/addons/minio/templates/_helper_create_bucket.txt @@ -1,8 +1,8 @@ #!/bin/sh set -e ; # Have script exit in the event of a failed command. -{{- if .Values.configPathmc }} -MC_CONFIG_DIR="{{ .Values.configPathmc }}" +{{- if .Values.mcConfigPath }} +MC_CONFIG_DIR="{{ .Values.mcConfigPath }}" MC="/usr/bin/mc --insecure --config-dir ${MC_CONFIG_DIR}" {{- else }} MC="/usr/bin/mc --insecure" diff --git a/addons/minio/templates/_helpers.tpl b/addons/minio/templates/_helpers.tpl index df4aa0cde..fac3bc732 100644 --- a/addons/minio/templates/_helpers.tpl +++ b/addons/minio/templates/_helpers.tpl @@ -52,162 +52,46 @@ app.kubernetes.io/instance: {{ .Release.Name }} {{- end }} {{/* -Return the appropriate apiVersion for poddisruptionbudget. +Common minio annotations */}} -{{- define "minio.pdb.apiVersion" -}} -{{- if semverCompare "<1.21-0" .Capabilities.KubeVersion.Version -}} -{{- print "policy/v1beta1" -}} -{{- else -}} -{{- print "policy/v1" -}} -{{- end -}} -{{- end -}} - -{{/* -Return the appropriate apiVersion for networkpolicy. -*/}} -{{- define "minio.networkPolicy.apiVersion" -}} -{{- if semverCompare ">=1.4-0, <1.7-0" .Capabilities.KubeVersion.Version -}} -{{- print "extensions/v1beta1" -}} -{{- else if semverCompare "^1.7-0" .Capabilities.KubeVersion.Version -}} -{{- print "networking.k8s.io/v1beta1" -}} -{{- end -}} -{{- end -}} - -{{/* -Return the appropriate apiVersion for deployment. -*/}} -{{- define "minio.deployment.apiVersion" -}} -{{- if semverCompare "<1.9-0" .Capabilities.KubeVersion.Version -}} -{{- print "apps/v1beta2" -}} -{{- else -}} -{{- print "apps/v1" -}} -{{- end -}} -{{- end -}} - -{{/* -Return the appropriate apiVersion for statefulset. -*/}} -{{- define "minio.statefulset.apiVersion" -}} -{{- if semverCompare "<1.16-0" .Capabilities.KubeVersion.Version -}} -{{- print "apps/v1beta2" -}} -{{- else -}} -{{- print "apps/v1" -}} -{{- end -}} -{{- end -}} - -{{/* -Return the appropriate apiVersion for ingress. -*/}} -{{- define "minio.ingress.apiVersion" -}} -{{- if semverCompare "<1.14-0" .Capabilities.KubeVersion.GitVersion -}} -{{- print "extensions/v1beta1" -}} -{{- else -}} -{{- print "networking.k8s.io/v1beta1" -}} -{{- end -}} -{{- end -}} - -{{/* -Determine secret name. -*/}} -{{- define "minio.secretName" -}} -{{- if .Values.existingSecret -}} -{{- .Values.existingSecret }} -{{- else -}} -{{- include "minio.fullname" . -}} -{{- end -}} -{{- end -}} - -{{/* -Determine service account name for deployment or statefulset. -*/}} -{{- define "minio.serviceAccountName" -}} -{{- if .Values.serviceAccount.create -}} -{{- default (include "minio.fullname" .) .Values.serviceAccount.name | replace "+" "_" | trunc 63 | trimSuffix "-" -}} -{{- else -}} -{{- default "default" .Values.serviceAccount.name -}} -{{- end -}} -{{- end -}} +{{- define "minio.annotations" -}} +helm.sh/resource-policy: keep +{{- end }} {{/* -Determine name for scc role and rolebinding +Generate minio scripts configmap */}} -{{- define "minio.sccRoleName" -}} -{{- printf "%s-%s" "scc" (include "minio.fullname" .) | trunc 63 | trimSuffix "-" -}} -{{- end -}} +{{- define "minio.extend.scripts" -}} +{{- range $path, $_ := $.Files.Glob "scripts/**" }} +{{ $path | base }}: |- +{{- $.Files.Get $path | nindent 2 }} +{{- end }} +{{- end }} {{/* -Properly format optional additional arguments to Minio binary +Define minio component definition name */}} -{{- define "minio.extraArgs" -}} -{{- range .Values.extraArgs -}} -{{ " " }}{{ . }} -{{- end -}} +{{- define "minio.cmpdName" -}} +minio-{{ .Chart.Version }} {{- end -}} {{/* -Return the proper Docker Image Registry Secret Names +Define minio component definition regular expression name prefix */}} -{{- define "minio.imagePullSecrets" -}} -{{/* -Helm 2.11 supports the assignment of a value to a variable defined in a different scope, -but Helm 2.9 and 2.10 does not support it, so we need to implement this if-else logic. -Also, we can not use a single if because lazy evaluation is not an option -*/}} -{{- if .Values.global }} -{{- if .Values.global.imagePullSecrets }} -imagePullSecrets: -{{- range .Values.global.imagePullSecrets }} - - name: {{ . }} -{{- end }} -{{- else if .Values.imagePullSecrets }} -imagePullSecrets: - {{ toYaml .Values.imagePullSecrets }} -{{- end -}} -{{- else if .Values.imagePullSecrets }} -imagePullSecrets: - {{ toYaml .Values.imagePullSecrets }} -{{- end -}} +{{- define "minio.cmpdRegexpPattern" -}} +^minio- {{- end -}} {{/* -Formats volumeMount for Minio tls keys and trusted certs +Define minio scripts template name */}} -{{- define "minio.tlsKeysVolumeMount" -}} -{{- if .Values.tls.enabled }} -- name: cert-secret-volume - mountPath: {{ .Values.certsPath }} -{{- end }} -{{- if or .Values.tls.enabled (ne .Values.trustedCertsSecret "") }} -{{- $casPath := printf "%s/CAs" .Values.certsPath | clean }} -- name: trusted-cert-secret-volume - mountPath: {{ $casPath }} -{{- end }} +{{- define "minio.scriptsTplName" -}} +minio-scripts {{- end -}} {{/* -Formats volume for Minio tls keys and trusted certs +Define minio config template name */}} -{{- define "minio.tlsKeysVolume" -}} -{{- if .Values.tls.enabled }} -- name: cert-secret-volume - secret: - secretName: {{ .Values.tls.certSecret }} - items: - - key: {{ .Values.tls.publicCrt }} - path: public.crt - - key: {{ .Values.tls.privateKey }} - path: private.key -{{- end }} -{{- if or .Values.tls.enabled (ne .Values.trustedCertsSecret "") }} -{{- $certSecret := eq .Values.trustedCertsSecret "" | ternary .Values.tls.certSecret .Values.trustedCertsSecret }} -{{- $publicCrt := eq .Values.trustedCertsSecret "" | ternary .Values.tls.publicCrt "" }} -- name: trusted-cert-secret-volume - secret: - secretName: {{ $certSecret }} - {{- if ne $publicCrt "" }} - items: - - key: {{ $publicCrt }} - path: public.crt - {{- end }} -{{- end }} +{{- define "minio.configTplName" -}} +minio-configuration {{- end -}} diff --git a/addons/minio/templates/cmpd.yaml b/addons/minio/templates/cmpd.yaml index 3ee66c9db..2c1944294 100644 --- a/addons/minio/templates/cmpd.yaml +++ b/addons/minio/templates/cmpd.yaml @@ -1,17 +1,16 @@ -{{ $scheme := .Values.tls.enabled | ternary "https" "http" }} - apiVersion: apps.kubeblocks.io/v1 kind: ComponentDefinition metadata: - name: minio + name: {{ include "minio.cmpdName" . }} labels: {{- include "minio.labels" . | nindent 4 }} + annotations: + {{- include "minio.annotations" . | nindent 4 }} spec: provider: kubeblocks description: Minio is a High Performance Object Storage. serviceKind: minio - serviceVersion: {{ .Values.componentServiceVersion.minio }} - + serviceVersion: {{ .Values.defaultServiceVersion.minio }} systemAccounts: - name: root initAccount: true @@ -19,7 +18,31 @@ spec: length: 16 numDigits: 8 letterCase: MixedCases - + services: + - name: default + spec: + ports: + - name: api + port: 9000 + targetPort: api + - name: console + port: 9001 + targetPort: console + configs: + - name: minio-configuration + templateRef: {{ include "minio.configTplName" . }} + namespace: {{ .Release.Namespace }} + volumeName: minio-config + defaultMode: 0777 + scripts: + - name: minio-scripts + templateRef: {{ include "minio.scriptsTplName" . }} + namespace: {{ .Release.Namespace }} + volumeName: scripts + defaultMode: 0555 + volumes: + - name: data + needSnapshot: true vars: - name: MINIO_ROOT_USER valueFrom: @@ -49,6 +72,16 @@ spec: valueFrom: clusterVarRef: namespace: Required + - name: CLUSTER_DOMAIN + value: {{ .Values.clusterDomain }} + - name: CERTS_PATH + value: {{ .Values.certsPath }} + - name: MINIO_API_PORT + value: {{ .Values.minioAPIPort }} + - name: MINIO_CONSOLE_PORT + value: {{ .Values.minioConsolePort }} + - name: HTTP_PROTOCOL + value: {{ .Values.tls.enabled | ternary "https" "http" }} lifecycleActions: roleProbe: exec: @@ -57,103 +90,26 @@ spec: - -c - | mc config host add minio http://127.0.0.1:9000 $MINIO_ROOT_USER $MINIO_ROOT_PASSWORD - - services: - - name: default - spec: - ports: - - name: api - port: 9000 - targetPort: api - - name: console - port: 9001 - targetPort: console - - configs: - - name: minio-configuration - templateRef: minio-configuration - namespace: {{ .Release.Namespace }} - volumeName: minio-config - defaultMode: 0777 - runtime: initContainers: - - command: - - /bin/sh - - -ce - - | - name="$MINIO_COMPONENT_NAME"-minio-configuration - namespace="$CLUSTER_NAMESPACE" - key="MINIO_REPLICAS_HISTORY" - cur=$(kubectl get configmaps "$name" -n "$namespace" -o jsonpath="{.data.$key}") - cur=$(echo "$cur" | tr -d '[]') - if [[ -z "$cur" ]]; then - new="[$MINIO_COMP_REPLICAS]" - else - IFS=',' set -- $cur - array="$@" - last=$(echo $array | awk '{print $NF}') - if [[ "$last" == "$MINIO_COMP_REPLICAS" ]]; then - new="[$cur]" - else - new="[$cur,$MINIO_COMP_REPLICAS]" - fi - fi - kubectl patch configmap "$name" -n "$namespace" --type strategic -p "{\"data\":{\"$key\":\"$new\"}}" - echo "ConfigMap $name updated successfully with $key=$new" + - name: init image: {{ .Values.image.registry | default "docker.io" }}/apecloud/kubeblocks-tools:0.8.2 imagePullPolicy: {{ default "IfNotPresent" .Values.image.pullPolicy }} - name: init + command: + - /bin/bash + - -c + - /scripts/initialize-patch-configmap.sh + volumeMounts: + - name: scripts + mountPath: /scripts containers: - name: minio image: {{ .Values.image.registry | default "docker.io" }}/{{ .Values.image.repository }}:{{ .Values.image.tag }} imagePullPolicy: {{ default "IfNotPresent" .Values.image.pullPolicy }} command: - - /bin/sh - - -ce - - | - FILE="/minio-config/MINIO_REPLICAS_HISTORY" - if [ ! -f "$FILE" ]; then - echo "minio config don't existed" - exit - fi - - # init bucket if set - buckets=$MINIO_BUCKETS - if [ -n "$buckets" ]; then - IFS=',' read -ra BUCKET_ARRAY <<< "$buckets" - for bucket in "${BUCKET_ARRAY[@]}"; do - bucket=$(echo $bucket) - - directory="/data/$bucket" - mkdir -p "$directory" - - if [ $? -eq 0 ]; then - echo "Successfully init bucket: $directory" - else - echo "Failed to init bucket: $directory" - fi - done - fi - - # read the content - content=$(cat "$FILE") - content=$(echo "$content" | tr -d '[]') - IFS=',' read -r -a replicas <<< "$content" - server="" - for ((i=0; i < ${#replicas[@]}; i++)); do - if [ $i -eq 0 ]; then - cur=${replicas[i]} - server+="{{ $scheme }}://$MINIO_COMPONENT_NAME-{0...$((cur-1))}.$MINIO_COMPONENT_NAME-headless.$CLUSTER_NAMESPACE.svc.{{ $.Values.clusterDomain }}/data" - else - prev=${replicas[i-1]} - cur=${replicas[i]} - server+=" {{ $scheme }}://$MINIO_COMPONENT_NAME-{$((prev))...$((cur-1))}.$MINIO_COMPONENT_NAME-headless.$CLUSTER_NAMESPACE.svc.{{ $.Values.clusterDomain }}/data" - fi - done - echo "the minio server pool is $server" - cmd="/usr/bin/docker-entrypoint.sh minio server "$server" -S {{ .Values.certsPath }} --address :{{ .Values.minioAPIPort }} --console-address :{{ .Values.minioConsolePort }} {{- template `minio.extraArgs` . }}" - $cmd + - /bin/bash + - -c + - /scripts/startup.sh ports: - name: api protocol: TCP @@ -171,8 +127,7 @@ spec: volumeMounts: - name: data mountPath: /data + - name: scripts + mountPath: /scripts - name: minio-config mountPath: /minio-config - volumes: - - name: data - needSnapshot: true diff --git a/addons/minio/templates/cpmv.yaml b/addons/minio/templates/cpmv.yaml index f804e0006..68e1a064e 100644 --- a/addons/minio/templates/cpmv.yaml +++ b/addons/minio/templates/cpmv.yaml @@ -1,4 +1,4 @@ -apiVersion: apps.kubeblocks.io/v1alpha1 +apiVersion: apps.kubeblocks.io/v1 kind: ComponentVersion metadata: name: minio @@ -9,7 +9,7 @@ spec: - releases: - 2024.6.29 compDefs: - - minio + - {{ include "minio.cmpdRegexpPattern" . }} releases: - name: 2024.6.29 serviceVersion: 2024.6.29 diff --git a/addons/minio/templates/configmap.yaml b/addons/minio/templates/minio-config-template.yaml similarity index 79% rename from addons/minio/templates/configmap.yaml rename to addons/minio/templates/minio-config-template.yaml index 4c745ce5a..219847f90 100644 --- a/addons/minio/templates/configmap.yaml +++ b/addons/minio/templates/minio-config-template.yaml @@ -1,7 +1,7 @@ apiVersion: v1 kind: ConfigMap metadata: - name: minio-configuration + name: {{ include "minio.configTplName" . }} namespace: {{ .Release.Namespace | quote }} labels: {{- include "minio.labels" . | nindent 4 }} diff --git a/addons/minio/templates/minio-scripts-template.yaml b/addons/minio/templates/minio-scripts-template.yaml new file mode 100644 index 000000000..b3346b999 --- /dev/null +++ b/addons/minio/templates/minio-scripts-template.yaml @@ -0,0 +1,10 @@ +apiVersion: v1 +kind: ConfigMap +metadata: + name: {{ include "minio.scriptsTplName" . }} + labels: + {{- include "minio.labels" . | nindent 4 }} +data: + {{- with include "minio.extend.scripts" . }} + {{- . | nindent 2 }} + {{- end }} \ No newline at end of file diff --git a/addons/minio/values.yaml b/addons/minio/values.yaml index af1c73b66..69862072d 100644 --- a/addons/minio/values.yaml +++ b/addons/minio/values.yaml @@ -2,6 +2,10 @@ ## clusterDomain: cluster.local +nameOverride: "" + +fullnameOverride: "" + ## Set default image, imageTag, and imagePullPolicy. mode is used to indicate the ## image: @@ -10,7 +14,8 @@ image: tag: RELEASE.2024-06-29T01-20-47Z pullPolicy: IfNotPresent -componentServiceVersion: +## @param define default serviceVersion +defaultServiceVersion: minio: 2024.6.29 ## Set default image, imageTag, and imagePullPolicy for the `mc` (the minio @@ -33,7 +38,7 @@ minioConsolePort: "9001" ## Directory on the MinIO pof certsPath: "/etc/minio/certs/" -configPathmc: "/etc/minio/mc/" +mcConfigPath: "/etc/minio/mc/" ## TLS Settings for MinIO tls: From db0a741f8893ca13eb9df696ad180d7fd9b09988 Mon Sep 17 00:00:00 2001 From: Y-Rookie Date: Fri, 18 Oct 2024 17:21:31 +0800 Subject: [PATCH 2/4] refactor minio addon --- .../scripts/initialize-patch-configmap.sh | 41 +++++++++---------- addons/minio/scripts/startup.sh | 23 ++++++----- addons/minio/templates/cmpd.yaml | 14 +++---- 3 files changed, 39 insertions(+), 39 deletions(-) diff --git a/addons/minio/scripts/initialize-patch-configmap.sh b/addons/minio/scripts/initialize-patch-configmap.sh index f648cef3e..d68507254 100644 --- a/addons/minio/scripts/initialize-patch-configmap.sh +++ b/addons/minio/scripts/initialize-patch-configmap.sh @@ -1,44 +1,43 @@ -#!/bin/bash +#!/bin/sh get_current_cm_key_value() { - local name=$1 - local namespace=$2 - local key=$3 + name="$1" + namespace="$2" + key="$3" kubectl get configmaps "$name" -n "$namespace" -o jsonpath="{.data.$key}" | tr -d '[]' } update_cm_key_value() { - local name=$1 - local namespace=$2 - local key=$3 - local new_value=$4 + name="$1" + namespace="$2" + key="$3" + new_value="$4" kubectl patch configmap "$name" -n "$namespace" --type strategic -p "{\"data\":{\"$key\":\"$new_value\"}}" } get_cm_key_new_value() { - local cur=$1 - local replicas=$2 + cur="$1" + replicas="$2" - if [[ -z "$cur" ]]; then - echo "[$replicas]" + if [ -z "$cur" ]; then + printf "[%s]" "$replicas" else - IFS=',' read -ra array <<< "$cur" - last=${array[-1]} - if [[ "$last" == "$replicas" ]]; then - echo "[$cur]" + last=$(echo "$cur" | awk -F, '{print $NF}') + if [ "$last" = "$replicas" ]; then + printf "[%s]" "$cur" else - echo "[$cur,$replicas]" + printf "[%s,%s]" "$cur" "$replicas" fi fi } update_configmap() { - local name="$MINIO_COMPONENT_NAME-minio-configuration" - local namespace="$CLUSTER_NAMESPACE" - local key="MINIO_REPLICAS_HISTORY" - local replicas="$MINIO_COMP_REPLICAS" + name="$MINIO_COMPONENT_NAME-minio-configuration" + namespace="$CLUSTER_NAMESPACE" + key="MINIO_REPLICAS_HISTORY" + replicas="$MINIO_COMP_REPLICAS" cur=$(get_current_cm_key_value "$name" "$namespace" "$key") new=$(get_cm_key_new_value "$cur" "$replicas") diff --git a/addons/minio/scripts/startup.sh b/addons/minio/scripts/startup.sh index d1769e44b..1e14bc9f4 100644 --- a/addons/minio/scripts/startup.sh +++ b/addons/minio/scripts/startup.sh @@ -20,22 +20,21 @@ read_replicas_history() { local file=$1 content=$(cat "$file") content=$(echo "$content" | tr -d '[]') - IFS=',' read -r -a replicas <<< "$content" - echo "${replicas[@]}" + echo "$content" } generate_server_pool() { - local replicas=("$@") + local replicas=$1 local server="" - for ((i=0; i < ${#replicas[@]}; i++)); do - if [ $i -eq 0 ]; then - cur=${replicas[i]} + prev=0 + IFS=',' read -ra REPLICAS_INDEX_ARRAY <<< "$replicas" + for cur in "${REPLICAS_INDEX_ARRAY[@]}"; do + if [ $prev -eq 0 ]; then server+=" $HTTP_PROTOCOL://$MINIO_COMPONENT_NAME-{0...$((cur-1))}.$MINIO_COMPONENT_NAME-headless.$CLUSTER_NAMESPACE.svc.$CLUSTER_DOMAIN/data" else - prev=${replicas[i-1]} - cur=${replicas[i]} - server+=" $HTTP_PROTOCOL://$MINIO_COMPONENT_NAME-{$((prev))...$((cur-1))}.$MINIO_COMPONENT_NAME-headless.$CLUSTER_NAMESPACE.svc.$CLUSTER_DOMAIN/data" + server+=" $HTTP_PROTOCOL://$MINIO_COMPONENT_NAME-{$prev...$((cur-1))}.$MINIO_COMPONENT_NAME-headless.$CLUSTER_NAMESPACE.svc.$CLUSTER_DOMAIN/data" fi + prev=$cur done echo "$server" } @@ -51,8 +50,10 @@ startup() { init_buckets "$buckets" fi - mapfile -t replicas < <(read_replicas_history "$replicas_history_file") - server=$(generate_server_pool "${replicas[@]}") + replicas=$(read_replicas_history "$replicas_history_file") + echo "the minio replicas history is $replicas" + + server=$(generate_server_pool $replicas) echo "the minio server pool is $server" cmd="/usr/bin/docker-entrypoint.sh minio server $server -S $CERTS_PATH --address :$MINIO_API_PORT --console-address :$MINIO_CONSOLE_PORT" diff --git a/addons/minio/templates/cmpd.yaml b/addons/minio/templates/cmpd.yaml index 2c1944294..b4f71406f 100644 --- a/addons/minio/templates/cmpd.yaml +++ b/addons/minio/templates/cmpd.yaml @@ -73,15 +73,15 @@ spec: clusterVarRef: namespace: Required - name: CLUSTER_DOMAIN - value: {{ .Values.clusterDomain }} + value: {{ .Values.clusterDomain | quote }} - name: CERTS_PATH - value: {{ .Values.certsPath }} + value: {{ .Values.certsPath | quote }} - name: MINIO_API_PORT - value: {{ .Values.minioAPIPort }} + value: {{ .Values.minioAPIPort | quote }} - name: MINIO_CONSOLE_PORT - value: {{ .Values.minioConsolePort }} + value: {{ .Values.minioConsolePort | quote}} - name: HTTP_PROTOCOL - value: {{ .Values.tls.enabled | ternary "https" "http" }} + value: {{ .Values.tls.enabled | ternary "https" "http" | quote }} lifecycleActions: roleProbe: exec: @@ -96,8 +96,8 @@ spec: image: {{ .Values.image.registry | default "docker.io" }}/apecloud/kubeblocks-tools:0.8.2 imagePullPolicy: {{ default "IfNotPresent" .Values.image.pullPolicy }} command: - - /bin/bash - - -c + - /bin/sh + - -ce - /scripts/initialize-patch-configmap.sh volumeMounts: - name: scripts From b5f9361a4597179203ffc4da7a7c9b808a61598d Mon Sep 17 00:00:00 2001 From: Y-Rookie Date: Fri, 18 Oct 2024 18:25:15 +0800 Subject: [PATCH 3/4] minio addon add ut --- addons/minio/Chart.yaml | 5 +- .../initialize_patch_configmap_spec.sh | 86 +++++++++++ addons/minio/scripts-ut-spec/startup_spec.sh | 146 ++++++++++++++++++ addons/minio/scripts/startup.sh | 21 ++- 4 files changed, 252 insertions(+), 6 deletions(-) create mode 100644 addons/minio/scripts-ut-spec/initialize_patch_configmap_spec.sh create mode 100644 addons/minio/scripts-ut-spec/startup_spec.sh diff --git a/addons/minio/Chart.yaml b/addons/minio/Chart.yaml index a68f4a1e2..8e14f3aec 100644 --- a/addons/minio/Chart.yaml +++ b/addons/minio/Chart.yaml @@ -1,8 +1,11 @@ apiVersion: v1 description: High Performance, Kubernetes Native Object Storage name: minio + version: 1.0.0-alpha.0 + appVersion: master + keywords: - storage - object-storage @@ -18,6 +21,6 @@ maintainers: url: https://github.com/apecloud/kubeblocks/ annotations: - addon.kubeblocks.io/kubeblocks-version: ">=0.8.0" + addon.kubeblocks.io/kubeblocks-version: ">=1.0.0" addon.kubeblocks.io/model: "object-storage" addon.kubeblocks.io/provider: "community" \ No newline at end of file diff --git a/addons/minio/scripts-ut-spec/initialize_patch_configmap_spec.sh b/addons/minio/scripts-ut-spec/initialize_patch_configmap_spec.sh new file mode 100644 index 000000000..f76af6a42 --- /dev/null +++ b/addons/minio/scripts-ut-spec/initialize_patch_configmap_spec.sh @@ -0,0 +1,86 @@ +# shellcheck shell=bash +# shellcheck disable=SC2034 + +# validate_shell_type_and_version defined in shellspec/spec_helper.sh used to validate the expected shell type and version this script needs to run. +if ! validate_shell_type_and_version "bash" 4 &>/dev/null; then + echo "initialize_patch_configmap_spec.sh skip cases because dependency bash version 4 or higher is not installed." + exit 0 +fi + +Describe "Minio init container bash script Tests" + # load the scripts to be tested and dependencies + Include ../scripts/initialize-patch-configmap.sh + + init() { + ut_mode="true" + } + BeforeAll "init" + + Describe "get_current_cm_key_value()" + It "returns the value of the specified key from the ConfigMap" + kubectl() { + echo "[1,2,3]" + } + + When call get_current_cm_key_value "my-configmap" "my-namespace" "replicas" + The output should eq "1,2,3" + End + End + + Describe "update_cm_key_value()" + It "updates the value of the specified key in the ConfigMap" + kubectl() { + return 0 + } + + When call update_cm_key_value "my-configmap" "my-namespace" "replicas" "[1,2,3,4]" + The status should be success + End + End + + Describe "get_cm_key_new_value()" + It "returns the replicas value when cur is empty" + When call get_cm_key_new_value "" "4" + The output should eq "[4]" + End + + It "returns the cur value when last equals replicas" + When call get_cm_key_new_value "1,2,3" "3" + The output should eq "[1,2,3]" + End + + It "appends the replicas value to cur when last does not equal replicas" + When call get_cm_key_new_value "1,2,3" "4" + The output should eq "[1,2,3,4]" + End + End + + Describe "update_configmap()" + setup() { + export MINIO_COMPONENT_NAME="minio" + export CLUSTER_NAMESPACE="default" + export MINIO_COMP_REPLICAS="4" + } + Before "setup" + + un_setup() { + unset MINIO_COMPONENT_NAME + unset CLUSTER_NAMESPACE + unset MINIO_COMP_REPLICAS + } + After "un_setup" + + It "updates the ConfigMap with the new replicas value" + get_current_cm_key_value() { + echo "1,2,3" + } + + update_cm_key_value() { + return 0 + } + + When run update_configmap + The output should eq "ConfigMap minio-minio-configuration updated successfully with MINIO_REPLICAS_HISTORY=[1,2,3,4]" + End + End +End \ No newline at end of file diff --git a/addons/minio/scripts-ut-spec/startup_spec.sh b/addons/minio/scripts-ut-spec/startup_spec.sh new file mode 100644 index 000000000..9597968f8 --- /dev/null +++ b/addons/minio/scripts-ut-spec/startup_spec.sh @@ -0,0 +1,146 @@ +# shellcheck shell=bash +# shellcheck disable=SC2034 + +# validate_shell_type_and_version defined in shellspec/spec_helper.sh used to validate the expected shell type and version this script needs to run. +if ! validate_shell_type_and_version "bash" 4 &>/dev/null; then + echo "initialize_patch_configmap_spec.sh skip cases because dependency bash version 4 or higher is not installed." + exit 0 +fi + +Describe "Minio startup bash script tests" + # load the scripts to be tested and dependencies + Include ../scripts/startup.sh + + init() { + replicas_history_file="./replicas_history" + ut_mode="true" + } + BeforeAll "init" + + cleanup() { + rm -f $replicas_history_file; + } + AfterAll 'cleanup' + + Describe "init_buckets()" + setup() { + bucket_dir="./data" + mkdir -p "$bucket_dir" + } + Before "setup" + + un_setup() { + rm -rf "$bucket_dir" + } + After "un_setup" + + It "creates directories for the specified buckets" + When call init_buckets "bucket1,bucket2,bucket3" + The output should include "Successfully init bucket: $bucket_dir/bucket1" + The output should include "Successfully init bucket: $bucket_dir/bucket2" + The output should include "Successfully init bucket: $bucket_dir/bucket3" + The directory "$bucket_dir/bucket1" should be exist + The directory "$bucket_dir/bucket2" should be exist + The directory "$bucket_dir/bucket3" should be exist + End + End + + Describe "read_replicas_history()" + setup() { + replicas_history_file="./replicas_history" + echo "[2,4,6]" > "$replicas_history_file" + } + Before "setup" + + un_setup() { + rm -f "$replicas_history_file" + } + After "un_setup" + + It "reads the replicas history from the specified file" + When call read_replicas_history "$replicas_history_file" + The output should eq "2,4,6" + End + End + + Describe "generate_server_pool()" + setup() { + export HTTP_PROTOCOL="http" + export MINIO_COMPONENT_NAME="minio-minio" + export CLUSTER_NAMESPACE="default" + export CLUSTER_DOMAIN="cluster.local" + } + Before "setup" + + un_setup() { + unset HTTP_PROTOCOL + unset MINIO_COMPONENT_NAME + unset CLUSTER_NAMESPACE + unset CLUSTER_DOMAIN + } + After "un_setup" + + It "generates the server pool based on the replicas" + When call generate_server_pool "2,4,6" + The output should eq " http://minio-minio-{0...1}.minio-minio-headless.default.svc.cluster.local/data http://minio-minio-{2...3}.minio-minio-headless.default.svc.cluster.local/data http://minio-minio-{4...5}.minio-minio-headless.default.svc.cluster.local/data" + End + End + + Describe "build_startup_cmd()" + setup() { + export HTTP_PROTOCOL="http" + export MINIO_COMPONENT_NAME="minio" + export MINIO_BUCKETS="bucket1,bucket2" + export CERTS_PATH="/certs" + export MINIO_API_PORT="9000" + export MINIO_CONSOLE_PORT="9001" + export CLUSTER_DOMAIN="cluster.local" + replicas_history_file="./replicas_history" + echo "[1,3,5]" > "$replicas_history_file" + } + Before "setup" + + un_setup() { + unset HTTP_PROTOCOL + unset MINIO_COMPONENT_NAME + unset MINIO_BUCKETS + unset CERTS_PATH + unset MINIO_API_PORT + unset MINIO_CONSOLE_PORT + unset CLUSTER_DOMAIN + rm -f "$replicas_history_file" + } + After "un_setup" + + It "builds the startup command with the generated server pool" + init_buckets() { + return 0 + } + + When call build_startup_cmd + The stderr should include "the minio replicas history is 1,3,5" + The output should eq "/usr/bin/docker-entrypoint.sh minio server http://minio-{0...0}.minio-headless..svc.cluster.local/data http://minio-{1...2}.minio-headless..svc.cluster.local/data http://minio-{3...4}.minio-headless..svc.cluster.local/data -S /certs --address :9000 --console-address :9001" + The status should be success + End + + It "returns status 1 when replicas history file does not exist" + replicas_history_file="/nonexistent" + + When run build_startup_cmd + The stderr should include "minio config don't existed" + The status should be failure + End + End + + Describe "startup()" + It "exits with status 1 when failed to build startup command" + build_startup_cmd() { + return 1 + } + + When run startup + The stderr should include "Failed to build startup command" + The status should be failure + End + End +End \ No newline at end of file diff --git a/addons/minio/scripts/startup.sh b/addons/minio/scripts/startup.sh index 1e14bc9f4..5a88dbf32 100644 --- a/addons/minio/scripts/startup.sh +++ b/addons/minio/scripts/startup.sh @@ -39,10 +39,10 @@ generate_server_pool() { echo "$server" } -startup() { +build_startup_cmd() { if [ ! -f "$replicas_history_file" ]; then - echo "minio config don't existed" - exit 1 + echo "minio config don't existed" >&2 + return 1 fi buckets="$MINIO_BUCKETS" @@ -51,12 +51,23 @@ startup() { fi replicas=$(read_replicas_history "$replicas_history_file") - echo "the minio replicas history is $replicas" + echo "the minio replicas history is $replicas" >&2 server=$(generate_server_pool $replicas) - echo "the minio server pool is $server" + echo "the minio server pool is $server" >&2 cmd="/usr/bin/docker-entrypoint.sh minio server $server -S $CERTS_PATH --address :$MINIO_API_PORT --console-address :$MINIO_CONSOLE_PORT" + echo "$cmd" + return 0 +} + +startup() { + cmd=$(build_startup_cmd) + status=$? + if [ $status -ne 0 ]; then + echo "Failed to build startup command" >&2 + exit 1 + fi echo "Starting minio server with command: $cmd" eval "$cmd" } From 70b53244c4c8fde1314188cc37a274903411725a Mon Sep 17 00:00:00 2001 From: Y-Rookie Date: Tue, 22 Oct 2024 20:10:53 +0800 Subject: [PATCH 4/4] fix pr commonet --- addons/minio/templates/_helpers.tpl | 10 ---------- addons/minio/templates/minio-scripts-template.yaml | 4 +--- 2 files changed, 1 insertion(+), 13 deletions(-) diff --git a/addons/minio/templates/_helpers.tpl b/addons/minio/templates/_helpers.tpl index fac3bc732..869ac34c0 100644 --- a/addons/minio/templates/_helpers.tpl +++ b/addons/minio/templates/_helpers.tpl @@ -58,16 +58,6 @@ Common minio annotations helm.sh/resource-policy: keep {{- end }} -{{/* -Generate minio scripts configmap -*/}} -{{- define "minio.extend.scripts" -}} -{{- range $path, $_ := $.Files.Glob "scripts/**" }} -{{ $path | base }}: |- -{{- $.Files.Get $path | nindent 2 }} -{{- end }} -{{- end }} - {{/* Define minio component definition name */}} diff --git a/addons/minio/templates/minio-scripts-template.yaml b/addons/minio/templates/minio-scripts-template.yaml index b3346b999..24aaf9182 100644 --- a/addons/minio/templates/minio-scripts-template.yaml +++ b/addons/minio/templates/minio-scripts-template.yaml @@ -5,6 +5,4 @@ metadata: labels: {{- include "minio.labels" . | nindent 4 }} data: - {{- with include "minio.extend.scripts" . }} - {{- . | nindent 2 }} - {{- end }} \ No newline at end of file +{{ (.Files.Glob "scripts/*.sh").AsConfig | indent 2 }} \ No newline at end of file