From e44e8b21a0f16e5008aa87ffc85491f8cbba0127 Mon Sep 17 00:00:00 2001
From: Y-Rookie <yong.huang@apecloud.com>
Date: Tue, 7 Jan 2025 16:31:42 +0800
Subject: [PATCH 1/6] pick redis sentinel acl & custom secret & redis cluster
 optimize to main

---
 addons-cluster/redis/templates/_helpers.tpl   | 20 ++++++
 addons-cluster/redis/values.schema.json       | 36 ++++++++++
 addons-cluster/redis/values.yaml              | 13 ++++
 .../redis-cluster-common.sh                   | 10 ++-
 .../redis-cluster-manage.sh                   | 72 +++++++++++++++++--
 .../redis-cluster-replica-pre-stop.sh         | 43 +++++++++++
 .../redis-cluster-server-start.sh             | 13 +++-
 .../scripts/redis-sentinel-post-start.sh      |  3 +-
 .../redis/scripts/redis-sentinel-start-v2.sh  | 17 ++++-
 .../redis/templates/cmpd-redis-cluster-7.yaml | 13 +++-
 10 files changed, 227 insertions(+), 13 deletions(-)
 create mode 100644 addons/redis/redis-cluster-scripts/redis-cluster-replica-pre-stop.sh

diff --git a/addons-cluster/redis/templates/_helpers.tpl b/addons-cluster/redis/templates/_helpers.tpl
index b39582264..39ba61fa7 100644
--- a/addons-cluster/redis/templates/_helpers.tpl
+++ b/addons-cluster/redis/templates/_helpers.tpl
@@ -18,12 +18,18 @@ Define redis cluster shardingSpec with ComponentDefinition.
     serviceVersion: {{ .Values.version }}
     systemAccounts:
     - name: default
+      {{- if and .Values.redisCluster.customSecretName .Values.redisCluster.customSecretNamespace }}
+      secretRef:
+        name: {{ .Values.redisCluster.customSecretName }}
+        namespace: {{ .Values.redisCluster.customSecretNamespace }}
+      {{- else }}
       passwordConfig:
         length: 10
         numDigits: 5
         numSymbols: 0
         letterCase: MixedCases
         seed: {{ include "kblib.clusterName" . }}
+      {{- end }}
     resources:
       limits:
         cpu: {{ .Values.cpu | quote }}
@@ -61,6 +67,13 @@ Define redis ComponentSpec with ComponentDefinition.
   {{- end }}
   serviceVersion: {{ .Values.version }}
   serviceAccountName: {{ include "kblib.serviceAccountName" . }}
+  {{- if and .Values.customSecretName .Values.customSecretNamespace }}
+  systemAccounts:
+    - name: default
+      secretRef:
+        name: {{ .Values.customSecretName }}
+        namespace: {{ .Values.customSecretNamespace }}
+  {{- end }}
   {{- include "kblib.componentResources" . | indent 2 }}
   {{- include "kblib.componentStorages" . | indent 2 }}
 {{- end }}
@@ -79,6 +92,13 @@ Define redis sentinel ComponentSpec with ComponentDefinition.
   {{- end }}
   serviceVersion: {{ .Values.version }}
   serviceAccountName: {{ include "kblib.serviceAccountName" . }}
+  {{- if and .Values.sentinel.customSecretName .Values.sentinel.customSecretNamespace }}
+  systemAccounts:
+    - name: default
+      secretRef:
+        name: {{ .Values.sentinel.customSecretName }}
+        namespace: {{ .Values.sentinel.customSecretNamespace }}
+  {{- end }}
   resources:
     limits:
       cpu: {{ .Values.sentinel.cpu | quote }}
diff --git a/addons-cluster/redis/values.schema.json b/addons-cluster/redis/values.schema.json
index ebdc82ac3..925d08ba1 100644
--- a/addons-cluster/redis/values.schema.json
+++ b/addons-cluster/redis/values.schema.json
@@ -73,6 +73,18 @@
       "title": "nodePortEnabled",
       "description":"Whether NodePort service is enabled, default is true"
     },
+    "customSecretName": {
+      "title": "the secret name for Redis default user's password",
+      "description": "the secret must contain keys named 'username' and 'password'",
+      "type": "string",
+      "default": ""
+    },
+    "customSecretNamespace": {
+      "title": "the secret namespace for Redis default user's password",
+      "description": "the secret must contain keys named 'username' and 'password'",
+      "type": "string",
+      "default": ""
+    },
     "twemproxy": {
       "title": "The redis twemproxy component",
       "type": "object",
@@ -119,6 +131,18 @@
           "type": "string",
           "default": ""
         },
+        "customSecretName": {
+          "title": "the secret name for Redis Sentinel default user's password",
+          "description": "the secret must contain keys named 'username' and 'password'",
+          "type": "string",
+          "default": ""
+        },
+        "customSecretNamespace": {
+          "title": "the secret namespace for Redis Sentinel default user's password",
+          "description": "the secret must contain keys named 'username' and 'password'",
+          "type": "string",
+          "default": ""
+        },
         "enabled": {
           "description":"Whether have sentinel component, default is true",
           "title": "sentinel.enable",
@@ -170,6 +194,18 @@
           "default": 3,
           "minimum": 3,
           "maximum": 2048
+        },
+        "customSecretName": {
+          "title": "the secret name for Redis Cluster default user's password",
+          "description": "the secret must contain keys named 'username' and 'password'",
+          "type": "string",
+          "default": ""
+        },
+        "customSecretNamespace": {
+          "title": "the secret namespace for Redis Cluster default user's password",
+          "description": "the secret must contain keys named 'username' and 'password'",
+          "type": "string",
+          "default": ""
         }
       }
     }
diff --git a/addons-cluster/redis/values.yaml b/addons-cluster/redis/values.yaml
index 66b6b8e20..d0c29a948 100644
--- a/addons-cluster/redis/values.yaml
+++ b/addons-cluster/redis/values.yaml
@@ -38,6 +38,11 @@ requests:
 ##
 storage: 20
 
+# custom password of default user for redis by passing a secret
+# the secret must contain keys named 'username' and 'password'
+customSecretName: ""
+customSecretNamespace: ""
+
 ## @param twemproxy.enable
 ## @param sentinel.cpu
 ## @param sentinel.memory
@@ -68,11 +73,19 @@ sentinel:
   storage: 20
   replicas: 3
   customMasterName: ""
+  # custom password of default user for redis sentinel by passing a secret
+  # the secret must contain keys named 'username' and 'password'
+  customSecretName: ""
+  customSecretNamespace: ""
 
 ## if mode is cluster, specify the redis cluster parameters
 ## @param redisCluster.shardCount specify the number of shards in the cluster, the minimum value is 3
 redisCluster:
   shardCount: 3
+  # custom password of default user for redis cluster by passing a secret
+  # the secret must contain keys named 'username' and 'password'
+  customSecretName: ""
+  customSecretNamespace: ""
 
 extra:
   rbacEnabled: true
diff --git a/addons/redis/redis-cluster-scripts/redis-cluster-common.sh b/addons/redis/redis-cluster-scripts/redis-cluster-common.sh
index 13cbd6be3..74cf56e58 100644
--- a/addons/redis/redis-cluster-scripts/redis-cluster-common.sh
+++ b/addons/redis/redis-cluster-scripts/redis-cluster-common.sh
@@ -151,11 +151,13 @@ shutdown_redis_server() {
 
 check_redis_server_ready() {
   unset_xtrace_when_ut_mode_false
+  local host="$1"
+  local port="$2"
   local max_retry=10
   local retry_interval=5
-  check_ready_cmd="redis-cli -h 127.0.0.1 -p $service_port ping"
+  check_ready_cmd="redis-cli -h $host -p $port ping"
   if ! is_empty "$REDIS_DEFAULT_PASSWORD"; then
-    check_ready_cmd="redis-cli -h 127.0.0.1 -p $service_port -a $REDIS_DEFAULT_PASSWORD ping"
+    check_ready_cmd="redis-cli -h $host -p $port -a $REDIS_DEFAULT_PASSWORD ping"
   fi
   set_xtrace_when_ut_mode_false
   output=$($check_ready_cmd)
@@ -408,8 +410,10 @@ check_node_in_cluster_with_retry() {
 }
 
 check_redis_server_ready_with_retry() {
+  local host="$1"
+  local port="$2"
   # call the execute_check_redis_server_ready_command function with call_func_with_retry function and get the output
-  check_result=$(call_func_with_retry $check_ready_times $retry_delay_second check_redis_server_ready)
+  check_result=$(call_func_with_retry $check_ready_times $retry_delay_second check_redis_server_ready "$host" "$port")
   status=$?
   if [ $status -ne 0 ]; then
     echo "Failed to check the redis server ready after retry" >&2
diff --git a/addons/redis/redis-cluster-scripts/redis-cluster-manage.sh b/addons/redis/redis-cluster-scripts/redis-cluster-manage.sh
index 90632f3e8..c3d5a9de7 100644
--- a/addons/redis/redis-cluster-scripts/redis-cluster-manage.sh
+++ b/addons/redis/redis-cluster-scripts/redis-cluster-manage.sh
@@ -41,6 +41,19 @@ load_redis_cluster_common_utils() {
   source "${redis_cluster_common_library_file}"
 }
 
+check_initialize_nodes_ready() {
+  local nodes=("$@")
+  for node in "${nodes[@]}"; do
+    local host port
+    host=$(echo "$node" | cut -d':' -f1)
+    port=$(echo "$node" | cut -d':' -f2)
+    if ! check_redis_server_ready_with_retry "$host" "$port"; then
+      return 1
+    fi
+  done
+  return 0
+}
+
 # initialize the other component and pods info
 init_other_components_and_pods_info() {
   local current_component="$1"
@@ -605,17 +618,40 @@ initialize_redis_cluster() {
     return 1
   fi
 
-  # initialize all the primary nodes
+  # generate primary and secondary nodes
   gen_initialize_redis_cluster_primary_node
+  gen_initialize_redis_cluster_secondary_nodes
+
   if [ ${#initialize_redis_cluster_primary_nodes[@]} -eq 0 ] || [ ${#initialize_redis_cluster_primary_nodes[@]} -lt 3 ]; then
     echo "Failed to get primary nodes or the primary nodes count is less than 3" >&2
     return 1
   fi
+
+  # check all the primary nodes are ready
   local primary_nodes=""
-  for primary_pod_name in "${!initialize_redis_cluster_primary_nodes[@]}"; do
-    primary_nodes+="${initialize_redis_cluster_primary_nodes[$primary_pod_name]} "
+  local primary_node_list=()
+  for pod_name in "${!initialize_redis_cluster_primary_nodes[@]}"; do
+    primary_nodes+="${initialize_redis_cluster_primary_nodes[$pod_name]} "
+    primary_node_list+=("${initialize_redis_cluster_primary_nodes[$pod_name]}")
   done
+  if ! check_initialize_nodes_ready "${primary_node_list[@]}"; then
+    echo "Primary nodes health check failed" >&2
+    return 1
+  fi
 
+  # check all the secondary nodes are ready
+  if [ ${#initialize_redis_cluster_secondary_nodes[@]} -gt 0 ]; then
+    secondary_node_list=()
+    for pod_name in "${!initialize_redis_cluster_secondary_nodes[@]}"; do
+      secondary_node_list+=("${initialize_redis_cluster_secondary_nodes[$pod_name]}")
+    done
+    if ! check_initialize_nodes_ready "${secondary_node_list[@]}"; then
+      echo "Secondary nodes health check failed" >&2
+      return 1
+    fi
+  fi
+
+  # initialize all the primary nodes
   if create_redis_cluster "$primary_nodes"; then
     echo "Redis cluster initialized primary nodes successfully, cluster nodes: $primary_nodes"
   else
@@ -633,7 +669,6 @@ initialize_redis_cluster() {
   fi
 
   # initialize all the secondary nodes
-  gen_initialize_redis_cluster_secondary_nodes
   if [ ${#initialize_redis_cluster_secondary_nodes[@]} -eq 0 ]; then
     echo "No secondary nodes to initialize"
     return 0
@@ -664,10 +699,39 @@ initialize_redis_cluster() {
     echo "Redis cluster initialized secondary node $secondary_pod_name successfully"
     # waiting for all nodes sync the information
     sleep_when_ut_mode_false 5
+
+    # verify secondary node is already in all primary nodes
+    if ! verify_secondary_in_all_primaries "$secondary_endpoint_with_port" "${primary_node_list[@]}"; then
+      echo "Failed to verify secondary node $secondary_endpoint_with_port in all primary nodes" >&2
+      return 1
+    fi
+    echo "Secondary node $secondary_endpoint_with_port successfully joined the cluster and verified in all primaries"
   done
   return 0
 }
 
+verify_secondary_in_all_primaries() {
+  local secondary_pod_name="$1"
+  local primary_nodes=("$@")
+  # Skip the first argument
+  shift
+  all_check_ok=true
+  for primary_node in "$@"; do
+    local primary_host primary_port
+    primary_host=$(echo "$primary_node" | cut -d':' -f1)
+    primary_port=$(echo "$primary_node" | cut -d':' -f2)
+    if ! check_node_in_cluster_with_retry "$primary_host" "$primary_port" "$secondary_pod_name"; then
+      echo "Secondary node $secondary_pod_name not found in primary $primary_node after retry" >&2
+      all_check_ok=false
+      contnue
+    fi
+  done
+  if ! all_check_ok; then
+    return 1
+  fi
+  return 0
+}
+
 scale_out_redis_cluster_shard() {
   if is_empty "$CURRENT_SHARD_COMPONENT_SHORT_NAME" || is_empty "$KB_CLUSTER_POD_NAME_LIST" || is_empty "$KB_CLUSTER_POD_HOST_IP_LIST" || is_empty "$KB_CLUSTER_COMPONENT_POD_NAME_LIST" || is_empty "$KB_CLUSTER_COMPONENT_POD_HOST_IP_LIST"; then
     echo "Error: Required environment variable CURRENT_SHARD_COMPONENT_SHORT_NAME, KB_CLUSTER_POD_NAME_LIST, KB_CLUSTER_POD_HOST_IP_LIST, KB_CLUSTER_COMPONENT_POD_NAME_LIST and KB_CLUSTER_COMPONENT_POD_HOST_IP_LIST are not set when scale out redis cluster shard" >&2
diff --git a/addons/redis/redis-cluster-scripts/redis-cluster-replica-pre-stop.sh b/addons/redis/redis-cluster-scripts/redis-cluster-replica-pre-stop.sh
new file mode 100644
index 000000000..b9630d627
--- /dev/null
+++ b/addons/redis/redis-cluster-scripts/redis-cluster-replica-pre-stop.sh
@@ -0,0 +1,43 @@
+#!/bin/bash
+
+# shellcheck disable=SC2034
+ut_mode="false"
+test || __() {
+  # when running in non-unit test mode, set the options "set -e".
+  set -ex;
+}
+
+load_common_library() {
+  # the common.sh scripts is mounted to the same path which is defined in the cmpd.spec.scripts
+  common_library_file="/scripts/common.sh"
+  # shellcheck disable=SC1090
+  source "${common_library_file}"
+}
+
+acl_save_before_stop() {
+  if ! is_empty "$REDIS_DEFAULT_PASSWORD"; then
+    acl_save_command="redis-cli -h localhost -p $SERVICE_PORT -a $REDIS_DEFAULT_PASSWORD acl save"
+    logging_mask_acl_save_command="${acl_save_command/$REDIS_DEFAULT_PASSWORD/********}"
+  else
+    acl_save_command="redis-cli -h localhost -p $SERVICE_PORT acl save"
+    logging_mask_acl_save_command="$acl_save_command"
+  fi
+  echo "acl save command: $logging_mask_acl_save_command"
+  if output=$($acl_save_command 2>&1); then
+    echo "acl save command executed successfully: $output"
+  else
+    echo "failed to execute acl save command: $output"
+    exit 1
+  fi
+}
+
+# This is magic for shellspec ut framework.
+# Sometime, functions are defined in a single shell script.
+# You will want to test it. but you do not want to run the script.
+# When included from shellspec, __SOURCED__ variable defined and script
+# end here. The script path is assigned to the __SOURCED__ variable.
+${__SOURCED__:+false} : || return 0
+
+# main
+load_common_library
+acl_save_before_stop
\ No newline at end of file
diff --git a/addons/redis/redis-cluster-scripts/redis-cluster-server-start.sh b/addons/redis/redis-cluster-scripts/redis-cluster-server-start.sh
index 872e86b2f..9e2cb9874 100755
--- a/addons/redis/redis-cluster-scripts/redis-cluster-server-start.sh
+++ b/addons/redis/redis-cluster-scripts/redis-cluster-server-start.sh
@@ -65,10 +65,17 @@ check_and_correct_other_primary_nodes() {
     while true; do
       current_announce_ip=$(get_cluster_announce_ip "$node_endpoint" "$node_port")
       echo "original_announce_ip: $original_announce_ip, node_endpoint_with_port: $node_endpoint_with_port, current_announce_ip: $current_announce_ip"
-      # if current_announce_ip is empty, we need to retry
+      # if current_announce_ip is empty, retry it
       if is_empty "$current_announce_ip"; then
+        echo "Error: current_announce_ip is empty"
         sleep_when_ut_mode_false 3
-        echo "current_announce_ip is empty, retry..."
+        continue
+      fi
+
+      if [ "$node_port" -eq 0 ] || [ "$node_bus_port" -eq 0 ]; then
+        echo "Error: node_port or node_bus_port is 0"
+        sleep_when_ut_mode_false 3
+        # TODO: get other_comp_primary_nodes again
         continue
       fi
 
@@ -221,7 +228,7 @@ get_current_comp_nodes_for_scale_out_replica() {
 # scale out replica of redis cluster shard if needed
 scale_redis_cluster_replica() {
   # Waiting for redis-server to start
-  if check_redis_server_ready_with_retry ; then
+  if check_redis_server_ready_with_retry "127.0.0.1" "$service_port"; then
     echo "Redis server is ready, continue to scale out replica..."
   else
     echo "Redis server is not ready, exit scale out replica..." >&2
diff --git a/addons/redis/scripts/redis-sentinel-post-start.sh b/addons/redis/scripts/redis-sentinel-post-start.sh
index 959a9b944..98d298458 100755
--- a/addons/redis/scripts/redis-sentinel-post-start.sh
+++ b/addons/redis/scripts/redis-sentinel-post-start.sh
@@ -30,7 +30,8 @@ acl_set_user_for_redis_sentinel() {
   # set default user password and replication user password
   if [ -n "$SENTINEL_PASSWORD" ]; then
     until redis-cli -h localhost -p $SENTINEL_SERVICE_PORT -a $SENTINEL_PASSWORD ping; do sleep 1; done
-    redis-cli -h localhost -p $SENTINEL_SERVICE_PORT ACL SETUSER $SENTINEL_USER ON \>$SENTINEL_PASSWORD allchannels +@all
+    redis-cli -h localhost -p $SENTINEL_SERVICE_PORT -a $SENTINEL_PASSWORD ACL SETUSER $SENTINEL_USER ON \>$SENTINEL_PASSWORD allchannels +@all
+    redis-cli -h localhost -p $SENTINEL_SERVICE_PORT -a $SENTINEL_PASSWORD ACL SAVE
     echo "redis sentinel user and password set successfully."
   fi
 }
diff --git a/addons/redis/scripts/redis-sentinel-start-v2.sh b/addons/redis/scripts/redis-sentinel-start-v2.sh
index e3e9de2e0..11d23380e 100755
--- a/addons/redis/scripts/redis-sentinel-start-v2.sh
+++ b/addons/redis/scripts/redis-sentinel-start-v2.sh
@@ -70,6 +70,14 @@ parse_redis_sentinel_announce_addr() {
   fi
 }
 
+rebuild_redis_sentinel_acl_file() {
+  if [ -f /data/users.acl ]; then
+    sed -i "/user default on/d" /data/users.acl
+  else
+    touch /data/users.acl
+  fi
+}
+
 reset_redis_sentinel_conf() {
   echo "reset redis sentinel conf"
   sentinel_port=26379
@@ -79,6 +87,7 @@ reset_redis_sentinel_conf() {
   mkdir -p $redis_sentinel_conf_dir
   if [ -f $redis_sentinel_real_conf ]; then
     sed "/sentinel announce-ip/d" $redis_sentinel_real_conf > $redis_sentinel_real_conf_bak && mv $redis_sentinel_real_conf_bak $redis_sentinel_real_conf
+    sed "/sentinel announce-port/d" $redis_sentinel_real_conf > $redis_sentinel_real_conf_bak && mv $redis_sentinel_real_conf_bak $redis_sentinel_real_conf
     sed "/sentinel resolve-hostnames/d" $redis_sentinel_real_conf > $redis_sentinel_real_conf_bak && mv $redis_sentinel_real_conf_bak $redis_sentinel_real_conf
     sed "/sentinel announce-hostnames/d" $redis_sentinel_real_conf > $redis_sentinel_real_conf_bak && mv $redis_sentinel_real_conf_bak $redis_sentinel_real_conf
     unset_xtrace_when_ut_mode_false
@@ -88,6 +97,9 @@ reset_redis_sentinel_conf() {
     fi
     set_xtrace_when_ut_mode_false
     sed "/port $sentinel_port/d" $redis_sentinel_real_conf > $redis_sentinel_real_conf_bak && mv $redis_sentinel_real_conf_bak $redis_sentinel_real_conf
+    sed "/aclfile \/data\/users.acl/d" $redis_sentinel_real_conf > $redis_sentinel_real_conf_bak && mv $redis_sentinel_real_conf_bak $redis_sentinel_real_conf
+    # backward compatible for previous versions without ACL
+    sed "/user default on/d" $redis_sentinel_real_conf > $redis_sentinel_real_conf_bak && mv $redis_sentinel_real_conf_bak $redis_sentinel_real_conf
   fi
 
   # hack for redis sentinel when nodeport is enabled, remove known-replica line which has the same nodeport port with master
@@ -98,7 +110,8 @@ reset_redis_sentinel_conf() {
       if [[ $line =~ ^sentinel[[:space:]]+monitor[[:space:]]+([^[:space:]]+)[[:space:]]+[^[:space:]]+[[:space:]]+([^[:space:]]+) ]]; then
         master_name="${BASH_REMATCH[1]}"
         master_port="${BASH_REMATCH[2]}"
-        sed -i "/^sentinel known-replica ${master_name} .* ${master_port}$/d" /data/sentinel/redis-sentinel.conf
+        sed "/^sentinel known-replica ${master_name} .* ${master_port}$/d" $redis_sentinel_real_conf > $redis_sentinel_real_conf_bak && mv $redis_sentinel_real_conf_bak $redis_sentinel_real_conf
+        sed "/^sentinel known-sentinel ${master_name}/d" $redis_sentinel_real_conf > $redis_sentinel_real_conf_bak && mv $redis_sentinel_real_conf_bak $redis_sentinel_real_conf
       fi
     done < "$temp_file"
     rm -f "$temp_file"
@@ -150,6 +163,7 @@ build_redis_sentinel_conf() {
     } >> $redis_sentinel_real_conf
   fi
   set_xtrace_when_ut_mode_false
+  echo "aclfile /data/users.acl">> $redis_sentinel_real_conf
   echo "build redis sentinel conf succeeded!"
 }
 
@@ -169,6 +183,7 @@ ${__SOURCED__:+false} : || return 0
 # main
 load_common_library
 parse_redis_sentinel_announce_addr "$CURRENT_POD_NAME"
+rebuild_redis_sentinel_acl_file
 reset_redis_sentinel_conf
 build_redis_sentinel_conf
 start_redis_sentinel_server
\ No newline at end of file
diff --git a/addons/redis/templates/cmpd-redis-cluster-7.yaml b/addons/redis/templates/cmpd-redis-cluster-7.yaml
index 22b96dbd2..06173eecd 100644
--- a/addons/redis/templates/cmpd-redis-cluster-7.yaml
+++ b/addons/redis/templates/cmpd-redis-cluster-7.yaml
@@ -27,6 +27,7 @@ spec:
       podService: true
       disableAutoProvision: true
   updateStrategy: BestEffortParallel
+  podManagementPolicy: OrderedReady
   volumes:
     - name: data
       needSnapshot: true
@@ -365,6 +366,16 @@ spec:
           - /scripts/redis-cluster-manage.sh --pre-terminate
       retryPolicy:
         maxRetries: 10
+    memberLeave:
+      exec:
+        image: {{ include "redis7.image" . }}
+        container: redis-cluster
+        command:
+          - /bin/bash
+          - -c
+          - /scripts/redis-cluster-replica-member-leave.sh
+      retryPolicy:
+        maxRetries: 10
   runtime:
     initContainers:
       - name: init-dbctl
@@ -433,7 +444,7 @@ spec:
               command:
                 - /bin/bash
                 - -c
-                - /scripts/redis-cluster-replica-member-leave.sh
+                - /scripts/redis-cluster-replica-pre-stop.sh
       - name: metrics
         image: {{ include "metrics.image" . }}
         imagePullPolicy: {{ .Values.metrics.image.pullPolicy | quote }}

From 6551b7677045077f4fc01dc133eea5a26c48c40a Mon Sep 17 00:00:00 2001
From: Y-Rookie <yong.huang@apecloud.com>
Date: Tue, 7 Jan 2025 16:41:03 +0800
Subject: [PATCH 2/6] pick redis sentinel acl & custom secret & redis cluster
 optimize to main

---
 .../redis-cluster-manage.sh                    | 18 +++++++++++-------
 1 file changed, 11 insertions(+), 7 deletions(-)

diff --git a/addons/redis/redis-cluster-scripts/redis-cluster-manage.sh b/addons/redis/redis-cluster-scripts/redis-cluster-manage.sh
index c3d5a9de7..7be741b4e 100644
--- a/addons/redis/redis-cluster-scripts/redis-cluster-manage.sh
+++ b/addons/redis/redis-cluster-scripts/redis-cluster-manage.sh
@@ -673,6 +673,8 @@ initialize_redis_cluster() {
     echo "No secondary nodes to initialize"
     return 0
   fi
+
+  all_secondaries_ready=true
   for secondary_pod_name in "${!initialize_redis_cluster_secondary_nodes[@]}"; do
     secondary_endpoint_with_port=${initialize_redis_cluster_secondary_nodes["$secondary_pod_name"]}
     # shellcheck disable=SC2001
@@ -703,10 +705,17 @@ initialize_redis_cluster() {
     # verify secondary node is already in all primary nodes
     if ! verify_secondary_in_all_primaries "$secondary_endpoint_with_port" "${primary_node_list[@]}"; then
       echo "Failed to verify secondary node $secondary_endpoint_with_port in all primary nodes" >&2
-      return 1
+      all_secondaries_ready=false
+      continue
     fi
     echo "Secondary node $secondary_endpoint_with_port successfully joined the cluster and verified in all primaries"
   done
+
+  if ! all_secondaries_ready; then
+    echo "Failed to initialize all secondary nodes" >&2
+    return 1
+  fi
+  echo "Redis cluster initialized all secondary nodes successfully"
   return 0
 }
 
@@ -715,20 +724,15 @@ verify_secondary_in_all_primaries() {
   local primary_nodes=("$@")
   # Skip the first argument
   shift
-  all_check_ok=true
   for primary_node in "$@"; do
     local primary_host primary_port
     primary_host=$(echo "$primary_node" | cut -d':' -f1)
     primary_port=$(echo "$primary_node" | cut -d':' -f2)
     if ! check_node_in_cluster_with_retry "$primary_host" "$primary_port" "$secondary_pod_name"; then
       echo "Secondary node $secondary_pod_name not found in primary $primary_node after retry" >&2
-      all_check_ok=false
-      contnue
+      return 1
     fi
   done
-  if ! all_check_ok; then
-    return 1
-  fi
   return 0
 }
 

From 7731267de0118c388b9a267e9a66de150c8b5606 Mon Sep 17 00:00:00 2001
From: Y-Rookie <yong.huang@apecloud.com>
Date: Tue, 7 Jan 2025 16:55:34 +0800
Subject: [PATCH 3/6] pick redis sentinel acl & custom secret & redis cluster
 optimize to main

---
 .../redis-cluster-scripts/redis-cluster-manage.sh      | 10 ++++++++--
 1 file changed, 8 insertions(+), 2 deletions(-)

diff --git a/addons/redis/redis-cluster-scripts/redis-cluster-manage.sh b/addons/redis/redis-cluster-scripts/redis-cluster-manage.sh
index 7be741b4e..a99bd582b 100644
--- a/addons/redis/redis-cluster-scripts/redis-cluster-manage.sh
+++ b/addons/redis/redis-cluster-scripts/redis-cluster-manage.sh
@@ -711,7 +711,7 @@ initialize_redis_cluster() {
     echo "Secondary node $secondary_endpoint_with_port successfully joined the cluster and verified in all primaries"
   done
 
-  if ! all_secondaries_ready; then
+  if [ "$all_secondaries_ready" = false ]; then
     echo "Failed to initialize all secondary nodes" >&2
     return 1
   fi
@@ -728,7 +728,13 @@ verify_secondary_in_all_primaries() {
     local primary_host primary_port
     primary_host=$(echo "$primary_node" | cut -d':' -f1)
     primary_port=$(echo "$primary_node" | cut -d':' -f2)
-    if ! check_node_in_cluster_with_retry "$primary_host" "$primary_port" "$secondary_pod_name"; then
+    retry_count=0
+    while ! check_node_in_cluster "$primary_host" "$primary_port" "$secondary_pod_name" && [ $retry_count -lt 30 ]; do
+      sleep 3
+      ((retry_count++))
+    done
+    # shellcheck disable=SC2086
+    if [ $retry_count -eq 30 ]; then
       echo "Secondary node $secondary_pod_name not found in primary $primary_node after retry" >&2
       return 1
     fi

From e4b459dcd41c07ab793c12d9f53cbea688d27db1 Mon Sep 17 00:00:00 2001
From: Y-Rookie <yong.huang@apecloud.com>
Date: Tue, 7 Jan 2025 17:02:33 +0800
Subject: [PATCH 4/6] pick redis sentinel acl & custom secret & redis cluster
 optimize to main

---
 addons/redis/redis-cluster-scripts/redis-cluster-manage.sh | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/addons/redis/redis-cluster-scripts/redis-cluster-manage.sh b/addons/redis/redis-cluster-scripts/redis-cluster-manage.sh
index a99bd582b..d834e22af 100644
--- a/addons/redis/redis-cluster-scripts/redis-cluster-manage.sh
+++ b/addons/redis/redis-cluster-scripts/redis-cluster-manage.sh
@@ -703,12 +703,12 @@ initialize_redis_cluster() {
     sleep_when_ut_mode_false 5
 
     # verify secondary node is already in all primary nodes
-    if ! verify_secondary_in_all_primaries "$secondary_endpoint_with_port" "${primary_node_list[@]}"; then
-      echo "Failed to verify secondary node $secondary_endpoint_with_port in all primary nodes" >&2
+    if ! verify_secondary_in_all_primaries "$secondary_pod_name" "${primary_node_list[@]}"; then
+      echo "Failed to verify secondary node $secondary_pod_name in all primary nodes" >&2
       all_secondaries_ready=false
       continue
     fi
-    echo "Secondary node $secondary_endpoint_with_port successfully joined the cluster and verified in all primaries"
+    echo "Secondary node $secondary_pod_name successfully joined the cluster and verified in all primaries"
   done
 
   if [ "$all_secondaries_ready" = false ]; then

From 09b9919ec67a8fc5af9954a07438e9caed3e2c08 Mon Sep 17 00:00:00 2001
From: Y-Rookie <yong.huang@apecloud.com>
Date: Tue, 7 Jan 2025 17:20:00 +0800
Subject: [PATCH 5/6] pick redis sentinel acl & custom secret & redis cluster
 optimize to main

---
 .../redis-cluster-scripts/redis-cluster-server-start.sh    | 7 +++++++
 1 file changed, 7 insertions(+)

diff --git a/addons/redis/redis-cluster-scripts/redis-cluster-server-start.sh b/addons/redis/redis-cluster-scripts/redis-cluster-server-start.sh
index 9e2cb9874..9ccc313fe 100755
--- a/addons/redis/redis-cluster-scripts/redis-cluster-server-start.sh
+++ b/addons/redis/redis-cluster-scripts/redis-cluster-server-start.sh
@@ -235,6 +235,13 @@ scale_redis_cluster_replica() {
     exit 1
   fi
 
+  if [ -f /data/nodes.conf ]; then
+    echo "the nodes.conf file after redis server start:"
+    cat /data/nodes.conf
+  else
+    echo "the nodes.conf file after redis server start is not exist"
+  fi
+
   # get the current component nodes for scale out replica
   target_node_name=$(min_lexicographical_order_pod "$CURRENT_SHARD_POD_NAME_LIST")
   if ! is_empty "$CURRENT_SHARD_PRIMARY_POD_NAME"; then

From a14f0592e0148d342c082347b2ea474a5e92a1ae Mon Sep 17 00:00:00 2001
From: Y-Rookie <yong.huang@apecloud.com>
Date: Tue, 7 Jan 2025 17:28:39 +0800
Subject: [PATCH 6/6] fix script ut

---
 .../redis-cluster-manage.sh                   |  2 +-
 .../redis_cluster_manage_spec.sh              | 24 +++++++++++++++++++
 2 files changed, 25 insertions(+), 1 deletion(-)

diff --git a/addons/redis/redis-cluster-scripts/redis-cluster-manage.sh b/addons/redis/redis-cluster-scripts/redis-cluster-manage.sh
index d834e22af..a75d39c64 100644
--- a/addons/redis/redis-cluster-scripts/redis-cluster-manage.sh
+++ b/addons/redis/redis-cluster-scripts/redis-cluster-manage.sh
@@ -730,7 +730,7 @@ verify_secondary_in_all_primaries() {
     primary_port=$(echo "$primary_node" | cut -d':' -f2)
     retry_count=0
     while ! check_node_in_cluster "$primary_host" "$primary_port" "$secondary_pod_name" && [ $retry_count -lt 30 ]; do
-      sleep 3
+      sleep_when_ut_mode_false 3
       ((retry_count++))
     done
     # shellcheck disable=SC2086
diff --git a/addons/redis/scripts-ut-spec/redis_cluster_manage_spec.sh b/addons/redis/scripts-ut-spec/redis_cluster_manage_spec.sh
index 13ed282a7..083726f7d 100644
--- a/addons/redis/scripts-ut-spec/redis_cluster_manage_spec.sh
+++ b/addons/redis/scripts-ut-spec/redis_cluster_manage_spec.sh
@@ -749,6 +749,10 @@ d-98x-redis-advertised-1:31318.shard-7hy@redis-shard-7hy-redis-advertised-0:3202
         initialize_redis_cluster_primary_nodes["redis-shard-jwl-0"]="10.42.0.5:6379"
       }
 
+      check_initialize_nodes_ready() {
+        return 0
+      }
+
       create_redis_cluster() {
         return 1
       }
@@ -780,6 +784,10 @@ d-98x-redis-advertised-1:31318.shard-7hy@redis-shard-7hy-redis-advertised-0:3202
         initialize_redis_cluster_primary_nodes["redis-shard-jwl-0"]="10.42.0.5:6379"
       }
 
+      check_initialize_nodes_ready() {
+        return 0
+      }
+
       create_redis_cluster() {
         return 0
       }
@@ -818,6 +826,10 @@ d-98x-redis-advertised-1:31318.shard-7hy@redis-shard-7hy-redis-advertised-0:3202
         initialize_redis_cluster_primary_nodes["redis-shard-jwl-0"]="10.42.0.5:6379"
       }
 
+      check_initialize_nodes_ready() {
+        return 0
+      }
+
       create_redis_cluster() {
         return 0
       }
@@ -859,6 +871,10 @@ d-98x-redis-advertised-1:31318.shard-7hy@redis-shard-7hy-redis-advertised-0:3202
         initialize_redis_cluster_primary_nodes["redis-shard-jwl-0"]="10.42.0.5:6379"
       }
 
+      check_initialize_nodes_ready() {
+        return 0
+      }
+
       create_redis_cluster() {
         return 0
       }
@@ -906,6 +922,10 @@ d-98x-redis-advertised-1:31318.shard-7hy@redis-shard-7hy-redis-advertised-0:3202
         initialize_redis_cluster_primary_nodes["redis-shard-jwl-0"]="10.42.0.5:6379"
       }
 
+      check_initialize_nodes_ready() {
+        return 0
+      }
+
       create_redis_cluster() {
         return 0
       }
@@ -955,6 +975,10 @@ d-98x-redis-advertised-1:31318.shard-7hy@redis-shard-7hy-redis-advertised-0:3202
         initialize_redis_cluster_primary_nodes["redis-shard-7hy-0"]="10.42.0.3:6379"
         initialize_redis_cluster_primary_nodes["redis-shard-jwl-0"]="10.42.0.5:6379"
       }
+      
+      check_initialize_nodes_ready() {
+        return 0
+      }
 
       create_redis_cluster() {
         return 0