chore: pick redis sentinel acl & custom secret password & redis cluster optimize to main

addons-cluster/redis/templates/_helpers.tpl

@@ -18,12 +18,18 @@ Define redis cluster shardingSpec with ComponentDefinition.
     serviceVersion: {{ .Values.version }}
     systemAccounts:
     - name: default
+      {{- if and .Values.redisCluster.customSecretName .Values.redisCluster.customSecretNamespace }}
+      secretRef:
+        name: {{ .Values.redisCluster.customSecretName }}
+        namespace: {{ .Values.redisCluster.customSecretNamespace }}
+      {{- else }}
       passwordConfig:
         length: 10
         numDigits: 5
         numSymbols: 0
         letterCase: MixedCases
         seed: {{ include "kblib.clusterName" . }}
+      {{- end }}
     resources:
       limits:
         cpu: {{ .Values.cpu | quote }}
@@ -61,6 +67,13 @@ Define redis ComponentSpec with ComponentDefinition.
   {{- end }}
   serviceVersion: {{ .Values.version }}
   serviceAccountName: {{ include "kblib.serviceAccountName" . }}
+  {{- if and .Values.customSecretName .Values.customSecretNamespace }}
+  systemAccounts:
+    - name: default
+      secretRef:
+        name: {{ .Values.customSecretName }}
+        namespace: {{ .Values.customSecretNamespace }}
+  {{- end }}
   {{- include "kblib.componentResources" . | indent 2 }}
   {{- include "kblib.componentStorages" . | indent 2 }}
 {{- end }}
@@ -79,6 +92,13 @@ Define redis sentinel ComponentSpec with ComponentDefinition.
   {{- end }}
   serviceVersion: {{ .Values.version }}
   serviceAccountName: {{ include "kblib.serviceAccountName" . }}
+  {{- if and .Values.sentinel.customSecretName .Values.sentinel.customSecretNamespace }}
+  systemAccounts:
+    - name: default
+      secretRef:
+        name: {{ .Values.sentinel.customSecretName }}
+        namespace: {{ .Values.sentinel.customSecretNamespace }}
+  {{- end }}
   resources:
     limits:
       cpu: {{ .Values.sentinel.cpu | quote }}

addons-cluster/redis/values.schema.json

@@ -73,6 +73,18 @@
       "title": "nodePortEnabled",
       "description":"Whether NodePort service is enabled, default is true"
     },
+    "customSecretName": {
+      "title": "the secret name for Redis default user's password",
+      "description": "the secret must contain keys named 'username' and 'password'",
+      "type": "string",
+      "default": ""
+    },
+    "customSecretNamespace": {
+      "title": "the secret namespace for Redis default user's password",
+      "description": "the secret must contain keys named 'username' and 'password'",
+      "type": "string",
+      "default": ""
+    },
     "twemproxy": {
       "title": "The redis twemproxy component",
       "type": "object",
@@ -119,6 +131,18 @@
           "type": "string",
           "default": ""
         },
+        "customSecretName": {
+          "title": "the secret name for Redis Sentinel default user's password",
+          "description": "the secret must contain keys named 'username' and 'password'",
+          "type": "string",
+          "default": ""
+        },
+        "customSecretNamespace": {
+          "title": "the secret namespace for Redis Sentinel default user's password",
+          "description": "the secret must contain keys named 'username' and 'password'",
+          "type": "string",
+          "default": ""
+        },
         "enabled": {
           "description":"Whether have sentinel component, default is true",
           "title": "sentinel.enable",
@@ -170,6 +194,18 @@
           "default": 3,
           "minimum": 3,
           "maximum": 2048
+        },
+        "customSecretName": {
+          "title": "the secret name for Redis Cluster default user's password",
+          "description": "the secret must contain keys named 'username' and 'password'",
+          "type": "string",
+          "default": ""
+        },
+        "customSecretNamespace": {
+          "title": "the secret namespace for Redis Cluster default user's password",
+          "description": "the secret must contain keys named 'username' and 'password'",
+          "type": "string",
+          "default": ""
         }
       }
     }

addons-cluster/redis/values.yaml

@@ -38,6 +38,11 @@ requests:
 ##
 storage: 20
 
+# custom password of default user for redis by passing a secret
+# the secret must contain keys named 'username' and 'password'
+customSecretName: ""
+customSecretNamespace: ""
+
 ## @param twemproxy.enable
 ## @param sentinel.cpu
 ## @param sentinel.memory
@@ -68,11 +73,19 @@ sentinel:
   storage: 20
   replicas: 3
   customMasterName: ""
+  # custom password of default user for redis sentinel by passing a secret
+  # the secret must contain keys named 'username' and 'password'
+  customSecretName: ""
+  customSecretNamespace: ""
 
 ## if mode is cluster, specify the redis cluster parameters
 ## @param redisCluster.shardCount specify the number of shards in the cluster, the minimum value is 3
 redisCluster:
   shardCount: 3
+  # custom password of default user for redis cluster by passing a secret
+  # the secret must contain keys named 'username' and 'password'
+  customSecretName: ""
+  customSecretNamespace: ""
 
 extra:
   rbacEnabled: true

addons/redis/redis-cluster-scripts/redis-cluster-common.sh

@@ -151,11 +151,13 @@ shutdown_redis_server() {
 
 check_redis_server_ready() {
   unset_xtrace_when_ut_mode_false
+  local host="$1"
+  local port="$2"
   local max_retry=10
   local retry_interval=5
-  check_ready_cmd="redis-cli -h 127.0.0.1 -p $service_port ping"
+  check_ready_cmd="redis-cli -h $host -p $port ping"
   if ! is_empty "$REDIS_DEFAULT_PASSWORD"; then
-    check_ready_cmd="redis-cli -h 127.0.0.1 -p $service_port -a $REDIS_DEFAULT_PASSWORD ping"
+    check_ready_cmd="redis-cli -h $host -p $port -a $REDIS_DEFAULT_PASSWORD ping"
   fi
   set_xtrace_when_ut_mode_false
   output=$($check_ready_cmd)
@@ -408,8 +410,10 @@ check_node_in_cluster_with_retry() {
 }
 
 check_redis_server_ready_with_retry() {
+  local host="$1"
+  local port="$2"
   # call the execute_check_redis_server_ready_command function with call_func_with_retry function and get the output
-  check_result=$(call_func_with_retry $check_ready_times $retry_delay_second check_redis_server_ready)
+  check_result=$(call_func_with_retry $check_ready_times $retry_delay_second check_redis_server_ready "$host" "$port")
   status=$?
   if [ $status -ne 0 ]; then
     echo "Failed to check the redis server ready after retry" >&2

addons/redis/redis-cluster-scripts/redis-cluster-manage.sh

@@ -41,6 +41,19 @@ load_redis_cluster_common_utils() {
   source "${redis_cluster_common_library_file}"
 }
 
+check_initialize_nodes_ready() {
+  local nodes=("$@")
+  for node in "${nodes[@]}"; do
+    local host port
+    host=$(echo "$node" | cut -d':' -f1)
+    port=$(echo "$node" | cut -d':' -f2)
+    if ! check_redis_server_ready_with_retry "$host" "$port"; then
+      return 1
+    fi
+  done
+  return 0
+}
+
 # initialize the other component and pods info
 init_other_components_and_pods_info() {
   local current_component="$1"
@@ -605,17 +618,40 @@ initialize_redis_cluster() {
     return 1
   fi
 
-  # initialize all the primary nodes
+  # generate primary and secondary nodes
   gen_initialize_redis_cluster_primary_node
+  gen_initialize_redis_cluster_secondary_nodes
+
   if [ ${#initialize_redis_cluster_primary_nodes[@]} -eq 0 ] || [ ${#initialize_redis_cluster_primary_nodes[@]} -lt 3 ]; then
     echo "Failed to get primary nodes or the primary nodes count is less than 3" >&2
     return 1
   fi
+
+  # check all the primary nodes are ready
   local primary_nodes=""
-  for primary_pod_name in "${!initialize_redis_cluster_primary_nodes[@]}"; do
-    primary_nodes+="${initialize_redis_cluster_primary_nodes[$primary_pod_name]} "
+  local primary_node_list=()
+  for pod_name in "${!initialize_redis_cluster_primary_nodes[@]}"; do
+    primary_nodes+="${initialize_redis_cluster_primary_nodes[$pod_name]} "
+    primary_node_list+=("${initialize_redis_cluster_primary_nodes[$pod_name]}")
   done
+  if ! check_initialize_nodes_ready "${primary_node_list[@]}"; then
+    echo "Primary nodes health check failed" >&2
+    return 1
+  fi
+
+  # check all the secondary nodes are ready
+  if [ ${#initialize_redis_cluster_secondary_nodes[@]} -gt 0 ]; then
+    secondary_node_list=()
+    for pod_name in "${!initialize_redis_cluster_secondary_nodes[@]}"; do
+      secondary_node_list+=("${initialize_redis_cluster_secondary_nodes[$pod_name]}")
+    done
+    if ! check_initialize_nodes_ready "${secondary_node_list[@]}"; then
+      echo "Secondary nodes health check failed" >&2
+      return 1
+    fi
+  fi
 
+  # initialize all the primary nodes
   if create_redis_cluster "$primary_nodes"; then
     echo "Redis cluster initialized primary nodes successfully, cluster nodes: $primary_nodes"
   else
@@ -633,11 +669,12 @@ initialize_redis_cluster() {
   fi
 
   # initialize all the secondary nodes
-  gen_initialize_redis_cluster_secondary_nodes
   if [ ${#initialize_redis_cluster_secondary_nodes[@]} -eq 0 ]; then
     echo "No secondary nodes to initialize"
     return 0
   fi
+
+  all_secondaries_ready=true
   for secondary_pod_name in "${!initialize_redis_cluster_secondary_nodes[@]}"; do
     secondary_endpoint_with_port=${initialize_redis_cluster_secondary_nodes["$secondary_pod_name"]}
     # shellcheck disable=SC2001
@@ -664,6 +701,43 @@ initialize_redis_cluster() {
     echo "Redis cluster initialized secondary node $secondary_pod_name successfully"
     # waiting for all nodes sync the information
     sleep_when_ut_mode_false 5
+
+    # verify secondary node is already in all primary nodes
+    if ! verify_secondary_in_all_primaries "$secondary_pod_name" "${primary_node_list[@]}"; then
+      echo "Failed to verify secondary node $secondary_pod_name in all primary nodes" >&2
+      all_secondaries_ready=false
+      continue
+    fi
+    echo "Secondary node $secondary_pod_name successfully joined the cluster and verified in all primaries"
+  done
+
+  if [ "$all_secondaries_ready" = false ]; then
+    echo "Failed to initialize all secondary nodes" >&2
+    return 1
+  fi
+  echo "Redis cluster initialized all secondary nodes successfully"
+  return 0
+}
+
+verify_secondary_in_all_primaries() {
+  local secondary_pod_name="$1"
+  local primary_nodes=("$@")
+  # Skip the first argument
+  shift
+  for primary_node in "$@"; do
+    local primary_host primary_port
+    primary_host=$(echo "$primary_node" | cut -d':' -f1)
+    primary_port=$(echo "$primary_node" | cut -d':' -f2)
+    retry_count=0
+    while ! check_node_in_cluster "$primary_host" "$primary_port" "$secondary_pod_name" && [ $retry_count -lt 30 ]; do
+      sleep_when_ut_mode_false 3
+      ((retry_count++))
+    done
+    # shellcheck disable=SC2086
+    if [ $retry_count -eq 30 ]; then
+      echo "Secondary node $secondary_pod_name not found in primary $primary_node after retry" >&2
+      return 1
+    fi
   done
   return 0
 }

addons/redis/redis-cluster-scripts/redis-cluster-replica-pre-stop.sh

@@ -0,0 +1,43 @@
+#!/bin/bash
+
+# shellcheck disable=SC2034
+ut_mode="false"
+test || __() {
+  # when running in non-unit test mode, set the options "set -e".
+  set -ex;
+}
+
+load_common_library() {
+  # the common.sh scripts is mounted to the same path which is defined in the cmpd.spec.scripts
+  common_library_file="/scripts/common.sh"
+  # shellcheck disable=SC1090
+  source "${common_library_file}"
+}
+
+acl_save_before_stop() {
+  if ! is_empty "$REDIS_DEFAULT_PASSWORD"; then
+    acl_save_command="redis-cli -h localhost -p $SERVICE_PORT -a $REDIS_DEFAULT_PASSWORD acl save"
+    logging_mask_acl_save_command="${acl_save_command/$REDIS_DEFAULT_PASSWORD/********}"
+  else
+    acl_save_command="redis-cli -h localhost -p $SERVICE_PORT acl save"
+    logging_mask_acl_save_command="$acl_save_command"
+  fi
+  echo "acl save command: $logging_mask_acl_save_command"
+  if output=$($acl_save_command 2>&1); then
+    echo "acl save command executed successfully: $output"
+  else
+    echo "failed to execute acl save command: $output"
+    exit 1
+  fi
+}
+
+# This is magic for shellspec ut framework.
+# Sometime, functions are defined in a single shell script.
+# You will want to test it. but you do not want to run the script.
+# When included from shellspec, __SOURCED__ variable defined and script
+# end here. The script path is assigned to the __SOURCED__ variable.
+${__SOURCED__:+false} : || return 0
+
+# main
+load_common_library
+acl_save_before_stop

addons/redis/redis-cluster-scripts/redis-cluster-server-start.sh

@@ -65,10 +65,17 @@ check_and_correct_other_primary_nodes() {
     while true; do
       current_announce_ip=$(get_cluster_announce_ip "$node_endpoint" "$node_port")
       echo "original_announce_ip: $original_announce_ip, node_endpoint_with_port: $node_endpoint_with_port, current_announce_ip: $current_announce_ip"
-      # if current_announce_ip is empty, we need to retry
+      # if current_announce_ip is empty, retry it
       if is_empty "$current_announce_ip"; then
+        echo "Error: current_announce_ip is empty"
         sleep_when_ut_mode_false 3
-        echo "current_announce_ip is empty, retry..."
+        continue
+      fi
+
+      if [ "$node_port" -eq 0 ] || [ "$node_bus_port" -eq 0 ]; then
+        echo "Error: node_port or node_bus_port is 0"
+        sleep_when_ut_mode_false 3
+        # TODO: get other_comp_primary_nodes again
         continue
       fi
 
@@ -221,13 +228,20 @@ get_current_comp_nodes_for_scale_out_replica() {
 # scale out replica of redis cluster shard if needed
 scale_redis_cluster_replica() {
   # Waiting for redis-server to start
-  if check_redis_server_ready_with_retry ; then
+  if check_redis_server_ready_with_retry "127.0.0.1" "$service_port"; then
     echo "Redis server is ready, continue to scale out replica..."
   else
     echo "Redis server is not ready, exit scale out replica..." >&2
     exit 1
   fi
 
+  if [ -f /data/nodes.conf ]; then
+    echo "the nodes.conf file after redis server start:"
+    cat /data/nodes.conf
+  else
+    echo "the nodes.conf file after redis server start is not exist"
+  fi
+
   # get the current component nodes for scale out replica
   target_node_name=$(min_lexicographical_order_pod "$CURRENT_SHARD_POD_NAME_LIST")
   if ! is_empty "$CURRENT_SHARD_PRIMARY_POD_NAME"; then