chore: support new switchover action (#1338)

apecloud · Dec 31, 2024 · b4cab1e · b4cab1e
1 parent af17f8a
commit b4cab1e
Show file tree

Hide file tree

Showing 14 changed files with 129 additions and 5 deletions.
diff --git a/addons/apecloud-mysql/templates/_helpers.tpl b/addons/apecloud-mysql/templates/_helpers.tpl
@@ -167,7 +167,16 @@ lifecycleActions:
         - /bin/sh
         - -c
         - |
-          /tools/syncerctl switchover --primary "$KB_LEADER_POD_NAME" ${KB_SWITCHOVER_CANDIDATE_NAME:+--candidate "$KB_SWITCHOVER_CANDIDATE_NAME"}
+          if [ -z "$KB_SWITCHOVER_ROLE" ]; then
+              echo "role can't be empty"
+              exit 1
+          fi
+
+          if [ "$KB_SWITCHOVER_ROLE" != "leader" ]; then
+              exit 0
+          fi
+
+          /tools/syncerctl switchover --primary "$KB_SWITCHOVER_CURRENT_NAME" ${KB_SWITCHOVER_CANDIDATE_NAME:+--candidate "$KB_SWITCHOVER_CANDIDATE_NAME"}
   accountProvision:
     exec:
       container: mysql

diff --git a/addons/apecloud-postgresql/templates/_helpers.tpl b/addons/apecloud-postgresql/templates/_helpers.tpl
@@ -237,6 +237,15 @@ lifecycleActions:
         - /bin/sh
         - -c
         - |
+          if [ -z "$KB_SWITCHOVER_ROLE" ]; then
+              echo "role can't be empty"
+              exit 1
+          fi
+
+          if [ "$KB_SWITCHOVER_ROLE" != "leader" ]; then
+              exit 0
+          fi
+
           /tools/syncerctl switchover --primary "$POSTGRES_LEADER_POD_NAME" ${KB_SWITCHOVER_CANDIDATE_NAME:+--candidate "$KB_SWITCHOVER_CANDIDATE_NAME"}
   memberLeave:
     exec:

diff --git a/addons/etcd/scripts/switchover.sh b/addons/etcd/scripts/switchover.sh
@@ -83,6 +83,11 @@ switchover_without_candidate() {
 }
 
 switchover() {
+  if [[ $LEADER_POD_FQDN != "$KB_SWITCHOVER_CURRENT_FQDN" ]]; then
+    echo "switchover action not triggered for leader pod. Exiting."
+    exit 0
+  fi
+
   if is_empty "$KB_SWITCHOVER_CANDIDATE_FQDN"; then
       switchover_without_candidate
   else

diff --git a/addons/mongodb/templates/componentdefinition.yaml b/addons/mongodb/templates/componentdefinition.yaml
@@ -95,7 +95,16 @@ spec:
           - /bin/sh
           - -c
           - |
-            /tools/syncerctl switchover --primary "$KB_LEADER_POD_NAME" ${KB_SWITCHOVER_CANDIDATE_NAME:+--candidate "$KB_SWITCHOVER_CANDIDATE_NAME"}
+            if [ -z "$KB_SWITCHOVER_ROLE" ]; then
+                echo "role can't be empty"
+                exit 1
+            fi
+
+            if [ "$KB_SWITCHOVER_ROLE" != "primary" ]; then
+                exit 0
+            fi
+
+            /tools/syncerctl switchover --primary "$KB_SWITCHOVER_CURRENT_NAME" ${KB_SWITCHOVER_CANDIDATE_NAME:+--candidate "$KB_SWITCHOVER_CANDIDATE_NAME"}
   runtime:
     initContainers:
       - command:

diff --git a/addons/mysql/templates/_helpers.tpl b/addons/mysql/templates/_helpers.tpl
@@ -152,7 +152,16 @@ lifecycleActions:
         - /bin/sh
         - -c
         - |
-          /tools/syncerctl switchover --primary "$KB_LEADER_POD_NAME" ${KB_SWITCHOVER_CANDIDATE_NAME:+--candidate "$KB_SWITCHOVER_CANDIDATE_NAME"}
+          if [ -z "$KB_SWITCHOVER_ROLE" ]; then
+              echo "role can't be empty"
+              exit 1
+          fi
+
+          if [ "$KB_SWITCHOVER_ROLE" != "primary" ]; then
+              exit 0
+          fi
+
+          /tools/syncerctl switchover --primary "$KB_SWITCHOVER_CURRENT_NAME" ${KB_SWITCHOVER_CANDIDATE_NAME:+--candidate "$KB_SWITCHOVER_CANDIDATE_NAME"}
 roles:
   - name: primary
     serviceable: true

diff --git a/addons/orioledb/scripts/switchover.sh b/addons/orioledb/scripts/switchover.sh
@@ -64,6 +64,11 @@ switchover() {
     exit 1
   fi
 
+  if [[ $POSTGRES_PRIMARY_POD_NAME != "$KB_SWITCHOVER_CURRENT_NAME" ]]; then
+    echo "switchover action not triggered for primary pod. Exiting."
+    exit 0
+  fi
+
   current_pod_fqdn=$(get_target_pod_fqdn_from_pod_fqdn_vars "$POSTGRES_POD_FQDN_LIST" "$CURRENT_POD_NAME")
   if is_empty "$current_pod_fqdn"; then
     echo "Error: Failed to get current pod: $CURRENT_POD_NAME fqdn from postgres pod fqdn list: $POSTGRES_POD_FQDN_LIST. Exiting."

diff --git a/addons/postgresql/scripts-ut-spec/switchover_spec.sh b/addons/postgresql/scripts-ut-spec/switchover_spec.sh
@@ -72,6 +72,7 @@ Describe "PostgreSQL Switchover Script Tests"
     Context "when the current pod FQDN is not found"
       setup() {
         CURRENT_POD_NAME="pod1"
+        KB_SWITCHOVER_CURRENT_NAME="pod1"
         POSTGRES_PRIMARY_POD_NAME="pod1"
         POSTGRES_POD_NAME_LIST="pod2,pod3"
         POSTGRES_POD_FQDN_LIST="pod2.example.com,pod3.example.com"
@@ -85,10 +86,27 @@ Describe "PostgreSQL Switchover Script Tests"
       End
     End
 
+    Context "when switchover action is called for a non-primary pod"
+      setup() {
+        CURRENT_POD_NAME="pod1"
+        POSTGRES_PRIMARY_POD_NAME="pod2"
+        KB_SWITCHOVER_CURRENT_NAME="pod1"
+        POSTGRES_POD_NAME_LIST="pod1,pod2"
+        POSTGRES_POD_FQDN_LIST="pod1.example.com,pod2.example.com"
+      }
+      Before 'setup'
+
+      It "exits without error"
+        When run switchover
+        The output should include "switchover action not triggered for primary pod. Exiting"
+      End
+    End
+
     Context "when KB_SWITCHOVER_CANDIDATE_NAME is set"
       setup() {
         CURRENT_POD_NAME="pod1"
         POSTGRES_PRIMARY_POD_NAME="pod1"
+        KB_SWITCHOVER_CURRENT_NAME="pod1"
         POSTGRES_POD_NAME_LIST="pod1,pod2"
         POSTGRES_POD_FQDN_LIST="pod1.example.com,pod2.example.com"
         KB_SWITCHOVER_CANDIDATE_NAME="pod2"
@@ -108,6 +126,7 @@ Describe "PostgreSQL Switchover Script Tests"
       setup() {
         CURRENT_POD_NAME="pod1"
         POSTGRES_PRIMARY_POD_NAME="pod1"
+        KB_SWITCHOVER_CURRENT_NAME="pod1"
         POSTGRES_POD_NAME_LIST="pod1,pod2"
         POSTGRES_POD_FQDN_LIST="pod1.example.com,pod2.example.com"
         unset KB_SWITCHOVER_CANDIDATE_NAME

diff --git a/addons/postgresql/scripts/switchover.sh b/addons/postgresql/scripts/switchover.sh
@@ -64,6 +64,11 @@ switchover() {
     exit 1
   fi
 
+  if [[ $POSTGRES_PRIMARY_POD_NAME != "$KB_SWITCHOVER_CURRENT_NAME" ]]; then
+    echo "switchover action not triggered for primary pod. Exiting."
+    exit 0
+  fi
+
   current_pod_fqdn=$(get_target_pod_fqdn_from_pod_fqdn_vars "$POSTGRES_POD_FQDN_LIST" "$CURRENT_POD_NAME")
   if is_empty "$current_pod_fqdn"; then
     echo "Error: Failed to get current pod: $CURRENT_POD_NAME fqdn from postgres pod fqdn list: $POSTGRES_POD_FQDN_LIST. Exiting."

diff --git a/addons/redis/scripts/redis-switchover.sh b/addons/redis/scripts/redis-switchover.sh
@@ -43,6 +43,16 @@ check_environment_exist() {
     echo "Error: Required environment variable REDIS_COMPONENT_NAME: $REDIS_COMPONENT_NAME is not set."
     exit 1
   fi
+
+  if is_empty "$KB_SWITCHOVER_ROLE"; then
+    echo "role can't be empty"
+    exit 1
+  fi
+
+  if [ "$KB_SWITCHOVER_ROLE" != "primary" ]; then
+    echo "swtichover not triggered for primary, nothing to do"
+    exit 0
+  fi
 }
 
 check_connectivity() {

diff --git a/addons/tidb/scripts/pd_switchover.sh b/addons/tidb/scripts/pd_switchover.sh
@@ -0,0 +1,23 @@
+#!/bin/bash
+
+set -exo pipefail
+
+if [[ -z $KB_SWITCHOVER_ROLE ]]; then
+    echo "role can't be empty"
+    exit 1
+fi
+
+if [[ $KB_SWITCHOVER_ROLE != "leader" ]]; then
+    exit 0
+fi
+
+if [[ -n $KB_SWITCHOVER_CANDIDATE_NAME ]]; then
+    result=$(/pd-ctl member leader transfer "$KB_SWITCHOVER_CANDIDATE_NAME")
+else
+    result=$(/pd-ctl member leader resign)
+fi
+
+if [[ $result != "Success!" ]]; then
+    echo "switchover failed"
+    exit 1
+fi
diff --git a/addons/tidb/scripts/tikv_member_leave.sh b/addons/tidb/scripts/tikv_member_leave.sh
@@ -2,9 +2,13 @@
 
 set -exo pipefail
 
-/pd-ctl -u "$PD_ADDRESS" store delete addr "$KB_LEAVE_MEMBER_POD_FQDN:20160"
+output=$(/pd-ctl -u "$PD_ADDRESS" store delete addr "$KB_LEAVE_MEMBER_POD_FQDN:20160")
+if [[ $output != "Success!" ]]; then
+    echo "leave member $KB_LEAVE_MEMBER_POD_FQDN failed"
+    exit 1
+fi
 
-until [ $(/pd-ctl -u "$PD_ADDRESS" store | jq "any(.stores[]; select(.store.address == \"$KB_LEAVE_MEMBER_POD_FQDN:20160\"))") == "false" ]
+until [[ $(/pd-ctl -u "$PD_ADDRESS" store | jq "any(.stores[]; select(.store.address == \"$KB_LEAVE_MEMBER_POD_FQDN:20160\"))") == "false" ]]
 do
     echo "waiting for tikv node to become tombstone"
     sleep 10

diff --git a/addons/tidb/templates/componentdefinition-pd.yaml b/addons/tidb/templates/componentdefinition-pd.yaml
@@ -92,6 +92,15 @@ spec:
             echo $KB_LEAVE_MEMBER_POD_NAME
             echo $ADDRESS
             /pd-ctl -u "$ADDRESS:2379" member delete name $KB_LEAVE_MEMBER_POD_NAME
+    switchover:
+      exec:
+        targetPodSelector: Role
+        matchingKey: leader
+        command:
+          - bash
+          - -c
+          - |
+            {{- .Files.Get "scripts/pd_switchover.sh" | nindent 14 }}
   exporter:
     containerName: pd
     scrapePort: "2379"

diff --git a/addons/tidb/templates/componentversion.yaml b/addons/tidb/templates/componentversion.yaml
@@ -14,18 +14,21 @@ spec:
         pd: {{ .Values.image.registry | default "docker.io" }}/{{ .Values.image.pd.repository }}:v7.5.2
         roleProbe: {{ .Values.image.registry | default "docker.io" }}/{{ .Values.image.pd.repository }}:v7.5.2
         memberLeave: {{ .Values.image.registry | default "docker.io" }}/{{ .Values.image.pd.repository }}:v7.5.2
+        switchover: {{ .Values.image.registry | default "docker.io" }}/{{ .Values.image.pd.repository }}:v7.5.2
     - name: "7.1.5"
       serviceVersion: "7.1.5"
       images:
         pd: {{ .Values.image.registry | default "docker.io" }}/{{ .Values.image.pd.repository }}:v7.1.5
         roleProbe: {{ .Values.image.registry | default "docker.io" }}/{{ .Values.image.pd.repository }}:v7.1.5
         memberLeave: {{ .Values.image.registry | default "docker.io" }}/{{ .Values.image.pd.repository }}:v7.1.5
+        switchover: {{ .Values.image.registry | default "docker.io" }}/{{ .Values.image.pd.repository }}:v7.1.5
     - name: "6.5.10"
       serviceVersion: "6.5.10"
       images:
         pd: {{ .Values.image.registry | default "docker.io" }}/{{ .Values.image.pd.repository }}:v6.5.10
         roleProbe: {{ .Values.image.registry | default "docker.io" }}/{{ .Values.image.pd.repository }}:v6.5.10
         memberLeave: {{ .Values.image.registry | default "docker.io" }}/{{ .Values.image.pd.repository }}:v6.5.10
+        switchover: {{ .Values.image.registry | default "docker.io" }}/{{ .Values.image.pd.repository }}:v6.5.10
   compatibilityRules:
     - compDefs:
         - {{ include "tidb.pd7.cmpdRegexpPattern" . }}

diff --git a/addons/vanilla-postgresql/scripts/switchover.sh b/addons/vanilla-postgresql/scripts/switchover.sh
@@ -1,3 +1,8 @@
 #!/bin/sh
 
+if [ "$POSTGRES_PRIMARY_POD_NAME" != "$KB_SWITCHOVER_CURRENT_NAME" ]; then
+  echo "switchover action not triggered for primary pod. Exiting."
+  exit 0
+fi
+
 /tools/syncerctl switchover --primary "$POSTGRES_PRIMARY_POD_NAME" ${KB_SWITCHOVER_CANDIDATE_NAME:+--candidate "$KB_SWITCHOVER_CANDIDATE_NAME"}