feat: xtrabackup incremental backup for mysql

addons/apecloud-mysql/dataprotection/backup.sh

@@ -22,11 +22,20 @@ if [ -f ${DATA_DIR}/mysql-bin.index ]; then
   echo "" | datasafed push - "apecloud-mysql.old"
 fi
 # do xtrabackup
+TMP_DIR=${DATA_MOUNT_DIR}/xtrabackup-temp
+mkdir -p ${TMP_DIR}
 START_TIME=$(date -u '+%Y-%m-%dT%H:%M:%SZ')
 xtrabackup --compress=zstd --backup --safe-slave-backup --slave-info --stream=xbstream \
   --host=${DP_DB_HOST} --port=${DP_DB_PORT} \
-  --user=${DP_DB_USER} --password=${DP_DB_PASSWORD} \
-  --datadir=${DATA_DIR} | datasafed push - "/${DP_BACKUP_NAME}.xbstream"
+  --user=${DP_DB_USER} --password=${DP_DB_PASSWORD} --datadir=${DATA_DIR} \
+  2> >(tee ${TMP_DIR}/xtrabackup.log >&2) \
+  | datasafed push - "/${DP_BACKUP_NAME}.xbstream"
+# record lsn for incremental backups
+cat "${TMP_DIR}/xtrabackup.log" \
+  | grep "The latest check point (for incremental)" \
+  | awk -F"'" '{print $2}' \
+  | datasafed push - "/${DP_BACKUP_NAME}.lsn"
 STOP_TIME=$(date -u '+%Y-%m-%dT%H:%M:%SZ')
 TOTAL_SIZE=$(datasafed stat / | grep TotalSize | awk '{print $2}')
 echo "{\"totalSize\":\"$TOTAL_SIZE\",\"timeRange\":{\"start\":\"${START_TIME}\",\"end\":\"${STOP_TIME}\"}}" >"${DP_BACKUP_INFO_FILE}"
+rm -rf ${TMP_DIR}

addons/apecloud-mysql/dataprotection/xtrabackup-incremental-backup.sh

@@ -0,0 +1,86 @@
+#!/bin/bash
+set -e
+set -o pipefail
+
+# if the script exits with a non-zero exit code, touch a file to indicate that the backup failed,
+# the sync progress container will check this file and exit if it exists
+function handle_exit() {
+  exit_code=$?
+  if [ $exit_code -ne 0 ]; then
+    echo "failed with exit code $exit_code"
+    touch "${DP_BACKUP_INFO_FILE}.exit"
+    exit 1
+  fi
+}
+trap handle_exit EXIT
+
+export PATH="$PATH:$DP_DATASAFED_BIN_PATH"
+
+# 1. check parent backup name
+if [[ -z ${DP_PARENT_BACKUP_NAME} ]]; then
+  echo "DP_PARENT_BACKUP_NAME is empty"
+  exit 1
+fi
+
+# 2. get parent backup lsn or download backup file
+mkdir -p ${DATA_DIR}
+PARENT_DIR=${DATA_MOUNT_DIR}/xtrabackup-parent
+
+# function get_last_lsn gets parent backup lsn from datasafed
+function get_last_lsn() {
+  export DATASAFED_BACKEND_BASE_PATH="${DP_BACKUP_ROOT_PATH}/${DP_PARENT_BACKUP_NAME}/${DP_TARGET_RELATIVE_PATH}"
+  lsnFile="${DP_PARENT_BACKUP_NAME}.lsn"
+  if [ "$(datasafed list ${lsnFile})" == "${lsnFile}" ]; then
+    echo "$(datasafed pull "/${lsnFile}" - | awk '{print $1}')"
+  fi
+}
+
+# function download_parent_backup_file downloads the parent backup file from datasafed
+function download_parent_backup_file() {
+  rm -rf ${PARENT_DIR}
+  mkdir -p ${PARENT_DIR} && cd ${PARENT_DIR}
+  export DATASAFED_BACKEND_BASE_PATH="${DP_BACKUP_ROOT_PATH}/${DP_PARENT_BACKUP_NAME}/${DP_TARGET_RELATIVE_PATH}"
+  datasafed pull "${DP_PARENT_BACKUP_NAME}.xbstream" - | xbstream -x
+  xtrabackup --decompress --remove-original --target-dir=${PARENT_DIR}
+}
+
+# build incremental cmd
+incremental_cmd=""
+last_lsn=$(get_last_lsn)
+if [ -n "${last_lsn}" ]; then
+  incremental_cmd="--incremental-lsn=${last_lsn}"
+  echo "create incremental backup based on parent backup lsn"
+else
+  download_parent_backup_file
+  incremental_cmd="--incremental-basedir=${PARENT_DIR}"
+  echo "create incremental backup based on parent backup file"
+fi
+
+# set the datasafed backend base path for the current backup
+# it is equal to ${DP_BACKUP_ROOT_PATH}/${DP_BACKUP_NAME}/${DP_TARGET_RELATIVE_PATH}
+export DATASAFED_BACKEND_BASE_PATH="$DP_BACKUP_BASE_PATH"
+# compatible with version 0.6
+if [ -f ${DATA_DIR}/mysql-bin.index ]; then
+  echo "" | datasafed push - "apecloud-mysql.old"
+fi
+
+# 3. do incremental xtrabackup
+TMP_DIR=${DATA_MOUNT_DIR}/xtrabackup-temp
+mkdir -p ${TMP_DIR}
+START_TIME=$(date -u '+%Y-%m-%dT%H:%M:%SZ')
+xtrabackup --compress=zstd --backup --safe-slave-backup --slave-info --stream=xbstream \
+  --host=${DP_DB_HOST} --port=${DP_DB_PORT} \
+  --user=${DP_DB_USER} --password=${DP_DB_PASSWORD} \
+  --datadir=${DATA_DIR} ${incremental_cmd} \
+  2> >(tee ${TMP_DIR}/xtrabackup.log >&2) \
+  | datasafed push - "/${DP_BACKUP_NAME}.xbstream"
+# record lsn for incremental backups
+cat "${TMP_DIR}/xtrabackup.log" \
+  | grep "The latest check point (for incremental)" \
+  | awk -F"'" '{print $2}' \
+  | datasafed push - "/${DP_BACKUP_NAME}.lsn"
+STOP_TIME=$(date -u '+%Y-%m-%dT%H:%M:%SZ')
+TOTAL_SIZE=$(datasafed stat / | grep TotalSize | awk '{print $2}')
+echo "{\"totalSize\":\"$TOTAL_SIZE\",\"timeRange\":{\"start\":\"${START_TIME}\",\"end\":\"${STOP_TIME}\"}}" >"${DP_BACKUP_INFO_FILE}"
+rm -rf ${PARENT_DIR}
+rm -rf ${TMP_DIR}

addons/apecloud-mysql/dataprotection/xtrabackup-incremental-restore.sh

@@ -0,0 +1,64 @@
+#!/bin/bash
+set -e
+set -o pipefail
+export PATH="$PATH:$DP_DATASAFED_BIN_PATH"
+
+# function change_backend_path changes the DATASAFED_BACKEND_BASE_PATH by backup name
+function change_backend_path() {
+  backup_name=$1
+  export DATASAFED_BACKEND_BASE_PATH="${DP_BACKUP_ROOT_PATH}/${backup_name}/${DP_TARGET_RELATIVE_PATH}"
+}
+
+# function download_backup_file downloads a backup file to a local target directory
+function download_backup_file() {
+  backup_name=$1
+  local_target_dir=$2
+  mkdir -p ${local_target_dir} && cd ${local_target_dir}
+  change_backend_path "${backup_name}"
+  datasafed pull "${backup_name}.xbstream" - | xbstream -x
+  xtrabackup --decompress --remove-original --target-dir=${local_target_dir}
+}
+
+# 1. check base backup name
+if [[ -z ${DP_BASE_BACKUP_NAME} ]]; then
+  echo "DP_BASE_BACKUP_NAME is empty"
+  exit 1
+fi
+
+# 2. download backup files
+# download base backup file
+mkdir -p ${DATA_DIR}
+BASE_DIR=${DATA_MOUNT_DIR}/xtrabackup-base
+download_backup_file "${DP_BASE_BACKUP_NAME}" "${BASE_DIR}"
+# download parent backup files
+if [ -n "${DP_ANCESTOR_INCREMENTAL_BACKUP_NAMES}" ]; then
+  read -r -a ANCESTOR_INCREMENTAL_BACKUP_NAMES <<< "${DP_ANCESTOR_INCREMENTAL_BACKUP_NAMES//,/ }"
+fi
+INCS_DIR=${DATA_MOUNT_DIR}/xtrabackup-incs
+mkdir -p ${INCS_DIR}
+for parent_name in "${ANCESTOR_INCREMENTAL_BACKUP_NAMES[@]}"; do
+  download_backup_file "${parent_name}" "${INCS_DIR}/${parent_name}"
+done
+# download target backup file
+download_backup_file "${DP_BACKUP_NAME}" "${INCS_DIR}/${DP_BACKUP_NAME}"
+
+old_signal="apecloud-mysql.old"
+log_bin=${LOG_BIN}
+if [ "$(datasafed list ${old_signal})" == "${old_signal}" ]; then
+   log_bin="${DATA_DIR}/mysql-bin"
+fi
+
+# 3. prepare data
+xtrabackup --prepare --apply-log-only --target-dir=${BASE_DIR}
+for parent_name in "${ANCESTOR_INCREMENTAL_BACKUP_NAMES[@]}"; do
+  xtrabackup --prepare --apply-log-only --target-dir=${BASE_DIR} --incremental-dir=${INCS_DIR}/${parent_name}
+done
+xtrabackup --prepare --target-dir=${BASE_DIR} --incremental-dir=${INCS_DIR}/${DP_BACKUP_NAME}
+
+# 4. restore
+xtrabackup --move-back --target-dir=${BASE_DIR} --datadir=${DATA_DIR}/ --log-bin=${log_bin}
+touch ${DATA_DIR}/${SIGNAL_FILE}
+rm -rf ${BASE_DIR}
+rm -rf ${INCS_DIR}
+chmod -R 0777 ${DATA_DIR}
+echo "Restore completed!"

addons/apecloud-mysql/templates/_names.tpl

@@ -9,6 +9,17 @@ apecloud-mysql-xtrabackup
 {{- end -}}
 {{- end -}}
 
+{{/*
+Define xtrabackup actionSet name for incremental backups
+*/}}
+{{- define "apecloud-mysql.xtrabackupIncActionSetName" -}}
+{{- if eq (len .Values.resourceNamePrefix) 0 -}}
+apecloud-mysql-xtrabackup-incremental
+{{- else -}}
+{{- .Values.resourceNamePrefix -}}-xtrabackup-incremental
+{{- end -}}
+{{- end -}}
+
 {{/*
 Define volume snapshot actionSet name
 */}}

addons/apecloud-mysql/templates/actionset-incremental.yaml

@@ -0,0 +1,42 @@
+apiVersion: dataprotection.kubeblocks.io/v1alpha1
+kind: ActionSet
+metadata:
+  name: {{ include "apecloud-mysql.xtrabackupIncActionSetName" . }}
+  labels:
+    clusterdefinition.kubeblocks.io/name: apecloud-mysql
+spec:
+  backupType: Incremental
+  env:
+    - name: DATA_DIR
+      value: {{ .Values.mysqlConfigs.dataDir }}
+    - name: LOG_BIN
+      value: {{ .Values.mysqlConfigs.logBin }}
+    - name: DP_DB_PORT
+      value: "3306"
+    - name: DATA_MOUNT_DIR
+      value: {{ .Values.mysqlConfigs.dataMountPath }}
+    - name: SIGNAL_FILE
+      value: .xtrabackup_restore_new_cluster
+  backup:
+    preBackup: []
+    postBackup: []
+    backupData:
+      image: {{ include "apecloud-mysql.bakcupToolImage" . }}
+      runOnTargetPodNode: true
+      command:
+      - bash
+      - -c
+      - |
+        {{- .Files.Get "dataprotection/xtrabackup-incremental-backup.sh" | nindent 8 }}
+      syncProgress:
+        enabled: true
+        intervalSeconds: 5
+  restore:
+    prepareData:
+      image: {{ include "apecloud-mysql.bakcupToolImage" . }}
+      command:
+      - sh
+      - -c
+      - |
+        {{- .Files.Get "dataprotection/xtrabackup-incremental-restore.sh" | nindent 8 }}
+    postReady: []

addons/apecloud-mysql/templates/actionset.yaml

@@ -24,7 +24,7 @@ spec:
       image: {{ include "apecloud-mysql.bakcupToolImage" . }}
       runOnTargetPodNode: true
       command:
-      - sh
+      - bash
       - -c
       - |
         {{- .Files.Get "dataprotection/backup.sh" | nindent 8 }}

addons/apecloud-mysql/templates/backuppolicytemplate.yaml

@@ -20,6 +20,14 @@ spec:
         volumeMounts:
         - name: data
           mountPath: {{ .Values.mysqlConfigs.dataMountPath }}
+    - name: xtrabackup-inc
+      compatibleMethod: xtrabackup
+      snapshotVolumes: false
+      actionSetName: {{ include "apecloud-mysql.xtrabackupIncActionSetName" . }}
+      targetVolumes:
+        volumeMounts:
+        - name: data
+          mountPath: {{ .Values.mysqlConfigs.dataMountPath }}
     - name: volume-snapshot
       snapshotVolumes: true
       actionSetName: {{ include "apecloud-mysql.vsActionSetName" . }}
@@ -34,6 +42,10 @@ spec:
       enabled: false
       cronExpression: "0 18 * * *"
       retentionPeriod: 7d
+    - backupMethod: xtrabackup-inc
+      enabled: false
+      cronExpression: "0 19 * * *"
+      retentionPeriod: 7d
     - backupMethod: volume-snapshot
       enabled: false
       cronExpression: "0 18 * * *"

addons/mysql/dataprotection/backup.sh

@@ -21,7 +21,17 @@ if [ "${IMAGE_TAG}" == "2.4" ]; then
   lock_per_table_ddl="--lock-ddl-per-table"
 fi
 
+TMP_DIR=${MYSQL_DIR}/xtrabackup-temp
+mkdir -p ${TMP_DIR}
 xtrabackup --backup --safe-slave-backup --slave-info ${lock_per_table_ddl} --stream=xbstream \
-  --host=${DP_DB_HOST} --user=${DP_DB_USER} --password=${DP_DB_PASSWORD} --datadir=${DATA_DIR} | datasafed push -z zstd-fastest - "/${DP_BACKUP_NAME}.xbstream.zst"
+  --host=${DP_DB_HOST} --user=${DP_DB_USER} --password=${DP_DB_PASSWORD} --datadir=${DATA_DIR} \
+  2> >(tee ${TMP_DIR}/xtrabackup.log >&2) \
+  | datasafed push -z zstd-fastest - "/${DP_BACKUP_NAME}.xbstream.zst"
+# record lsn for incremental backups
+cat "${TMP_DIR}/xtrabackup.log" \
+  | grep "The latest check point (for incremental)" \
+  | awk -F"'" '{print $2}' \
+  | datasafed push - "/${DP_BACKUP_NAME}.lsn"
 TOTAL_SIZE=$(datasafed stat / | grep TotalSize | awk '{print $2}')
 echo "{\"totalSize\":\"$TOTAL_SIZE\"}" >"${DP_BACKUP_INFO_FILE}"
+rm -rf ${TMP_DIR}

addons/mysql/dataprotection/xtrabackup-incremental-backup.sh

@@ -0,0 +1,91 @@
+#!/bin/bash
+set -e
+set -o pipefail
+
+# if the script exits with a non-zero exit code, touch a file to indicate that the backup failed,
+# the sync progress container will check this file and exit if it exists
+function handle_exit() {
+  exit_code=$?
+  if [ $exit_code -ne 0 ]; then
+    echo "failed with exit code $exit_code"
+    touch "${DP_BACKUP_INFO_FILE}.exit"
+    exit 1
+  fi
+}
+trap handle_exit EXIT
+
+export PATH="$PATH:$DP_DATASAFED_BIN_PATH"
+
+# 1. check parent backup name
+if [[ -z ${DP_PARENT_BACKUP_NAME} ]]; then
+  echo "DP_PARENT_BACKUP_NAME is empty"
+  exit 1
+fi
+
+# 2. get parent backup lsn or download backup file
+mkdir -p ${DATA_DIR}
+PARENT_DIR=${MYSQL_DIR}/xtrabackup-parent
+
+# function get_last_lsn gets parent backup lsn from datasafed
+function get_last_lsn() {
+  export DATASAFED_BACKEND_BASE_PATH="${DP_BACKUP_ROOT_PATH}/${DP_PARENT_BACKUP_NAME}/${DP_TARGET_RELATIVE_PATH}"
+  lsnFile="${DP_PARENT_BACKUP_NAME}.lsn"
+  if [ "$(datasafed list ${lsnFile})" == "${lsnFile}" ]; then
+    echo "$(datasafed pull "/${lsnFile}" - | awk '{print $1}')"
+  fi
+}
+
+# function download_parent_backup_file downloads the parent backup file from datasafed
+function download_parent_backup_file() {
+  rm -rf ${PARENT_DIR}
+  mkdir -p ${PARENT_DIR} && cd ${PARENT_DIR}
+  export DATASAFED_BACKEND_BASE_PATH="${DP_BACKUP_ROOT_PATH}/${DP_PARENT_BACKUP_NAME}/${DP_TARGET_RELATIVE_PATH}"
+  xbstreamFile="${DP_PARENT_BACKUP_NAME}.xbstream.zst"
+  if [ "$(datasafed list ${xbstreamFile})" == "${xbstreamFile}" ]; then
+    datasafed pull -d zstd-fastest "${xbstreamFile}" - | xbstream -x
+  else
+    datasafed pull "${DP_PARENT_BACKUP_NAME}.xbstream" - | xbstream -x
+  fi
+  xtrabackup --decompress --remove-original --target-dir=${PARENT_DIR}
+}
+
+# build incremental cmd
+incremental_cmd=""
+last_lsn=$(get_last_lsn)
+if [ -n "${last_lsn}" ]; then
+  incremental_cmd="--incremental-lsn=${last_lsn}"
+  echo "create incremental backup based on parent backup lsn"
+else
+  download_parent_backup_file
+  incremental_cmd="--incremental-basedir=${PARENT_DIR}"
+  echo "create incremental backup based on parent backup file"
+fi
+
+# set the datasafed backend base path for the current backup
+# it is equal to ${DP_BACKUP_ROOT_PATH}/${DP_BACKUP_NAME}/${DP_TARGET_RELATIVE_PATH}
+export DATASAFED_BACKEND_BASE_PATH="$DP_BACKUP_BASE_PATH"
+
+# compatible with version 2.4
+lock_per_table_ddl=""
+if [ "${IMAGE_TAG}" == "2.4" ]; then
+  lock_per_table_ddl="--lock-ddl-per-table"
+fi
+
+# 3. do incremental xtrabackup
+TMP_DIR=${MYSQL_DIR}/xtrabackup-temp
+mkdir -p ${TMP_DIR}
+xtrabackup --backup --safe-slave-backup --slave-info ${lock_per_table_ddl} --stream=xbstream \
+  --host=${DP_DB_HOST} --port=${DP_DB_PORT} \
+  --user=${DP_DB_USER} --password=${DP_DB_PASSWORD} \
+  --datadir=${DATA_DIR} ${incremental_cmd} \
+  2> >(tee ${TMP_DIR}/xtrabackup.log >&2) \
+  | datasafed push -z zstd-fastest - "/${DP_BACKUP_NAME}.xbstream.zst"
+# record lsn for incremental backups
+cat "${TMP_DIR}/xtrabackup.log" \
+  | grep "The latest check point (for incremental)" \
+  | awk -F"'" '{print $2}' \
+  | datasafed push - "/${DP_BACKUP_NAME}.lsn"
+TOTAL_SIZE=$(datasafed stat / | grep TotalSize | awk '{print $2}')
+echo "{\"totalSize\":\"$TOTAL_SIZE\"}" >"${DP_BACKUP_INFO_FILE}"
+rm -rf ${PARENT_DIR}
+rm -rf ${TMP_DIR}