feat: switch to using the datasafed tool for backup/restore scripts (#…

…5451)

(cherry picked from commit 59e81b5)

apis/dataprotection/v1alpha1/backuprepo_types.go

@@ -109,6 +109,7 @@ type BackupRepoStatus struct {
 // +kubebuilder:resource:path=backuprepos,categories={kubeblocks},scope=Cluster
 // +kubebuilder:printcolumn:name="STATUS",type="string",JSONPath=".status.phase"
 // +kubebuilder:printcolumn:name="STORAGEPROVIDER",type="string",JSONPath=".spec.storageProviderRef"
+// +kubebuilder:printcolumn:name="ACCESSMETHOD",type="string",JSONPath=".spec.accessMethod"
 // +kubebuilder:printcolumn:name="DEFAULT",type="boolean",JSONPath=`.status.isDefault`
 // +kubebuilder:printcolumn:name="AGE",type="date",JSONPath=".metadata.creationTimestamp"
 

config/crd/bases/dataprotection.kubeblocks.io_backuprepos.yaml

@@ -24,6 +24,9 @@ spec:
     - jsonPath: .spec.storageProviderRef
       name: STORAGEPROVIDER
       type: string
+    - jsonPath: .spec.accessMethod
+      name: ACCESSMETHOD
+      type: string
     - jsonPath: .status.isDefault
       name: DEFAULT
       type: boolean

controllers/dataprotection/backup_controller.go

@@ -302,20 +302,42 @@ func (r *BackupReconciler) handleBackupRepo(request *dpbackup.Request) error {
 	}
 	request.BackupRepo = repo
 
-	pvcName := repo.Status.BackupPVCName
-	if pvcName == "" {
-		return dperrors.NewBackupPVCNameIsEmpty(repo.Name, request.Spec.BackupPolicyName)
+	if repo.Status.Phase != dpv1alpha1.BackupRepoReady {
+		return dperrors.NewBackupRepoIsNotReady(repo.Name)
 	}
 
-	pvc := &corev1.PersistentVolumeClaim{}
-	pvcKey := client.ObjectKey{Namespace: request.Req.Namespace, Name: pvcName}
-	if err = r.Client.Get(request.Ctx, pvcKey, pvc); err != nil {
-		return client.IgnoreNotFound(err)
-	}
-
-	// backupRepo PVC exists, record the PVC name
-	if err == nil {
-		request.BackupRepoPVC = pvc
+	switch {
+	case repo.AccessByMount():
+		pvcName := repo.Status.BackupPVCName
+		if pvcName == "" {
+			return dperrors.NewBackupPVCNameIsEmpty(repo.Name, request.Spec.BackupPolicyName)
+		}
+		pvc := &corev1.PersistentVolumeClaim{}
+		pvcKey := client.ObjectKey{Namespace: request.Req.Namespace, Name: pvcName}
+		if err = r.Client.Get(request.Ctx, pvcKey, pvc); err != nil {
+			// will wait for the backuprepo controller to create the PVC,
+			// so ignore the NotFound error
+			return client.IgnoreNotFound(err)
+		}
+		// backupRepo PVC exists, record the PVC name
+		if err == nil {
+			request.BackupRepoPVC = pvc
+		}
+	case repo.AccessByTool():
+		toolConfigSecretName := repo.Status.ToolConfigSecretName
+		if toolConfigSecretName == "" {
+			return dperrors.NewToolConfigSecretNameIsEmpty(repo.Name)
+		}
+		secret := &corev1.Secret{}
+		secretKey := client.ObjectKey{Namespace: request.Req.Namespace, Name: toolConfigSecretName}
+		if err = r.Client.Get(request.Ctx, secretKey, secret); err != nil {
+			// will wait for the backuprepo controller to create the secret,
+			// so ignore the NotFound error
+			return client.IgnoreNotFound(err)
+		}
+		if err == nil {
+			request.ToolConfigSecret = secret
+		}
 	}
 	return nil
 }
@@ -327,8 +349,10 @@ func (r *BackupReconciler) patchBackupStatus(
 	request.Status.Path = dpbackup.BuildBackupPath(request.Backup, request.BackupPolicy.Spec.PathPrefix)
 	request.Status.Target = request.BackupPolicy.Spec.Target
 	request.Status.BackupMethod = request.BackupMethod
-	request.Status.PersistentVolumeClaimName = request.BackupRepoPVC.Name
 	request.Status.BackupRepoName = request.BackupRepo.Name
+	if request.BackupRepoPVC != nil {
+		request.Status.PersistentVolumeClaimName = request.BackupRepoPVC.Name
+	}
 
 	// init action status
 	actions, err := request.BuildActions()
@@ -383,10 +407,10 @@ func (r *BackupReconciler) patchBackupObjectMeta(
 	request.Labels[constant.AppManagedByLabelKey] = constant.AppName
 	request.Labels[dataProtectionLabelBackupTypeKey] = request.GetBackupType()
 
-	// if the backupRepo PVC is not present, add a special label and wait for the
-	// backup repo controller to create the PVC.
+	// wait for the backup repo controller to prepare the essential resource.
 	wait := false
-	if request.BackupRepoPVC == nil {
+	if (request.BackupRepo.AccessByMount() && request.BackupRepoPVC == nil) ||
+		(request.BackupRepo.AccessByTool() && request.ToolConfigSecret == nil) {
 		request.Labels[dataProtectionWaitRepoPreparationKey] = trueVal
 		wait = true
 	}

controllers/dataprotection/backup_controller_test.go

@@ -208,7 +208,7 @@ var _ = Describe("Backup Controller test", func() {
 				jobKey := dpbackup.BuildDeleteBackupFilesJobKey(backup)
 				job := &batchv1.Job{}
 				Eventually(testapps.CheckObjExists(&testCtx, jobKey, job, true)).Should(Succeed())
-				volumeName := dpbackup.GenerateBackupRepoVolumeName(repoPVCName)
+				volumeName := "dp-backup-data"
 				Eventually(testapps.CheckObj(&testCtx, jobKey, func(g Gomega, job *batchv1.Job) {
 					Expect(job.Spec.Template.Spec.Volumes).
 						Should(ContainElement(corev1.Volume{
@@ -476,6 +476,7 @@ var _ = Describe("Backup Controller test", func() {
 					Eventually(testapps.GetAndChangeObjStatus(&testCtx, client.ObjectKeyFromObject(sp),
 						func(fetched *storagev1alpha1.StorageProvider) {
 							fetched.Status.Phase = storagev1alpha1.StorageProviderNotReady
+							fetched.Status.Conditions = nil
 						})).ShouldNot(HaveOccurred())
 					Eventually(testapps.CheckObj(&testCtx, client.ObjectKeyFromObject(repo),
 						func(g Gomega, repo *dpv1alpha1.BackupRepo) {

controllers/dataprotection/backuprepo_controller.go

@@ -266,6 +266,11 @@ func (r *BackupRepoReconciler) checkStorageProvider(
 			reason = ReasonInvalidStorageProvider
 			return provider, newDependencyError("both StorageClassTemplate and PersistentVolumeClaimTemplate are empty")
 		}
+		csiInstalledCond := meta.FindStatusCondition(provider.Status.Conditions, storagev1alpha1.ConditionTypeCSIDriverInstalled)
+		if csiInstalledCond == nil || csiInstalledCond.Status != metav1.ConditionTrue {
+			reason = ReasonStorageProviderNotReady
+			return provider, newDependencyError("CSI driver is not installed")
+		}
 	case repo.AccessByTool():
 		if provider.Spec.DatasafedConfigTemplate == "" {
 			reason = ReasonInvalidStorageProvider
@@ -274,15 +279,8 @@ func (r *BackupRepoReconciler) checkStorageProvider(
 	}
 
 	// check its status
-	if provider.Status.Phase == storagev1alpha1.StorageProviderReady {
-		reason = ReasonStorageProviderReady
-		return provider, nil
-	} else {
-		reason = ReasonStorageProviderNotReady
-		err = newDependencyError(fmt.Sprintf("storage provider %s is not ready, status: %s",
-			provider.Name, provider.Status.Phase))
-		return provider, err
-	}
+	reason = ReasonStorageProviderReady
+	return provider, nil
 }
 
 func (r *BackupRepoReconciler) checkParameters(reqCtx intctrlutil.RequestCtx,

controllers/dataprotection/backuprepo_controller_test.go

@@ -185,6 +185,11 @@ parameters:
     secret-namespace: {{ .CSIDriverSecretRef.Namespace }}
 `
 			obj.Status.Phase = storagev1alpha1.StorageProviderReady
+			meta.SetStatusCondition(&obj.Status.Conditions, metav1.Condition{
+				Type:   storagev1alpha1.ConditionTypeCSIDriverInstalled,
+				Status: metav1.ConditionTrue,
+				Reason: "CSIDriverInstalled",
+			})
 			if mutateFunc != nil {
 				mutateFunc(obj)
 			}
@@ -306,6 +311,11 @@ parameters:
 			By("updating the status of the storage provider to not ready")
 			Eventually(testapps.GetAndChangeObjStatus(&testCtx, providerKey, func(provider *storagev1alpha1.StorageProvider) {
 				provider.Status.Phase = storagev1alpha1.StorageProviderNotReady
+				meta.SetStatusCondition(&provider.Status.Conditions, metav1.Condition{
+					Type:   storagev1alpha1.ConditionTypeCSIDriverInstalled,
+					Status: metav1.ConditionFalse,
+					Reason: "CSINotInstalled",
+				})
 			})).Should(Succeed())
 			By("checking the status of the BackupRepo, should become failed")
 			Eventually(testapps.CheckObj(&testCtx, repoKey, func(g Gomega, repo *dpv1alpha1.BackupRepo) {
@@ -664,8 +674,7 @@ spec:
 			var backup *dpv1alpha1.Backup
 			var toolConfigSecretKey types.NamespacedName
 
-			BeforeEach(func() {
-				By("preparing")
+			createStorageProviderSpecForToolAccessMethod := func(mutateFunc func(provider *storagev1alpha1.StorageProvider)) {
 				createStorageProviderSpec(func(provider *storagev1alpha1.StorageProvider) {
 					provider.Spec.DatasafedConfigTemplate = `
 [storage]
@@ -675,7 +684,15 @@ key2={{ index .Parameters "key2" }}
 cred-key1={{ index .Parameters "cred-key1" }}
 cred-key2={{ index .Parameters "cred-key2" }}
 `
+					if mutateFunc != nil {
+						mutateFunc(provider)
+					}
 				})
+			}
+
+			BeforeEach(func() {
+				By("preparing")
+				createStorageProviderSpecForToolAccessMethod(nil)
 				createBackupRepoSpec(func(repo *dpv1alpha1.BackupRepo) {
 					repo.Spec.AccessMethod = dpv1alpha1.AccessMethodTool
 				})
@@ -695,7 +712,7 @@ cred-key2={{ index .Parameters "cred-key2" }}
 
 			It("should check that the storage provider has a non-empty datasafedConfigTemplate", func() {
 				By("preparing")
-				createStorageProviderSpec(func(provider *storagev1alpha1.StorageProvider) {
+				createStorageProviderSpecForToolAccessMethod(func(provider *storagev1alpha1.StorageProvider) {
 					provider.Spec.DatasafedConfigTemplate = ""
 				})
 				createBackupRepoSpec(func(repo *dpv1alpha1.BackupRepo) {
@@ -714,7 +731,7 @@ cred-key2={{ index .Parameters "cred-key2" }}
 
 			It("should fail if the datasafedConfigTemplate is invalid", func() {
 				By("preparing")
-				createStorageProviderSpec(func(provider *storagev1alpha1.StorageProvider) {
+				createStorageProviderSpecForToolAccessMethod(func(provider *storagev1alpha1.StorageProvider) {
 					provider.Spec.DatasafedConfigTemplate = "bad template {{"
 				})
 				createBackupRepoSpec(func(repo *dpv1alpha1.BackupRepo) {
@@ -730,6 +747,25 @@ cred-key2={{ index .Parameters "cred-key2" }}
 				})).Should(Succeed())
 			})
 
+			It("should work even if the CSI driver required by the storage provider is not installed", func() {
+				By("preparing")
+				createStorageProviderSpecForToolAccessMethod(func(provider *storagev1alpha1.StorageProvider) {
+					provider.Status.Phase = storagev1alpha1.StorageProviderNotReady
+					meta.SetStatusCondition(&provider.Status.Conditions, metav1.Condition{
+						Type:   storagev1alpha1.ConditionTypeCSIDriverInstalled,
+						Status: metav1.ConditionFalse,
+						Reason: "NotInstalled",
+					})
+				})
+				createBackupRepoSpec(func(repo *dpv1alpha1.BackupRepo) {
+					repo.Spec.AccessMethod = dpv1alpha1.AccessMethodTool
+				})
+				By("checking")
+				Eventually(testapps.CheckObj(&testCtx, repoKey, func(g Gomega, repo *dpv1alpha1.BackupRepo) {
+					g.Expect(repo.Status.Phase).Should(Equal(dpv1alpha1.BackupRepoReady))
+				})).Should(Succeed())
+			})
+
 			It("should create the secret containing the tool config", func() {
 				Eventually(testapps.CheckObj(&testCtx, toolConfigSecretKey, func(g Gomega, secret *corev1.Secret) {
 					g.Expect(secret.Data).Should(HaveKeyWithValue("datasafed.conf", []byte(`

controllers/dataprotection/volumepopulator_controller.go

@@ -173,7 +173,7 @@ func (r *VolumePopulatorReconciler) populate(reqCtx intctrlutil.RequestCtx, pvc
 		}
 
 		// 1. build populate job
-		job, err := restoreMgr.BuildVolumePopulateJob(v, populatePVC, i)
+		job, err := restoreMgr.BuildVolumePopulateJob(reqCtx, r.Client, v, populatePVC, i)
 		if err != nil {
 			return err
 		}

deploy/apecloud-mysql/dataprotection/backup.sh

@@ -1,12 +1,13 @@
 #!/bin/bash
 set -e
-if [ -d ${DP_BACKUP_DIR} ]; then
-    rm -rf ${DP_BACKUP_DIR}
-fi
-mkdir -p ${DP_BACKUP_DIR}
+set -o pipefail
+export PATH="$PATH:$DP_DATASAFED_BIN_PATH"
+export DATASAFED_BACKEND_BASE_PATH="$DP_BACKUP_BASE_PATH"
 START_TIME=$(date -u '+%Y-%m-%dT%H:%M:%SZ')
 xtrabackup --compress=zstd --backup --safe-slave-backup --slave-info --stream=xbstream \
-    --host=${DP_DB_HOST} --user=${DP_DB_USER} --port=${DP_DB_PORT} --password=${DP_DB_PASSWORD} --datadir=${DATA_DIR} >${DP_BACKUP_DIR}/${DP_BACKUP_NAME}.xbstream
+    --host=${DP_DB_HOST} --port=${DP_DB_PORT} \
+    --user=${DP_DB_USER} --password=${DP_DB_PASSWORD} \
+    --datadir=${DATA_DIR} | datasafed push - "/${DP_BACKUP_NAME}.xbstream"
 STOP_TIME=$(date -u '+%Y-%m-%dT%H:%M:%SZ')
-TOTAL_SIZE=$(du -shx ${DP_BACKUP_DIR} | awk '{print $1}')
-echo "{\"totalSize\":\"$TOTAL_SIZE\",\"timeRange\":{\"start\":\"${START_TIME}\",\"end\":\"${STOP_TIME}\"}}" >${DP_BACKUP_DIR}/backup.info
+TOTAL_SIZE=$(datasafed stat / | grep TotalSize | awk '{print $2}')
+echo "{\"totalSize\":\"$TOTAL_SIZE\",\"timeRange\":{\"start\":\"${START_TIME}\",\"end\":\"${STOP_TIME}\"}}" > "${DP_BACKUP_INFO_FILE}"

deploy/apecloud-mysql/dataprotection/restore.sh

@@ -1,9 +1,12 @@
 #!/bin/bash
 set -e
+set -o pipefail
+export PATH="$PATH:$DP_DATASAFED_BIN_PATH"
+export DATASAFED_BACKEND_BASE_PATH="$DP_BACKUP_BASE_PATH"
 mkdir -p ${DATA_DIR}
 TMP_DIR=${DATA_MOUNT_DIR}/temp
 mkdir -p ${TMP_DIR} && cd ${TMP_DIR}
-xbstream -x <${DP_BACKUP_DIR}/${DP_BACKUP_NAME}.xbstream
+datasafed pull "${DP_BACKUP_NAME}.xbstream" - | xbstream -x
 xtrabackup --decompress --remove-original --target-dir=${TMP_DIR}
 xtrabackup --prepare --target-dir=${TMP_DIR}
 xtrabackup --move-back --target-dir=${TMP_DIR} --datadir=${DATA_DIR}/ --log-bin=${LOG_BIN}

deploy/helm/crds/dataprotection.kubeblocks.io_backuprepos.yaml

@@ -24,6 +24,9 @@ spec:
     - jsonPath: .spec.storageProviderRef
       name: STORAGEPROVIDER
       type: string
+    - jsonPath: .spec.accessMethod
+      name: ACCESSMETHOD
+      type: string
     - jsonPath: .status.isDefault
       name: DEFAULT
       type: boolean

deploy/helm/templates/addons/csi-s3-addon.yaml

@@ -51,8 +51,10 @@ spec:
     {{- $autoInstall := (get ( get ( .Values | toYaml | fromYaml ) "csi-s3" ) "enabled") }}
     {{- /* auto install csi-s3 if it's required by backup repos */ -}}
     {{- if .Values.backupRepo.create }}
-      {{- if eq .Values.backupRepo.storageProvider "s3" "oss" "minio" "obs" "cos" "gcs" }}
-        {{- $autoInstall = true }}
+      {{- if eq .Values.backupRepo.accessMethod "Mount" }}
+        {{- if eq .Values.backupRepo.storageProvider "s3" "oss" "minio" "obs" "cos" "gcs" }}
+          {{- $autoInstall = true }}
+        {{- end }}
       {{- end }}
     {{- end }}
     autoInstall: {{ $autoInstall }}

deploy/helm/templates/dataprotection.yaml

@@ -87,6 +87,8 @@ spec:
             - name: ENABLE_WEBHOOKS
               value: "true"
             {{- end }}
+            - name: DATASAFED_IMAGE
+              value: "{{ .Values.dataProtection.image.registry | default "docker.io" }}/{{ .Values.dataProtection.image.datasafed.repository }}:{{ .Values.dataProtection.image.datasafed.tag | default "latest" }}"
           {{- with .Values.securityContext }}
           securityContext:
             {{- toYaml . | nindent 12 }}

deploy/helm/templates/storageprovider/cos.yaml

@@ -42,6 +42,7 @@ spec:
     secret_access_key = {{ `{{ index .Parameters "secretAccessKey" }}` }}
     endpoint = {{ `{{ printf "cos.%s.myqcloud.com" .Parameters.region }}` }}
     root = {{ `{{ index .Parameters "bucket" }}` }}
+    chunk_size = 50Mi
 
   parametersSchema:
     openAPIV3Schema:

deploy/helm/templates/storageprovider/gcs.yaml

@@ -45,6 +45,7 @@ spec:
     {{ `{{- end }}` }}
     endpoint = {{ `{{ $endpoint }}` }}
     root = {{ `{{ index .Parameters "bucket" }}` }}
+    chunk_size = 50Mi
 
   parametersSchema:
     openAPIV3Schema:

deploy/helm/templates/storageprovider/minio.yaml

@@ -36,6 +36,7 @@ spec:
     secret_access_key = {{ `{{ index .Parameters "secretAccessKey" }}` }}
     endpoint = {{ `{{ index .Parameters "endpoint" }}` }}
     root = {{ `{{ index .Parameters "bucket" }}` }}
+    chunk_size = 50Mi
 
   parametersSchema:
     openAPIV3Schema:

deploy/helm/templates/storageprovider/obs.yaml

@@ -43,6 +43,7 @@ spec:
     region = {{ `{{ index .Parameters "region" }}` }}
     endpoint = {{ `{{ printf "obs.%s.myhuaweicloud.com" .Parameters.region }}` }}
     root = {{ `{{ index .Parameters "bucket" }}` }}
+    chunk_size = 50Mi
 
   parametersSchema:
     openAPIV3Schema:

deploy/helm/templates/storageprovider/oss.yaml

@@ -41,6 +41,7 @@ spec:
     secret_access_key = {{ `{{ index .Parameters "secretAccessKey" }}` }}
     endpoint = {{ `{{- printf "oss-%s.aliyuncs.com" .Parameters.region) }}` }}
     root = {{ `{{ index .Parameters "bucket" }}` }}
+    chunk_size = 50Mi
 
   parametersSchema:
     openAPIV3Schema:

deploy/helm/templates/storageprovider/s3.yaml

@@ -46,6 +46,7 @@ spec:
     region = {{ `{{ index .Parameters "region" }}` }}
     endpoint = {{ `{{ index .Parameters "endpoint" }}` }}
     root = {{ `{{ index .Parameters "bucket" }}` }}
+    chunk_size = 50Mi
 
   parametersSchema:
     openAPIV3Schema:

deploy/helm/values.yaml

@@ -322,11 +322,15 @@ dataProtection:
     # Overrides the image tag whose default is the chart appVersion.
     tag: ""
     imagePullSecrets: []
+    datasafed:
+      repository: apecloud/datasafed
+      tag: "0.0.3"
 
 ## BackupRepo settings
 ##
 ## @param backupRepo.create - creates a backup repo during installation
 ## @param backupRepo.default - set the created repo as the default
+## @param backupRepo.accessMethod - the access method for the backup repo, options: [Mount, Tool]
 ## @param backupRepo.storageProvider - the storage provider used by the repo, options: [s3, oss, minio]
 ## @param backupRepo.pvReclaimPolicy - the PV reclaim policy, options: [Retain, Delete]
 ## @param backupRepo.volumeCapacity - the capacity for creating PVC
@@ -335,6 +339,7 @@ dataProtection:
 backupRepo:
   create: false
   default: true
+  accessMethod: Tool
   storageProvider: ""
   pvReclaimPolicy: "Retain"
   volumeCapacity: ""