From 2880e70751a9202472c8da2b1e1b1c1592e8bf88 Mon Sep 17 00:00:00 2001 From: Abhisheka Date: Wed, 31 Jul 2024 13:42:35 +0000 Subject: [PATCH] PB-7645: Add pause to cleanup of resources if env is set in kdmp-config configmap Signed-off-by: Abhisheka --- pkg/controllers/dataexport/reconcile.go | 22 +++++++++++++++-- pkg/controllers/resourceexport/reconcile.go | 7 ++++++ pkg/drivers/utils/utils.go | 27 ++++++++++++++++++--- 3 files changed, 50 insertions(+), 6 deletions(-) diff --git a/pkg/controllers/dataexport/reconcile.go b/pkg/controllers/dataexport/reconcile.go index 25ab340bf..7e8630537 100644 --- a/pkg/controllers/dataexport/reconcile.go +++ b/pkg/controllers/dataexport/reconcile.go @@ -1131,7 +1131,13 @@ func (c *Controller) stageLocalSnapshotRestore(ctx context.Context, dataExport * } return true, c.updateStatus(dataExport, data) } else if dataExport.Status.Status == kdmpapi.DataExportStatusFailed { - err := c.cleanupLocalRestoredSnapshotResources(dataExport, false) + val, err := utils.PauseCleanupResource() + if err == nil && val != 0 { + logrus.Debugf("Starting to wait for %v before cleanup of local restored snapshot resources for dataexport:%v", val, dataExport.Name) + time.Sleep(val) + } + logrus.Debugf("Starting the cleanup of local restored snapshot resources for dataexport:%v", dataExport.Name) + err = c.cleanupLocalRestoredSnapshotResources(dataExport, false) if err != nil { logrus.Errorf("cleaning up temporary resources for restoring from snapshot failed for data export %s/%s: %v", dataExport.Namespace, dataExport.Name, err) } @@ -1284,7 +1290,13 @@ func (c *Controller) stageLocalSnapshotRestoreInProgress(ctx context.Context, da } return true, c.updateStatus(dataExport, data) } else if dataExport.Status.Status == kdmpapi.DataExportStatusFailed { - err := c.cleanupLocalRestoredSnapshotResources(dataExport, false) + val, err := utils.PauseCleanupResource() + if err == nil && val != 0 { + logrus.Debugf("Starting to wait for %v before cleanup of local restored snapshots for dataexport:%v", val, dataExport.Name) + time.Sleep(val) + } + logrus.Debugf("Starting the cleanup of local restored snapshot resources for dataexport:%v", dataExport.Name) + err = c.cleanupLocalRestoredSnapshotResources(dataExport, false) // Already done with max retries, so moving to kdmp restore anyway if err != nil { logrus.Errorf("cleaning up temporary resources for restoring from snapshot failed for data export %s/%s: %v", dataExport.Namespace, dataExport.Name, err) @@ -1455,6 +1467,12 @@ func (c *Controller) cleanUp(driver drivers.Interface, de *kdmpapi.DataExport) e // No cleanup needed for rsync return nil } + val, err := utils.PauseCleanupResource() + if err == nil && val != 0 { + logrus.Infof("Starting to wait for %v before cleanup of dataexport:%v", val, de.Name) + time.Sleep(val) + } + logrus.Debugf("Starting the cleanup for dataexport:%v", de.Name) if hasLocalRestoreStage(de) { err := c.cleanupLocalRestoredSnapshotResources(de, true) if err != nil { diff --git a/pkg/controllers/resourceexport/reconcile.go b/pkg/controllers/resourceexport/reconcile.go index 56fb7b4a9..9edff22a9 100644 --- a/pkg/controllers/resourceexport/reconcile.go +++ b/pkg/controllers/resourceexport/reconcile.go @@ -4,6 +4,7 @@ import ( "context" "fmt" "reflect" + "time" storkapi "github.com/libopenstorage/stork/pkg/apis/stork/v1alpha1" "github.com/libopenstorage/stork/pkg/controllers" @@ -245,6 +246,12 @@ func (c *Controller) process(ctx context.Context, in *kdmpapi.ResourceExport) (b } func (c *Controller) cleanupResources(resourceExport *kdmpapi.ResourceExport) error { + val, err := utils.PauseCleanupResource() + if err == nil && val != 0 { + logrus.Debugf("Starting to wait for %v before cleanup of resourceExport:%v", val, resourceExport.Name) + time.Sleep(val) + } + logrus.Debugf("Starting the cleanup of resourceExport:%v", resourceExport.Name) // clean up resources rbNamespace, rbName, err := utils.ParseJobID(resourceExport.Status.TransferID) if err != nil { diff --git a/pkg/drivers/utils/utils.go b/pkg/drivers/utils/utils.go index dcbacac03..ab566c84b 100644 --- a/pkg/drivers/utils/utils.go +++ b/pkg/drivers/utils/utils.go @@ -55,10 +55,12 @@ const ( ResourceCleanupKey = "RESOURCE_CLEANUP" // ResourceCleanupDefaultValue is true as resource cleanup process is enabled by default for debugging user can set to false. ResourceCleanupDefaultValue = "true" - volumeinitialDelay = 2 * time.Second - volumeFactor = 1.5 - volumeSteps = 15 - nfsVolumeSize = "10Gi" + // PauseResourceCleanupKey - this key pauses the resource cleanup process. + PauseResourceCleanupKey = "PAUSE_RESOURCE_CLEANUP" + volumeinitialDelay = 2 * time.Second + volumeFactor = 1.5 + volumeSteps = 15 + nfsVolumeSize = "10Gi" // ResourceUploadSuccessMsg - resource update success message ResourceUploadSuccessMsg = "upload resource Successfully" // PvcBoundSuccessMsg - pvc bound success message @@ -1108,3 +1110,20 @@ func IsGcpHostedCluster() (bool, error) { } return false, nil } + +// PauseCleanupResource returns whether to pause the cleanup of the CRs & other resources. +func PauseCleanupResource() (time.Duration, error) { + pauseCleanupVal := time.Duration(0) + pauseCleanupValStr, err := k8sutils.GetConfigValue(KdmpConfig, defaultPXNamespace, PauseResourceCleanupKey) + if err != nil { + logrus.Errorf("Failed to get %s key from kdmp-config-map: %v", PauseResourceCleanupKey, err) + return pauseCleanupVal, err + } + if pauseCleanupValStr != "" { + pauseCleanupVal, err = time.ParseDuration(pauseCleanupValStr) + if err != nil { + return pauseCleanupVal, err + } + } + return pauseCleanupVal, nil +}