Merge pull request #390 from portworx/PB-7645

PB-7645: Add pause to cleanup of resources if env is set in kdmp-config configmap

pkg/controllers/dataexport/reconcile.go

@@ -1131,7 +1131,13 @@ func (c *Controller) stageLocalSnapshotRestore(ctx context.Context, dataExport *
 		}
 		return true, c.updateStatus(dataExport, data)
 	} else if dataExport.Status.Status == kdmpapi.DataExportStatusFailed {
-		err := c.cleanupLocalRestoredSnapshotResources(dataExport, false)
+		val, err := utils.PauseCleanupResource()
+		if err == nil && val != 0 {
+			logrus.Debugf("Starting to wait for %v before cleanup of local restored snapshot resources for dataexport:%v", val, dataExport.Name)
+			time.Sleep(val)
+		}
+		logrus.Debugf("Starting the cleanup of local restored snapshot resources for dataexport:%v", dataExport.Name)
+		err = c.cleanupLocalRestoredSnapshotResources(dataExport, false)
 		if err != nil {
 			logrus.Errorf("cleaning up temporary resources for restoring from snapshot failed for data export %s/%s: %v", dataExport.Namespace, dataExport.Name, err)
 		}
@@ -1284,7 +1290,13 @@ func (c *Controller) stageLocalSnapshotRestoreInProgress(ctx context.Context, da
 		}
 		return true, c.updateStatus(dataExport, data)
 	} else if dataExport.Status.Status == kdmpapi.DataExportStatusFailed {
-		err := c.cleanupLocalRestoredSnapshotResources(dataExport, false)
+		val, err := utils.PauseCleanupResource()
+		if err == nil && val != 0 {
+			logrus.Debugf("Starting to wait for %v before cleanup of local restored snapshots for dataexport:%v", val, dataExport.Name)
+			time.Sleep(val)
+		}
+		logrus.Debugf("Starting the cleanup of local restored snapshot resources for dataexport:%v", dataExport.Name)
+		err = c.cleanupLocalRestoredSnapshotResources(dataExport, false)
 		// Already done with max retries, so moving to kdmp restore anyway
 		if err != nil {
 			logrus.Errorf("cleaning up temporary resources for restoring from snapshot failed for data export %s/%s: %v", dataExport.Namespace, dataExport.Name, err)
@@ -1455,6 +1467,12 @@ func (c *Controller) cleanUp(driver drivers.Interface, de *kdmpapi.DataExport) e
 		// No cleanup needed for rsync
 		return nil
 	}
+	val, err := utils.PauseCleanupResource()
+	if err == nil && val != 0 {
+		logrus.Infof("Starting to wait for %v before cleanup of dataexport:%v", val, de.Name)
+		time.Sleep(val)
+	}
+	logrus.Debugf("Starting the cleanup for dataexport:%v", de.Name)
 	if hasLocalRestoreStage(de) {
 		err := c.cleanupLocalRestoredSnapshotResources(de, true)
 		if err != nil {

pkg/controllers/resourceexport/reconcile.go

@@ -4,6 +4,7 @@ import (
 	"context"
 	"fmt"
 	"reflect"
+	"time"
 
 	storkapi "github.com/libopenstorage/stork/pkg/apis/stork/v1alpha1"
 	"github.com/libopenstorage/stork/pkg/controllers"
@@ -245,6 +246,12 @@ func (c *Controller) process(ctx context.Context, in *kdmpapi.ResourceExport) (b
 }
 
 func (c *Controller) cleanupResources(resourceExport *kdmpapi.ResourceExport) error {
+	val, err := utils.PauseCleanupResource()
+	if err == nil && val != 0 {
+		logrus.Debugf("Starting to wait for %v before cleanup of resourceExport:%v", val, resourceExport.Name)
+		time.Sleep(val)
+	}
+	logrus.Debugf("Starting the cleanup of resourceExport:%v", resourceExport.Name)
 	// clean up resources
 	rbNamespace, rbName, err := utils.ParseJobID(resourceExport.Status.TransferID)
 	if err != nil {

pkg/drivers/utils/utils.go

@@ -55,10 +55,12 @@ const (
 	ResourceCleanupKey = "RESOURCE_CLEANUP"
 	// ResourceCleanupDefaultValue is true as resource cleanup process is enabled by default for debugging user can set to false.
 	ResourceCleanupDefaultValue = "true"
-	volumeinitialDelay          = 2 * time.Second
-	volumeFactor                = 1.5
-	volumeSteps                 = 15
-	nfsVolumeSize               = "10Gi"
+	// PauseResourceCleanupKey - this key pauses the resource cleanup process.
+	PauseResourceCleanupKey = "PAUSE_RESOURCE_CLEANUP"
+	volumeinitialDelay      = 2 * time.Second
+	volumeFactor            = 1.5
+	volumeSteps             = 15
+	nfsVolumeSize           = "10Gi"
 	// ResourceUploadSuccessMsg - resource update success message
 	ResourceUploadSuccessMsg = "upload resource Successfully"
 	// PvcBoundSuccessMsg - pvc bound success message
@@ -1108,3 +1110,20 @@ func IsGcpHostedCluster() (bool, error) {
 	}
 	return false, nil
 }
+
+// PauseCleanupResource returns whether to pause the cleanup of the CRs & other resources.
+func PauseCleanupResource() (time.Duration, error) {
+	pauseCleanupVal := time.Duration(0)
+	pauseCleanupValStr, err := k8sutils.GetConfigValue(KdmpConfig, defaultPXNamespace, PauseResourceCleanupKey)
+	if err != nil {
+		logrus.Errorf("Failed to get %s key from kdmp-config-map: %v", PauseResourceCleanupKey, err)
+		return pauseCleanupVal, err
+	}
+	if pauseCleanupValStr != "" {
+		pauseCleanupVal, err = time.ParseDuration(pauseCleanupValStr)
+		if err != nil {
+			return pauseCleanupVal, err
+		}
+	}
+	return pauseCleanupVal, nil
+}