fix(NfsVolumeBackup): Short circuit reconciliation loop if completed

api/cloud-resources/v1beta1/condition.go

@@ -70,4 +70,5 @@ const (
 	ReasonUnknownSchedule       = "UnknownSchedule"
 	ReasonInvalidStartTime      = "InvalidStartTime"
 	ReasonInvalidEndTime        = "InvalidEndTime"
+	ReasonBackupFailed          = "BackupFailed"
 )

api/cloud-resources/v1beta1/gcpnfsvolumebackup_types.go

@@ -41,6 +41,9 @@ const (
 
 	// GcpNfsBackupDeleted signifies backup delete operation is complete.
 	GcpNfsBackupDeleted GcpNfsBackupState = "Deleted"
+
+	// GcpNfsBackupFailed signifies backup operation failed, and it will not be retried again.
+	GcpNfsBackupFailed GcpNfsBackupState = "Failed"
 )
 
 type GcpNfsVolumeBackupSource struct {

api/cloud-resources/v1beta1/state.go

@@ -10,6 +10,7 @@ const (
 	StateDeleting          = "Deleting"
 	StateUpdating          = "Updating"
 	StateDeleted           = "Deleted"
+	StateFailed            = "Failed"
 )
 
 const (

pkg/skr/awsnfsvolumebackup/markFailed.go

@@ -0,0 +1,89 @@
+package awsnfsvolumebackup
+
+import (
+	"context"
+	"fmt"
+	"github.com/kyma-project/cloud-manager/api/cloud-resources/v1beta1"
+	"github.com/kyma-project/cloud-manager/pkg/composed"
+	"k8s.io/apimachinery/pkg/api/meta"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"sigs.k8s.io/controller-runtime/pkg/client"
+)
+
+func markFailed(ctx context.Context, st composed.State) (error, context.Context) {
+	state := st.(*State)
+
+	//If deletion, continue.
+	if composed.MarkedForDeletionPredicate(ctx, st) {
+		return nil, nil
+	}
+
+	backup := state.ObjAsAwsNfsVolumeBackup()
+	backupState := backup.Status.State
+
+	//If not in error state, continue
+	if backupState != v1beta1.StateError {
+		return nil, ctx
+	}
+
+	//If this backup doesn't belong to a schedule, continue
+	scheduleName, exists := backup.GetLabels()[v1beta1.LabelScheduleName]
+	if !exists {
+		return nil, ctx
+	}
+
+	//createdOn := backup.GetCreationTimestamp().Format(time.RFC3339)
+	list := &v1beta1.AwsNfsVolumeBackupList{}
+
+	//List subsequent backups for this schedule.
+	err := state.Cluster().K8sClient().List(
+		ctx,
+		list,
+		client.MatchingLabels{
+			v1beta1.LabelScheduleName:      scheduleName,
+			v1beta1.LabelScheduleNamespace: backup.GetNamespace(),
+		},
+		client.InNamespace(backup.GetNamespace()),
+	)
+
+	if err != nil {
+
+		return composed.PatchStatus(backup).
+			SetExclusiveConditions(metav1.Condition{
+				Type:    v1beta1.ConditionTypeError,
+				Status:  metav1.ConditionTrue,
+				Reason:  v1beta1.ReasonBackupListFailed,
+				Message: fmt.Sprintf("Error listing subsequent backup(s) : %s", err.Error()),
+			}).
+			SuccessError(composed.StopWithRequeue).
+			Run(ctx, state)
+	}
+
+	//If there are subsequent backups exist,
+	//mark this backup object state as failed.
+	for _, item := range list.Items {
+
+		if item.CreationTimestamp.Time.After(backup.CreationTimestamp.Time) {
+			errCondition := meta.FindStatusCondition(backup.Status.Conditions, v1beta1.ConditionTypeError)
+			message := "Backup moved to Failed state as more recent backup(s) is available."
+			if errCondition != nil {
+				message = errCondition.Message + "\n" + message
+			}
+
+			backup.Status.State = v1beta1.StateFailed
+			return composed.PatchStatus(backup).
+				SetExclusiveConditions(metav1.Condition{
+					Type:    v1beta1.ConditionTypeError,
+					Status:  metav1.ConditionTrue,
+					Reason:  v1beta1.ReasonBackupFailed,
+					Message: message,
+				}).
+				SuccessLogMsg("AwsNfsVolumeBackup status updated with Failed state. ").
+				SuccessError(composed.StopAndForget).
+				Run(ctx, state)
+		}
+	}
+
+	//continue if there are no subsequent backups exist
+	return nil, ctx
+}

pkg/skr/awsnfsvolumebackup/markFailed_test.go

@@ -0,0 +1,215 @@
+package awsnfsvolumebackup
+
+import (
+	"context"
+	"github.com/go-logr/logr"
+	"github.com/kyma-project/cloud-manager/api/cloud-resources/v1beta1"
+	"github.com/kyma-project/cloud-manager/pkg/composed"
+	"github.com/stretchr/testify/suite"
+	v1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/types"
+	"sigs.k8s.io/controller-runtime/pkg/log"
+	"testing"
+	"time"
+)
+
+type markFailedSuite struct {
+	suite.Suite
+	ctx context.Context
+}
+
+func (suite *markFailedSuite) SetupTest() {
+	suite.ctx = log.IntoContext(context.Background(), logr.Discard())
+}
+
+func (suite *markFailedSuite) TestWhenBackupIsDeleting() {
+
+	obj := deletingAwsNfsVolumeBackup.DeepCopy()
+	factory, err := newStateFactoryWithObj(obj)
+	suite.Nil(err)
+
+	ctx, cancel := context.WithCancel(context.Background())
+	defer cancel()
+	//Get state object with AwsNfsVolume
+	state, err := factory.newStateWith(obj)
+	suite.Nil(err)
+
+	err, _ctx := markFailed(ctx, state)
+
+	//validate expected return values
+	suite.Nil(err)
+	suite.Nil(_ctx)
+}
+
+func (suite *markFailedSuite) TestWhenBackupIsReady() {
+
+	obj := awsNfsVolumeBackup.DeepCopy()
+	obj.Status.State = v1beta1.StateReady
+	factory, err := newStateFactoryWithObj(obj)
+	suite.Nil(err)
+
+	ctx, cancel := context.WithCancel(context.Background())
+	defer cancel()
+	//Get state object with AwsNfsVolume
+	state, err := factory.newStateWith(obj)
+	suite.Nil(err)
+
+	err, _ctx := markFailed(ctx, state)
+
+	//validate expected return values
+	suite.Nil(err)
+	suite.Equal(ctx, _ctx)
+
+	fromK8s := &v1beta1.AwsNfsVolumeBackup{}
+	err = factory.skrCluster.K8sClient().Get(ctx,
+		types.NamespacedName{Name: obj.Name,
+			Namespace: obj.Namespace},
+		fromK8s)
+	suite.Nil(err)
+
+	suite.Equal(v1beta1.StateReady, fromK8s.Status.State)
+}
+
+func (suite *markFailedSuite) TestWhenBackupIsFailed() {
+
+	obj := awsNfsVolumeBackup.DeepCopy()
+	obj.Status.State = v1beta1.StateFailed
+	factory, err := newStateFactoryWithObj(obj)
+	suite.Nil(err)
+
+	ctx, cancel := context.WithCancel(context.Background())
+	defer cancel()
+	//Get state object with AwsNfsVolume
+	state, err := factory.newStateWith(obj)
+	suite.Nil(err)
+
+	err, _ctx := markFailed(ctx, state)
+
+	//validate expected return values
+	suite.Nil(err)
+	suite.Equal(ctx, _ctx)
+
+	fromK8s := &v1beta1.AwsNfsVolumeBackup{}
+	err = factory.skrCluster.K8sClient().Get(ctx,
+		types.NamespacedName{Name: obj.Name,
+			Namespace: obj.Namespace},
+		fromK8s)
+	suite.Nil(err)
+
+	suite.Equal(v1beta1.StateFailed, fromK8s.Status.State)
+}
+
+func (suite *markFailedSuite) TestWhenBackupIsCreating() {
+
+	obj := awsNfsVolumeBackup.DeepCopy()
+	obj.Status.State = v1beta1.StateCreating
+	factory, err := newStateFactoryWithObj(obj)
+	suite.Nil(err)
+
+	ctx, cancel := context.WithCancel(context.Background())
+	defer cancel()
+	//Get state object with AwsNfsVolume
+	state, err := factory.newStateWith(obj)
+	suite.Nil(err)
+
+	err, _ctx := markFailed(ctx, state)
+
+	//validate expected return values
+	suite.Nil(err)
+	suite.Equal(ctx, _ctx)
+
+	fromK8s := &v1beta1.AwsNfsVolumeBackup{}
+	err = factory.skrCluster.K8sClient().Get(ctx,
+		types.NamespacedName{Name: obj.Name,
+			Namespace: obj.Namespace},
+		fromK8s)
+	suite.Nil(err)
+
+	suite.Equal(v1beta1.StateCreating, fromK8s.Status.State)
+}
+
+func (suite *markFailedSuite) TestWhenBackupIsLatestAndInError() {
+
+	obj := awsNfsVolumeBackup.DeepCopy()
+	obj.Status.State = v1beta1.StateError
+	factory, err := newStateFactoryWithObj(obj)
+	suite.Nil(err)
+
+	ctx, cancel := context.WithCancel(context.Background())
+	defer cancel()
+	//Get state object with AwsNfsVolume
+	state, err := factory.newStateWith(obj)
+	suite.Nil(err)
+
+	err, _ctx := markFailed(ctx, state)
+
+	//validate expected return values
+	suite.Nil(err)
+	suite.Equal(ctx, _ctx)
+
+	fromK8s := &v1beta1.AwsNfsVolumeBackup{}
+	err = factory.skrCluster.K8sClient().Get(ctx,
+		types.NamespacedName{Name: obj.Name,
+			Namespace: obj.Namespace},
+		fromK8s)
+	suite.Nil(err)
+
+	suite.Equal(v1beta1.StateError, fromK8s.Status.State)
+}
+
+func (suite *markFailedSuite) TestWhenBackupIsNotLatestAndInError() {
+
+	labels := map[string]string{
+		v1beta1.LabelScheduleName:      "test-schedule",
+		v1beta1.LabelScheduleNamespace: "test",
+	}
+
+	obj := awsNfsVolumeBackup.DeepCopy()
+	obj.CreationTimestamp = v1.Time{Time: time.Now().Add(-1 * time.Minute)}
+	obj.Labels = labels
+	factory, err := newStateFactoryWithObj(obj)
+	suite.Nil(err)
+
+	ctx, cancel := context.WithCancel(context.Background())
+	defer cancel()
+
+	//Get state object with AwsNfsVolume
+	state, err := factory.newStateWith(obj)
+	suite.Nil(err)
+
+	obj.Status.State = v1beta1.StateError
+	err = state.Cluster().K8sClient().Status().Update(ctx, obj)
+	suite.Nil(err)
+
+	//Create another backup object for the same schedule
+	obj2 := awsNfsVolumeBackup.DeepCopy()
+	obj2.Name = "test-backup-02"
+	obj2.Namespace = "test"
+	obj2.CreationTimestamp = v1.Time{Time: time.Now()}
+	obj2.Labels = labels
+	obj2.Status.State = v1beta1.StateReady
+	err = factory.skrCluster.K8sClient().Create(ctx, obj2)
+	suite.Nil(err)
+
+	err, _ctx := markFailed(ctx, state)
+
+	//validate expected return values
+	suite.Equal(composed.StopAndForget, err)
+	suite.Equal(ctx, _ctx)
+
+	fromK8s := &v1beta1.AwsNfsVolumeBackup{}
+	err = factory.skrCluster.K8sClient().Get(ctx,
+		types.NamespacedName{Name: obj.Name,
+			Namespace: obj.Namespace},
+		fromK8s)
+	suite.Nil(err)
+
+	suite.Equal(v1beta1.StateFailed, fromK8s.Status.State)
+	suite.Equal(v1beta1.ConditionTypeError, fromK8s.Status.Conditions[0].Type)
+	suite.Equal(v1.ConditionTrue, fromK8s.Status.Conditions[0].Status)
+	suite.Equal(v1beta1.ReasonBackupFailed, fromK8s.Status.Conditions[0].Reason)
+}
+
+func TestMarkFailed(t *testing.T) {
+	suite.Run(t, new(markFailedSuite))
+}

pkg/skr/awsnfsvolumebackup/reconciler.go

@@ -63,6 +63,8 @@ func (r *reconciler) newAction() composed.Action {
 		"AwsNfsVolumeBackupMain",
 		feature.LoadFeatureContextFromObj(&cloudresourcesv1beta1.AwsNfsVolumeBackup{}),
 		commonScope.New(),
+		shortCircuitCompleted,
+		markFailed,
 		addFinalizer,
 		loadSkrAwsNfsVolume,
 		stopIfVolumeNotReady,

pkg/skr/awsnfsvolumebackup/shortCircuit.go

@@ -0,0 +1,25 @@
+package awsnfsvolumebackup
+
+import (
+	"context"
+	"github.com/kyma-project/cloud-manager/api/cloud-resources/v1beta1"
+	"github.com/kyma-project/cloud-manager/pkg/composed"
+)
+
+func shortCircuitCompleted(ctx context.Context, st composed.State) (error, context.Context) {
+	state := st.(*State)
+
+	//If deletion, continue.
+	if composed.MarkedForDeletionPredicate(ctx, st) {
+		return nil, nil
+	}
+
+	backup := state.ObjAsAwsNfsVolumeBackup()
+	backupState := backup.Status.State
+	if backupState == v1beta1.StateReady || backupState == v1beta1.StateFailed {
+		composed.LoggerFromCtx(ctx).Info("NfsVolumeBackup is complete , short-circuiting into StopAndForget")
+		return composed.StopAndForget, nil
+	}
+
+	return nil, ctx
+}