Delete manifestwork for application namespace

cmd/main.go

@@ -17,9 +17,6 @@ import (
 	volrep "github.com/csi-addons/kubernetes-csi-addons/api/replication.storage/v1alpha1"
 	groupsnapv1alpha1 "github.com/kubernetes-csi/external-snapshotter/client/v7/apis/volumegroupsnapshot/v1alpha1"
 	snapv1 "github.com/kubernetes-csi/external-snapshotter/client/v7/apis/volumesnapshot/v1"
-	clrapiv1beta1 "github.com/open-cluster-management-io/api/cluster/v1beta1"
-	ocmv1 "github.com/open-cluster-management/api/cluster/v1"
-	ocmworkv1 "github.com/open-cluster-management/api/work/v1"
 	viewv1beta1 "github.com/stolostron/multicloud-operators-foundation/pkg/apis/view/v1beta1"
 	plrv1 "github.com/stolostron/multicloud-operators-placementrule/pkg/apis/apps/v1"
 	velero "github.com/vmware-tanzu/velero/pkg/apis/velero/v1"
@@ -29,6 +26,9 @@ import (
 	"k8s.io/apimachinery/pkg/runtime"
 	utilruntime "k8s.io/apimachinery/pkg/util/runtime"
 	clientgoscheme "k8s.io/client-go/kubernetes/scheme"
+	ocmv1 "open-cluster-management.io/api/cluster/v1"
+	clrapiv1beta1 "open-cluster-management.io/api/cluster/v1beta1"
+	ocmworkv1 "open-cluster-management.io/api/work/v1"
 	cpcv1 "open-cluster-management.io/config-policy-controller/api/v1"
 	gppv1 "open-cluster-management.io/governance-policy-propagator/api/v1"
 	ctrl "sigs.k8s.io/controller-runtime"

go.mod

@@ -14,8 +14,6 @@ require (
 	github.com/kubernetes-csi/external-snapshotter/client/v7 v7.0.0
 	github.com/onsi/ginkgo/v2 v2.20.0
 	github.com/onsi/gomega v1.34.1
-	github.com/open-cluster-management-io/api v0.0.0-00010101000000-000000000000
-	github.com/open-cluster-management/api v0.0.0-20210527013639-a6845f2ebcb1
 	github.com/operator-framework/api v0.17.6
 	github.com/pkg/errors v0.9.1
 	github.com/prometheus/client_golang v1.18.0
@@ -33,6 +31,7 @@ require (
 	k8s.io/client-go v12.0.0+incompatible
 	k8s.io/component-base v0.30.1
 	k8s.io/kube-openapi v0.0.0-20240228011516-70dd3763d340
+	open-cluster-management.io/api v0.13.0
 	open-cluster-management.io/config-policy-controller v0.12.0
 	open-cluster-management.io/governance-policy-propagator v0.12.0
 	sigs.k8s.io/controller-runtime v0.18.4
@@ -99,13 +98,10 @@ require (
 	gopkg.in/yaml.v3 v3.0.1 // indirect
 	k8s.io/klog/v2 v2.130.1 // indirect
 	k8s.io/utils v0.0.0-20230726121419-3b25d923346b // indirect
-	open-cluster-management.io/api v0.11.1-0.20230905055724-cf1ead467a83 // indirect
 	open-cluster-management.io/multicloud-operators-subscription v0.12.0 // indirect
 	sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd // indirect
 	sigs.k8s.io/structured-merge-diff/v4 v4.4.1 // indirect
 )
 
 // replace directives to accommodate for stolostron
 replace k8s.io/client-go v12.0.0+incompatible => k8s.io/client-go v0.30.2
-
-replace github.com/open-cluster-management-io/api => open-cluster-management.io/api v0.10.0

internal/controller/controllers_utils_test.go

@@ -11,8 +11,8 @@ import (
 	. "github.com/onsi/gomega"
 	. "github.com/onsi/gomega/gstruct"
 	gomegaTypes "github.com/onsi/gomega/types"
-	ocmv1 "github.com/open-cluster-management/api/cluster/v1"
-	workv1 "github.com/open-cluster-management/api/work/v1"
+	ocmv1 "open-cluster-management.io/api/cluster/v1"
+	workv1 "open-cluster-management.io/api/work/v1"
 	"sigs.k8s.io/controller-runtime/pkg/client"
 
 	ramen "github.com/ramendr/ramen/api/v1alpha1"

internal/controller/drcluster_controller.go

@@ -11,7 +11,6 @@ import (
 	"strings"
 
 	"github.com/google/uuid"
-	ocmworkv1 "github.com/open-cluster-management/api/work/v1"
 	viewv1beta1 "github.com/stolostron/multicloud-operators-foundation/pkg/apis/view/v1beta1"
 	corev1 "k8s.io/api/core/v1"
 	"k8s.io/apimachinery/pkg/api/errors"
@@ -20,6 +19,7 @@ import (
 	"k8s.io/apimachinery/pkg/runtime"
 	"k8s.io/apimachinery/pkg/types"
 	"k8s.io/client-go/util/workqueue"
+	ocmworkv1 "open-cluster-management.io/api/work/v1"
 	ctrl "sigs.k8s.io/controller-runtime"
 	"sigs.k8s.io/controller-runtime/pkg/builder"
 	"sigs.k8s.io/controller-runtime/pkg/client"

internal/controller/drcluster_controller_test.go

@@ -10,7 +10,6 @@ import (
 	. "github.com/onsi/ginkgo/v2"
 	. "github.com/onsi/gomega"
 	. "github.com/onsi/gomega/gstruct"
-	workv1 "github.com/open-cluster-management/api/work/v1"
 	ramen "github.com/ramendr/ramen/api/v1alpha1"
 	controllers "github.com/ramendr/ramen/internal/controller"
 	"github.com/ramendr/ramen/internal/controller/util"
@@ -19,6 +18,7 @@ import (
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/runtime/schema"
 	"k8s.io/apimachinery/pkg/types"
+	workv1 "open-cluster-management.io/api/work/v1"
 	ctrl "sigs.k8s.io/controller-runtime"
 )
 

internal/controller/drcluster_drcconfig_test.go

@@ -12,7 +12,7 @@ import (
 	. "github.com/onsi/ginkgo/v2"
 	. "github.com/onsi/gomega"
 	. "github.com/onsi/gomega/gstruct"
-	ocmv1 "github.com/open-cluster-management/api/cluster/v1"
+	ocmv1 "open-cluster-management.io/api/cluster/v1"
 	ctrl "sigs.k8s.io/controller-runtime"
 	"sigs.k8s.io/controller-runtime/pkg/client"
 	"sigs.k8s.io/controller-runtime/pkg/envtest"

internal/controller/drcluster_mmode.go

@@ -5,10 +5,10 @@ package controllers
 
 import (
 	"github.com/go-logr/logr"
-	ocmworkv1 "github.com/open-cluster-management/api/work/v1"
 	viewv1beta1 "github.com/stolostron/multicloud-operators-foundation/pkg/apis/view/v1beta1"
 	"k8s.io/apimachinery/pkg/api/errors"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	ocmworkv1 "open-cluster-management.io/api/work/v1"
 
 	ramen "github.com/ramendr/ramen/api/v1alpha1"
 	"github.com/ramendr/ramen/internal/controller/util"

internal/controller/drcluster_mmode_test.go

@@ -11,8 +11,8 @@ import (
 
 	. "github.com/onsi/ginkgo/v2"
 	. "github.com/onsi/gomega"
-	ocmworkv1 "github.com/open-cluster-management/api/work/v1"
 	plrv1 "github.com/stolostron/multicloud-operators-placementrule/pkg/apis/apps/v1"
+	ocmworkv1 "open-cluster-management.io/api/work/v1"
 	ctrl "sigs.k8s.io/controller-runtime"
 	"sigs.k8s.io/controller-runtime/pkg/client"
 	"sigs.k8s.io/controller-runtime/pkg/envtest"

internal/controller/drplacementcontrol.go

@@ -10,12 +10,12 @@ import (
 	"time"
 
 	"github.com/go-logr/logr"
-	clrapiv1beta1 "github.com/open-cluster-management-io/api/cluster/v1beta1"
 	errorswrapper "github.com/pkg/errors"
 	"golang.org/x/exp/slices"
 	corev1 "k8s.io/api/core/v1"
 	"k8s.io/apimachinery/pkg/api/errors"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	clrapiv1beta1 "open-cluster-management.io/api/cluster/v1beta1"
 
 	rmn "github.com/ramendr/ramen/api/v1alpha1"
 	rmnutil "github.com/ramendr/ramen/internal/controller/util"
@@ -1842,7 +1842,8 @@ func (d *DRPCInstance) ensureVRGManifestWorkOnClusterDeleted(clusterName string)
 	}
 
 	if d.ensureVRGIsSecondaryOnCluster(clusterName) {
-		err := d.mwu.DeleteManifestWorksForCluster(clusterName)
+		// delete VRG manifest work
+		err = d.mwu.DeleteManifestWork(d.mwu.BuildManifestWorkName(rmnutil.MWTypeVRG), clusterName)
 		if err != nil {
 			return !done, fmt.Errorf("%w", err)
 		}

internal/controller/drplacementcontrol_controller.go

@@ -13,7 +13,6 @@ import (
 
 	"github.com/go-logr/logr"
 	"github.com/google/uuid"
-	ocmworkv1 "github.com/open-cluster-management/api/work/v1"
 	errorswrapper "github.com/pkg/errors"
 	viewv1beta1 "github.com/stolostron/multicloud-operators-foundation/pkg/apis/view/v1beta1"
 	plrv1 "github.com/stolostron/multicloud-operators-placementrule/pkg/apis/apps/v1"
@@ -24,6 +23,7 @@ import (
 	"k8s.io/apimachinery/pkg/types"
 	"k8s.io/apimachinery/pkg/util/sets"
 	"k8s.io/client-go/util/workqueue"
+	ocmworkv1 "open-cluster-management.io/api/work/v1"
 	ctrl "sigs.k8s.io/controller-runtime"
 	"sigs.k8s.io/controller-runtime/pkg/builder"
 	"sigs.k8s.io/controller-runtime/pkg/client"
@@ -34,11 +34,11 @@ import (
 	"sigs.k8s.io/controller-runtime/pkg/predicate"
 	"sigs.k8s.io/controller-runtime/pkg/reconcile"
 
-	clrapiv1beta1 "github.com/open-cluster-management-io/api/cluster/v1beta1"
 	rmn "github.com/ramendr/ramen/api/v1alpha1"
 	argocdv1alpha1hack "github.com/ramendr/ramen/internal/controller/argocd"
 	rmnutil "github.com/ramendr/ramen/internal/controller/util"
 	"github.com/ramendr/ramen/internal/controller/volsync"
+	clrapiv1beta1 "open-cluster-management.io/api/cluster/v1beta1"
 )
 
 const (
@@ -1206,18 +1206,11 @@ func (r *DRPlacementControlReconciler) finalizeDRPC(ctx context.Context, drpc *r
 	clonedPlRuleName := fmt.Sprintf(ClonedPlacementRuleNameFormat, drpc.Name, drpc.Namespace)
 	// delete cloned placementrule, if one created.
 	if drpc.Spec.PreferredCluster == "" {
-		err := r.deleteClonedPlacementRule(ctx, clonedPlRuleName, drpc.Namespace, log)
-		if err != nil {
+		if err := r.deleteClonedPlacementRule(ctx, clonedPlRuleName, drpc.Namespace, log); err != nil {
 			return err
 		}
 	}
 
-	// Cleanup volsync secret-related resources (policy/plrule/binding)
-	err := volsync.CleanupSecretPropagation(ctx, r.Client, drpc, r.Log)
-	if err != nil {
-		return fmt.Errorf("failed to clean up volsync secret-related resources (%w)", err)
-	}
-
 	vrgNamespace, err := selectVRGNamespace(r.Client, r.Log, drpc, placementObj)
 	if err != nil {
 		return err
@@ -1237,6 +1230,51 @@ func (r *DRPlacementControlReconciler) finalizeDRPC(ctx context.Context, drpc *r
 		return fmt.Errorf("failed to get DRPolicy while finalizing DRPC (%w)", err)
 	}
 
+	// Cleanup volsync secret-related resources (policy/plrule/binding)
+	if err := volsync.CleanupSecretPropagation(ctx, r.Client, drpc, r.Log); err != nil {
+		return fmt.Errorf("failed to clean up volsync secret-related resources (%w)", err)
+	}
+
+	// cleanup for VRG artifacts
+	if err = r.cleanupVRGs(ctx, drPolicy, log, mwu, drpc, vrgNamespace); err != nil {
+		return err
+	}
+
+	// delete namespace manifestwork
+	for _, drClusterName := range rmnutil.DRPolicyClusterNames(drPolicy) {
+		annotations := make(map[string]string)
+		annotations[DRPCNameAnnotation] = drpc.Name
+		annotations[DRPCNamespaceAnnotation] = drpc.Namespace
+
+		if err := mwu.DeleteNamespaceManifestWork(drClusterName, annotations); err != nil {
+			return err
+		}
+	}
+
+	// delete metrics if matching labels are found
+	syncTimeMetricLabels := SyncTimeMetricLabels(drPolicy, drpc)
+	DeleteSyncTimeMetric(syncTimeMetricLabels)
+
+	syncDurationMetricLabels := SyncDurationMetricLabels(drPolicy, drpc)
+	DeleteSyncDurationMetric(syncDurationMetricLabels)
+
+	syncDataBytesMetricLabels := SyncDataBytesMetricLabels(drPolicy, drpc)
+	DeleteSyncDataBytesMetric(syncDataBytesMetricLabels)
+
+	workloadProtectionLabels := WorkloadProtectionStatusLabels(drpc)
+	DeleteWorkloadProtectionStatusMetric(workloadProtectionLabels)
+
+	return nil
+}
+
+func (r *DRPlacementControlReconciler) cleanupVRGs(
+	ctx context.Context,
+	drPolicy *rmn.DRPolicy,
+	log logr.Logger,
+	mwu rmnutil.MWUtil,
+	drpc *rmn.DRPlacementControl,
+	vrgNamespace string,
+) error {
 	drClusters, err := GetDRClusters(ctx, r.Client, drPolicy)
 	if err != nil {
 		return fmt.Errorf("failed to get drclusters. Error (%w)", err)
@@ -1252,10 +1290,9 @@ func (r *DRPlacementControlReconciler) finalizeDRPC(ctx context.Context, drpc *r
 		return fmt.Errorf("VRG adoption in progress")
 	}
 
-	// delete manifestworks (VRGs)
+	// delete VRG manifestwork
 	for _, drClusterName := range rmnutil.DRPolicyClusterNames(drPolicy) {
-		err := mwu.DeleteManifestWorksForCluster(drClusterName)
-		if err != nil {
+		if err := mwu.DeleteManifestWork(mwu.BuildManifestWorkName(rmnutil.MWTypeVRG), drClusterName); err != nil {
 			return fmt.Errorf("%w", err)
 		}
 	}
@@ -1264,24 +1301,11 @@ func (r *DRPlacementControlReconciler) finalizeDRPC(ctx context.Context, drpc *r
 		return fmt.Errorf("waiting for VRGs count to go to zero")
 	}
 
-	// delete MCVs used in the previous call
+	// delete MCVs
 	if err := r.deleteAllManagedClusterViews(drpc, rmnutil.DRPolicyClusterNames(drPolicy)); err != nil {
 		return fmt.Errorf("error in deleting MCV (%w)", err)
 	}
 
-	// delete metrics if matching labels are found
-	syncTimeMetricLabels := SyncTimeMetricLabels(drPolicy, drpc)
-	DeleteSyncTimeMetric(syncTimeMetricLabels)
-
-	syncDurationMetricLabels := SyncDurationMetricLabels(drPolicy, drpc)
-	DeleteSyncDurationMetric(syncDurationMetricLabels)
-
-	syncDataBytesMetricLabels := SyncDataBytesMetricLabels(drPolicy, drpc)
-	DeleteSyncDataBytesMetric(syncDataBytesMetricLabels)
-
-	workloadProtectionLabels := WorkloadProtectionStatusLabels(drpc)
-	DeleteWorkloadProtectionStatusMetric(workloadProtectionLabels)
-
 	return nil
 }