clean omap details

Signed-off-by: yati1998 <[email protected]>

.github/workflows/go-test.yaml

@@ -77,7 +77,7 @@ jobs:
 
       - name: Subvolume command
         run: |
-          set -ex
+          set -e
           kubectl rook-ceph ceph fs subvolume create myfs test-subvol group-a
           kubectl rook-ceph subvolume ls
           kubectl rook-ceph subvolume ls --stale
@@ -233,7 +233,7 @@ jobs:
 
       - name: Subvolume command
         run: |
-          set -ex
+          set -e
           kubectl rook-ceph --operator-namespace test-operator -n test-cluster ceph fs subvolume create myfs test-subvol group-a
           kubectl rook-ceph --operator-namespace test-operator -n test-cluster subvolume ls
           kubectl rook-ceph --operator-namespace test-operator -n test-cluster subvolume ls --stale

docs/subvolume.md

@@ -40,16 +40,20 @@ kubectl rook-ceph subvolume ls --stale
 ## delete
 
 ```bash
-kubectl rook-ceph subvolume delete csi-vol-427774b4-340b-11ed-8d66-0242ac110004 ocs-storagecluster csi
+kubectl rook-ceph subvolume delete ocs-storagecluster-cephfilesystem csi-vol-427774b4-340b-11ed-8d66-0242ac110005 csi
 
 # Info: subvolume csi-vol-427774b4-340b-11ed-8d66-0242ac110004 deleted
+# Info: omap object:"csi.volume.427774b4-340b-11ed-8d66-0242ac110005" deleted
+# Info: omap key:"csi.volume.pvc-fca205e5-8788-4132-979c-e210c0133182" deleted
 
 ```
 
 ```bash
-kubectl rook-ceph subvolume delete csi-vol-427774b4-340b-11ed-8d66-0242ac110004,csi-vol-427774b4-340b-11ed-8d66-0242ac110005 ocs-storagecluster csi
+kubectl rook-ceph subvolume delete ocs-storagecluster-cephfilesystem csi-vol-test csi
 
-# Info: subvolume csi-vol-427774b4-340b-11ed-8d66-0242ac110004 deleted
-# Info: subvolume csi-vol-427774b4-340b-11ed-8d66-0242ac110004 deleted
+# Info: subvolume "csi-vol-test" deleted
+# Error: error getting omap keys ocs-storagecluster-cephfilesystem-metadata/test.subvolume:
+# (2) No such file or directory
+# . failed to run command. command terminated with exit code 1%!(EXTRA string=failed to list omapval)
 
 ```

pkg/exec/exec.go

@@ -127,6 +127,8 @@ func execCmdInPod(ctx context.Context, clientsets *k8sutil.Clientsets,
 		cmd = append(cmd, "--connect-timeout=10", fmt.Sprintf("--conf=/var/lib/rook/%s/%s.config", clusterNamespace, clusterNamespace))
 	} else if cmd[0] == "rbd" {
 		cmd = append(cmd, fmt.Sprintf("--conf=/var/lib/rook/%s/%s.config", clusterNamespace, clusterNamespace))
+	} else if cmd[0] == "rados" {
+		cmd = append(cmd, fmt.Sprintf("--conf=/var/lib/rook/%s/%s.config", clusterNamespace, clusterNamespace))
 	}
 
 	// Prepare the API URL used to execute another process within the Pod.  In

pkg/filesystem/subvolume.go

@@ -38,6 +38,11 @@ type subVolumeInfo struct {
 	fs  string
 }
 
+type OmapDetails struct {
+	omapval string
+	omapkey string
+}
+
 func List(ctx context.Context, clientsets *k8sutil.Clientsets, operatorNamespace, clusterNamespace string, includeStaleOnly bool) {
 
 	subvolumeNames := getK8sRefSubvolume(ctx, clientsets)
@@ -178,12 +183,28 @@ func Delete(ctx context.Context, clientsets *k8sutil.Clientsets, OperatorNamespa
 				continue
 			}
 			logging.Info("subvolume %q deleted", subvolume)
+			DeleteOmap(ctx, clientsets, OperatorNamespace, CephClusterNamespace, subvolume, fs)
 		} else {
 			logging.Info("subvolume %q is not stale", subvolume)
 		}
 	}
 }
 
+// DeleteOmap deletes omap object and key for the given subvolume.
+func DeleteOmap(ctx context.Context, clientsets *k8sutil.Clientsets, OperatorNamespace, CephClusterNamespace, subVol, fs string) {
+	omapdetails, err := getOmapDetails(ctx, clientsets, OperatorNamespace, CephClusterNamespace, subVol, fs)
+	if err != nil {
+		logging.Fatal(err)
+	}
+	poolName := fs + "-metadata"
+	// remove omap object.
+	exec.RunCommandInOperatorPod(ctx, clientsets, "rados", []string{"rm", omapdetails.omapval, "-p", poolName, "--namespace", "csi"}, OperatorNamespace, CephClusterNamespace, true)
+	logging.Info("omap object:%q deleted", omapdetails.omapval)
+	// remove omap key.
+	exec.RunCommandInOperatorPod(ctx, clientsets, "rados", []string{"rmomapkey", "csi.volumes.default", omapdetails.omapkey, "-p", poolName, "--namespace", "csi"}, OperatorNamespace, CephClusterNamespace, true)
+	logging.Info("omap key:%q deleted", omapdetails.omapkey)
+}
+
 // checkStaleSubvolume checks if there are any stale subvolume to be deleted
 func checkStaleSubvolume(ctx context.Context, clientsets *k8sutil.Clientsets, OperatorNamespace, CephClusterNamespace, fs, subvolume, svg string, k8sSubvolume map[string]subVolumeInfo) bool {
 	_, ok := k8sSubvolume[subvolume]
@@ -199,3 +220,25 @@ func checkStaleSubvolume(ctx context.Context, clientsets *k8sutil.Clientsets, Op
 	logging.Error(fmt.Errorf("Subvolume %s is referenced by a PV", subvolume))
 	return false
 }
+
+func getOmapDetails(ctx context.Context, clientsets *k8sutil.Clientsets, OperatorNamespace, CephClusterNamespace, subVol, fs string) (OmapDetails, error) {
+
+	var details OmapDetails
+	poolName := fs + "-metadata"
+	omapval := strings.Replace(subVol, "-", ".", 2)
+	omapval1 := strings.Replace(omapval, "vol", "volume", 1)
+
+	ls, err := exec.RunCommandInOperatorPod(ctx, clientsets, "rados", []string{"listomapvals", omapval1, "-p", poolName, "--namespace", "csi"}, OperatorNamespace, CephClusterNamespace, true)
+	if err != nil {
+		logging.Error(err, "failed to list omapval")
+		return OmapDetails{}, err
+	}
+	if ls != "" {
+		volname := strings.Split(ls, "|")
+		pv := "csi.volume." + volname[7] + volname[9] + volname[11]
+		details = OmapDetails{omapval1, pv}
+	} else {
+		logging.Fatal(fmt.Errorf("omap details not found"))
+	}
+	return details, nil
+}