csi: clean omap details

this commit deletes the omap presence of the
stale subvolume

Signed-off-by: yati1998 <[email protected]>

docs/subvolume.md

@@ -39,8 +39,18 @@ kubectl rook-ceph subvolume ls --stale
 ## delete
 
 ```bash
-kubectl rook-ceph subvolume delete ocs-storagecluster csi-vol-427774b4-340b-11ed-8d66-0242ac110004
+kubectl rook-ceph subvolume delete ocs-storagecluster-cephfilesystem csi-vol-427774b4-340b-11ed-8d66-0242ac110005
 
-# Info: subvolume "csi-vol-427774b4-340b-11ed-8d66-0242ac110004" deleted
+# Info: subvolume csi-vol-427774b4-340b-11ed-8d66-0242ac110004 deleted
+# Info: omap object:"csi.volume.427774b4-340b-11ed-8d66-0242ac110005" deleted
+# Info: omap key:"csi.volume.pvc-fca205e5-8788-4132-979c-e210c0133182" deleted
+
+```
+
+```bash
+kubectl rook-ceph subvolume delete ocs-storagecluster-cephfilesystem csi-vol-test
+
+# Info: subvolume "csi-vol-test" deleted
+# Info: No omapvals found for subvolume csi-vol-test
 
 ```

pkg/exec/exec.go

@@ -127,6 +127,8 @@ func execCmdInPod(ctx context.Context, clientsets *k8sutil.Clientsets,
 		cmd = append(cmd, "--connect-timeout=10", fmt.Sprintf("--conf=/var/lib/rook/%s/%s.config", clusterNamespace, clusterNamespace))
 	} else if cmd[0] == "rbd" {
 		cmd = append(cmd, fmt.Sprintf("--conf=/var/lib/rook/%s/%s.config", clusterNamespace, clusterNamespace))
+	} else if cmd[0] == "rados" {
+		cmd = append(cmd, fmt.Sprintf("--conf=/var/lib/rook/%s/%s.config", clusterNamespace, clusterNamespace))
 	}
 
 	// Prepare the API URL used to execute another process within the Pod.  In

pkg/filesystem/subvolume.go

@@ -20,6 +20,7 @@ import (
 	"context"
 	"encoding/json"
 	"fmt"
+	"strings"
 
 	"github.com/rook/kubectl-rook-ceph/pkg/exec"
 	"github.com/rook/kubectl-rook-ceph/pkg/k8sutil"
@@ -44,6 +45,11 @@ const (
 	staleWithSnapshot = "stale-with-snapshot"
 )
 
+type omapDetails struct {
+	omapval string
+	omapkey string
+}
+
 func List(ctx context.Context, clientsets *k8sutil.Clientsets, operatorNamespace, clusterNamespace string, includeStaleOnly bool) {
 
 	subvolumeNames := getK8sRefSubvolume(ctx, clientsets)
@@ -211,7 +217,86 @@ func Delete(ctx context.Context, clientsets *k8sutil.Clientsets, OperatorNamespa
 	if !check {
 		exec.RunCommandInOperatorPod(ctx, clientsets, "ceph", []string{"fs", "subvolume", "rm", fs, subvol, svg, "--retain-snapshots"}, OperatorNamespace, CephClusterNamespace, true)
 		logging.Info("subvolume %q deleted", subvol)
+		deleteOmap(ctx, clientsets, OperatorNamespace, CephClusterNamespace, subvol, fs)
 	} else {
 		logging.Info("subvolume %q is not stale", subvol)
 	}
 }
+
+// deleteOmap deletes omap object and key for the given subvolume.
+func deleteOmap(ctx context.Context, clientsets *k8sutil.Clientsets, OperatorNamespace, CephClusterNamespace, subVol, fs string) {
+	omapdetails := getOmapDetails(ctx, clientsets, OperatorNamespace, CephClusterNamespace, subVol, fs)
+	poolName := fs + "-metadata"
+	if omapdetails.omapval != "" {
+		// remove omap object.
+		exec.RunCommandInOperatorPod(ctx, clientsets, "rados", []string{"rm", omapdetails.omapval, "-p", poolName, "--namespace", "csi"}, OperatorNamespace, CephClusterNamespace, true)
+		logging.Info("omap object:%q deleted", omapdetails.omapval)
+	}
+	if omapdetails.omapkey != "" {
+		// remove omap key.
+		exec.RunCommandInOperatorPod(ctx, clientsets, "rados", []string{"rmomapkey", "csi.volumes.default", omapdetails.omapkey, "-p", poolName, "--namespace", "csi"}, OperatorNamespace, CephClusterNamespace, true)
+		logging.Info("omap key:%q deleted", omapdetails.omapkey)
+	}
+}
+
+// getOmapDetails gets the omap key and value details for a given subvolume.
+// Deletion of omap object required the rbd-image-name which is of format
+// csi.volume.subvolume, where subvolume is the name of subvolume that needs to be
+// deleted.
+// similarly to delete of omap key requires csi.volume.ompakey, where
+// omapkey is the pv name which is extracted the omap object.
+func getOmapDetails(ctx context.Context, clientsets *k8sutil.Clientsets, OperatorNamespace, CephClusterNamespace, subVol, fs string) omapDetails {
+
+	var details omapDetails
+	var omapval, pv string
+	poolName := fs + "-metadata"
+	replace := strings.Replace(subVol, "-", ".", 2)
+	checknfs := strings.Contains(subVol, "nfs-export")
+	// check if it is cephfs or nfs subvolume
+	if checknfs {
+		omapval = strings.Replace(replace, "export", "volume", 1)
+		omapval = strings.Replace(omapval, "nfs", "csi", 1)
+	} else {
+		omapval = strings.Replace(replace, "vol", "volume", 1)
+
+	}
+	ls, err := exec.RunCommandInOperatorPod(ctx, clientsets, "rados", []string{"listomapvals", omapval, "-p", poolName, "--namespace", "csi"}, OperatorNamespace, CephClusterNamespace, true)
+	if err != nil {
+		if !strings.Contains(err.Error(), "No such file or directory") {
+			logging.Fatal(fmt.Errorf("failed to list omapvals: %q", err))
+		} else {
+			logging.Info("No omapvals found for subvolume %s", subVol)
+			return omapDetails{}
+		}
+	}
+	if ls != "" {
+
+		// Below is how the output of listmapvals looks, and when we split it w.r.t
+		// '|' we can get the pv name at 7,9 and 11th part of the output.
+		// Below example is for cephfs, we have similar case for nfs subvolumes.
+		//csi.imagename
+		//value (44 bytes) :
+		//00000000  63 73 69 2d 76 6f 6c 2d  35 66 62 32 61 38 30 32  |csi-vol-5fb2a802|
+		//00000010  2d 38 37 31 31 2d 34 33  63 33 2d 62 39 36 35 2d  |-8711-43c3-b965-|
+		//00000020  66 66 37 65 37 39 35 62  33 35 33 65              |ff7e795b353e|
+		//0000002c
+		//csi.volname
+		//value (40 bytes) :
+		//00000000  70 76 63 2d 65 63 62 62  34 34 38 61 2d 30 35 37  |pvc-ecbb448a-057|
+		//00000010  36 2d 34 35 30 35 2d 62  32 39 35 2d 30 32 39 33  |6-4505-b295-0293|
+		//00000020  34 37 37 36 34 65 61 64                           |47764ead|
+		//0000002
+		volname := strings.Split(ls, "|")
+		if len(volname) > 16 {
+			if checknfs {
+				pv = "csi.volume." + volname[12] + volname[14] + volname[16]
+			} else {
+				pv = "csi.volume." + volname[7] + volname[9] + volname[11]
+			}
+		}
+		details = omapDetails{omapval, pv}
+	} else {
+		logging.Fatal(fmt.Errorf("omap details not found"))
+	}
+	return details
+}