csi: clean omap details

this commit deletes the omap presence of the
stale subvolume

Signed-off-by: yati1998 <[email protected]>

.github/workflows/go-test.yaml

@@ -77,7 +77,7 @@ jobs:
 
       - name: Subvolume command
         run: |
-          set -ex
+          set -e
           kubectl rook-ceph ceph fs subvolume create myfs test-subvol group-a
           kubectl rook-ceph ceph fs subvolume create myfs test-subvol-1 group-a
           kubectl rook-ceph subvolume ls
@@ -235,7 +235,7 @@ jobs:
 
       - name: Subvolume command
         run: |
-          set -ex
+          set -e
           kubectl rook-ceph --operator-namespace test-operator -n test-cluster ceph fs subvolume create myfs test-subvol-1 group-a
           kubectl rook-ceph --operator-namespace test-operator -n test-cluster ceph fs subvolume create myfs test-subvol group-a
           kubectl rook-ceph --operator-namespace test-operator -n test-cluster subvolume ls

docs/subvolume.md

@@ -39,8 +39,20 @@ kubectl rook-ceph subvolume ls --stale
 ## delete
 
 ```bash
-kubectl rook-ceph subvolume delete ocs-storagecluster csi-vol-427774b4-340b-11ed-8d66-0242ac110004
+kubectl rook-ceph subvolume delete ocs-storagecluster-cephfilesystem csi-vol-427774b4-340b-11ed-8d66-0242ac110005
 
-# Info: subvolume "csi-vol-427774b4-340b-11ed-8d66-0242ac110004" deleted
+# Info: subvolume csi-vol-427774b4-340b-11ed-8d66-0242ac110004 deleted
+# Info: omap object:"csi.volume.427774b4-340b-11ed-8d66-0242ac110005" deleted
+# Info: omap key:"csi.volume.pvc-fca205e5-8788-4132-979c-e210c0133182" deleted
+
+```
+
+```bash
+kubectl rook-ceph subvolume delete ocs-storagecluster-cephfilesystem csi-vol-test
+
+# Info: subvolume "csi-vol-test" deleted
+# Error: error getting omap keys ocs-storagecluster-cephfilesystem-metadata/test.subvolume:
+# (2) No such file or directory
+# . failed to run command. command terminated with exit code 1%!(EXTRA string=failed to list omapval)
 
 ```

pkg/exec/exec.go

@@ -127,6 +127,8 @@ func execCmdInPod(ctx context.Context, clientsets *k8sutil.Clientsets,
 		cmd = append(cmd, "--connect-timeout=10", fmt.Sprintf("--conf=/var/lib/rook/%s/%s.config", clusterNamespace, clusterNamespace))
 	} else if cmd[0] == "rbd" {
 		cmd = append(cmd, fmt.Sprintf("--conf=/var/lib/rook/%s/%s.config", clusterNamespace, clusterNamespace))
+	} else if cmd[0] == "rados" {
+		cmd = append(cmd, fmt.Sprintf("--conf=/var/lib/rook/%s/%s.config", clusterNamespace, clusterNamespace))
 	}
 
 	// Prepare the API URL used to execute another process within the Pod.  In

pkg/filesystem/subvolume.go

@@ -20,6 +20,7 @@ import (
 	"context"
 	"encoding/json"
 	"fmt"
+	"strings"
 
 	"github.com/rook/kubectl-rook-ceph/pkg/exec"
 	"github.com/rook/kubectl-rook-ceph/pkg/k8sutil"
@@ -44,6 +45,11 @@ const (
 	staleWithSnapshot = "stale-with-snapshot"
 )
 
+type OmapDetails struct {
+	omapval string
+	omapkey string
+}
+
 func List(ctx context.Context, clientsets *k8sutil.Clientsets, operatorNamespace, clusterNamespace string, includeStaleOnly bool) {
 
 	subvolumeNames := getK8sRefSubvolume(ctx, clientsets)
@@ -211,7 +217,73 @@ func Delete(ctx context.Context, clientsets *k8sutil.Clientsets, OperatorNamespa
 	if !check {
 		exec.RunCommandInOperatorPod(ctx, clientsets, "ceph", []string{"fs", "subvolume", "rm", fs, subvol, svg, "--retain-snapshots"}, OperatorNamespace, CephClusterNamespace, true)
 		logging.Info("subvolume %q deleted", subvol)
+		deleteOmap(ctx, clientsets, OperatorNamespace, CephClusterNamespace, subvol, fs)
 	} else {
 		logging.Info("subvolume %q is not stale", subvol)
 	}
 }
+
+// deleteOmap deletes omap object and key for the given subvolume.
+func deleteOmap(ctx context.Context, clientsets *k8sutil.Clientsets, OperatorNamespace, CephClusterNamespace, subVol, fs string) {
+	omapdetails, err := getOmapDetails(ctx, clientsets, OperatorNamespace, CephClusterNamespace, subVol, fs)
+	if err != nil {
+		logging.Fatal(fmt.Errorf("failed to get omap details: %q", err))
+	}
+	poolName := fs + "-metadata"
+	// remove omap object.
+	exec.RunCommandInOperatorPod(ctx, clientsets, "rados", []string{"rm", omapdetails.omapval, "-p", poolName, "--namespace", "csi"}, OperatorNamespace, CephClusterNamespace, true)
+	logging.Info("omap object:%q deleted", omapdetails.omapval)
+	// remove omap key.
+	exec.RunCommandInOperatorPod(ctx, clientsets, "rados", []string{"rmomapkey", "csi.volumes.default", omapdetails.omapkey, "-p", poolName, "--namespace", "csi"}, OperatorNamespace, CephClusterNamespace, true)
+	logging.Info("omap key:%q deleted", omapdetails.omapkey)
+}
+
+// checkStaleSubvolume checks if there are any stale subvolume to be deleted
+func checkStaleSubvolume(ctx context.Context, clientsets *k8sutil.Clientsets, OperatorNamespace, CephClusterNamespace, fs, subvolume, svg string, k8sSubvolume map[string]subVolumeInfo) bool {
+	_, ok := k8sSubvolume[subvolume]
+	if !ok {
+		snapshot := checkSnapshot(ctx, clientsets, OperatorNamespace, CephClusterNamespace, fs, subvolume, svg)
+		if snapshot {
+			logging.Error(fmt.Errorf("subvolume %s has snapshots", subvolume))
+			return false
+		} else {
+			return true
+		}
+	}
+	logging.Error(fmt.Errorf("Subvolume %s is referenced by a PV", subvolume))
+	return false
+}
+
+func getOmapDetails(ctx context.Context, clientsets *k8sutil.Clientsets, OperatorNamespace, CephClusterNamespace, subVol, fs string) (OmapDetails, error) {
+
+	var details OmapDetails
+	var omapval1, pv string
+	poolName := fs + "-metadata"
+	omapval := strings.Replace(subVol, "-", ".", 2)
+	checknfs := strings.Contains(subVol, "nfs-export")
+	if checknfs {
+		omapval1 = strings.Replace(omapval, "export", "volume", 1)
+		omapval1 = strings.Replace(omapval1, "nfs", "csi", 1)
+	} else {
+		omapval1 = strings.Replace(omapval, "vol", "volume", 1)
+
+	}
+
+	ls, err := exec.RunCommandInOperatorPod(ctx, clientsets, "rados", []string{"listomapvals", omapval1, "-p", poolName, "--namespace", "csi"}, OperatorNamespace, CephClusterNamespace, true)
+	if err != nil {
+		logging.Error(err, "failed to list omapval")
+		return OmapDetails{}, err
+	}
+	if ls != "" {
+		volname := strings.Split(ls, "|")
+		if checknfs {
+			pv = "csi.volume." + volname[12] + volname[14] + volname[16]
+		} else {
+			pv = "csi.volume." + volname[7] + volname[9] + volname[11]
+		}
+		details = OmapDetails{omapval1, pv}
+	} else {
+		logging.Fatal(fmt.Errorf("omap details not found"))
+	}
+	return details, nil
+}