csi: clean cephFS omap details

this commit deletes the omap presence of the stale subvolume Signed-off-by: yati1998 <[email protected]>
rook · Mar 20, 2024 · d86b70d · d86b70d
1 parent 2cf7872
commit d86b70d
Show file tree

Hide file tree

Showing 6 changed files with 225 additions and 24 deletions.
diff --git a/.github/workflows/go-test.yaml b/.github/workflows/go-test.yaml
@@ -77,11 +77,13 @@ jobs:
         run: |
           set -ex
           kubectl rook-ceph ceph fs subvolume create myfs test-subvol group-a
-          kubectl rook-ceph ceph fs subvolume create myfs test-subvol-1 group-a
           kubectl rook-ceph subvolume ls
           kubectl rook-ceph subvolume ls --stale
           kubectl rook-ceph subvolume delete myfs test-subvol group-a
-          kubectl rook-ceph subvolume delete myfs test-subvol-1
+          tests/github-action-helper.sh create_sc_with_retain_policy
+          tests/github-action-helper.sh create_stale_subvolume
+          subVol=$(kubectl rook-ceph subvolume ls --stale | awk '{print $2}' | grep csi-vol)
+          kubectl rook_ceph subvolume delete myfs $subVol
 
       - name: Get mon endpoints
         run: |
@@ -232,12 +234,14 @@ jobs:
       - name: Subvolume command
         run: |
           set -ex
-          kubectl rook-ceph --operator-namespace test-operator -n test-cluster ceph fs subvolume create myfs test-subvol-1 group-a
           kubectl rook-ceph --operator-namespace test-operator -n test-cluster ceph fs subvolume create myfs test-subvol group-a
           kubectl rook-ceph --operator-namespace test-operator -n test-cluster subvolume ls
           kubectl rook-ceph --operator-namespace test-operator -n test-cluster subvolume ls --stale
           kubectl rook-ceph --operator-namespace test-operator -n test-cluster subvolume delete myfs test-subvol group-a
-          kubectl rook-ceph --operator-namespace test-operator -n test-cluster subvolume delete myfs test-subvol-1
+          tests/github-action-helper.sh create_sc_with_retain_policy_custom_ns test-operator test-cluster
+          tests/github-action-helper.sh create_stale_subvolume
+          subVol=$(kubectl rook-ceph --operator-namespace test-operator -n test-cluster subvolume ls --stale | awk '{print $2}' | grep csi-vol)
+          kubectl rook_ceph --operator-namespace test-operator -n test-cluster subvolume delete myfs $subVol
 
       - name: Get mon endpoints
         run: |

diff --git a/docs/subvolume.md b/docs/subvolume.md
@@ -17,30 +17,37 @@ The subvolume command will require the following sub commands:
 ## ls
 
 ```bash
-kubectl rook-ceph subvolume ls
-
-# Filesystem  Subvolume  SubvolumeGroup  State
-# ocs-storagecluster-cephfilesystem csi-vol-427774b4-340b-11ed-8d66-0242ac110004 csi in-use
-# ocs-storagecluster-cephfilesystem csi-vol-427774b4-340b-11ed-8d66-0242ac110005 csi in-use
-# ocs-storagecluster-cephfilesystem csi-vol-427774b4-340b-11ed-8d66-0242ac110007 csi stale
-# ocs-storagecluster-cephfilesystem csi-vol-427774b4-340b-11ed-8d66-0242ac110007 csi stale-with-snapshot
+$ kubectl rook-ceph subvolume ls
 
+Filesystem  Subvolume  SubvolumeGroup  State
+myfs csi-vol-427774b4-340b-11ed-8d66-0242ac110004 csi in-use
+myfs csi-vol-427774b4-340b-11ed-8d66-0242ac110005 csi in-use
+myfs csi-vol-427774b4-340b-11ed-8d66-0242ac110007 csi stale
+myfs csi-vol-427774b4-340b-11ed-8d66-0242ac110007 csi stale-with-snapshot
 ```
 
 ```bash
-kubectl rook-ceph subvolume ls --stale
-
-# Filesystem  Subvolume  SubvolumeGroup state
-# ocs-storagecluster-cephfilesystem csi-vol-427774b4-340b-11ed-8d66-0242ac110004 csi stale
-# ocs-storagecluster-cephfilesystem csi-vol-427774b4-340b-11ed-8d66-0242ac110005 csi stale-with-snapshot
+$ kubectl rook-ceph subvolume ls --stale
 
+Filesystem  Subvolume  SubvolumeGroup state
+myfs csi-vol-427774b4-340b-11ed-8d66-0242ac110004 csi stale
+myfs csi-vol-427774b4-340b-11ed-8d66-0242ac110005 csi stale-with-snapshot
 ```
 
 ## delete
 
 ```bash
-kubectl rook-ceph subvolume delete ocs-storagecluster csi-vol-427774b4-340b-11ed-8d66-0242ac110004
+$ kubectl rook-ceph subvolume delete myfs csi-vol-427774b4-340b-11ed-8d66-0242ac110005
 
-# Info: subvolume "csi-vol-427774b4-340b-11ed-8d66-0242ac110004" deleted
+Info: Deleting the omap object and key for subvolume "csi-vol-0c91ba82-5a63-4117-88a4-690acd86cbbd"
+Info: omap object:"csi.volume.0c91ba82-5a63-4117-88a4-690acd86cbbd" deleted
+Info: omap key:"csi.volume.pvc-78abf81c-5381-42ee-8d75-dc17cd0cf5de" deleted
+Info: subvolume "csi-vol-0c91ba82-5a63-4117-88a4-690acd86cbbd" deleted
+```
+
+```bash
+$ kubectl rook-ceph subvolume delete myfs csi-vol-427774b4-340b-11ed-8d66-0242ac110005
 
+Info: No omapvals found for subvolume csi-vol-427774b4-340b-11ed-8d66-0242ac110005
+Info: subvolume "csi-vol-427774b4-340b-11ed-8d66-0242ac110005" deleted
 ```
diff --git a/pkg/exec/exec.go b/pkg/exec/exec.go
@@ -130,6 +130,8 @@ func execCmdInPod(ctx context.Context, clientsets *k8sutil.Clientsets,
 		cmd = append(cmd, "--connect-timeout=10", fmt.Sprintf("--conf=/var/lib/rook/%s/%s.config", clusterNamespace, clusterNamespace))
 	} else if cmd[0] == "rbd" {
 		cmd = append(cmd, fmt.Sprintf("--conf=/var/lib/rook/%s/%s.config", clusterNamespace, clusterNamespace))
+	} else if cmd[0] == "rados" {
+		cmd = append(cmd, fmt.Sprintf("--conf=/var/lib/rook/%s/%s.config", clusterNamespace, clusterNamespace))
 	}
 
 	// Prepare the API URL used to execute another process within the Pod.  In

diff --git a/pkg/filesystem/subvolume.go b/pkg/filesystem/subvolume.go
@@ -20,6 +20,7 @@ import (
 	"context"
 	"encoding/json"
 	"fmt"
+	"strings"
 
 	"github.com/rook/kubectl-rook-ceph/pkg/exec"
 	"github.com/rook/kubectl-rook-ceph/pkg/k8sutil"
@@ -28,8 +29,9 @@ import (
 )
 
 type fsStruct struct {
-	Name string
-	data []string
+	Name         string
+	MetadataName string `json:"metadata_pool"`
+	data         []string
 }
 
 type subVolumeInfo struct {
@@ -201,17 +203,107 @@ func unMarshaljson(list string) []fsStruct {
 	if errg != nil {
 		logging.Fatal(errg)
 	}
-
 	return unmarshal
 }
 
 func Delete(ctx context.Context, clientsets *k8sutil.Clientsets, OperatorNamespace, CephClusterNamespace, fs, subvol, svg string) {
 	k8sSubvolume := getK8sRefSubvolume(ctx, clientsets)
 	_, check := k8sSubvolume[subvol]
 	if !check {
-		exec.RunCommandInOperatorPod(ctx, clientsets, "ceph", []string{"fs", "subvolume", "rm", fs, subvol, svg, "--retain-snapshots"}, OperatorNamespace, CephClusterNamespace, true)
-		logging.Info("subvolume %q deleted", subvol)
+		deleteOmapForSubvolume(ctx, clientsets, OperatorNamespace, CephClusterNamespace, subvol, fs)
+		_, err := exec.RunCommandInOperatorPod(ctx, clientsets, "ceph", []string{"fs", "subvolume", "rm", fs, subvol, svg, "--retain-snapshots"}, OperatorNamespace, CephClusterNamespace, true)
+		if err != nil {
+			logging.Fatal(err, "failed to delete subvolume of %s/%s/%s", fs, svg, subvol)
+		}
+		logging.Info("subvolume %s/%s/%s deleted", fs, svg, subvol)
+
 	} else {
-		logging.Info("subvolume %q is not stale", subvol)
+		logging.Info("subvolume %s/%s/%s is not stale", fs, svg, subvol)
+	}
+}
+
+func getMetadataPoolName(ctx context.Context, clientsets *k8sutil.Clientsets, OperatorNamespace, CephClusterNamespace, fs string) (string, error) {
+	fsstruct, err := getFileSystem(ctx, clientsets, OperatorNamespace, CephClusterNamespace)
+	if err != nil {
+		return "", err
+	}
+
+	for _, pool := range fsstruct {
+		if pool.Name == fs {
+			return pool.MetadataName, nil
+		}
+	}
+	return "", fmt.Errorf("metadataPool not found for %q filesystem", fs)
+}
+
+// deleteOmap deletes omap object and key for the given subvolume.
+func deleteOmapForSubvolume(ctx context.Context, clientsets *k8sutil.Clientsets, OperatorNamespace, CephClusterNamespace, subVol, fs string) {
+	logging.Info("Deleting the omap object and key for subvolume %q", subVol)
+	omapkey := getOmapKey(ctx, clientsets, OperatorNamespace, CephClusterNamespace, subVol, fs)
+	omapval := getOmapVal(subVol)
+	poolName, err := getMetadataPoolName(ctx, clientsets, OperatorNamespace, CephClusterNamespace, fs)
+	if err != nil || poolName == "" {
+		logging.Fatal(fmt.Errorf("pool name not found: %q", err))
+	}
+	if omapval != "" {
+		// remove omap object.
+		_, err := exec.RunCommandInOperatorPod(ctx, clientsets, "rados", []string{"rm", omapval, "-p", poolName, "--namespace", "csi"}, OperatorNamespace, CephClusterNamespace, true)
+		if err != nil {
+			logging.Fatal(err, "failed to remove omap object for subvolume %q", subVol)
+		}
+		logging.Info("omap object:%q deleted", omapval)
+
 	}
+	if omapkey != "" {
+		// remove omap key.
+		_, err := exec.RunCommandInOperatorPod(ctx, clientsets, "rados", []string{"rmomapkey", "csi.volumes.default", omapkey, "-p", poolName, "--namespace", "csi"}, OperatorNamespace, CephClusterNamespace, true)
+		if err != nil {
+			logging.Fatal(err, "failed to remove omap key for subvolume %q", subVol)
+		}
+		logging.Info("omap key:%q deleted", omapkey)
+
+	}
+}
+
+// getOmapKey gets the omap key and value details for a given subvolume.
+// Deletion of omap object required the subvolumeName which is of format
+// csi.volume.subvolume, where subvolume is the name of subvolume that needs to be
+// deleted.
+// similarly to delete of omap key requires csi.volume.ompakey, where
+// omapkey is the pv name which is extracted the omap object.
+func getOmapKey(ctx context.Context, clientsets *k8sutil.Clientsets, OperatorNamespace, CephClusterNamespace, subVol, fs string) string {
+
+	poolName, err := getMetadataPoolName(ctx, clientsets, OperatorNamespace, CephClusterNamespace, fs)
+	if err != nil || poolName == "" {
+		logging.Fatal(fmt.Errorf("pool name not found: %q", err))
+	}
+	omapval := getOmapVal(subVol)
+
+	cmd := []string{"getomapval", omapval, "csi.volname", "-p", poolName, "--namespace", "csi", "/dev/stdout"}
+	pvname, err := exec.RunCommandInOperatorPod(ctx, clientsets, "rados", cmd, OperatorNamespace, CephClusterNamespace, true)
+	if err != nil || pvname == "" {
+		logging.Info("No PV found for subvolume %s: %s", subVol, err)
+		return ""
+	}
+	// omap key is for format csi.volume.pvc-fca205e5-8788-4132-979c-e210c0133182
+	// hence, attaching pvname to required prefix.
+	omapkey := "csi.volume." + pvname
+
+	return omapkey
+}
+
+// func getOmapVal is used to get the omapval from the given subvolume
+// omapval is of format csi.volume.427774b4-340b-11ed-8d66-0242ac110005
+// which is similar to volume name csi-vol-427774b4-340b-11ed-8d66-0242ac110005
+// hence, replacing 'csi-vol-' to 'csi.volume.' to get the omapval
+func getOmapVal(subVol string) string {
+
+	splitSubvol := strings.SplitAfterN(subVol, "-", 3)
+	if len(splitSubvol) < 3 {
+		return ""
+	}
+	subvol_id := splitSubvol[len(splitSubvol)-1]
+	omapval := "csi.volume." + subvol_id
+
+	return omapval
 }
diff --git a/pkg/filesystem/subvolume_test.go b/pkg/filesystem/subvolume_test.go
@@ -0,0 +1,55 @@
+/*
+Copyright 2024 The Rook Authors. All rights reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+	http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package subvolume
+
+import "testing"
+
+func TestGetOmapVal(t *testing.T) {
+
+	tests := []struct {
+		name string
+		want string
+	}{
+		{
+			name: "csi-vol-427774b4-340b-11ed-8d66-0242ac110005",
+			want: "csi.volume.427774b4-340b-11ed-8d66-0242ac110005",
+		},
+		{
+			name: "nfs-export-427774b4-340b-11ed-8d66-0242ac110005",
+			want: "csi.volume.427774b4-340b-11ed-8d66-0242ac110005",
+		},
+		{
+			name: "",
+			want: "",
+		},
+		{
+			name: "csi-427774b4-340b-11ed-8d66-0242ac11000",
+			want: "csi.volume.340b-11ed-8d66-0242ac11000",
+		},
+		{
+			name: "csi-427774b440b11ed8d660242ac11000",
+			want: "",
+		},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			if got := getOmapVal(tt.name); got != tt.want {
+				t.Errorf("getOmapVal() = %v, want %v", got, tt.want)
+			}
+		})
+	}
+}
diff --git a/tests/github-action-helper.sh b/tests/github-action-helper.sh
@@ -76,6 +76,37 @@ deploy_rook_in_custom_namespace() {
   deploy_csi_driver_custom_ns "$1" "$2"
 }
 
+create_sc_with_retain_policy(){
+  curl https://raw.githubusercontent.com/rook/rook/master/deploy/examples/csi/cephfs/storageclass.yaml -o storageclass.yaml
+  sed -i "s|name: rook-cephfs|name: rook-cephfs-retain|g" storageclass.yaml
+  sed -i "s|reclaimPolicy: Delete|reclaimPolicy: Retain|g" storageclass.yaml
+  kubectl create -f storageclass.yaml 
+}
+
+create_sc_with_retain_policy_custom_ns(){
+  export OPERATOR_NS=$1
+  export CLUSTER_NS=$2
+
+  curl https://raw.githubusercontent.com/rook/rook/master/deploy/examples/csi/cephfs/storageclass.yaml -o storageclass.yaml
+  sed -i "s|name: rook-cephfs|name: rook-cephfs-retain|g" storageclass.yaml
+  sed -i "s|reclaimPolicy: Delete|reclaimPolicy: Retain|g" storageclass.yaml
+  sed -i "s|provisioner: rook-ceph.cephfs.csi.ceph.com |provisioner: test-operator.cephfs.csi.ceph.com |g" storageclass.yaml
+  deploy_with_custom_ns $OPERATOR_NS $CLUSTER_NS storageclass.yaml
+}
+
+create_stale_subvolume() {
+  curl https://raw.githubusercontent.com/rook/rook/master/deploy/examples/csi/cephfs/pvc.yaml -o pvc.yaml
+  sed -i "s|name: cephfs-pvc|name: cephfs-pvc-retain|g" pvc.yaml
+  sed -i "s|storageClassName: rook-cephfs|storageClassName: rook-cephfs-retain|g" pvc.yaml
+  kubectl create -f pvc.yaml
+  kubectl get pvc cephfs-pvc-retain
+  : "${PVNAME:=$(kubectl get pvc cephfs-pvc-retain -o=jsonpath='{.spec.volumeName}')}"
+  wait_for_pvc_to_be_bound_state_default
+  kubectl get pvc cephfs-pvc-retain
+  kubectl delete pvc cephfs-pvc-retain
+  kubectl delete pv "$PVNAME"
+}
+
 deploy_with_custom_ns() {
   sed -i "s|rook-ceph # namespace:operator|$1 # namespace:operator|g" "$3"
   sed -i "s|rook-ceph # namespace:cluster|$2 # namespace:cluster|g" "$3"
@@ -109,6 +140,16 @@ deploy_csi_driver_custom_ns() {
   kubectl create -f https://raw.githubusercontent.com/rook/rook/master/deploy/examples/csi/cephfs/pvc.yaml
 }
 
+wait_for_pvc_to_be_bound_state_default() {
+  timeout 100 bash <<-'EOF'
+    until [ $(kubectl get pvc cephfs-pvc-retain -o jsonpath='{.status.phase}') == "Bound" ]; do
+      echo "waiting for the pvc to be in bound state"
+      sleep 1
+    done
+EOF
+  timeout_command_exit_code
+}
+
 wait_for_pod_to_be_ready_state_default() {
   timeout 200 bash <<-'EOF'
     until [ $(kubectl get pod -l app=rook-ceph-osd -n rook-ceph -o jsonpath='{.items[*].metadata.name}' -o custom-columns=READY:status.containerStatuses[*].ready | grep -c true) -eq 1 ]; do