From 63b18803bf136f9400471c2d3ed6fd2b1db7fd0c Mon Sep 17 00:00:00 2001 From: yati1998 Date: Tue, 6 Feb 2024 18:19:06 +0530 Subject: [PATCH] clean omap details Signed-off-by: yati1998 --- .github/workflows/go-test.yaml | 4 ++-- docs/subvolume.md | 12 ++++++---- pkg/exec/exec.go | 2 ++ pkg/filesystem/subvolume.go | 43 ++++++++++++++++++++++++++++++++++ 4 files changed, 55 insertions(+), 6 deletions(-) diff --git a/.github/workflows/go-test.yaml b/.github/workflows/go-test.yaml index 928e5084..33b61215 100644 --- a/.github/workflows/go-test.yaml +++ b/.github/workflows/go-test.yaml @@ -77,7 +77,7 @@ jobs: - name: Subvolume command run: | - set -ex + set -e kubectl rook-ceph ceph fs subvolume create myfs test-subvol group-a kubectl rook-ceph subvolume ls kubectl rook-ceph subvolume ls --stale @@ -233,7 +233,7 @@ jobs: - name: Subvolume command run: | - set -ex + set -e kubectl rook-ceph --operator-namespace test-operator -n test-cluster ceph fs subvolume create myfs test-subvol group-a kubectl rook-ceph --operator-namespace test-operator -n test-cluster subvolume ls kubectl rook-ceph --operator-namespace test-operator -n test-cluster subvolume ls --stale diff --git a/docs/subvolume.md b/docs/subvolume.md index bf771927..9e130bb9 100644 --- a/docs/subvolume.md +++ b/docs/subvolume.md @@ -40,16 +40,20 @@ kubectl rook-ceph subvolume ls --stale ## delete ```bash -kubectl rook-ceph subvolume delete csi-vol-427774b4-340b-11ed-8d66-0242ac110004 ocs-storagecluster csi +kubectl rook-ceph subvolume delete ocs-storagecluster-cephfilesystem csi-vol-427774b4-340b-11ed-8d66-0242ac110005 csi # Info: subvolume csi-vol-427774b4-340b-11ed-8d66-0242ac110004 deleted +# Info: omap object:"csi.volume.427774b4-340b-11ed-8d66-0242ac110005" deleted +# Info: omap key:"csi.volume.pvc-fca205e5-8788-4132-979c-e210c0133182" deleted ``` ```bash -kubectl rook-ceph subvolume delete csi-vol-427774b4-340b-11ed-8d66-0242ac110004,csi-vol-427774b4-340b-11ed-8d66-0242ac110005 ocs-storagecluster csi +kubectl rook-ceph subvolume delete ocs-storagecluster-cephfilesystem csi-vol-test csi -# Info: subvolume csi-vol-427774b4-340b-11ed-8d66-0242ac110004 deleted -# Info: subvolume csi-vol-427774b4-340b-11ed-8d66-0242ac110004 deleted +# Info: subvolume "csi-vol-test" deleted +# Error: error getting omap keys ocs-storagecluster-cephfilesystem-metadata/test.subvolume: +# (2) No such file or directory +# . failed to run command. command terminated with exit code 1%!(EXTRA string=failed to list omapval) ``` \ No newline at end of file diff --git a/pkg/exec/exec.go b/pkg/exec/exec.go index 0cd409ea..ffd977a1 100644 --- a/pkg/exec/exec.go +++ b/pkg/exec/exec.go @@ -127,6 +127,8 @@ func execCmdInPod(ctx context.Context, clientsets *k8sutil.Clientsets, cmd = append(cmd, "--connect-timeout=10", fmt.Sprintf("--conf=/var/lib/rook/%s/%s.config", clusterNamespace, clusterNamespace)) } else if cmd[0] == "rbd" { cmd = append(cmd, fmt.Sprintf("--conf=/var/lib/rook/%s/%s.config", clusterNamespace, clusterNamespace)) + } else if cmd[0] == "rados" { + cmd = append(cmd, fmt.Sprintf("--conf=/var/lib/rook/%s/%s.config", clusterNamespace, clusterNamespace)) } // Prepare the API URL used to execute another process within the Pod. In diff --git a/pkg/filesystem/subvolume.go b/pkg/filesystem/subvolume.go index dcbe4197..e3ef2367 100644 --- a/pkg/filesystem/subvolume.go +++ b/pkg/filesystem/subvolume.go @@ -38,6 +38,11 @@ type subVolumeInfo struct { fs string } +type OmapDetails struct { + omapval string + omapkey string +} + func List(ctx context.Context, clientsets *k8sutil.Clientsets, operatorNamespace, clusterNamespace string, includeStaleOnly bool) { subvolumeNames := getK8sRefSubvolume(ctx, clientsets) @@ -178,12 +183,28 @@ func Delete(ctx context.Context, clientsets *k8sutil.Clientsets, OperatorNamespa continue } logging.Info("subvolume %q deleted", subvolume) + DeleteOmap(ctx, clientsets, OperatorNamespace, CephClusterNamespace, subvolume, fs) } else { logging.Info("subvolume %q is not stale", subvolume) } } } +// DeleteOmap deletes omap object and key for the given subvolume. +func DeleteOmap(ctx context.Context, clientsets *k8sutil.Clientsets, OperatorNamespace, CephClusterNamespace, subVol, fs string) { + omapdetails, err := getOmapDetails(ctx, clientsets, OperatorNamespace, CephClusterNamespace, subVol, fs) + if err != nil { + logging.Fatal(err) + } + poolName := fs + "-metadata" + // remove omap object. + exec.RunCommandInOperatorPod(ctx, clientsets, "rados", []string{"rm", omapdetails.omapval, "-p", poolName, "--namespace", "csi"}, OperatorNamespace, CephClusterNamespace, true) + logging.Info("omap object:%q deleted", omapdetails.omapval) + // remove omap key. + exec.RunCommandInOperatorPod(ctx, clientsets, "rados", []string{"rmomapkey", "csi.volumes.default", omapdetails.omapkey, "-p", poolName, "--namespace", "csi"}, OperatorNamespace, CephClusterNamespace, true) + logging.Info("omap key:%q deleted", omapdetails.omapkey) +} + // checkStaleSubvolume checks if there are any stale subvolume to be deleted func checkStaleSubvolume(ctx context.Context, clientsets *k8sutil.Clientsets, OperatorNamespace, CephClusterNamespace, fs, subvolume, svg string, k8sSubvolume map[string]subVolumeInfo) bool { _, ok := k8sSubvolume[subvolume] @@ -199,3 +220,25 @@ func checkStaleSubvolume(ctx context.Context, clientsets *k8sutil.Clientsets, Op logging.Error(fmt.Errorf("Subvolume %s is referenced by a PV", subvolume)) return false } + +func getOmapDetails(ctx context.Context, clientsets *k8sutil.Clientsets, OperatorNamespace, CephClusterNamespace, subVol, fs string) (OmapDetails, error) { + + var details OmapDetails + poolName := fs + "-metadata" + omapval := strings.Replace(subVol, "-", ".", 2) + omapval1 := strings.Replace(omapval, "vol", "volume", 1) + + ls, err := exec.RunCommandInOperatorPod(ctx, clientsets, "rados", []string{"listomapvals", omapval1, "-p", poolName, "--namespace", "csi"}, OperatorNamespace, CephClusterNamespace, true) + if err != nil { + logging.Error(err, "failed to list omapval") + return OmapDetails{}, err + } + if ls != "" { + volname := strings.Split(ls, "|") + pv := "csi.volume." + volname[7] + volname[9] + volname[11] + details = OmapDetails{omapval1, pv} + } else { + logging.Fatal(fmt.Errorf("omap details not found")) + } + return details, nil +}