From d86b70d8b5a07a0cd112f0d8e3e12a8c49369a72 Mon Sep 17 00:00:00 2001 From: yati1998 Date: Tue, 6 Feb 2024 18:19:06 +0530 Subject: [PATCH] csi: clean cephFS omap details this commit deletes the omap presence of the stale subvolume Signed-off-by: yati1998 --- .github/workflows/go-test.yaml | 12 ++-- docs/subvolume.md | 35 ++++++----- pkg/exec/exec.go | 2 + pkg/filesystem/subvolume.go | 104 +++++++++++++++++++++++++++++-- pkg/filesystem/subvolume_test.go | 55 ++++++++++++++++ tests/github-action-helper.sh | 41 ++++++++++++ 6 files changed, 225 insertions(+), 24 deletions(-) create mode 100644 pkg/filesystem/subvolume_test.go diff --git a/.github/workflows/go-test.yaml b/.github/workflows/go-test.yaml index a4710917..c4a0ffde 100644 --- a/.github/workflows/go-test.yaml +++ b/.github/workflows/go-test.yaml @@ -77,11 +77,13 @@ jobs: run: | set -ex kubectl rook-ceph ceph fs subvolume create myfs test-subvol group-a - kubectl rook-ceph ceph fs subvolume create myfs test-subvol-1 group-a kubectl rook-ceph subvolume ls kubectl rook-ceph subvolume ls --stale kubectl rook-ceph subvolume delete myfs test-subvol group-a - kubectl rook-ceph subvolume delete myfs test-subvol-1 + tests/github-action-helper.sh create_sc_with_retain_policy + tests/github-action-helper.sh create_stale_subvolume + subVol=$(kubectl rook-ceph subvolume ls --stale | awk '{print $2}' | grep csi-vol) + kubectl rook_ceph subvolume delete myfs $subVol - name: Get mon endpoints run: | @@ -232,12 +234,14 @@ jobs: - name: Subvolume command run: | set -ex - kubectl rook-ceph --operator-namespace test-operator -n test-cluster ceph fs subvolume create myfs test-subvol-1 group-a kubectl rook-ceph --operator-namespace test-operator -n test-cluster ceph fs subvolume create myfs test-subvol group-a kubectl rook-ceph --operator-namespace test-operator -n test-cluster subvolume ls kubectl rook-ceph --operator-namespace test-operator -n test-cluster subvolume ls --stale kubectl rook-ceph --operator-namespace test-operator -n test-cluster subvolume delete myfs test-subvol group-a - kubectl rook-ceph --operator-namespace test-operator -n test-cluster subvolume delete myfs test-subvol-1 + tests/github-action-helper.sh create_sc_with_retain_policy_custom_ns test-operator test-cluster + tests/github-action-helper.sh create_stale_subvolume + subVol=$(kubectl rook-ceph --operator-namespace test-operator -n test-cluster subvolume ls --stale | awk '{print $2}' | grep csi-vol) + kubectl rook_ceph --operator-namespace test-operator -n test-cluster subvolume delete myfs $subVol - name: Get mon endpoints run: | diff --git a/docs/subvolume.md b/docs/subvolume.md index 02dc8ed6..fce73a6c 100644 --- a/docs/subvolume.md +++ b/docs/subvolume.md @@ -17,30 +17,37 @@ The subvolume command will require the following sub commands: ## ls ```bash -kubectl rook-ceph subvolume ls - -# Filesystem Subvolume SubvolumeGroup State -# ocs-storagecluster-cephfilesystem csi-vol-427774b4-340b-11ed-8d66-0242ac110004 csi in-use -# ocs-storagecluster-cephfilesystem csi-vol-427774b4-340b-11ed-8d66-0242ac110005 csi in-use -# ocs-storagecluster-cephfilesystem csi-vol-427774b4-340b-11ed-8d66-0242ac110007 csi stale -# ocs-storagecluster-cephfilesystem csi-vol-427774b4-340b-11ed-8d66-0242ac110007 csi stale-with-snapshot +$ kubectl rook-ceph subvolume ls +Filesystem Subvolume SubvolumeGroup State +myfs csi-vol-427774b4-340b-11ed-8d66-0242ac110004 csi in-use +myfs csi-vol-427774b4-340b-11ed-8d66-0242ac110005 csi in-use +myfs csi-vol-427774b4-340b-11ed-8d66-0242ac110007 csi stale +myfs csi-vol-427774b4-340b-11ed-8d66-0242ac110007 csi stale-with-snapshot ``` ```bash -kubectl rook-ceph subvolume ls --stale - -# Filesystem Subvolume SubvolumeGroup state -# ocs-storagecluster-cephfilesystem csi-vol-427774b4-340b-11ed-8d66-0242ac110004 csi stale -# ocs-storagecluster-cephfilesystem csi-vol-427774b4-340b-11ed-8d66-0242ac110005 csi stale-with-snapshot +$ kubectl rook-ceph subvolume ls --stale +Filesystem Subvolume SubvolumeGroup state +myfs csi-vol-427774b4-340b-11ed-8d66-0242ac110004 csi stale +myfs csi-vol-427774b4-340b-11ed-8d66-0242ac110005 csi stale-with-snapshot ``` ## delete ```bash -kubectl rook-ceph subvolume delete ocs-storagecluster csi-vol-427774b4-340b-11ed-8d66-0242ac110004 +$ kubectl rook-ceph subvolume delete myfs csi-vol-427774b4-340b-11ed-8d66-0242ac110005 -# Info: subvolume "csi-vol-427774b4-340b-11ed-8d66-0242ac110004" deleted +Info: Deleting the omap object and key for subvolume "csi-vol-0c91ba82-5a63-4117-88a4-690acd86cbbd" +Info: omap object:"csi.volume.0c91ba82-5a63-4117-88a4-690acd86cbbd" deleted +Info: omap key:"csi.volume.pvc-78abf81c-5381-42ee-8d75-dc17cd0cf5de" deleted +Info: subvolume "csi-vol-0c91ba82-5a63-4117-88a4-690acd86cbbd" deleted +``` + +```bash +$ kubectl rook-ceph subvolume delete myfs csi-vol-427774b4-340b-11ed-8d66-0242ac110005 +Info: No omapvals found for subvolume csi-vol-427774b4-340b-11ed-8d66-0242ac110005 +Info: subvolume "csi-vol-427774b4-340b-11ed-8d66-0242ac110005" deleted ``` \ No newline at end of file diff --git a/pkg/exec/exec.go b/pkg/exec/exec.go index d342bc23..074182a5 100644 --- a/pkg/exec/exec.go +++ b/pkg/exec/exec.go @@ -130,6 +130,8 @@ func execCmdInPod(ctx context.Context, clientsets *k8sutil.Clientsets, cmd = append(cmd, "--connect-timeout=10", fmt.Sprintf("--conf=/var/lib/rook/%s/%s.config", clusterNamespace, clusterNamespace)) } else if cmd[0] == "rbd" { cmd = append(cmd, fmt.Sprintf("--conf=/var/lib/rook/%s/%s.config", clusterNamespace, clusterNamespace)) + } else if cmd[0] == "rados" { + cmd = append(cmd, fmt.Sprintf("--conf=/var/lib/rook/%s/%s.config", clusterNamespace, clusterNamespace)) } // Prepare the API URL used to execute another process within the Pod. In diff --git a/pkg/filesystem/subvolume.go b/pkg/filesystem/subvolume.go index aed7b5f9..8e7673bf 100644 --- a/pkg/filesystem/subvolume.go +++ b/pkg/filesystem/subvolume.go @@ -20,6 +20,7 @@ import ( "context" "encoding/json" "fmt" + "strings" "github.com/rook/kubectl-rook-ceph/pkg/exec" "github.com/rook/kubectl-rook-ceph/pkg/k8sutil" @@ -28,8 +29,9 @@ import ( ) type fsStruct struct { - Name string - data []string + Name string + MetadataName string `json:"metadata_pool"` + data []string } type subVolumeInfo struct { @@ -201,7 +203,6 @@ func unMarshaljson(list string) []fsStruct { if errg != nil { logging.Fatal(errg) } - return unmarshal } @@ -209,9 +210,100 @@ func Delete(ctx context.Context, clientsets *k8sutil.Clientsets, OperatorNamespa k8sSubvolume := getK8sRefSubvolume(ctx, clientsets) _, check := k8sSubvolume[subvol] if !check { - exec.RunCommandInOperatorPod(ctx, clientsets, "ceph", []string{"fs", "subvolume", "rm", fs, subvol, svg, "--retain-snapshots"}, OperatorNamespace, CephClusterNamespace, true) - logging.Info("subvolume %q deleted", subvol) + deleteOmapForSubvolume(ctx, clientsets, OperatorNamespace, CephClusterNamespace, subvol, fs) + _, err := exec.RunCommandInOperatorPod(ctx, clientsets, "ceph", []string{"fs", "subvolume", "rm", fs, subvol, svg, "--retain-snapshots"}, OperatorNamespace, CephClusterNamespace, true) + if err != nil { + logging.Fatal(err, "failed to delete subvolume of %s/%s/%s", fs, svg, subvol) + } + logging.Info("subvolume %s/%s/%s deleted", fs, svg, subvol) + } else { - logging.Info("subvolume %q is not stale", subvol) + logging.Info("subvolume %s/%s/%s is not stale", fs, svg, subvol) + } +} + +func getMetadataPoolName(ctx context.Context, clientsets *k8sutil.Clientsets, OperatorNamespace, CephClusterNamespace, fs string) (string, error) { + fsstruct, err := getFileSystem(ctx, clientsets, OperatorNamespace, CephClusterNamespace) + if err != nil { + return "", err + } + + for _, pool := range fsstruct { + if pool.Name == fs { + return pool.MetadataName, nil + } + } + return "", fmt.Errorf("metadataPool not found for %q filesystem", fs) +} + +// deleteOmap deletes omap object and key for the given subvolume. +func deleteOmapForSubvolume(ctx context.Context, clientsets *k8sutil.Clientsets, OperatorNamespace, CephClusterNamespace, subVol, fs string) { + logging.Info("Deleting the omap object and key for subvolume %q", subVol) + omapkey := getOmapKey(ctx, clientsets, OperatorNamespace, CephClusterNamespace, subVol, fs) + omapval := getOmapVal(subVol) + poolName, err := getMetadataPoolName(ctx, clientsets, OperatorNamespace, CephClusterNamespace, fs) + if err != nil || poolName == "" { + logging.Fatal(fmt.Errorf("pool name not found: %q", err)) + } + if omapval != "" { + // remove omap object. + _, err := exec.RunCommandInOperatorPod(ctx, clientsets, "rados", []string{"rm", omapval, "-p", poolName, "--namespace", "csi"}, OperatorNamespace, CephClusterNamespace, true) + if err != nil { + logging.Fatal(err, "failed to remove omap object for subvolume %q", subVol) + } + logging.Info("omap object:%q deleted", omapval) + } + if omapkey != "" { + // remove omap key. + _, err := exec.RunCommandInOperatorPod(ctx, clientsets, "rados", []string{"rmomapkey", "csi.volumes.default", omapkey, "-p", poolName, "--namespace", "csi"}, OperatorNamespace, CephClusterNamespace, true) + if err != nil { + logging.Fatal(err, "failed to remove omap key for subvolume %q", subVol) + } + logging.Info("omap key:%q deleted", omapkey) + + } +} + +// getOmapKey gets the omap key and value details for a given subvolume. +// Deletion of omap object required the subvolumeName which is of format +// csi.volume.subvolume, where subvolume is the name of subvolume that needs to be +// deleted. +// similarly to delete of omap key requires csi.volume.ompakey, where +// omapkey is the pv name which is extracted the omap object. +func getOmapKey(ctx context.Context, clientsets *k8sutil.Clientsets, OperatorNamespace, CephClusterNamespace, subVol, fs string) string { + + poolName, err := getMetadataPoolName(ctx, clientsets, OperatorNamespace, CephClusterNamespace, fs) + if err != nil || poolName == "" { + logging.Fatal(fmt.Errorf("pool name not found: %q", err)) + } + omapval := getOmapVal(subVol) + + cmd := []string{"getomapval", omapval, "csi.volname", "-p", poolName, "--namespace", "csi", "/dev/stdout"} + pvname, err := exec.RunCommandInOperatorPod(ctx, clientsets, "rados", cmd, OperatorNamespace, CephClusterNamespace, true) + if err != nil || pvname == "" { + logging.Info("No PV found for subvolume %s: %s", subVol, err) + return "" + } + // omap key is for format csi.volume.pvc-fca205e5-8788-4132-979c-e210c0133182 + // hence, attaching pvname to required prefix. + omapkey := "csi.volume." + pvname + + return omapkey +} + +// func getOmapVal is used to get the omapval from the given subvolume +// omapval is of format csi.volume.427774b4-340b-11ed-8d66-0242ac110005 +// which is similar to volume name csi-vol-427774b4-340b-11ed-8d66-0242ac110005 +// hence, replacing 'csi-vol-' to 'csi.volume.' to get the omapval +func getOmapVal(subVol string) string { + + splitSubvol := strings.SplitAfterN(subVol, "-", 3) + if len(splitSubvol) < 3 { + return "" + } + subvol_id := splitSubvol[len(splitSubvol)-1] + omapval := "csi.volume." + subvol_id + + return omapval } diff --git a/pkg/filesystem/subvolume_test.go b/pkg/filesystem/subvolume_test.go new file mode 100644 index 00000000..6ce3d8c3 --- /dev/null +++ b/pkg/filesystem/subvolume_test.go @@ -0,0 +1,55 @@ +/* +Copyright 2024 The Rook Authors. All rights reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package subvolume + +import "testing" + +func TestGetOmapVal(t *testing.T) { + + tests := []struct { + name string + want string + }{ + { + name: "csi-vol-427774b4-340b-11ed-8d66-0242ac110005", + want: "csi.volume.427774b4-340b-11ed-8d66-0242ac110005", + }, + { + name: "nfs-export-427774b4-340b-11ed-8d66-0242ac110005", + want: "csi.volume.427774b4-340b-11ed-8d66-0242ac110005", + }, + { + name: "", + want: "", + }, + { + name: "csi-427774b4-340b-11ed-8d66-0242ac11000", + want: "csi.volume.340b-11ed-8d66-0242ac11000", + }, + { + name: "csi-427774b440b11ed8d660242ac11000", + want: "", + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if got := getOmapVal(tt.name); got != tt.want { + t.Errorf("getOmapVal() = %v, want %v", got, tt.want) + } + }) + } +} diff --git a/tests/github-action-helper.sh b/tests/github-action-helper.sh index c365d407..e48a1a23 100755 --- a/tests/github-action-helper.sh +++ b/tests/github-action-helper.sh @@ -76,6 +76,37 @@ deploy_rook_in_custom_namespace() { deploy_csi_driver_custom_ns "$1" "$2" } +create_sc_with_retain_policy(){ + curl https://raw.githubusercontent.com/rook/rook/master/deploy/examples/csi/cephfs/storageclass.yaml -o storageclass.yaml + sed -i "s|name: rook-cephfs|name: rook-cephfs-retain|g" storageclass.yaml + sed -i "s|reclaimPolicy: Delete|reclaimPolicy: Retain|g" storageclass.yaml + kubectl create -f storageclass.yaml +} + +create_sc_with_retain_policy_custom_ns(){ + export OPERATOR_NS=$1 + export CLUSTER_NS=$2 + + curl https://raw.githubusercontent.com/rook/rook/master/deploy/examples/csi/cephfs/storageclass.yaml -o storageclass.yaml + sed -i "s|name: rook-cephfs|name: rook-cephfs-retain|g" storageclass.yaml + sed -i "s|reclaimPolicy: Delete|reclaimPolicy: Retain|g" storageclass.yaml + sed -i "s|provisioner: rook-ceph.cephfs.csi.ceph.com |provisioner: test-operator.cephfs.csi.ceph.com |g" storageclass.yaml + deploy_with_custom_ns $OPERATOR_NS $CLUSTER_NS storageclass.yaml +} + +create_stale_subvolume() { + curl https://raw.githubusercontent.com/rook/rook/master/deploy/examples/csi/cephfs/pvc.yaml -o pvc.yaml + sed -i "s|name: cephfs-pvc|name: cephfs-pvc-retain|g" pvc.yaml + sed -i "s|storageClassName: rook-cephfs|storageClassName: rook-cephfs-retain|g" pvc.yaml + kubectl create -f pvc.yaml + kubectl get pvc cephfs-pvc-retain + : "${PVNAME:=$(kubectl get pvc cephfs-pvc-retain -o=jsonpath='{.spec.volumeName}')}" + wait_for_pvc_to_be_bound_state_default + kubectl get pvc cephfs-pvc-retain + kubectl delete pvc cephfs-pvc-retain + kubectl delete pv "$PVNAME" +} + deploy_with_custom_ns() { sed -i "s|rook-ceph # namespace:operator|$1 # namespace:operator|g" "$3" sed -i "s|rook-ceph # namespace:cluster|$2 # namespace:cluster|g" "$3" @@ -109,6 +140,16 @@ deploy_csi_driver_custom_ns() { kubectl create -f https://raw.githubusercontent.com/rook/rook/master/deploy/examples/csi/cephfs/pvc.yaml } +wait_for_pvc_to_be_bound_state_default() { + timeout 100 bash <<-'EOF' + until [ $(kubectl get pvc cephfs-pvc-retain -o jsonpath='{.status.phase}') == "Bound" ]; do + echo "waiting for the pvc to be in bound state" + sleep 1 + done +EOF + timeout_command_exit_code +} + wait_for_pod_to_be_ready_state_default() { timeout 200 bash <<-'EOF' until [ $(kubectl get pod -l app=rook-ceph-osd -n rook-ceph -o jsonpath='{.items[*].metadata.name}' -o custom-columns=READY:status.containerStatuses[*].ready | grep -c true) -eq 1 ]; do