Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

clean cephFS omap details #237

Merged
merged 1 commit into from
Mar 20, 2024
Merged

clean cephFS omap details #237

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
12 changes: 8 additions & 4 deletions .github/workflows/go-test.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -77,11 +77,13 @@ jobs:
run: |
set -ex
kubectl rook-ceph ceph fs subvolume create myfs test-subvol group-a
kubectl rook-ceph ceph fs subvolume create myfs test-subvol-1 group-a
kubectl rook-ceph subvolume ls
kubectl rook-ceph subvolume ls --stale
kubectl rook-ceph subvolume delete myfs test-subvol group-a
kubectl rook-ceph subvolume delete myfs test-subvol-1
tests/github-action-helper.sh create_sc_with_retain_policy
tests/github-action-helper.sh create_stale_subvolume
subVol=$(kubectl rook-ceph subvolume ls --stale | awk '{print $2}' | grep csi-vol)
kubectl rook_ceph subvolume delete myfs $subVol

- name: Get mon endpoints
run: |
Expand Down Expand Up @@ -232,12 +234,14 @@ jobs:
- name: Subvolume command
run: |
set -ex
kubectl rook-ceph --operator-namespace test-operator -n test-cluster ceph fs subvolume create myfs test-subvol-1 group-a
kubectl rook-ceph --operator-namespace test-operator -n test-cluster ceph fs subvolume create myfs test-subvol group-a
kubectl rook-ceph --operator-namespace test-operator -n test-cluster subvolume ls
kubectl rook-ceph --operator-namespace test-operator -n test-cluster subvolume ls --stale
kubectl rook-ceph --operator-namespace test-operator -n test-cluster subvolume delete myfs test-subvol group-a
kubectl rook-ceph --operator-namespace test-operator -n test-cluster subvolume delete myfs test-subvol-1
tests/github-action-helper.sh create_sc_with_retain_policy_custom_ns test-operator test-cluster
tests/github-action-helper.sh create_stale_subvolume
subVol=$(kubectl rook-ceph --operator-namespace test-operator -n test-cluster subvolume ls --stale | awk '{print $2}' | grep csi-vol)
kubectl rook_ceph --operator-namespace test-operator -n test-cluster subvolume delete myfs $subVol

- name: Get mon endpoints
run: |
Expand Down
35 changes: 21 additions & 14 deletions docs/subvolume.md
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -17,30 +17,37 @@ The subvolume command will require the following sub commands:
## ls

```bash
kubectl rook-ceph subvolume ls

# Filesystem Subvolume SubvolumeGroup State
# ocs-storagecluster-cephfilesystem csi-vol-427774b4-340b-11ed-8d66-0242ac110004 csi in-use
# ocs-storagecluster-cephfilesystem csi-vol-427774b4-340b-11ed-8d66-0242ac110005 csi in-use
# ocs-storagecluster-cephfilesystem csi-vol-427774b4-340b-11ed-8d66-0242ac110007 csi stale
# ocs-storagecluster-cephfilesystem csi-vol-427774b4-340b-11ed-8d66-0242ac110007 csi stale-with-snapshot
$ kubectl rook-ceph subvolume ls

Filesystem Subvolume SubvolumeGroup State
myfs csi-vol-427774b4-340b-11ed-8d66-0242ac110004 csi in-use
myfs csi-vol-427774b4-340b-11ed-8d66-0242ac110005 csi in-use
myfs csi-vol-427774b4-340b-11ed-8d66-0242ac110007 csi stale
myfs csi-vol-427774b4-340b-11ed-8d66-0242ac110007 csi stale-with-snapshot
```

```bash
kubectl rook-ceph subvolume ls --stale

# Filesystem Subvolume SubvolumeGroup state
# ocs-storagecluster-cephfilesystem csi-vol-427774b4-340b-11ed-8d66-0242ac110004 csi stale
# ocs-storagecluster-cephfilesystem csi-vol-427774b4-340b-11ed-8d66-0242ac110005 csi stale-with-snapshot
$ kubectl rook-ceph subvolume ls --stale

Filesystem Subvolume SubvolumeGroup state
myfs csi-vol-427774b4-340b-11ed-8d66-0242ac110004 csi stale
myfs csi-vol-427774b4-340b-11ed-8d66-0242ac110005 csi stale-with-snapshot
```

## delete

```bash
kubectl rook-ceph subvolume delete ocs-storagecluster csi-vol-427774b4-340b-11ed-8d66-0242ac110004
$ kubectl rook-ceph subvolume delete myfs csi-vol-427774b4-340b-11ed-8d66-0242ac110005

# Info: subvolume "csi-vol-427774b4-340b-11ed-8d66-0242ac110004" deleted
Info: Deleting the omap object and key for subvolume "csi-vol-0c91ba82-5a63-4117-88a4-690acd86cbbd"
Info: omap object:"csi.volume.0c91ba82-5a63-4117-88a4-690acd86cbbd" deleted
Info: omap key:"csi.volume.pvc-78abf81c-5381-42ee-8d75-dc17cd0cf5de" deleted
Info: subvolume "csi-vol-0c91ba82-5a63-4117-88a4-690acd86cbbd" deleted
```

```bash
$ kubectl rook-ceph subvolume delete myfs csi-vol-427774b4-340b-11ed-8d66-0242ac110005

Info: No omapvals found for subvolume csi-vol-427774b4-340b-11ed-8d66-0242ac110005
Info: subvolume "csi-vol-427774b4-340b-11ed-8d66-0242ac110005" deleted
```
2 changes: 2 additions & 0 deletions pkg/exec/exec.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -130,6 +130,8 @@ func execCmdInPod(ctx context.Context, clientsets *k8sutil.Clientsets,
cmd = append(cmd, "--connect-timeout=10", fmt.Sprintf("--conf=/var/lib/rook/%s/%s.config", clusterNamespace, clusterNamespace))
} else if cmd[0] == "rbd" {
cmd = append(cmd, fmt.Sprintf("--conf=/var/lib/rook/%s/%s.config", clusterNamespace, clusterNamespace))
} else if cmd[0] == "rados" {
cmd = append(cmd, fmt.Sprintf("--conf=/var/lib/rook/%s/%s.config", clusterNamespace, clusterNamespace))
}

// Prepare the API URL used to execute another process within the Pod. In
Expand Down
104 changes: 98 additions & 6 deletions pkg/filesystem/subvolume.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -20,6 +20,7 @@ import (
"context"
"encoding/json"
"fmt"
"strings"

"github.com/rook/kubectl-rook-ceph/pkg/exec"
"github.com/rook/kubectl-rook-ceph/pkg/k8sutil"
Expand All @@ -28,8 +29,9 @@ import (
)

type fsStruct struct {
Name string
data []string
Name string
MetadataName string `json:"metadata_pool"`
data []string
}

type subVolumeInfo struct {
Expand Down Expand Up @@ -201,17 +203,107 @@ func unMarshaljson(list string) []fsStruct {
if errg != nil {
logging.Fatal(errg)
}

return unmarshal
}

func Delete(ctx context.Context, clientsets *k8sutil.Clientsets, OperatorNamespace, CephClusterNamespace, fs, subvol, svg string) {
k8sSubvolume := getK8sRefSubvolume(ctx, clientsets)
_, check := k8sSubvolume[subvol]
if !check {
exec.RunCommandInOperatorPod(ctx, clientsets, "ceph", []string{"fs", "subvolume", "rm", fs, subvol, svg, "--retain-snapshots"}, OperatorNamespace, CephClusterNamespace, true)
logging.Info("subvolume %q deleted", subvol)
deleteOmapForSubvolume(ctx, clientsets, OperatorNamespace, CephClusterNamespace, subvol, fs)
_, err := exec.RunCommandInOperatorPod(ctx, clientsets, "ceph", []string{"fs", "subvolume", "rm", fs, subvol, svg, "--retain-snapshots"}, OperatorNamespace, CephClusterNamespace, true)
if err != nil {
logging.Fatal(err, "failed to delete subvolume of %s/%s/%s", fs, svg, subvol)
}
logging.Info("subvolume %s/%s/%s deleted", fs, svg, subvol)

} else {
logging.Info("subvolume %q is not stale", subvol)
logging.Info("subvolume %s/%s/%s is not stale", fs, svg, subvol)
}
}

func getMetadataPoolName(ctx context.Context, clientsets *k8sutil.Clientsets, OperatorNamespace, CephClusterNamespace, fs string) (string, error) {
fsstruct, err := getFileSystem(ctx, clientsets, OperatorNamespace, CephClusterNamespace)
if err != nil {
return "", err
}

for _, pool := range fsstruct {
if pool.Name == fs {
return pool.MetadataName, nil
}
}
return "", fmt.Errorf("metadataPool not found for %q filesystem", fs)
}

// deleteOmap deletes omap object and key for the given subvolume.
func deleteOmapForSubvolume(ctx context.Context, clientsets *k8sutil.Clientsets, OperatorNamespace, CephClusterNamespace, subVol, fs string) {
logging.Info("Deleting the omap object and key for subvolume %q", subVol)
omapkey := getOmapKey(ctx, clientsets, OperatorNamespace, CephClusterNamespace, subVol, fs)
omapval := getOmapVal(subVol)
poolName, err := getMetadataPoolName(ctx, clientsets, OperatorNamespace, CephClusterNamespace, fs)
if err != nil || poolName == "" {
logging.Fatal(fmt.Errorf("pool name not found: %q", err))
}
if omapval != "" {
// remove omap object.
_, err := exec.RunCommandInOperatorPod(ctx, clientsets, "rados", []string{"rm", omapval, "-p", poolName, "--namespace", "csi"}, OperatorNamespace, CephClusterNamespace, true)
if err != nil {
logging.Fatal(err, "failed to remove omap object for subvolume %q", subVol)
}
logging.Info("omap object:%q deleted", omapval)

}
if omapkey != "" {
// remove omap key.
_, err := exec.RunCommandInOperatorPod(ctx, clientsets, "rados", []string{"rmomapkey", "csi.volumes.default", omapkey, "-p", poolName, "--namespace", "csi"}, OperatorNamespace, CephClusterNamespace, true)
if err != nil {
logging.Fatal(err, "failed to remove omap key for subvolume %q", subVol)
}
logging.Info("omap key:%q deleted", omapkey)

}
}

// getOmapKey gets the omap key and value details for a given subvolume.
// Deletion of omap object required the subvolumeName which is of format
// csi.volume.subvolume, where subvolume is the name of subvolume that needs to be
// deleted.
// similarly to delete of omap key requires csi.volume.ompakey, where
// omapkey is the pv name which is extracted the omap object.
func getOmapKey(ctx context.Context, clientsets *k8sutil.Clientsets, OperatorNamespace, CephClusterNamespace, subVol, fs string) string {

poolName, err := getMetadataPoolName(ctx, clientsets, OperatorNamespace, CephClusterNamespace, fs)
if err != nil || poolName == "" {
logging.Fatal(fmt.Errorf("pool name not found: %q", err))
Comment on lines +277 to +278
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

same here as well

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

yes, it's not required.

}
omapval := getOmapVal(subVol)

cmd := []string{"getomapval", omapval, "csi.volname", "-p", poolName, "--namespace", "csi", "/dev/stdout"}
pvname, err := exec.RunCommandInOperatorPod(ctx, clientsets, "rados", cmd, OperatorNamespace, CephClusterNamespace, true)
if err != nil || pvname == "" {
logging.Info("No PV found for subvolume %s: %s", subVol, err)
return ""
}
// omap key is for format csi.volume.pvc-fca205e5-8788-4132-979c-e210c0133182
// hence, attaching pvname to required prefix.
omapkey := "csi.volume." + pvname

return omapkey
}

// func getOmapVal is used to get the omapval from the given subvolume
// omapval is of format csi.volume.427774b4-340b-11ed-8d66-0242ac110005
// which is similar to volume name csi-vol-427774b4-340b-11ed-8d66-0242ac110005
// hence, replacing 'csi-vol-' to 'csi.volume.' to get the omapval
func getOmapVal(subVol string) string {

splitSubvol := strings.SplitAfterN(subVol, "-", 3)
if len(splitSubvol) < 3 {
return ""
}
subvol_id := splitSubvol[len(splitSubvol)-1]
omapval := "csi.volume." + subvol_id

return omapval
}
55 changes: 55 additions & 0 deletions pkg/filesystem/subvolume_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,55 @@
/*
Copyright 2024 The Rook Authors. All rights reserved.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package subvolume

import "testing"

func TestGetOmapVal(t *testing.T) {

tests := []struct {
name string
want string
}{
Copy link

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Maybe consider adding negative tests as well? An empty string, a string with only no, one or two - in there probably are problematic and cause issues. It may be unlikely that these get passed to the function, but you should be prepared for weird issues in a cleanup process, some values may be unexpectedly missing or somehow got corrupted.

Copy link

@nixpanic nixpanic Mar 18, 2024 •
edited
Loading

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I'm not sure if the added tests provide the needed or expected results to whatever calls getOmapVal(). But this at least confirms that there are no panics happening.

{
name: "csi-vol-427774b4-340b-11ed-8d66-0242ac110005",
want: "csi.volume.427774b4-340b-11ed-8d66-0242ac110005",
},
{
name: "nfs-export-427774b4-340b-11ed-8d66-0242ac110005",
want: "csi.volume.427774b4-340b-11ed-8d66-0242ac110005",
},
{
name: "",
want: "",
},
{
name: "csi-427774b4-340b-11ed-8d66-0242ac11000",
want: "csi.volume.340b-11ed-8d66-0242ac11000",
},
{
name: "csi-427774b440b11ed8d660242ac11000",
want: "",
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
if got := getOmapVal(tt.name); got != tt.want {
t.Errorf("getOmapVal() = %v, want %v", got, tt.want)
}
})
}
}
41 changes: 41 additions & 0 deletions tests/github-action-helper.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -76,6 +76,37 @@ deploy_rook_in_custom_namespace() {
deploy_csi_driver_custom_ns "$1" "$2"
}

create_sc_with_retain_policy(){
curl https://raw.githubusercontent.com/rook/rook/master/deploy/examples/csi/cephfs/storageclass.yaml -o storageclass.yaml
sed -i "s|name: rook-cephfs|name: rook-cephfs-retain|g" storageclass.yaml
sed -i "s|reclaimPolicy: Delete|reclaimPolicy: Retain|g" storageclass.yaml
kubectl create -f storageclass.yaml
}

create_sc_with_retain_policy_custom_ns(){
export OPERATOR_NS=$1
export CLUSTER_NS=$2

curl https://raw.githubusercontent.com/rook/rook/master/deploy/examples/csi/cephfs/storageclass.yaml -o storageclass.yaml
sed -i "s|name: rook-cephfs|name: rook-cephfs-retain|g" storageclass.yaml
sed -i "s|reclaimPolicy: Delete|reclaimPolicy: Retain|g" storageclass.yaml
sed -i "s|provisioner: rook-ceph.cephfs.csi.ceph.com |provisioner: test-operator.cephfs.csi.ceph.com |g" storageclass.yaml
deploy_with_custom_ns $OPERATOR_NS $CLUSTER_NS storageclass.yaml
}

create_stale_subvolume() {
curl https://raw.githubusercontent.com/rook/rook/master/deploy/examples/csi/cephfs/pvc.yaml -o pvc.yaml
sed -i "s|name: cephfs-pvc|name: cephfs-pvc-retain|g" pvc.yaml
sed -i "s|storageClassName: rook-cephfs|storageClassName: rook-cephfs-retain|g" pvc.yaml
kubectl create -f pvc.yaml
kubectl get pvc cephfs-pvc-retain
: "${PVNAME:=$(kubectl get pvc cephfs-pvc-retain -o=jsonpath='{.spec.volumeName}')}"
wait_for_pvc_to_be_bound_state_default
kubectl get pvc cephfs-pvc-retain
kubectl delete pvc cephfs-pvc-retain
kubectl delete pv "$PVNAME"
}

deploy_with_custom_ns() {
sed -i "s|rook-ceph # namespace:operator|$1 # namespace:operator|g" "$3"
sed -i "s|rook-ceph # namespace:cluster|$2 # namespace:cluster|g" "$3"
Expand Down Expand Up @@ -109,6 +140,16 @@ deploy_csi_driver_custom_ns() {
kubectl create -f https://raw.githubusercontent.com/rook/rook/master/deploy/examples/csi/cephfs/pvc.yaml
}

wait_for_pvc_to_be_bound_state_default() {
timeout 100 bash <<-'EOF'
until [ $(kubectl get pvc cephfs-pvc-retain -o jsonpath='{.status.phase}') == "Bound" ]; do
echo "waiting for the pvc to be in bound state"
sleep 1
done
EOF
timeout_command_exit_code
}

wait_for_pod_to_be_ready_state_default() {
timeout 200 bash <<-'EOF'
until [ $(kubectl get pod -l app=rook-ceph-osd -n rook-ceph -o jsonpath='{.items[*].metadata.name}' -o custom-columns=READY:status.containerStatuses[*].ready | grep -c true) -eq 1 ]; do
Expand Down
Loading