From ebc56887cdfe425844f30b96a7d8bf604179d5c5 Mon Sep 17 00:00:00 2001
From: Niraj Yadav <niryadav@redhat.com>
Date: Fri, 21 Jun 2024 15:49:06 +0530
Subject: [PATCH 01/14] rbd: implement pv key rotation

This patch implements the EncryptionKeyRotation spec for ceph-csi

Signed-off-by: Niraj Yadav <niryadav@redhat.com>
---
 go.mod                                        |   2 +-
 go.sum                                        |   4 +-
 .../csi-addons/rbd/encryptionkeyrotation.go   | 101 ++++++
 internal/csi-addons/rbd/identity.go           |   7 +
 internal/kms/vault.go                         |  34 +-
 internal/kms/vault_test.go                    |  18 -
 internal/kms/vault_tokens.go                  |  18 +-
 internal/rbd/driver/driver.go                 |   3 +
 internal/rbd/encryption.go                    |  73 ++++
 internal/util/crypto.go                       |   5 +
 internal/util/cryptsetup.go                   | 133 ++++++++
 internal/util/file/file.go                    |  54 +++
 internal/util/file/file_test.go               | 100 ++++++
 .../encryptionkeyrotation.pb.go               | 317 ++++++++++++++++++
 .../encryptionkeyrotation_grpc.pb.go          | 116 +++++++
 vendor/modules.txt                            |   3 +-
 16 files changed, 930 insertions(+), 58 deletions(-)
 create mode 100644 internal/csi-addons/rbd/encryptionkeyrotation.go
 create mode 100644 internal/util/file/file.go
 create mode 100644 internal/util/file/file_test.go
 create mode 100644 vendor/github.com/csi-addons/spec/lib/go/encryptionkeyrotation/encryptionkeyrotation.pb.go
 create mode 100644 vendor/github.com/csi-addons/spec/lib/go/encryptionkeyrotation/encryptionkeyrotation_grpc.pb.go

diff --git a/go.mod b/go.mod
index 7e469a7ccb5..de27daf5016 100644
--- a/go.mod
+++ b/go.mod
@@ -9,7 +9,7 @@ require (
 	github.com/ceph/ceph-csi/api v0.0.0-00010101000000-000000000000
 	github.com/ceph/go-ceph v0.28.0
 	github.com/container-storage-interface/spec v1.10.0
-	github.com/csi-addons/spec v0.2.1-0.20240627093359-0dd74d521e67
+	github.com/csi-addons/spec v0.2.1-0.20240718113938-dc98b454ba65
 	github.com/gemalto/kmip-go v0.0.10
 	github.com/golang/protobuf v1.5.4
 	github.com/google/fscrypt v0.3.6-0.20240502174735-068b9f8f5dec
diff --git a/go.sum b/go.sum
index d10fdbd7298..6f3afc0d380 100644
--- a/go.sum
+++ b/go.sum
@@ -911,8 +911,8 @@ github.com/cpuguy83/go-md2man/v2 v2.0.2/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46t
 github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E=
 github.com/creack/pty v1.1.11/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E=
 github.com/creack/pty v1.1.18/go.mod h1:MOBLtS5ELjhRRrroQr9kyvTxUAFNvYEK993ew/Vr4O4=
-github.com/csi-addons/spec v0.2.1-0.20240627093359-0dd74d521e67 h1:UAcAhE1pTkWaFBS0kvhHUcUsoEv5fsieD0tl8psQMCs=
-github.com/csi-addons/spec v0.2.1-0.20240627093359-0dd74d521e67/go.mod h1:Mwq4iLiUV4s+K1bszcWU6aMsR5KPsbIYzzszJ6+56vI=
+github.com/csi-addons/spec v0.2.1-0.20240718113938-dc98b454ba65 h1:i9JGGQTEmRQXSpQQPR96+DV4D4o+V1+gjAWf+bpxQxk=
+github.com/csi-addons/spec v0.2.1-0.20240718113938-dc98b454ba65/go.mod h1:Mwq4iLiUV4s+K1bszcWU6aMsR5KPsbIYzzszJ6+56vI=
 github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
 github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
 github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc h1:U9qPSI2PIWSS1VwoXQT9A3Wy9MM3WgvqSxFWenqJduM=
diff --git a/internal/csi-addons/rbd/encryptionkeyrotation.go b/internal/csi-addons/rbd/encryptionkeyrotation.go
new file mode 100644
index 00000000000..42ca620e41b
--- /dev/null
+++ b/internal/csi-addons/rbd/encryptionkeyrotation.go
@@ -0,0 +1,101 @@
+/*
+Copyright 2024 The Ceph-CSI Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package rbd
+
+import (
+	"context"
+	"errors"
+
+	csicommon "github.com/ceph/ceph-csi/internal/csi-common"
+	"github.com/ceph/ceph-csi/internal/rbd"
+	"github.com/ceph/ceph-csi/internal/util"
+	"github.com/ceph/ceph-csi/internal/util/log"
+
+	"github.com/container-storage-interface/spec/lib/go/csi"
+	ekr "github.com/csi-addons/spec/lib/go/encryptionkeyrotation"
+	"google.golang.org/grpc"
+	"google.golang.org/grpc/codes"
+	"google.golang.org/grpc/status"
+)
+
+type EncryptionKeyRotationServer struct {
+	*ekr.UnimplementedEncryptionKeyRotationControllerServer
+	volLock *util.VolumeLocks
+}
+
+func NewEncryptionKeyRotationServer(volLock *util.VolumeLocks) *EncryptionKeyRotationServer {
+	return &EncryptionKeyRotationServer{volLock: volLock}
+}
+
+func (ekrs *EncryptionKeyRotationServer) RegisterService(svc grpc.ServiceRegistrar) {
+	ekr.RegisterEncryptionKeyRotationControllerServer(svc, ekrs)
+}
+
+func (ekrs *EncryptionKeyRotationServer) EncryptionKeyRotate(
+	ctx context.Context,
+	req *ekr.EncryptionKeyRotateRequest,
+) (*ekr.EncryptionKeyRotateResponse, error) {
+	// Get the volume ID from the request
+	volID := req.GetVolumeId()
+	if volID == "" {
+		return nil, status.Error(codes.InvalidArgument, "empty volume ID in request")
+	}
+
+	// Block key rotation for RWX/ROX volumes
+	_, isMultiNode := csicommon.IsBlockMultiNode([]*csi.VolumeCapability{req.GetVolumeCapability()})
+	if isMultiNode {
+		return nil, status.Error(codes.Unimplemented, "multi-node key rotation is not supported")
+	}
+
+	if acquired := ekrs.volLock.TryAcquire(volID); !acquired {
+		return nil, status.Errorf(codes.Aborted, util.VolumeOperationAlreadyExistsFmt, volID)
+	}
+	defer ekrs.volLock.Release(volID)
+
+	// Get the credentials required to authenticate
+	// against a ceph cluster
+	creds, err := util.NewUserCredentials(req.GetSecrets())
+	if err != nil {
+		return nil, status.Error(codes.Internal, err.Error())
+	}
+	defer creds.DeleteCredentials()
+
+	rbdVol, err := rbd.GenVolFromVolID(ctx, volID, creds, req.GetSecrets())
+	if err != nil {
+		switch {
+		case errors.Is(err, rbd.ErrImageNotFound):
+			err = status.Errorf(codes.NotFound, "volume ID %s not found", volID)
+		case errors.Is(err, util.ErrPoolNotFound):
+			log.ErrorLog(ctx, "failed to get backend volume for %s: %v", volID, err)
+			err = status.Errorf(codes.NotFound, err.Error())
+		default:
+			err = status.Errorf(codes.Internal, err.Error())
+		}
+
+		return nil, err
+	}
+	defer rbdVol.Destroy(ctx)
+
+	err = rbdVol.RotateEncryptionKey(ctx)
+	if err != nil {
+		return nil, status.Errorf(
+			codes.Internal, "failed to rotate the key for volume with ID %q: %s", volID, err.Error())
+	}
+
+	// Success
+	return &ekr.EncryptionKeyRotateResponse{}, nil
+}
diff --git a/internal/csi-addons/rbd/identity.go b/internal/csi-addons/rbd/identity.go
index 65dc39afe61..749d708bd0e 100644
--- a/internal/csi-addons/rbd/identity.go
+++ b/internal/csi-addons/rbd/identity.go
@@ -133,6 +133,13 @@ func (is *IdentityServer) GetCapabilities(
 						Type: identity.Capability_ReclaimSpace_ONLINE,
 					},
 				},
+			},
+			&identity.Capability{
+				Type: &identity.Capability_EncryptionKeyRotation_{
+					EncryptionKeyRotation: &identity.Capability_EncryptionKeyRotation{
+						Type: identity.Capability_EncryptionKeyRotation_ENCRYPTIONKEYROTATION,
+					},
+				},
 			})
 	}
 
diff --git a/internal/kms/vault.go b/internal/kms/vault.go
index 8735d93c88d..66da402ccbd 100644
--- a/internal/kms/vault.go
+++ b/internal/kms/vault.go
@@ -28,6 +28,8 @@ import (
 	"github.com/hashicorp/vault/api"
 	loss "github.com/libopenstorage/secrets"
 	"github.com/libopenstorage/secrets/vault"
+
+	"github.com/ceph/ceph-csi/internal/util/file"
 )
 
 const (
@@ -269,10 +271,12 @@ func (vc *vaultConnection) initCertificates(config map[string]interface{}, secre
 			return fmt.Errorf("missing vault CA in secret %s", vaultCAFromSecret)
 		}
 
-		vaultConfig[api.EnvVaultCACert], err = createTempFile("vault-ca-cert", []byte(caPEM))
+		tf, err := file.CreateTempFile("vault-ca-cert", caPEM)
 		if err != nil {
 			return fmt.Errorf("failed to create temporary file for Vault CA: %w", err)
 		}
+		vaultConfig[api.EnvVaultCACert] = tf.Name()
+
 		// update the existing config
 		for key, value := range vaultConfig {
 			vc.vaultConfig[key] = value
@@ -480,31 +484,3 @@ func detectAuthMountPath(path string) (string, error) {
 
 	return authMountPath, nil
 }
-
-// createTempFile writes data to a temporary file that contains the pattern in
-// the filename (see os.CreateTemp for details).
-func createTempFile(pattern string, data []byte) (string, error) {
-	t, err := os.CreateTemp("", pattern)
-	if err != nil {
-		return "", fmt.Errorf("failed to create temporary file: %w", err)
-	}
-
-	// delete the tmpfile on error
-	defer func() {
-		if err != nil {
-			// ignore error on failure to remove tmpfile (gosec complains)
-			_ = os.Remove(t.Name())
-		}
-	}()
-
-	s, err := t.Write(data)
-	if err != nil || s != len(data) {
-		return "", fmt.Errorf("failed to write temporary file: %w", err)
-	}
-	err = t.Close()
-	if err != nil {
-		return "", fmt.Errorf("failed to close temporary file: %w", err)
-	}
-
-	return t.Name(), nil
-}
diff --git a/internal/kms/vault_test.go b/internal/kms/vault_test.go
index 6a059d81564..b1408b3b3b2 100644
--- a/internal/kms/vault_test.go
+++ b/internal/kms/vault_test.go
@@ -18,7 +18,6 @@ package kms
 
 import (
 	"errors"
-	"os"
 	"testing"
 
 	loss "github.com/libopenstorage/secrets"
@@ -44,23 +43,6 @@ func TestDetectAuthMountPath(t *testing.T) {
 	}
 }
 
-func TestCreateTempFile(t *testing.T) {
-	t.Parallel()
-	data := []byte("Hello World!")
-	tmpfile, err := createTempFile("my-file", data)
-	if err != nil {
-		t.Errorf("createTempFile() failed: %s", err)
-	}
-	if tmpfile == "" {
-		t.Errorf("createTempFile() returned an empty filename")
-	}
-
-	err = os.Remove(tmpfile)
-	if err != nil {
-		t.Errorf("failed to remove tmpfile (%s): %s", tmpfile, err)
-	}
-}
-
 func TestSetConfigString(t *testing.T) {
 	t.Parallel()
 	const defaultValue = "default-value"
diff --git a/internal/kms/vault_tokens.go b/internal/kms/vault_tokens.go
index e6a2d305651..c0fcc431ef9 100644
--- a/internal/kms/vault_tokens.go
+++ b/internal/kms/vault_tokens.go
@@ -24,6 +24,7 @@ import (
 	"os"
 	"strconv"
 
+	"github.com/ceph/ceph-csi/internal/util/file"
 	"github.com/ceph/ceph-csi/internal/util/k8s"
 
 	"github.com/hashicorp/vault/api"
@@ -378,10 +379,11 @@ func (vtc *vaultTenantConnection) initCertificates(config map[string]interface{}
 				return fmt.Errorf("failed to get CA certificate from secret %s: %w", vaultCAFromSecret, cErr)
 			}
 		}
-		vaultConfig[api.EnvVaultCACert], err = createTempFile("vault-ca-cert", []byte(cert))
-		if err != nil {
-			return fmt.Errorf("failed to create temporary file for Vault CA: %w", err)
+		cer, ferr := file.CreateTempFile("vault-ca-cert", cert)
+		if ferr != nil {
+			return fmt.Errorf("failed to create temporary file for Vault CA: %w", ferr)
 		}
+		vaultConfig[api.EnvVaultCACert] = cer.Name()
 	}
 
 	vaultClientCertFromSecret := "" // optional
@@ -403,10 +405,11 @@ func (vtc *vaultTenantConnection) initCertificates(config map[string]interface{}
 				return fmt.Errorf("failed to get client certificate from secret %s: %w", vaultCAFromSecret, cErr)
 			}
 		}
-		vaultConfig[api.EnvVaultClientCert], err = createTempFile("vault-ca-cert", []byte(cert))
-		if err != nil {
-			return fmt.Errorf("failed to create temporary file for Vault client certificate: %w", err)
+		cer, ferr := file.CreateTempFile("vault-ca-cert", cert)
+		if ferr != nil {
+			return fmt.Errorf("failed to create temporary file for Vault client certificate: %w", ferr)
 		}
+		vaultConfig[api.EnvVaultClientCert] = cer.Name()
 	}
 
 	vaultClientCertKeyFromSecret := "" // optional
@@ -432,10 +435,11 @@ func (vtc *vaultTenantConnection) initCertificates(config map[string]interface{}
 				return fmt.Errorf("failed to get client certificate key from secret %s: %w", vaultCAFromSecret, err)
 			}
 		}
-		vaultConfig[api.EnvVaultClientKey], err = createTempFile("vault-client-cert-key", []byte(certKey))
+		ckey, err := file.CreateTempFile("vault-client-cert-key", certKey)
 		if err != nil {
 			return fmt.Errorf("failed to create temporary file for Vault client cert key: %w", err)
 		}
+		vaultConfig[api.EnvVaultClientKey] = ckey.Name()
 	}
 
 	for key, value := range vaultConfig {
diff --git a/internal/rbd/driver/driver.go b/internal/rbd/driver/driver.go
index a8bea2e15a8..e682884286a 100644
--- a/internal/rbd/driver/driver.go
+++ b/internal/rbd/driver/driver.go
@@ -229,6 +229,9 @@ func (r *Driver) setupCSIAddonsServer(conf *util.Config) error {
 	if conf.IsNodeServer {
 		rs := casrbd.NewReclaimSpaceNodeServer()
 		r.cas.RegisterService(rs)
+
+		ekr := casrbd.NewEncryptionKeyRotationServer(r.ns.VolumeLocks)
+		r.cas.RegisterService(ekr)
 	}
 
 	// start the server, this does not block, it runs a new go-routine
diff --git a/internal/rbd/encryption.go b/internal/rbd/encryption.go
index 015c1e5642f..9cb87cb5dbd 100644
--- a/internal/rbd/encryption.go
+++ b/internal/rbd/encryption.go
@@ -63,6 +63,10 @@ const (
 	// user did not specify an "encryptionType", but set
 	// "encryption": true.
 	rbdDefaultEncryptionType = util.EncryptionTypeBlock
+
+	// Luks slots.
+	luksSlot0 = "0"
+	luksSlot1 = "1"
 )
 
 // checkRbdImageEncrypted verifies if rbd image was encrypted when created.
@@ -437,3 +441,72 @@ func (ri *rbdImage) RemoveDEK(ctx context.Context, volumeID string) error {
 
 	return nil
 }
+
+// GetEncryptionPassphraseSize returns the value of `encryptionPassphraseSize`.
+func GetEncryptionPassphraseSize() int {
+	return encryptionPassphraseSize
+}
+
+// RotateEncryptionKey processes the key rotation for the RBD Volume.
+func (rv *rbdVolume) RotateEncryptionKey(ctx context.Context) error {
+	if !rv.isBlockEncrypted() {
+		return errors.New("key rotation unsupported for non block encrypted device")
+	}
+
+	// Verify that the underlying device has been setup for encryption
+	currState, err := rv.checkRbdImageEncrypted(ctx)
+	if err != nil {
+		return fmt.Errorf("failed to check encryption state: %w", err)
+	}
+
+	if currState != rbdImageEncrypted {
+		return errors.New("key rotation not supported for unencrypted device")
+	}
+
+	// Get the device path for the underlying image
+	useNbd := rv.Mounter == rbdNbdMounter && hasNBD
+	devicePath, found := waitForPath(ctx, rv.Pool, rv.RadosNamespace, rv.RbdImageName, 1, useNbd)
+	if !found {
+		return fmt.Errorf("failed to get the device path for %q: %w", rv, err)
+	}
+
+	// Step 1: Get the current passphrase
+	oldPassphrase, err := rv.blockEncryption.GetCryptoPassphrase(ctx, rv.VolID)
+	if err != nil {
+		return fmt.Errorf("failed to fetch the current passphrase for %q: %w", rv, err)
+	}
+
+	// Step 2: Add current key to slot 1
+	err = util.LuksAddKey(devicePath, oldPassphrase, oldPassphrase, luksSlot1)
+	if err != nil {
+		return fmt.Errorf("failed to add curr key to luksSlot1: %w", err)
+	}
+
+	// Step 3: Generate new key and add it to slot 0
+	newPassphrase, err := rv.blockEncryption.GetNewCryptoPassphrase(
+		GetEncryptionPassphraseSize())
+	if err != nil {
+		return fmt.Errorf("failed to generate a new passphrase: %w", err)
+	}
+
+	err = util.LuksAddKey(devicePath, oldPassphrase, newPassphrase, luksSlot0)
+	if err != nil {
+		return fmt.Errorf("failed to add the new key to luksSlot0: %w", err)
+	}
+
+	// Step 4: Add the new key to KMS
+	err = rv.blockEncryption.StoreCryptoPassphrase(ctx, rv.VolID, newPassphrase)
+	if err != nil {
+		return fmt.Errorf("failed to update the new key into the KMS: %w", err)
+	}
+
+	// Step 5: Remove the old key from slot 1
+	// We use the newPassphrase to authenticate LUKS here
+	err = util.LuksRemoveKey(devicePath, newPassphrase, luksSlot1)
+	if err != nil {
+		return fmt.Errorf("failed to remove the backup key from luksSlot1: %w", err)
+	}
+
+	// Return error accordingly.
+	return nil
+}
diff --git a/internal/util/crypto.go b/internal/util/crypto.go
index 989788cc8b5..995c8282f6d 100644
--- a/internal/util/crypto.go
+++ b/internal/util/crypto.go
@@ -237,6 +237,11 @@ func (ve *VolumeEncryption) GetCryptoPassphrase(ctx context.Context, volumeID st
 	return ve.KMS.DecryptDEK(ctx, volumeID, passphrase)
 }
 
+// GetNewCryptoPassphrase returns a random passphrase of given length.
+func (ve *VolumeEncryption) GetNewCryptoPassphrase(length int) (string, error) {
+	return generateNewEncryptionPassphrase(length)
+}
+
 // generateNewEncryptionPassphrase generates a random passphrase for encryption.
 func generateNewEncryptionPassphrase(length int) (string, error) {
 	bytesPassphrase := make([]byte, length)
diff --git a/internal/util/cryptsetup.go b/internal/util/cryptsetup.go
index e5669b42533..06e2028f3f6 100644
--- a/internal/util/cryptsetup.go
+++ b/internal/util/cryptsetup.go
@@ -19,9 +19,13 @@ package util
 import (
 	"bytes"
 	"fmt"
+	"os"
 	"os/exec"
 	"strconv"
 	"strings"
+
+	"github.com/ceph/ceph-csi/internal/util/file"
+	"github.com/ceph/ceph-csi/internal/util/log"
 )
 
 // Limit memory used by Argon2i PBKDF to 32 MiB.
@@ -66,6 +70,135 @@ func LuksStatus(mapperFile string) (string, string, error) {
 	return execCryptsetupCommand(nil, "status", mapperFile)
 }
 
+// LuksAddKey adds a new key to the specified slot.
+func LuksAddKey(devicePath, passphrase, newPassphrase, slot string) error {
+	passFile, err := file.CreateTempFile("luks-", passphrase)
+	if err != nil {
+		return err
+	}
+	defer os.Remove(passFile.Name())
+
+	newPassFile, err := file.CreateTempFile("luks-", newPassphrase)
+	if err != nil {
+		return err
+	}
+	defer os.Remove(newPassFile.Name())
+
+	_, stderr, err := execCryptsetupCommand(
+		nil,
+		"--verbose",
+		"--key-file="+passFile.Name(),
+		"--key-slot="+slot,
+		"luksAddKey",
+		devicePath,
+		newPassFile.Name(),
+	)
+
+	// Return early if no error to save us some time
+	if err == nil {
+		return nil
+	}
+
+	// Possible scenarios
+	// 1. The provided passphrase to unlock the disk is wrong
+	// 2. The key slot is already in use
+	// 	  If so, check if the key we want to add to the slot is already there
+	//    If not, remove it and then add the new key to the slot
+	if strings.Contains(stderr, fmt.Sprintf("Key slot %s is full", slot)) {
+		// The given slot already has a key
+		// Check if it is the one that we want to update with
+		exists, fErr := LuksVerifyKey(devicePath, newPassphrase, slot)
+		if fErr != nil {
+			return fErr
+		}
+
+		// Verification passed, return early
+		if exists {
+			return nil
+		}
+
+		// Else, we remove the key from the given slot and add the new one
+		// Note: we use existing passphrase here as we are not yet sure if
+		// the newPassphrase is present in the headers
+		fErr = LuksRemoveKey(devicePath, passphrase, slot)
+		if fErr != nil {
+			return fErr
+		}
+
+		// Now the slot is free, add the new key to it
+		fErr = LuksAddKey(devicePath, passphrase, newPassphrase, slot)
+		if fErr != nil {
+			return fErr
+		}
+
+		// No errors, we good.
+		return nil
+	}
+
+	// The existing passphrase is wrong and the slot is empty
+	return err
+}
+
+// LuksRemoveKey removes the key by killing the specified slot.
+func LuksRemoveKey(devicePath, passphrase, slot string) error {
+	keyFile, err := file.CreateTempFile("luks-", passphrase)
+	if err != nil {
+		return err
+	}
+	defer os.Remove(keyFile.Name())
+
+	_, stderr, err := execCryptsetupCommand(
+		nil,
+		"--verbose",
+		"--key-file="+keyFile.Name(),
+		"luksKillSlot",
+		devicePath,
+		slot,
+	)
+	if err != nil {
+		// If a slot is not active, don't treat that as an error
+		if !strings.Contains(stderr, fmt.Sprintf("Keyslot %s is not active.", slot)) {
+			return fmt.Errorf("failed to kill slot %s for device %s: %w", slot, devicePath, err)
+		}
+	}
+
+	return nil
+}
+
+// LuksVerifyKey verifies that a key exists in a given slot.
+func LuksVerifyKey(devicePath, passphrase, slot string) (bool, error) {
+	// Create a temp file that we will use to open the device
+	keyFile, err := file.CreateTempFile("luks-", passphrase)
+	if err != nil {
+		return false, err
+	}
+	defer os.Remove(keyFile.Name())
+
+	_, stderr, err := execCryptsetupCommand(
+		nil,
+		"--verbose",
+		"--key-file="+keyFile.Name(),
+		"--key-slot="+slot,
+		"luksChangeKey",
+		devicePath,
+		keyFile.Name(),
+	)
+	if err != nil {
+		// If the passphrase doesn't match the key in given slot
+		if strings.Contains(stderr, "No key available with this passphrase.") {
+			// No match, no error
+			return false, nil
+		}
+
+		// Otherwise it was something else, return the wrapped error
+		log.ErrorLogMsg("failed to verify key in slot %s. stderr: %s. err: %v", slot, stderr, err)
+
+		return false, fmt.Errorf("failed to verify key in slot %s for device %s: %w", slot, devicePath, err)
+	}
+
+	return true, nil
+}
+
 func execCryptsetupCommand(stdin *string, args ...string) (string, string, error) {
 	var (
 		program       = "cryptsetup"
diff --git a/internal/util/file/file.go b/internal/util/file/file.go
new file mode 100644
index 00000000000..2f32b882207
--- /dev/null
+++ b/internal/util/file/file.go
@@ -0,0 +1,54 @@
+/*
+Copyright 2024 The Ceph-CSI Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package file
+
+import (
+	"fmt"
+	"os"
+)
+
+// CreateTempFile create a temporary file with the given string
+// content and returns the reference to the file.
+// The caller is responsible for disposing the file.
+func CreateTempFile(prefix, contents string) (*os.File, error) {
+	// Create a temp file
+	file, err := os.CreateTemp("", prefix)
+	if err != nil {
+		return nil, fmt.Errorf("failed to create temporary file: %w", err)
+	}
+
+	// In case of error, remove the file if it was created
+	defer func() {
+		if err != nil {
+			_ = os.Remove(file.Name())
+		}
+	}()
+
+	// Write the contents
+	var c int
+	c, err = file.WriteString(contents)
+	if err != nil || c != len(contents) {
+		return nil, fmt.Errorf("failed to write temporary file: %w", err)
+	}
+
+	// Close the handle
+	if err = file.Close(); err != nil {
+		return nil, fmt.Errorf("failed to close temporary file: %w", err)
+	}
+
+	return file, nil
+}
diff --git a/internal/util/file/file_test.go b/internal/util/file/file_test.go
new file mode 100644
index 00000000000..9daf142f402
--- /dev/null
+++ b/internal/util/file/file_test.go
@@ -0,0 +1,100 @@
+/*
+Copyright 2024 The Ceph-CSI Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package file
+
+import (
+	"os"
+	"testing"
+)
+
+func TestCreateTempFile_WithValidContent(t *testing.T) {
+	t.Parallel()
+
+	content := "Valid Content"
+
+	file, err := CreateTempFile("test-", content)
+	if err != nil {
+		t.Fatalf("Unexpected error: %v", err)
+	}
+	defer func() {
+		err = os.Remove(file.Name())
+		if err != nil {
+			t.Fatalf("Unexpected error: %v", err)
+		}
+	}()
+
+	readContent, err := os.ReadFile(file.Name())
+	if err != nil {
+		t.Fatalf("Unexpected error: %v", err)
+	}
+
+	if string(readContent) != content {
+		t.Fatalf("Content mismatch: got %v, want %v", string(readContent), content)
+	}
+}
+
+func TestCreateTempFile_WithEmptyContent(t *testing.T) {
+	t.Parallel()
+
+	content := ""
+
+	file, err := CreateTempFile("test-", content)
+	if err != nil {
+		t.Fatalf("Unexpected error: %v", err)
+	}
+	defer func() {
+		err = os.Remove(file.Name())
+		if err != nil {
+			t.Fatalf("Unexpected error: %v", err)
+		}
+	}()
+
+	readContent, err := os.ReadFile(file.Name())
+	if err != nil {
+		t.Fatalf("Unexpected error: %v", err)
+	}
+
+	if string(readContent) != content {
+		t.Fatalf("Content mismatch: got %v, want %v", string(readContent), content)
+	}
+}
+
+func TestCreateTempFile_WithLargeContent(t *testing.T) {
+	t.Parallel()
+
+	content := string(make([]byte, 1<<20))
+
+	file, err := CreateTempFile("test-", content)
+	if err != nil {
+		t.Fatalf("Unexpected error: %v", err)
+	}
+	defer func() {
+		err = os.Remove(file.Name())
+		if err != nil {
+			t.Fatalf("Unexpected error: %v", err)
+		}
+	}()
+
+	readContent, err := os.ReadFile(file.Name())
+	if err != nil {
+		t.Fatalf("Unexpected error: %v", err)
+	}
+
+	if string(readContent) != content {
+		t.Fatalf("Content mismatch: got %v, want %v", string(readContent), content)
+	}
+}
diff --git a/vendor/github.com/csi-addons/spec/lib/go/encryptionkeyrotation/encryptionkeyrotation.pb.go b/vendor/github.com/csi-addons/spec/lib/go/encryptionkeyrotation/encryptionkeyrotation.pb.go
new file mode 100644
index 00000000000..d060d4a7544
--- /dev/null
+++ b/vendor/github.com/csi-addons/spec/lib/go/encryptionkeyrotation/encryptionkeyrotation.pb.go
@@ -0,0 +1,317 @@
+// Code generated by make; DO NOT EDIT.
+
+// Code generated by protoc-gen-go. DO NOT EDIT.
+// versions:
+// 	protoc-gen-go v1.28.1
+// 	protoc        v3.20.2
+// source: encryptionkeyrotation/encryptionkeyrotation.proto
+
+package encryptionkeyrotation
+
+import (
+	csi "github.com/container-storage-interface/spec/lib/go/csi"
+	protoreflect "google.golang.org/protobuf/reflect/protoreflect"
+	protoimpl "google.golang.org/protobuf/runtime/protoimpl"
+	_ "google.golang.org/protobuf/types/descriptorpb"
+	reflect "reflect"
+	sync "sync"
+)
+
+const (
+	// Verify that this generated code is sufficiently up-to-date.
+	_ = protoimpl.EnforceVersion(20 - protoimpl.MinVersion)
+	// Verify that runtime/protoimpl is sufficiently up-to-date.
+	_ = protoimpl.EnforceVersion(protoimpl.MaxVersion - 20)
+)
+
+// EncryptionKeyRotateRequest contains the information needed to identify
+// the volume by the SP and access any backend services so that the key can be
+// rotated.
+type EncryptionKeyRotateRequest struct {
+	state         protoimpl.MessageState
+	sizeCache     protoimpl.SizeCache
+	unknownFields protoimpl.UnknownFields
+
+	// The ID of the volume for which the key is to be rotated.
+	// This field is required
+	VolumeId string `protobuf:"bytes,1,opt,name=volume_id,json=volumeId,proto3" json:"volume_id,omitempty"`
+	// The path where the volume is available.
+	// This field is OPTIONAL
+	// Useful if you are implementing the RPC on CSI Driver NodePlugin
+	VolumePath string `protobuf:"bytes,2,opt,name=volume_path,json=volumePath,proto3" json:"volume_path,omitempty"`
+	// Provide the encryption key to be set
+	// This field is OPTIONAL
+	EncryptionKey string `protobuf:"bytes,3,opt,name=encryption_key,json=encryptionKey,proto3" json:"encryption_key,omitempty"`
+	// Plugin specific parameters passed in as opaque key-value pairs.
+	Parameters map[string]string `protobuf:"bytes,4,rep,name=parameters,proto3" json:"parameters,omitempty" protobuf_key:"bytes,1,opt,name=key,proto3" protobuf_val:"bytes,2,opt,name=value,proto3"`
+	// Secrets required by the plugin to complete the request.
+	Secrets map[string]string `protobuf:"bytes,5,rep,name=secrets,proto3" json:"secrets,omitempty" protobuf_key:"bytes,1,opt,name=key,proto3" protobuf_val:"bytes,2,opt,name=value,proto3"`
+	// Volume capability describing how the CO intends to use this volume.
+	// This allows SP to determine if volume is being used as a block
+	// device or mounted file system. This is OPTIONAL.
+	VolumeCapability *csi.VolumeCapability `protobuf:"bytes,6,opt,name=volume_capability,json=volumeCapability,proto3" json:"volume_capability,omitempty"`
+}
+
+func (x *EncryptionKeyRotateRequest) Reset() {
+	*x = EncryptionKeyRotateRequest{}
+	if protoimpl.UnsafeEnabled {
+		mi := &file_encryptionkeyrotation_encryptionkeyrotation_proto_msgTypes[0]
+		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+		ms.StoreMessageInfo(mi)
+	}
+}
+
+func (x *EncryptionKeyRotateRequest) String() string {
+	return protoimpl.X.MessageStringOf(x)
+}
+
+func (*EncryptionKeyRotateRequest) ProtoMessage() {}
+
+func (x *EncryptionKeyRotateRequest) ProtoReflect() protoreflect.Message {
+	mi := &file_encryptionkeyrotation_encryptionkeyrotation_proto_msgTypes[0]
+	if protoimpl.UnsafeEnabled && x != nil {
+		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+		if ms.LoadMessageInfo() == nil {
+			ms.StoreMessageInfo(mi)
+		}
+		return ms
+	}
+	return mi.MessageOf(x)
+}
+
+// Deprecated: Use EncryptionKeyRotateRequest.ProtoReflect.Descriptor instead.
+func (*EncryptionKeyRotateRequest) Descriptor() ([]byte, []int) {
+	return file_encryptionkeyrotation_encryptionkeyrotation_proto_rawDescGZIP(), []int{0}
+}
+
+func (x *EncryptionKeyRotateRequest) GetVolumeId() string {
+	if x != nil {
+		return x.VolumeId
+	}
+	return ""
+}
+
+func (x *EncryptionKeyRotateRequest) GetVolumePath() string {
+	if x != nil {
+		return x.VolumePath
+	}
+	return ""
+}
+
+func (x *EncryptionKeyRotateRequest) GetEncryptionKey() string {
+	if x != nil {
+		return x.EncryptionKey
+	}
+	return ""
+}
+
+func (x *EncryptionKeyRotateRequest) GetParameters() map[string]string {
+	if x != nil {
+		return x.Parameters
+	}
+	return nil
+}
+
+func (x *EncryptionKeyRotateRequest) GetSecrets() map[string]string {
+	if x != nil {
+		return x.Secrets
+	}
+	return nil
+}
+
+func (x *EncryptionKeyRotateRequest) GetVolumeCapability() *csi.VolumeCapability {
+	if x != nil {
+		return x.VolumeCapability
+	}
+	return nil
+}
+
+// EncryptionKeyRotateResponse holds the information about the result of the
+// EncryptionKeyRotateRequest call.
+type EncryptionKeyRotateResponse struct {
+	state         protoimpl.MessageState
+	sizeCache     protoimpl.SizeCache
+	unknownFields protoimpl.UnknownFields
+}
+
+func (x *EncryptionKeyRotateResponse) Reset() {
+	*x = EncryptionKeyRotateResponse{}
+	if protoimpl.UnsafeEnabled {
+		mi := &file_encryptionkeyrotation_encryptionkeyrotation_proto_msgTypes[1]
+		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+		ms.StoreMessageInfo(mi)
+	}
+}
+
+func (x *EncryptionKeyRotateResponse) String() string {
+	return protoimpl.X.MessageStringOf(x)
+}
+
+func (*EncryptionKeyRotateResponse) ProtoMessage() {}
+
+func (x *EncryptionKeyRotateResponse) ProtoReflect() protoreflect.Message {
+	mi := &file_encryptionkeyrotation_encryptionkeyrotation_proto_msgTypes[1]
+	if protoimpl.UnsafeEnabled && x != nil {
+		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+		if ms.LoadMessageInfo() == nil {
+			ms.StoreMessageInfo(mi)
+		}
+		return ms
+	}
+	return mi.MessageOf(x)
+}
+
+// Deprecated: Use EncryptionKeyRotateResponse.ProtoReflect.Descriptor instead.
+func (*EncryptionKeyRotateResponse) Descriptor() ([]byte, []int) {
+	return file_encryptionkeyrotation_encryptionkeyrotation_proto_rawDescGZIP(), []int{1}
+}
+
+var File_encryptionkeyrotation_encryptionkeyrotation_proto protoreflect.FileDescriptor
+
+var file_encryptionkeyrotation_encryptionkeyrotation_proto_rawDesc = []byte{
+	0x0a, 0x31, 0x65, 0x6e, 0x63, 0x72, 0x79, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x6b, 0x65, 0x79, 0x72,
+	0x6f, 0x74, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x2f, 0x65, 0x6e, 0x63, 0x72, 0x79, 0x70, 0x74, 0x69,
+	0x6f, 0x6e, 0x6b, 0x65, 0x79, 0x72, 0x6f, 0x74, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x2e, 0x70, 0x72,
+	0x6f, 0x74, 0x6f, 0x12, 0x15, 0x65, 0x6e, 0x63, 0x72, 0x79, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x6b,
+	0x65, 0x79, 0x72, 0x6f, 0x74, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x1a, 0x40, 0x67, 0x69, 0x74, 0x68,
+	0x75, 0x62, 0x2e, 0x63, 0x6f, 0x6d, 0x2f, 0x63, 0x6f, 0x6e, 0x74, 0x61, 0x69, 0x6e, 0x65, 0x72,
+	0x2d, 0x73, 0x74, 0x6f, 0x72, 0x61, 0x67, 0x65, 0x2d, 0x69, 0x6e, 0x74, 0x65, 0x72, 0x66, 0x61,
+	0x63, 0x65, 0x2f, 0x73, 0x70, 0x65, 0x63, 0x2f, 0x6c, 0x69, 0x62, 0x2f, 0x67, 0x6f, 0x2f, 0x63,
+	0x73, 0x69, 0x2f, 0x63, 0x73, 0x69, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x1a, 0x20, 0x67, 0x6f,
+	0x6f, 0x67, 0x6c, 0x65, 0x2f, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2f, 0x64, 0x65,
+	0x73, 0x63, 0x72, 0x69, 0x70, 0x74, 0x6f, 0x72, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x22, 0x8a,
+	0x04, 0x0a, 0x1a, 0x45, 0x6e, 0x63, 0x72, 0x79, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x4b, 0x65, 0x79,
+	0x52, 0x6f, 0x74, 0x61, 0x74, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x1b, 0x0a,
+	0x09, 0x76, 0x6f, 0x6c, 0x75, 0x6d, 0x65, 0x5f, 0x69, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09,
+	0x52, 0x08, 0x76, 0x6f, 0x6c, 0x75, 0x6d, 0x65, 0x49, 0x64, 0x12, 0x1f, 0x0a, 0x0b, 0x76, 0x6f,
+	0x6c, 0x75, 0x6d, 0x65, 0x5f, 0x70, 0x61, 0x74, 0x68, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52,
+	0x0a, 0x76, 0x6f, 0x6c, 0x75, 0x6d, 0x65, 0x50, 0x61, 0x74, 0x68, 0x12, 0x2a, 0x0a, 0x0e, 0x65,
+	0x6e, 0x63, 0x72, 0x79, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x5f, 0x6b, 0x65, 0x79, 0x18, 0x03, 0x20,
+	0x01, 0x28, 0x09, 0x42, 0x03, 0x98, 0x42, 0x01, 0x52, 0x0d, 0x65, 0x6e, 0x63, 0x72, 0x79, 0x70,
+	0x74, 0x69, 0x6f, 0x6e, 0x4b, 0x65, 0x79, 0x12, 0x61, 0x0a, 0x0a, 0x70, 0x61, 0x72, 0x61, 0x6d,
+	0x65, 0x74, 0x65, 0x72, 0x73, 0x18, 0x04, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x41, 0x2e, 0x65, 0x6e,
+	0x63, 0x72, 0x79, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x6b, 0x65, 0x79, 0x72, 0x6f, 0x74, 0x61, 0x74,
+	0x69, 0x6f, 0x6e, 0x2e, 0x45, 0x6e, 0x63, 0x72, 0x79, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x4b, 0x65,
+	0x79, 0x52, 0x6f, 0x74, 0x61, 0x74, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x2e, 0x50,
+	0x61, 0x72, 0x61, 0x6d, 0x65, 0x74, 0x65, 0x72, 0x73, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x52, 0x0a,
+	0x70, 0x61, 0x72, 0x61, 0x6d, 0x65, 0x74, 0x65, 0x72, 0x73, 0x12, 0x5d, 0x0a, 0x07, 0x73, 0x65,
+	0x63, 0x72, 0x65, 0x74, 0x73, 0x18, 0x05, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x3e, 0x2e, 0x65, 0x6e,
+	0x63, 0x72, 0x79, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x6b, 0x65, 0x79, 0x72, 0x6f, 0x74, 0x61, 0x74,
+	0x69, 0x6f, 0x6e, 0x2e, 0x45, 0x6e, 0x63, 0x72, 0x79, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x4b, 0x65,
+	0x79, 0x52, 0x6f, 0x74, 0x61, 0x74, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x2e, 0x53,
+	0x65, 0x63, 0x72, 0x65, 0x74, 0x73, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x42, 0x03, 0x98, 0x42, 0x01,
+	0x52, 0x07, 0x73, 0x65, 0x63, 0x72, 0x65, 0x74, 0x73, 0x12, 0x45, 0x0a, 0x11, 0x76, 0x6f, 0x6c,
+	0x75, 0x6d, 0x65, 0x5f, 0x63, 0x61, 0x70, 0x61, 0x62, 0x69, 0x6c, 0x69, 0x74, 0x79, 0x18, 0x06,
+	0x20, 0x01, 0x28, 0x0b, 0x32, 0x18, 0x2e, 0x63, 0x73, 0x69, 0x2e, 0x76, 0x31, 0x2e, 0x56, 0x6f,
+	0x6c, 0x75, 0x6d, 0x65, 0x43, 0x61, 0x70, 0x61, 0x62, 0x69, 0x6c, 0x69, 0x74, 0x79, 0x52, 0x10,
+	0x76, 0x6f, 0x6c, 0x75, 0x6d, 0x65, 0x43, 0x61, 0x70, 0x61, 0x62, 0x69, 0x6c, 0x69, 0x74, 0x79,
+	0x1a, 0x3d, 0x0a, 0x0f, 0x50, 0x61, 0x72, 0x61, 0x6d, 0x65, 0x74, 0x65, 0x72, 0x73, 0x45, 0x6e,
+	0x74, 0x72, 0x79, 0x12, 0x10, 0x0a, 0x03, 0x6b, 0x65, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09,
+	0x52, 0x03, 0x6b, 0x65, 0x79, 0x12, 0x14, 0x0a, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x18, 0x02,
+	0x20, 0x01, 0x28, 0x09, 0x52, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x3a, 0x02, 0x38, 0x01, 0x1a,
+	0x3a, 0x0a, 0x0c, 0x53, 0x65, 0x63, 0x72, 0x65, 0x74, 0x73, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x12,
+	0x10, 0x0a, 0x03, 0x6b, 0x65, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x03, 0x6b, 0x65,
+	0x79, 0x12, 0x14, 0x0a, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09,
+	0x52, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x3a, 0x02, 0x38, 0x01, 0x22, 0x1d, 0x0a, 0x1b, 0x45,
+	0x6e, 0x63, 0x72, 0x79, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x4b, 0x65, 0x79, 0x52, 0x6f, 0x74, 0x61,
+	0x74, 0x65, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x32, 0xa1, 0x01, 0x0a, 0x1f, 0x45,
+	0x6e, 0x63, 0x72, 0x79, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x4b, 0x65, 0x79, 0x52, 0x6f, 0x74, 0x61,
+	0x74, 0x69, 0x6f, 0x6e, 0x43, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x6c, 0x65, 0x72, 0x12, 0x7e,
+	0x0a, 0x13, 0x45, 0x6e, 0x63, 0x72, 0x79, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x4b, 0x65, 0x79, 0x52,
+	0x6f, 0x74, 0x61, 0x74, 0x65, 0x12, 0x31, 0x2e, 0x65, 0x6e, 0x63, 0x72, 0x79, 0x70, 0x74, 0x69,
+	0x6f, 0x6e, 0x6b, 0x65, 0x79, 0x72, 0x6f, 0x74, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x2e, 0x45, 0x6e,
+	0x63, 0x72, 0x79, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x4b, 0x65, 0x79, 0x52, 0x6f, 0x74, 0x61, 0x74,
+	0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x32, 0x2e, 0x65, 0x6e, 0x63, 0x72, 0x79,
+	0x70, 0x74, 0x69, 0x6f, 0x6e, 0x6b, 0x65, 0x79, 0x72, 0x6f, 0x74, 0x61, 0x74, 0x69, 0x6f, 0x6e,
+	0x2e, 0x45, 0x6e, 0x63, 0x72, 0x79, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x4b, 0x65, 0x79, 0x52, 0x6f,
+	0x74, 0x61, 0x74, 0x65, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x00, 0x42, 0x39,
+	0x5a, 0x37, 0x67, 0x69, 0x74, 0x68, 0x75, 0x62, 0x2e, 0x63, 0x6f, 0x6d, 0x2f, 0x63, 0x73, 0x69,
+	0x2d, 0x61, 0x64, 0x64, 0x6f, 0x6e, 0x73, 0x2f, 0x73, 0x70, 0x65, 0x63, 0x2f, 0x6c, 0x69, 0x62,
+	0x2f, 0x67, 0x6f, 0x2f, 0x65, 0x6e, 0x63, 0x72, 0x79, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x6b, 0x65,
+	0x79, 0x72, 0x6f, 0x74, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f,
+	0x33,
+}
+
+var (
+	file_encryptionkeyrotation_encryptionkeyrotation_proto_rawDescOnce sync.Once
+	file_encryptionkeyrotation_encryptionkeyrotation_proto_rawDescData = file_encryptionkeyrotation_encryptionkeyrotation_proto_rawDesc
+)
+
+func file_encryptionkeyrotation_encryptionkeyrotation_proto_rawDescGZIP() []byte {
+	file_encryptionkeyrotation_encryptionkeyrotation_proto_rawDescOnce.Do(func() {
+		file_encryptionkeyrotation_encryptionkeyrotation_proto_rawDescData = protoimpl.X.CompressGZIP(file_encryptionkeyrotation_encryptionkeyrotation_proto_rawDescData)
+	})
+	return file_encryptionkeyrotation_encryptionkeyrotation_proto_rawDescData
+}
+
+var file_encryptionkeyrotation_encryptionkeyrotation_proto_msgTypes = make([]protoimpl.MessageInfo, 4)
+var file_encryptionkeyrotation_encryptionkeyrotation_proto_goTypes = []interface{}{
+	(*EncryptionKeyRotateRequest)(nil),  // 0: encryptionkeyrotation.EncryptionKeyRotateRequest
+	(*EncryptionKeyRotateResponse)(nil), // 1: encryptionkeyrotation.EncryptionKeyRotateResponse
+	nil,                                 // 2: encryptionkeyrotation.EncryptionKeyRotateRequest.ParametersEntry
+	nil,                                 // 3: encryptionkeyrotation.EncryptionKeyRotateRequest.SecretsEntry
+	(*csi.VolumeCapability)(nil),        // 4: csi.v1.VolumeCapability
+}
+var file_encryptionkeyrotation_encryptionkeyrotation_proto_depIdxs = []int32{
+	2, // 0: encryptionkeyrotation.EncryptionKeyRotateRequest.parameters:type_name -> encryptionkeyrotation.EncryptionKeyRotateRequest.ParametersEntry
+	3, // 1: encryptionkeyrotation.EncryptionKeyRotateRequest.secrets:type_name -> encryptionkeyrotation.EncryptionKeyRotateRequest.SecretsEntry
+	4, // 2: encryptionkeyrotation.EncryptionKeyRotateRequest.volume_capability:type_name -> csi.v1.VolumeCapability
+	0, // 3: encryptionkeyrotation.EncryptionKeyRotationController.EncryptionKeyRotate:input_type -> encryptionkeyrotation.EncryptionKeyRotateRequest
+	1, // 4: encryptionkeyrotation.EncryptionKeyRotationController.EncryptionKeyRotate:output_type -> encryptionkeyrotation.EncryptionKeyRotateResponse
+	4, // [4:5] is the sub-list for method output_type
+	3, // [3:4] is the sub-list for method input_type
+	3, // [3:3] is the sub-list for extension type_name
+	3, // [3:3] is the sub-list for extension extendee
+	0, // [0:3] is the sub-list for field type_name
+}
+
+func init() { file_encryptionkeyrotation_encryptionkeyrotation_proto_init() }
+func file_encryptionkeyrotation_encryptionkeyrotation_proto_init() {
+	if File_encryptionkeyrotation_encryptionkeyrotation_proto != nil {
+		return
+	}
+	if !protoimpl.UnsafeEnabled {
+		file_encryptionkeyrotation_encryptionkeyrotation_proto_msgTypes[0].Exporter = func(v interface{}, i int) interface{} {
+			switch v := v.(*EncryptionKeyRotateRequest); i {
+			case 0:
+				return &v.state
+			case 1:
+				return &v.sizeCache
+			case 2:
+				return &v.unknownFields
+			default:
+				return nil
+			}
+		}
+		file_encryptionkeyrotation_encryptionkeyrotation_proto_msgTypes[1].Exporter = func(v interface{}, i int) interface{} {
+			switch v := v.(*EncryptionKeyRotateResponse); i {
+			case 0:
+				return &v.state
+			case 1:
+				return &v.sizeCache
+			case 2:
+				return &v.unknownFields
+			default:
+				return nil
+			}
+		}
+	}
+	type x struct{}
+	out := protoimpl.TypeBuilder{
+		File: protoimpl.DescBuilder{
+			GoPackagePath: reflect.TypeOf(x{}).PkgPath(),
+			RawDescriptor: file_encryptionkeyrotation_encryptionkeyrotation_proto_rawDesc,
+			NumEnums:      0,
+			NumMessages:   4,
+			NumExtensions: 0,
+			NumServices:   1,
+		},
+		GoTypes:           file_encryptionkeyrotation_encryptionkeyrotation_proto_goTypes,
+		DependencyIndexes: file_encryptionkeyrotation_encryptionkeyrotation_proto_depIdxs,
+		MessageInfos:      file_encryptionkeyrotation_encryptionkeyrotation_proto_msgTypes,
+	}.Build()
+	File_encryptionkeyrotation_encryptionkeyrotation_proto = out.File
+	file_encryptionkeyrotation_encryptionkeyrotation_proto_rawDesc = nil
+	file_encryptionkeyrotation_encryptionkeyrotation_proto_goTypes = nil
+	file_encryptionkeyrotation_encryptionkeyrotation_proto_depIdxs = nil
+}
diff --git a/vendor/github.com/csi-addons/spec/lib/go/encryptionkeyrotation/encryptionkeyrotation_grpc.pb.go b/vendor/github.com/csi-addons/spec/lib/go/encryptionkeyrotation/encryptionkeyrotation_grpc.pb.go
new file mode 100644
index 00000000000..63f2afe1b10
--- /dev/null
+++ b/vendor/github.com/csi-addons/spec/lib/go/encryptionkeyrotation/encryptionkeyrotation_grpc.pb.go
@@ -0,0 +1,116 @@
+// Code generated by make; DO NOT EDIT.
+
+// Code generated by protoc-gen-go-grpc. DO NOT EDIT.
+// versions:
+// - protoc-gen-go-grpc v1.3.0
+// - protoc             v3.20.2
+// source: encryptionkeyrotation/encryptionkeyrotation.proto
+
+package encryptionkeyrotation
+
+import (
+	context "context"
+	grpc "google.golang.org/grpc"
+	codes "google.golang.org/grpc/codes"
+	status "google.golang.org/grpc/status"
+)
+
+// This is a compile-time assertion to ensure that this generated file
+// is compatible with the grpc package it is being compiled against.
+// Requires gRPC-Go v1.32.0 or later.
+const _ = grpc.SupportPackageIsVersion7
+
+const (
+	EncryptionKeyRotationController_EncryptionKeyRotate_FullMethodName = "/encryptionkeyrotation.EncryptionKeyRotationController/EncryptionKeyRotate"
+)
+
+// EncryptionKeyRotationControllerClient is the client API for EncryptionKeyRotationController service.
+//
+// For semantics around ctx use and closing/ending streaming RPCs, please refer to https://pkg.go.dev/google.golang.org/grpc/?tab=doc#ClientConn.NewStream.
+type EncryptionKeyRotationControllerClient interface {
+	// EncryptionKeyRotate is a procedure that is called
+	// on the CSI ControllerPlugin or NodePlugin
+	EncryptionKeyRotate(ctx context.Context, in *EncryptionKeyRotateRequest, opts ...grpc.CallOption) (*EncryptionKeyRotateResponse, error)
+}
+
+type encryptionKeyRotationControllerClient struct {
+	cc grpc.ClientConnInterface
+}
+
+func NewEncryptionKeyRotationControllerClient(cc grpc.ClientConnInterface) EncryptionKeyRotationControllerClient {
+	return &encryptionKeyRotationControllerClient{cc}
+}
+
+func (c *encryptionKeyRotationControllerClient) EncryptionKeyRotate(ctx context.Context, in *EncryptionKeyRotateRequest, opts ...grpc.CallOption) (*EncryptionKeyRotateResponse, error) {
+	out := new(EncryptionKeyRotateResponse)
+	err := c.cc.Invoke(ctx, EncryptionKeyRotationController_EncryptionKeyRotate_FullMethodName, in, out, opts...)
+	if err != nil {
+		return nil, err
+	}
+	return out, nil
+}
+
+// EncryptionKeyRotationControllerServer is the server API for EncryptionKeyRotationController service.
+// All implementations must embed UnimplementedEncryptionKeyRotationControllerServer
+// for forward compatibility
+type EncryptionKeyRotationControllerServer interface {
+	// EncryptionKeyRotate is a procedure that is called
+	// on the CSI ControllerPlugin or NodePlugin
+	EncryptionKeyRotate(context.Context, *EncryptionKeyRotateRequest) (*EncryptionKeyRotateResponse, error)
+	mustEmbedUnimplementedEncryptionKeyRotationControllerServer()
+}
+
+// UnimplementedEncryptionKeyRotationControllerServer must be embedded to have forward compatible implementations.
+type UnimplementedEncryptionKeyRotationControllerServer struct {
+}
+
+func (UnimplementedEncryptionKeyRotationControllerServer) EncryptionKeyRotate(context.Context, *EncryptionKeyRotateRequest) (*EncryptionKeyRotateResponse, error) {
+	return nil, status.Errorf(codes.Unimplemented, "method EncryptionKeyRotate not implemented")
+}
+func (UnimplementedEncryptionKeyRotationControllerServer) mustEmbedUnimplementedEncryptionKeyRotationControllerServer() {
+}
+
+// UnsafeEncryptionKeyRotationControllerServer may be embedded to opt out of forward compatibility for this service.
+// Use of this interface is not recommended, as added methods to EncryptionKeyRotationControllerServer will
+// result in compilation errors.
+type UnsafeEncryptionKeyRotationControllerServer interface {
+	mustEmbedUnimplementedEncryptionKeyRotationControllerServer()
+}
+
+func RegisterEncryptionKeyRotationControllerServer(s grpc.ServiceRegistrar, srv EncryptionKeyRotationControllerServer) {
+	s.RegisterService(&EncryptionKeyRotationController_ServiceDesc, srv)
+}
+
+func _EncryptionKeyRotationController_EncryptionKeyRotate_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) {
+	in := new(EncryptionKeyRotateRequest)
+	if err := dec(in); err != nil {
+		return nil, err
+	}
+	if interceptor == nil {
+		return srv.(EncryptionKeyRotationControllerServer).EncryptionKeyRotate(ctx, in)
+	}
+	info := &grpc.UnaryServerInfo{
+		Server:     srv,
+		FullMethod: EncryptionKeyRotationController_EncryptionKeyRotate_FullMethodName,
+	}
+	handler := func(ctx context.Context, req interface{}) (interface{}, error) {
+		return srv.(EncryptionKeyRotationControllerServer).EncryptionKeyRotate(ctx, req.(*EncryptionKeyRotateRequest))
+	}
+	return interceptor(ctx, in, info, handler)
+}
+
+// EncryptionKeyRotationController_ServiceDesc is the grpc.ServiceDesc for EncryptionKeyRotationController service.
+// It's only intended for direct use with grpc.RegisterService,
+// and not to be introspected or modified (even as a copy)
+var EncryptionKeyRotationController_ServiceDesc = grpc.ServiceDesc{
+	ServiceName: "encryptionkeyrotation.EncryptionKeyRotationController",
+	HandlerType: (*EncryptionKeyRotationControllerServer)(nil),
+	Methods: []grpc.MethodDesc{
+		{
+			MethodName: "EncryptionKeyRotate",
+			Handler:    _EncryptionKeyRotationController_EncryptionKeyRotate_Handler,
+		},
+	},
+	Streams:  []grpc.StreamDesc{},
+	Metadata: "encryptionkeyrotation/encryptionkeyrotation.proto",
+}
diff --git a/vendor/modules.txt b/vendor/modules.txt
index 119aed65e61..33d4bebd4b2 100644
--- a/vendor/modules.txt
+++ b/vendor/modules.txt
@@ -236,8 +236,9 @@ github.com/coreos/go-semver/semver
 ## explicit; go 1.12
 github.com/coreos/go-systemd/v22/daemon
 github.com/coreos/go-systemd/v22/journal
-# github.com/csi-addons/spec v0.2.1-0.20240627093359-0dd74d521e67
+# github.com/csi-addons/spec v0.2.1-0.20240718113938-dc98b454ba65
 ## explicit
+github.com/csi-addons/spec/lib/go/encryptionkeyrotation
 github.com/csi-addons/spec/lib/go/fence
 github.com/csi-addons/spec/lib/go/identity
 github.com/csi-addons/spec/lib/go/reclaimspace

From 141da9af42dfd9374b0588ce3bed6c1a210e37f4 Mon Sep 17 00:00:00 2001
From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com>
Date: Mon, 22 Jul 2024 20:36:09 +0000
Subject: [PATCH 02/14] rebase: bump k8s.io/api in /api in the k8s-dependencies
 group

Bumps the k8s-dependencies group in /api with 1 update: [k8s.io/api](https://github.com/kubernetes/api).

Updates `k8s.io/api` from 0.30.2 to 0.30.3
- [Commits](https://github.com/kubernetes/api/compare/v0.30.2...v0.30.3)

---
updated-dependencies:
- dependency-name: k8s.io/api
  dependency-type: direct:production
  update-type: version-update:semver-patch
  dependency-group: k8s-dependencies
...

Signed-off-by: dependabot[bot] <support@github.com>
---
 api/go.mod             | 4 ++--
 api/go.sum             | 8 ++++----
 api/vendor/modules.txt | 4 ++--
 go.mod                 | 4 ++--
 vendor/modules.txt     | 4 ++--
 5 files changed, 12 insertions(+), 12 deletions(-)

diff --git a/api/go.mod b/api/go.mod
index e0ffe73b59a..93cd4daaa2e 100644
--- a/api/go.mod
+++ b/api/go.mod
@@ -6,7 +6,7 @@ require (
 	github.com/ghodss/yaml v1.0.0
 	github.com/openshift/api v0.0.0-20240115183315-0793e918179d
 	github.com/stretchr/testify v1.9.0
-	k8s.io/api v0.30.2
+	k8s.io/api v0.30.3
 )
 
 require (
@@ -23,7 +23,7 @@ require (
 	gopkg.in/inf.v0 v0.9.1 // indirect
 	gopkg.in/yaml.v2 v2.4.0 // indirect
 	gopkg.in/yaml.v3 v3.0.1 // indirect
-	k8s.io/apimachinery v0.30.2 // indirect
+	k8s.io/apimachinery v0.30.3 // indirect
 	k8s.io/klog/v2 v2.120.1 // indirect
 	k8s.io/utils v0.0.0-20230726121419-3b25d923346b // indirect
 	sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd // indirect
diff --git a/api/go.sum b/api/go.sum
index dde255efea4..ad5efc1fd04 100644
--- a/api/go.sum
+++ b/api/go.sum
@@ -79,10 +79,10 @@ gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY=
 gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ=
 gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
 gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
-k8s.io/api v0.30.2 h1:+ZhRj+28QT4UOH+BKznu4CBgPWgkXO7XAvMcMl0qKvI=
-k8s.io/api v0.30.2/go.mod h1:ULg5g9JvOev2dG0u2hig4Z7tQ2hHIuS+m8MNZ+X6EmI=
-k8s.io/apimachinery v0.30.2 h1:fEMcnBj6qkzzPGSVsAZtQThU62SmQ4ZymlXRC5yFSCg=
-k8s.io/apimachinery v0.30.2/go.mod h1:iexa2somDaxdnj7bha06bhb43Zpa6eWH8N8dbqVjTUc=
+k8s.io/api v0.30.3 h1:ImHwK9DCsPA9uoU3rVh4QHAHHK5dTSv1nxJUapx8hoQ=
+k8s.io/api v0.30.3/go.mod h1:GPc8jlzoe5JG3pb0KJCSLX5oAFIW3/qNJITlDj8BH04=
+k8s.io/apimachinery v0.30.3 h1:q1laaWCmrszyQuSQCfNB8cFgCuDAoPszKY4ucAjDwHc=
+k8s.io/apimachinery v0.30.3/go.mod h1:iexa2somDaxdnj7bha06bhb43Zpa6eWH8N8dbqVjTUc=
 k8s.io/klog/v2 v2.120.1 h1:QXU6cPEOIslTGvZaXvFWiP9VKyeet3sawzTOvdXb4Vw=
 k8s.io/klog/v2 v2.120.1/go.mod h1:3Jpz1GvMt720eyJH1ckRHK1EDfpxISzJ7I9OYgaDtPE=
 k8s.io/utils v0.0.0-20230726121419-3b25d923346b h1:sgn3ZU783SCgtaSJjpcVVlRqd6GSnlTLKgpAAttJvpI=
diff --git a/api/vendor/modules.txt b/api/vendor/modules.txt
index 5bc91bf19de..915f8019440 100644
--- a/api/vendor/modules.txt
+++ b/api/vendor/modules.txt
@@ -55,12 +55,12 @@ gopkg.in/yaml.v2
 # gopkg.in/yaml.v3 v3.0.1
 ## explicit
 gopkg.in/yaml.v3
-# k8s.io/api v0.30.2
+# k8s.io/api v0.30.3
 ## explicit; go 1.22.0
 k8s.io/api/core/v1
 k8s.io/api/rbac/v1
 k8s.io/api/storage/v1
-# k8s.io/apimachinery v0.30.2
+# k8s.io/apimachinery v0.30.3
 ## explicit; go 1.22.0
 k8s.io/apimachinery/pkg/api/resource
 k8s.io/apimachinery/pkg/apis/meta/v1
diff --git a/go.mod b/go.mod
index de27daf5016..1503a0bdad2 100644
--- a/go.mod
+++ b/go.mod
@@ -33,8 +33,8 @@ require (
 	//
 	// when updating k8s.io/kubernetes, make sure to update the replace section too
 	//
-	k8s.io/api v0.30.2
-	k8s.io/apimachinery v0.30.2
+	k8s.io/api v0.30.3
+	k8s.io/apimachinery v0.30.3
 	k8s.io/client-go v12.0.0+incompatible
 	k8s.io/cloud-provider v0.30.2
 	k8s.io/klog/v2 v2.130.1
diff --git a/vendor/modules.txt b/vendor/modules.txt
index 33d4bebd4b2..70ba2a65438 100644
--- a/vendor/modules.txt
+++ b/vendor/modules.txt
@@ -932,7 +932,7 @@ gopkg.in/yaml.v2
 # gopkg.in/yaml.v3 v3.0.1
 ## explicit
 gopkg.in/yaml.v3
-# k8s.io/api v0.30.2 => k8s.io/api v0.30.2
+# k8s.io/api v0.30.3 => k8s.io/api v0.30.3
 ## explicit; go 1.22.0
 k8s.io/api/admission/v1
 k8s.io/api/admission/v1beta1
@@ -995,7 +995,7 @@ k8s.io/api/storagemigration/v1alpha1
 k8s.io/apiextensions-apiserver/pkg/apis/apiextensions
 k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1
 k8s.io/apiextensions-apiserver/pkg/features
-# k8s.io/apimachinery v0.30.2 => k8s.io/apimachinery v0.30.2
+# k8s.io/apimachinery v0.30.3 => k8s.io/apimachinery v0.30.3
 ## explicit; go 1.22.0
 k8s.io/apimachinery/pkg/api/equality
 k8s.io/apimachinery/pkg/api/errors

From dce8561f33d01738bdcdffcb0d2b14ee9b7ab57d Mon Sep 17 00:00:00 2001
From: Niels de Vos <ndevos@ibm.com>
Date: Fri, 19 Jul 2024 16:03:39 +0200
Subject: [PATCH 03/14] doc: update coding guideline to suggest to place local
 imports last

It seems very common in other Go based projects to place the local
packages in the import statement last. Currently Ceph-CSI expects the
imports to group the local packages immediately after standard packages.
This exception compared to other projects often requires new
contributors to 'correct' their PR.

Following a more common convention for grouping imports should make it a
little easier to contribute to the project.

Signed-off-by: Niels de Vos <ndevos@ibm.com>
---
 docs/coding.md | 10 +++++-----
 1 file changed, 5 insertions(+), 5 deletions(-)

diff --git a/docs/coding.md b/docs/coding.md
index fc96b628da2..ae6d37806b3 100644
--- a/docs/coding.md
+++ b/docs/coding.md
@@ -18,14 +18,14 @@ the code and will be pointed out in the review process:
 
 ### Imports
 
-We use the following convention for specifying imports:
+We prefer the following convention for specifying imports:
 
 ```
 <import standard library packages>
 
-<import ceph-csi packages>
-
 <import third-party packages>
+
+<import ceph-csi packages>
 ```
 
 Example:
@@ -37,9 +37,9 @@ import (
  "strings"
  "time"
 
- "github.com/ceph/ceph-csi/internal/util"
-
  "github.com/pborman/uuid"
+
+ "github.com/ceph/ceph-csi/internal/util"
 )
 ```
 

From c875483f8aa1b1a139e04837be3f9d0216d88742 Mon Sep 17 00:00:00 2001
From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com>
Date: Mon, 22 Jul 2024 20:55:31 +0000
Subject: [PATCH 04/14] rebase: bump the github-dependencies group with 2
 updates

Bumps the github-dependencies group with 2 updates: [github.com/aws/aws-sdk-go](https://github.com/aws/aws-sdk-go) and [github.com/pkg/xattr](https://github.com/pkg/xattr).


Updates `github.com/aws/aws-sdk-go` from 1.54.19 to 1.55.0
- [Release notes](https://github.com/aws/aws-sdk-go/releases)
- [Commits](https://github.com/aws/aws-sdk-go/compare/v1.54.19...v1.55.0)

Updates `github.com/pkg/xattr` from 0.4.9 to 0.4.10
- [Release notes](https://github.com/pkg/xattr/releases)
- [Commits](https://github.com/pkg/xattr/compare/v0.4.9...v0.4.10)

---
updated-dependencies:
- dependency-name: github.com/aws/aws-sdk-go
  dependency-type: direct:production
  update-type: version-update:semver-minor
  dependency-group: github-dependencies
- dependency-name: github.com/pkg/xattr
  dependency-type: direct:production
  update-type: version-update:semver-patch
  dependency-group: github-dependencies
...

Signed-off-by: dependabot[bot] <support@github.com>
---
 go.mod                                        |  4 +-
 go.sum                                        |  8 +--
 .../aws/aws-sdk-go/aws/endpoints/defaults.go  | 72 ++++++++++++++-----
 .../github.com/aws/aws-sdk-go/aws/version.go  |  2 +-
 vendor/github.com/pkg/xattr/xattr.go          |  9 +--
 vendor/github.com/pkg/xattr/xattr_solaris.go  | 20 ++++--
 vendor/modules.txt                            |  4 +-
 7 files changed, 85 insertions(+), 34 deletions(-)

diff --git a/go.mod b/go.mod
index 1503a0bdad2..14f5071775d 100644
--- a/go.mod
+++ b/go.mod
@@ -4,7 +4,7 @@ go 1.22.0
 
 require (
 	github.com/IBM/keyprotect-go-client v0.14.3
-	github.com/aws/aws-sdk-go v1.54.19
+	github.com/aws/aws-sdk-go v1.55.0
 	github.com/aws/aws-sdk-go-v2/service/sts v1.30.3
 	github.com/ceph/ceph-csi/api v0.0.0-00010101000000-000000000000
 	github.com/ceph/go-ceph v0.28.0
@@ -22,7 +22,7 @@ require (
 	github.com/libopenstorage/secrets v0.0.0-20231011182615-5f4b25ceede1
 	github.com/onsi/ginkgo/v2 v2.19.0
 	github.com/onsi/gomega v1.33.1
-	github.com/pkg/xattr v0.4.9
+	github.com/pkg/xattr v0.4.10
 	github.com/prometheus/client_golang v1.19.1
 	github.com/stretchr/testify v1.9.0
 	golang.org/x/crypto v0.25.0
diff --git a/go.sum b/go.sum
index 6f3afc0d380..149c794f52d 100644
--- a/go.sum
+++ b/go.sum
@@ -828,8 +828,8 @@ github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5/go.mod h1:wHh0iHkY
 github.com/asaskevich/govalidator v0.0.0-20190424111038-f61b66f89f4a h1:idn718Q4B6AGu/h5Sxe66HYVdqdGu2l9Iebqhi/AEoA=
 github.com/asaskevich/govalidator v0.0.0-20190424111038-f61b66f89f4a/go.mod h1:lB+ZfQJz7igIIfQNfa7Ml4HSf2uFQQRzpGGRXenZAgY=
 github.com/aws/aws-sdk-go v1.44.164/go.mod h1:aVsgQcEevwlmQ7qHE9I3h+dtQgpqhFB+i8Phjh7fkwI=
-github.com/aws/aws-sdk-go v1.54.19 h1:tyWV+07jagrNiCcGRzRhdtVjQs7Vy41NwsuOcl0IbVI=
-github.com/aws/aws-sdk-go v1.54.19/go.mod h1:eRwEWoyTWFMVYVQzKMNHWP5/RV4xIUGMQfXQHfHkpNU=
+github.com/aws/aws-sdk-go v1.55.0 h1:hVALKPjXz33kP1R9nTyJpUK7qF59dO2mleQxUW9mCVE=
+github.com/aws/aws-sdk-go v1.55.0/go.mod h1:eRwEWoyTWFMVYVQzKMNHWP5/RV4xIUGMQfXQHfHkpNU=
 github.com/aws/aws-sdk-go-v2 v1.30.3 h1:jUeBtG0Ih+ZIFH0F4UkmL9w3cSpaMv9tYYDbzILP8dY=
 github.com/aws/aws-sdk-go-v2 v1.30.3/go.mod h1:nIQjQVp5sfpQcTc9mPSr1B0PaWK5ByX9MOoDadSN4lc=
 github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.15 h1:SoNJ4RlFEQEbtDcCEt+QG56MY4fm4W8rYirAmq+/DdU=
@@ -1481,8 +1481,8 @@ github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4=
 github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
 github.com/pkg/sftp v1.10.1/go.mod h1:lYOWFsE0bwd1+KfKJaKeuokY15vzFx25BLbzYYoAxZI=
 github.com/pkg/sftp v1.13.1/go.mod h1:3HaPG6Dq1ILlpPZRO0HVMrsydcdLt6HRDccSgb87qRg=
-github.com/pkg/xattr v0.4.9 h1:5883YPCtkSd8LFbs13nXplj9g9tlrwoJRjgpgMu1/fE=
-github.com/pkg/xattr v0.4.9/go.mod h1:di8WF84zAKk8jzR1UBTEWh9AUlIZZ7M/JNt8e9B6ktU=
+github.com/pkg/xattr v0.4.10 h1:Qe0mtiNFHQZ296vRgUjRCoPHPqH7VdTOrZx3g0T+pGA=
+github.com/pkg/xattr v0.4.10/go.mod h1:di8WF84zAKk8jzR1UBTEWh9AUlIZZ7M/JNt8e9B6ktU=
 github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
 github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 h1:Jamvg5psRIccs7FGNTlIRMkT8wgtp5eCXdBlqhYGL6U=
 github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
diff --git a/vendor/github.com/aws/aws-sdk-go/aws/endpoints/defaults.go b/vendor/github.com/aws/aws-sdk-go/aws/endpoints/defaults.go
index 84dc7dc08e7..d517a35a409 100644
--- a/vendor/github.com/aws/aws-sdk-go/aws/endpoints/defaults.go
+++ b/vendor/github.com/aws/aws-sdk-go/aws/endpoints/defaults.go
@@ -13068,6 +13068,9 @@ var awsPartition = partition{
 				endpointKey{
 					Region: "eu-central-1",
 				}: endpoint{},
+				endpointKey{
+					Region: "eu-central-2",
+				}: endpoint{},
 				endpointKey{
 					Region: "eu-north-1",
 				}: endpoint{},
@@ -22522,6 +22525,9 @@ var awsPartition = partition{
 				}: endpoint{
 					Hostname: "network-firewall-fips.ca-central-1.amazonaws.com",
 				},
+				endpointKey{
+					Region: "ca-west-1",
+				}: endpoint{},
 				endpointKey{
 					Region: "eu-central-1",
 				}: endpoint{},
@@ -24479,6 +24485,14 @@ var awsPartition = partition{
 						Region: "ca-central-1",
 					},
 				},
+				endpointKey{
+					Region: "ca-west-1",
+				}: endpoint{
+					Hostname: "portal.sso.ca-west-1.amazonaws.com",
+					CredentialScope: credentialScope{
+						Region: "ca-west-1",
+					},
+				},
 				endpointKey{
 					Region: "eu-central-1",
 				}: endpoint{
@@ -33621,6 +33635,20 @@ var awsPartition = partition{
 				}: endpoint{},
 			},
 		},
+		"tax": service{
+			PartitionEndpoint: "aws-global",
+			IsRegionalized:    boxedFalse,
+			Endpoints: serviceEndpoints{
+				endpointKey{
+					Region: "aws-global",
+				}: endpoint{
+					Hostname: "tax.us-east-1.amazonaws.com",
+					CredentialScope: credentialScope{
+						Region: "us-east-1",
+					},
+				},
+			},
+		},
 		"textract": service{
 			Endpoints: serviceEndpoints{
 				endpointKey{
@@ -39973,16 +40001,12 @@ var awsusgovPartition = partition{
 			Endpoints: serviceEndpoints{
 				endpointKey{
 					Region: "us-gov-east-1",
-				}: endpoint{
-					Hostname:  "autoscaling-plans.us-gov-east-1.amazonaws.com",
-					Protocols: []string{"http", "https"},
-				},
+				}: endpoint{},
 				endpointKey{
 					Region:  "us-gov-east-1",
 					Variant: fipsVariant,
 				}: endpoint{
-					Hostname:  "autoscaling-plans.us-gov-east-1.amazonaws.com",
-					Protocols: []string{"http", "https"},
+					Hostname: "autoscaling-plans.us-gov-east-1.amazonaws.com",
 				},
 				endpointKey{
 					Region: "us-gov-east-1-fips",
@@ -39994,16 +40018,12 @@ var awsusgovPartition = partition{
 				},
 				endpointKey{
 					Region: "us-gov-west-1",
-				}: endpoint{
-					Hostname:  "autoscaling-plans.us-gov-west-1.amazonaws.com",
-					Protocols: []string{"http", "https"},
-				},
+				}: endpoint{},
 				endpointKey{
 					Region:  "us-gov-west-1",
 					Variant: fipsVariant,
 				}: endpoint{
-					Hostname:  "autoscaling-plans.us-gov-west-1.amazonaws.com",
-					Protocols: []string{"http", "https"},
+					Hostname: "autoscaling-plans.us-gov-west-1.amazonaws.com",
 				},
 				endpointKey{
 					Region: "us-gov-west-1-fips",
@@ -40969,20 +40989,40 @@ var awsusgovPartition = partition{
 		"directconnect": service{
 			Endpoints: serviceEndpoints{
 				endpointKey{
-					Region: "us-gov-east-1",
+					Region: "fips-us-gov-east-1",
 				}: endpoint{
-					Hostname: "directconnect.us-gov-east-1.amazonaws.com",
+					Hostname: "directconnect-fips.us-gov-east-1.amazonaws.com",
 					CredentialScope: credentialScope{
 						Region: "us-gov-east-1",
 					},
+					Deprecated: boxedTrue,
 				},
 				endpointKey{
-					Region: "us-gov-west-1",
+					Region: "fips-us-gov-west-1",
 				}: endpoint{
-					Hostname: "directconnect.us-gov-west-1.amazonaws.com",
+					Hostname: "directconnect-fips.us-gov-west-1.amazonaws.com",
 					CredentialScope: credentialScope{
 						Region: "us-gov-west-1",
 					},
+					Deprecated: boxedTrue,
+				},
+				endpointKey{
+					Region: "us-gov-east-1",
+				}: endpoint{},
+				endpointKey{
+					Region:  "us-gov-east-1",
+					Variant: fipsVariant,
+				}: endpoint{
+					Hostname: "directconnect-fips.us-gov-east-1.amazonaws.com",
+				},
+				endpointKey{
+					Region: "us-gov-west-1",
+				}: endpoint{},
+				endpointKey{
+					Region:  "us-gov-west-1",
+					Variant: fipsVariant,
+				}: endpoint{
+					Hostname: "directconnect-fips.us-gov-west-1.amazonaws.com",
 				},
 			},
 		},
diff --git a/vendor/github.com/aws/aws-sdk-go/aws/version.go b/vendor/github.com/aws/aws-sdk-go/aws/version.go
index b2040b05e5b..0734268e6ac 100644
--- a/vendor/github.com/aws/aws-sdk-go/aws/version.go
+++ b/vendor/github.com/aws/aws-sdk-go/aws/version.go
@@ -5,4 +5,4 @@ package aws
 const SDKName = "aws-sdk-go"
 
 // SDKVersion is the version of this SDK
-const SDKVersion = "1.54.19"
+const SDKVersion = "1.55.0"
diff --git a/vendor/github.com/pkg/xattr/xattr.go b/vendor/github.com/pkg/xattr/xattr.go
index 8b2b5fe80e8..e34e274d513 100644
--- a/vendor/github.com/pkg/xattr/xattr.go
+++ b/vendor/github.com/pkg/xattr/xattr.go
@@ -87,8 +87,8 @@ func get(path string, name string, getxattrFunc getxattrFunc) ([]byte, error) {
 		initialBufSize = 1024
 
 		// The theoretical maximum xattr value size on MacOS is 64 MB. On Linux it's
-		// much smaller at 64 KB. Unless the kernel is evil or buggy, we should never
-		// hit the limit.
+		// much smaller: documented at 64 KB. However, at least on TrueNAS SCALE, a
+		// Debian-based Linux distro, it can be larger.
 		maxBufSize = 64 * 1024 * 1024
 
 		// Function name as reported in error messages
@@ -102,14 +102,15 @@ func get(path string, name string, getxattrFunc getxattrFunc) ([]byte, error) {
 
 		// If the buffer was too small to fit the value, Linux and MacOS react
 		// differently:
-		// Linux: returns an ERANGE error and "-1" bytes.
+		// Linux: returns an ERANGE error and "-1" bytes. However, the TrueNAS
+		//   SCALE distro sometimes returns E2BIG.
 		// MacOS: truncates the value and returns "size" bytes. If the value
 		//   happens to be exactly as big as the buffer, we cannot know if it was
 		//   truncated, and we retry with a bigger buffer. Contrary to documentation,
 		//   MacOS never seems to return ERANGE!
 		// To keep the code simple, we always check both conditions, and sometimes
 		// double the buffer size without it being strictly necessary.
-		if err == syscall.ERANGE || read == size {
+		if err == syscall.ERANGE || err == syscall.E2BIG || read == size {
 			// The buffer was too small. Try again.
 			size <<= 1
 			if size >= maxBufSize {
diff --git a/vendor/github.com/pkg/xattr/xattr_solaris.go b/vendor/github.com/pkg/xattr/xattr_solaris.go
index 8d65b8d8d69..7c98b4afbac 100644
--- a/vendor/github.com/pkg/xattr/xattr_solaris.go
+++ b/vendor/github.com/pkg/xattr/xattr_solaris.go
@@ -24,7 +24,7 @@ const (
 )
 
 func getxattr(path string, name string, data []byte) (int, error) {
-	f, err := os.OpenFile(path, os.O_RDONLY, 0)
+	f, err := openNonblock(path)
 	if err != nil {
 		return 0, err
 	}
@@ -50,7 +50,7 @@ func fgetxattr(f *os.File, name string, data []byte) (int, error) {
 }
 
 func setxattr(path string, name string, data []byte, flags int) error {
-	f, err := os.OpenFile(path, os.O_RDONLY, 0)
+	f, err := openNonblock(path)
 	if err != nil {
 		return err
 	}
@@ -87,7 +87,8 @@ func fsetxattr(f *os.File, name string, data []byte, flags int) error {
 }
 
 func removexattr(path string, name string) error {
-	fd, err := unix.Open(path, unix.O_RDONLY|unix.O_XATTR, 0)
+	mode := unix.O_RDONLY | unix.O_XATTR | unix.O_NONBLOCK | unix.O_CLOEXEC
+	fd, err := unix.Open(path, mode, 0)
 	if err != nil {
 		return err
 	}
@@ -114,7 +115,7 @@ func fremovexattr(f *os.File, name string) error {
 }
 
 func listxattr(path string, data []byte) (int, error) {
-	f, err := os.OpenFile(path, os.O_RDONLY, 0)
+	f, err := openNonblock(path)
 	if err != nil {
 		return 0, err
 	}
@@ -151,8 +152,17 @@ func flistxattr(f *os.File, data []byte) (int, error) {
 	return copy(data, buf), nil
 }
 
+// Like os.Open, but passes O_NONBLOCK to the open(2) syscall.
+func openNonblock(path string) (*os.File, error) {
+	fd, err := unix.Open(path, unix.O_RDONLY|unix.O_CLOEXEC|unix.O_NONBLOCK, 0)
+	if err != nil {
+		return nil, err
+	}
+	return os.NewFile(uintptr(fd), path), err
+}
+
 // stringsFromByteSlice converts a sequence of attributes to a []string.
-// On Darwin and Linux, each entry is a NULL-terminated string.
+// We simulate Linux/Darwin, where each entry is a NULL-terminated string.
 func stringsFromByteSlice(buf []byte) (result []string) {
 	offset := 0
 	for index, b := range buf {
diff --git a/vendor/modules.txt b/vendor/modules.txt
index 70ba2a65438..1ffec3c15a4 100644
--- a/vendor/modules.txt
+++ b/vendor/modules.txt
@@ -81,7 +81,7 @@ github.com/antlr/antlr4/runtime/Go/antlr/v4
 # github.com/asaskevich/govalidator v0.0.0-20190424111038-f61b66f89f4a
 ## explicit
 github.com/asaskevich/govalidator
-# github.com/aws/aws-sdk-go v1.54.19
+# github.com/aws/aws-sdk-go v1.55.0
 ## explicit; go 1.19
 github.com/aws/aws-sdk-go/aws
 github.com/aws/aws-sdk-go/aws/auth/bearer
@@ -568,7 +568,7 @@ github.com/pkg/browser
 # github.com/pkg/errors v0.9.1
 ## explicit
 github.com/pkg/errors
-# github.com/pkg/xattr v0.4.9
+# github.com/pkg/xattr v0.4.10
 ## explicit; go 1.14
 github.com/pkg/xattr
 # github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2

From d46b7d7ff4d0dc992b8feb5e9d9fd54c6424a7f4 Mon Sep 17 00:00:00 2001
From: Sunnatillo <sunnat.samadov@est.tech>
Date: Tue, 23 Jul 2024 15:18:10 +0300
Subject: [PATCH 05/14] cephfs: Avoid hanging lock in volume mutex lock

This patch allows to avoid hanging mutex lock scenario when
fscrypt fails to unlock. Prevents uncessary delays

Signed-off-by: Sunnatillo <sunnat.samadov@est.tech>
---
 internal/cephfs/nodeserver.go | 24 +++++++++++++-----------
 1 file changed, 13 insertions(+), 11 deletions(-)

diff --git a/internal/cephfs/nodeserver.go b/internal/cephfs/nodeserver.go
index a73f48cf35c..cfeefa0bd67 100644
--- a/internal/cephfs/nodeserver.go
+++ b/internal/cephfs/nodeserver.go
@@ -164,23 +164,25 @@ func maybeUnlockFileEncryption(
 	}
 	log.DebugLog(ctx, "Lock successfully created for volume ID %s", volID)
 
+	defer func() {
+		ret, unlockErr := ioctx.Unlock(string(volID), lockName, lockCookie)
+		switch ret {
+		case 0:
+			log.DebugLog(ctx, "Lock %s successfully released ", lockName)
+		case -int(syscall.ENOENT):
+			log.DebugLog(ctx, "Lock is not held by the specified %s, %s pair", lockCookie, lockName)
+		default:
+			log.ErrorLog(ctx, "Failed to release following lock, this will lead to orphan lock %s: %v",
+				lockName, unlockErr)
+		}
+	}()
+
 	log.DebugLog(ctx, "cephfs: unlocking fscrypt on volume %q path %s", volID, stagingTargetPath)
 	err = fscrypt.Unlock(ctx, volOptions.Encryption, stagingTargetPath, string(volID))
 	if err != nil {
 		return err
 	}
 
-	ret, err := ioctx.Unlock(string(volID), lockName, lockCookie)
-	switch ret {
-	case 0:
-		log.DebugLog(ctx, "Lock %s successfully released ", lockName)
-	case -int(syscall.ENOENT):
-		log.DebugLog(ctx, "Lock is not held by the specified %s, %s pair", lockCookie, lockName)
-	default:
-		log.ErrorLog(ctx, "Failed to release following lock, this will lead to orphan lock %s: %v",
-			lockName, err)
-	}
-
 	return nil
 }
 

From 435e26d948c1c7cec7a16e1f39877a8139d36c4c Mon Sep 17 00:00:00 2001
From: Niels de Vos <ndevos@ibm.com>
Date: Tue, 16 Jul 2024 17:23:49 +0200
Subject: [PATCH 06/14] cleanup: make VolumeGroupJournalConnection a private
 type

VolumeGroupJournalConnection is not used outside the internal/journal
package. There is no need to expose the type outside of the package, it
causes only confusion about the usage of the journalling API.

Signed-off-by: Niels de Vos <ndevos@ibm.com>
---
 internal/journal/volumegroupjournal.go | 22 +++++++++++++---------
 1 file changed, 13 insertions(+), 9 deletions(-)

diff --git a/internal/journal/volumegroupjournal.go b/internal/journal/volumegroupjournal.go
index 949cc9145f3..6d36c9d9af2 100644
--- a/internal/journal/volumegroupjournal.go
+++ b/internal/journal/volumegroupjournal.go
@@ -77,11 +77,15 @@ type VolumeGroupJournalConfig struct {
 	Config
 }
 
-type VolumeGroupJournalConnection struct {
+type volumeGroupJournalConnection struct {
 	config     *VolumeGroupJournalConfig
 	connection *Connection
 }
 
+// assert that volumeGroupJournalConnection implements the VolumeGroupJournal
+// interface.
+var _ VolumeGroupJournal = &volumeGroupJournalConnection{}
+
 // NewCSIVolumeGroupJournal returns an instance of VolumeGroupJournal for groups.
 func NewCSIVolumeGroupJournal(suffix string) VolumeGroupJournalConfig {
 	return VolumeGroupJournalConfig{
@@ -116,7 +120,7 @@ func (vgc *VolumeGroupJournalConfig) Connect(
 	namespace string,
 	cr *util.Credentials,
 ) (VolumeGroupJournal, error) {
-	vgjc := &VolumeGroupJournalConnection{}
+	vgjc := &volumeGroupJournalConnection{}
 	vgjc.config = &VolumeGroupJournalConfig{
 		Config: vgc.Config,
 	}
@@ -130,7 +134,7 @@ func (vgc *VolumeGroupJournalConfig) Connect(
 }
 
 // Destroy frees any resources and invalidates the journal connection.
-func (vgjc *VolumeGroupJournalConnection) Destroy() {
+func (vgjc *volumeGroupJournalConnection) Destroy() {
 	vgjc.connection.Destroy()
 }
 
@@ -167,7 +171,7 @@ Return values:
     reservation found.
   - error: non-nil in case of any errors.
 */
-func (vgjc *VolumeGroupJournalConnection) CheckReservation(ctx context.Context,
+func (vgjc *volumeGroupJournalConnection) CheckReservation(ctx context.Context,
 	journalPool, reqName, namePrefix string,
 ) (*VolumeGroupData, error) {
 	var (
@@ -244,7 +248,7 @@ Input arguments:
   - groupID: ID of the volume group, generated from the UUID
   - reqName: Request name for the volume group
 */
-func (vgjc *VolumeGroupJournalConnection) UndoReservation(ctx context.Context,
+func (vgjc *volumeGroupJournalConnection) UndoReservation(ctx context.Context,
 	csiJournalPool, groupID, reqName string,
 ) error {
 	// delete volume UUID omap (first, inverse of create order)
@@ -303,7 +307,7 @@ Return values:
   - string: Contains the VolumeGroup name that was reserved for the passed in reqName
   - error: non-nil in case of any errors
 */
-func (vgjc *VolumeGroupJournalConnection) ReserveName(ctx context.Context,
+func (vgjc *volumeGroupJournalConnection) ReserveName(ctx context.Context,
 	journalPool, reqName, namePrefix string,
 ) (string, string, error) {
 	cj := vgjc.config
@@ -366,7 +370,7 @@ type VolumeGroupAttributes struct {
 	VolumeMap   map[string]string // Contains the volumeID and the corresponding value mapping
 }
 
-func (vgjc *VolumeGroupJournalConnection) GetVolumeGroupAttributes(
+func (vgjc *volumeGroupJournalConnection) GetVolumeGroupAttributes(
 	ctx context.Context,
 	pool, objectUUID string,
 ) (*VolumeGroupAttributes, error) {
@@ -401,7 +405,7 @@ func (vgjc *VolumeGroupJournalConnection) GetVolumeGroupAttributes(
 	return groupAttributes, nil
 }
 
-func (vgjc *VolumeGroupJournalConnection) AddVolumesMapping(
+func (vgjc *volumeGroupJournalConnection) AddVolumesMapping(
 	ctx context.Context,
 	pool,
 	reservedUUID string,
@@ -418,7 +422,7 @@ func (vgjc *VolumeGroupJournalConnection) AddVolumesMapping(
 	return nil
 }
 
-func (vgjc *VolumeGroupJournalConnection) RemoveVolumesMapping(
+func (vgjc *volumeGroupJournalConnection) RemoveVolumesMapping(
 	ctx context.Context,
 	pool,
 	reservedUUID string,

From a98edab480c1c3c6da27cb97cebe1d7effecbb9e Mon Sep 17 00:00:00 2001
From: Niels de Vos <ndevos@ibm.com>
Date: Wed, 17 Jul 2024 10:08:27 +0200
Subject: [PATCH 07/14] rbd: pass CSI-instanceID to CSI-Addons
 VolumeGroupServer

Signed-off-by: Niels de Vos <ndevos@ibm.com>
---
 internal/csi-addons/rbd/volumegroup.go | 9 +++++++--
 internal/rbd/driver/driver.go          | 2 +-
 2 files changed, 8 insertions(+), 3 deletions(-)

diff --git a/internal/csi-addons/rbd/volumegroup.go b/internal/csi-addons/rbd/volumegroup.go
index a47b47feb3d..261bc67787a 100644
--- a/internal/csi-addons/rbd/volumegroup.go
+++ b/internal/csi-addons/rbd/volumegroup.go
@@ -37,12 +37,17 @@ type VolumeGroupServer struct {
 	// if volumegroup spec add more RPC services in the proto file, then we
 	// don't need to add all RPC methods leading to forward compatibility.
 	*volumegroup.UnimplementedControllerServer
+
+	// csiID is the unique ID for this CSI-driver deployment.
+	csiID string
 }
 
 // NewVolumeGroupServer creates a new VolumeGroupServer which handles the
 // VolumeGroup Service requests from the CSI-Addons specification.
-func NewVolumeGroupServer() *VolumeGroupServer {
-	return &VolumeGroupServer{}
+func NewVolumeGroupServer(instanceID string) *VolumeGroupServer {
+	return &VolumeGroupServer{
+		csiID: instanceID,
+	}
 }
 
 func (vs *VolumeGroupServer) RegisterService(server grpc.ServiceRegistrar) {
diff --git a/internal/rbd/driver/driver.go b/internal/rbd/driver/driver.go
index e682884286a..979da3b1c5a 100644
--- a/internal/rbd/driver/driver.go
+++ b/internal/rbd/driver/driver.go
@@ -222,7 +222,7 @@ func (r *Driver) setupCSIAddonsServer(conf *util.Config) error {
 		rcs := casrbd.NewReplicationServer(NewControllerServer(r.cd))
 		r.cas.RegisterService(rcs)
 
-		vgcs := casrbd.NewVolumeGroupServer()
+		vgcs := casrbd.NewVolumeGroupServer(conf.InstanceID)
 		r.cas.RegisterService(vgcs)
 	}
 

From fbf9ffcac4fe6fb3a9a517be99d3f7e86491c9c1 Mon Sep 17 00:00:00 2001
From: Niels de Vos <ndevos@ibm.com>
Date: Thu, 18 Jul 2024 20:40:28 +0200
Subject: [PATCH 08/14] rbd: update Volume interface implementation for
 VolumeGroup APIs

Add support for adding and removing the RBD-image from a group.

Signed-off-by: Niels de Vos <ndevos@ibm.com>
---
 internal/rbd/controllerserver.go | 21 ++++++++----
 internal/rbd/group.go            | 58 ++++++++++++++++++++++++++++++++
 internal/rbd/types/volume.go     |  8 ++++-
 3 files changed, 79 insertions(+), 8 deletions(-)
 create mode 100644 internal/rbd/group.go

diff --git a/internal/rbd/controllerserver.go b/internal/rbd/controllerserver.go
index 1f32e6dec7e..2ad9fe85464 100644
--- a/internal/rbd/controllerserver.go
+++ b/internal/rbd/controllerserver.go
@@ -239,7 +239,7 @@ func (cs *ControllerServer) parseVolCreateRequest(
 	return rbdVol, nil
 }
 
-func (rbdVol *rbdVolume) ToCSI(ctx context.Context) *csi.Volume {
+func (rbdVol *rbdVolume) ToCSI(ctx context.Context) (*csi.Volume, error) {
 	vol := &csi.Volume{
 		VolumeId:      rbdVol.VolID,
 		CapacityBytes: rbdVol.VolSize,
@@ -266,22 +266,29 @@ func (rbdVol *rbdVolume) ToCSI(ctx context.Context) *csi.Volume {
 		}
 	}
 
-	return vol
+	return vol, nil
 }
 
 func buildCreateVolumeResponse(
 	ctx context.Context,
 	req *csi.CreateVolumeRequest,
 	rbdVol *rbdVolume,
-) *csi.CreateVolumeResponse {
-	volume := rbdVol.ToCSI(ctx)
+) (*csi.CreateVolumeResponse, error) {
+	volume, err := rbdVol.ToCSI(ctx)
+	if err != nil {
+		return nil, status.Errorf(
+			codes.Internal,
+			"BUG, can not happen: failed to convert volume %q to CSI type: %v",
+			rbdVol, err)
+	}
+
 	volume.ContentSource = req.GetVolumeContentSource()
 
 	for param, value := range util.GetVolumeContext(req.GetParameters()) {
 		volume.VolumeContext[param] = value
 	}
 
-	return &csi.CreateVolumeResponse{Volume: volume}
+	return &csi.CreateVolumeResponse{Volume: volume}, nil
 }
 
 // getGRPCErrorForCreateVolume converts the returns the GRPC errors based on
@@ -424,7 +431,7 @@ func (cs *ControllerServer) CreateVolume(
 		return nil, status.Error(codes.Internal, err.Error())
 	}
 
-	return buildCreateVolumeResponse(ctx, req, rbdVol), nil
+	return buildCreateVolumeResponse(ctx, req, rbdVol)
 }
 
 // flattenParentImage is to be called before proceeding with creating volume,
@@ -559,7 +566,7 @@ func (cs *ControllerServer) repairExistingVolume(ctx context.Context, req *csi.C
 		return nil, err
 	}
 
-	return buildCreateVolumeResponse(ctx, req, rbdVol), nil
+	return buildCreateVolumeResponse(ctx, req, rbdVol)
 }
 
 // check snapshots on the rbd image, as we have limit from krbd that an image
diff --git a/internal/rbd/group.go b/internal/rbd/group.go
new file mode 100644
index 00000000000..af59776aa08
--- /dev/null
+++ b/internal/rbd/group.go
@@ -0,0 +1,58 @@
+/*
+Copyright 2024 The Ceph-CSI Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package rbd
+
+import (
+	"context"
+	"fmt"
+
+	librbd "github.com/ceph/go-ceph/rbd"
+
+	"github.com/ceph/ceph-csi/internal/rbd/types"
+)
+
+// AddToGroup adds the image to the group. This is called from the rbd_group
+// package.
+func (rv *rbdVolume) AddToGroup(ctx context.Context, vg types.VolumeGroup) error {
+	ioctx, err := vg.GetIOContext(ctx)
+	if err != nil {
+		return fmt.Errorf("could not get iocontext for volume group %q: %w", vg, err)
+	}
+
+	name, err := vg.GetName(ctx)
+	if err != nil {
+		return fmt.Errorf("could not get name for volume group %q: %w", vg, err)
+	}
+
+	return librbd.GroupImageAdd(ioctx, name, rv.ioctx, rv.RbdImageName)
+}
+
+// RemoveFromGroup removes the image from the group. This is called from the
+// rbd_group package.
+func (rv *rbdVolume) RemoveFromGroup(ctx context.Context, vg types.VolumeGroup) error {
+	ioctx, err := vg.GetIOContext(ctx)
+	if err != nil {
+		return fmt.Errorf("could not get iocontext for volume group %q: %w", vg, err)
+	}
+
+	name, err := vg.GetName(ctx)
+	if err != nil {
+		return fmt.Errorf("could not get name for volume group %q: %w", vg, err)
+	}
+
+	return librbd.GroupImageRemove(ioctx, name, rv.ioctx, rv.RbdImageName)
+}
diff --git a/internal/rbd/types/volume.go b/internal/rbd/types/volume.go
index 99961a73382..2ea6e54caff 100644
--- a/internal/rbd/types/volume.go
+++ b/internal/rbd/types/volume.go
@@ -33,5 +33,11 @@ type Volume interface {
 	GetID(ctx context.Context) (string, error)
 
 	// ToCSI creates a CSI protocol formatted struct of the volume.
-	ToCSI(ctx context.Context) *csi.Volume
+	ToCSI(ctx context.Context) (*csi.Volume, error)
+
+	// AddToGroup adds the Volume to the VolumeGroup.
+	AddToGroup(ctx context.Context, vg VolumeGroup) error
+
+	// RemoveFromGroup removes the Volume from the VolumeGroup.
+	RemoveFromGroup(ctx context.Context, vg VolumeGroup) error
 }

From 40b0526f643b0b6604285af4fdf62be9f4f25398 Mon Sep 17 00:00:00 2001
From: Niels de Vos <ndevos@ibm.com>
Date: Fri, 19 Jul 2024 16:25:21 +0200
Subject: [PATCH 09/14] rbd: implement the VolumeGroup interface

Signed-off-by: Niels de Vos <ndevos@ibm.com>
---
 internal/rbd/group/volume_group.go | 402 +++++++++++++++++++++++++++++
 internal/rbd/types/group.go        |  14 +-
 2 files changed, 415 insertions(+), 1 deletion(-)
 create mode 100644 internal/rbd/group/volume_group.go

diff --git a/internal/rbd/group/volume_group.go b/internal/rbd/group/volume_group.go
new file mode 100644
index 00000000000..4d0a05d58f3
--- /dev/null
+++ b/internal/rbd/group/volume_group.go
@@ -0,0 +1,402 @@
+/*
+Copyright 2024 The Ceph-CSI Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package group
+
+import (
+	"context"
+	"errors"
+	"fmt"
+
+	"github.com/ceph/go-ceph/rados"
+	librbd "github.com/ceph/go-ceph/rbd"
+	"github.com/container-storage-interface/spec/lib/go/csi"
+	"github.com/csi-addons/spec/lib/go/volumegroup"
+
+	"github.com/ceph/ceph-csi/internal/journal"
+	"github.com/ceph/ceph-csi/internal/rbd/types"
+	"github.com/ceph/ceph-csi/internal/util"
+	"github.com/ceph/ceph-csi/internal/util/log"
+)
+
+var ErrRBDGroupNotConnected = errors.New("RBD group is not connected")
+
+// volumeGroup handles all requests for 'rbd group' operations.
+type volumeGroup struct {
+	// id is a unique value for this volume group in the Ceph cluster, it
+	// is used to find the group in the journal.
+	id string
+
+	// name is used in RBD API calls as the name of this object
+	name string
+
+	clusterID string
+
+	credentials *util.Credentials
+
+	// temporary connection attributes
+	conn  *util.ClusterConnection
+	ioctx *rados.IOContext
+
+	// required details to perform operations on the group
+	monitors  string
+	pool      string
+	namespace string
+
+	// volumes is a list of rbd-images that are part of the group. The ID
+	// of each volume is stored in the journal.
+	volumes []types.Volume
+
+	journal journal.VolumeGroupJournal
+}
+
+// verify that volumeGroup implements the VolumeGroup and Stringer interfaces.
+var (
+	_ types.VolumeGroup = &volumeGroup{}
+	_ fmt.Stringer      = &volumeGroup{}
+)
+
+// GetVolumeGroup initializes a new VolumeGroup object that can be used
+// to manage an `rbd group`.
+func GetVolumeGroup(
+	ctx context.Context,
+	id string,
+	j journal.VolumeGroupJournal,
+	creds *util.Credentials,
+) (types.VolumeGroup, error) {
+	csiID := util.CSIIdentifier{}
+	err := csiID.DecomposeCSIID(id)
+	if err != nil {
+		return nil, fmt.Errorf("failed to decompose volume group id %q: %w", id, err)
+	}
+
+	mons, err := util.Mons(util.CsiConfigFile, csiID.ClusterID)
+	if err != nil {
+		return nil, fmt.Errorf("failed to get MONs for cluster id %q: %w", csiID.ClusterID, err)
+	}
+
+	namespace, err := util.GetRadosNamespace(util.CsiConfigFile, csiID.ClusterID)
+	if err != nil {
+		return nil, fmt.Errorf("failed to get RADOS namespace for cluster id %q: %w", csiID.ClusterID, err)
+	}
+
+	pool, err := util.GetPoolName(mons, creds, csiID.LocationID)
+	if err != nil {
+		return nil, fmt.Errorf("failed to get pool for volume group id %q: %w", id, err)
+	}
+
+	attrs, err := j.GetVolumeGroupAttributes(ctx, pool, csiID.ObjectUUID)
+	if err != nil {
+		return nil, fmt.Errorf("failed to get attributes for volume group id %q: %w", id, err)
+	}
+
+	// TODO: get the volumes that are part of this volume group
+
+	return &volumeGroup{
+		journal:     j,
+		credentials: creds,
+		id:          id,
+		name:        attrs.GroupName,
+		clusterID:   csiID.ClusterID,
+		monitors:    mons,
+		pool:        pool,
+		namespace:   namespace,
+	}, nil
+}
+
+// String makes it easy to include the volumeGroup object in log and error
+// messages.
+func (vg *volumeGroup) String() string {
+	if vg.name != "" {
+		return vg.name
+	}
+
+	if vg.id != "" {
+		return vg.id
+	}
+
+	return fmt.Sprintf("<unidentified volume %v>", *vg)
+}
+
+// GetID returns the CSI-Addons VolumeGroupId of the VolumeGroup.
+func (vg *volumeGroup) GetID(ctx context.Context) (string, error) {
+	if vg.id == "" {
+		return "", errors.New("BUG: ID is not set")
+	}
+
+	return vg.id, nil
+}
+
+// GetName returns the name in the backend storage for the VolumeGroup.
+func (vg *volumeGroup) GetName(ctx context.Context) (string, error) {
+	if vg.name == "" {
+		return "", errors.New("BUG: name is not set")
+	}
+
+	return vg.name, nil
+}
+
+// ToCSI creates a CSI-Addons type for the VolumeGroup.
+func (vg *volumeGroup) ToCSI(ctx context.Context) (*volumegroup.VolumeGroup, error) {
+	volumes, err := vg.ListVolumes(ctx)
+	if err != nil {
+		return nil, fmt.Errorf("failed to list volumes for volume group %q: %w", vg, err)
+	}
+
+	csiVolumes := make([]*csi.Volume, len(volumes))
+	for i, vol := range volumes {
+		csiVolumes[i], err = vol.ToCSI(ctx)
+		if err != nil {
+			return nil, fmt.Errorf("failed to convert volume %q to CSI type: %w", vol, err)
+		}
+	}
+
+	id, err := vg.GetID(ctx)
+	if err != nil {
+		return nil, fmt.Errorf("failed to get id for volume group %q: %w", vg, err)
+	}
+
+	// TODO: maybe store the VolumeContext in the journal?
+	vgContext := map[string]string{}
+
+	return &volumegroup.VolumeGroup{
+		VolumeGroupId:      id,
+		VolumeGroupContext: vgContext,
+		Volumes:            csiVolumes,
+	}, nil
+}
+
+// getConnection returns the ClusterConnection for the volume group if it
+// exists, otherwise it will open a new one.
+// Destroy should be used to close the ClusterConnection.
+func (vg *volumeGroup) getConnection(ctx context.Context) (*util.ClusterConnection, error) {
+	if vg.conn != nil {
+		return vg.conn, nil
+	}
+
+	conn := &util.ClusterConnection{}
+	err := conn.Connect(vg.monitors, vg.credentials)
+	if err != nil {
+		return nil, fmt.Errorf("failed to connect to MONs %q: %w", vg.monitors, err)
+	}
+
+	vg.conn = conn
+	log.DebugLog(ctx, "connection established for volume group %q", vg.id)
+
+	return conn, nil
+}
+
+// GetIOContext returns the IOContext for the volume group if it exists,
+// otherwise it will allocate a new one.
+// Destroy should be used to free the IOContext.
+func (vg *volumeGroup) GetIOContext(ctx context.Context) (*rados.IOContext, error) {
+	if vg.ioctx != nil {
+		return vg.ioctx, nil
+	}
+
+	conn, err := vg.getConnection(ctx)
+	if err != nil {
+		return nil, fmt.Errorf("%w: failed to connect: %w", ErrRBDGroupNotConnected, err)
+	}
+
+	ioctx, err := conn.GetIoctx(vg.pool)
+	if err != nil {
+		return nil, fmt.Errorf("%w: failed to get IOContext: %w", ErrRBDGroupNotConnected, err)
+	}
+
+	if vg.namespace != "" {
+		ioctx.SetNamespace(vg.namespace)
+	}
+
+	vg.ioctx = ioctx
+	log.DebugLog(ctx, "iocontext created for volume group %q in pool %q", vg.id, vg.pool)
+
+	return ioctx, nil
+}
+
+// Destroy frees the resources used by the volumeGroup.
+func (vg *volumeGroup) Destroy(ctx context.Context) {
+	if vg.ioctx != nil {
+		vg.ioctx.Destroy()
+		vg.ioctx = nil
+	}
+
+	if vg.conn != nil {
+		vg.conn.Destroy()
+		vg.conn = nil
+	}
+
+	if vg.credentials != nil {
+		vg.credentials.DeleteCredentials()
+		vg.credentials = nil
+	}
+
+	// FIXME: maybe need to .Destroy() all vg.volumes?
+	log.DebugLog(ctx, "destroyed volume group instance with id %q", vg.id)
+}
+
+func (vg *volumeGroup) Create(ctx context.Context, name string) error {
+	// TODO: if the group already exists, resolve details and use that
+	ioctx, err := vg.GetIOContext(ctx)
+	if err != nil {
+		return err
+	}
+
+	err = librbd.GroupCreate(ioctx, name)
+	if err != nil {
+		if !errors.Is(rados.ErrObjectExists, err) {
+			return fmt.Errorf("failed to create volume group %q: %w", name, err)
+		}
+
+		log.DebugLog(ctx, "ignoring error while creating volume group %q: %v", vg, err)
+	}
+
+	log.DebugLog(ctx, "volume group %q has been created", name)
+
+	return nil
+}
+
+func (vg *volumeGroup) Delete(ctx context.Context) error {
+	ioctx, err := vg.GetIOContext(ctx)
+	if err != nil {
+		return err
+	}
+
+	err = librbd.GroupRemove(ioctx, vg.name)
+	if err != nil {
+		return fmt.Errorf("failed to remove volume group %q: %w", vg, err)
+	}
+
+	log.DebugLog(ctx, "volume group %q has been removed", vg)
+
+	return nil
+}
+
+func (vg *volumeGroup) AddVolume(ctx context.Context, vol types.Volume) error {
+	err := vol.AddToGroup(ctx, vg)
+	if err != nil {
+		// rados.ErrObjectExists does not match the rbd error (there is no EEXISTS error)
+		if errors.Is(rados.ErrObjectExists, err) || strings.Contains(err.Error(), "ret=-17") {
+			log.DebugLog(ctx, "volume %q is already part of volume group %q: %v", vol, vg, err)
+			return nil
+		}
+
+		return fmt.Errorf("failed to add volume %q to volume group %q: %w", vol, vg, err)
+	}
+
+	vg.volumes = append(vg.volumes, vol)
+
+	volID, err := vol.GetID(ctx)
+	if err != nil {
+		return err
+	}
+
+	pool, err := vg.GetPool(ctx)
+	if err != nil {
+		return err
+	}
+
+	id, err := vg.GetID(ctx)
+	if err != nil {
+		return err
+	}
+
+	csiID := util.CSIIdentifier{}
+	err = csiID.DecomposeCSIID(id)
+	if err != nil {
+		return fmt.Errorf("failed to decompose volume group id %q: %w", id, err)
+	}
+
+	toAdd := map[string]string{
+		volID: "",
+	}
+
+	err = vg.journal.AddVolumesMapping(ctx, pool, csiID.ObjectUUID, toAdd)
+	if err != nil {
+		return fmt.Errorf("failed to add mapping for volume %q to volume group id %q: %w", volID, id, err)
+	}
+
+	return nil
+}
+
+func (vg *volumeGroup) RemoveVolume(ctx context.Context, vol types.Volume) error {
+	// volume was already removed from the group
+	if len(vg.volumes) == 0 {
+		return nil
+	}
+
+	err := vol.RemoveFromGroup(ctx, vg)
+	if err != nil {
+		return fmt.Errorf("failed to remove volume %q from volume group %q: %w", vol, vg, err)
+	}
+
+	// toRemove contain the ID of the volume that is removed from the group
+	toRemove, err := vol.GetID(ctx)
+	if err != nil {
+		return fmt.Errorf("failed to get volume id for %q: %w", vol, err)
+	}
+
+	// volumes is the updated list, without the volume that was removed
+	volumes := make([]types.Volume, 0)
+	var id string
+	for _, v := range vg.volumes {
+		id, err = v.GetID(ctx)
+		if err != nil {
+			return err
+		}
+
+		if id == toRemove {
+			// do not add the volume to the list
+			continue
+		}
+
+		volumes = append(volumes, v)
+	}
+
+	// update the list of volumes
+	vg.volumes = volumes
+
+	pool, err := vg.GetPool(ctx)
+	if err != nil {
+		return err
+	}
+
+	id, err = vg.GetID(ctx)
+	if err != nil {
+		return err
+	}
+
+	csiID := util.CSIIdentifier{}
+	err = csiID.DecomposeCSIID(id)
+	if err != nil {
+		return fmt.Errorf("failed to decompose volume group id %q: %w", id, err)
+	}
+
+	mapping := []string{
+		toRemove,
+	}
+
+	err = vg.journal.RemoveVolumesMapping(ctx, pool, csiID.ObjectUUID, mapping)
+	if err != nil {
+		return fmt.Errorf("failed to remove mapping for volume %q to volume group id %q: %w", toRemove, id, err)
+	}
+
+	return nil
+}
+
+func (vg *volumeGroup) ListVolumes(ctx context.Context) ([]types.Volume, error) {
+	return vg.volumes, nil
+}
diff --git a/internal/rbd/types/group.go b/internal/rbd/types/group.go
index 3beb4e9530c..d3c806a3b1e 100644
--- a/internal/rbd/types/group.go
+++ b/internal/rbd/types/group.go
@@ -19,6 +19,7 @@ package types
 import (
 	"context"
 
+	"github.com/ceph/go-ceph/rados"
 	"github.com/csi-addons/spec/lib/go/volumegroup"
 )
 
@@ -28,11 +29,22 @@ type VolumeGroup interface {
 	// Destroy frees the resources used by the VolumeGroup.
 	Destroy(ctx context.Context)
 
+	// GetIOContext returns the IOContext for performing librbd operations
+	// on the VolumeGroup. This is used by the rbdVolume struct when it
+	// needs to add/remove itself from the VolumeGroup.
+	GetIOContext(ctx context.Context) (*rados.IOContext, error)
+
 	// GetID returns the CSI-Addons VolumeGroupId of the VolumeGroup.
 	GetID(ctx context.Context) (string, error)
 
+	// GetName returns the name in the backend storage for the VolumeGroup.
+	GetName(ctx context.Context) (string, error)
+
 	// ToCSI creates a CSI-Addons type for the VolumeGroup.
-	ToCSI(ctx context.Context) *volumegroup.VolumeGroup
+	ToCSI(ctx context.Context) (*volumegroup.VolumeGroup, error)
+
+	// Create makes a new group in the backend storage.
+	Create(ctx context.Context, name string) error
 
 	// Delete removes the VolumeGroup from the backend storage.
 	Delete(ctx context.Context) error

From a82ae15f1abca7801ccf8e222259b42fc608f47f Mon Sep 17 00:00:00 2001
From: Niels de Vos <ndevos@ibm.com>
Date: Mon, 22 Jul 2024 17:51:17 +0200
Subject: [PATCH 10/14] rbd: use the Manager to handle CSI-Addons VolumeGroup
 requests

Signed-off-by: Niels de Vos <ndevos@ibm.com>
---
 internal/csi-addons/rbd/volumegroup.go |  15 +-
 internal/rbd/group/volume_group.go     |  61 ++++++--
 internal/rbd/manager.go                | 201 ++++++++++++++++++++++---
 internal/rbd/types/manager.go          |  12 +-
 4 files changed, 257 insertions(+), 32 deletions(-)

diff --git a/internal/csi-addons/rbd/volumegroup.go b/internal/csi-addons/rbd/volumegroup.go
index 261bc67787a..beda20d5bf1 100644
--- a/internal/csi-addons/rbd/volumegroup.go
+++ b/internal/csi-addons/rbd/volumegroup.go
@@ -82,7 +82,7 @@ func (vs *VolumeGroupServer) CreateVolumeGroup(
 	ctx context.Context,
 	req *volumegroup.CreateVolumeGroupRequest,
 ) (*volumegroup.CreateVolumeGroupResponse, error) {
-	mgr := rbd.NewManager(req.GetParameters(), req.GetSecrets())
+	mgr := rbd.NewManager(vs.csiID, req.GetParameters(), req.GetSecrets())
 	defer mgr.Destroy(ctx)
 
 	// resolve all volumes
@@ -132,8 +132,17 @@ func (vs *VolumeGroupServer) CreateVolumeGroup(
 
 	log.DebugLog(ctx, fmt.Sprintf("all %d Volumes have been added to for VolumeGroup %q", len(volumes), req.GetName()))
 
+	csiVG, err := vg.ToCSI(ctx)
+	if err != nil {
+		return nil, status.Errorf(
+			codes.Internal,
+			"failed to convert volume group %q to CSI type: %s",
+			req.GetName(),
+			err.Error())
+	}
+
 	return &volumegroup.CreateVolumeGroupResponse{
-		VolumeGroup: vg.ToCSI(ctx),
+		VolumeGroup: csiVG,
 	}, nil
 }
 
@@ -159,7 +168,7 @@ func (vs *VolumeGroupServer) DeleteVolumeGroup(
 	ctx context.Context,
 	req *volumegroup.DeleteVolumeGroupRequest,
 ) (*volumegroup.DeleteVolumeGroupResponse, error) {
-	mgr := rbd.NewManager(nil, req.GetSecrets())
+	mgr := rbd.NewManager(vs.csiID, nil, req.GetSecrets())
 	defer mgr.Destroy(ctx)
 
 	// resolve the volume group
diff --git a/internal/rbd/group/volume_group.go b/internal/rbd/group/volume_group.go
index 4d0a05d58f3..1a9e1963c4c 100644
--- a/internal/rbd/group/volume_group.go
+++ b/internal/rbd/group/volume_group.go
@@ -14,12 +14,13 @@ See the License for the specific language governing permissions and
 limitations under the License.
 */
 
-package group
+package rbd_group
 
 import (
 	"context"
 	"errors"
 	"fmt"
+	"strings"
 
 	"github.com/ceph/go-ceph/rados"
 	librbd "github.com/ceph/go-ceph/rbd"
@@ -56,11 +57,17 @@ type volumeGroup struct {
 	pool      string
 	namespace string
 
+	journal journal.VolumeGroupJournal
+
 	// volumes is a list of rbd-images that are part of the group. The ID
 	// of each volume is stored in the journal.
 	volumes []types.Volume
 
-	journal journal.VolumeGroupJournal
+	// volumeToFree contains Volumes that were resolved during
+	// GetVolumeGroup. The volumes slice can be updated independently of
+	// this by calling AddVolume (Volumes are allocated elsewhere), and
+	// RemoveVolume (need to keep track of the allocated Volume).
+	volumesToFree []types.Volume
 }
 
 // verify that volumeGroup implements the VolumeGroup and Stringer interfaces.
@@ -71,11 +78,14 @@ var (
 
 // GetVolumeGroup initializes a new VolumeGroup object that can be used
 // to manage an `rbd group`.
+// If the .GetName() function returns an error, the VolumeGroup does not exist
+// yet. It is needed to call .Create() in that case first.
 func GetVolumeGroup(
 	ctx context.Context,
 	id string,
 	j journal.VolumeGroupJournal,
 	creds *util.Credentials,
+	volumeResolver types.VolumeResolver,
 ) (types.VolumeGroup, error) {
 	csiID := util.CSIIdentifier{}
 	err := csiID.DecomposeCSIID(id)
@@ -100,10 +110,27 @@ func GetVolumeGroup(
 
 	attrs, err := j.GetVolumeGroupAttributes(ctx, pool, csiID.ObjectUUID)
 	if err != nil {
-		return nil, fmt.Errorf("failed to get attributes for volume group id %q: %w", id, err)
+		if !errors.Is(err, util.ErrKeyNotFound) && !errors.Is(err, util.ErrPoolNotFound) {
+			return nil, fmt.Errorf("failed to get attributes for volume group id %q: %w", id, err)
+		}
+
+		attrs = &journal.VolumeGroupAttributes{}
 	}
 
-	// TODO: get the volumes that are part of this volume group
+	var volumes []types.Volume
+	for volID := range attrs.VolumeMap {
+		vol, err := volumeResolver.GetVolumeByID(ctx, volID)
+		if err != nil {
+			// free the previously allocated volumes
+			for _, v := range volumes {
+				v.Destroy(ctx)
+			}
+
+			return nil, fmt.Errorf("failed to get attributes for volume group id %q: %w", id, err)
+		}
+
+		volumes = append(volumes, vol)
+	}
 
 	return &volumeGroup{
 		journal:     j,
@@ -114,6 +141,9 @@ func GetVolumeGroup(
 		monitors:    mons,
 		pool:        pool,
 		namespace:   namespace,
+		volumes:     volumes,
+		// all allocated volumes need to be free'd at Destroy() time
+		volumesToFree: volumes,
 	}, nil
 }
 
@@ -229,6 +259,14 @@ func (vg *volumeGroup) GetIOContext(ctx context.Context) (*rados.IOContext, erro
 
 // Destroy frees the resources used by the volumeGroup.
 func (vg *volumeGroup) Destroy(ctx context.Context) {
+	// free the volumes that were allocated in GetVolumeGroup()
+	if len(vg.volumesToFree) > 0 {
+		for _, volume := range vg.volumesToFree {
+			volume.Destroy(ctx)
+		}
+		vg.volumesToFree = make([]types.Volume, 0)
+	}
+
 	if vg.ioctx != nil {
 		vg.ioctx.Destroy()
 		vg.ioctx = nil
@@ -244,7 +282,6 @@ func (vg *volumeGroup) Destroy(ctx context.Context) {
 		vg.credentials = nil
 	}
 
-	// FIXME: maybe need to .Destroy() all vg.volumes?
 	log.DebugLog(ctx, "destroyed volume group instance with id %q", vg.id)
 }
 
@@ -257,26 +294,32 @@ func (vg *volumeGroup) Create(ctx context.Context, name string) error {
 
 	err = librbd.GroupCreate(ioctx, name)
 	if err != nil {
-		if !errors.Is(rados.ErrObjectExists, err) {
+		if !errors.Is(rados.ErrObjectExists, err) && !strings.Contains(err.Error(), "rbd: ret=-17, File exists") {
 			return fmt.Errorf("failed to create volume group %q: %w", name, err)
 		}
 
 		log.DebugLog(ctx, "ignoring error while creating volume group %q: %v", vg, err)
 	}
 
-	log.DebugLog(ctx, "volume group %q has been created", name)
+	vg.name = name
+	log.DebugLog(ctx, "volume group %q has been created", vg)
 
 	return nil
 }
 
 func (vg *volumeGroup) Delete(ctx context.Context) error {
-	ioctx, err := vg.GetIOContext(ctx)
+	name, err := vg.GetName(ctx)
 	if err != nil {
 		return err
 	}
 
-	err = librbd.GroupRemove(ioctx, vg.name)
+	ioctx, err := vg.GetIOContext(ctx)
 	if err != nil {
+		return err
+	}
+
+	err = librbd.GroupRemove(ioctx, name)
+	if err != nil && !errors.Is(rados.ErrNotFound, err) {
 		return fmt.Errorf("failed to remove volume group %q: %w", vg, err)
 	}
 
diff --git a/internal/rbd/manager.go b/internal/rbd/manager.go
index 518a914d073..cf19e0b01b8 100644
--- a/internal/rbd/manager.go
+++ b/internal/rbd/manager.go
@@ -19,24 +19,36 @@ package rbd
 import (
 	"context"
 	"errors"
+	"fmt"
 
+	"github.com/ceph/ceph-csi/internal/journal"
+	"github.com/ceph/ceph-csi/internal/rbd/group"
 	"github.com/ceph/ceph-csi/internal/rbd/types"
 	"github.com/ceph/ceph-csi/internal/util"
+	"github.com/ceph/ceph-csi/internal/util/log"
 )
 
 var _ types.Manager = &rbdManager{}
 
 type rbdManager struct {
+	// csiID is the instance id of the CSI-driver (driver name).
+	csiID string
+	// parameters can contain the parameters of a create request.
 	parameters map[string]string
-	secrets    map[string]string
+	// secrets contain the credentials to connect to the Ceph cluster.
+	secrets map[string]string
 
+	// creds are the cached credentials, will be freed on Destroy()
 	creds *util.Credentials
+	// vgJournal is the journal that is used during opetations, it will be freed on Destroy().
+	vgJournal journal.VolumeGroupJournal
 }
 
 // NewManager returns a new manager for handling Volume and Volume Group
 // operations, combining the requests for RBD and the journalling in RADOS.
-func NewManager(parameters, secrets map[string]string) types.Manager {
+func NewManager(csiID string, parameters, secrets map[string]string) types.Manager {
 	return &rbdManager{
+		csiID:      csiID,
 		parameters: parameters,
 		secrets:    secrets,
 	}
@@ -47,43 +59,198 @@ func (mgr *rbdManager) Destroy(ctx context.Context) {
 		mgr.creds.DeleteCredentials()
 		mgr.creds = nil
 	}
+
+	if mgr.vgJournal != nil {
+		mgr.vgJournal.Destroy()
+		mgr.vgJournal = nil
+	}
 }
 
-// connect sets up credentials and connects to the journal.
-func (mgr *rbdManager) connect() error {
-	if mgr.creds == nil {
-		creds, err := util.NewUserCredentials(mgr.secrets)
-		if err != nil {
-			return err
-		}
+// getCredentials sets up credentials and connects to the journal.
+func (mgr *rbdManager) getCredentials() (*util.Credentials, error) {
+	if mgr.creds != nil {
+		return mgr.creds, nil
+	}
 
-		mgr.creds = creds
+	creds, err := util.NewUserCredentials(mgr.secrets)
+	if err != nil {
+		return nil, fmt.Errorf("failed to get credentials: %w", err)
 	}
 
-	return nil
+	mgr.creds = creds
+
+	return creds, nil
 }
 
-func (mgr *rbdManager) GetVolumeByID(ctx context.Context, id string) (types.Volume, error) {
-	if err := mgr.connect(); err != nil {
+func (mgr *rbdManager) getVolumeGroupJournal(clusterID string) (journal.VolumeGroupJournal, error) {
+	if mgr.vgJournal != nil {
+		return mgr.vgJournal, nil
+	}
+
+	creds, err := mgr.getCredentials()
+	if err != nil {
 		return nil, err
 	}
 
-	volume, err := GenVolFromVolID(ctx, id, mgr.creds, mgr.secrets)
+	monitors, err := util.Mons(util.CsiConfigFile, clusterID)
+	if err != nil {
+		return nil, fmt.Errorf("failed to find MONs for cluster %q: %w", clusterID, err)
+	}
+
+	ns, err := util.GetRadosNamespace(util.CsiConfigFile, clusterID)
+	if err != nil {
+		return nil, fmt.Errorf("failed to find the RADOS namespace for cluster %q: %w", clusterID, err)
+	}
+
+	vgJournalConfig := journal.NewCSIVolumeGroupJournalWithNamespace(mgr.csiID, ns)
+
+	vgJournal, err := vgJournalConfig.Connect(monitors, ns, creds)
+	if err != nil {
+		return nil, fmt.Errorf("failed to connect to journal: %w", err)
+	}
+
+	mgr.vgJournal = vgJournal
+
+	return vgJournal, nil
+}
+
+func (mgr *rbdManager) GetVolumeByID(ctx context.Context, id string) (types.Volume, error) {
+	creds, err := mgr.getCredentials()
 	if err != nil {
 		return nil, err
 	}
 
+	volume, err := GenVolFromVolID(ctx, id, creds, mgr.secrets)
+	if err != nil {
+		return nil, fmt.Errorf("failed to get volume from id %q: %w", id, err)
+	}
+
 	return volume, nil
 }
 
 func (mgr *rbdManager) GetVolumeGroupByID(ctx context.Context, id string) (types.VolumeGroup, error) {
-	return nil, errors.New("rbdManager.GetVolumeGroupByID() is not implemented yet")
+	vi := &util.CSIIdentifier{}
+	if err := vi.DecomposeCSIID(id); err != nil {
+		return nil, fmt.Errorf("failed to parse volume group id %q: %w", id, err)
+	}
+
+	vgJournal, err := mgr.getVolumeGroupJournal(vi.ClusterID)
+	if err != nil {
+		return nil, err
+	}
+
+	creds, err := mgr.getCredentials()
+	if err != nil {
+		return nil, err
+	}
+
+	vg, err := rbd_group.GetVolumeGroup(ctx, id, vgJournal, creds, mgr)
+	if err != nil {
+		return nil, fmt.Errorf("failed to get volume group with id %q: %w", id, err)
+	}
+
+	return vg, nil
 }
 
 func (mgr *rbdManager) CreateVolumeGroup(ctx context.Context, name string) (types.VolumeGroup, error) {
-	return nil, errors.New("rbdManager.CreateVolumeGroup() is not implemented yet")
+	creds, err := mgr.getCredentials()
+	if err != nil {
+		return nil, err
+	}
+
+	clusterID, err := util.GetClusterID(mgr.parameters)
+	if err != nil {
+		return nil, fmt.Errorf("failed to get cluster-id: %w", err)
+	}
+
+	vgJournal, err := mgr.getVolumeGroupJournal(clusterID)
+	if err != nil {
+		return nil, err
+	}
+
+	// pool is a required parameter
+	pool, ok := mgr.parameters["pool"]
+	if !ok || pool == "" {
+		return nil, errors.New("required 'pool' option missing in volume group parameters")
+	}
+
+	// journalPool is an optional parameter, use pool if it is not set
+	journalPool, ok := mgr.parameters["journalPool"]
+	if !ok || journalPool == "" {
+		journalPool = pool
+	}
+
+	// volumeNamePrefix is an optional parameter, can be an empty string
+	prefix := mgr.parameters["volumeNamePrefix"]
+
+	// check if the journal contains a generated name for the group already
+	vgData, err := vgJournal.CheckReservation(ctx, journalPool, name, prefix)
+	if err != nil {
+		return nil, fmt.Errorf("failed to reserve volume group for name %q: %w", name, err)
+	}
+
+	var uuid string
+	if vgData != nil && vgData.GroupName != "" {
+		uuid = vgData.GroupUUID
+	} else {
+		log.DebugLog(ctx, "the journal does not contain a reservation for a volume group with name %q yet", name)
+
+		var vgName string
+		uuid, vgName, err = vgJournal.ReserveName(ctx, journalPool, name, prefix)
+		if err != nil {
+			return nil, fmt.Errorf("failed to reserve volume group for name %q: %w", name, err)
+		}
+		defer func() {
+			if err != nil {
+				err = vgJournal.UndoReservation(ctx, pool, vgName, name)
+				if err != nil {
+					log.ErrorLog(ctx, "failed to undo the reservation for volume group %q: %w", name, err)
+				}
+			}
+		}()
+	}
+
+	monitors, err := util.Mons(util.CsiConfigFile, clusterID)
+	if err != nil {
+		return nil, fmt.Errorf("failed to find MONs for cluster %q: %w", clusterID, err)
+	}
+
+	_ /*journalPoolID*/, poolID, err := util.GetPoolIDs(ctx, monitors, journalPool, pool, creds)
+	if err != nil {
+		return nil, fmt.Errorf("failed to generate a unique CSI volume group with uuid for %q: %w", uuid, err)
+	}
+
+	csiID, err := util.GenerateVolID(ctx, monitors, creds, poolID, pool, clusterID, uuid)
+	if err != nil {
+		return nil, fmt.Errorf("failed to generate a unique CSI volume group with uuid for %q: %w", uuid, err)
+	}
+
+	vg, err := rbd_group.GetVolumeGroup(ctx, csiID, vgJournal, creds, mgr)
+	if err != nil {
+		return nil, fmt.Errorf("failed to get volume group %q at cluster %q: %w", name, clusterID, err)
+	}
+	defer func() {
+		if err != nil {
+			vg.Destroy(ctx)
+		}
+	}()
+
+	// check if the volume group exists in the backend
+	existingName, err := vg.GetName(ctx)
+	if err != nil {
+		// the volume group does not exist yet
+		err = vg.Create(ctx, vgName)
+		if err != nil {
+			return nil, fmt.Errorf("failed to create volume group %q: %w", name, err)
+		}
+	} else if existingName != vgName {
+		return nil, fmt.Errorf("volume group id %q has a name mismatch, expected %q, not %q", name, vgName, existingName)
+	}
+
+	return vg, nil
 }
 
 func (mgr *rbdManager) DeleteVolumeGroup(ctx context.Context, vg types.VolumeGroup) error {
-	return errors.New("rbdManager.CreateVolumeGroup() is not implemented yet")
+	// TODO: remove from journal
+	return vg.Delete(ctx)
 }
diff --git a/internal/rbd/types/manager.go b/internal/rbd/types/manager.go
index 7744eb18a6f..33497493226 100644
--- a/internal/rbd/types/manager.go
+++ b/internal/rbd/types/manager.go
@@ -20,16 +20,22 @@ import (
 	"context"
 )
 
+// VolumeResolver can be used to construct a Volume from a CSI VolumeId.
+type VolumeResolver interface {
+	// GetVolumeByID uses the CSI VolumeId to resolve the returned Volume.
+	GetVolumeByID(ctx context.Context, id string) (Volume, error)
+}
+
 // Manager provides a way for other packages to get Volumes and VolumeGroups.
 // It handles the operations on the backend, and makes sure the journal
 // reflects the expected state.
 type Manager interface {
+	// VolumeResolver is fully implemented by the Manager.
+	VolumeResolver
+
 	// Destroy frees all resources that the Manager allocated.
 	Destroy(ctx context.Context)
 
-	// GetVolumeByID uses the CSI VolumeId to resolve the returned Volume.
-	GetVolumeByID(ctx context.Context, id string) (Volume, error)
-
 	// GetVolumeGroupByID uses the CSI-Addons VolumeGroupId to resolve the
 	// returned VolumeGroup.
 	GetVolumeGroupByID(ctx context.Context, id string) (VolumeGroup, error)

From fd2053666227fa7546535a37df9c4667171a5f1a Mon Sep 17 00:00:00 2001
From: Niels de Vos <ndevos@ibm.com>
Date: Fri, 19 Jul 2024 20:21:57 +0200
Subject: [PATCH 11/14] rbd: add journalledObject as base for VolumeGroup
 interface

Signed-off-by: Niels de Vos <ndevos@ibm.com>
---
 internal/rbd/types/group.go | 25 +++++++++++++++++--------
 1 file changed, 17 insertions(+), 8 deletions(-)

diff --git a/internal/rbd/types/group.go b/internal/rbd/types/group.go
index d3c806a3b1e..322230952bc 100644
--- a/internal/rbd/types/group.go
+++ b/internal/rbd/types/group.go
@@ -23,9 +23,24 @@ import (
 	"github.com/csi-addons/spec/lib/go/volumegroup"
 )
 
-// VolumeGroup contains a number of volumes, and can be used to create a
-// VolumeGroupSnapshot.
+type journalledObject interface {
+	// GetID returns the CSI-Addons VolumeGroupId of the VolumeGroup.
+	GetID(ctx context.Context) (string, error)
+
+	// GetName returns the name in the backend storage for the VolumeGroup.
+	GetName(ctx context.Context) (string, error)
+
+	// GetPool returns the name of the pool that holds the VolumeGroup.
+	GetPool(ctx context.Context) (string, error)
+
+	// GetClusterID returns the ID of the cluster of the VolumeGroup.
+	GetClusterID(ctx context.Context) (string, error)
+}
+
+// VolumeGroup contains a number of volumes.
 type VolumeGroup interface {
+	journalledObject
+
 	// Destroy frees the resources used by the VolumeGroup.
 	Destroy(ctx context.Context)
 
@@ -34,12 +49,6 @@ type VolumeGroup interface {
 	// needs to add/remove itself from the VolumeGroup.
 	GetIOContext(ctx context.Context) (*rados.IOContext, error)
 
-	// GetID returns the CSI-Addons VolumeGroupId of the VolumeGroup.
-	GetID(ctx context.Context) (string, error)
-
-	// GetName returns the name in the backend storage for the VolumeGroup.
-	GetName(ctx context.Context) (string, error)
-
 	// ToCSI creates a CSI-Addons type for the VolumeGroup.
 	ToCSI(ctx context.Context) (*volumegroup.VolumeGroup, error)
 

From 382d70893d5224127fa8ebfc1657756f697bb7b7 Mon Sep 17 00:00:00 2001
From: Niels de Vos <ndevos@ibm.com>
Date: Fri, 19 Jul 2024 20:22:53 +0200
Subject: [PATCH 12/14] rbd: remove the VolumeGroup from the journal on
 DeleteVolumeGroup

Signed-off-by: Niels de Vos <ndevos@ibm.com>
---
 internal/rbd/group/volume_group.go | 18 +++++++++++++
 internal/rbd/manager.go            | 42 +++++++++++++++++++++++++++---
 2 files changed, 57 insertions(+), 3 deletions(-)

diff --git a/internal/rbd/group/volume_group.go b/internal/rbd/group/volume_group.go
index 1a9e1963c4c..294a05cf06d 100644
--- a/internal/rbd/group/volume_group.go
+++ b/internal/rbd/group/volume_group.go
@@ -179,6 +179,24 @@ func (vg *volumeGroup) GetName(ctx context.Context) (string, error) {
 	return vg.name, nil
 }
 
+// GetPool returns the name of the pool that holds the VolumeGroup.
+func (vg *volumeGroup) GetPool(ctx context.Context) (string, error) {
+	if vg.pool == "" {
+		return "", errors.New("BUG: pool is not set")
+	}
+
+	return vg.pool, nil
+}
+
+// GetClusterID returns the name of the pool that holds the VolumeGroup.
+func (vg *volumeGroup) GetClusterID(ctx context.Context) (string, error) {
+	if vg.clusterID == "" {
+		return "", errors.New("BUG: clusterID is not set")
+	}
+
+	return vg.clusterID, nil
+}
+
 // ToCSI creates a CSI-Addons type for the VolumeGroup.
 func (vg *volumeGroup) ToCSI(ctx context.Context) (*volumegroup.VolumeGroup, error) {
 	volumes, err := vg.ListVolumes(ctx)
diff --git a/internal/rbd/manager.go b/internal/rbd/manager.go
index cf19e0b01b8..e487b1c269c 100644
--- a/internal/rbd/manager.go
+++ b/internal/rbd/manager.go
@@ -21,8 +21,10 @@ import (
 	"errors"
 	"fmt"
 
+	"github.com/ceph/go-ceph/rados"
+
 	"github.com/ceph/ceph-csi/internal/journal"
-	"github.com/ceph/ceph-csi/internal/rbd/group"
+	rbd_group "github.com/ceph/ceph-csi/internal/rbd/group"
 	"github.com/ceph/ceph-csi/internal/rbd/types"
 	"github.com/ceph/ceph-csi/internal/util"
 	"github.com/ceph/ceph-csi/internal/util/log"
@@ -251,6 +253,40 @@ func (mgr *rbdManager) CreateVolumeGroup(ctx context.Context, name string) (type
 }
 
 func (mgr *rbdManager) DeleteVolumeGroup(ctx context.Context, vg types.VolumeGroup) error {
-	// TODO: remove from journal
-	return vg.Delete(ctx)
+	err := vg.Delete(ctx)
+	if err != nil && !errors.Is(rados.ErrNotFound, err) {
+		return fmt.Errorf("failed to delete volume group %q: %w", vg, err)
+	}
+
+	clusterID, err := vg.GetClusterID(ctx)
+	if err != nil {
+		return fmt.Errorf("failed to get cluster id for volume group %q: %w", vg, err)
+	}
+
+	vgJournal, err := mgr.getVolumeGroupJournal(clusterID)
+	if err != nil {
+		return err
+	}
+
+	name, err := vg.GetName(ctx)
+	if err != nil {
+		return fmt.Errorf("failed to get name for volume group %q: %w", vg, err)
+	}
+
+	csiID, err := vg.GetID(ctx)
+	if err != nil {
+		return fmt.Errorf("failed to get id for volume group %q: %w", vg, err)
+	}
+
+	pool, err := vg.GetPool(ctx)
+	if err != nil {
+		return fmt.Errorf("failed to get pool for volume group %q: %w", vg, err)
+	}
+
+	err = vgJournal.UndoReservation(ctx, pool, name, csiID)
+	if err != nil /* TODO? !errors.Is(..., err) */ {
+		return fmt.Errorf("failed to undo the reservation for volume group %q: %w", vg, err)
+	}
+
+	return nil
 }

From 4acffb5548d79936f37935be37fb6b42e5709c85 Mon Sep 17 00:00:00 2001
From: Niels de Vos <ndevos@ibm.com>
Date: Mon, 22 Jul 2024 16:59:07 +0200
Subject: [PATCH 13/14] rbd: make VolumeGroup
 Create/Delete/AddVolume/RemoveVolume idempotent

Add extra error checking to make sure trying to create an existing
volume group does not result in a failure. The same counts for deleting
a non-existing volume group, and adding/removing volumes to/from the
volume group.

Signed-off-by: Niels de Vos <ndevos@ibm.com>
---
 internal/csi-addons/rbd/volumegroup.go | 13 ++++++-------
 internal/rbd/group/volume_group.go     | 21 ++++++++++++++++-----
 internal/rbd/manager.go                | 11 ++---------
 internal/rbd/types/group.go            |  2 +-
 4 files changed, 25 insertions(+), 22 deletions(-)

diff --git a/internal/csi-addons/rbd/volumegroup.go b/internal/csi-addons/rbd/volumegroup.go
index beda20d5bf1..9844c93c465 100644
--- a/internal/csi-addons/rbd/volumegroup.go
+++ b/internal/csi-addons/rbd/volumegroup.go
@@ -18,7 +18,6 @@ package rbd
 
 import (
 	"context"
-	"fmt"
 
 	"github.com/ceph/ceph-csi/internal/rbd"
 	"github.com/ceph/ceph-csi/internal/rbd/types"
@@ -103,7 +102,7 @@ func (vs *VolumeGroupServer) CreateVolumeGroup(
 		volumes[i] = vol
 	}
 
-	log.DebugLog(ctx, fmt.Sprintf("all %d Volumes for VolumeGroup %q have been found", len(volumes), req.GetName()))
+	log.DebugLog(ctx, "all %d Volumes for VolumeGroup %q have been found", len(volumes), req.GetName())
 
 	// create a RBDVolumeGroup
 	vg, err := mgr.CreateVolumeGroup(ctx, req.GetName())
@@ -115,7 +114,7 @@ func (vs *VolumeGroupServer) CreateVolumeGroup(
 			err.Error())
 	}
 
-	log.DebugLog(ctx, fmt.Sprintf("VolumeGroup %q had been created", req.GetName()))
+	log.DebugLog(ctx, "VolumeGroup %q has been created: %+v", req.GetName(), vg)
 
 	// add each rbd-image to the RBDVolumeGroup
 	for _, vol := range volumes {
@@ -130,7 +129,7 @@ func (vs *VolumeGroupServer) CreateVolumeGroup(
 		}
 	}
 
-	log.DebugLog(ctx, fmt.Sprintf("all %d Volumes have been added to for VolumeGroup %q", len(volumes), req.GetName()))
+	log.DebugLog(ctx, "all %d Volumes have been added to for VolumeGroup %q", len(volumes), req.GetName())
 
 	csiVG, err := vg.ToCSI(ctx)
 	if err != nil {
@@ -182,7 +181,7 @@ func (vs *VolumeGroupServer) DeleteVolumeGroup(
 	}
 	defer vg.Destroy(ctx)
 
-	log.DebugLog(ctx, fmt.Sprintf("VolumeGroup %q has been found", req.GetVolumeGroupId()))
+	log.DebugLog(ctx, "VolumeGroup %q has been found", req.GetVolumeGroupId())
 
 	// verify that the volume group is empty
 	volumes, err := vg.ListVolumes(ctx)
@@ -194,7 +193,7 @@ func (vs *VolumeGroupServer) DeleteVolumeGroup(
 			err.Error())
 	}
 
-	log.DebugLog(ctx, fmt.Sprintf("VolumeGroup %q contains %d volumes", req.GetVolumeGroupId(), len(volumes)))
+	log.DebugLog(ctx, "VolumeGroup %q contains %d volumes", req.GetVolumeGroupId(), len(volumes))
 
 	if len(volumes) != 0 {
 		return nil, status.Errorf(
@@ -212,7 +211,7 @@ func (vs *VolumeGroupServer) DeleteVolumeGroup(
 			err.Error())
 	}
 
-	log.DebugLog(ctx, fmt.Sprintf("VolumeGroup %q has been deleted", req.GetVolumeGroupId()))
+	log.DebugLog(ctx, "VolumeGroup %q has been deleted", req.GetVolumeGroupId())
 
 	return &volumegroup.DeleteVolumeGroupResponse{}, nil
 }
diff --git a/internal/rbd/group/volume_group.go b/internal/rbd/group/volume_group.go
index 294a05cf06d..c41db02d02e 100644
--- a/internal/rbd/group/volume_group.go
+++ b/internal/rbd/group/volume_group.go
@@ -132,7 +132,7 @@ func GetVolumeGroup(
 		volumes = append(volumes, vol)
 	}
 
-	return &volumeGroup{
+	vg := &volumeGroup{
 		journal:     j,
 		credentials: creds,
 		id:          id,
@@ -144,7 +144,11 @@ func GetVolumeGroup(
 		volumes:     volumes,
 		// all allocated volumes need to be free'd at Destroy() time
 		volumesToFree: volumes,
-	}, nil
+	}
+
+	log.DebugLog(ctx, "GetVolumeGroup(%s) returns %+v", id, *vg)
+
+	return vg, nil
 }
 
 // String makes it easy to include the volumeGroup object in log and error
@@ -303,8 +307,12 @@ func (vg *volumeGroup) Destroy(ctx context.Context) {
 	log.DebugLog(ctx, "destroyed volume group instance with id %q", vg.id)
 }
 
-func (vg *volumeGroup) Create(ctx context.Context, name string) error {
-	// TODO: if the group already exists, resolve details and use that
+func (vg *volumeGroup) Create(ctx context.Context) error {
+	name, err := vg.GetName(ctx)
+	if err != nil {
+		return fmt.Errorf("missing name to create volume group: %w", err)
+	}
+
 	ioctx, err := vg.GetIOContext(ctx)
 	if err != nil {
 		return err
@@ -319,7 +327,6 @@ func (vg *volumeGroup) Create(ctx context.Context, name string) error {
 		log.DebugLog(ctx, "ignoring error while creating volume group %q: %v", vg, err)
 	}
 
-	vg.name = name
 	log.DebugLog(ctx, "volume group %q has been created", vg)
 
 	return nil
@@ -401,6 +408,10 @@ func (vg *volumeGroup) RemoveVolume(ctx context.Context, vol types.Volume) error
 
 	err := vol.RemoveFromGroup(ctx, vg)
 	if err != nil {
+		if errors.Is(librbd.ErrNotExist, err) {
+			return nil
+		}
+
 		return fmt.Errorf("failed to remove volume %q from volume group %q: %w", vol, vg, err)
 	}
 
diff --git a/internal/rbd/manager.go b/internal/rbd/manager.go
index e487b1c269c..50e278ce63a 100644
--- a/internal/rbd/manager.go
+++ b/internal/rbd/manager.go
@@ -237,16 +237,9 @@ func (mgr *rbdManager) CreateVolumeGroup(ctx context.Context, name string) (type
 		}
 	}()
 
-	// check if the volume group exists in the backend
-	existingName, err := vg.GetName(ctx)
+	err = vg.Create(ctx)
 	if err != nil {
-		// the volume group does not exist yet
-		err = vg.Create(ctx, vgName)
-		if err != nil {
-			return nil, fmt.Errorf("failed to create volume group %q: %w", name, err)
-		}
-	} else if existingName != vgName {
-		return nil, fmt.Errorf("volume group id %q has a name mismatch, expected %q, not %q", name, vgName, existingName)
+		return nil, fmt.Errorf("failed to create volume group %q: %w", name, err)
 	}
 
 	return vg, nil
diff --git a/internal/rbd/types/group.go b/internal/rbd/types/group.go
index 322230952bc..83d0adea376 100644
--- a/internal/rbd/types/group.go
+++ b/internal/rbd/types/group.go
@@ -53,7 +53,7 @@ type VolumeGroup interface {
 	ToCSI(ctx context.Context) (*volumegroup.VolumeGroup, error)
 
 	// Create makes a new group in the backend storage.
-	Create(ctx context.Context, name string) error
+	Create(ctx context.Context) error
 
 	// Delete removes the VolumeGroup from the backend storage.
 	Delete(ctx context.Context) error

From f9ab14e82685c2d3ba4709c17823c653a58e1584 Mon Sep 17 00:00:00 2001
From: Niels de Vos <ndevos@ibm.com>
Date: Mon, 22 Jul 2024 18:18:26 +0200
Subject: [PATCH 14/14] rbd: check if an image is part of a group before adding
 it

A RBD image can only be part of a single group. While an image is added
to a group, check if the image is already part of a group, and return an
error in case it is.

Signed-off-by: Niels de Vos <ndevos@ibm.com>
---
 internal/rbd/group.go              | 23 ++++++++++++++++++++++-
 internal/rbd/group/volume_group.go |  6 ------
 2 files changed, 22 insertions(+), 7 deletions(-)

diff --git a/internal/rbd/group.go b/internal/rbd/group.go
index af59776aa08..46c7739d7d5 100644
--- a/internal/rbd/group.go
+++ b/internal/rbd/group.go
@@ -38,7 +38,28 @@ func (rv *rbdVolume) AddToGroup(ctx context.Context, vg types.VolumeGroup) error
 		return fmt.Errorf("could not get name for volume group %q: %w", vg, err)
 	}
 
-	return librbd.GroupImageAdd(ioctx, name, rv.ioctx, rv.RbdImageName)
+	// check if the image is already part of a group
+	// "rbd: ret=-17, File exists" is returned if the image is part of ANY group
+	image, err := rv.open()
+	if err != nil {
+		return fmt.Errorf("failed to open image %q: %w", rv, err)
+	}
+
+	info, err := image.GetGroup()
+	if err != nil {
+		return fmt.Errorf("could not get group information for image %q: %w", rv, err)
+	}
+
+	if info.Name != "" && info.Name != name {
+		return fmt.Errorf("image %q is already part of volume group %q", rv, info.Name)
+	}
+
+	err = librbd.GroupImageAdd(ioctx, name, rv.ioctx, rv.RbdImageName)
+	if err != nil {
+		return fmt.Errorf("failed to add image %q to volume group %q: %w", rv, vg, err)
+	}
+
+	return nil
 }
 
 // RemoveFromGroup removes the image from the group. This is called from the
diff --git a/internal/rbd/group/volume_group.go b/internal/rbd/group/volume_group.go
index c41db02d02e..e4f818b3ca4 100644
--- a/internal/rbd/group/volume_group.go
+++ b/internal/rbd/group/volume_group.go
@@ -356,12 +356,6 @@ func (vg *volumeGroup) Delete(ctx context.Context) error {
 func (vg *volumeGroup) AddVolume(ctx context.Context, vol types.Volume) error {
 	err := vol.AddToGroup(ctx, vg)
 	if err != nil {
-		// rados.ErrObjectExists does not match the rbd error (there is no EEXISTS error)
-		if errors.Is(rados.ErrObjectExists, err) || strings.Contains(err.Error(), "ret=-17") {
-			log.DebugLog(ctx, "volume %q is already part of volume group %q: %v", vol, vg, err)
-			return nil
-		}
-
 		return fmt.Errorf("failed to add volume %q to volume group %q: %w", vol, vg, err)
 	}