e2e: test vgs beta api

testing vgs beta api

Signed-off-by: Madhu Rajanna <[email protected]>

deploy/cephfs/kubernetes/csi-cephfsplugin-provisioner.yaml

@@ -145,14 +145,14 @@ spec:
             - name: socket-dir
               mountPath: /csi
         - name: csi-snapshotter
-          image: registry.k8s.io/sig-storage/csi-snapshotter:v8.0.1
+          image: quay.io/madhupr001/csi-snapshotter:latest
           args:
             - "--csi-address=$(ADDRESS)"
             - "--v=1"
             - "--timeout=150s"
             - "--leader-election=true"
             - "--extra-create-metadata=true"
-            - "--enable-volume-group-snapshots=true"
+            - "--feature-gates=CSIVolumeGroupSnapshot=true"
             - "--http-endpoint=$(POD_IP):8092"
           env:
             - name: ADDRESS
@@ -195,9 +195,7 @@ spec:
           imagePullPolicy: "IfNotPresent"
       volumes:
         - name: socket-dir
-          emptyDir: {
-            medium: "Memory"
-          }
+          emptyDir: { medium: "Memory" }
         - name: host-sys
           hostPath:
             path: /sys
@@ -214,9 +212,7 @@ spec:
           configMap:
             name: ceph-csi-config
         - name: keys-tmp-dir
-          emptyDir: {
-            medium: "Memory"
-          }
+          emptyDir: { medium: "Memory" }
         - name: ceph-csi-encryption-kms-config
           configMap:
             name: ceph-csi-encryption-kms-config

deploy/rbd/kubernetes/csi-rbdplugin-provisioner.yaml

@@ -134,14 +134,14 @@ spec:
             - name: socket-dir
               mountPath: /csi
         - name: csi-snapshotter
-          image: registry.k8s.io/sig-storage/csi-snapshotter:v8.0.1
+          image: quay.io/madhupr001/csi-snapshotter:latest
           args:
             - "--csi-address=$(ADDRESS)"
             - "--v=1"
             - "--timeout=150s"
             - "--leader-election=true"
             - "--extra-create-metadata=true"
-            - "--enable-volume-group-snapshots=true"
+            - "--feature-gates=CSIVolumeGroupSnapshot=true"
             - "--http-endpoint=$(POD_IP):8092"
           env:
             - name: ADDRESS
@@ -265,9 +265,7 @@ spec:
           hostPath:
             path: /lib/modules
         - name: socket-dir
-          emptyDir: {
-            medium: "Memory"
-          }
+          emptyDir: { medium: "Memory" }
         - name: ceph-config
           configMap:
             name: ceph-config
@@ -278,9 +276,7 @@ spec:
           configMap:
             name: ceph-csi-encryption-kms-config
         - name: keys-tmp-dir
-          emptyDir: {
-            medium: "Memory"
-          }
+          emptyDir: { medium: "Memory" }
         - name: oidc-token
           projected:
             sources:

e2e/volumegroupsnapshot.go

@@ -18,9 +18,10 @@ package e2e
 
 import (
 	"context"
+	"crypto/sha256"
 	"fmt"
 
-	groupsnapapi "github.com/kubernetes-csi/external-snapshotter/client/v8/apis/volumegroupsnapshot/v1alpha1"
+	groupsnapapi "github.com/kubernetes-csi/external-snapshotter/client/v8/apis/volumegroupsnapshot/v1beta1"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/kubernetes/test/e2e/framework"
 )
@@ -78,17 +79,29 @@ func (c *cephFSVolumeGroupSnapshot) ValidateResourcesForCreate(vgs *groupsnapapi
 		return fmt.Errorf("failed getting cephFS metadata pool name: %w", err)
 	}
 
-	sourcePVCCount := len(vgs.Status.PVCVolumeSnapshotRefList)
+	vgsc, err := c.groupclient.VolumeGroupSnapshotContents().Get(ctx, *vgs.Status.BoundVolumeGroupSnapshotContentName, metav1.GetOptions{})
+	if err != nil {
+		return fmt.Errorf("failed to get VolumeGroupSnapshotContent: %w", err)
+	}
+
+	sourcePVCCount := len(vgsc.Status.VolumeSnapshotHandlePairList)
 	// we are creating clones for each source PVC
-	clonePVCCount := len(vgs.Status.PVCVolumeSnapshotRefList)
+	clonePVCCount := len(vgsc.Status.VolumeSnapshotHandlePairList)
 	totalPVCCount := sourcePVCCount + clonePVCCount
 	validateSubvolumeCount(c.framework, totalPVCCount, fileSystemName, subvolumegroup)
 
 	// we are creating 1 snapshot for each source PVC, validate the snapshot count
-	for _, pvcSnap := range vgs.Status.PVCVolumeSnapshotRefList {
+	for _, snapshot := range vgsc.Status.VolumeSnapshotHandlePairList {
+		volumeHandle := snapshot.VolumeHandle
+		volumeSnapshotName := fmt.Sprintf("snapshot-%x", sha256.Sum256([]byte(
+			string(vgsc.UID)+volumeHandle)))
+		volumeSnapshot, err := c.snapClient.VolumeSnapshots(vgs.Namespace).Get(ctx, volumeSnapshotName, metav1.GetOptions{})
+		if err != nil {
+			return fmt.Errorf("failed to get VolumeSnapshot: %w", err)
+		}
+		pvcName := *volumeSnapshot.Spec.Source.PersistentVolumeClaimName
 		pvc, err := c.framework.ClientSet.CoreV1().PersistentVolumeClaims(vgs.Namespace).Get(ctx,
-			pvcSnap.PersistentVolumeClaimRef.Name,
-			metav1.GetOptions{})
+			pvcName, metav1.GetOptions{})
 		if err != nil {
 			return fmt.Errorf("failed to get PVC: %w", err)
 		}
@@ -165,8 +178,14 @@ func (rvgs *rbdVolumeGroupSnapshot) GetVolumeGroupSnapshotClass() (*groupsnapapi
 }
 
 func (rvgs *rbdVolumeGroupSnapshot) ValidateResourcesForCreate(vgs *groupsnapapi.VolumeGroupSnapshot) error {
-	sourcePVCCount := len(vgs.Status.PVCVolumeSnapshotRefList)
-	clonePVCCount := len(vgs.Status.PVCVolumeSnapshotRefList)
+
+	vgsc, err := rvgs.groupclient.VolumeGroupSnapshotContents().Get(context.TODO(), *vgs.Status.BoundVolumeGroupSnapshotContentName, metav1.GetOptions{})
+	if err != nil {
+		return fmt.Errorf("failed to get VolumeGroupSnapshotContent: %w", err)
+	}
+
+	sourcePVCCount := len(vgsc.Status.VolumeSnapshotHandlePairList)
+	clonePVCCount := len(vgsc.Status.VolumeSnapshotHandlePairList)
 	totalPVCCount := sourcePVCCount + clonePVCCount
 
 	validateOmapCount(rvgs.framework, totalPVCCount, rbdType, defaultRBDPool, volumesType)

e2e/volumegroupsnapshot_base.go

@@ -18,12 +18,14 @@ package e2e
 
 import (
 	"context"
+	"crypto/sha256"
 	"fmt"
 	"time"
 
-	groupsnapapi "github.com/kubernetes-csi/external-snapshotter/client/v8/apis/volumegroupsnapshot/v1alpha1"
+	groupsnapapi "github.com/kubernetes-csi/external-snapshotter/client/v8/apis/volumegroupsnapshot/v1beta1"
 	snapapi "github.com/kubernetes-csi/external-snapshotter/client/v8/apis/volumesnapshot/v1"
-	groupsnapclient "github.com/kubernetes-csi/external-snapshotter/client/v8/clientset/versioned/typed/volumegroupsnapshot/v1alpha1"
+	groupsnapclient "github.com/kubernetes-csi/external-snapshotter/client/v8/clientset/versioned/typed/volumegroupsnapshot/v1beta1"
+	snapclient "github.com/kubernetes-csi/external-snapshotter/client/v8/clientset/versioned/typed/volumesnapshot/v1"
 	v1 "k8s.io/api/core/v1"
 	apierrs "k8s.io/apimachinery/pkg/api/errors"
 	"k8s.io/apimachinery/pkg/api/resource"
@@ -75,7 +77,8 @@ type VolumeGroupSnapshotter interface {
 type volumeGroupSnapshotterBase struct {
 	timeout          int
 	framework        *framework.Framework
-	groupclient      *groupsnapclient.GroupsnapshotV1alpha1Client
+	groupclient      *groupsnapclient.GroupsnapshotV1beta1Client
+	snapClient       *snapclient.SnapshotV1Client
 	storageClassName string
 	blockPVC         bool
 	totalPVCCount    int
@@ -96,9 +99,15 @@ func newVolumeGroupSnapshotBase(f *framework.Framework, namespace,
 		return nil, fmt.Errorf("error creating group snapshot client: %w", err)
 	}
 
+	s, err := snapclient.NewForConfig(config)
+	if err != nil {
+		return nil, fmt.Errorf("error creating snapshot client: %w", err)
+	}
+
 	return &volumeGroupSnapshotterBase{
 		framework:        f,
 		groupclient:      c,
+		snapClient:       s,
 		namespace:        namespace,
 		storageClassName: storageClass,
 		blockPVC:         blockPVC,
@@ -160,14 +169,27 @@ func (v *volumeGroupSnapshotterBase) DeletePVCs(pvcs []*v1.PersistentVolumeClaim
 func (v *volumeGroupSnapshotterBase) CreatePVCClones(
 	vgs *groupsnapapi.VolumeGroupSnapshot,
 ) ([]*v1.PersistentVolumeClaim, error) {
-	pvcSnapRef := vgs.Status.PVCVolumeSnapshotRefList
+
+	groupSnapshotContent, err := v.groupclient.VolumeGroupSnapshotContents().Get(context.TODO(), *vgs.Status.BoundVolumeGroupSnapshotContentName, metav1.GetOptions{})
+
+	if err != nil {
+		return nil, fmt.Errorf("failed to get VolumeGroupSnapshotContent: %w", err)
+	}
+
 	namespace := vgs.Namespace
 	ctx := context.TODO()
-	pvcs := make([]*v1.PersistentVolumeClaim, len(pvcSnapRef))
-	for i, pvcSnap := range pvcSnapRef {
+	pvcs := make([]*v1.PersistentVolumeClaim, len(groupSnapshotContent.Status.VolumeSnapshotHandlePairList))
+	for i, snapshot := range groupSnapshotContent.Status.VolumeSnapshotHandlePairList {
+		volumeHandle := snapshot.VolumeHandle
+		volumeSnapshotName := fmt.Sprintf("snapshot-%x", sha256.Sum256([]byte(
+			string(groupSnapshotContent.UID)+volumeHandle)))
+		volumeSnapshot, err := v.snapClient.VolumeSnapshots(namespace).Get(ctx, volumeSnapshotName, metav1.GetOptions{})
+		if err != nil {
+			return nil, fmt.Errorf("failed to get VolumeSnapshot: %w", err)
+		}
+		pvcName := *volumeSnapshot.Spec.Source.PersistentVolumeClaimName
 		pvc, err := v.framework.ClientSet.CoreV1().PersistentVolumeClaims(namespace).Get(ctx,
-			pvcSnap.PersistentVolumeClaimRef.Name,
-			metav1.GetOptions{})
+			pvcName, metav1.GetOptions{})
 		if err != nil {
 			return nil, fmt.Errorf("failed to get PVC: %w", err)
 		}
@@ -179,12 +201,11 @@ func (v *volumeGroupSnapshotterBase) CreatePVCClones(
 			Spec: *pvc.Spec.DeepCopy(),
 		}
 
-		snap := pvcSnap.VolumeSnapshotRef
 		apiGroup := snapapi.GroupName
 		pvcs[i].Spec.DataSource = &v1.TypedLocalObjectReference{
 			APIGroup: &apiGroup,
 			Kind:     "VolumeSnapshot",
-			Name:     snap.Name,
+			Name:     volumeSnapshot.Name,
 		}
 		pvcs[i].Spec.StorageClassName = &v.storageClassName
 		// cleanup the VolumeName as we are creating a new PVC

scripts/install-snapshot.sh

@@ -25,9 +25,9 @@ VOLUME_SNAPSHOT_CONTENT="${SNAPSHOTTER_URL}/client/config/crd/snapshot.storage.k
 VOLUME_SNAPSHOT="${SNAPSHOTTER_URL}/client/config/crd/snapshot.storage.k8s.io_volumesnapshots.yaml"
 
 # volumegroupsnapshot CRD
-VOLUME_GROUP_SNAPSHOTCLASS="${SNAPSHOTTER_URL}/client/config/crd/groupsnapshot.storage.k8s.io_volumegroupsnapshotclasses.yaml"
-VOLUME_GROUP_SNAPSHOT_CONTENT="${SNAPSHOTTER_URL}/client/config/crd/groupsnapshot.storage.k8s.io_volumegroupsnapshotcontents.yaml"
-VOLUME_GROUP_SNAPSHOT="${SNAPSHOTTER_URL}/client/config/crd/groupsnapshot.storage.k8s.io_volumegroupsnapshots.yaml"
+VOLUME_GROUP_SNAPSHOTCLASS=https://github.com/kubernetes-csi/external-snapshotter/blob/a4754736ebb2dc7475b28d1e4bf7a68e330070fe/client/config/crd/groupsnapshot.storage.k8s.io_volumegroupsnapshotclasses.yaml
+VOLUME_GROUP_SNAPSHOT_CONTENT=https://github.com/kubernetes-csi/external-snapshotter/blob/a4754736ebb2dc7475b28d1e4bf7a68e330070fe/client/config/crd/groupsnapshot.storage.k8s.io_volumegroupsnapshotcontents.yaml
+VOLUME_GROUP_SNAPSHOT=https://github.com/kubernetes-csi/external-snapshotter/blob/a4754736ebb2dc7475b28d1e4bf7a68e330070fe/client/config/crd/groupsnapshot.storage.k8s.io_volumegroupsnapshots.yaml
 
 function install_snapshot_controller() {
     local namespace=$1
@@ -74,21 +74,14 @@ function create_or_delete_resource() {
     curl -o "${temp_snap_controller}" "${SNAPSHOT_CONTROLLER}"
     sed -i "s/namespace: kube-system/namespace: ${namespace}/g" "${temp_rbac}"
     sed -i "s/namespace: kube-system/namespace: ${namespace}/g" "${temp_snap_controller}"
-    sed -i -E "s/(image: registry\.k8s\.io\/sig-storage\/snapshot-controller:).*$/\1$SNAPSHOT_VERSION/g" "${temp_snap_controller}"
+    sed -i 's|image:.*|image: quay.io/madhupr001/csi-snapshotter:latest|' "${temp_snap_controller}"
 
     if [ "${operation}" == "create" ]; then
         # Argument to add/update
-        ARGUMENT="--enable-volume-group-snapshots=true"
+        ARGUMENT="--feature-gates=CSIVolumeGroupSnapshot=true"
         # Check if the argument is already present and set to false
-        if grep -q -E "^\s+-\s+--enable-volume-group-snapshots=false" "${temp_snap_controller}"; then
-            sed -i -E "s/^\s+-\s+--enable-volume-group-snapshots=false$/      - $ARGUMENT/" "${temp_snap_controller}"
-            # Check if the argument is already present and set to true
-        elif grep -q -E "^\s+-\s+--enable-volume-group-snapshots=true" "${temp_snap_controller}"; then
-            echo "Argument already present and matching."
-        else
-            # Add the argument if it's not present
-            sed -i -E "/^(\s+)args:/a\           \ - $ARGUMENT" "${temp_snap_controller}"
-        fi
+        # Add the argument if it's not present
+        sed -i -E "/^(\s+)args:/a\           \ - $ARGUMENT" "${temp_snap_controller}"
     fi
 
     kubectl_retry "${operation}" -f "${VOLUME_GROUP_SNAPSHOTCLASS}"