Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Upgrade ramen to latest snapshotter crd and enforce VGS name length limit #1554

Merged
merged 3 commits into from
Sep 17, 2024
Merged

Upgrade ramen to latest snapshotter crd and enforce VGS name length limit #1554

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
f51b389
Update Ramen to use latest CRD from external-snapshotter
Sep 10, 2024
643e4dd
Ensure VGS name contains at most 63 characters
Sep 12, 2024
24650ae
Ensure the Secondary VRG is updated when DRPC is updated
Sep 12, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 2 additions & 2 deletions cmd/main.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -15,8 +15,8 @@ import (

volsyncv1alpha1 "github.com/backube/volsync/api/v1alpha1"
volrep "github.com/csi-addons/kubernetes-csi-addons/api/replication.storage/v1alpha1"
groupsnapv1alpha1 "github.com/kubernetes-csi/external-snapshotter/client/v7/apis/volumegroupsnapshot/v1alpha1"
snapv1 "github.com/kubernetes-csi/external-snapshotter/client/v7/apis/volumesnapshot/v1"
groupsnapv1alpha1 "github.com/kubernetes-csi/external-snapshotter/client/v8/apis/volumegroupsnapshot/v1alpha1"
snapv1 "github.com/kubernetes-csi/external-snapshotter/client/v8/apis/volumesnapshot/v1"
viewv1beta1 "github.com/stolostron/multicloud-operators-foundation/pkg/apis/view/v1beta1"
plrv1 "github.com/stolostron/multicloud-operators-placementrule/pkg/apis/apps/v1"
velero "github.com/vmware-tanzu/velero/pkg/apis/velero/v1"
Expand Down
2 changes: 1 addition & 1 deletion go.mod
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -11,7 +11,7 @@ require (
github.com/csi-addons/kubernetes-csi-addons v0.8.1-0.20240822090723-89d4c5b45a32
github.com/go-logr/logr v1.4.2
github.com/google/uuid v1.6.0
github.com/kubernetes-csi/external-snapshotter/client/v7 v7.0.0
github.com/kubernetes-csi/external-snapshotter/client/v8 v8.0.0
github.com/onsi/ginkgo/v2 v2.20.0
github.com/onsi/gomega v1.34.1
github.com/operator-framework/api v0.17.6
Expand Down
4 changes: 2 additions & 2 deletions go.sum
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -196,8 +196,8 @@ github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY=
github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE=
github.com/kubernetes-csi/external-snapshotter/client/v6 v6.2.0 h1:cMM5AB37e9aRGjErygVT6EuBPB6s5a+l95OPERmSlVM=
github.com/kubernetes-csi/external-snapshotter/client/v6 v6.2.0/go.mod h1:VQVLCPGDX5l6V5PezjlDXLa+SpCbWSVU7B16cFWVVeE=
github.com/kubernetes-csi/external-snapshotter/client/v7 v7.0.0 h1:j3YK74myEQRxR/srciTpOrm221SAvz6J5OVWbyfeXFo=
github.com/kubernetes-csi/external-snapshotter/client/v7 v7.0.0/go.mod h1:FlyYFe32mPxKEPaRXKNxfX576d1AoCzstYDoOOnyMA4=
github.com/kubernetes-csi/external-snapshotter/client/v8 v8.0.0 h1:mjQG0Vakr2h246kEDR85U8y8ZhPgT3bguTCajRa/jaw=
github.com/kubernetes-csi/external-snapshotter/client/v8 v8.0.0/go.mod h1:E3vdYxHj2C2q6qo8/Da4g7P+IcwqRZyy3gJBzYybV9Y=
github.com/magiconair/properties v1.8.7 h1:IeQXZAiQcpL9mgcAe1Nu6cX9LLw6ExEHKjN0VQdvPDY=
github.com/magiconair/properties v1.8.7/go.mod h1:Dhd985XPs7jluiymwWYZ0G4Z61jb3vdS329zhj2hYo0=
github.com/mailru/easyjson v0.7.7 h1:UGYAvKxe3sBsEDzO8ZeWOSlIQfWFlxbzLZe7hwFURr0=
Expand Down
3 changes: 1 addition & 2 deletions hack/test/groupsnapshot.storage.k8s.io_volumegroupsnapshotclasses.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,4 +1,3 @@

---
apiVersion: apiextensions.k8s.io/v1
kind: CustomResourceDefinition
Expand Down Expand Up @@ -92,4 +91,4 @@ spec:
type: object
served: true
storage: true
subresources: {}
subresources: {}
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

@BenamarMk , this file is not actually changed?

Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Ok, thanks. I'll remove it.

291 changes: 149 additions & 142 deletions hack/test/groupsnapshot.storage.k8s.io_volumegroupsnapshots.yaml
View file
Open in desktop

Large diffs are not rendered by default.

4 changes: 2 additions & 2 deletions internal/controller/cephfscg/cephfscg_suite_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -9,8 +9,8 @@ import (

volsyncv1alpha1 "github.com/backube/volsync/api/v1alpha1"
"github.com/go-logr/logr"
groupsnapv1alpha1 "github.com/kubernetes-csi/external-snapshotter/client/v7/apis/volumegroupsnapshot/v1alpha1"
snapv1 "github.com/kubernetes-csi/external-snapshotter/client/v7/apis/volumesnapshot/v1"
groupsnapv1alpha1 "github.com/kubernetes-csi/external-snapshotter/client/v8/apis/volumegroupsnapshot/v1alpha1"
snapv1 "github.com/kubernetes-csi/external-snapshotter/client/v8/apis/volumesnapshot/v1"
. "github.com/onsi/ginkgo/v2"
. "github.com/onsi/gomega"
"github.com/onsi/gomega/format"
Expand Down
6 changes: 3 additions & 3 deletions internal/controller/cephfscg/cghandler.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -6,7 +6,7 @@ import (

volsyncv1alpha1 "github.com/backube/volsync/api/v1alpha1"
"github.com/go-logr/logr"
snapv1 "github.com/kubernetes-csi/external-snapshotter/client/v7/apis/volumesnapshot/v1"
snapv1 "github.com/kubernetes-csi/external-snapshotter/client/v8/apis/volumesnapshot/v1"
ramendrv1alpha1 "github.com/ramendr/ramen/api/v1alpha1"
"github.com/ramendr/ramen/internal/controller/util"
"github.com/ramendr/ramen/internal/controller/volsync"
Expand Down Expand Up @@ -95,7 +95,7 @@ func (c *cgHandler) CreateOrUpdateReplicationGroupDestination(
replicationGroupDestinationName, replicationGroupDestinationNamespace string,
rdSpecsInGroup []ramendrv1alpha1.VolSyncReplicationDestinationSpec,
) (*ramendrv1alpha1.ReplicationGroupDestination, error) {
replicationGroupDestinationName += c.cgName
replicationGroupDestinationName = util.TrimToK8sResourceNameLength(replicationGroupDestinationName + c.cgName)

log := c.logger.WithName("CreateOrUpdateReplicationGroupDestination").
WithValues("ReplicationGroupDestinationName", replicationGroupDestinationName,
Expand Down Expand Up @@ -144,7 +144,7 @@ func (c *cgHandler) CreateOrUpdateReplicationGroupSource(
replicationGroupSourceName, replicationGroupSourceNamespace string,
runFinalSync bool,
) (*ramendrv1alpha1.ReplicationGroupSource, bool, error) {
replicationGroupSourceName += c.cgName
replicationGroupSourceName = util.TrimToK8sResourceNameLength(replicationGroupSourceName + c.cgName)

log := c.logger.WithName("CreateOrUpdateReplicationGroupSource").
WithValues("ReplicationGroupSourceName", replicationGroupSourceName,
Expand Down
4 changes: 2 additions & 2 deletions internal/controller/cephfscg/cghandler_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -4,8 +4,8 @@ import (
"context"

volsyncv1alpha1 "github.com/backube/volsync/api/v1alpha1"
groupsnapv1alpha1 "github.com/kubernetes-csi/external-snapshotter/client/v7/apis/volumegroupsnapshot/v1alpha1"
vsv1 "github.com/kubernetes-csi/external-snapshotter/client/v7/apis/volumesnapshot/v1"
groupsnapv1alpha1 "github.com/kubernetes-csi/external-snapshotter/client/v8/apis/volumegroupsnapshot/v1alpha1"
vsv1 "github.com/kubernetes-csi/external-snapshotter/client/v8/apis/volumesnapshot/v1"
. "github.com/onsi/ginkgo/v2"
. "github.com/onsi/gomega"
ramendrv1alpha1 "github.com/ramendr/ramen/api/v1alpha1"
Expand Down
2 changes: 1 addition & 1 deletion internal/controller/cephfscg/replicationgroupdestination_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -6,7 +6,7 @@ import (

volsyncv1alpha1 "github.com/backube/volsync/api/v1alpha1"
"github.com/backube/volsync/controllers/statemachine"
snapv1 "github.com/kubernetes-csi/external-snapshotter/client/v7/apis/volumesnapshot/v1"
snapv1 "github.com/kubernetes-csi/external-snapshotter/client/v8/apis/volumesnapshot/v1"
. "github.com/onsi/ginkgo/v2"
. "github.com/onsi/gomega"
corev1 "k8s.io/api/core/v1"
Expand Down
106 changes: 29 additions & 77 deletions internal/controller/cephfscg/volumegroupsourcehandler.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -7,8 +7,8 @@ import (

volsyncv1alpha1 "github.com/backube/volsync/api/v1alpha1"
"github.com/go-logr/logr"
vgsv1alphfa1 "github.com/kubernetes-csi/external-snapshotter/client/v7/apis/volumegroupsnapshot/v1alpha1"
vsv1 "github.com/kubernetes-csi/external-snapshotter/client/v7/apis/volumesnapshot/v1"
vgsv1alphfa1 "github.com/kubernetes-csi/external-snapshotter/client/v8/apis/volumegroupsnapshot/v1alpha1"
vsv1 "github.com/kubernetes-csi/external-snapshotter/client/v8/apis/volumesnapshot/v1"
ramendrv1alpha1 "github.com/ramendr/ramen/api/v1alpha1"
"github.com/ramendr/ramen/internal/controller/util"
"github.com/ramendr/ramen/internal/controller/volsync"
Expand Down Expand Up @@ -84,16 +84,18 @@ func NewVolumeGroupSourceHandler(
) VolumeGroupSourceHandler {
vrgName := rgs.GetLabels()[volsync.VRGOwnerNameLabel]

vgsName := util.TrimToK8sResourceNameLength(fmt.Sprintf(VolumeGroupSnapshotNameFormat, rgs.Name))

return &volumeGroupSourceHandler{
Client: client,
VolumeGroupSnapshotName: fmt.Sprintf(VolumeGroupSnapshotNameFormat, rgs.Name),
VolumeGroupSnapshotName: vgsName,
VolumeGroupSnapshotNamespace: rgs.Namespace,
VolumeGroupSnapshotClassName: rgs.Spec.VolumeGroupSnapshotClassName,
VolumeGroupLabel: rgs.Spec.VolumeGroupSnapshotSource,
VolsyncKeySecretName: volsync.GetVolSyncPSKSecretNameFromVRGName(vrgName),
DefaultCephFSCSIDriverName: defaultCephFSCSIDriverName,
Logger: logger.WithName("VolumeGroupSourceHandler").
WithValues("VolumeGroupSnapshotName", fmt.Sprintf(VolumeGroupSnapshotNameFormat, rgs.Name)).
WithValues("VolumeGroupSnapshotName", vgsName).
WithValues("VolumeGroupSnapshotNamespace", rgs.Namespace),
}
}
Expand Down Expand Up @@ -136,7 +138,7 @@ func (h *volumeGroupSourceHandler) CreateOrUpdateVolumeGroupSnapshot(
return err
}

logger.Info("VolumeGroupSnapshot successfully be created or updated", "operation", op)
logger.Info("VolumeGroupSnapshot successfully created or updated", "operation", op)

return nil
}
Expand Down Expand Up @@ -166,20 +168,21 @@ func (h *volumeGroupSourceHandler) CleanVolumeGroupSnapshot(
}

if volumeGroupSnapshot.Status != nil {
for _, vsRef := range volumeGroupSnapshot.Status.VolumeSnapshotRefList {
for _, pvcVSRef := range volumeGroupSnapshot.Status.PVCVolumeSnapshotRefList {
logger.Info("Get PVCName from volume snapshot",
"VolumeSnapshotName", vsRef.Name, "VolumeSnapshotNamespace", vsRef.Namespace)
"vsName", pvcVSRef.VolumeSnapshotRef.Name, "vsNamespace", volumeGroupSnapshot.Namespace)

pvc, err := GetPVCFromVolumeSnapshot(ctx, h.Client, vsRef.Name, vsRef.Namespace, volumeGroupSnapshot)
pvc, err := util.GetPVC(ctx, h.Client,
types.NamespacedName{Name: pvcVSRef.PersistentVolumeClaimRef.Name, Namespace: volumeGroupSnapshot.Namespace})
if err != nil {
logger.Error(err, "Failed to get PVC name from volume snapshot",
"VolumeSnapshotName", vsRef.Name, "VolumeSnapshotNamespace", vsRef.Namespace)
"pvcName", pvcVSRef.PersistentVolumeClaimRef.Name, "vsNamespace", volumeGroupSnapshot.Namespace)

return err
}

restoredPVCName := fmt.Sprintf(RestorePVCinCGNameFormat, pvc.Name)
restoredPVCNamespace := vsRef.Namespace
restoredPVCNamespace := pvc.Namespace

logger.Info("Delete restored PVCs", "PVCName", restoredPVCName, "PVCNamespace", restoredPVCNamespace)

Expand Down Expand Up @@ -229,28 +232,32 @@ func (h *volumeGroupSourceHandler) RestoreVolumesFromVolumeGroupSnapshot(

restoredPVCs := []RestoredPVC{}

for _, vsRef := range volumeGroupSnapshot.Status.VolumeSnapshotRefList {
for _, pvcVSRef := range volumeGroupSnapshot.Status.PVCVolumeSnapshotRefList {
logger.Info("Get PVCName from volume snapshot",
"VolumeSnapshotName", vsRef.Name, "VolumeSnapshotNamespace", vsRef.Namespace)
"PVCName", pvcVSRef.PersistentVolumeClaimRef.Name, "VolumeSnapshotName", pvcVSRef.VolumeSnapshotRef.Name)

pvc, err := GetPVCFromVolumeSnapshot(ctx, h.Client, vsRef.Name, vsRef.Namespace, volumeGroupSnapshot)
pvc, err := util.GetPVC(ctx, h.Client,
types.NamespacedName{Name: pvcVSRef.PersistentVolumeClaimRef.Name, Namespace: volumeGroupSnapshot.Namespace})
if err != nil {
return nil, fmt.Errorf("failed to get PVC name from volume snapshot %s: %w", vsRef.Namespace+"/"+vsRef.Name, err)
return nil, fmt.Errorf("failed to get PVC from VGS %s: %w",
volumeGroupSnapshot.Namespace+"/"+pvcVSRef.PersistentVolumeClaimRef.Name, err)
}

restoreStorageClass, err := GetRestoreStorageClass(ctx, h.Client,
*pvc.Spec.StorageClassName, h.DefaultCephFSCSIDriverName)
if err != nil {
return nil, fmt.Errorf("failed to get Restore Storage Class from PVC %s: %w", pvc.Name+"/"+vsRef.Namespace, err)
return nil, fmt.Errorf("failed to get Restore Storage Class from PVC %s: %w", pvc.Name+"/"+pvc.Namespace, err)
}

RestoredPVCNamespacedName := types.NamespacedName{
Namespace: vsRef.Namespace,
Namespace: pvc.Namespace,
Name: fmt.Sprintf(RestorePVCinCGNameFormat, pvc.Name),
}
if err := h.RestoreVolumesFromSnapshot(
ctx, vsRef, pvc, RestoredPVCNamespacedName, restoreStorageClass.GetName(), owner); err != nil {
return nil, fmt.Errorf("failed to restore volumes from snapshot %s: %w", vsRef.Name+"/"+vsRef.Namespace, err)
ctx, pvcVSRef.VolumeSnapshotRef.Name, pvc, RestoredPVCNamespacedName,
restoreStorageClass.GetName(), owner); err != nil {
return nil, fmt.Errorf("failed to restore volumes from snapshot %s: %w",
pvcVSRef.VolumeSnapshotRef.Name+"/"+pvc.Namespace, err)
}

logger.Info("Successfully restore volumes from snapshot",
Expand All @@ -259,7 +266,7 @@ func (h *volumeGroupSourceHandler) RestoreVolumesFromVolumeGroupSnapshot(
restoredPVCs = append(restoredPVCs, RestoredPVC{
SourcePVCName: pvc.Name,
RestoredPVCName: RestoredPVCNamespacedName.Name,
VolumeSnapshotName: vsRef.Name,
VolumeSnapshotName: pvcVSRef.VolumeSnapshotRef.Name,
})
}

Expand All @@ -273,7 +280,7 @@ func (h *volumeGroupSourceHandler) RestoreVolumesFromVolumeGroupSnapshot(
//nolint:funlen,gocognit,cyclop
func (h *volumeGroupSourceHandler) RestoreVolumesFromSnapshot(
ctx context.Context,
vsRef corev1.ObjectReference,
vsName string,
pvc *corev1.PersistentVolumeClaim,
restoredPVCNamespacedname types.NamespacedName,
restoreStorageClassName string,
Expand All @@ -285,13 +292,13 @@ func (h *volumeGroupSourceHandler) RestoreVolumesFromSnapshot(

volumeSnapshot := &vsv1.VolumeSnapshot{}
if err := h.Client.Get(ctx,
types.NamespacedName{Name: vsRef.Name, Namespace: vsRef.Namespace},
types.NamespacedName{Name: vsName, Namespace: pvc.Namespace},
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Maybe it's better to look for volume snapshot by vs name and vs namespace? What if one day vs namespace will be different from pvc namespace?

Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

The VolumeGroupSnapshot status looks like this:

status:
  boundVolumeGroupSnapshotContentName: groupsnapcontent-66e442b9-d739-48cc-9158-a6472c3a2022
  creationTime: "2024-09-13T13:05:00Z"
  pvcVolumeSnapshotRefList:
  - persistentVolumeClaimRef:
      name: busybox-cg2-pvc1
    volumeSnapshotRef:
      name: snapshot-bc3b0f7a8da345fd3d423eebe43cf6023ad34b03d7ff1df3fdf227da330303e6-2024-09-13-1.5.0
  - persistentVolumeClaimRef:
      name: busybox-cg2-pvc2
    volumeSnapshotRef:
      name: snapshot-811a028ea7008016c74c082430686955ae178d56cc4f0fdc3e5394da2cc20a1f-2024-09-13-1.5.3
  readyToUse: true

We have only the names in the status. The snapshot is for the PVC and we already have the PVC for it. Therefore we use the PVC namespace. The PVC namespace is the same as the VolumeGroupSnapshot.

volumeSnapshot,
); err != nil {
return fmt.Errorf("failed to get volume snapshot: %w", err)
}

snapshotRef := corev1.TypedLocalObjectReference{Name: vsRef.Name, APIGroup: &SnapshotGroup, Kind: SnapshotGroupKind}
snapshotRef := corev1.TypedLocalObjectReference{Name: vsName, APIGroup: &SnapshotGroup, Kind: SnapshotGroupKind}
restoredPVC := &corev1.PersistentVolumeClaim{
ObjectMeta: metav1.ObjectMeta{
Name: restoredPVCNamespacedname.Name,
Expand Down Expand Up @@ -497,61 +504,6 @@ func (h *volumeGroupSourceHandler) CheckReplicationSourceForRestoredPVCsComplete
return true, nil
}

var GetPVCFromVolumeSnapshot func(
ctx context.Context, k8sClient client.Client, vsName string,
vsNamespace string, vgs *vgsv1alphfa1.VolumeGroupSnapshot,
) (*corev1.PersistentVolumeClaim, error)

func init() {
GetPVCFromVolumeSnapshot = FakeGetPVCFromVolumeSnapshot
}

// TODO(wangyouhang): https://github.com/kubernetes-csi/external-snapshotter/issues/969
// Fake func, need to be changed
func FakeGetPVCFromVolumeSnapshot(
ctx context.Context, k8sClient client.Client, vsName string,
vsNamespace string, vgs *vgsv1alphfa1.VolumeGroupSnapshot,
) (*corev1.PersistentVolumeClaim, error) {
if vgs.Status.BoundVolumeGroupSnapshotContentName == nil {
return nil, fmt.Errorf("BoundVolumeGroupSnapshotContentName is nil")
}

// get vs index in vgs
var index int

for i, VolumeSnapshotRef := range vgs.Status.VolumeSnapshotRefList {
if VolumeSnapshotRef.Name == vsName && VolumeSnapshotRef.Namespace == vsNamespace {
index = i
}
}

// get storageHandle based on index
vgsc := &vgsv1alphfa1.VolumeGroupSnapshotContent{}

err := k8sClient.Get(ctx,
types.NamespacedName{
Name: *vgs.Status.BoundVolumeGroupSnapshotContentName,
Namespace: vgs.GetNamespace(),
},
vgsc)
if err != nil {
return nil, err
}

if len(vgs.Status.VolumeSnapshotRefList) != len(vgsc.Spec.Source.VolumeHandles) {
return nil, fmt.Errorf("len of vgs.Status.VolumeSnapshotRefList != len of vgsc.Spec.Source.VolumeHandles")
}

storageHandle := vgsc.Spec.Source.VolumeHandles[index]

pvc, err := GetPVCfromStorageHandle(ctx, k8sClient, storageHandle)
if err != nil {
return nil, fmt.Errorf("PVC is not found with storageHandle %s: %w", storageHandle, err)
}

return pvc, nil
}

func GetPVCfromStorageHandle(
ctx context.Context,
k8sClient client.Client,
Expand Down
Loading
Loading