From 52b07b483bfc637ff06530ddbbbb95eee7e5ab6b Mon Sep 17 00:00:00 2001 From: Lalatendu Das Date: Wed, 15 May 2024 07:21:28 +0000 Subject: [PATCH] pb-6681: vendor strok master to kdmp master for PSA support - added make target called vendor to eliminate compilation issue Signed-off-by: Lalatendu Das --- Makefile | 10 ++- go.mod | 2 +- go.sum | 4 +- .../apis/stork/v1alpha1/applicationbackup.go | 10 +++ .../controllers/applicationbackup.go | 87 +++++++++++++++---- .../controllers/applicationbackupschedule.go | 14 ++- .../controllers/applicationrestore.go | 2 +- .../libopenstorage/stork/pkg/log/log.go | 7 -- vendor/modules.txt | 2 +- 9 files changed, 105 insertions(+), 33 deletions(-) diff --git a/Makefile b/Makefile index a03e275de..e226e0ae8 100644 --- a/Makefile +++ b/Makefile @@ -44,7 +44,15 @@ GO_FILES := $(shell find . -name '*.go' | grep -v 'vendor' | \ grep -v 'generated') .DEFAULT_GOAL: all -.PHONY: test deploy build container +.PHONY: test vendor vendor-update deploy build container + +vendor-update: + go mod download + +vendor: + go mod tidy + go mod vendor + sed -i '1 i\// +build skipcompile\n' vendor/kubevirt.io/client-go/kubecli/kubevirt_test_utils.go all: do-fmt pretest test-container build container diff --git a/go.mod b/go.mod index bbe16632a..7a6fbcd85 100644 --- a/go.mod +++ b/go.mod @@ -8,7 +8,7 @@ require ( github.com/hashicorp/go-version v1.6.0 github.com/kubernetes-csi/external-snapshotter/client/v4 v4.2.0 github.com/kubernetes-incubator/external-storage v0.20.4-openstorage-rc7 - github.com/libopenstorage/stork v1.4.1-0.20240513102605-2340238c7664 + github.com/libopenstorage/stork v1.4.1-0.20240515065340-81de2400f2b1 github.com/portworx/pxc v0.33.0 github.com/portworx/sched-ops v1.20.4-rc1.0.20240424153814-f3083bdb4578 github.com/sirupsen/logrus v1.9.3 diff --git a/go.sum b/go.sum index 20ff0a4db..2cf17892b 100644 --- a/go.sum +++ b/go.sum @@ -3362,8 +3362,8 @@ github.com/libopenstorage/stork v1.4.1-0.20230502135851-9cacb19e1df5/go.mod h1:R github.com/libopenstorage/stork v1.4.1-0.20230519043154-cbc10dffaf19/go.mod h1:Xm4DHoViynFXMQKBXGj3IkA77LY2RBFkNtv6vbo3wNw= github.com/libopenstorage/stork v1.4.1-0.20230601053837-5dd68f026569/go.mod h1:+mKPMCPNhS/XOF2RPcNFijkr67CCCWp0o8OXVG6xxAk= github.com/libopenstorage/stork v1.4.1-0.20230610103146-72cf75320066/go.mod h1:Yst+fnOYjWk6SA5pXZBKm19wtiinjxQ/vgYTXI3k80Q= -github.com/libopenstorage/stork v1.4.1-0.20240513102605-2340238c7664 h1:bABYni9x1xTkaIzIvfeYj1MpubMp+kjV5K0qP9k1Xb4= -github.com/libopenstorage/stork v1.4.1-0.20240513102605-2340238c7664/go.mod h1:kp5qtpq+BgjL5WqiOpDvbPH1WGReO5AaqXDbb+XpvzM= +github.com/libopenstorage/stork v1.4.1-0.20240515065340-81de2400f2b1 h1:9KerP/OdPqEkJB7lQcNOh78PVmGdrUU+KomH4f5TvxM= +github.com/libopenstorage/stork v1.4.1-0.20240515065340-81de2400f2b1/go.mod h1:R5w3M262y8SDwJEPN9Ln7Z1ffNecXkcM8Vn2w+rpdig= github.com/libopenstorage/systemutils v0.0.0-20160208220149-44ac83be3ce1/go.mod h1:xwNGC7xiz/BQ/wbMkvHujL8Gjgseg+x41xMek7sKRRQ= github.com/liggitt/tabwriter v0.0.0-20181228230101-89fcab3d43de h1:9TO3cAIGXtEhnIaL+V+BEER86oLrvS+kWobKpbJuye0= github.com/liggitt/tabwriter v0.0.0-20181228230101-89fcab3d43de/go.mod h1:zAbeS9B/r2mtpb6U+EI2rYA5OAXxsYw6wTamcNW+zcE= diff --git a/vendor/github.com/libopenstorage/stork/pkg/apis/stork/v1alpha1/applicationbackup.go b/vendor/github.com/libopenstorage/stork/pkg/apis/stork/v1alpha1/applicationbackup.go index 795d45b7c..460eff209 100644 --- a/vendor/github.com/libopenstorage/stork/pkg/apis/stork/v1alpha1/applicationbackup.go +++ b/vendor/github.com/libopenstorage/stork/pkg/apis/stork/v1alpha1/applicationbackup.go @@ -93,6 +93,13 @@ type ApplicationBackupResourceInfo struct { ObjectInfo `json:",inline"` } +// This object is used in VolumeInfo for PSA enabled cluster to retain the runAsUser ID and runAsGroup ID used by +// Job pod(KDMP/NFS)during backup. We will use the same IDs to spin up Job Pods during restore. +type VolumeJobSecurityContext struct { + RunAsUser int64 `json:"runAsUser"` + RunAsGroup int64 `json:"runAsGroup"` +} + // ApplicationBackupVolumeInfo is the info for the backup of a volume type ApplicationBackupVolumeInfo struct { PersistentVolumeClaim string `json:"persistentVolumeClaim"` @@ -110,6 +117,9 @@ type ApplicationBackupVolumeInfo struct { StorageClass string `json:"storageClass"` Provisioner string `json:"provisioner"` VolumeSnapshot string `json:"volumeSnapshot"` + // It preserves the uid and gid of the pod that is run by the backup job + // that helps in restore operation. this is required only when PSA is enforced. + VolumeJobSecurityContext VolumeJobSecurityContext `json:",inline"` } // ApplicationBackupStatusType is the status of the application backup diff --git a/vendor/github.com/libopenstorage/stork/pkg/applicationmanager/controllers/applicationbackup.go b/vendor/github.com/libopenstorage/stork/pkg/applicationmanager/controllers/applicationbackup.go index 3da6b4452..c6575d0e9 100644 --- a/vendor/github.com/libopenstorage/stork/pkg/applicationmanager/controllers/applicationbackup.go +++ b/vendor/github.com/libopenstorage/stork/pkg/applicationmanager/controllers/applicationbackup.go @@ -33,6 +33,7 @@ import ( "github.com/portworx/sched-ops/k8s/core" "github.com/portworx/sched-ops/k8s/externalsnapshotter" kdmpShedOps "github.com/portworx/sched-ops/k8s/kdmp" + "github.com/portworx/sched-ops/k8s/storage" storkops "github.com/portworx/sched-ops/k8s/stork" "github.com/sirupsen/logrus" "gocloud.dev/blob" @@ -103,7 +104,8 @@ const ( createdByValue = annotationKeyPrefix + "stork" lastUpdateKey = annotationKeyPrefix + "last-update" // optCSISnapshotClassName is an option for providing a snapshot class name - optCSISnapshotClassName = "stork.libopenstorage.org/csi-snapshot-class-name" + optCSISnapshotClassName = "stork.libopenstorage.org/csi-snapshot-class-name" + defaultVolumeSnapshotClassAnnotation = "snapshot.storage.kubernetes.io/is-default-class" ) var ( @@ -413,22 +415,14 @@ func (a *ApplicationBackupController) handle(ctx context.Context, backup *stork_ var err error - // Check whether if VolumeSnapshotClassName is given. If yes, check it's using the older way of requestParams. If yes, then migrate - // to new map in case of csi based backups - if snapshotClassName, ok := backup.Spec.Options[optCSISnapshotClassName]; ok && len(backup.Spec.CSISnapshotClassMap) == 0 { - vsc, err := externalsnapshotter.Instance().GetSnapshotClass(snapshotClassName) - if err != nil { - log.ApplicationBackupLog(backup).Errorf("Error getting volumesnapshotclass: %v", err) - a.recorder.Event(backup, - v1.EventTypeWarning, - string(stork_api.ApplicationBackupStatusFailed), - err.Error()) - return nil - } - if backup.Spec.CSISnapshotClassMap == nil { - backup.Spec.CSISnapshotClassMap = make(map[string]string) - } - backup.Spec.CSISnapshotClassMap[vsc.Driver] = vsc.Name + err = handleCSINametoCSIMapMigration(&backup.Spec) + if err != nil { + log.ApplicationBackupLog(backup).Errorf("Error creating CSISnapshotMap: %v", err) + a.recorder.Event(backup, + v1.EventTypeWarning, + string(stork_api.ApplicationBackupStatusFailed), + err.Error()) + return nil } if a.setDefaults(backup) { @@ -707,7 +701,7 @@ func (a *ApplicationBackupController) updateBackupCRInVolumeStage( backup := &stork_api.ApplicationBackup{} var err error for i := 0; i < maxRetry; i++ { - err := a.client.Get(context.TODO(), namespacedName, backup) + err = a.client.Get(context.TODO(), namespacedName, backup) if err != nil { time.Sleep(retrySleep) continue @@ -2556,3 +2550,60 @@ func (a *ApplicationBackupController) validateApplicationBackupParameters(backup } return nil } + +// handleCSINametoCSIMapMigration migrates the CSISnapshotName variable to CSISnapshotMap if "Default" is given +func handleCSINametoCSIMapMigration(spec *stork_api.ApplicationBackupSpec) error { + // Check whether if VolumeSnapshotClassName is given. If yes, check it's using the older way of requestParams. If yes, then migrate + // to new map in case of csi based backups + if spec.Options != nil { + if snapshotClassName, ok := spec.Options[optCSISnapshotClassName]; ok && len(spec.CSISnapshotClassMap) == 0 { + vsc, err := externalsnapshotter.Instance().GetSnapshotClass(snapshotClassName) + // In Case of Default given, list out all csi drivers and their snapshot classes and make a map of default vsc to its respective csi driver + // make empty map if no default vsc found + if k8s_errors.IsNotFound(err) && strings.EqualFold(snapshotClassName, "default") { + spec.CSISnapshotClassMap = make(map[string]string) + + // list all csi drivers in the cluster + drivers, dErr := storage.Instance().ListCsiDrivers() + if dErr != nil { + return dErr + } + + // list all snapshot classes in the cluster + snapshotClasses, sErr := externalsnapshotter.Instance().ListSnapshotClasses() + if sErr != nil { + return sErr + } + + for _, driver := range drivers.Items { + spec.CSISnapshotClassMap[driver.Name] = "" + onlySnapshotClass := "" + driverCount := 0 + for _, vsc := range snapshotClasses.Items { + if vsc.Driver == driver.Name { + driverCount++ + if isTrue, exist := vsc.GetAnnotations()[defaultVolumeSnapshotClassAnnotation]; isTrue == "true" && exist { + spec.CSISnapshotClassMap[driver.Name] = vsc.Name + break + } + if driverCount == 1 { + onlySnapshotClass = vsc.Name + } + } + } + if spec.CSISnapshotClassMap[driver.Name] == "" && driverCount == 1 { + spec.CSISnapshotClassMap[driver.Name] = onlySnapshotClass + } + } + } else if err != nil { + return err + } else { + if spec.CSISnapshotClassMap == nil { + spec.CSISnapshotClassMap = make(map[string]string) + } + spec.CSISnapshotClassMap[vsc.Driver] = vsc.Name + } + } + } + return nil +} diff --git a/vendor/github.com/libopenstorage/stork/pkg/applicationmanager/controllers/applicationbackupschedule.go b/vendor/github.com/libopenstorage/stork/pkg/applicationmanager/controllers/applicationbackupschedule.go index 719a496a3..1aa4ec04e 100644 --- a/vendor/github.com/libopenstorage/stork/pkg/applicationmanager/controllers/applicationbackupschedule.go +++ b/vendor/github.com/libopenstorage/stork/pkg/applicationmanager/controllers/applicationbackupschedule.go @@ -101,14 +101,24 @@ func (s *ApplicationBackupScheduleController) Reconcile(ctx context.Context, req } // Handle updates for ApplicationBackupSchedule objects -func (s *ApplicationBackupScheduleController) handle(ctx context.Context, backupSchedule *stork_api.ApplicationBackupSchedule) error { +func (s *ApplicationBackupScheduleController) handle(_ context.Context, backupSchedule *stork_api.ApplicationBackupSchedule) error { if backupSchedule.DeletionTimestamp != nil { return nil } + err := handleCSINametoCSIMapMigration(&backupSchedule.Spec.Template.Spec) + if err != nil { + log.ApplicationBackupScheduleLog(backupSchedule).Errorf("Error creating CSISnapshotMap: %v", err) + s.recorder.Event(backupSchedule, + v1.EventTypeWarning, + string(stork_api.ApplicationBackupStatusFailed), + err.Error()) + return nil + } + s.setDefaults(backupSchedule) // First update the status of any pending backups - err := s.updateApplicationBackupStatus(backupSchedule) + err = s.updateApplicationBackupStatus(backupSchedule) if err != nil { msg := fmt.Sprintf("Error updating backup status: %v", err) s.recorder.Event(backupSchedule, diff --git a/vendor/github.com/libopenstorage/stork/pkg/applicationmanager/controllers/applicationrestore.go b/vendor/github.com/libopenstorage/stork/pkg/applicationmanager/controllers/applicationrestore.go index 817233ea8..49ff79922 100644 --- a/vendor/github.com/libopenstorage/stork/pkg/applicationmanager/controllers/applicationrestore.go +++ b/vendor/github.com/libopenstorage/stork/pkg/applicationmanager/controllers/applicationrestore.go @@ -532,7 +532,7 @@ func (a *ApplicationRestoreController) updateRestoreCRInVolumeStage( restore := &storkapi.ApplicationRestore{} var err error for i := 0; i < maxRetry; i++ { - err := a.client.Get(context.TODO(), namespacedName, restore) + err = a.client.Get(context.TODO(), namespacedName, restore) if err != nil { time.Sleep(retrySleep) continue diff --git a/vendor/github.com/libopenstorage/stork/pkg/log/log.go b/vendor/github.com/libopenstorage/stork/pkg/log/log.go index c89e91df8..ac061df9a 100644 --- a/vendor/github.com/libopenstorage/stork/pkg/log/log.go +++ b/vendor/github.com/libopenstorage/stork/pkg/log/log.go @@ -5,7 +5,6 @@ import ( "fmt" "io" - defaultLogger "log" "net/http" "os" "strings" @@ -159,36 +158,30 @@ func Error(format string, args ...interface{}) { verifyDashboard() Dash.Errorf(format, args...) Dash.TestLog.Errorf(format, args...) - defaultLogger.Printf("ERROR: %s", fmt.Sprintf(format, args...)) } func Debug(msg string) { Dash.TestLog.Debug(msg) - defaultLogger.Printf("DEBUG: %s", msg) } func Debugf(format string, args ...interface{}) { Dash.TestLog.Debugf(format, args...) - defaultLogger.Printf("DEBUG: %s", fmt.Sprintf(format, args...)) } func Warn(format string, args ...interface{}) { verifyDashboard() Dash.TestLog.Warnf(format, args...) Dash.Warnf(format, args...) - defaultLogger.Printf("WARN: %s", fmt.Sprintf(format, args...)) } func Info(format string, args ...interface{}) { Dash.TestLog.Infof(format, args...) - defaultLogger.Printf("INFO: %s", fmt.Sprintf(format, args...)) } func InfoD(format string, args ...interface{}) { verifyDashboard() Dash.TestLog.Infof(format, args...) Dash.Infof(format, args...) - defaultLogger.Printf("INFO: %s", fmt.Sprintf(format, args...)) } // GetLogInstance returns the logrus instance diff --git a/vendor/modules.txt b/vendor/modules.txt index a1cfac03f..1583ccc9b 100644 --- a/vendor/modules.txt +++ b/vendor/modules.txt @@ -441,7 +441,7 @@ github.com/libopenstorage/openstorage-sdk-clients/sdk/golang github.com/libopenstorage/secrets github.com/libopenstorage/secrets/aws/credentials github.com/libopenstorage/secrets/k8s -# github.com/libopenstorage/stork v1.4.1-0.20240513102605-2340238c7664 +# github.com/libopenstorage/stork v1.4.1-0.20240515065340-81de2400f2b1 ## explicit; go 1.21 github.com/libopenstorage/stork/drivers github.com/libopenstorage/stork/drivers/volume