From f38d8bb174fa53b550e7e03e5bb9ca11cbf1e7ec Mon Sep 17 00:00:00 2001 From: Oksana Grishchenko <91597950+oksana-grishchenko@users.noreply.github.com> Date: Thu, 25 Jan 2024 17:16:20 +0200 Subject: [PATCH] EVEREST-638 DataSource validation (#405) --- api/validation.go | 128 ++++++++++++++++++++++++++++++++++------- api/validation_test.go | 68 ++++++++++++++++++++++ go.mod | 2 +- go.sum | 4 +- 4 files changed, 179 insertions(+), 23 deletions(-) diff --git a/api/validation.go b/api/validation.go index c94ce34d..f8b3cc8c 100644 --- a/api/validation.go +++ b/api/validation.go @@ -23,6 +23,7 @@ import ( "fmt" "net/url" "regexp" + "time" "github.com/Azure/azure-sdk-for-go/sdk/storage/azblob" "github.com/aws/aws-sdk-go/aws" @@ -44,6 +45,7 @@ const ( pxcDeploymentName = "percona-xtradb-cluster-operator" psmdbDeploymentName = "percona-server-mongodb-operator" pgDeploymentName = "percona-postgresql-operator" + dateFormat = "2006-01-02T15:04:05Z" ) var ( @@ -51,25 +53,32 @@ var ( minCPUQuantity = resource.MustParse("600m") //nolint:gochecknoglobals minMemQuantity = resource.MustParse("512M") //nolint:gochecknoglobals - errDBCEmptyMetadata = errors.New("databaseCluster's Metadata should not be empty") - errDBCNameEmpty = errors.New("databaseCluster's metadata.name should not be empty") - errDBCNameWrongFormat = errors.New("databaseCluster's metadata.name should be a string") - errNotEnoughMemory = fmt.Errorf("memory limits should be above %s", minMemQuantity.String()) - errInt64NotSupported = errors.New("specifying resources using int64 data type is not supported. Please use string format for that") - errNotEnoughCPU = fmt.Errorf("CPU limits should be above %s", minCPUQuantity.String()) - errNotEnoughDiskSize = fmt.Errorf("storage size should be above %s", minStorageQuantity.String()) - errUnsupportedPXCProxy = errors.New("you can use either HAProxy or Proxy SQL for PXC clusters") - errUnsupportedPGProxy = errors.New("you can use only PGBouncer as a proxy type for Postgres clusters") - errUnsupportedPSMDBProxy = errors.New("you can use only Mongos as a proxy type for MongoDB clusters") - errNoSchedules = errors.New("please specify at least one backup schedule") - errNoNameInSchedule = errors.New("'name' field for the backup schedules cannot be empty") - errScheduleNoBackupStorageName = errors.New("'backupStorageName' field cannot be empty when schedule is enabled") - errPitrNoBackupStorageName = errors.New("'backupStorageName' field cannot be empty when pitr is enabled") - errNoResourceDefined = errors.New("please specify resource limits for the cluster") - errPitrUploadInterval = errors.New("'uploadIntervalSec' should be more than 0") - errPXCPitrS3Only = errors.New("point-in-time recovery only supported for s3 compatible storages") - errPSMDBMultipleStorages = errors.New("can't use more than one backup storage for PSMDB clusters") - errPSMDBViolateActiveStorage = errors.New("can't change the active storage for PSMDB clusters") + errDBCEmptyMetadata = errors.New("databaseCluster's Metadata should not be empty") + errDBCNameEmpty = errors.New("databaseCluster's metadata.name should not be empty") + errDBCNameWrongFormat = errors.New("databaseCluster's metadata.name should be a string") + errNotEnoughMemory = fmt.Errorf("memory limits should be above %s", minMemQuantity.String()) + errInt64NotSupported = errors.New("specifying resources using int64 data type is not supported. Please use string format for that") + errNotEnoughCPU = fmt.Errorf("CPU limits should be above %s", minCPUQuantity.String()) + errNotEnoughDiskSize = fmt.Errorf("storage size should be above %s", minStorageQuantity.String()) + errUnsupportedPXCProxy = errors.New("you can use either HAProxy or Proxy SQL for PXC clusters") + errUnsupportedPGProxy = errors.New("you can use only PGBouncer as a proxy type for Postgres clusters") + errUnsupportedPSMDBProxy = errors.New("you can use only Mongos as a proxy type for MongoDB clusters") + errNoSchedules = errors.New("please specify at least one backup schedule") + errNoNameInSchedule = errors.New("'name' field for the backup schedules cannot be empty") + errScheduleNoBackupStorageName = errors.New("'backupStorageName' field cannot be empty when schedule is enabled") + errPitrNoBackupStorageName = errors.New("'backupStorageName' field cannot be empty when pitr is enabled") + errNoResourceDefined = errors.New("please specify resource limits for the cluster") + errPitrUploadInterval = errors.New("'uploadIntervalSec' should be more than 0") + errPXCPitrS3Only = errors.New("point-in-time recovery only supported for s3 compatible storages") + errPSMDBMultipleStorages = errors.New("can't use more than one backup storage for PSMDB clusters") + errPSMDBViolateActiveStorage = errors.New("can't change the active storage for PSMDB clusters") + errDataSourceConfig = errors.New("either DBClusterBackupName or BackupSource must be specified in the DataSource field") + errDataSourceNoPitrDateSpecified = errors.New("pitr Date must be specified for type Date") + errDataSourceWrongDateFormat = errors.New("failed to parse .Spec.DataSource.Pitr.Date as 2006-01-02T15:04:05Z") + errDataSourceNoBackupStorageName = errors.New("'backupStorageName' should be specified in .Spec.DataSource.BackupSource") + errDataSourceNoPath = errors.New("'path' should be specified in .Spec.DataSource.BackupSource") + errIncorrectDataSourceStruct = errors.New("incorrect data source struct") + errUnsupportedPitrType = errors.New("the given point-in-time recovery type is not supported") //nolint:gochecknoglobals operatorEngine = map[everestv1alpha1.EngineType]string{ everestv1alpha1.DatabaseEnginePXC: pxcDeploymentName, @@ -409,7 +418,7 @@ func validateCreateDatabaseClusterRequest(dbc DatabaseCluster) error { return validateRFC1035(strName, "metadata.name") } -func (e *EverestServer) validateDatabaseClusterCR(ctx echo.Context, databaseCluster *DatabaseCluster) error { +func (e *EverestServer) validateDatabaseClusterCR(ctx echo.Context, databaseCluster *DatabaseCluster) error { //nolint:cyclop if err := validateCreateDatabaseClusterRequest(*databaseCluster); err != nil { return err } @@ -446,6 +455,12 @@ func (e *EverestServer) validateDatabaseClusterCR(ctx echo.Context, databaseClus return err } + if databaseCluster.Spec.DataSource != nil { + if err := validateDBDataSource(databaseCluster); err != nil { + return err + } + } + return validateResourceLimits(databaseCluster) } @@ -615,6 +630,64 @@ func validateResourceLimits(cluster *DatabaseCluster) error { return validateStorageSize(cluster) } +func validateDBDataSource(db *DatabaseCluster) error { + bytes, err := json.Marshal(db.Spec.DataSource) + if err != nil { + return errIncorrectDataSourceStruct + } + return validateCommonDataSourceStruct(bytes) +} + +func validateRestoreDataSource(restore *DatabaseClusterRestore) error { + bytes, err := json.Marshal(restore.Spec.DataSource) + if err != nil { + return errIncorrectDataSourceStruct + } + return validateCommonDataSourceStruct(bytes) +} + +func validateCommonDataSourceStruct(data []byte) error { + // marshal and unmarshal to use the same validation func to validate DataSource for both db and restore + ds := &dataSourceStruct{} + err := json.Unmarshal(data, ds) + if err != nil { + return errIncorrectDataSourceStruct + } + return validateDataSource(*ds) +} + +func validateDataSource(dataSource dataSourceStruct) error { + if (dataSource.DbClusterBackupName == nil && dataSource.BackupSource == nil) || + (dataSource.DbClusterBackupName != nil && *dataSource.DbClusterBackupName != "" && dataSource.BackupSource != nil) { + return errDataSourceConfig + } + + if dataSource.BackupSource != nil { + if dataSource.BackupSource.BackupStorageName == "" { + return errDataSourceNoBackupStorageName + } + + if dataSource.BackupSource.Path == "" { + return errDataSourceNoPath + } + } + + if dataSource.Pitr != nil { //nolint:nestif + if dataSource.Pitr.Type == nil || *dataSource.Pitr.Type == string(DatabaseClusterSpecDataSourcePitrTypeDate) { + if dataSource.Pitr.Date == nil { + return errDataSourceNoPitrDateSpecified + } + + if _, err := time.Parse(dateFormat, *dataSource.Pitr.Date); err != nil { + return errDataSourceWrongDateFormat + } + } else { + return errUnsupportedPitrType + } + } + return nil +} + func ensureNonEmptyResources(cluster *DatabaseCluster) error { if cluster.Spec.Engine.Resources == nil { return errNoResourceDefined @@ -790,5 +863,20 @@ func validateDatabaseClusterRestore(ctx context.Context, restore *DatabaseCluste } return err } + if err = validateRestoreDataSource(restore); err != nil { + return err + } return err } + +type dataSourceStruct struct { + BackupSource *struct { + BackupStorageName string `json:"backupStorageName"` + Path string `json:"path"` + } `json:"backupSource,omitempty"` + DbClusterBackupName *string `json:"dbClusterBackupName,omitempty"` //nolint:stylecheck + Pitr *struct { + Date *string `json:"date,omitempty"` + Type *string `json:"type,omitempty"` + } `json:"pitr,omitempty"` +} diff --git a/api/validation_test.go b/api/validation_test.go index 44cc1894..4990f94b 100644 --- a/api/validation_test.go +++ b/api/validation_test.go @@ -573,6 +573,7 @@ func TestValidatePitrSpec(t *testing.T) { require.NoError(t, err) return } + require.Error(t, err) assert.Equal(t, err.Error(), tc.err.Error()) }) } @@ -648,6 +649,73 @@ func TestValidateResourceLimits(t *testing.T) { require.NoError(t, err) return } + require.Error(t, err) + assert.Equal(t, err.Error(), tc.err.Error()) + }) + } +} + +func TestValidateDataSource(t *testing.T) { + t.Parallel() + cases := []struct { + name string + cluster []byte + err error + }{ + { + name: "err none of the data source specified", + cluster: []byte(`{}`), + err: errDataSourceConfig, + }, + { + name: "err both of the data source specified", + cluster: []byte(`{"dbClusterBackupName":"some-backup", "backupSource": {"backupStorageName":"some-name","path":"some-path"}}`), + err: errDataSourceConfig, + }, + { + name: "err no date in pitr", + cluster: []byte(`{"dbClusterBackupName":"some-backup","pitr":{}}`), + err: errDataSourceNoPitrDateSpecified, + }, + { + name: "wrong pitr date format", + cluster: []byte(`{"dbClusterBackupName":"some-backup","pitr":{"date":"2006-06-07 14:06:07"}}`), + err: errDataSourceWrongDateFormat, + }, + { + name: "wrong pitr date format", + cluster: []byte(`{"dbClusterBackupName":"some-backup","pitr":{"date":""}}`), + err: errDataSourceWrongDateFormat, + }, + { + name: "correct minimal", + cluster: []byte(`{"dbClusterBackupName":"some-backup","pitr":{"date":"2006-06-07T14:06:07Z"}}`), + err: nil, + }, + { + name: "correct with pitr type", + cluster: []byte(`{"dbClusterBackupName":"some-backup","pitr":{"type":"date","date":"2006-06-07T14:06:07Z"}}`), + err: nil, + }, + { + name: "unsupported pitr type", + cluster: []byte(`{"backupSource":{"backupStorageName":"some-name","path":"some-path"},"pitr":{"type":"latest"}}`), + err: errUnsupportedPitrType, + }, + } + for _, tc := range cases { + tc := tc + t.Run(tc.name, func(t *testing.T) { + t.Parallel() + dsDB := &dataSourceStruct{} + err := json.Unmarshal(tc.cluster, dsDB) + require.NoError(t, err) + err = validateDataSource(*dsDB) + if tc.err == nil { + require.NoError(t, err) + return + } + require.Error(t, err) assert.Equal(t, err.Error(), tc.err.Error()) }) } diff --git a/go.mod b/go.mod index 944bd378..c4396a80 100644 --- a/go.mod +++ b/go.mod @@ -13,7 +13,7 @@ require ( github.com/labstack/echo/v4 v4.11.3 github.com/oapi-codegen/echo-middleware v1.0.1 github.com/oapi-codegen/runtime v1.1.0 - github.com/percona/everest-operator v0.6.0-dev1.0.20240119104008-aeb868d82769 + github.com/percona/everest-operator v0.6.0-dev1.0.20240125150540-298621412982 github.com/stretchr/testify v1.8.4 go.uber.org/zap v1.26.0 golang.org/x/crypto v0.17.0 diff --git a/go.sum b/go.sum index bd4aaf10..787cab16 100644 --- a/go.sum +++ b/go.sum @@ -422,8 +422,8 @@ github.com/onsi/gomega v1.30.0/go.mod h1:9sxs+SwGrKI0+PWe4Fxa9tFQQBG5xSsSbMXOI8P github.com/pborman/uuid v1.2.0/go.mod h1:X/NO0urCmaxf9VXbdlT7C2Yzkj2IKimNn4k+gtPdI/k= github.com/pelletier/go-toml v1.2.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/94hg7ilaic= github.com/pelletier/go-toml v1.7.0/go.mod h1:vwGMzjaWMwyfHwgIBhI2YUM4fB6nL6lVAvS1LBMMhTE= -github.com/percona/everest-operator v0.6.0-dev1.0.20240119104008-aeb868d82769 h1:IrF61ks3spy9i7r7C94pcHGwf1HL1R4+pLP9rnU0g3s= -github.com/percona/everest-operator v0.6.0-dev1.0.20240119104008-aeb868d82769/go.mod h1:o84NcJlAImYMpKK9+PIjS4V8SSREt1uZOqNhHt5qXMg= +github.com/percona/everest-operator v0.6.0-dev1.0.20240125150540-298621412982 h1:rb3XM3Ce544WoX1Z41E7R0sL4KFEuidn0fYRhHen6Lg= +github.com/percona/everest-operator v0.6.0-dev1.0.20240125150540-298621412982/go.mod h1:o84NcJlAImYMpKK9+PIjS4V8SSREt1uZOqNhHt5qXMg= github.com/percona/percona-backup-mongodb v1.8.1-0.20230920143330-3b1c2e263901 h1:BDgsZRCjEuxl2/z4yWBqB0s8d20shuIDks7/RVdZiLs= github.com/percona/percona-backup-mongodb v1.8.1-0.20230920143330-3b1c2e263901/go.mod h1:fZRCMpUqkWlLVdRKqqaj001LoVP2eo6F0ZhoMPeXDng= github.com/percona/percona-postgresql-operator v0.0.0-20231220140959-ad5eef722609 h1:+UOK4gcHrRgqjo4smgfwT7/0apF6PhAJdQIdAV4ub/M=