Skip to content
This repository has been archived by the owner on Mar 4, 2024. It is now read-only.

Commit

Permalink
EVEREST-638 DataSource validation (#405)
Browse files Browse the repository at this point in the history
  • Loading branch information
@oksana-grishchenko
oksana-grishchenko authored Jan 25, 2024
1 parent 9c8e5d2 commit f38d8bb
Show file tree
Hide file tree
Showing 4 changed files with 179 additions and 23 deletions.
128 changes: 108 additions & 20 deletions api/validation.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -23,6 +23,7 @@ import (
"fmt"
"net/url"
"regexp"
"time"

"github.com/Azure/azure-sdk-for-go/sdk/storage/azblob"
"github.com/aws/aws-sdk-go/aws"
Expand All @@ -44,32 +45,40 @@ const (
pxcDeploymentName = "percona-xtradb-cluster-operator"
psmdbDeploymentName = "percona-server-mongodb-operator"
pgDeploymentName = "percona-postgresql-operator"
dateFormat = "2006-01-02T15:04:05Z"
)

var (
minStorageQuantity = resource.MustParse("1G") //nolint:gochecknoglobals
minCPUQuantity = resource.MustParse("600m") //nolint:gochecknoglobals
minMemQuantity = resource.MustParse("512M") //nolint:gochecknoglobals

errDBCEmptyMetadata = errors.New("databaseCluster's Metadata should not be empty")
errDBCNameEmpty = errors.New("databaseCluster's metadata.name should not be empty")
errDBCNameWrongFormat = errors.New("databaseCluster's metadata.name should be a string")
errNotEnoughMemory = fmt.Errorf("memory limits should be above %s", minMemQuantity.String())
errInt64NotSupported = errors.New("specifying resources using int64 data type is not supported. Please use string format for that")
errNotEnoughCPU = fmt.Errorf("CPU limits should be above %s", minCPUQuantity.String())
errNotEnoughDiskSize = fmt.Errorf("storage size should be above %s", minStorageQuantity.String())
errUnsupportedPXCProxy = errors.New("you can use either HAProxy or Proxy SQL for PXC clusters")
errUnsupportedPGProxy = errors.New("you can use only PGBouncer as a proxy type for Postgres clusters")
errUnsupportedPSMDBProxy = errors.New("you can use only Mongos as a proxy type for MongoDB clusters")
errNoSchedules = errors.New("please specify at least one backup schedule")
errNoNameInSchedule = errors.New("'name' field for the backup schedules cannot be empty")
errScheduleNoBackupStorageName = errors.New("'backupStorageName' field cannot be empty when schedule is enabled")
errPitrNoBackupStorageName = errors.New("'backupStorageName' field cannot be empty when pitr is enabled")
errNoResourceDefined = errors.New("please specify resource limits for the cluster")
errPitrUploadInterval = errors.New("'uploadIntervalSec' should be more than 0")
errPXCPitrS3Only = errors.New("point-in-time recovery only supported for s3 compatible storages")
errPSMDBMultipleStorages = errors.New("can't use more than one backup storage for PSMDB clusters")
errPSMDBViolateActiveStorage = errors.New("can't change the active storage for PSMDB clusters")
errDBCEmptyMetadata = errors.New("databaseCluster's Metadata should not be empty")
errDBCNameEmpty = errors.New("databaseCluster's metadata.name should not be empty")
errDBCNameWrongFormat = errors.New("databaseCluster's metadata.name should be a string")
errNotEnoughMemory = fmt.Errorf("memory limits should be above %s", minMemQuantity.String())
errInt64NotSupported = errors.New("specifying resources using int64 data type is not supported. Please use string format for that")
errNotEnoughCPU = fmt.Errorf("CPU limits should be above %s", minCPUQuantity.String())
errNotEnoughDiskSize = fmt.Errorf("storage size should be above %s", minStorageQuantity.String())
errUnsupportedPXCProxy = errors.New("you can use either HAProxy or Proxy SQL for PXC clusters")
errUnsupportedPGProxy = errors.New("you can use only PGBouncer as a proxy type for Postgres clusters")
errUnsupportedPSMDBProxy = errors.New("you can use only Mongos as a proxy type for MongoDB clusters")
errNoSchedules = errors.New("please specify at least one backup schedule")
errNoNameInSchedule = errors.New("'name' field for the backup schedules cannot be empty")
errScheduleNoBackupStorageName = errors.New("'backupStorageName' field cannot be empty when schedule is enabled")
errPitrNoBackupStorageName = errors.New("'backupStorageName' field cannot be empty when pitr is enabled")
errNoResourceDefined = errors.New("please specify resource limits for the cluster")
errPitrUploadInterval = errors.New("'uploadIntervalSec' should be more than 0")
errPXCPitrS3Only = errors.New("point-in-time recovery only supported for s3 compatible storages")
errPSMDBMultipleStorages = errors.New("can't use more than one backup storage for PSMDB clusters")
errPSMDBViolateActiveStorage = errors.New("can't change the active storage for PSMDB clusters")
errDataSourceConfig = errors.New("either DBClusterBackupName or BackupSource must be specified in the DataSource field")
errDataSourceNoPitrDateSpecified = errors.New("pitr Date must be specified for type Date")
errDataSourceWrongDateFormat = errors.New("failed to parse .Spec.DataSource.Pitr.Date as 2006-01-02T15:04:05Z")
errDataSourceNoBackupStorageName = errors.New("'backupStorageName' should be specified in .Spec.DataSource.BackupSource")
errDataSourceNoPath = errors.New("'path' should be specified in .Spec.DataSource.BackupSource")
errIncorrectDataSourceStruct = errors.New("incorrect data source struct")
errUnsupportedPitrType = errors.New("the given point-in-time recovery type is not supported")
//nolint:gochecknoglobals
operatorEngine = map[everestv1alpha1.EngineType]string{
everestv1alpha1.DatabaseEnginePXC: pxcDeploymentName,
Expand Down Expand Up @@ -409,7 +418,7 @@ func validateCreateDatabaseClusterRequest(dbc DatabaseCluster) error {
return validateRFC1035(strName, "metadata.name")
}

func (e *EverestServer) validateDatabaseClusterCR(ctx echo.Context, databaseCluster *DatabaseCluster) error {
func (e *EverestServer) validateDatabaseClusterCR(ctx echo.Context, databaseCluster *DatabaseCluster) error { //nolint:cyclop
if err := validateCreateDatabaseClusterRequest(*databaseCluster); err != nil {
return err
}
Expand Down Expand Up @@ -446,6 +455,12 @@ func (e *EverestServer) validateDatabaseClusterCR(ctx echo.Context, databaseClus
return err
}

if databaseCluster.Spec.DataSource != nil {
if err := validateDBDataSource(databaseCluster); err != nil {
return err
}
}

return validateResourceLimits(databaseCluster)
}

Expand Down Expand Up @@ -615,6 +630,64 @@ func validateResourceLimits(cluster *DatabaseCluster) error {
return validateStorageSize(cluster)
}

func validateDBDataSource(db *DatabaseCluster) error {
bytes, err := json.Marshal(db.Spec.DataSource)
if err != nil {
return errIncorrectDataSourceStruct
}
return validateCommonDataSourceStruct(bytes)
}

func validateRestoreDataSource(restore *DatabaseClusterRestore) error {
bytes, err := json.Marshal(restore.Spec.DataSource)
if err != nil {
return errIncorrectDataSourceStruct
}
return validateCommonDataSourceStruct(bytes)
}

func validateCommonDataSourceStruct(data []byte) error {
// marshal and unmarshal to use the same validation func to validate DataSource for both db and restore
ds := &dataSourceStruct{}
err := json.Unmarshal(data, ds)
if err != nil {
return errIncorrectDataSourceStruct
}
return validateDataSource(*ds)
}

func validateDataSource(dataSource dataSourceStruct) error {
if (dataSource.DbClusterBackupName == nil && dataSource.BackupSource == nil) ||
(dataSource.DbClusterBackupName != nil && *dataSource.DbClusterBackupName != "" && dataSource.BackupSource != nil) {
return errDataSourceConfig
}

if dataSource.BackupSource != nil {
if dataSource.BackupSource.BackupStorageName == "" {
return errDataSourceNoBackupStorageName
}

if dataSource.BackupSource.Path == "" {
return errDataSourceNoPath
}
}

if dataSource.Pitr != nil { //nolint:nestif
if dataSource.Pitr.Type == nil || *dataSource.Pitr.Type == string(DatabaseClusterSpecDataSourcePitrTypeDate) {
if dataSource.Pitr.Date == nil {
return errDataSourceNoPitrDateSpecified
}

if _, err := time.Parse(dateFormat, *dataSource.Pitr.Date); err != nil {
return errDataSourceWrongDateFormat
}
} else {
return errUnsupportedPitrType
}
}
return nil
}

func ensureNonEmptyResources(cluster *DatabaseCluster) error {
if cluster.Spec.Engine.Resources == nil {
return errNoResourceDefined
Expand Down Expand Up @@ -790,5 +863,20 @@ func validateDatabaseClusterRestore(ctx context.Context, restore *DatabaseCluste
}
return err
}
if err = validateRestoreDataSource(restore); err != nil {
return err
}
return err
}

type dataSourceStruct struct {
BackupSource *struct {
BackupStorageName string `json:"backupStorageName"`
Path string `json:"path"`
} `json:"backupSource,omitempty"`
DbClusterBackupName *string `json:"dbClusterBackupName,omitempty"` //nolint:stylecheck
Pitr *struct {
Date *string `json:"date,omitempty"`
Type *string `json:"type,omitempty"`
} `json:"pitr,omitempty"`
}
68 changes: 68 additions & 0 deletions api/validation_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -573,6 +573,7 @@ func TestValidatePitrSpec(t *testing.T) {
require.NoError(t, err)
return
}
require.Error(t, err)
assert.Equal(t, err.Error(), tc.err.Error())
})
}
Expand Down Expand Up @@ -648,6 +649,73 @@ func TestValidateResourceLimits(t *testing.T) {
require.NoError(t, err)
return
}
require.Error(t, err)
assert.Equal(t, err.Error(), tc.err.Error())
})
}
}

func TestValidateDataSource(t *testing.T) {
t.Parallel()
cases := []struct {
name string
cluster []byte
err error
}{
{
name: "err none of the data source specified",
cluster: []byte(`{}`),
err: errDataSourceConfig,
},
{
name: "err both of the data source specified",
cluster: []byte(`{"dbClusterBackupName":"some-backup", "backupSource": {"backupStorageName":"some-name","path":"some-path"}}`),
err: errDataSourceConfig,
},
{
name: "err no date in pitr",
cluster: []byte(`{"dbClusterBackupName":"some-backup","pitr":{}}`),
err: errDataSourceNoPitrDateSpecified,
},
{
name: "wrong pitr date format",
cluster: []byte(`{"dbClusterBackupName":"some-backup","pitr":{"date":"2006-06-07 14:06:07"}}`),
err: errDataSourceWrongDateFormat,
},
{
name: "wrong pitr date format",
cluster: []byte(`{"dbClusterBackupName":"some-backup","pitr":{"date":""}}`),
err: errDataSourceWrongDateFormat,
},
{
name: "correct minimal",
cluster: []byte(`{"dbClusterBackupName":"some-backup","pitr":{"date":"2006-06-07T14:06:07Z"}}`),
err: nil,
},
{
name: "correct with pitr type",
cluster: []byte(`{"dbClusterBackupName":"some-backup","pitr":{"type":"date","date":"2006-06-07T14:06:07Z"}}`),
err: nil,
},
{
name: "unsupported pitr type",
cluster: []byte(`{"backupSource":{"backupStorageName":"some-name","path":"some-path"},"pitr":{"type":"latest"}}`),
err: errUnsupportedPitrType,
},
}
for _, tc := range cases {
tc := tc
t.Run(tc.name, func(t *testing.T) {
t.Parallel()
dsDB := &dataSourceStruct{}
err := json.Unmarshal(tc.cluster, dsDB)
require.NoError(t, err)
err = validateDataSource(*dsDB)
if tc.err == nil {
require.NoError(t, err)
return
}
require.Error(t, err)
assert.Equal(t, err.Error(), tc.err.Error())
})
}
Expand Down
2 changes: 1 addition & 1 deletion go.mod
View file
Original file line number Diff line number Diff line change
Expand Up @@ -13,7 +13,7 @@ require (
github.com/labstack/echo/v4 v4.11.3
github.com/oapi-codegen/echo-middleware v1.0.1
github.com/oapi-codegen/runtime v1.1.0
github.com/percona/everest-operator v0.6.0-dev1.0.20240119104008-aeb868d82769
github.com/percona/everest-operator v0.6.0-dev1.0.20240125150540-298621412982
github.com/stretchr/testify v1.8.4
go.uber.org/zap v1.26.0
golang.org/x/crypto v0.17.0
Expand Down
4 changes: 2 additions & 2 deletions go.sum
View file
Original file line number Diff line number Diff line change
Expand Up @@ -422,8 +422,8 @@ github.com/onsi/gomega v1.30.0/go.mod h1:9sxs+SwGrKI0+PWe4Fxa9tFQQBG5xSsSbMXOI8P
github.com/pborman/uuid v1.2.0/go.mod h1:X/NO0urCmaxf9VXbdlT7C2Yzkj2IKimNn4k+gtPdI/k=
github.com/pelletier/go-toml v1.2.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/94hg7ilaic=
github.com/pelletier/go-toml v1.7.0/go.mod h1:vwGMzjaWMwyfHwgIBhI2YUM4fB6nL6lVAvS1LBMMhTE=
github.com/percona/everest-operator v0.6.0-dev1.0.20240119104008-aeb868d82769 h1:IrF61ks3spy9i7r7C94pcHGwf1HL1R4+pLP9rnU0g3s=
github.com/percona/everest-operator v0.6.0-dev1.0.20240119104008-aeb868d82769/go.mod h1:o84NcJlAImYMpKK9+PIjS4V8SSREt1uZOqNhHt5qXMg=
github.com/percona/everest-operator v0.6.0-dev1.0.20240125150540-298621412982 h1:rb3XM3Ce544WoX1Z41E7R0sL4KFEuidn0fYRhHen6Lg=
github.com/percona/everest-operator v0.6.0-dev1.0.20240125150540-298621412982/go.mod h1:o84NcJlAImYMpKK9+PIjS4V8SSREt1uZOqNhHt5qXMg=
github.com/percona/percona-backup-mongodb v1.8.1-0.20230920143330-3b1c2e263901 h1:BDgsZRCjEuxl2/z4yWBqB0s8d20shuIDks7/RVdZiLs=
github.com/percona/percona-backup-mongodb v1.8.1-0.20230920143330-3b1c2e263901/go.mod h1:fZRCMpUqkWlLVdRKqqaj001LoVP2eo6F0ZhoMPeXDng=
github.com/percona/percona-postgresql-operator v0.0.0-20231220140959-ad5eef722609 h1:+UOK4gcHrRgqjo4smgfwT7/0apF6PhAJdQIdAV4ub/M=
Expand Down

0 comments on commit f38d8bb

Please sign in to comment.