From bf6e13c0d0006e3eabf3d914cc90d5d1dfa4f007 Mon Sep 17 00:00:00 2001 From: illia-li Date: Fri, 7 Jul 2023 13:09:40 -0400 Subject: [PATCH 1/3] add(typedef) add marshal and unmarshal test for full cases schema --- cmd/gemini/root.go | 14 - cmd/gemini/schema.go | 3 +- cmd/gemini/strategies_test.go | 3 +- pkg/replication/replication.go | 17 + pkg/tableopts/options.go | 14 + pkg/typedef/schema_test.go | 310 ++++++++ .../test_expected_data/full_schema.json | 668 ++++++++++++++++++ 7 files changed, 1013 insertions(+), 16 deletions(-) create mode 100644 pkg/typedef/test_expected_data/full_schema.json diff --git a/cmd/gemini/root.go b/cmd/gemini/root.go index 5508e00f..90d2fb08 100644 --- a/cmd/gemini/root.go +++ b/cmd/gemini/root.go @@ -33,7 +33,6 @@ import ( "github.com/scylladb/gemini/pkg/jobs" "github.com/scylladb/gemini/pkg/replication" "github.com/scylladb/gemini/pkg/store" - "github.com/scylladb/gemini/pkg/tableopts" "github.com/scylladb/gemini/pkg/typedef" "github.com/scylladb/gemini/pkg/utils" @@ -392,19 +391,6 @@ func createClusters( return testCluster, oracleCluster } -func createTableOptions(tableOptionStrings []string, logger *zap.Logger) []tableopts.Option { - var tableOptions []tableopts.Option - for _, optionString := range tableOptionStrings { - o, err := tableopts.FromCQL(optionString) - if err != nil { - logger.Warn("invalid table option", zap.String("option", optionString), zap.Error(err)) - continue - } - tableOptions = append(tableOptions, o) - } - return tableOptions -} - func getReplicationStrategy(rs string, fallback *replication.Replication, logger *zap.Logger) *replication.Replication { switch rs { case "network": diff --git a/cmd/gemini/schema.go b/cmd/gemini/schema.go index 58d8a6d0..09cb6b48 100644 --- a/cmd/gemini/schema.go +++ b/cmd/gemini/schema.go @@ -18,6 +18,7 @@ import ( "strings" "github.com/scylladb/gemini/pkg/replication" + "github.com/scylladb/gemini/pkg/tableopts" "github.com/scylladb/gemini/pkg/typedef" "go.uber.org/zap" @@ -67,7 +68,7 @@ func createDefaultSchemaConfig(logger *zap.Logger) typedef.SchemaConfig { return typedef.SchemaConfig{ ReplicationStrategy: rs, OracleReplicationStrategy: ors, - TableOptions: createTableOptions(tableOptions, logger), + TableOptions: tableopts.CreateTableOptions(tableOptions, logger), MaxTables: maxTables, MaxPartitionKeys: maxPartitionKeys, MinPartitionKeys: minPartitionKeys, diff --git a/cmd/gemini/strategies_test.go b/cmd/gemini/strategies_test.go index ad25ef4f..6859024e 100644 --- a/cmd/gemini/strategies_test.go +++ b/cmd/gemini/strategies_test.go @@ -60,7 +60,8 @@ func TestGetReplicationStrategy(t *testing.T) { } } -func TestReadSchema(t *testing.T) { +// TestReadExampleSchema main task of this test to be sure that schema example (schema.json) is correct and have correct marshal, unmarshal +func TestReadExampleSchema(t *testing.T) { filePath := "schema.json" testSchema, err := readSchema(filePath, typedef.SchemaConfig{}) diff --git a/pkg/replication/replication.go b/pkg/replication/replication.go index ab143e3b..8bd80d66 100644 --- a/pkg/replication/replication.go +++ b/pkg/replication/replication.go @@ -39,3 +39,20 @@ func NewNetworkTopologyStrategy() *Replication { "datacenter1": 1, } } + +func (r *Replication) UnmarshalJSON(data []byte) error { + dataMap := make(map[string]interface{}) + if err := json.Unmarshal(data, &dataMap); err != nil { + return err + } + out := Replication{} + for idx := range dataMap { + val := dataMap[idx] + if fVal, ok := val.(float64); ok { + dataMap[idx] = int(fVal) + } + out[idx] = dataMap[idx] + } + *r = out + return nil +} diff --git a/pkg/tableopts/options.go b/pkg/tableopts/options.go index db3c8927..7b02cedc 100644 --- a/pkg/tableopts/options.go +++ b/pkg/tableopts/options.go @@ -19,6 +19,7 @@ import ( "strings" "github.com/pkg/errors" + "go.uber.org/zap" ) type Option interface { @@ -65,3 +66,16 @@ func FromCQL(cql string) (Option, error) { val: valPart, }, nil } + +func CreateTableOptions(tableOptionStrings []string, logger *zap.Logger) []Option { + var tableOptions []Option + for _, optionString := range tableOptionStrings { + o, err := FromCQL(optionString) + if err != nil { + logger.Warn("invalid table option", zap.String("option", optionString), zap.Error(err)) + continue + } + tableOptions = append(tableOptions, o) + } + return tableOptions +} diff --git a/pkg/typedef/schema_test.go b/pkg/typedef/schema_test.go index 2da47816..64b149e0 100644 --- a/pkg/typedef/schema_test.go +++ b/pkg/typedef/schema_test.go @@ -16,9 +16,15 @@ package typedef import ( + "encoding/json" + "os" "testing" "github.com/google/go-cmp/cmp" + "github.com/google/go-cmp/cmp/cmpopts" + + "github.com/scylladb/gemini/pkg/replication" + "github.com/scylladb/gemini/pkg/tableopts" ) func TestSchemaConfigValidate(t *testing.T) { @@ -100,3 +106,307 @@ func TestSchemaConfigValidate(t *testing.T) { }) } } + +// TestSchemaMarshalUnmarshalNotChanged main task of this test catch all possible changes in json representation of schema and notify QA about this. +// +// If any change was catch and if you sure that this changes really needed - you should notify QA about this changes by create new issue https://github.com/scylladb/scylla-cluster-tests/issues/new +// Only then you can rewrite expected data file +func TestSchemaMarshalUnmarshalNotChanged(t *testing.T) { + expectedFilePath := "./test_expected_data/full_schema.json" + + fullSchemaExpected, err := os.ReadFile(expectedFilePath) + if err != nil { + t.Fatalf("failed to open schema json file %s, error:%s", expectedFilePath, err) + } + + fullSchemaMarshaled, err := json.MarshalIndent(fullSchema, "", " ") + if err != nil { + t.Fatalf("unable to marshal schema json, error=%s\n", err) + } + + if diff := cmp.Diff(fullSchemaExpected, fullSchemaMarshaled); diff != "" { + t.Errorf("schema not the same after marshal, diff=%s", diff) + t.Error("if you sure that this changes really needed - you should notify QA about this changes by create new issue https://github.com/scylladb/scylla-cluster-tests/issues/new\n" + + "Only then you can rewrite expected data file") + } + + fullSchemaExpectedUnmarshalled := Schema{} + + if err = json.Unmarshal(fullSchemaExpected, &fullSchemaExpectedUnmarshalled); err != nil { + t.Fatalf("unable to marshal schema example json, error=%s\n", err) + } + + opts := cmp.Options{ + cmp.AllowUnexported(Table{}, MaterializedView{}), + cmpopts.IgnoreUnexported(Table{}, MaterializedView{}), + cmpopts.EquateEmpty(), + } + + fullSchema.Config = SchemaConfig{} + if diff := cmp.Diff(fullSchema, fullSchemaExpectedUnmarshalled, opts); diff != "" { + t.Errorf("schema not the same after unmarshal, diff=%s", diff) + t.Error("if you sure that this changes really needed - you should notify QA about this changes by create new issue https://github.com/scylladb/scylla-cluster-tests/issues/new\n" + + "Only then you can rewrite expected data file") + } +} + +// nolint: revive +var fullSchema = Schema{ + Keyspace: Keyspace{ + Replication: replication.NewSimpleStrategy(), + OracleReplication: replication.NewSimpleStrategy(), + Name: "ks1", + }, + Tables: []*Table{ + { + Name: "tb0", + PartitionKeys: Columns{ + {Name: "pk0", Type: TYPE_ASCII}, + {Name: "pk1", Type: TYPE_BIGINT}, + {Name: "pk2", Type: TYPE_BLOB}, + {Name: "pk3", Type: TYPE_BOOLEAN}, + {Name: "pk4", Type: TYPE_DATE}, + {Name: "pk5", Type: TYPE_DECIMAL}, + {Name: "pk6", Type: TYPE_DOUBLE}, + }, + ClusteringKeys: Columns{ + {Name: "ck0", Type: TYPE_FLOAT}, + {Name: "ck1", Type: TYPE_INET}, + {Name: "ck2", Type: TYPE_INT}, + {Name: "ck3", Type: TYPE_SMALLINT}, + {Name: "ck4", Type: TYPE_TEXT}, + {Name: "ck5", Type: TYPE_TIMESTAMP}, + {Name: "ck6", Type: TYPE_TIMEUUID}, + }, + Columns: Columns{ + {Name: "col0", Type: TYPE_ASCII}, + {Name: "col1", Type: TYPE_BIGINT}, + {Name: "col2", Type: TYPE_BLOB}, + {Name: "col3", Type: TYPE_BOOLEAN}, + {Name: "col4", Type: TYPE_DATE}, + {Name: "col5", Type: TYPE_DECIMAL}, + {Name: "col6", Type: TYPE_DOUBLE}, + {Name: "col7", Type: TYPE_FLOAT}, + {Name: "col8", Type: TYPE_INET}, + {Name: "col9", Type: TYPE_INT}, + {Name: "col10", Type: TYPE_SMALLINT}, + {Name: "col11", Type: TYPE_TEXT}, + {Name: "col12", Type: TYPE_TIMESTAMP}, + {Name: "col13", Type: TYPE_TIMEUUID}, + {Name: "col14", Type: TYPE_TINYINT}, + {Name: "col15", Type: TYPE_UUID}, + {Name: "col16", Type: TYPE_VARCHAR}, + {Name: "col17", Type: TYPE_VARINT}, + {Name: "col18", Type: TYPE_TIME}, + {Name: "col19", Type: TYPE_DURATION}, + }, + Indexes: []IndexDef{ + {IndexName: "col0_idx", ColumnName: "col0"}, + {IndexName: "col12_idx", ColumnName: "col12"}, + }, + MaterializedViews: nil, + KnownIssues: map[string]bool{KnownIssuesJSONWithTuples: true}, + TableOptions: []string{"compression = {'sstable_compression':'LZ4Compressor'}", "read_repair_chance = 1.0", "comment = 'Important records'", "cdc = {'enabled':'true','preimage':'true'}", "compaction = {'class':'LeveledCompactionStrategy','enabled':true,'sstable_size_in_mb':160,'tombstone_compaction_interval':86400,'tombstone_threshold':0.2}"}, + }, { + Name: "tb1", + PartitionKeys: Columns{ + {Name: "pk0", Type: TYPE_FLOAT}, + {Name: "pk1", Type: TYPE_INET}, + {Name: "pk2", Type: TYPE_INT}, + {Name: "pk3", Type: TYPE_SMALLINT}, + {Name: "pk4", Type: TYPE_TEXT}, + {Name: "pk5", Type: TYPE_TIMESTAMP}, + {Name: "pk6", Type: TYPE_TIMEUUID}, + }, + ClusteringKeys: Columns{ + {Name: "ck0", Type: TYPE_ASCII}, + {Name: "ck1", Type: TYPE_BIGINT}, + {Name: "ck2", Type: TYPE_BLOB}, + {Name: "ck3", Type: TYPE_BOOLEAN}, + {Name: "ck4", Type: TYPE_DATE}, + {Name: "ck5", Type: TYPE_DECIMAL}, + {Name: "ck6", Type: TYPE_DOUBLE}, + }, + Columns: Columns{ + {Name: "col0", Type: TYPE_ASCII}, + {Name: "col1", Type: TYPE_BIGINT}, + {Name: "col2", Type: TYPE_BLOB}, + {Name: "col3", Type: TYPE_BOOLEAN}, + {Name: "col4", Type: TYPE_DATE}, + {Name: "col5", Type: TYPE_DECIMAL}, + {Name: "col6", Type: TYPE_DOUBLE}, + {Name: "col7", Type: TYPE_FLOAT}, + {Name: "col8", Type: TYPE_INET}, + {Name: "col9", Type: TYPE_INT}, + {Name: "col10", Type: TYPE_SMALLINT}, + {Name: "col11", Type: TYPE_TEXT}, + {Name: "col12", Type: TYPE_TIMESTAMP}, + {Name: "col13", Type: TYPE_TIMEUUID}, + {Name: "col14", Type: TYPE_TINYINT}, + {Name: "col15", Type: TYPE_UUID}, + {Name: "col16", Type: TYPE_VARCHAR}, + {Name: "col17", Type: TYPE_VARINT}, + {Name: "col18", Type: TYPE_TIME}, + {Name: "col19", Type: TYPE_DURATION}, + }, + Indexes: []IndexDef{ + {IndexName: "col0_idx", ColumnName: "col0"}, + {IndexName: "col12_idx", ColumnName: "col12"}, + }, + MaterializedViews: []MaterializedView{ + { + NonPrimaryKey: &ColumnDef{ + Type: TYPE_DECIMAL, + Name: "col5", + }, + Name: "mv0", + PartitionKeys: Columns{ + {Name: "pk0", Type: TYPE_FLOAT}, + {Name: "pk1", Type: TYPE_INET}, + {Name: "pk2", Type: TYPE_INT}, + {Name: "pk3", Type: TYPE_SMALLINT}, + {Name: "pk4", Type: TYPE_TEXT}, + {Name: "pk5", Type: TYPE_TIMESTAMP}, + {Name: "pk6", Type: TYPE_TIMEUUID}, + }, + ClusteringKeys: Columns{ + {Name: "ck0", Type: TYPE_ASCII}, + {Name: "ck1", Type: TYPE_BIGINT}, + {Name: "ck2", Type: TYPE_BLOB}, + {Name: "ck3", Type: TYPE_BOOLEAN}, + {Name: "ck4", Type: TYPE_DATE}, + {Name: "ck5", Type: TYPE_DECIMAL}, + {Name: "ck6", Type: TYPE_DOUBLE}, + }, + }, { + NonPrimaryKey: &ColumnDef{ + Type: TYPE_ASCII, + Name: "col0_idx", + }, + Name: "mv1", + PartitionKeys: Columns{ + {Name: "pk0", Type: TYPE_FLOAT}, + {Name: "pk1", Type: TYPE_INET}, + {Name: "pk2", Type: TYPE_INT}, + {Name: "pk3", Type: TYPE_SMALLINT}, + {Name: "pk4", Type: TYPE_TEXT}, + {Name: "pk5", Type: TYPE_TIMESTAMP}, + {Name: "pk6", Type: TYPE_TIMEUUID}, + }, + ClusteringKeys: Columns{ + {Name: "ck0", Type: TYPE_ASCII}, + {Name: "ck1", Type: TYPE_BIGINT}, + {Name: "ck2", Type: TYPE_BLOB}, + {Name: "ck3", Type: TYPE_BOOLEAN}, + {Name: "ck4", Type: TYPE_DATE}, + {Name: "ck5", Type: TYPE_DECIMAL}, + {Name: "ck6", Type: TYPE_DOUBLE}, + }, + }, + }, + KnownIssues: map[string]bool{KnownIssuesJSONWithTuples: true}, + TableOptions: []string{"compression = {'sstable_compression':'LZ4Compressor'}", "read_repair_chance = 1.0", "comment = 'Important records'", "cdc = {'enabled':'true','preimage':'true'}", "compaction = {'class':'LeveledCompactionStrategy','enabled':true,'sstable_size_in_mb':160,'tombstone_compaction_interval':86400,'tombstone_threshold':0.2}"}, + }, { + Name: "tb2", + PartitionKeys: Columns{ + {Name: "pk0", Type: TYPE_TINYINT}, + {Name: "pk1", Type: TYPE_UUID}, + {Name: "pk2", Type: TYPE_VARCHAR}, + {Name: "pk3", Type: TYPE_VARINT}, + {Name: "pk4", Type: TYPE_TIME}, + }, + ClusteringKeys: Columns{ + {Name: "ck0", Type: TYPE_TINYINT}, + {Name: "ck1", Type: TYPE_UUID}, + {Name: "ck2", Type: TYPE_VARCHAR}, + {Name: "ck3", Type: TYPE_VARINT}, + {Name: "ck4", Type: TYPE_TIME}, + {Name: "ck5", Type: TYPE_DURATION}, + }, + Columns: Columns{ + {Name: "col0", Type: &UDTType{ + ComplexType: "udt", + ValueTypes: map[string]SimpleType{"udt_10.1": TYPE_BIGINT, "udt_10.2": TYPE_DATE, "udt_10.3": TYPE_BLOB}, + TypeName: "udt_10", + Frozen: false, + }}, + {Name: "col1", Type: &MapType{ + ComplexType: "map", + KeyType: TYPE_INET, + ValueType: TYPE_TIME, + Frozen: true, + }}, + {Name: "col2", Type: &TupleType{ + ComplexType: "tuple", + ValueTypes: []SimpleType{TYPE_FLOAT, TYPE_DATE, TYPE_VARCHAR}, + Frozen: false, + }}, + {Name: "col3", Type: &BagType{ + ComplexType: "list", + ValueType: TYPE_UUID, + Frozen: true, + }}, + {Name: "col4", Type: &BagType{ + ComplexType: "set", + ValueType: TYPE_TIMESTAMP, + Frozen: false, + }}, + {Name: "col5", Type: TYPE_DECIMAL}, + }, + Indexes: []IndexDef{ + {IndexName: "col0_idx", ColumnName: "col0"}, + {IndexName: "col12_idx", ColumnName: "col12"}, + }, + MaterializedViews: []MaterializedView{ + { + NonPrimaryKey: nil, + Name: "mv0", + PartitionKeys: Columns{ + {Name: "pk0", Type: TYPE_TINYINT}, + {Name: "pk1", Type: TYPE_UUID}, + {Name: "pk2", Type: TYPE_VARCHAR}, + {Name: "pk3", Type: TYPE_VARINT}, + {Name: "pk4", Type: TYPE_TIME}, + }, + ClusteringKeys: Columns{ + {Name: "ck0", Type: TYPE_TINYINT}, + {Name: "ck1", Type: TYPE_UUID}, + {Name: "ck2", Type: TYPE_VARCHAR}, + {Name: "ck3", Type: TYPE_VARINT}, + {Name: "ck4", Type: TYPE_TIME}, + {Name: "ck5", Type: TYPE_DURATION}, + }, + }, + }, + KnownIssues: map[string]bool{KnownIssuesJSONWithTuples: true}, + TableOptions: []string{"compression = {'sstable_compression':'LZ4Compressor'}", "read_repair_chance = 1.0", "comment = 'Important records'", "cdc = {'enabled':'true','preimage':'true'}", "compaction = {'class':'LeveledCompactionStrategy','enabled':true,'sstable_size_in_mb':160,'tombstone_compaction_interval':86400,'tombstone_threshold':0.2}"}, + }, + }, + Config: SchemaConfig{ + ReplicationStrategy: replication.NewSimpleStrategy(), + OracleReplicationStrategy: replication.NewSimpleStrategy(), + TableOptions: tableopts.CreateTableOptions([]string{ + "compression = {'sstable_compression':'LZ4Compressor'}", + "read_repair_chance = 1.0", "comment = 'Important records'", "cdc = {'enabled':'true','preimage':'true'}", + "compaction = {'class':'LeveledCompactionStrategy','enabled':true,'sstable_size_in_mb':160,'tombstone_compaction_interval':86400,'tombstone_threshold':0.2}", + }, nil), + MaxTables: 10, + MaxPartitionKeys: 10, + MinPartitionKeys: 1, + MaxClusteringKeys: 10, + MinClusteringKeys: 1, + MaxColumns: 25, + MinColumns: 1, + MaxUDTParts: 20, + MaxTupleParts: 20, + MaxBlobLength: 1e4, + MaxStringLength: 1000, + MinBlobLength: 0, + MinStringLength: 0, + UseCounters: false, + UseLWT: false, + CQLFeature: CQL_FEATURE_NORMAL, + AsyncObjectStabilizationAttempts: 10, + AsyncObjectStabilizationDelay: 100000, + }, +} diff --git a/pkg/typedef/test_expected_data/full_schema.json b/pkg/typedef/test_expected_data/full_schema.json new file mode 100644 index 00000000..a30174d7 --- /dev/null +++ b/pkg/typedef/test_expected_data/full_schema.json @@ -0,0 +1,668 @@ +{ + "keyspace": { + "replication": { + "class": "SimpleStrategy", + "replication_factor": 1 + }, + "oracle_replication": { + "class": "SimpleStrategy", + "replication_factor": 1 + }, + "name": "ks1" + }, + "tables": [ + { + "name": "tb0", + "partition_keys": [ + { + "type": "ascii", + "name": "pk0" + }, + { + "type": "bigint", + "name": "pk1" + }, + { + "type": "blob", + "name": "pk2" + }, + { + "type": "boolean", + "name": "pk3" + }, + { + "type": "date", + "name": "pk4" + }, + { + "type": "decimal", + "name": "pk5" + }, + { + "type": "double", + "name": "pk6" + } + ], + "clustering_keys": [ + { + "type": "float", + "name": "ck0" + }, + { + "type": "inet", + "name": "ck1" + }, + { + "type": "int", + "name": "ck2" + }, + { + "type": "smallint", + "name": "ck3" + }, + { + "type": "text", + "name": "ck4" + }, + { + "type": "timestamp", + "name": "ck5" + }, + { + "type": "timeuuid", + "name": "ck6" + } + ], + "columns": [ + { + "type": "ascii", + "name": "col0" + }, + { + "type": "bigint", + "name": "col1" + }, + { + "type": "blob", + "name": "col2" + }, + { + "type": "boolean", + "name": "col3" + }, + { + "type": "date", + "name": "col4" + }, + { + "type": "decimal", + "name": "col5" + }, + { + "type": "double", + "name": "col6" + }, + { + "type": "float", + "name": "col7" + }, + { + "type": "inet", + "name": "col8" + }, + { + "type": "int", + "name": "col9" + }, + { + "type": "smallint", + "name": "col10" + }, + { + "type": "text", + "name": "col11" + }, + { + "type": "timestamp", + "name": "col12" + }, + { + "type": "timeuuid", + "name": "col13" + }, + { + "type": "tinyint", + "name": "col14" + }, + { + "type": "uuid", + "name": "col15" + }, + { + "type": "varchar", + "name": "col16" + }, + { + "type": "varint", + "name": "col17" + }, + { + "type": "time", + "name": "col18" + }, + { + "type": "duration", + "name": "col19" + } + ], + "indexes": [ + { + "Column": null, + "index_name": "col0_idx", + "column_name": "col0" + }, + { + "Column": null, + "index_name": "col12_idx", + "column_name": "col12" + } + ], + "known_issues": { + "https://github.com/scylladb/scylla/issues/3708": true + }, + "table_options": [ + "compression = {'sstable_compression':'LZ4Compressor'}", + "read_repair_chance = 1.0", + "comment = 'Important records'", + "cdc = {'enabled':'true','preimage':'true'}", + "compaction = {'class':'LeveledCompactionStrategy','enabled':true,'sstable_size_in_mb':160,'tombstone_compaction_interval':86400,'tombstone_threshold':0.2}" + ] + }, + { + "name": "tb1", + "partition_keys": [ + { + "type": "float", + "name": "pk0" + }, + { + "type": "inet", + "name": "pk1" + }, + { + "type": "int", + "name": "pk2" + }, + { + "type": "smallint", + "name": "pk3" + }, + { + "type": "text", + "name": "pk4" + }, + { + "type": "timestamp", + "name": "pk5" + }, + { + "type": "timeuuid", + "name": "pk6" + } + ], + "clustering_keys": [ + { + "type": "ascii", + "name": "ck0" + }, + { + "type": "bigint", + "name": "ck1" + }, + { + "type": "blob", + "name": "ck2" + }, + { + "type": "boolean", + "name": "ck3" + }, + { + "type": "date", + "name": "ck4" + }, + { + "type": "decimal", + "name": "ck5" + }, + { + "type": "double", + "name": "ck6" + } + ], + "columns": [ + { + "type": "ascii", + "name": "col0" + }, + { + "type": "bigint", + "name": "col1" + }, + { + "type": "blob", + "name": "col2" + }, + { + "type": "boolean", + "name": "col3" + }, + { + "type": "date", + "name": "col4" + }, + { + "type": "decimal", + "name": "col5" + }, + { + "type": "double", + "name": "col6" + }, + { + "type": "float", + "name": "col7" + }, + { + "type": "inet", + "name": "col8" + }, + { + "type": "int", + "name": "col9" + }, + { + "type": "smallint", + "name": "col10" + }, + { + "type": "text", + "name": "col11" + }, + { + "type": "timestamp", + "name": "col12" + }, + { + "type": "timeuuid", + "name": "col13" + }, + { + "type": "tinyint", + "name": "col14" + }, + { + "type": "uuid", + "name": "col15" + }, + { + "type": "varchar", + "name": "col16" + }, + { + "type": "varint", + "name": "col17" + }, + { + "type": "time", + "name": "col18" + }, + { + "type": "duration", + "name": "col19" + } + ], + "indexes": [ + { + "Column": null, + "index_name": "col0_idx", + "column_name": "col0" + }, + { + "Column": null, + "index_name": "col12_idx", + "column_name": "col12" + } + ], + "materialized_views": [ + { + "NonPrimaryKey": { + "type": "decimal", + "name": "col5" + }, + "name": "mv0", + "partition_keys": [ + { + "type": "float", + "name": "pk0" + }, + { + "type": "inet", + "name": "pk1" + }, + { + "type": "int", + "name": "pk2" + }, + { + "type": "smallint", + "name": "pk3" + }, + { + "type": "text", + "name": "pk4" + }, + { + "type": "timestamp", + "name": "pk5" + }, + { + "type": "timeuuid", + "name": "pk6" + } + ], + "clustering_keys": [ + { + "type": "ascii", + "name": "ck0" + }, + { + "type": "bigint", + "name": "ck1" + }, + { + "type": "blob", + "name": "ck2" + }, + { + "type": "boolean", + "name": "ck3" + }, + { + "type": "date", + "name": "ck4" + }, + { + "type": "decimal", + "name": "ck5" + }, + { + "type": "double", + "name": "ck6" + } + ] + }, + { + "NonPrimaryKey": { + "type": "ascii", + "name": "col0_idx" + }, + "name": "mv1", + "partition_keys": [ + { + "type": "float", + "name": "pk0" + }, + { + "type": "inet", + "name": "pk1" + }, + { + "type": "int", + "name": "pk2" + }, + { + "type": "smallint", + "name": "pk3" + }, + { + "type": "text", + "name": "pk4" + }, + { + "type": "timestamp", + "name": "pk5" + }, + { + "type": "timeuuid", + "name": "pk6" + } + ], + "clustering_keys": [ + { + "type": "ascii", + "name": "ck0" + }, + { + "type": "bigint", + "name": "ck1" + }, + { + "type": "blob", + "name": "ck2" + }, + { + "type": "boolean", + "name": "ck3" + }, + { + "type": "date", + "name": "ck4" + }, + { + "type": "decimal", + "name": "ck5" + }, + { + "type": "double", + "name": "ck6" + } + ] + } + ], + "known_issues": { + "https://github.com/scylladb/scylla/issues/3708": true + }, + "table_options": [ + "compression = {'sstable_compression':'LZ4Compressor'}", + "read_repair_chance = 1.0", + "comment = 'Important records'", + "cdc = {'enabled':'true','preimage':'true'}", + "compaction = {'class':'LeveledCompactionStrategy','enabled':true,'sstable_size_in_mb':160,'tombstone_compaction_interval':86400,'tombstone_threshold':0.2}" + ] + }, + { + "name": "tb2", + "partition_keys": [ + { + "type": "tinyint", + "name": "pk0" + }, + { + "type": "uuid", + "name": "pk1" + }, + { + "type": "varchar", + "name": "pk2" + }, + { + "type": "varint", + "name": "pk3" + }, + { + "type": "time", + "name": "pk4" + } + ], + "clustering_keys": [ + { + "type": "tinyint", + "name": "ck0" + }, + { + "type": "uuid", + "name": "ck1" + }, + { + "type": "varchar", + "name": "ck2" + }, + { + "type": "varint", + "name": "ck3" + }, + { + "type": "time", + "name": "ck4" + }, + { + "type": "duration", + "name": "ck5" + } + ], + "columns": [ + { + "type": { + "complex_type": "udt", + "value_types": { + "udt_10.1": "bigint", + "udt_10.2": "date", + "udt_10.3": "blob" + }, + "type_name": "udt_10", + "frozen": false + }, + "name": "col0" + }, + { + "type": { + "complex_type": "map", + "key_type": "inet", + "value_type": "time", + "frozen": true + }, + "name": "col1" + }, + { + "type": { + "complex_type": "tuple", + "value_types": [ + "float", + "date", + "varchar" + ], + "frozen": false + }, + "name": "col2" + }, + { + "type": { + "complex_type": "list", + "value_type": "uuid", + "frozen": true + }, + "name": "col3" + }, + { + "type": { + "complex_type": "set", + "value_type": "timestamp", + "frozen": false + }, + "name": "col4" + }, + { + "type": "decimal", + "name": "col5" + } + ], + "indexes": [ + { + "Column": null, + "index_name": "col0_idx", + "column_name": "col0" + }, + { + "Column": null, + "index_name": "col12_idx", + "column_name": "col12" + } + ], + "materialized_views": [ + { + "NonPrimaryKey": null, + "name": "mv0", + "partition_keys": [ + { + "type": "tinyint", + "name": "pk0" + }, + { + "type": "uuid", + "name": "pk1" + }, + { + "type": "varchar", + "name": "pk2" + }, + { + "type": "varint", + "name": "pk3" + }, + { + "type": "time", + "name": "pk4" + } + ], + "clustering_keys": [ + { + "type": "tinyint", + "name": "ck0" + }, + { + "type": "uuid", + "name": "ck1" + }, + { + "type": "varchar", + "name": "ck2" + }, + { + "type": "varint", + "name": "ck3" + }, + { + "type": "time", + "name": "ck4" + }, + { + "type": "duration", + "name": "ck5" + } + ] + } + ], + "known_issues": { + "https://github.com/scylladb/scylla/issues/3708": true + }, + "table_options": [ + "compression = {'sstable_compression':'LZ4Compressor'}", + "read_repair_chance = 1.0", + "comment = 'Important records'", + "cdc = {'enabled':'true','preimage':'true'}", + "compaction = {'class':'LeveledCompactionStrategy','enabled':true,'sstable_size_in_mb':160,'tombstone_compaction_interval':86400,'tombstone_threshold':0.2}" + ] + } + ] +} \ No newline at end of file From ae1d81cd5981db04d655c4ffd08447db48e9f880 Mon Sep 17 00:00:00 2001 From: Dmitry Kropachev Date: Fri, 7 Jul 2023 06:13:23 -0400 Subject: [PATCH 2/3] fix(generotrs): update schema generation to use standalone seed --- cmd/gemini/root.go | 2 +- pkg/generators/column_generator.go | 68 +++++++++++++-------------- pkg/generators/statement_generator.go | 31 ++++++------ pkg/jobs/gen_ddl_stmt.go | 38 ++------------- pkg/typedef/columns.go | 4 +- pkg/typedef/columns_test.go | 29 +++++++----- pkg/typedef/simple_type.go | 4 +- pkg/utils/utils.go | 7 --- 8 files changed, 78 insertions(+), 105 deletions(-) diff --git a/cmd/gemini/root.go b/cmd/gemini/root.go index 90d2fb08..3a95c3d5 100644 --- a/cmd/gemini/root.go +++ b/cmd/gemini/root.go @@ -197,7 +197,7 @@ func run(_ *cobra.Command, _ []string) error { return errors.Wrap(err, "cannot create schema") } } else { - schema = generators.GenSchema(schemaConfig) + schema = generators.GenSchema(schemaConfig, seed) } jsonSchema, _ := json.MarshalIndent(schema, "", " ") diff --git a/pkg/generators/column_generator.go b/pkg/generators/column_generator.go index bcd00faa..254d1f80 100644 --- a/pkg/generators/column_generator.go +++ b/pkg/generators/column_generator.go @@ -26,51 +26,51 @@ func GenColumnName(prefix string, idx int) string { return fmt.Sprintf("%s%d", prefix, idx) } -func GenColumnType(numColumns int, sc *typedef.SchemaConfig) typedef.Type { - n := rand.Intn(numColumns + 5) +func GenColumnType(numColumns int, sc *typedef.SchemaConfig, r *rand.Rand) typedef.Type { + n := r.Intn(numColumns + 5) switch n { case numColumns: - return GenTupleType(sc) + return GenTupleType(sc, r) case numColumns + 1: - return GenUDTType(sc) + return GenUDTType(sc, r) case numColumns + 2: - return GenSetType(sc) + return GenSetType(sc, r) case numColumns + 3: - return GenListType(sc) + return GenListType(sc, r) case numColumns + 4: - return GenMapType(sc) + return GenMapType(sc, r) default: - return GenSimpleType(sc) + return GenSimpleType(sc, r) } } -func GenSimpleType(_ *typedef.SchemaConfig) typedef.SimpleType { - return typedef.AllTypes[rand.Intn(len(typedef.AllTypes))] +func GenSimpleType(_ *typedef.SchemaConfig, r *rand.Rand) typedef.SimpleType { + return typedef.AllTypes[r.Intn(len(typedef.AllTypes))] } -func GenTupleType(sc *typedef.SchemaConfig) typedef.Type { - n := rand.Intn(sc.MaxTupleParts) +func GenTupleType(sc *typedef.SchemaConfig, r *rand.Rand) typedef.Type { + n := r.Intn(sc.MaxTupleParts) if n < 2 { n = 2 } typeList := make([]typedef.SimpleType, n) for i := 0; i < n; i++ { - typeList[i] = GenSimpleType(sc) + typeList[i] = GenSimpleType(sc, r) } return &typedef.TupleType{ ComplexType: typedef.TYPE_TUPLE, ValueTypes: typeList, - Frozen: rand.Uint32()%2 == 0, + Frozen: r.Uint32()%2 == 0, } } -func GenUDTType(sc *typedef.SchemaConfig) *typedef.UDTType { - udtNum := rand.Uint32() +func GenUDTType(sc *typedef.SchemaConfig, r *rand.Rand) *typedef.UDTType { + udtNum := r.Uint32() typeName := fmt.Sprintf("udt_%d", udtNum) ts := make(map[string]typedef.SimpleType) - for i := 0; i < rand.Intn(sc.MaxUDTParts)+1; i++ { - ts[typeName+fmt.Sprintf("_%d", i)] = GenSimpleType(sc) + for i := 0; i < r.Intn(sc.MaxUDTParts)+1; i++ { + ts[typeName+fmt.Sprintf("_%d", i)] = GenSimpleType(sc, r) } return &typedef.UDTType{ @@ -81,18 +81,18 @@ func GenUDTType(sc *typedef.SchemaConfig) *typedef.UDTType { } } -func GenSetType(sc *typedef.SchemaConfig) *typedef.BagType { - return genBagType(typedef.TYPE_SET, sc) +func GenSetType(sc *typedef.SchemaConfig, r *rand.Rand) *typedef.BagType { + return genBagType(typedef.TYPE_SET, sc, r) } -func GenListType(sc *typedef.SchemaConfig) *typedef.BagType { - return genBagType(typedef.TYPE_LIST, sc) +func GenListType(sc *typedef.SchemaConfig, r *rand.Rand) *typedef.BagType { + return genBagType(typedef.TYPE_LIST, sc, r) } -func genBagType(kind string, sc *typedef.SchemaConfig) *typedef.BagType { +func genBagType(kind string, sc *typedef.SchemaConfig, r *rand.Rand) *typedef.BagType { var t typedef.SimpleType for { - t = GenSimpleType(sc) + t = GenSimpleType(sc, r) if t != typedef.TYPE_DURATION { break } @@ -100,32 +100,32 @@ func genBagType(kind string, sc *typedef.SchemaConfig) *typedef.BagType { return &typedef.BagType{ ComplexType: kind, ValueType: t, - Frozen: rand.Uint32()%2 == 0, + Frozen: r.Uint32()%2 == 0, } } -func GenMapType(sc *typedef.SchemaConfig) *typedef.MapType { - t := GenSimpleType(sc) +func GenMapType(sc *typedef.SchemaConfig, r *rand.Rand) *typedef.MapType { + t := GenSimpleType(sc, r) for { if _, ok := typedef.TypesMapKeyBlacklist[t]; !ok { break } - t = GenSimpleType(sc) + t = GenSimpleType(sc, r) } return &typedef.MapType{ ComplexType: typedef.TYPE_MAP, KeyType: t, - ValueType: GenSimpleType(sc), - Frozen: rand.Uint32()%2 == 0, + ValueType: GenSimpleType(sc, r), + Frozen: r.Uint32()%2 == 0, } } -func GenPartitionKeyColumnType() typedef.Type { - return typedef.PartitionKeyTypes[rand.Intn(len(typedef.PartitionKeyTypes))] +func GenPartitionKeyColumnType(r *rand.Rand) typedef.Type { + return typedef.PartitionKeyTypes[r.Intn(len(typedef.PartitionKeyTypes))] } -func GenPrimaryKeyColumnType() typedef.Type { - return typedef.PkTypes[rand.Intn(len(typedef.PkTypes))] +func GenPrimaryKeyColumnType(r *rand.Rand) typedef.Type { + return typedef.PkTypes[r.Intn(len(typedef.PkTypes))] } func GenIndexName(prefix string, idx int) string { diff --git a/pkg/generators/statement_generator.go b/pkg/generators/statement_generator.go index 6a084994..c5dd3f97 100644 --- a/pkg/generators/statement_generator.go +++ b/pkg/generators/statement_generator.go @@ -18,12 +18,15 @@ import ( "fmt" "strings" + "golang.org/x/exp/rand" + "github.com/scylladb/gemini/pkg/builders" "github.com/scylladb/gemini/pkg/typedef" "github.com/scylladb/gemini/pkg/utils" ) -func GenSchema(sc typedef.SchemaConfig) *typedef.Schema { +func GenSchema(sc typedef.SchemaConfig, seed uint64) *typedef.Schema { + r := rand.New(rand.NewSource(seed)) builder := builders.NewSchemaBuilder() builder.Config(sc) keyspace := typedef.Keyspace{ @@ -32,22 +35,22 @@ func GenSchema(sc typedef.SchemaConfig) *typedef.Schema { OracleReplication: sc.OracleReplicationStrategy, } builder.Keyspace(keyspace) - numTables := utils.RandInt(1, sc.GetMaxTables()) + numTables := utils.RandInt2(r, 1, sc.GetMaxTables()) for i := 0; i < numTables; i++ { - table := genTable(sc, fmt.Sprintf("table%d", i+1)) + table := genTable(sc, fmt.Sprintf("table%d", i+1), r) builder.Table(table) } return builder.Build() } -func genTable(sc typedef.SchemaConfig, tableName string) *typedef.Table { - partitionKeys := make(typedef.Columns, utils.RandInt(sc.GetMinPartitionKeys(), sc.GetMaxPartitionKeys())) +func genTable(sc typedef.SchemaConfig, tableName string, r *rand.Rand) *typedef.Table { + partitionKeys := make(typedef.Columns, utils.RandInt2(r, sc.GetMinPartitionKeys(), sc.GetMaxPartitionKeys())) for i := 0; i < len(partitionKeys); i++ { - partitionKeys[i] = &typedef.ColumnDef{Name: GenColumnName("pk", i), Type: GenPartitionKeyColumnType()} + partitionKeys[i] = &typedef.ColumnDef{Name: GenColumnName("pk", i), Type: GenPartitionKeyColumnType(r)} } - clusteringKeys := make(typedef.Columns, utils.RandInt(sc.GetMinClusteringKeys(), sc.GetMaxClusteringKeys())) + clusteringKeys := make(typedef.Columns, utils.RandInt2(r, sc.GetMinClusteringKeys(), sc.GetMaxClusteringKeys())) for i := 0; i < len(clusteringKeys); i++ { - clusteringKeys[i] = &typedef.ColumnDef{Name: GenColumnName("ck", i), Type: GenPrimaryKeyColumnType()} + clusteringKeys[i] = &typedef.ColumnDef{Name: GenColumnName("ck", i), Type: GenPrimaryKeyColumnType(r)} } table := typedef.Table{ Name: tableName, @@ -71,21 +74,21 @@ func genTable(sc typedef.SchemaConfig, tableName string) *typedef.Table { } return &table } - columns := make(typedef.Columns, utils.RandInt(sc.GetMinColumns(), sc.GetMaxColumns())) + columns := make(typedef.Columns, utils.RandInt2(r, sc.GetMinColumns(), sc.GetMaxColumns())) for i := 0; i < len(columns); i++ { - columns[i] = &typedef.ColumnDef{Name: GenColumnName("col", i), Type: GenColumnType(len(columns), &sc)} + columns[i] = &typedef.ColumnDef{Name: GenColumnName("col", i), Type: GenColumnType(len(columns), &sc, r)} } table.Columns = columns var indexes []typedef.IndexDef if sc.CQLFeature > typedef.CQL_FEATURE_BASIC && len(columns) > 0 { - indexes = CreateIndexesForColumn(&table, utils.RandInt(1, len(columns))) + indexes = CreateIndexesForColumn(&table, utils.RandInt2(r, 1, len(columns))) } table.Indexes = indexes var mvs []typedef.MaterializedView if sc.CQLFeature > typedef.CQL_FEATURE_BASIC && len(clusteringKeys) > 0 { - mvs = CreateMaterializedViews(columns, table.Name, partitionKeys, clusteringKeys) + mvs = CreateMaterializedViews(columns, table.Name, partitionKeys, clusteringKeys, r) } table.MaterializedViews = mvs @@ -143,12 +146,12 @@ func GetDropSchema(s *typedef.Schema) []string { } } -func CreateMaterializedViews(c typedef.Columns, tableName string, partitionKeys, clusteringKeys typedef.Columns) []typedef.MaterializedView { +func CreateMaterializedViews(c typedef.Columns, tableName string, partitionKeys, clusteringKeys typedef.Columns, r *rand.Rand) []typedef.MaterializedView { validColumns := c.ValidColumnsForPrimaryKey() var mvs []typedef.MaterializedView numMvs := 1 for i := 0; i < numMvs; i++ { - col := validColumns.Random() + col := validColumns.Random(r) if col == nil { fmt.Printf("unable to generate valid columns for materialized view") continue diff --git a/pkg/jobs/gen_ddl_stmt.go b/pkg/jobs/gen_ddl_stmt.go index 4d3fed50..9d14fc73 100644 --- a/pkg/jobs/gen_ddl_stmt.go +++ b/pkg/jobs/gen_ddl_stmt.go @@ -18,7 +18,6 @@ import ( "fmt" "strings" - "github.com/pkg/errors" "golang.org/x/exp/rand" "github.com/scylladb/gemini/pkg/builders" @@ -36,9 +35,12 @@ func GenDDLStmt(s *typedef.Schema, t *typedef.Table, r *rand.Rand, _ *typedef.Pa // case 0: // Alter column not supported in Cassandra from 3.0.11 // return t.alterColumn(s.Keyspace.Name) case 2: - return genDropColumnStmt(t, s.Keyspace.Name, validCols.Random()) + return genDropColumnStmt(t, s.Keyspace.Name, validCols.Random(r)) default: - column := typedef.ColumnDef{Name: generators.GenColumnName("col", len(t.Columns)+1), Type: generators.GenColumnType(len(t.Columns)+1, sc)} + column := typedef.ColumnDef{ + Name: generators.GenColumnName("col", len(t.Columns)+1), + Type: generators.GenColumnType(len(t.Columns)+1, sc, r), + } return genAddColumnStmt(t, s.Keyspace.Name, &column) } } @@ -82,36 +84,6 @@ func genAddColumnStmt(t *typedef.Table, keyspace string, column *typedef.ColumnD }, nil } -//nolint:unused -func alterColumn(t *typedef.Table, keyspace string) ([]*typedef.Stmt, func(), error) { - var stmts []*typedef.Stmt - idx := rand.Intn(len(t.Columns)) - column := t.Columns[idx] - oldType, isSimpleType := column.Type.(typedef.SimpleType) - if !isSimpleType { - return nil, func() {}, errors.Errorf("complex type=%s cannot be altered", column.Name) - } - compatTypes := typedef.CompatibleColumnTypes[oldType] - if len(compatTypes) == 0 { - return nil, func() {}, errors.Errorf("simple type=%s has no compatible coltypes so it cannot be altered", column.Name) - } - newType := compatTypes.Random() - newColumn := typedef.ColumnDef{Name: column.Name, Type: newType} - stmt := "ALTER TABLE " + keyspace + "." + t.Name + " ALTER " + column.Name + " TYPE " + column.Type.CQLDef() - stmts = append(stmts, &typedef.Stmt{ - StmtCache: &typedef.StmtCache{ - Query: &builders.AlterTableBuilder{ - Stmt: stmt, - }, - QueryType: typedef.AlterColumnStatementType, - }, - }) - return stmts, func() { - t.Columns[idx] = &newColumn - t.ResetQueryCache() - }, nil -} - func genDropColumnStmt(t *typedef.Table, keyspace string, column *typedef.ColumnDef) (*typedef.Stmts, error) { var stmts []*typedef.Stmt diff --git a/pkg/typedef/columns.go b/pkg/typedef/columns.go index dbf267e6..cb00f9ea 100644 --- a/pkg/typedef/columns.go +++ b/pkg/typedef/columns.go @@ -124,8 +124,8 @@ func (c Columns) ValidColumnsForPrimaryKey() Columns { return validCols } -func (c Columns) Random() *ColumnDef { - return c[rand.Intn(len(c))] +func (c Columns) Random(r *rand.Rand) *ColumnDef { + return c[r.Intn(len(c))] } func (c Columns) LenValues() int { diff --git a/pkg/typedef/columns_test.go b/pkg/typedef/columns_test.go index b8390762..0bfb3ac6 100644 --- a/pkg/typedef/columns_test.go +++ b/pkg/typedef/columns_test.go @@ -22,6 +22,7 @@ import ( "github.com/google/go-cmp/cmp" "github.com/google/go-cmp/cmp/cmpopts" + "golang.org/x/exp/rand" "github.com/scylladb/gemini/pkg/generators" "github.com/scylladb/gemini/pkg/typedef" @@ -110,8 +111,9 @@ func TestColumnMarshalUnmarshal(t *testing.T) { func TestMarshalUnmarshal(t *testing.T) { t.Parallel() + r := rand.New(rand.NewSource(rand.Uint64())) - s1 := getTestSchema() + s1 := getTestSchema(r) opts := cmp.Options{ cmp.AllowUnexported(typedef.Table{}, typedef.MaterializedView{}), @@ -134,6 +136,8 @@ func TestMarshalUnmarshal(t *testing.T) { } func TestPrimitives(t *testing.T) { + r := rand.New(rand.NewSource(rand.Uint64())) + t.Parallel() sc := &typedef.SchemaConfig{ @@ -150,11 +154,11 @@ func TestPrimitives(t *testing.T) { cols := typedef.Columns{ &typedef.ColumnDef{ Name: "pk_mv_0", - Type: generators.GenListType(sc), + Type: generators.GenListType(sc, r), }, &typedef.ColumnDef{ Name: "pk_mv_1", - Type: generators.GenTupleType(sc), + Type: generators.GenTupleType(sc, r), }, &typedef.ColumnDef{ Name: "ct_1", @@ -207,7 +211,8 @@ func TestPrimitives(t *testing.T) { func TestValidColumnsForDelete(t *testing.T) { t.Parallel() - s1 := getTestSchema() + r := rand.New(rand.NewSource(rand.Uint64())) + s1 := getTestSchema(r) expected := typedef.Columns{ s1.Tables[0].Columns[2], s1.Tables[0].Columns[3], @@ -241,7 +246,7 @@ func TestValidColumnsForDelete(t *testing.T) { } } -func getTestSchema() *typedef.Schema { +func getTestSchema(r *rand.Rand) *typedef.Schema { sc := &typedef.SchemaConfig{ MaxPartitionKeys: 3, MinPartitionKeys: 2, @@ -255,23 +260,23 @@ func getTestSchema() *typedef.Schema { columns := typedef.Columns{ &typedef.ColumnDef{ Name: generators.GenColumnName("col", 0), - Type: generators.GenMapType(sc), + Type: generators.GenMapType(sc, r), }, &typedef.ColumnDef{ Name: generators.GenColumnName("col", 1), - Type: generators.GenSetType(sc), + Type: generators.GenSetType(sc, r), }, &typedef.ColumnDef{ Name: generators.GenColumnName("col", 2), - Type: generators.GenListType(sc), + Type: generators.GenListType(sc, r), }, &typedef.ColumnDef{ Name: generators.GenColumnName("col", 3), - Type: generators.GenTupleType(sc), + Type: generators.GenTupleType(sc, r), }, &typedef.ColumnDef{ Name: generators.GenColumnName("col", 4), - Type: generators.GenUDTType(sc), + Type: generators.GenUDTType(sc, r), }, } @@ -282,13 +287,13 @@ func getTestSchema() *typedef.Schema { PartitionKeys: typedef.Columns{ &typedef.ColumnDef{ Name: generators.GenColumnName("pk", 0), - Type: generators.GenSimpleType(sc), + Type: generators.GenSimpleType(sc, r), }, }, ClusteringKeys: typedef.Columns{ &typedef.ColumnDef{ Name: generators.GenColumnName("ck", 0), - Type: generators.GenSimpleType(sc), + Type: generators.GenSimpleType(sc, r), }, }, Columns: columns, diff --git a/pkg/typedef/simple_type.go b/pkg/typedef/simple_type.go index 0f7295f6..b507cad6 100644 --- a/pkg/typedef/simple_type.go +++ b/pkg/typedef/simple_type.go @@ -44,8 +44,8 @@ func (l SimpleTypes) Contains(colType Type) bool { return false } -func (l SimpleTypes) Random() SimpleType { - return l[rand.Intn(len(l))] +func (l SimpleTypes) Random(r *rand.Rand) SimpleType { + return l[r.Intn(len(l))] } type SimpleType string diff --git a/pkg/utils/utils.go b/pkg/utils/utils.go index 83973361..14616c2c 100644 --- a/pkg/utils/utils.go +++ b/pkg/utils/utils.go @@ -74,13 +74,6 @@ func RandInt2(rnd *rand.Rand, min, max int) int { return min + rnd.Intn(max-min) } -func RandInt(min, max int) int { - if max <= min { - return min - } - return min + rand.Intn(max-min) -} - func IgnoreError(fn func() error) { _ = fn() } From e64e8e74f718bc312030d1408c02b3fb20d4ec7d Mon Sep 17 00:00:00 2001 From: illia-li Date: Fri, 7 Jul 2023 13:12:31 -0400 Subject: [PATCH 3/3] add(generators) add test for GenSchema with different seeds --- pkg/generators/statement_generator.go | 2 +- pkg/generators/statement_generator_test.go | 77 ++++++++++++++++++++++ 2 files changed, 78 insertions(+), 1 deletion(-) diff --git a/pkg/generators/statement_generator.go b/pkg/generators/statement_generator.go index c5dd3f97..6d2cb84b 100644 --- a/pkg/generators/statement_generator.go +++ b/pkg/generators/statement_generator.go @@ -87,7 +87,7 @@ func genTable(sc typedef.SchemaConfig, tableName string, r *rand.Rand) *typedef. table.Indexes = indexes var mvs []typedef.MaterializedView - if sc.CQLFeature > typedef.CQL_FEATURE_BASIC && len(clusteringKeys) > 0 { + if sc.CQLFeature > typedef.CQL_FEATURE_BASIC && len(clusteringKeys) > 0 && columns.ValidColumnsForPrimaryKey().Len() != 0 { mvs = CreateMaterializedViews(columns, table.Name, partitionKeys, clusteringKeys, r) } diff --git a/pkg/generators/statement_generator_test.go b/pkg/generators/statement_generator_test.go index e4bd3a4f..405abb06 100644 --- a/pkg/generators/statement_generator_test.go +++ b/pkg/generators/statement_generator_test.go @@ -15,11 +15,16 @@ package generators_test import ( + "encoding/json" "testing" + "time" "github.com/google/go-cmp/cmp" + "github.com/google/go-cmp/cmp/cmpopts" + "golang.org/x/exp/rand" "github.com/scylladb/gemini/pkg/generators" + "github.com/scylladb/gemini/pkg/replication" "github.com/scylladb/gemini/pkg/tableopts" "github.com/scylladb/gemini/pkg/typedef" ) @@ -180,6 +185,49 @@ func TestGetCreateSchema(t *testing.T) { } } +func TestGenSchema(t *testing.T) { + seeds := [10]uint64{ + uint64(10 + rand.Intn(10)), + uint64(100 + rand.Intn(100)), + uint64(1000 + rand.Intn(1000)), + uint64(10000 + rand.Intn(10000)), + uint64(100000 + rand.Intn(100000)), + uint64(1000000 + rand.Intn(1000000)), + uint64(10000000 + rand.Intn(10000000)), + uint64(100000000 + rand.Intn(100000000)), + uint64(1000000000 + rand.Intn(1000000000)), + uint64(time.Now().Nanosecond()), + } + + for idx := range seeds { + testSchema := generators.GenSchema(testSchemaConfig, seeds[idx]) + testSchema.Config = typedef.SchemaConfig{} + transformAndDiff(t, testSchema) + } +} + +func transformAndDiff(t *testing.T, testSchema *typedef.Schema) { + t.Helper() + opts := cmp.Options{ + cmp.AllowUnexported(typedef.Table{}, typedef.MaterializedView{}), + cmpopts.IgnoreUnexported(typedef.Table{}, typedef.MaterializedView{}), + cmpopts.EquateEmpty(), + } + + testSchemaMarshaled, err := json.MarshalIndent(testSchema, " ", " ") + if err != nil { + t.Fatalf("unable to marshal schema example json, error=%s\n", err) + } + testSchemaTransformed := typedef.Schema{} + if err = json.Unmarshal(testSchemaMarshaled, &testSchemaTransformed); err != nil { + t.Fatalf("unable to unmarshal json, error=%s\n", err) + } + + if diff := cmp.Diff(*testSchema, testSchemaTransformed, opts); diff != "" { + t.Fatalf("schema not the same after marshal/unmarshal, diff=%s", diff) + } +} + func createColumns(cnt int, prefix string) typedef.Columns { var cols typedef.Columns for i := 0; i < cnt; i++ { @@ -190,3 +238,32 @@ func createColumns(cnt int, prefix string) typedef.Columns { } return cols } + +var testSchemaConfig = typedef.SchemaConfig{ + ReplicationStrategy: replication.NewSimpleStrategy(), + OracleReplicationStrategy: replication.NewSimpleStrategy(), + TableOptions: tableopts.CreateTableOptions([]string{ + "compression = {'sstable_compression':'LZ4Compressor'}", + "read_repair_chance = 1.0", "comment = 'Important records'", "cdc = {'enabled':'true','preimage':'true'}", + "compaction = {'class':'LeveledCompactionStrategy','enabled':true,'sstable_size_in_mb':160,'tombstone_compaction_interval':86400,'tombstone_threshold':0.2}", + }, + nil), + MaxTables: 10, + MaxPartitionKeys: 10, + MinPartitionKeys: 1, + MaxClusteringKeys: 10, + MinClusteringKeys: 1, + MaxColumns: 25, + MinColumns: 1, + MaxUDTParts: 20, + MaxTupleParts: 20, + MaxBlobLength: 1e4, + MaxStringLength: 1000, + MinBlobLength: 0, + MinStringLength: 0, + UseCounters: false, + UseLWT: false, + CQLFeature: typedef.CQL_FEATURE_NORMAL, + AsyncObjectStabilizationAttempts: 10, + AsyncObjectStabilizationDelay: 100000, +}