Skip to content

Commit

Permalink
Merge pull request #381 from illia-li/il/add-typedef_generators-schem…
Browse files Browse the repository at this point in the history 
…a_tests_add

add(typedef,generators) schema tests add
  • Loading branch information
@dkropachev
dkropachev authored Jul 7, 2023
2 parents 2a125f3 + e64e8e7 commit 8442ad5
Show file tree
Hide file tree
Showing 15 changed files with 1,169 additions and 122 deletions.
16 changes: 1 addition & 15 deletions cmd/gemini/root.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -33,7 +33,6 @@ import (
"github.com/scylladb/gemini/pkg/jobs"
"github.com/scylladb/gemini/pkg/replication"
"github.com/scylladb/gemini/pkg/store"
"github.com/scylladb/gemini/pkg/tableopts"
"github.com/scylladb/gemini/pkg/typedef"
"github.com/scylladb/gemini/pkg/utils"

Expand Down Expand Up @@ -198,7 +197,7 @@ func run(_ *cobra.Command, _ []string) error {
return errors.Wrap(err, "cannot create schema")
}
} else {
schema = generators.GenSchema(schemaConfig)
schema = generators.GenSchema(schemaConfig, seed)
}

jsonSchema, _ := json.MarshalIndent(schema, "", " ")
Expand Down Expand Up @@ -393,19 +392,6 @@ func createClusters(
return testCluster, oracleCluster
}

func createTableOptions(tableOptionStrings []string, logger *zap.Logger) []tableopts.Option {
var tableOptions []tableopts.Option
for _, optionString := range tableOptionStrings {
o, err := tableopts.FromCQL(optionString)
if err != nil {
logger.Warn("invalid table option", zap.String("option", optionString), zap.Error(err))
continue
}
tableOptions = append(tableOptions, o)
}
return tableOptions
}

func getReplicationStrategy(rs string, fallback *replication.Replication, logger *zap.Logger) *replication.Replication {
switch rs {
case "network":
Expand Down
3 changes: 2 additions & 1 deletion cmd/gemini/schema.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,7 @@ import (
"strings"

"github.com/scylladb/gemini/pkg/replication"
"github.com/scylladb/gemini/pkg/tableopts"
"github.com/scylladb/gemini/pkg/typedef"

"go.uber.org/zap"
Expand Down Expand Up @@ -67,7 +68,7 @@ func createDefaultSchemaConfig(logger *zap.Logger) typedef.SchemaConfig {
return typedef.SchemaConfig{
ReplicationStrategy: rs,
OracleReplicationStrategy: ors,
TableOptions: createTableOptions(tableOptions, logger),
TableOptions: tableopts.CreateTableOptions(tableOptions, logger),
MaxTables: maxTables,
MaxPartitionKeys: maxPartitionKeys,
MinPartitionKeys: minPartitionKeys,
Expand Down
3 changes: 2 additions & 1 deletion cmd/gemini/strategies_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -60,7 +60,8 @@ func TestGetReplicationStrategy(t *testing.T) {
}
}

func TestReadSchema(t *testing.T) {
// TestReadExampleSchema main task of this test to be sure that schema example (schema.json) is correct and have correct marshal, unmarshal
func TestReadExampleSchema(t *testing.T) {
filePath := "schema.json"

testSchema, err := readSchema(filePath, typedef.SchemaConfig{})
Expand Down
68 changes: 34 additions & 34 deletions pkg/generators/column_generator.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -26,51 +26,51 @@ func GenColumnName(prefix string, idx int) string {
return fmt.Sprintf("%s%d", prefix, idx)
}

func GenColumnType(numColumns int, sc *typedef.SchemaConfig) typedef.Type {
n := rand.Intn(numColumns + 5)
func GenColumnType(numColumns int, sc *typedef.SchemaConfig, r *rand.Rand) typedef.Type {
n := r.Intn(numColumns + 5)
switch n {
case numColumns:
return GenTupleType(sc)
return GenTupleType(sc, r)
case numColumns + 1:
return GenUDTType(sc)
return GenUDTType(sc, r)
case numColumns + 2:
return GenSetType(sc)
return GenSetType(sc, r)
case numColumns + 3:
return GenListType(sc)
return GenListType(sc, r)
case numColumns + 4:
return GenMapType(sc)
return GenMapType(sc, r)
default:
return GenSimpleType(sc)
return GenSimpleType(sc, r)
}
}

func GenSimpleType(_ *typedef.SchemaConfig) typedef.SimpleType {
return typedef.AllTypes[rand.Intn(len(typedef.AllTypes))]
func GenSimpleType(_ *typedef.SchemaConfig, r *rand.Rand) typedef.SimpleType {
return typedef.AllTypes[r.Intn(len(typedef.AllTypes))]
}

func GenTupleType(sc *typedef.SchemaConfig) typedef.Type {
n := rand.Intn(sc.MaxTupleParts)
func GenTupleType(sc *typedef.SchemaConfig, r *rand.Rand) typedef.Type {
n := r.Intn(sc.MaxTupleParts)
if n < 2 {
n = 2
}
typeList := make([]typedef.SimpleType, n)
for i := 0; i < n; i++ {
typeList[i] = GenSimpleType(sc)
typeList[i] = GenSimpleType(sc, r)
}
return &typedef.TupleType{
ComplexType: typedef.TYPE_TUPLE,
ValueTypes: typeList,
Frozen: rand.Uint32()%2 == 0,
Frozen: r.Uint32()%2 == 0,
}
}

func GenUDTType(sc *typedef.SchemaConfig) *typedef.UDTType {
udtNum := rand.Uint32()
func GenUDTType(sc *typedef.SchemaConfig, r *rand.Rand) *typedef.UDTType {
udtNum := r.Uint32()
typeName := fmt.Sprintf("udt_%d", udtNum)
ts := make(map[string]typedef.SimpleType)

for i := 0; i < rand.Intn(sc.MaxUDTParts)+1; i++ {
ts[typeName+fmt.Sprintf("_%d", i)] = GenSimpleType(sc)
for i := 0; i < r.Intn(sc.MaxUDTParts)+1; i++ {
ts[typeName+fmt.Sprintf("_%d", i)] = GenSimpleType(sc, r)
}

return &typedef.UDTType{
Expand All @@ -81,51 +81,51 @@ func GenUDTType(sc *typedef.SchemaConfig) *typedef.UDTType {
}
}

func GenSetType(sc *typedef.SchemaConfig) *typedef.BagType {
return genBagType(typedef.TYPE_SET, sc)
func GenSetType(sc *typedef.SchemaConfig, r *rand.Rand) *typedef.BagType {
return genBagType(typedef.TYPE_SET, sc, r)
}

func GenListType(sc *typedef.SchemaConfig) *typedef.BagType {
return genBagType(typedef.TYPE_LIST, sc)
func GenListType(sc *typedef.SchemaConfig, r *rand.Rand) *typedef.BagType {
return genBagType(typedef.TYPE_LIST, sc, r)
}

func genBagType(kind string, sc *typedef.SchemaConfig) *typedef.BagType {
func genBagType(kind string, sc *typedef.SchemaConfig, r *rand.Rand) *typedef.BagType {
var t typedef.SimpleType
for {
t = GenSimpleType(sc)
t = GenSimpleType(sc, r)
if t != typedef.TYPE_DURATION {
break
}
}
return &typedef.BagType{
ComplexType: kind,
ValueType: t,
Frozen: rand.Uint32()%2 == 0,
Frozen: r.Uint32()%2 == 0,
}
}

func GenMapType(sc *typedef.SchemaConfig) *typedef.MapType {
t := GenSimpleType(sc)
func GenMapType(sc *typedef.SchemaConfig, r *rand.Rand) *typedef.MapType {
t := GenSimpleType(sc, r)
for {
if _, ok := typedef.TypesMapKeyBlacklist[t]; !ok {
break
}
t = GenSimpleType(sc)
t = GenSimpleType(sc, r)
}
return &typedef.MapType{
ComplexType: typedef.TYPE_MAP,
KeyType: t,
ValueType: GenSimpleType(sc),
Frozen: rand.Uint32()%2 == 0,
ValueType: GenSimpleType(sc, r),
Frozen: r.Uint32()%2 == 0,
}
}

func GenPartitionKeyColumnType() typedef.Type {
return typedef.PartitionKeyTypes[rand.Intn(len(typedef.PartitionKeyTypes))]
func GenPartitionKeyColumnType(r *rand.Rand) typedef.Type {
return typedef.PartitionKeyTypes[r.Intn(len(typedef.PartitionKeyTypes))]
}

func GenPrimaryKeyColumnType() typedef.Type {
return typedef.PkTypes[rand.Intn(len(typedef.PkTypes))]
func GenPrimaryKeyColumnType(r *rand.Rand) typedef.Type {
return typedef.PkTypes[r.Intn(len(typedef.PkTypes))]
}

func GenIndexName(prefix string, idx int) string {
Expand Down
33 changes: 18 additions & 15 deletions pkg/generators/statement_generator.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -18,12 +18,15 @@ import (
"fmt"
"strings"

"golang.org/x/exp/rand"

"github.com/scylladb/gemini/pkg/builders"
"github.com/scylladb/gemini/pkg/typedef"
"github.com/scylladb/gemini/pkg/utils"
)

func GenSchema(sc typedef.SchemaConfig) *typedef.Schema {
func GenSchema(sc typedef.SchemaConfig, seed uint64) *typedef.Schema {
r := rand.New(rand.NewSource(seed))
builder := builders.NewSchemaBuilder()
builder.Config(sc)
keyspace := typedef.Keyspace{
Expand All @@ -32,22 +35,22 @@ func GenSchema(sc typedef.SchemaConfig) *typedef.Schema {
OracleReplication: sc.OracleReplicationStrategy,
}
builder.Keyspace(keyspace)
numTables := utils.RandInt(1, sc.GetMaxTables())
numTables := utils.RandInt2(r, 1, sc.GetMaxTables())
for i := 0; i < numTables; i++ {
table := genTable(sc, fmt.Sprintf("table%d", i+1))
table := genTable(sc, fmt.Sprintf("table%d", i+1), r)
builder.Table(table)
}
return builder.Build()
}

func genTable(sc typedef.SchemaConfig, tableName string) *typedef.Table {
partitionKeys := make(typedef.Columns, utils.RandInt(sc.GetMinPartitionKeys(), sc.GetMaxPartitionKeys()))
func genTable(sc typedef.SchemaConfig, tableName string, r *rand.Rand) *typedef.Table {
partitionKeys := make(typedef.Columns, utils.RandInt2(r, sc.GetMinPartitionKeys(), sc.GetMaxPartitionKeys()))
for i := 0; i < len(partitionKeys); i++ {
partitionKeys[i] = &typedef.ColumnDef{Name: GenColumnName("pk", i), Type: GenPartitionKeyColumnType()}
partitionKeys[i] = &typedef.ColumnDef{Name: GenColumnName("pk", i), Type: GenPartitionKeyColumnType(r)}
}
clusteringKeys := make(typedef.Columns, utils.RandInt(sc.GetMinClusteringKeys(), sc.GetMaxClusteringKeys()))
clusteringKeys := make(typedef.Columns, utils.RandInt2(r, sc.GetMinClusteringKeys(), sc.GetMaxClusteringKeys()))
for i := 0; i < len(clusteringKeys); i++ {
clusteringKeys[i] = &typedef.ColumnDef{Name: GenColumnName("ck", i), Type: GenPrimaryKeyColumnType()}
clusteringKeys[i] = &typedef.ColumnDef{Name: GenColumnName("ck", i), Type: GenPrimaryKeyColumnType(r)}
}
table := typedef.Table{
Name: tableName,
Expand All @@ -71,21 +74,21 @@ func genTable(sc typedef.SchemaConfig, tableName string) *typedef.Table {
}
return &table
}
columns := make(typedef.Columns, utils.RandInt(sc.GetMinColumns(), sc.GetMaxColumns()))
columns := make(typedef.Columns, utils.RandInt2(r, sc.GetMinColumns(), sc.GetMaxColumns()))
for i := 0; i < len(columns); i++ {
columns[i] = &typedef.ColumnDef{Name: GenColumnName("col", i), Type: GenColumnType(len(columns), &sc)}
columns[i] = &typedef.ColumnDef{Name: GenColumnName("col", i), Type: GenColumnType(len(columns), &sc, r)}
}
table.Columns = columns

var indexes []typedef.IndexDef
if sc.CQLFeature > typedef.CQL_FEATURE_BASIC && len(columns) > 0 {
indexes = CreateIndexesForColumn(&table, utils.RandInt(1, len(columns)))
indexes = CreateIndexesForColumn(&table, utils.RandInt2(r, 1, len(columns)))
}
table.Indexes = indexes

var mvs []typedef.MaterializedView
if sc.CQLFeature > typedef.CQL_FEATURE_BASIC && len(clusteringKeys) > 0 {
mvs = CreateMaterializedViews(columns, table.Name, partitionKeys, clusteringKeys)
if sc.CQLFeature > typedef.CQL_FEATURE_BASIC && len(clusteringKeys) > 0 && columns.ValidColumnsForPrimaryKey().Len() != 0 {
mvs = CreateMaterializedViews(columns, table.Name, partitionKeys, clusteringKeys, r)
}

table.MaterializedViews = mvs
Expand Down Expand Up @@ -143,12 +146,12 @@ func GetDropSchema(s *typedef.Schema) []string {
}
}

func CreateMaterializedViews(c typedef.Columns, tableName string, partitionKeys, clusteringKeys typedef.Columns) []typedef.MaterializedView {
func CreateMaterializedViews(c typedef.Columns, tableName string, partitionKeys, clusteringKeys typedef.Columns, r *rand.Rand) []typedef.MaterializedView {
validColumns := c.ValidColumnsForPrimaryKey()
var mvs []typedef.MaterializedView
numMvs := 1
for i := 0; i < numMvs; i++ {
col := validColumns.Random()
col := validColumns.Random(r)
if col == nil {
fmt.Printf("unable to generate valid columns for materialized view")
continue
Expand Down
77 changes: 77 additions & 0 deletions pkg/generators/statement_generator_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -15,11 +15,16 @@
package generators_test

import (
"encoding/json"
"testing"
"time"

"github.com/google/go-cmp/cmp"
"github.com/google/go-cmp/cmp/cmpopts"
"golang.org/x/exp/rand"

"github.com/scylladb/gemini/pkg/generators"
"github.com/scylladb/gemini/pkg/replication"
"github.com/scylladb/gemini/pkg/tableopts"
"github.com/scylladb/gemini/pkg/typedef"
)
Expand Down Expand Up @@ -180,6 +185,49 @@ func TestGetCreateSchema(t *testing.T) {
}
}

func TestGenSchema(t *testing.T) {
seeds := [10]uint64{
uint64(10 + rand.Intn(10)),
uint64(100 + rand.Intn(100)),
uint64(1000 + rand.Intn(1000)),
uint64(10000 + rand.Intn(10000)),
uint64(100000 + rand.Intn(100000)),
uint64(1000000 + rand.Intn(1000000)),
uint64(10000000 + rand.Intn(10000000)),
uint64(100000000 + rand.Intn(100000000)),
uint64(1000000000 + rand.Intn(1000000000)),
uint64(time.Now().Nanosecond()),
}

for idx := range seeds {
testSchema := generators.GenSchema(testSchemaConfig, seeds[idx])
testSchema.Config = typedef.SchemaConfig{}
transformAndDiff(t, testSchema)
}
}

func transformAndDiff(t *testing.T, testSchema *typedef.Schema) {
t.Helper()
opts := cmp.Options{
cmp.AllowUnexported(typedef.Table{}, typedef.MaterializedView{}),
cmpopts.IgnoreUnexported(typedef.Table{}, typedef.MaterializedView{}),
cmpopts.EquateEmpty(),
}

testSchemaMarshaled, err := json.MarshalIndent(testSchema, " ", " ")
if err != nil {
t.Fatalf("unable to marshal schema example json, error=%s\n", err)
}
testSchemaTransformed := typedef.Schema{}
if err = json.Unmarshal(testSchemaMarshaled, &testSchemaTransformed); err != nil {
t.Fatalf("unable to unmarshal json, error=%s\n", err)
}

if diff := cmp.Diff(*testSchema, testSchemaTransformed, opts); diff != "" {
t.Fatalf("schema not the same after marshal/unmarshal, diff=%s", diff)
}
}

func createColumns(cnt int, prefix string) typedef.Columns {
var cols typedef.Columns
for i := 0; i < cnt; i++ {
Expand All @@ -190,3 +238,32 @@ func createColumns(cnt int, prefix string) typedef.Columns {
}
return cols
}

var testSchemaConfig = typedef.SchemaConfig{
ReplicationStrategy: replication.NewSimpleStrategy(),
OracleReplicationStrategy: replication.NewSimpleStrategy(),
TableOptions: tableopts.CreateTableOptions([]string{
"compression = {'sstable_compression':'LZ4Compressor'}",
"read_repair_chance = 1.0", "comment = 'Important records'", "cdc = {'enabled':'true','preimage':'true'}",
"compaction = {'class':'LeveledCompactionStrategy','enabled':true,'sstable_size_in_mb':160,'tombstone_compaction_interval':86400,'tombstone_threshold':0.2}",
},
nil),
MaxTables: 10,
MaxPartitionKeys: 10,
MinPartitionKeys: 1,
MaxClusteringKeys: 10,
MinClusteringKeys: 1,
MaxColumns: 25,
MinColumns: 1,
MaxUDTParts: 20,
MaxTupleParts: 20,
MaxBlobLength: 1e4,
MaxStringLength: 1000,
MinBlobLength: 0,
MinStringLength: 0,
UseCounters: false,
UseLWT: false,
CQLFeature: typedef.CQL_FEATURE_NORMAL,
AsyncObjectStabilizationAttempts: 10,
AsyncObjectStabilizationDelay: 100000,
}
Loading

0 comments on commit 8442ad5

Please sign in to comment.