diff --git a/go/mysql/collations/env.go b/go/mysql/collations/env.go index 9fe87230649..a3439d49ef5 100644 --- a/go/mysql/collations/env.go +++ b/go/mysql/collations/env.go @@ -180,7 +180,7 @@ func makeEnv(version collver) *Environment { } } - for from, to := range charsetAliases() { + for from, to := range charsetAliases(env.version) { env.byCharset[from] = env.byCharset[to] } @@ -212,14 +212,12 @@ var SystemCollation = TypedCollation{ // this mapping will change, so it's important to use this helper so that // Vitess code has a consistent mapping for the active collations environment. func (env *Environment) CharsetAlias(charset string) (alias string, ok bool) { - alias, ok = charsetAliases()[charset] + alias, ok = charsetAliases(env.version)[charset] return } // CollationAlias returns the internal collaction name for the given charset. -// For now, this maps all `utf8` to `utf8mb3` collation names; in future versions of MySQL, -// this mapping will change, so it's important to use this helper so that -// Vitess code has a consistent mapping for the active collations environment. +// For now, this maps `utf8` to `utf8mb4` collation names for MySQL 8.0 and maps the rest to `utf8mb3`. func (env *Environment) CollationAlias(collation string) (string, bool) { col := env.LookupByName(collation) if col == Unknown { @@ -233,7 +231,7 @@ func (env *Environment) CollationAlias(collation string) (string, bool) { return collation, false } for _, alias := range allCols.alias { - for source, dest := range charsetAliases() { + for source, dest := range charsetAliases(env.version) { if strings.HasPrefix(collation, fmt.Sprintf("%s_", source)) && strings.HasPrefix(alias.name, fmt.Sprintf("%s_", dest)) { return alias.name, true diff --git a/go/mysql/collations/mysqlversion.go b/go/mysql/collations/mysqlversion.go index 93d1add9b6a..fe5a8b4314b 100644 --- a/go/mysql/collations/mysqlversion.go +++ b/go/mysql/collations/mysqlversion.go @@ -58,7 +58,15 @@ func (v collver) String() string { panic("invalid version identifier") } } -func charsetAliases() map[string]string { return map[string]string{"utf8": "utf8mb3"} } + +func charsetAliases(version collver) map[string]string { + switch version { + case collverMySQL8: + return map[string]string{"utf8": "utf8mb4"} + default: + return map[string]string{"utf8": "utf8mb3"} + } +} var globalVersionInfo = map[ID]struct { alias []collalias @@ -108,7 +116,6 @@ var globalVersionInfo = map[ID]struct { 43: {alias: []collalias{{0b01111111, "macce_bin", "macce"}}, isdefault: 0b00000000}, 44: {alias: []collalias{{0b01111111, "cp1250_croatian_ci", "cp1250"}}, isdefault: 0b00000000}, 45: {alias: []collalias{{0b01111111, "utf8mb4_general_ci", "utf8mb4"}}, isdefault: 0b00111111}, - 46: {alias: []collalias{{0b01111111, "utf8mb4_bin", "utf8mb4"}}, isdefault: 0b00000000}, 47: {alias: []collalias{{0b01111111, "latin1_bin", "latin1"}}, isdefault: 0b00000000}, 48: {alias: []collalias{{0b01111111, "latin1_general_ci", "latin1"}}, isdefault: 0b00000000}, 49: {alias: []collalias{{0b01111111, "latin1_general_cs", "latin1"}}, isdefault: 0b00000000}, @@ -145,7 +152,7 @@ var globalVersionInfo = map[ID]struct { 80: {alias: []collalias{{0b01111111, "cp850_bin", "cp850"}}, isdefault: 0b00000000}, 81: {alias: []collalias{{0b01111111, "cp852_bin", "cp852"}}, isdefault: 0b00000000}, 82: {alias: []collalias{{0b01111111, "swe7_bin", "swe7"}}, isdefault: 0b00000000}, - 83: {alias: []collalias{{0b01111111, "utf8_bin", "utf8"}, {0b01111111, "utf8mb3_bin", "utf8mb3"}}, isdefault: 0b00000000}, + 83: {alias: []collalias{{0b01111111, "utf8_bin", "utf8"}, {0b01111111, "utf8mb3_bin", "utf8mb3"}, {0b01111111, "utf8mb4_bin", "utf8mb4"}}, isdefault: 0b00000000}, 84: {alias: []collalias{{0b01111111, "big5_bin", "big5"}}, isdefault: 0b00000000}, 85: {alias: []collalias{{0b01111111, "euckr_bin", "euckr"}}, isdefault: 0b00000000}, 86: {alias: []collalias{{0b01111111, "gb2312_bin", "gb2312"}}, isdefault: 0b00000000}, diff --git a/go/vt/schemadiff/column.go b/go/vt/schemadiff/column.go index d36d226294d..be143d7b6b8 100644 --- a/go/vt/schemadiff/column.go +++ b/go/vt/schemadiff/column.go @@ -130,7 +130,7 @@ func (c *ColumnDefinitionEntity) ColumnDiff( }() c.columnDefinition.Type.Options.Collate = t1cc.collate } - if c.columnDefinition.Type.Options.Collate = t1cc.collate; c.columnDefinition.Type.Options.Collate == "" { + if c.columnDefinition.Type.Options.Collate = t1cc.collate; t1cc.charset != "" && c.columnDefinition.Type.Options.Collate == "" { collation := env.CollationEnv().DefaultCollationForCharset(t1cc.charset) if collation == collations.Unknown { return nil, vterrors.Errorf(vtrpcpb.Code_INVALID_ARGUMENT, "cannot match collation to charset %v", t1cc.charset) @@ -157,7 +157,7 @@ func (c *ColumnDefinitionEntity) ColumnDiff( other.columnDefinition.Type.Options.Collate = "" }() other.columnDefinition.Type.Charset.Name = t2cc.charset - if other.columnDefinition.Type.Options.Collate = t2cc.collate; other.columnDefinition.Type.Options.Collate == "" { + if other.columnDefinition.Type.Options.Collate = t2cc.collate; t2cc.charset != "" && other.columnDefinition.Type.Options.Collate == "" { collation := env.CollationEnv().DefaultCollationForCharset(t2cc.charset) if collation == collations.Unknown { return nil, vterrors.Errorf(vtrpcpb.Code_INVALID_ARGUMENT, "cannot match collation to charset %v", t2cc.charset) diff --git a/go/vt/sidecardb/schema/schemaengine/tables.sql b/go/vt/sidecardb/schema/schemaengine/tables.sql index 3aadc7c9635..5192fe7d95e 100644 --- a/go/vt/sidecardb/schema/schemaengine/tables.sql +++ b/go/vt/sidecardb/schema/schemaengine/tables.sql @@ -16,8 +16,8 @@ limitations under the License. CREATE TABLE IF NOT EXISTS tables ( - TABLE_SCHEMA varchar(64) CHARACTER SET `utf8mb3` COLLATE `utf8mb3_bin` NOT NULL, - TABLE_NAME varchar(64) CHARACTER SET `utf8mb3` COLLATE `utf8mb3_bin` NOT NULL, + TABLE_SCHEMA varchar(64) CHARACTER SET `utf8` COLLATE `utf8_bin` NOT NULL, + TABLE_NAME varchar(64) CHARACTER SET `utf8` COLLATE `utf8_bin` NOT NULL, CREATE_STATEMENT longtext, CREATE_TIME BIGINT, PRIMARY KEY (TABLE_SCHEMA, TABLE_NAME) diff --git a/go/vt/sidecardb/schema/schemaengine/views.sql b/go/vt/sidecardb/schema/schemaengine/views.sql index dd242e6567f..5bed46b31a4 100644 --- a/go/vt/sidecardb/schema/schemaengine/views.sql +++ b/go/vt/sidecardb/schema/schemaengine/views.sql @@ -16,8 +16,8 @@ limitations under the License. CREATE TABLE IF NOT EXISTS views ( - TABLE_SCHEMA varchar(64) CHARACTER SET `utf8mb3` COLLATE `utf8mb3_bin` NOT NULL, - TABLE_NAME varchar(64) CHARACTER SET `utf8mb3` COLLATE `utf8mb3_bin` NOT NULL, + TABLE_SCHEMA varchar(64) CHARACTER SET `utf8` COLLATE `utf8_bin` NOT NULL, + TABLE_NAME varchar(64) CHARACTER SET `utf8` COLLATE `utf8_bin` NOT NULL, CREATE_STATEMENT longtext, VIEW_DEFINITION longtext NOT NULL, PRIMARY KEY (TABLE_SCHEMA, TABLE_NAME) diff --git a/go/vt/sidecardb/sidecardb_test.go b/go/vt/sidecardb/sidecardb_test.go index 55c2c6cd6b5..f49007b4976 100644 --- a/go/vt/sidecardb/sidecardb_test.go +++ b/go/vt/sidecardb/sidecardb_test.go @@ -24,13 +24,13 @@ import ( "strings" "testing" + "github.com/stretchr/testify/require" + "vitess.io/vitess/go/constants/sidecar" "vitess.io/vitess/go/vt/dbconfigs" "vitess.io/vitess/go/vt/sqlparser" "vitess.io/vitess/go/vt/vtenv" - "github.com/stretchr/testify/require" - "vitess.io/vitess/go/stats" "vitess.io/vitess/go/mysql/fakesqldb" @@ -249,3 +249,102 @@ func TestAlterTableAlgorithm(t *testing.T) { }) } } + +// TestCollationsForSchemaEngineTables tests that the correct collation and character set is used for +// schema engine tables based on the MySQL version being used. +func TestCollationsForSchemaEngineTables(t *testing.T) { + testcases := []struct { + name string + mysqlVersion string + tableName string + currentSchema string + expectedDiff string + }{ + { + name: "tables schema in MySQL 5.7", + mysqlVersion: "5.7.31", + tableName: "tables", + currentSchema: "", + expectedDiff: "CREATE TABLE IF NOT EXISTS `_vt`.`tables` (\n\t`TABLE_SCHEMA` varchar(64) CHARACTER SET utf8mb3 COLLATE utf8mb3_bin NOT NULL,\n\t`TABLE_NAME` varchar(64) CHARACTER SET utf8mb3 COLLATE utf8mb3_bin NOT NULL,\n\t`CREATE_STATEMENT` longtext,\n\t`CREATE_TIME` bigint,\n\tPRIMARY KEY (`TABLE_SCHEMA`, `TABLE_NAME`)\n) ENGINE InnoDB", + }, + { + name: "tables schema in MySQL 8.0", + mysqlVersion: "8.0.30", + tableName: "tables", + currentSchema: "", + expectedDiff: "CREATE TABLE IF NOT EXISTS `_vt`.`tables` (\n\t`TABLE_SCHEMA` varchar(64) CHARACTER SET utf8mb4 COLLATE utf8mb4_bin NOT NULL,\n\t`TABLE_NAME` varchar(64) CHARACTER SET utf8mb4 COLLATE utf8mb4_bin NOT NULL,\n\t`CREATE_STATEMENT` longtext,\n\t`CREATE_TIME` bigint,\n\tPRIMARY KEY (`TABLE_SCHEMA`, `TABLE_NAME`)\n) ENGINE InnoDB", + }, + { + name: "views schema in MySQL 5.7", + mysqlVersion: "5.7.31", + tableName: "views", + currentSchema: "", + expectedDiff: "CREATE TABLE IF NOT EXISTS `_vt`.`views` (\n\t`TABLE_SCHEMA` varchar(64) CHARACTER SET utf8mb3 COLLATE utf8mb3_bin NOT NULL,\n\t`TABLE_NAME` varchar(64) CHARACTER SET utf8mb3 COLLATE utf8mb3_bin NOT NULL,\n\t`CREATE_STATEMENT` longtext,\n\t`VIEW_DEFINITION` longtext NOT NULL,\n\tPRIMARY KEY (`TABLE_SCHEMA`, `TABLE_NAME`)\n) ENGINE InnoDB", + }, + { + name: "views schema in MySQL 8.0", + mysqlVersion: "8.0.30", + tableName: "views", + currentSchema: "", + expectedDiff: "CREATE TABLE IF NOT EXISTS `_vt`.`views` (\n\t`TABLE_SCHEMA` varchar(64) CHARACTER SET utf8mb4 COLLATE utf8mb4_bin NOT NULL,\n\t`TABLE_NAME` varchar(64) CHARACTER SET utf8mb4 COLLATE utf8mb4_bin NOT NULL,\n\t`CREATE_STATEMENT` longtext,\n\t`VIEW_DEFINITION` longtext NOT NULL,\n\tPRIMARY KEY (`TABLE_SCHEMA`, `TABLE_NAME`)\n) ENGINE InnoDB", + }, + { + name: "tables upgrade from MySQL 5.7 to MySQL 8.0", + mysqlVersion: "8.0.30", + tableName: "tables", + currentSchema: "CREATE TABLE IF NOT EXISTS `_vt`.`tables` (\n\t`TABLE_SCHEMA` varchar(64) CHARACTER SET utf8mb3 COLLATE utf8mb3_bin NOT NULL,\n\t`TABLE_NAME` varchar(64) CHARACTER SET utf8mb3 COLLATE utf8mb3_bin NOT NULL,\n\t`CREATE_STATEMENT` longtext,\n\t`CREATE_TIME` bigint,\n\tPRIMARY KEY (`TABLE_SCHEMA`, `TABLE_NAME`)\n) ENGINE InnoDB", + expectedDiff: "ALTER TABLE `_vt`.`tables` MODIFY COLUMN `TABLE_SCHEMA` varchar(64) CHARACTER SET utf8mb4 COLLATE utf8mb4_bin NOT NULL, MODIFY COLUMN `TABLE_NAME` varchar(64) CHARACTER SET utf8mb4 COLLATE utf8mb4_bin NOT NULL, ALGORITHM = COPY", + }, + { + name: "views upgrade from MySQL 5.7 to MySQL 8.0", + mysqlVersion: "8.0.30", + tableName: "views", + currentSchema: "CREATE TABLE IF NOT EXISTS `_vt`.`views` (\n\t`TABLE_SCHEMA` varchar(64) CHARACTER SET utf8mb3 COLLATE utf8mb3_bin NOT NULL,\n\t`TABLE_NAME` varchar(64) CHARACTER SET utf8mb3 COLLATE utf8mb3_bin NOT NULL,\n\t`CREATE_STATEMENT` longtext,\n\t`VIEW_DEFINITION` longtext NOT NULL,\n\tPRIMARY KEY (`TABLE_SCHEMA`, `TABLE_NAME`)\n) ENGINE InnoDB", + expectedDiff: "ALTER TABLE `_vt`.`views` MODIFY COLUMN `TABLE_SCHEMA` varchar(64) CHARACTER SET utf8mb4 COLLATE utf8mb4_bin NOT NULL, MODIFY COLUMN `TABLE_NAME` varchar(64) CHARACTER SET utf8mb4 COLLATE utf8mb4_bin NOT NULL, ALGORITHM = COPY", + }, + { + name: "tables downgrade from MySQL 8.0 to MySQL 5.7", + mysqlVersion: "5.7.31", + tableName: "tables", + currentSchema: "CREATE TABLE IF NOT EXISTS `_vt`.`tables` (\n\t`TABLE_SCHEMA` varchar(64) CHARACTER SET utf8mb4 COLLATE utf8mb4_bin NOT NULL,\n\t`TABLE_NAME` varchar(64) CHARACTER SET utf8mb4 COLLATE utf8mb4_bin NOT NULL,\n\t`CREATE_STATEMENT` longtext,\n\t`CREATE_TIME` bigint,\n\tPRIMARY KEY (`TABLE_SCHEMA`, `TABLE_NAME`)\n) ENGINE InnoDB", + expectedDiff: "ALTER TABLE `_vt`.`tables` MODIFY COLUMN `TABLE_SCHEMA` varchar(64) CHARACTER SET utf8mb3 COLLATE utf8mb3_bin NOT NULL, MODIFY COLUMN `TABLE_NAME` varchar(64) CHARACTER SET utf8mb3 COLLATE utf8mb3_bin NOT NULL, ALGORITHM = COPY", + }, + { + name: "views downgrade from MySQL 8.0 to MySQL 5.7", + mysqlVersion: "5.7.31", + tableName: "views", + currentSchema: "CREATE TABLE IF NOT EXISTS `_vt`.`views` (\n\t`TABLE_SCHEMA` varchar(64) CHARACTER SET utf8mb4 COLLATE utf8mb4_bin NOT NULL,\n\t`TABLE_NAME` varchar(64) CHARACTER SET utf8mb4 COLLATE utf8mb4_bin NOT NULL,\n\t`CREATE_STATEMENT` longtext,\n\t`VIEW_DEFINITION` longtext NOT NULL,\n\tPRIMARY KEY (`TABLE_SCHEMA`, `TABLE_NAME`)\n) ENGINE InnoDB", + expectedDiff: "ALTER TABLE `_vt`.`views` MODIFY COLUMN `TABLE_SCHEMA` varchar(64) CHARACTER SET utf8mb3 COLLATE utf8mb3_bin NOT NULL, MODIFY COLUMN `TABLE_NAME` varchar(64) CHARACTER SET utf8mb3 COLLATE utf8mb3_bin NOT NULL, ALGORITHM = COPY", + }, + } + + for _, tt := range testcases { + t.Run(tt.name, func(t *testing.T) { + oldSidecarTables := sidecarTables + defer func() { + sidecarTables = oldSidecarTables + }() + + env, err := vtenv.New(vtenv.Options{ + MySQLServerVersion: tt.mysqlVersion, + }) + require.NoError(t, err) + + loadSchemaDefinitions(env.Parser()) + + si := &schemaInit{ + env: env, + } + + for _, table := range sidecarTables { + if table.name != tt.tableName { + continue + } + ddl, err := si.findTableSchemaDiff(tt.tableName, tt.currentSchema, table.schema) + require.NoError(t, err) + require.Equal(t, tt.expectedDiff, ddl) + } + + }) + } +}