Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Fix character set mapping for utf8 and use it in schema-tracking tables #15130

Closed
wants to merge 2 commits into from
Closed

Fix character set mapping for utf8 and use it in schema-tracking tables #15130

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
3e39c8c
test: add failing test for expected collation and character set
GuptaManan100 Feb 5, 2024
4e92949
feat: update collations to fix the alias for utf8
GuptaManan100 Feb 5, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
10 changes: 4 additions & 6 deletions go/mysql/collations/env.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -180,7 +180,7 @@ func makeEnv(version collver) *Environment {
}
}

for from, to := range charsetAliases() {
for from, to := range charsetAliases(env.version) {
env.byCharset[from] = env.byCharset[to]
}

Expand Down Expand Up @@ -212,14 +212,12 @@ var SystemCollation = TypedCollation{
// this mapping will change, so it's important to use this helper so that
// Vitess code has a consistent mapping for the active collations environment.
func (env *Environment) CharsetAlias(charset string) (alias string, ok bool) {
alias, ok = charsetAliases()[charset]
alias, ok = charsetAliases(env.version)[charset]
return
}

// CollationAlias returns the internal collaction name for the given charset.
// For now, this maps all `utf8` to `utf8mb3` collation names; in future versions of MySQL,
// this mapping will change, so it's important to use this helper so that
// Vitess code has a consistent mapping for the active collations environment.
// For now, this maps `utf8` to `utf8mb4` collation names for MySQL 8.0 and maps the rest to `utf8mb3`.
func (env *Environment) CollationAlias(collation string) (string, bool) {
col := env.LookupByName(collation)
if col == Unknown {
Expand All @@ -233,7 +231,7 @@ func (env *Environment) CollationAlias(collation string) (string, bool) {
return collation, false
}
for _, alias := range allCols.alias {
for source, dest := range charsetAliases() {
for source, dest := range charsetAliases(env.version) {
if strings.HasPrefix(collation, fmt.Sprintf("%s_", source)) &&
strings.HasPrefix(alias.name, fmt.Sprintf("%s_", dest)) {
return alias.name, true
Expand Down
13 changes: 10 additions & 3 deletions go/mysql/collations/mysqlversion.go
View file
Open in desktop

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

4 changes: 2 additions & 2 deletions go/vt/schemadiff/column.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -130,7 +130,7 @@ func (c *ColumnDefinitionEntity) ColumnDiff(
}()
c.columnDefinition.Type.Options.Collate = t1cc.collate
}
if c.columnDefinition.Type.Options.Collate = t1cc.collate; c.columnDefinition.Type.Options.Collate == "" {
if c.columnDefinition.Type.Options.Collate = t1cc.collate; t1cc.charset != "" && c.columnDefinition.Type.Options.Collate == "" {
collation := env.CollationEnv().DefaultCollationForCharset(t1cc.charset)
if collation == collations.Unknown {
return nil, vterrors.Errorf(vtrpcpb.Code_INVALID_ARGUMENT, "cannot match collation to charset %v", t1cc.charset)
Expand All @@ -157,7 +157,7 @@ func (c *ColumnDefinitionEntity) ColumnDiff(
other.columnDefinition.Type.Options.Collate = ""
}()
other.columnDefinition.Type.Charset.Name = t2cc.charset
if other.columnDefinition.Type.Options.Collate = t2cc.collate; other.columnDefinition.Type.Options.Collate == "" {
if other.columnDefinition.Type.Options.Collate = t2cc.collate; t2cc.charset != "" && other.columnDefinition.Type.Options.Collate == "" {
collation := env.CollationEnv().DefaultCollationForCharset(t2cc.charset)
if collation == collations.Unknown {
return nil, vterrors.Errorf(vtrpcpb.Code_INVALID_ARGUMENT, "cannot match collation to charset %v", t2cc.charset)
Expand Down
4 changes: 2 additions & 2 deletions go/vt/sidecardb/schema/schemaengine/tables.sql
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -16,8 +16,8 @@ limitations under the License.

CREATE TABLE IF NOT EXISTS tables
(
TABLE_SCHEMA varchar(64) CHARACTER SET `utf8mb3` COLLATE `utf8mb3_bin` NOT NULL,
TABLE_NAME varchar(64) CHARACTER SET `utf8mb3` COLLATE `utf8mb3_bin` NOT NULL,
TABLE_SCHEMA varchar(64) CHARACTER SET `utf8` COLLATE `utf8_bin` NOT NULL,
TABLE_NAME varchar(64) CHARACTER SET `utf8` COLLATE `utf8_bin` NOT NULL,
CREATE_STATEMENT longtext,
CREATE_TIME BIGINT,
PRIMARY KEY (TABLE_SCHEMA, TABLE_NAME)
Expand Down
4 changes: 2 additions & 2 deletions go/vt/sidecardb/schema/schemaengine/views.sql
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -16,8 +16,8 @@ limitations under the License.

CREATE TABLE IF NOT EXISTS views
(
TABLE_SCHEMA varchar(64) CHARACTER SET `utf8mb3` COLLATE `utf8mb3_bin` NOT NULL,
TABLE_NAME varchar(64) CHARACTER SET `utf8mb3` COLLATE `utf8mb3_bin` NOT NULL,
TABLE_SCHEMA varchar(64) CHARACTER SET `utf8` COLLATE `utf8_bin` NOT NULL,
TABLE_NAME varchar(64) CHARACTER SET `utf8` COLLATE `utf8_bin` NOT NULL,
CREATE_STATEMENT longtext,
VIEW_DEFINITION longtext NOT NULL,
PRIMARY KEY (TABLE_SCHEMA, TABLE_NAME)
Expand Down
103 changes: 101 additions & 2 deletions go/vt/sidecardb/sidecardb_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -24,13 +24,13 @@ import (
"strings"
"testing"

"github.com/stretchr/testify/require"

"vitess.io/vitess/go/constants/sidecar"
"vitess.io/vitess/go/vt/dbconfigs"
"vitess.io/vitess/go/vt/sqlparser"
"vitess.io/vitess/go/vt/vtenv"

"github.com/stretchr/testify/require"

"vitess.io/vitess/go/stats"

"vitess.io/vitess/go/mysql/fakesqldb"
Expand Down Expand Up @@ -249,3 +249,102 @@ func TestAlterTableAlgorithm(t *testing.T) {
})
}
}

// TestCollationsForSchemaEngineTables tests that the correct collation and character set is used for
// schema engine tables based on the MySQL version being used.
func TestCollationsForSchemaEngineTables(t *testing.T) {
testcases := []struct {
name string
mysqlVersion string
tableName string
currentSchema string
expectedDiff string
}{
{
name: "tables schema in MySQL 5.7",
mysqlVersion: "5.7.31",
tableName: "tables",
currentSchema: "",
expectedDiff: "CREATE TABLE IF NOT EXISTS `_vt`.`tables` (\n\t`TABLE_SCHEMA` varchar(64) CHARACTER SET utf8mb3 COLLATE utf8mb3_bin NOT NULL,\n\t`TABLE_NAME` varchar(64) CHARACTER SET utf8mb3 COLLATE utf8mb3_bin NOT NULL,\n\t`CREATE_STATEMENT` longtext,\n\t`CREATE_TIME` bigint,\n\tPRIMARY KEY (`TABLE_SCHEMA`, `TABLE_NAME`)\n) ENGINE InnoDB",
},
{
name: "tables schema in MySQL 8.0",
mysqlVersion: "8.0.30",
tableName: "tables",
currentSchema: "",
expectedDiff: "CREATE TABLE IF NOT EXISTS `_vt`.`tables` (\n\t`TABLE_SCHEMA` varchar(64) CHARACTER SET utf8mb4 COLLATE utf8mb4_bin NOT NULL,\n\t`TABLE_NAME` varchar(64) CHARACTER SET utf8mb4 COLLATE utf8mb4_bin NOT NULL,\n\t`CREATE_STATEMENT` longtext,\n\t`CREATE_TIME` bigint,\n\tPRIMARY KEY (`TABLE_SCHEMA`, `TABLE_NAME`)\n) ENGINE InnoDB",
},
{
name: "views schema in MySQL 5.7",
mysqlVersion: "5.7.31",
tableName: "views",
currentSchema: "",
expectedDiff: "CREATE TABLE IF NOT EXISTS `_vt`.`views` (\n\t`TABLE_SCHEMA` varchar(64) CHARACTER SET utf8mb3 COLLATE utf8mb3_bin NOT NULL,\n\t`TABLE_NAME` varchar(64) CHARACTER SET utf8mb3 COLLATE utf8mb3_bin NOT NULL,\n\t`CREATE_STATEMENT` longtext,\n\t`VIEW_DEFINITION` longtext NOT NULL,\n\tPRIMARY KEY (`TABLE_SCHEMA`, `TABLE_NAME`)\n) ENGINE InnoDB",
},
{
name: "views schema in MySQL 8.0",
mysqlVersion: "8.0.30",
tableName: "views",
currentSchema: "",
expectedDiff: "CREATE TABLE IF NOT EXISTS `_vt`.`views` (\n\t`TABLE_SCHEMA` varchar(64) CHARACTER SET utf8mb4 COLLATE utf8mb4_bin NOT NULL,\n\t`TABLE_NAME` varchar(64) CHARACTER SET utf8mb4 COLLATE utf8mb4_bin NOT NULL,\n\t`CREATE_STATEMENT` longtext,\n\t`VIEW_DEFINITION` longtext NOT NULL,\n\tPRIMARY KEY (`TABLE_SCHEMA`, `TABLE_NAME`)\n) ENGINE InnoDB",
},
{
name: "tables upgrade from MySQL 5.7 to MySQL 8.0",
mysqlVersion: "8.0.30",
tableName: "tables",
currentSchema: "CREATE TABLE IF NOT EXISTS `_vt`.`tables` (\n\t`TABLE_SCHEMA` varchar(64) CHARACTER SET utf8mb3 COLLATE utf8mb3_bin NOT NULL,\n\t`TABLE_NAME` varchar(64) CHARACTER SET utf8mb3 COLLATE utf8mb3_bin NOT NULL,\n\t`CREATE_STATEMENT` longtext,\n\t`CREATE_TIME` bigint,\n\tPRIMARY KEY (`TABLE_SCHEMA`, `TABLE_NAME`)\n) ENGINE InnoDB",
expectedDiff: "ALTER TABLE `_vt`.`tables` MODIFY COLUMN `TABLE_SCHEMA` varchar(64) CHARACTER SET utf8mb4 COLLATE utf8mb4_bin NOT NULL, MODIFY COLUMN `TABLE_NAME` varchar(64) CHARACTER SET utf8mb4 COLLATE utf8mb4_bin NOT NULL, ALGORITHM = COPY",
},
{
name: "views upgrade from MySQL 5.7 to MySQL 8.0",
mysqlVersion: "8.0.30",
tableName: "views",
currentSchema: "CREATE TABLE IF NOT EXISTS `_vt`.`views` (\n\t`TABLE_SCHEMA` varchar(64) CHARACTER SET utf8mb3 COLLATE utf8mb3_bin NOT NULL,\n\t`TABLE_NAME` varchar(64) CHARACTER SET utf8mb3 COLLATE utf8mb3_bin NOT NULL,\n\t`CREATE_STATEMENT` longtext,\n\t`VIEW_DEFINITION` longtext NOT NULL,\n\tPRIMARY KEY (`TABLE_SCHEMA`, `TABLE_NAME`)\n) ENGINE InnoDB",
expectedDiff: "ALTER TABLE `_vt`.`views` MODIFY COLUMN `TABLE_SCHEMA` varchar(64) CHARACTER SET utf8mb4 COLLATE utf8mb4_bin NOT NULL, MODIFY COLUMN `TABLE_NAME` varchar(64) CHARACTER SET utf8mb4 COLLATE utf8mb4_bin NOT NULL, ALGORITHM = COPY",
},
{
name: "tables downgrade from MySQL 8.0 to MySQL 5.7",
mysqlVersion: "5.7.31",
tableName: "tables",
currentSchema: "CREATE TABLE IF NOT EXISTS `_vt`.`tables` (\n\t`TABLE_SCHEMA` varchar(64) CHARACTER SET utf8mb4 COLLATE utf8mb4_bin NOT NULL,\n\t`TABLE_NAME` varchar(64) CHARACTER SET utf8mb4 COLLATE utf8mb4_bin NOT NULL,\n\t`CREATE_STATEMENT` longtext,\n\t`CREATE_TIME` bigint,\n\tPRIMARY KEY (`TABLE_SCHEMA`, `TABLE_NAME`)\n) ENGINE InnoDB",
expectedDiff: "ALTER TABLE `_vt`.`tables` MODIFY COLUMN `TABLE_SCHEMA` varchar(64) CHARACTER SET utf8mb3 COLLATE utf8mb3_bin NOT NULL, MODIFY COLUMN `TABLE_NAME` varchar(64) CHARACTER SET utf8mb3 COLLATE utf8mb3_bin NOT NULL, ALGORITHM = COPY",
},
{
name: "views downgrade from MySQL 8.0 to MySQL 5.7",
mysqlVersion: "5.7.31",
tableName: "views",
currentSchema: "CREATE TABLE IF NOT EXISTS `_vt`.`views` (\n\t`TABLE_SCHEMA` varchar(64) CHARACTER SET utf8mb4 COLLATE utf8mb4_bin NOT NULL,\n\t`TABLE_NAME` varchar(64) CHARACTER SET utf8mb4 COLLATE utf8mb4_bin NOT NULL,\n\t`CREATE_STATEMENT` longtext,\n\t`VIEW_DEFINITION` longtext NOT NULL,\n\tPRIMARY KEY (`TABLE_SCHEMA`, `TABLE_NAME`)\n) ENGINE InnoDB",
expectedDiff: "ALTER TABLE `_vt`.`views` MODIFY COLUMN `TABLE_SCHEMA` varchar(64) CHARACTER SET utf8mb3 COLLATE utf8mb3_bin NOT NULL, MODIFY COLUMN `TABLE_NAME` varchar(64) CHARACTER SET utf8mb3 COLLATE utf8mb3_bin NOT NULL, ALGORITHM = COPY",
},
}

for _, tt := range testcases {
t.Run(tt.name, func(t *testing.T) {
oldSidecarTables := sidecarTables
defer func() {
sidecarTables = oldSidecarTables
}()

env, err := vtenv.New(vtenv.Options{
MySQLServerVersion: tt.mysqlVersion,
})
require.NoError(t, err)

loadSchemaDefinitions(env.Parser())

si := &schemaInit{
env: env,
}

for _, table := range sidecarTables {
if table.name != tt.tableName {
continue
}
ddl, err := si.findTableSchemaDiff(tt.tableName, tt.currentSchema, table.schema)
require.NoError(t, err)
require.Equal(t, tt.expectedDiff, ddl)
}

})
}
}
Loading