Skip to content

Commit

Permalink
feat: update collations to fix the alias for utf8
Browse files Browse the repository at this point in the history 
Signed-off-by: Manan Gupta <[email protected]>
  • Loading branch information
@GuptaManan100
GuptaManan100 committed Feb 5, 2024
1 parent 3e39c8c commit 4e92949
Show file tree
Hide file tree
Showing 6 changed files with 62 additions and 46 deletions.
10 changes: 4 additions & 6 deletions go/mysql/collations/env.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -180,7 +180,7 @@ func makeEnv(version collver) *Environment {
}
}

for from, to := range charsetAliases() {
for from, to := range charsetAliases(env.version) {
env.byCharset[from] = env.byCharset[to]
}

Expand Down Expand Up @@ -212,14 +212,12 @@ var SystemCollation = TypedCollation{
// this mapping will change, so it's important to use this helper so that
// Vitess code has a consistent mapping for the active collations environment.
func (env *Environment) CharsetAlias(charset string) (alias string, ok bool) {
alias, ok = charsetAliases()[charset]
alias, ok = charsetAliases(env.version)[charset]
return
}

// CollationAlias returns the internal collaction name for the given charset.
// For now, this maps all `utf8` to `utf8mb3` collation names; in future versions of MySQL,
// this mapping will change, so it's important to use this helper so that
// Vitess code has a consistent mapping for the active collations environment.
// For now, this maps `utf8` to `utf8mb4` collation names for MySQL 8.0 and maps the rest to `utf8mb3`.
func (env *Environment) CollationAlias(collation string) (string, bool) {
col := env.LookupByName(collation)
if col == Unknown {
Expand All @@ -233,7 +231,7 @@ func (env *Environment) CollationAlias(collation string) (string, bool) {
return collation, false
}
for _, alias := range allCols.alias {
for source, dest := range charsetAliases() {
for source, dest := range charsetAliases(env.version) {
if strings.HasPrefix(collation, fmt.Sprintf("%s_", source)) &&
strings.HasPrefix(alias.name, fmt.Sprintf("%s_", dest)) {
return alias.name, true
Expand Down
13 changes: 10 additions & 3 deletions go/mysql/collations/mysqlversion.go
View file

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

4 changes: 2 additions & 2 deletions go/vt/schemadiff/column.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -130,7 +130,7 @@ func (c *ColumnDefinitionEntity) ColumnDiff(
}()
c.columnDefinition.Type.Options.Collate = t1cc.collate
}
if c.columnDefinition.Type.Options.Collate = t1cc.collate; c.columnDefinition.Type.Options.Collate == "" {
if c.columnDefinition.Type.Options.Collate = t1cc.collate; t1cc.charset != "" && c.columnDefinition.Type.Options.Collate == "" {
collation := env.CollationEnv().DefaultCollationForCharset(t1cc.charset)
if collation == collations.Unknown {
return nil, vterrors.Errorf(vtrpcpb.Code_INVALID_ARGUMENT, "cannot match collation to charset %v", t1cc.charset)
Expand All @@ -157,7 +157,7 @@ func (c *ColumnDefinitionEntity) ColumnDiff(
other.columnDefinition.Type.Options.Collate = ""
}()
other.columnDefinition.Type.Charset.Name = t2cc.charset
if other.columnDefinition.Type.Options.Collate = t2cc.collate; other.columnDefinition.Type.Options.Collate == "" {
if other.columnDefinition.Type.Options.Collate = t2cc.collate; t2cc.charset != "" && other.columnDefinition.Type.Options.Collate == "" {
collation := env.CollationEnv().DefaultCollationForCharset(t2cc.charset)
if collation == collations.Unknown {
return nil, vterrors.Errorf(vtrpcpb.Code_INVALID_ARGUMENT, "cannot match collation to charset %v", t2cc.charset)
Expand Down
4 changes: 2 additions & 2 deletions go/vt/sidecardb/schema/schemaengine/tables.sql
View file
Original file line number Diff line number Diff line change
Expand Up @@ -16,8 +16,8 @@ limitations under the License.

CREATE TABLE IF NOT EXISTS tables
(
TABLE_SCHEMA varchar(64) CHARACTER SET `utf8mb3` COLLATE `utf8mb3_bin` NOT NULL,
TABLE_NAME varchar(64) CHARACTER SET `utf8mb3` COLLATE `utf8mb3_bin` NOT NULL,
TABLE_SCHEMA varchar(64) CHARACTER SET `utf8` COLLATE `utf8_bin` NOT NULL,
TABLE_NAME varchar(64) CHARACTER SET `utf8` COLLATE `utf8_bin` NOT NULL,
CREATE_STATEMENT longtext,
CREATE_TIME BIGINT,
PRIMARY KEY (TABLE_SCHEMA, TABLE_NAME)
Expand Down
4 changes: 2 additions & 2 deletions go/vt/sidecardb/schema/schemaengine/views.sql
View file
Original file line number Diff line number Diff line change
Expand Up @@ -16,8 +16,8 @@ limitations under the License.

CREATE TABLE IF NOT EXISTS views
(
TABLE_SCHEMA varchar(64) CHARACTER SET `utf8mb3` COLLATE `utf8mb3_bin` NOT NULL,
TABLE_NAME varchar(64) CHARACTER SET `utf8mb3` COLLATE `utf8mb3_bin` NOT NULL,
TABLE_SCHEMA varchar(64) CHARACTER SET `utf8` COLLATE `utf8_bin` NOT NULL,
TABLE_NAME varchar(64) CHARACTER SET `utf8` COLLATE `utf8_bin` NOT NULL,
CREATE_STATEMENT longtext,
VIEW_DEFINITION longtext NOT NULL,
PRIMARY KEY (TABLE_SCHEMA, TABLE_NAME)
Expand Down
73 changes: 42 additions & 31 deletions go/vt/sidecardb/sidecardb_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -20,8 +20,6 @@ import (
"context"
"expvar"
"fmt"
"io/fs"
"path/filepath"
"sort"
"strings"
"testing"
Expand Down Expand Up @@ -276,6 +274,48 @@ func TestCollationsForSchemaEngineTables(t *testing.T) {
currentSchema: "",
expectedDiff: "CREATE TABLE IF NOT EXISTS `_vt`.`tables` (\n\t`TABLE_SCHEMA` varchar(64) CHARACTER SET utf8mb4 COLLATE utf8mb4_bin NOT NULL,\n\t`TABLE_NAME` varchar(64) CHARACTER SET utf8mb4 COLLATE utf8mb4_bin NOT NULL,\n\t`CREATE_STATEMENT` longtext,\n\t`CREATE_TIME` bigint,\n\tPRIMARY KEY (`TABLE_SCHEMA`, `TABLE_NAME`)\n) ENGINE InnoDB",
},
{
name: "views schema in MySQL 5.7",
mysqlVersion: "5.7.31",
tableName: "views",
currentSchema: "",
expectedDiff: "CREATE TABLE IF NOT EXISTS `_vt`.`views` (\n\t`TABLE_SCHEMA` varchar(64) CHARACTER SET utf8mb3 COLLATE utf8mb3_bin NOT NULL,\n\t`TABLE_NAME` varchar(64) CHARACTER SET utf8mb3 COLLATE utf8mb3_bin NOT NULL,\n\t`CREATE_STATEMENT` longtext,\n\t`VIEW_DEFINITION` longtext NOT NULL,\n\tPRIMARY KEY (`TABLE_SCHEMA`, `TABLE_NAME`)\n) ENGINE InnoDB",
},
{
name: "views schema in MySQL 8.0",
mysqlVersion: "8.0.30",
tableName: "views",
currentSchema: "",
expectedDiff: "CREATE TABLE IF NOT EXISTS `_vt`.`views` (\n\t`TABLE_SCHEMA` varchar(64) CHARACTER SET utf8mb4 COLLATE utf8mb4_bin NOT NULL,\n\t`TABLE_NAME` varchar(64) CHARACTER SET utf8mb4 COLLATE utf8mb4_bin NOT NULL,\n\t`CREATE_STATEMENT` longtext,\n\t`VIEW_DEFINITION` longtext NOT NULL,\n\tPRIMARY KEY (`TABLE_SCHEMA`, `TABLE_NAME`)\n) ENGINE InnoDB",
},
{
name: "tables upgrade from MySQL 5.7 to MySQL 8.0",
mysqlVersion: "8.0.30",
tableName: "tables",
currentSchema: "CREATE TABLE IF NOT EXISTS `_vt`.`tables` (\n\t`TABLE_SCHEMA` varchar(64) CHARACTER SET utf8mb3 COLLATE utf8mb3_bin NOT NULL,\n\t`TABLE_NAME` varchar(64) CHARACTER SET utf8mb3 COLLATE utf8mb3_bin NOT NULL,\n\t`CREATE_STATEMENT` longtext,\n\t`CREATE_TIME` bigint,\n\tPRIMARY KEY (`TABLE_SCHEMA`, `TABLE_NAME`)\n) ENGINE InnoDB",
expectedDiff: "ALTER TABLE `_vt`.`tables` MODIFY COLUMN `TABLE_SCHEMA` varchar(64) CHARACTER SET utf8mb4 COLLATE utf8mb4_bin NOT NULL, MODIFY COLUMN `TABLE_NAME` varchar(64) CHARACTER SET utf8mb4 COLLATE utf8mb4_bin NOT NULL, ALGORITHM = COPY",
},
{
name: "views upgrade from MySQL 5.7 to MySQL 8.0",
mysqlVersion: "8.0.30",
tableName: "views",
currentSchema: "CREATE TABLE IF NOT EXISTS `_vt`.`views` (\n\t`TABLE_SCHEMA` varchar(64) CHARACTER SET utf8mb3 COLLATE utf8mb3_bin NOT NULL,\n\t`TABLE_NAME` varchar(64) CHARACTER SET utf8mb3 COLLATE utf8mb3_bin NOT NULL,\n\t`CREATE_STATEMENT` longtext,\n\t`VIEW_DEFINITION` longtext NOT NULL,\n\tPRIMARY KEY (`TABLE_SCHEMA`, `TABLE_NAME`)\n) ENGINE InnoDB",
expectedDiff: "ALTER TABLE `_vt`.`views` MODIFY COLUMN `TABLE_SCHEMA` varchar(64) CHARACTER SET utf8mb4 COLLATE utf8mb4_bin NOT NULL, MODIFY COLUMN `TABLE_NAME` varchar(64) CHARACTER SET utf8mb4 COLLATE utf8mb4_bin NOT NULL, ALGORITHM = COPY",
},
{
name: "tables downgrade from MySQL 8.0 to MySQL 5.7",
mysqlVersion: "5.7.31",
tableName: "tables",
currentSchema: "CREATE TABLE IF NOT EXISTS `_vt`.`tables` (\n\t`TABLE_SCHEMA` varchar(64) CHARACTER SET utf8mb4 COLLATE utf8mb4_bin NOT NULL,\n\t`TABLE_NAME` varchar(64) CHARACTER SET utf8mb4 COLLATE utf8mb4_bin NOT NULL,\n\t`CREATE_STATEMENT` longtext,\n\t`CREATE_TIME` bigint,\n\tPRIMARY KEY (`TABLE_SCHEMA`, `TABLE_NAME`)\n) ENGINE InnoDB",
expectedDiff: "ALTER TABLE `_vt`.`tables` MODIFY COLUMN `TABLE_SCHEMA` varchar(64) CHARACTER SET utf8mb3 COLLATE utf8mb3_bin NOT NULL, MODIFY COLUMN `TABLE_NAME` varchar(64) CHARACTER SET utf8mb3 COLLATE utf8mb3_bin NOT NULL, ALGORITHM = COPY",
},
{
name: "views downgrade from MySQL 8.0 to MySQL 5.7",
mysqlVersion: "5.7.31",
tableName: "views",
currentSchema: "CREATE TABLE IF NOT EXISTS `_vt`.`views` (\n\t`TABLE_SCHEMA` varchar(64) CHARACTER SET utf8mb4 COLLATE utf8mb4_bin NOT NULL,\n\t`TABLE_NAME` varchar(64) CHARACTER SET utf8mb4 COLLATE utf8mb4_bin NOT NULL,\n\t`CREATE_STATEMENT` longtext,\n\t`VIEW_DEFINITION` longtext NOT NULL,\n\tPRIMARY KEY (`TABLE_SCHEMA`, `TABLE_NAME`)\n) ENGINE InnoDB",
expectedDiff: "ALTER TABLE `_vt`.`views` MODIFY COLUMN `TABLE_SCHEMA` varchar(64) CHARACTER SET utf8mb3 COLLATE utf8mb3_bin NOT NULL, MODIFY COLUMN `TABLE_NAME` varchar(64) CHARACTER SET utf8mb3 COLLATE utf8mb3_bin NOT NULL, ALGORITHM = COPY",
},
}

for _, tt := range testcases {
Expand Down Expand Up @@ -308,32 +348,3 @@ func TestCollationsForSchemaEngineTables(t *testing.T) {
})
}
}

func findTablesAndViewsSchema() (string, string, error) {
var tablesSchema, viewsSchema string
err := fs.WalkDir(schemaLocation, ".", func(path string, entry fs.DirEntry, err error) error {
if err != nil {
return err
}
if !entry.IsDir() {
_, fname := filepath.Split(path)

if fname == "tables.sql" {
schema, err := schemaLocation.ReadFile(path)
if err != nil {
panic(err)
}
tablesSchema = string(schema)
}
if fname == "views.sql" {
schema, err := schemaLocation.ReadFile(path)
if err != nil {
panic(err)
}
viewsSchema = string(schema)
}
}
return nil
})
return tablesSchema, viewsSchema, err
}

0 comments on commit 4e92949

Please sign in to comment.