engine.Reload(): read InnoDB tables sizes including FULLTEXT index volume

go/mysql/collations/charset/filename_test.go

@@ -0,0 +1,62 @@
+/*
+Copyright 2024 The Vitess Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package charset
+
+import (
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+)
+
+func TestTablenameToFilename(t *testing.T) {
+	testCases := []struct {
+		tablename string
+		filename  string
+	}{
+		{
+			tablename: "my_table123",
+			filename:  "my_table123",
+		},
+		{
+			tablename: "my-table",
+			filename:  "my@002dtable",
+		},
+		{
+			tablename: "my$table",
+			filename:  "my@0024table",
+		},
+		{
+			tablename: "myát",
+			filename:  "my@0ht",
+		},
+		{
+			tablename: "myÃt",
+			filename:  "my@0jt",
+		},
+		{
+			tablename: "myאt",
+			filename:  "my@05d0t",
+		},
+	}
+
+	for _, tc := range testCases {
+		t.Run(tc.tablename, func(t *testing.T) {
+			filename := TablenameToFilename(tc.tablename)
+			assert.Equal(t, tc.filename, filename, "original bytes: %x", []byte(tc.tablename))
+		})
+	}
+}

go/mysql/config/config.go

@@ -2,3 +2,4 @@ package config
 
 const DefaultSQLMode = "ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_ENGINE_SUBSTITUTION"
 const DefaultMySQLVersion = "8.0.30"
+const LegacyMySQLVersion = "5.7.31"

go/mysql/fakesqldb/server.go

@@ -166,6 +166,11 @@ type ExpectedExecuteFetch struct {
 
 // New creates a server, and starts listening.
 func New(t testing.TB) *DB {
+	return NewWithEnv(t, vtenv.NewTestEnv())
+}
+
+// NewWithEnv creates a server, and starts listening.
+func NewWithEnv(t testing.TB, env *vtenv.Environment) *DB {
 	// Pick a path for our socket.
 	socketDir, err := os.MkdirTemp("", "fakesqldb")
 	if err != nil {
@@ -185,7 +190,7 @@ func New(t testing.TB) *DB {
 		queryPatternUserCallback: make(map[*regexp.Regexp]func(string)),
 		patternData:              make(map[string]exprResult),
 		lastErrorMu:              sync.Mutex{},
-		env:                      vtenv.NewTestEnv(),
+		env:                      env,
 	}
 
 	db.Handler = db

go/mysql/flavor.go

@@ -151,6 +151,7 @@ type flavor interface {
 
 	baseShowTables() string
 	baseShowTablesWithSizes() string
+	baseShowInnodbTableSizes() string
 
 	supportsCapability(capability capabilities.FlavorCapability) (bool, error)
 }
@@ -454,6 +455,11 @@ func (c *Conn) BaseShowTablesWithSizes() string {
 	return c.flavor.baseShowTablesWithSizes()
 }
 
+// BaseShowInnodbTableSizes returns a query that shows innodb-internal FULLTEXT index tables and their sizes
+func (c *Conn) BaseShowInnodbTableSizes() string {
+	return c.flavor.baseShowInnodbTableSizes()
+}
+
 // SupportsCapability checks if the database server supports the given capability
 func (c *Conn) SupportsCapability(capability capabilities.FlavorCapability) (bool, error) {
 	return c.flavor.supportsCapability(capability)

go/mysql/flavor_filepos.go

@@ -366,6 +366,10 @@ func (*filePosFlavor) baseShowTablesWithSizes() string {
 	return TablesWithSize56
 }
 
+func (filePosFlavor) baseShowInnodbTableSizes() string {
+	return ""
+}
+
 // supportsCapability is part of the Flavor interface.
 func (f *filePosFlavor) supportsCapability(capability capabilities.FlavorCapability) (bool, error) {
 	switch capability {

go/mysql/flavor_mariadb_binlog_playback.go

@@ -22,6 +22,10 @@ func (mariadbFlavor) baseShowTables() string {
 	return mysqlFlavor{}.baseShowTables()
 }
 
+func (mariadbFlavor) baseShowInnodbTableSizes() string {
+	return ""
+}
+
 // baseShowTablesWithSizes is part of the Flavor interface.
 func (mariadbFlavor101) baseShowTablesWithSizes() string {
 	return TablesWithSize56

go/mysql/flavor_mysql.go

@@ -412,63 +412,92 @@ const BaseShowTables = `SELECT t.table_name,
 		t.table_schema = database()
 `
 
-// TablesWithSize80 is a query to select table along with size for mysql 8.0
-// Note the following:
-//   - `TABLES`.`TABLE_NAME` has `utf8mb4_0900_ai_ci` collation.  `INNODB_TABLESPACES`.`NAME` has `utf8mb3_general_ci`.
-//     We normalize the collation to get better query performance (we force the casting at the time of our choosing)
-//   - InnoDB has different table names than MySQL does, in particular for partitioned tables. As far as InnoDB
-//     is concerned, each partition is its own table.
-//   - We use a `UNION ALL` approach to handle two distinct scenarios: tables that are partitioned and those that are not.
-//     Since we `LEFT JOIN` from `TABLES` to `INNODB_TABLESPACES`, we know we already do full table scan on `TABLES`. We therefore
-//     don't mind spending some extra computation time (as in `CONCAT(t.table_schema, '/', t.table_name, '#p#%') COLLATE utf8mb3_general_ci`)
-//     to make things easier for the JOIN.
-//   - We utilize `INFORMATION_SCHEMA`.`TABLES`.`CREATE_OPTIONS` column to tell if the table is partitioned or not. The column
-//     may be `NULL` or may have multiple attributes, one of which is "partitioned", which we are looking for.
-//   - In a partitioned table, InnoDB will return multiple rows for the same table name, one for each partition, which we successively SUM.
-//     We also `SUM` the sizes in the non-partitioned case. This is not because we need to, but because it makes the query
-//     symmetric and less prone to future edit errors.
-const TablesWithSize80 = `SELECT t.table_name,
-		t.table_type,
-		UNIX_TIMESTAMP(t.create_time),
-		t.table_comment,
-		SUM(i.file_size),
-		SUM(i.allocated_size)
-	FROM information_schema.tables t
-	LEFT JOIN (SELECT name, file_size, allocated_size FROM information_schema.innodb_tablespaces WHERE name LIKE CONCAT(database(), '/%')) i
-	ON i.name = CONCAT(t.table_schema, '/', t.table_name) COLLATE utf8mb3_general_ci
-	WHERE
-		t.table_schema = database() AND IFNULL(t.create_options, '') NOT LIKE '%partitioned%'
-	GROUP BY
-		t.table_schema, t.table_name, t.table_type, t.create_time, t.table_comment
-UNION ALL
-SELECT t.table_name,
-		t.table_type,
-		UNIX_TIMESTAMP(t.create_time),
-		t.table_comment,
-		SUM(i.file_size),
-		SUM(i.allocated_size)
-	FROM information_schema.tables t
-	LEFT JOIN (SELECT name, file_size, allocated_size FROM information_schema.innodb_tablespaces WHERE name LIKE CONCAT(database(), '/%')) i
-		ON i.name LIKE (CONCAT(t.table_schema, '/', t.table_name, '#p#%') COLLATE utf8mb3_general_ci)
-	WHERE
-		t.table_schema = database() AND t.create_options LIKE '%partitioned%'
-	GROUP BY
-		t.table_schema, t.table_name, t.table_type, t.create_time, t.table_comment
+// InnoDBTableSizes: a query to return file/allocated sizes for InnoDB tables.
+// File sizes and allocated sizes are found in information_schema.innodb_tablespaces
+// Table names in information_schema.innodb_tablespaces match those in information_schema.tables, even for table names
+// with special characters. This, a innodb_tablespaces.name could be `my-db/my-table`.
+// These tablespaces will have one entry for every InnoDB table, hidden or internal. This means:
+// - One entry for every partition in a partitioned table.
+// - Several entries for any FULLTEXT index (FULLTEXT indexes are not BTREEs and are implemented using multiple hidden tables)
+// So a single table wih a FULLTEXT index will have one entry for the "normal" table, plus multiple more entries for
+// every FTS index hidden tables.
+// Thankfully FULLTEXT does not work with Partitioning so this does not explode too much.
+// Next thing is that FULLTEXT hidden table names do not resemble the original table name, and could look like:
+// `a-b/fts_000000000000075e_00000000000005f9_index_2`.
+// To unlock the identify of this table we turn to information_schema.innodb_tables. These table similarly has one entry for
+// every InnoDB table, normal or hidden. It also has a `TABLE_ID` value. Given some table with FULLTEXT keys, its TABLE_ID
+// is encoded in the names of the hidden tables in information_schema.innodb_tablespaces: `000000000000075e` in the
+// example above.
+//
+// The query below is a two part:
+//  1. Finding the "normal" tables only, those that the user created. We note their file size and allocated size.
+//  2. Finding the hidden tables only, those that implement FTS keys. We aggregate their file size and allocated size grouping
+//     by the original table name with which they're associated.
+//
+// A table that has a FULLTEXT index will have two entries in the result set:
+// - one for the "normal" table size (actual rows, texts, etc.)
+// - and one for the aggregated hidden table size
+// The code that reads the results of this query will need to add the two.
+// Similarly, the code will need to know how to aggregate the sizes of partitioned tables, which could appear as:
+// - `mydb/tbl_part#p#p0`
+// - `mydb/tbl_part#p#p1`
+// - `mydb/tbl_part#p#p2`
+// - `mydb/tbl_part#p#p3`
+//
+// Lastly, we note that table name in information_schema.innodb_tables are encoded. A table that shows as
+// `my-db/my-table` in information_schema.innodb_tablespaces will show as `my@002ddb/my@002dtable` in information_schema.innodb_tables.
+// So this query returns InnoDB-encoded table names. The golang code reading those will have to decode the names.
+const InnoDBTableSizes = `
+	SELECT
+		it.name,
+		its.file_size as normal_tables_sum_file_size,
+		its.allocated_size as normal_tables_sum_allocated_size
+	FROM
+		information_schema.innodb_tables it
+		JOIN information_schema.innodb_tablespaces its
+		ON (its.space = it.space)
+		WHERE
+					its.name LIKE CONCAT(database(), '/%')
+			AND	its.name NOT LIKE CONCAT(database(), '/fts_%')
+	UNION ALL
+	SELECT
+		it.name,
+		SUM(its.file_size) as hidden_tables_sum_file_size,
+		SUM(its.allocated_size) as hidden_tables_sum_allocated_size
+	FROM
+		information_schema.innodb_tables it
+		JOIN information_schema.innodb_tablespaces its
+		ON (
+				 its.name LIKE CONCAT(database(), '/fts_', CONVERT(LPAD(HEX(table_id), 16, '0') USING utf8mb3) COLLATE utf8mb3_general_ci, '_%')
+		)
+		WHERE
+				 its.name LIKE CONCAT(database(), '/fts_%')
+		GROUP BY it.name
 `
 
 // baseShowTablesWithSizes is part of the Flavor interface.
 func (mysqlFlavor57) baseShowTablesWithSizes() string {
 	return TablesWithSize57
 }
 
+// baseShowInnodbTableSizes is part of the Flavor interface.
+func (mysqlFlavor57) baseShowInnodbTableSizes() string {
+	return ""
+}
+
 // supportsCapability is part of the Flavor interface.
 func (f mysqlFlavor) supportsCapability(capability capabilities.FlavorCapability) (bool, error) {
 	return capabilities.MySQLVersionHasCapability(f.serverVersion, capability)
 }
 
 // baseShowTablesWithSizes is part of the Flavor interface.
 func (mysqlFlavor) baseShowTablesWithSizes() string {
-	return TablesWithSize80
+	return "" // Won't be used, as InnoDBTableSizes is defined, and schema.Engine will use that, instead.
+}
+
+// baseShowInnodbTableSizes is part of the Flavor interface.
+func (mysqlFlavor) baseShowInnodbTableSizes() string {
+	return InnoDBTableSizes
 }
 
 func (mysqlFlavor) setReplicationSourceCommand(params *ConnParams, host string, port int32, heartbeatInterval float64, connectRetry int) string {

go/mysql/flavor_mysql_legacy.go

@@ -72,12 +72,13 @@ GROUP BY table_name,
 // We join with a subquery that materializes the data from `information_schema.innodb_sys_tablespaces`
 // early for performance reasons. This effectively causes only a single read of `information_schema.innodb_sys_tablespaces`
 // per query.
+// Note that 5.7 has NULL for a VIEW's create_time, so we use IFNULL to make it 1 (non NULL and non zero).
 const TablesWithSize57 = `SELECT t.table_name,
 	t.table_type,
-	UNIX_TIMESTAMP(t.create_time),
+	IFNULL(UNIX_TIMESTAMP(t.create_time), 1),
 	t.table_comment,
-	IFNULL(SUM(i.file_size), SUM(t.data_length + t.index_length)),
-	IFNULL(SUM(i.allocated_size), SUM(t.data_length + t.index_length))
+	IFNULL(SUM(i.file_size), SUM(t.data_length + t.index_length)) AS file_size,
+	IFNULL(SUM(i.allocated_size), SUM(t.data_length + t.index_length)) AS allocated_size
 FROM information_schema.tables t
 LEFT OUTER JOIN (
 	SELECT space, file_size, allocated_size, name

go/mysql/flavor_mysqlgr.go

@@ -254,7 +254,12 @@ func (mysqlGRFlavor) baseShowTables() string {
 }
 
 func (mysqlGRFlavor) baseShowTablesWithSizes() string {
-	return TablesWithSize80
+	return "" // Won't be used, as InnoDBTableSizes is defined, and schema.Engine will use that, instead.
+}
+
+// baseShowInnodbTableSizes is part of the Flavor interface.
+func (mysqlGRFlavor) baseShowInnodbTableSizes() string {
+	return InnoDBTableSizes
 }
 
 // supportsCapability is part of the Flavor interface.

go/mysql/schema.go

@@ -93,6 +93,29 @@ var BaseShowTablesWithSizesFields = append(BaseShowTablesFields, &querypb.Field{
 	Charset:      collations.CollationBinaryID,
 	Flags:        uint32(querypb.MySqlFlag_BINARY_FLAG | querypb.MySqlFlag_NUM_FLAG),
 })
+var BaseInnoDBTableSizesFields = []*querypb.Field{{
+	Name:         "it.name",
+	Type:         querypb.Type_VARCHAR,
+	Table:        "tables",
+	OrgTable:     "TABLES",
+	Database:     "information_schema",
+	OrgName:      "TABLE_NAME",
+	ColumnLength: 192,
+	Charset:      uint32(collations.SystemCollation.Collation),
+	Flags:        uint32(querypb.MySqlFlag_NOT_NULL_FLAG),
+}, {
+	Name:         "i.file_size",
+	Type:         querypb.Type_INT64,
+	ColumnLength: 11,
+	Charset:      collations.CollationBinaryID,
+	Flags:        uint32(querypb.MySqlFlag_BINARY_FLAG | querypb.MySqlFlag_NUM_FLAG),
+}, {
+	Name:         "i.allocated_size",
+	Type:         querypb.Type_INT64,
+	ColumnLength: 11,
+	Charset:      collations.CollationBinaryID,
+	Flags:        uint32(querypb.MySqlFlag_BINARY_FLAG | querypb.MySqlFlag_NUM_FLAG),
+}}
 
 // BaseShowTablesRow returns the fields from a BaseShowTables or
 // BaseShowTablesForTable command.
@@ -116,6 +139,14 @@ func BaseShowTablesWithSizesRow(tableName string, isView bool, comment string) [
 	)
 }
 
+func BaseInnoDBTableSizesRow(dbName string, tableName string) []sqltypes.Value {
+	return []sqltypes.Value{
+		sqltypes.MakeTrusted(sqltypes.VarChar, []byte(dbName+"/"+tableName)),
+		sqltypes.MakeTrusted(sqltypes.Int64, []byte("100")), // file_size
+		sqltypes.MakeTrusted(sqltypes.Int64, []byte("150")), // allocated_size
+	}
+}
+
 // ShowPrimaryFields contains the fields for a BaseShowPrimary.
 var ShowPrimaryFields = []*querypb.Field{{
 	Name: "table_name",

go/vt/vtenv/vtenv.go

@@ -67,6 +67,12 @@ func NewTestEnv() *Environment {
 	}
 }
 
+func NewLegacyTestEnv() *Environment {
+	env := NewTestEnv()
+	env.mysqlVersion = config.LegacyMySQLVersion
+	return env
+}
+
 func (e *Environment) CollationEnv() *collations.Environment {
 	return e.collationEnv
 }

go/vt/vtexplain/vtexplain_vttablet.go

@@ -430,6 +430,7 @@ func newTabletEnvironment(ddls []sqlparser.DDLStatement, opts *Options, collatio
 
 	showTableRows := make([][]sqltypes.Value, 0, len(ddls))
 	showTableWithSizesRows := make([][]sqltypes.Value, 0, len(ddls))
+	innodbTableSizesRows := make([][]sqltypes.Value, 0, len(ddls))
 
 	for _, ddl := range ddls {
 		table := ddl.GetTable().Name.String()
@@ -455,9 +456,9 @@ func newTabletEnvironment(ddls []sqlparser.DDLStatement, opts *Options, collatio
 		Fields: mysql.BaseShowTablesWithSizesFields,
 		Rows:   showTableWithSizesRows,
 	})
-	tEnv.addResult(mysql.TablesWithSize80, &sqltypes.Result{
-		Fields: mysql.BaseShowTablesWithSizesFields,
-		Rows:   showTableWithSizesRows,
+	tEnv.addResult(mysql.InnoDBTableSizes, &sqltypes.Result{
+		Fields: mysql.BaseInnoDBTableSizesFields,
+		Rows:   innodbTableSizesRows,
 	})
 
 	indexRows := make([][]sqltypes.Value, 0, 4)