Merge remote-tracking branch 'origin/main' into vstream_enum_strings

Signed-off-by: Matt Lord <[email protected]>

config/init_db.sql

@@ -1,30 +1,22 @@
 # This file is executed immediately after initializing a fresh data directory.
 
-###############################################################################
-# WARNING: This sql is *NOT* safe for production use,
-#          as it contains default well-known users and passwords.
-#          Care should be taken to change these users and passwords
-#          for production.
-###############################################################################
-
 ###############################################################################
 # Equivalent of mysql_secure_installation
 ###############################################################################
 # We need to ensure that super_read_only is disabled so that we can execute
 # these commands. Note that disabling it does NOT disable read_only.
 # We save the current value so that we only re-enable it at the end if it was
 # enabled before.
+
 SET @original_super_read_only=IF(@@global.super_read_only=1, 'ON', 'OFF');
 SET GLOBAL super_read_only='OFF';
 
 # Changes during the init db should not make it to the binlog.
 # They could potentially create errant transactions on replicas.
 SET sql_log_bin = 0;
-# Remove anonymous users.
-DELETE FROM mysql.user WHERE User = '';
 
-# Disable remote root access (only allow UNIX socket).
-DELETE FROM mysql.user WHERE User = 'root' AND Host != 'localhost';
+# Remove anonymous users & disable remote root access (only allow UNIX socket).
+DROP USER IF EXISTS ''@'%', ''@'localhost', 'root'@'%';
 
 # Remove test database.
 DROP DATABASE IF EXISTS test;
@@ -78,8 +70,6 @@ GRANT SELECT, PROCESS, SUPER, REPLICATION CLIENT, RELOAD
 GRANT SELECT, UPDATE, DELETE, DROP
   ON performance_schema.* TO 'vt_monitoring'@'localhost';
 
-FLUSH PRIVILEGES;
-
 RESET SLAVE ALL;
 RESET MASTER;
 

examples/compose/config/init_db.sql

@@ -12,10 +12,8 @@ SET GLOBAL super_read_only='OFF';
 # Changes during the init db should not make it to the binlog.
 # They could potentially create errant transactions on replicas.
 SET sql_log_bin = 0;
-# Remove anonymous users.
-DELETE FROM mysql.user WHERE User = '';
-# Disable remote root access (only allow UNIX socket).
-DELETE FROM mysql.user WHERE User = 'root' AND Host != 'localhost';
+# Remove anonymous users & disable remote root access (only allow UNIX socket).
+DROP USER IF EXISTS ''@'%', ''@'localhost', 'root'@'%';
 # Remove test database.
 DROP DATABASE IF EXISTS test;
 ###############################################################################
@@ -70,7 +68,7 @@ GRANT SELECT, INSERT, UPDATE, DELETE, CREATE, DROP, RELOAD, PROCESS, FILE,
   LOCK TABLES, EXECUTE, REPLICATION SLAVE, REPLICATION CLIENT, CREATE VIEW,
   SHOW VIEW, CREATE ROUTINE, ALTER ROUTINE, CREATE USER, EVENT, TRIGGER
   ON *.* TO 'vt_filtered'@'localhost';
-FLUSH PRIVILEGES;
+
 RESET SLAVE ALL;
 RESET MASTER;
 

examples/compose/external_db/mysql/grant.sh

@@ -3,5 +3,5 @@ echo '**********GRANTING PRIVILEGES START*******************'
 echo ${mysql[@]}
 # PURGE BINARY LOGS BEFORE DATE(NOW());
 mysql --protocol=socket -uroot -hlocalhost --socket=/var/run/mysqld/mysqld.sock -p$MYSQL_ROOT_PASSWORD -e \
-"GRANT ALL PRIVILEGES ON *.* TO '$MYSQL_USER'@'%' IDENTIFIED BY '$MYSQL_PASSWORD'; FLUSH PRIVILEGES;"
-echo '*************GRANTING PRIVILEGES END****************'
+"GRANT ALL PRIVILEGES ON *.* TO '$MYSQL_USER'@'%' IDENTIFIED BY '$MYSQL_PASSWORD'"
+echo '*************GRANTING PRIVILEGES END****************'

examples/compose/vttablet-up.sh

@@ -68,7 +68,7 @@ if [ "$external" = "1" ]; then
   # We need a common user for the unmanaged and managed tablets else tools like orchestrator will not function correctly
   echo "Creating matching user for managed tablets..."
   echo "CREATE USER IF NOT EXISTS '$DB_USER'@'%' IDENTIFIED BY '$DB_PASS';" >> $init_db_sql_file
-  echo "GRANT ALL ON *.* TO '$DB_USER'@'%';FLUSH PRIVILEGES;" >> $init_db_sql_file
+  echo "GRANT ALL ON *.* TO '$DB_USER'@'%';" >> $init_db_sql_file
 fi
 echo "##[CUSTOM_SQL_END]##" >> $init_db_sql_file
 

examples/operator/101_initial_cluster.yaml

@@ -145,11 +145,8 @@ stringData:
     # Changes during the init db should not make it to the binlog.
     # They could potentially create errant transactions on replicas.
     SET sql_log_bin = 0;
-    # Remove anonymous users.
-    DELETE FROM mysql.user WHERE User = '';
-
-    # Disable remote root access (only allow UNIX socket).
-    DELETE FROM mysql.user WHERE User = 'root' AND Host != 'localhost';
+    # Remove anonymous users & disable remote root access (only allow UNIX socket).
+    DROP USER IF EXISTS ''@'%', ''@'localhost', 'root'@'%';
 
     # Remove test database.
     DROP DATABASE IF EXISTS test;
@@ -215,8 +212,6 @@ stringData:
       SHOW VIEW, CREATE ROUTINE, ALTER ROUTINE, CREATE USER, EVENT, TRIGGER
       ON *.* TO 'vt_filtered'@'localhost';
 
-    FLUSH PRIVILEGES;
-
     RESET SLAVE ALL;
     RESET MASTER;
 

go/cmd/vtctldclient/command/vreplication/movetables/movetables.go

@@ -48,7 +48,6 @@ func registerCommands(root *cobra.Command) {
 	create.Flags().BoolVar(&createOptions.NoRoutingRules, "no-routing-rules", false, "(Advanced) Do not create routing rules while creating the workflow. See the reference documentation for limitations if you use this flag.")
 	create.Flags().BoolVar(&createOptions.AtomicCopy, "atomic-copy", false, "(EXPERIMENTAL) A single copy phase is run for all tables from the source. Use this, for example, if your source keyspace has tables which use foreign key constraints.")
 	create.Flags().StringVar(&createOptions.WorkflowOptions.TenantId, "tenant-id", "", "(EXPERIMENTAL) The tenant ID to use for the MoveTables workflow into a multi-tenant keyspace.")
-	create.Flags().StringVar(&createOptions.WorkflowOptions.SourceKeyspaceAlias, "source-keyspace-alias", "", "(EXPERIMENTAL) Used currently only for multi-tenant migrations. This value will be used instead of the source keyspace name in the keyspace routing rules.")
 	create.Flags().BoolVar(&createOptions.WorkflowOptions.StripShardedAutoIncrement, "remove-sharded-auto-increment", true, "If moving the table(s) to a sharded keyspace, remove any auto_increment clauses when copying the schema to the target as sharded keyspaces should rely on either user/application generated values or Vitess sequences to ensure uniqueness.")
 	base.AddCommand(create)
 

go/test/endtoend/cluster/cluster_util.go

@@ -351,7 +351,6 @@ func GetPasswordUpdateSQL(localCluster *LocalProcessCluster) string {
 					SET PASSWORD FOR 'vt_repl'@'%' = 'VtReplPass';
 					SET PASSWORD FOR 'vt_filtered'@'localhost' = 'VtFilteredPass';
 					SET PASSWORD FOR 'vt_appdebug'@'localhost' = 'VtDebugPass';
-					FLUSH PRIVILEGES;
 					`
 	return pwdChangeCmd
 }

go/test/endtoend/throttler/util.go

@@ -34,9 +34,10 @@ import (
 	"vitess.io/vitess/go/test/endtoend/cluster"
 	"vitess.io/vitess/go/vt/concurrency"
 	"vitess.io/vitess/go/vt/log"
-	topodatapb "vitess.io/vitess/go/vt/proto/topodata"
 	"vitess.io/vitess/go/vt/vttablet/tabletserver/throttle/base"
 	"vitess.io/vitess/go/vt/vttablet/tabletserver/throttle/throttlerapp"
+
+	topodatapb "vitess.io/vitess/go/vt/proto/topodata"
 )
 
 type Config struct {

go/test/endtoend/vreplication/config_test.go

@@ -144,6 +144,22 @@ create table nopk (name varchar(128), age int unsigned);
         "sequence": "customer_seq"
       }
     },
+    "customer_name": {
+      "column_vindexes": [
+        {
+          "column": "cid",
+          "name": "xxhash"
+        }
+      ]
+    },
+    "customer_type": {
+      "column_vindexes": [
+        {
+          "column": "cid",
+          "name": "xxhash"
+        }
+      ]
+    },
     "customer2": {
       "column_vindexes": [
         {
@@ -401,6 +417,32 @@ create table nopk (name varchar(128), age int unsigned);
 		"create_ddl": "create table cproduct(pid bigint, description varchar(128), date1 datetime not null default '0000-00-00 00:00:00', date2 datetime not null default '2021-00-01 00:00:00', primary key(pid)) CHARSET=utf8mb4"
 	}]
 }
+`
+
+	materializeCustomerNameSpec = `
+{
+  "workflow": "customer_name",
+  "source_keyspace": "customer",
+  "target_keyspace": "customer",
+  "table_settings": [{
+    "target_table": "customer_name",
+    "source_expression": "select cid, name from customer",
+    "create_ddl": "create table if not exists customer_name (cid bigint not null, name varchar(128), primary key(cid), key(name))"
+  }]
+}
+`
+
+	materializeCustomerTypeSpec = `
+{
+  "workflow": "customer_type",
+  "source_keyspace": "customer",
+  "target_keyspace": "customer",
+  "table_settings": [{
+    "target_table": "customer_type",
+    "source_expression": "select cid, typ from customer",
+    "create_ddl": "create table if not exists customer_type (cid bigint not null, typ enum('individual','soho','enterprise'), primary key(cid), key(typ))"
+  }]
+}
 `
 
 	merchantOrdersVSchema = `

go/test/endtoend/vreplication/multi_tenant_test.go

@@ -52,9 +52,8 @@ const (
 	tenantMigrationStatusMigrating
 	tenantMigrationStatusMigrated
 
-	sourceKeyspaceTemplate      = "s%d"
-	sourceAliasKeyspaceTemplate = "a%d"
-	targetKeyspaceName          = "mt"
+	sourceKeyspaceTemplate = "s%d"
+	targetKeyspaceName     = "mt"
 
 	numTenants                 = 10
 	numInitialRowsPerTenant    = 10
@@ -130,7 +129,6 @@ func TestMultiTenantSimple(t *testing.T) {
 
 	tenantId := int64(1)
 	sourceKeyspace := getSourceKeyspace(tenantId)
-	sourceAliasKeyspace := getSourceAliasKeyspace(tenantId)
 	_, err = vc.AddKeyspace(t, []*Cell{vc.Cells["zone1"]}, sourceKeyspace, "0", stVSchema, stSchema, 1, 0, getInitialTabletIdForTenant(tenantId), nil)
 	require.NoError(t, err)
 
@@ -163,19 +161,16 @@ func TestMultiTenantSimple(t *testing.T) {
 		sourceKeyspace: sourceKeyspace,
 		createFlags: []string{
 			"--tenant-id", strconv.FormatInt(tenantId, 10),
-			"--source-keyspace-alias", sourceAliasKeyspace,
 		},
 	})
 
 	preSwitchRules := &vschemapb.KeyspaceRoutingRules{
 		Rules: []*vschemapb.KeyspaceRoutingRule{
-			{FromKeyspace: "a1", ToKeyspace: "s1"},
 			{FromKeyspace: "s1", ToKeyspace: "s1"},
 		},
 	}
 	postSwitchRules := &vschemapb.KeyspaceRoutingRules{
 		Rules: []*vschemapb.KeyspaceRoutingRule{
-			{FromKeyspace: "a1", ToKeyspace: "mt"},
 			{FromKeyspace: "s1", ToKeyspace: "mt"},
 		},
 	}
@@ -187,25 +182,21 @@ func TestMultiTenantSimple(t *testing.T) {
 	mt.Create()
 	validateKeyspaceRoutingRules(t, vc, primaries, rulesMap, false)
 	// Note: we cannot insert into the target keyspace since that is never routed to the source keyspace.
-	for _, ks := range []string{sourceKeyspace, sourceAliasKeyspace} {
-		lastIndex = insertRows(lastIndex, ks)
-	}
+	lastIndex = insertRows(lastIndex, sourceKeyspace)
 	waitForWorkflowState(t, vc, fmt.Sprintf("%s.%s", targetKeyspace, mt.workflowName), binlogdatapb.VReplicationWorkflowState_Running.String())
 	mt.SwitchReadsAndWrites()
 	validateKeyspaceRoutingRules(t, vc, primaries, rulesMap, true)
-	// Note: here we have already switched and we can insert into the target keyspace and it should get reverse
-	// replicated to the source keyspace. The source and alias are also routed to the target keyspace at this point.
-	for _, ks := range []string{sourceKeyspace, sourceAliasKeyspace, targetKeyspace} {
-		lastIndex = insertRows(lastIndex, ks)
-	}
+	// Note: here we have already switched, and we can insert into the target keyspace, and it should get reverse
+	// replicated to the source keyspace. The source keyspace is routed to the target keyspace at this point.
+	lastIndex = insertRows(lastIndex, sourceKeyspace)
 	mt.Complete()
 	require.Zero(t, len(getKeyspaceRoutingRules(t, vc).Rules))
 	actualRowsInserted := getRowCount(t, vtgateConn, fmt.Sprintf("%s.%s", targetKeyspace, "t1"))
 	log.Infof("Migration completed, total rows in target: %d", actualRowsInserted)
 	require.Equal(t, lastIndex, int64(actualRowsInserted))
 }
 
-// If switched queries with source/alias qualifiers should execute on target, else on source. Confirm that
+// If switched, queries with source qualifiers should execute on target, else on source. Confirm that
 // the routing rules are as expected and that the query executes on the expected tablet.
 func validateKeyspaceRoutingRules(t *testing.T, vc *VitessCluster, primaries map[string]*cluster.VttabletProcess, rulesMap map[string]*vschemapb.KeyspaceRoutingRules, switched bool) {
 	currentRules := getKeyspaceRoutingRules(t, vc)
@@ -224,25 +215,19 @@ func validateKeyspaceRoutingRules(t *testing.T, vc *VitessCluster, primaries map
 		require.ElementsMatch(t, rulesMap["post"].Rules, currentRules.Rules)
 		validateQueryRoute("mt", "target")
 		validateQueryRoute("s1", "target")
-		validateQueryRoute("a1", "target")
 	} else {
 		require.ElementsMatch(t, rulesMap["pre"].Rules, currentRules.Rules)
 		// Note that with multi-tenant migration, we cannot redirect the target keyspace since
 		// there are multiple source keyspaces and the target has the aggregate of all the tenants.
 		validateQueryRoute("mt", "target")
 		validateQueryRoute("s1", "source")
-		validateQueryRoute("a1", "source")
 	}
 }
 
 func getSourceKeyspace(tenantId int64) string {
 	return fmt.Sprintf(sourceKeyspaceTemplate, tenantId)
 }
 
-func getSourceAliasKeyspace(tenantId int64) string {
-	return fmt.Sprintf(sourceAliasKeyspaceTemplate, tenantId)
-}
-
 func (mtm *multiTenantMigration) insertSomeData(t *testing.T, tenantId int64, keyspace string, numRows int64) {
 	vtgateConn, closeConn := getVTGateConn()
 	defer closeConn()
@@ -362,7 +347,7 @@ func (mtm *multiTenantMigration) getLastID(tenantId int64) int64 {
 	return mtm.lastIDs[tenantId]
 }
 
-func (mtm *multiTenantMigration) initTenantData(t *testing.T, tenantId int64, sourceAliasKeyspace string) {
+func (mtm *multiTenantMigration) initTenantData(t *testing.T, tenantId int64) {
 	mtm.insertSomeData(t, tenantId, getSourceKeyspace(tenantId), numInitialRowsPerTenant)
 }
 
@@ -374,16 +359,14 @@ func (mtm *multiTenantMigration) setup(tenantId int64) {
 	log.Infof("Creating MoveTables for tenant %d", tenantId)
 	mtm.setLastID(tenantId, 0)
 	sourceKeyspace := getSourceKeyspace(tenantId)
-	sourceAliasKeyspace := getSourceAliasKeyspace(tenantId)
 	_, err := vc.AddKeyspace(mtm.t, []*Cell{vc.Cells["zone1"]}, sourceKeyspace, "0", stVSchema, stSchema,
 		1, 0, getInitialTabletIdForTenant(tenantId), nil)
 	require.NoError(mtm.t, err)
-	mtm.initTenantData(mtm.t, tenantId, sourceAliasKeyspace)
+	mtm.initTenantData(mtm.t, tenantId)
 }
 
 func (mtm *multiTenantMigration) start(tenantId int64) {
 	sourceKeyspace := getSourceKeyspace(tenantId)
-	sourceAliasKeyspace := getSourceAliasKeyspace(tenantId)
 	mtm.setTenantMigrationStatus(tenantId, tenantMigrationStatusMigrating)
 	mt := newVtctldMoveTables(&moveTablesWorkflow{
 		workflowInfo: &workflowInfo{
@@ -395,7 +378,6 @@ func (mtm *multiTenantMigration) start(tenantId int64) {
 		tables:         mtm.tables,
 		createFlags: []string{
 			"--tenant-id", strconv.FormatInt(tenantId, 10),
-			"--source-keyspace-alias", sourceAliasKeyspace,
 		},
 	})
 	mtm.setActiveMoveTables(tenantId, mt)
@@ -404,13 +386,11 @@ func (mtm *multiTenantMigration) start(tenantId int64) {
 
 func (mtm *multiTenantMigration) switchTraffic(tenantId int64) {
 	t := mtm.t
-	sourceAliasKeyspace := getSourceAliasKeyspace(tenantId)
 	sourceKeyspaceName := getSourceKeyspace(tenantId)
 	mt := mtm.getActiveMoveTables(tenantId)
 	ksWorkflow := fmt.Sprintf("%s.%s", mtm.targetKeyspace, mt.workflowName)
 	waitForWorkflowState(t, vc, ksWorkflow, binlogdatapb.VReplicationWorkflowState_Running.String())
-	// we intentionally insert first into the source alias keyspace and then the source keyspace to test routing rules for both.
-	mtm.insertSomeData(t, tenantId, sourceAliasKeyspace, numAdditionalRowsPerTenant)
+	mtm.insertSomeData(t, tenantId, sourceKeyspaceName, numAdditionalRowsPerTenant)
 	mt.SwitchReadsAndWrites()
 	mtm.insertSomeData(t, tenantId, sourceKeyspaceName, numAdditionalRowsPerTenant)
 }