Initialize Tablet with super_read_only mode (#12206)

* Adding super read only in cnf file Signed-off-by: Rameez Sajwani <[email protected]> * add new init sql file for unit test Signed-off-by: Rameez Sajwani <[email protected]> * enable super-readonly by default Signed-off-by: Rameez Sajwani <[email protected]> * Fixing some arrangement for super-read-only Signed-off-by: Rameez Sajwani <[email protected]> * Fix end to end tests Signed-off-by: Rameez Sajwani <[email protected]> * fix rpc_replication issue Signed-off-by: Rameez Sajwani <[email protected]> * Fixing backup tests Signed-off-by: Rameez Sajwani <[email protected]> * fixing error with spacing in init sql file Signed-off-by: Rameez Sajwani <[email protected]> * vtgate end to end fixes Signed-off-by: Rameez Sajwani <[email protected]> * some more test fixes Signed-off-by: Rameez Sajwani <[email protected]> * fixing flag tests and more Signed-off-by: Rameez Sajwani <[email protected]> * fixing schema sql Signed-off-by: Rameez Sajwani <[email protected]> * fixing unit tests Signed-off-by: Rameez Sajwani <[email protected]> * fixing vttest server test cases Signed-off-by: Rameez Sajwani <[email protected]> * fixing more downgrad upgrade test Signed-off-by: Rameez Sajwani <[email protected]> * fixing reparent upgrade downgrade test Signed-off-by: Rameez Sajwani <[email protected]> * Bug fix for replication position Signed-off-by: Rameez Sajwani <[email protected]> * Fixing vtexplain tests Signed-off-by: Rameez Sajwani <[email protected]> * Fixing mariadb initialization Signed-off-by: Rameez Sajwani <[email protected]> * fixing upgrade downgrade e2e test Signed-off-by: Rameez Sajwani <[email protected]> * Fixing assignment of init sql db file Signed-off-by: Rameez Sajwani <[email protected]> * code clean up Signed-off-by: Rameez Sajwani <[email protected]> * Fixing backup test Signed-off-by: Rameez Sajwani <[email protected]> * adding super read only api for sql demon Signed-off-by: Rameez Sajwani <[email protected]> * converting read-only calls to super-read-only calls Signed-off-by: Rameez Sajwani <[email protected]> * restructure vtexplain code to be more generic Signed-off-by: Rameez Sajwani <[email protected]> * forgot to make proto vtadmin Signed-off-by: Rameez Sajwani <[email protected]> * Code review part-1 Signed-off-by: Rameez Sajwani <[email protected]> * remove code for redolog in test Signed-off-by: Rameez Sajwani <[email protected]> * fix ensureDB test Signed-off-by: Rameez Sajwani <[email protected]> * code review part-2 Signed-off-by: Rameez Sajwani <[email protected]> * Remove setSuperReadOnly checks Signed-off-by: Rameez Sajwani <[email protected]> * code review part-3 Signed-off-by: Rameez Sajwani <[email protected]> * Try restoring without super_read_only Signed-off-by: Rameez Sajwani <[email protected]> * test: simplify vtorc test to not require turning off super-read-only Signed-off-by: Manan Gupta <[email protected]> * code review part-4 Signed-off-by: Rameez Sajwani <[email protected]> * Fixing typo Signed-off-by: Rameez Sajwani <[email protected]> * code review part-5 Signed-off-by: Rameez Sajwani <[email protected]> * clean up some code Signed-off-by: Rameez Sajwani <[email protected]> * test: fix vtorc test Signed-off-by: Manan Gupta <[email protected]> * Code clean up - part 2 Signed-off-by: Rameez Sajwani <[email protected]> * Adding summary Signed-off-by: Rameez Sajwani <[email protected]> * code cleanup Signed-off-by: Rameez Sajwani <[email protected]> * Removing init_testserver to test its dependenciesy Signed-off-by: Rameez Sajwani <[email protected]> * fixing downgrade/upgrade and bringing back mariadb check Signed-off-by: Rameez Sajwani <[email protected]> * Removing mariadb check from vttest Signed-off-by: Rameez Sajwani <[email protected]> * deprecating `use_super_read_only` flag Signed-off-by: Rameez Sajwani <[email protected]> * Modify FullStatus test Signed-off-by: Rameez Sajwani <[email protected]> * code clean-up Signed-off-by: Rameez Sajwani <[email protected]> * code cleanup - part 2 Signed-off-by: Rameez Sajwani <[email protected]> * fixing fullstatus test Signed-off-by: Rameez Sajwani <[email protected]> * code review feedback Signed-off-by: Rameez Sajwani <[email protected]> * fixing summary and some minor cleanup Signed-off-by: Rameez Sajwani <[email protected]> * remove extra logging Signed-off-by: Rameez Sajwani <[email protected]> * moving init_testserver_db under test folder Signed-off-by: Rameez Sajwani <[email protected]> * fixing code comment Signed-off-by: Rameez Sajwani <[email protected]> * trying out password db init file for PITR test Signed-off-by: Rameez Sajwani <[email protected]> * remove read-only flag from default.cnf Signed-off-by: Rameez Sajwani <[email protected]> * Removing QueryTabletWithSuperReadOnlyHandling method Signed-off-by: Rameez Sajwani <[email protected]> * moving flag at right place Signed-off-by: Rameez Sajwani <[email protected]> * code review feedback Signed-off-by: Rameez Sajwani <[email protected]> * fixing require.noerror check Signed-off-by: Rameez Sajwani <[email protected]> * code review - part 2 Signed-off-by: Rameez Sajwani <[email protected]> * code review phase 3 Signed-off-by: Rameez Sajwani <[email protected]> * revert uncommented code Signed-off-by: Rameez Sajwani <[email protected]> --------- Signed-off-by: Rameez Sajwani <[email protected]> Signed-off-by: Manan Gupta <[email protected]> Co-authored-by: Manan Gupta <[email protected]>
vitessio · Mar 14, 2023 · 3490385 · 3490385
1 parent 1d18df4
commit 3490385
Show file tree

Hide file tree

Showing 70 changed files with 924 additions and 252 deletions.
diff --git a/.github/workflows/upgrade_downgrade_test_backups_e2e.yml b/.github/workflows/upgrade_downgrade_test_backups_e2e.yml
@@ -161,8 +161,10 @@ jobs:
       run: |
         source build.env
 
-        rm -f $PWD/bin/vttablet
+        rm -f $PWD/bin/vttablet $PWD/bin/mysqlctl $PWD/bin/mysqlctld
         cp /tmp/vitess-build-other/bin/vttablet $PWD/bin/vttablet
+        cp /tmp/vitess-build-other/bin/mysqlctl $PWD/bin/mysqlctl
+        cp /tmp/vitess-build-other/bin/mysqlctld $PWD/bin/mysqlctld
         vttablet --version
 
     # Run test with VTTablet at version N-1 and VTBackup at version N
@@ -181,9 +183,11 @@ jobs:
       run: |
         source build.env
 
-        rm -f $PWD/bin/vtbackup $PWD/bin/vttablet
+        rm -f $PWD/bin/vtbackup $PWD/bin/vttablet $PWD/bin/mysqlctl $PWD/bin/mysqlctld
         cp /tmp/vitess-build-current/bin/vtbackup $PWD/bin/vtbackup
         cp /tmp/vitess-build-other/bin/vttablet $PWD/bin/vttablet
+        cp /tmp/vitess-build-other/bin/mysqlctl $PWD/bin/mysqlctl
+        cp /tmp/vitess-build-other/bin/mysqlctld $PWD/bin/mysqlctld
         vtbackup --version
         vttablet --version
 

diff --git a/.github/workflows/upgrade_downgrade_test_backups_e2e_next_release.yml b/.github/workflows/upgrade_downgrade_test_backups_e2e_next_release.yml
@@ -164,8 +164,10 @@ jobs:
       run: |
         source build.env
 
-        rm -f $PWD/bin/vttablet
+        rm -f $PWD/bin/vttablet $PWD/bin/mysqlctl $PWD/bin/mysqlctld
         cp /tmp/vitess-build-other/bin/vttablet $PWD/bin/vttablet
+        cp /tmp/vitess-build-other/bin/mysqlctl $PWD/bin/mysqlctl
+        cp /tmp/vitess-build-other/bin/mysqlctld $PWD/bin/mysqlctld
         vttablet --version
 
     # Run test with VTTablet at version N+1 and VTBackup at version N
@@ -184,9 +186,11 @@ jobs:
       run: |
         source build.env
 
-        rm -f $PWD/bin/vtbackup $PWD/bin/vttablet
-        cp /tmp/vitess-build-current/bin/vtbackup $PWD/bin/vtbackup
-        cp /tmp/vitess-build-other/bin/vttablet $PWD/bin/vttablet
+        rm -f $PWD/bin/vtbackup $PWD/bin/vttablet $PWD/bin/mysqlctl $PWD/bin/mysqlctld
+        cp /tmp/vitess-build-other/bin/vtbackup $PWD/bin/vtbackup
+        cp /tmp/vitess-build-current/bin/vttablet $PWD/bin/vttablet
+        cp /tmp/vitess-build-current/bin/mysqlctl $PWD/bin/mysqlctl
+        cp /tmp/vitess-build-current/bin/mysqlctld $PWD/bin/mysqlctld
         vtbackup --version
         vttablet --version
 

diff --git a/.github/workflows/upgrade_downgrade_test_backups_manual.yml b/.github/workflows/upgrade_downgrade_test_backups_manual.yml
@@ -234,8 +234,10 @@ jobs:
       run: |
         source build.env
 
-        rm -f $PWD/bin/vttablet
+        rm -f $PWD/bin/vttablet $PWD/bin/mysqlctl $PWD/bin/mysqlctld
         cp /tmp/vitess-build-other/bin/vttablet $PWD/bin/vttablet
+        cp /tmp/vitess-build-other/bin/mysqlctl $PWD/bin/mysqlctl
+        cp /tmp/vitess-build-other/bin/mysqlctld $PWD/bin/mysqlctld
         vttablet --version
 
     # Starting the tablets again, they will automatically start restoring the last backup.
@@ -290,8 +292,10 @@ jobs:
       run: |
         source build.env
 
-        rm -f $PWD/bin/vttablet
+        rm -f $PWD/bin/vttablet $PWD/bin/mysqlctl $PWD/bin/mysqlctld
         cp /tmp/vitess-build-current/bin/vttablet $PWD/bin/vttablet
+        cp /tmp/vitess-build-current/bin/mysqlctl $PWD/bin/mysqlctl
+        cp /tmp/vitess-build-current/bin/mysqlctld $PWD/bin/mysqlctld
         vttablet --version
 
     # Starting the tablets again and restoring the previous backup.

diff --git a/.github/workflows/upgrade_downgrade_test_backups_manual_next_release.yml b/.github/workflows/upgrade_downgrade_test_backups_manual_next_release.yml
@@ -237,8 +237,10 @@ jobs:
       run: |
         source build.env
 
-        rm -f $PWD/bin/vttablet
+        rm -f $PWD/bin/vttablet $PWD/bin/mysqlctl $PWD/bin/mysqlctld
         cp /tmp/vitess-build-other/bin/vttablet $PWD/bin/vttablet
+        cp /tmp/vitess-build-other/bin/mysqlctl $PWD/bin/mysqlctl
+        cp /tmp/vitess-build-other/bin/mysqlctld $PWD/bin/mysqlctld
         vttablet --version
 
     # Starting the tablets again, they will automatically start restoring the last backup.
@@ -293,8 +295,10 @@ jobs:
       run: |
         source build.env
 
-        rm -f $PWD/bin/vttablet
+        rm -f $PWD/bin/vttablet $PWD/bin/mysqlctl $PWD/bin/mysqlctld
         cp /tmp/vitess-build-current/bin/vttablet $PWD/bin/vttablet
+        cp /tmp/vitess-build-current/bin/mysqlctl $PWD/bin/mysqlctl
+        cp /tmp/vitess-build-current/bin/mysqlctld $PWD/bin/mysqlctld
         vttablet --version
 
     # Starting the tablets again and restoring the next backup.

diff --git a/.github/workflows/upgrade_downgrade_test_query_serving_queries.yml b/.github/workflows/upgrade_downgrade_test_query_serving_queries.yml
@@ -209,9 +209,11 @@ jobs:
       run: |
         source build.env
 
-        rm -f $PWD/bin/vtgate $PWD/bin/vttablet
+        rm -f $PWD/bin/vtgate $PWD/bin/vttablet $PWD/bin/mysqlctl $PWD/bin/mysqlctld
         cp /tmp/vitess-build-current/bin/vtgate $PWD/bin/vtgate
         cp /tmp/vitess-build-other/bin/vttablet $PWD/bin/vttablet
+        cp /tmp/vitess-build-other/bin/mysqlctl $PWD/bin/mysqlctl
+        cp /tmp/vitess-build-other/bin/mysqlctld $PWD/bin/mysqlctld
         vtgate --version
         vttablet --version
 

diff --git a/.github/workflows/upgrade_downgrade_test_query_serving_queries_next_release.yml b/.github/workflows/upgrade_downgrade_test_query_serving_queries_next_release.yml
@@ -212,9 +212,11 @@ jobs:
       run: |
         source build.env
 
-        rm -f $PWD/bin/vtgate $PWD/bin/vttablet
+        rm -f $PWD/bin/vtgate $PWD/bin/vttablet $PWD/bin/mysqlctl $PWD/bin/mysqlctld
         cp /tmp/vitess-build-current/bin/vtgate $PWD/bin/vtgate
         cp /tmp/vitess-build-other/bin/vttablet $PWD/bin/vttablet
+        cp /tmp/vitess-build-other/bin/mysqlctl $PWD/bin/mysqlctl
+        cp /tmp/vitess-build-other/bin/mysqlctld $PWD/bin/mysqlctld
         vtgate --version
         vttablet --version
 

diff --git a/.github/workflows/upgrade_downgrade_test_query_serving_schema.yml b/.github/workflows/upgrade_downgrade_test_query_serving_schema.yml
@@ -209,9 +209,11 @@ jobs:
       run: |
         source build.env
 
-        rm -f $PWD/bin/vtgate $PWD/bin/vttablet
+        rm -f $PWD/bin/vtgate $PWD/bin/vttablet $PWD/bin/mysqlctl $PWD/bin/mysqlctld
         cp /tmp/vitess-build-current/bin/vtgate $PWD/bin/vtgate
         cp /tmp/vitess-build-other/bin/vttablet $PWD/bin/vttablet
+        cp /tmp/vitess-build-other/bin/mysqlctl $PWD/bin/mysqlctl
+        cp /tmp/vitess-build-other/bin/mysqlctld $PWD/bin/mysqlctld
         vtgate --version
         vttablet --version
 

diff --git a/.github/workflows/upgrade_downgrade_test_query_serving_schema_next_release.yml b/.github/workflows/upgrade_downgrade_test_query_serving_schema_next_release.yml
@@ -212,9 +212,11 @@ jobs:
       run: |
         source build.env
 
-        rm -f $PWD/bin/vtgate $PWD/bin/vttablet
+        rm -f $PWD/bin/vtgate $PWD/bin/vttablet $PWD/bin/mysqlctl $PWD/bin/mysqlctld
         cp /tmp/vitess-build-current/bin/vtgate $PWD/bin/vtgate
         cp /tmp/vitess-build-other/bin/vttablet $PWD/bin/vttablet
+        cp /tmp/vitess-build-other/bin/mysqlctl $PWD/bin/mysqlctl
+        cp /tmp/vitess-build-other/bin/mysqlctld $PWD/bin/mysqlctld
         vtgate --version
         vttablet --version
 

diff --git a/.github/workflows/upgrade_downgrade_test_reparent_new_vttablet.yml b/.github/workflows/upgrade_downgrade_test_reparent_new_vttablet.yml
@@ -182,8 +182,10 @@ jobs:
       run: |
         source build.env
 
-        rm -f $PWD/bin/vttablet
+        rm -f $PWD/bin/vttablet $PWD/bin/mysqlctl $PWD/bin/mysqlctld
         cp /tmp/vitess-build-other/bin/vttablet $PWD/bin/vttablet
+        cp /tmp/vitess-build-other/bin/mysqlctl $PWD/bin/mysqlctl
+        cp /tmp/vitess-build-other/bin/mysqlctld $PWD/bin/mysqlctld
         vtctl --version
         vttablet --version
 

diff --git a/.github/workflows/upgrade_downgrade_test_reparent_old_vttablet.yml b/.github/workflows/upgrade_downgrade_test_reparent_old_vttablet.yml
@@ -179,8 +179,10 @@ jobs:
       run: |
         source build.env
 
-        rm -f $PWD/bin/vttablet
+        rm -f $PWD/bin/vttablet $PWD/bin/mysqlctl $PWD/bin/mysqlctld
         cp /tmp/vitess-build-other/bin/vttablet $PWD/bin/vttablet
+        cp /tmp/vitess-build-other/bin/mysqlctl $PWD/bin/mysqlctl
+        cp /tmp/vitess-build-other/bin/mysqlctld $PWD/bin/mysqlctld
         vtctl --version
         vttablet --version
 

diff --git a/config/init_db.sql b/config/init_db.sql
@@ -1,5 +1,4 @@
-# This file is executed immediately after mysql_install_db,
-# to initialize a fresh data directory.
+# This file is executed immediately after initializing a fresh data directory.
 
 ###############################################################################
 # WARNING: This sql is *NOT* safe for production use,
@@ -11,6 +10,12 @@
 ###############################################################################
 # Equivalent of mysql_secure_installation
 ###############################################################################
+# We need to ensure that super_read_only is disabled so that we can execute
+# these commands. Note that disabling it does NOT disable read_only.
+# We save the current value so that we only re-enable it at the end if it was
+# enabled before.
+SET @original_super_read_only=IF(@@global.super_read_only=1, 'ON', 'OFF');
+SET GLOBAL super_read_only='OFF';
 
 # Changes during the init db should not make it to the binlog.
 # They could potentially create errant transactions on replicas.
@@ -77,3 +82,9 @@ FLUSH PRIVILEGES;
 
 RESET SLAVE ALL;
 RESET MASTER;
+
+# custom sql is used to add custom scripts like creating users/passwords. We use it in our tests
+# {{custom_sql}}
+
+# We need to set super_read_only back to what it was before
+SET GLOBAL super_read_only=IFNULL(@original_super_read_only, 'ON');
diff --git a/config/mycnf/default.cnf b/config/mycnf/default.cnf
@@ -14,9 +14,6 @@ port = {{.MysqlPort}}
 secure-file-priv = {{.SecureFilePriv}}
 {{end}}
 
-# all db instances should start in read-only mode - once the db is started and
-# fully functional, we'll push it into read-write mode
-read-only
 server-id = {{.ServerID}}
 
 # all db instances should skip starting replication threads - that way we can do any

diff --git a/config/mycnf/mariadb10.cnf b/config/mycnf/mariadb10.cnf
@@ -39,3 +39,5 @@ slave_net_timeout = 60
 character_set_server = utf8
 collation_server = utf8_general_ci
 
+# All MariaDB instances should start in read-only mode
+read-only
diff --git a/config/mycnf/mysql57.cnf b/config/mycnf/mysql57.cnf
@@ -32,3 +32,7 @@ plugin-load = rpl_semi_sync_master=semisync_master.so;rpl_semi_sync_slave=semisy
 rpl_semi_sync_master_timeout = 1000000000000000000
 rpl_semi_sync_master_wait_no_slave = 1
 
+# In order to protect against any errand GTIDs we will start the mysql instance
+# in super-read-only mode.
+super-read-only
+
diff --git a/config/mycnf/mysql80.cnf b/config/mycnf/mysql80.cnf
@@ -28,3 +28,7 @@ plugin-load = rpl_semi_sync_master=semisync_master.so;rpl_semi_sync_slave=semisy
 loose_rpl_semi_sync_master_timeout = 1000000000000000000
 loose_rpl_semi_sync_master_wait_no_slave = 1
 
+# In order to protect against any errand GTIDs we will start the mysql instance
+# in super-read-only mode.
+super-read-only
+
diff --git a/config/mycnf/test-suite.cnf b/config/mycnf/test-suite.cnf
@@ -23,3 +23,6 @@ sql_mode = STRICT_TRANS_TABLES
 
 # set a short heartbeat interval in order to detect failures quickly
 slave_net_timeout = 4
+# Disabling `super-read-only`. `test-suite` is mainly used for `vttestserver`. Since `vttestserver` uses a single MySQL for primary and replicas,
+# so it is not possible to run it with `super-read-only`. Therefore, we are disabling it.
+super-read-only = false
diff --git a/doc/releasenotes/17_0_0_summary.md b/doc/releasenotes/17_0_0_summary.md
@@ -12,6 +12,9 @@
     - [VTtablet Error count with code ](#vttablet-error-count-with-code)
   - **[Deprecations and Deletions](#deprecations-and-deletions)**
     - [Deprecated Stats](#deprecated-stats)
+  - **[VTTablet](#vttablet)**
+    - [VTTablet: Initializing all replicas with super_read_only](#vttablet-initialization)
+    - [Deprecated Flags](#deprecated-flags)
 
 ## <a id="major-changes"/> Major Changes
 
@@ -205,4 +208,15 @@ These stats are deprecated in v17.
 | Deprecated stat | Supported alternatives |
 |-|-|
 | `backup_duration_seconds` | `BackupDurationNanoseconds` |
-| `restore_duration_seconds` | `RestoreDurationNanoseconds` |
+| `restore_duration_seconds` | `RestoreDurationNanoseconds` |
+### <a id="vttablet"/> VTTablet
+#### <a id="vttablet-initialization"/> Initializing all replicas with super_read_only
+In order to prevent SUPER privileged users like `root` or `vt_dba` from producing errant GTIDs on replicas, all the replica MySQL servers are initialized with the MySQL
+global variable `super_read_only` value set to `ON`. During failovers, we set `super_read_only` to `OFF` for the promoted primary tablet. This will allow the
+primary to accept writes. All of the shard's tablets, except the current primary, will still have their global variable `super_read_only` set to `ON`. This will make sure that apart from
+MySQL replication no other component, offline system or operator can write directly to a replica.
+
+Reference PR for this change is [PR #12206](https://github.com/vitessio/vitess/pull/12206)
+
+#### <a id="deprecated-flags"/> Deprecated Flags
+The flag `use_super_read_only` is deprecated and will be removed in a later release.
diff --git a/go/cmd/vtbackup/vtbackup.go b/go/cmd/vtbackup/vtbackup.go
@@ -312,6 +312,19 @@ func takeBackup(ctx context.Context, topoServer *topo.Server, backupStorage back
 		if err := mysqld.ResetReplication(ctx); err != nil {
 			return fmt.Errorf("can't reset replication: %v", err)
 		}
+		// We need to switch off super_read_only before we create the database.
+		resetFunc, err := mysqld.SetSuperReadOnly(false)
+		if err != nil {
+			return fmt.Errorf("failed to disable super_read_only during backup: %v", err)
+		}
+		if resetFunc != nil {
+			defer func() {
+				err := resetFunc()
+				if err != nil {
+					log.Error("Failed to set super_read_only back to its original value during backup")
+				}
+			}()
+		}
 		cmd := mysqlctl.GenerateInitialBinlogEntry()
 		if err := mysqld.ExecuteSuperQueryList(ctx, []string{cmd}); err != nil {
 			return err
@@ -405,6 +418,8 @@ func takeBackup(ctx context.Context, topoServer *topo.Server, backupStorage back
 		return fmt.Errorf("can't get the primary replication position after all retries: %v", err)
 	}
 
+	log.Infof("takeBackup: primary position is: %s", primaryPos.String())
+
 	// Remember the time when we fetched the primary position, not when we caught
 	// up to it, so the timestamp on our backup is honest (assuming we make it
 	// to the goal position).

diff --git a/go/flags/endtoend/vttablet.txt b/go/flags/endtoend/vttablet.txt
@@ -349,7 +349,6 @@ Usage of vttablet:
       --tx_throttler_config string                                       The configuration of the transaction throttler as a text formatted throttlerdata.Configuration protocol buffer message (default "target_replication_lag_sec: 2\nmax_replication_lag_sec: 10\ninitial_rate: 100\nmax_increase: 1\nemergency_decrease: 0.5\nmin_duration_between_increases_sec: 40\nmax_duration_between_increases_sec: 62\nmin_duration_between_decreases_sec: 20\nspread_backlog_across_sec: 20\nage_bad_rate_after_sec: 180\nbad_rate_increase: 0.1\nmax_rate_approach_threshold: 0.9\n")
       --tx_throttler_healthcheck_cells strings                           A comma-separated list of cells. Only tabletservers running in these cells will be monitored for replication lag by the transaction throttler.
       --unhealthy_threshold duration                                     replication lag after which a replica is considered unhealthy (default 2h0m0s)
-      --use_super_read_only                                              Set super_read_only flag when performing planned failover.
       --v Level                                                          log level for V logs
   -v, --version                                                          print binary version
       --vmodule moduleSpec                                               comma-separated list of pattern=N settings for file-filtered logging

diff --git a/go/mysql/fakesqldb/server.go b/go/mysql/fakesqldb/server.go
@@ -783,3 +783,40 @@ func (db *DB) MockQueriesForTable(table string, result *sqltypes.Result) {
 		cols...,
 	))
 }
+
+// GetRejectedQueryResult checks if we should reject the query.
+func (db *DB) GetRejectedQueryResult(key string) error {
+	if err, ok := db.rejectedData[key]; ok {
+		return err
+	}
+
+	return nil
+}
+
+// GetQueryResult checks for explicit queries add through AddQuery().
+func (db *DB) GetQueryResult(key string) *ExpectedResult {
+	result, ok := db.data[key]
+	if ok {
+		return result
+	}
+	return nil
+}
+
+// GetQueryPatternResult checks if a query matches any pattern previously added using AddQueryPattern().
+func (db *DB) GetQueryPatternResult(key string) (func(string), ExpectedResult, bool, error) {
+	for _, pat := range db.patternData {
+		if pat.expr.MatchString(key) {
+			userCallback, ok := db.queryPatternUserCallback[pat.expr]
+			if ok {
+				if pat.err != "" {
+					return userCallback, ExpectedResult{pat.result, nil}, true, fmt.Errorf(pat.err)
+				}
+				return userCallback, ExpectedResult{pat.result, nil}, true, nil
+			}
+
+			return nil, ExpectedResult{nil, nil}, false, nil
+		}
+	}
+
+	return nil, ExpectedResult{nil, nil}, false, nil
+}
diff --git a/go/test/endtoend/backup/pitr/backup_mysqlctld_pitr_test.go b/go/test/endtoend/backup/pitr/backup_mysqlctld_pitr_test.go
@@ -138,9 +138,9 @@ func TestIncrementalBackupMysqlctld(t *testing.T) {
 			if tc.writeBeforeBackup {
 				backup.InsertRowOnPrimary(t, "")
 			}
-			// we wait for 1 second because backups ar ewritten to a directory named after the current timestamp,
-			// in 1 second resolution. We want to aoid two backups that have the same pathname. Realistically this
-			// is only ever a problem in this endtoend test, not in production.
+			// we wait for 1 second because backups are written to a directory named after the current timestamp,
+			// in 1 second resolution. We want to avoid two backups that have the same pathname. Realistically this
+			// is only ever a problem in this end-to-end test, not in production.
 			// Also, we gie the replica a chance to catch up.
 			time.Sleep(1100 * time.Millisecond)
 			waitForReplica(t)