From 1f70afb5c7bd48304ad301817bd60058657878f5 Mon Sep 17 00:00:00 2001 From: Renan Rangel Date: Thu, 27 Jun 2024 07:08:24 -0700 Subject: [PATCH 01/34] adding new mysql shell backup engine Signed-off-by: Renan Rangel --- go/vt/mysqlctl/backup.go | 20 ++ go/vt/mysqlctl/mysqlshellbackupengine.go | 327 ++++++++++++++++++ go/vt/mysqlctl/mysqlshellbackupengine_test.go | 69 ++++ go/vt/mysqlctl/xtrabackupengine.go | 17 - 4 files changed, 416 insertions(+), 17 deletions(-) create mode 100644 go/vt/mysqlctl/mysqlshellbackupengine.go create mode 100644 go/vt/mysqlctl/mysqlshellbackupengine_test.go diff --git a/go/vt/mysqlctl/backup.go b/go/vt/mysqlctl/backup.go index 7052dcbdf87..fd067b02887 100644 --- a/go/vt/mysqlctl/backup.go +++ b/go/vt/mysqlctl/backup.go @@ -17,9 +17,11 @@ limitations under the License. package mysqlctl import ( + "bufio" "context" "errors" "fmt" + "io" "os" "path/filepath" "strings" @@ -29,6 +31,7 @@ import ( "vitess.io/vitess/go/textutil" "vitess.io/vitess/go/vt/log" + "vitess.io/vitess/go/vt/logutil" "vitess.io/vitess/go/vt/mysqlctl/backupstats" "vitess.io/vitess/go/vt/mysqlctl/backupstorage" "vitess.io/vitess/go/vt/proto/vtrpc" @@ -500,3 +503,20 @@ func Restore(ctx context.Context, params RestoreParams) (*BackupManifest, error) params.Logger.Infof("Restore: complete") return manifest, nil } + +// scanLinesToLogger scans full lines from the given Reader and sends them to +// the given Logger until EOF. +func scanLinesToLogger(prefix string, reader io.Reader, logger logutil.Logger, doneFunc func()) { + defer doneFunc() + + scanner := bufio.NewScanner(reader) + for scanner.Scan() { + line := scanner.Text() + logger.Infof("%s: %s", prefix, line) + } + if err := scanner.Err(); err != nil { + // This is usually run in a background goroutine, so there's no point + // returning an error. Just log it. + logger.Warningf("error scanning lines from %s: %v", prefix, err) + } +} diff --git a/go/vt/mysqlctl/mysqlshellbackupengine.go b/go/vt/mysqlctl/mysqlshellbackupengine.go new file mode 100644 index 00000000000..d1085742dc5 --- /dev/null +++ b/go/vt/mysqlctl/mysqlshellbackupengine.go @@ -0,0 +1,327 @@ +package mysqlctl + +import ( + "context" + "encoding/json" + "errors" + "flag" + "fmt" + "os/exec" + "path" + "strings" + "sync" + "time" + + "vitess.io/vitess/go/mysql/replication" + "vitess.io/vitess/go/vt/mysqlctl/backupstorage" + tabletmanagerdatapb "vitess.io/vitess/go/vt/proto/tabletmanagerdata" + "vitess.io/vitess/go/vt/vterrors" +) + +var ( + // location to store the mysql shell backup + mysqlShellBackupLocation = flag.String("mysql_shell_backup_location", "", "location where the backup will be stored") + mysqlShellFlags = flag.String("mysql_shell_flags", "--defaults-file=/dev/null --js -h localhost", "execution flags to pass to mysqlsh binary") + // flags to pass through to backup phase + mysqlShellDumpFlags = flag.String("mysql_shell_dump_flags", + `{"threads": 2}`, + "flags to pass to mysql shell dump utility. This should be a JSON string and will be saved in the MANIFEST") + // flags to pass through to extract phase of restore + mysqlShellLoadFlags = flag.String("mysql_shell_load_flags", + `{"threads": 2, "updateGtidSet": "replace", "skipBinlog": true, "progressFile": ""}`, + "flags to pass to mysql shell load utility. This should be a JSON string") + // additional flags + mysqlShellBackupShouldDrain = flag.Bool("mysql_shell_should_drain", + false, "decide if we should drain while taking a backup or continue to serving traffic") + mysqlShellSpeedUpRestore = flag.Bool("mysql_shell_speedup_restore", + false, "speed up restore by disabling redo logging and double write buffer during the restore process") + + MySQLShellPreCheckError = errors.New("MySQLShellPreCheckError") +) + +// MySQLShellBackupManifest represents a backup. +type MySQLShellBackupManifest struct { + // BackupManifest is an anonymous embedding of the base manifest struct. + // Note that the manifest itself doesn't fill the Position field, as we have + // no way of fetching that information from mysqlsh at the moment. + BackupManifest + + // Location of the backup directory + BackupLocation string + // Params are the parameters that backup was created with + Params string +} + +// MySQLShellBackupEngine encapsulates the logic to implement the restoration +// of a mysql-shell based backup. +type MySQLShellBackupEngine struct { +} + +const ( + mysqlShellBackupBinaryName = "mysqlsh" + mysqlShellBackupEngineName = "mysqlshell" +) + +func (be *MySQLShellBackupEngine) ExecuteBackup(ctx context.Context, params BackupParams, bh backupstorage.BackupHandle) (result BackupResult, finalErr error) { + params.Logger.Infof("Starting ExecuteBackup in %s", params.TabletAlias) + + err := be.backupPreCheck() + if err != nil { + return BackupUnusable, vterrors.Wrap(err, "failed backup precheck") + } + + start := time.Now().UTC() + location := path.Join(*mysqlShellBackupLocation, params.Keyspace, params.Shard, start.Format("2006-01-02_15-04-05")) + + args := []string{} + + if *mysqlShellFlags != "" { + args = append(args, strings.Fields(*mysqlShellFlags)...) + } + + args = append(args, "-e", fmt.Sprintf("util.dumpSchemas([\"vt_%s\"], %q, %s)", + params.Keyspace, + location, + *mysqlShellDumpFlags, + )) + + cmd := exec.CommandContext(ctx, mysqlShellBackupBinaryName, args...) + + params.Logger.Infof("running %s", cmd.String()) + + cmdOut, err := cmd.StdoutPipe() + if err != nil { + return BackupUnusable, vterrors.Wrap(err, "cannot create stdout pipe") + } + cmdErr, err := cmd.StderrPipe() + if err != nil { + return BackupUnusable, vterrors.Wrap(err, "cannot create stderr pipe") + } + if err := cmd.Start(); err != nil { + return BackupUnusable, vterrors.Wrap(err, "can't start xbstream") + } + + cmdWg := &sync.WaitGroup{} + cmdWg.Add(2) + go scanLinesToLogger(mysqlShellBackupEngineName+" stdout", cmdOut, params.Logger, cmdWg.Done) + go scanLinesToLogger(mysqlShellBackupEngineName+" stderr", cmdErr, params.Logger, cmdWg.Done) + cmdWg.Wait() + + // Get exit status. + if err := cmd.Wait(); err != nil { + return BackupUnusable, vterrors.Wrap(err, mysqlShellBackupEngineName+" failed") + } + + // open the MANIFEST + params.Logger.Infof("Writing backup MANIFEST") + mwc, err := bh.AddFile(ctx, backupManifestFileName, backupstorage.FileSizeUnknown) + if err != nil { + return BackupUnusable, vterrors.Wrapf(err, "cannot add %v to backup", backupManifestFileName) + } + defer closeFile(mwc, backupManifestFileName, params.Logger, &finalErr) + + // JSON-encode and write the MANIFEST + bm := &MySQLShellBackupManifest{ + // Common base fields + BackupManifest: BackupManifest{ + BackupMethod: mysqlShellBackupEngineName, + // the position is empty here because we have no way of capturing it from mysqlsh + // we will capture it when doing the restore as mysqlsh can replace the GTIDs with + // what it has stored in the backup. + Position: replication.Position{}, + BackupTime: start.Format(time.RFC3339), + FinishedTime: time.Now().UTC().Format(time.RFC3339), + }, + + // mysql shell backup specific fields + BackupLocation: location, + Params: *mysqlShellLoadFlags, + } + + data, err := json.MarshalIndent(bm, "", " ") + if err != nil { + return BackupUnusable, vterrors.Wrapf(err, "cannot JSON encode %v", backupManifestFileName) + } + if _, err := mwc.Write([]byte(data)); err != nil { + return BackupUnusable, vterrors.Wrapf(err, "cannot write %v", backupManifestFileName) + } + + params.Logger.Infof("Backup completed") + return BackupUsable, nil +} + +func (be *MySQLShellBackupEngine) ExecuteRestore(ctx context.Context, params RestoreParams, bh backupstorage.BackupHandle) (*BackupManifest, error) { + params.Logger.Infof("Calling ExecuteRestore for %s (DeleteBeforeRestore: %v)", params.DbName, params.DeleteBeforeRestore) + + err := be.restorePreCheck() + if err != nil { + return nil, vterrors.Wrap(err, "failed restore precheck") + } + + var bm MySQLShellBackupManifest + if err := getBackupManifestInto(ctx, bh, &bm); err != nil { + return nil, err + } + + // mark restore as in progress + if err := createStateFile(params.Cnf); err != nil { + return nil, err + } + + // make sure semi-sync is disabled, otherwise we will wait forever for acknowledgements + err = params.Mysqld.SetSemiSyncEnabled(ctx, false, false) + if err != nil { + return nil, vterrors.Wrap(err, "disable semi-sync failed") + } + + // if we received a RestoreFromBackup API call instead of it being a command line argument, + // we need to first clean the host before we start the restore. + if params.DeleteBeforeRestore { + params.Logger.Infof("restoring on an existing tablet, so dropping database %q", params.DbName) + + err = params.Mysqld.ExecuteSuperQueryList(ctx, + []string{fmt.Sprintf("DROP DATABASE IF EXISTS `%s`", params.DbName)}, + ) + if err != nil { + return nil, vterrors.Wrap(err, fmt.Sprintf("dropping database %q failed", params.DbName)) + } + + } + + // we need to get rid of all the current replication information on the host. + err = params.Mysqld.ResetReplication(ctx) + if err != nil { + return nil, vterrors.Wrap(err, "unable to reset replication") + } + + // this is required so we can load the backup generated by MySQL Shell. we will disable it afterwards. + err = params.Mysqld.ExecuteSuperQueryList(ctx, []string{"SET GLOBAL LOCAL_INFILE=1"}) + if err != nil { + return nil, vterrors.Wrap(err, "unable to set local_infile=1") + } + + if *mysqlShellSpeedUpRestore { + // disable redo logging and double write buffer if we are configured to do so. + err = params.Mysqld.ExecuteSuperQueryList(ctx, []string{"ALTER INSTANCE DISABLE INNODB REDO_LOG"}) + if err != nil { + return nil, vterrors.Wrap(err, "unable to disable REDO_LOG") + } + params.Logger.Infof("Disabled REDO_LOG") + + defer func() { // re-enable once we are done with the restore. + err := params.Mysqld.ExecuteSuperQueryList(ctx, []string{"ALTER INSTANCE ENABLE INNODB REDO_LOG"}) + if err != nil { + params.Logger.Errorf("unable to re-enable REDO_LOG: %v", err) + } else { + params.Logger.Infof("Disabled REDO_LOG") + } + }() + } + + args := []string{} + + if *mysqlShellFlags != "" { + args = append(args, strings.Fields(*mysqlShellFlags)...) + } + + args = append(args, "-e", fmt.Sprintf("util.loadDump(%q, %s)", + bm.BackupLocation, + *mysqlShellLoadFlags, + )) + + cmd := exec.CommandContext(ctx, "mysqlsh", args...) + + params.Logger.Infof("running %s", cmd.String()) + + cmdOut, err := cmd.StdoutPipe() + if err != nil { + return nil, vterrors.Wrap(err, "cannot create stdout pipe") + } + cmdErr, err := cmd.StderrPipe() + if err != nil { + return nil, vterrors.Wrap(err, "cannot create stderr pipe") + } + if err := cmd.Start(); err != nil { + return nil, vterrors.Wrap(err, "can't start xbstream") + } + + cmdWg := &sync.WaitGroup{} + cmdWg.Add(2) + go scanLinesToLogger(mysqlShellBackupEngineName+" stdout", cmdOut, params.Logger, cmdWg.Done) + go scanLinesToLogger(mysqlShellBackupEngineName+" stderr", cmdErr, params.Logger, cmdWg.Done) + cmdWg.Wait() + + // Get the exit status. + if err := cmd.Wait(); err != nil { + return nil, vterrors.Wrap(err, mysqlShellBackupEngineName+" failed") + } + params.Logger.Infof("%s completed successfully", mysqlShellBackupBinaryName) + + // disable local_infile now that the restore is done. + err = params.Mysqld.ExecuteSuperQueryList(ctx, []string{ + "SET GLOBAL LOCAL_INFILE=0", + }) + if err != nil { + return nil, vterrors.Wrap(err, "unable to set local_infile=0") + } + params.Logger.Infof("set local_infile=0") + + // since MySQL Shell backups do not store the Executed GTID position in the manifest, we need to + // fetch it and override in the manifest we are returning so Vitess can set it again. alternatively + // we could have a flag in the future where the backup engine controls if it needs to be set or not. + pos, err := params.Mysqld.PrimaryPosition(ctx) + if err != nil { + return nil, vterrors.Wrap(err, "failure getting restored position") + } + + params.Logger.Infof("retrieved primary position after restore") + bm.BackupManifest.Position = pos + + params.Logger.Infof("Restore completed") + + return &bm.BackupManifest, nil +} + +// ShouldDrainForBackup satisfies the BackupEngine interface +// MySQL Shell backups can be taken while MySQL is running so we can control this via a flag. +func (be *MySQLShellBackupEngine) ShouldDrainForBackup(req *tabletmanagerdatapb.BackupRequest) bool { + return *mysqlShellBackupShouldDrain +} + +func (be *MySQLShellBackupEngine) backupPreCheck() error { + if *mysqlShellBackupLocation == "" { + return fmt.Errorf("%w: no backup location set via --mysql_shell_location", MySQLShellPreCheckError) + } + + if *mysqlShellFlags == "" || !strings.Contains(*mysqlShellFlags, "--js") { + return fmt.Errorf("%w: at least the --js flag is required", MySQLShellPreCheckError) + } + + return nil +} + +func (be *MySQLShellBackupEngine) restorePreCheck() error { + if *mysqlShellFlags == "" { + return fmt.Errorf("%w: at least the --js flag is required", MySQLShellPreCheckError) + } + + loadFlags := map[string]interface{}{} + err := json.Unmarshal([]byte(*mysqlShellLoadFlags), &loadFlags) + if err != nil { + return fmt.Errorf("%w: unable to parse JSON of load flags", MySQLShellPreCheckError) + } + + if val, ok := loadFlags["updateGtidSet"]; !ok || val != "replace" { + return fmt.Errorf("%w: mysql-shell needs to restore with updateGtidSet set to \"replace\" to work with Vitess", MySQLShellPreCheckError) + } + + if val, ok := loadFlags["progressFile"]; !ok || val != "" { + return fmt.Errorf("%w: \"progressFile\" needs to be empty as vitess always starts a restore from scratch", MySQLShellPreCheckError) + } + + return nil +} + +func init() { + BackupRestoreEngineMap[mysqlShellBackupEngineName] = &MySQLShellBackupEngine{} +} diff --git a/go/vt/mysqlctl/mysqlshellbackupengine_test.go b/go/vt/mysqlctl/mysqlshellbackupengine_test.go new file mode 100644 index 00000000000..4b4c6030605 --- /dev/null +++ b/go/vt/mysqlctl/mysqlshellbackupengine_test.go @@ -0,0 +1,69 @@ +package mysqlctl + +import ( + "testing" + + "github.com/stretchr/testify/assert" + tabletmanagerdatapb "vitess.io/vitess/go/vt/proto/tabletmanagerdata" +) + +func TestMySQLShellBackupRestorePreCheck(t *testing.T) { + original := *mysqlShellLoadFlags + defer func() { *mysqlShellLoadFlags = original }() + + engine := MySQLShellBackupEngine{} + tests := []struct { + name string + flags string + err error + }{ + { + "empty load flags", + `{}`, + MySQLShellPreCheckError, + }, + { + "only updateGtidSet", + `{"updateGtidSet": "replace"}`, + MySQLShellPreCheckError, + }, + { + "only progressFile", + `{"progressFile": ""}`, + MySQLShellPreCheckError, + }, + { + "both values but unsupported values", + `{"updateGtidSet": "append", "progressFile": "/tmp/test1"}`, + MySQLShellPreCheckError, + }, + { + "supported values", + `{"updateGtidSet": "replace", "progressFile": ""}`, + nil, + }, + } + + for _, tt := range tests { + *mysqlShellLoadFlags = tt.flags + assert.ErrorIs(t, engine.restorePreCheck(), tt.err) + } + +} + +func TestShouldDrainForBackupMySQLShell(t *testing.T) { + original := *mysqlShellBackupShouldDrain + defer func() { *mysqlShellBackupShouldDrain = original }() + + engine := MySQLShellBackupEngine{} + + *mysqlShellBackupShouldDrain = false + + assert.False(t, engine.ShouldDrainForBackup(nil)) + assert.False(t, engine.ShouldDrainForBackup(&tabletmanagerdatapb.BackupRequest{})) + + *mysqlShellBackupShouldDrain = true + + assert.True(t, engine.ShouldDrainForBackup(nil)) + assert.True(t, engine.ShouldDrainForBackup(&tabletmanagerdatapb.BackupRequest{})) +} diff --git a/go/vt/mysqlctl/xtrabackupengine.go b/go/vt/mysqlctl/xtrabackupengine.go index 3f8491fdfb6..87f3f4cee6a 100644 --- a/go/vt/mysqlctl/xtrabackupengine.go +++ b/go/vt/mysqlctl/xtrabackupengine.go @@ -776,23 +776,6 @@ func findReplicationPosition(input, flavor string, logger logutil.Logger) (repli return replicationPosition, nil } -// scanLinesToLogger scans full lines from the given Reader and sends them to -// the given Logger until EOF. -func scanLinesToLogger(prefix string, reader io.Reader, logger logutil.Logger, doneFunc func()) { - defer doneFunc() - - scanner := bufio.NewScanner(reader) - for scanner.Scan() { - line := scanner.Text() - logger.Infof("%s: %s", prefix, line) - } - if err := scanner.Err(); err != nil { - // This is usually run in a background goroutine, so there's no point - // returning an error. Just log it. - logger.Warningf("error scanning lines from %s: %v", prefix, err) - } -} - func stripeFileName(baseFileName string, index int) string { return fmt.Sprintf("%s-%03d", baseFileName, index) } From 58005ab4a766ffeb12152794d0a9a365dcb7aa67 Mon Sep 17 00:00:00 2001 From: Renan Rangel Date: Tue, 2 Jul 2024 01:23:03 -0700 Subject: [PATCH 02/34] change flags Signed-off-by: Renan Rangel --- go/vt/mysqlctl/mysqlshellbackupengine.go | 73 ++++++++++-------- go/vt/mysqlctl/mysqlshellbackupengine_test.go | 77 +++++++++++++++++-- 2 files changed, 112 insertions(+), 38 deletions(-) diff --git a/go/vt/mysqlctl/mysqlshellbackupengine.go b/go/vt/mysqlctl/mysqlshellbackupengine.go index d1085742dc5..bdbe4c3ea62 100644 --- a/go/vt/mysqlctl/mysqlshellbackupengine.go +++ b/go/vt/mysqlctl/mysqlshellbackupengine.go @@ -4,7 +4,6 @@ import ( "context" "encoding/json" "errors" - "flag" "fmt" "os/exec" "path" @@ -12,29 +11,28 @@ import ( "sync" "time" + "github.com/spf13/pflag" + "vitess.io/vitess/go/mysql/replication" "vitess.io/vitess/go/vt/mysqlctl/backupstorage" tabletmanagerdatapb "vitess.io/vitess/go/vt/proto/tabletmanagerdata" + "vitess.io/vitess/go/vt/servenv" "vitess.io/vitess/go/vt/vterrors" ) var ( // location to store the mysql shell backup - mysqlShellBackupLocation = flag.String("mysql_shell_backup_location", "", "location where the backup will be stored") - mysqlShellFlags = flag.String("mysql_shell_flags", "--defaults-file=/dev/null --js -h localhost", "execution flags to pass to mysqlsh binary") - // flags to pass through to backup phase - mysqlShellDumpFlags = flag.String("mysql_shell_dump_flags", - `{"threads": 2}`, - "flags to pass to mysql shell dump utility. This should be a JSON string and will be saved in the MANIFEST") - // flags to pass through to extract phase of restore - mysqlShellLoadFlags = flag.String("mysql_shell_load_flags", - `{"threads": 2, "updateGtidSet": "replace", "skipBinlog": true, "progressFile": ""}`, - "flags to pass to mysql shell load utility. This should be a JSON string") - // additional flags - mysqlShellBackupShouldDrain = flag.Bool("mysql_shell_should_drain", - false, "decide if we should drain while taking a backup or continue to serving traffic") - mysqlShellSpeedUpRestore = flag.Bool("mysql_shell_speedup_restore", - false, "speed up restore by disabling redo logging and double write buffer during the restore process") + mysqlShellBackupLocation = "" + // flags passed to the mysql shell utility, used both on dump/restore + mysqlShellFlags = "--defaults-file=/dev/null --js -h localhost" + // flags passed to the Dump command, as a JSON string + mysqlShellDumpFlags = `{"threads": 2}` + // flags passed to the Load command, as a JSON string + mysqlShellLoadFlags = `{"threads": 4, "updateGtidSet": "replace", "skipBinlog": true, "progressFile": ""}` + // drain a tablet when taking a backup + mysqlShellBackupShouldDrain = false + // disable redo logging and double write buffer + mysqlShellSpeedUpRestore = false MySQLShellPreCheckError = errors.New("MySQLShellPreCheckError") ) @@ -52,6 +50,21 @@ type MySQLShellBackupManifest struct { Params string } +func init() { + for _, cmd := range []string{"vtcombo", "vttablet", "vtbackup", "vttestserver", "vtctldclient"} { + servenv.OnParseFor(cmd, registerMysqlShellBackupEngineFlags) + } +} + +func registerMysqlShellBackupEngineFlags(fs *pflag.FlagSet) { + fs.StringVar(&mysqlShellBackupLocation, "mysql_shell_backup_location", mysqlShellBackupLocation, "location where the backup will be stored") + fs.StringVar(&mysqlShellFlags, "mysql_shell_flags", mysqlShellFlags, "execution flags to pass to mysqlsh binary to be used during dump/load") + fs.StringVar(&mysqlShellDumpFlags, "mysql_shell_dump_flags", mysqlShellDumpFlags, "flags to pass to mysql shell dump utility. This should be a JSON string and will be saved in the MANIFEST") + fs.StringVar(&mysqlShellLoadFlags, "mysql_shell_load_flags", mysqlShellLoadFlags, "flags to pass to mysql shell load utility. This should be a JSON string") + fs.BoolVar(&mysqlShellBackupShouldDrain, "mysql_shell_should_drain", mysqlShellBackupShouldDrain, "decide if we should drain while taking a backup or continue to serving traffic") + fs.BoolVar(&mysqlShellSpeedUpRestore, "mysql_shell_speedup_restore", mysqlShellSpeedUpRestore, "speed up restore by disabling redo logging and double write buffer during the restore process") +} + // MySQLShellBackupEngine encapsulates the logic to implement the restoration // of a mysql-shell based backup. type MySQLShellBackupEngine struct { @@ -71,18 +84,18 @@ func (be *MySQLShellBackupEngine) ExecuteBackup(ctx context.Context, params Back } start := time.Now().UTC() - location := path.Join(*mysqlShellBackupLocation, params.Keyspace, params.Shard, start.Format("2006-01-02_15-04-05")) + location := path.Join(mysqlShellBackupLocation, params.Keyspace, params.Shard, start.Format("2006-01-02_15-04-05")) args := []string{} - if *mysqlShellFlags != "" { - args = append(args, strings.Fields(*mysqlShellFlags)...) + if mysqlShellFlags != "" { + args = append(args, strings.Fields(mysqlShellFlags)...) } args = append(args, "-e", fmt.Sprintf("util.dumpSchemas([\"vt_%s\"], %q, %s)", params.Keyspace, location, - *mysqlShellDumpFlags, + mysqlShellDumpFlags, )) cmd := exec.CommandContext(ctx, mysqlShellBackupBinaryName, args...) @@ -135,7 +148,7 @@ func (be *MySQLShellBackupEngine) ExecuteBackup(ctx context.Context, params Back // mysql shell backup specific fields BackupLocation: location, - Params: *mysqlShellLoadFlags, + Params: mysqlShellLoadFlags, } data, err := json.MarshalIndent(bm, "", " ") @@ -200,7 +213,7 @@ func (be *MySQLShellBackupEngine) ExecuteRestore(ctx context.Context, params Res return nil, vterrors.Wrap(err, "unable to set local_infile=1") } - if *mysqlShellSpeedUpRestore { + if mysqlShellSpeedUpRestore { // disable redo logging and double write buffer if we are configured to do so. err = params.Mysqld.ExecuteSuperQueryList(ctx, []string{"ALTER INSTANCE DISABLE INNODB REDO_LOG"}) if err != nil { @@ -220,13 +233,13 @@ func (be *MySQLShellBackupEngine) ExecuteRestore(ctx context.Context, params Res args := []string{} - if *mysqlShellFlags != "" { - args = append(args, strings.Fields(*mysqlShellFlags)...) + if mysqlShellFlags != "" { + args = append(args, strings.Fields(mysqlShellFlags)...) } args = append(args, "-e", fmt.Sprintf("util.loadDump(%q, %s)", bm.BackupLocation, - *mysqlShellLoadFlags, + mysqlShellLoadFlags, )) cmd := exec.CommandContext(ctx, "mysqlsh", args...) @@ -285,15 +298,15 @@ func (be *MySQLShellBackupEngine) ExecuteRestore(ctx context.Context, params Res // ShouldDrainForBackup satisfies the BackupEngine interface // MySQL Shell backups can be taken while MySQL is running so we can control this via a flag. func (be *MySQLShellBackupEngine) ShouldDrainForBackup(req *tabletmanagerdatapb.BackupRequest) bool { - return *mysqlShellBackupShouldDrain + return mysqlShellBackupShouldDrain } func (be *MySQLShellBackupEngine) backupPreCheck() error { - if *mysqlShellBackupLocation == "" { + if mysqlShellBackupLocation == "" { return fmt.Errorf("%w: no backup location set via --mysql_shell_location", MySQLShellPreCheckError) } - if *mysqlShellFlags == "" || !strings.Contains(*mysqlShellFlags, "--js") { + if mysqlShellFlags == "" || !strings.Contains(mysqlShellFlags, "--js") { return fmt.Errorf("%w: at least the --js flag is required", MySQLShellPreCheckError) } @@ -301,12 +314,12 @@ func (be *MySQLShellBackupEngine) backupPreCheck() error { } func (be *MySQLShellBackupEngine) restorePreCheck() error { - if *mysqlShellFlags == "" { + if mysqlShellFlags == "" { return fmt.Errorf("%w: at least the --js flag is required", MySQLShellPreCheckError) } loadFlags := map[string]interface{}{} - err := json.Unmarshal([]byte(*mysqlShellLoadFlags), &loadFlags) + err := json.Unmarshal([]byte(mysqlShellLoadFlags), &loadFlags) if err != nil { return fmt.Errorf("%w: unable to parse JSON of load flags", MySQLShellPreCheckError) } diff --git a/go/vt/mysqlctl/mysqlshellbackupengine_test.go b/go/vt/mysqlctl/mysqlshellbackupengine_test.go index 4b4c6030605..e5a107d245f 100644 --- a/go/vt/mysqlctl/mysqlshellbackupengine_test.go +++ b/go/vt/mysqlctl/mysqlshellbackupengine_test.go @@ -4,12 +4,71 @@ import ( "testing" "github.com/stretchr/testify/assert" + tabletmanagerdatapb "vitess.io/vitess/go/vt/proto/tabletmanagerdata" ) +func TestMySQLShellBackupBackupPreCheck(t *testing.T) { + originalLocation := mysqlShellBackupLocation + originalFlags := mysqlShellFlags + defer func() { + mysqlShellBackupLocation = originalLocation + mysqlShellFlags = originalFlags + }() + + engine := MySQLShellBackupEngine{} + tests := []struct { + name string + location string + flags string + err error + }{ + { + "empty flags", + "", + `{}`, + MySQLShellPreCheckError, + }, + { + "only location", + "/dev/null", + "", + MySQLShellPreCheckError, + }, + { + "only flags", + "", + "--js", + MySQLShellPreCheckError, + }, + { + "both values present but without --js", + "", + "-h localhost", + MySQLShellPreCheckError, + }, + { + "supported values", + "/tmp/backup/", + "--js -h localhost", + nil, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + + mysqlShellBackupLocation = tt.location + mysqlShellFlags = tt.flags + assert.ErrorIs(t, engine.backupPreCheck(), tt.err) + }) + } + +} + func TestMySQLShellBackupRestorePreCheck(t *testing.T) { - original := *mysqlShellLoadFlags - defer func() { *mysqlShellLoadFlags = original }() + original := mysqlShellLoadFlags + defer func() { mysqlShellLoadFlags = original }() engine := MySQLShellBackupEngine{} tests := []struct { @@ -45,24 +104,26 @@ func TestMySQLShellBackupRestorePreCheck(t *testing.T) { } for _, tt := range tests { - *mysqlShellLoadFlags = tt.flags - assert.ErrorIs(t, engine.restorePreCheck(), tt.err) + t.Run(tt.name, func(t *testing.T) { + mysqlShellLoadFlags = tt.flags + assert.ErrorIs(t, engine.restorePreCheck(), tt.err) + }) } } func TestShouldDrainForBackupMySQLShell(t *testing.T) { - original := *mysqlShellBackupShouldDrain - defer func() { *mysqlShellBackupShouldDrain = original }() + original := mysqlShellBackupShouldDrain + defer func() { mysqlShellBackupShouldDrain = original }() engine := MySQLShellBackupEngine{} - *mysqlShellBackupShouldDrain = false + mysqlShellBackupShouldDrain = false assert.False(t, engine.ShouldDrainForBackup(nil)) assert.False(t, engine.ShouldDrainForBackup(&tabletmanagerdatapb.BackupRequest{})) - *mysqlShellBackupShouldDrain = true + mysqlShellBackupShouldDrain = true assert.True(t, engine.ShouldDrainForBackup(nil)) assert.True(t, engine.ShouldDrainForBackup(&tabletmanagerdatapb.BackupRequest{})) From b74a87f6bb292fd0d9707b0878d440bba27cd65f Mon Sep 17 00:00:00 2001 From: Renan Rangel Date: Wed, 10 Jul 2024 08:22:19 -0700 Subject: [PATCH 03/34] fix TestHelpOutput Signed-off-by: Renan Rangel --- go/flags/endtoend/vtbackup.txt | 6 ++++++ go/flags/endtoend/vtcombo.txt | 6 ++++++ go/flags/endtoend/vttablet.txt | 6 ++++++ go/flags/endtoend/vttestserver.txt | 6 ++++++ 4 files changed, 24 insertions(+) diff --git a/go/flags/endtoend/vtbackup.txt b/go/flags/endtoend/vtbackup.txt index 004871d7c09..3d4087fe1f3 100644 --- a/go/flags/endtoend/vtbackup.txt +++ b/go/flags/endtoend/vtbackup.txt @@ -178,6 +178,12 @@ Flags: --mysql-shutdown-timeout duration how long to wait for mysqld shutdown (default 5m0s) --mysql_port int mysql port (default 3306) --mysql_server_version string MySQL server version to advertise. (default "8.0.30-Vitess") + --mysql_shell_backup_location string location where the backup will be stored + --mysql_shell_dump_flags string flags to pass to mysql shell dump utility. This should be a JSON string and will be saved in the MANIFEST (default "{\"threads\": 2}") + --mysql_shell_flags string execution flags to pass to mysqlsh binary to be used during dump/load (default "--defaults-file=/dev/null --js -h localhost") + --mysql_shell_load_flags string flags to pass to mysql shell load utility. This should be a JSON string (default "{\"threads\": 4, \"updateGtidSet\": \"replace\", \"skipBinlog\": true, \"progressFile\": \"\"}") + --mysql_shell_should_drain decide if we should drain while taking a backup or continue to serving traffic + --mysql_shell_speedup_restore speed up restore by disabling redo logging and double write buffer during the restore process --mysql_socket string path to the mysql socket --mysql_timeout duration how long to wait for mysqld startup (default 5m0s) --opentsdb_uri string URI of opentsdb /api/put method diff --git a/go/flags/endtoend/vtcombo.txt b/go/flags/endtoend/vtcombo.txt index 8d868e9f49c..c1eab3890a0 100644 --- a/go/flags/endtoend/vtcombo.txt +++ b/go/flags/endtoend/vtcombo.txt @@ -245,6 +245,12 @@ Flags: --mysql_server_tls_min_version string Configures the minimal TLS version negotiated when SSL is enabled. Defaults to TLSv1.2. Options: TLSv1.0, TLSv1.1, TLSv1.2, TLSv1.3. --mysql_server_version string MySQL server version to advertise. (default "8.0.30-Vitess") --mysql_server_write_timeout duration connection write timeout + --mysql_shell_backup_location string location where the backup will be stored + --mysql_shell_dump_flags string flags to pass to mysql shell dump utility. This should be a JSON string and will be saved in the MANIFEST (default "{\"threads\": 2}") + --mysql_shell_flags string execution flags to pass to mysqlsh binary to be used during dump/load (default "--defaults-file=/dev/null --js -h localhost") + --mysql_shell_load_flags string flags to pass to mysql shell load utility. This should be a JSON string (default "{\"threads\": 4, \"updateGtidSet\": \"replace\", \"skipBinlog\": true, \"progressFile\": \"\"}") + --mysql_shell_should_drain decide if we should drain while taking a backup or continue to serving traffic + --mysql_shell_speedup_restore speed up restore by disabling redo logging and double write buffer during the restore process --mysql_slow_connect_warn_threshold duration Warn if it takes more than the given threshold for a mysql connection to establish --mysql_tcp_version string Select tcp, tcp4, or tcp6 to control the socket type. (default "tcp") --mysqlctl_mycnf_template string template file to use for generating the my.cnf file during server init diff --git a/go/flags/endtoend/vttablet.txt b/go/flags/endtoend/vttablet.txt index d160968e014..118bdf00f85 100644 --- a/go/flags/endtoend/vttablet.txt +++ b/go/flags/endtoend/vttablet.txt @@ -245,6 +245,12 @@ Flags: --mycnf_tmp_dir string mysql tmp directory --mysql-shutdown-timeout duration timeout to use when MySQL is being shut down. (default 5m0s) --mysql_server_version string MySQL server version to advertise. (default "8.0.30-Vitess") + --mysql_shell_backup_location string location where the backup will be stored + --mysql_shell_dump_flags string flags to pass to mysql shell dump utility. This should be a JSON string and will be saved in the MANIFEST (default "{\"threads\": 2}") + --mysql_shell_flags string execution flags to pass to mysqlsh binary to be used during dump/load (default "--defaults-file=/dev/null --js -h localhost") + --mysql_shell_load_flags string flags to pass to mysql shell load utility. This should be a JSON string (default "{\"threads\": 4, \"updateGtidSet\": \"replace\", \"skipBinlog\": true, \"progressFile\": \"\"}") + --mysql_shell_should_drain decide if we should drain while taking a backup or continue to serving traffic + --mysql_shell_speedup_restore speed up restore by disabling redo logging and double write buffer during the restore process --mysqlctl_mycnf_template string template file to use for generating the my.cnf file during server init --mysqlctl_socket string socket file to use for remote mysqlctl actions (empty for local actions) --onclose_timeout duration wait no more than this for OnClose handlers before stopping (default 10s) diff --git a/go/flags/endtoend/vttestserver.txt b/go/flags/endtoend/vttestserver.txt index 8cce76afc65..b9f97e0b28b 100644 --- a/go/flags/endtoend/vttestserver.txt +++ b/go/flags/endtoend/vttestserver.txt @@ -90,6 +90,12 @@ Flags: --mysql_bind_host string which host to bind vtgate mysql listener to (default "localhost") --mysql_only If this flag is set only mysql is initialized. The rest of the vitess components are not started. Also, the output specifies the mysql unix socket instead of the vtgate port. --mysql_server_version string MySQL server version to advertise. (default "8.0.30-Vitess") + --mysql_shell_backup_location string location where the backup will be stored + --mysql_shell_dump_flags string flags to pass to mysql shell dump utility. This should be a JSON string and will be saved in the MANIFEST (default "{\"threads\": 2}") + --mysql_shell_flags string execution flags to pass to mysqlsh binary to be used during dump/load (default "--defaults-file=/dev/null --js -h localhost") + --mysql_shell_load_flags string flags to pass to mysql shell load utility. This should be a JSON string (default "{\"threads\": 4, \"updateGtidSet\": \"replace\", \"skipBinlog\": true, \"progressFile\": \"\"}") + --mysql_shell_should_drain decide if we should drain while taking a backup or continue to serving traffic + --mysql_shell_speedup_restore speed up restore by disabling redo logging and double write buffer during the restore process --mysqlctl_mycnf_template string template file to use for generating the my.cnf file during server init --mysqlctl_socket string socket file to use for remote mysqlctl actions (empty for local actions) --no_scatter when set to true, the planner will fail instead of producing a plan that includes scatter queries From c51ee4b7e3da381843ca6ae37fa404715bf8d459 Mon Sep 17 00:00:00 2001 From: Renan Rangel Date: Fri, 19 Jul 2024 03:55:43 -0700 Subject: [PATCH 04/34] add unit test for scanLinesToLogger Signed-off-by: Renan Rangel --- go/vt/mysqlctl/backup_test.go | 32 +++++++++++++++++++++++++++----- 1 file changed, 27 insertions(+), 5 deletions(-) diff --git a/go/vt/mysqlctl/backup_test.go b/go/vt/mysqlctl/backup_test.go index d1d6a73b7bd..e1ded4d1316 100644 --- a/go/vt/mysqlctl/backup_test.go +++ b/go/vt/mysqlctl/backup_test.go @@ -26,19 +26,18 @@ import ( "path" "reflect" "sort" + "sync" "testing" "time" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" - "vitess.io/vitess/go/sqltypes" - "vitess.io/vitess/go/test/utils" - - "vitess.io/vitess/go/mysql/replication" - "vitess.io/vitess/go/mysql" "vitess.io/vitess/go/mysql/fakesqldb" + "vitess.io/vitess/go/mysql/replication" + "vitess.io/vitess/go/sqltypes" + "vitess.io/vitess/go/test/utils" "vitess.io/vitess/go/vt/logutil" "vitess.io/vitess/go/vt/mysqlctl/backupstats" "vitess.io/vitess/go/vt/mysqlctl/backupstorage" @@ -714,3 +713,26 @@ func TestShouldRestore(t *testing.T) { assert.False(t, b) assert.NoError(t, err) } + +func Test_scanLinesToLogger(t *testing.T) { + reader, writer := io.Pipe() + logger := logutil.NewMemoryLogger() + var wg sync.WaitGroup + + wg.Add(1) + go scanLinesToLogger("test", reader, logger, wg.Done) + + for i := range 100 { + _, err := writer.Write([]byte(fmt.Sprintf("foobar %d\n", i))) + require.NoError(t, err) + } + + writer.Close() + wg.Wait() + + require.Equal(t, 100, len(logger.Events)) + + for i, event := range logger.Events { + require.Equal(t, fmt.Sprintf("test: foobar %d", i), event.Value) + } +} From 6e59215d34e9d526e937047409ecea5a263e02c3 Mon Sep 17 00:00:00 2001 From: Renan Rangel Date: Tue, 23 Jul 2024 08:17:50 -0700 Subject: [PATCH 05/34] PR feedback Signed-off-by: Renan Rangel --- docker/lite/Dockerfile | 2 +- docker/utils/install_dependencies.sh | 2 + go/flags/endtoend/vtbackup.txt | 12 +++--- go/flags/endtoend/vtcombo.txt | 12 +++--- go/flags/endtoend/vttablet.txt | 12 +++--- go/flags/endtoend/vttestserver.txt | 12 +++--- go/vt/mysqlctl/mysqlshellbackupengine.go | 37 ++++++++++++++----- go/vt/mysqlctl/mysqlshellbackupengine_test.go | 28 +++++++++++++- 8 files changed, 82 insertions(+), 35 deletions(-) diff --git a/docker/lite/Dockerfile b/docker/lite/Dockerfile index d5c46cac133..5fc83123b1a 100644 --- a/docker/lite/Dockerfile +++ b/docker/lite/Dockerfile @@ -42,7 +42,7 @@ RUN /vt/dist/install_dependencies.sh mysql80 # Set up Vitess user and directory tree. RUN groupadd -r vitess && useradd -r -g vitess vitess -RUN mkdir -p /vt/vtdataroot && chown -R vitess:vitess /vt +RUN mkdir -p /vt/vtdataroot /home/vitess && chown -R vitess:vitess /vt /home/vitess # Set up Vitess environment (just enough to run pre-built Go binaries) ENV VTROOT /vt/src/vitess.io/vitess diff --git a/docker/utils/install_dependencies.sh b/docker/utils/install_dependencies.sh index b686c2418bf..91e6e2b8c76 100755 --- a/docker/utils/install_dependencies.sh +++ b/docker/utils/install_dependencies.sh @@ -86,6 +86,7 @@ mysql57) /tmp/mysql-client_${VERSION}-1debian10_amd64.deb /tmp/mysql-community-server_${VERSION}-1debian10_amd64.deb /tmp/mysql-server_${VERSION}-1debian10_amd64.deb + mysql-shell percona-xtrabackup-24 ) ;; @@ -112,6 +113,7 @@ mysql80) /tmp/mysql-community-server-core_${VERSION}-1debian11_amd64.deb /tmp/mysql-community-server_${VERSION}-1debian11_amd64.deb /tmp/mysql-server_${VERSION}-1debian11_amd64.deb + mysql-shell percona-xtrabackup-80 ) ;; diff --git a/go/flags/endtoend/vtbackup.txt b/go/flags/endtoend/vtbackup.txt index 3d4087fe1f3..647245b7a1d 100644 --- a/go/flags/endtoend/vtbackup.txt +++ b/go/flags/endtoend/vtbackup.txt @@ -178,12 +178,12 @@ Flags: --mysql-shutdown-timeout duration how long to wait for mysqld shutdown (default 5m0s) --mysql_port int mysql port (default 3306) --mysql_server_version string MySQL server version to advertise. (default "8.0.30-Vitess") - --mysql_shell_backup_location string location where the backup will be stored - --mysql_shell_dump_flags string flags to pass to mysql shell dump utility. This should be a JSON string and will be saved in the MANIFEST (default "{\"threads\": 2}") - --mysql_shell_flags string execution flags to pass to mysqlsh binary to be used during dump/load (default "--defaults-file=/dev/null --js -h localhost") - --mysql_shell_load_flags string flags to pass to mysql shell load utility. This should be a JSON string (default "{\"threads\": 4, \"updateGtidSet\": \"replace\", \"skipBinlog\": true, \"progressFile\": \"\"}") - --mysql_shell_should_drain decide if we should drain while taking a backup or continue to serving traffic - --mysql_shell_speedup_restore speed up restore by disabling redo logging and double write buffer during the restore process + --mysql-shell-backup-location string location where the backup will be stored + --mysql-shell-dump-flags string flags to pass to mysql shell dump utility. This should be a JSON string and will be saved in the MANIFEST (default "{\"threads\": 2}") + --mysql-shell-flags string execution flags to pass to mysqlsh binary to be used during dump/load (default "--defaults-file=/dev/null --js -h localhost") + --mysql-shell-load-flags string flags to pass to mysql shell load utility. This should be a JSON string (default "{\"threads\": 4, \"updateGtidSet\": \"replace\", \"skipBinlog\": true, \"progressFile\": \"\"}") + --mysql-shell-should-drain decide if we should drain while taking a backup or continue to serving traffic + --mysql-shell-speedup-restore speed up restore by disabling redo logging and double write buffer during the restore process --mysql_socket string path to the mysql socket --mysql_timeout duration how long to wait for mysqld startup (default 5m0s) --opentsdb_uri string URI of opentsdb /api/put method diff --git a/go/flags/endtoend/vtcombo.txt b/go/flags/endtoend/vtcombo.txt index c1eab3890a0..73fe180aa5e 100644 --- a/go/flags/endtoend/vtcombo.txt +++ b/go/flags/endtoend/vtcombo.txt @@ -245,12 +245,12 @@ Flags: --mysql_server_tls_min_version string Configures the minimal TLS version negotiated when SSL is enabled. Defaults to TLSv1.2. Options: TLSv1.0, TLSv1.1, TLSv1.2, TLSv1.3. --mysql_server_version string MySQL server version to advertise. (default "8.0.30-Vitess") --mysql_server_write_timeout duration connection write timeout - --mysql_shell_backup_location string location where the backup will be stored - --mysql_shell_dump_flags string flags to pass to mysql shell dump utility. This should be a JSON string and will be saved in the MANIFEST (default "{\"threads\": 2}") - --mysql_shell_flags string execution flags to pass to mysqlsh binary to be used during dump/load (default "--defaults-file=/dev/null --js -h localhost") - --mysql_shell_load_flags string flags to pass to mysql shell load utility. This should be a JSON string (default "{\"threads\": 4, \"updateGtidSet\": \"replace\", \"skipBinlog\": true, \"progressFile\": \"\"}") - --mysql_shell_should_drain decide if we should drain while taking a backup or continue to serving traffic - --mysql_shell_speedup_restore speed up restore by disabling redo logging and double write buffer during the restore process + --mysql-shell-backup-location string location where the backup will be stored + --mysql-shell-dump-flags string flags to pass to mysql shell dump utility. This should be a JSON string and will be saved in the MANIFEST (default "{\"threads\": 2}") + --mysql-shell-flags string execution flags to pass to mysqlsh binary to be used during dump/load (default "--defaults-file=/dev/null --js -h localhost") + --mysql-shell-load-flags string flags to pass to mysql shell load utility. This should be a JSON string (default "{\"threads\": 4, \"updateGtidSet\": \"replace\", \"skipBinlog\": true, \"progressFile\": \"\"}") + --mysql-shell-should-drain decide if we should drain while taking a backup or continue to serving traffic + --mysql-shell-speedup-restore speed up restore by disabling redo logging and double write buffer during the restore process --mysql_slow_connect_warn_threshold duration Warn if it takes more than the given threshold for a mysql connection to establish --mysql_tcp_version string Select tcp, tcp4, or tcp6 to control the socket type. (default "tcp") --mysqlctl_mycnf_template string template file to use for generating the my.cnf file during server init diff --git a/go/flags/endtoend/vttablet.txt b/go/flags/endtoend/vttablet.txt index 118bdf00f85..07ec5fee516 100644 --- a/go/flags/endtoend/vttablet.txt +++ b/go/flags/endtoend/vttablet.txt @@ -245,12 +245,12 @@ Flags: --mycnf_tmp_dir string mysql tmp directory --mysql-shutdown-timeout duration timeout to use when MySQL is being shut down. (default 5m0s) --mysql_server_version string MySQL server version to advertise. (default "8.0.30-Vitess") - --mysql_shell_backup_location string location where the backup will be stored - --mysql_shell_dump_flags string flags to pass to mysql shell dump utility. This should be a JSON string and will be saved in the MANIFEST (default "{\"threads\": 2}") - --mysql_shell_flags string execution flags to pass to mysqlsh binary to be used during dump/load (default "--defaults-file=/dev/null --js -h localhost") - --mysql_shell_load_flags string flags to pass to mysql shell load utility. This should be a JSON string (default "{\"threads\": 4, \"updateGtidSet\": \"replace\", \"skipBinlog\": true, \"progressFile\": \"\"}") - --mysql_shell_should_drain decide if we should drain while taking a backup or continue to serving traffic - --mysql_shell_speedup_restore speed up restore by disabling redo logging and double write buffer during the restore process + --mysql-shell-backup-location string location where the backup will be stored + --mysql-shell-dump-flags string flags to pass to mysql shell dump utility. This should be a JSON string and will be saved in the MANIFEST (default "{\"threads\": 2}") + --mysql-shell-flags string execution flags to pass to mysqlsh binary to be used during dump/load (default "--defaults-file=/dev/null --js -h localhost") + --mysql-shell-load-flags string flags to pass to mysql shell load utility. This should be a JSON string (default "{\"threads\": 4, \"updateGtidSet\": \"replace\", \"skipBinlog\": true, \"progressFile\": \"\"}") + --mysql-shell-should-drain decide if we should drain while taking a backup or continue to serving traffic + --mysql-shell-speedup-restore speed up restore by disabling redo logging and double write buffer during the restore process --mysqlctl_mycnf_template string template file to use for generating the my.cnf file during server init --mysqlctl_socket string socket file to use for remote mysqlctl actions (empty for local actions) --onclose_timeout duration wait no more than this for OnClose handlers before stopping (default 10s) diff --git a/go/flags/endtoend/vttestserver.txt b/go/flags/endtoend/vttestserver.txt index b9f97e0b28b..ebb74229b56 100644 --- a/go/flags/endtoend/vttestserver.txt +++ b/go/flags/endtoend/vttestserver.txt @@ -90,12 +90,12 @@ Flags: --mysql_bind_host string which host to bind vtgate mysql listener to (default "localhost") --mysql_only If this flag is set only mysql is initialized. The rest of the vitess components are not started. Also, the output specifies the mysql unix socket instead of the vtgate port. --mysql_server_version string MySQL server version to advertise. (default "8.0.30-Vitess") - --mysql_shell_backup_location string location where the backup will be stored - --mysql_shell_dump_flags string flags to pass to mysql shell dump utility. This should be a JSON string and will be saved in the MANIFEST (default "{\"threads\": 2}") - --mysql_shell_flags string execution flags to pass to mysqlsh binary to be used during dump/load (default "--defaults-file=/dev/null --js -h localhost") - --mysql_shell_load_flags string flags to pass to mysql shell load utility. This should be a JSON string (default "{\"threads\": 4, \"updateGtidSet\": \"replace\", \"skipBinlog\": true, \"progressFile\": \"\"}") - --mysql_shell_should_drain decide if we should drain while taking a backup or continue to serving traffic - --mysql_shell_speedup_restore speed up restore by disabling redo logging and double write buffer during the restore process + --mysql-shell-backup-location string location where the backup will be stored + --mysql-shell-dump-flags string flags to pass to mysql shell dump utility. This should be a JSON string and will be saved in the MANIFEST (default "{\"threads\": 2}") + --mysql-shell-flags string execution flags to pass to mysqlsh binary to be used during dump/load (default "--defaults-file=/dev/null --js -h localhost") + --mysql-shell-load-flags string flags to pass to mysql shell load utility. This should be a JSON string (default "{\"threads\": 4, \"updateGtidSet\": \"replace\", \"skipBinlog\": true, \"progressFile\": \"\"}") + --mysql-shell-should-drain decide if we should drain while taking a backup or continue to serving traffic + --mysql-shell-speedup-restore speed up restore by disabling redo logging and double write buffer during the restore process --mysqlctl_mycnf_template string template file to use for generating the my.cnf file during server init --mysqlctl_socket string socket file to use for remote mysqlctl actions (empty for local actions) --no_scatter when set to true, the planner will fail instead of producing a plan that includes scatter queries diff --git a/go/vt/mysqlctl/mysqlshellbackupengine.go b/go/vt/mysqlctl/mysqlshellbackupengine.go index bdbe4c3ea62..3ca6b1d7481 100644 --- a/go/vt/mysqlctl/mysqlshellbackupengine.go +++ b/go/vt/mysqlctl/mysqlshellbackupengine.go @@ -13,6 +13,8 @@ import ( "github.com/spf13/pflag" + "vitess.io/vitess/go/mysql" + "vitess.io/vitess/go/mysql/capabilities" "vitess.io/vitess/go/mysql/replication" "vitess.io/vitess/go/vt/mysqlctl/backupstorage" tabletmanagerdatapb "vitess.io/vitess/go/vt/proto/tabletmanagerdata" @@ -57,12 +59,12 @@ func init() { } func registerMysqlShellBackupEngineFlags(fs *pflag.FlagSet) { - fs.StringVar(&mysqlShellBackupLocation, "mysql_shell_backup_location", mysqlShellBackupLocation, "location where the backup will be stored") - fs.StringVar(&mysqlShellFlags, "mysql_shell_flags", mysqlShellFlags, "execution flags to pass to mysqlsh binary to be used during dump/load") - fs.StringVar(&mysqlShellDumpFlags, "mysql_shell_dump_flags", mysqlShellDumpFlags, "flags to pass to mysql shell dump utility. This should be a JSON string and will be saved in the MANIFEST") - fs.StringVar(&mysqlShellLoadFlags, "mysql_shell_load_flags", mysqlShellLoadFlags, "flags to pass to mysql shell load utility. This should be a JSON string") - fs.BoolVar(&mysqlShellBackupShouldDrain, "mysql_shell_should_drain", mysqlShellBackupShouldDrain, "decide if we should drain while taking a backup or continue to serving traffic") - fs.BoolVar(&mysqlShellSpeedUpRestore, "mysql_shell_speedup_restore", mysqlShellSpeedUpRestore, "speed up restore by disabling redo logging and double write buffer during the restore process") + fs.StringVar(&mysqlShellBackupLocation, "mysql-shell-backup_location", mysqlShellBackupLocation, "location where the backup will be stored") + fs.StringVar(&mysqlShellFlags, "mysql-shell-flags", mysqlShellFlags, "execution flags to pass to mysqlsh binary to be used during dump/load") + fs.StringVar(&mysqlShellDumpFlags, "mysql-shell-dump-flags", mysqlShellDumpFlags, "flags to pass to mysql shell dump utility. This should be a JSON string and will be saved in the MANIFEST") + fs.StringVar(&mysqlShellLoadFlags, "mysql-shell-load-flags", mysqlShellLoadFlags, "flags to pass to mysql shell load utility. This should be a JSON string") + fs.BoolVar(&mysqlShellBackupShouldDrain, "mysql-shell-should-drain", mysqlShellBackupShouldDrain, "decide if we should drain while taking a backup or continue to serving traffic") + fs.BoolVar(&mysqlShellSpeedUpRestore, "mysql-shell-speedup-restore", mysqlShellSpeedUpRestore, "speed up restore by disabling redo logging and double write buffer during the restore process") } // MySQLShellBackupEngine encapsulates the logic to implement the restoration @@ -84,7 +86,7 @@ func (be *MySQLShellBackupEngine) ExecuteBackup(ctx context.Context, params Back } start := time.Now().UTC() - location := path.Join(mysqlShellBackupLocation, params.Keyspace, params.Shard, start.Format("2006-01-02_15-04-05")) + location := path.Join(mysqlShellBackupLocation, params.Keyspace, params.Shard, start.Format(BackupTimestampFormat)) args := []string{} @@ -166,7 +168,7 @@ func (be *MySQLShellBackupEngine) ExecuteBackup(ctx context.Context, params Back func (be *MySQLShellBackupEngine) ExecuteRestore(ctx context.Context, params RestoreParams, bh backupstorage.BackupHandle) (*BackupManifest, error) { params.Logger.Infof("Calling ExecuteRestore for %s (DeleteBeforeRestore: %v)", params.DbName, params.DeleteBeforeRestore) - err := be.restorePreCheck() + err := be.restorePreCheck(ctx, params) if err != nil { return nil, vterrors.Wrap(err, "failed restore precheck") } @@ -313,7 +315,7 @@ func (be *MySQLShellBackupEngine) backupPreCheck() error { return nil } -func (be *MySQLShellBackupEngine) restorePreCheck() error { +func (be *MySQLShellBackupEngine) restorePreCheck(ctx context.Context, params RestoreParams) error { if mysqlShellFlags == "" { return fmt.Errorf("%w: at least the --js flag is required", MySQLShellPreCheckError) } @@ -332,6 +334,23 @@ func (be *MySQLShellBackupEngine) restorePreCheck() error { return fmt.Errorf("%w: \"progressFile\" needs to be empty as vitess always starts a restore from scratch", MySQLShellPreCheckError) } + if mysqlShellSpeedUpRestore { + version, err := params.Mysqld.GetVersionString(ctx) + if err != nil { + return fmt.Errorf("%w: failed to fetch MySQL version: %v", MySQLShellPreCheckError, err) + } + + capableOf := mysql.ServerVersionCapableOf(version) + capable, err := capableOf(capabilities.DisableRedoLogFlavorCapability) + if err != nil { + return fmt.Errorf("%w: error checking if server supports disabling redo log: %v", MySQLShellPreCheckError, err) + } + + if !capable { + return fmt.Errorf("%w: MySQL version doesn't support disabling the redo log (must be >=8.0.21)", MySQLShellPreCheckError) + } + } + return nil } diff --git a/go/vt/mysqlctl/mysqlshellbackupengine_test.go b/go/vt/mysqlctl/mysqlshellbackupengine_test.go index e5a107d245f..b538351d5d1 100644 --- a/go/vt/mysqlctl/mysqlshellbackupengine_test.go +++ b/go/vt/mysqlctl/mysqlshellbackupengine_test.go @@ -1,10 +1,13 @@ package mysqlctl import ( + "context" "testing" "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "vitess.io/vitess/go/mysql/fakesqldb" tabletmanagerdatapb "vitess.io/vitess/go/vt/proto/tabletmanagerdata" ) @@ -106,12 +109,35 @@ func TestMySQLShellBackupRestorePreCheck(t *testing.T) { for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { mysqlShellLoadFlags = tt.flags - assert.ErrorIs(t, engine.restorePreCheck(), tt.err) + assert.ErrorIs(t, engine.restorePreCheck(context.Background(), RestoreParams{}), tt.err) }) } } +func TestMySQLShellBackupRestorePreCheckDisableRedolog(t *testing.T) { + original := mysqlShellSpeedUpRestore + defer func() { mysqlShellSpeedUpRestore = original }() + + mysqlShellSpeedUpRestore = true + engine := MySQLShellBackupEngine{} + + fakeMysqld := NewFakeMysqlDaemon(fakesqldb.New(t)) // defaults to 8.0.32 + params := RestoreParams{ + Mysqld: fakeMysqld, + } + + // this should work as it is supported since 8.0.21 + require.NoError(t, engine.restorePreCheck(context.Background(), params)) + + // it should error out if we change to an older version + fakeMysqld.Version = "8.0.20" + + err := engine.restorePreCheck(context.Background(), params) + require.ErrorIs(t, err, MySQLShellPreCheckError) + require.ErrorContains(t, err, "doesn't support disabling the redo log") +} + func TestShouldDrainForBackupMySQLShell(t *testing.T) { original := mysqlShellBackupShouldDrain defer func() { mysqlShellBackupShouldDrain = original }() From 3ca4b3bbadd28a2854a7ecd44dfeae25cb9e632f Mon Sep 17 00:00:00 2001 From: Renan Rangel Date: Thu, 25 Jul 2024 07:22:36 -0700 Subject: [PATCH 06/34] PR feedback + fix tests Signed-off-by: Renan Rangel --- docker/lite/Dockerfile | 5 +++++ go/flags/endtoend/vtbackup.txt | 6 +++--- go/flags/endtoend/vtcombo.txt | 12 ++++++------ go/flags/endtoend/vttablet.txt | 4 ++-- go/flags/endtoend/vttestserver.txt | 6 +++--- go/vt/mysqlctl/mysqlshellbackupengine.go | 18 +++++++++++++++++- go/vt/mysqlctl/mysqlshellbackupengine_test.go | 16 ++++++++++++++++ 7 files changed, 52 insertions(+), 15 deletions(-) diff --git a/docker/lite/Dockerfile b/docker/lite/Dockerfile index 4ae6400bcab..a3aef833226 100644 --- a/docker/lite/Dockerfile +++ b/docker/lite/Dockerfile @@ -33,6 +33,11 @@ RUN make install PREFIX=/vt/install # Start over and build the final image. FROM --platform=linux/amd64 debian:bullseye-slim +# Install locale required for mysqlsh +RUN apt-get update && apt-get install -y locales \ + && echo "en_US.UTF-8 UTF-8" > /etc/locale.gen \ + && locale-gen en_US.UTF-8 + # Install dependencies COPY docker/utils/install_dependencies.sh /vt/dist/install_dependencies.sh RUN /vt/dist/install_dependencies.sh mysql80 diff --git a/go/flags/endtoend/vtbackup.txt b/go/flags/endtoend/vtbackup.txt index 647245b7a1d..5c9a7114519 100644 --- a/go/flags/endtoend/vtbackup.txt +++ b/go/flags/endtoend/vtbackup.txt @@ -175,15 +175,15 @@ Flags: --mycnf_slow_log_path string mysql slow query log path --mycnf_socket_file string mysql socket file --mycnf_tmp_dir string mysql tmp directory - --mysql-shutdown-timeout duration how long to wait for mysqld shutdown (default 5m0s) - --mysql_port int mysql port (default 3306) - --mysql_server_version string MySQL server version to advertise. (default "8.0.30-Vitess") --mysql-shell-backup-location string location where the backup will be stored --mysql-shell-dump-flags string flags to pass to mysql shell dump utility. This should be a JSON string and will be saved in the MANIFEST (default "{\"threads\": 2}") --mysql-shell-flags string execution flags to pass to mysqlsh binary to be used during dump/load (default "--defaults-file=/dev/null --js -h localhost") --mysql-shell-load-flags string flags to pass to mysql shell load utility. This should be a JSON string (default "{\"threads\": 4, \"updateGtidSet\": \"replace\", \"skipBinlog\": true, \"progressFile\": \"\"}") --mysql-shell-should-drain decide if we should drain while taking a backup or continue to serving traffic --mysql-shell-speedup-restore speed up restore by disabling redo logging and double write buffer during the restore process + --mysql-shutdown-timeout duration how long to wait for mysqld shutdown (default 5m0s) + --mysql_port int mysql port (default 3306) + --mysql_server_version string MySQL server version to advertise. (default "8.0.30-Vitess") --mysql_socket string path to the mysql socket --mysql_timeout duration how long to wait for mysqld startup (default 5m0s) --opentsdb_uri string URI of opentsdb /api/put method diff --git a/go/flags/endtoend/vtcombo.txt b/go/flags/endtoend/vtcombo.txt index 778217528de..0254942df39 100644 --- a/go/flags/endtoend/vtcombo.txt +++ b/go/flags/endtoend/vtcombo.txt @@ -226,6 +226,12 @@ Flags: --mysql-server-drain-onterm If set, the server waits for --onterm_timeout for already connected clients to complete their in flight work --mysql-server-keepalive-period duration TCP period between keep-alives --mysql-server-pool-conn-read-buffers If set, the server will pool incoming connection read buffers + --mysql-shell-backup-location string location where the backup will be stored + --mysql-shell-dump-flags string flags to pass to mysql shell dump utility. This should be a JSON string and will be saved in the MANIFEST (default "{\"threads\": 2}") + --mysql-shell-flags string execution flags to pass to mysqlsh binary to be used during dump/load (default "--defaults-file=/dev/null --js -h localhost") + --mysql-shell-load-flags string flags to pass to mysql shell load utility. This should be a JSON string (default "{\"threads\": 4, \"updateGtidSet\": \"replace\", \"skipBinlog\": true, \"progressFile\": \"\"}") + --mysql-shell-should-drain decide if we should drain while taking a backup or continue to serving traffic + --mysql-shell-speedup-restore speed up restore by disabling redo logging and double write buffer during the restore process --mysql-shutdown-timeout duration timeout to use when MySQL is being shut down. (default 5m0s) --mysql_allow_clear_text_without_tls If set, the server will allow the use of a clear text password over non-SSL connections. --mysql_auth_server_impl string Which auth server implementation to use. Options: none, ldap, clientcert, static, vault. (default "static") @@ -246,12 +252,6 @@ Flags: --mysql_server_tls_min_version string Configures the minimal TLS version negotiated when SSL is enabled. Defaults to TLSv1.2. Options: TLSv1.0, TLSv1.1, TLSv1.2, TLSv1.3. --mysql_server_version string MySQL server version to advertise. (default "8.0.30-Vitess") --mysql_server_write_timeout duration connection write timeout - --mysql-shell-backup-location string location where the backup will be stored - --mysql-shell-dump-flags string flags to pass to mysql shell dump utility. This should be a JSON string and will be saved in the MANIFEST (default "{\"threads\": 2}") - --mysql-shell-flags string execution flags to pass to mysqlsh binary to be used during dump/load (default "--defaults-file=/dev/null --js -h localhost") - --mysql-shell-load-flags string flags to pass to mysql shell load utility. This should be a JSON string (default "{\"threads\": 4, \"updateGtidSet\": \"replace\", \"skipBinlog\": true, \"progressFile\": \"\"}") - --mysql-shell-should-drain decide if we should drain while taking a backup or continue to serving traffic - --mysql-shell-speedup-restore speed up restore by disabling redo logging and double write buffer during the restore process --mysql_slow_connect_warn_threshold duration Warn if it takes more than the given threshold for a mysql connection to establish --mysql_tcp_version string Select tcp, tcp4, or tcp6 to control the socket type. (default "tcp") --mysqlctl_mycnf_template string template file to use for generating the my.cnf file during server init diff --git a/go/flags/endtoend/vttablet.txt b/go/flags/endtoend/vttablet.txt index 596445c740b..cea972984e5 100644 --- a/go/flags/endtoend/vttablet.txt +++ b/go/flags/endtoend/vttablet.txt @@ -243,14 +243,14 @@ Flags: --mycnf_slow_log_path string mysql slow query log path --mycnf_socket_file string mysql socket file --mycnf_tmp_dir string mysql tmp directory - --mysql-shutdown-timeout duration timeout to use when MySQL is being shut down. (default 5m0s) - --mysql_server_version string MySQL server version to advertise. (default "8.0.30-Vitess") --mysql-shell-backup-location string location where the backup will be stored --mysql-shell-dump-flags string flags to pass to mysql shell dump utility. This should be a JSON string and will be saved in the MANIFEST (default "{\"threads\": 2}") --mysql-shell-flags string execution flags to pass to mysqlsh binary to be used during dump/load (default "--defaults-file=/dev/null --js -h localhost") --mysql-shell-load-flags string flags to pass to mysql shell load utility. This should be a JSON string (default "{\"threads\": 4, \"updateGtidSet\": \"replace\", \"skipBinlog\": true, \"progressFile\": \"\"}") --mysql-shell-should-drain decide if we should drain while taking a backup or continue to serving traffic --mysql-shell-speedup-restore speed up restore by disabling redo logging and double write buffer during the restore process + --mysql-shutdown-timeout duration timeout to use when MySQL is being shut down. (default 5m0s) + --mysql_server_version string MySQL server version to advertise. (default "8.0.30-Vitess") --mysqlctl_mycnf_template string template file to use for generating the my.cnf file during server init --mysqlctl_socket string socket file to use for remote mysqlctl actions (empty for local actions) --onclose_timeout duration wait no more than this for OnClose handlers before stopping (default 10s) diff --git a/go/flags/endtoend/vttestserver.txt b/go/flags/endtoend/vttestserver.txt index 6ad529334e7..278f8885c77 100644 --- a/go/flags/endtoend/vttestserver.txt +++ b/go/flags/endtoend/vttestserver.txt @@ -87,15 +87,15 @@ Flags: --max-stack-size int configure the maximum stack size in bytes (default 67108864) --max_table_shard_size int The maximum number of initial rows in a table shard. Ignored if--initialize_with_random_data is false. The actual number is chosen randomly (default 10000) --min_table_shard_size int The minimum number of initial rows in a table shard. Ignored if--initialize_with_random_data is false. The actual number is chosen randomly. (default 1000) - --mysql_bind_host string which host to bind vtgate mysql listener to (default "localhost") - --mysql_only If this flag is set only mysql is initialized. The rest of the vitess components are not started. Also, the output specifies the mysql unix socket instead of the vtgate port. - --mysql_server_version string MySQL server version to advertise. (default "8.0.30-Vitess") --mysql-shell-backup-location string location where the backup will be stored --mysql-shell-dump-flags string flags to pass to mysql shell dump utility. This should be a JSON string and will be saved in the MANIFEST (default "{\"threads\": 2}") --mysql-shell-flags string execution flags to pass to mysqlsh binary to be used during dump/load (default "--defaults-file=/dev/null --js -h localhost") --mysql-shell-load-flags string flags to pass to mysql shell load utility. This should be a JSON string (default "{\"threads\": 4, \"updateGtidSet\": \"replace\", \"skipBinlog\": true, \"progressFile\": \"\"}") --mysql-shell-should-drain decide if we should drain while taking a backup or continue to serving traffic --mysql-shell-speedup-restore speed up restore by disabling redo logging and double write buffer during the restore process + --mysql_bind_host string which host to bind vtgate mysql listener to (default "localhost") + --mysql_only If this flag is set only mysql is initialized. The rest of the vitess components are not started. Also, the output specifies the mysql unix socket instead of the vtgate port. + --mysql_server_version string MySQL server version to advertise. (default "8.0.30-Vitess") --mysqlctl_mycnf_template string template file to use for generating the my.cnf file during server init --mysqlctl_socket string socket file to use for remote mysqlctl actions (empty for local actions) --no_scatter when set to true, the planner will fail instead of producing a plan that includes scatter queries diff --git a/go/vt/mysqlctl/mysqlshellbackupengine.go b/go/vt/mysqlctl/mysqlshellbackupengine.go index 3ca6b1d7481..7922b51eb1e 100644 --- a/go/vt/mysqlctl/mysqlshellbackupengine.go +++ b/go/vt/mysqlctl/mysqlshellbackupengine.go @@ -1,3 +1,19 @@ +/* +Copyright 2019 The Vitess Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + package mysqlctl import ( @@ -59,7 +75,7 @@ func init() { } func registerMysqlShellBackupEngineFlags(fs *pflag.FlagSet) { - fs.StringVar(&mysqlShellBackupLocation, "mysql-shell-backup_location", mysqlShellBackupLocation, "location where the backup will be stored") + fs.StringVar(&mysqlShellBackupLocation, "mysql-shell-backup-location", mysqlShellBackupLocation, "location where the backup will be stored") fs.StringVar(&mysqlShellFlags, "mysql-shell-flags", mysqlShellFlags, "execution flags to pass to mysqlsh binary to be used during dump/load") fs.StringVar(&mysqlShellDumpFlags, "mysql-shell-dump-flags", mysqlShellDumpFlags, "flags to pass to mysql shell dump utility. This should be a JSON string and will be saved in the MANIFEST") fs.StringVar(&mysqlShellLoadFlags, "mysql-shell-load-flags", mysqlShellLoadFlags, "flags to pass to mysql shell load utility. This should be a JSON string") diff --git a/go/vt/mysqlctl/mysqlshellbackupengine_test.go b/go/vt/mysqlctl/mysqlshellbackupengine_test.go index b538351d5d1..9d5ccc087e1 100644 --- a/go/vt/mysqlctl/mysqlshellbackupengine_test.go +++ b/go/vt/mysqlctl/mysqlshellbackupengine_test.go @@ -1,3 +1,19 @@ +/* +Copyright 2019 The Vitess Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + package mysqlctl import ( From d26b5c31f901068951c29da60cb50b03a049bacb Mon Sep 17 00:00:00 2001 From: Renan Rangel Date: Wed, 7 Aug 2024 14:59:14 -0700 Subject: [PATCH 07/34] more improvements Signed-off-by: Renan Rangel --- .../backup_pitr_mysqlshell_test.go | 61 ++++++ .../backup/vtctlbackup/backup_utils.go | 83 +++++-- go/vt/mysqlctl/fakemysqldaemon.go | 16 ++ go/vt/mysqlctl/mysql_daemon.go | 10 + go/vt/mysqlctl/mysqld.go | 7 +- go/vt/mysqlctl/mysqlshellbackupengine.go | 203 +++++++++++++++--- go/vt/mysqlctl/mysqlshellbackupengine_test.go | 7 +- go/vt/mysqlctl/query.go | 37 ++++ test/config.json | 9 + 9 files changed, 374 insertions(+), 59 deletions(-) create mode 100644 go/test/endtoend/backup/pitr_mysqlshell/backup_pitr_mysqlshell_test.go diff --git a/go/test/endtoend/backup/pitr_mysqlshell/backup_pitr_mysqlshell_test.go b/go/test/endtoend/backup/pitr_mysqlshell/backup_pitr_mysqlshell_test.go new file mode 100644 index 00000000000..b323927483e --- /dev/null +++ b/go/test/endtoend/backup/pitr_mysqlshell/backup_pitr_mysqlshell_test.go @@ -0,0 +1,61 @@ +/* +Copyright 2024 The Vitess Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package mysqlctld + +import ( + "testing" + + backup "vitess.io/vitess/go/test/endtoend/backup/vtctlbackup" +) + +// TestIncrementalBackupAndRestoreToPos +func TestIncrementalBackupAndRestoreToPos(t *testing.T) { + tcase := &backup.PITRTestCase{ + Name: "MySQLShell", + SetupType: backup.MySQLShell, + ComprssDetails: &backup.CompressionDetails{ + CompressorEngineName: "pgzip", + }, + } + backup.ExecTestIncrementalBackupAndRestoreToPos(t, tcase) +} + +// TestIncrementalBackupAndRestoreToTimestamp - tests incremental backups and restores. +// The general outline of the test: +// - Generate some schema with data +// - Take a full backup +// - Proceed to take a series of inremental backups. In between, inject data (insert rows), and keep record +// of which data (number of rows) is present in each backup, and at which timestamp. +// - Expect backups success/failure per scenario +// - Next up, we start testing restores. Randomly pick recorded timestamps and restore to those points in time. +// - In each restore, excpect to find the data (number of rows) recorded for said timestamp +// - Some restores should fail because the timestamp exceeds the last binlog +// - Do so for all recorded tiemstamps. +// - Then, a 2nd round where some backups are purged -- this tests to see that we're still able to find a restore path +// (of course we only delete backups that still leave us with valid restore paths). +// +// All of the above is done for BuiltinBackup, XtraBackup, Mysqlctld (which is technically builtin) +func TestIncrementalBackupAndRestoreToTimestamp(t *testing.T) { + tcase := &backup.PITRTestCase{ + Name: "MySQLShell", + SetupType: backup.XtraBackup, + ComprssDetails: &backup.CompressionDetails{ + CompressorEngineName: "pgzip", + }, + } + backup.ExecTestIncrementalBackupAndRestoreToTimestamp(t, tcase) +} diff --git a/go/test/endtoend/backup/vtctlbackup/backup_utils.go b/go/test/endtoend/backup/vtctlbackup/backup_utils.go index 707c9010b0c..722c06937b2 100644 --- a/go/test/endtoend/backup/vtctlbackup/backup_utils.go +++ b/go/test/endtoend/backup/vtctlbackup/backup_utils.go @@ -54,19 +54,21 @@ const ( XtraBackup = iota BuiltinBackup Mysqlctld + MySQLShell timeout = time.Duration(60 * time.Second) topoConsistencyTimeout = 20 * time.Second ) var ( - primary *cluster.Vttablet - replica1 *cluster.Vttablet - replica2 *cluster.Vttablet - replica3 *cluster.Vttablet - localCluster *cluster.LocalProcessCluster - newInitDBFile string - useXtrabackup bool - cell = cluster.DefaultCell + primary *cluster.Vttablet + replica1 *cluster.Vttablet + replica2 *cluster.Vttablet + replica3 *cluster.Vttablet + localCluster *cluster.LocalProcessCluster + newInitDBFile string + useXtrabackup bool + currentSetupType int + cell = cluster.DefaultCell hostname = "localhost" keyspaceName = "ks" @@ -103,6 +105,7 @@ type CompressionDetails struct { // LaunchCluster : starts the cluster as per given params. func LaunchCluster(setupType int, streamMode string, stripes int, cDetails *CompressionDetails) (int, error) { + currentSetupType = setupType localCluster = cluster.NewCluster(cell, hostname) // Start topo server @@ -144,8 +147,9 @@ func LaunchCluster(setupType int, streamMode string, stripes int, cDetails *Comp extraArgs := []string{"--db-credentials-file", dbCredentialFile} commonTabletArg = append(commonTabletArg, "--db-credentials-file", dbCredentialFile) - // Update arguments for xtrabackup - if setupType == XtraBackup { + // Update arguments for different backup engines + switch setupType { + case XtraBackup: useXtrabackup = true xtrabackupArgs := []string{ @@ -162,6 +166,18 @@ func LaunchCluster(setupType int, streamMode string, stripes int, cDetails *Comp } commonTabletArg = append(commonTabletArg, xtrabackupArgs...) + case MySQLShell: + mysqlShellBackupLocation := path.Join(localCluster.CurrentVTDATAROOT, "backups-mysqlshell") + err = os.MkdirAll(mysqlShellBackupLocation, 0o777) + if err != nil { + return 0, err + } + + mysqlShellArgs := []string{ + "--backup_engine_implementation", "mysqlshell", + "--mysql-shell-backup-location", mysqlShellBackupLocation, + } + commonTabletArg = append(commonTabletArg, mysqlShellArgs...) } commonTabletArg = append(commonTabletArg, getCompressorArgs(cDetails)...) @@ -178,9 +194,19 @@ func LaunchCluster(setupType int, streamMode string, stripes int, cDetails *Comp tablet := localCluster.NewVttabletInstance(tabletType, 0, cell) tablet.VttabletProcess = localCluster.VtprocessInstanceFromVttablet(tablet, shard.Name, keyspaceName) tablet.VttabletProcess.DbPassword = dbPassword - tablet.VttabletProcess.ExtraArgs = commonTabletArg tablet.VttabletProcess.SupportsBackup = true + // since we spin different mysqld processes, we need to pass exactly the socket of the this particular + // one when running mysql shell dump/loads + if setupType == MySQLShell { + commonTabletArg = append(commonTabletArg, + "--mysql-shell-flags", fmt.Sprintf("--js -u vt_dba -p%s -S %s", dbPassword, + path.Join(os.Getenv("VTDATAROOT"), fmt.Sprintf("/vt_%010d", tablet.TabletUID), "mysql.sock"), + ), + ) + } + tablet.VttabletProcess.ExtraArgs = commonTabletArg + if setupType == Mysqlctld { mysqlctldProcess, err := cluster.MysqlCtldProcessInstance(tablet.TabletUID, tablet.MySQLPort, localCluster.TmpDirectory) if err != nil { @@ -1345,9 +1371,10 @@ func TestReplicaRestoreToTimestamp(t *testing.T, restoreToTimestamp time.Time, e } func verifyTabletBackupStats(t *testing.T, vars map[string]any) { - // Currently only the builtin backup engine instruments bytes-processed - // counts. - if !useXtrabackup { + switch currentSetupType { + // Currently only the builtin backup engine instruments bytes-processed counts. + case XtraBackup, MySQLShell: + default: require.Contains(t, vars, "BackupBytes") bb := vars["BackupBytes"].(map[string]any) require.Contains(t, bb, "BackupEngine.Builtin.Compressor:Write") @@ -1361,8 +1388,11 @@ func verifyTabletBackupStats(t *testing.T, vars map[string]any) { require.Contains(t, vars, "BackupCount") bc := vars["BackupCount"].(map[string]any) require.Contains(t, bc, "-.-.Backup") - // Currently only the builtin backup engine implements operation counts. - if !useXtrabackup { + + switch currentSetupType { + // Currently only the builtin backup engine instruments bytes-processed counts. + case XtraBackup, MySQLShell: + default: require.Contains(t, bc, "BackupEngine.Builtin.Compressor:Close") require.Contains(t, bc, "BackupEngine.Builtin.Destination:Close") require.Contains(t, bc, "BackupEngine.Builtin.Destination:Open") @@ -1373,8 +1403,11 @@ func verifyTabletBackupStats(t *testing.T, vars map[string]any) { require.Contains(t, vars, "BackupDurationNanoseconds") bd := vars["BackupDurationNanoseconds"] require.Contains(t, bd, "-.-.Backup") + + switch currentSetupType { // Currently only the builtin backup engine emits timings. - if !useXtrabackup { + case XtraBackup, MySQLShell: + default: require.Contains(t, bd, "BackupEngine.Builtin.Compressor:Close") require.Contains(t, bd, "BackupEngine.Builtin.Compressor:Write") require.Contains(t, bd, "BackupEngine.Builtin.Destination:Close") @@ -1384,6 +1417,7 @@ func verifyTabletBackupStats(t *testing.T, vars map[string]any) { require.Contains(t, bd, "BackupEngine.Builtin.Source:Open") require.Contains(t, bd, "BackupEngine.Builtin.Source:Read") } + if backupstorage.BackupStorageImplementation == "file" { require.Contains(t, bd, "BackupStorage.File.File:Write") } @@ -1408,7 +1442,9 @@ func verifyTabletRestoreStats(t *testing.T, vars map[string]any) { verifyRestorePositionAndTimeStats(t, vars) - if !useXtrabackup { + switch currentSetupType { + case XtraBackup, MySQLShell: + default: require.Contains(t, vars, "RestoreBytes") bb := vars["RestoreBytes"].(map[string]any) require.Contains(t, bb, "BackupEngine.Builtin.Decompressor:Read") @@ -1420,8 +1456,11 @@ func verifyTabletRestoreStats(t *testing.T, vars map[string]any) { require.Contains(t, vars, "RestoreCount") bc := vars["RestoreCount"].(map[string]any) require.Contains(t, bc, "-.-.Restore") + + switch currentSetupType { // Currently only the builtin backup engine emits operation counts. - if !useXtrabackup { + case XtraBackup, MySQLShell: + default: require.Contains(t, bc, "BackupEngine.Builtin.Decompressor:Close") require.Contains(t, bc, "BackupEngine.Builtin.Destination:Close") require.Contains(t, bc, "BackupEngine.Builtin.Destination:Open") @@ -1432,8 +1471,11 @@ func verifyTabletRestoreStats(t *testing.T, vars map[string]any) { require.Contains(t, vars, "RestoreDurationNanoseconds") bd := vars["RestoreDurationNanoseconds"] require.Contains(t, bd, "-.-.Restore") + + switch currentSetupType { // Currently only the builtin backup engine emits timings. - if !useXtrabackup { + case XtraBackup, MySQLShell: + default: require.Contains(t, bd, "BackupEngine.Builtin.Decompressor:Close") require.Contains(t, bd, "BackupEngine.Builtin.Decompressor:Read") require.Contains(t, bd, "BackupEngine.Builtin.Destination:Close") @@ -1443,5 +1485,6 @@ func verifyTabletRestoreStats(t *testing.T, vars map[string]any) { require.Contains(t, bd, "BackupEngine.Builtin.Source:Open") require.Contains(t, bd, "BackupEngine.Builtin.Source:Read") } + require.Contains(t, bd, "BackupStorage.File.File:Read") } diff --git a/go/vt/mysqlctl/fakemysqldaemon.go b/go/vt/mysqlctl/fakemysqldaemon.go index 317aed4f578..aaed3a1e51c 100644 --- a/go/vt/mysqlctl/fakemysqldaemon.go +++ b/go/vt/mysqlctl/fakemysqldaemon.go @@ -18,6 +18,7 @@ package mysqlctl import ( "context" + "errors" "fmt" "reflect" "regexp" @@ -544,6 +545,11 @@ func (fmd *FakeMysqlDaemon) Promote(ctx context.Context, hookExtraEnv map[string return fmd.PromoteResult, nil } +// ExecuteSuperQuery is part of the MysqlDaemon interface +func (fmd *FakeMysqlDaemon) ExecuteSuperQuery(ctx context.Context, query string) error { + return fmd.ExecuteSuperQueryList(ctx, []string{query}) +} + // ExecuteSuperQueryList is part of the MysqlDaemon interface func (fmd *FakeMysqlDaemon) ExecuteSuperQueryList(ctx context.Context, queryList []string) error { for _, query := range queryList { @@ -736,3 +742,13 @@ func (fmd *FakeMysqlDaemon) GetVersionString(ctx context.Context) (string, error func (fmd *FakeMysqlDaemon) GetVersionComment(ctx context.Context) (string, error) { return "", nil } + +// AcquireGlobalReadLock is part of the MysqlDaemon interface. +func (fmd *FakeMysqlDaemon) AcquireGlobalReadLock(ctx context.Context) error { + return errors.New("not implemented") +} + +// ReleaseGlobalReadLock is part of the MysqlDaemon interface. +func (fmd *FakeMysqlDaemon) ReleaseGlobalReadLock(ctx context.Context) error { + return errors.New("not implemented") +} diff --git a/go/vt/mysqlctl/mysql_daemon.go b/go/vt/mysqlctl/mysql_daemon.go index 6c7df04786c..051a8660025 100644 --- a/go/vt/mysqlctl/mysql_daemon.go +++ b/go/vt/mysqlctl/mysql_daemon.go @@ -119,12 +119,22 @@ type MysqlDaemon interface { // GetVersionComment returns the version comment GetVersionComment(ctx context.Context) (string, error) + // ExecuteSuperQuery executes a single query, no result + ExecuteSuperQuery(ctx context.Context, query string) error + // ExecuteSuperQueryList executes a list of queries, no result ExecuteSuperQueryList(ctx context.Context, queryList []string) error // FetchSuperQuery executes one query, returns the result FetchSuperQuery(ctx context.Context, query string) (*sqltypes.Result, error) + // AcquireGlobalReadLock acquires a global read lock and keeps the connection so + // as to release it with the function below. + AcquireGlobalReadLock(ctx context.Context) error + + // ReleaseGlobalReadLock release a lock acquired with the connection from the above function. + ReleaseGlobalReadLock(ctx context.Context) error + // Close will close this instance of Mysqld. It will wait for all dba // queries to be finished. Close() diff --git a/go/vt/mysqlctl/mysqld.go b/go/vt/mysqlctl/mysqld.go index 952c0987c82..a228459c49d 100644 --- a/go/vt/mysqlctl/mysqld.go +++ b/go/vt/mysqlctl/mysqld.go @@ -108,9 +108,10 @@ var ( // Mysqld is the object that represents a mysqld daemon running on this server. type Mysqld struct { - dbcfgs *dbconfigs.DBConfigs - dbaPool *dbconnpool.ConnectionPool - appPool *dbconnpool.ConnectionPool + dbcfgs *dbconfigs.DBConfigs + dbaPool *dbconnpool.ConnectionPool + appPool *dbconnpool.ConnectionPool + lockConn *dbconnpool.PooledDBConnection capabilities capabilitySet diff --git a/go/vt/mysqlctl/mysqlshellbackupengine.go b/go/vt/mysqlctl/mysqlshellbackupengine.go index 7922b51eb1e..624aba138aa 100644 --- a/go/vt/mysqlctl/mysqlshellbackupengine.go +++ b/go/vt/mysqlctl/mysqlshellbackupengine.go @@ -1,5 +1,5 @@ /* -Copyright 2019 The Vitess Authors. +Copyright 2024 The Vitess Authors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -17,10 +17,13 @@ limitations under the License. package mysqlctl import ( + "bufio" "context" "encoding/json" "errors" "fmt" + "io" + "os" "os/exec" "path" "strings" @@ -29,9 +32,9 @@ import ( "github.com/spf13/pflag" + "vitess.io/vitess/go/constants/sidecar" "vitess.io/vitess/go/mysql" "vitess.io/vitess/go/mysql/capabilities" - "vitess.io/vitess/go/mysql/replication" "vitess.io/vitess/go/vt/mysqlctl/backupstorage" tabletmanagerdatapb "vitess.io/vitess/go/vt/proto/tabletmanagerdata" "vitess.io/vitess/go/vt/servenv" @@ -52,6 +55,9 @@ var ( // disable redo logging and double write buffer mysqlShellSpeedUpRestore = false + // use when checking if we need to create the directory on the local filesystem or not. + knownObjectStoreParams = []string{"s3BucketName", "osBucketName", "azureContainerName"} + MySQLShellPreCheckError = errors.New("MySQLShellPreCheckError") ) @@ -96,26 +102,51 @@ const ( func (be *MySQLShellBackupEngine) ExecuteBackup(ctx context.Context, params BackupParams, bh backupstorage.BackupHandle) (result BackupResult, finalErr error) { params.Logger.Infof("Starting ExecuteBackup in %s", params.TabletAlias) - err := be.backupPreCheck() + location := path.Join(mysqlShellBackupLocation, bh.Directory(), bh.Name()) + + err := be.backupPreCheck(location) if err != nil { return BackupUnusable, vterrors.Wrap(err, "failed backup precheck") } - start := time.Now().UTC() - location := path.Join(mysqlShellBackupLocation, params.Keyspace, params.Shard, start.Format(BackupTimestampFormat)) + serverUUID, err := params.Mysqld.GetServerUUID(ctx) + if err != nil { + return BackupUnusable, vterrors.Wrap(err, "can't get server uuid") + } - args := []string{} + mysqlVersion, err := params.Mysqld.GetVersionString(ctx) + if err != nil { + return BackupUnusable, vterrors.Wrap(err, "can't get MySQL version") + } + args := []string{} if mysqlShellFlags != "" { args = append(args, strings.Fields(mysqlShellFlags)...) } - args = append(args, "-e", fmt.Sprintf("util.dumpSchemas([\"vt_%s\"], %q, %s)", + args = append(args, "-e", fmt.Sprintf("util.dumpSchemas([\"%s\", \"vt_%s\"], %q, %s)", + sidecar.GetName(), params.Keyspace, location, mysqlShellDumpFlags, )) + // to be able to get the consistent GTID sets, we will acquire a global read lock before starting mysql shell. + // oncce we have the lock, we start it and wait unti it has acquired and release its global read lock, which + // should guarantee that both use and mysql shell are seeing the same executed GTID sets. + // after this we release the lock so that replication can continue. this usually should take just a few seconds. + params.Logger.Infof("acquiring a global read lock before fetching the executed GTID sets") + err = params.Mysqld.AcquireGlobalReadLock(ctx) + if err != nil { + return BackupUnusable, vterrors.Wrap(err, "failed to acquire read lock to start backup") + } + lockAcquired := time.Now() // we will report how long we hold the lock for + + posBeforeBackup, err := params.Mysqld.PrimaryPosition(ctx) + if err != nil { + return BackupUnusable, vterrors.Wrap(err, "failed to fetch position") + } + cmd := exec.CommandContext(ctx, mysqlShellBackupBinaryName, args...) params.Logger.Infof("running %s", cmd.String()) @@ -124,25 +155,32 @@ func (be *MySQLShellBackupEngine) ExecuteBackup(ctx context.Context, params Back if err != nil { return BackupUnusable, vterrors.Wrap(err, "cannot create stdout pipe") } - cmdErr, err := cmd.StderrPipe() + cmdOriginalErr, err := cmd.StderrPipe() if err != nil { return BackupUnusable, vterrors.Wrap(err, "cannot create stderr pipe") } if err := cmd.Start(); err != nil { - return BackupUnusable, vterrors.Wrap(err, "can't start xbstream") + return BackupUnusable, vterrors.Wrap(err, "can't start mysqlshell") } + pipeReader, pipeWriter := io.Pipe() + cmdErr := io.TeeReader(cmdOriginalErr, pipeWriter) + cmdWg := &sync.WaitGroup{} - cmdWg.Add(2) + cmdWg.Add(3) + go releaseReadLock(ctx, pipeReader, params, cmdWg, lockAcquired) go scanLinesToLogger(mysqlShellBackupEngineName+" stdout", cmdOut, params.Logger, cmdWg.Done) go scanLinesToLogger(mysqlShellBackupEngineName+" stderr", cmdErr, params.Logger, cmdWg.Done) - cmdWg.Wait() // Get exit status. if err := cmd.Wait(); err != nil { return BackupUnusable, vterrors.Wrap(err, mysqlShellBackupEngineName+" failed") } + // close the pipeWriter and wait for the goroutines to have read all the logs + pipeWriter.Close() + cmdWg.Wait() + // open the MANIFEST params.Logger.Infof("Writing backup MANIFEST") mwc, err := bh.AddFile(ctx, backupManifestFileName, backupstorage.FileSizeUnknown) @@ -159,9 +197,16 @@ func (be *MySQLShellBackupEngine) ExecuteBackup(ctx context.Context, params Back // the position is empty here because we have no way of capturing it from mysqlsh // we will capture it when doing the restore as mysqlsh can replace the GTIDs with // what it has stored in the backup. - Position: replication.Position{}, - BackupTime: start.Format(time.RFC3339), - FinishedTime: time.Now().UTC().Format(time.RFC3339), + Position: posBeforeBackup, + PurgedPosition: posBeforeBackup, + BackupTime: FormatRFC3339(params.BackupTime.UTC()), + FinishedTime: FormatRFC3339(time.Now().UTC()), + ServerUUID: serverUUID, + TabletAlias: params.TabletAlias, + Keyspace: params.Keyspace, + Shard: params.Shard, + MySQLVersion: mysqlVersion, + UpgradeSafe: true, }, // mysql shell backup specific fields @@ -210,8 +255,28 @@ func (be *MySQLShellBackupEngine) ExecuteRestore(ctx context.Context, params Res if params.DeleteBeforeRestore { params.Logger.Infof("restoring on an existing tablet, so dropping database %q", params.DbName) + readonly, err := params.Mysqld.IsSuperReadOnly(ctx) + if err != nil { + return nil, vterrors.Wrap(err, fmt.Sprintf("checking if mysqld has super_read_only=enable: %v", err)) + } + + if readonly { + resetFunc, err := params.Mysqld.SetSuperReadOnly(ctx, false) + if err != nil { + return nil, vterrors.Wrap(err, fmt.Sprintf("unable to disable super-read-only: %v", err)) + } + + defer func() { + err := resetFunc() + if err != nil { + params.Logger.Errorf("Not able to set super_read_only to its original value after restore") + } + }() + } + err = params.Mysqld.ExecuteSuperQueryList(ctx, - []string{fmt.Sprintf("DROP DATABASE IF EXISTS `%s`", params.DbName)}, + []string{fmt.Sprintf("DROP DATABASE IF EXISTS `%s`", params.DbName), + fmt.Sprintf("DROP DATABASE IF EXISTS `%s`", sidecar.GetName())}, ) if err != nil { return nil, vterrors.Wrap(err, fmt.Sprintf("dropping database %q failed", params.DbName)) @@ -226,21 +291,21 @@ func (be *MySQLShellBackupEngine) ExecuteRestore(ctx context.Context, params Res } // this is required so we can load the backup generated by MySQL Shell. we will disable it afterwards. - err = params.Mysqld.ExecuteSuperQueryList(ctx, []string{"SET GLOBAL LOCAL_INFILE=1"}) + err = params.Mysqld.ExecuteSuperQuery(ctx, "SET GLOBAL LOCAL_INFILE=1") if err != nil { return nil, vterrors.Wrap(err, "unable to set local_infile=1") } if mysqlShellSpeedUpRestore { // disable redo logging and double write buffer if we are configured to do so. - err = params.Mysqld.ExecuteSuperQueryList(ctx, []string{"ALTER INSTANCE DISABLE INNODB REDO_LOG"}) + err = params.Mysqld.ExecuteSuperQuery(ctx, "ALTER INSTANCE DISABLE INNODB REDO_LOG") if err != nil { return nil, vterrors.Wrap(err, "unable to disable REDO_LOG") } params.Logger.Infof("Disabled REDO_LOG") defer func() { // re-enable once we are done with the restore. - err := params.Mysqld.ExecuteSuperQueryList(ctx, []string{"ALTER INSTANCE ENABLE INNODB REDO_LOG"}) + err := params.Mysqld.ExecuteSuperQuery(ctx, "ALTER INSTANCE ENABLE INNODB REDO_LOG") if err != nil { params.Logger.Errorf("unable to re-enable REDO_LOG: %v", err) } else { @@ -249,6 +314,14 @@ func (be *MySQLShellBackupEngine) ExecuteRestore(ctx context.Context, params Res }() } + // we need to disable SuperReadOnly otherwise we won't be able to restore the backup properly. + // once the backups is complete, we will restore it to its previous state. + resetFunc, err := be.handleSuperReadOnly(ctx, params) + if err != nil { + return nil, vterrors.Wrap(err, "unable to disable super-read-only") + } + defer resetFunc() + args := []string{} if mysqlShellFlags != "" { @@ -289,25 +362,12 @@ func (be *MySQLShellBackupEngine) ExecuteRestore(ctx context.Context, params Res params.Logger.Infof("%s completed successfully", mysqlShellBackupBinaryName) // disable local_infile now that the restore is done. - err = params.Mysqld.ExecuteSuperQueryList(ctx, []string{ - "SET GLOBAL LOCAL_INFILE=0", - }) + err = params.Mysqld.ExecuteSuperQuery(ctx, "SET GLOBAL LOCAL_INFILE=0") if err != nil { return nil, vterrors.Wrap(err, "unable to set local_infile=0") } params.Logger.Infof("set local_infile=0") - // since MySQL Shell backups do not store the Executed GTID position in the manifest, we need to - // fetch it and override in the manifest we are returning so Vitess can set it again. alternatively - // we could have a flag in the future where the backup engine controls if it needs to be set or not. - pos, err := params.Mysqld.PrimaryPosition(ctx) - if err != nil { - return nil, vterrors.Wrap(err, "failure getting restored position") - } - - params.Logger.Infof("retrieved primary position after restore") - bm.BackupManifest.Position = pos - params.Logger.Infof("Restore completed") return &bm.BackupManifest, nil @@ -319,7 +379,7 @@ func (be *MySQLShellBackupEngine) ShouldDrainForBackup(req *tabletmanagerdatapb. return mysqlShellBackupShouldDrain } -func (be *MySQLShellBackupEngine) backupPreCheck() error { +func (be *MySQLShellBackupEngine) backupPreCheck(location string) error { if mysqlShellBackupLocation == "" { return fmt.Errorf("%w: no backup location set via --mysql_shell_location", MySQLShellPreCheckError) } @@ -328,6 +388,23 @@ func (be *MySQLShellBackupEngine) backupPreCheck() error { return fmt.Errorf("%w: at least the --js flag is required", MySQLShellPreCheckError) } + // make sure the targe directory exists if the target location for the backup is not an object store + // (e.g. is the local filesystem) as MySQL Shell doesn't create the entire path beforehand: + isObjectStorage := false + for _, objStore := range knownObjectStoreParams { + if strings.Contains(mysqlShellDumpFlags, objStore) { + isObjectStorage = true + break + } + } + + if !isObjectStorage { + err := os.MkdirAll(location, 0o750) + if err != nil { + return fmt.Errorf("failure creating directory %s: %w", location, err) + } + } + return nil } @@ -350,6 +427,10 @@ func (be *MySQLShellBackupEngine) restorePreCheck(ctx context.Context, params Re return fmt.Errorf("%w: \"progressFile\" needs to be empty as vitess always starts a restore from scratch", MySQLShellPreCheckError) } + if val, ok := loadFlags["skipBinlog"]; !ok || val != true { + return fmt.Errorf("%w: \"skipBinlog\" needs to set to true", MySQLShellPreCheckError) + } + if mysqlShellSpeedUpRestore { version, err := params.Mysqld.GetVersionString(ctx) if err != nil { @@ -370,6 +451,62 @@ func (be *MySQLShellBackupEngine) restorePreCheck(ctx context.Context, params Re return nil } +func (be *MySQLShellBackupEngine) handleSuperReadOnly(ctx context.Context, params RestoreParams) (func(), error) { + readonly, err := params.Mysqld.IsSuperReadOnly(ctx) + if err != nil { + return nil, vterrors.Wrap(err, fmt.Sprintf("checking if mysqld has super_read_only=enable: %v", err)) + } + + params.Logger.Infof("Is Super Read Only: %v", readonly) + + if readonly { + resetFunc, err := params.Mysqld.SetSuperReadOnly(ctx, false) + if err != nil { + return nil, vterrors.Wrap(err, fmt.Sprintf("unable to disable super-read-only: %v", err)) + } + + return func() { + err := resetFunc() + if err != nil { + params.Logger.Errorf("Not able to set super_read_only to its original value after restore") + } + }, nil + } + + return func() {}, nil +} + +// releaseReadLock will keep reading the MySQL Shell STDERR waiting until the point it has acquired its lock +func releaseReadLock(ctx context.Context, reader io.Reader, params BackupParams, wg *sync.WaitGroup, lockAcquired time.Time) { + defer wg.Done() + + scanner := bufio.NewScanner(reader) + released := false + for scanner.Scan() { + line := scanner.Text() + + if !released { + + if !strings.Contains(line, "Global read lock has been released") { + continue + } + released = true + + params.Logger.Infof("mysql shell released its global read lock, doing the same") + + err := params.Mysqld.ReleaseGlobalReadLock(ctx) + if err != nil { + params.Logger.Errorf("unable to release global read lock: %v", err) + } + + params.Logger.Infof("global read lock released after %v", time.Since(lockAcquired)) + } + } + if err := scanner.Err(); err != nil { + params.Logger.Errorf("error reading from reader: %v", err) + } +} + func init() { BackupRestoreEngineMap[mysqlShellBackupEngineName] = &MySQLShellBackupEngine{} } diff --git a/go/vt/mysqlctl/mysqlshellbackupengine_test.go b/go/vt/mysqlctl/mysqlshellbackupengine_test.go index 9d5ccc087e1..fdf1e07af0d 100644 --- a/go/vt/mysqlctl/mysqlshellbackupengine_test.go +++ b/go/vt/mysqlctl/mysqlshellbackupengine_test.go @@ -1,5 +1,5 @@ /* -Copyright 2019 The Vitess Authors. +Copyright 2024 The Vitess Authors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -18,6 +18,7 @@ package mysqlctl import ( "context" + "path" "testing" "github.com/stretchr/testify/assert" @@ -68,7 +69,7 @@ func TestMySQLShellBackupBackupPreCheck(t *testing.T) { }, { "supported values", - "/tmp/backup/", + t.TempDir(), "--js -h localhost", nil, }, @@ -79,7 +80,7 @@ func TestMySQLShellBackupBackupPreCheck(t *testing.T) { mysqlShellBackupLocation = tt.location mysqlShellFlags = tt.flags - assert.ErrorIs(t, engine.backupPreCheck(), tt.err) + assert.ErrorIs(t, engine.backupPreCheck(path.Join(mysqlShellBackupLocation, "test")), tt.err) }) } diff --git a/go/vt/mysqlctl/query.go b/go/vt/mysqlctl/query.go index 7a1816e3cfb..7b1e1f0094b 100644 --- a/go/vt/mysqlctl/query.go +++ b/go/vt/mysqlctl/query.go @@ -18,6 +18,7 @@ package mysqlctl import ( "context" + "errors" "fmt" "strings" "time" @@ -226,6 +227,42 @@ func (mysqld *Mysqld) fetchStatuses(ctx context.Context, pattern string) (map[st return varMap, nil } +// ExecuteSuperQuery allows the user to execute a query as a super user. +func (mysqld *Mysqld) AcquireGlobalReadLock(ctx context.Context) error { + if mysqld.lockConn != nil { + return errors.New("lock already acquired") + } + + conn, err := getPoolReconnect(ctx, mysqld.dbaPool) + if err != nil { + return err + } + + err = mysqld.executeSuperQueryListConn(ctx, conn, []string{"FLUSH TABLES WITH READ LOCK"}) + if err != nil { + conn.Recycle() + return err + } + + mysqld.lockConn = conn + return nil +} + +func (mysqld *Mysqld) ReleaseGlobalReadLock(ctx context.Context) error { + if mysqld.lockConn == nil { + return errors.New("no read locks acquired yet") + } + + err := mysqld.executeSuperQueryListConn(ctx, mysqld.lockConn, []string{"UNLOCK TABLES"}) + if err != nil { + return err + } + + mysqld.lockConn.Recycle() + mysqld.lockConn = nil + return nil +} + const ( sourcePasswordStart = " SOURCE_PASSWORD = '" sourcePasswordEnd = "',\n" diff --git a/test/config.json b/test/config.json index 1cdf92127ef..644cac597a1 100644 --- a/test/config.json +++ b/test/config.json @@ -109,6 +109,15 @@ "RetryMax": 1, "Tags": [] }, + "backup_pitr_mysqlshell": { + "File": "unused.go", + "Args": ["vitess.io/vitess/go/test/endtoend/backup/pitr_mysqlshell", "-timeout", "30m"], + "Command": [], + "Manual": false, + "Shard": "backup_pitr_mysqlshell", + "RetryMax": 1, + "Tags": [] + }, "backup": { "File": "unused.go", "Args": ["vitess.io/vitess/go/test/endtoend/backup/vtctlbackup", "-timeout", "30m"], From 0a5fda445041c184d93231f69f6dc701c29e43fc Mon Sep 17 00:00:00 2001 From: Renan Rangel Date: Wed, 7 Aug 2024 15:24:21 -0700 Subject: [PATCH 08/34] fix unit test Signed-off-by: Renan Rangel --- go/vt/mysqlctl/mysqlshellbackupengine_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/go/vt/mysqlctl/mysqlshellbackupengine_test.go b/go/vt/mysqlctl/mysqlshellbackupengine_test.go index fdf1e07af0d..cf138429d5e 100644 --- a/go/vt/mysqlctl/mysqlshellbackupengine_test.go +++ b/go/vt/mysqlctl/mysqlshellbackupengine_test.go @@ -118,7 +118,7 @@ func TestMySQLShellBackupRestorePreCheck(t *testing.T) { }, { "supported values", - `{"updateGtidSet": "replace", "progressFile": ""}`, + `{"updateGtidSet": "replace", "progressFile": "", "skipBinlog": true}`, nil, }, } From 7116317f661b2d71ea332c6c4f3825d2c2457436 Mon Sep 17 00:00:00 2001 From: Renan Rangel Date: Tue, 13 Aug 2024 06:34:00 -0700 Subject: [PATCH 09/34] fix tests Signed-off-by: Renan Rangel --- go/vt/mysqlctl/mysqlshellbackupengine_test.go | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/go/vt/mysqlctl/mysqlshellbackupengine_test.go b/go/vt/mysqlctl/mysqlshellbackupengine_test.go index cf138429d5e..bb5aec65a4c 100644 --- a/go/vt/mysqlctl/mysqlshellbackupengine_test.go +++ b/go/vt/mysqlctl/mysqlshellbackupengine_test.go @@ -139,7 +139,11 @@ func TestMySQLShellBackupRestorePreCheckDisableRedolog(t *testing.T) { mysqlShellSpeedUpRestore = true engine := MySQLShellBackupEngine{} - fakeMysqld := NewFakeMysqlDaemon(fakesqldb.New(t)) // defaults to 8.0.32 + fakedb := fakesqldb.New(t) + defer fakedb.Close() + fakeMysqld := NewFakeMysqlDaemon(fakedb) // defaults to 8.0.32 + defer fakeMysqld.Close() + params := RestoreParams{ Mysqld: fakeMysqld, } From c2d41acc97e6aabba902ff5a7f5b2300612b7b7c Mon Sep 17 00:00:00 2001 From: Renan Rangel Date: Wed, 21 Aug 2024 02:28:48 -0700 Subject: [PATCH 10/34] PR feedback Signed-off-by: Renan Rangel --- .../cluster_endtoend_pitr_mysqlshell.yml | 157 ++++++++++++++++++ .../backup_pitr_mysqlshell_test.go | 2 +- .../backup/vtctlbackup/backup_utils.go | 15 +- go/vt/mysqlctl/backup_test.go | 2 +- test/ci_workflow_gen.go | 1 + 5 files changed, 162 insertions(+), 15 deletions(-) create mode 100644 .github/workflows/cluster_endtoend_pitr_mysqlshell.yml diff --git a/.github/workflows/cluster_endtoend_pitr_mysqlshell.yml b/.github/workflows/cluster_endtoend_pitr_mysqlshell.yml new file mode 100644 index 00000000000..4f39e7b0b4b --- /dev/null +++ b/.github/workflows/cluster_endtoend_pitr_mysqlshell.yml @@ -0,0 +1,157 @@ +# DO NOT MODIFY: THIS FILE IS GENERATED USING "make generate_ci_workflows" + +name: Cluster (pitr_mysqlshell) +on: [push, pull_request] +concurrency: + group: format('{0}-{1}', ${{ github.ref }}, 'Cluster (pitr_mysqlshell)') + cancel-in-progress: true + +permissions: read-all + +env: + LAUNCHABLE_ORGANIZATION: "vitess" + LAUNCHABLE_WORKSPACE: "vitess-app" + GITHUB_PR_HEAD_SHA: "${{ github.event.pull_request.head.sha }}" + +jobs: + build: + name: Run endtoend tests on Cluster (pitr_mysqlshell) + runs-on: gh-hosted-runners-4cores-1 + + steps: + - name: Skip CI + run: | + if [[ "${{contains( github.event.pull_request.labels.*.name, 'Skip CI')}}" == "true" ]]; then + echo "skipping CI due to the 'Skip CI' label" + exit 1 + fi + + - name: Check if workflow needs to be skipped + id: skip-workflow + run: | + skip='false' + if [[ "${{github.event.pull_request}}" == "" ]] && [[ "${{github.ref}}" != "refs/heads/main" ]] && [[ ! "${{github.ref}}" =~ ^refs/heads/release-[0-9]+\.[0-9]$ ]] && [[ ! "${{github.ref}}" =~ "refs/tags/.*" ]]; then + skip='true' + fi + echo Skip ${skip} + echo "skip-workflow=${skip}" >> $GITHUB_OUTPUT + + PR_DATA=$(curl -s\ + -H "Authorization: token ${{ secrets.GITHUB_TOKEN }}" \ + -H "Accept: application/vnd.github.v3+json" \ + "https://api.github.com/repos/${{ github.repository }}/pulls/${{ github.event.pull_request.number }}") + draft=$(echo "$PR_DATA" | jq .draft -r) + echo "is_draft=${draft}" >> $GITHUB_OUTPUT + + - name: Check out code + if: steps.skip-workflow.outputs.skip-workflow == 'false' + uses: actions/checkout@v4 + + - name: Check for changes in relevant files + if: steps.skip-workflow.outputs.skip-workflow == 'false' + uses: dorny/paths-filter@v3.0.1 + id: changes + with: + token: '' + filters: | + end_to_end: + - 'go/**/*.go' + - 'go/vt/sidecardb/**/*.sql' + - 'go/test/endtoend/onlineddl/vrepl_suite/**' + - 'test.go' + - 'Makefile' + - 'build.env' + - 'go.sum' + - 'go.mod' + - 'proto/*.proto' + - 'tools/**' + - 'config/**' + - 'bootstrap.sh' + - '.github/workflows/cluster_endtoend_pitr_mysqlshell.yml' + + - name: Set up Go + if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' + uses: actions/setup-go@v5 + with: + go-version: 1.22.5 + + - name: Set up python + if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' + uses: actions/setup-python@v5 + + - name: Tune the OS + if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' + run: | + # Limit local port range to not use ports that overlap with server side + # ports that we listen on. + sudo sysctl -w net.ipv4.ip_local_port_range="22768 65535" + # Increase the asynchronous non-blocking I/O. More information at https://dev.mysql.com/doc/refman/5.7/en/innodb-parameters.html#sysvar_innodb_use_native_aio + echo "fs.aio-max-nr = 1048576" | sudo tee -a /etc/sysctl.conf + sudo sysctl -p /etc/sysctl.conf + + - name: Get dependencies + if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' + run: | + + # Get key to latest MySQL repo + sudo apt-key adv --keyserver keyserver.ubuntu.com --recv-keys A8D3785C + # Setup MySQL 8.0 + wget -c https://dev.mysql.com/get/mysql-apt-config_0.8.32-1_all.deb + echo mysql-apt-config mysql-apt-config/select-server select mysql-8.0 | sudo debconf-set-selections + sudo DEBIAN_FRONTEND="noninteractive" dpkg -i mysql-apt-config* + sudo apt-get -qq update + # Install everything else we need, and configure + sudo apt-get -qq install -y mysql-server mysql-client make unzip g++ etcd curl git wget eatmydata xz-utils libncurses5 + + sudo service mysql stop + sudo service etcd stop + sudo ln -s /etc/apparmor.d/usr.sbin.mysqld /etc/apparmor.d/disable/ + sudo apparmor_parser -R /etc/apparmor.d/usr.sbin.mysqld + go mod download + + # install JUnit report formatter + go install github.com/vitessio/go-junit-report@HEAD + + - name: Setup launchable dependencies + if: steps.skip-workflow.outputs.is_draft == 'false' && steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' && github.base_ref == 'main' + run: | + # Get Launchable CLI installed. If you can, make it a part of the builder image to speed things up + pip3 install --user launchable~=1.0 > /dev/null + + # verify that launchable setup is all correct. + launchable verify || true + + # Tell Launchable about the build you are producing and testing + launchable record build --name "$GITHUB_RUN_ID" --no-commit-collection --source . + + - name: Run cluster endtoend test + if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' + timeout-minutes: 45 + run: | + # We set the VTDATAROOT to the /tmp folder to reduce the file path of mysql.sock file + # which musn't be more than 107 characters long. + export VTDATAROOT="/tmp/" + source build.env + + set -exo pipefail + + # run the tests however you normally do, then produce a JUnit XML file + eatmydata -- go run test.go -docker=false -follow -shard pitr_mysqlshell | tee -a output.txt | go-junit-report -set-exit-code > report.xml + + - name: Print test output and Record test result in launchable if PR is not a draft + if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' && always() + run: | + if [[ "${{steps.skip-workflow.outputs.is_draft}}" == "false" ]]; then + # send recorded tests to launchable + launchable record tests --build "$GITHUB_RUN_ID" go-test . || true + fi + + # print test output + cat output.txt + + - name: Test Summary + if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' && always() + uses: test-summary/action@v2 + with: + paths: "report.xml" + show: "fail, skip" diff --git a/go/test/endtoend/backup/pitr_mysqlshell/backup_pitr_mysqlshell_test.go b/go/test/endtoend/backup/pitr_mysqlshell/backup_pitr_mysqlshell_test.go index b323927483e..fe6fa0b7dcd 100644 --- a/go/test/endtoend/backup/pitr_mysqlshell/backup_pitr_mysqlshell_test.go +++ b/go/test/endtoend/backup/pitr_mysqlshell/backup_pitr_mysqlshell_test.go @@ -52,7 +52,7 @@ func TestIncrementalBackupAndRestoreToPos(t *testing.T) { func TestIncrementalBackupAndRestoreToTimestamp(t *testing.T) { tcase := &backup.PITRTestCase{ Name: "MySQLShell", - SetupType: backup.XtraBackup, + SetupType: backup.MySQLShell, ComprssDetails: &backup.CompressionDetails{ CompressorEngineName: "pgzip", }, diff --git a/go/test/endtoend/backup/vtctlbackup/backup_utils.go b/go/test/endtoend/backup/vtctlbackup/backup_utils.go index 0a938d57218..a278e25e2e6 100644 --- a/go/test/endtoend/backup/vtctlbackup/backup_utils.go +++ b/go/test/endtoend/backup/vtctlbackup/backup_utils.go @@ -66,7 +66,6 @@ var ( replica3 *cluster.Vttablet localCluster *cluster.LocalProcessCluster newInitDBFile string - useXtrabackup bool currentSetupType int cell = cluster.DefaultCell @@ -150,8 +149,6 @@ func LaunchCluster(setupType int, streamMode string, stripes int, cDetails *Comp // Update arguments for different backup engines switch setupType { case XtraBackup: - useXtrabackup = true - xtrabackupArgs := []string{ "--backup_engine_implementation", "xtrabackup", fmt.Sprintf("--xtrabackup_stream_mode=%s", streamMode), @@ -1059,12 +1056,8 @@ func verifySemiSyncStatus(t *testing.T, vttablet *cluster.Vttablet, expectedStat func terminateBackup(t *testing.T, alias string) { stopBackupMsg := "Completed backing up" - if useXtrabackup { + if currentSetupType == XtraBackup { stopBackupMsg = "Starting backup with" - useXtrabackup = false - defer func() { - useXtrabackup = true - }() } args := append([]string{"--server", localCluster.VtctldClientProcess.Server, "--alsologtostderr"}, "Backup", alias) @@ -1093,12 +1086,8 @@ func terminateBackup(t *testing.T, alias string) { func terminateRestore(t *testing.T) { stopRestoreMsg := "Copying file 10" - if useXtrabackup { + if currentSetupType == XtraBackup { stopRestoreMsg = "Restore: Preparing" - useXtrabackup = false - defer func() { - useXtrabackup = true - }() } args := append([]string{"--server", localCluster.VtctldClientProcess.Server, "--alsologtostderr"}, "RestoreFromBackup", primary.Alias) diff --git a/go/vt/mysqlctl/backup_test.go b/go/vt/mysqlctl/backup_test.go index e1ded4d1316..86b34fbbf99 100644 --- a/go/vt/mysqlctl/backup_test.go +++ b/go/vt/mysqlctl/backup_test.go @@ -714,7 +714,7 @@ func TestShouldRestore(t *testing.T) { assert.NoError(t, err) } -func Test_scanLinesToLogger(t *testing.T) { +func TestScanLinesToLogger(t *testing.T) { reader, writer := io.Pipe() logger := logutil.NewMemoryLogger() var wg sync.WaitGroup diff --git a/test/ci_workflow_gen.go b/test/ci_workflow_gen.go index 09e2fbb6900..af8b0a84d47 100644 --- a/test/ci_workflow_gen.go +++ b/test/ci_workflow_gen.go @@ -75,6 +75,7 @@ var ( "xb_backup", "backup_pitr", "backup_pitr_xtrabackup", + "backup_pitr_mysqlshell", "21", "mysql_server_vault", "vstream", From 8c6c665497553bb10406c6207964147c1f05890f Mon Sep 17 00:00:00 2001 From: Renan Rangel Date: Wed, 21 Aug 2024 05:24:50 -0700 Subject: [PATCH 11/34] regenerate ci workflows Signed-off-by: Renan Rangel --- ...yml => cluster_endtoend_backup_pitr_mysqlshell.yml} | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) rename .github/workflows/{cluster_endtoend_pitr_mysqlshell.yml => cluster_endtoend_backup_pitr_mysqlshell.yml} (94%) diff --git a/.github/workflows/cluster_endtoend_pitr_mysqlshell.yml b/.github/workflows/cluster_endtoend_backup_pitr_mysqlshell.yml similarity index 94% rename from .github/workflows/cluster_endtoend_pitr_mysqlshell.yml rename to .github/workflows/cluster_endtoend_backup_pitr_mysqlshell.yml index 4f39e7b0b4b..8095f1f8bbd 100644 --- a/.github/workflows/cluster_endtoend_pitr_mysqlshell.yml +++ b/.github/workflows/cluster_endtoend_backup_pitr_mysqlshell.yml @@ -1,9 +1,9 @@ # DO NOT MODIFY: THIS FILE IS GENERATED USING "make generate_ci_workflows" -name: Cluster (pitr_mysqlshell) +name: Cluster (backup_pitr_mysqlshell) on: [push, pull_request] concurrency: - group: format('{0}-{1}', ${{ github.ref }}, 'Cluster (pitr_mysqlshell)') + group: format('{0}-{1}', ${{ github.ref }}, 'Cluster (backup_pitr_mysqlshell)') cancel-in-progress: true permissions: read-all @@ -15,7 +15,7 @@ env: jobs: build: - name: Run endtoend tests on Cluster (pitr_mysqlshell) + name: Run endtoend tests on Cluster (backup_pitr_mysqlshell) runs-on: gh-hosted-runners-4cores-1 steps: @@ -67,7 +67,7 @@ jobs: - 'tools/**' - 'config/**' - 'bootstrap.sh' - - '.github/workflows/cluster_endtoend_pitr_mysqlshell.yml' + - '.github/workflows/cluster_endtoend_backup_pitr_mysqlshell.yml' - name: Set up Go if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' @@ -136,7 +136,7 @@ jobs: set -exo pipefail # run the tests however you normally do, then produce a JUnit XML file - eatmydata -- go run test.go -docker=false -follow -shard pitr_mysqlshell | tee -a output.txt | go-junit-report -set-exit-code > report.xml + eatmydata -- go run test.go -docker=false -follow -shard backup_pitr_mysqlshell | tee -a output.txt | go-junit-report -set-exit-code > report.xml - name: Print test output and Record test result in launchable if PR is not a draft if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' && always() From 0bb563199dc82920c89e56e1d8994f0f9ac111b5 Mon Sep 17 00:00:00 2001 From: Renan Rangel Date: Wed, 21 Aug 2024 06:09:16 -0700 Subject: [PATCH 12/34] add mysql-shell package as part of the standard mysql CI dependencies Signed-off-by: Renan Rangel --- .github/workflows/cluster_endtoend_12.yml | 2 +- .github/workflows/cluster_endtoend_13.yml | 2 +- .github/workflows/cluster_endtoend_15.yml | 2 +- .github/workflows/cluster_endtoend_18.yml | 2 +- .github/workflows/cluster_endtoend_21.yml | 2 +- .github/workflows/cluster_endtoend_backup_pitr.yml | 2 +- .github/workflows/cluster_endtoend_backup_pitr_mysqlshell.yml | 2 +- .../workflows/cluster_endtoend_ers_prs_newfeatures_heavy.yml | 2 +- .github/workflows/cluster_endtoend_mysql80.yml | 2 +- .github/workflows/cluster_endtoend_mysql_server_vault.yml | 2 +- .github/workflows/cluster_endtoend_onlineddl_revert.yml | 2 +- .github/workflows/cluster_endtoend_onlineddl_scheduler.yml | 2 +- .github/workflows/cluster_endtoend_onlineddl_vrepl.yml | 2 +- .github/workflows/cluster_endtoend_onlineddl_vrepl_stress.yml | 2 +- .../workflows/cluster_endtoend_onlineddl_vrepl_stress_suite.yml | 2 +- .github/workflows/cluster_endtoend_onlineddl_vrepl_suite.yml | 2 +- .github/workflows/cluster_endtoend_schemadiff_vrepl.yml | 2 +- .github/workflows/cluster_endtoend_tabletmanager_consul.yml | 2 +- .github/workflows/cluster_endtoend_tabletmanager_tablegc.yml | 2 +- .../workflows/cluster_endtoend_tabletmanager_throttler_topo.yml | 2 +- .github/workflows/cluster_endtoend_topo_connection_cache.yml | 2 +- .../cluster_endtoend_vreplication_across_db_versions.yml | 2 +- .github/workflows/cluster_endtoend_vreplication_basic.yml | 2 +- .github/workflows/cluster_endtoend_vreplication_cellalias.yml | 2 +- .../workflows/cluster_endtoend_vreplication_copy_parallel.yml | 2 +- .../cluster_endtoend_vreplication_foreign_key_stress.yml | 2 +- .../cluster_endtoend_vreplication_mariadb_to_mysql.yml | 2 +- .../cluster_endtoend_vreplication_migrate_vdiff2_convert_tz.yml | 2 +- .../workflows/cluster_endtoend_vreplication_multi_tenant.yml | 2 +- ...endtoend_vreplication_partial_movetables_and_materialize.yml | 2 +- .github/workflows/cluster_endtoend_vreplication_v2.yml | 2 +- .github/workflows/cluster_endtoend_vstream.yml | 2 +- .github/workflows/cluster_endtoend_vtbackup.yml | 2 +- .../cluster_endtoend_vtctlbackup_sharded_clustertest_heavy.yml | 2 +- .github/workflows/cluster_endtoend_vtgate_concurrentdml.yml | 2 +- .github/workflows/cluster_endtoend_vtgate_foreignkey_stress.yml | 2 +- .github/workflows/cluster_endtoend_vtgate_gen4.yml | 2 +- .github/workflows/cluster_endtoend_vtgate_general_heavy.yml | 2 +- .github/workflows/cluster_endtoend_vtgate_godriver.yml | 2 +- .github/workflows/cluster_endtoend_vtgate_partial_keyspace.yml | 2 +- .github/workflows/cluster_endtoend_vtgate_queries.yml | 2 +- .github/workflows/cluster_endtoend_vtgate_readafterwrite.yml | 2 +- .github/workflows/cluster_endtoend_vtgate_reservedconn.yml | 2 +- .github/workflows/cluster_endtoend_vtgate_schema.yml | 2 +- .github/workflows/cluster_endtoend_vtgate_schema_tracker.yml | 2 +- .../cluster_endtoend_vtgate_tablet_healthcheck_cache.yml | 2 +- .github/workflows/cluster_endtoend_vtgate_topo.yml | 2 +- .github/workflows/cluster_endtoend_vtgate_topo_consul.yml | 2 +- .github/workflows/cluster_endtoend_vtgate_topo_etcd.yml | 2 +- .github/workflows/cluster_endtoend_vtgate_transaction.yml | 2 +- .github/workflows/cluster_endtoend_vtgate_unsharded.yml | 2 +- .github/workflows/cluster_endtoend_vtgate_vindex_heavy.yml | 2 +- .github/workflows/cluster_endtoend_vtgate_vschema.yml | 2 +- .github/workflows/cluster_endtoend_vtorc.yml | 2 +- .github/workflows/cluster_endtoend_vttablet_prscomplex.yml | 2 +- test/templates/cluster_endtoend_test.tpl | 2 +- 56 files changed, 56 insertions(+), 56 deletions(-) diff --git a/.github/workflows/cluster_endtoend_12.yml b/.github/workflows/cluster_endtoend_12.yml index 3c984f4d403..0a5a86b2b9a 100644 --- a/.github/workflows/cluster_endtoend_12.yml +++ b/.github/workflows/cluster_endtoend_12.yml @@ -101,7 +101,7 @@ jobs: sudo DEBIAN_FRONTEND="noninteractive" dpkg -i mysql-apt-config* sudo apt-get -qq update # Install everything else we need, and configure - sudo apt-get -qq install -y mysql-server mysql-client make unzip g++ etcd curl git wget eatmydata xz-utils libncurses5 + sudo apt-get -qq install -y mysql-server mysql-shell mysql-client make unzip g++ etcd curl git wget eatmydata xz-utils libncurses5 sudo service mysql stop sudo service etcd stop diff --git a/.github/workflows/cluster_endtoend_13.yml b/.github/workflows/cluster_endtoend_13.yml index c1a92a8a4b7..3f62fbd2e49 100644 --- a/.github/workflows/cluster_endtoend_13.yml +++ b/.github/workflows/cluster_endtoend_13.yml @@ -101,7 +101,7 @@ jobs: sudo DEBIAN_FRONTEND="noninteractive" dpkg -i mysql-apt-config* sudo apt-get -qq update # Install everything else we need, and configure - sudo apt-get -qq install -y mysql-server mysql-client make unzip g++ etcd curl git wget eatmydata xz-utils libncurses5 + sudo apt-get -qq install -y mysql-server mysql-shell mysql-client make unzip g++ etcd curl git wget eatmydata xz-utils libncurses5 sudo service mysql stop sudo service etcd stop diff --git a/.github/workflows/cluster_endtoend_15.yml b/.github/workflows/cluster_endtoend_15.yml index 3d2881b7676..f3955de1465 100644 --- a/.github/workflows/cluster_endtoend_15.yml +++ b/.github/workflows/cluster_endtoend_15.yml @@ -101,7 +101,7 @@ jobs: sudo DEBIAN_FRONTEND="noninteractive" dpkg -i mysql-apt-config* sudo apt-get -qq update # Install everything else we need, and configure - sudo apt-get -qq install -y mysql-server mysql-client make unzip g++ etcd curl git wget eatmydata xz-utils libncurses5 + sudo apt-get -qq install -y mysql-server mysql-shell mysql-client make unzip g++ etcd curl git wget eatmydata xz-utils libncurses5 sudo service mysql stop sudo service etcd stop diff --git a/.github/workflows/cluster_endtoend_18.yml b/.github/workflows/cluster_endtoend_18.yml index 1f35fbf0431..b08be8b278e 100644 --- a/.github/workflows/cluster_endtoend_18.yml +++ b/.github/workflows/cluster_endtoend_18.yml @@ -101,7 +101,7 @@ jobs: sudo DEBIAN_FRONTEND="noninteractive" dpkg -i mysql-apt-config* sudo apt-get -qq update # Install everything else we need, and configure - sudo apt-get -qq install -y mysql-server mysql-client make unzip g++ etcd curl git wget eatmydata xz-utils libncurses5 + sudo apt-get -qq install -y mysql-server mysql-shell mysql-client make unzip g++ etcd curl git wget eatmydata xz-utils libncurses5 sudo service mysql stop sudo service etcd stop diff --git a/.github/workflows/cluster_endtoend_21.yml b/.github/workflows/cluster_endtoend_21.yml index d47ad86dfd1..26c56f9aa7e 100644 --- a/.github/workflows/cluster_endtoend_21.yml +++ b/.github/workflows/cluster_endtoend_21.yml @@ -101,7 +101,7 @@ jobs: sudo DEBIAN_FRONTEND="noninteractive" dpkg -i mysql-apt-config* sudo apt-get -qq update # Install everything else we need, and configure - sudo apt-get -qq install -y mysql-server mysql-client make unzip g++ etcd curl git wget eatmydata xz-utils libncurses5 + sudo apt-get -qq install -y mysql-server mysql-shell mysql-client make unzip g++ etcd curl git wget eatmydata xz-utils libncurses5 sudo service mysql stop sudo service etcd stop diff --git a/.github/workflows/cluster_endtoend_backup_pitr.yml b/.github/workflows/cluster_endtoend_backup_pitr.yml index 560b3029158..598821bc33f 100644 --- a/.github/workflows/cluster_endtoend_backup_pitr.yml +++ b/.github/workflows/cluster_endtoend_backup_pitr.yml @@ -101,7 +101,7 @@ jobs: sudo DEBIAN_FRONTEND="noninteractive" dpkg -i mysql-apt-config* sudo apt-get -qq update # Install everything else we need, and configure - sudo apt-get -qq install -y mysql-server mysql-client make unzip g++ etcd curl git wget eatmydata xz-utils libncurses5 + sudo apt-get -qq install -y mysql-server mysql-shell mysql-client make unzip g++ etcd curl git wget eatmydata xz-utils libncurses5 sudo service mysql stop sudo service etcd stop diff --git a/.github/workflows/cluster_endtoend_backup_pitr_mysqlshell.yml b/.github/workflows/cluster_endtoend_backup_pitr_mysqlshell.yml index 8095f1f8bbd..7dcd070f1de 100644 --- a/.github/workflows/cluster_endtoend_backup_pitr_mysqlshell.yml +++ b/.github/workflows/cluster_endtoend_backup_pitr_mysqlshell.yml @@ -101,7 +101,7 @@ jobs: sudo DEBIAN_FRONTEND="noninteractive" dpkg -i mysql-apt-config* sudo apt-get -qq update # Install everything else we need, and configure - sudo apt-get -qq install -y mysql-server mysql-client make unzip g++ etcd curl git wget eatmydata xz-utils libncurses5 + sudo apt-get -qq install -y mysql-server mysql-shell mysql-client make unzip g++ etcd curl git wget eatmydata xz-utils libncurses5 sudo service mysql stop sudo service etcd stop diff --git a/.github/workflows/cluster_endtoend_ers_prs_newfeatures_heavy.yml b/.github/workflows/cluster_endtoend_ers_prs_newfeatures_heavy.yml index a3f41d78264..ed82cff2302 100644 --- a/.github/workflows/cluster_endtoend_ers_prs_newfeatures_heavy.yml +++ b/.github/workflows/cluster_endtoend_ers_prs_newfeatures_heavy.yml @@ -101,7 +101,7 @@ jobs: sudo DEBIAN_FRONTEND="noninteractive" dpkg -i mysql-apt-config* sudo apt-get -qq update # Install everything else we need, and configure - sudo apt-get -qq install -y mysql-server mysql-client make unzip g++ etcd curl git wget eatmydata xz-utils libncurses5 + sudo apt-get -qq install -y mysql-server mysql-shell mysql-client make unzip g++ etcd curl git wget eatmydata xz-utils libncurses5 sudo service mysql stop sudo service etcd stop diff --git a/.github/workflows/cluster_endtoend_mysql80.yml b/.github/workflows/cluster_endtoend_mysql80.yml index f5dc2abad55..cee3dbcbc37 100644 --- a/.github/workflows/cluster_endtoend_mysql80.yml +++ b/.github/workflows/cluster_endtoend_mysql80.yml @@ -101,7 +101,7 @@ jobs: sudo DEBIAN_FRONTEND="noninteractive" dpkg -i mysql-apt-config* sudo apt-get -qq update # Install everything else we need, and configure - sudo apt-get -qq install -y mysql-server mysql-client make unzip g++ etcd curl git wget eatmydata xz-utils libncurses5 + sudo apt-get -qq install -y mysql-server mysql-shell mysql-client make unzip g++ etcd curl git wget eatmydata xz-utils libncurses5 sudo service mysql stop sudo service etcd stop diff --git a/.github/workflows/cluster_endtoend_mysql_server_vault.yml b/.github/workflows/cluster_endtoend_mysql_server_vault.yml index d41b6f014f5..0f46968a343 100644 --- a/.github/workflows/cluster_endtoend_mysql_server_vault.yml +++ b/.github/workflows/cluster_endtoend_mysql_server_vault.yml @@ -101,7 +101,7 @@ jobs: sudo DEBIAN_FRONTEND="noninteractive" dpkg -i mysql-apt-config* sudo apt-get -qq update # Install everything else we need, and configure - sudo apt-get -qq install -y mysql-server mysql-client make unzip g++ etcd curl git wget eatmydata xz-utils libncurses5 + sudo apt-get -qq install -y mysql-server mysql-shell mysql-client make unzip g++ etcd curl git wget eatmydata xz-utils libncurses5 sudo service mysql stop sudo service etcd stop diff --git a/.github/workflows/cluster_endtoend_onlineddl_revert.yml b/.github/workflows/cluster_endtoend_onlineddl_revert.yml index 10e6c4e670f..f7ec07b2184 100644 --- a/.github/workflows/cluster_endtoend_onlineddl_revert.yml +++ b/.github/workflows/cluster_endtoend_onlineddl_revert.yml @@ -102,7 +102,7 @@ jobs: sudo DEBIAN_FRONTEND="noninteractive" dpkg -i mysql-apt-config* sudo apt-get -qq update # Install everything else we need, and configure - sudo apt-get -qq install -y mysql-server mysql-client make unzip g++ etcd curl git wget eatmydata xz-utils libncurses5 + sudo apt-get -qq install -y mysql-server mysql-shell mysql-client make unzip g++ etcd curl git wget eatmydata xz-utils libncurses5 sudo service mysql stop sudo service etcd stop diff --git a/.github/workflows/cluster_endtoend_onlineddl_scheduler.yml b/.github/workflows/cluster_endtoend_onlineddl_scheduler.yml index 4b76d62b923..bca7bb6caf5 100644 --- a/.github/workflows/cluster_endtoend_onlineddl_scheduler.yml +++ b/.github/workflows/cluster_endtoend_onlineddl_scheduler.yml @@ -102,7 +102,7 @@ jobs: sudo DEBIAN_FRONTEND="noninteractive" dpkg -i mysql-apt-config* sudo apt-get -qq update # Install everything else we need, and configure - sudo apt-get -qq install -y mysql-server mysql-client make unzip g++ etcd curl git wget eatmydata xz-utils libncurses5 + sudo apt-get -qq install -y mysql-server mysql-shell mysql-client make unzip g++ etcd curl git wget eatmydata xz-utils libncurses5 sudo service mysql stop sudo service etcd stop diff --git a/.github/workflows/cluster_endtoend_onlineddl_vrepl.yml b/.github/workflows/cluster_endtoend_onlineddl_vrepl.yml index f2e7d2939f0..3708519250f 100644 --- a/.github/workflows/cluster_endtoend_onlineddl_vrepl.yml +++ b/.github/workflows/cluster_endtoend_onlineddl_vrepl.yml @@ -102,7 +102,7 @@ jobs: sudo DEBIAN_FRONTEND="noninteractive" dpkg -i mysql-apt-config* sudo apt-get -qq update # Install everything else we need, and configure - sudo apt-get -qq install -y mysql-server mysql-client make unzip g++ etcd curl git wget eatmydata xz-utils libncurses5 + sudo apt-get -qq install -y mysql-server mysql-shell mysql-client make unzip g++ etcd curl git wget eatmydata xz-utils libncurses5 sudo service mysql stop sudo service etcd stop diff --git a/.github/workflows/cluster_endtoend_onlineddl_vrepl_stress.yml b/.github/workflows/cluster_endtoend_onlineddl_vrepl_stress.yml index 7574a4bcea2..dfccb1f5bcf 100644 --- a/.github/workflows/cluster_endtoend_onlineddl_vrepl_stress.yml +++ b/.github/workflows/cluster_endtoend_onlineddl_vrepl_stress.yml @@ -102,7 +102,7 @@ jobs: sudo DEBIAN_FRONTEND="noninteractive" dpkg -i mysql-apt-config* sudo apt-get -qq update # Install everything else we need, and configure - sudo apt-get -qq install -y mysql-server mysql-client make unzip g++ etcd curl git wget eatmydata xz-utils libncurses5 + sudo apt-get -qq install -y mysql-server mysql-shell mysql-client make unzip g++ etcd curl git wget eatmydata xz-utils libncurses5 sudo service mysql stop sudo service etcd stop diff --git a/.github/workflows/cluster_endtoend_onlineddl_vrepl_stress_suite.yml b/.github/workflows/cluster_endtoend_onlineddl_vrepl_stress_suite.yml index c6576ed2927..75efc18779c 100644 --- a/.github/workflows/cluster_endtoend_onlineddl_vrepl_stress_suite.yml +++ b/.github/workflows/cluster_endtoend_onlineddl_vrepl_stress_suite.yml @@ -102,7 +102,7 @@ jobs: sudo DEBIAN_FRONTEND="noninteractive" dpkg -i mysql-apt-config* sudo apt-get -qq update # Install everything else we need, and configure - sudo apt-get -qq install -y mysql-server mysql-client make unzip g++ etcd curl git wget eatmydata xz-utils libncurses5 + sudo apt-get -qq install -y mysql-server mysql-shell mysql-client make unzip g++ etcd curl git wget eatmydata xz-utils libncurses5 sudo service mysql stop sudo service etcd stop diff --git a/.github/workflows/cluster_endtoend_onlineddl_vrepl_suite.yml b/.github/workflows/cluster_endtoend_onlineddl_vrepl_suite.yml index ce528f797ea..edaae8cf9c1 100644 --- a/.github/workflows/cluster_endtoend_onlineddl_vrepl_suite.yml +++ b/.github/workflows/cluster_endtoend_onlineddl_vrepl_suite.yml @@ -102,7 +102,7 @@ jobs: sudo DEBIAN_FRONTEND="noninteractive" dpkg -i mysql-apt-config* sudo apt-get -qq update # Install everything else we need, and configure - sudo apt-get -qq install -y mysql-server mysql-client make unzip g++ etcd curl git wget eatmydata xz-utils libncurses5 + sudo apt-get -qq install -y mysql-server mysql-shell mysql-client make unzip g++ etcd curl git wget eatmydata xz-utils libncurses5 sudo service mysql stop sudo service etcd stop diff --git a/.github/workflows/cluster_endtoend_schemadiff_vrepl.yml b/.github/workflows/cluster_endtoend_schemadiff_vrepl.yml index 07224c415db..b04b13b788f 100644 --- a/.github/workflows/cluster_endtoend_schemadiff_vrepl.yml +++ b/.github/workflows/cluster_endtoend_schemadiff_vrepl.yml @@ -102,7 +102,7 @@ jobs: sudo DEBIAN_FRONTEND="noninteractive" dpkg -i mysql-apt-config* sudo apt-get -qq update # Install everything else we need, and configure - sudo apt-get -qq install -y mysql-server mysql-client make unzip g++ etcd curl git wget eatmydata xz-utils libncurses5 + sudo apt-get -qq install -y mysql-server mysql-shell mysql-client make unzip g++ etcd curl git wget eatmydata xz-utils libncurses5 sudo service mysql stop sudo service etcd stop diff --git a/.github/workflows/cluster_endtoend_tabletmanager_consul.yml b/.github/workflows/cluster_endtoend_tabletmanager_consul.yml index e80c8598f44..de5191e2b26 100644 --- a/.github/workflows/cluster_endtoend_tabletmanager_consul.yml +++ b/.github/workflows/cluster_endtoend_tabletmanager_consul.yml @@ -101,7 +101,7 @@ jobs: sudo DEBIAN_FRONTEND="noninteractive" dpkg -i mysql-apt-config* sudo apt-get -qq update # Install everything else we need, and configure - sudo apt-get -qq install -y mysql-server mysql-client make unzip g++ etcd curl git wget eatmydata xz-utils libncurses5 + sudo apt-get -qq install -y mysql-server mysql-shell mysql-client make unzip g++ etcd curl git wget eatmydata xz-utils libncurses5 sudo service mysql stop sudo service etcd stop diff --git a/.github/workflows/cluster_endtoend_tabletmanager_tablegc.yml b/.github/workflows/cluster_endtoend_tabletmanager_tablegc.yml index 14973bd1f2d..33fb740a24a 100644 --- a/.github/workflows/cluster_endtoend_tabletmanager_tablegc.yml +++ b/.github/workflows/cluster_endtoend_tabletmanager_tablegc.yml @@ -101,7 +101,7 @@ jobs: sudo DEBIAN_FRONTEND="noninteractive" dpkg -i mysql-apt-config* sudo apt-get -qq update # Install everything else we need, and configure - sudo apt-get -qq install -y mysql-server mysql-client make unzip g++ etcd curl git wget eatmydata xz-utils libncurses5 + sudo apt-get -qq install -y mysql-server mysql-shell mysql-client make unzip g++ etcd curl git wget eatmydata xz-utils libncurses5 sudo service mysql stop sudo service etcd stop diff --git a/.github/workflows/cluster_endtoend_tabletmanager_throttler_topo.yml b/.github/workflows/cluster_endtoend_tabletmanager_throttler_topo.yml index a647e443e61..9ecf23720da 100644 --- a/.github/workflows/cluster_endtoend_tabletmanager_throttler_topo.yml +++ b/.github/workflows/cluster_endtoend_tabletmanager_throttler_topo.yml @@ -101,7 +101,7 @@ jobs: sudo DEBIAN_FRONTEND="noninteractive" dpkg -i mysql-apt-config* sudo apt-get -qq update # Install everything else we need, and configure - sudo apt-get -qq install -y mysql-server mysql-client make unzip g++ etcd curl git wget eatmydata xz-utils libncurses5 + sudo apt-get -qq install -y mysql-server mysql-shell mysql-client make unzip g++ etcd curl git wget eatmydata xz-utils libncurses5 sudo service mysql stop sudo service etcd stop diff --git a/.github/workflows/cluster_endtoend_topo_connection_cache.yml b/.github/workflows/cluster_endtoend_topo_connection_cache.yml index 37c932621d9..957b33332a7 100644 --- a/.github/workflows/cluster_endtoend_topo_connection_cache.yml +++ b/.github/workflows/cluster_endtoend_topo_connection_cache.yml @@ -101,7 +101,7 @@ jobs: sudo DEBIAN_FRONTEND="noninteractive" dpkg -i mysql-apt-config* sudo apt-get -qq update # Install everything else we need, and configure - sudo apt-get -qq install -y mysql-server mysql-client make unzip g++ etcd curl git wget eatmydata xz-utils libncurses5 + sudo apt-get -qq install -y mysql-server mysql-shell mysql-client make unzip g++ etcd curl git wget eatmydata xz-utils libncurses5 sudo service mysql stop sudo service etcd stop diff --git a/.github/workflows/cluster_endtoend_vreplication_across_db_versions.yml b/.github/workflows/cluster_endtoend_vreplication_across_db_versions.yml index 9116e1d3fc2..5b0c3e62e60 100644 --- a/.github/workflows/cluster_endtoend_vreplication_across_db_versions.yml +++ b/.github/workflows/cluster_endtoend_vreplication_across_db_versions.yml @@ -101,7 +101,7 @@ jobs: sudo DEBIAN_FRONTEND="noninteractive" dpkg -i mysql-apt-config* sudo apt-get -qq update # Install everything else we need, and configure - sudo apt-get -qq install -y mysql-server mysql-client make unzip g++ etcd curl git wget eatmydata xz-utils libncurses5 + sudo apt-get -qq install -y mysql-server mysql-shell mysql-client make unzip g++ etcd curl git wget eatmydata xz-utils libncurses5 sudo service mysql stop sudo service etcd stop diff --git a/.github/workflows/cluster_endtoend_vreplication_basic.yml b/.github/workflows/cluster_endtoend_vreplication_basic.yml index 7843ad05192..dd52ca51a1e 100644 --- a/.github/workflows/cluster_endtoend_vreplication_basic.yml +++ b/.github/workflows/cluster_endtoend_vreplication_basic.yml @@ -101,7 +101,7 @@ jobs: sudo DEBIAN_FRONTEND="noninteractive" dpkg -i mysql-apt-config* sudo apt-get -qq update # Install everything else we need, and configure - sudo apt-get -qq install -y mysql-server mysql-client make unzip g++ etcd curl git wget eatmydata xz-utils libncurses5 + sudo apt-get -qq install -y mysql-server mysql-shell mysql-client make unzip g++ etcd curl git wget eatmydata xz-utils libncurses5 sudo service mysql stop sudo service etcd stop diff --git a/.github/workflows/cluster_endtoend_vreplication_cellalias.yml b/.github/workflows/cluster_endtoend_vreplication_cellalias.yml index 9a179229f1e..b648e08cb0e 100644 --- a/.github/workflows/cluster_endtoend_vreplication_cellalias.yml +++ b/.github/workflows/cluster_endtoend_vreplication_cellalias.yml @@ -101,7 +101,7 @@ jobs: sudo DEBIAN_FRONTEND="noninteractive" dpkg -i mysql-apt-config* sudo apt-get -qq update # Install everything else we need, and configure - sudo apt-get -qq install -y mysql-server mysql-client make unzip g++ etcd curl git wget eatmydata xz-utils libncurses5 + sudo apt-get -qq install -y mysql-server mysql-shell mysql-client make unzip g++ etcd curl git wget eatmydata xz-utils libncurses5 sudo service mysql stop sudo service etcd stop diff --git a/.github/workflows/cluster_endtoend_vreplication_copy_parallel.yml b/.github/workflows/cluster_endtoend_vreplication_copy_parallel.yml index 37eafd948c4..16bbe95d593 100644 --- a/.github/workflows/cluster_endtoend_vreplication_copy_parallel.yml +++ b/.github/workflows/cluster_endtoend_vreplication_copy_parallel.yml @@ -101,7 +101,7 @@ jobs: sudo DEBIAN_FRONTEND="noninteractive" dpkg -i mysql-apt-config* sudo apt-get -qq update # Install everything else we need, and configure - sudo apt-get -qq install -y mysql-server mysql-client make unzip g++ etcd curl git wget eatmydata xz-utils libncurses5 + sudo apt-get -qq install -y mysql-server mysql-shell mysql-client make unzip g++ etcd curl git wget eatmydata xz-utils libncurses5 sudo service mysql stop sudo service etcd stop diff --git a/.github/workflows/cluster_endtoend_vreplication_foreign_key_stress.yml b/.github/workflows/cluster_endtoend_vreplication_foreign_key_stress.yml index 9fb4287e0b1..da426532cb8 100644 --- a/.github/workflows/cluster_endtoend_vreplication_foreign_key_stress.yml +++ b/.github/workflows/cluster_endtoend_vreplication_foreign_key_stress.yml @@ -101,7 +101,7 @@ jobs: sudo DEBIAN_FRONTEND="noninteractive" dpkg -i mysql-apt-config* sudo apt-get -qq update # Install everything else we need, and configure - sudo apt-get -qq install -y mysql-server mysql-client make unzip g++ etcd curl git wget eatmydata xz-utils libncurses5 + sudo apt-get -qq install -y mysql-server mysql-shell mysql-client make unzip g++ etcd curl git wget eatmydata xz-utils libncurses5 sudo service mysql stop sudo service etcd stop diff --git a/.github/workflows/cluster_endtoend_vreplication_mariadb_to_mysql.yml b/.github/workflows/cluster_endtoend_vreplication_mariadb_to_mysql.yml index 86f2fa69e30..7e75595bc8b 100644 --- a/.github/workflows/cluster_endtoend_vreplication_mariadb_to_mysql.yml +++ b/.github/workflows/cluster_endtoend_vreplication_mariadb_to_mysql.yml @@ -101,7 +101,7 @@ jobs: sudo DEBIAN_FRONTEND="noninteractive" dpkg -i mysql-apt-config* sudo apt-get -qq update # Install everything else we need, and configure - sudo apt-get -qq install -y mysql-server mysql-client make unzip g++ etcd curl git wget eatmydata xz-utils libncurses5 + sudo apt-get -qq install -y mysql-server mysql-shell mysql-client make unzip g++ etcd curl git wget eatmydata xz-utils libncurses5 sudo service mysql stop sudo service etcd stop diff --git a/.github/workflows/cluster_endtoend_vreplication_migrate_vdiff2_convert_tz.yml b/.github/workflows/cluster_endtoend_vreplication_migrate_vdiff2_convert_tz.yml index d7d385f1e9f..3a31ac94750 100644 --- a/.github/workflows/cluster_endtoend_vreplication_migrate_vdiff2_convert_tz.yml +++ b/.github/workflows/cluster_endtoend_vreplication_migrate_vdiff2_convert_tz.yml @@ -101,7 +101,7 @@ jobs: sudo DEBIAN_FRONTEND="noninteractive" dpkg -i mysql-apt-config* sudo apt-get -qq update # Install everything else we need, and configure - sudo apt-get -qq install -y mysql-server mysql-client make unzip g++ etcd curl git wget eatmydata xz-utils libncurses5 + sudo apt-get -qq install -y mysql-server mysql-shell mysql-client make unzip g++ etcd curl git wget eatmydata xz-utils libncurses5 sudo service mysql stop sudo service etcd stop diff --git a/.github/workflows/cluster_endtoend_vreplication_multi_tenant.yml b/.github/workflows/cluster_endtoend_vreplication_multi_tenant.yml index e4e5ee17452..b82cf96dcdd 100644 --- a/.github/workflows/cluster_endtoend_vreplication_multi_tenant.yml +++ b/.github/workflows/cluster_endtoend_vreplication_multi_tenant.yml @@ -101,7 +101,7 @@ jobs: sudo DEBIAN_FRONTEND="noninteractive" dpkg -i mysql-apt-config* sudo apt-get -qq update # Install everything else we need, and configure - sudo apt-get -qq install -y mysql-server mysql-client make unzip g++ etcd curl git wget eatmydata xz-utils libncurses5 + sudo apt-get -qq install -y mysql-server mysql-shell mysql-client make unzip g++ etcd curl git wget eatmydata xz-utils libncurses5 sudo service mysql stop sudo service etcd stop diff --git a/.github/workflows/cluster_endtoend_vreplication_partial_movetables_and_materialize.yml b/.github/workflows/cluster_endtoend_vreplication_partial_movetables_and_materialize.yml index a7b2c31b420..2bebd1a1951 100644 --- a/.github/workflows/cluster_endtoend_vreplication_partial_movetables_and_materialize.yml +++ b/.github/workflows/cluster_endtoend_vreplication_partial_movetables_and_materialize.yml @@ -101,7 +101,7 @@ jobs: sudo DEBIAN_FRONTEND="noninteractive" dpkg -i mysql-apt-config* sudo apt-get -qq update # Install everything else we need, and configure - sudo apt-get -qq install -y mysql-server mysql-client make unzip g++ etcd curl git wget eatmydata xz-utils libncurses5 + sudo apt-get -qq install -y mysql-server mysql-shell mysql-client make unzip g++ etcd curl git wget eatmydata xz-utils libncurses5 sudo service mysql stop sudo service etcd stop diff --git a/.github/workflows/cluster_endtoend_vreplication_v2.yml b/.github/workflows/cluster_endtoend_vreplication_v2.yml index fb1247fabfb..0937295e21f 100644 --- a/.github/workflows/cluster_endtoend_vreplication_v2.yml +++ b/.github/workflows/cluster_endtoend_vreplication_v2.yml @@ -101,7 +101,7 @@ jobs: sudo DEBIAN_FRONTEND="noninteractive" dpkg -i mysql-apt-config* sudo apt-get -qq update # Install everything else we need, and configure - sudo apt-get -qq install -y mysql-server mysql-client make unzip g++ etcd curl git wget eatmydata xz-utils libncurses5 + sudo apt-get -qq install -y mysql-server mysql-shell mysql-client make unzip g++ etcd curl git wget eatmydata xz-utils libncurses5 sudo service mysql stop sudo service etcd stop diff --git a/.github/workflows/cluster_endtoend_vstream.yml b/.github/workflows/cluster_endtoend_vstream.yml index 5d13e6d3a41..d8d0a537888 100644 --- a/.github/workflows/cluster_endtoend_vstream.yml +++ b/.github/workflows/cluster_endtoend_vstream.yml @@ -101,7 +101,7 @@ jobs: sudo DEBIAN_FRONTEND="noninteractive" dpkg -i mysql-apt-config* sudo apt-get -qq update # Install everything else we need, and configure - sudo apt-get -qq install -y mysql-server mysql-client make unzip g++ etcd curl git wget eatmydata xz-utils libncurses5 + sudo apt-get -qq install -y mysql-server mysql-shell mysql-client make unzip g++ etcd curl git wget eatmydata xz-utils libncurses5 sudo service mysql stop sudo service etcd stop diff --git a/.github/workflows/cluster_endtoend_vtbackup.yml b/.github/workflows/cluster_endtoend_vtbackup.yml index 9bf411cc328..82e237622b6 100644 --- a/.github/workflows/cluster_endtoend_vtbackup.yml +++ b/.github/workflows/cluster_endtoend_vtbackup.yml @@ -101,7 +101,7 @@ jobs: sudo DEBIAN_FRONTEND="noninteractive" dpkg -i mysql-apt-config* sudo apt-get -qq update # Install everything else we need, and configure - sudo apt-get -qq install -y mysql-server mysql-client make unzip g++ etcd curl git wget eatmydata xz-utils libncurses5 + sudo apt-get -qq install -y mysql-server mysql-shell mysql-client make unzip g++ etcd curl git wget eatmydata xz-utils libncurses5 sudo service mysql stop sudo service etcd stop diff --git a/.github/workflows/cluster_endtoend_vtctlbackup_sharded_clustertest_heavy.yml b/.github/workflows/cluster_endtoend_vtctlbackup_sharded_clustertest_heavy.yml index 09fbfa8e451..5d124f02bf3 100644 --- a/.github/workflows/cluster_endtoend_vtctlbackup_sharded_clustertest_heavy.yml +++ b/.github/workflows/cluster_endtoend_vtctlbackup_sharded_clustertest_heavy.yml @@ -101,7 +101,7 @@ jobs: sudo DEBIAN_FRONTEND="noninteractive" dpkg -i mysql-apt-config* sudo apt-get -qq update # Install everything else we need, and configure - sudo apt-get -qq install -y mysql-server mysql-client make unzip g++ etcd curl git wget eatmydata xz-utils libncurses5 + sudo apt-get -qq install -y mysql-server mysql-shell mysql-client make unzip g++ etcd curl git wget eatmydata xz-utils libncurses5 sudo service mysql stop sudo service etcd stop diff --git a/.github/workflows/cluster_endtoend_vtgate_concurrentdml.yml b/.github/workflows/cluster_endtoend_vtgate_concurrentdml.yml index b8cb2d77ad9..994905b7acf 100644 --- a/.github/workflows/cluster_endtoend_vtgate_concurrentdml.yml +++ b/.github/workflows/cluster_endtoend_vtgate_concurrentdml.yml @@ -101,7 +101,7 @@ jobs: sudo DEBIAN_FRONTEND="noninteractive" dpkg -i mysql-apt-config* sudo apt-get -qq update # Install everything else we need, and configure - sudo apt-get -qq install -y mysql-server mysql-client make unzip g++ etcd curl git wget eatmydata xz-utils libncurses5 + sudo apt-get -qq install -y mysql-server mysql-shell mysql-client make unzip g++ etcd curl git wget eatmydata xz-utils libncurses5 sudo service mysql stop sudo service etcd stop diff --git a/.github/workflows/cluster_endtoend_vtgate_foreignkey_stress.yml b/.github/workflows/cluster_endtoend_vtgate_foreignkey_stress.yml index 5ad1025debd..2d929cea328 100644 --- a/.github/workflows/cluster_endtoend_vtgate_foreignkey_stress.yml +++ b/.github/workflows/cluster_endtoend_vtgate_foreignkey_stress.yml @@ -101,7 +101,7 @@ jobs: sudo DEBIAN_FRONTEND="noninteractive" dpkg -i mysql-apt-config* sudo apt-get -qq update # Install everything else we need, and configure - sudo apt-get -qq install -y mysql-server mysql-client make unzip g++ etcd curl git wget eatmydata xz-utils libncurses5 + sudo apt-get -qq install -y mysql-server mysql-shell mysql-client make unzip g++ etcd curl git wget eatmydata xz-utils libncurses5 sudo service mysql stop sudo service etcd stop diff --git a/.github/workflows/cluster_endtoend_vtgate_gen4.yml b/.github/workflows/cluster_endtoend_vtgate_gen4.yml index 43bbe666186..eafbecfe0b0 100644 --- a/.github/workflows/cluster_endtoend_vtgate_gen4.yml +++ b/.github/workflows/cluster_endtoend_vtgate_gen4.yml @@ -101,7 +101,7 @@ jobs: sudo DEBIAN_FRONTEND="noninteractive" dpkg -i mysql-apt-config* sudo apt-get -qq update # Install everything else we need, and configure - sudo apt-get -qq install -y mysql-server mysql-client make unzip g++ etcd curl git wget eatmydata xz-utils libncurses5 + sudo apt-get -qq install -y mysql-server mysql-shell mysql-client make unzip g++ etcd curl git wget eatmydata xz-utils libncurses5 sudo service mysql stop sudo service etcd stop diff --git a/.github/workflows/cluster_endtoend_vtgate_general_heavy.yml b/.github/workflows/cluster_endtoend_vtgate_general_heavy.yml index d63edf72ae7..2259a00e948 100644 --- a/.github/workflows/cluster_endtoend_vtgate_general_heavy.yml +++ b/.github/workflows/cluster_endtoend_vtgate_general_heavy.yml @@ -101,7 +101,7 @@ jobs: sudo DEBIAN_FRONTEND="noninteractive" dpkg -i mysql-apt-config* sudo apt-get -qq update # Install everything else we need, and configure - sudo apt-get -qq install -y mysql-server mysql-client make unzip g++ etcd curl git wget eatmydata xz-utils libncurses5 + sudo apt-get -qq install -y mysql-server mysql-shell mysql-client make unzip g++ etcd curl git wget eatmydata xz-utils libncurses5 sudo service mysql stop sudo service etcd stop diff --git a/.github/workflows/cluster_endtoend_vtgate_godriver.yml b/.github/workflows/cluster_endtoend_vtgate_godriver.yml index c7c2d0bf494..4436f8ae2e6 100644 --- a/.github/workflows/cluster_endtoend_vtgate_godriver.yml +++ b/.github/workflows/cluster_endtoend_vtgate_godriver.yml @@ -101,7 +101,7 @@ jobs: sudo DEBIAN_FRONTEND="noninteractive" dpkg -i mysql-apt-config* sudo apt-get -qq update # Install everything else we need, and configure - sudo apt-get -qq install -y mysql-server mysql-client make unzip g++ etcd curl git wget eatmydata xz-utils libncurses5 + sudo apt-get -qq install -y mysql-server mysql-shell mysql-client make unzip g++ etcd curl git wget eatmydata xz-utils libncurses5 sudo service mysql stop sudo service etcd stop diff --git a/.github/workflows/cluster_endtoend_vtgate_partial_keyspace.yml b/.github/workflows/cluster_endtoend_vtgate_partial_keyspace.yml index 3510e8f2144..0ac5cd93c70 100644 --- a/.github/workflows/cluster_endtoend_vtgate_partial_keyspace.yml +++ b/.github/workflows/cluster_endtoend_vtgate_partial_keyspace.yml @@ -101,7 +101,7 @@ jobs: sudo DEBIAN_FRONTEND="noninteractive" dpkg -i mysql-apt-config* sudo apt-get -qq update # Install everything else we need, and configure - sudo apt-get -qq install -y mysql-server mysql-client make unzip g++ etcd curl git wget eatmydata xz-utils libncurses5 + sudo apt-get -qq install -y mysql-server mysql-shell mysql-client make unzip g++ etcd curl git wget eatmydata xz-utils libncurses5 sudo service mysql stop sudo service etcd stop diff --git a/.github/workflows/cluster_endtoend_vtgate_queries.yml b/.github/workflows/cluster_endtoend_vtgate_queries.yml index 40f20e2177e..d8c7377d2af 100644 --- a/.github/workflows/cluster_endtoend_vtgate_queries.yml +++ b/.github/workflows/cluster_endtoend_vtgate_queries.yml @@ -101,7 +101,7 @@ jobs: sudo DEBIAN_FRONTEND="noninteractive" dpkg -i mysql-apt-config* sudo apt-get -qq update # Install everything else we need, and configure - sudo apt-get -qq install -y mysql-server mysql-client make unzip g++ etcd curl git wget eatmydata xz-utils libncurses5 + sudo apt-get -qq install -y mysql-server mysql-shell mysql-client make unzip g++ etcd curl git wget eatmydata xz-utils libncurses5 sudo service mysql stop sudo service etcd stop diff --git a/.github/workflows/cluster_endtoend_vtgate_readafterwrite.yml b/.github/workflows/cluster_endtoend_vtgate_readafterwrite.yml index c913e05810a..a1ba3a62e03 100644 --- a/.github/workflows/cluster_endtoend_vtgate_readafterwrite.yml +++ b/.github/workflows/cluster_endtoend_vtgate_readafterwrite.yml @@ -101,7 +101,7 @@ jobs: sudo DEBIAN_FRONTEND="noninteractive" dpkg -i mysql-apt-config* sudo apt-get -qq update # Install everything else we need, and configure - sudo apt-get -qq install -y mysql-server mysql-client make unzip g++ etcd curl git wget eatmydata xz-utils libncurses5 + sudo apt-get -qq install -y mysql-server mysql-shell mysql-client make unzip g++ etcd curl git wget eatmydata xz-utils libncurses5 sudo service mysql stop sudo service etcd stop diff --git a/.github/workflows/cluster_endtoend_vtgate_reservedconn.yml b/.github/workflows/cluster_endtoend_vtgate_reservedconn.yml index aa173b32ca3..78d0948f8c9 100644 --- a/.github/workflows/cluster_endtoend_vtgate_reservedconn.yml +++ b/.github/workflows/cluster_endtoend_vtgate_reservedconn.yml @@ -101,7 +101,7 @@ jobs: sudo DEBIAN_FRONTEND="noninteractive" dpkg -i mysql-apt-config* sudo apt-get -qq update # Install everything else we need, and configure - sudo apt-get -qq install -y mysql-server mysql-client make unzip g++ etcd curl git wget eatmydata xz-utils libncurses5 + sudo apt-get -qq install -y mysql-server mysql-shell mysql-client make unzip g++ etcd curl git wget eatmydata xz-utils libncurses5 sudo service mysql stop sudo service etcd stop diff --git a/.github/workflows/cluster_endtoend_vtgate_schema.yml b/.github/workflows/cluster_endtoend_vtgate_schema.yml index 5a5ab725b8a..800be37a0b0 100644 --- a/.github/workflows/cluster_endtoend_vtgate_schema.yml +++ b/.github/workflows/cluster_endtoend_vtgate_schema.yml @@ -101,7 +101,7 @@ jobs: sudo DEBIAN_FRONTEND="noninteractive" dpkg -i mysql-apt-config* sudo apt-get -qq update # Install everything else we need, and configure - sudo apt-get -qq install -y mysql-server mysql-client make unzip g++ etcd curl git wget eatmydata xz-utils libncurses5 + sudo apt-get -qq install -y mysql-server mysql-shell mysql-client make unzip g++ etcd curl git wget eatmydata xz-utils libncurses5 sudo service mysql stop sudo service etcd stop diff --git a/.github/workflows/cluster_endtoend_vtgate_schema_tracker.yml b/.github/workflows/cluster_endtoend_vtgate_schema_tracker.yml index 00244b8223a..0e824c2bc50 100644 --- a/.github/workflows/cluster_endtoend_vtgate_schema_tracker.yml +++ b/.github/workflows/cluster_endtoend_vtgate_schema_tracker.yml @@ -101,7 +101,7 @@ jobs: sudo DEBIAN_FRONTEND="noninteractive" dpkg -i mysql-apt-config* sudo apt-get -qq update # Install everything else we need, and configure - sudo apt-get -qq install -y mysql-server mysql-client make unzip g++ etcd curl git wget eatmydata xz-utils libncurses5 + sudo apt-get -qq install -y mysql-server mysql-shell mysql-client make unzip g++ etcd curl git wget eatmydata xz-utils libncurses5 sudo service mysql stop sudo service etcd stop diff --git a/.github/workflows/cluster_endtoend_vtgate_tablet_healthcheck_cache.yml b/.github/workflows/cluster_endtoend_vtgate_tablet_healthcheck_cache.yml index ee14848cd79..89a36475d9e 100644 --- a/.github/workflows/cluster_endtoend_vtgate_tablet_healthcheck_cache.yml +++ b/.github/workflows/cluster_endtoend_vtgate_tablet_healthcheck_cache.yml @@ -101,7 +101,7 @@ jobs: sudo DEBIAN_FRONTEND="noninteractive" dpkg -i mysql-apt-config* sudo apt-get -qq update # Install everything else we need, and configure - sudo apt-get -qq install -y mysql-server mysql-client make unzip g++ etcd curl git wget eatmydata xz-utils libncurses5 + sudo apt-get -qq install -y mysql-server mysql-shell mysql-client make unzip g++ etcd curl git wget eatmydata xz-utils libncurses5 sudo service mysql stop sudo service etcd stop diff --git a/.github/workflows/cluster_endtoend_vtgate_topo.yml b/.github/workflows/cluster_endtoend_vtgate_topo.yml index 9517ad965b5..b35726078fb 100644 --- a/.github/workflows/cluster_endtoend_vtgate_topo.yml +++ b/.github/workflows/cluster_endtoend_vtgate_topo.yml @@ -101,7 +101,7 @@ jobs: sudo DEBIAN_FRONTEND="noninteractive" dpkg -i mysql-apt-config* sudo apt-get -qq update # Install everything else we need, and configure - sudo apt-get -qq install -y mysql-server mysql-client make unzip g++ etcd curl git wget eatmydata xz-utils libncurses5 + sudo apt-get -qq install -y mysql-server mysql-shell mysql-client make unzip g++ etcd curl git wget eatmydata xz-utils libncurses5 sudo service mysql stop sudo service etcd stop diff --git a/.github/workflows/cluster_endtoend_vtgate_topo_consul.yml b/.github/workflows/cluster_endtoend_vtgate_topo_consul.yml index 6d585fb41f3..b55503a7090 100644 --- a/.github/workflows/cluster_endtoend_vtgate_topo_consul.yml +++ b/.github/workflows/cluster_endtoend_vtgate_topo_consul.yml @@ -101,7 +101,7 @@ jobs: sudo DEBIAN_FRONTEND="noninteractive" dpkg -i mysql-apt-config* sudo apt-get -qq update # Install everything else we need, and configure - sudo apt-get -qq install -y mysql-server mysql-client make unzip g++ etcd curl git wget eatmydata xz-utils libncurses5 + sudo apt-get -qq install -y mysql-server mysql-shell mysql-client make unzip g++ etcd curl git wget eatmydata xz-utils libncurses5 sudo service mysql stop sudo service etcd stop diff --git a/.github/workflows/cluster_endtoend_vtgate_topo_etcd.yml b/.github/workflows/cluster_endtoend_vtgate_topo_etcd.yml index d1ae0e96afd..60cfbf90e9d 100644 --- a/.github/workflows/cluster_endtoend_vtgate_topo_etcd.yml +++ b/.github/workflows/cluster_endtoend_vtgate_topo_etcd.yml @@ -101,7 +101,7 @@ jobs: sudo DEBIAN_FRONTEND="noninteractive" dpkg -i mysql-apt-config* sudo apt-get -qq update # Install everything else we need, and configure - sudo apt-get -qq install -y mysql-server mysql-client make unzip g++ etcd curl git wget eatmydata xz-utils libncurses5 + sudo apt-get -qq install -y mysql-server mysql-shell mysql-client make unzip g++ etcd curl git wget eatmydata xz-utils libncurses5 sudo service mysql stop sudo service etcd stop diff --git a/.github/workflows/cluster_endtoend_vtgate_transaction.yml b/.github/workflows/cluster_endtoend_vtgate_transaction.yml index 50f6a57b551..840456aff00 100644 --- a/.github/workflows/cluster_endtoend_vtgate_transaction.yml +++ b/.github/workflows/cluster_endtoend_vtgate_transaction.yml @@ -101,7 +101,7 @@ jobs: sudo DEBIAN_FRONTEND="noninteractive" dpkg -i mysql-apt-config* sudo apt-get -qq update # Install everything else we need, and configure - sudo apt-get -qq install -y mysql-server mysql-client make unzip g++ etcd curl git wget eatmydata xz-utils libncurses5 + sudo apt-get -qq install -y mysql-server mysql-shell mysql-client make unzip g++ etcd curl git wget eatmydata xz-utils libncurses5 sudo service mysql stop sudo service etcd stop diff --git a/.github/workflows/cluster_endtoend_vtgate_unsharded.yml b/.github/workflows/cluster_endtoend_vtgate_unsharded.yml index b0d8348aaab..a6b68b702a4 100644 --- a/.github/workflows/cluster_endtoend_vtgate_unsharded.yml +++ b/.github/workflows/cluster_endtoend_vtgate_unsharded.yml @@ -101,7 +101,7 @@ jobs: sudo DEBIAN_FRONTEND="noninteractive" dpkg -i mysql-apt-config* sudo apt-get -qq update # Install everything else we need, and configure - sudo apt-get -qq install -y mysql-server mysql-client make unzip g++ etcd curl git wget eatmydata xz-utils libncurses5 + sudo apt-get -qq install -y mysql-server mysql-shell mysql-client make unzip g++ etcd curl git wget eatmydata xz-utils libncurses5 sudo service mysql stop sudo service etcd stop diff --git a/.github/workflows/cluster_endtoend_vtgate_vindex_heavy.yml b/.github/workflows/cluster_endtoend_vtgate_vindex_heavy.yml index 330ee506517..5cea5d09c18 100644 --- a/.github/workflows/cluster_endtoend_vtgate_vindex_heavy.yml +++ b/.github/workflows/cluster_endtoend_vtgate_vindex_heavy.yml @@ -101,7 +101,7 @@ jobs: sudo DEBIAN_FRONTEND="noninteractive" dpkg -i mysql-apt-config* sudo apt-get -qq update # Install everything else we need, and configure - sudo apt-get -qq install -y mysql-server mysql-client make unzip g++ etcd curl git wget eatmydata xz-utils libncurses5 + sudo apt-get -qq install -y mysql-server mysql-shell mysql-client make unzip g++ etcd curl git wget eatmydata xz-utils libncurses5 sudo service mysql stop sudo service etcd stop diff --git a/.github/workflows/cluster_endtoend_vtgate_vschema.yml b/.github/workflows/cluster_endtoend_vtgate_vschema.yml index 4ddf20fa9c5..23f9dd06565 100644 --- a/.github/workflows/cluster_endtoend_vtgate_vschema.yml +++ b/.github/workflows/cluster_endtoend_vtgate_vschema.yml @@ -101,7 +101,7 @@ jobs: sudo DEBIAN_FRONTEND="noninteractive" dpkg -i mysql-apt-config* sudo apt-get -qq update # Install everything else we need, and configure - sudo apt-get -qq install -y mysql-server mysql-client make unzip g++ etcd curl git wget eatmydata xz-utils libncurses5 + sudo apt-get -qq install -y mysql-server mysql-shell mysql-client make unzip g++ etcd curl git wget eatmydata xz-utils libncurses5 sudo service mysql stop sudo service etcd stop diff --git a/.github/workflows/cluster_endtoend_vtorc.yml b/.github/workflows/cluster_endtoend_vtorc.yml index 77493692395..2827685c6de 100644 --- a/.github/workflows/cluster_endtoend_vtorc.yml +++ b/.github/workflows/cluster_endtoend_vtorc.yml @@ -110,7 +110,7 @@ jobs: sudo DEBIAN_FRONTEND="noninteractive" dpkg -i mysql-apt-config* sudo apt-get -qq update # Install everything else we need, and configure - sudo apt-get -qq install -y mysql-server mysql-client make unzip g++ etcd curl git wget eatmydata xz-utils libncurses5 + sudo apt-get -qq install -y mysql-server mysql-shell mysql-client make unzip g++ etcd curl git wget eatmydata xz-utils libncurses5 sudo service mysql stop sudo service etcd stop diff --git a/.github/workflows/cluster_endtoend_vttablet_prscomplex.yml b/.github/workflows/cluster_endtoend_vttablet_prscomplex.yml index 58ff4074c0c..630912ff9c1 100644 --- a/.github/workflows/cluster_endtoend_vttablet_prscomplex.yml +++ b/.github/workflows/cluster_endtoend_vttablet_prscomplex.yml @@ -101,7 +101,7 @@ jobs: sudo DEBIAN_FRONTEND="noninteractive" dpkg -i mysql-apt-config* sudo apt-get -qq update # Install everything else we need, and configure - sudo apt-get -qq install -y mysql-server mysql-client make unzip g++ etcd curl git wget eatmydata xz-utils libncurses5 + sudo apt-get -qq install -y mysql-server mysql-shell mysql-client make unzip g++ etcd curl git wget eatmydata xz-utils libncurses5 sudo service mysql stop sudo service etcd stop diff --git a/test/templates/cluster_endtoend_test.tpl b/test/templates/cluster_endtoend_test.tpl index 332cb67fedc..c7afa2d5cb3 100644 --- a/test/templates/cluster_endtoend_test.tpl +++ b/test/templates/cluster_endtoend_test.tpl @@ -129,7 +129,7 @@ jobs: sudo DEBIAN_FRONTEND="noninteractive" dpkg -i mysql-apt-config* sudo apt-get -qq update # Install everything else we need, and configure - sudo apt-get -qq install -y mysql-server mysql-client make unzip g++ etcd curl git wget eatmydata xz-utils libncurses5 + sudo apt-get -qq install -y mysql-server mysql-shell mysql-client make unzip g++ etcd curl git wget eatmydata xz-utils libncurses5 {{end}} From 82a288ca49558874f9c9163f5fb58c02d495a4dc Mon Sep 17 00:00:00 2001 From: Renan Rangel Date: Wed, 21 Aug 2024 08:03:14 -0700 Subject: [PATCH 13/34] use dumpInstances() instead Signed-off-by: Renan Rangel --- .../backup/vtctlbackup/backup_utils.go | 24 +++++++++---------- go/vt/mysqlctl/mysqlshellbackupengine.go | 4 +--- 2 files changed, 13 insertions(+), 15 deletions(-) diff --git a/go/test/endtoend/backup/vtctlbackup/backup_utils.go b/go/test/endtoend/backup/vtctlbackup/backup_utils.go index a278e25e2e6..a658fb04d20 100644 --- a/go/test/endtoend/backup/vtctlbackup/backup_utils.go +++ b/go/test/endtoend/backup/vtctlbackup/backup_utils.go @@ -1362,8 +1362,7 @@ func TestReplicaRestoreToTimestamp(t *testing.T, restoreToTimestamp time.Time, e func verifyTabletBackupStats(t *testing.T, vars map[string]any) { switch currentSetupType { // Currently only the builtin backup engine instruments bytes-processed counts. - case XtraBackup, MySQLShell: - default: + case BuiltinBackup: require.Contains(t, vars, "BackupBytes") bb := vars["BackupBytes"].(map[string]any) require.Contains(t, bb, "BackupEngine.Builtin.Compressor:Write") @@ -1372,6 +1371,7 @@ func verifyTabletBackupStats(t *testing.T, vars map[string]any) { if backupstorage.BackupStorageImplementation == "file" { require.Contains(t, bb, "BackupStorage.File.File:Write") } + case XtraBackup, MySQLShell: } require.Contains(t, vars, "BackupCount") @@ -1380,13 +1380,13 @@ func verifyTabletBackupStats(t *testing.T, vars map[string]any) { switch currentSetupType { // Currently only the builtin backup engine instruments bytes-processed counts. - case XtraBackup, MySQLShell: - default: + case BuiltinBackup: require.Contains(t, bc, "BackupEngine.Builtin.Compressor:Close") require.Contains(t, bc, "BackupEngine.Builtin.Destination:Close") require.Contains(t, bc, "BackupEngine.Builtin.Destination:Open") require.Contains(t, bc, "BackupEngine.Builtin.Source:Close") require.Contains(t, bc, "BackupEngine.Builtin.Source:Open") + case XtraBackup, MySQLShell: } require.Contains(t, vars, "BackupDurationNanoseconds") @@ -1395,8 +1395,7 @@ func verifyTabletBackupStats(t *testing.T, vars map[string]any) { switch currentSetupType { // Currently only the builtin backup engine emits timings. - case XtraBackup, MySQLShell: - default: + case BuiltinBackup: require.Contains(t, bd, "BackupEngine.Builtin.Compressor:Close") require.Contains(t, bd, "BackupEngine.Builtin.Compressor:Write") require.Contains(t, bd, "BackupEngine.Builtin.Destination:Close") @@ -1405,6 +1404,7 @@ func verifyTabletBackupStats(t *testing.T, vars map[string]any) { require.Contains(t, bd, "BackupEngine.Builtin.Source:Close") require.Contains(t, bd, "BackupEngine.Builtin.Source:Open") require.Contains(t, bd, "BackupEngine.Builtin.Source:Read") + case XtraBackup, MySQLShell: } if backupstorage.BackupStorageImplementation == "file" { @@ -1432,14 +1432,14 @@ func verifyTabletRestoreStats(t *testing.T, vars map[string]any) { verifyRestorePositionAndTimeStats(t, vars) switch currentSetupType { - case XtraBackup, MySQLShell: - default: + case BuiltinBackup: require.Contains(t, vars, "RestoreBytes") bb := vars["RestoreBytes"].(map[string]any) require.Contains(t, bb, "BackupEngine.Builtin.Decompressor:Read") require.Contains(t, bb, "BackupEngine.Builtin.Destination:Write") require.Contains(t, bb, "BackupEngine.Builtin.Source:Read") require.Contains(t, bb, "BackupStorage.File.File:Read") + case XtraBackup, MySQLShell: } require.Contains(t, vars, "RestoreCount") @@ -1448,13 +1448,13 @@ func verifyTabletRestoreStats(t *testing.T, vars map[string]any) { switch currentSetupType { // Currently only the builtin backup engine emits operation counts. - case XtraBackup, MySQLShell: - default: + case BuiltinBackup: require.Contains(t, bc, "BackupEngine.Builtin.Decompressor:Close") require.Contains(t, bc, "BackupEngine.Builtin.Destination:Close") require.Contains(t, bc, "BackupEngine.Builtin.Destination:Open") require.Contains(t, bc, "BackupEngine.Builtin.Source:Close") require.Contains(t, bc, "BackupEngine.Builtin.Source:Open") + case XtraBackup, MySQLShell: } require.Contains(t, vars, "RestoreDurationNanoseconds") @@ -1463,8 +1463,7 @@ func verifyTabletRestoreStats(t *testing.T, vars map[string]any) { switch currentSetupType { // Currently only the builtin backup engine emits timings. - case XtraBackup, MySQLShell: - default: + case BuiltinBackup: require.Contains(t, bd, "BackupEngine.Builtin.Decompressor:Close") require.Contains(t, bd, "BackupEngine.Builtin.Decompressor:Read") require.Contains(t, bd, "BackupEngine.Builtin.Destination:Close") @@ -1473,6 +1472,7 @@ func verifyTabletRestoreStats(t *testing.T, vars map[string]any) { require.Contains(t, bd, "BackupEngine.Builtin.Source:Close") require.Contains(t, bd, "BackupEngine.Builtin.Source:Open") require.Contains(t, bd, "BackupEngine.Builtin.Source:Read") + case XtraBackup, MySQLShell: } require.Contains(t, bd, "BackupStorage.File.File:Read") diff --git a/go/vt/mysqlctl/mysqlshellbackupengine.go b/go/vt/mysqlctl/mysqlshellbackupengine.go index 624aba138aa..7b80f221406 100644 --- a/go/vt/mysqlctl/mysqlshellbackupengine.go +++ b/go/vt/mysqlctl/mysqlshellbackupengine.go @@ -124,9 +124,7 @@ func (be *MySQLShellBackupEngine) ExecuteBackup(ctx context.Context, params Back args = append(args, strings.Fields(mysqlShellFlags)...) } - args = append(args, "-e", fmt.Sprintf("util.dumpSchemas([\"%s\", \"vt_%s\"], %q, %s)", - sidecar.GetName(), - params.Keyspace, + args = append(args, "-e", fmt.Sprintf("util.dumpInstance(%q, %s)", location, mysqlShellDumpFlags, )) From d9407f2e442accfc3ae8213fa6ab9bb45f0087aa Mon Sep 17 00:00:00 2001 From: Renan Rangel Date: Wed, 21 Aug 2024 08:58:56 -0700 Subject: [PATCH 14/34] remove case XtraBackup, MySQLShell Signed-off-by: Renan Rangel --- go/test/endtoend/backup/vtctlbackup/backup_utils.go | 6 ------ 1 file changed, 6 deletions(-) diff --git a/go/test/endtoend/backup/vtctlbackup/backup_utils.go b/go/test/endtoend/backup/vtctlbackup/backup_utils.go index a658fb04d20..7b471b3bf21 100644 --- a/go/test/endtoend/backup/vtctlbackup/backup_utils.go +++ b/go/test/endtoend/backup/vtctlbackup/backup_utils.go @@ -1371,7 +1371,6 @@ func verifyTabletBackupStats(t *testing.T, vars map[string]any) { if backupstorage.BackupStorageImplementation == "file" { require.Contains(t, bb, "BackupStorage.File.File:Write") } - case XtraBackup, MySQLShell: } require.Contains(t, vars, "BackupCount") @@ -1386,7 +1385,6 @@ func verifyTabletBackupStats(t *testing.T, vars map[string]any) { require.Contains(t, bc, "BackupEngine.Builtin.Destination:Open") require.Contains(t, bc, "BackupEngine.Builtin.Source:Close") require.Contains(t, bc, "BackupEngine.Builtin.Source:Open") - case XtraBackup, MySQLShell: } require.Contains(t, vars, "BackupDurationNanoseconds") @@ -1404,7 +1402,6 @@ func verifyTabletBackupStats(t *testing.T, vars map[string]any) { require.Contains(t, bd, "BackupEngine.Builtin.Source:Close") require.Contains(t, bd, "BackupEngine.Builtin.Source:Open") require.Contains(t, bd, "BackupEngine.Builtin.Source:Read") - case XtraBackup, MySQLShell: } if backupstorage.BackupStorageImplementation == "file" { @@ -1439,7 +1436,6 @@ func verifyTabletRestoreStats(t *testing.T, vars map[string]any) { require.Contains(t, bb, "BackupEngine.Builtin.Destination:Write") require.Contains(t, bb, "BackupEngine.Builtin.Source:Read") require.Contains(t, bb, "BackupStorage.File.File:Read") - case XtraBackup, MySQLShell: } require.Contains(t, vars, "RestoreCount") @@ -1454,7 +1450,6 @@ func verifyTabletRestoreStats(t *testing.T, vars map[string]any) { require.Contains(t, bc, "BackupEngine.Builtin.Destination:Open") require.Contains(t, bc, "BackupEngine.Builtin.Source:Close") require.Contains(t, bc, "BackupEngine.Builtin.Source:Open") - case XtraBackup, MySQLShell: } require.Contains(t, vars, "RestoreDurationNanoseconds") @@ -1472,7 +1467,6 @@ func verifyTabletRestoreStats(t *testing.T, vars map[string]any) { require.Contains(t, bd, "BackupEngine.Builtin.Source:Close") require.Contains(t, bd, "BackupEngine.Builtin.Source:Open") require.Contains(t, bd, "BackupEngine.Builtin.Source:Read") - case XtraBackup, MySQLShell: } require.Contains(t, bd, "BackupStorage.File.File:Read") From 31ba57e30af1fd3dc648a7005b04b1a30d51ce33 Mon Sep 17 00:00:00 2001 From: Renan Rangel Date: Wed, 21 Aug 2024 09:41:55 -0700 Subject: [PATCH 15/34] make generate_ci_workflows Signed-off-by: Renan Rangel --- .github/workflows/cluster_endtoend_backup_pitr_mysqlshell.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/cluster_endtoend_backup_pitr_mysqlshell.yml b/.github/workflows/cluster_endtoend_backup_pitr_mysqlshell.yml index 7dcd070f1de..d7fd48761e2 100644 --- a/.github/workflows/cluster_endtoend_backup_pitr_mysqlshell.yml +++ b/.github/workflows/cluster_endtoend_backup_pitr_mysqlshell.yml @@ -73,7 +73,7 @@ jobs: if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' uses: actions/setup-go@v5 with: - go-version: 1.22.5 + go-version: 1.23.0 - name: Set up python if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' From 31c27b51da7f52808a0fd731c3a77bcd4c2b27c5 Mon Sep 17 00:00:00 2001 From: Renan Rangel Date: Wed, 4 Sep 2024 02:59:17 -0700 Subject: [PATCH 16/34] update flags and operator.yaml Signed-off-by: Renan Rangel --- examples/operator/operator.yaml | 1 + go/vt/mysqlctl/mysqlshellbackupengine.go | 6 +++--- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/examples/operator/operator.yaml b/examples/operator/operator.yaml index 1ebf69da491..6795f37becd 100644 --- a/examples/operator/operator.yaml +++ b/examples/operator/operator.yaml @@ -1523,6 +1523,7 @@ spec: enum: - builtin - xtrabackup + - mysqlshell type: string locations: items: diff --git a/go/vt/mysqlctl/mysqlshellbackupengine.go b/go/vt/mysqlctl/mysqlshellbackupengine.go index 7b80f221406..6ee276693b2 100644 --- a/go/vt/mysqlctl/mysqlshellbackupengine.go +++ b/go/vt/mysqlctl/mysqlshellbackupengine.go @@ -379,11 +379,11 @@ func (be *MySQLShellBackupEngine) ShouldDrainForBackup(req *tabletmanagerdatapb. func (be *MySQLShellBackupEngine) backupPreCheck(location string) error { if mysqlShellBackupLocation == "" { - return fmt.Errorf("%w: no backup location set via --mysql_shell_location", MySQLShellPreCheckError) + return fmt.Errorf("%w: no backup location set via --mysql-shell-backup-location", MySQLShellPreCheckError) } if mysqlShellFlags == "" || !strings.Contains(mysqlShellFlags, "--js") { - return fmt.Errorf("%w: at least the --js flag is required", MySQLShellPreCheckError) + return fmt.Errorf("%w: at least the --js flag is required in the value of the flag --mysql-shell-flags", MySQLShellPreCheckError) } // make sure the targe directory exists if the target location for the backup is not an object store @@ -408,7 +408,7 @@ func (be *MySQLShellBackupEngine) backupPreCheck(location string) error { func (be *MySQLShellBackupEngine) restorePreCheck(ctx context.Context, params RestoreParams) error { if mysqlShellFlags == "" { - return fmt.Errorf("%w: at least the --js flag is required", MySQLShellPreCheckError) + return fmt.Errorf("%w: at least the --js flag is required in the value of the flag --mysql-shell-flags", MySQLShellPreCheckError) } loadFlags := map[string]interface{}{} From e62ae955ea9514fa52ecb14fccd112e50c1d2940 Mon Sep 17 00:00:00 2001 From: Renan Rangel Date: Thu, 5 Sep 2024 03:30:32 -0700 Subject: [PATCH 17/34] removing DeleteBeforeRestore Signed-off-by: Renan Rangel --- go/cmd/vtbackup/cli/vtbackup.go | 1 - go/vt/mysqlctl/backup.go | 28 +------------ go/vt/mysqlctl/backup_blackbox_test.go | 1 - go/vt/mysqlctl/backup_test.go | 32 +++++++------- go/vt/mysqlctl/backupengine.go | 5 --- go/vt/mysqlctl/fakemysqldaemon.go | 18 +++++++- go/vt/mysqlctl/mysqlshellbackupengine.go | 49 ++++++++++------------ go/vt/vttablet/tabletmanager/restore.go | 6 +-- go/vt/vttablet/tabletmanager/rpc_backup.go | 2 +- go/vt/vttablet/tabletmanager/tm_init.go | 2 +- go/vt/wrangler/testlib/backup_test.go | 12 +++--- 11 files changed, 65 insertions(+), 91 deletions(-) diff --git a/go/cmd/vtbackup/cli/vtbackup.go b/go/cmd/vtbackup/cli/vtbackup.go index 1b61c886ae7..c4379dac09d 100644 --- a/go/cmd/vtbackup/cli/vtbackup.go +++ b/go/cmd/vtbackup/cli/vtbackup.go @@ -428,7 +428,6 @@ func takeBackup(ctx, backgroundCtx context.Context, topoServer *topo.Server, bac Logger: logutil.NewConsoleLogger(), Concurrency: concurrency, HookExtraEnv: extraEnv, - DeleteBeforeRestore: true, DbName: dbName, Keyspace: initKeyspace, Shard: initShard, diff --git a/go/vt/mysqlctl/backup.go b/go/vt/mysqlctl/backup.go index fd067b02887..8d10c31f847 100644 --- a/go/vt/mysqlctl/backup.go +++ b/go/vt/mysqlctl/backup.go @@ -237,29 +237,6 @@ func ParseBackupName(dir string, name string) (backupTime *time.Time, alias *top return backupTime, alias, nil } -// checkNoDB makes sure there is no user data already there. -// Used by Restore, as we do not want to destroy an existing DB. -// The user's database name must be given since we ignore all others. -// Returns (true, nil) if the specified DB doesn't exist. -// Returns (false, nil) if the check succeeds but the condition is not -// satisfied (there is a DB). -// Returns (false, non-nil error) if one occurs while trying to perform the check. -func checkNoDB(ctx context.Context, mysqld MysqlDaemon, dbName string) (bool, error) { - qr, err := mysqld.FetchSuperQuery(ctx, "SHOW DATABASES") - if err != nil { - return false, vterrors.Wrap(err, "checkNoDB failed") - } - - for _, row := range qr.Rows { - if row[0].ToString() == dbName { - // found active db - log.Warningf("checkNoDB failed, found active db %v", dbName) - return false, nil - } - } - return true, nil -} - // removeExistingFiles will delete existing files in the data dir to prevent // conflicts with the restored archive. In particular, binlogs can be created // even during initial bootstrap, and these can interfere with configuring @@ -312,10 +289,9 @@ func removeExistingFiles(cnf *Mycnf) error { // ShouldRestore checks whether a database with tables already exists // and returns whether a restore action should be performed func ShouldRestore(ctx context.Context, params RestoreParams) (bool, error) { - if params.DeleteBeforeRestore || RestoreWasInterrupted(params.Cnf) { + if RestoreWasInterrupted(params.Cnf) { return true, nil } - params.Logger.Infof("Restore: No %v file found, checking no existing data is present", RestoreState) // Wait for mysqld to be ready, in case it was launched in parallel with us. // If this doesn't succeed, we should not attempt a restore if err := params.Mysqld.Wait(ctx, params.Cnf); err != nil { @@ -325,7 +301,7 @@ func ShouldRestore(ctx context.Context, params RestoreParams) (bool, error) { params.Logger.Errorf("error waiting for the grants: %v", err) return false, err } - return checkNoDB(ctx, params.Mysqld, params.DbName) + return true, nil } // ensureRestoredGTIDPurgedMatchesManifest sees the following: when you restore a full backup, you want the MySQL server to have diff --git a/go/vt/mysqlctl/backup_blackbox_test.go b/go/vt/mysqlctl/backup_blackbox_test.go index 15244fb8782..ad583b047de 100644 --- a/go/vt/mysqlctl/backup_blackbox_test.go +++ b/go/vt/mysqlctl/backup_blackbox_test.go @@ -512,7 +512,6 @@ func TestExecuteRestoreWithTimedOutContext(t *testing.T) { Mysqld: mysqld, Concurrency: 2, HookExtraEnv: map[string]string{}, - DeleteBeforeRestore: false, DbName: "test", Keyspace: "test", Shard: "-", diff --git a/go/vt/mysqlctl/backup_test.go b/go/vt/mysqlctl/backup_test.go index 86b34fbbf99..5a7feafbdf5 100644 --- a/go/vt/mysqlctl/backup_test.go +++ b/go/vt/mysqlctl/backup_test.go @@ -36,7 +36,6 @@ import ( "vitess.io/vitess/go/mysql" "vitess.io/vitess/go/mysql/fakesqldb" "vitess.io/vitess/go/mysql/replication" - "vitess.io/vitess/go/sqltypes" "vitess.io/vitess/go/test/utils" "vitess.io/vitess/go/vt/logutil" "vitess.io/vitess/go/vt/mysqlctl/backupstats" @@ -585,7 +584,6 @@ func createFakeBackupRestoreEnv(t *testing.T) *fakeBackupRestoreEnv { Mysqld: mysqld, Concurrency: 1, HookExtraEnv: map[string]string{}, - DeleteBeforeRestore: false, DbName: "test", Keyspace: "test", Shard: "-", @@ -690,28 +688,26 @@ func TestShouldRestore(t *testing.T) { env := createFakeBackupRestoreEnv(t) b, err := ShouldRestore(env.ctx, env.restoreParams) - assert.False(t, b) - assert.Error(t, err) - - env.restoreParams.DeleteBeforeRestore = true - b, err = ShouldRestore(env.ctx, env.restoreParams) assert.True(t, b) assert.NoError(t, err) - env.restoreParams.DeleteBeforeRestore = false - env.mysqld.FetchSuperQueryMap = map[string]*sqltypes.Result{ - "SHOW DATABASES": {Rows: [][]sqltypes.Value{{sqltypes.NewVarBinary("any_db")}}}, - } - b, err = ShouldRestore(env.ctx, env.restoreParams) - assert.NoError(t, err) + // if MySQL is online on time, we should continue the restore + env.mysqld.WaitDuration = time.Second + ctx, cancel := context.WithTimeout(context.Background(), time.Second*3) + defer cancel() + + b, err = ShouldRestore(ctx, env.restoreParams) assert.True(t, b) + assert.NoError(t, err) - env.mysqld.FetchSuperQueryMap = map[string]*sqltypes.Result{ - "SHOW DATABASES": {Rows: [][]sqltypes.Value{{sqltypes.NewVarBinary("test")}}}, - } - b, err = ShouldRestore(env.ctx, env.restoreParams) + // but should return an error if waiting for MySQL longer than the context expected + env.mysqld.WaitDuration = time.Second * 2 + ctx, cancel = context.WithTimeout(context.Background(), time.Second) + defer cancel() + + b, err = ShouldRestore(ctx, env.restoreParams) assert.False(t, b) - assert.NoError(t, err) + assert.ErrorIs(t, err, context.DeadlineExceeded) } func TestScanLinesToLogger(t *testing.T) { diff --git a/go/vt/mysqlctl/backupengine.go b/go/vt/mysqlctl/backupengine.go index c483aff3d78..dcd41e8eb26 100644 --- a/go/vt/mysqlctl/backupengine.go +++ b/go/vt/mysqlctl/backupengine.go @@ -118,10 +118,6 @@ type RestoreParams struct { Concurrency int // Extra env variables for pre-restore and post-restore transform hooks HookExtraEnv map[string]string - // DeleteBeforeRestore tells us whether existing data should be deleted before - // restoring. This is always set to false when starting a tablet with -restore_from_backup, - // but is set to true when executing a RestoreFromBackup command on an already running vttablet - DeleteBeforeRestore bool // DbName is the name of the managed database / schema DbName string // Keyspace and Shard are used to infer the directory where backups are stored @@ -152,7 +148,6 @@ func (p *RestoreParams) Copy() RestoreParams { Logger: p.Logger, Concurrency: p.Concurrency, HookExtraEnv: p.HookExtraEnv, - DeleteBeforeRestore: p.DeleteBeforeRestore, DbName: p.DbName, Keyspace: p.Keyspace, Shard: p.Shard, diff --git a/go/vt/mysqlctl/fakemysqldaemon.go b/go/vt/mysqlctl/fakemysqldaemon.go index b9175a32779..cfa3188c049 100644 --- a/go/vt/mysqlctl/fakemysqldaemon.go +++ b/go/vt/mysqlctl/fakemysqldaemon.go @@ -139,6 +139,10 @@ type FakeMysqlDaemon struct { // function returns an error. WaitPrimaryPositions []replication.Position + // WaitDuration is used by the Wait() and WaitForDBAGrants() calls to + // simulate and arbitrary amount of time it would take waiting for mysql + WaitDuration time.Duration + // PromoteResult is returned by Promote. PromoteResult replication.Position @@ -271,11 +275,21 @@ func (fmd *FakeMysqlDaemon) RefreshConfig(ctx context.Context, cnf *Mycnf) error // Wait is part of the MysqlDaemon interface. func (fmd *FakeMysqlDaemon) Wait(ctx context.Context, cnf *Mycnf) error { - return nil + select { + case <-ctx.Done(): + return ctx.Err() + case <-time.After(fmd.WaitDuration): + return nil + } } func (fmd *FakeMysqlDaemon) WaitForDBAGrants(ctx context.Context, waitTime time.Duration) (err error) { - return nil + select { + case <-ctx.Done(): + return ctx.Err() + case <-time.After(fmd.WaitDuration): + return nil + } } // GetMysqlPort is part of the MysqlDaemon interface. diff --git a/go/vt/mysqlctl/mysqlshellbackupengine.go b/go/vt/mysqlctl/mysqlshellbackupengine.go index 6ee276693b2..2549b29651d 100644 --- a/go/vt/mysqlctl/mysqlshellbackupengine.go +++ b/go/vt/mysqlctl/mysqlshellbackupengine.go @@ -100,7 +100,7 @@ const ( ) func (be *MySQLShellBackupEngine) ExecuteBackup(ctx context.Context, params BackupParams, bh backupstorage.BackupHandle) (result BackupResult, finalErr error) { - params.Logger.Infof("Starting ExecuteBackup in %s", params.TabletAlias) + params.Logger.Infof("running ExecuteBackup in %s", params.TabletAlias) location := path.Join(mysqlShellBackupLocation, bh.Directory(), bh.Name()) @@ -225,7 +225,7 @@ func (be *MySQLShellBackupEngine) ExecuteBackup(ctx context.Context, params Back } func (be *MySQLShellBackupEngine) ExecuteRestore(ctx context.Context, params RestoreParams, bh backupstorage.BackupHandle) (*BackupManifest, error) { - params.Logger.Infof("Calling ExecuteRestore for %s (DeleteBeforeRestore: %v)", params.DbName, params.DeleteBeforeRestore) + params.Logger.Infof("running ExecuteRestore for %s", params.DbName) err := be.restorePreCheck(ctx, params) if err != nil { @@ -248,38 +248,35 @@ func (be *MySQLShellBackupEngine) ExecuteRestore(ctx context.Context, params Res return nil, vterrors.Wrap(err, "disable semi-sync failed") } - // if we received a RestoreFromBackup API call instead of it being a command line argument, - // we need to first clean the host before we start the restore. - if params.DeleteBeforeRestore { - params.Logger.Infof("restoring on an existing tablet, so dropping database %q", params.DbName) + // if super read only is enabled, we need to disable it first so we can then remove existing databases + readonly, err := params.Mysqld.IsSuperReadOnly(ctx) + if err != nil { + return nil, vterrors.Wrap(err, fmt.Sprintf("checking if mysqld has super_read_only=enable: %v", err)) + } - readonly, err := params.Mysqld.IsSuperReadOnly(ctx) + if readonly { + resetFunc, err := params.Mysqld.SetSuperReadOnly(ctx, false) if err != nil { - return nil, vterrors.Wrap(err, fmt.Sprintf("checking if mysqld has super_read_only=enable: %v", err)) + return nil, vterrors.Wrap(err, fmt.Sprintf("unable to disable super-read-only: %v", err)) } - if readonly { - resetFunc, err := params.Mysqld.SetSuperReadOnly(ctx, false) + defer func() { // make sure we enable it back on after the restore is done + err := resetFunc() if err != nil { - return nil, vterrors.Wrap(err, fmt.Sprintf("unable to disable super-read-only: %v", err)) + params.Logger.Errorf("Not able to set super_read_only to its original value after restore") } + }() + } - defer func() { - err := resetFunc() - if err != nil { - params.Logger.Errorf("Not able to set super_read_only to its original value after restore") - } - }() - } - - err = params.Mysqld.ExecuteSuperQueryList(ctx, - []string{fmt.Sprintf("DROP DATABASE IF EXISTS `%s`", params.DbName), - fmt.Sprintf("DROP DATABASE IF EXISTS `%s`", sidecar.GetName())}, - ) - if err != nil { - return nil, vterrors.Wrap(err, fmt.Sprintf("dropping database %q failed", params.DbName)) - } + // we first need to drop existing databases so mysql shell can properly restore them from backup + params.Logger.Infof("restoring on an existing tablet, so dropping database %q", params.DbName) + err = params.Mysqld.ExecuteSuperQueryList(ctx, + []string{fmt.Sprintf("DROP DATABASE IF EXISTS `%s`", params.DbName), + fmt.Sprintf("DROP DATABASE IF EXISTS `%s`", sidecar.GetName())}, + ) + if err != nil { + return nil, vterrors.Wrap(err, fmt.Sprintf("dropping database %q failed", params.DbName)) } // we need to get rid of all the current replication information on the host. diff --git a/go/vt/vttablet/tabletmanager/restore.go b/go/vt/vttablet/tabletmanager/restore.go index 236d048340b..6fdf04c86ce 100644 --- a/go/vt/vttablet/tabletmanager/restore.go +++ b/go/vt/vttablet/tabletmanager/restore.go @@ -128,7 +128,6 @@ func (tm *TabletManager) RestoreData( ctx context.Context, logger logutil.Logger, waitForBackupInterval time.Duration, - deleteBeforeRestore bool, backupTime time.Time, restoreToTimetamp time.Time, restoreToPos string, @@ -181,14 +180,14 @@ func (tm *TabletManager) RestoreData( RestoreToPos: restoreToPos, RestoreToTimestamp: protoutil.TimeToProto(restoreToTimetamp), } - err = tm.restoreDataLocked(ctx, logger, waitForBackupInterval, deleteBeforeRestore, req, mysqlShutdownTimeout) + err = tm.restoreDataLocked(ctx, logger, waitForBackupInterval, req, mysqlShutdownTimeout) if err != nil { return err } return nil } -func (tm *TabletManager) restoreDataLocked(ctx context.Context, logger logutil.Logger, waitForBackupInterval time.Duration, deleteBeforeRestore bool, request *tabletmanagerdatapb.RestoreFromBackupRequest, mysqlShutdownTimeout time.Duration) error { +func (tm *TabletManager) restoreDataLocked(ctx context.Context, logger logutil.Logger, waitForBackupInterval time.Duration, request *tabletmanagerdatapb.RestoreFromBackupRequest, mysqlShutdownTimeout time.Duration) error { tablet := tm.Tablet() originalType := tablet.Type @@ -223,7 +222,6 @@ func (tm *TabletManager) restoreDataLocked(ctx context.Context, logger logutil.L Logger: logger, Concurrency: restoreConcurrency, HookExtraEnv: tm.hookExtraEnv(), - DeleteBeforeRestore: deleteBeforeRestore, DbName: topoproto.TabletDbName(tablet), Keyspace: keyspace, Shard: tablet.Shard, diff --git a/go/vt/vttablet/tabletmanager/rpc_backup.go b/go/vt/vttablet/tabletmanager/rpc_backup.go index 906e34ca9d7..7543ddc3312 100644 --- a/go/vt/vttablet/tabletmanager/rpc_backup.go +++ b/go/vt/vttablet/tabletmanager/rpc_backup.go @@ -190,7 +190,7 @@ func (tm *TabletManager) RestoreFromBackup(ctx context.Context, logger logutil.L l := logutil.NewTeeLogger(logutil.NewConsoleLogger(), logger) // Now we can run restore. - err = tm.restoreDataLocked(ctx, l, 0 /* waitForBackupInterval */, true /* deleteBeforeRestore */, request, mysqlShutdownTimeout) + err = tm.restoreDataLocked(ctx, l, 0 /* waitForBackupInterval */, request, mysqlShutdownTimeout) // Re-run health check to be sure to capture any replication delay. tm.QueryServiceControl.BroadcastHealth() diff --git a/go/vt/vttablet/tabletmanager/tm_init.go b/go/vt/vttablet/tabletmanager/tm_init.go index 2e70596b686..713f724dfd9 100644 --- a/go/vt/vttablet/tabletmanager/tm_init.go +++ b/go/vt/vttablet/tabletmanager/tm_init.go @@ -843,7 +843,7 @@ func (tm *TabletManager) handleRestore(ctx context.Context, config *tabletenv.Ta } // restoreFromBackup will just be a regular action // (same as if it was triggered remotely) - if err := tm.RestoreData(ctx, logutil.NewConsoleLogger(), waitForBackupInterval, false /* deleteBeforeRestore */, backupTime, restoreToTimestamp, restoreToPos, mysqlShutdownTimeout); err != nil { + if err := tm.RestoreData(ctx, logutil.NewConsoleLogger(), waitForBackupInterval, backupTime, restoreToTimestamp, restoreToPos, mysqlShutdownTimeout); err != nil { log.Exitf("RestoreFromBackup failed: %v", err) } diff --git a/go/vt/wrangler/testlib/backup_test.go b/go/vt/wrangler/testlib/backup_test.go index 5e73d266705..75c5673b5f8 100644 --- a/go/vt/wrangler/testlib/backup_test.go +++ b/go/vt/wrangler/testlib/backup_test.go @@ -264,7 +264,7 @@ func testBackupRestore(t *testing.T, cDetails *compressionDetails) error { RelayLogInfoPath: path.Join(root, "relay-log.info"), } - err = destTablet.TM.RestoreData(ctx, logutil.NewConsoleLogger(), 0 /* waitForBackupInterval */, false /* deleteBeforeRestore */, time.Time{} /* backupTime */, time.Time{} /* restoreToTimestamp */, "", mysqlShutdownTimeout) + err = destTablet.TM.RestoreData(ctx, logutil.NewConsoleLogger(), 0 /* waitForBackupInterval */, time.Time{} /* backupTime */, time.Time{} /* restoreToTimestamp */, "", mysqlShutdownTimeout) if err != nil { return err } @@ -304,7 +304,7 @@ func testBackupRestore(t *testing.T, cDetails *compressionDetails) error { primary.FakeMysqlDaemon.SetReplicationPositionPos = primary.FakeMysqlDaemon.CurrentPrimaryPosition // restore primary from latest backup - require.NoError(t, primary.TM.RestoreData(ctx, logutil.NewConsoleLogger(), 0 /* waitForBackupInterval */, false /* deleteBeforeRestore */, time.Time{} /* restoreFromBackupTs */, time.Time{} /* restoreToTimestamp */, "", mysqlShutdownTimeout), + require.NoError(t, primary.TM.RestoreData(ctx, logutil.NewConsoleLogger(), 0 /* waitForBackupInterval */, time.Time{} /* restoreFromBackupTs */, time.Time{} /* restoreToTimestamp */, "", mysqlShutdownTimeout), "RestoreData failed") // tablet was created as PRIMARY, so it's baseTabletType is PRIMARY assert.Equal(t, topodatapb.TabletType_PRIMARY, primary.Tablet.Type) @@ -320,7 +320,7 @@ func testBackupRestore(t *testing.T, cDetails *compressionDetails) error { } // Test restore with the backup timestamp - require.NoError(t, primary.TM.RestoreData(ctx, logutil.NewConsoleLogger(), 0 /* waitForBackupInterval */, false /* deleteBeforeRestore */, backupTime, time.Time{} /* restoreToTimestamp */, "", mysqlShutdownTimeout), + require.NoError(t, primary.TM.RestoreData(ctx, logutil.NewConsoleLogger(), 0 /* waitForBackupInterval */, backupTime, time.Time{} /* restoreToTimestamp */, "", mysqlShutdownTimeout), "RestoreData with backup timestamp failed") assert.Equal(t, topodatapb.TabletType_PRIMARY, primary.Tablet.Type) assert.False(t, primary.FakeMysqlDaemon.Replicating) @@ -522,7 +522,7 @@ func TestBackupRestoreLagged(t *testing.T) { errCh = make(chan error, 1) go func(ctx context.Context, tablet *FakeTablet) { - errCh <- tablet.TM.RestoreData(ctx, logutil.NewConsoleLogger(), 0 /* waitForBackupInterval */, false /* deleteBeforeRestore */, time.Time{} /* restoreFromBackupTs */, time.Time{} /* restoreToTimestamp */, "", mysqlShutdownTimeout) + errCh <- tablet.TM.RestoreData(ctx, logutil.NewConsoleLogger(), 0 /* waitForBackupInterval */, time.Time{} /* restoreFromBackupTs */, time.Time{} /* restoreToTimestamp */, "", mysqlShutdownTimeout) }(ctx, destTablet) timer = time.NewTicker(1 * time.Second) @@ -717,7 +717,7 @@ func TestRestoreUnreachablePrimary(t *testing.T) { // set a short timeout so that we don't have to wait 30 seconds topo.RemoteOperationTimeout = 2 * time.Second // Restore should still succeed - require.NoError(t, destTablet.TM.RestoreData(ctx, logutil.NewConsoleLogger(), 0 /* waitForBackupInterval */, false /* deleteBeforeRestore */, time.Time{} /* restoreFromBackupTs */, time.Time{} /* restoreToTimestamp */, "", mysqlShutdownTimeout)) + require.NoError(t, destTablet.TM.RestoreData(ctx, logutil.NewConsoleLogger(), 0 /* waitForBackupInterval */, time.Time{} /* restoreFromBackupTs */, time.Time{} /* restoreToTimestamp */, "", mysqlShutdownTimeout)) // verify the full status require.NoError(t, destTablet.FakeMysqlDaemon.CheckSuperQueryList(), "destTablet.FakeMysqlDaemon.CheckSuperQueryList failed") assert.True(t, destTablet.FakeMysqlDaemon.Replicating) @@ -872,7 +872,7 @@ func TestDisableActiveReparents(t *testing.T) { RelayLogInfoPath: path.Join(root, "relay-log.info"), } - require.NoError(t, destTablet.TM.RestoreData(ctx, logutil.NewConsoleLogger(), 0 /* waitForBackupInterval */, false /* deleteBeforeRestore */, time.Time{} /* restoreFromBackupTs */, time.Time{} /* restoreToTimestamp */, "", mysqlShutdownTimeout)) + require.NoError(t, destTablet.TM.RestoreData(ctx, logutil.NewConsoleLogger(), 0 /* waitForBackupInterval */, time.Time{} /* restoreFromBackupTs */, time.Time{} /* restoreToTimestamp */, "", mysqlShutdownTimeout)) // verify the full status require.NoError(t, destTablet.FakeMysqlDaemon.CheckSuperQueryList(), "destTablet.FakeMysqlDaemon.CheckSuperQueryList failed") assert.False(t, destTablet.FakeMysqlDaemon.Replicating) From 99f08ea055df71b9a9a07f1fe7216fd73c39d391 Mon Sep 17 00:00:00 2001 From: Renan Rangel Date: Mon, 9 Sep 2024 04:40:46 -0700 Subject: [PATCH 18/34] fixing tests Signed-off-by: Renan Rangel --- go/vt/wrangler/testlib/backup_test.go | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/go/vt/wrangler/testlib/backup_test.go b/go/vt/wrangler/testlib/backup_test.go index 75c5673b5f8..44d7d1c01f3 100644 --- a/go/vt/wrangler/testlib/backup_test.go +++ b/go/vt/wrangler/testlib/backup_test.go @@ -312,12 +312,21 @@ func testBackupRestore(t *testing.T, cDetails *compressionDetails) error { assert.True(t, primary.FakeMysqlDaemon.Running) // restore primary when database already exists - // checkNoDb should return false - // so fake the necessary queries + // we expect it to wipe the current database and + // do the restore so fake the necessary queries primary.FakeMysqlDaemon.FetchSuperQueryMap = map[string]*sqltypes.Result{ "SHOW DATABASES": {Rows: [][]sqltypes.Value{{sqltypes.NewVarBinary("vt_test_keyspace")}}}, "SHOW TABLES FROM `vt_test_keyspace`": {Rows: [][]sqltypes.Value{{sqltypes.NewVarBinary("a")}}}, } + primary.FakeMysqlDaemon.ExpectedExecuteSuperQueryCurrent = 0 + primary.FakeMysqlDaemon.ExpectedExecuteSuperQueryList = []string{ + "FAKE RESET BINARY LOGS AND GTIDS", + "FAKE SET GLOBAL gtid_purged", + "STOP REPLICA", + "FAKE RESET REPLICA ALL", + "FAKE RESET BINARY LOGS AND GTIDS", + "FAKE SET GLOBAL gtid_purged", + } // Test restore with the backup timestamp require.NoError(t, primary.TM.RestoreData(ctx, logutil.NewConsoleLogger(), 0 /* waitForBackupInterval */, backupTime, time.Time{} /* restoreToTimestamp */, "", mysqlShutdownTimeout), From 4257ae453438e5a811d397d0b5752a3c0f78a84e Mon Sep 17 00:00:00 2001 From: Renan Rangel Date: Tue, 10 Sep 2024 08:27:24 -0700 Subject: [PATCH 19/34] Revert "fixing tests" This reverts commit 99f08ea055df71b9a9a07f1fe7216fd73c39d391. Signed-off-by: Renan Rangel --- go/vt/wrangler/testlib/backup_test.go | 13 ++----------- 1 file changed, 2 insertions(+), 11 deletions(-) diff --git a/go/vt/wrangler/testlib/backup_test.go b/go/vt/wrangler/testlib/backup_test.go index 44d7d1c01f3..75c5673b5f8 100644 --- a/go/vt/wrangler/testlib/backup_test.go +++ b/go/vt/wrangler/testlib/backup_test.go @@ -312,21 +312,12 @@ func testBackupRestore(t *testing.T, cDetails *compressionDetails) error { assert.True(t, primary.FakeMysqlDaemon.Running) // restore primary when database already exists - // we expect it to wipe the current database and - // do the restore so fake the necessary queries + // checkNoDb should return false + // so fake the necessary queries primary.FakeMysqlDaemon.FetchSuperQueryMap = map[string]*sqltypes.Result{ "SHOW DATABASES": {Rows: [][]sqltypes.Value{{sqltypes.NewVarBinary("vt_test_keyspace")}}}, "SHOW TABLES FROM `vt_test_keyspace`": {Rows: [][]sqltypes.Value{{sqltypes.NewVarBinary("a")}}}, } - primary.FakeMysqlDaemon.ExpectedExecuteSuperQueryCurrent = 0 - primary.FakeMysqlDaemon.ExpectedExecuteSuperQueryList = []string{ - "FAKE RESET BINARY LOGS AND GTIDS", - "FAKE SET GLOBAL gtid_purged", - "STOP REPLICA", - "FAKE RESET REPLICA ALL", - "FAKE RESET BINARY LOGS AND GTIDS", - "FAKE SET GLOBAL gtid_purged", - } // Test restore with the backup timestamp require.NoError(t, primary.TM.RestoreData(ctx, logutil.NewConsoleLogger(), 0 /* waitForBackupInterval */, backupTime, time.Time{} /* restoreToTimestamp */, "", mysqlShutdownTimeout), From e958558a90cff7406fd631a2e9e31f68555f0e47 Mon Sep 17 00:00:00 2001 From: Renan Rangel Date: Tue, 10 Sep 2024 08:27:43 -0700 Subject: [PATCH 20/34] Revert "removing DeleteBeforeRestore" This reverts commit e62ae955ea9514fa52ecb14fccd112e50c1d2940. Signed-off-by: Renan Rangel --- go/cmd/vtbackup/cli/vtbackup.go | 1 + go/vt/mysqlctl/backup.go | 28 ++++++++++++- go/vt/mysqlctl/backup_blackbox_test.go | 1 + go/vt/mysqlctl/backup_test.go | 32 +++++++------- go/vt/mysqlctl/backupengine.go | 5 +++ go/vt/mysqlctl/fakemysqldaemon.go | 18 +------- go/vt/mysqlctl/mysqlshellbackupengine.go | 49 ++++++++++++---------- go/vt/vttablet/tabletmanager/restore.go | 6 ++- go/vt/vttablet/tabletmanager/rpc_backup.go | 2 +- go/vt/vttablet/tabletmanager/tm_init.go | 2 +- go/vt/wrangler/testlib/backup_test.go | 12 +++--- 11 files changed, 91 insertions(+), 65 deletions(-) diff --git a/go/cmd/vtbackup/cli/vtbackup.go b/go/cmd/vtbackup/cli/vtbackup.go index c4379dac09d..1b61c886ae7 100644 --- a/go/cmd/vtbackup/cli/vtbackup.go +++ b/go/cmd/vtbackup/cli/vtbackup.go @@ -428,6 +428,7 @@ func takeBackup(ctx, backgroundCtx context.Context, topoServer *topo.Server, bac Logger: logutil.NewConsoleLogger(), Concurrency: concurrency, HookExtraEnv: extraEnv, + DeleteBeforeRestore: true, DbName: dbName, Keyspace: initKeyspace, Shard: initShard, diff --git a/go/vt/mysqlctl/backup.go b/go/vt/mysqlctl/backup.go index 8d10c31f847..fd067b02887 100644 --- a/go/vt/mysqlctl/backup.go +++ b/go/vt/mysqlctl/backup.go @@ -237,6 +237,29 @@ func ParseBackupName(dir string, name string) (backupTime *time.Time, alias *top return backupTime, alias, nil } +// checkNoDB makes sure there is no user data already there. +// Used by Restore, as we do not want to destroy an existing DB. +// The user's database name must be given since we ignore all others. +// Returns (true, nil) if the specified DB doesn't exist. +// Returns (false, nil) if the check succeeds but the condition is not +// satisfied (there is a DB). +// Returns (false, non-nil error) if one occurs while trying to perform the check. +func checkNoDB(ctx context.Context, mysqld MysqlDaemon, dbName string) (bool, error) { + qr, err := mysqld.FetchSuperQuery(ctx, "SHOW DATABASES") + if err != nil { + return false, vterrors.Wrap(err, "checkNoDB failed") + } + + for _, row := range qr.Rows { + if row[0].ToString() == dbName { + // found active db + log.Warningf("checkNoDB failed, found active db %v", dbName) + return false, nil + } + } + return true, nil +} + // removeExistingFiles will delete existing files in the data dir to prevent // conflicts with the restored archive. In particular, binlogs can be created // even during initial bootstrap, and these can interfere with configuring @@ -289,9 +312,10 @@ func removeExistingFiles(cnf *Mycnf) error { // ShouldRestore checks whether a database with tables already exists // and returns whether a restore action should be performed func ShouldRestore(ctx context.Context, params RestoreParams) (bool, error) { - if RestoreWasInterrupted(params.Cnf) { + if params.DeleteBeforeRestore || RestoreWasInterrupted(params.Cnf) { return true, nil } + params.Logger.Infof("Restore: No %v file found, checking no existing data is present", RestoreState) // Wait for mysqld to be ready, in case it was launched in parallel with us. // If this doesn't succeed, we should not attempt a restore if err := params.Mysqld.Wait(ctx, params.Cnf); err != nil { @@ -301,7 +325,7 @@ func ShouldRestore(ctx context.Context, params RestoreParams) (bool, error) { params.Logger.Errorf("error waiting for the grants: %v", err) return false, err } - return true, nil + return checkNoDB(ctx, params.Mysqld, params.DbName) } // ensureRestoredGTIDPurgedMatchesManifest sees the following: when you restore a full backup, you want the MySQL server to have diff --git a/go/vt/mysqlctl/backup_blackbox_test.go b/go/vt/mysqlctl/backup_blackbox_test.go index ad583b047de..15244fb8782 100644 --- a/go/vt/mysqlctl/backup_blackbox_test.go +++ b/go/vt/mysqlctl/backup_blackbox_test.go @@ -512,6 +512,7 @@ func TestExecuteRestoreWithTimedOutContext(t *testing.T) { Mysqld: mysqld, Concurrency: 2, HookExtraEnv: map[string]string{}, + DeleteBeforeRestore: false, DbName: "test", Keyspace: "test", Shard: "-", diff --git a/go/vt/mysqlctl/backup_test.go b/go/vt/mysqlctl/backup_test.go index 5a7feafbdf5..86b34fbbf99 100644 --- a/go/vt/mysqlctl/backup_test.go +++ b/go/vt/mysqlctl/backup_test.go @@ -36,6 +36,7 @@ import ( "vitess.io/vitess/go/mysql" "vitess.io/vitess/go/mysql/fakesqldb" "vitess.io/vitess/go/mysql/replication" + "vitess.io/vitess/go/sqltypes" "vitess.io/vitess/go/test/utils" "vitess.io/vitess/go/vt/logutil" "vitess.io/vitess/go/vt/mysqlctl/backupstats" @@ -584,6 +585,7 @@ func createFakeBackupRestoreEnv(t *testing.T) *fakeBackupRestoreEnv { Mysqld: mysqld, Concurrency: 1, HookExtraEnv: map[string]string{}, + DeleteBeforeRestore: false, DbName: "test", Keyspace: "test", Shard: "-", @@ -688,26 +690,28 @@ func TestShouldRestore(t *testing.T) { env := createFakeBackupRestoreEnv(t) b, err := ShouldRestore(env.ctx, env.restoreParams) - assert.True(t, b) - assert.NoError(t, err) - - // if MySQL is online on time, we should continue the restore - env.mysqld.WaitDuration = time.Second - ctx, cancel := context.WithTimeout(context.Background(), time.Second*3) - defer cancel() + assert.False(t, b) + assert.Error(t, err) - b, err = ShouldRestore(ctx, env.restoreParams) + env.restoreParams.DeleteBeforeRestore = true + b, err = ShouldRestore(env.ctx, env.restoreParams) assert.True(t, b) assert.NoError(t, err) + env.restoreParams.DeleteBeforeRestore = false - // but should return an error if waiting for MySQL longer than the context expected - env.mysqld.WaitDuration = time.Second * 2 - ctx, cancel = context.WithTimeout(context.Background(), time.Second) - defer cancel() + env.mysqld.FetchSuperQueryMap = map[string]*sqltypes.Result{ + "SHOW DATABASES": {Rows: [][]sqltypes.Value{{sqltypes.NewVarBinary("any_db")}}}, + } + b, err = ShouldRestore(env.ctx, env.restoreParams) + assert.NoError(t, err) + assert.True(t, b) - b, err = ShouldRestore(ctx, env.restoreParams) + env.mysqld.FetchSuperQueryMap = map[string]*sqltypes.Result{ + "SHOW DATABASES": {Rows: [][]sqltypes.Value{{sqltypes.NewVarBinary("test")}}}, + } + b, err = ShouldRestore(env.ctx, env.restoreParams) assert.False(t, b) - assert.ErrorIs(t, err, context.DeadlineExceeded) + assert.NoError(t, err) } func TestScanLinesToLogger(t *testing.T) { diff --git a/go/vt/mysqlctl/backupengine.go b/go/vt/mysqlctl/backupengine.go index dcd41e8eb26..c483aff3d78 100644 --- a/go/vt/mysqlctl/backupengine.go +++ b/go/vt/mysqlctl/backupengine.go @@ -118,6 +118,10 @@ type RestoreParams struct { Concurrency int // Extra env variables for pre-restore and post-restore transform hooks HookExtraEnv map[string]string + // DeleteBeforeRestore tells us whether existing data should be deleted before + // restoring. This is always set to false when starting a tablet with -restore_from_backup, + // but is set to true when executing a RestoreFromBackup command on an already running vttablet + DeleteBeforeRestore bool // DbName is the name of the managed database / schema DbName string // Keyspace and Shard are used to infer the directory where backups are stored @@ -148,6 +152,7 @@ func (p *RestoreParams) Copy() RestoreParams { Logger: p.Logger, Concurrency: p.Concurrency, HookExtraEnv: p.HookExtraEnv, + DeleteBeforeRestore: p.DeleteBeforeRestore, DbName: p.DbName, Keyspace: p.Keyspace, Shard: p.Shard, diff --git a/go/vt/mysqlctl/fakemysqldaemon.go b/go/vt/mysqlctl/fakemysqldaemon.go index cfa3188c049..b9175a32779 100644 --- a/go/vt/mysqlctl/fakemysqldaemon.go +++ b/go/vt/mysqlctl/fakemysqldaemon.go @@ -139,10 +139,6 @@ type FakeMysqlDaemon struct { // function returns an error. WaitPrimaryPositions []replication.Position - // WaitDuration is used by the Wait() and WaitForDBAGrants() calls to - // simulate and arbitrary amount of time it would take waiting for mysql - WaitDuration time.Duration - // PromoteResult is returned by Promote. PromoteResult replication.Position @@ -275,21 +271,11 @@ func (fmd *FakeMysqlDaemon) RefreshConfig(ctx context.Context, cnf *Mycnf) error // Wait is part of the MysqlDaemon interface. func (fmd *FakeMysqlDaemon) Wait(ctx context.Context, cnf *Mycnf) error { - select { - case <-ctx.Done(): - return ctx.Err() - case <-time.After(fmd.WaitDuration): - return nil - } + return nil } func (fmd *FakeMysqlDaemon) WaitForDBAGrants(ctx context.Context, waitTime time.Duration) (err error) { - select { - case <-ctx.Done(): - return ctx.Err() - case <-time.After(fmd.WaitDuration): - return nil - } + return nil } // GetMysqlPort is part of the MysqlDaemon interface. diff --git a/go/vt/mysqlctl/mysqlshellbackupengine.go b/go/vt/mysqlctl/mysqlshellbackupengine.go index 2549b29651d..6ee276693b2 100644 --- a/go/vt/mysqlctl/mysqlshellbackupengine.go +++ b/go/vt/mysqlctl/mysqlshellbackupengine.go @@ -100,7 +100,7 @@ const ( ) func (be *MySQLShellBackupEngine) ExecuteBackup(ctx context.Context, params BackupParams, bh backupstorage.BackupHandle) (result BackupResult, finalErr error) { - params.Logger.Infof("running ExecuteBackup in %s", params.TabletAlias) + params.Logger.Infof("Starting ExecuteBackup in %s", params.TabletAlias) location := path.Join(mysqlShellBackupLocation, bh.Directory(), bh.Name()) @@ -225,7 +225,7 @@ func (be *MySQLShellBackupEngine) ExecuteBackup(ctx context.Context, params Back } func (be *MySQLShellBackupEngine) ExecuteRestore(ctx context.Context, params RestoreParams, bh backupstorage.BackupHandle) (*BackupManifest, error) { - params.Logger.Infof("running ExecuteRestore for %s", params.DbName) + params.Logger.Infof("Calling ExecuteRestore for %s (DeleteBeforeRestore: %v)", params.DbName, params.DeleteBeforeRestore) err := be.restorePreCheck(ctx, params) if err != nil { @@ -248,35 +248,38 @@ func (be *MySQLShellBackupEngine) ExecuteRestore(ctx context.Context, params Res return nil, vterrors.Wrap(err, "disable semi-sync failed") } - // if super read only is enabled, we need to disable it first so we can then remove existing databases - readonly, err := params.Mysqld.IsSuperReadOnly(ctx) - if err != nil { - return nil, vterrors.Wrap(err, fmt.Sprintf("checking if mysqld has super_read_only=enable: %v", err)) - } + // if we received a RestoreFromBackup API call instead of it being a command line argument, + // we need to first clean the host before we start the restore. + if params.DeleteBeforeRestore { + params.Logger.Infof("restoring on an existing tablet, so dropping database %q", params.DbName) - if readonly { - resetFunc, err := params.Mysqld.SetSuperReadOnly(ctx, false) + readonly, err := params.Mysqld.IsSuperReadOnly(ctx) if err != nil { - return nil, vterrors.Wrap(err, fmt.Sprintf("unable to disable super-read-only: %v", err)) + return nil, vterrors.Wrap(err, fmt.Sprintf("checking if mysqld has super_read_only=enable: %v", err)) } - defer func() { // make sure we enable it back on after the restore is done - err := resetFunc() + if readonly { + resetFunc, err := params.Mysqld.SetSuperReadOnly(ctx, false) if err != nil { - params.Logger.Errorf("Not able to set super_read_only to its original value after restore") + return nil, vterrors.Wrap(err, fmt.Sprintf("unable to disable super-read-only: %v", err)) } - }() - } - // we first need to drop existing databases so mysql shell can properly restore them from backup - params.Logger.Infof("restoring on an existing tablet, so dropping database %q", params.DbName) + defer func() { + err := resetFunc() + if err != nil { + params.Logger.Errorf("Not able to set super_read_only to its original value after restore") + } + }() + } + + err = params.Mysqld.ExecuteSuperQueryList(ctx, + []string{fmt.Sprintf("DROP DATABASE IF EXISTS `%s`", params.DbName), + fmt.Sprintf("DROP DATABASE IF EXISTS `%s`", sidecar.GetName())}, + ) + if err != nil { + return nil, vterrors.Wrap(err, fmt.Sprintf("dropping database %q failed", params.DbName)) + } - err = params.Mysqld.ExecuteSuperQueryList(ctx, - []string{fmt.Sprintf("DROP DATABASE IF EXISTS `%s`", params.DbName), - fmt.Sprintf("DROP DATABASE IF EXISTS `%s`", sidecar.GetName())}, - ) - if err != nil { - return nil, vterrors.Wrap(err, fmt.Sprintf("dropping database %q failed", params.DbName)) } // we need to get rid of all the current replication information on the host. diff --git a/go/vt/vttablet/tabletmanager/restore.go b/go/vt/vttablet/tabletmanager/restore.go index 6fdf04c86ce..236d048340b 100644 --- a/go/vt/vttablet/tabletmanager/restore.go +++ b/go/vt/vttablet/tabletmanager/restore.go @@ -128,6 +128,7 @@ func (tm *TabletManager) RestoreData( ctx context.Context, logger logutil.Logger, waitForBackupInterval time.Duration, + deleteBeforeRestore bool, backupTime time.Time, restoreToTimetamp time.Time, restoreToPos string, @@ -180,14 +181,14 @@ func (tm *TabletManager) RestoreData( RestoreToPos: restoreToPos, RestoreToTimestamp: protoutil.TimeToProto(restoreToTimetamp), } - err = tm.restoreDataLocked(ctx, logger, waitForBackupInterval, req, mysqlShutdownTimeout) + err = tm.restoreDataLocked(ctx, logger, waitForBackupInterval, deleteBeforeRestore, req, mysqlShutdownTimeout) if err != nil { return err } return nil } -func (tm *TabletManager) restoreDataLocked(ctx context.Context, logger logutil.Logger, waitForBackupInterval time.Duration, request *tabletmanagerdatapb.RestoreFromBackupRequest, mysqlShutdownTimeout time.Duration) error { +func (tm *TabletManager) restoreDataLocked(ctx context.Context, logger logutil.Logger, waitForBackupInterval time.Duration, deleteBeforeRestore bool, request *tabletmanagerdatapb.RestoreFromBackupRequest, mysqlShutdownTimeout time.Duration) error { tablet := tm.Tablet() originalType := tablet.Type @@ -222,6 +223,7 @@ func (tm *TabletManager) restoreDataLocked(ctx context.Context, logger logutil.L Logger: logger, Concurrency: restoreConcurrency, HookExtraEnv: tm.hookExtraEnv(), + DeleteBeforeRestore: deleteBeforeRestore, DbName: topoproto.TabletDbName(tablet), Keyspace: keyspace, Shard: tablet.Shard, diff --git a/go/vt/vttablet/tabletmanager/rpc_backup.go b/go/vt/vttablet/tabletmanager/rpc_backup.go index 7543ddc3312..906e34ca9d7 100644 --- a/go/vt/vttablet/tabletmanager/rpc_backup.go +++ b/go/vt/vttablet/tabletmanager/rpc_backup.go @@ -190,7 +190,7 @@ func (tm *TabletManager) RestoreFromBackup(ctx context.Context, logger logutil.L l := logutil.NewTeeLogger(logutil.NewConsoleLogger(), logger) // Now we can run restore. - err = tm.restoreDataLocked(ctx, l, 0 /* waitForBackupInterval */, request, mysqlShutdownTimeout) + err = tm.restoreDataLocked(ctx, l, 0 /* waitForBackupInterval */, true /* deleteBeforeRestore */, request, mysqlShutdownTimeout) // Re-run health check to be sure to capture any replication delay. tm.QueryServiceControl.BroadcastHealth() diff --git a/go/vt/vttablet/tabletmanager/tm_init.go b/go/vt/vttablet/tabletmanager/tm_init.go index 713f724dfd9..2e70596b686 100644 --- a/go/vt/vttablet/tabletmanager/tm_init.go +++ b/go/vt/vttablet/tabletmanager/tm_init.go @@ -843,7 +843,7 @@ func (tm *TabletManager) handleRestore(ctx context.Context, config *tabletenv.Ta } // restoreFromBackup will just be a regular action // (same as if it was triggered remotely) - if err := tm.RestoreData(ctx, logutil.NewConsoleLogger(), waitForBackupInterval, backupTime, restoreToTimestamp, restoreToPos, mysqlShutdownTimeout); err != nil { + if err := tm.RestoreData(ctx, logutil.NewConsoleLogger(), waitForBackupInterval, false /* deleteBeforeRestore */, backupTime, restoreToTimestamp, restoreToPos, mysqlShutdownTimeout); err != nil { log.Exitf("RestoreFromBackup failed: %v", err) } diff --git a/go/vt/wrangler/testlib/backup_test.go b/go/vt/wrangler/testlib/backup_test.go index 75c5673b5f8..5e73d266705 100644 --- a/go/vt/wrangler/testlib/backup_test.go +++ b/go/vt/wrangler/testlib/backup_test.go @@ -264,7 +264,7 @@ func testBackupRestore(t *testing.T, cDetails *compressionDetails) error { RelayLogInfoPath: path.Join(root, "relay-log.info"), } - err = destTablet.TM.RestoreData(ctx, logutil.NewConsoleLogger(), 0 /* waitForBackupInterval */, time.Time{} /* backupTime */, time.Time{} /* restoreToTimestamp */, "", mysqlShutdownTimeout) + err = destTablet.TM.RestoreData(ctx, logutil.NewConsoleLogger(), 0 /* waitForBackupInterval */, false /* deleteBeforeRestore */, time.Time{} /* backupTime */, time.Time{} /* restoreToTimestamp */, "", mysqlShutdownTimeout) if err != nil { return err } @@ -304,7 +304,7 @@ func testBackupRestore(t *testing.T, cDetails *compressionDetails) error { primary.FakeMysqlDaemon.SetReplicationPositionPos = primary.FakeMysqlDaemon.CurrentPrimaryPosition // restore primary from latest backup - require.NoError(t, primary.TM.RestoreData(ctx, logutil.NewConsoleLogger(), 0 /* waitForBackupInterval */, time.Time{} /* restoreFromBackupTs */, time.Time{} /* restoreToTimestamp */, "", mysqlShutdownTimeout), + require.NoError(t, primary.TM.RestoreData(ctx, logutil.NewConsoleLogger(), 0 /* waitForBackupInterval */, false /* deleteBeforeRestore */, time.Time{} /* restoreFromBackupTs */, time.Time{} /* restoreToTimestamp */, "", mysqlShutdownTimeout), "RestoreData failed") // tablet was created as PRIMARY, so it's baseTabletType is PRIMARY assert.Equal(t, topodatapb.TabletType_PRIMARY, primary.Tablet.Type) @@ -320,7 +320,7 @@ func testBackupRestore(t *testing.T, cDetails *compressionDetails) error { } // Test restore with the backup timestamp - require.NoError(t, primary.TM.RestoreData(ctx, logutil.NewConsoleLogger(), 0 /* waitForBackupInterval */, backupTime, time.Time{} /* restoreToTimestamp */, "", mysqlShutdownTimeout), + require.NoError(t, primary.TM.RestoreData(ctx, logutil.NewConsoleLogger(), 0 /* waitForBackupInterval */, false /* deleteBeforeRestore */, backupTime, time.Time{} /* restoreToTimestamp */, "", mysqlShutdownTimeout), "RestoreData with backup timestamp failed") assert.Equal(t, topodatapb.TabletType_PRIMARY, primary.Tablet.Type) assert.False(t, primary.FakeMysqlDaemon.Replicating) @@ -522,7 +522,7 @@ func TestBackupRestoreLagged(t *testing.T) { errCh = make(chan error, 1) go func(ctx context.Context, tablet *FakeTablet) { - errCh <- tablet.TM.RestoreData(ctx, logutil.NewConsoleLogger(), 0 /* waitForBackupInterval */, time.Time{} /* restoreFromBackupTs */, time.Time{} /* restoreToTimestamp */, "", mysqlShutdownTimeout) + errCh <- tablet.TM.RestoreData(ctx, logutil.NewConsoleLogger(), 0 /* waitForBackupInterval */, false /* deleteBeforeRestore */, time.Time{} /* restoreFromBackupTs */, time.Time{} /* restoreToTimestamp */, "", mysqlShutdownTimeout) }(ctx, destTablet) timer = time.NewTicker(1 * time.Second) @@ -717,7 +717,7 @@ func TestRestoreUnreachablePrimary(t *testing.T) { // set a short timeout so that we don't have to wait 30 seconds topo.RemoteOperationTimeout = 2 * time.Second // Restore should still succeed - require.NoError(t, destTablet.TM.RestoreData(ctx, logutil.NewConsoleLogger(), 0 /* waitForBackupInterval */, time.Time{} /* restoreFromBackupTs */, time.Time{} /* restoreToTimestamp */, "", mysqlShutdownTimeout)) + require.NoError(t, destTablet.TM.RestoreData(ctx, logutil.NewConsoleLogger(), 0 /* waitForBackupInterval */, false /* deleteBeforeRestore */, time.Time{} /* restoreFromBackupTs */, time.Time{} /* restoreToTimestamp */, "", mysqlShutdownTimeout)) // verify the full status require.NoError(t, destTablet.FakeMysqlDaemon.CheckSuperQueryList(), "destTablet.FakeMysqlDaemon.CheckSuperQueryList failed") assert.True(t, destTablet.FakeMysqlDaemon.Replicating) @@ -872,7 +872,7 @@ func TestDisableActiveReparents(t *testing.T) { RelayLogInfoPath: path.Join(root, "relay-log.info"), } - require.NoError(t, destTablet.TM.RestoreData(ctx, logutil.NewConsoleLogger(), 0 /* waitForBackupInterval */, time.Time{} /* restoreFromBackupTs */, time.Time{} /* restoreToTimestamp */, "", mysqlShutdownTimeout)) + require.NoError(t, destTablet.TM.RestoreData(ctx, logutil.NewConsoleLogger(), 0 /* waitForBackupInterval */, false /* deleteBeforeRestore */, time.Time{} /* restoreFromBackupTs */, time.Time{} /* restoreToTimestamp */, "", mysqlShutdownTimeout)) // verify the full status require.NoError(t, destTablet.FakeMysqlDaemon.CheckSuperQueryList(), "destTablet.FakeMysqlDaemon.CheckSuperQueryList failed") assert.False(t, destTablet.FakeMysqlDaemon.Replicating) From 2e8c4415cbf7de01312042c1e18e67107c142624 Mon Sep 17 00:00:00 2001 From: Renan Rangel Date: Tue, 10 Sep 2024 08:28:51 -0700 Subject: [PATCH 21/34] make generate_ci_workflows Signed-off-by: Renan Rangel --- .../cluster_endtoend_backup_pitr_mysqlshell.yml | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/.github/workflows/cluster_endtoend_backup_pitr_mysqlshell.yml b/.github/workflows/cluster_endtoend_backup_pitr_mysqlshell.yml index d7fd48761e2..7421a1ed135 100644 --- a/.github/workflows/cluster_endtoend_backup_pitr_mysqlshell.yml +++ b/.github/workflows/cluster_endtoend_backup_pitr_mysqlshell.yml @@ -16,7 +16,7 @@ env: jobs: build: name: Run endtoend tests on Cluster (backup_pitr_mysqlshell) - runs-on: gh-hosted-runners-4cores-1 + runs-on: ubuntu-latest steps: - name: Skip CI @@ -45,11 +45,11 @@ jobs: - name: Check out code if: steps.skip-workflow.outputs.skip-workflow == 'false' - uses: actions/checkout@v4 + uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7 - name: Check for changes in relevant files if: steps.skip-workflow.outputs.skip-workflow == 'false' - uses: dorny/paths-filter@v3.0.1 + uses: dorny/paths-filter@ebc4d7e9ebcb0b1eb21480bb8f43113e996ac77a # v3.0.1 id: changes with: token: '' @@ -71,13 +71,13 @@ jobs: - name: Set up Go if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' - uses: actions/setup-go@v5 + uses: actions/setup-go@0a12ed9d6a96ab950c8f026ed9f722fe0da7ef32 # v5.0.2 with: go-version: 1.23.0 - name: Set up python if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' - uses: actions/setup-python@v5 + uses: actions/setup-python@39cd14951b08e74b54015e9e001cdefcf80e669f # v5.1.1 - name: Tune the OS if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' @@ -151,7 +151,7 @@ jobs: - name: Test Summary if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' && always() - uses: test-summary/action@v2 + uses: test-summary/action@31493c76ec9e7aa675f1585d3ed6f1da69269a86 # v2.4 with: paths: "report.xml" show: "fail, skip" From 75b2b76a3ff25fc7ab4dc682dd88323e01e002b9 Mon Sep 17 00:00:00 2001 From: Renan Rangel Date: Tue, 10 Sep 2024 09:30:12 -0700 Subject: [PATCH 22/34] make generate_ci_workflows Signed-off-by: Renan Rangel --- .github/workflows/cluster_endtoend_backup_pitr_mysqlshell.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/cluster_endtoend_backup_pitr_mysqlshell.yml b/.github/workflows/cluster_endtoend_backup_pitr_mysqlshell.yml index 7421a1ed135..8d26ccb6d93 100644 --- a/.github/workflows/cluster_endtoend_backup_pitr_mysqlshell.yml +++ b/.github/workflows/cluster_endtoend_backup_pitr_mysqlshell.yml @@ -73,7 +73,7 @@ jobs: if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' uses: actions/setup-go@0a12ed9d6a96ab950c8f026ed9f722fe0da7ef32 # v5.0.2 with: - go-version: 1.23.0 + go-version: 1.23.1 - name: Set up python if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' From 85fba81544f41d28c9854a63ce8585e4c6844162 Mon Sep 17 00:00:00 2001 From: Renan Rangel Date: Thu, 12 Sep 2024 08:25:03 -0700 Subject: [PATCH 23/34] deleting users + other dbs Signed-off-by: Renan Rangel --- go/vt/mysqlctl/mysqlshellbackupengine.go | 117 ++++++++++++++---- go/vt/mysqlctl/mysqlshellbackupengine_test.go | 93 ++++++++++++++ 2 files changed, 183 insertions(+), 27 deletions(-) diff --git a/go/vt/mysqlctl/mysqlshellbackupengine.go b/go/vt/mysqlctl/mysqlshellbackupengine.go index 6ee276693b2..8d2b3f61b0f 100644 --- a/go/vt/mysqlctl/mysqlshellbackupengine.go +++ b/go/vt/mysqlctl/mysqlshellbackupengine.go @@ -26,15 +26,17 @@ import ( "os" "os/exec" "path" + "slices" "strings" "sync" "time" "github.com/spf13/pflag" - "vitess.io/vitess/go/constants/sidecar" "vitess.io/vitess/go/mysql" "vitess.io/vitess/go/mysql/capabilities" + "vitess.io/vitess/go/vt/log" + "vitess.io/vitess/go/vt/logutil" "vitess.io/vitess/go/vt/mysqlctl/backupstorage" tabletmanagerdatapb "vitess.io/vitess/go/vt/proto/tabletmanagerdata" "vitess.io/vitess/go/vt/servenv" @@ -49,7 +51,7 @@ var ( // flags passed to the Dump command, as a JSON string mysqlShellDumpFlags = `{"threads": 2}` // flags passed to the Load command, as a JSON string - mysqlShellLoadFlags = `{"threads": 4, "updateGtidSet": "replace", "skipBinlog": true, "progressFile": ""}` + mysqlShellLoadFlags = `{"threads": 4, "loadUsers": true, "excludeUsers": ["'root'@'localhost'"], "updateGtidSet": "replace", "skipBinlog": true, "progressFile": ""}` // drain a tablet when taking a backup mysqlShellBackupShouldDrain = false // disable redo logging and double write buffer @@ -59,6 +61,15 @@ var ( knownObjectStoreParams = []string{"s3BucketName", "osBucketName", "azureContainerName"} MySQLShellPreCheckError = errors.New("MySQLShellPreCheckError") + + // internal databases not backed up by MySQL Shell + internalDBs = []string{ + "information_schema", "mysql", "ndbinfo", "performance_schema", "sys", + } + // reserved MySQL users https://dev.mysql.com/doc/refman/8.0/en/reserved-accounts.html + reservedUsers = []string{ + "mysql.sys@localhost", "mysql.session@localhost", "mysql.infoschema@localhost", + } ) // MySQLShellBackupManifest represents a backup. @@ -248,38 +259,32 @@ func (be *MySQLShellBackupEngine) ExecuteRestore(ctx context.Context, params Res return nil, vterrors.Wrap(err, "disable semi-sync failed") } - // if we received a RestoreFromBackup API call instead of it being a command line argument, - // we need to first clean the host before we start the restore. - if params.DeleteBeforeRestore { - params.Logger.Infof("restoring on an existing tablet, so dropping database %q", params.DbName) + params.Logger.Infof("restoring on an existing tablet, so dropping database %q", params.DbName) + + readonly, err := params.Mysqld.IsSuperReadOnly(ctx) + if err != nil { + return nil, vterrors.Wrap(err, fmt.Sprintf("checking if mysqld has super_read_only=enable: %v", err)) + } - readonly, err := params.Mysqld.IsSuperReadOnly(ctx) + if readonly { + resetFunc, err := params.Mysqld.SetSuperReadOnly(ctx, false) if err != nil { - return nil, vterrors.Wrap(err, fmt.Sprintf("checking if mysqld has super_read_only=enable: %v", err)) + return nil, vterrors.Wrap(err, fmt.Sprintf("unable to disable super-read-only: %v", err)) } - if readonly { - resetFunc, err := params.Mysqld.SetSuperReadOnly(ctx, false) + defer func() { + err := resetFunc() if err != nil { - return nil, vterrors.Wrap(err, fmt.Sprintf("unable to disable super-read-only: %v", err)) + params.Logger.Errorf("Not able to set super_read_only to its original value after restore") } + }() + } - defer func() { - err := resetFunc() - if err != nil { - params.Logger.Errorf("Not able to set super_read_only to its original value after restore") - } - }() - } - - err = params.Mysqld.ExecuteSuperQueryList(ctx, - []string{fmt.Sprintf("DROP DATABASE IF EXISTS `%s`", params.DbName), - fmt.Sprintf("DROP DATABASE IF EXISTS `%s`", sidecar.GetName())}, - ) - if err != nil { - return nil, vterrors.Wrap(err, fmt.Sprintf("dropping database %q failed", params.DbName)) - } - + err = cleanupMySQL(ctx, params.Mysqld, params.Logger) + if err != nil { + log.Errorf(err.Error()) + // time.Sleep(time.Minute * 2) + return nil, vterrors.Wrap(err, "error cleaning MySQL") } // we need to get rid of all the current replication information on the host. @@ -505,6 +510,64 @@ func releaseReadLock(ctx context.Context, reader io.Reader, params BackupParams, } } +func cleanupMySQL(ctx context.Context, mysql MysqlDaemon, logger logutil.Logger) error { + logger.Infof("Cleaning up MySQL ahead of a restore") + result, err := mysql.FetchSuperQuery(ctx, "SHOW DATABASES") + if err != nil { + return err + } + + // drop all databases + for _, row := range result.Rows { + dbName := row[0].ToString() + if slices.Contains(internalDBs, dbName) { + continue // not dropping internal DBs + } + + logger.Infof("Dropping DB %q", dbName) + err = mysql.ExecuteSuperQuery(ctx, fmt.Sprintf("DROP DATABASE IF EXISTS `%s`", row[0].ToString())) + if err != nil { + return fmt.Errorf("error droppping database %q: %w", row[0].ToString(), err) + } + } + + // get current user + var currentUser string + result, err = mysql.FetchSuperQuery(ctx, "SELECT user()") + if err != nil { + return fmt.Errorf("error fetching current user: %w", err) + } + + for _, row := range result.Rows { + currentUser = row[0].ToString() + } + + // drop all users except reserved ones + result, err = mysql.FetchSuperQuery(ctx, "SELECT user, host FROM mysql.user") + if err != nil { + return err + } + + for _, row := range result.Rows { + user := fmt.Sprintf("%s@%s", row[0].ToString(), row[1].ToString()) + + if user == currentUser { + continue // we don't drop the current user + } + if slices.Contains(reservedUsers, user) { + continue // we skip reserved MySQL users + } + + logger.Infof("Dropping User %q", user) + err = mysql.ExecuteSuperQuery(ctx, fmt.Sprintf("DROP USER '%s'@'%s'", row[0].ToString(), row[1].ToString())) + if err != nil { + return fmt.Errorf("error droppping user %q: %w", user, err) + } + } + + return err +} + func init() { BackupRestoreEngineMap[mysqlShellBackupEngineName] = &MySQLShellBackupEngine{} } diff --git a/go/vt/mysqlctl/mysqlshellbackupengine_test.go b/go/vt/mysqlctl/mysqlshellbackupengine_test.go index bb5aec65a4c..5f1f13e5438 100644 --- a/go/vt/mysqlctl/mysqlshellbackupengine_test.go +++ b/go/vt/mysqlctl/mysqlshellbackupengine_test.go @@ -18,6 +18,7 @@ package mysqlctl import ( "context" + "fmt" "path" "testing" @@ -25,6 +26,8 @@ import ( "github.com/stretchr/testify/require" "vitess.io/vitess/go/mysql/fakesqldb" + "vitess.io/vitess/go/sqltypes" + "vitess.io/vitess/go/vt/logutil" tabletmanagerdatapb "vitess.io/vitess/go/vt/proto/tabletmanagerdata" ) @@ -175,3 +178,93 @@ func TestShouldDrainForBackupMySQLShell(t *testing.T) { assert.True(t, engine.ShouldDrainForBackup(nil)) assert.True(t, engine.ShouldDrainForBackup(&tabletmanagerdatapb.BackupRequest{})) } + +func TestCleanupMySQL(t *testing.T) { + type userRecord struct { + user, host string + } + + tests := []struct { + name string + existingDBs []string + expectedDropDBs []string + currentUser string + existingUsers []userRecord + expectedDropUsers []string + }{ + { + name: "testing only specific DBs", + existingDBs: []string{"_vt", "vt_test"}, + expectedDropDBs: []string{"_vt", "vt_test"}, + }, + { + name: "testing with internal dbs", + existingDBs: []string{"_vt", "mysql", "vt_test", "performance_schema"}, + expectedDropDBs: []string{"_vt", "vt_test"}, + }, + { + name: "with users", + existingDBs: []string{"_vt", "mysql", "vt_test", "performance_schema"}, + expectedDropDBs: []string{"_vt", "vt_test"}, + existingUsers: []userRecord{ + {"test", "localhost"}, + {"app", "10.0.0.1"}, + }, + expectedDropUsers: []string{"'test'@'localhost'", "'app'@'10.0.0.1'"}, + }, + { + name: "with reserved users", + existingDBs: []string{"_vt", "mysql", "vt_test", "performance_schema"}, + expectedDropDBs: []string{"_vt", "vt_test"}, + existingUsers: []userRecord{ + {"mysql.sys", "localhost"}, + {"mysql.infoschema", "localhost"}, + {"mysql.session", "localhost"}, + {"test", "localhost"}, + {"app", "10.0.0.1"}, + }, + expectedDropUsers: []string{"'test'@'localhost'", "'app'@'10.0.0.1'"}, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + fakedb := fakesqldb.New(t) + defer fakedb.Close() + mysql := NewFakeMysqlDaemon(fakedb) + defer mysql.Close() + + databases := [][]sqltypes.Value{} + for _, db := range tt.existingDBs { + databases = append(databases, []sqltypes.Value{sqltypes.NewVarChar(db)}) + } + + users := [][]sqltypes.Value{} + for _, record := range tt.existingUsers { + users = append(users, []sqltypes.Value{sqltypes.NewVarChar(record.user), sqltypes.NewVarChar(record.host)}) + } + + mysql.FetchSuperQueryMap = map[string]*sqltypes.Result{ + "SHOW DATABASES": {Rows: databases}, + "SELECT user()": {Rows: [][]sqltypes.Value{{sqltypes.NewVarChar(tt.currentUser)}}}, + "SELECT user, host FROM mysql.user": {Rows: users}, + } + + for _, drop := range tt.expectedDropDBs { + mysql.ExpectedExecuteSuperQueryList = append(mysql.ExpectedExecuteSuperQueryList, + fmt.Sprintf("DROP DATABASE IF EXISTS `%s`", drop), + ) + } + + for _, drop := range tt.expectedDropUsers { + mysql.ExpectedExecuteSuperQueryList = append(mysql.ExpectedExecuteSuperQueryList, + fmt.Sprintf("DROP USER %s", drop), + ) + } + + err := cleanupMySQL(context.Background(), mysql, logutil.NewMemoryLogger()) + require.NoError(t, err) + }) + } + +} From 21f2604538a0389329db06adae07072e4bb80397 Mon Sep 17 00:00:00 2001 From: Renan Rangel Date: Thu, 12 Sep 2024 08:45:33 -0700 Subject: [PATCH 24/34] update flags Signed-off-by: Renan Rangel --- go/flags/endtoend/vtbackup.txt | 2 +- go/flags/endtoend/vtcombo.txt | 2 +- go/flags/endtoend/vttablet.txt | 2 +- go/flags/endtoend/vttestserver.txt | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/go/flags/endtoend/vtbackup.txt b/go/flags/endtoend/vtbackup.txt index 2cbf0abb078..b52d381d63b 100644 --- a/go/flags/endtoend/vtbackup.txt +++ b/go/flags/endtoend/vtbackup.txt @@ -178,7 +178,7 @@ Flags: --mysql-shell-backup-location string location where the backup will be stored --mysql-shell-dump-flags string flags to pass to mysql shell dump utility. This should be a JSON string and will be saved in the MANIFEST (default "{\"threads\": 2}") --mysql-shell-flags string execution flags to pass to mysqlsh binary to be used during dump/load (default "--defaults-file=/dev/null --js -h localhost") - --mysql-shell-load-flags string flags to pass to mysql shell load utility. This should be a JSON string (default "{\"threads\": 4, \"updateGtidSet\": \"replace\", \"skipBinlog\": true, \"progressFile\": \"\"}") + --mysql-shell-load-flags string flags to pass to mysql shell load utility. This should be a JSON string (default "{\"threads\": 4, \"loadUsers\": true, \"excludeUsers\": [\"'root'@'localhost'\"], \"updateGtidSet\": \"replace\", \"skipBinlog\": true, \"progressFile\": \"\"}") --mysql-shell-should-drain decide if we should drain while taking a backup or continue to serving traffic --mysql-shell-speedup-restore speed up restore by disabling redo logging and double write buffer during the restore process --mysql-shutdown-timeout duration how long to wait for mysqld shutdown (default 5m0s) diff --git a/go/flags/endtoend/vtcombo.txt b/go/flags/endtoend/vtcombo.txt index 13949c5a87b..7d00392a87e 100644 --- a/go/flags/endtoend/vtcombo.txt +++ b/go/flags/endtoend/vtcombo.txt @@ -229,7 +229,7 @@ Flags: --mysql-shell-backup-location string location where the backup will be stored --mysql-shell-dump-flags string flags to pass to mysql shell dump utility. This should be a JSON string and will be saved in the MANIFEST (default "{\"threads\": 2}") --mysql-shell-flags string execution flags to pass to mysqlsh binary to be used during dump/load (default "--defaults-file=/dev/null --js -h localhost") - --mysql-shell-load-flags string flags to pass to mysql shell load utility. This should be a JSON string (default "{\"threads\": 4, \"updateGtidSet\": \"replace\", \"skipBinlog\": true, \"progressFile\": \"\"}") + --mysql-shell-load-flags string flags to pass to mysql shell load utility. This should be a JSON string (default "{\"threads\": 4, \"loadUsers\": true, \"excludeUsers\": [\"'root'@'localhost'\"], \"updateGtidSet\": \"replace\", \"skipBinlog\": true, \"progressFile\": \"\"}") --mysql-shell-should-drain decide if we should drain while taking a backup or continue to serving traffic --mysql-shell-speedup-restore speed up restore by disabling redo logging and double write buffer during the restore process --mysql-shutdown-timeout duration timeout to use when MySQL is being shut down. (default 5m0s) diff --git a/go/flags/endtoend/vttablet.txt b/go/flags/endtoend/vttablet.txt index 4f8a344015d..18433aad3dc 100644 --- a/go/flags/endtoend/vttablet.txt +++ b/go/flags/endtoend/vttablet.txt @@ -246,7 +246,7 @@ Flags: --mysql-shell-backup-location string location where the backup will be stored --mysql-shell-dump-flags string flags to pass to mysql shell dump utility. This should be a JSON string and will be saved in the MANIFEST (default "{\"threads\": 2}") --mysql-shell-flags string execution flags to pass to mysqlsh binary to be used during dump/load (default "--defaults-file=/dev/null --js -h localhost") - --mysql-shell-load-flags string flags to pass to mysql shell load utility. This should be a JSON string (default "{\"threads\": 4, \"updateGtidSet\": \"replace\", \"skipBinlog\": true, \"progressFile\": \"\"}") + --mysql-shell-load-flags string flags to pass to mysql shell load utility. This should be a JSON string (default "{\"threads\": 4, \"loadUsers\": true, \"excludeUsers\": [\"'root'@'localhost'\"], \"updateGtidSet\": \"replace\", \"skipBinlog\": true, \"progressFile\": \"\"}") --mysql-shell-should-drain decide if we should drain while taking a backup or continue to serving traffic --mysql-shell-speedup-restore speed up restore by disabling redo logging and double write buffer during the restore process --mysql-shutdown-timeout duration timeout to use when MySQL is being shut down. (default 5m0s) diff --git a/go/flags/endtoend/vttestserver.txt b/go/flags/endtoend/vttestserver.txt index 278f8885c77..bcba552b26b 100644 --- a/go/flags/endtoend/vttestserver.txt +++ b/go/flags/endtoend/vttestserver.txt @@ -90,7 +90,7 @@ Flags: --mysql-shell-backup-location string location where the backup will be stored --mysql-shell-dump-flags string flags to pass to mysql shell dump utility. This should be a JSON string and will be saved in the MANIFEST (default "{\"threads\": 2}") --mysql-shell-flags string execution flags to pass to mysqlsh binary to be used during dump/load (default "--defaults-file=/dev/null --js -h localhost") - --mysql-shell-load-flags string flags to pass to mysql shell load utility. This should be a JSON string (default "{\"threads\": 4, \"updateGtidSet\": \"replace\", \"skipBinlog\": true, \"progressFile\": \"\"}") + --mysql-shell-load-flags string flags to pass to mysql shell load utility. This should be a JSON string (default "{\"threads\": 4, \"loadUsers\": true, \"excludeUsers\": [\"'root'@'localhost'\"], \"updateGtidSet\": \"replace\", \"skipBinlog\": true, \"progressFile\": \"\"}") --mysql-shell-should-drain decide if we should drain while taking a backup or continue to serving traffic --mysql-shell-speedup-restore speed up restore by disabling redo logging and double write buffer during the restore process --mysql_bind_host string which host to bind vtgate mysql listener to (default "localhost") From 47dc753d6c50244ad044a4b74c7cf5f07e7fe7fb Mon Sep 17 00:00:00 2001 From: Renan Rangel Date: Fri, 13 Sep 2024 05:36:59 -0700 Subject: [PATCH 25/34] make sure vitess default matches mysqlshell's Signed-off-by: Renan Rangel --- go/flags/endtoend/vtbackup.txt | 2 +- go/flags/endtoend/vtcombo.txt | 2 +- go/flags/endtoend/vttablet.txt | 2 +- go/flags/endtoend/vttestserver.txt | 2 +- go/vt/mysqlctl/mysqlshellbackupengine.go | 2 +- 5 files changed, 5 insertions(+), 5 deletions(-) diff --git a/go/flags/endtoend/vtbackup.txt b/go/flags/endtoend/vtbackup.txt index b52d381d63b..76f7842563b 100644 --- a/go/flags/endtoend/vtbackup.txt +++ b/go/flags/endtoend/vtbackup.txt @@ -176,7 +176,7 @@ Flags: --mycnf_socket_file string mysql socket file --mycnf_tmp_dir string mysql tmp directory --mysql-shell-backup-location string location where the backup will be stored - --mysql-shell-dump-flags string flags to pass to mysql shell dump utility. This should be a JSON string and will be saved in the MANIFEST (default "{\"threads\": 2}") + --mysql-shell-dump-flags string flags to pass to mysql shell dump utility. This should be a JSON string and will be saved in the MANIFEST (default "{\"threads\": 4}") --mysql-shell-flags string execution flags to pass to mysqlsh binary to be used during dump/load (default "--defaults-file=/dev/null --js -h localhost") --mysql-shell-load-flags string flags to pass to mysql shell load utility. This should be a JSON string (default "{\"threads\": 4, \"loadUsers\": true, \"excludeUsers\": [\"'root'@'localhost'\"], \"updateGtidSet\": \"replace\", \"skipBinlog\": true, \"progressFile\": \"\"}") --mysql-shell-should-drain decide if we should drain while taking a backup or continue to serving traffic diff --git a/go/flags/endtoend/vtcombo.txt b/go/flags/endtoend/vtcombo.txt index 7d00392a87e..500615b95c9 100644 --- a/go/flags/endtoend/vtcombo.txt +++ b/go/flags/endtoend/vtcombo.txt @@ -227,7 +227,7 @@ Flags: --mysql-server-keepalive-period duration TCP period between keep-alives --mysql-server-pool-conn-read-buffers If set, the server will pool incoming connection read buffers --mysql-shell-backup-location string location where the backup will be stored - --mysql-shell-dump-flags string flags to pass to mysql shell dump utility. This should be a JSON string and will be saved in the MANIFEST (default "{\"threads\": 2}") + --mysql-shell-dump-flags string flags to pass to mysql shell dump utility. This should be a JSON string and will be saved in the MANIFEST (default "{\"threads\": 4}") --mysql-shell-flags string execution flags to pass to mysqlsh binary to be used during dump/load (default "--defaults-file=/dev/null --js -h localhost") --mysql-shell-load-flags string flags to pass to mysql shell load utility. This should be a JSON string (default "{\"threads\": 4, \"loadUsers\": true, \"excludeUsers\": [\"'root'@'localhost'\"], \"updateGtidSet\": \"replace\", \"skipBinlog\": true, \"progressFile\": \"\"}") --mysql-shell-should-drain decide if we should drain while taking a backup or continue to serving traffic diff --git a/go/flags/endtoend/vttablet.txt b/go/flags/endtoend/vttablet.txt index 18433aad3dc..695cb70680d 100644 --- a/go/flags/endtoend/vttablet.txt +++ b/go/flags/endtoend/vttablet.txt @@ -244,7 +244,7 @@ Flags: --mycnf_socket_file string mysql socket file --mycnf_tmp_dir string mysql tmp directory --mysql-shell-backup-location string location where the backup will be stored - --mysql-shell-dump-flags string flags to pass to mysql shell dump utility. This should be a JSON string and will be saved in the MANIFEST (default "{\"threads\": 2}") + --mysql-shell-dump-flags string flags to pass to mysql shell dump utility. This should be a JSON string and will be saved in the MANIFEST (default "{\"threads\": 4}") --mysql-shell-flags string execution flags to pass to mysqlsh binary to be used during dump/load (default "--defaults-file=/dev/null --js -h localhost") --mysql-shell-load-flags string flags to pass to mysql shell load utility. This should be a JSON string (default "{\"threads\": 4, \"loadUsers\": true, \"excludeUsers\": [\"'root'@'localhost'\"], \"updateGtidSet\": \"replace\", \"skipBinlog\": true, \"progressFile\": \"\"}") --mysql-shell-should-drain decide if we should drain while taking a backup or continue to serving traffic diff --git a/go/flags/endtoend/vttestserver.txt b/go/flags/endtoend/vttestserver.txt index bcba552b26b..538fdd3ea47 100644 --- a/go/flags/endtoend/vttestserver.txt +++ b/go/flags/endtoend/vttestserver.txt @@ -88,7 +88,7 @@ Flags: --max_table_shard_size int The maximum number of initial rows in a table shard. Ignored if--initialize_with_random_data is false. The actual number is chosen randomly (default 10000) --min_table_shard_size int The minimum number of initial rows in a table shard. Ignored if--initialize_with_random_data is false. The actual number is chosen randomly. (default 1000) --mysql-shell-backup-location string location where the backup will be stored - --mysql-shell-dump-flags string flags to pass to mysql shell dump utility. This should be a JSON string and will be saved in the MANIFEST (default "{\"threads\": 2}") + --mysql-shell-dump-flags string flags to pass to mysql shell dump utility. This should be a JSON string and will be saved in the MANIFEST (default "{\"threads\": 4}") --mysql-shell-flags string execution flags to pass to mysqlsh binary to be used during dump/load (default "--defaults-file=/dev/null --js -h localhost") --mysql-shell-load-flags string flags to pass to mysql shell load utility. This should be a JSON string (default "{\"threads\": 4, \"loadUsers\": true, \"excludeUsers\": [\"'root'@'localhost'\"], \"updateGtidSet\": \"replace\", \"skipBinlog\": true, \"progressFile\": \"\"}") --mysql-shell-should-drain decide if we should drain while taking a backup or continue to serving traffic diff --git a/go/vt/mysqlctl/mysqlshellbackupengine.go b/go/vt/mysqlctl/mysqlshellbackupengine.go index 8d2b3f61b0f..dbe834974d8 100644 --- a/go/vt/mysqlctl/mysqlshellbackupengine.go +++ b/go/vt/mysqlctl/mysqlshellbackupengine.go @@ -49,7 +49,7 @@ var ( // flags passed to the mysql shell utility, used both on dump/restore mysqlShellFlags = "--defaults-file=/dev/null --js -h localhost" // flags passed to the Dump command, as a JSON string - mysqlShellDumpFlags = `{"threads": 2}` + mysqlShellDumpFlags = `{"threads": 4}` // flags passed to the Load command, as a JSON string mysqlShellLoadFlags = `{"threads": 4, "loadUsers": true, "excludeUsers": ["'root'@'localhost'"], "updateGtidSet": "replace", "skipBinlog": true, "progressFile": ""}` // drain a tablet when taking a backup From 5c08cc79c0410b951ebd007f4abb9976763ca6ef Mon Sep 17 00:00:00 2001 From: Renan Rangel Date: Fri, 13 Sep 2024 07:54:07 -0700 Subject: [PATCH 26/34] do not remove users if not loading them Signed-off-by: Renan Rangel --- go/vt/mysqlctl/mysqlshellbackupengine.go | 95 ++++++++++--------- go/vt/mysqlctl/mysqlshellbackupengine_test.go | 65 ++++++++++--- 2 files changed, 102 insertions(+), 58 deletions(-) diff --git a/go/vt/mysqlctl/mysqlshellbackupengine.go b/go/vt/mysqlctl/mysqlshellbackupengine.go index dbe834974d8..98f1230c9eb 100644 --- a/go/vt/mysqlctl/mysqlshellbackupengine.go +++ b/go/vt/mysqlctl/mysqlshellbackupengine.go @@ -36,7 +36,6 @@ import ( "vitess.io/vitess/go/mysql" "vitess.io/vitess/go/mysql/capabilities" "vitess.io/vitess/go/vt/log" - "vitess.io/vitess/go/vt/logutil" "vitess.io/vitess/go/vt/mysqlctl/backupstorage" tabletmanagerdatapb "vitess.io/vitess/go/vt/proto/tabletmanagerdata" "vitess.io/vitess/go/vt/servenv" @@ -238,7 +237,7 @@ func (be *MySQLShellBackupEngine) ExecuteBackup(ctx context.Context, params Back func (be *MySQLShellBackupEngine) ExecuteRestore(ctx context.Context, params RestoreParams, bh backupstorage.BackupHandle) (*BackupManifest, error) { params.Logger.Infof("Calling ExecuteRestore for %s (DeleteBeforeRestore: %v)", params.DbName, params.DeleteBeforeRestore) - err := be.restorePreCheck(ctx, params) + shouldDeleteUsers, err := be.restorePreCheck(ctx, params) if err != nil { return nil, vterrors.Wrap(err, "failed restore precheck") } @@ -280,7 +279,7 @@ func (be *MySQLShellBackupEngine) ExecuteRestore(ctx context.Context, params Res }() } - err = cleanupMySQL(ctx, params.Mysqld, params.Logger) + err = cleanupMySQL(ctx, params, shouldDeleteUsers) if err != nil { log.Errorf(err.Error()) // time.Sleep(time.Minute * 2) @@ -411,47 +410,51 @@ func (be *MySQLShellBackupEngine) backupPreCheck(location string) error { return nil } -func (be *MySQLShellBackupEngine) restorePreCheck(ctx context.Context, params RestoreParams) error { +func (be *MySQLShellBackupEngine) restorePreCheck(ctx context.Context, params RestoreParams) (shouldDeleteUsers bool, err error) { if mysqlShellFlags == "" { - return fmt.Errorf("%w: at least the --js flag is required in the value of the flag --mysql-shell-flags", MySQLShellPreCheckError) + return shouldDeleteUsers, fmt.Errorf("%w: at least the --js flag is required in the value of the flag --mysql-shell-flags", MySQLShellPreCheckError) } loadFlags := map[string]interface{}{} - err := json.Unmarshal([]byte(mysqlShellLoadFlags), &loadFlags) + err = json.Unmarshal([]byte(mysqlShellLoadFlags), &loadFlags) if err != nil { - return fmt.Errorf("%w: unable to parse JSON of load flags", MySQLShellPreCheckError) + return false, fmt.Errorf("%w: unable to parse JSON of load flags", MySQLShellPreCheckError) } if val, ok := loadFlags["updateGtidSet"]; !ok || val != "replace" { - return fmt.Errorf("%w: mysql-shell needs to restore with updateGtidSet set to \"replace\" to work with Vitess", MySQLShellPreCheckError) + return false, fmt.Errorf("%w: mysql-shell needs to restore with updateGtidSet set to \"replace\" to work with Vitess", MySQLShellPreCheckError) } if val, ok := loadFlags["progressFile"]; !ok || val != "" { - return fmt.Errorf("%w: \"progressFile\" needs to be empty as vitess always starts a restore from scratch", MySQLShellPreCheckError) + return false, fmt.Errorf("%w: \"progressFile\" needs to be empty as vitess always starts a restore from scratch", MySQLShellPreCheckError) } if val, ok := loadFlags["skipBinlog"]; !ok || val != true { - return fmt.Errorf("%w: \"skipBinlog\" needs to set to true", MySQLShellPreCheckError) + return false, fmt.Errorf("%w: \"skipBinlog\" needs to set to true", MySQLShellPreCheckError) + } + + if val, ok := loadFlags["loadUsers"]; ok && val == true { + shouldDeleteUsers = true } if mysqlShellSpeedUpRestore { version, err := params.Mysqld.GetVersionString(ctx) if err != nil { - return fmt.Errorf("%w: failed to fetch MySQL version: %v", MySQLShellPreCheckError, err) + return false, fmt.Errorf("%w: failed to fetch MySQL version: %v", MySQLShellPreCheckError, err) } capableOf := mysql.ServerVersionCapableOf(version) capable, err := capableOf(capabilities.DisableRedoLogFlavorCapability) if err != nil { - return fmt.Errorf("%w: error checking if server supports disabling redo log: %v", MySQLShellPreCheckError, err) + return false, fmt.Errorf("%w: error checking if server supports disabling redo log: %v", MySQLShellPreCheckError, err) } if !capable { - return fmt.Errorf("%w: MySQL version doesn't support disabling the redo log (must be >=8.0.21)", MySQLShellPreCheckError) + return false, fmt.Errorf("%w: MySQL version doesn't support disabling the redo log (must be >=8.0.21)", MySQLShellPreCheckError) } } - return nil + return shouldDeleteUsers, nil } func (be *MySQLShellBackupEngine) handleSuperReadOnly(ctx context.Context, params RestoreParams) (func(), error) { @@ -510,9 +513,9 @@ func releaseReadLock(ctx context.Context, reader io.Reader, params BackupParams, } } -func cleanupMySQL(ctx context.Context, mysql MysqlDaemon, logger logutil.Logger) error { - logger.Infof("Cleaning up MySQL ahead of a restore") - result, err := mysql.FetchSuperQuery(ctx, "SHOW DATABASES") +func cleanupMySQL(ctx context.Context, params RestoreParams, shouldDeleteUsers bool) error { + params.Logger.Infof("Cleaning up MySQL ahead of a restore") + result, err := params.Mysqld.FetchSuperQuery(ctx, "SHOW DATABASES") if err != nil { return err } @@ -524,44 +527,46 @@ func cleanupMySQL(ctx context.Context, mysql MysqlDaemon, logger logutil.Logger) continue // not dropping internal DBs } - logger.Infof("Dropping DB %q", dbName) - err = mysql.ExecuteSuperQuery(ctx, fmt.Sprintf("DROP DATABASE IF EXISTS `%s`", row[0].ToString())) + params.Logger.Infof("Dropping DB %q", dbName) + err = params.Mysqld.ExecuteSuperQuery(ctx, fmt.Sprintf("DROP DATABASE IF EXISTS `%s`", row[0].ToString())) if err != nil { return fmt.Errorf("error droppping database %q: %w", row[0].ToString(), err) } } - // get current user - var currentUser string - result, err = mysql.FetchSuperQuery(ctx, "SELECT user()") - if err != nil { - return fmt.Errorf("error fetching current user: %w", err) - } + if shouldDeleteUsers { + // get current user + var currentUser string + result, err = params.Mysqld.FetchSuperQuery(ctx, "SELECT user()") + if err != nil { + return fmt.Errorf("error fetching current user: %w", err) + } - for _, row := range result.Rows { - currentUser = row[0].ToString() - } + for _, row := range result.Rows { + currentUser = row[0].ToString() + } - // drop all users except reserved ones - result, err = mysql.FetchSuperQuery(ctx, "SELECT user, host FROM mysql.user") - if err != nil { - return err - } + // drop all users except reserved ones + result, err = params.Mysqld.FetchSuperQuery(ctx, "SELECT user, host FROM mysql.user") + if err != nil { + return err + } - for _, row := range result.Rows { - user := fmt.Sprintf("%s@%s", row[0].ToString(), row[1].ToString()) + for _, row := range result.Rows { + user := fmt.Sprintf("%s@%s", row[0].ToString(), row[1].ToString()) - if user == currentUser { - continue // we don't drop the current user - } - if slices.Contains(reservedUsers, user) { - continue // we skip reserved MySQL users - } + if user == currentUser { + continue // we don't drop the current user + } + if slices.Contains(reservedUsers, user) { + continue // we skip reserved MySQL users + } - logger.Infof("Dropping User %q", user) - err = mysql.ExecuteSuperQuery(ctx, fmt.Sprintf("DROP USER '%s'@'%s'", row[0].ToString(), row[1].ToString())) - if err != nil { - return fmt.Errorf("error droppping user %q: %w", user, err) + params.Logger.Infof("Dropping User %q", user) + err = params.Mysqld.ExecuteSuperQuery(ctx, fmt.Sprintf("DROP USER '%s'@'%s'", row[0].ToString(), row[1].ToString())) + if err != nil { + return fmt.Errorf("error droppping user %q: %w", user, err) + } } } diff --git a/go/vt/mysqlctl/mysqlshellbackupengine_test.go b/go/vt/mysqlctl/mysqlshellbackupengine_test.go index 5f1f13e5438..a0d0c8d7a1d 100644 --- a/go/vt/mysqlctl/mysqlshellbackupengine_test.go +++ b/go/vt/mysqlctl/mysqlshellbackupengine_test.go @@ -95,41 +95,55 @@ func TestMySQLShellBackupRestorePreCheck(t *testing.T) { engine := MySQLShellBackupEngine{} tests := []struct { - name string - flags string - err error + name string + flags string + err error + shouldDeleteUsers bool }{ { "empty load flags", `{}`, MySQLShellPreCheckError, + false, }, { "only updateGtidSet", `{"updateGtidSet": "replace"}`, MySQLShellPreCheckError, + false, }, { "only progressFile", `{"progressFile": ""}`, MySQLShellPreCheckError, + false, }, { "both values but unsupported values", `{"updateGtidSet": "append", "progressFile": "/tmp/test1"}`, MySQLShellPreCheckError, + false, }, { "supported values", - `{"updateGtidSet": "replace", "progressFile": "", "skipBinlog": true}`, + `{"updateGtidSet": "replace", "progressFile": "", "skipBinlog": true, "loadUsers": false}`, nil, + false, + }, + { + "should delete users", + `{"updateGtidSet": "replace", "progressFile": "", "skipBinlog": true, "loadUsers": true}`, + nil, + true, }, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { mysqlShellLoadFlags = tt.flags - assert.ErrorIs(t, engine.restorePreCheck(context.Background(), RestoreParams{}), tt.err) + shouldDeleteUsers, err := engine.restorePreCheck(context.Background(), RestoreParams{}) + assert.ErrorIs(t, err, tt.err) + assert.Equal(t, tt.shouldDeleteUsers, shouldDeleteUsers) }) } @@ -152,12 +166,13 @@ func TestMySQLShellBackupRestorePreCheckDisableRedolog(t *testing.T) { } // this should work as it is supported since 8.0.21 - require.NoError(t, engine.restorePreCheck(context.Background(), params)) + _, err := engine.restorePreCheck(context.Background(), params) + require.NoError(t, err, params) // it should error out if we change to an older version fakeMysqld.Version = "8.0.20" - err := engine.restorePreCheck(context.Background(), params) + _, err = engine.restorePreCheck(context.Background(), params) require.ErrorIs(t, err, MySQLShellPreCheckError) require.ErrorContains(t, err, "doesn't support disabling the redo log") } @@ -191,6 +206,7 @@ func TestCleanupMySQL(t *testing.T) { currentUser string existingUsers []userRecord expectedDropUsers []string + shouldDeleteUsers bool }{ { name: "testing only specific DBs", @@ -203,7 +219,18 @@ func TestCleanupMySQL(t *testing.T) { expectedDropDBs: []string{"_vt", "vt_test"}, }, { - name: "with users", + name: "with users but without delete", + existingDBs: []string{"_vt", "mysql", "vt_test", "performance_schema"}, + expectedDropDBs: []string{"_vt", "vt_test"}, + existingUsers: []userRecord{ + {"test", "localhost"}, + {"app", "10.0.0.1"}, + }, + expectedDropUsers: []string{}, + shouldDeleteUsers: false, + }, + { + name: "with users and delete", existingDBs: []string{"_vt", "mysql", "vt_test", "performance_schema"}, expectedDropDBs: []string{"_vt", "vt_test"}, existingUsers: []userRecord{ @@ -211,6 +238,7 @@ func TestCleanupMySQL(t *testing.T) { {"app", "10.0.0.1"}, }, expectedDropUsers: []string{"'test'@'localhost'", "'app'@'10.0.0.1'"}, + shouldDeleteUsers: true, }, { name: "with reserved users", @@ -224,6 +252,7 @@ func TestCleanupMySQL(t *testing.T) { {"app", "10.0.0.1"}, }, expectedDropUsers: []string{"'test'@'localhost'", "'app'@'10.0.0.1'"}, + shouldDeleteUsers: true, }, } @@ -256,14 +285,24 @@ func TestCleanupMySQL(t *testing.T) { ) } - for _, drop := range tt.expectedDropUsers { - mysql.ExpectedExecuteSuperQueryList = append(mysql.ExpectedExecuteSuperQueryList, - fmt.Sprintf("DROP USER %s", drop), - ) + if tt.shouldDeleteUsers { + for _, drop := range tt.expectedDropUsers { + mysql.ExpectedExecuteSuperQueryList = append(mysql.ExpectedExecuteSuperQueryList, + fmt.Sprintf("DROP USER %s", drop), + ) + } + } + + params := RestoreParams{ + Mysqld: mysql, + Logger: logutil.NewMemoryLogger(), } - err := cleanupMySQL(context.Background(), mysql, logutil.NewMemoryLogger()) + err := cleanupMySQL(context.Background(), params, tt.shouldDeleteUsers) require.NoError(t, err) + + require.Equal(t, len(tt.expectedDropDBs)+len(tt.expectedDropUsers), mysql.ExpectedExecuteSuperQueryCurrent, + "unexpected number of queries executed") }) } From e37dbbbd0be466138468f55a0101c23e2686e68b Mon Sep 17 00:00:00 2001 From: Renan Rangel Date: Fri, 13 Sep 2024 08:05:46 -0700 Subject: [PATCH 27/34] changelog notice Signed-off-by: Renan Rangel --- changelog/21.0/21.0.0/summary.md | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/changelog/21.0/21.0.0/summary.md b/changelog/21.0/21.0.0/summary.md index d5fefd74f0c..27f0c0f9abf 100644 --- a/changelog/21.0/21.0.0/summary.md +++ b/changelog/21.0/21.0.0/summary.md @@ -14,6 +14,7 @@ - **[Allow Cross Cell Promotion in PRS](#allow-cross-cell)** - **[Support for recursive CTEs](#recursive-cte)** - **[VTGate Tablet Balancer](#tablet-balancer)** + - **[New Backup Engine](#new-backup-engine)** ## Major Changes @@ -113,4 +114,11 @@ When a VTGate routes a query and has multiple available tablets for a given shar The tablet balancer is enabled by a new flag `--enable-balancer` and configured by `--balancer-vtgate-cells` and `--balancer-keyspaces`. -See [RFC for details](https://github.com/vitessio/vitess/issues/12241). \ No newline at end of file +See [RFC for details](https://github.com/vitessio/vitess/issues/12241). + +### New Backup Engine + +We are introducing a the backup engine supporting logical backups starting on v21 to support use cases that require something else besides physical backups. This is experimental and is based on the +[MySQL Shell](https://dev.mysql.com/doc/mysql-shell/8.0/en/). + +The new engine is enabled by using `--backup_engine_implementation=mysqlshell`. There are other options that are required, so check the docs on which options are required and how to use it. From 9243f2256943db39c2428144503504a1220e36e5 Mon Sep 17 00:00:00 2001 From: Renan Rangel Date: Wed, 18 Sep 2024 05:22:03 -0700 Subject: [PATCH 28/34] add ShouldStartMySQLAfterRestore() Signed-off-by: Renan Rangel --- go/vt/mysqlctl/backup.go | 24 +++++++++++++----------- go/vt/mysqlctl/backupengine.go | 1 + go/vt/mysqlctl/builtinbackupengine.go | 5 +++++ go/vt/mysqlctl/fakebackupengine.go | 4 ++++ go/vt/mysqlctl/mysqlshellbackupengine.go | 6 ++++++ go/vt/mysqlctl/xtrabackupengine.go | 5 +++++ 6 files changed, 34 insertions(+), 11 deletions(-) diff --git a/go/vt/mysqlctl/backup.go b/go/vt/mysqlctl/backup.go index fd067b02887..4a89add9c9e 100644 --- a/go/vt/mysqlctl/backup.go +++ b/go/vt/mysqlctl/backup.go @@ -447,17 +447,19 @@ func Restore(ctx context.Context, params RestoreParams) (*BackupManifest, error) return nil, err } - // mysqld needs to be running in order for mysql_upgrade to work. - // If we've just restored from a backup from previous MySQL version then mysqld - // may fail to start due to a different structure of mysql.* tables. The flag - // --skip-grant-tables ensures that these tables are not read until mysql_upgrade - // is executed. And since with --skip-grant-tables anyone can connect to MySQL - // without password, we are passing --skip-networking to greatly reduce the set - // of those who can connect. - params.Logger.Infof("Restore: starting mysqld for mysql_upgrade") - // Note Start will use dba user for waiting, this is fine, it will be allowed. - if err := params.Mysqld.Start(context.Background(), params.Cnf, "--skip-grant-tables", "--skip-networking"); err != nil { - return nil, err + if re.ShouldStartMySQLAfterRestore() { // all engines except mysqlshell since MySQL is always running there + // mysqld needs to be running in order for mysql_upgrade to work. + // If we've just restored from a backup from previous MySQL version then mysqld + // may fail to start due to a different structure of mysql.* tables. The flag + // --skip-grant-tables ensures that these tables are not read until mysql_upgrade + // is executed. And since with --skip-grant-tables anyone can connect to MySQL + // without password, we are passing --skip-networking to greatly reduce the set + // of those who can connect. + params.Logger.Infof("Restore: starting mysqld for mysql_upgrade") + // Note Start will use dba user for waiting, this is fine, it will be allowed. + if err := params.Mysqld.Start(context.Background(), params.Cnf, "--skip-grant-tables", "--skip-networking"); err != nil { + return nil, err + } } params.Logger.Infof("Restore: running mysql_upgrade") diff --git a/go/vt/mysqlctl/backupengine.go b/go/vt/mysqlctl/backupengine.go index c483aff3d78..1401c7e2f84 100644 --- a/go/vt/mysqlctl/backupengine.go +++ b/go/vt/mysqlctl/backupengine.go @@ -179,6 +179,7 @@ func (p *RestoreParams) IsIncrementalRecovery() bool { // Returns the manifest of a backup if successful, otherwise returns an error type RestoreEngine interface { ExecuteRestore(ctx context.Context, params RestoreParams, bh backupstorage.BackupHandle) (*BackupManifest, error) + ShouldStartMySQLAfterRestore() bool } // BackupRestoreEngine is a combination of BackupEngine and RestoreEngine. diff --git a/go/vt/mysqlctl/builtinbackupengine.go b/go/vt/mysqlctl/builtinbackupengine.go index 494d765f2a9..9876de36098 100644 --- a/go/vt/mysqlctl/builtinbackupengine.go +++ b/go/vt/mysqlctl/builtinbackupengine.go @@ -1185,6 +1185,11 @@ func (be *BuiltinBackupEngine) ShouldDrainForBackup(req *tabletmanagerdatapb.Bac return true } +// ShouldStartMySQLAfterRestore signifies if this backup engine needs to restart MySQL once the restore is completed. +func (be *BuiltinBackupEngine) ShouldStartMySQLAfterRestore() bool { + return true +} + func getPrimaryPosition(ctx context.Context, tmc tmclient.TabletManagerClient, ts *topo.Server, keyspace, shard string) (replication.Position, error) { si, err := ts.GetShard(ctx, keyspace, shard) if err != nil { diff --git a/go/vt/mysqlctl/fakebackupengine.go b/go/vt/mysqlctl/fakebackupengine.go index d78282e6aff..180922347e1 100644 --- a/go/vt/mysqlctl/fakebackupengine.go +++ b/go/vt/mysqlctl/fakebackupengine.go @@ -91,3 +91,7 @@ func (be *FakeBackupEngine) ShouldDrainForBackup(req *tabletmanagerdatapb.Backup be.ShouldDrainForBackupCalls = be.ShouldDrainForBackupCalls + 1 return be.ShouldDrainForBackupReturn } + +func (be *FakeBackupEngine) ShouldStartMySQLAfterRestore() bool { + return true +} diff --git a/go/vt/mysqlctl/mysqlshellbackupengine.go b/go/vt/mysqlctl/mysqlshellbackupengine.go index 98f1230c9eb..a4210518ac8 100644 --- a/go/vt/mysqlctl/mysqlshellbackupengine.go +++ b/go/vt/mysqlctl/mysqlshellbackupengine.go @@ -381,6 +381,12 @@ func (be *MySQLShellBackupEngine) ShouldDrainForBackup(req *tabletmanagerdatapb. return mysqlShellBackupShouldDrain } +// ShouldStartMySQLAfterRestore signifies if this backup engine needs to restart MySQL once the restore is completed. +// Since MySQL Shell operates on a live MySQL instance, there is no need to start it once the restore is completed +func (be *MySQLShellBackupEngine) ShouldStartMySQLAfterRestore() bool { + return false +} + func (be *MySQLShellBackupEngine) backupPreCheck(location string) error { if mysqlShellBackupLocation == "" { return fmt.Errorf("%w: no backup location set via --mysql-shell-backup-location", MySQLShellPreCheckError) diff --git a/go/vt/mysqlctl/xtrabackupengine.go b/go/vt/mysqlctl/xtrabackupengine.go index 5dcca37c984..9e4917abb0f 100644 --- a/go/vt/mysqlctl/xtrabackupengine.go +++ b/go/vt/mysqlctl/xtrabackupengine.go @@ -942,6 +942,11 @@ func (be *XtrabackupEngine) ShouldDrainForBackup(req *tabletmanagerdatapb.Backup return false } +// ShouldStartMySQLAfterRestore signifies if this backup engine needs to restart MySQL once the restore is completed. +func (be *XtrabackupEngine) ShouldStartMySQLAfterRestore() bool { + return true +} + func init() { BackupRestoreEngineMap[xtrabackupEngineName] = &XtrabackupEngine{} } From 4e79a02d4fbc7136ec3d4097f3d4797bd3b0f558 Mon Sep 17 00:00:00 2001 From: Shlomi Noach <2607934+shlomi-noach@users.noreply.github.com> Date: Thu, 19 Sep 2024 12:19:20 +0300 Subject: [PATCH 29/34] backup_pitr testing: validate rejoining replication stream Signed-off-by: Shlomi Noach <2607934+shlomi-noach@users.noreply.github.com> --- .../endtoend/backup/vtctlbackup/backup_utils.go | 11 +++++++++++ .../backup/vtctlbackup/pitr_test_framework.go | 17 +++++++++++++---- 2 files changed, 24 insertions(+), 4 deletions(-) diff --git a/go/test/endtoend/backup/vtctlbackup/backup_utils.go b/go/test/endtoend/backup/vtctlbackup/backup_utils.go index 3a0150cc87c..5c69a110e15 100644 --- a/go/test/endtoend/backup/vtctlbackup/backup_utils.go +++ b/go/test/endtoend/backup/vtctlbackup/backup_utils.go @@ -1133,6 +1133,17 @@ func GetReplicaGtidPurged(t *testing.T, replicaIndex int) string { return row.AsString("gtid_purged", "") } +func ReconnectReplicaToPrimary(t *testing.T, replicaIndex int) { + query := fmt.Sprintf("CHANGE REPLICATION SOURCE TO SOURCE_HOST='localhost', SOURCE_PORT=%d, SOURCE_USER='vt_repl', SOURCE_AUTO_POSITION = 1", primary.MySQLPort) + replica := getReplica(t, replicaIndex) + _, err := replica.VttabletProcess.QueryTablet("stop replica", keyspaceName, true) + require.NoError(t, err) + _, err = replica.VttabletProcess.QueryTablet(query, keyspaceName, true) + require.NoError(t, err) + _, err = replica.VttabletProcess.QueryTablet("start replica", keyspaceName, true) + require.NoError(t, err) +} + func InsertRowOnPrimary(t *testing.T, hint string) { if hint == "" { hint = textutil.RandomHash()[:12] diff --git a/go/test/endtoend/backup/vtctlbackup/pitr_test_framework.go b/go/test/endtoend/backup/vtctlbackup/pitr_test_framework.go index 7f611d81ad6..2970e915ac1 100644 --- a/go/test/endtoend/backup/vtctlbackup/pitr_test_framework.go +++ b/go/test/endtoend/backup/vtctlbackup/pitr_test_framework.go @@ -68,21 +68,24 @@ type testedBackupTimestampInfo struct { postTimestamp time.Time } -func waitForReplica(t *testing.T, replicaIndex int) { +// waitForReplica waits for the replica to have same row set as on primary. +func waitForReplica(t *testing.T, replicaIndex int) int { ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) defer cancel() + ticker := time.NewTicker(1 * time.Second) + defer ticker.Stop() pMsgs := ReadRowsFromPrimary(t) for { rMsgs := ReadRowsFromReplica(t, replicaIndex) if len(pMsgs) == len(rMsgs) { // success - return + return len(pMsgs) } select { case <-ctx.Done(): assert.FailNow(t, "timeout waiting for replica to catch up") - return - case <-time.After(time.Second): + return 0 + case <-ticker.C: // } } @@ -289,6 +292,12 @@ func ExecTestIncrementalBackupAndRestoreToPos(t *testing.T, tcase *PITRTestCase) if sampleTestedBackupPos == "" { sampleTestedBackupPos = pos } + t.Run("post-pitr, wait for replica to catch up", func(t *testing.T) { + // Replica is DRAINED and does not have replication configuration. + // We now connect the replica to the primary and validate it's able to catch up. + ReconnectReplicaToPrimary(t, 0) + waitForReplica(t, 0) + }) }) } } From 7e77e864be51b2c6cf395a04f0ab8db7d5d2e239 Mon Sep 17 00:00:00 2001 From: Shlomi Noach <2607934+shlomi-noach@users.noreply.github.com> Date: Thu, 19 Sep 2024 12:26:43 +0300 Subject: [PATCH 30/34] test replication stream in ExecTestIncrementalBackupAndRestoreToTimestamp Signed-off-by: Shlomi Noach <2607934+shlomi-noach@users.noreply.github.com> --- go/test/endtoend/backup/vtctlbackup/pitr_test_framework.go | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/go/test/endtoend/backup/vtctlbackup/pitr_test_framework.go b/go/test/endtoend/backup/vtctlbackup/pitr_test_framework.go index 2970e915ac1..4c84c3e63bc 100644 --- a/go/test/endtoend/backup/vtctlbackup/pitr_test_framework.go +++ b/go/test/endtoend/backup/vtctlbackup/pitr_test_framework.go @@ -548,6 +548,12 @@ func ExecTestIncrementalBackupAndRestoreToTimestamp(t *testing.T, tcase *PITRTes if sampleTestedBackupIndex < 0 { sampleTestedBackupIndex = backupIndex } + t.Run("post-pitr, wait for replica to catch up", func(t *testing.T) { + // Replica is DRAINED and does not have replication configuration. + // We now connect the replica to the primary and validate it's able to catch up. + ReconnectReplicaToPrimary(t, 0) + waitForReplica(t, 0) + }) } else { numFailedRestores++ } From 3c655a16108b1dd79c1173d4ee3e0f84e15140fe Mon Sep 17 00:00:00 2001 From: Renan Rangel Date: Fri, 20 Sep 2024 05:19:57 -0700 Subject: [PATCH 31/34] add missing grant Signed-off-by: Renan Rangel --- config/init_db.sql | 1 + go/flags/endtoend/vtbackup.txt | 2 +- go/flags/endtoend/vtcombo.txt | 2 +- go/flags/endtoend/vttablet.txt | 2 +- go/flags/endtoend/vttestserver.txt | 2 +- go/vt/mysqlctl/mysqlshellbackupengine.go | 2 +- 6 files changed, 6 insertions(+), 5 deletions(-) diff --git a/config/init_db.sql b/config/init_db.sql index 25ea2a42f3a..118d8d02687 100644 --- a/config/init_db.sql +++ b/config/init_db.sql @@ -29,6 +29,7 @@ DROP DATABASE IF EXISTS test; CREATE USER 'vt_dba'@'localhost'; GRANT ALL ON *.* TO 'vt_dba'@'localhost'; GRANT GRANT OPTION ON *.* TO 'vt_dba'@'localhost'; +GRANT PROXY ON ''@'' TO 'vt_dba'@'localhost' WITH GRANT OPTION; # User for app traffic, with global read-write access. CREATE USER 'vt_app'@'localhost'; diff --git a/go/flags/endtoend/vtbackup.txt b/go/flags/endtoend/vtbackup.txt index 76f7842563b..fba7c794fc0 100644 --- a/go/flags/endtoend/vtbackup.txt +++ b/go/flags/endtoend/vtbackup.txt @@ -178,7 +178,7 @@ Flags: --mysql-shell-backup-location string location where the backup will be stored --mysql-shell-dump-flags string flags to pass to mysql shell dump utility. This should be a JSON string and will be saved in the MANIFEST (default "{\"threads\": 4}") --mysql-shell-flags string execution flags to pass to mysqlsh binary to be used during dump/load (default "--defaults-file=/dev/null --js -h localhost") - --mysql-shell-load-flags string flags to pass to mysql shell load utility. This should be a JSON string (default "{\"threads\": 4, \"loadUsers\": true, \"excludeUsers\": [\"'root'@'localhost'\"], \"updateGtidSet\": \"replace\", \"skipBinlog\": true, \"progressFile\": \"\"}") + --mysql-shell-load-flags string flags to pass to mysql shell load utility. This should be a JSON string (default "{\"threads\": 4, \"loadUsers\": true, \"updateGtidSet\": \"replace\", \"skipBinlog\": true, \"progressFile\": \"\"}") --mysql-shell-should-drain decide if we should drain while taking a backup or continue to serving traffic --mysql-shell-speedup-restore speed up restore by disabling redo logging and double write buffer during the restore process --mysql-shutdown-timeout duration how long to wait for mysqld shutdown (default 5m0s) diff --git a/go/flags/endtoend/vtcombo.txt b/go/flags/endtoend/vtcombo.txt index 77d825dee9e..5e82ad5b28e 100644 --- a/go/flags/endtoend/vtcombo.txt +++ b/go/flags/endtoend/vtcombo.txt @@ -229,7 +229,7 @@ Flags: --mysql-shell-backup-location string location where the backup will be stored --mysql-shell-dump-flags string flags to pass to mysql shell dump utility. This should be a JSON string and will be saved in the MANIFEST (default "{\"threads\": 4}") --mysql-shell-flags string execution flags to pass to mysqlsh binary to be used during dump/load (default "--defaults-file=/dev/null --js -h localhost") - --mysql-shell-load-flags string flags to pass to mysql shell load utility. This should be a JSON string (default "{\"threads\": 4, \"loadUsers\": true, \"excludeUsers\": [\"'root'@'localhost'\"], \"updateGtidSet\": \"replace\", \"skipBinlog\": true, \"progressFile\": \"\"}") + --mysql-shell-load-flags string flags to pass to mysql shell load utility. This should be a JSON string (default "{\"threads\": 4, \"loadUsers\": true, \"updateGtidSet\": \"replace\", \"skipBinlog\": true, \"progressFile\": \"\"}") --mysql-shell-should-drain decide if we should drain while taking a backup or continue to serving traffic --mysql-shell-speedup-restore speed up restore by disabling redo logging and double write buffer during the restore process --mysql-shutdown-timeout duration timeout to use when MySQL is being shut down. (default 5m0s) diff --git a/go/flags/endtoend/vttablet.txt b/go/flags/endtoend/vttablet.txt index bbf5174ae98..5ff9a4b25aa 100644 --- a/go/flags/endtoend/vttablet.txt +++ b/go/flags/endtoend/vttablet.txt @@ -246,7 +246,7 @@ Flags: --mysql-shell-backup-location string location where the backup will be stored --mysql-shell-dump-flags string flags to pass to mysql shell dump utility. This should be a JSON string and will be saved in the MANIFEST (default "{\"threads\": 4}") --mysql-shell-flags string execution flags to pass to mysqlsh binary to be used during dump/load (default "--defaults-file=/dev/null --js -h localhost") - --mysql-shell-load-flags string flags to pass to mysql shell load utility. This should be a JSON string (default "{\"threads\": 4, \"loadUsers\": true, \"excludeUsers\": [\"'root'@'localhost'\"], \"updateGtidSet\": \"replace\", \"skipBinlog\": true, \"progressFile\": \"\"}") + --mysql-shell-load-flags string flags to pass to mysql shell load utility. This should be a JSON string (default "{\"threads\": 4, \"loadUsers\": true, \"updateGtidSet\": \"replace\", \"skipBinlog\": true, \"progressFile\": \"\"}") --mysql-shell-should-drain decide if we should drain while taking a backup or continue to serving traffic --mysql-shell-speedup-restore speed up restore by disabling redo logging and double write buffer during the restore process --mysql-shutdown-timeout duration timeout to use when MySQL is being shut down. (default 5m0s) diff --git a/go/flags/endtoend/vttestserver.txt b/go/flags/endtoend/vttestserver.txt index 538fdd3ea47..0aa69dfb204 100644 --- a/go/flags/endtoend/vttestserver.txt +++ b/go/flags/endtoend/vttestserver.txt @@ -90,7 +90,7 @@ Flags: --mysql-shell-backup-location string location where the backup will be stored --mysql-shell-dump-flags string flags to pass to mysql shell dump utility. This should be a JSON string and will be saved in the MANIFEST (default "{\"threads\": 4}") --mysql-shell-flags string execution flags to pass to mysqlsh binary to be used during dump/load (default "--defaults-file=/dev/null --js -h localhost") - --mysql-shell-load-flags string flags to pass to mysql shell load utility. This should be a JSON string (default "{\"threads\": 4, \"loadUsers\": true, \"excludeUsers\": [\"'root'@'localhost'\"], \"updateGtidSet\": \"replace\", \"skipBinlog\": true, \"progressFile\": \"\"}") + --mysql-shell-load-flags string flags to pass to mysql shell load utility. This should be a JSON string (default "{\"threads\": 4, \"loadUsers\": true, \"updateGtidSet\": \"replace\", \"skipBinlog\": true, \"progressFile\": \"\"}") --mysql-shell-should-drain decide if we should drain while taking a backup or continue to serving traffic --mysql-shell-speedup-restore speed up restore by disabling redo logging and double write buffer during the restore process --mysql_bind_host string which host to bind vtgate mysql listener to (default "localhost") diff --git a/go/vt/mysqlctl/mysqlshellbackupengine.go b/go/vt/mysqlctl/mysqlshellbackupengine.go index a4210518ac8..d1a7bd0fee0 100644 --- a/go/vt/mysqlctl/mysqlshellbackupengine.go +++ b/go/vt/mysqlctl/mysqlshellbackupengine.go @@ -50,7 +50,7 @@ var ( // flags passed to the Dump command, as a JSON string mysqlShellDumpFlags = `{"threads": 4}` // flags passed to the Load command, as a JSON string - mysqlShellLoadFlags = `{"threads": 4, "loadUsers": true, "excludeUsers": ["'root'@'localhost'"], "updateGtidSet": "replace", "skipBinlog": true, "progressFile": ""}` + mysqlShellLoadFlags = `{"threads": 4, "loadUsers": true, "updateGtidSet": "replace", "skipBinlog": true, "progressFile": ""}` // drain a tablet when taking a backup mysqlShellBackupShouldDrain = false // disable redo logging and double write buffer From 4e0a0e2ad930800128028356b9ccc6fdb25a787b Mon Sep 17 00:00:00 2001 From: Renan Rangel Date: Fri, 20 Sep 2024 05:55:13 -0700 Subject: [PATCH 32/34] update vitess operator example Signed-off-by: Renan Rangel --- examples/operator/101_initial_cluster.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/examples/operator/101_initial_cluster.yaml b/examples/operator/101_initial_cluster.yaml index de627c61c50..c26219254f1 100644 --- a/examples/operator/101_initial_cluster.yaml +++ b/examples/operator/101_initial_cluster.yaml @@ -176,6 +176,7 @@ stringData: CREATE USER 'vt_dba'@'localhost'; GRANT ALL ON *.* TO 'vt_dba'@'localhost'; GRANT GRANT OPTION ON *.* TO 'vt_dba'@'localhost'; + GRANT PROXY ON ''@'' TO 'vt_dba'@'localhost' WITH GRANT OPTION; # User for app traffic, with global read-write access. CREATE USER 'vt_app'@'localhost'; From 92181860dd8307f763d33657a8701963583f099e Mon Sep 17 00:00:00 2001 From: Renan Rangel Date: Mon, 23 Sep 2024 03:49:44 -0700 Subject: [PATCH 33/34] PR feedback Signed-off-by: Renan Rangel --- changelog/21.0/21.0.0/summary.md | 4 ++-- go/vt/mysqlctl/mysqlshellbackupengine.go | 6 ++---- 2 files changed, 4 insertions(+), 6 deletions(-) diff --git a/changelog/21.0/21.0.0/summary.md b/changelog/21.0/21.0.0/summary.md index b17cfb890eb..f314a0be8df 100644 --- a/changelog/21.0/21.0.0/summary.md +++ b/changelog/21.0/21.0.0/summary.md @@ -135,10 +135,10 @@ Example usage: ### New Backup Engine (EXPERIMENTAL) -We are introducing a the backup engine supporting logical backups starting on v21 to support use cases that require something else besides physical backups. This is experimental and is based on the +We are introducing a backup engine supporting logical backups starting on v21 to support use cases that require something else besides physical backups. This is experimental and is based on the [MySQL Shell](https://dev.mysql.com/doc/mysql-shell/8.0/en/). -The new engine is enabled by using `--backup_engine_implementation=mysqlshell`. There are other options that are required, so check the docs on which options are required and how to use it. +The new engine is enabled by using `--backup_engine_implementation=mysqlshell`. There are other options that are required, so [check the docs](https://vitess.io/docs/21.0/user-guides/operating-vitess/backup-and-restore/creating-a-backup/) on which options are required and how to use it. ### Dynamic VReplication Configuration Currently many of the configuration options for VReplication Workflows are vttablet flags. This means that any change diff --git a/go/vt/mysqlctl/mysqlshellbackupengine.go b/go/vt/mysqlctl/mysqlshellbackupengine.go index d1a7bd0fee0..38dccc8c622 100644 --- a/go/vt/mysqlctl/mysqlshellbackupengine.go +++ b/go/vt/mysqlctl/mysqlshellbackupengine.go @@ -85,6 +85,8 @@ type MySQLShellBackupManifest struct { } func init() { + BackupRestoreEngineMap[mysqlShellBackupEngineName] = &MySQLShellBackupEngine{} + for _, cmd := range []string{"vtcombo", "vttablet", "vtbackup", "vttestserver", "vtctldclient"} { servenv.OnParseFor(cmd, registerMysqlShellBackupEngineFlags) } @@ -578,7 +580,3 @@ func cleanupMySQL(ctx context.Context, params RestoreParams, shouldDeleteUsers b return err } - -func init() { - BackupRestoreEngineMap[mysqlShellBackupEngineName] = &MySQLShellBackupEngine{} -} From bc2cff24acb5d90adc68f0242a457ecf1b9bb850 Mon Sep 17 00:00:00 2001 From: Renan Rangel Date: Tue, 24 Sep 2024 09:15:24 -0700 Subject: [PATCH 34/34] add missing grant Signed-off-by: Renan Rangel --- examples/compose/config/init_db.sql | 1 + .../endtoend/vreplication/testdata/config/init_testserver_db.sql | 1 + 2 files changed, 2 insertions(+) diff --git a/examples/compose/config/init_db.sql b/examples/compose/config/init_db.sql index b567faf1722..8c0ba4ebdd9 100644 --- a/examples/compose/config/init_db.sql +++ b/examples/compose/config/init_db.sql @@ -39,6 +39,7 @@ CREATE TABLE IF NOT EXISTS _vt.shard_metadata ( CREATE USER 'vt_dba'@'localhost'; GRANT ALL ON *.* TO 'vt_dba'@'localhost'; GRANT GRANT OPTION ON *.* TO 'vt_dba'@'localhost'; +GRANT PROXY ON ''@'' TO 'vt_dba'@'localhost' WITH GRANT OPTION; # User for app traffic, with global read-write access. CREATE USER 'vt_app'@'localhost'; GRANT SELECT, INSERT, UPDATE, DELETE, CREATE, DROP, RELOAD, PROCESS, FILE, diff --git a/go/test/endtoend/vreplication/testdata/config/init_testserver_db.sql b/go/test/endtoend/vreplication/testdata/config/init_testserver_db.sql index fc78f6b414a..71bfba11ed8 100644 --- a/go/test/endtoend/vreplication/testdata/config/init_testserver_db.sql +++ b/go/test/endtoend/vreplication/testdata/config/init_testserver_db.sql @@ -38,6 +38,7 @@ DROP DATABASE IF EXISTS test; CREATE USER 'vt_dba'@'localhost'; GRANT ALL ON *.* TO 'vt_dba'@'localhost'; GRANT GRANT OPTION ON *.* TO 'vt_dba'@'localhost'; +GRANT PROXY ON ''@'' TO 'vt_dba'@'localhost' WITH GRANT OPTION; # User for app traffic, with global read-write access. CREATE USER 'vt_app'@'localhost';