From b3d45fdb5be936f3815ff8e3efcb3af4c1afe06d Mon Sep 17 00:00:00 2001 From: Shlomi Noach <2607934+shlomi-noach@users.noreply.github.com> Date: Mon, 25 Mar 2024 12:19:58 +0200 Subject: [PATCH] Configurable incremental restore files path (#15451) Signed-off-by: Shlomi Noach <2607934+shlomi-noach@users.noreply.github.com> --- go/flags/endtoend/vtbackup.txt | 1 + go/flags/endtoend/vtcombo.txt | 1 + go/flags/endtoend/vtctld.txt | 1 + go/flags/endtoend/vttablet.txt | 1 + go/flags/endtoend/vttestserver.txt | 1 + go/vt/mysqlctl/builtinbackupengine.go | 9 ++++++++- 6 files changed, 13 insertions(+), 1 deletion(-) diff --git a/go/flags/endtoend/vtbackup.txt b/go/flags/endtoend/vtbackup.txt index 1b9f5aca61f..a814cd80342 100644 --- a/go/flags/endtoend/vtbackup.txt +++ b/go/flags/endtoend/vtbackup.txt @@ -58,6 +58,7 @@ Flags: --bind-address string Bind address for the server. If empty, the server will listen on all available unicast and anycast IP addresses of the local system. --builtinbackup-file-read-buffer-size uint read files using an IO buffer of this many bytes. Golang defaults are used when set to 0. --builtinbackup-file-write-buffer-size uint write files using an IO buffer of this many bytes. Golang defaults are used when set to 0. (default 2097152) + --builtinbackup-incremental-restore-path string the directory where incremental restore files, namely binlog files, are extracted to. In k8s environments, this should be set to a directory that is shared between the vttablet and mysqld pods. The path should exist. When empty, the default OS temp dir is assumed. --builtinbackup_mysqld_timeout duration how long to wait for mysqld to shutdown at the start of the backup. (default 10m0s) --builtinbackup_progress duration how often to send progress updates when backing up large files. (default 5s) --ceph_backup_storage_config string Path to JSON config file for ceph backup storage. (default "ceph_backup_config.json") diff --git a/go/flags/endtoend/vtcombo.txt b/go/flags/endtoend/vtcombo.txt index ee759218d9c..763f91b0ca9 100644 --- a/go/flags/endtoend/vtcombo.txt +++ b/go/flags/endtoend/vtcombo.txt @@ -38,6 +38,7 @@ Flags: --buffer_window duration Duration for how long a request should be buffered at most. (default 10s) --builtinbackup-file-read-buffer-size uint read files using an IO buffer of this many bytes. Golang defaults are used when set to 0. --builtinbackup-file-write-buffer-size uint write files using an IO buffer of this many bytes. Golang defaults are used when set to 0. (default 2097152) + --builtinbackup-incremental-restore-path string the directory where incremental restore files, namely binlog files, are extracted to. In k8s environments, this should be set to a directory that is shared between the vttablet and mysqld pods. The path should exist. When empty, the default OS temp dir is assumed. --builtinbackup_mysqld_timeout duration how long to wait for mysqld to shutdown at the start of the backup. (default 10m0s) --builtinbackup_progress duration how often to send progress updates when backing up large files. (default 5s) --catch-sigpipe catch and ignore SIGPIPE on stdout and stderr if specified diff --git a/go/flags/endtoend/vtctld.txt b/go/flags/endtoend/vtctld.txt index e0f552d2b00..969e4f9774e 100644 --- a/go/flags/endtoend/vtctld.txt +++ b/go/flags/endtoend/vtctld.txt @@ -37,6 +37,7 @@ Flags: --bind-address string Bind address for the server. If empty, the server will listen on all available unicast and anycast IP addresses of the local system. --builtinbackup-file-read-buffer-size uint read files using an IO buffer of this many bytes. Golang defaults are used when set to 0. --builtinbackup-file-write-buffer-size uint write files using an IO buffer of this many bytes. Golang defaults are used when set to 0. (default 2097152) + --builtinbackup-incremental-restore-path string the directory where incremental restore files, namely binlog files, are extracted to. In k8s environments, this should be set to a directory that is shared between the vttablet and mysqld pods. The path should exist. When empty, the default OS temp dir is assumed. --builtinbackup_mysqld_timeout duration how long to wait for mysqld to shutdown at the start of the backup. (default 10m0s) --builtinbackup_progress duration how often to send progress updates when backing up large files. (default 5s) --catch-sigpipe catch and ignore SIGPIPE on stdout and stderr if specified diff --git a/go/flags/endtoend/vttablet.txt b/go/flags/endtoend/vttablet.txt index 9f089275aba..86883ec544b 100644 --- a/go/flags/endtoend/vttablet.txt +++ b/go/flags/endtoend/vttablet.txt @@ -72,6 +72,7 @@ Flags: --binlog_user string PITR restore parameter: username of binlog server. --builtinbackup-file-read-buffer-size uint read files using an IO buffer of this many bytes. Golang defaults are used when set to 0. --builtinbackup-file-write-buffer-size uint write files using an IO buffer of this many bytes. Golang defaults are used when set to 0. (default 2097152) + --builtinbackup-incremental-restore-path string the directory where incremental restore files, namely binlog files, are extracted to. In k8s environments, this should be set to a directory that is shared between the vttablet and mysqld pods. The path should exist. When empty, the default OS temp dir is assumed. --builtinbackup_mysqld_timeout duration how long to wait for mysqld to shutdown at the start of the backup. (default 10m0s) --builtinbackup_progress duration how often to send progress updates when backing up large files. (default 5s) --catch-sigpipe catch and ignore SIGPIPE on stdout and stderr if specified diff --git a/go/flags/endtoend/vttestserver.txt b/go/flags/endtoend/vttestserver.txt index 7c0df363c15..2683fb70171 100644 --- a/go/flags/endtoend/vttestserver.txt +++ b/go/flags/endtoend/vttestserver.txt @@ -13,6 +13,7 @@ Flags: --backup_storage_number_blocks int if backup_storage_compress is true, backup_storage_number_blocks sets the number of blocks that can be processed, in parallel, before the writer blocks, during compression (default is 2). It should be equal to the number of CPUs available for compression. (default 2) --builtinbackup-file-read-buffer-size uint read files using an IO buffer of this many bytes. Golang defaults are used when set to 0. --builtinbackup-file-write-buffer-size uint write files using an IO buffer of this many bytes. Golang defaults are used when set to 0. (default 2097152) + --builtinbackup-incremental-restore-path string the directory where incremental restore files, namely binlog files, are extracted to. In k8s environments, this should be set to a directory that is shared between the vttablet and mysqld pods. The path should exist. When empty, the default OS temp dir is assumed. --builtinbackup_mysqld_timeout duration how long to wait for mysqld to shutdown at the start of the backup. (default 10m0s) --builtinbackup_progress duration how often to send progress updates when backing up large files. (default 5s) --catch-sigpipe catch and ignore SIGPIPE on stdout and stderr if specified diff --git a/go/vt/mysqlctl/builtinbackupengine.go b/go/vt/mysqlctl/builtinbackupengine.go index 0ffcd1aba76..5cdf39084da 100644 --- a/go/vt/mysqlctl/builtinbackupengine.go +++ b/go/vt/mysqlctl/builtinbackupengine.go @@ -80,6 +80,12 @@ var ( // engines during backups. The backupstorage may be a physical file, // network, or something else. builtinBackupStorageWriteBufferSize = 2 * 1024 * 1024 /* 2 MiB */ + + // The directory where incremental restore files, namely binlog files, are extracted to. + // In k8s environments, this should be set to a directory that is shared between the vttablet and mysqld pods. + // The path should exist. + // When empty, the default OS temp dir is assumed. + builtinIncrementalRestorePath = "" ) // BuiltinBackupEngine encapsulates the logic of the builtin engine @@ -157,6 +163,7 @@ func registerBuiltinBackupEngineFlags(fs *pflag.FlagSet) { fs.DurationVar(&builtinBackupProgress, "builtinbackup_progress", builtinBackupProgress, "how often to send progress updates when backing up large files.") fs.UintVar(&builtinBackupFileReadBufferSize, "builtinbackup-file-read-buffer-size", builtinBackupFileReadBufferSize, "read files using an IO buffer of this many bytes. Golang defaults are used when set to 0.") fs.UintVar(&builtinBackupFileWriteBufferSize, "builtinbackup-file-write-buffer-size", builtinBackupFileWriteBufferSize, "write files using an IO buffer of this many bytes. Golang defaults are used when set to 0.") + fs.StringVar(&builtinIncrementalRestorePath, "builtinbackup-incremental-restore-path", builtinIncrementalRestorePath, "the directory where incremental restore files, namely binlog files, are extracted to. In k8s environments, this should be set to a directory that is shared between the vttablet and mysqld pods. The path should exist. When empty, the default OS temp dir is assumed.") } // fullPath returns the full path of the entry, based on its type @@ -1006,7 +1013,7 @@ func (be *BuiltinBackupEngine) restoreFiles(ctx context.Context, params RestoreP } if bm.Incremental { - createdDir, err = os.MkdirTemp("", "restore-incremental-*") + createdDir, err = os.MkdirTemp(builtinIncrementalRestorePath, "restore-incremental-*") if err != nil { return "", err }