From 8ee3294f37de22eaa7fdb3b43dd34d161a813d3e Mon Sep 17 00:00:00 2001 From: "vitess-bot[bot]" <108069721+vitess-bot[bot]@users.noreply.github.com> Date: Mon, 25 Mar 2024 13:02:32 +0200 Subject: [PATCH] [release-19.0] Configurable incremental restore files path (#15451) (#15564) Signed-off-by: Shlomi Noach <2607934+shlomi-noach@users.noreply.github.com> Co-authored-by: vitess-bot[bot] <108069721+vitess-bot[bot]@users.noreply.github.com> --- go/flags/endtoend/vtbackup.txt | 1 + go/flags/endtoend/vtcombo.txt | 1 + go/flags/endtoend/vtctld.txt | 1 + go/flags/endtoend/vttablet.txt | 1 + go/flags/endtoend/vttestserver.txt | 1 + go/vt/mysqlctl/builtinbackupengine.go | 9 ++++++++- 6 files changed, 13 insertions(+), 1 deletion(-) diff --git a/go/flags/endtoend/vtbackup.txt b/go/flags/endtoend/vtbackup.txt index ea98ad70b88..fd6a771dbeb 100644 --- a/go/flags/endtoend/vtbackup.txt +++ b/go/flags/endtoend/vtbackup.txt @@ -58,6 +58,7 @@ Flags: --bind-address string Bind address for the server. If empty, the server will listen on all available unicast and anycast IP addresses of the local system. --builtinbackup-file-read-buffer-size uint read files using an IO buffer of this many bytes. Golang defaults are used when set to 0. --builtinbackup-file-write-buffer-size uint write files using an IO buffer of this many bytes. Golang defaults are used when set to 0. (default 2097152) + --builtinbackup-incremental-restore-path string the directory where incremental restore files, namely binlog files, are extracted to. In k8s environments, this should be set to a directory that is shared between the vttablet and mysqld pods. The path should exist. When empty, the default OS temp dir is assumed. --builtinbackup_mysqld_timeout duration how long to wait for mysqld to shutdown at the start of the backup. (default 10m0s) --builtinbackup_progress duration how often to send progress updates when backing up large files. (default 5s) --ceph_backup_storage_config string Path to JSON config file for ceph backup storage. (default "ceph_backup_config.json") diff --git a/go/flags/endtoend/vtcombo.txt b/go/flags/endtoend/vtcombo.txt index 337a0e40454..340251d2b76 100644 --- a/go/flags/endtoend/vtcombo.txt +++ b/go/flags/endtoend/vtcombo.txt @@ -38,6 +38,7 @@ Flags: --buffer_window duration Duration for how long a request should be buffered at most. (default 10s) --builtinbackup-file-read-buffer-size uint read files using an IO buffer of this many bytes. Golang defaults are used when set to 0. --builtinbackup-file-write-buffer-size uint write files using an IO buffer of this many bytes. Golang defaults are used when set to 0. (default 2097152) + --builtinbackup-incremental-restore-path string the directory where incremental restore files, namely binlog files, are extracted to. In k8s environments, this should be set to a directory that is shared between the vttablet and mysqld pods. The path should exist. When empty, the default OS temp dir is assumed. --builtinbackup_mysqld_timeout duration how long to wait for mysqld to shutdown at the start of the backup. (default 10m0s) --builtinbackup_progress duration how often to send progress updates when backing up large files. (default 5s) --catch-sigpipe catch and ignore SIGPIPE on stdout and stderr if specified diff --git a/go/flags/endtoend/vtctld.txt b/go/flags/endtoend/vtctld.txt index 288954e71cb..5d5bf2bcfc3 100644 --- a/go/flags/endtoend/vtctld.txt +++ b/go/flags/endtoend/vtctld.txt @@ -37,6 +37,7 @@ Flags: --bind-address string Bind address for the server. If empty, the server will listen on all available unicast and anycast IP addresses of the local system. --builtinbackup-file-read-buffer-size uint read files using an IO buffer of this many bytes. Golang defaults are used when set to 0. --builtinbackup-file-write-buffer-size uint write files using an IO buffer of this many bytes. Golang defaults are used when set to 0. (default 2097152) + --builtinbackup-incremental-restore-path string the directory where incremental restore files, namely binlog files, are extracted to. In k8s environments, this should be set to a directory that is shared between the vttablet and mysqld pods. The path should exist. When empty, the default OS temp dir is assumed. --builtinbackup_mysqld_timeout duration how long to wait for mysqld to shutdown at the start of the backup. (default 10m0s) --builtinbackup_progress duration how often to send progress updates when backing up large files. (default 5s) --catch-sigpipe catch and ignore SIGPIPE on stdout and stderr if specified diff --git a/go/flags/endtoend/vttablet.txt b/go/flags/endtoend/vttablet.txt index 8f18c859992..71f1f42cf72 100644 --- a/go/flags/endtoend/vttablet.txt +++ b/go/flags/endtoend/vttablet.txt @@ -72,6 +72,7 @@ Flags: --binlog_user string PITR restore parameter: username of binlog server. --builtinbackup-file-read-buffer-size uint read files using an IO buffer of this many bytes. Golang defaults are used when set to 0. --builtinbackup-file-write-buffer-size uint write files using an IO buffer of this many bytes. Golang defaults are used when set to 0. (default 2097152) + --builtinbackup-incremental-restore-path string the directory where incremental restore files, namely binlog files, are extracted to. In k8s environments, this should be set to a directory that is shared between the vttablet and mysqld pods. The path should exist. When empty, the default OS temp dir is assumed. --builtinbackup_mysqld_timeout duration how long to wait for mysqld to shutdown at the start of the backup. (default 10m0s) --builtinbackup_progress duration how often to send progress updates when backing up large files. (default 5s) --catch-sigpipe catch and ignore SIGPIPE on stdout and stderr if specified diff --git a/go/flags/endtoend/vttestserver.txt b/go/flags/endtoend/vttestserver.txt index 70929bf6e4a..ff9fa98af28 100644 --- a/go/flags/endtoend/vttestserver.txt +++ b/go/flags/endtoend/vttestserver.txt @@ -13,6 +13,7 @@ Flags: --backup_storage_number_blocks int if backup_storage_compress is true, backup_storage_number_blocks sets the number of blocks that can be processed, in parallel, before the writer blocks, during compression (default is 2). It should be equal to the number of CPUs available for compression. (default 2) --builtinbackup-file-read-buffer-size uint read files using an IO buffer of this many bytes. Golang defaults are used when set to 0. --builtinbackup-file-write-buffer-size uint write files using an IO buffer of this many bytes. Golang defaults are used when set to 0. (default 2097152) + --builtinbackup-incremental-restore-path string the directory where incremental restore files, namely binlog files, are extracted to. In k8s environments, this should be set to a directory that is shared between the vttablet and mysqld pods. The path should exist. When empty, the default OS temp dir is assumed. --builtinbackup_mysqld_timeout duration how long to wait for mysqld to shutdown at the start of the backup. (default 10m0s) --builtinbackup_progress duration how often to send progress updates when backing up large files. (default 5s) --catch-sigpipe catch and ignore SIGPIPE on stdout and stderr if specified diff --git a/go/vt/mysqlctl/builtinbackupengine.go b/go/vt/mysqlctl/builtinbackupengine.go index 023439d1fff..94ed7bdee6a 100644 --- a/go/vt/mysqlctl/builtinbackupengine.go +++ b/go/vt/mysqlctl/builtinbackupengine.go @@ -80,6 +80,12 @@ var ( // engines during backups. The backupstorage may be a physical file, // network, or something else. builtinBackupStorageWriteBufferSize = 2 * 1024 * 1024 /* 2 MiB */ + + // The directory where incremental restore files, namely binlog files, are extracted to. + // In k8s environments, this should be set to a directory that is shared between the vttablet and mysqld pods. + // The path should exist. + // When empty, the default OS temp dir is assumed. + builtinIncrementalRestorePath = "" ) // BuiltinBackupEngine encapsulates the logic of the builtin engine @@ -157,6 +163,7 @@ func registerBuiltinBackupEngineFlags(fs *pflag.FlagSet) { fs.DurationVar(&builtinBackupProgress, "builtinbackup_progress", builtinBackupProgress, "how often to send progress updates when backing up large files.") fs.UintVar(&builtinBackupFileReadBufferSize, "builtinbackup-file-read-buffer-size", builtinBackupFileReadBufferSize, "read files using an IO buffer of this many bytes. Golang defaults are used when set to 0.") fs.UintVar(&builtinBackupFileWriteBufferSize, "builtinbackup-file-write-buffer-size", builtinBackupFileWriteBufferSize, "write files using an IO buffer of this many bytes. Golang defaults are used when set to 0.") + fs.StringVar(&builtinIncrementalRestorePath, "builtinbackup-incremental-restore-path", builtinIncrementalRestorePath, "the directory where incremental restore files, namely binlog files, are extracted to. In k8s environments, this should be set to a directory that is shared between the vttablet and mysqld pods. The path should exist. When empty, the default OS temp dir is assumed.") } // fullPath returns the full path of the entry, based on its type @@ -1005,7 +1012,7 @@ func (be *BuiltinBackupEngine) restoreFiles(ctx context.Context, params RestoreP } if bm.Incremental { - createdDir, err = os.MkdirTemp("", "restore-incremental-*") + createdDir, err = os.MkdirTemp(builtinIncrementalRestorePath, "restore-incremental-*") if err != nil { return "", err }