From a4f11b000037bcaecbe49986187f7e3f286a312e Mon Sep 17 00:00:00 2001 From: Simon Kollberg Date: Wed, 29 May 2024 15:59:18 +0200 Subject: [PATCH 1/8] Choose WAL-G backup by start time when cloning --- .../bootstrap/clone_with_wale.py | 22 +++++++++++++------ 1 file changed, 15 insertions(+), 7 deletions(-) diff --git a/postgres-appliance/bootstrap/clone_with_wale.py b/postgres-appliance/bootstrap/clone_with_wale.py index e8d31962d..041fde197 100755 --- a/postgres-appliance/bootstrap/clone_with_wale.py +++ b/postgres-appliance/bootstrap/clone_with_wale.py @@ -2,6 +2,7 @@ import argparse import csv +import json import logging import os import re @@ -40,7 +41,7 @@ def read_configuration(): return options(args.scope, args.datadir, recovery_target_time, args.dry_run) -def build_wale_command(command, datadir=None, backup=None): +def build_wale_command(command, datadir=None, backup=None, extra_args=None): cmd = ['wal-g' if os.getenv('USE_WALG_RESTORE') == 'true' else 'wal-e'] + [command] if command == 'backup-fetch': if datadir is None or backup is None: @@ -48,6 +49,8 @@ def build_wale_command(command, datadir=None, backup=None): cmd.extend([datadir, backup]) elif command != 'backup-list': raise Exception("invalid {0} command {1}".format(cmd[0], command)) + if extra_args: + cmd.extend(extra_args) return cmd @@ -70,20 +73,25 @@ def choose_backup(backup_list, recovery_target_time): match_timestamp = match = None for backup in backup_list: - last_modified = parse(backup['last_modified']) + last_modified = parse(backup['start_time' if os.getenv('USE_WALG_RESTORE') == 'true' else 'last_modified']) if last_modified < recovery_target_time: if match is None or last_modified > match_timestamp: match = backup match_timestamp = last_modified if match is not None: - return match['name'] + return match['backup_name' if os.getenv('USE_WALG_RESTORE') == 'true' else 'name'] def list_backups(env): - backup_list_cmd = build_wale_command('backup-list') - output = subprocess.check_output(backup_list_cmd, env=env) - reader = csv.DictReader(fix_output(output), dialect='excel-tab') - return list(reader) + if os.getenv('USE_WALG_RESTORE') == 'true': + backup_list_cmd = build_wale_command('backup-list', extra_args=['--detail', '--json']) + output = subprocess.check_output(backup_list_cmd, env=env) + return json.loads(output or 'null') + else: + backup_list_cmd = build_wale_command('backup-list') + output = subprocess.check_output(backup_list_cmd, env=env) + reader = csv.DictReader(fix_output(output), dialect='excel-tab') + return list(reader) def get_clone_envdir(): From 30b4d13c48f9833a8d9fa22efc6d8604d3a4f850 Mon Sep 17 00:00:00 2001 From: Simon Kollberg Date: Thu, 20 Jun 2024 11:12:37 +0200 Subject: [PATCH 2/8] Use wal-g delete retain to clean up old backups --- postgres-appliance/Dockerfile | 2 +- postgres-appliance/scripts/postgres_backup.sh | 40 ++++++++++--------- 2 files changed, 22 insertions(+), 20 deletions(-) diff --git a/postgres-appliance/Dockerfile b/postgres-appliance/Dockerfile index 89298181b..ae0946959 100644 --- a/postgres-appliance/Dockerfile +++ b/postgres-appliance/Dockerfile @@ -19,7 +19,7 @@ FROM $BASE_IMAGE as dependencies-builder ARG DEMO -ENV WALG_VERSION=v2.0.1 +ENV WALG_VERSION=v3.0.0 COPY build_scripts/dependencies.sh /builddeps/ diff --git a/postgres-appliance/scripts/postgres_backup.sh b/postgres-appliance/scripts/postgres_backup.sh index 3216ae4ed..f26cc08e9 100755 --- a/postgres-appliance/scripts/postgres_backup.sh +++ b/postgres-appliance/scripts/postgres_backup.sh @@ -39,29 +39,31 @@ else POOL_SIZE=(--pool-size "$POOL_SIZE") fi -BEFORE="" -LEFT=0 - NOW=$(date +%s -u) readonly NOW -while read -r name last_modified rest; do - last_modified=$(date +%s -ud "$last_modified") - if [ $(((NOW-last_modified)/86400)) -ge $DAYS_TO_RETAIN ]; then - if [ -z "$BEFORE" ] || [ "$last_modified" -gt "$BEFORE_TIME" ]; then - BEFORE_TIME=$last_modified - BEFORE=$name + +if [[ "$USE_WALG_BACKUP" == "true" ]]; then + AFTER=$((NOW-(DAYS_TO_RETAIN*86400))) + $WAL_E delete --use-sentinel-time --confirm retain FIND_FULL "$DAYS_TO_RETAIN" --after "$(date --iso-8601=seconds -d @"${AFTER}")" +else + BEFORE="" + LEFT=0 + + while read -r name last_modified rest; do + last_modified=$(date +%s -ud "$last_modified") + if [ $(((NOW-last_modified)/86400)) -ge $DAYS_TO_RETAIN ]; then + if [ -z "$BEFORE" ] || [ "$last_modified" -gt "$BEFORE_TIME" ]; then + BEFORE_TIME=$last_modified + BEFORE=$name + fi + else + # count how many backups will remain after we remove everything up to certain date + ((LEFT=LEFT+1)) fi - else - # count how many backups will remain after we remove everything up to certain date - ((LEFT=LEFT+1)) - fi -done < <($WAL_E backup-list 2> /dev/null | sed '0,/^name\s*\(last_\)\?modified\s*/d') + done < <($WAL_E backup-list 2> /dev/null | sed '0,/^name\s*\(last_\)\?modified\s*/d') -# we want keep at least N backups even if the number of days exceeded -if [ -n "$BEFORE" ] && [ $LEFT -ge $DAYS_TO_RETAIN ]; then - if [[ "$USE_WALG_BACKUP" == "true" ]]; then - $WAL_E delete before FIND_FULL "$BEFORE" --confirm - else + # we want keep at least N backups even if the number of days exceeded + if [ -n "$BEFORE" ] && [ $LEFT -ge $DAYS_TO_RETAIN ]; then $WAL_E delete --confirm before "$BEFORE" fi fi From 8416ba1e2bfcc2e877320478bfa1cac4e3bc2c1e Mon Sep 17 00:00:00 2001 From: Simon Kollberg Date: Mon, 24 Jun 2024 16:05:37 +0200 Subject: [PATCH 3/8] tmpfix --- postgres-appliance/Dockerfile | 2 +- postgres-appliance/build_scripts/dependencies.sh | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/postgres-appliance/Dockerfile b/postgres-appliance/Dockerfile index ae0946959..5f70715eb 100644 --- a/postgres-appliance/Dockerfile +++ b/postgres-appliance/Dockerfile @@ -19,7 +19,7 @@ FROM $BASE_IMAGE as dependencies-builder ARG DEMO -ENV WALG_VERSION=v3.0.0 +ENV WALG_VERSION=fix-libsodium COPY build_scripts/dependencies.sh /builddeps/ diff --git a/postgres-appliance/build_scripts/dependencies.sh b/postgres-appliance/build_scripts/dependencies.sh index 33fe897db..58645a4f1 100644 --- a/postgres-appliance/build_scripts/dependencies.sh +++ b/postgres-appliance/build_scripts/dependencies.sh @@ -29,7 +29,7 @@ apt-get update apt-get install -y golang-go liblzo2-dev brotli libsodium-dev git make cmake gcc libc-dev go version -git clone -b "$WALG_VERSION" --recurse-submodules https://github.com/wal-g/wal-g.git +git clone -b "$WALG_VERSION" --recurse-submodules https://github.com/garry-t/wal-g.git cd /wal-g go get -v -t -d ./... go mod vendor From 65a49d7b453139d048c2de76319b127095f390b3 Mon Sep 17 00:00:00 2001 From: Simon Kollberg Date: Wed, 21 Aug 2024 15:47:21 +0200 Subject: [PATCH 4/8] Revert "tmpfix" This reverts commit 8416ba1e2bfcc2e877320478bfa1cac4e3bc2c1e. --- postgres-appliance/Dockerfile | 2 +- postgres-appliance/build_scripts/dependencies.sh | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/postgres-appliance/Dockerfile b/postgres-appliance/Dockerfile index 5f70715eb..ae0946959 100644 --- a/postgres-appliance/Dockerfile +++ b/postgres-appliance/Dockerfile @@ -19,7 +19,7 @@ FROM $BASE_IMAGE as dependencies-builder ARG DEMO -ENV WALG_VERSION=fix-libsodium +ENV WALG_VERSION=v3.0.0 COPY build_scripts/dependencies.sh /builddeps/ diff --git a/postgres-appliance/build_scripts/dependencies.sh b/postgres-appliance/build_scripts/dependencies.sh index 58645a4f1..33fe897db 100644 --- a/postgres-appliance/build_scripts/dependencies.sh +++ b/postgres-appliance/build_scripts/dependencies.sh @@ -29,7 +29,7 @@ apt-get update apt-get install -y golang-go liblzo2-dev brotli libsodium-dev git make cmake gcc libc-dev go version -git clone -b "$WALG_VERSION" --recurse-submodules https://github.com/garry-t/wal-g.git +git clone -b "$WALG_VERSION" --recurse-submodules https://github.com/wal-g/wal-g.git cd /wal-g go get -v -t -d ./... go mod vendor From c8fcca23959df162c548cd71c4590635e4ab5c39 Mon Sep 17 00:00:00 2001 From: Simon Kollberg Date: Wed, 21 Aug 2024 15:47:59 +0200 Subject: [PATCH 5/8] Bump wal-g v3.0.3 --- postgres-appliance/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/postgres-appliance/Dockerfile b/postgres-appliance/Dockerfile index ae0946959..51fe19d1b 100644 --- a/postgres-appliance/Dockerfile +++ b/postgres-appliance/Dockerfile @@ -19,7 +19,7 @@ FROM $BASE_IMAGE as dependencies-builder ARG DEMO -ENV WALG_VERSION=v3.0.0 +ENV WALG_VERSION=v3.0.3 COPY build_scripts/dependencies.sh /builddeps/ From 899bc941316dff85b0577094792487c90daed057 Mon Sep 17 00:00:00 2001 From: Simon Kollberg Date: Wed, 21 Aug 2024 16:25:20 +0200 Subject: [PATCH 6/8] Conform with upstream changes --- postgres-appliance/bootstrap/clone_with_wale.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/postgres-appliance/bootstrap/clone_with_wale.py b/postgres-appliance/bootstrap/clone_with_wale.py index 041fde197..dbc0b38f5 100755 --- a/postgres-appliance/bootstrap/clone_with_wale.py +++ b/postgres-appliance/bootstrap/clone_with_wale.py @@ -60,8 +60,7 @@ def fix_output(output): started = None for line in output.decode('utf-8').splitlines(): if not started: - started = re.match(r'^name\s+last_modified\s+', line) - started = re.match(r'^name\s+last_modified\s+', line) or re.match(r'^name\s+modified\s+', line) + started = re.match(r'^(backup_)?name\s+(last_)?modified\s+', line) if started: line = line.replace(' modified ', ' last_modified ') if started: @@ -79,7 +78,7 @@ def choose_backup(backup_list, recovery_target_time): match = backup match_timestamp = last_modified if match is not None: - return match['backup_name' if os.getenv('USE_WALG_RESTORE') == 'true' else 'name'] + return match.get('name', match['backup_name']) def list_backups(env): From 272fbf171997f2c158a028938c7e4aabcfac3582 Mon Sep 17 00:00:00 2001 From: Simon Kollberg Date: Tue, 27 Aug 2024 09:10:59 +0200 Subject: [PATCH 7/8] Pick base backup based on finish_time instead of start_time --- postgres-appliance/bootstrap/clone_with_wale.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/postgres-appliance/bootstrap/clone_with_wale.py b/postgres-appliance/bootstrap/clone_with_wale.py index dbc0b38f5..595d19e38 100755 --- a/postgres-appliance/bootstrap/clone_with_wale.py +++ b/postgres-appliance/bootstrap/clone_with_wale.py @@ -72,7 +72,7 @@ def choose_backup(backup_list, recovery_target_time): match_timestamp = match = None for backup in backup_list: - last_modified = parse(backup['start_time' if os.getenv('USE_WALG_RESTORE') == 'true' else 'last_modified']) + last_modified = parse(backup['finish_time' if os.getenv('USE_WALG_RESTORE') == 'true' else 'last_modified']) if last_modified < recovery_target_time: if match is None or last_modified > match_timestamp: match = backup From c788be701a2afd65d757907a47e87b32b125ad59 Mon Sep 17 00:00:00 2001 From: Simon Kollberg Date: Tue, 27 Aug 2024 09:34:46 +0200 Subject: [PATCH 8/8] Another upstream change required for WAL-G v3 --- postgres-appliance/scripts/wale_restore.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/postgres-appliance/scripts/wale_restore.sh b/postgres-appliance/scripts/wale_restore.sh index 138dfecae..d641efb91 100755 --- a/postgres-appliance/scripts/wale_restore.sh +++ b/postgres-appliance/scripts/wale_restore.sh @@ -43,7 +43,7 @@ ATTEMPT=0 server_version="-1" while true; do [[ -z $wal_segment_backup_start ]] && wal_segment_backup_start=$($WAL_E backup-list 2> /dev/null \ - | sed '0,/^name\s*\(last_\)\?modified\s*/d' | sort -bk2 | tail -n1 | awk '{print $3;}' | sed 's/_.*$//') + | sed '0,/^(backup_\)?name\s*\(last_\)\?modified\s*/d' | sort -bk2 | tail -n1 | awk '{print $3;}' | sed 's/_.*$//') [[ -n "$CONNSTR" && $server_version == "-1" ]] && server_version=$(psql -d "$CONNSTR" -tAc 'show server_version_num' 2> /dev/null || echo "-1")