Choose WAL-G backup by start time when cloning and deleting (#3)

* Choose WAL-G backup by start time when cloning * Use wal-g delete retain to clean up old backups * Pick base backup based on finish_time instead of start_time --------- Co-authored-by: Simon Kollberg <[email protected]>
elastisys · Dec 11, 2024 · 4a18050 · 4a18050
1 parent 74c927d
commit 4a18050
Show file tree

Hide file tree

Showing 2 changed files with 35 additions and 25 deletions.
diff --git a/postgres-appliance/bootstrap/clone_with_wale.py b/postgres-appliance/bootstrap/clone_with_wale.py
@@ -2,6 +2,7 @@
 
 import argparse
 import csv
+import json
 import logging
 import os
 import re
@@ -40,14 +41,16 @@ def read_configuration():
     return options(args.scope, args.datadir, recovery_target_time, args.dry_run)
 
 
-def build_wale_command(command, datadir=None, backup=None):
+def build_wale_command(command, datadir=None, backup=None, extra_args=None):
     cmd = ['wal-g' if os.getenv('USE_WALG_RESTORE') == 'true' else 'wal-e'] + [command]
     if command == 'backup-fetch':
         if datadir is None or backup is None:
             raise Exception("backup-fetch requires datadir and backup arguments")
         cmd.extend([datadir, backup])
     elif command != 'backup-list':
         raise Exception("invalid {0} command {1}".format(cmd[0], command))
+    if extra_args:
+        cmd.extend(extra_args)
     return cmd
 
 
@@ -69,7 +72,7 @@ def choose_backup(backup_list, recovery_target_time):
 
     match_timestamp = match = None
     for backup in backup_list:
-        last_modified = parse(backup['last_modified'])
+        last_modified = parse(backup['finish_time' if os.getenv('USE_WALG_RESTORE') == 'true' else 'last_modified'])
         if last_modified < recovery_target_time:
             if match is None or last_modified > match_timestamp:
                 match = backup
@@ -79,10 +82,15 @@ def choose_backup(backup_list, recovery_target_time):
 
 
 def list_backups(env):
-    backup_list_cmd = build_wale_command('backup-list')
-    output = subprocess.check_output(backup_list_cmd, env=env)
-    reader = csv.DictReader(fix_output(output), dialect='excel-tab')
-    return list(reader)
+    if os.getenv('USE_WALG_RESTORE') == 'true':
+        backup_list_cmd = build_wale_command('backup-list', extra_args=['--detail', '--json'])
+        output = subprocess.check_output(backup_list_cmd, env=env)
+        return json.loads(output or 'null')
+    else:
+        backup_list_cmd = build_wale_command('backup-list')
+        output = subprocess.check_output(backup_list_cmd, env=env)
+        reader = csv.DictReader(fix_output(output), dialect='excel-tab')
+        return list(reader)
 
 
 def get_clone_envdir():

diff --git a/postgres-appliance/scripts/postgres_backup.sh b/postgres-appliance/scripts/postgres_backup.sh
@@ -39,29 +39,31 @@ else
     POOL_SIZE=(--pool-size "$POOL_SIZE")
 fi
 
-BEFORE=""
-LEFT=0
-
 NOW=$(date +%s -u)
 readonly NOW
-while read -r name last_modified rest; do
-    last_modified=$(date +%s -ud "$last_modified")
-    if [ $(((NOW-last_modified)/86400)) -ge $DAYS_TO_RETAIN ]; then
-        if [ -z "$BEFORE" ] || [ "$last_modified" -gt "$BEFORE_TIME" ]; then
-            BEFORE_TIME=$last_modified
-            BEFORE=$name
+
+if [[ "$USE_WALG_BACKUP" == "true" ]]; then
+    AFTER=$((NOW-(DAYS_TO_RETAIN*86400)))
+    $WAL_E delete --use-sentinel-time --confirm retain FIND_FULL "$DAYS_TO_RETAIN" --after "$(date --iso-8601=seconds -d @"${AFTER}")"
+else
+    BEFORE=""
+    LEFT=0
+
+    while read -r name last_modified rest; do
+        last_modified=$(date +%s -ud "$last_modified")
+        if [ $(((NOW-last_modified)/86400)) -ge $DAYS_TO_RETAIN ]; then
+            if [ -z "$BEFORE" ] || [ "$last_modified" -gt "$BEFORE_TIME" ]; then
+                BEFORE_TIME=$last_modified
+                BEFORE=$name
+            fi
+        else
+            # count how many backups will remain after we remove everything up to certain date
+            ((LEFT=LEFT+1))
         fi
-    else
-        # count how many backups will remain after we remove everything up to certain date
-        ((LEFT=LEFT+1))
-    fi
-done < <($WAL_E backup-list 2> /dev/null | sed '0,/^\(backup_\)\?name\s*\(last_\)\?modified\s*/d')
+    done < <($WAL_E backup-list 2> /dev/null | sed '0,/^name\s*\(last_\)\?modified\s*/d')
 
-# we want keep at least N backups even if the number of days exceeded
-if [ -n "$BEFORE" ] && [ $LEFT -ge $DAYS_TO_RETAIN ]; then
-    if [[ "$USE_WALG_BACKUP" == "true" ]]; then
-        $WAL_E delete before FIND_FULL "$BEFORE" --confirm
-    else
+    # we want keep at least N backups even if the number of days exceeded
+    if [ -n "$BEFORE" ] && [ $LEFT -ge $DAYS_TO_RETAIN ]; then
         $WAL_E delete --confirm before "$BEFORE"
     fi
 fi