diff --git a/postgres-appliance/bootstrap/clone_with_wale.py b/postgres-appliance/bootstrap/clone_with_wale.py index 9e0adc1c5..595d19e38 100755 --- a/postgres-appliance/bootstrap/clone_with_wale.py +++ b/postgres-appliance/bootstrap/clone_with_wale.py @@ -2,6 +2,7 @@ import argparse import csv +import json import logging import os import re @@ -40,7 +41,7 @@ def read_configuration(): return options(args.scope, args.datadir, recovery_target_time, args.dry_run) -def build_wale_command(command, datadir=None, backup=None): +def build_wale_command(command, datadir=None, backup=None, extra_args=None): cmd = ['wal-g' if os.getenv('USE_WALG_RESTORE') == 'true' else 'wal-e'] + [command] if command == 'backup-fetch': if datadir is None or backup is None: @@ -48,6 +49,8 @@ def build_wale_command(command, datadir=None, backup=None): cmd.extend([datadir, backup]) elif command != 'backup-list': raise Exception("invalid {0} command {1}".format(cmd[0], command)) + if extra_args: + cmd.extend(extra_args) return cmd @@ -69,7 +72,7 @@ def choose_backup(backup_list, recovery_target_time): match_timestamp = match = None for backup in backup_list: - last_modified = parse(backup['last_modified']) + last_modified = parse(backup['finish_time' if os.getenv('USE_WALG_RESTORE') == 'true' else 'last_modified']) if last_modified < recovery_target_time: if match is None or last_modified > match_timestamp: match = backup @@ -79,10 +82,15 @@ def choose_backup(backup_list, recovery_target_time): def list_backups(env): - backup_list_cmd = build_wale_command('backup-list') - output = subprocess.check_output(backup_list_cmd, env=env) - reader = csv.DictReader(fix_output(output), dialect='excel-tab') - return list(reader) + if os.getenv('USE_WALG_RESTORE') == 'true': + backup_list_cmd = build_wale_command('backup-list', extra_args=['--detail', '--json']) + output = subprocess.check_output(backup_list_cmd, env=env) + return json.loads(output or 'null') + else: + backup_list_cmd = build_wale_command('backup-list') + output = subprocess.check_output(backup_list_cmd, env=env) + reader = csv.DictReader(fix_output(output), dialect='excel-tab') + return list(reader) def get_clone_envdir(): diff --git a/postgres-appliance/scripts/postgres_backup.sh b/postgres-appliance/scripts/postgres_backup.sh index 9b9a4723a..f26cc08e9 100755 --- a/postgres-appliance/scripts/postgres_backup.sh +++ b/postgres-appliance/scripts/postgres_backup.sh @@ -39,29 +39,31 @@ else POOL_SIZE=(--pool-size "$POOL_SIZE") fi -BEFORE="" -LEFT=0 - NOW=$(date +%s -u) readonly NOW -while read -r name last_modified rest; do - last_modified=$(date +%s -ud "$last_modified") - if [ $(((NOW-last_modified)/86400)) -ge $DAYS_TO_RETAIN ]; then - if [ -z "$BEFORE" ] || [ "$last_modified" -gt "$BEFORE_TIME" ]; then - BEFORE_TIME=$last_modified - BEFORE=$name + +if [[ "$USE_WALG_BACKUP" == "true" ]]; then + AFTER=$((NOW-(DAYS_TO_RETAIN*86400))) + $WAL_E delete --use-sentinel-time --confirm retain FIND_FULL "$DAYS_TO_RETAIN" --after "$(date --iso-8601=seconds -d @"${AFTER}")" +else + BEFORE="" + LEFT=0 + + while read -r name last_modified rest; do + last_modified=$(date +%s -ud "$last_modified") + if [ $(((NOW-last_modified)/86400)) -ge $DAYS_TO_RETAIN ]; then + if [ -z "$BEFORE" ] || [ "$last_modified" -gt "$BEFORE_TIME" ]; then + BEFORE_TIME=$last_modified + BEFORE=$name + fi + else + # count how many backups will remain after we remove everything up to certain date + ((LEFT=LEFT+1)) fi - else - # count how many backups will remain after we remove everything up to certain date - ((LEFT=LEFT+1)) - fi -done < <($WAL_E backup-list 2> /dev/null | sed '0,/^\(backup_\)\?name\s*\(last_\)\?modified\s*/d') + done < <($WAL_E backup-list 2> /dev/null | sed '0,/^name\s*\(last_\)\?modified\s*/d') -# we want keep at least N backups even if the number of days exceeded -if [ -n "$BEFORE" ] && [ $LEFT -ge $DAYS_TO_RETAIN ]; then - if [[ "$USE_WALG_BACKUP" == "true" ]]; then - $WAL_E delete before FIND_FULL "$BEFORE" --confirm - else + # we want keep at least N backups even if the number of days exceeded + if [ -n "$BEFORE" ] && [ $LEFT -ge $DAYS_TO_RETAIN ]; then $WAL_E delete --confirm before "$BEFORE" fi fi