-
Notifications
You must be signed in to change notification settings - Fork 43
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
feat: logging for PostgreSQL wal-g scripts (#1120)
- Loading branch information
1 parent
cfe6878
commit 81459e5
Showing
10 changed files
with
257 additions
and
67 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,46 @@ | ||
| {{/* | ||
| This function is used to format a line to the log format, so that it can be saved into a log file. | ||
| Usage: | ||
| format_log_content SOME_TYPE SOME_COMPONENT | ||
| Result: | ||
| The line to be logged will be formatted with some logging information, and be printed to the stdout | ||
| Example: | ||
| echo "some line content from stdout" | format_log_content STDOUT SOME_COMPONENT > /path/to/logfile | ||
| echo "some line content from stderr" | format_log_content STDERR SOME_COMPONENT > /path/to/logfile | ||
| */}} | ||
| {{- define "kblib.logs.format_log_content" }} | ||
| format_log_content() { | ||
| local content_type #STDOUT or STDERR | ||
| local component | ||
| content_type=$1 | ||
| component=$2 | ||
| while IFS= read -r line; do | ||
| echo "$(date -u '+%Y-%m-%dT%H:%M:%SZ') [$content_type] [$component] $line" | ||
| done | ||
| } | ||
| {{- end }} | ||
|
|
||
| {{/* | ||
| This function is used to setup the logging of the script, | ||
| so that the STDOUT/STDERR of the scripts can be recorded into a dedicated file. | ||
| Usage: | ||
| setup_logging LOG_COMPONENT LOG_FILE | ||
| Result: | ||
| The stdout / stderr of the script will be saved into a dedicated file | ||
| Example: | ||
| setup_logging SOME_COMPONENT dummy.log | ||
| */}} | ||
| {{- define "kblib.logs.setup_logging" }} | ||
| setup_logging() { | ||
| local component | ||
| local log_file | ||
| component=$1 | ||
| log_file=$2 | ||
| # redirect all the stdout and stderr to files | ||
| exec > >( tee >( format_log_content STDOUT "${component}" | stdbuf -oL cat >> "${log_file}" ) ) \ | ||
| 2> >( tee >( format_log_content STDERR "${component}" | stdbuf -oL cat >> "${log_file}" ) >&2 ) | ||
| } | ||
| {{- end }} |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -1,26 +1,34 @@ | ||
| # shellcheck disable=SC2148 | ||
| set -e | ||
|
|
||
| postgres_log_dir="${VOLUME_DATA_DIR}/logs" | ||
| postgres_scripts_log_file="${postgres_log_dir}/scripts.log" | ||
| setup_logging WALG_CONFIG "${postgres_scripts_log_file}" | ||
|
|
||
| if [[ ! -f /etc/datasafed/datasafed.conf ]]; then | ||
| echo "ERROR: backupRepo should use Tool accessMode" | ||
| exit 1 | ||
| echo "ERROR: backupRepo should use Tool accessMode" | ||
| exit 1 | ||
| fi | ||
|
|
||
| function config_wal_g() { | ||
| walg_dir=${VOLUME_DATA_DIR}/wal-g | ||
| walg_env=${walg_dir}/env | ||
| mkdir -p ${walg_dir}/env | ||
| cp /etc/datasafed/datasafed.conf ${walg_dir}/datasafed.conf | ||
| cp /usr/bin/wal-g ${walg_dir}/wal-g | ||
| walg_dir="${VOLUME_DATA_DIR}/wal-g" | ||
| walg_env="${walg_dir}/env" | ||
| mkdir -p "${walg_env}" | ||
| cp /etc/datasafed/datasafed.conf "${walg_dir}/datasafed.conf" | ||
| cp /usr/bin/wal-g "${walg_dir}/wal-g" | ||
| datasafed_base_path=${1:?missing datasafed_base_path} | ||
| # config wal-g env | ||
| # config WALG_PG_WAL_SIZE with wal_segment_size which fetched by psql | ||
| # echo "" > ${walg_env}/WALG_PG_WAL_SIZE | ||
| echo "${walg_dir}/datasafed.conf" > ${walg_env}/WALG_DATASAFED_CONFIG | ||
| echo "${datasafed_base_path}" > ${walg_env}/DATASAFED_BACKEND_BASE_PATH | ||
| echo "true" > ${walg_env}/PG_READY_RENAME | ||
| echo "zstd" > ${walg_env}/WALG_COMPRESSION_METHOD | ||
| echo "${walg_dir}/datasafed.conf" > "${walg_env}/WALG_DATASAFED_CONFIG" | ||
| echo "${datasafed_base_path}" > "${walg_env}/DATASAFED_BACKEND_BASE_PATH" | ||
| echo "true" > "${walg_env}/PG_READY_RENAME" | ||
| echo "zstd" > "${walg_env}/WALG_COMPRESSION_METHOD" | ||
| } | ||
|
|
||
| config_wal_g "$(dirname $(dirname $DP_BACKUP_BASE_PATH))/wal-g" | ||
| echo "Configure WAL-G : BEGIN" | ||
| config_wal_g "$(dirname "$(dirname "${DP_BACKUP_BASE_PATH}")")/wal-g" | ||
| echo "Configure WAL-G : DONE" | ||
|
|
||
| echo "{}" >"${DP_BACKUP_INFO_FILE}" | ||
| sync | ||
| sync |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -1,44 +1,54 @@ | ||
| # shellcheck disable=SC2148 | ||
|
|
||
| postgres_log_dir="${VOLUME_DATA_DIR}/logs" | ||
| postgres_scripts_log_file="${postgres_log_dir}/scripts.log" | ||
| setup_logging WALG_DELETE "${postgres_scripts_log_file}" | ||
|
|
||
| export WALG_DATASAFED_CONFIG="" | ||
| export PATH="$PATH:$DP_DATASAFED_BIN_PATH" | ||
| export DATASAFED_BACKEND_BASE_PATH="$DP_BACKUP_BASE_PATH" | ||
|
|
||
| function getWalGSentinelInfo() { | ||
| local sentinelFile=${1} | ||
| local out=$(datasafed list ${sentinelFile}) | ||
| local sentinelFile | ||
| local out | ||
| sentinelFile=${1} | ||
| out=$(datasafed list "${sentinelFile}") | ||
| if [ "${out}" == "${sentinelFile}" ]; then | ||
| datasafed pull "${sentinelFile}" ${sentinelFile} | ||
| echo "$(cat ${sentinelFile})" | ||
| datasafed pull "${sentinelFile}" "${sentinelFile}" | ||
| cat "${sentinelFile}" | ||
| return | ||
| fi | ||
| } | ||
|
|
||
| # 1. get backup repo path of wal-g | ||
| backupRepoPath=$(getWalGSentinelInfo "wal-g-backup-repo.path") | ||
| if [[ -z ${backupRepoPath} ]]; then | ||
| if [[ -z "${backupRepoPath}" ]]; then | ||
| echo "INFO: nothing to delete." | ||
| exit 0 | ||
| fi | ||
|
|
||
| # 2. get backup name of this backup | ||
| backupName=$(getWalGSentinelInfo "wal-g-backup-name") | ||
| if [[ -z ${backupName} ]]; then | ||
| if [[ -z "${backupName}" ]]; then | ||
| echo "INFO: delete unsuccessfully backup files and outdated WAL archive." | ||
| export DATASAFED_BACKEND_BASE_PATH=${backupRepoPath} | ||
| wal-g delete garbage --confirm | ||
| exit 0 | ||
| fi | ||
|
|
||
| # 3. cleanup outdated wal logs, only effective when existing at least one full backup | ||
| export DATASAFED_BACKEND_BASE_PATH=${backupRepoPath} | ||
| echo "cleanup garbage archives in backup repo" | ||
| export DATASAFED_BACKEND_BASE_PATH="${backupRepoPath}" | ||
| wal-g delete garbage ARCHIVES | ||
|
|
||
| # 4. delete wal-g | ||
| echo "cleanup garbage full backups in backup repo" | ||
| dpBackupFilesCount=$(datasafed list --name "${backupName}_dp_*" /basebackups_005 | wc -l) | ||
| if [[ ${dpBackupFilesCount} -le 1 ]]; then | ||
| if [[ "${dpBackupFilesCount}" -le 1 ]]; then | ||
| # if this base backup only belongs to a backup CR, delete it. | ||
| echo "INFO: delete ${backupName}, backupRepo: ${backupRepo}" | ||
| wal-g delete target ${backupName} --confirm && wal-g delete garbage ARCHIVES --confirm | ||
| echo "INFO: delete ${backupName}" | ||
| wal-g delete target "${backupName}" --confirm && wal-g delete garbage ARCHIVES --confirm | ||
| fi | ||
| datasafed rm "/basebackups_005/${backupName}_dp_${DP_BACKUP_NAME}" | ||
|
|
||
|
|
||
| echo "backup cleanup DONE" | ||
| sync |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Oops, something went wrong.