From 66bca3c27729b7935062c9d4235b6797124ef25e Mon Sep 17 00:00:00 2001 From: Akis Kesoglou Date: Tue, 15 Feb 2022 18:36:09 +0200 Subject: [PATCH 1/2] Fix a few typos --- balena-entrypoint.sh | 47 +++++++++++++++++++------------------------- 1 file changed, 20 insertions(+), 27 deletions(-) diff --git a/balena-entrypoint.sh b/balena-entrypoint.sh index 774de3d..20ccd65 100755 --- a/balena-entrypoint.sh +++ b/balena-entrypoint.sh @@ -2,13 +2,7 @@ set -e -# creates a backup archive of the dir $1... -backup_data () { - gosu postgres tar --exclude backup.tar.gz -zcf /tmp/backup.tar.gz -C "$1" . - mv /tmp/backup.tar.gz "${1}/backup.tar.gz" -} - -# copies all the data from $1 to $2... +# copies all the data from $1 to $2 restore_volume () { echo "=== Restoring data volume structure" find "$1" -maxdepth 1 \ @@ -16,7 +10,7 @@ restore_volume () { | xargs mv -t "$2" } -# install the old version of Postgres, required when doing an upgrade... +# install the old version of Postgres, required when doing an upgrade install_old_version () { if [ ! -d "/usr/lib/postgresql/$1/bin" ]; then sed -i 's/$/ '"$1"'/' /etc/apt/sources.list.d/pgdg.list @@ -28,7 +22,7 @@ install_old_version () { fi } -# creates a dir at $1 which is valid for Postgres data to live in... +# creates a dir at $1 which is valid for Postgres data to live in create_postgres_data_dir () { echo "=== Creating data dir $1" mkdir -p "$1" @@ -36,53 +30,52 @@ create_postgres_data_dir () { chown -R postgres:postgres "$1" } -# echo a message and die... +# echo a message and die die_with_message () { echo "[FATAL] $1" exit 2 } -# check that this is running from within a valid Postgres container environment... +# check that this is running from within a valid Postgres container environment [ ! -z "$PG_MAJOR" ] || die_with_message "Not a compatible Postgres runtime environment" -# set our target version... +# set our target version TARGET_VERSION="$PG_MAJOR" -# ensure we have a versioned data directory... +# ensure we have a versioned data directory [ -d "${PGDATA}/${TARGET_VERSION}" ] || create_postgres_data_dir "${PGDATA}/${TARGET_VERSION}" -# check for Postgres data in the root of the $PGDATA directory... +# check for Postgres data in the root of the $PGDATA directory if [ -f "${PGDATA}/PG_VERSION" ]; then SOURCE_VERSION="$(cat ${PGDATA}/PG_VERSION)" - - # does the data need upgrading... + + # does the data need upgrading? if [ "$SOURCE_VERSION" -ne "$TARGET_VERSION" ]; then echo "=== Upgrading data from v${SOURCE_VERSION} to v${TARGET_VERSION}" - # define our directories... PGDATAOLD="${PGDATA}/${SOURCE_VERSION}" PGDATANEW="${PGDATA}/${TARGET_VERSION}" PGBINOLD="/usr/lib/postgresql/$SOURCE_VERSION/bin" PGBINNEW="/usr/lib/postgresql/$TARGET_VERSION/bin" - + echo "=== Installing tools for Postgres v${SOURCE_VERSION}" install_old_version "$SOURCE_VERSION" - echo "=== Moving extisting data to ${PGDATAOLD}" + echo "=== Moving existing data to ${PGDATAOLD}" create_postgres_data_dir "${PGDATAOLD}" find "$PGDATA" -maxdepth 1 \ ! -path "$PGDATA" \ ! -path "$PGDATAOLD*" \ | xargs mv -t "${PGDATAOLD}" - + trap "restore_volume ${PGDATAOLD} ${PGDATA}" ERR - echo "=== Initialising new data directory ${PGDATANEW}" + echo "=== Initializing new data directory ${PGDATANEW}" rm -rf "${PGDATANEW}" create_postgres_data_dir "${PGDATANEW}" gosu postgres initdb -D "$PGDATANEW" -U "$POSTGRES_USER" $POSTGRES_INITDB_ARGS - - echo "=== Beginning pg_upgrade" + + echo "=== Running pg_upgrade" cd /tmp gosu postgres pg_upgrade \ -U "$POSTGRES_USER" \ @@ -98,21 +91,21 @@ if [ -f "${PGDATA}/PG_VERSION" ]; then --new-datadir="$PGDATANEW" \ --old-bindir="$PGBINOLD" \ --new-bindir="$PGBINNEW" - + echo "=== Restoring configuration files" cp "${PGDATAOLD}/pg_hba.conf" "${PGDATANEW}/pg_hba.conf" cp "${PGDATAOLD}/pg_ident.conf" "${PGDATANEW}/pg_ident.conf" else - echo "=== Moving extisting data to directory "${PGDATA}/${TARGET_VERSION}"" + echo "=== Moving existing data to directory "${PGDATA}/${TARGET_VERSION}"" find "$PGDATA" -maxdepth 1 \ ! -path "$PGDATA" \ | xargs mv -t "${PGDATA}/${TARGET_VERSION}" fi fi -# set our runtime data directory to the versioned one... +# set our runtime data directory to the target version export PGDATA="${PGDATA}/${TARGET_VERSION}" -# run the existing Postgres entrypoint script... +# run the existing Postgres entrypoint script . /usr/local/bin/docker-entrypoint.sh _main "$@" From a569d60ceba85f65664ac48e165c295f7627dbf0 Mon Sep 17 00:00:00 2001 From: Akis Kesoglou Date: Tue, 15 Feb 2022 20:51:56 +0200 Subject: [PATCH 2/2] Extend migrator to support upgrading between versioned data directories The migrator previously would only migrate data stored directly under PGDATA (as was the case in previous versions of this project before switching to versioned directories). This adds support for upgrading between versioned data directories as well. Change-type: minor --- balena-entrypoint.sh | 136 ++++++++++++++++++++++++++++++------------- 1 file changed, 97 insertions(+), 39 deletions(-) diff --git a/balena-entrypoint.sh b/balena-entrypoint.sh index 20ccd65..8c37e40 100755 --- a/balena-entrypoint.sh +++ b/balena-entrypoint.sh @@ -2,14 +2,6 @@ set -e -# copies all the data from $1 to $2 -restore_volume () { - echo "=== Restoring data volume structure" - find "$1" -maxdepth 1 \ - ! -path "$1" \ - | xargs mv -t "$2" -} - # install the old version of Postgres, required when doing an upgrade install_old_version () { if [ ! -d "/usr/lib/postgresql/$1/bin" ]; then @@ -23,8 +15,8 @@ install_old_version () { } # creates a dir at $1 which is valid for Postgres data to live in -create_postgres_data_dir () { - echo "=== Creating data dir $1" +create_postgres_dir () { + echo "=== Creating postgres directory $1" mkdir -p "$1" chmod 700 "$1" chown -R postgres:postgres "$1" @@ -41,42 +33,65 @@ die_with_message () { # set our target version TARGET_VERSION="$PG_MAJOR" +PGDATANEW="${PGDATA}/${TARGET_VERSION}" + +WORKDIR=/usr/lib/postgresql/upgrade + +# check there's no previously interrupted upgrade, it would be dangerous to continue +[ ! -d "${WORKDIR}" ] || die_with_message "Found possibly interrupted upgrade in ${WORKDIR}; not continuing" + +migrate_pgdata_root_if_needed () { + # perform a one-time migration from storing data under $PGDATA root, + # to storing it under $PGDATA/$PG_VERSION + if [ -f "${PGDATA}/PG_VERSION" ]; then + local version="$(cat "${PGDATA}/PG_VERSION")" + if [ ! -d "${PGDATA}/${version}" ]; then + create_postgres_dir "${PGDATA}/${version}" + echo "=== Moving existing data to directory "${PGDATA}/${version}"" + find "$PGDATA" -maxdepth 1 \ + ! -path "$PGDATA" \ + ! -path "${PGDATA}/${version}*" \ + | xargs mv -t "${PGDATA}/${version}" + fi + fi +} -# ensure we have a versioned data directory -[ -d "${PGDATA}/${TARGET_VERSION}" ] || create_postgres_data_dir "${PGDATA}/${TARGET_VERSION}" - -# check for Postgres data in the root of the $PGDATA directory -if [ -f "${PGDATA}/PG_VERSION" ]; then - SOURCE_VERSION="$(cat ${PGDATA}/PG_VERSION)" +find_source_version () { + # find our source version by looking for a PG_VERSION file + # in every directory under $PGDATA + for src in $(find "$PGDATA" -maxdepth 2 -type f -name 'PG_VERSION' | sort -rn); do + local version="$(cat "$src")" + if [ "$version" -ne "$TARGET_VERSION" ]; then + SOURCE_VERSION="$version" + PGDATAOLD="$(dirname "$src")" # drop 'PG_VERSION' from path + break + fi + done +} +perform_upgrade_if_needed () { # does the data need upgrading? - if [ "$SOURCE_VERSION" -ne "$TARGET_VERSION" ]; then + if [ -n "${SOURCE_VERSION}" ]; then echo "=== Upgrading data from v${SOURCE_VERSION} to v${TARGET_VERSION}" - PGDATAOLD="${PGDATA}/${SOURCE_VERSION}" - PGDATANEW="${PGDATA}/${TARGET_VERSION}" + create_postgres_dir "${WORKDIR}" + PGBINOLD="/usr/lib/postgresql/$SOURCE_VERSION/bin" PGBINNEW="/usr/lib/postgresql/$TARGET_VERSION/bin" echo "=== Installing tools for Postgres v${SOURCE_VERSION}" install_old_version "$SOURCE_VERSION" - echo "=== Moving existing data to ${PGDATAOLD}" - create_postgres_data_dir "${PGDATAOLD}" - find "$PGDATA" -maxdepth 1 \ - ! -path "$PGDATA" \ - ! -path "$PGDATAOLD*" \ - | xargs mv -t "${PGDATAOLD}" - - trap "restore_volume ${PGDATAOLD} ${PGDATA}" ERR - echo "=== Initializing new data directory ${PGDATANEW}" - rm -rf "${PGDATANEW}" - create_postgres_data_dir "${PGDATANEW}" - gosu postgres initdb -D "$PGDATANEW" -U "$POSTGRES_USER" $POSTGRES_INITDB_ARGS + create_postgres_dir "${PGDATANEW}" + gosu postgres initdb \ + -D "$PGDATANEW" \ + -U "$POSTGRES_USER" \ + $POSTGRES_INITDB_ARGS echo "=== Running pg_upgrade" - cd /tmp + pushd "${WORKDIR}" >/dev/null + gosu postgres pg_upgrade \ -U "$POSTGRES_USER" \ --old-datadir="$PGDATAOLD" \ @@ -95,16 +110,59 @@ if [ -f "${PGDATA}/PG_VERSION" ]; then echo "=== Restoring configuration files" cp "${PGDATAOLD}/pg_hba.conf" "${PGDATANEW}/pg_hba.conf" cp "${PGDATAOLD}/pg_ident.conf" "${PGDATANEW}/pg_ident.conf" - else - echo "=== Moving existing data to directory "${PGDATA}/${TARGET_VERSION}"" - find "$PGDATA" -maxdepth 1 \ - ! -path "$PGDATA" \ - | xargs mv -t "${PGDATA}/${TARGET_VERSION}" + + popd >/dev/null + + echo "=== Data upgraded successfully" fi -fi +} + +complete_upgrade_if_needed () { + if [ -d "${WORKDIR}" ]; then + echo "=== Performing post-upgrade steps" + pushd "${WORKDIR}" >/dev/null + + echo "=== Starting temp server" + gosu postgres bash -c '. /usr/local/bin/docker-entrypoint.sh; docker_temp_server_start' + + if [ -f "${WORKDIR}/update_extensions.sql" ]; then + echo "=== Updating extensions" + gosu postgres psql \ + -U "$POSTGRES_USER" \ + -f "${WORKDIR}/update_extensions.sql" + fi + if [ -f "${WORKDIR}/analyze_new_cluster.sh" ]; then + echo "=== Analyzing new cluster" + gosu postgres "${WORKDIR}/analyze_new_cluster.sh" + fi + if [ -f "${WORKDIR}/delete_old_cluster.sh" ]; then + echo "=== Deleting old cluster" + gosu postgres "${WORKDIR}/delete_old_cluster.sh" + else + rm -rf "${PGDATAOLD}" + fi + + echo "=== Stopping temp server" + gosu postgres bash -c '. /usr/local/bin/docker-entrypoint.sh; docker_temp_server_stop' + + popd >/dev/null + rm -rf "${WORKDIR}" + + echo "=== Data upgrade complete" + fi +} + +migrate_pgdata_root_if_needed +find_source_version +perform_upgrade_if_needed + +# ensure we have a versioned data directory +[ -d "${PGDATANEW}" ] || create_postgres_dir "${PGDATANEW}" # set our runtime data directory to the target version -export PGDATA="${PGDATA}/${TARGET_VERSION}" +export PGDATA="${PGDATANEW}" + +complete_upgrade_if_needed # run the existing Postgres entrypoint script . /usr/local/bin/docker-entrypoint.sh