Merge branch 'master' into patroni-4x

zalando · Dec 6, 2024 · d096478 · d096478
2 parents 1207622 + 39a6c8f
commit d096478
Show file tree

Hide file tree

Showing 10 changed files with 194 additions and 110 deletions.
diff --git a/delivery.yaml b/delivery.yaml
@@ -3,7 +3,7 @@ allow_concurrent_steps: true
 
 build_env: &BUILD_ENV
   BASE_IMAGE: container-registry.zalando.net/library/ubuntu-22.04
-  PGVERSION: 16
+  PGVERSION: 17
   MULTI_ARCH_REGISTRY: container-registry-test.zalando.net/acid
 
 pipeline:
@@ -33,6 +33,8 @@ pipeline:
       docker buildx build --platform "linux/amd64" \
                           --build-arg PGVERSION="$PGVERSION" \
                           --build-arg BASE_IMAGE="$BASE_IMAGE" \
+                          --build-arg PGOLDVERSIONS="14 15 16" \
+                          --build-arg TIMESCALEDB="2.17.2" \
                           -t "$ECR_TEST_IMAGE" \
                           --push .
 
@@ -61,6 +63,8 @@ pipeline:
       docker buildx build --platform "linux/amd64,linux/arm64" \
                           --build-arg PGVERSION="$PGVERSION" \
                           --build-arg BASE_IMAGE="$BASE_IMAGE" \
+                          --build-arg PGOLDVERSIONS="14 15 16" \
+                          --build-arg TIMESCALEDB="2.17.2" \
                           -t "$ECR_TEST_IMAGE" \
                           --push .
       cdp-promote-image "$ECR_TEST_IMAGE"
@@ -91,6 +95,8 @@ pipeline:
       docker buildx build --platform "linux/amd64,linux/arm64" \
                           --build-arg PGVERSION="$PGVERSION" \
                           --build-arg BASE_IMAGE="$BASE_IMAGE" \
+                          --build-arg PGOLDVERSIONS="14 15 16" \
+                          --build-arg TIMESCALEDB="2.17.2" \
                           -t "$ECR_TEST_IMAGE" \
                           --push .
       cdp-promote-image "$ECR_TEST_IMAGE"

diff --git a/postgres-appliance/Dockerfile b/postgres-appliance/Dockerfile
@@ -1,6 +1,6 @@
 ARG BASE_IMAGE=ubuntu:22.04
-ARG PGVERSION=16
-ARG TIMESCALEDB="2.11.2 2.14.2"
+ARG PGVERSION=17
+ARG TIMESCALEDB="2.15.3 2.17.2"
 ARG DEMO=false
 ARG COMPRESS=false
 ARG ADDITIONAL_LOCALES=
@@ -48,21 +48,21 @@ ARG TIMESCALEDB
 ARG TIMESCALEDB_APACHE_ONLY=true
 ARG TIMESCALEDB_TOOLKIT=true
 ARG COMPRESS
-ARG PGOLDVERSIONS="12 13 14 15"
+ARG PGOLDVERSIONS="13 14 15 16"
 ARG WITH_PERL=false
 
 ARG DEB_PG_SUPPORTED_VERSIONS="$PGOLDVERSIONS $PGVERSION"
 
 # Install PostgreSQL, extensions and contribs
 ENV POSTGIS_VERSION=3.5 \
-    BG_MON_COMMIT=ef60961eff92672b1e21f5260dc1211367da6f1f \
-    PG_AUTH_MON_COMMIT=3d010e5959285c32b155e8064c9c9b57869aeca7 \
-    PG_MON_COMMIT=a6c5982368edd876edbee01e51b91e7387071e21 \
-    SET_USER=REL4_0_1 \
-    PLPROFILER=REL4_2_4 \
-    PG_PROFILE=4.6 \
+    BG_MON_COMMIT=7f5887218790b263fe3f42f85f4ddc9c8400b154 \
+    PG_AUTH_MON_COMMIT=fe099eef7662cbc85b0b79191f47f52f1e96b779 \
+    PG_MON_COMMIT=ead1de70794ed62ca1e34d4022f6165ff36e9a91 \
+    SET_USER=REL4_1_0 \
+    PLPROFILER=REL4_2_5 \
+    PG_PROFILE=4.7 \
     PAM_OAUTH2=v1.0.1 \
-    PG_PERMISSIONS_COMMIT=314b9359e3d77c0b2ef7dbbde97fa4be80e31925
+    PG_PERMISSIONS_COMMIT=f4b7c18676fa64236a1c8e28d34a35764e4a70e2
 
 WORKDIR /builddeps
 RUN bash base.sh
@@ -92,7 +92,6 @@ FROM builder-${COMPRESS}
 LABEL maintainer="Team ACID @ Zalando <[email protected]>"
 
 ARG PGVERSION
-ARG TIMESCALEDB
 ARG DEMO
 ARG COMPRESS
 
@@ -102,7 +101,6 @@ ENV LC_ALL=en_US.utf-8 \
     PATH=$PATH:/usr/lib/postgresql/$PGVERSION/bin \
     PGHOME=/home/postgres \
     RW_DIR=/run \
-    TIMESCALEDB=$TIMESCALEDB \
     DEMO=$DEMO
 
 ENV WALE_ENV_DIR=$RW_DIR/etc/wal-e.d/env \

diff --git a/postgres-appliance/build_scripts/base.sh b/postgres-appliance/build_scripts/base.sh
@@ -56,7 +56,6 @@ curl -sL "https://github.com/zalando-pg/pg_auth_mon/archive/$PG_AUTH_MON_COMMIT.
 curl -sL "https://github.com/cybertec-postgresql/pg_permissions/archive/$PG_PERMISSIONS_COMMIT.tar.gz" | tar xz
 curl -sL "https://github.com/zubkov-andrei/pg_profile/archive/$PG_PROFILE.tar.gz" | tar xz
 git clone -b "$SET_USER" https://github.com/pgaudit/set_user.git
-git clone https://github.com/timescale/timescaledb.git
 
 apt-get install -y \
     postgresql-common \
@@ -81,8 +80,8 @@ for version in $DEB_PG_SUPPORTED_VERSIONS; do
                 "postgresql-${version}-first-last-agg"
                 "postgresql-${version}-hll"
                 "postgresql-${version}-hypopg"
-                "postgresql-${version}-plproxy"
                 "postgresql-${version}-partman"
+                "postgresql-${version}-plproxy"
                 "postgresql-${version}-pgaudit"
                 "postgresql-${version}-pldebugger"
                 "postgresql-${version}-pglogical"
@@ -105,6 +104,12 @@ for version in $DEB_PG_SUPPORTED_VERSIONS; do
 
     fi
 
+    if [ "${TIMESCALEDB_APACHE_ONLY}" = "true" ]; then
+        EXTRAS+=("timescaledb-2-oss-postgresql-${version}")
+    else
+        EXTRAS+=("timescaledb-2-postgresql-${version}")
+    fi
+
     # Install PostgreSQL binaries, contrib, plproxy and multiple pl's
     apt-get install --allow-downgrades -y \
         "postgresql-${version}-cron" \
@@ -116,39 +121,28 @@ for version in $DEB_PG_SUPPORTED_VERSIONS; do
         "postgresql-${version}-pg-stat-kcache" \
         "${EXTRAS[@]}"
 
-    # Install 3rd party stuff
+    # Clean up timescaledb versions except the highest compatible version
+    exclude_patterns=()
+    exclude_patterns_tsl=()
+    for ts_version in ${TIMESCALEDB}; do
+        exclude_patterns+=(! -name timescaledb-"${ts_version}".so)
+        exclude_patterns_tsl+=(! -name timescaledb-tsl-"${ts_version}".so)
+    done
+    find /usr/lib/postgresql/"${version}"/lib/ -name 'timescaledb-2.*.so' "${exclude_patterns[@]}" -delete;
 
-    # use subshell to avoid having to cd back (SC2103)
-    (
-        cd timescaledb
-        for v in $TIMESCALEDB; do
-            git checkout "$v"
-            sed -i "s/VERSION 3.11/VERSION 3.10/" CMakeLists.txt
-            if BUILD_FORCE_REMOVE=true ./bootstrap -DREGRESS_CHECKS=OFF -DWARNINGS_AS_ERRORS=OFF \
-                    -DTAP_CHECKS=OFF -DPG_CONFIG="/usr/lib/postgresql/$version/bin/pg_config" \
-                    -DAPACHE_ONLY="$TIMESCALEDB_APACHE_ONLY" -DSEND_TELEMETRY_DEFAULT=NO; then
-                make -C build install
-                strip /usr/lib/postgresql/"$version"/lib/timescaledb*.so
-            fi
-            git reset --hard
-            git clean -f -d
-        done
-    )
+    if [ "${TIMESCALEDB_APACHE_ONLY}" != "true" ]; then
+        find /usr/lib/postgresql/"${version}"/lib/ -name 'timescaledb-tsl-2.*.so' "${exclude_patterns_tsl[@]}" -delete;
+    fi
 
-    if [ "${TIMESCALEDB_APACHE_ONLY}" != "true" ] && [ "${TIMESCALEDB_TOOLKIT}" = "true" ]; then
-        __versionCodename=$(sed </etc/os-release -ne 's/^VERSION_CODENAME=//p')
-        echo "deb [signed-by=/usr/share/keyrings/timescale_E7391C94080429FF.gpg] https://packagecloud.io/timescale/timescaledb/ubuntu/ ${__versionCodename} main" | tee /etc/apt/sources.list.d/timescaledb.list
-        curl -L https://packagecloud.io/timescale/timescaledb/gpgkey | gpg --dearmor > /usr/share/keyrings/timescale_E7391C94080429FF.gpg
+    # Install 3rd party stuff
 
+    if [ "${TIMESCALEDB_APACHE_ONLY}" != "true" ] && [ "${TIMESCALEDB_TOOLKIT}" = "true" ]; then
         apt-get update
         if [ "$(apt-cache search --names-only "^timescaledb-toolkit-postgresql-${version}$" | wc -l)" -eq 1 ]; then
             apt-get install "timescaledb-toolkit-postgresql-$version"
         else
             echo "Skipping timescaledb-toolkit-postgresql-$version as it's not found in the repository"
         fi
-
-        rm /etc/apt/sources.list.d/timescaledb.list
-        rm /usr/share/keyrings/timescale_E7391C94080429FF.gpg
     fi
 
     EXTRA_EXTENSIONS=()

diff --git a/postgres-appliance/build_scripts/prepare.sh b/postgres-appliance/build_scripts/prepare.sh
@@ -40,6 +40,10 @@ for t in deb deb-src; do
 done
 curl -s -o - https://www.postgresql.org/media/keys/ACCC4CF8.asc | gpg --dearmor > /etc/apt/trusted.gpg.d/apt.postgresql.org.gpg
 
+# add TimescaleDB repository
+echo "deb [signed-by=/etc/apt/keyrings/timescale_timescaledb-archive-keyring.gpg] https://packagecloud.io/timescale/timescaledb/ubuntu/ ${DISTRIB_CODENAME} main" | tee /etc/apt/sources.list.d/timescaledb.list
+curl -fsSL https://packagecloud.io/timescale/timescaledb/gpgkey | gpg --dearmor | tee /etc/apt/keyrings/timescale_timescaledb-archive-keyring.gpg > /dev/null
+
 # Clean up
 apt-get purge -y libcap2-bin
 apt-get autoremove -y

diff --git a/postgres-appliance/major_upgrade/pg_upgrade.py b/postgres-appliance/major_upgrade/pg_upgrade.py
@@ -204,7 +204,7 @@ def pg_upgrade(self, check=False):
     def prepare_new_pgdata(self, version):
         from spilo_commons import append_extensions
 
-        locale = self.query('SHOW lc_collate')[0][0]
+        locale = self.query("SELECT datcollate FROM pg_database WHERE datname='template1';")[0][0]
         encoding = self.query('SHOW server_encoding')[0][0]
         initdb_config = [{'locale': locale}, {'encoding': encoding}]
         if self.query("SELECT current_setting('data_checksums')::bool")[0][0]:

diff --git a/postgres-appliance/scripts/post_init.sh b/postgres-appliance/scripts/post_init.sh
@@ -5,7 +5,11 @@ cd "$(dirname "${BASH_SOURCE[0]}")" || exit 1
 export PGOPTIONS="-c synchronous_commit=local -c search_path=pg_catalog"
 
 PGVER=$(psql -d "$2" -XtAc "SELECT pg_catalog.current_setting('server_version_num')::int/10000")
-RESET_ARGS="oid, oid, bigint"
+if [ "$PGVER" -lt 17 ]; then
+    RESET_ARGS="oid, oid, bigint"
+else
+    RESET_ARGS="oid, oid, bigint, bool"
+fi
 
 (echo "\set ON_ERROR_STOP on"
 echo "DO \$\$
@@ -213,6 +217,88 @@ while IFS= read -r db_name; do
     UPGRADE_TIMESCALEDB=$(echo -e "SELECT NULL;\nSELECT default_version != installed_version FROM pg_catalog.pg_available_extensions WHERE name = 'timescaledb'" | psql -tAX -d "${db_name}" 2> /dev/null | tail -n 1)
     if [ "$UPGRADE_TIMESCALEDB" = "t" ]; then
         echo "ALTER EXTENSION timescaledb UPDATE;"
+        IS_VERSION_BELOW_215=$(echo -e "SELECT (installed_version < '2.15')::bool FROM pg_catalog.pg_available_extensions WHERE name = 'timescaledb'" | psql -tAX -d "${db_name}" 2> /dev/null | tail -n 1)
+        if [ "$IS_VERSION_BELOW_215" = "t" ]; then
+            echo """
+                    -- Fix compressed hypertables with FOREIGN KEY constraints that were created with TimescaleDB versions before 2.15.0
+                    CREATE OR REPLACE FUNCTION pg_temp.constraint_columns(regclass, int2[]) RETURNS text[] AS
+                    $$
+                    SELECT array_agg(attname) FROM unnest($2) un(attnum) LEFT JOIN pg_attribute att ON att.attrelid=$1 AND att.attnum = un.attnum;
+                    $$ LANGUAGE SQL SET search_path TO pg_catalog, pg_temp;
+                    DO $$
+                        DECLARE
+                        ht_id int;
+                        ht regclass;
+                        chunk regclass;
+                        con_oid oid;
+                        con_frelid regclass;
+                        con_name text;
+                        con_columns text[];
+                        chunk_id int;
+
+                        BEGIN
+
+                        -- iterate over all hypertables that have foreign key constraints
+                        FOR ht_id, ht in
+                            SELECT
+                            ht.id,
+                            format('%I.%I',ht.schema_name,ht.table_name)::regclass
+                            FROM _timescaledb_catalog.hypertable ht
+                            WHERE
+                            EXISTS (
+                                SELECT FROM pg_constraint con
+                                WHERE
+                                con.contype='f' AND
+                                con.conrelid=format('%I.%I',ht.schema_name,ht.table_name)::regclass
+                            )
+                        LOOP
+                            RAISE NOTICE 'Hypertable % has foreign key constraint', ht;
+
+                            -- iterate over all foreign key constraints on the hypertable
+                            -- and check that they are present on every chunk
+                            FOR con_oid, con_frelid, con_name, con_columns IN
+                            SELECT con.oid, con.confrelid, con.conname, pg_temp.constraint_columns(con.conrelid,con.conkey)
+                            FROM pg_constraint con
+                            WHERE
+                                con.contype='f' AND
+                                con.conrelid=ht
+                            LOOP
+                                RAISE NOTICE 'Checking constraint % %', con_name, con_columns;
+                                -- check that the foreign key constraint is present on the chunk
+
+                                FOR chunk_id, chunk IN
+                                    SELECT
+                                    ch.id,
+                                    format('%I.%I',ch.schema_name,ch.table_name)::regclass
+                                    FROM _timescaledb_catalog.chunk ch
+                                    WHERE
+                                    ch.hypertable_id=ht_id
+                                LOOP
+                                    RAISE NOTICE 'Checking chunk %', chunk;
+                                    IF NOT EXISTS (
+                                    SELECT FROM pg_constraint con
+                                    WHERE
+                                        con.contype='f' AND
+                                        con.conrelid=chunk AND
+                                        con.confrelid=con_frelid  AND
+                                        pg_temp.constraint_columns(con.conrelid,con.conkey) = con_columns
+                                    ) THEN
+                                    RAISE WARNING 'Restoring constraint % on chunk %', con_name, chunk;
+                                    PERFORM _timescaledb_functions.constraint_clone(con_oid, chunk);
+                                    INSERT INTO _timescaledb_catalog.chunk_constraint(chunk_id, dimension_slice_id, constraint_name, hypertable_constraint_name) VALUES (chunk_id, NULL, con_name, con_name);
+                                    END IF;
+
+                                END LOOP;
+                            END LOOP;
+
+                        END LOOP;
+
+                    END
+                    $$;
+
+                    DROP FUNCTION pg_temp.constraint_columns(regclass, int2[]);
+                """
+        fi
     fi
     UPGRADE_TIMESCALEDB_TOOLKIT=$(echo -e "SELECT NULL;\nSELECT default_version != installed_version FROM pg_catalog.pg_available_extensions WHERE name = 'timescaledb_toolkit'" | psql -tAX -d "${db_name}" 2> /dev/null | tail -n 1)
     if [ "$UPGRADE_TIMESCALEDB_TOOLKIT" = "t" ]; then

diff --git a/postgres-appliance/scripts/postgres_backup.sh b/postgres-appliance/scripts/postgres_backup.sh
@@ -55,7 +55,7 @@ while read -r name last_modified rest; do
         # count how many backups will remain after we remove everything up to certain date
         ((LEFT=LEFT+1))
     fi
-done < <($WAL_E backup-list 2> /dev/null | sed '0,/^name\s*\(last_\)\?modified\s*/d')
+done < <($WAL_E backup-list 2> /dev/null | sed '0,/^\(backup_\)\?name\s*\(last_\)\?modified\s*/d')
 
 # we want keep at least N backups even if the number of days exceeded
 if [ -n "$BEFORE" ] && [ $LEFT -ge $DAYS_TO_RETAIN ]; then

diff --git a/postgres-appliance/scripts/spilo_commons.py b/postgres-appliance/scripts/spilo_commons.py
@@ -12,13 +12,13 @@
 
 # (min_version, max_version, shared_preload_libraries, extwlist.extensions)
 extensions = {
-    'timescaledb':    (9.6, 16, True,  True),
-    'pg_cron':        (9.5, 16, True,  False),
-    'pg_stat_kcache': (9.4, 16, True,  False),
-    'pg_partman':     (9.4, 16, False, True)
+    'timescaledb':    (9.6, 17, True,  True),
+    'pg_cron':        (9.5, 17, True,  False),
+    'pg_stat_kcache': (9.4, 17, True,  False),
+    'pg_partman':     (9.4, 17, False, True)
 }
 if os.environ.get('ENABLE_PG_MON') == 'true':
-    extensions['pg_mon'] = (11,  16, True,  False)
+    extensions['pg_mon'] = (11,  17, True,  False)
 
 
 def adjust_extensions(old, version, extwlist=False):

diff --git a/postgres-appliance/tests/docker-compose.yml b/postgres-appliance/tests/docker-compose.yml
@@ -51,7 +51,7 @@ services:
               postgresql:
                 parameters:
                   shared_buffers: 32MB
-            PGVERSION: '12'
+            PGVERSION: '13'
             # Just to test upgrade with clone. Without CLONE_SCOPE they don't work
             CLONE_WAL_S3_BUCKET: *bucket
             CLONE_AWS_ACCESS_KEY_ID: *access_key