From 975becbc535f2e33e2c5dd4cd88d4d4d4b43d6ba Mon Sep 17 00:00:00 2001 From: Ida Novindasari Date: Mon, 14 Oct 2024 08:52:08 +0200 Subject: [PATCH] Add timescaledb extension to support pg17 (#1032) --- postgres-appliance/Dockerfile | 2 +- postgres-appliance/build_scripts/base.sh | 26 +++---- postgres-appliance/scripts/post_init.sh | 82 +++++++++++++++++++++ postgres-appliance/scripts/spilo_commons.py | 2 +- postgres-appliance/tests/test_spilo.sh | 4 +- 5 files changed, 98 insertions(+), 18 deletions(-) diff --git a/postgres-appliance/Dockerfile b/postgres-appliance/Dockerfile index 9b3cf313..e3882007 100644 --- a/postgres-appliance/Dockerfile +++ b/postgres-appliance/Dockerfile @@ -1,6 +1,6 @@ ARG BASE_IMAGE=ubuntu:22.04 ARG PGVERSION=17 -ARG TIMESCALEDB="2.14.2" +ARG TIMESCALEDB="2.15.3 2.17.0" ARG DEMO=false ARG COMPRESS=false ARG ADDITIONAL_LOCALES= diff --git a/postgres-appliance/build_scripts/base.sh b/postgres-appliance/build_scripts/base.sh index 968bc587..9b6f2845 100644 --- a/postgres-appliance/build_scripts/base.sh +++ b/postgres-appliance/build_scripts/base.sh @@ -124,20 +124,18 @@ for version in $DEB_PG_SUPPORTED_VERSIONS; do # use subshell to avoid having to cd back (SC2103) ( cd timescaledb - if [ "$version" != "17" ]; then - for v in $TIMESCALEDB; do - git checkout "$v" - sed -i "s/VERSION 3.11/VERSION 3.10/" CMakeLists.txt - if BUILD_FORCE_REMOVE=true ./bootstrap -DREGRESS_CHECKS=OFF -DWARNINGS_AS_ERRORS=OFF \ - -DTAP_CHECKS=OFF -DPG_CONFIG="/usr/lib/postgresql/$version/bin/pg_config" \ - -DAPACHE_ONLY="$TIMESCALEDB_APACHE_ONLY" -DSEND_TELEMETRY_DEFAULT=NO; then - make -C build install - strip /usr/lib/postgresql/"$version"/lib/timescaledb*.so - fi - git reset --hard - git clean -f -d - done - fi + for v in $TIMESCALEDB; do + git checkout "$v" + sed -i "s/VERSION 3.11/VERSION 3.10/" CMakeLists.txt + if BUILD_FORCE_REMOVE=true ./bootstrap -DREGRESS_CHECKS=OFF -DWARNINGS_AS_ERRORS=OFF \ + -DTAP_CHECKS=OFF -DPG_CONFIG="/usr/lib/postgresql/$version/bin/pg_config" \ + -DAPACHE_ONLY="$TIMESCALEDB_APACHE_ONLY" -DSEND_TELEMETRY_DEFAULT=NO; then + make -C build install + strip /usr/lib/postgresql/"$version"/lib/timescaledb*.so + fi + git reset --hard + git clean -f -d + done ) if [ "${TIMESCALEDB_APACHE_ONLY}" != "true" ] && [ "${TIMESCALEDB_TOOLKIT}" = "true" ]; then diff --git a/postgres-appliance/scripts/post_init.sh b/postgres-appliance/scripts/post_init.sh index 281de8f6..c07730bd 100755 --- a/postgres-appliance/scripts/post_init.sh +++ b/postgres-appliance/scripts/post_init.sh @@ -175,6 +175,88 @@ while IFS= read -r db_name; do UPGRADE_TIMESCALEDB=$(echo -e "SELECT NULL;\nSELECT default_version != installed_version FROM pg_catalog.pg_available_extensions WHERE name = 'timescaledb'" | psql -tAX -d "${db_name}" 2> /dev/null | tail -n 1) if [ "$UPGRADE_TIMESCALEDB" = "t" ]; then echo "ALTER EXTENSION timescaledb UPDATE;" + IS_VERSION_BELOW_215=$(echo -e "SELECT (installed_version < '2.15')::bool FROM pg_catalog.pg_available_extensions WHERE name = 'timescaledb'" | psql -tAX -d "${db_name}" 2> /dev/null | tail -n 1) + if [ "$IS_VERSION_BELOW_215" = "t" ]; then + echo """ + -- Fix compressed hypertables with FOREIGN KEY constraints that were created with TimescaleDB versions before 2.15.0 + CREATE OR REPLACE FUNCTION pg_temp.constraint_columns(regclass, int2[]) RETURNS text[] AS + $$ + SELECT array_agg(attname) FROM unnest($2) un(attnum) LEFT JOIN pg_attribute att ON att.attrelid=$1 AND att.attnum = un.attnum; + $$ LANGUAGE SQL SET search_path TO pg_catalog, pg_temp; + DO $$ + DECLARE + ht_id int; + ht regclass; + chunk regclass; + con_oid oid; + con_frelid regclass; + con_name text; + con_columns text[]; + chunk_id int; + + BEGIN + + -- iterate over all hypertables that have foreign key constraints + FOR ht_id, ht in + SELECT + ht.id, + format('%I.%I',ht.schema_name,ht.table_name)::regclass + FROM _timescaledb_catalog.hypertable ht + WHERE + EXISTS ( + SELECT FROM pg_constraint con + WHERE + con.contype='f' AND + con.conrelid=format('%I.%I',ht.schema_name,ht.table_name)::regclass + ) + LOOP + RAISE NOTICE 'Hypertable % has foreign key constraint', ht; + + -- iterate over all foreign key constraints on the hypertable + -- and check that they are present on every chunk + FOR con_oid, con_frelid, con_name, con_columns IN + SELECT con.oid, con.confrelid, con.conname, pg_temp.constraint_columns(con.conrelid,con.conkey) + FROM pg_constraint con + WHERE + con.contype='f' AND + con.conrelid=ht + LOOP + RAISE NOTICE 'Checking constraint % %', con_name, con_columns; + -- check that the foreign key constraint is present on the chunk + + FOR chunk_id, chunk IN + SELECT + ch.id, + format('%I.%I',ch.schema_name,ch.table_name)::regclass + FROM _timescaledb_catalog.chunk ch + WHERE + ch.hypertable_id=ht_id + LOOP + RAISE NOTICE 'Checking chunk %', chunk; + IF NOT EXISTS ( + SELECT FROM pg_constraint con + WHERE + con.contype='f' AND + con.conrelid=chunk AND + con.confrelid=con_frelid AND + pg_temp.constraint_columns(con.conrelid,con.conkey) = con_columns + ) THEN + RAISE WARNING 'Restoring constraint % on chunk %', con_name, chunk; + PERFORM _timescaledb_functions.constraint_clone(con_oid, chunk); + INSERT INTO _timescaledb_catalog.chunk_constraint(chunk_id, dimension_slice_id, constraint_name, hypertable_constraint_name) VALUES (chunk_id, NULL, con_name, con_name); + END IF; + + END LOOP; + END LOOP; + + END LOOP; + + END + $$; + + DROP FUNCTION pg_temp.constraint_columns(regclass, int2[]); + """ + fi fi UPGRADE_TIMESCALEDB_TOOLKIT=$(echo -e "SELECT NULL;\nSELECT default_version != installed_version FROM pg_catalog.pg_available_extensions WHERE name = 'timescaledb_toolkit'" | psql -tAX -d "${db_name}" 2> /dev/null | tail -n 1) if [ "$UPGRADE_TIMESCALEDB_TOOLKIT" = "t" ]; then diff --git a/postgres-appliance/scripts/spilo_commons.py b/postgres-appliance/scripts/spilo_commons.py index 7b782366..0543bf77 100644 --- a/postgres-appliance/scripts/spilo_commons.py +++ b/postgres-appliance/scripts/spilo_commons.py @@ -12,7 +12,7 @@ # (min_version, max_version, shared_preload_libraries, extwlist.extensions) extensions = { - 'timescaledb': (9.6, 16, True, True), + 'timescaledb': (9.6, 17, True, True), 'pg_cron': (9.5, 17, True, False), 'pg_stat_kcache': (9.4, 17, True, False), 'pg_partman': (9.4, 17, False, True) diff --git a/postgres-appliance/tests/test_spilo.sh b/postgres-appliance/tests/test_spilo.sh index 7a063ad2..a5583d14 100755 --- a/postgres-appliance/tests/test_spilo.sh +++ b/postgres-appliance/tests/test_spilo.sh @@ -343,11 +343,11 @@ function test_spilo() { # TEST SUITE 1 - run_test test_pg_upgrade_to_17_check_failed "$container" # pg_upgrade --check complains about timescaledb + # run_test test_pg_upgrade_to_17_check_failed "$container" # pg_upgrade --check complains about timescaledb wait_backup "$container" - drop_timescaledb "$container" + # drop_timescaledb "$container" log_info "[TS1] Testing in-place major upgrade 16->17" run_test test_successful_inplace_upgrade_to_17 "$container" wait_all_streaming "$container"