Reflect refactor tests from master branch

postgres-appliance/tests/schema.sql

@@ -1,3 +1,6 @@
+CREATE EXTENSION pg_repack;
+CREATE EXTENSION amcheck_next;  /* the upgrade script must delete it before running pg_upgrade --check! */
+
 CREATE DATABASE test_db;
 \c test_db
 

postgres-appliance/tests/test_spilo.sh

@@ -9,6 +9,7 @@ readonly PREFIX="demo-"
 readonly UPGRADE_SCRIPT="python3 /scripts/inplace_upgrade.py"
 readonly TIMEOUT=120
 
+
 function cleanup() {
     stop_containers
     docker ps -q --filter="ancestor=${SPILO_TEST_IMAGE:-spilo}" --filter="name=${PREFIX}" | xargs docker rm -f
@@ -99,27 +100,33 @@ function wait_query() {
 }
 
 function wait_all_streaming() {
-    log_info "Waiting for all replicas to start streaming from the leader..."
-    wait_query "$1" "SELECT COUNT(*) FROM pg_stat_replication WHERE application_name LIKE 'spilo_'" 2
+    local repl_count=${2:-2}
+    log_info "Waiting for all replicas to start streaming from the leader ($1)..."
+    wait_query "$1" "SELECT COUNT(*) FROM pg_stat_replication WHERE application_name LIKE 'spilo_'" "$repl_count"
 }
 
 function wait_zero_lag() {
+    local repl_count=${2:-2}
     log_info "Waiting for all replicas to catch up with WAL replay..."
-    wait_query "$1" "SELECT COUNT(*) FROM pg_stat_replication WHERE application_name LIKE 'spilo_' AND pg_catalog.pg_wal_lsn_diff(pg_catalog.pg_current_wal_lsn(), COALESCE(replay_lsn, '0/0')) < 16*1024*1024" 2
+    wait_query "$1" "SELECT COUNT(*) FROM pg_stat_replication WHERE application_name LIKE 'spilo_' AND pg_catalog.pg_wal_lsn_diff(pg_catalog.pg_current_wal_lsn(), COALESCE(replay_lsn, '0/0')) < 16*1024*1024" "$repl_count"
 }
 
 function create_schema() {
     docker_exec -i "$1" "psql -U postgres" < schema.sql
 }
 
-function create_schema2() {
-    docker_exec -i "$1" "psql -U postgres" < schema2.sql
+function create_table_with_oids() {
+    docker_exec "$1" "psql -U postgres -d test_db -c 'CREATE TABLE with_oids() WITH OIDS'"
 }
 
 function drop_table_with_oids() {
     docker_exec "$1" "psql -U postgres -d test_db -c 'DROP TABLE with_oids'"
 }
 
+function create_timescaledb() {
+    docker_exec -i "$1" "psql -U postgres" < timescaledb.sql
+}
+
 function drop_timescaledb() {
     docker_exec "$1" "psql -U postgres -d test_db -c 'DROP EXTENSION timescaledb CASCADE'"
 }
@@ -184,6 +191,7 @@ function start_clone_with_wale_upgrade_container() {
         -e CLONE_SCOPE=demo \
         -e CLONE_METHOD=CLONE_WITH_WALE \
         -e CLONE_TARGET_TIME="$(next_minute)" \
+        -e WALE_BACKUP_THRESHOLD_PERCENTAGE=80 \
         --name "${PREFIX}upgrade$ID" \
         -d "spilo$ID"
 }
@@ -199,7 +207,7 @@ function start_clone_with_wale_upgrade_to_16_container() {
         -e CLONE_SCOPE=demo \
         -e CLONE_PGVERSION=11 \
         -e CLONE_METHOD=CLONE_WITH_WALE \
-        -e CLONE_TARGET_TIME="$(next_minute)" \
+        -e CLONE_TARGET_TIME="$(next_hour)" \
         --name "${PREFIX}upgrade4" \
         -d "spilo3"
 }
@@ -231,20 +239,13 @@ function start_clone_with_basebackup_upgrade_container() {
         -d spilo3
 }
 
-function verify_clone_with_wale_upgrade() {
-    wait_query "$1" "SELECT current_setting('server_version_num')::int/10000" 12 2> /dev/null
-}
-
-function verify_clone_with_basebackup_upgrade() {
-    log_info "Waiting for clone with basebackup and upgrade 11->12 to complete..."
+function verify_clone_upgrade() {
+    local type=$2
+    local from_version=$3
+    local to_version=$4
+    log_info "Waiting for clone with $type and upgrade $from_version->$to_version to complete..."
     find_leader "$1" 1
-    wait_query "$1" "SELECT current_setting('server_version_num')::int/10000" 13 2> /dev/null
-}
-
-function verify_clone_with_wale_upgrade_to_16() {
-    log_info "Waiting for clone with wal-e and upgrade 11->16 to complete..."
-    find_leader "$1" 1
-    wait_query "$1" "SELECT current_setting('server_version_num')::int/10000" 16 2> /dev/null
+    wait_query "$1" "SELECT current_setting('server_version_num')::int/10000" "$to_version" 2> /dev/null
 }
 
 function verify_archive_mode_is_on() {
@@ -254,22 +255,23 @@ function verify_archive_mode_is_on() {
 
 
 # TEST SUITE 1 - In-place major upgrade 11->12->...->16
-# TEST SUITE 2 - Major upgrade 11->16 after wal-e clone
+# TEST SUITE 2 - Major upgrade 11->16 after wal-e clone (with CLONE_PGVERSION set)
 # TEST SUITE 3 - PITR (clone with wal-e) with unreachable target (13+)
-# TEST SUITE 4 - Major upgrade 11->12 after wal-e clone
+# TEST SUITE 4 - Major upgrade 11->12 after wal-e clone (no CLONE_PGVERSION)
 # TEST SUITE 5 - Replica bootstrap with wal-e
+# TEST SUITE 6 - Major upgrade 11->12 after clone with basebackup
 function test_spilo() {
     # TEST SUITE 1
     local container=$1
 
     run_test test_envdir_suffix "$container" 11
 
+    log_info "[TS1] Testing wrong upgrade setups"
     run_test test_inplace_upgrade_wrong_version "$container"
     run_test test_inplace_upgrade_wrong_capacity "$container"
 
     wait_all_streaming "$container"
-
-    create_schema "$container" || exit 1
+    create_schema "$container" || exit 1 # incompatible upgrade exts, custom tbl with statistics and data
 
     # run_test test_failed_inplace_upgrade_big_replication_lag "$container"
 
@@ -278,50 +280,46 @@ function test_spilo() {
     wait_backup "$container"
 
     # TEST SUITE 2
+    local upgrade3_container
+    upgrade3_container=$(start_clone_with_wale_upgrade_to_16_container) # SCOPE=upgrade3 PGVERSION=16 CLONE: _SCOPE=demo _PGVERSION=11 _TARGET_TIME=<next_min>
+    log_info "[TS2] Started $upgrade3_container for testing major upgrade 11->16 after clone with wal-e"
+
+    # TEST SUITE 4
     local upgrade_container
-    upgrade_container=$(start_clone_with_wale_upgrade_to_16_container)
-    log_info "Started $upgrade_container for testing major upgrade 11->16 after clone with wal-e"
+    upgrade_container=$(start_clone_with_wale_upgrade_container) # SCOPE=upgrade PGVERSION=12 CLONE: _SCOPE=demo _TARGET_TIME=<next_min>
+    log_info "[TS4] Started $upgrade_container for testing major upgrade 11->12 after clone with wal-e"
 
     # TEST SUITE 1
-    wait_backup "$container"
     find_leader "$upgrade_container" # wait clone to finish and prevent timescale installation gets cloned
 
-    create_schema2 "$container" || exit 1
+    log_info "[TS1] Testing in-place major upgrade 11->13 with failing check"
+    create_timescaledb "$container" # we don't install it at the beginning, as we do 11->16 in a clone
+    create_table_with_oids "$container"
     run_test test_pg_upgrade_to_13_check_failed "$container"  # pg_upgrade --check complains about OID
     drop_table_with_oids "$container"
 
     log_info "Testing in-place major upgrade 11->12"
     run_test test_successful_inplace_upgrade_to_12 "$container"
-
     wait_all_streaming "$container"
-
     run_test test_envdir_updated_to_x 12
 
     # TEST SUITE 2
-    run_test verify_clone_with_wale_upgrade_to_16 "$upgrade_container"
+    log_info "[TS2] Testing in-place major upgrade 11->16 after wal-e clone"
+    run_test verify_clone_upgrade "$upgrade3_container" "wal-e" 11 16
 
-    run_test verify_archive_mode_is_on "$upgrade_container"
-    wait_backup "$upgrade_container"
-    rm_container "$upgrade_container"
+    run_test verify_archive_mode_is_on "$upgrade3_container"
+    wait_backup "$upgrade3_container"
 
     # TEST SUITE 3
     local clone16_container
-    clone16_container=$(start_clone_with_wale_16_container)
+    clone16_container=$(start_clone_with_wale_16_container) # SCOPE=clone15 CLONE: _SCOPE=upgrade3 _PGVERSION=16 _TARGET_TIME=<next_hour
     log_info "Started $clone16_container for testing point-in-time recovery (clone with wal-e) with unreachable target on 13+"
 
-    # TEST SUITE 4
-    wait_backup "$container"
-    wait_zero_lag "$container"
-
-    upgrade_container=$(start_clone_with_wale_upgrade_container)
-    log_info "Started $upgrade_container for testing major upgrade 11->12 after clone with wal-e"
-
     # TEST SUITE 1
-    log_info "Testing in-place major upgrade 12->14"
+    log_info "[TS1] Testing in-place major upgrade 12->14"
     run_test test_successful_inplace_upgrade_to_14 "$container"
 
     wait_all_streaming "$container"
-
     run_test test_envdir_updated_to_x 14
 
     # TEST SUITE 3
@@ -331,52 +329,46 @@ function test_spilo() {
     # TEST SUITE 1
     wait_backup "$container"
 
-    log_info "Testing in-place major upgrade to 14->15"
+    log_info "[TS1] Testing in-place major upgrade to 14->15"
     run_test test_successful_inplace_upgrade_to_15 "$container"
 
     wait_all_streaming "$container"
-
     run_test test_envdir_updated_to_x 15
 
     # TEST SUITE 4
-    log_info "Waiting for clone with wal-e and upgrade 11->12 to complete..."
-    find_leader "$upgrade_container" 1
-    run_test verify_clone_with_wale_upgrade "$upgrade_container"
+    log_info "[TS4] Testing in-place major upgrade 10->11 after clone with wal-e"
+    run_test verify_clone_upgrade "$upgrade_container" "wal-e" 11 12
 
     wait_backup "$upgrade_container"
 
     # TEST SUITE 5
     local upgrade_replica_container
-    upgrade_replica_container=$(start_clone_with_wale_upgrade_replica_container)
-    log_info "Started $upgrade_replica_container for testing replica bootstrap with wal-e"
+    upgrade_replica_container=$(start_clone_with_wale_upgrade_replica_container) # SCOPE=upgrade
+    log_info "[TS5] Started $upgrade_replica_container for testing replica bootstrap with wal-e"
 
-    # TEST SUITE 4
+    # TEST SUITE 6
     local basebackup_container
-    basebackup_container=$(start_clone_with_basebackup_upgrade_container "$upgrade_container")
-    log_info "Started $basebackup_container for testing major upgrade 12->13 after clone with basebackup"
+    basebackup_container=$(start_clone_with_basebackup_upgrade_container "$upgrade_container")  # SCOPE=upgrade2 PGVERSION=13 CLONE: _SCOPE=upgrade
+    log_info "[TS6] Started $basebackup_container for testing major upgrade 12->13 after clone with basebackup"
 
     # TEST SUITE 1
     run_test test_pg_upgrade_to_16_check_failed "$container"  # pg_upgrade --check complains about timescaledb
 
     wait_backup "$container"
 
     drop_timescaledb "$container"
-
-    log_info "Testing in-place major upgrade to 15->16"
+    log_info "[TS1] Testing in-place major upgrade 15->16"
     run_test test_successful_inplace_upgrade_to_16 "$container"
-
     wait_all_streaming "$container"
-
     run_test test_envdir_updated_to_x 16
 
-    wait_backup "$container"
-
     # TEST SUITE 5
-    log_info "Waiting for postgres to start in the $upgrade_replica_container..."
-    run_test verify_clone_with_wale_upgrade "$upgrade_replica_container"
+    log_info "[TS5] Waiting for postgres to start in the $upgrade_replica_container and stream from primary..."
+    wait_all_streaming "$upgrade_container" 1
 
-    # TEST SUITE 4
-    run_test verify_clone_with_basebackup_upgrade "$basebackup_container"
+    # TEST SUITE 6
+    log_info "[TS6] Testing in-place major upgrade 12->13 after clone with basebackup"
+    run_test verify_clone_upgrade "$basebackup_container" "basebackup" 12 13
     run_test verify_archive_mode_is_on "$basebackup_container"
 }
 

postgres-appliance/tests/schema2.sql → postgres-appliance/tests/timescaledb.sql

@@ -1,9 +1,5 @@
-CREATE EXTENSION amcheck_next;  /* the upgrade script must delete it before running pg_upgrade --check! */
-
 \c test_db
 
-CREATE TABLE with_oids() WITH OIDS;
-
 CREATE EXTENSION timescaledb;
 
 CREATE TABLE "fOo" (id bigint NOT NULL PRIMARY KEY);