PG16: Enable CI

.github/ci_settings.py

@@ -22,4 +22,8 @@
 PG15_LATEST = "15.4"
 PG15_ABI_MIN = "15.0"
 
-PG_LATEST = [PG13_LATEST, PG14_LATEST, PG15_LATEST]
+PG16_EARLIEST = "16.0"
+PG16_LATEST = "16.0"
+PG16_ABI_MIN = "16.0"
+
+PG_LATEST = [PG13_LATEST, PG14_LATEST, PG15_LATEST, PG16_LATEST]

.github/gh_matrix_builder.py

@@ -32,6 +32,8 @@
     PG14_LATEST,
     PG15_EARLIEST,
     PG15_LATEST,
+    PG16_EARLIEST,
+    PG16_LATEST,
     PG_LATEST,
 )
 
@@ -43,6 +45,7 @@
     "include": [],
 }
 
+
 # helper functions to generate matrix entries
 # the release and apache config inherit from the
 # debug config to reduce repetition
@@ -172,6 +175,18 @@ def macos_config(overrides):
         "telemetry",
     }
 
+# temporary ignored tests for PG16
+# it should be removed once we fix the remaining issues
+ignored_tests_pg16 = ignored_tests | {
+    "compression_sorted_merge-16",
+    "transparent_decompression-16",
+    "transparent_decompression_ordered_index-16",
+    "transparent_decompress_chunk-16",
+    "transparent_decompression_join_index",
+    "ordered_append_join-16",
+    "pg_join",
+}
+
 # always test debug build on latest of all supported pg versions
 m["include"].append(
     build_debug_config(
@@ -192,17 +207,32 @@ def macos_config(overrides):
     build_debug_config({"pg": PG15_LATEST, "ignored_tests": ignored_tests})
 )
 
+m["include"].append(
+    build_debug_config({"pg": PG16_LATEST, "ignored_tests": ignored_tests_pg16})
+)
+
 # test timescaledb with release config on latest postgres release in MacOS
 m["include"].append(
     build_release_config(
         macos_config({"pg": PG15_LATEST, "ignored_tests": ignored_tests})
     )
 )
+
+m["include"].append(
+    build_release_config(
+        macos_config({"pg": PG16_LATEST, "ignored_tests": ignored_tests_pg16})
+    )
+)
+
 # test latest postgres release without telemetry
 m["include"].append(
     build_without_telemetry({"pg": PG15_LATEST, "ignored_tests": ignored_tests})
 )
 
+m["include"].append(
+    build_without_telemetry({"pg": PG16_LATEST, "ignored_tests": ignored_tests_pg16})
+)
+
 # if this is not a pull request e.g. a scheduled run or a push
 # to a specific branch like prerelease_test we add additional
 # entries to the matrix
@@ -238,19 +268,33 @@ def macos_config(overrides):
         build_debug_config({"pg": PG15_EARLIEST, "ignored_tests": ignored_tests})
     )
 
+    # add debug test for first supported PG16 version
+    m["include"].append(
+        build_debug_config({"pg": PG16_EARLIEST, "ignored_tests": ignored_tests_pg16})
+    )
+
     # add debug tests for timescaledb on latest postgres release in MacOS
     m["include"].append(
         build_debug_config(
             macos_config({"pg": PG15_LATEST, "ignored_tests": ignored_tests})
         )
     )
 
+    m["include"].append(
+        build_debug_config(
+            macos_config({"pg": PG16_LATEST, "ignored_tests": ignored_tests_pg16})
+        )
+    )
+
     # add release test for latest pg releases
     m["include"].append(build_release_config({"pg": PG13_LATEST}))
     m["include"].append(build_release_config({"pg": PG14_LATEST}))
     m["include"].append(
         build_release_config({"pg": PG15_LATEST, "ignored_tests": ignored_tests})
     )
+    m["include"].append(
+        build_release_config({"pg": PG16_LATEST, "ignored_tests": ignored_tests_pg16})
+    )
 
     # add apache only test for latest pg versions
     for PG_LATEST_VER in PG_LATEST:
@@ -277,6 +321,15 @@ def macos_config(overrides):
             }
         )
     )
+    m["include"].append(
+        build_debug_config(
+            {
+                "pg": 16,
+                "snapshot": "snapshot",
+                "ignored_tests": ignored_tests_pg16,
+            }
+        )
+    )
 elif len(sys.argv) > 2:
     # Check if we need to check for the flaky tests. Determine which test files
     # have been changed in the PR. The sql files might include other files that

.github/workflows/abi.yaml

@@ -21,9 +21,11 @@ jobs:
       pg13_abi_min: ${{ steps.config.outputs.pg13_abi_min }}
       pg14_abi_min: ${{ steps.config.outputs.pg14_abi_min }}
       pg15_abi_min: ${{ steps.config.outputs.pg15_abi_min }}
+      pg16_abi_min: ${{ steps.config.outputs.pg16_abi_min }}
       pg13_latest: ${{ steps.config.outputs.pg13_latest }}
       pg14_latest: ${{ steps.config.outputs.pg14_latest }}
       pg15_latest: ${{ steps.config.outputs.pg15_latest }}
+      pg16_latest: ${{ steps.config.outputs.pg16_latest }}
 
     steps:
     - name: Checkout source code
@@ -40,7 +42,7 @@ jobs:
       fail-fast: false
       matrix:
         test: [ "13backward", "13forward", "14backward", "14forward",
-          "15backward", "15forward" ]
+          "15backward", "15forward", "16backward", "16forward" ]
         os: [ windows-2019 ]
         include:
           - test: 13backward
@@ -70,6 +72,16 @@ jobs:
             builder: ${{ fromJson(needs.config.outputs.pg15_abi_min) }}
             tester: ${{ fromJson(needs.config.outputs.pg15_latest) }}
             ignores: partialize_finalize
+          - test: 16backward
+            pg: 16
+            builder: ${{ fromJson(needs.config.outputs.pg16_latest) }}
+            tester: ${{ fromJson(needs.config.outputs.pg16_abi_min) }}
+            ignores: partialize_finalize
+          - test: 16forward
+            pg: 16
+            builder: ${{ fromJson(needs.config.outputs.pg16_abi_min) }}
+            tester: ${{ fromJson(needs.config.outputs.pg16_latest) }}
+            ignores: partialize_finalize
 
     steps:
 

.github/workflows/coverity.yaml

@@ -15,8 +15,8 @@ jobs:
       fail-fast: false
       matrix:
         # run only on the 3 latest PG versions as we have rate limit on coverity
-        pg: [13, 14, 15]
-        os: [ubuntu-20.04]
+        pg: [13, 14, 15, 16]
+        os: [ubuntu-22.04]
     steps:
     - name: Install Dependencies
       run: |

.github/workflows/linux-32bit-build-and-test-ignored.yaml

@@ -38,8 +38,6 @@ jobs:
       matrix:
         pg: ${{ fromJson(needs.config.outputs.pg_latest) }}
         build_type: [ Debug ]
-        include:
-          - pg: ${{ fromJson(needs.config.outputs.pg15_latest) }}
     steps:
       - run: |
           echo "No build required"

.github/workflows/linux-32bit-build-and-test.yaml

@@ -65,7 +65,7 @@ jobs:
         echo '/tmp/core.%h.%e.%t' > /proc/sys/kernel/core_pattern
         apt-get install -y gcc make cmake libssl-dev libkrb5-dev libipc-run-perl \
           libtest-most-perl sudo gdb git wget gawk lbzip2 flex bison lcov base-files \
-          locales clang-14 llvm-14 llvm-14-dev llvm-14-tools postgresql-client
+          locales clang-14 llvm-14 llvm-14-dev llvm-14-tools postgresql-client icu-devtools
 
     - name: Checkout TimescaleDB
       uses: actions/checkout@v3

.github/workflows/linux-build-and-test.yaml

@@ -64,15 +64,15 @@ jobs:
         # If needed, install them before opening the core dump.
         sudo apt-get update
         sudo apt-get install flex bison lcov systemd-coredump gdb libipc-run-perl \
-          libtest-most-perl ${{ matrix.extra_packages }}
+          libtest-most-perl icu-devtools ${{ matrix.extra_packages }}
 
     - name: Install macOS Dependencies
       if: runner.os == 'macOS'
       run: |
         # This is needed because GitHub image macos-10.15 version
         # 20210927.1 did not install OpenSSL so we install openssl
         # explicitly.
-        brew install openssl gawk
+        brew install openssl gawk icu4c
         sudo perl -MCPAN -e "CPAN::Shell->notest('install', 'IPC::Run')"
         sudo perl -MCPAN -e "CPAN::Shell->notest('install', 'Test::Most')"
 

.github/workflows/memory-tests.yaml

@@ -9,10 +9,10 @@ name: Memory tests
 jobs:
   memory_leak:
     name: Memory leak on insert PG${{ matrix.pg }}
-    runs-on: ubuntu-20.04
+    runs-on: ubuntu-22.04
     strategy:
       matrix:
-        pg: [13, 14, 15]
+        pg: [13, 14, 15, 16]
       fail-fast: false
 
     steps:

.github/workflows/sqlsmith.yaml

@@ -20,7 +20,7 @@ jobs:
     strategy:
       matrix:
         os: ["ubuntu-22.04"]
-        pg: [ "15" ]
+        pg: [ "15", "16" ]
         build_type: ["Debug"]
       fail-fast: false
     env:

.github/workflows/windows-build-and-test-ignored.yaml

@@ -45,7 +45,7 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        pg: [ 13, 14, 15 ]
+        pg: [ 13, 14, 15, 16 ]
         os: [ windows-2022 ]
         build_type: ${{ fromJson(needs.config.outputs.build_type) }}
     steps:

.github/workflows/windows-build-and-test.yaml

@@ -52,7 +52,7 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        pg: [ 13, 14, 15 ]
+        pg: [ 13, 14, 15, 16 ]
         os: [ windows-2022 ]
         build_type: ${{ fromJson(needs.config.outputs.build_type) }}
         ignores: ["chunk_adaptive metadata telemetry"]
@@ -72,6 +72,11 @@ jobs:
             # pkg_version: ${{ fromJson(needs.config.outputs.pg15_latest) }}
             pkg_version: 15.0.1 # hardcoded due to issues with PG15.1 on chocolatey
             tsl_skips_version: dist_partial_agg-15 dist_grant-15
+          - pg: 16
+            pkg_version: ${{ fromJson(needs.config.outputs.pg16_latest) }}
+            tsl_skips_version: compression_sorted_merge-16 transparent_decompression-16 transparent_decompression_ordered_index-16
+              transparent_decompress_chunk-16 transparent_decompression_join_index ordered_append_join-16 pg_join
+
     env:
       # PostgreSQL configuration
       PGPORT: 55432

scripts/backport.py

@@ -226,6 +226,7 @@ def git_returncode(command):
 ]
 branch_commit_titles = {x[1] for x in branch_commits}
 
+
 # We will do backports per-PR, because one PR, though not often, might contain
 # many commits. So as the first step, go through the commits unique to main, find
 # out which of them have to be backported, and remember the corresponding PRs.

scripts/test_memory_spikes.py

@@ -16,6 +16,7 @@
 CHECK_INTERVAL = 15
 DEBUG = False
 
+
 # finds processes with name as argument
 def find_procs_by_name(name):
     # Return a list of processes matching 'name'

scripts/test_updates_pg16.sh

@@ -0,0 +1,12 @@
+#!/usr/bin/env bash
+
+set -e
+
+SCRIPT_DIR=$(dirname $0)
+
+# shellcheck source=scripts/test_functions.inc
+source ${SCRIPT_DIR}/test_functions.inc
+
+# Also run repair tests for >=2.10.x versions due to PR #5441
+# run_tests "$@" -r -v8 \
+#           2.13.0-pg16