From 74d0f3e98e777c939bdbec9b02b0aec4f96b311a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fabr=C3=ADzio=20de=20Royes=20Mello?= Date: Sun, 8 Oct 2023 13:58:22 -0300 Subject: [PATCH] PG16: Enable CI --- .github/ci_settings.py | 6 ++- .github/gh_matrix_builder.py | 53 +++++++++++++++++++ .github/workflows/abi.yaml | 14 ++++- .github/workflows/coverity.yaml | 4 +- .../linux-32bit-build-and-test-ignored.yaml | 2 - .../workflows/linux-32bit-build-and-test.yaml | 2 +- .github/workflows/linux-build-and-test.yaml | 4 +- .github/workflows/memory-tests.yaml | 4 +- .github/workflows/sqlsmith.yaml | 2 +- .../windows-build-and-test-ignored.yaml | 2 +- .github/workflows/windows-build-and-test.yaml | 7 ++- scripts/backport.py | 1 + scripts/test_memory_spikes.py | 1 + scripts/test_updates_pg16.sh | 12 +++++ 14 files changed, 100 insertions(+), 14 deletions(-) create mode 100755 scripts/test_updates_pg16.sh diff --git a/.github/ci_settings.py b/.github/ci_settings.py index 5d0da10554e..4dfbf0c15ea 100644 --- a/.github/ci_settings.py +++ b/.github/ci_settings.py @@ -22,4 +22,8 @@ PG15_LATEST = "15.4" PG15_ABI_MIN = "15.0" -PG_LATEST = [PG13_LATEST, PG14_LATEST, PG15_LATEST] +PG16_EARLIEST = "16.0" +PG16_LATEST = "16.0" +PG16_ABI_MIN = "16.0" + +PG_LATEST = [PG13_LATEST, PG14_LATEST, PG15_LATEST, PG16_LATEST] diff --git a/.github/gh_matrix_builder.py b/.github/gh_matrix_builder.py index 8feb179c22e..978f59b495c 100755 --- a/.github/gh_matrix_builder.py +++ b/.github/gh_matrix_builder.py @@ -32,6 +32,8 @@ PG14_LATEST, PG15_EARLIEST, PG15_LATEST, + PG16_EARLIEST, + PG16_LATEST, PG_LATEST, ) @@ -43,6 +45,7 @@ "include": [], } + # helper functions to generate matrix entries # the release and apache config inherit from the # debug config to reduce repetition @@ -172,6 +175,18 @@ def macos_config(overrides): "telemetry", } +# temporary ignored tests for PG16 +# it should be removed once we fix the remaining issues +ignored_tests_pg16 = ignored_tests | { + "compression_sorted_merge-16", + "transparent_decompression-16", + "transparent_decompression_ordered_index-16", + "transparent_decompress_chunk-16", + "transparent_decompression_join_index", + "ordered_append_join-16", + "pg_join", +} + # always test debug build on latest of all supported pg versions m["include"].append( build_debug_config( @@ -192,17 +207,32 @@ def macos_config(overrides): build_debug_config({"pg": PG15_LATEST, "ignored_tests": ignored_tests}) ) +m["include"].append( + build_debug_config({"pg": PG16_LATEST, "ignored_tests": ignored_tests_pg16}) +) + # test timescaledb with release config on latest postgres release in MacOS m["include"].append( build_release_config( macos_config({"pg": PG15_LATEST, "ignored_tests": ignored_tests}) ) ) + +m["include"].append( + build_release_config( + macos_config({"pg": PG16_LATEST, "ignored_tests": ignored_tests_pg16}) + ) +) + # test latest postgres release without telemetry m["include"].append( build_without_telemetry({"pg": PG15_LATEST, "ignored_tests": ignored_tests}) ) +m["include"].append( + build_without_telemetry({"pg": PG16_LATEST, "ignored_tests": ignored_tests_pg16}) +) + # if this is not a pull request e.g. a scheduled run or a push # to a specific branch like prerelease_test we add additional # entries to the matrix @@ -238,6 +268,11 @@ def macos_config(overrides): build_debug_config({"pg": PG15_EARLIEST, "ignored_tests": ignored_tests}) ) + # add debug test for first supported PG16 version + m["include"].append( + build_debug_config({"pg": PG16_EARLIEST, "ignored_tests": ignored_tests_pg16}) + ) + # add debug tests for timescaledb on latest postgres release in MacOS m["include"].append( build_debug_config( @@ -245,12 +280,21 @@ def macos_config(overrides): ) ) + m["include"].append( + build_debug_config( + macos_config({"pg": PG16_LATEST, "ignored_tests": ignored_tests_pg16}) + ) + ) + # add release test for latest pg releases m["include"].append(build_release_config({"pg": PG13_LATEST})) m["include"].append(build_release_config({"pg": PG14_LATEST})) m["include"].append( build_release_config({"pg": PG15_LATEST, "ignored_tests": ignored_tests}) ) + m["include"].append( + build_release_config({"pg": PG16_LATEST, "ignored_tests": ignored_tests_pg16}) + ) # add apache only test for latest pg versions for PG_LATEST_VER in PG_LATEST: @@ -277,6 +321,15 @@ def macos_config(overrides): } ) ) + m["include"].append( + build_debug_config( + { + "pg": 16, + "snapshot": "snapshot", + "ignored_tests": ignored_tests_pg16, + } + ) + ) elif len(sys.argv) > 2: # Check if we need to check for the flaky tests. Determine which test files # have been changed in the PR. The sql files might include other files that diff --git a/.github/workflows/abi.yaml b/.github/workflows/abi.yaml index 0e3473afa13..f3311e7c1e7 100644 --- a/.github/workflows/abi.yaml +++ b/.github/workflows/abi.yaml @@ -21,9 +21,11 @@ jobs: pg13_abi_min: ${{ steps.config.outputs.pg13_abi_min }} pg14_abi_min: ${{ steps.config.outputs.pg14_abi_min }} pg15_abi_min: ${{ steps.config.outputs.pg15_abi_min }} + pg16_abi_min: ${{ steps.config.outputs.pg16_abi_min }} pg13_latest: ${{ steps.config.outputs.pg13_latest }} pg14_latest: ${{ steps.config.outputs.pg14_latest }} pg15_latest: ${{ steps.config.outputs.pg15_latest }} + pg16_latest: ${{ steps.config.outputs.pg16_latest }} steps: - name: Checkout source code @@ -40,7 +42,7 @@ jobs: fail-fast: false matrix: test: [ "13backward", "13forward", "14backward", "14forward", - "15backward", "15forward" ] + "15backward", "15forward", "16backward", "16forward" ] os: [ windows-2019 ] include: - test: 13backward @@ -70,6 +72,16 @@ jobs: builder: ${{ fromJson(needs.config.outputs.pg15_abi_min) }} tester: ${{ fromJson(needs.config.outputs.pg15_latest) }} ignores: partialize_finalize + - test: 16backward + pg: 16 + builder: ${{ fromJson(needs.config.outputs.pg16_latest) }} + tester: ${{ fromJson(needs.config.outputs.pg16_abi_min) }} + ignores: partialize_finalize + - test: 16forward + pg: 16 + builder: ${{ fromJson(needs.config.outputs.pg16_abi_min) }} + tester: ${{ fromJson(needs.config.outputs.pg16_latest) }} + ignores: partialize_finalize steps: diff --git a/.github/workflows/coverity.yaml b/.github/workflows/coverity.yaml index 265ee9e92b3..87e93975e23 100644 --- a/.github/workflows/coverity.yaml +++ b/.github/workflows/coverity.yaml @@ -15,8 +15,8 @@ jobs: fail-fast: false matrix: # run only on the 3 latest PG versions as we have rate limit on coverity - pg: [13, 14, 15] - os: [ubuntu-20.04] + pg: [13, 14, 15, 16] + os: [ubuntu-22.04] steps: - name: Install Dependencies run: | diff --git a/.github/workflows/linux-32bit-build-and-test-ignored.yaml b/.github/workflows/linux-32bit-build-and-test-ignored.yaml index d353ae25e5b..e48ce17faf5 100644 --- a/.github/workflows/linux-32bit-build-and-test-ignored.yaml +++ b/.github/workflows/linux-32bit-build-and-test-ignored.yaml @@ -38,8 +38,6 @@ jobs: matrix: pg: ${{ fromJson(needs.config.outputs.pg_latest) }} build_type: [ Debug ] - include: - - pg: ${{ fromJson(needs.config.outputs.pg15_latest) }} steps: - run: | echo "No build required" diff --git a/.github/workflows/linux-32bit-build-and-test.yaml b/.github/workflows/linux-32bit-build-and-test.yaml index 93d22fab814..f4e144825f1 100644 --- a/.github/workflows/linux-32bit-build-and-test.yaml +++ b/.github/workflows/linux-32bit-build-and-test.yaml @@ -65,7 +65,7 @@ jobs: echo '/tmp/core.%h.%e.%t' > /proc/sys/kernel/core_pattern apt-get install -y gcc make cmake libssl-dev libkrb5-dev libipc-run-perl \ libtest-most-perl sudo gdb git wget gawk lbzip2 flex bison lcov base-files \ - locales clang-14 llvm-14 llvm-14-dev llvm-14-tools postgresql-client + locales clang-14 llvm-14 llvm-14-dev llvm-14-tools postgresql-client icu-devtools - name: Checkout TimescaleDB uses: actions/checkout@v3 diff --git a/.github/workflows/linux-build-and-test.yaml b/.github/workflows/linux-build-and-test.yaml index 967648db03c..a67cdc343f9 100644 --- a/.github/workflows/linux-build-and-test.yaml +++ b/.github/workflows/linux-build-and-test.yaml @@ -64,7 +64,7 @@ jobs: # If needed, install them before opening the core dump. sudo apt-get update sudo apt-get install flex bison lcov systemd-coredump gdb libipc-run-perl \ - libtest-most-perl ${{ matrix.extra_packages }} + libtest-most-perl icu-devtools ${{ matrix.extra_packages }} - name: Install macOS Dependencies if: runner.os == 'macOS' @@ -72,7 +72,7 @@ jobs: # This is needed because GitHub image macos-10.15 version # 20210927.1 did not install OpenSSL so we install openssl # explicitly. - brew install openssl gawk + brew install openssl gawk icu4c sudo perl -MCPAN -e "CPAN::Shell->notest('install', 'IPC::Run')" sudo perl -MCPAN -e "CPAN::Shell->notest('install', 'Test::Most')" diff --git a/.github/workflows/memory-tests.yaml b/.github/workflows/memory-tests.yaml index 61460fa1e17..9e3b962640a 100644 --- a/.github/workflows/memory-tests.yaml +++ b/.github/workflows/memory-tests.yaml @@ -9,10 +9,10 @@ name: Memory tests jobs: memory_leak: name: Memory leak on insert PG${{ matrix.pg }} - runs-on: ubuntu-20.04 + runs-on: ubuntu-22.04 strategy: matrix: - pg: [13, 14, 15] + pg: [13, 14, 15, 16] fail-fast: false steps: diff --git a/.github/workflows/sqlsmith.yaml b/.github/workflows/sqlsmith.yaml index 68177ac6f3e..d21a34b62ce 100644 --- a/.github/workflows/sqlsmith.yaml +++ b/.github/workflows/sqlsmith.yaml @@ -20,7 +20,7 @@ jobs: strategy: matrix: os: ["ubuntu-22.04"] - pg: [ "15" ] + pg: [ "15", "16" ] build_type: ["Debug"] fail-fast: false env: diff --git a/.github/workflows/windows-build-and-test-ignored.yaml b/.github/workflows/windows-build-and-test-ignored.yaml index bccf663924b..7491839d0c6 100644 --- a/.github/workflows/windows-build-and-test-ignored.yaml +++ b/.github/workflows/windows-build-and-test-ignored.yaml @@ -45,7 +45,7 @@ jobs: strategy: fail-fast: false matrix: - pg: [ 13, 14, 15 ] + pg: [ 13, 14, 15, 16 ] os: [ windows-2022 ] build_type: ${{ fromJson(needs.config.outputs.build_type) }} steps: diff --git a/.github/workflows/windows-build-and-test.yaml b/.github/workflows/windows-build-and-test.yaml index 53aad8c61a8..a65ea7c2583 100644 --- a/.github/workflows/windows-build-and-test.yaml +++ b/.github/workflows/windows-build-and-test.yaml @@ -52,7 +52,7 @@ jobs: strategy: fail-fast: false matrix: - pg: [ 13, 14, 15 ] + pg: [ 13, 14, 15, 16 ] os: [ windows-2022 ] build_type: ${{ fromJson(needs.config.outputs.build_type) }} ignores: ["chunk_adaptive metadata telemetry"] @@ -72,6 +72,11 @@ jobs: # pkg_version: ${{ fromJson(needs.config.outputs.pg15_latest) }} pkg_version: 15.0.1 # hardcoded due to issues with PG15.1 on chocolatey tsl_skips_version: dist_partial_agg-15 dist_grant-15 + - pg: 16 + pkg_version: ${{ fromJson(needs.config.outputs.pg16_latest) }} + tsl_skips_version: compression_sorted_merge-16 transparent_decompression-16 transparent_decompression_ordered_index-16 + transparent_decompress_chunk-16 transparent_decompression_join_index ordered_append_join-16 pg_join + env: # PostgreSQL configuration PGPORT: 55432 diff --git a/scripts/backport.py b/scripts/backport.py index f7be4968f3d..569647923b9 100755 --- a/scripts/backport.py +++ b/scripts/backport.py @@ -226,6 +226,7 @@ def git_returncode(command): ] branch_commit_titles = {x[1] for x in branch_commits} + # We will do backports per-PR, because one PR, though not often, might contain # many commits. So as the first step, go through the commits unique to main, find # out which of them have to be backported, and remember the corresponding PRs. diff --git a/scripts/test_memory_spikes.py b/scripts/test_memory_spikes.py index 2fb65a4d3cf..c7bc000f493 100644 --- a/scripts/test_memory_spikes.py +++ b/scripts/test_memory_spikes.py @@ -16,6 +16,7 @@ CHECK_INTERVAL = 15 DEBUG = False + # finds processes with name as argument def find_procs_by_name(name): # Return a list of processes matching 'name' diff --git a/scripts/test_updates_pg16.sh b/scripts/test_updates_pg16.sh new file mode 100755 index 00000000000..b1928ce70fc --- /dev/null +++ b/scripts/test_updates_pg16.sh @@ -0,0 +1,12 @@ +#!/usr/bin/env bash + +set -e + +SCRIPT_DIR=$(dirname $0) + +# shellcheck source=scripts/test_functions.inc +source ${SCRIPT_DIR}/test_functions.inc + +# Also run repair tests for >=2.10.x versions due to PR #5441 +# run_tests "$@" -r -v8 \ +# 2.13.0-pg16