diff --git a/.github/gh_matrix_builder.py b/.github/gh_matrix_builder.py index 2cd5eef8d4b..8feb179c22e 100755 --- a/.github/gh_matrix_builder.py +++ b/.github/gh_matrix_builder.py @@ -37,6 +37,7 @@ # github event type which is either push, pull_request or schedule event_type = sys.argv[1] +pull_request = event_type == "pull_request" m = { "include": [], @@ -75,6 +76,8 @@ def build_debug_config(overrides): "tsdb_build_args": "-DWARNINGS_AS_ERRORS=ON -DREQUIRE_ALL_TESTS=ON", } ) + if not pull_request: + base_config["tsdb_build_args"] += " -DENABLE_MULTINODETESTS=ON" base_config.update(overrides) return base_config @@ -93,6 +96,8 @@ def build_release_config(overrides): "coverage": False, } ) + if not pull_request: + release_config["tsdb_build_args"] += " -DENABLE_MULTINODETESTS=ON" base_config.update(release_config) base_config.update(overrides) return base_config @@ -148,6 +153,8 @@ def macos_config(overrides): "tsdb_build_args": "-DASSERTIONS=ON -DREQUIRE_ALL_TESTS=ON -DOPENSSL_ROOT_DIR=/usr/local/opt/openssl", } ) + if not pull_request: + base_config["tsdb_build_args"] += " -DENABLE_MULTINODETESTS=ON" base_config.update(overrides) return base_config @@ -159,13 +166,9 @@ def macos_config(overrides): # common ignored tests for all non-scheduled pg15 tests (e.g. PRs) # partialize_finalize is ignored due to #4937 # dist_move_chunk, dist_param, dist_insert, and remote_txn ignored due to flakiness -if event_type == "pull_request": +if pull_request: ignored_tests = { - "dist_insert", - "dist_move_chunk", - "dist_param", "partialize_finalize", - "remote_txn", "telemetry", } @@ -203,7 +206,7 @@ def macos_config(overrides): # if this is not a pull request e.g. a scheduled run or a push # to a specific branch like prerelease_test we add additional # entries to the matrix -if event_type != "pull_request": +if not pull_request: # add debug test for first supported PG13 version pg13_debug_earliest = { "pg": PG13_EARLIEST, @@ -214,7 +217,7 @@ def macos_config(overrides): "dist_gapfill_pushdown-13", "transparent_decompress_chunk-13", }, - "tsdb_build_args": "-DWARNINGS_AS_ERRORS=ON -DASSERTIONS=ON -DPG_ISOLATION_REGRESS=OFF", + "tsdb_build_args": "-DWARNINGS_AS_ERRORS=ON -DASSERTIONS=ON -DPG_ISOLATION_REGRESS=OFF -DENABLE_MULTINODETESTS=ON", } m["include"].append(build_debug_config(pg13_debug_earliest)) @@ -274,7 +277,7 @@ def macos_config(overrides): } ) ) -else: +elif len(sys.argv) > 2: # Check if we need to check for the flaky tests. Determine which test files # have been changed in the PR. The sql files might include other files that # change independently, and might be .in templates, so it's easier to look diff --git a/.github/workflows/abi.yaml b/.github/workflows/abi.yaml index a9184816755..0e3473afa13 100644 --- a/.github/workflows/abi.yaml +++ b/.github/workflows/abi.yaml @@ -95,7 +95,7 @@ jobs: apk add cmake gcc make build-base krb5-dev git ${EXTRA_PKGS} git config --global --add safe.directory /mnt cd /mnt - BUILD_DIR=build_abi BUILD_FORCE_REMOVE=true ./bootstrap + BUILD_DIR=build_abi BUILD_FORCE_REMOVE=true ./bootstrap -DENABLE_MULTINODE_TESTS=ON make -C build_abi install mkdir -p build_abi/install_ext build_abi/install_lib cp `pg_config --sharedir`/extension/timescaledb*.{control,sql} build_abi/install_ext diff --git a/.github/workflows/sanitizer-build-and-test.yaml b/.github/workflows/sanitizer-build-and-test.yaml index 8b22d99acac..37fa97bad0d 100644 --- a/.github/workflows/sanitizer-build-and-test.yaml +++ b/.github/workflows/sanitizer-build-and-test.yaml @@ -145,7 +145,7 @@ jobs: - name: Build TimescaleDB run: | - ./bootstrap -DCMAKE_BUILD_TYPE=Debug -DPG_SOURCE_DIR=~/$PG_SRC_DIR \ + ./bootstrap -DCMAKE_BUILD_TYPE=Debug -DPG_SOURCE_DIR=~/$PG_SRC_DIR -DENABLE_MULTINODE_TESTS=ON \ -DPG_PATH=~/$PG_INSTALL_DIR -DCODECOVERAGE=OFF -DREQUIRE_ALL_TESTS=ON -DTEST_GROUP_SIZE=5 make -j$(nproc) -C build make -C build install diff --git a/CMakeLists.txt b/CMakeLists.txt index 1e2ad56390e..e61a335d6fb 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -26,6 +26,7 @@ option( "Enable OPTIMIZER_DEBUG when building. Requires Postgres server to be built with OPTIMIZER_DEBUG." OFF) option(ENABLE_DEBUG_UTILS "Enable debug utilities for the extension." ON) +option(ENABLE_MULTINODE_TESTS "Enable multinode-specific tests" OFF) # Option to enable assertions. Note that if we include headers from a PostgreSQL # build that has assertions enabled, we might inherit that setting without diff --git a/tsl/test/isolation/specs/CMakeLists.txt b/tsl/test/isolation/specs/CMakeLists.txt index db5e80c2920..5b231d5d67b 100644 --- a/tsl/test/isolation/specs/CMakeLists.txt +++ b/tsl/test/isolation/specs/CMakeLists.txt @@ -15,27 +15,21 @@ list( cagg_insert.spec cagg_multi_iso.spec cagg_concurrent_refresh.spec - cagg_concurrent_refresh_dist_ht.spec deadlock_drop_chunks_compress.spec) +if(ENABLE_MULTINODE_TESTS) + list(APPEND TEST_FILES cagg_concurrent_refresh_dist_ht.spec) +endif() + if(PG_VERSION VERSION_GREATER_EQUAL "14.0") list(APPEND TEST_FILES concurrent_decompress_update.spec) endif() if(CMAKE_BUILD_TYPE MATCHES Debug) list(APPEND TEST_TEMPLATES_MODULE ${TEST_TEMPLATES_MODULE_DEBUG}) - list( - APPEND - TEST_FILES - cagg_drop_chunks_iso.spec - cagg_multi_dist_ht.spec - compression_chunk_race.spec - compression_merge_race.spec - decompression_chunk_and_parallel_query_wo_idx.spec - dist_ha_chunk_drop.spec - dist_ha_chunk_drop.spec - dist_restore_point.spec - remote_create_chunk.spec) + list(APPEND TEST_FILES cagg_drop_chunks_iso.spec compression_chunk_race.spec + compression_merge_race.spec + decompression_chunk_and_parallel_query_wo_idx.spec) if(PG_VERSION VERSION_GREATER_EQUAL "14.0") list(APPEND TEST_FILES freeze_chunk.spec compression_dml_iso.spec) endif() @@ -48,6 +42,17 @@ if(CMAKE_BUILD_TYPE MATCHES Debug) list(APPEND TEST_FILES deadlock_recompress_chunk.spec) endif() + if(ENABLE_MULTINODE_TESTS) + list( + APPEND + TEST_FILES + cagg_multi_dist_ht.spec + dist_ha_chunk_drop.spec + dist_ha_chunk_drop.spec + dist_restore_point.spec + remote_create_chunk.spec) + endif() + endif(CMAKE_BUILD_TYPE MATCHES Debug) # need to generate MODULE name for the .spec files diff --git a/tsl/test/shared/sql/CMakeLists.txt b/tsl/test/shared/sql/CMakeLists.txt index 228d84211e3..e162bedf507 100644 --- a/tsl/test/shared/sql/CMakeLists.txt +++ b/tsl/test/shared/sql/CMakeLists.txt @@ -6,21 +6,19 @@ set(TEST_FILES_SHARED constraint_exclusion_prepared.sql decompress_join.sql decompress_placeholdervar.sql - dist_chunk.sql - dist_distinct_pushdown.sql - dist_gapfill.sql - dist_insert.sql gapfill.sql subtract_integer_from_now.sql) set(TEST_TEMPLATES_SHARED - dist_fetcher_type.sql.in - generated_columns.sql.in - ordered_append.sql.in - ordered_append_join.sql.in - transparent_decompress_chunk.sql.in - dist_distinct.sql.in - space_constraint.sql.in) + generated_columns.sql.in ordered_append.sql.in ordered_append_join.sql.in + transparent_decompress_chunk.sql.in space_constraint.sql.in) + +if(ENABLE_MULTINODE_TESTS) + list(APPEND TEST_FILES_SHARED dist_chunk.sql dist_distinct_pushdown.sql + dist_gapfill.sql dist_insert.sql) + list(APPEND TEST_TEMPLATES_SHARED dist_fetcher_type.sql.in + dist_distinct.sql.in) +endif() if((${PG_VERSION_MAJOR} GREATER_EQUAL "14")) list(APPEND TEST_FILES_SHARED compression_dml.sql memoize.sql) @@ -33,15 +31,12 @@ if((${PG_VERSION_MAJOR} GREATER_EQUAL "15")) endif() if(CMAKE_BUILD_TYPE MATCHES Debug) - list( - APPEND - TEST_FILES_SHARED - dist_parallel_agg.sql - dist_queries.sql - extension.sql - timestamp_limits.sql - with_clause_parser.sql) + list(APPEND TEST_FILES_SHARED extension.sql timestamp_limits.sql + with_clause_parser.sql) list(APPEND TEST_TEMPLATES_SHARED constify_now.sql.in) + if(ENABLE_MULTINODE_TESTS) + list(APPEND TEST_FILES_SHARED dist_parallel_agg.sql dist_queries.sql) + endif() endif(CMAKE_BUILD_TYPE MATCHES Debug) # Regression tests that vary with PostgreSQL version. Generated test files are diff --git a/tsl/test/sql/CMakeLists.txt b/tsl/test/sql/CMakeLists.txt index 3b384493c64..aae0ccaa603 100644 --- a/tsl/test/sql/CMakeLists.txt +++ b/tsl/test/sql/CMakeLists.txt @@ -18,8 +18,6 @@ set(TEST_FILES compression_conflicts.sql compression_insert.sql compression_qualpushdown.sql - dist_param.sql - dist_views.sql exp_cagg_monthly.sql exp_cagg_next_gen.sql exp_cagg_origin.sql @@ -30,6 +28,10 @@ set(TEST_FILES skip_scan.sql size_utils_tsl.sql) +if(ENABLE_MULTINODE_TESTS) + list(APPEND TEST_FILES dist_param.sql dist_views.sql) +endif() + if(USE_TELEMETRY) list(APPEND TEST_FILES bgw_telemetry.sql) endif() @@ -59,20 +61,15 @@ if(CMAKE_BUILD_TYPE MATCHES Debug) compress_table.sql cagg_bgw_drop_chunks.sql cagg_bgw.sql - cagg_bgw_dist_ht.sql cagg_ddl.sql - cagg_ddl_dist_ht.sql cagg_drop_chunks.sql cagg_dump.sql cagg_errors_deprecated.sql cagg_joins.sql cagg_migrate.sql - cagg_migrate_dist_ht.sql cagg_multi.sql cagg_on_cagg.sql - cagg_on_cagg_dist_ht.sql cagg_on_cagg_joins.sql - cagg_on_cagg_joins_dist_ht.sql cagg_tableam.sql cagg_usage.sql cagg_policy_run.sql @@ -83,29 +80,9 @@ if(CMAKE_BUILD_TYPE MATCHES Debug) ddl_hook.sql debug_notice.sql deparse.sql - dist_api_calls.sql - dist_commands.sql - dist_compression.sql - dist_copy_available_dns.sql - dist_copy_format_long.sql - dist_copy_long.sql - dist_ddl.sql - dist_cagg.sql - dist_move_chunk.sql - dist_policy.sql - dist_util.sql - dist_triggers.sql - dist_backup.sql insert_memory_usage.sql information_view_chunk_count.sql read_only.sql - remote_connection_cache.sql - remote_connection.sql - remote_copy.sql - remote_stmt_params.sql - remote_txn_id.sql - remote_txn_resolve.sql - remote_txn.sql transparent_decompression_queries.sql tsl_tables.sql license_tsl.sql @@ -113,6 +90,38 @@ if(CMAKE_BUILD_TYPE MATCHES Debug) recompress_chunk_segmentwise.sql transparent_decompression_join_index.sql feature_flags.sql) + + if(ENABLE_MULTINODE_TESTS) + list( + APPEND + TEST_FILES + cagg_bgw_dist_ht.sql + cagg_migrate_dist_ht.sql + cagg_ddl_dist_ht.sql + cagg_on_cagg_dist_ht.sql + cagg_on_cagg_joins_dist_ht.sql + dist_api_calls.sql + dist_commands.sql + dist_compression.sql + dist_copy_available_dns.sql + dist_copy_format_long.sql + dist_copy_long.sql + dist_ddl.sql + dist_cagg.sql + dist_move_chunk.sql + dist_policy.sql + dist_util.sql + dist_triggers.sql + dist_backup.sql + remote_connection_cache.sql + remote_connection.sql + remote_copy.sql + remote_stmt_params.sql + remote_txn_id.sql + remote_txn_resolve.sql + remote_txn.sql) + endif() + endif(CMAKE_BUILD_TYPE MATCHES Debug) if((${PG_VERSION_MAJOR} GREATER_EQUAL "14")) @@ -172,19 +181,20 @@ if((${PG_VERSION_MAJOR} GREATER_EQUAL "14")) endif() if(CMAKE_BUILD_TYPE MATCHES Debug) - list( - APPEND - TEST_TEMPLATES - cagg_query.sql.in - dist_hypertable.sql.in - dist_grant.sql.in - dist_ref_table_join.sql.in - dist_remote_error.sql.in - dist_partial_agg.sql.in - dist_query.sql.in - cagg_invalidation_dist_ht.sql.in - continuous_aggs.sql.in - continuous_aggs_deprecated.sql.in) + list(APPEND TEST_TEMPLATES cagg_query.sql.in continuous_aggs.sql.in + continuous_aggs_deprecated.sql.in) + if(ENABLE_MULTINODE_TESTS) + list( + APPEND + TEST_TEMPLATES + cagg_invalidation_dist_ht.sql.in + dist_hypertable.sql.in + dist_grant.sql.in + dist_ref_table_join.sql.in + dist_remote_error.sql.in + dist_partial_agg.sql.in + dist_query.sql.in) + endif() if(USE_TELEMETRY) list(APPEND TEST_TEMPLATES telemetry_stats.sql.in) endif()