Make multinode tests conditional

Since multinode tests take a long time to run we dont want to run
them in CI on individual PRs but only on nightly runs.

.github/gh_matrix_builder.py

@@ -37,6 +37,7 @@ from ci_settings import (
 
 # github event type which is either push, pull_request or schedule
 event_type = sys.argv[1]
+pull_request = event_type == "pull_request"
 
 m = {
     "include": [],
@@ -75,6 +76,8 @@ def build_debug_config(overrides):
             "tsdb_build_args": "-DWARNINGS_AS_ERRORS=ON -DREQUIRE_ALL_TESTS=ON",
         }
     )
+    if not pull_request:
+        base_config["tsdb_build_args"] += " -DENABLE_MULTINODETESTS=ON"
     base_config.update(overrides)
     return base_config
 
@@ -93,6 +96,8 @@ def build_release_config(overrides):
             "coverage": False,
         }
     )
+    if not pull_request:
+        release_config["tsdb_build_args"] += " -DENABLE_MULTINODETESTS=ON"
     base_config.update(release_config)
     base_config.update(overrides)
     return base_config
@@ -148,6 +153,8 @@ def macos_config(overrides):
             "tsdb_build_args": "-DASSERTIONS=ON -DREQUIRE_ALL_TESTS=ON -DOPENSSL_ROOT_DIR=/usr/local/opt/openssl",
         }
     )
+    if not pull_request:
+        base_config["tsdb_build_args"] += " -DENABLE_MULTINODETESTS=ON"
     base_config.update(overrides)
     return base_config
 
@@ -159,13 +166,9 @@ ignored_tests = {"partialize_finalize"}
 # common ignored tests for all non-scheduled pg15 tests (e.g. PRs)
 # partialize_finalize is ignored due to #4937
 # dist_move_chunk, dist_param, dist_insert, and remote_txn ignored due to flakiness
-if event_type == "pull_request":
+if pull_request:
     ignored_tests = {
-        "dist_insert",
-        "dist_move_chunk",
-        "dist_param",
         "partialize_finalize",
-        "remote_txn",
         "telemetry",
     }
 
@@ -203,7 +206,7 @@ m["include"].append(
 # if this is not a pull request e.g. a scheduled run or a push
 # to a specific branch like prerelease_test we add additional
 # entries to the matrix
-if event_type != "pull_request":
+if not pull_request:
     # add debug test for first supported PG13 version
     pg13_debug_earliest = {
         "pg": PG13_EARLIEST,
@@ -214,7 +217,7 @@ if event_type != "pull_request":
             "dist_gapfill_pushdown-13",
             "transparent_decompress_chunk-13",
         },
-        "tsdb_build_args": "-DWARNINGS_AS_ERRORS=ON -DASSERTIONS=ON -DPG_ISOLATION_REGRESS=OFF",
+        "tsdb_build_args": "-DWARNINGS_AS_ERRORS=ON -DASSERTIONS=ON -DPG_ISOLATION_REGRESS=OFF -DENABLE_MULTINODETESTS=ON",
     }
     m["include"].append(build_debug_config(pg13_debug_earliest))
 
@@ -274,7 +277,7 @@ if event_type != "pull_request":
             }
         )
     )
-else:
+elif len(sys.argv) > 2:
     # Check if we need to check for the flaky tests. Determine which test files
     # have been changed in the PR. The sql files might include other files that
     # change independently, and might be .in templates, so it's easier to look

.github/workflows/abi.yaml

@@ -95,7 +95,7 @@ jobs:
           apk add cmake gcc make build-base krb5-dev git ${EXTRA_PKGS}
           git config --global --add safe.directory /mnt
           cd /mnt
-          BUILD_DIR=build_abi BUILD_FORCE_REMOVE=true ./bootstrap
+          BUILD_DIR=build_abi BUILD_FORCE_REMOVE=true ./bootstrap -DENABLE_MULTINODE_TESTS=ON
           make -C build_abi install
           mkdir -p build_abi/install_ext build_abi/install_lib
           cp `pg_config --sharedir`/extension/timescaledb*.{control,sql} build_abi/install_ext

.github/workflows/sanitizer-build-and-test.yaml

@@ -145,7 +145,7 @@ jobs:
 
     - name: Build TimescaleDB
       run: |
-        ./bootstrap -DCMAKE_BUILD_TYPE=Debug -DPG_SOURCE_DIR=~/$PG_SRC_DIR \
+        ./bootstrap -DCMAKE_BUILD_TYPE=Debug -DPG_SOURCE_DIR=~/$PG_SRC_DIR -DENABLE_MULTINODE_TESTS=ON \
           -DPG_PATH=~/$PG_INSTALL_DIR -DCODECOVERAGE=OFF -DREQUIRE_ALL_TESTS=ON -DTEST_GROUP_SIZE=5
         make -j$(nproc) -C build
         make -C build install

CMakeLists.txt

@@ -26,6 +26,7 @@ option(
   "Enable OPTIMIZER_DEBUG when building. Requires Postgres server to be built with OPTIMIZER_DEBUG."
   OFF)
 option(ENABLE_DEBUG_UTILS "Enable debug utilities for the extension." ON)
+option(ENABLE_MULTINODE_TESTS "Enable multinode-specific tests" OFF)
 
 # Option to enable assertions. Note that if we include headers from a PostgreSQL
 # build that has assertions enabled, we might inherit that setting without

tsl/test/isolation/specs/CMakeLists.txt

@@ -15,27 +15,21 @@ list(
   cagg_insert.spec
   cagg_multi_iso.spec
   cagg_concurrent_refresh.spec
-  cagg_concurrent_refresh_dist_ht.spec
   deadlock_drop_chunks_compress.spec)
 
+if(ENABLE_MULTINODE_TESTS)
+  list(APPEND TEST_FILES cagg_concurrent_refresh_dist_ht.spec)
+endif()
+
 if(PG_VERSION VERSION_GREATER_EQUAL "14.0")
   list(APPEND TEST_FILES concurrent_decompress_update.spec)
 endif()
 
 if(CMAKE_BUILD_TYPE MATCHES Debug)
   list(APPEND TEST_TEMPLATES_MODULE ${TEST_TEMPLATES_MODULE_DEBUG})
-  list(
-    APPEND
-    TEST_FILES
-    cagg_drop_chunks_iso.spec
-    cagg_multi_dist_ht.spec
-    compression_chunk_race.spec
-    compression_merge_race.spec
-    decompression_chunk_and_parallel_query_wo_idx.spec
-    dist_ha_chunk_drop.spec
-    dist_ha_chunk_drop.spec
-    dist_restore_point.spec
-    remote_create_chunk.spec)
+  list(APPEND TEST_FILES cagg_drop_chunks_iso.spec compression_chunk_race.spec
+       compression_merge_race.spec
+       decompression_chunk_and_parallel_query_wo_idx.spec)
   if(PG_VERSION VERSION_GREATER_EQUAL "14.0")
     list(APPEND TEST_FILES freeze_chunk.spec compression_dml_iso.spec)
   endif()
@@ -48,6 +42,17 @@ if(CMAKE_BUILD_TYPE MATCHES Debug)
     list(APPEND TEST_FILES deadlock_recompress_chunk.spec)
   endif()
 
+  if(ENABLE_MULTINODE_TESTS)
+    list(
+      APPEND
+      TEST_FILES
+      cagg_multi_dist_ht.spec
+      dist_ha_chunk_drop.spec
+      dist_ha_chunk_drop.spec
+      dist_restore_point.spec
+      remote_create_chunk.spec)
+  endif()
+
 endif(CMAKE_BUILD_TYPE MATCHES Debug)
 
 # need to generate MODULE name for the .spec files

tsl/test/shared/sql/CMakeLists.txt

@@ -6,21 +6,19 @@ set(TEST_FILES_SHARED
     constraint_exclusion_prepared.sql
     decompress_join.sql
     decompress_placeholdervar.sql
-    dist_chunk.sql
-    dist_distinct_pushdown.sql
-    dist_gapfill.sql
-    dist_insert.sql
     gapfill.sql
     subtract_integer_from_now.sql)
 
 set(TEST_TEMPLATES_SHARED
-    dist_fetcher_type.sql.in
-    generated_columns.sql.in
-    ordered_append.sql.in
-    ordered_append_join.sql.in
-    transparent_decompress_chunk.sql.in
-    dist_distinct.sql.in
-    space_constraint.sql.in)
+    generated_columns.sql.in ordered_append.sql.in ordered_append_join.sql.in
+    transparent_decompress_chunk.sql.in space_constraint.sql.in)
+
+if(ENABLE_MULTINODE_TESTS)
+  list(APPEND TEST_FILES_SHARED dist_chunk.sql dist_distinct_pushdown.sql
+       dist_gapfill.sql dist_insert.sql)
+  list(APPEND TEST_TEMPLATES_SHARED dist_fetcher_type.sql.in
+       dist_distinct.sql.in)
+endif()
 
 if((${PG_VERSION_MAJOR} GREATER_EQUAL "14"))
   list(APPEND TEST_FILES_SHARED compression_dml.sql memoize.sql)
@@ -33,15 +31,12 @@ if((${PG_VERSION_MAJOR} GREATER_EQUAL "15"))
 endif()
 
 if(CMAKE_BUILD_TYPE MATCHES Debug)
-  list(
-    APPEND
-    TEST_FILES_SHARED
-    dist_parallel_agg.sql
-    dist_queries.sql
-    extension.sql
-    timestamp_limits.sql
-    with_clause_parser.sql)
+  list(APPEND TEST_FILES_SHARED extension.sql timestamp_limits.sql
+       with_clause_parser.sql)
   list(APPEND TEST_TEMPLATES_SHARED constify_now.sql.in)
+  if(ENABLE_MULTINODE_TESTS)
+    list(APPEND TEST_FILES_SHARED dist_parallel_agg.sql dist_queries.sql)
+  endif()
 endif(CMAKE_BUILD_TYPE MATCHES Debug)
 
 # Regression tests that vary with PostgreSQL version. Generated test files are

tsl/test/sql/CMakeLists.txt

@@ -18,8 +18,6 @@ set(TEST_FILES
     compression_conflicts.sql
     compression_insert.sql
     compression_qualpushdown.sql
-    dist_param.sql
-    dist_views.sql
     exp_cagg_monthly.sql
     exp_cagg_next_gen.sql
     exp_cagg_origin.sql
@@ -30,6 +28,10 @@ set(TEST_FILES
     skip_scan.sql
     size_utils_tsl.sql)
 
+if(ENABLE_MULTINODE_TESTS)
+  list(APPEND TEST_FILES dist_param.sql dist_views.sql)
+endif()
+
 if(USE_TELEMETRY)
   list(APPEND TEST_FILES bgw_telemetry.sql)
 endif()
@@ -59,20 +61,15 @@ if(CMAKE_BUILD_TYPE MATCHES Debug)
     compress_table.sql
     cagg_bgw_drop_chunks.sql
     cagg_bgw.sql
-    cagg_bgw_dist_ht.sql
     cagg_ddl.sql
-    cagg_ddl_dist_ht.sql
     cagg_drop_chunks.sql
     cagg_dump.sql
     cagg_errors_deprecated.sql
     cagg_joins.sql
     cagg_migrate.sql
-    cagg_migrate_dist_ht.sql
     cagg_multi.sql
     cagg_on_cagg.sql
-    cagg_on_cagg_dist_ht.sql
     cagg_on_cagg_joins.sql
-    cagg_on_cagg_joins_dist_ht.sql
     cagg_tableam.sql
     cagg_usage.sql
     cagg_policy_run.sql
@@ -83,29 +80,9 @@ if(CMAKE_BUILD_TYPE MATCHES Debug)
     ddl_hook.sql
     debug_notice.sql
     deparse.sql
-    dist_api_calls.sql
-    dist_commands.sql
-    dist_compression.sql
-    dist_copy_available_dns.sql
-    dist_copy_format_long.sql
-    dist_copy_long.sql
-    dist_ddl.sql
-    dist_cagg.sql
-    dist_move_chunk.sql
-    dist_policy.sql
-    dist_util.sql
-    dist_triggers.sql
-    dist_backup.sql
     insert_memory_usage.sql
     information_view_chunk_count.sql
     read_only.sql
-    remote_connection_cache.sql
-    remote_connection.sql
-    remote_copy.sql
-    remote_stmt_params.sql
-    remote_txn_id.sql
-    remote_txn_resolve.sql
-    remote_txn.sql
     transparent_decompression_queries.sql
     tsl_tables.sql
     license_tsl.sql
@@ -113,6 +90,38 @@ if(CMAKE_BUILD_TYPE MATCHES Debug)
     recompress_chunk_segmentwise.sql
     transparent_decompression_join_index.sql
     feature_flags.sql)
+
+  if(ENABLE_MULTINODE_TESTS)
+    list(
+      APPEND
+      TEST_FILES
+      cagg_bgw_dist_ht.sql
+      cagg_migrate_dist_ht.sql
+      cagg_ddl_dist_ht.sql
+      cagg_on_cagg_dist_ht.sql
+      cagg_on_cagg_joins_dist_ht.sql
+      dist_api_calls.sql
+      dist_commands.sql
+      dist_compression.sql
+      dist_copy_available_dns.sql
+      dist_copy_format_long.sql
+      dist_copy_long.sql
+      dist_ddl.sql
+      dist_cagg.sql
+      dist_move_chunk.sql
+      dist_policy.sql
+      dist_util.sql
+      dist_triggers.sql
+      dist_backup.sql
+      remote_connection_cache.sql
+      remote_connection.sql
+      remote_copy.sql
+      remote_stmt_params.sql
+      remote_txn_id.sql
+      remote_txn_resolve.sql
+      remote_txn.sql)
+  endif()
+
 endif(CMAKE_BUILD_TYPE MATCHES Debug)
 
 if((${PG_VERSION_MAJOR} GREATER_EQUAL "14"))
@@ -172,19 +181,20 @@ if((${PG_VERSION_MAJOR} GREATER_EQUAL "14"))
 endif()
 
 if(CMAKE_BUILD_TYPE MATCHES Debug)
-  list(
-    APPEND
-    TEST_TEMPLATES
-    cagg_query.sql.in
-    dist_hypertable.sql.in
-    dist_grant.sql.in
-    dist_ref_table_join.sql.in
-    dist_remote_error.sql.in
-    dist_partial_agg.sql.in
-    dist_query.sql.in
-    cagg_invalidation_dist_ht.sql.in
-    continuous_aggs.sql.in
-    continuous_aggs_deprecated.sql.in)
+  list(APPEND TEST_TEMPLATES cagg_query.sql.in continuous_aggs.sql.in
+       continuous_aggs_deprecated.sql.in)
+  if(ENABLE_MULTINODE_TESTS)
+    list(
+      APPEND
+      TEST_TEMPLATES
+      cagg_invalidation_dist_ht.sql.in
+      dist_hypertable.sql.in
+      dist_grant.sql.in
+      dist_ref_table_join.sql.in
+      dist_remote_error.sql.in
+      dist_partial_agg.sql.in
+      dist_query.sql.in)
+  endif()
   if(USE_TELEMETRY)
     list(APPEND TEST_TEMPLATES telemetry_stats.sql.in)
   endif()