From 60ecd4605212acdf4c2b643a415a8f1d0b3b5ea7 Mon Sep 17 00:00:00 2001 From: Ildar Musin Date: Tue, 14 Jan 2025 17:41:46 +0100 Subject: [PATCH] Add `include_tiered_data` option to cagg policy The `include_tiered_data` option allows user to override the value of `timescaledb.enable_tiered_reads` defined on instance level for a particular continuous aggregate policy. --- .unreleased/pr_7587 | 1 + sql/policy_api.sql | 3 +- sql/updates/latest-dev.sql | 19 ++ sql/updates/reverse-dev.sql | 20 ++ src/guc.h | 2 +- tsl/src/bgw_policy/continuous_aggregate_api.c | 23 +- tsl/src/bgw_policy/continuous_aggregate_api.h | 4 +- tsl/src/bgw_policy/job.c | 30 +++ tsl/src/bgw_policy/job.h | 3 +- tsl/src/bgw_policy/policies_v2.c | 5 +- tsl/src/bgw_policy/policies_v2.h | 1 + tsl/test/expected/chunk_utils_internal.out | 207 ++++++++++++------ tsl/test/shared/expected/extension.out | 2 +- tsl/test/sql/chunk_utils_internal.sql | 48 ++++ 14 files changed, 297 insertions(+), 71 deletions(-) create mode 100644 .unreleased/pr_7587 diff --git a/.unreleased/pr_7587 b/.unreleased/pr_7587 new file mode 100644 index 000000000..bd14aaf1c --- /dev/null +++ b/.unreleased/pr_7587 @@ -0,0 +1 @@ +Implements: #7587 Add `include_tiered_data` parameter to `add_continuous_aggregate_policy` API diff --git a/sql/policy_api.sql b/sql/policy_api.sql index 83a8a3904..be8cc33cc 100644 --- a/sql/policy_api.sql +++ b/sql/policy_api.sql @@ -85,7 +85,8 @@ CREATE OR REPLACE FUNCTION @extschema@.add_continuous_aggregate_policy( end_offset "any", schedule_interval INTERVAL, if_not_exists BOOL = false, initial_start TIMESTAMPTZ = NULL, - timezone TEXT = NULL + timezone TEXT = NULL, + include_tiered_data BOOL = NULL ) RETURNS INTEGER AS '@MODULE_PATHNAME@', 'ts_policy_refresh_cagg_add' diff --git a/sql/updates/latest-dev.sql b/sql/updates/latest-dev.sql index 5255e9e39..33d290e37 100644 --- a/sql/updates/latest-dev.sql +++ b/sql/updates/latest-dev.sql @@ -125,3 +125,22 @@ CREATE PROCEDURE @extschema@.refresh_continuous_aggregate( force BOOLEAN = FALSE ) LANGUAGE C AS '@MODULE_PATHNAME@', 'ts_update_placeholder'; +-- Add `include_tiered_data` argument to `add_continuous_aggregate_policy` +DROP FUNCTION @extschema@.add_continuous_aggregate_policy( + continuous_aggregate REGCLASS, start_offset "any", + end_offset "any", schedule_interval INTERVAL, + if_not_exists BOOL, + initial_start TIMESTAMPTZ, + timezone TEXT +); +CREATE FUNCTION @extschema@.add_continuous_aggregate_policy( + continuous_aggregate REGCLASS, start_offset "any", + end_offset "any", schedule_interval INTERVAL, + if_not_exists BOOL = false, + initial_start TIMESTAMPTZ = NULL, + timezone TEXT = NULL, + include_tiered_data BOOL = NULL +) +RETURNS INTEGER +AS '@MODULE_PATHNAME@', 'ts_update_placeholder' +LANGUAGE C VOLATILE; diff --git a/sql/updates/reverse-dev.sql b/sql/updates/reverse-dev.sql index bff21e0a9..37f3659dd 100644 --- a/sql/updates/reverse-dev.sql +++ b/sql/updates/reverse-dev.sql @@ -67,3 +67,23 @@ CREATE PROCEDURE @extschema@.refresh_continuous_aggregate( window_start "any", window_end "any" ) LANGUAGE C AS '@MODULE_PATHNAME@', 'ts_continuous_agg_refresh'; + +-- Remove `include_tiered_data` argument from `add_continuous_aggregate_policy` +DROP FUNCTION @extschema@.add_continuous_aggregate_policy( + continuous_aggregate REGCLASS, start_offset "any", + end_offset "any", schedule_interval INTERVAL, + if_not_exists BOOL, + initial_start TIMESTAMPTZ, + timezone TEXT, + include_tiered_data BOOL +); +CREATE FUNCTION @extschema@.add_continuous_aggregate_policy( + continuous_aggregate REGCLASS, start_offset "any", + end_offset "any", schedule_interval INTERVAL, + if_not_exists BOOL = false, + initial_start TIMESTAMPTZ = NULL, + timezone TEXT = NULL +) +RETURNS INTEGER +AS '@MODULE_PATHNAME@', 'ts_policy_refresh_cagg_add' +LANGUAGE C VOLATILE; diff --git a/src/guc.h b/src/guc.h index 54a341b7a..7691be198 100644 --- a/src/guc.h +++ b/src/guc.h @@ -44,7 +44,7 @@ extern bool ts_guc_enable_cagg_reorder_groupby; extern TSDLLEXPORT int ts_guc_cagg_max_individual_materializations; extern bool ts_guc_enable_now_constify; extern bool ts_guc_enable_foreign_key_propagation; -extern bool ts_guc_enable_osm_reads; +extern TSDLLEXPORT bool ts_guc_enable_osm_reads; #if PG16_GE extern TSDLLEXPORT bool ts_guc_enable_cagg_sort_pushdown; #endif diff --git a/tsl/src/bgw_policy/continuous_aggregate_api.c b/tsl/src/bgw_policy/continuous_aggregate_api.c index 245c12006..e400e4ccf 100644 --- a/tsl/src/bgw_policy/continuous_aggregate_api.c +++ b/tsl/src/bgw_policy/continuous_aggregate_api.c @@ -136,6 +136,16 @@ policy_refresh_cagg_get_refresh_end(const Dimension *dim, const Jsonb *config, b return res; } +bool +policy_refresh_cagg_get_include_tiered_data(const Jsonb *config, bool *isnull) +{ + bool found; + bool res = ts_jsonb_get_bool_field(config, POL_REFRESH_CONF_KEY_INCLUDE_TIERED_DATA, &found); + + *isnull = !found; + return res; +} + /* returns false if a policy could not be found */ bool policy_refresh_cagg_exists(int32 materialization_id) @@ -519,7 +529,8 @@ Datum policy_refresh_cagg_add_internal(Oid cagg_oid, Oid start_offset_type, NullableDatum start_offset, Oid end_offset_type, NullableDatum end_offset, Interval refresh_interval, bool if_not_exists, bool fixed_schedule, - TimestampTz initial_start, const char *timezone) + TimestampTz initial_start, const char *timezone, + NullableDatum include_tiered_data) { NameData application_name; NameData proc_name, proc_schema, check_name, check_schema, owner; @@ -630,6 +641,10 @@ policy_refresh_cagg_add_internal(Oid cagg_oid, Oid start_offset_type, NullableDa policyconf.offset_end.value); else ts_jsonb_add_null(parse_state, POL_REFRESH_CONF_KEY_END_OFFSET); + if (!include_tiered_data.isnull) + ts_jsonb_add_bool(parse_state, + POL_REFRESH_CONF_KEY_INCLUDE_TIERED_DATA, + include_tiered_data.value); JsonbValue *result = pushJsonbValue(&parse_state, WJB_END_OBJECT, NULL); Jsonb *config = JsonbValueToJsonb(result); @@ -660,6 +675,7 @@ policy_refresh_cagg_add(PG_FUNCTION_ARGS) Interval refresh_interval; bool if_not_exists; NullableDatum start_offset, end_offset; + NullableDatum include_tiered_data; ts_feature_flag_check(FEATURE_POLICY); @@ -682,6 +698,8 @@ policy_refresh_cagg_add(PG_FUNCTION_ARGS) bool fixed_schedule = !PG_ARGISNULL(5); text *timezone = PG_ARGISNULL(6) ? NULL : PG_GETARG_TEXT_PP(6); char *valid_timezone = NULL; + include_tiered_data.value = PG_GETARG_DATUM(7); + include_tiered_data.isnull = PG_ARGISNULL(7); Datum retval; /* if users pass in -infinity for initial_start, then use the current_timestamp instead */ @@ -704,7 +722,8 @@ policy_refresh_cagg_add(PG_FUNCTION_ARGS) if_not_exists, fixed_schedule, initial_start, - valid_timezone); + valid_timezone, + include_tiered_data); if (!TIMESTAMP_NOT_FINITE(initial_start)) { int32 job_id = DatumGetInt32(retval); diff --git a/tsl/src/bgw_policy/continuous_aggregate_api.h b/tsl/src/bgw_policy/continuous_aggregate_api.h index 160d7bd67..50588b424 100644 --- a/tsl/src/bgw_policy/continuous_aggregate_api.h +++ b/tsl/src/bgw_policy/continuous_aggregate_api.h @@ -20,6 +20,7 @@ int64 policy_refresh_cagg_get_refresh_start(const ContinuousAgg *cagg, const Dim const Jsonb *config, bool *start_isnull); int64 policy_refresh_cagg_get_refresh_end(const Dimension *dim, const Jsonb *config, bool *end_isnull); +bool policy_refresh_cagg_get_include_tiered_data(const Jsonb *config, bool *isnull); bool policy_refresh_cagg_refresh_start_lt(int32 materialization_id, Oid cmp_type, Datum cmp_interval); bool policy_refresh_cagg_exists(int32 materialization_id); @@ -28,5 +29,6 @@ Datum policy_refresh_cagg_add_internal(Oid cagg_oid, Oid start_offset_type, NullableDatum start_offset, Oid end_offset_type, NullableDatum end_offset, Interval refresh_interval, bool if_not_exists, bool fixed_schedule, - TimestampTz initial_start, const char *timezone); + TimestampTz initial_start, const char *timezone, + NullableDatum include_tiered_data); Datum policy_refresh_cagg_remove_internal(Oid cagg_oid, bool if_exists); diff --git a/tsl/src/bgw_policy/job.c b/tsl/src/bgw_policy/job.c index f4a9dfdd9..2e2dc5473 100644 --- a/tsl/src/bgw_policy/job.c +++ b/tsl/src/bgw_policy/job.c @@ -19,6 +19,7 @@ #include #include #include +#include #include #include #include @@ -51,6 +52,7 @@ #include "dimension_slice.h" #include "dimension_vector.h" #include "errors.h" +#include "guc.h" #include "job.h" #include "reorder.h" #include "utils.h" @@ -372,7 +374,21 @@ policy_refresh_cagg_execute(int32 job_id, Jsonb *config) { PolicyContinuousAggData policy_data; + StringInfo str = makeStringInfo(); + JsonbToCStringIndent(str, &config->root, VARSIZE(config)); + policy_refresh_cagg_read_and_validate_config(config, &policy_data); + + bool enable_osm_reads_old = ts_guc_enable_osm_reads; + + if (!policy_data.include_tiered_data_isnull) + { + SetConfigOption("timescaledb.enable_tiered_reads", + policy_data.include_tiered_data ? "on" : "off", + PGC_USERSET, + PGC_S_SESSION); + } + continuous_agg_refresh_internal(policy_data.cagg, &policy_data.refresh_window, CAGG_REFRESH_POLICY, @@ -380,6 +396,14 @@ policy_refresh_cagg_execute(int32 job_id, Jsonb *config) policy_data.end_is_null, false); + if (!policy_data.include_tiered_data_isnull) + { + SetConfigOption("timescaledb.enable_tiered_reads", + enable_osm_reads_old ? "on" : "off", + PGC_USERSET, + PGC_S_SESSION); + } + return true; } @@ -392,6 +416,7 @@ policy_refresh_cagg_read_and_validate_config(Jsonb *config, PolicyContinuousAggD Oid dim_type; int64 refresh_start, refresh_end; bool start_isnull, end_isnull; + bool include_tiered_data, include_tiered_data_isnull; materialization_id = policy_continuous_aggregate_get_mat_hypertable_id(config); mat_ht = ts_hypertable_get_by_id(materialization_id); @@ -418,6 +443,9 @@ policy_refresh_cagg_read_and_validate_config(Jsonb *config, PolicyContinuousAggD ts_internal_to_time_string(refresh_end, dim_type)), errhint("The start of the window must be before the end."))); + include_tiered_data = + policy_refresh_cagg_get_include_tiered_data(config, &include_tiered_data_isnull); + if (policy_data) { policy_data->refresh_window.type = dim_type; @@ -426,6 +454,8 @@ policy_refresh_cagg_read_and_validate_config(Jsonb *config, PolicyContinuousAggD policy_data->cagg = cagg; policy_data->start_is_null = start_isnull; policy_data->end_is_null = end_isnull; + policy_data->include_tiered_data = include_tiered_data; + policy_data->include_tiered_data_isnull = include_tiered_data_isnull; } } diff --git a/tsl/src/bgw_policy/job.h b/tsl/src/bgw_policy/job.h index 70e488cd4..404d97640 100644 --- a/tsl/src/bgw_policy/job.h +++ b/tsl/src/bgw_policy/job.h @@ -36,7 +36,8 @@ typedef struct PolicyContinuousAggData { InternalTimeRange refresh_window; ContinuousAgg *cagg; - bool start_is_null, end_is_null; + bool include_tiered_data; + bool start_is_null, end_is_null, include_tiered_data_isnull; } PolicyContinuousAggData; typedef struct PolicyCompressionData diff --git a/tsl/src/bgw_policy/policies_v2.c b/tsl/src/bgw_policy/policies_v2.c index 9bfbe9e3a..eeb1643ab 100644 --- a/tsl/src/bgw_policy/policies_v2.c +++ b/tsl/src/bgw_policy/policies_v2.c @@ -206,6 +206,8 @@ validate_and_create_policies(policies_info all_policies, bool if_exists) /* Create policies as required, delete the old ones if coming from alter */ if (all_policies.refresh && all_policies.refresh->create_policy) { + NullableDatum include_tiered_data = { .isnull = true }; + if (all_policies.is_alter_policy) policy_refresh_cagg_remove_internal(all_policies.rel_oid, if_exists); refresh_job_id = policy_refresh_cagg_add_internal(all_policies.rel_oid, @@ -217,7 +219,8 @@ validate_and_create_policies(policies_info all_policies, bool if_exists) false, false, DT_NOBEGIN, - NULL); + NULL, + include_tiered_data); } if (all_policies.compress && all_policies.compress->create_policy) { diff --git a/tsl/src/bgw_policy/policies_v2.h b/tsl/src/bgw_policy/policies_v2.h index 4a5eeb852..03535adb4 100644 --- a/tsl/src/bgw_policy/policies_v2.h +++ b/tsl/src/bgw_policy/policies_v2.h @@ -19,6 +19,7 @@ #define POL_REFRESH_CONF_KEY_MAT_HYPERTABLE_ID "mat_hypertable_id" #define POL_REFRESH_CONF_KEY_START_OFFSET "start_offset" #define POL_REFRESH_CONF_KEY_END_OFFSET "end_offset" +#define POL_REFRESH_CONF_KEY_INCLUDE_TIERED_DATA "include_tiered_data" #define POLICY_COMPRESSION_PROC_NAME "policy_compression" #define POLICY_COMPRESSION_CHECK_NAME "policy_compression_check" diff --git a/tsl/test/expected/chunk_utils_internal.out b/tsl/test/expected/chunk_utils_internal.out index 520484a86..d03c131a7 100644 --- a/tsl/test/expected/chunk_utils_internal.out +++ b/tsl/test/expected/chunk_utils_internal.out @@ -819,6 +819,87 @@ SELECT * FROM ht_try_weekly; DROP MATERIALIZED VIEW ht_try_weekly; NOTICE: drop cascades to table _timescaledb_internal._hyper_6_12_chunk +-- Test refresh policy with different settings of `include_tiered_data` parameter +CREATE FUNCTION create_test_cagg(include_tiered_data BOOL) +RETURNS INTEGER AS +$$ +DECLARE + cfg jsonb; + job_id INTEGER; +BEGIN + CREATE MATERIALIZED VIEW ht_try_weekly + WITH (timescaledb.continuous) AS + SELECT time_bucket(interval '1 week', timec) AS ts_bucket, avg(value) + FROM ht_try + GROUP BY 1 + WITH NO DATA; + + job_id := add_continuous_aggregate_policy( + 'ht_try_weekly', + start_offset => NULL, + end_offset => INTERVAL '1 hour', + schedule_interval => INTERVAL '1 hour', + include_tiered_data => include_tiered_data + ); + + cfg := config FROM _timescaledb_config.bgw_job WHERE id = job_id; + RAISE NOTICE 'config: %', jsonb_pretty(cfg); + + RETURN job_id; +END +$$ LANGUAGE plpgsql; +-- include tiered data +SELECT create_test_cagg(true) AS job_id \gset +NOTICE: config: { + "end_offset": "@ 1 hour", + "start_offset": null, + "mat_hypertable_id": 7, + "include_tiered_data": true +} +CALL run_job(:job_id); +SELECT * FROM ht_try_weekly ORDER BY 1; + ts_bucket | avg +------------------------------+----------------------- + Sun Dec 29 16:00:00 2019 PST | 1000.0000000000000000 + Sun May 01 17:00:00 2022 PDT | 222.0000000000000000 +(2 rows) + +DROP MATERIALIZED VIEW ht_try_weekly; +NOTICE: drop cascades to 2 other objects +-- exclude tiered data +SELECT create_test_cagg(false) AS job_id \gset +NOTICE: config: { + "end_offset": "@ 1 hour", + "start_offset": null, + "mat_hypertable_id": 8, + "include_tiered_data": false +} +CALL run_job(:job_id); +SELECT * FROM ht_try_weekly ORDER BY 1; + ts_bucket | avg +------------------------------+---------------------- + Sun May 01 17:00:00 2022 PDT | 222.0000000000000000 +(1 row) + +DROP MATERIALIZED VIEW ht_try_weekly; +NOTICE: drop cascades to table _timescaledb_internal._hyper_8_15_chunk +-- default behavior: use instance-wide GUC value +SELECT create_test_cagg(null) AS job_id \gset +NOTICE: config: { + "end_offset": "@ 1 hour", + "start_offset": null, + "mat_hypertable_id": 9 +} +CALL run_job(:job_id); +SELECT * FROM ht_try_weekly ORDER BY 1; + ts_bucket | avg +------------------------------+----------------------- + Sun Dec 29 16:00:00 2019 PST | 1000.0000000000000000 + Sun May 01 17:00:00 2022 PDT | 222.0000000000000000 +(2 rows) + +DROP MATERIALIZED VIEW ht_try_weekly; +NOTICE: drop cascades to 2 other objects -- This test verifies that a bugfix regarding the way `ROWID_VAR`s are adjusted -- in the chunks' targetlists on DELETE/UPDATE works (including partially -- compressed chunks) @@ -831,7 +912,7 @@ SELECT compress_chunk(show_chunks('ht_try', newer_than => '2021-01-01'::timestam compress_chunk ----------------------------------------- _timescaledb_internal._hyper_5_10_chunk - _timescaledb_internal._hyper_5_13_chunk + _timescaledb_internal._hyper_5_18_chunk (2 rows) INSERT INTO ht_try VALUES ('2021-06-05 01:00', 10, 222); @@ -934,7 +1015,7 @@ Indexes: Triggers: ts_insert_blocker BEFORE INSERT ON ht_try FOR EACH ROW EXECUTE FUNCTION _timescaledb_functions.insert_blocker() Child tables: _timescaledb_internal._hyper_5_10_chunk, - _timescaledb_internal._hyper_5_13_chunk + _timescaledb_internal._hyper_5_18_chunk -- verify that still can read from the table after catalog manipulations EXPLAIN (ANALYZE, COSTS OFF, TIMING OFF, SUMMARY OFF) SELECT * FROM ht_try; @@ -942,10 +1023,10 @@ EXPLAIN (ANALYZE, COSTS OFF, TIMING OFF, SUMMARY OFF) SELECT * FROM ht_try; ---------------------------------------------------------------------------------- Append (actual rows=3 loops=1) -> Custom Scan (DecompressChunk) on _hyper_5_10_chunk (actual rows=1 loops=1) - -> Seq Scan on compress_hyper_7_14_chunk (actual rows=1 loops=1) - -> Custom Scan (DecompressChunk) on _hyper_5_13_chunk (actual rows=1 loops=1) - -> Seq Scan on compress_hyper_7_15_chunk (actual rows=1 loops=1) - -> Seq Scan on _hyper_5_13_chunk (actual rows=1 loops=1) + -> Seq Scan on compress_hyper_10_19_chunk (actual rows=1 loops=1) + -> Custom Scan (DecompressChunk) on _hyper_5_18_chunk (actual rows=1 loops=1) + -> Seq Scan on compress_hyper_10_20_chunk (actual rows=1 loops=1) + -> Seq Scan on _hyper_5_18_chunk (actual rows=1 loops=1) (6 rows) ROLLBACK; @@ -996,9 +1077,9 @@ RESTRICT ,CHECK ( temp > 10) ); SELECT create_hypertable('hyper_constr', 'time', chunk_time_interval => 10); - create_hypertable ---------------------------- - (8,public,hyper_constr,t) + create_hypertable +---------------------------- + (11,public,hyper_constr,t) (1 row) INSERT INTO hyper_constr VALUES( 10, 200, 22, 1, 111, 44); @@ -1031,7 +1112,7 @@ WHERE hypertable_id IN (SELECT id from _timescaledb_catalog.hypertable ORDER BY table_name; table_name | status | osm_chunk --------------------+--------+----------- - _hyper_8_16_chunk | 0 | f + _hyper_11_21_chunk | 0 | f child_hyper_constr | 0 | t (2 rows) @@ -1119,15 +1200,15 @@ where hypertable_id = (Select id from _timescaledb_catalog.hypertable where tabl ORDER BY id; id | table_name ----+-------------------- - 16 | _hyper_8_16_chunk - 17 | child_hyper_constr + 21 | _hyper_11_21_chunk + 22 | child_hyper_constr (2 rows) -- show_chunks will not show the OSM chunk which is visible via the above query SELECT show_chunks('hyper_constr'); - show_chunks ------------------------------------------ - _timescaledb_internal._hyper_8_16_chunk + show_chunks +------------------------------------------ + _timescaledb_internal._hyper_11_21_chunk (1 row) ROLLBACK; @@ -1157,9 +1238,9 @@ CREATE TABLE test1.copy_test ( "value" double precision NOT NULL ); SELECT create_hypertable('test1.copy_test', 'time', chunk_time_interval => interval '1 day'); - create_hypertable ------------------------ - (9,test1,copy_test,t) + create_hypertable +------------------------ + (12,test1,copy_test,t) (1 row) COPY test1.copy_test FROM STDIN DELIMITER ','; @@ -1178,15 +1259,15 @@ SELECT _timescaledb_functions.freeze_chunk( :'COPY_CHNAME'); -- Check state SELECT table_name, status FROM _timescaledb_catalog.chunk WHERE table_name = :'COPY_CHUNK_NAME'; - table_name | status --------------------+-------- - _hyper_9_18_chunk | 4 + table_name | status +--------------------+-------- + _hyper_12_23_chunk | 4 (1 row) \set ON_ERROR_STOP 0 -- Copy should fail because one of che chunks is frozen COPY test1.copy_test FROM STDIN DELIMITER ','; -ERROR: cannot INSERT into frozen chunk "_hyper_9_18_chunk" +ERROR: cannot INSERT into frozen chunk "_hyper_12_23_chunk" \set ON_ERROR_STOP 1 -- Count existing rows SELECT COUNT(*) FROM test1.copy_test; @@ -1198,15 +1279,15 @@ SELECT COUNT(*) FROM test1.copy_test; -- Check state SELECT table_name, status FROM _timescaledb_catalog.chunk WHERE table_name = :'COPY_CHUNK_NAME'; - table_name | status --------------------+-------- - _hyper_9_18_chunk | 4 + table_name | status +--------------------+-------- + _hyper_12_23_chunk | 4 (1 row) \set ON_ERROR_STOP 0 -- Copy should fail because one of che chunks is frozen COPY test1.copy_test FROM STDIN DELIMITER ','; -ERROR: cannot INSERT into frozen chunk "_hyper_9_18_chunk" +ERROR: cannot INSERT into frozen chunk "_hyper_12_23_chunk" \set ON_ERROR_STOP 1 -- Count existing rows SELECT COUNT(*) FROM test1.copy_test; @@ -1225,9 +1306,9 @@ SELECT _timescaledb_functions.unfreeze_chunk( :'COPY_CHNAME'); -- Check state SELECT table_name, status FROM _timescaledb_catalog.chunk WHERE table_name = :'COPY_CHUNK_NAME'; - table_name | status --------------------+-------- - _hyper_9_18_chunk | 0 + table_name | status +--------------------+-------- + _hyper_12_23_chunk | 0 (1 row) -- Copy should work now @@ -1330,12 +1411,12 @@ WHERE ht.table_name LIKE 'osm%' ORDER BY 2,3; table_name | id | dimension_id | range_start | range_end ------------+----+--------------+---------------------+--------------------- - osm_int2 | 17 | 8 | 9223372036854775806 | 9223372036854775807 - osm_int4 | 18 | 9 | 9223372036854775806 | 9223372036854775807 - osm_int8 | 19 | 10 | 9223372036854775806 | 9223372036854775807 - osm_date | 20 | 11 | 9223372036854775806 | 9223372036854775807 - osm_ts | 21 | 12 | 9223372036854775806 | 9223372036854775807 - osm_tstz | 22 | 13 | 9223372036854775806 | 9223372036854775807 + osm_int2 | 22 | 11 | 9223372036854775806 | 9223372036854775807 + osm_int4 | 23 | 12 | 9223372036854775806 | 9223372036854775807 + osm_int8 | 24 | 13 | 9223372036854775806 | 9223372036854775807 + osm_date | 25 | 14 | 9223372036854775806 | 9223372036854775807 + osm_ts | 26 | 15 | 9223372036854775806 | 9223372036854775807 + osm_tstz | 27 | 16 | 9223372036854775806 | 9223372036854775807 (6 rows) -- test that correct slice is found and updated for table with multiple chunk constraints @@ -1348,8 +1429,8 @@ _timescaledb_catalog.chunk c, _timescaledb_catalog.chunk_constraint cc WHERE c.h AND c.id = cc.chunk_id; id | hypertable_id | schema_name | table_name | compressed_chunk_id | dropped | status | osm_chunk | chunk_id | dimension_slice_id | constraint_name | hypertable_constraint_name ----+---------------+-----------------------+--------------------+---------------------+---------+--------+-----------+----------+--------------------+-----------------------------+---------------------------- - 26 | 16 | _timescaledb_internal | _hyper_16_26_chunk | | f | 0 | f | 26 | | 26_5_test_multicon_time_key | test_multicon_time_key - 26 | 16 | _timescaledb_internal | _hyper_16_26_chunk | | f | 0 | f | 26 | 23 | constraint_23 | + 31 | 19 | _timescaledb_internal | _hyper_19_31_chunk | | f | 0 | f | 31 | | 31_5_test_multicon_time_key | test_multicon_time_key + 31 | 19 | _timescaledb_internal | _hyper_19_31_chunk | | f | 0 | f | 31 | 28 | constraint_28 | (2 rows) \c :TEST_DBNAME :ROLE_SUPERUSER ; @@ -1367,7 +1448,7 @@ FROM _timescaledb_catalog.chunk c, _timescaledb_catalog.chunk_constraint cc, _ti WHERE c.hypertable_id = :htid AND cc.chunk_id = c.id AND ds.id = cc.dimension_slice_id; chunk_id | table_name | status | osm_chunk | dimension_slice_id | range_start | range_end ----------+--------------------+--------+-----------+--------------------+------------------+------------------ - 26 | _hyper_16_26_chunk | 0 | t | 23 | 1577955600000000 | 1578128400000000 + 31 | _hyper_19_31_chunk | 0 | t | 28 | 1577955600000000 | 1578128400000000 (1 row) -- check that range was reset to default - infinity @@ -1395,7 +1476,7 @@ FROM _timescaledb_catalog.chunk c, _timescaledb_catalog.chunk_constraint cc, _ti WHERE c.hypertable_id = :htid AND cc.chunk_id = c.id AND ds.id = cc.dimension_slice_id ORDER BY cc.chunk_id; chunk_id | table_name | status | osm_chunk | dimension_slice_id | range_start | range_end ----------+--------------------+--------+-----------+--------------------+---------------------+--------------------- - 26 | _hyper_16_26_chunk | 0 | t | 23 | 9223372036854775806 | 9223372036854775807 + 31 | _hyper_19_31_chunk | 0 | t | 28 | 9223372036854775806 | 9223372036854775807 (1 row) -- TEST for orderedappend that depends on hypertable_osm_range_update functionality @@ -1420,9 +1501,9 @@ FROM _timescaledb_catalog.chunk c, _timescaledb_catalog.chunk_constraint cc, _ti WHERE c.hypertable_id = :htid AND cc.chunk_id = c.id AND ds.id = cc.dimension_slice_id ORDER BY cc.chunk_id; chunk_id | table_name | status | osm_chunk | dimension_slice_id | range_start | range_end ----------+-------------------------+--------+-----------+--------------------+---------------------+--------------------- - 27 | _hyper_17_27_chunk | 0 | f | 24 | 1577836800000000 | 1577923200000000 - 28 | _hyper_17_28_chunk | 0 | f | 25 | 1577923200000000 | 1578009600000000 - 29 | test_chunkapp_fdw_child | 0 | t | 26 | 9223372036854775806 | 9223372036854775807 + 32 | _hyper_20_32_chunk | 0 | f | 29 | 1577836800000000 | 1577923200000000 + 33 | _hyper_20_33_chunk | 0 | f | 30 | 1577923200000000 | 1578009600000000 + 34 | test_chunkapp_fdw_child | 0 | t | 31 | 9223372036854775806 | 9223372036854775807 (3 rows) -- attempt to update overlapping range, should fail @@ -1443,9 +1524,9 @@ FROM _timescaledb_catalog.chunk c, _timescaledb_catalog.chunk_constraint cc, _ti WHERE c.hypertable_id = :htid AND cc.chunk_id = c.id AND ds.id = cc.dimension_slice_id ORDER BY cc.chunk_id; chunk_id | table_name | status | osm_chunk | dimension_slice_id | range_start | range_end ----------+-------------------------+--------+-----------+--------------------+------------------+------------------ - 27 | _hyper_17_27_chunk | 0 | f | 24 | 1577836800000000 | 1577923200000000 - 28 | _hyper_17_28_chunk | 0 | f | 25 | 1577923200000000 | 1578009600000000 - 29 | test_chunkapp_fdw_child | 0 | t | 26 | 1578038400000000 | 1578124800000000 + 32 | _hyper_20_32_chunk | 0 | f | 29 | 1577836800000000 | 1577923200000000 + 33 | _hyper_20_33_chunk | 0 | f | 30 | 1577923200000000 | 1578009600000000 + 34 | test_chunkapp_fdw_child | 0 | t | 31 | 1578038400000000 | 1578124800000000 (3 rows) -- ordered append should be possible as ranges do not overlap @@ -1454,8 +1535,8 @@ WHERE c.hypertable_id = :htid AND cc.chunk_id = c.id AND ds.id = cc.dimension_sl ------------------------------------------------------------------------------------------------- Custom Scan (ChunkAppend) on test_chunkapp Order: test_chunkapp."time" - -> Index Scan Backward using _hyper_17_27_chunk_test_chunkapp_time_idx on _hyper_17_27_chunk - -> Index Scan Backward using _hyper_17_28_chunk_test_chunkapp_time_idx on _hyper_17_28_chunk + -> Index Scan Backward using _hyper_20_32_chunk_test_chunkapp_time_idx on _hyper_20_32_chunk + -> Index Scan Backward using _hyper_20_33_chunk_test_chunkapp_time_idx on _hyper_20_33_chunk -> Foreign Scan on test_chunkapp_fdw_child (5 rows) @@ -1496,9 +1577,9 @@ SELECT _timescaledb_functions.hypertable_osm_range_update('test_chunkapp',empty: QUERY PLAN ------------------------------------------------------------------------------------------------- Merge Append - Sort Key: _hyper_17_27_chunk."time" - -> Index Scan Backward using _hyper_17_27_chunk_test_chunkapp_time_idx on _hyper_17_27_chunk - -> Index Scan Backward using _hyper_17_28_chunk_test_chunkapp_time_idx on _hyper_17_28_chunk + Sort Key: _hyper_20_32_chunk."time" + -> Index Scan Backward using _hyper_20_32_chunk_test_chunkapp_time_idx on _hyper_20_32_chunk + -> Index Scan Backward using _hyper_20_33_chunk_test_chunkapp_time_idx on _hyper_20_33_chunk -> Foreign Scan on test_chunkapp_fdw_child (5 rows) @@ -1515,9 +1596,9 @@ FROM _timescaledb_catalog.chunk c, _timescaledb_catalog.chunk_constraint cc, _ti WHERE c.hypertable_id = :htid AND cc.chunk_id = c.id AND ds.id = cc.dimension_slice_id ORDER BY cc.chunk_id; chunk_id | table_name | status | osm_chunk | dimension_slice_id | range_start | range_end ----------+-------------------------+--------+-----------+--------------------+---------------------+--------------------- - 27 | _hyper_17_27_chunk | 0 | f | 24 | 1577836800000000 | 1577923200000000 - 28 | _hyper_17_28_chunk | 0 | f | 25 | 1577923200000000 | 1578009600000000 - 29 | test_chunkapp_fdw_child | 0 | t | 26 | 9223372036854775806 | 9223372036854775807 + 32 | _hyper_20_32_chunk | 0 | f | 29 | 1577836800000000 | 1577923200000000 + 33 | _hyper_20_33_chunk | 0 | f | 30 | 1577923200000000 | 1578009600000000 + 34 | test_chunkapp_fdw_child | 0 | t | 31 | 9223372036854775806 | 9223372036854775807 (3 rows) -- but also, OSM chunk should be included in the scan, since range is invalid and chunk is not empty @@ -1525,10 +1606,10 @@ WHERE c.hypertable_id = :htid AND cc.chunk_id = c.id AND ds.id = cc.dimension_sl QUERY PLAN ------------------------------------------------------------------------------------------------- Merge Append - Sort Key: _hyper_17_27_chunk."time" - -> Index Scan Backward using _hyper_17_27_chunk_test_chunkapp_time_idx on _hyper_17_27_chunk + Sort Key: _hyper_20_32_chunk."time" + -> Index Scan Backward using _hyper_20_32_chunk_test_chunkapp_time_idx on _hyper_20_32_chunk Index Cond: ("time" < 'Sun Jan 01 00:00:00 2023 PST'::timestamp with time zone) - -> Index Scan Backward using _hyper_17_28_chunk_test_chunkapp_time_idx on _hyper_17_28_chunk + -> Index Scan Backward using _hyper_20_33_chunk_test_chunkapp_time_idx on _hyper_20_33_chunk Index Cond: ("time" < 'Sun Jan 01 00:00:00 2023 PST'::timestamp with time zone) -> Foreign Scan on test_chunkapp_fdw_child (7 rows) @@ -1556,8 +1637,8 @@ SELECT _timescaledb_functions.hypertable_osm_range_update('test_chunkapp', NULL: ------------------------------------------------------------------------------------------------- Custom Scan (ChunkAppend) on test_chunkapp Order: test_chunkapp."time" - -> Index Scan Backward using _hyper_17_27_chunk_test_chunkapp_time_idx on _hyper_17_27_chunk - -> Index Scan Backward using _hyper_17_28_chunk_test_chunkapp_time_idx on _hyper_17_28_chunk + -> Index Scan Backward using _hyper_20_32_chunk_test_chunkapp_time_idx on _hyper_20_32_chunk + -> Index Scan Backward using _hyper_20_33_chunk_test_chunkapp_time_idx on _hyper_20_33_chunk -> Foreign Scan on test_chunkapp_fdw_child (5 rows) @@ -1574,9 +1655,9 @@ SELECT * FROM test_chunkapp ORDER BY 1; ------------------------------------------------------------------------------------------------- Custom Scan (ChunkAppend) on test_chunkapp Order: test_chunkapp."time" - -> Index Scan Backward using _hyper_17_27_chunk_test_chunkapp_time_idx on _hyper_17_27_chunk + -> Index Scan Backward using _hyper_20_32_chunk_test_chunkapp_time_idx on _hyper_20_32_chunk Index Cond: ("time" < 'Sun Jan 01 00:00:00 2023 PST'::timestamp with time zone) - -> Index Scan Backward using _hyper_17_28_chunk_test_chunkapp_time_idx on _hyper_17_28_chunk + -> Index Scan Backward using _hyper_20_33_chunk_test_chunkapp_time_idx on _hyper_20_33_chunk Index Cond: ("time" < 'Sun Jan 01 00:00:00 2023 PST'::timestamp with time zone) (6 rows) @@ -1613,7 +1694,7 @@ CREATE TABLE test2(time timestamptz not null, a int); SELECT create_hypertable('test2', 'time'); create_hypertable --------------------- - (18,public,test2,t) + (21,public,test2,t) (1 row) INSERT INTO test2 VALUES ('2020-01-01'::timestamptz, 1); @@ -1624,7 +1705,7 @@ psql:include/chunk_utils_internal_orderedappend.sql:138: NOTICE: default order SELECT compress_chunk(show_chunks('test2')); compress_chunk ------------------------------------------ - _timescaledb_internal._hyper_18_30_chunk + _timescaledb_internal._hyper_21_35_chunk (1 row) -- find internal compression table, call API function on it @@ -1633,7 +1714,7 @@ FROM _timescaledb_catalog.hypertable ht, _timescaledb_catalog.hypertable cht WHERE ht.table_name = 'test2' and cht.id = ht.compressed_hypertable_id \gset \set ON_ERROR_STOP 0 SELECT _timescaledb_functions.hypertable_osm_range_update(:'COMPRESSION_TBLNM'::regclass, '2020-01-01'::timestamptz); -psql:include/chunk_utils_internal_orderedappend.sql:145: ERROR: could not find time dimension for hypertable _timescaledb_internal._compressed_hypertable_19 +psql:include/chunk_utils_internal_orderedappend.sql:145: ERROR: could not find time dimension for hypertable _timescaledb_internal._compressed_hypertable_22 \set ON_ERROR_STOP 1 -- test wrong/incompatible data types with hypertable time dimension -- update range of int2 with int4 diff --git a/tsl/test/shared/expected/extension.out b/tsl/test/shared/expected/extension.out index 08e275bd2..fc8c70768 100644 --- a/tsl/test/shared/expected/extension.out +++ b/tsl/test/shared/expected/extension.out @@ -213,7 +213,7 @@ ORDER BY pronamespace::regnamespace::text COLLATE "C", p.oid::regprocedure::text ts_now_mock() add_columnstore_policy(regclass,"any",boolean,interval,timestamp with time zone,text,interval,boolean) add_compression_policy(regclass,"any",boolean,interval,timestamp with time zone,text,interval,boolean) - add_continuous_aggregate_policy(regclass,"any","any",interval,boolean,timestamp with time zone,text) + add_continuous_aggregate_policy(regclass,"any","any",interval,boolean,timestamp with time zone,text,boolean) add_dimension(regclass,_timescaledb_internal.dimension_info,boolean) add_dimension(regclass,name,integer,anyelement,regproc,boolean) add_job(regproc,interval,jsonb,timestamp with time zone,boolean,regproc,boolean,text) diff --git a/tsl/test/sql/chunk_utils_internal.sql b/tsl/test/sql/chunk_utils_internal.sql index feeb83bc5..ebb234e13 100644 --- a/tsl/test/sql/chunk_utils_internal.sql +++ b/tsl/test/sql/chunk_utils_internal.sql @@ -440,6 +440,54 @@ CALL refresh_continuous_aggregate('ht_try_weekly', '2019-12-29', '2020-01-10', f SELECT * FROM ht_try_weekly; DROP MATERIALIZED VIEW ht_try_weekly; +-- Test refresh policy with different settings of `include_tiered_data` parameter +CREATE FUNCTION create_test_cagg(include_tiered_data BOOL) +RETURNS INTEGER AS +$$ +DECLARE + cfg jsonb; + job_id INTEGER; +BEGIN + CREATE MATERIALIZED VIEW ht_try_weekly + WITH (timescaledb.continuous) AS + SELECT time_bucket(interval '1 week', timec) AS ts_bucket, avg(value) + FROM ht_try + GROUP BY 1 + WITH NO DATA; + + job_id := add_continuous_aggregate_policy( + 'ht_try_weekly', + start_offset => NULL, + end_offset => INTERVAL '1 hour', + schedule_interval => INTERVAL '1 hour', + include_tiered_data => include_tiered_data + ); + + cfg := config FROM _timescaledb_config.bgw_job WHERE id = job_id; + RAISE NOTICE 'config: %', jsonb_pretty(cfg); + + RETURN job_id; +END +$$ LANGUAGE plpgsql; + +-- include tiered data +SELECT create_test_cagg(true) AS job_id \gset +CALL run_job(:job_id); +SELECT * FROM ht_try_weekly ORDER BY 1; +DROP MATERIALIZED VIEW ht_try_weekly; + +-- exclude tiered data +SELECT create_test_cagg(false) AS job_id \gset +CALL run_job(:job_id); +SELECT * FROM ht_try_weekly ORDER BY 1; +DROP MATERIALIZED VIEW ht_try_weekly; + +-- default behavior: use instance-wide GUC value +SELECT create_test_cagg(null) AS job_id \gset +CALL run_job(:job_id); +SELECT * FROM ht_try_weekly ORDER BY 1; +DROP MATERIALIZED VIEW ht_try_weekly; + -- This test verifies that a bugfix regarding the way `ROWID_VAR`s are adjusted -- in the chunks' targetlists on DELETE/UPDATE works (including partially -- compressed chunks)