mirror of
https://github.com/timescale/timescaledb.git
synced 2025-05-17 19:13:16 +08:00
f49492b83d
When refreshing with an "infinite" refresh window going forward in time, the invalidation threshold is also moved forward to the end of the valid time range. This effectively renders the invalidation threshold useless, leading to unnecessary write amplification. To handle infinite refreshes better, this change caps the refresh window at the end of the last bucket of data in the underlying hypertable, as to not move the invalidation threshold further than necessary. For instance, if the max time value in the hypertable is 11, a refresh command such as: ``` CALL refresh_continuous_aggregate(NULL, NULL); ``` would be turned into ``` CALL refresh_continuous_aggregate(NULL, 20); ``` assuming that a bucket starts at 10 and ends at 20 (exclusive). Thus the invalidation threshold would at most move to 20, allowing the threshold to still do its work once time again moves forward and beyond it. Note that one must never process invalidations beyond the invalidation threshold without also moving it, as that would clear that area from invalidations and thus prohibit refreshing that region once the invalidation threshold is moved forward. Therefore, if we do not move the threshold further than a certain point, we cannot refresh beyond it either. An alternative, and perhaps safer, approach would be to always invalidate the region over which the invalidation threshold is moved (i.e., new_threshold - old_threshold). However, that is left for a future change. It would be possible to also cap non-infinite refreshes, e.g., refreshes that end at a higher time value than the max time value in the hypertable. However, when an explicit end is specified, it might be on purpose so optimizing this case is also left for the future. Closes #2333
650 lines
21 KiB
PL/PgSQL
650 lines
21 KiB
PL/PgSQL
-- This file and its contents are licensed under the Timescale License.
|
|
-- Please see the included NOTICE for copyright information and
|
|
-- LICENSE-TIMESCALE for a copy of the license.
|
|
|
|
-- Disable background workers since we are testing manual refresh
|
|
\c :TEST_DBNAME :ROLE_SUPERUSER
|
|
SELECT _timescaledb_internal.stop_background_workers();
|
|
SET ROLE :ROLE_DEFAULT_PERM_USER;
|
|
SET datestyle TO 'ISO, YMD';
|
|
SET timezone TO 'UTC';
|
|
|
|
CREATE TABLE conditions (time bigint NOT NULL, device int, temp float);
|
|
SELECT create_hypertable('conditions', 'time', chunk_time_interval => 10);
|
|
|
|
CREATE TABLE measurements (time int NOT NULL, device int, temp float);
|
|
SELECT create_hypertable('measurements', 'time', chunk_time_interval => 10);
|
|
|
|
CREATE OR REPLACE FUNCTION bigint_now()
|
|
RETURNS bigint LANGUAGE SQL STABLE AS
|
|
$$
|
|
SELECT coalesce(max(time), 0)
|
|
FROM conditions
|
|
$$;
|
|
|
|
CREATE OR REPLACE FUNCTION int_now()
|
|
RETURNS int LANGUAGE SQL STABLE AS
|
|
$$
|
|
SELECT coalesce(max(time), 0)
|
|
FROM measurements
|
|
$$;
|
|
|
|
SELECT set_integer_now_func('conditions', 'bigint_now');
|
|
SELECT set_integer_now_func('measurements', 'int_now');
|
|
|
|
INSERT INTO conditions
|
|
SELECT t, ceil(abs(timestamp_hash(to_timestamp(t)::timestamp))%4)::int,
|
|
abs(timestamp_hash(to_timestamp(t)::timestamp))%40
|
|
FROM generate_series(1, 100, 1) t;
|
|
|
|
INSERT INTO measurements
|
|
SELECT * FROM conditions;
|
|
|
|
-- Show the most recent data
|
|
SELECT * FROM conditions
|
|
ORDER BY time DESC, device
|
|
LIMIT 10;
|
|
|
|
-- Create two continuous aggregates on the same hypertable to test
|
|
-- that invalidations are handled correctly across both of them.
|
|
CREATE MATERIALIZED VIEW cond_10
|
|
WITH (timescaledb.continuous,
|
|
timescaledb.materialized_only=true)
|
|
AS
|
|
SELECT time_bucket(BIGINT '10', time) AS bucket, device, avg(temp) AS avg_temp
|
|
FROM conditions
|
|
GROUP BY 1,2;
|
|
|
|
CREATE MATERIALIZED VIEW cond_20
|
|
WITH (timescaledb.continuous,
|
|
timescaledb.materialized_only=true)
|
|
AS
|
|
SELECT time_bucket(BIGINT '20', time) AS bucket, device, avg(temp) AS avg_temp
|
|
FROM conditions
|
|
GROUP BY 1,2;
|
|
|
|
CREATE MATERIALIZED VIEW measure_10
|
|
WITH (timescaledb.continuous,
|
|
timescaledb.materialized_only=true)
|
|
AS
|
|
SELECT time_bucket(10, time) AS bucket, device, avg(temp) AS avg_temp
|
|
FROM measurements
|
|
GROUP BY 1,2;
|
|
|
|
-- There should be three continuous aggregates, two on one hypertable
|
|
-- and one on the other:
|
|
SELECT mat_hypertable_id, raw_hypertable_id, user_view_name
|
|
FROM _timescaledb_catalog.continuous_agg;
|
|
|
|
-- The continuous aggregates should be empty
|
|
SELECT * FROM cond_10
|
|
ORDER BY 1 DESC, 2;
|
|
|
|
SELECT * FROM cond_20
|
|
ORDER BY 1 DESC, 2;
|
|
|
|
SELECT * FROM measure_10
|
|
ORDER BY 1 DESC, 2;
|
|
|
|
|
|
-- Must refresh to move the invalidation threshold, or no
|
|
-- invalidations will be generated. Initially, there is no threshold
|
|
-- set:
|
|
SELECT * FROM _timescaledb_catalog.continuous_aggs_invalidation_threshold
|
|
ORDER BY 1,2;
|
|
|
|
-- There should be only "infinite" invalidations in the cagg
|
|
-- invalidation log:
|
|
SELECT * FROM _timescaledb_catalog.continuous_aggs_materialization_invalidation_log
|
|
ORDER BY 1,2,3;
|
|
|
|
-- Now refresh up to 50, and the threshold should be updated accordingly:
|
|
CALL refresh_continuous_aggregate('cond_10', 1, 50);
|
|
SELECT * FROM _timescaledb_catalog.continuous_aggs_invalidation_threshold
|
|
ORDER BY 1,2;
|
|
|
|
-- Invalidations should be cleared for the refresh window:
|
|
SELECT * FROM _timescaledb_catalog.continuous_aggs_materialization_invalidation_log
|
|
ORDER BY 1,2,3;
|
|
|
|
-- Refreshing below the threshold does not move it:
|
|
CALL refresh_continuous_aggregate('cond_10', 20, 49);
|
|
SELECT * FROM _timescaledb_catalog.continuous_aggs_invalidation_threshold
|
|
ORDER BY 1,2;
|
|
|
|
-- Nothing changes with invalidations either since the region was
|
|
-- already refreshed and no new invalidations have been generated:
|
|
SELECT * FROM _timescaledb_catalog.continuous_aggs_materialization_invalidation_log
|
|
ORDER BY 1,2,3;
|
|
|
|
-- Refreshing measure_10 moves the threshold only for the other hypertable:
|
|
CALL refresh_continuous_aggregate('measure_10', 1, 30);
|
|
SELECT * FROM _timescaledb_catalog.continuous_aggs_invalidation_threshold
|
|
ORDER BY 1,2;
|
|
SELECT * FROM _timescaledb_catalog.continuous_aggs_materialization_invalidation_log
|
|
ORDER BY 1,2,3;
|
|
|
|
-- Refresh on the second continuous aggregate, cond_20, on the first
|
|
-- hypertable moves the same threshold as when refreshing cond_10:
|
|
CALL refresh_continuous_aggregate('cond_20', 60, 100);
|
|
SELECT * FROM _timescaledb_catalog.continuous_aggs_invalidation_threshold
|
|
ORDER BY 1,2;
|
|
SELECT * FROM _timescaledb_catalog.continuous_aggs_materialization_invalidation_log
|
|
ORDER BY 1,2,3;
|
|
|
|
-- There should be no hypertable invalidations initially:
|
|
SELECT hypertable_id AS hyper_id,
|
|
lowest_modified_value AS start,
|
|
greatest_modified_value AS end
|
|
FROM _timescaledb_catalog.continuous_aggs_hypertable_invalidation_log
|
|
ORDER BY 1,2,3;
|
|
|
|
SELECT materialization_id AS cagg_id,
|
|
lowest_modified_value AS start,
|
|
greatest_modified_value AS end
|
|
FROM _timescaledb_catalog.continuous_aggs_materialization_invalidation_log
|
|
ORDER BY 1,2,3;
|
|
|
|
-- Create invalidations across different ranges. Some of these should
|
|
-- be deleted and others cut in different ways when a refresh is
|
|
-- run. Note that the refresh window is inclusive in the start of the
|
|
-- window but exclusive at the end.
|
|
|
|
-- Entries that should be left unmodified:
|
|
INSERT INTO conditions VALUES (10, 4, 23.7);
|
|
INSERT INTO conditions VALUES (10, 5, 23.8), (19, 3, 23.6);
|
|
INSERT INTO conditions VALUES (60, 3, 23.7), (70, 4, 23.7);
|
|
|
|
-- Should see some invaliations in the hypertable invalidation log:
|
|
SELECT hypertable_id AS hyper_id,
|
|
lowest_modified_value AS start,
|
|
greatest_modified_value AS end
|
|
FROM _timescaledb_catalog.continuous_aggs_hypertable_invalidation_log
|
|
ORDER BY 1,2,3;
|
|
|
|
-- Generate some invalidations for the other hypertable
|
|
INSERT INTO measurements VALUES (20, 4, 23.7);
|
|
INSERT INTO measurements VALUES (30, 5, 23.8), (80, 3, 23.6);
|
|
|
|
-- Should now see invalidations for both hypertables
|
|
SELECT hypertable_id AS hyper_id,
|
|
lowest_modified_value AS start,
|
|
greatest_modified_value AS end
|
|
FROM _timescaledb_catalog.continuous_aggs_hypertable_invalidation_log
|
|
ORDER BY 1,2,3;
|
|
|
|
-- First refresh a window where we don't have any invalidations. This
|
|
-- allows us to see only the copying of the invalidations to the per
|
|
-- cagg log without additional processing.
|
|
CALL refresh_continuous_aggregate('cond_10', 20, 60);
|
|
-- Invalidation threshold remains at 100:
|
|
SELECT * FROM _timescaledb_catalog.continuous_aggs_invalidation_threshold
|
|
ORDER BY 1,2;
|
|
|
|
-- Invalidations should be moved from the hypertable invalidation log
|
|
-- to the continuous aggregate log, but only for the hypertable that
|
|
-- the refreshed aggregate belongs to:
|
|
SELECT hypertable_id AS hyper_id,
|
|
lowest_modified_value AS start,
|
|
greatest_modified_value AS end
|
|
FROM _timescaledb_catalog.continuous_aggs_hypertable_invalidation_log
|
|
ORDER BY 1,2,3;
|
|
|
|
SELECT materialization_id AS cagg_id,
|
|
lowest_modified_value AS start,
|
|
greatest_modified_value AS end
|
|
FROM _timescaledb_catalog.continuous_aggs_materialization_invalidation_log
|
|
ORDER BY 1,2,3;
|
|
|
|
-- Now add more invalidations to test a refresh that overlaps with them.
|
|
-- Entries that should be deleted:
|
|
INSERT INTO conditions VALUES (30, 1, 23.4), (59, 1, 23.4);
|
|
INSERT INTO conditions VALUES (20, 1, 23.4), (30, 1, 23.4);
|
|
-- Entries that should be cut to the right, leaving an invalidation to
|
|
-- the left of the refresh window:
|
|
INSERT INTO conditions VALUES (1, 4, 23.7), (25, 1, 23.4);
|
|
INSERT INTO conditions VALUES (19, 4, 23.7), (59, 1, 23.4);
|
|
-- Entries that should be cut to the left and right, leaving two
|
|
-- invalidation entries on each side of the refresh window:
|
|
INSERT INTO conditions VALUES (2, 2, 23.5), (60, 1, 23.4);
|
|
INSERT INTO conditions VALUES (3, 2, 23.5), (80, 1, 23.4);
|
|
-- Entries that should be cut to the left, leaving an invalidation to
|
|
-- the right of the refresh window:
|
|
INSERT INTO conditions VALUES (60, 3, 23.6), (90, 3, 23.6);
|
|
INSERT INTO conditions VALUES (20, 5, 23.8), (100, 3, 23.6);
|
|
|
|
-- New invalidations in the hypertable invalidation log:
|
|
SELECT hypertable_id AS hyper_id,
|
|
lowest_modified_value AS start,
|
|
greatest_modified_value AS end
|
|
FROM _timescaledb_catalog.continuous_aggs_hypertable_invalidation_log
|
|
ORDER BY 1,2,3;
|
|
|
|
-- But nothing has yet changed in the cagg invalidation log:
|
|
SELECT materialization_id AS cagg_id,
|
|
lowest_modified_value AS start,
|
|
greatest_modified_value AS end
|
|
FROM _timescaledb_catalog.continuous_aggs_materialization_invalidation_log
|
|
ORDER BY 1,2,3;
|
|
|
|
-- Refresh to process invalidations for daily temperature:
|
|
CALL refresh_continuous_aggregate('cond_10', 20, 60);
|
|
|
|
-- Invalidations should be moved from the hypertable invalidation log
|
|
-- to the continuous aggregate log.
|
|
SELECT hypertable_id AS hyper_id,
|
|
lowest_modified_value AS start,
|
|
greatest_modified_value AS end
|
|
FROM _timescaledb_catalog.continuous_aggs_hypertable_invalidation_log
|
|
ORDER BY 1,2,3;
|
|
|
|
-- Only the cond_10 cagg should have its entries cut:
|
|
SELECT materialization_id AS cagg_id,
|
|
lowest_modified_value AS start,
|
|
greatest_modified_value AS end
|
|
FROM _timescaledb_catalog.continuous_aggs_materialization_invalidation_log
|
|
ORDER BY 1,2,3;
|
|
|
|
-- Refresh also cond_20:
|
|
CALL refresh_continuous_aggregate('cond_20', 20, 60);
|
|
|
|
-- The cond_20 cagg should also have its entries cut:
|
|
SELECT materialization_id AS cagg_id,
|
|
lowest_modified_value AS start,
|
|
greatest_modified_value AS end
|
|
FROM _timescaledb_catalog.continuous_aggs_materialization_invalidation_log
|
|
ORDER BY 1,2,3;
|
|
|
|
-- Refresh cond_10 to completely remove an invalidation:
|
|
CALL refresh_continuous_aggregate('cond_10', 1, 20);
|
|
|
|
-- The 1-19 invalidation should be deleted:
|
|
SELECT materialization_id AS cagg_id,
|
|
lowest_modified_value AS start,
|
|
greatest_modified_value AS end
|
|
FROM _timescaledb_catalog.continuous_aggs_materialization_invalidation_log
|
|
ORDER BY 1,2,3;
|
|
|
|
-- Clear everything between 0 and 100 to make way for new
|
|
-- invalidations
|
|
CALL refresh_continuous_aggregate('cond_10', 0, 100);
|
|
|
|
-- Test refreshing with non-overlapping invalidations
|
|
INSERT INTO conditions VALUES (20, 1, 23.4), (25, 1, 23.4);
|
|
INSERT INTO conditions VALUES (30, 1, 23.4), (46, 1, 23.4);
|
|
|
|
CALL refresh_continuous_aggregate('cond_10', 1, 40);
|
|
|
|
SELECT materialization_id AS cagg_id,
|
|
lowest_modified_value AS start,
|
|
greatest_modified_value AS end
|
|
FROM _timescaledb_catalog.continuous_aggs_materialization_invalidation_log
|
|
ORDER BY 1,2,3;
|
|
|
|
-- Refresh whithout cutting (in area where there are no
|
|
-- invalidations). Merging of overlapping entries should still happen:
|
|
INSERT INTO conditions VALUES (15, 1, 23.4), (42, 1, 23.4);
|
|
|
|
CALL refresh_continuous_aggregate('cond_10', 90, 100);
|
|
|
|
SELECT materialization_id AS cagg_id,
|
|
lowest_modified_value AS start,
|
|
greatest_modified_value AS end
|
|
FROM _timescaledb_catalog.continuous_aggs_materialization_invalidation_log
|
|
ORDER BY 1,2,3;
|
|
|
|
-- Test max refresh window
|
|
CALL refresh_continuous_aggregate('cond_10', NULL, NULL);
|
|
|
|
SELECT materialization_id AS cagg_id,
|
|
lowest_modified_value AS start,
|
|
greatest_modified_value AS end
|
|
FROM _timescaledb_catalog.continuous_aggs_materialization_invalidation_log
|
|
ORDER BY 1,2,3;
|
|
|
|
SELECT hypertable_id AS hyper_id,
|
|
lowest_modified_value AS start,
|
|
greatest_modified_value AS end
|
|
FROM _timescaledb_catalog.continuous_aggs_hypertable_invalidation_log
|
|
ORDER BY 1,2,3;
|
|
|
|
-- TRUNCATE the hypertable to invalidate all its continuous aggregates
|
|
TRUNCATE conditions;
|
|
|
|
-- Now empty
|
|
SELECT * FROM conditions;
|
|
|
|
-- Should see an infinite invalidation entry for conditions
|
|
SELECT hypertable_id AS hyper_id,
|
|
lowest_modified_value AS start,
|
|
greatest_modified_value AS end
|
|
FROM _timescaledb_catalog.continuous_aggs_hypertable_invalidation_log
|
|
ORDER BY 1,2,3;
|
|
|
|
-- Aggregates still hold data
|
|
SELECT * FROM cond_10
|
|
ORDER BY 1,2
|
|
LIMIT 5;
|
|
|
|
SELECT * FROM cond_20
|
|
ORDER BY 1,2
|
|
LIMIT 5;
|
|
|
|
CALL refresh_continuous_aggregate('cond_10', NULL, NULL);
|
|
CALL refresh_continuous_aggregate('cond_20', NULL, NULL);
|
|
|
|
-- Both should now be empty after refresh
|
|
SELECT * FROM cond_10
|
|
ORDER BY 1,2;
|
|
|
|
SELECT * FROM cond_20
|
|
ORDER BY 1,2;
|
|
|
|
-- Insert new data again and refresh
|
|
INSERT INTO conditions VALUES
|
|
(1, 1, 23.4), (4, 3, 14.3), (5, 1, 13.6),
|
|
(6, 2, 17.9), (12, 1, 18.3), (19, 3, 28.2),
|
|
(10, 3, 22.3), (11, 2, 34.9), (15, 2, 45.6),
|
|
(21, 1, 15.3), (22, 2, 12.3), (29, 3, 16.3);
|
|
|
|
CALL refresh_continuous_aggregate('cond_10', NULL, NULL);
|
|
CALL refresh_continuous_aggregate('cond_20', NULL, NULL);
|
|
|
|
-- Should now hold data again
|
|
SELECT * FROM cond_10
|
|
ORDER BY 1,2;
|
|
|
|
SELECT * FROM cond_20
|
|
ORDER BY 1,2;
|
|
|
|
-- Truncate one of the aggregates, but first test that we block
|
|
-- TRUNCATE ONLY
|
|
\set ON_ERROR_STOP 0
|
|
TRUNCATE ONLY cond_20;
|
|
\set ON_ERROR_STOP 1
|
|
TRUNCATE cond_20;
|
|
|
|
-- Should now be empty
|
|
SELECT * FROM cond_20
|
|
ORDER BY 1,2;
|
|
|
|
-- Other aggregate is not affected
|
|
SELECT * FROM cond_10
|
|
ORDER BY 1,2;
|
|
|
|
-- Refresh again to bring data back
|
|
CALL refresh_continuous_aggregate('cond_20', NULL, NULL);
|
|
|
|
-- The aggregate should be populated again
|
|
SELECT * FROM cond_20
|
|
ORDER BY 1,2;
|
|
|
|
-------------------------------------------------------
|
|
-- Test corner cases against a minimal bucket aggregate
|
|
-------------------------------------------------------
|
|
|
|
-- First, clear the table and aggregate
|
|
TRUNCATE conditions;
|
|
SELECT * FROM conditions;
|
|
|
|
CALL refresh_continuous_aggregate('cond_10', NULL, NULL);
|
|
|
|
SELECT * FROM cond_10
|
|
ORDER BY 1,2;
|
|
|
|
CREATE MATERIALIZED VIEW cond_1
|
|
WITH (timescaledb.continuous,
|
|
timescaledb.materialized_only=true)
|
|
AS
|
|
SELECT time_bucket(BIGINT '1', time) AS bucket, device, avg(temp) AS avg_temp
|
|
FROM conditions
|
|
GROUP BY 1,2;
|
|
|
|
SELECT mat_hypertable_id AS cond_1_id
|
|
FROM _timescaledb_catalog.continuous_agg
|
|
WHERE user_view_name = 'cond_1' \gset
|
|
|
|
-- Test invalidations with bucket size 1
|
|
INSERT INTO conditions VALUES (0, 1, 1.0);
|
|
|
|
SELECT hypertable_id AS hyper_id,
|
|
lowest_modified_value AS start,
|
|
greatest_modified_value AS end
|
|
FROM _timescaledb_catalog.continuous_aggs_hypertable_invalidation_log
|
|
ORDER BY 1,2,3;
|
|
|
|
-- Refreshing around the bucket should not update the aggregate
|
|
CALL refresh_continuous_aggregate('cond_1', -1, 0);
|
|
SELECT * FROM cond_1
|
|
ORDER BY 1,2;
|
|
CALL refresh_continuous_aggregate('cond_1', 1, 2);
|
|
SELECT * FROM cond_1
|
|
ORDER BY 1,2;
|
|
|
|
-- Refresh only the invalidated bucket
|
|
CALL refresh_continuous_aggregate('cond_1', 0, 1);
|
|
|
|
SELECT materialization_id AS cagg_id,
|
|
lowest_modified_value AS start,
|
|
greatest_modified_value AS end
|
|
FROM _timescaledb_catalog.continuous_aggs_materialization_invalidation_log
|
|
WHERE materialization_id = :cond_1_id
|
|
ORDER BY 1,2,3;
|
|
|
|
SELECT * FROM cond_1
|
|
ORDER BY 1,2;
|
|
|
|
-- Refresh 1 extra bucket on the left
|
|
INSERT INTO conditions VALUES (0, 1, 2.0);
|
|
CALL refresh_continuous_aggregate('cond_1', -1, 1);
|
|
SELECT * FROM cond_1
|
|
ORDER BY 1,2;
|
|
|
|
-- Refresh 1 extra bucket on the right
|
|
INSERT INTO conditions VALUES (0, 1, 3.0);
|
|
CALL refresh_continuous_aggregate('cond_1', 0, 2);
|
|
SELECT * FROM cond_1
|
|
ORDER BY 1,2;
|
|
|
|
-- Refresh 1 extra bucket on each side
|
|
INSERT INTO conditions VALUES (0, 1, 4.0);
|
|
CALL refresh_continuous_aggregate('cond_1', -1, 2);
|
|
SELECT * FROM cond_1
|
|
ORDER BY 1,2;
|
|
|
|
-- Clear to reset aggregate
|
|
TRUNCATE conditions;
|
|
CALL refresh_continuous_aggregate('cond_1', NULL, NULL);
|
|
|
|
-- Test invalidation of size 2
|
|
INSERT INTO conditions VALUES (0, 1, 1.0), (1, 1, 2.0);
|
|
|
|
-- Refresh one bucket at a time
|
|
CALL refresh_continuous_aggregate('cond_1', 0, 1);
|
|
SELECT * FROM cond_1
|
|
ORDER BY 1,2;
|
|
|
|
CALL refresh_continuous_aggregate('cond_1', 1, 2);
|
|
SELECT * FROM cond_1
|
|
ORDER BY 1,2;
|
|
|
|
-- Repeat the same thing but refresh the whole invalidation at once
|
|
TRUNCATE conditions;
|
|
CALL refresh_continuous_aggregate('cond_1', NULL, NULL);
|
|
|
|
INSERT INTO conditions VALUES (0, 1, 1.0), (1, 1, 2.0);
|
|
CALL refresh_continuous_aggregate('cond_1', 0, 2);
|
|
SELECT * FROM cond_1
|
|
ORDER BY 1,2;
|
|
|
|
-- Test invalidation of size 3
|
|
TRUNCATE conditions;
|
|
CALL refresh_continuous_aggregate('cond_1', NULL, NULL);
|
|
|
|
INSERT INTO conditions VALUES (0, 1, 1.0), (1, 1, 2.0), (2, 1, 3.0);
|
|
|
|
-- Invalidation extends beyond the refresh window on both ends
|
|
CALL refresh_continuous_aggregate('cond_1', 1, 2);
|
|
SELECT * FROM cond_1
|
|
ORDER BY 1,2;
|
|
|
|
-- Should leave one invalidation on each side of the refresh window
|
|
SELECT materialization_id AS cagg_id,
|
|
lowest_modified_value AS start,
|
|
greatest_modified_value AS end
|
|
FROM _timescaledb_catalog.continuous_aggs_materialization_invalidation_log
|
|
WHERE materialization_id = :cond_1_id
|
|
ORDER BY 1,2,3;
|
|
|
|
-- Refresh the two remaining invalidations
|
|
CALL refresh_continuous_aggregate('cond_1', 0, 1);
|
|
SELECT * FROM cond_1
|
|
ORDER BY 1,2;
|
|
|
|
CALL refresh_continuous_aggregate('cond_1', 2, 3);
|
|
SELECT * FROM cond_1
|
|
ORDER BY 1,2;
|
|
|
|
-- Clear and repeat but instead refresh the whole range in one go. The
|
|
-- result should be the same as the three partial refreshes. Use
|
|
-- DELETE instead of TRUNCATE to clear this time.
|
|
DELETE FROM conditions;
|
|
CALL refresh_continuous_aggregate('cond_1', NULL, NULL);
|
|
INSERT INTO conditions VALUES (0, 1, 1.0), (1, 1, 2.0), (2, 1, 3.0);
|
|
|
|
CALL refresh_continuous_aggregate('cond_1', 0, 3);
|
|
SELECT * FROM cond_1
|
|
ORDER BY 1,2;
|
|
|
|
----------------------------------------------
|
|
-- Test that invalidation threshold is capped
|
|
----------------------------------------------
|
|
CREATE table threshold_test (time int, value int);
|
|
SELECT create_hypertable('threshold_test', 'time', chunk_time_interval => 4);
|
|
SELECT set_integer_now_func('threshold_test', 'int_now');
|
|
|
|
CREATE MATERIALIZED VIEW thresh_2
|
|
WITH (timescaledb.continuous,
|
|
timescaledb.materialized_only=true)
|
|
AS
|
|
SELECT time_bucket(2, time) AS bucket, max(value) AS max
|
|
FROM threshold_test
|
|
GROUP BY 1;
|
|
|
|
SELECT raw_hypertable_id AS thresh_hyper_id, mat_hypertable_id AS thresh_cagg_id
|
|
FROM _timescaledb_catalog.continuous_agg
|
|
WHERE user_view_name = 'thresh_2' \gset
|
|
|
|
-- There's no invalidation threshold initially
|
|
SELECT * FROM _timescaledb_catalog.continuous_aggs_invalidation_threshold
|
|
WHERE hypertable_id = :thresh_hyper_id
|
|
ORDER BY 1,2;
|
|
|
|
-- Test that threshold is initilized to min value when there's no data
|
|
-- and we specify an infinite end. Note that the min value may differ
|
|
-- depending on time type.
|
|
CALL refresh_continuous_aggregate('thresh_2', 0, NULL);
|
|
|
|
SELECT * FROM _timescaledb_catalog.continuous_aggs_invalidation_threshold
|
|
WHERE hypertable_id = :thresh_hyper_id
|
|
ORDER BY 1,2;
|
|
|
|
INSERT INTO threshold_test
|
|
SELECT v, v FROM generate_series(1, 10) v;
|
|
|
|
CALL refresh_continuous_aggregate('thresh_2', 0, 5);
|
|
|
|
-- Threshold should move to end of refresh window (note that window
|
|
-- expands to end of bucket).
|
|
SELECT * FROM _timescaledb_catalog.continuous_aggs_invalidation_threshold
|
|
WHERE hypertable_id = :thresh_hyper_id
|
|
ORDER BY 1,2;
|
|
|
|
-- Refresh where both the start and end of the window is above the
|
|
-- max data value
|
|
CALL refresh_continuous_aggregate('thresh_2', 14, NULL);
|
|
|
|
SELECT watermark AS thresh_hyper_id_watermark
|
|
FROM _timescaledb_catalog.continuous_aggs_invalidation_threshold
|
|
WHERE hypertable_id = :thresh_hyper_id \gset
|
|
|
|
-- Refresh where we start from the current watermark to infinity
|
|
CALL refresh_continuous_aggregate('thresh_2', :thresh_hyper_id_watermark, NULL);
|
|
|
|
-- Now refresh with max end of the window to test that the
|
|
-- invalidation threshold is capped at the last bucket of data
|
|
CALL refresh_continuous_aggregate('thresh_2', 0, NULL);
|
|
|
|
SELECT * FROM _timescaledb_catalog.continuous_aggs_invalidation_threshold
|
|
WHERE hypertable_id = :thresh_hyper_id
|
|
ORDER BY 1,2;
|
|
|
|
-- Should not have processed invalidations beyond the invalidation
|
|
-- threshold.
|
|
SELECT materialization_id AS cagg_id,
|
|
lowest_modified_value AS start,
|
|
greatest_modified_value AS end
|
|
FROM _timescaledb_catalog.continuous_aggs_materialization_invalidation_log
|
|
WHERE materialization_id = :thresh_cagg_id
|
|
ORDER BY 1,2,3;
|
|
|
|
-- Check that things are properly materialized
|
|
SELECT * FROM thresh_2
|
|
ORDER BY 1;
|
|
|
|
-- Delete the last data
|
|
SELECT show_chunks AS chunk_to_drop
|
|
FROM show_chunks('threshold_test')
|
|
ORDER BY 1 DESC
|
|
LIMIT 1 \gset
|
|
|
|
DELETE FROM threshold_test
|
|
WHERE time > 6;
|
|
|
|
-- The last data in the hypertable is gone
|
|
SELECT time_bucket(2, time) AS bucket, max(value) AS max
|
|
FROM threshold_test
|
|
GROUP BY 1
|
|
ORDER BY 1;
|
|
|
|
-- The aggregate still holds data
|
|
SELECT * FROM thresh_2
|
|
ORDER BY 1;
|
|
|
|
-- Refresh the aggregate to bring it up-to-date
|
|
CALL refresh_continuous_aggregate('thresh_2', 0, NULL);
|
|
|
|
-- Data also gone from the aggregate
|
|
SELECT * FROM thresh_2
|
|
ORDER BY 1;
|
|
|
|
-- The invalidation threshold remains the same
|
|
SELECT * FROM _timescaledb_catalog.continuous_aggs_invalidation_threshold
|
|
WHERE hypertable_id = :thresh_hyper_id
|
|
ORDER BY 1,2;
|
|
|
|
-- Insert new data beyond the invalidation threshold to move it
|
|
-- forward
|
|
INSERT INTO threshold_test
|
|
SELECT v, v FROM generate_series(7, 15) v;
|
|
|
|
CALL refresh_continuous_aggregate('thresh_2', 0, NULL);
|
|
|
|
-- Aggregate now updated to reflect newly aggregated data
|
|
SELECT * FROM thresh_2
|
|
ORDER BY 1;
|
|
|
|
-- The invalidation threshold should have moved forward to the end of
|
|
-- the new data
|
|
SELECT * FROM _timescaledb_catalog.continuous_aggs_invalidation_threshold
|
|
WHERE hypertable_id = :thresh_hyper_id
|
|
ORDER BY 1,2;
|
|
|
|
-- The aggregate remains invalid beyond the invalidation threshold
|
|
SELECT materialization_id AS cagg_id,
|
|
lowest_modified_value AS start,
|
|
greatest_modified_value AS end
|
|
FROM _timescaledb_catalog.continuous_aggs_materialization_invalidation_log
|
|
WHERE materialization_id = :thresh_cagg_id
|
|
ORDER BY 1,2,3;
|