mirror of
https://github.com/timescale/timescaledb.git
synced 2025-05-25 15:50:27 +08:00
706a3c0e50
Remove 'client_min_messages = LOG' where not needed, and add the 'LOG: statement' output otherwise.
298 lines
12 KiB
MySQL
298 lines
12 KiB
MySQL
-- This file and its contents are licensed under the Timescale License.
|
|
-- Please see the included NOTICE for copyright information and
|
|
-- LICENSE-TIMESCALE for a copy of the license.
|
|
|
|
\set TEST_BASE_NAME cagg_query
|
|
SELECT
|
|
format('%s/results/%s_results_view.out', :'TEST_OUTPUT_DIR', :'TEST_BASE_NAME') as "TEST_RESULTS_VIEW",
|
|
format('%s/results/%s_results_view_hashagg.out', :'TEST_OUTPUT_DIR', :'TEST_BASE_NAME') as "TEST_RESULTS_VIEW_HASHAGG",
|
|
format('%s/results/%s_results_table.out', :'TEST_OUTPUT_DIR', :'TEST_BASE_NAME') as "TEST_RESULTS_TABLE"
|
|
\gset
|
|
SELECT format('\! diff %s %s', :'TEST_RESULTS_VIEW', :'TEST_RESULTS_TABLE') as "DIFF_CMD",
|
|
format('\! diff %s %s', :'TEST_RESULTS_VIEW_HASHAGG', :'TEST_RESULTS_TABLE') as "DIFF_CMD2"
|
|
\gset
|
|
|
|
|
|
\set EXPLAIN 'EXPLAIN (VERBOSE, COSTS OFF)'
|
|
|
|
SET client_min_messages TO NOTICE;
|
|
|
|
CREATE TABLE conditions (
|
|
timec TIMESTAMPTZ NOT NULL,
|
|
location TEXT NOT NULL,
|
|
temperature DOUBLE PRECISION NULL,
|
|
humidity DOUBLE PRECISION NULL
|
|
);
|
|
|
|
select table_name from create_hypertable( 'conditions', 'timec');
|
|
|
|
insert into conditions values ( '2018-01-01 09:20:00-08', 'SFO', 55, 45);
|
|
insert into conditions values ( '2018-01-02 09:30:00-08', 'por', 100, 100);
|
|
insert into conditions values ( '2018-01-02 09:20:00-08', 'SFO', 65, 45);
|
|
insert into conditions values ( '2018-01-02 09:10:00-08', 'NYC', 65, 45);
|
|
insert into conditions values ( '2018-11-01 09:20:00-08', 'NYC', 45, 30);
|
|
insert into conditions values ( '2018-11-01 10:40:00-08', 'NYC', 55, 35);
|
|
insert into conditions values ( '2018-11-01 11:50:00-08', 'NYC', 65, 40);
|
|
insert into conditions values ( '2018-11-01 12:10:00-08', 'NYC', 75, 45);
|
|
insert into conditions values ( '2018-11-01 13:10:00-08', 'NYC', 85, 50);
|
|
insert into conditions values ( '2018-11-02 09:20:00-08', 'NYC', 10, 10);
|
|
insert into conditions values ( '2018-11-02 10:30:00-08', 'NYC', 20, 15);
|
|
insert into conditions values ( '2018-11-02 11:40:00-08', 'NYC', null, null);
|
|
insert into conditions values ( '2018-11-03 09:50:00-08', 'NYC', null, null);
|
|
|
|
create table location_tab( locid integer, locname text );
|
|
insert into location_tab values( 1, 'SFO');
|
|
insert into location_tab values( 2, 'NYC');
|
|
insert into location_tab values( 3, 'por');
|
|
|
|
create materialized view mat_m1( location, timec, minl, sumt , sumh)
|
|
WITH (timescaledb.continuous)
|
|
as
|
|
select location, time_bucket('1day', timec), min(location), sum(temperature),sum(humidity)
|
|
from conditions
|
|
group by time_bucket('1day', timec), location WITH NO DATA;
|
|
|
|
--compute time_bucketted max+bucket_width for the materialized view
|
|
SELECT time_bucket('1day' , q.timeval+ '1day'::interval)
|
|
FROM ( select max(timec)as timeval from conditions ) as q;
|
|
CALL refresh_continuous_aggregate('mat_m1', NULL, NULL);
|
|
|
|
--test first/last
|
|
create materialized view mat_m2(location, timec, firsth, lasth, maxtemp, mintemp)
|
|
WITH (timescaledb.continuous)
|
|
as
|
|
select location, time_bucket('1day', timec), first(humidity, timec), last(humidity, timec), max(temperature), min(temperature)
|
|
from conditions
|
|
group by time_bucket('1day', timec), location WITH NO DATA;
|
|
--time that refresh assumes as now() for repeatability
|
|
SELECT time_bucket('1day' , q.timeval+ '1day'::interval)
|
|
FROM ( select max(timec)as timeval from conditions ) as q;
|
|
CALL refresh_continuous_aggregate('mat_m2', NULL, NULL);
|
|
|
|
--normal view --
|
|
create or replace view regview( location, timec, minl, sumt , sumh)
|
|
as
|
|
select location, time_bucket('1day', timec), min(location), sum(temperature),sum(humidity)
|
|
from conditions
|
|
group by location, time_bucket('1day', timec);
|
|
|
|
set enable_hashagg = false;
|
|
|
|
-- NO pushdown cases ---
|
|
--when we have addl. attrs in order by that are not in the
|
|
-- group by, we will still need a sort
|
|
:EXPLAIN
|
|
select * from mat_m1 order by sumh, sumt, minl, timec ;
|
|
:EXPLAIN
|
|
select * from regview order by timec desc;
|
|
|
|
-- PUSHDOWN cases --
|
|
-- all group by elts in order by , reorder group by elts to match
|
|
-- group by order
|
|
-- This should prevent an additional sort after GroupAggregate
|
|
:EXPLAIN
|
|
select * from mat_m1 order by timec desc, location;
|
|
|
|
:EXPLAIN
|
|
select * from mat_m1 order by location, timec desc;
|
|
|
|
:EXPLAIN
|
|
select * from mat_m1 order by location, timec asc;
|
|
:EXPLAIN
|
|
select * from mat_m1 where timec > '2018-10-01' order by timec desc;
|
|
-- outer sort is used by mat_m1 for grouping. But doesn't avoid a sort after the join ---
|
|
:EXPLAIN
|
|
select l.locid, mat_m1.* from mat_m1 , location_tab l where timec > '2018-10-01' and l.locname = mat_m1.location order by timec desc;
|
|
|
|
:EXPLAIN
|
|
select * from mat_m2 where timec > '2018-10-01' order by timec desc;
|
|
|
|
:EXPLAIN
|
|
select * from (select * from mat_m2 where timec > '2018-10-01' order by timec desc ) as q limit 1;
|
|
|
|
:EXPLAIN
|
|
select * from (select * from mat_m2 where timec > '2018-10-01' order by timec desc , location asc nulls first) as q limit 1;
|
|
|
|
--plans with CTE
|
|
:EXPLAIN
|
|
with m1 as (
|
|
Select * from mat_m2 where timec > '2018-10-01' order by timec desc )
|
|
select * from m1;
|
|
|
|
-- should reorder mat_m1 group by only based on mat_m1 order-by
|
|
:EXPLAIN
|
|
select * from mat_m1, mat_m2 where mat_m1.timec > '2018-10-01' and mat_m1.timec = mat_m2.timec order by mat_m1.timec desc;
|
|
--should reorder only for mat_m1.
|
|
:EXPLAIN
|
|
select * from mat_m1, regview where mat_m1.timec > '2018-10-01' and mat_m1.timec = regview.timec order by mat_m1.timec desc;
|
|
|
|
select l.locid, mat_m1.* from mat_m1 , location_tab l where timec > '2018-10-01' and l.locname = mat_m1.location order by timec desc;
|
|
|
|
\set ECHO none
|
|
SET client_min_messages TO error;
|
|
\o :TEST_RESULTS_VIEW
|
|
select * from mat_m1 order by timec desc, location;
|
|
select * from mat_m1 order by location, timec desc;
|
|
select * from mat_m1 order by location, timec asc;
|
|
select * from mat_m1 where timec > '2018-10-01' order by timec desc;
|
|
select * from mat_m2 where timec > '2018-10-01' order by timec desc;
|
|
\o
|
|
RESET client_min_messages;
|
|
\set ECHO all
|
|
|
|
---- Run the same queries with hash agg enabled now
|
|
set enable_hashagg = true;
|
|
\set ECHO none
|
|
SET client_min_messages TO error;
|
|
\o :TEST_RESULTS_VIEW_HASHAGG
|
|
select * from mat_m1 order by timec desc, location;
|
|
select * from mat_m1 order by location, timec desc;
|
|
select * from mat_m1 order by location, timec asc;
|
|
select * from mat_m1 where timec > '2018-10-01' order by timec desc;
|
|
select * from mat_m2 where timec > '2018-10-01' order by timec desc;
|
|
\o
|
|
RESET client_min_messages;
|
|
\set ECHO all
|
|
|
|
--- Run the queries directly on the table now
|
|
set enable_hashagg = true;
|
|
\set ECHO none
|
|
SET client_min_messages TO error;
|
|
\o :TEST_RESULTS_TABLE
|
|
SELECT location, time_bucket('1day', timec) as timec, min(location) as minl, sum(temperature) as sumt, sum(humidity) as sumh from conditions group by time_bucket('1day', timec) , location
|
|
order by timec desc, location;
|
|
SELECT location, time_bucket('1day', timec) as timec, min(location) as minl, sum(temperature) as sumt, sum(humidity) as sumh from conditions group by time_bucket('1day', timec) , location
|
|
order by location, timec desc;
|
|
SELECT location, time_bucket('1day', timec) as timec, min(location) as minl, sum(temperature) as sumt, sum(humidity) as sumh from conditions group by time_bucket('1day', timec) , location
|
|
order by location, timec asc;
|
|
select * from (SELECT location, time_bucket('1day', timec) as timec, min(location) as minl, sum(temperature) as sumt, sum(humidity) as sumh from conditions
|
|
group by time_bucket('1day', timec) , location ) as q
|
|
where timec > '2018-10-01' order by timec desc;
|
|
--comparison for mat_m2 queries
|
|
select * from (
|
|
select location, time_bucket('1day', timec) as timec, first(humidity, timec) firsth, last(humidity, timec) lasth, max(temperature) maxtemp, min(temperature) mintemp
|
|
from conditions
|
|
group by time_bucket('1day', timec), location) as q
|
|
where timec > '2018-10-01' order by timec desc limit 10;
|
|
\o
|
|
RESET client_min_messages;
|
|
\set ECHO all
|
|
|
|
-- diff results view select and table select
|
|
:DIFF_CMD
|
|
:DIFF_CMD2
|
|
|
|
--check if the guc works , reordering will not work
|
|
set timescaledb.enable_cagg_reorder_groupby = false;
|
|
set enable_hashagg = false;
|
|
:EXPLAIN
|
|
select * from mat_m1 order by timec desc, location;
|
|
|
|
-----------------------------------------------------------------------
|
|
-- Test the cagg_watermark function. The watermark gives the point
|
|
-- where to UNION raw and materialized data in real-time
|
|
-- aggregation. Specifically, test that the watermark caching works as
|
|
-- expected.
|
|
-----------------------------------------------------------------------
|
|
|
|
-- Insert some more data so that there is something to UNION in
|
|
-- real-time aggregation.
|
|
|
|
insert into conditions values ( '2018-12-02 20:10:00-08', 'SFO', 55, 45);
|
|
insert into conditions values ( '2018-12-02 21:20:00-08', 'SFO', 65, 45);
|
|
insert into conditions values ( '2018-12-02 20:30:00-08', 'NYC', 65, 45);
|
|
insert into conditions values ( '2018-12-02 21:50:00-08', 'NYC', 45, 30);
|
|
|
|
-- Test join of two caggs. Joining two caggs will force the cache to
|
|
-- reset every time the watermark function is invoked on a different
|
|
-- cagg in the same query.
|
|
SELECT mat_hypertable_id AS mat_id,
|
|
raw_hypertable_id AS raw_id,
|
|
schema_name AS mat_schema,
|
|
table_name AS mat_name,
|
|
format('%I.%I', schema_name, table_name) AS mat_table
|
|
FROM _timescaledb_catalog.continuous_agg ca, _timescaledb_catalog.hypertable h
|
|
WHERE user_view_name='mat_m1'
|
|
AND h.id = ca.mat_hypertable_id \gset
|
|
|
|
BEGIN;
|
|
|
|
-- Query without join
|
|
SELECT m1.location, m1.timec, sumt, sumh
|
|
FROM mat_m1 m1
|
|
ORDER BY m1.location COLLATE "C", m1.timec DESC
|
|
LIMIT 10;
|
|
|
|
-- Query that joins two caggs. This should force the watermark cache
|
|
-- to reset when the materialized hypertable ID changes. A hash join
|
|
-- could potentially read all values from mat_m1 then all values from
|
|
-- mat_m2. This would be the optimal situation for cagg_watermark
|
|
-- caching. We want to avoid it in tests to see that caching doesn't
|
|
-- do anything wrong in worse situations (e.g., a nested loop join).
|
|
SET enable_hashjoin=false;
|
|
|
|
SELECT m1.location, m1.timec, sumt, sumh, firsth, lasth, maxtemp, mintemp
|
|
FROM mat_m1 m1 RIGHT JOIN mat_m2 m2
|
|
ON (m1.location = m2.location
|
|
AND m1.timec = m2.timec)
|
|
ORDER BY m1.location COLLATE "C", m1.timec DESC
|
|
LIMIT 10;
|
|
|
|
-- Show the current watermark
|
|
SELECT _timescaledb_internal.to_timestamp(_timescaledb_internal.cagg_watermark(:mat_id));
|
|
|
|
-- The watermark should, in this case, be the same as the invalidation
|
|
-- threshold
|
|
SELECT _timescaledb_internal.to_timestamp(watermark)
|
|
FROM _timescaledb_catalog.continuous_aggs_invalidation_threshold
|
|
WHERE hypertable_id = :raw_id;
|
|
|
|
-- The watermark is the end of materialization (end of last bucket)
|
|
-- while the MAX is the start of the last bucket
|
|
SELECT max(timec) FROM :mat_table;
|
|
|
|
-- Drop the most recent chunk
|
|
SELECT chunk_name, range_start, range_end
|
|
FROM timescaledb_information.chunks
|
|
WHERE hypertable_name = :'mat_name';
|
|
|
|
SELECT drop_chunks('mat_m1', newer_than=>'2018-01-01'::timestamptz);
|
|
|
|
SELECT chunk_name, range_start, range_end
|
|
FROM timescaledb_information.chunks
|
|
WHERE hypertable_name = :'mat_name';
|
|
|
|
-- The watermark should be updated to reflect the dropped data (i.e.,
|
|
-- the cache should be reset)
|
|
SELECT _timescaledb_internal.to_timestamp(_timescaledb_internal.cagg_watermark(:mat_id));
|
|
|
|
-- Since we removed the last chunk, the invalidation threshold doesn't
|
|
-- move back, while the watermark does.
|
|
SELECT _timescaledb_internal.to_timestamp(watermark)
|
|
FROM _timescaledb_catalog.continuous_aggs_invalidation_threshold
|
|
WHERE hypertable_id = :raw_id;
|
|
|
|
-- Compare the new watermark to the MAX time in the table
|
|
SELECT max(timec) FROM :mat_table;
|
|
|
|
-- Try a subtransaction
|
|
SAVEPOINT clear_cagg;
|
|
|
|
SELECT m1.location, m1.timec, sumt, sumh, firsth, lasth, maxtemp, mintemp
|
|
FROM mat_m1 m1 RIGHT JOIN mat_m2 m2
|
|
ON (m1.location = m2.location
|
|
AND m1.timec = m2.timec)
|
|
ORDER BY m1.location COLLATE "C", m1.timec DESC
|
|
LIMIT 10;
|
|
|
|
ALTER MATERIALIZED VIEW mat_m1 SET (timescaledb.materialized_only=true);
|
|
|
|
SELECT m1.location, m1.timec, sumt, sumh, firsth, lasth, maxtemp, mintemp
|
|
FROM mat_m1 m1 RIGHT JOIN mat_m2 m2
|
|
ON (m1.location = m2.location
|
|
AND m1.timec = m2.timec)
|
|
ORDER BY m1.location COLLATE "C" NULLS LAST, m1.timec DESC NULLS LAST, firsth NULLS LAST
|
|
LIMIT 10;
|
|
|
|
ROLLBACK;
|