mirror of
https://github.com/timescale/timescaledb.git
synced 2025-05-19 12:13:24 +08:00
e5e94960d0
Changing from using the `compress_using` parameter with a table access method name to use the boolean parameter `hypercore_use_access_method` instead to avoid having to provide a name when using the table access method for compression.
471 lines
20 KiB
Plaintext
471 lines
20 KiB
Plaintext
-- This file and its contents are licensed under the Timescale License.
|
|
-- Please see the included NOTICE for copyright information and
|
|
-- LICENSE-TIMESCALE for a copy of the license.
|
|
\ir include/setup_hypercore.sql
|
|
-- This file and its contents are licensed under the Timescale License.
|
|
-- Please see the included NOTICE for copyright information and
|
|
-- LICENSE-TIMESCALE for a copy of the license.
|
|
\set hypertable readings
|
|
\ir hypercore_helpers.sql
|
|
-- This file and its contents are licensed under the Timescale License.
|
|
-- Please see the included NOTICE for copyright information and
|
|
-- LICENSE-TIMESCALE for a copy of the license.
|
|
-- Function to run an explain analyze with and do replacements on the
|
|
-- emitted plan. This is intended to be used when the structure of the
|
|
-- plan is important, but not the specific chunks scanned nor the
|
|
-- number of heap fetches, rows, loops, etc.
|
|
create function explain_analyze_anonymize(text) returns setof text
|
|
language plpgsql as
|
|
$$
|
|
declare
|
|
ln text;
|
|
begin
|
|
for ln in
|
|
execute format('explain (analyze, costs off, summary off, timing off, decompress_cache_stats) %s', $1)
|
|
loop
|
|
if trim(both from ln) like 'Group Key:%' then
|
|
continue;
|
|
end if;
|
|
ln := regexp_replace(ln, 'Array Cache Hits: \d+', 'Array Cache Hits: N');
|
|
ln := regexp_replace(ln, 'Array Cache Misses: \d+', 'Array Cache Misses: N');
|
|
ln := regexp_replace(ln, 'Array Cache Evictions: \d+', 'Array Cache Evictions: N');
|
|
ln := regexp_replace(ln, 'Heap Fetches: \d+', 'Heap Fetches: N');
|
|
ln := regexp_replace(ln, 'Workers Launched: \d+', 'Workers Launched: N');
|
|
ln := regexp_replace(ln, 'actual rows=\d+ loops=\d+', 'actual rows=N loops=N');
|
|
ln := regexp_replace(ln, '_hyper_\d+_\d+_chunk', '_hyper_I_N_chunk', 1, 0);
|
|
return next ln;
|
|
end loop;
|
|
end;
|
|
$$;
|
|
create function explain_anonymize(text) returns setof text
|
|
language plpgsql as
|
|
$$
|
|
declare
|
|
ln text;
|
|
begin
|
|
for ln in
|
|
execute format('explain (costs off, summary off, timing off) %s', $1)
|
|
loop
|
|
ln := regexp_replace(ln, 'Array Cache Hits: \d+', 'Array Cache Hits: N');
|
|
ln := regexp_replace(ln, 'Array Cache Misses: \d+', 'Array Cache Misses: N');
|
|
ln := regexp_replace(ln, 'Array Cache Evictions: \d+', 'Array Cache Evictions: N');
|
|
ln := regexp_replace(ln, 'Heap Fetches: \d+', 'Heap Fetches: N');
|
|
ln := regexp_replace(ln, 'Workers Launched: \d+', 'Workers Launched: N');
|
|
ln := regexp_replace(ln, 'actual rows=\d+ loops=\d+', 'actual rows=N loops=N');
|
|
ln := regexp_replace(ln, '_hyper_\d+_\d+_chunk', '_hyper_I_N_chunk', 1, 0);
|
|
return next ln;
|
|
end loop;
|
|
end;
|
|
$$;
|
|
create table :hypertable(
|
|
metric_id serial,
|
|
created_at timestamptz not null unique,
|
|
location_id smallint, --segmentby attribute with index
|
|
owner_id bigint, --segmentby attribute without index
|
|
device_id bigint, --non-segmentby attribute
|
|
temp float8,
|
|
humidity float4
|
|
);
|
|
create index hypertable_location_id_idx on :hypertable (location_id);
|
|
create index hypertable_device_id_idx on :hypertable (device_id);
|
|
select create_hypertable(:'hypertable', by_range('created_at'));
|
|
create_hypertable
|
|
-------------------
|
|
(1,t)
|
|
(1 row)
|
|
|
|
-- Disable incremental sort to make tests stable
|
|
set enable_incremental_sort = false;
|
|
select setseed(1);
|
|
setseed
|
|
---------
|
|
|
|
(1 row)
|
|
|
|
-- Insert rows into the tables.
|
|
--
|
|
-- The timestamps for the original rows will have timestamps every 10
|
|
-- seconds. Any other timestamps are inserted as part of the test.
|
|
insert into :hypertable (created_at, location_id, device_id, owner_id, temp, humidity)
|
|
select t, ceil(random()*10), ceil(random()*30), ceil(random() * 5), random()*40, random()*100
|
|
from generate_series('2022-06-01'::timestamptz, '2022-07-01', '5m') t;
|
|
alter table :hypertable set (
|
|
timescaledb.compress,
|
|
timescaledb.compress_orderby = 'created_at',
|
|
timescaledb.compress_segmentby = 'location_id, owner_id'
|
|
);
|
|
-- Get some test chunks as global variables (first and last chunk here)
|
|
select format('%I.%I', chunk_schema, chunk_name)::regclass as chunk1
|
|
from timescaledb_information.chunks
|
|
where format('%I.%I', hypertable_schema, hypertable_name)::regclass = :'hypertable'::regclass
|
|
order by chunk1 asc
|
|
limit 1 \gset
|
|
select format('%I.%I', chunk_schema, chunk_name)::regclass as chunk2
|
|
from timescaledb_information.chunks
|
|
where format('%I.%I', hypertable_schema, hypertable_name)::regclass = :'hypertable'::regclass
|
|
order by chunk2 asc
|
|
limit 1 offset 1 \gset
|
|
-- Set parallel cost to zero to force parallel plans and avoid flaky test.
|
|
set parallel_tuple_cost to 0;
|
|
set parallel_setup_cost to 0;
|
|
-- We need to drop the index to trigger parallel plans. Otherwise they
|
|
-- will use the index.
|
|
drop index hypertable_device_id_idx;
|
|
-- Show parallel plan and count on uncompressed (non-hypercore)
|
|
-- hypertable
|
|
set max_parallel_workers_per_gather=2;
|
|
select explain_anonymize(format($$
|
|
select device_id, count(*) from %s where device_id=1 group by device_id
|
|
$$, :'hypertable'));
|
|
explain_anonymize
|
|
---------------------------------------------------------------
|
|
Finalize GroupAggregate
|
|
-> Gather
|
|
Workers Planned: 2
|
|
-> Parallel Append
|
|
-> Partial GroupAggregate
|
|
-> Parallel Seq Scan on _hyper_I_N_chunk
|
|
Filter: (device_id = 1)
|
|
-> Partial GroupAggregate
|
|
-> Parallel Seq Scan on _hyper_I_N_chunk
|
|
Filter: (device_id = 1)
|
|
-> Partial GroupAggregate
|
|
-> Parallel Seq Scan on _hyper_I_N_chunk
|
|
Filter: (device_id = 1)
|
|
-> Partial GroupAggregate
|
|
-> Parallel Seq Scan on _hyper_I_N_chunk
|
|
Filter: (device_id = 1)
|
|
-> Partial GroupAggregate
|
|
-> Parallel Seq Scan on _hyper_I_N_chunk
|
|
Filter: (device_id = 1)
|
|
-> Partial GroupAggregate
|
|
-> Parallel Seq Scan on _hyper_I_N_chunk
|
|
Filter: (device_id = 1)
|
|
(22 rows)
|
|
|
|
select device_id, count(*) from :hypertable where device_id=1 group by device_id;
|
|
device_id | count
|
|
-----------+-------
|
|
1 | 312
|
|
(1 row)
|
|
|
|
-- Save counts collected over entire hypertable
|
|
select device_id, count(*) into orig from :hypertable group by device_id;
|
|
-- Save counts over a single chunk
|
|
select device_id, count(*) into orig_chunk from :chunk1 group by device_id;
|
|
-----------------------
|
|
-- Enable hypercore --
|
|
-----------------------
|
|
select compress_chunk(show_chunks(:'hypertable'), hypercore_use_access_method => true);
|
|
compress_chunk
|
|
----------------------------------------
|
|
_timescaledb_internal._hyper_1_1_chunk
|
|
_timescaledb_internal._hyper_1_2_chunk
|
|
_timescaledb_internal._hyper_1_3_chunk
|
|
_timescaledb_internal._hyper_1_4_chunk
|
|
_timescaledb_internal._hyper_1_5_chunk
|
|
_timescaledb_internal._hyper_1_6_chunk
|
|
(6 rows)
|
|
|
|
-- Show count without parallel plan and without ColumnarScan
|
|
set timescaledb.enable_columnarscan=false;
|
|
set max_parallel_workers_per_gather=0;
|
|
select explain_anonymize(format($$
|
|
select device_id, count(*) from %s where device_id=1 group by device_id
|
|
$$, :'hypertable'));
|
|
explain_anonymize
|
|
------------------------------------------------
|
|
Finalize GroupAggregate
|
|
-> Append
|
|
-> Partial GroupAggregate
|
|
-> Seq Scan on _hyper_I_N_chunk
|
|
Filter: (device_id = 1)
|
|
-> Partial GroupAggregate
|
|
-> Seq Scan on _hyper_I_N_chunk
|
|
Filter: (device_id = 1)
|
|
-> Partial GroupAggregate
|
|
-> Seq Scan on _hyper_I_N_chunk
|
|
Filter: (device_id = 1)
|
|
-> Partial GroupAggregate
|
|
-> Seq Scan on _hyper_I_N_chunk
|
|
Filter: (device_id = 1)
|
|
-> Partial GroupAggregate
|
|
-> Seq Scan on _hyper_I_N_chunk
|
|
Filter: (device_id = 1)
|
|
-> Partial GroupAggregate
|
|
-> Seq Scan on _hyper_I_N_chunk
|
|
Filter: (device_id = 1)
|
|
(20 rows)
|
|
|
|
select device_id, count(*) from :hypertable where device_id=1 group by device_id;
|
|
device_id | count
|
|
-----------+-------
|
|
1 | 312
|
|
(1 row)
|
|
|
|
-- Filter on segmentby column
|
|
select explain_anonymize(format($$
|
|
select owner_id, count(*) from %s where owner_id=1 group by owner_id
|
|
$$, :'hypertable'));
|
|
explain_anonymize
|
|
------------------------------------------------
|
|
Finalize GroupAggregate
|
|
-> Append
|
|
-> Partial GroupAggregate
|
|
-> Seq Scan on _hyper_I_N_chunk
|
|
Filter: (owner_id = 1)
|
|
-> Partial GroupAggregate
|
|
-> Seq Scan on _hyper_I_N_chunk
|
|
Filter: (owner_id = 1)
|
|
-> Partial GroupAggregate
|
|
-> Seq Scan on _hyper_I_N_chunk
|
|
Filter: (owner_id = 1)
|
|
-> Partial GroupAggregate
|
|
-> Seq Scan on _hyper_I_N_chunk
|
|
Filter: (owner_id = 1)
|
|
-> Partial GroupAggregate
|
|
-> Seq Scan on _hyper_I_N_chunk
|
|
Filter: (owner_id = 1)
|
|
-> Partial GroupAggregate
|
|
-> Seq Scan on _hyper_I_N_chunk
|
|
Filter: (owner_id = 1)
|
|
(20 rows)
|
|
|
|
select owner_id, count(*) from :hypertable where owner_id=1 group by owner_id;
|
|
owner_id | count
|
|
----------+-------
|
|
1 | 1729
|
|
(1 row)
|
|
|
|
-- Enable parallel on SeqScan and check for same result
|
|
set max_parallel_workers_per_gather=2;
|
|
select explain_anonymize(format($$
|
|
select device_id, count(*) from %s where device_id=1 group by device_id
|
|
$$, :'hypertable'));
|
|
explain_anonymize
|
|
---------------------------------------------------------------
|
|
Finalize GroupAggregate
|
|
-> Gather
|
|
Workers Planned: 2
|
|
-> Parallel Append
|
|
-> Partial GroupAggregate
|
|
-> Parallel Seq Scan on _hyper_I_N_chunk
|
|
Filter: (device_id = 1)
|
|
-> Partial GroupAggregate
|
|
-> Parallel Seq Scan on _hyper_I_N_chunk
|
|
Filter: (device_id = 1)
|
|
-> Partial GroupAggregate
|
|
-> Parallel Seq Scan on _hyper_I_N_chunk
|
|
Filter: (device_id = 1)
|
|
-> Partial GroupAggregate
|
|
-> Parallel Seq Scan on _hyper_I_N_chunk
|
|
Filter: (device_id = 1)
|
|
-> Partial GroupAggregate
|
|
-> Parallel Seq Scan on _hyper_I_N_chunk
|
|
Filter: (device_id = 1)
|
|
-> Partial GroupAggregate
|
|
-> Parallel Seq Scan on _hyper_I_N_chunk
|
|
Filter: (device_id = 1)
|
|
(22 rows)
|
|
|
|
select device_id, count(*) from :hypertable where device_id=1 group by device_id;
|
|
device_id | count
|
|
-----------+-------
|
|
1 | 312
|
|
(1 row)
|
|
|
|
select explain_anonymize(format($$
|
|
select owner_id, count(*) from %s where owner_id=1 group by owner_id
|
|
$$, :'hypertable'));
|
|
explain_anonymize
|
|
---------------------------------------------------------------
|
|
Finalize GroupAggregate
|
|
-> Gather
|
|
Workers Planned: 2
|
|
-> Parallel Append
|
|
-> Partial GroupAggregate
|
|
-> Parallel Seq Scan on _hyper_I_N_chunk
|
|
Filter: (owner_id = 1)
|
|
-> Partial GroupAggregate
|
|
-> Parallel Seq Scan on _hyper_I_N_chunk
|
|
Filter: (owner_id = 1)
|
|
-> Partial GroupAggregate
|
|
-> Parallel Seq Scan on _hyper_I_N_chunk
|
|
Filter: (owner_id = 1)
|
|
-> Partial GroupAggregate
|
|
-> Parallel Seq Scan on _hyper_I_N_chunk
|
|
Filter: (owner_id = 1)
|
|
-> Partial GroupAggregate
|
|
-> Parallel Seq Scan on _hyper_I_N_chunk
|
|
Filter: (owner_id = 1)
|
|
-> Partial GroupAggregate
|
|
-> Parallel Seq Scan on _hyper_I_N_chunk
|
|
Filter: (owner_id = 1)
|
|
(22 rows)
|
|
|
|
select owner_id, count(*) from :hypertable where owner_id=1 group by owner_id;
|
|
owner_id | count
|
|
----------+-------
|
|
1 | 1729
|
|
(1 row)
|
|
|
|
-- Enable ColumnarScan and check for same result
|
|
set timescaledb.enable_columnarscan=true;
|
|
select explain_anonymize(format($$
|
|
select device_id, count(*) from %s where device_id=1 group by device_id
|
|
$$, :'hypertable'));
|
|
explain_anonymize
|
|
---------------------------------------------------------------------------------
|
|
Finalize GroupAggregate
|
|
-> Gather
|
|
Workers Planned: 2
|
|
-> Parallel Append
|
|
-> Partial GroupAggregate
|
|
-> Parallel Custom Scan (ColumnarScan) on _hyper_I_N_chunk
|
|
Vectorized Filter: (device_id = 1)
|
|
-> Partial GroupAggregate
|
|
-> Parallel Custom Scan (ColumnarScan) on _hyper_I_N_chunk
|
|
Vectorized Filter: (device_id = 1)
|
|
-> Partial GroupAggregate
|
|
-> Parallel Custom Scan (ColumnarScan) on _hyper_I_N_chunk
|
|
Vectorized Filter: (device_id = 1)
|
|
-> Partial GroupAggregate
|
|
-> Parallel Custom Scan (ColumnarScan) on _hyper_I_N_chunk
|
|
Vectorized Filter: (device_id = 1)
|
|
-> Partial GroupAggregate
|
|
-> Parallel Custom Scan (ColumnarScan) on _hyper_I_N_chunk
|
|
Vectorized Filter: (device_id = 1)
|
|
-> Partial GroupAggregate
|
|
-> Parallel Custom Scan (ColumnarScan) on _hyper_I_N_chunk
|
|
Vectorized Filter: (device_id = 1)
|
|
(22 rows)
|
|
|
|
select device_id, count(*) from :hypertable where device_id=1 group by device_id;
|
|
device_id | count
|
|
-----------+-------
|
|
1 | 312
|
|
(1 row)
|
|
|
|
-- Filter on segmentby column (pushing down scankeys in parallel mode)
|
|
select explain_anonymize(format($$
|
|
select owner_id, count(*) from %s where owner_id=1 group by owner_id
|
|
$$, :'hypertable'));
|
|
explain_anonymize
|
|
---------------------------------------------------------------------------------
|
|
Finalize GroupAggregate
|
|
-> Gather
|
|
Workers Planned: 2
|
|
-> Parallel Append
|
|
-> Partial GroupAggregate
|
|
-> Parallel Custom Scan (ColumnarScan) on _hyper_I_N_chunk
|
|
Scankey: (owner_id = 1)
|
|
-> Partial GroupAggregate
|
|
-> Parallel Custom Scan (ColumnarScan) on _hyper_I_N_chunk
|
|
Scankey: (owner_id = 1)
|
|
-> Partial GroupAggregate
|
|
-> Parallel Custom Scan (ColumnarScan) on _hyper_I_N_chunk
|
|
Scankey: (owner_id = 1)
|
|
-> Partial GroupAggregate
|
|
-> Parallel Custom Scan (ColumnarScan) on _hyper_I_N_chunk
|
|
Scankey: (owner_id = 1)
|
|
-> Partial GroupAggregate
|
|
-> Parallel Custom Scan (ColumnarScan) on _hyper_I_N_chunk
|
|
Scankey: (owner_id = 1)
|
|
-> Partial GroupAggregate
|
|
-> Parallel Custom Scan (ColumnarScan) on _hyper_I_N_chunk
|
|
Scankey: (owner_id = 1)
|
|
(22 rows)
|
|
|
|
select owner_id, count(*) from :hypertable where owner_id=1 group by owner_id;
|
|
owner_id | count
|
|
----------+-------
|
|
1 | 1729
|
|
(1 row)
|
|
|
|
-- Parallel plan with hypercore on single chunk
|
|
select explain_anonymize(format($$
|
|
select device_id, count(*) from %s where device_id=1 group by device_id
|
|
$$, :'hypertable'));
|
|
explain_anonymize
|
|
---------------------------------------------------------------------------------
|
|
Finalize GroupAggregate
|
|
-> Gather
|
|
Workers Planned: 2
|
|
-> Parallel Append
|
|
-> Partial GroupAggregate
|
|
-> Parallel Custom Scan (ColumnarScan) on _hyper_I_N_chunk
|
|
Vectorized Filter: (device_id = 1)
|
|
-> Partial GroupAggregate
|
|
-> Parallel Custom Scan (ColumnarScan) on _hyper_I_N_chunk
|
|
Vectorized Filter: (device_id = 1)
|
|
-> Partial GroupAggregate
|
|
-> Parallel Custom Scan (ColumnarScan) on _hyper_I_N_chunk
|
|
Vectorized Filter: (device_id = 1)
|
|
-> Partial GroupAggregate
|
|
-> Parallel Custom Scan (ColumnarScan) on _hyper_I_N_chunk
|
|
Vectorized Filter: (device_id = 1)
|
|
-> Partial GroupAggregate
|
|
-> Parallel Custom Scan (ColumnarScan) on _hyper_I_N_chunk
|
|
Vectorized Filter: (device_id = 1)
|
|
-> Partial GroupAggregate
|
|
-> Parallel Custom Scan (ColumnarScan) on _hyper_I_N_chunk
|
|
Vectorized Filter: (device_id = 1)
|
|
(22 rows)
|
|
|
|
select device_id, count(*) from :chunk1 where device_id=1 group by device_id;
|
|
device_id | count
|
|
-----------+-------
|
|
1 | 3
|
|
(1 row)
|
|
|
|
select explain_anonymize(format($$
|
|
select owner_id, count(*) from %s where owner_id=1 group by owner_id
|
|
$$, :'hypertable'));
|
|
explain_anonymize
|
|
---------------------------------------------------------------------------------
|
|
Finalize GroupAggregate
|
|
-> Gather
|
|
Workers Planned: 2
|
|
-> Parallel Append
|
|
-> Partial GroupAggregate
|
|
-> Parallel Custom Scan (ColumnarScan) on _hyper_I_N_chunk
|
|
Scankey: (owner_id = 1)
|
|
-> Partial GroupAggregate
|
|
-> Parallel Custom Scan (ColumnarScan) on _hyper_I_N_chunk
|
|
Scankey: (owner_id = 1)
|
|
-> Partial GroupAggregate
|
|
-> Parallel Custom Scan (ColumnarScan) on _hyper_I_N_chunk
|
|
Scankey: (owner_id = 1)
|
|
-> Partial GroupAggregate
|
|
-> Parallel Custom Scan (ColumnarScan) on _hyper_I_N_chunk
|
|
Scankey: (owner_id = 1)
|
|
-> Partial GroupAggregate
|
|
-> Parallel Custom Scan (ColumnarScan) on _hyper_I_N_chunk
|
|
Scankey: (owner_id = 1)
|
|
-> Partial GroupAggregate
|
|
-> Parallel Custom Scan (ColumnarScan) on _hyper_I_N_chunk
|
|
Scankey: (owner_id = 1)
|
|
(22 rows)
|
|
|
|
select owner_id, count(*) from :chunk1 where owner_id=1 group by owner_id;
|
|
owner_id | count
|
|
----------+-------
|
|
1 | 44
|
|
(1 row)
|
|
|
|
-- Compare hypercore per-location counts with original counts without
|
|
-- hypercore
|
|
select device_id, count(*) into comp from :hypertable group by device_id;
|
|
select * from orig join comp using (device_id) where orig.count != comp.count;
|
|
device_id | count | count
|
|
-----------+-------+-------
|
|
(0 rows)
|
|
|
|
-- Compare counts on single chunk
|
|
select device_id, count(*) into comp_chunk from :chunk1 group by device_id;
|
|
select * from orig_chunk join comp_chunk using (device_id) where orig_chunk.count != comp_chunk.count;
|
|
device_id | count | count
|
|
-----------+-------+-------
|
|
(0 rows)
|
|
|