mirror of
https://github.com/timescale/timescaledb.git
synced 2025-05-16 02:23:49 +08:00
e5e94960d0
Changing from using the `compress_using` parameter with a table access method name to use the boolean parameter `hypercore_use_access_method` instead to avoid having to provide a name when using the table access method for compression.
383 lines
15 KiB
Plaintext
383 lines
15 KiB
Plaintext
-- This file and its contents are licensed under the Timescale License.
|
|
-- Please see the included NOTICE for copyright information and
|
|
-- LICENSE-TIMESCALE for a copy of the license.
|
|
\ir include/setup_hypercore.sql
|
|
-- This file and its contents are licensed under the Timescale License.
|
|
-- Please see the included NOTICE for copyright information and
|
|
-- LICENSE-TIMESCALE for a copy of the license.
|
|
\set hypertable readings
|
|
\ir hypercore_helpers.sql
|
|
-- This file and its contents are licensed under the Timescale License.
|
|
-- Please see the included NOTICE for copyright information and
|
|
-- LICENSE-TIMESCALE for a copy of the license.
|
|
-- Function to run an explain analyze with and do replacements on the
|
|
-- emitted plan. This is intended to be used when the structure of the
|
|
-- plan is important, but not the specific chunks scanned nor the
|
|
-- number of heap fetches, rows, loops, etc.
|
|
create function explain_analyze_anonymize(text) returns setof text
|
|
language plpgsql as
|
|
$$
|
|
declare
|
|
ln text;
|
|
begin
|
|
for ln in
|
|
execute format('explain (analyze, costs off, summary off, timing off, decompress_cache_stats) %s', $1)
|
|
loop
|
|
if trim(both from ln) like 'Group Key:%' then
|
|
continue;
|
|
end if;
|
|
ln := regexp_replace(ln, 'Array Cache Hits: \d+', 'Array Cache Hits: N');
|
|
ln := regexp_replace(ln, 'Array Cache Misses: \d+', 'Array Cache Misses: N');
|
|
ln := regexp_replace(ln, 'Array Cache Evictions: \d+', 'Array Cache Evictions: N');
|
|
ln := regexp_replace(ln, 'Heap Fetches: \d+', 'Heap Fetches: N');
|
|
ln := regexp_replace(ln, 'Workers Launched: \d+', 'Workers Launched: N');
|
|
ln := regexp_replace(ln, 'actual rows=\d+ loops=\d+', 'actual rows=N loops=N');
|
|
ln := regexp_replace(ln, '_hyper_\d+_\d+_chunk', '_hyper_I_N_chunk', 1, 0);
|
|
return next ln;
|
|
end loop;
|
|
end;
|
|
$$;
|
|
create function explain_anonymize(text) returns setof text
|
|
language plpgsql as
|
|
$$
|
|
declare
|
|
ln text;
|
|
begin
|
|
for ln in
|
|
execute format('explain (costs off, summary off, timing off) %s', $1)
|
|
loop
|
|
ln := regexp_replace(ln, 'Array Cache Hits: \d+', 'Array Cache Hits: N');
|
|
ln := regexp_replace(ln, 'Array Cache Misses: \d+', 'Array Cache Misses: N');
|
|
ln := regexp_replace(ln, 'Array Cache Evictions: \d+', 'Array Cache Evictions: N');
|
|
ln := regexp_replace(ln, 'Heap Fetches: \d+', 'Heap Fetches: N');
|
|
ln := regexp_replace(ln, 'Workers Launched: \d+', 'Workers Launched: N');
|
|
ln := regexp_replace(ln, 'actual rows=\d+ loops=\d+', 'actual rows=N loops=N');
|
|
ln := regexp_replace(ln, '_hyper_\d+_\d+_chunk', '_hyper_I_N_chunk', 1, 0);
|
|
return next ln;
|
|
end loop;
|
|
end;
|
|
$$;
|
|
create table :hypertable(
|
|
metric_id serial,
|
|
created_at timestamptz not null unique,
|
|
location_id smallint, --segmentby attribute with index
|
|
owner_id bigint, --segmentby attribute without index
|
|
device_id bigint, --non-segmentby attribute
|
|
temp float8,
|
|
humidity float4
|
|
);
|
|
create index hypertable_location_id_idx on :hypertable (location_id);
|
|
create index hypertable_device_id_idx on :hypertable (device_id);
|
|
select create_hypertable(:'hypertable', by_range('created_at'));
|
|
create_hypertable
|
|
-------------------
|
|
(1,t)
|
|
(1 row)
|
|
|
|
-- Disable incremental sort to make tests stable
|
|
set enable_incremental_sort = false;
|
|
select setseed(1);
|
|
setseed
|
|
---------
|
|
|
|
(1 row)
|
|
|
|
-- Insert rows into the tables.
|
|
--
|
|
-- The timestamps for the original rows will have timestamps every 10
|
|
-- seconds. Any other timestamps are inserted as part of the test.
|
|
insert into :hypertable (created_at, location_id, device_id, owner_id, temp, humidity)
|
|
select t, ceil(random()*10), ceil(random()*30), ceil(random() * 5), random()*40, random()*100
|
|
from generate_series('2022-06-01'::timestamptz, '2022-07-01', '5m') t;
|
|
alter table :hypertable set (
|
|
timescaledb.compress,
|
|
timescaledb.compress_orderby = 'created_at',
|
|
timescaledb.compress_segmentby = 'location_id, owner_id'
|
|
);
|
|
-- Get some test chunks as global variables (first and last chunk here)
|
|
select format('%I.%I', chunk_schema, chunk_name)::regclass as chunk1
|
|
from timescaledb_information.chunks
|
|
where format('%I.%I', hypertable_schema, hypertable_name)::regclass = :'hypertable'::regclass
|
|
order by chunk1 asc
|
|
limit 1 \gset
|
|
select format('%I.%I', chunk_schema, chunk_name)::regclass as chunk2
|
|
from timescaledb_information.chunks
|
|
where format('%I.%I', hypertable_schema, hypertable_name)::regclass = :'hypertable'::regclass
|
|
order by chunk2 asc
|
|
limit 1 offset 1 \gset
|
|
-- To generate plans consistently.
|
|
set max_parallel_workers_per_gather to 0;
|
|
-- Create a function that uses a cursor to scan the the Hypercore
|
|
-- table. This should work equivalent to a query on the same table.
|
|
create function location_humidity_for(
|
|
in p_owner integer,
|
|
out p_location integer,
|
|
out p_humidity float)
|
|
returns setof record as
|
|
$$
|
|
declare
|
|
location_record record;
|
|
location_cursor cursor for
|
|
select location_id,
|
|
avg(humidity) as avg_humidity
|
|
from readings
|
|
where owner_id = p_owner
|
|
group by location_id;
|
|
begin
|
|
open location_cursor;
|
|
|
|
loop
|
|
fetch next from location_cursor into location_record;
|
|
exit when not found;
|
|
|
|
p_location = location_record.location_id;
|
|
p_humidity = location_record.avg_humidity;
|
|
return next;
|
|
end loop;
|
|
|
|
close location_cursor;
|
|
end;
|
|
$$
|
|
language plpgsql;
|
|
select compress_chunk(show_chunks(:'hypertable'), hypercore_use_access_method => true);
|
|
compress_chunk
|
|
----------------------------------------
|
|
_timescaledb_internal._hyper_1_1_chunk
|
|
_timescaledb_internal._hyper_1_2_chunk
|
|
_timescaledb_internal._hyper_1_3_chunk
|
|
_timescaledb_internal._hyper_1_4_chunk
|
|
_timescaledb_internal._hyper_1_5_chunk
|
|
_timescaledb_internal._hyper_1_6_chunk
|
|
(6 rows)
|
|
|
|
-- Compare executing the function with a cursor with a query fetching
|
|
-- the same data directly from the hypertable.
|
|
select p_location, lhs.p_humidity, rhs.p_humidity
|
|
from (select * from location_humidity_for(1)) lhs
|
|
join (select location_id as p_location,
|
|
avg(humidity) as p_humidity
|
|
from :hypertable
|
|
where owner_id = 1
|
|
group by location_id) rhs
|
|
using (p_location)
|
|
where lhs.p_humidity != rhs.p_humidity
|
|
order by p_location;
|
|
p_location | p_humidity | p_humidity
|
|
------------+------------+------------
|
|
(0 rows)
|
|
|
|
-- Create a function that will use a cursor to iterate through a table
|
|
-- and update the humidity for a location using a cursor.
|
|
create function update_location_humidity(
|
|
in p_location integer,
|
|
in p_humidity float)
|
|
returns setof record as
|
|
$$
|
|
declare
|
|
location_record record;
|
|
location_cursor cursor for
|
|
select location_id, humidity from readings where location_id = p_location;
|
|
begin
|
|
open location_cursor;
|
|
|
|
loop
|
|
move next in location_cursor;
|
|
exit when not found;
|
|
update readings set humidity = p_humidity where current of location_cursor;
|
|
end loop;
|
|
|
|
close location_cursor;
|
|
end;
|
|
$$
|
|
language plpgsql;
|
|
set timescaledb.max_tuples_decompressed_per_dml_transaction to 0;
|
|
create table saved as select * from :hypertable;
|
|
-- These two should generate the same result
|
|
update saved set humidity = 100.0 where location_id = 10;
|
|
select update_location_humidity(10, 100.0);
|
|
update_location_humidity
|
|
--------------------------
|
|
(0 rows)
|
|
|
|
-- This should show no rows, but if there are differences we limit
|
|
-- this to 10 rows to not waste electrons.
|
|
--
|
|
-- Note that update of compressed tables through a cursor does not
|
|
-- work for all compressed tables right now because of the way the
|
|
-- local ExecModifyTable is implemented, so this will show rows.
|
|
select metric_id, lhs.humidity, rhs.humidity
|
|
from saved lhs full join :hypertable rhs using (metric_id)
|
|
where lhs.humidity != rhs.humidity
|
|
order by metric_id limit 10;
|
|
metric_id | humidity | humidity
|
|
-----------+----------+----------
|
|
15 | 100 | 96.6755
|
|
16 | 100 | 62.7184
|
|
23 | 100 | 1.22822
|
|
30 | 100 | 73.4355
|
|
31 | 100 | 84.7107
|
|
79 | 100 | 72.8738
|
|
82 | 100 | 60.9366
|
|
101 | 100 | 42.5581
|
|
111 | 100 | 93.1651
|
|
113 | 100 | 1.58061
|
|
(10 rows)
|
|
|
|
drop function location_humidity_for;
|
|
drop function update_location_humidity;
|
|
-- Test cursor going backwards
|
|
create table backward_cursor (time timestamptz, location_id bigint, temp float8);
|
|
select create_hypertable('backward_cursor', 'time', create_default_indexes=>false);
|
|
NOTICE: adding not-null constraint to column "time"
|
|
create_hypertable
|
|
------------------------------
|
|
(3,public,backward_cursor,t)
|
|
(1 row)
|
|
|
|
alter table backward_cursor set (timescaledb.compress, timescaledb.compress_segmentby='location_id', timescaledb.compress_orderby='time asc');
|
|
insert into backward_cursor values ('2024-01-01 01:00', 1, 1.0), ('2024-01-01 02:00', 1, 2.0), ('2024-01-01 03:00', 2, 3.0), ('2024-01-01 04:00', 2, 4.0);
|
|
select compress_chunk(ch, hypercore_use_access_method => true) from show_chunks('backward_cursor') ch;
|
|
compress_chunk
|
|
-----------------------------------------
|
|
_timescaledb_internal._hyper_3_13_chunk
|
|
(1 row)
|
|
|
|
insert into backward_cursor values ('2024-01-01 05:00', 3, 5.0), ('2024-01-01 06:00', 3, 6.0);
|
|
begin;
|
|
-- This needs to be a simple scan on top of the baserel, without a
|
|
-- materialization. For scan nodes that don't support backwards scans,
|
|
-- or where a sort or similar happens, the query is typically
|
|
-- materialized first, thus not really testing the TAMs ability to do
|
|
-- backwards scanning.
|
|
explain (costs off)
|
|
declare curs1 cursor for
|
|
select _timescaledb_debug.is_compressed_tid(ctid), * from backward_cursor;
|
|
QUERY PLAN
|
|
-------------------------------------------------------
|
|
Result
|
|
-> Custom Scan (ColumnarScan) on _hyper_3_13_chunk
|
|
(2 rows)
|
|
|
|
declare curs1 cursor for
|
|
select _timescaledb_debug.is_compressed_tid(ctid), * from backward_cursor;
|
|
-- Immediately fetching backward should return nothing
|
|
fetch backward 1 from curs1;
|
|
is_compressed_tid | time | location_id | temp
|
|
-------------------+------+-------------+------
|
|
(0 rows)
|
|
|
|
-- Now read some values forward
|
|
fetch forward 1 from curs1;
|
|
is_compressed_tid | time | location_id | temp
|
|
-------------------+------------------------------+-------------+------
|
|
t | Mon Jan 01 01:00:00 2024 PST | 1 | 1
|
|
(1 row)
|
|
|
|
fetch forward 1 from curs1;
|
|
is_compressed_tid | time | location_id | temp
|
|
-------------------+------------------------------+-------------+------
|
|
t | Mon Jan 01 02:00:00 2024 PST | 1 | 2
|
|
(1 row)
|
|
|
|
-- The next fetch should move into a new segment with location_id=2
|
|
fetch forward 1 from curs1;
|
|
is_compressed_tid | time | location_id | temp
|
|
-------------------+------------------------------+-------------+------
|
|
t | Mon Jan 01 03:00:00 2024 PST | 2 | 3
|
|
(1 row)
|
|
|
|
-- Last compressed entry
|
|
fetch forward 1 from curs1;
|
|
is_compressed_tid | time | location_id | temp
|
|
-------------------+------------------------------+-------------+------
|
|
t | Mon Jan 01 04:00:00 2024 PST | 2 | 4
|
|
(1 row)
|
|
|
|
-- Now should move into non-compressed
|
|
fetch forward 1 from curs1;
|
|
is_compressed_tid | time | location_id | temp
|
|
-------------------+------------------------------+-------------+------
|
|
f | Mon Jan 01 05:00:00 2024 PST | 3 | 5
|
|
(1 row)
|
|
|
|
-- Last entry in non-compressed
|
|
fetch forward 1 from curs1;
|
|
is_compressed_tid | time | location_id | temp
|
|
-------------------+------------------------------+-------------+------
|
|
f | Mon Jan 01 06:00:00 2024 PST | 3 | 6
|
|
(1 row)
|
|
|
|
-- Should return nothing since at end
|
|
fetch forward 1 from curs1;
|
|
is_compressed_tid | time | location_id | temp
|
|
-------------------+------+-------------+------
|
|
(0 rows)
|
|
|
|
-- Now move backwards
|
|
fetch backward 1 from curs1;
|
|
is_compressed_tid | time | location_id | temp
|
|
-------------------+------------------------------+-------------+------
|
|
f | Mon Jan 01 06:00:00 2024 PST | 3 | 6
|
|
(1 row)
|
|
|
|
-- Now backwards into the old segment
|
|
fetch backward 5 from curs1;
|
|
is_compressed_tid | time | location_id | temp
|
|
-------------------+------------------------------+-------------+------
|
|
f | Mon Jan 01 05:00:00 2024 PST | 3 | 5
|
|
t | Mon Jan 01 04:00:00 2024 PST | 2 | 4
|
|
t | Mon Jan 01 03:00:00 2024 PST | 2 | 3
|
|
t | Mon Jan 01 02:00:00 2024 PST | 1 | 2
|
|
t | Mon Jan 01 01:00:00 2024 PST | 1 | 1
|
|
(5 rows)
|
|
|
|
-- Next fetch should return nothing since at start
|
|
fetch backward 1 from curs1;
|
|
is_compressed_tid | time | location_id | temp
|
|
-------------------+------+-------------+------
|
|
(0 rows)
|
|
|
|
-- Fetch first value again
|
|
fetch forward 1 from curs1;
|
|
is_compressed_tid | time | location_id | temp
|
|
-------------------+------------------------------+-------------+------
|
|
t | Mon Jan 01 01:00:00 2024 PST | 1 | 1
|
|
(1 row)
|
|
|
|
-- Jump to last value
|
|
fetch last from curs1;
|
|
is_compressed_tid | time | location_id | temp
|
|
-------------------+------------------------------+-------------+------
|
|
f | Mon Jan 01 06:00:00 2024 PST | 3 | 6
|
|
(1 row)
|
|
|
|
-- Back to first
|
|
fetch first from curs1;
|
|
is_compressed_tid | time | location_id | temp
|
|
-------------------+------------------------------+-------------+------
|
|
t | Mon Jan 01 01:00:00 2024 PST | 1 | 1
|
|
(1 row)
|
|
|
|
-- Get the values at position 2 and 5 from the start
|
|
fetch absolute 2 from curs1;
|
|
is_compressed_tid | time | location_id | temp
|
|
-------------------+------------------------------+-------------+------
|
|
t | Mon Jan 01 02:00:00 2024 PST | 1 | 2
|
|
(1 row)
|
|
|
|
fetch absolute 5 from curs1;
|
|
is_compressed_tid | time | location_id | temp
|
|
-------------------+------------------------------+-------------+------
|
|
f | Mon Jan 01 05:00:00 2024 PST | 3 | 5
|
|
(1 row)
|
|
|
|
-- Get the value at position 3 from the end (which should be 4 from
|
|
-- the start)
|
|
fetch absolute -3 from curs1;
|
|
is_compressed_tid | time | location_id | temp
|
|
-------------------+------------------------------+-------------+------
|
|
t | Mon Jan 01 04:00:00 2024 PST | 2 | 4
|
|
(1 row)
|
|
|
|
commit;
|