mirror of
https://github.com/timescale/timescaledb.git
synced 2025-05-15 01:53:41 +08:00
e5e94960d0
Changing from using the `compress_using` parameter with a table access method name to use the boolean parameter `hypercore_use_access_method` instead to avoid having to provide a name when using the table access method for compression.
338 lines
16 KiB
Plaintext
338 lines
16 KiB
Plaintext
-- This file and its contents are licensed under the Timescale License.
|
|
-- Please see the included NOTICE for copyright information and
|
|
-- LICENSE-TIMESCALE for a copy of the license.
|
|
\ir include/setup_hypercore.sql
|
|
-- This file and its contents are licensed under the Timescale License.
|
|
-- Please see the included NOTICE for copyright information and
|
|
-- LICENSE-TIMESCALE for a copy of the license.
|
|
\set hypertable readings
|
|
\ir hypercore_helpers.sql
|
|
-- This file and its contents are licensed under the Timescale License.
|
|
-- Please see the included NOTICE for copyright information and
|
|
-- LICENSE-TIMESCALE for a copy of the license.
|
|
-- Function to run an explain analyze with and do replacements on the
|
|
-- emitted plan. This is intended to be used when the structure of the
|
|
-- plan is important, but not the specific chunks scanned nor the
|
|
-- number of heap fetches, rows, loops, etc.
|
|
create function explain_analyze_anonymize(text) returns setof text
|
|
language plpgsql as
|
|
$$
|
|
declare
|
|
ln text;
|
|
begin
|
|
for ln in
|
|
execute format('explain (analyze, costs off, summary off, timing off, decompress_cache_stats) %s', $1)
|
|
loop
|
|
if trim(both from ln) like 'Group Key:%' then
|
|
continue;
|
|
end if;
|
|
ln := regexp_replace(ln, 'Array Cache Hits: \d+', 'Array Cache Hits: N');
|
|
ln := regexp_replace(ln, 'Array Cache Misses: \d+', 'Array Cache Misses: N');
|
|
ln := regexp_replace(ln, 'Array Cache Evictions: \d+', 'Array Cache Evictions: N');
|
|
ln := regexp_replace(ln, 'Heap Fetches: \d+', 'Heap Fetches: N');
|
|
ln := regexp_replace(ln, 'Workers Launched: \d+', 'Workers Launched: N');
|
|
ln := regexp_replace(ln, 'actual rows=\d+ loops=\d+', 'actual rows=N loops=N');
|
|
ln := regexp_replace(ln, '_hyper_\d+_\d+_chunk', '_hyper_I_N_chunk', 1, 0);
|
|
return next ln;
|
|
end loop;
|
|
end;
|
|
$$;
|
|
create function explain_anonymize(text) returns setof text
|
|
language plpgsql as
|
|
$$
|
|
declare
|
|
ln text;
|
|
begin
|
|
for ln in
|
|
execute format('explain (costs off, summary off, timing off) %s', $1)
|
|
loop
|
|
ln := regexp_replace(ln, 'Array Cache Hits: \d+', 'Array Cache Hits: N');
|
|
ln := regexp_replace(ln, 'Array Cache Misses: \d+', 'Array Cache Misses: N');
|
|
ln := regexp_replace(ln, 'Array Cache Evictions: \d+', 'Array Cache Evictions: N');
|
|
ln := regexp_replace(ln, 'Heap Fetches: \d+', 'Heap Fetches: N');
|
|
ln := regexp_replace(ln, 'Workers Launched: \d+', 'Workers Launched: N');
|
|
ln := regexp_replace(ln, 'actual rows=\d+ loops=\d+', 'actual rows=N loops=N');
|
|
ln := regexp_replace(ln, '_hyper_\d+_\d+_chunk', '_hyper_I_N_chunk', 1, 0);
|
|
return next ln;
|
|
end loop;
|
|
end;
|
|
$$;
|
|
create table :hypertable(
|
|
metric_id serial,
|
|
created_at timestamptz not null unique,
|
|
location_id smallint, --segmentby attribute with index
|
|
owner_id bigint, --segmentby attribute without index
|
|
device_id bigint, --non-segmentby attribute
|
|
temp float8,
|
|
humidity float4
|
|
);
|
|
create index hypertable_location_id_idx on :hypertable (location_id);
|
|
create index hypertable_device_id_idx on :hypertable (device_id);
|
|
select create_hypertable(:'hypertable', by_range('created_at'));
|
|
create_hypertable
|
|
-------------------
|
|
(1,t)
|
|
(1 row)
|
|
|
|
-- Disable incremental sort to make tests stable
|
|
set enable_incremental_sort = false;
|
|
select setseed(1);
|
|
setseed
|
|
---------
|
|
|
|
(1 row)
|
|
|
|
-- Insert rows into the tables.
|
|
--
|
|
-- The timestamps for the original rows will have timestamps every 10
|
|
-- seconds. Any other timestamps are inserted as part of the test.
|
|
insert into :hypertable (created_at, location_id, device_id, owner_id, temp, humidity)
|
|
select t, ceil(random()*10), ceil(random()*30), ceil(random() * 5), random()*40, random()*100
|
|
from generate_series('2022-06-01'::timestamptz, '2022-07-01', '5m') t;
|
|
alter table :hypertable set (
|
|
timescaledb.compress,
|
|
timescaledb.compress_orderby = 'created_at',
|
|
timescaledb.compress_segmentby = 'location_id, owner_id'
|
|
);
|
|
-- Get some test chunks as global variables (first and last chunk here)
|
|
select format('%I.%I', chunk_schema, chunk_name)::regclass as chunk1
|
|
from timescaledb_information.chunks
|
|
where format('%I.%I', hypertable_schema, hypertable_name)::regclass = :'hypertable'::regclass
|
|
order by chunk1 asc
|
|
limit 1 \gset
|
|
select format('%I.%I', chunk_schema, chunk_name)::regclass as chunk2
|
|
from timescaledb_information.chunks
|
|
where format('%I.%I', hypertable_schema, hypertable_name)::regclass = :'hypertable'::regclass
|
|
order by chunk2 asc
|
|
limit 1 offset 1 \gset
|
|
-- Compress the chunks and check that the counts are the same
|
|
select location_id, count(*) into orig from :hypertable GROUP BY location_id;
|
|
select compress_chunk(show_chunks(:'hypertable'), hypercore_use_access_method => true);
|
|
compress_chunk
|
|
----------------------------------------
|
|
_timescaledb_internal._hyper_1_1_chunk
|
|
_timescaledb_internal._hyper_1_2_chunk
|
|
_timescaledb_internal._hyper_1_3_chunk
|
|
_timescaledb_internal._hyper_1_4_chunk
|
|
_timescaledb_internal._hyper_1_5_chunk
|
|
_timescaledb_internal._hyper_1_6_chunk
|
|
(6 rows)
|
|
|
|
select location_id, count(*) into comp from :hypertable GROUP BY location_id;
|
|
select * from orig join comp using (location_id) where orig.count != comp.count;
|
|
location_id | count | count
|
|
-------------+-------+-------
|
|
(0 rows)
|
|
|
|
drop table orig, comp;
|
|
-- Check that all chunks are compressed
|
|
select chunk_name, compression_status from chunk_compression_stats(:'hypertable');
|
|
chunk_name | compression_status
|
|
------------------+--------------------
|
|
_hyper_1_1_chunk | Compressed
|
|
_hyper_1_2_chunk | Compressed
|
|
_hyper_1_3_chunk | Compressed
|
|
_hyper_1_4_chunk | Compressed
|
|
_hyper_1_5_chunk | Compressed
|
|
_hyper_1_6_chunk | Compressed
|
|
(6 rows)
|
|
|
|
--
|
|
-- Test that a conflict happens when inserting a value that already
|
|
-- exists in the compressed part of the chunk.
|
|
--
|
|
-- Check that we have at least one timestamp that we want to test.
|
|
SELECT count(*) FROM :chunk1 WHERE created_at = '2022-06-01'::timestamptz;
|
|
count
|
|
-------
|
|
1
|
|
(1 row)
|
|
|
|
\set ON_ERROR_STOP 0
|
|
insert into :hypertable(created_at, location_id, device_id, temp, humidity)
|
|
values ('2022-06-01', 1, 1, 1.0, 1.0);
|
|
ERROR: duplicate key value violates unique constraint "1_1_readings_created_at_key"
|
|
insert into :chunk1(created_at, location_id, device_id, temp, humidity)
|
|
values ('2022-06-01', 1, 1, 1.0, 1.0);
|
|
ERROR: duplicate key value violates unique constraint "1_1_readings_created_at_key"
|
|
\set ON_ERROR_STOP 1
|
|
-- Insert values, which will end up in the uncompressed parts
|
|
INSERT INTO :chunk1(created_at, location_id, device_id, temp, humidity)
|
|
VALUES ('2022-06-01 00:00:02', 1, 1, 1.0, 1.0);
|
|
INSERT INTO :hypertable(created_at, location_id, device_id, temp, humidity)
|
|
VALUES ('2022-06-01 00:00:03', 1, 1, 1.0, 1.0);
|
|
-- Should still generate an error
|
|
\set ON_ERROR_STOP 0
|
|
insert into :chunk1(created_at, location_id, device_id, temp, humidity)
|
|
values ('2022-06-01 00:00:02', 1, 1, 1.0, 1.0);
|
|
ERROR: duplicate key value violates unique constraint "1_1_readings_created_at_key"
|
|
insert into :hypertable(created_at, location_id, device_id, temp, humidity)
|
|
values ('2022-06-01 00:00:02', 1, 1, 1.0, 1.0);
|
|
ERROR: duplicate key value violates unique constraint "1_1_readings_created_at_key"
|
|
insert into :chunk1(created_at, location_id, device_id, temp, humidity)
|
|
values ('2022-06-01 00:00:03', 1, 1, 1.0, 1.0);
|
|
ERROR: duplicate key value violates unique constraint "1_1_readings_created_at_key"
|
|
insert into :hypertable(created_at, location_id, device_id, temp, humidity)
|
|
values ('2022-06-01 00:00:03', 1, 1, 1.0, 1.0);
|
|
ERROR: duplicate key value violates unique constraint "1_1_readings_created_at_key"
|
|
\set ON_ERROR_STOP 1
|
|
set session characteristics as transaction isolation level repeatable read;
|
|
--
|
|
-- Testing speculative inserts and upserts
|
|
--
|
|
-- Speculative insert with conflicting row should succeed and not
|
|
-- insert anything
|
|
select location_id, count(*) into orig from :hypertable GROUP BY location_id;
|
|
-- Inserting into compressed part
|
|
insert into :hypertable(created_at, location_id, device_id, temp, humidity)
|
|
values ('2022-06-01', 1, 1, 1.0, 1.0)
|
|
on conflict (created_at) do nothing;
|
|
insert into :chunk1(created_at, location_id, device_id, temp, humidity)
|
|
values ('2022-06-01', 1, 1, 1.0, 1.0)
|
|
on conflict (created_at) do nothing;
|
|
-- Inserting into non-compressed part
|
|
insert into :chunk1(created_at, location_id, device_id, temp, humidity)
|
|
values ('2022-06-01 00:00:02', 1, 1, 1.0, 1.0)
|
|
on conflict (created_at) do nothing;
|
|
insert into :hypertable(created_at, location_id, device_id, temp, humidity)
|
|
values ('2022-06-01 00:00:02', 1, 1, 1.0, 1.0)
|
|
on conflict (created_at) do nothing;
|
|
insert into :chunk1(created_at, location_id, device_id, temp, humidity)
|
|
values ('2022-06-01 00:00:03', 1, 1, 1.0, 1.0)
|
|
on conflict (created_at) do nothing;
|
|
insert into :hypertable(created_at, location_id, device_id, temp, humidity)
|
|
values ('2022-06-01 00:00:03', 1, 1, 1.0, 1.0)
|
|
on conflict (created_at) do nothing;
|
|
-- Check the count after the operations above. We count each location
|
|
-- here since it is easier to see what went wrong if something did.
|
|
select location_id, count(*) into curr from :hypertable GROUP BY location_id;
|
|
select * from :hypertable where created_at between '2022-06-01 00:00:01' and '2022-06-01 00:00:09';
|
|
metric_id | created_at | location_id | owner_id | device_id | temp | humidity
|
|
-----------+------------------------------+-------------+----------+-----------+------+----------
|
|
8644 | Wed Jun 01 00:00:02 2022 PDT | 1 | | 1 | 1 | 1
|
|
8645 | Wed Jun 01 00:00:03 2022 PDT | 1 | | 1 | 1 | 1
|
|
(2 rows)
|
|
|
|
select * from orig join curr using (location_id) where orig.count != curr.count;
|
|
location_id | count | count
|
|
-------------+-------+-------
|
|
(0 rows)
|
|
|
|
drop table curr;
|
|
-- Speculative insert with non-conflicting rows should succeed.
|
|
insert into :hypertable(created_at, location_id, device_id, temp, humidity)
|
|
values ('2022-06-01 00:00:06', 1, 1, 1.0, 1.0)
|
|
on conflict (created_at) do nothing;
|
|
insert into :chunk1(created_at, location_id, device_id, temp, humidity)
|
|
values ('2022-06-01 00:00:07', 1, 1, 1.0, 1.0)
|
|
on conflict (created_at) do nothing;
|
|
select location_id, count(*) into curr from :hypertable GROUP BY location_id;
|
|
select location_id, curr.count - orig.count as increase
|
|
from orig join curr using (location_id)
|
|
where orig.count != curr.count;
|
|
location_id | increase
|
|
-------------+----------
|
|
1 | 2
|
|
(1 row)
|
|
|
|
drop table orig, curr;
|
|
-- Upserts with conflicting and non-conflicting rows should work
|
|
-- correctly also when inserting directly into the chunks.
|
|
--
|
|
-- We have tested this above, but since different code paths are used
|
|
-- for DO UPDATE, DO NOTHING, and plain inserts, we test this as well
|
|
-- to be safe.
|
|
-- find a compressed tuple in a deterministic manner and get the
|
|
-- timestamp. Make sure to find one in chunk1 since we will use that
|
|
-- later.
|
|
select created_at
|
|
from :chunk1 where _timescaledb_debug.is_compressed_tid(ctid)
|
|
order by created_at limit 1 \gset
|
|
select * from :hypertable where created_at = :'created_at';
|
|
metric_id | created_at | location_id | owner_id | device_id | temp | humidity
|
|
-----------+------------------------------+-------------+----------+-----------+------------------+----------
|
|
1 | Wed Jun 01 00:00:00 2022 PDT | 4 | 2 | 23 | 16.4320374922463 | 55.2454
|
|
(1 row)
|
|
|
|
-- Insert of a value that exists in the compressed part should work
|
|
-- when done through the hypertable.
|
|
insert into :hypertable(created_at, location_id, device_id, temp, humidity)
|
|
values (:'created_at', 11, 1, 1.0, 1.0)
|
|
on conflict (created_at) do update set location_id = 12;
|
|
select * from :hypertable where created_at = :'created_at';
|
|
metric_id | created_at | location_id | owner_id | device_id | temp | humidity
|
|
-----------+------------------------------+-------------+----------+-----------+------------------+----------
|
|
1 | Wed Jun 01 00:00:00 2022 PDT | 12 | 2 | 23 | 16.4320374922463 | 55.2454
|
|
(1 row)
|
|
|
|
-- TODO(timescale/timescaledb-private#1087): Inserts directly into a
|
|
-- compressed tuple in a chunk do not work.
|
|
select created_at
|
|
from :chunk1 where _timescaledb_debug.is_compressed_tid(ctid)
|
|
order by created_at limit 1 \gset
|
|
\set ON_ERROR_STOP 0
|
|
insert into :chunk1(created_at, location_id, device_id, temp, humidity)
|
|
values (:'created_at', 13, 1, 1.0, 1.0)
|
|
on conflict (created_at) do update set location_id = 14;
|
|
ERROR: cannot update compressed tuple
|
|
\set ON_ERROR_STOP 1
|
|
-- Insert of a value that exists in the non-compressed part. (These
|
|
-- were inserted above.)
|
|
insert into :chunk1(created_at, location_id, device_id, temp, humidity)
|
|
values ('2022-06-01 00:00:02', 15, 1, 1.0, 1.0)
|
|
on conflict (created_at) do update set location_id = 16;
|
|
insert into :hypertable(created_at, location_id, device_id, temp, humidity)
|
|
values ('2022-06-01 00:00:03', 17, 1, 1.0, 1.0)
|
|
on conflict (created_at) do update set location_id = 18;
|
|
-- Inserting them again should still update.
|
|
insert into :chunk1(created_at, location_id, device_id, temp, humidity)
|
|
values ('2022-06-01 00:00:02', 19, 1, 1.0, 1.0)
|
|
on conflict (created_at) do update set location_id = 20;
|
|
insert into :hypertable(created_at, location_id, device_id, temp, humidity)
|
|
values ('2022-06-01 00:00:03', 21, 1, 1.0, 1.0)
|
|
on conflict (created_at) do update set location_id = 22;
|
|
select * from :hypertable where location_id between 11 and 22
|
|
order by location_id;
|
|
metric_id | created_at | location_id | owner_id | device_id | temp | humidity
|
|
-----------+------------------------------+-------------+----------+-----------+------------------+----------
|
|
1 | Wed Jun 01 00:00:00 2022 PDT | 12 | 2 | 23 | 16.4320374922463 | 55.2454
|
|
8644 | Wed Jun 01 00:00:02 2022 PDT | 20 | | 1 | 1 | 1
|
|
8645 | Wed Jun 01 00:00:03 2022 PDT | 22 | | 1 | 1 | 1
|
|
(3 rows)
|
|
|
|
drop table :hypertable;
|
|
-- Check that we can write to a hypercore table from another kind of
|
|
-- slot even if we have dropped and added attributes.
|
|
create table test2 (itime integer, b bigint, t text);
|
|
select create_hypertable('test2', by_range('itime', 10));
|
|
NOTICE: adding not-null constraint to column "itime"
|
|
create_hypertable
|
|
-------------------
|
|
(3,t)
|
|
(1 row)
|
|
|
|
create table test2_source(itime integer, d int, t text);
|
|
insert into test2_source values (9, '9', 90), (17, '17', 1700);
|
|
-- this will create a single chunk.
|
|
insert into test2 select t, 10, 'first'::text from generate_series(1, 7) t;
|
|
alter table test2 drop column b;
|
|
alter table test2 add column c int default -15;
|
|
alter table test2 add column d int;
|
|
-- Since we have chunk sizes of 10, this will create a second chunk
|
|
-- with a second set of attributes where one is dropped.
|
|
insert into test2 select t, 'second'::text, 120, 1 from generate_series(11, 15) t;
|
|
alter table test2
|
|
set access method hypercore,
|
|
set (timescaledb.compress_segmentby = '', timescaledb.compress_orderby = 'c, itime desc');
|
|
WARNING: there was some uncertainty picking the default segment by for the hypertable: You do not have any indexes on columns that can be used for segment_by and thus we are not using segment_by for compression. Please make sure you are not missing any indexes
|
|
NOTICE: default segment by for hypertable "test2" is set to ""
|
|
select compress_chunk(show_chunks('test2'));
|
|
compress_chunk
|
|
-----------------------------------------
|
|
_timescaledb_internal._hyper_3_13_chunk
|
|
_timescaledb_internal._hyper_3_14_chunk
|
|
(2 rows)
|
|
|
|
-- Insert into both chunks using a select.
|
|
insert into test2(itime ,t , d) select itime, t, d from test2_source;
|