mirror of
https://github.com/timescale/timescaledb.git
synced 2025-05-15 18:13:18 +08:00
e5e94960d0
Changing from using the `compress_using` parameter with a table access method name to use the boolean parameter `hypercore_use_access_method` instead to avoid having to provide a name when using the table access method for compression.
256 lines
11 KiB
Plaintext
256 lines
11 KiB
Plaintext
-- This file and its contents are licensed under the Timescale License.
|
|
-- Please see the included NOTICE for copyright information and
|
|
-- LICENSE-TIMESCALE for a copy of the license.
|
|
\ir include/setup_hypercore.sql
|
|
-- This file and its contents are licensed under the Timescale License.
|
|
-- Please see the included NOTICE for copyright information and
|
|
-- LICENSE-TIMESCALE for a copy of the license.
|
|
\set hypertable readings
|
|
\ir hypercore_helpers.sql
|
|
-- This file and its contents are licensed under the Timescale License.
|
|
-- Please see the included NOTICE for copyright information and
|
|
-- LICENSE-TIMESCALE for a copy of the license.
|
|
-- Function to run an explain analyze with and do replacements on the
|
|
-- emitted plan. This is intended to be used when the structure of the
|
|
-- plan is important, but not the specific chunks scanned nor the
|
|
-- number of heap fetches, rows, loops, etc.
|
|
create function explain_analyze_anonymize(text) returns setof text
|
|
language plpgsql as
|
|
$$
|
|
declare
|
|
ln text;
|
|
begin
|
|
for ln in
|
|
execute format('explain (analyze, costs off, summary off, timing off, decompress_cache_stats) %s', $1)
|
|
loop
|
|
if trim(both from ln) like 'Group Key:%' then
|
|
continue;
|
|
end if;
|
|
ln := regexp_replace(ln, 'Array Cache Hits: \d+', 'Array Cache Hits: N');
|
|
ln := regexp_replace(ln, 'Array Cache Misses: \d+', 'Array Cache Misses: N');
|
|
ln := regexp_replace(ln, 'Array Cache Evictions: \d+', 'Array Cache Evictions: N');
|
|
ln := regexp_replace(ln, 'Heap Fetches: \d+', 'Heap Fetches: N');
|
|
ln := regexp_replace(ln, 'Workers Launched: \d+', 'Workers Launched: N');
|
|
ln := regexp_replace(ln, 'actual rows=\d+ loops=\d+', 'actual rows=N loops=N');
|
|
ln := regexp_replace(ln, '_hyper_\d+_\d+_chunk', '_hyper_I_N_chunk', 1, 0);
|
|
return next ln;
|
|
end loop;
|
|
end;
|
|
$$;
|
|
create function explain_anonymize(text) returns setof text
|
|
language plpgsql as
|
|
$$
|
|
declare
|
|
ln text;
|
|
begin
|
|
for ln in
|
|
execute format('explain (costs off, summary off, timing off) %s', $1)
|
|
loop
|
|
ln := regexp_replace(ln, 'Array Cache Hits: \d+', 'Array Cache Hits: N');
|
|
ln := regexp_replace(ln, 'Array Cache Misses: \d+', 'Array Cache Misses: N');
|
|
ln := regexp_replace(ln, 'Array Cache Evictions: \d+', 'Array Cache Evictions: N');
|
|
ln := regexp_replace(ln, 'Heap Fetches: \d+', 'Heap Fetches: N');
|
|
ln := regexp_replace(ln, 'Workers Launched: \d+', 'Workers Launched: N');
|
|
ln := regexp_replace(ln, 'actual rows=\d+ loops=\d+', 'actual rows=N loops=N');
|
|
ln := regexp_replace(ln, '_hyper_\d+_\d+_chunk', '_hyper_I_N_chunk', 1, 0);
|
|
return next ln;
|
|
end loop;
|
|
end;
|
|
$$;
|
|
create table :hypertable(
|
|
metric_id serial,
|
|
created_at timestamptz not null unique,
|
|
location_id smallint, --segmentby attribute with index
|
|
owner_id bigint, --segmentby attribute without index
|
|
device_id bigint, --non-segmentby attribute
|
|
temp float8,
|
|
humidity float4
|
|
);
|
|
create index hypertable_location_id_idx on :hypertable (location_id);
|
|
create index hypertable_device_id_idx on :hypertable (device_id);
|
|
select create_hypertable(:'hypertable', by_range('created_at'));
|
|
create_hypertable
|
|
-------------------
|
|
(1,t)
|
|
(1 row)
|
|
|
|
-- Disable incremental sort to make tests stable
|
|
set enable_incremental_sort = false;
|
|
select setseed(1);
|
|
setseed
|
|
---------
|
|
|
|
(1 row)
|
|
|
|
-- Insert rows into the tables.
|
|
--
|
|
-- The timestamps for the original rows will have timestamps every 10
|
|
-- seconds. Any other timestamps are inserted as part of the test.
|
|
insert into :hypertable (created_at, location_id, device_id, owner_id, temp, humidity)
|
|
select t, ceil(random()*10), ceil(random()*30), ceil(random() * 5), random()*40, random()*100
|
|
from generate_series('2022-06-01'::timestamptz, '2022-07-01', '5m') t;
|
|
alter table :hypertable set (
|
|
timescaledb.compress,
|
|
timescaledb.compress_orderby = 'created_at',
|
|
timescaledb.compress_segmentby = 'location_id, owner_id'
|
|
);
|
|
-- Get some test chunks as global variables (first and last chunk here)
|
|
select format('%I.%I', chunk_schema, chunk_name)::regclass as chunk1
|
|
from timescaledb_information.chunks
|
|
where format('%I.%I', hypertable_schema, hypertable_name)::regclass = :'hypertable'::regclass
|
|
order by chunk1 asc
|
|
limit 1 \gset
|
|
select format('%I.%I', chunk_schema, chunk_name)::regclass as chunk2
|
|
from timescaledb_information.chunks
|
|
where format('%I.%I', hypertable_schema, hypertable_name)::regclass = :'hypertable'::regclass
|
|
order by chunk2 asc
|
|
limit 1 offset 1 \gset
|
|
create view amrels as
|
|
select cl.oid::regclass as rel, am.amname, inh.inhparent::regclass as relparent
|
|
from pg_class cl
|
|
inner join pg_am am on (cl.relam = am.oid)
|
|
left join pg_inherits inh on (inh.inhrelid = cl.oid);
|
|
-- Compress the chunks and check that the counts are the same
|
|
select location_id, count(*) into orig from :hypertable GROUP BY location_id;
|
|
select compress_chunk(show_chunks(:'hypertable'), hypercore_use_access_method => true);
|
|
compress_chunk
|
|
----------------------------------------
|
|
_timescaledb_internal._hyper_1_1_chunk
|
|
_timescaledb_internal._hyper_1_2_chunk
|
|
_timescaledb_internal._hyper_1_3_chunk
|
|
_timescaledb_internal._hyper_1_4_chunk
|
|
_timescaledb_internal._hyper_1_5_chunk
|
|
_timescaledb_internal._hyper_1_6_chunk
|
|
(6 rows)
|
|
|
|
select location_id, count(*) into comp from :hypertable GROUP BY location_id;
|
|
select * from orig join comp using (location_id) where orig.count != comp.count;
|
|
location_id | count | count
|
|
-------------+-------+-------
|
|
(0 rows)
|
|
|
|
drop table orig, comp;
|
|
-- Check that all chunks are compressed
|
|
select chunk_name, compression_status from chunk_compression_stats(:'hypertable');
|
|
chunk_name | compression_status
|
|
------------------+--------------------
|
|
_hyper_1_1_chunk | Compressed
|
|
_hyper_1_2_chunk | Compressed
|
|
_hyper_1_3_chunk | Compressed
|
|
_hyper_1_4_chunk | Compressed
|
|
_hyper_1_5_chunk | Compressed
|
|
_hyper_1_6_chunk | Compressed
|
|
(6 rows)
|
|
|
|
-- Check that a simple copy will work. This will insert into uncompressed region.
|
|
copy :hypertable(created_at, location_id, device_id, temp, humidity) from stdin;
|
|
select created_at, location_id, device_id, temp, humidity
|
|
from :hypertable where created_at between '2022-06-01 00:00:01' and '2022-06-01 00:00:09';
|
|
created_at | location_id | device_id | temp | humidity
|
|
------------------------------+-------------+-----------+------+----------
|
|
Wed Jun 01 00:00:02 2022 PDT | 1 | 1 | 1 | 1
|
|
Wed Jun 01 00:00:03 2022 PDT | 1 | 1 | 1 | 1
|
|
Wed Jun 01 00:00:04 2022 PDT | 1 | 1 | 1 | 1
|
|
Wed Jun 01 00:00:05 2022 PDT | 1 | 1 | 1 | 1
|
|
(4 rows)
|
|
|
|
select created_at, location_id, device_id, temp, humidity into orig
|
|
from :hypertable where created_at between '2022-06-01 00:00:01' and '2022-06-01 00:00:09';
|
|
-- Insert a batch of rows conflicting with compressed rows
|
|
\set ON_ERROR_STOP 0
|
|
copy :hypertable(created_at, location_id, device_id, temp, humidity) from stdin;
|
|
ERROR: duplicate key value violates unique constraint "1_1_readings_created_at_key"
|
|
\set ON_ERROR_STOP 1
|
|
-- Insert a batch of rows conflicting with non-compressed rows
|
|
\set ON_ERROR_STOP 0
|
|
copy :hypertable(created_at, location_id, device_id, temp, humidity) from stdin;
|
|
ERROR: duplicate key value violates unique constraint "1_1_readings_created_at_key"
|
|
\set ON_ERROR_STOP 1
|
|
select created_at, location_id, device_id, temp, humidity into curr
|
|
from :hypertable where created_at between '2022-06-01 00:00:01' and '2022-06-01 00:00:09';
|
|
select * from orig join curr using (created_at) where row(orig) != row(curr);
|
|
created_at | location_id | device_id | temp | humidity | location_id | device_id | temp | humidity
|
|
------------+-------------+-----------+------+----------+-------------+-----------+------+----------
|
|
(0 rows)
|
|
|
|
-- Read data from a file and make sure that it is inserted properly.
|
|
create table copy_test1(
|
|
metric_id int,
|
|
created_at timestamptz,
|
|
device_id int,
|
|
temp float,
|
|
humidity float
|
|
);
|
|
select create_hypertable('copy_test1', 'created_at');
|
|
NOTICE: adding not-null constraint to column "created_at"
|
|
create_hypertable
|
|
-------------------------
|
|
(3,public,copy_test1,t)
|
|
(1 row)
|
|
|
|
alter table copy_test1 set access method hypercore;
|
|
WARNING: there was some uncertainty picking the default segment by for the hypertable: You do not have any indexes on columns that can be used for segment_by and thus we are not using segment_by for compression. Please make sure you are not missing any indexes
|
|
NOTICE: default segment by for hypertable "copy_test1" is set to ""
|
|
NOTICE: default order by for hypertable "copy_test1" is set to "created_at DESC"
|
|
\copy copy_test1 from 'data/magic.csv' with csv header
|
|
select * from copy_test1 order by metric_id;
|
|
metric_id | created_at | device_id | temp | humidity
|
|
-----------+------------------------------+-----------+-------------------+--------------------
|
|
164 | Wed Jun 01 00:27:10 2022 PDT | 1 | 22.1571424036187 | 95.7589111064385
|
|
298 | Wed Jun 01 00:49:30 2022 PDT | 1 | 26.5904140796662 | 25.6517507790162
|
|
345 | Wed Jun 01 00:57:20 2022 PDT | 1 | 14.2112895008574 | 0.252265718039268
|
|
353 | Wed Jun 01 00:58:40 2022 PDT | 5 | 5.02598214446964 | 40.2435007149558
|
|
363 | Wed Jun 01 01:00:20 2022 PDT | 13 | 30.4508743614474 | 56.0536696009606
|
|
511 | Wed Jun 01 01:25:00 2022 PDT | 1 | 30.5256995663874 | 59.3760333752531
|
|
655 | Wed Jun 01 01:49:00 2022 PDT | 8 | 29.1506488040468 | 98.2462821993139
|
|
977 | Wed Jun 01 02:42:40 2022 PDT | 1 | 0.949462224935731 | 62.1551090171878
|
|
1005 | Wed Jun 01 02:47:20 2022 PDT | 2 | 32.7607481200711 | 0.0814742363326237
|
|
1096 | Wed Jun 01 03:02:30 2022 PDT | 1 | 16.7951263617964 | 76.8274813106784
|
|
2298 | Wed Jun 01 06:22:50 2022 PDT | 6 | 25.1019351511395 | 19.2896631866349
|
|
3246 | Wed Jun 01 09:00:50 2022 PDT | 7 | 39.3508344139115 | 95.1677745558139
|
|
(12 rows)
|
|
|
|
-- Here I wanted to write a subset of the data to a csv file and then
|
|
-- load it again, but this statement fails.
|
|
--
|
|
-- select * into subset from :hypertable where device_id between 0 and 10;
|
|
-- Testing copy into a table that has the access method set but has
|
|
-- already existing (compressed) heap-chunks in the table.
|
|
CREATE TABLE test1 (timec timestamptz , i integer, b bigint, t text);
|
|
SELECT table_name from create_hypertable('test1', 'timec');
|
|
NOTICE: adding not-null constraint to column "timec"
|
|
table_name
|
|
------------
|
|
test1
|
|
(1 row)
|
|
|
|
INSERT INTO test1 SELECT q, 10, 11, 'hello'
|
|
FROM generate_series( '2020-01-03 10:00:00-05', '2020-01-03 12:00:00-05' , '5 min'::interval) q;
|
|
alter table test1 set (timescaledb.compress_segmentby = 'b', timescaledb.compress_orderby = 'timec DESC');
|
|
SELECT compress_chunk(show_chunks('test1'));
|
|
compress_chunk
|
|
-----------------------------------------
|
|
_timescaledb_internal._hyper_5_15_chunk
|
|
(1 row)
|
|
|
|
select count(*) from test1;
|
|
count
|
|
-------
|
|
25
|
|
(1 row)
|
|
|
|
select * from amrels where relparent = 'test1'::regclass;
|
|
rel | amname | relparent
|
|
-----------------------------------------+--------+-----------
|
|
_timescaledb_internal._hyper_5_15_chunk | heap | test1
|
|
(1 row)
|
|
|
|
alter table test1 set access method hypercore;
|
|
copy test1 from stdin delimiter ',';
|
|
select count(*) from test1;
|
|
count
|
|
-------
|
|
26
|
|
(1 row)
|
|
|