postgres/timescaledb
1
0
Fork 0
mirror of https://github.com/timescale/timescaledb.git synced 2025-05-22 22:11:29 +08:00
Code Issues Packages Projects Releases Wiki Activity
timescaledb/tsl/test/expected/compression.out
Bharathy b869f91e25 Show warnings during create_hypertable(). 
The schema of base table on which hypertables are created, should define
columns with proper data types. As per postgres best practices Wiki
(https://wiki.postgresql.org/wiki/Don't_Do_This), one should not define
columns with CHAR, VARCHAR, VARCHAR(N), instead use TEXT data type.
Similarly instead of using timestamp, one should use timestamptz.
This patch reports a WARNING to end user when creating hypertables,
if underlying parent table, has columns of above mentioned data types.

Fixes #4335
2022-09-12 18:47:47 +05:30

1640 lines
68 KiB
Plaintext
Raw Blame History

-- This file and its contents are licensed under the Timescale License.
-- Please see the included NOTICE for copyright information and
-- LICENSE-TIMESCALE for a copy of the license.
SET timescaledb.enable_transparent_decompression to OFF;
\ir include/rand_generator.sql
-- This file and its contents are licensed under the Timescale License.
-- Please see the included NOTICE for copyright information and
-- LICENSE-TIMESCALE for a copy of the license.
--------------------------
-- cheap rand generator --
--------------------------
create table rand_minstd_state(i bigint);
create function rand_minstd_advance(bigint) returns bigint
language sql immutable as
$$
select (16807 * $1) % 2147483647
$$;
create function gen_rand_minstd() returns bigint
language sql security definer as
$$
update rand_minstd_state set i = rand_minstd_advance(i) returning i
$$;
-- seed the random num generator
insert into rand_minstd_state values (321);
--test_collation ---
--basic test with count
create table foo (a integer, b integer, c integer, d integer);
select table_name from create_hypertable('foo', 'a', chunk_time_interval=> 10);
NOTICE: adding not-null constraint to column "a"
table_name
------------
foo
(1 row)
create unique index foo_uniq ON foo (a, b);
--note that the "d" order by column is all NULL
insert into foo values( 3 , 16 , 20, NULL);
insert into foo values( 10 , 10 , 20, NULL);
insert into foo values( 20 , 11 , 20, NULL);
insert into foo values( 30 , 12 , 20, NULL);
alter table foo set (timescaledb.compress, timescaledb.compress_segmentby = 'a,b', timescaledb.compress_orderby = 'c desc, d asc nulls last');
--test self-refencing updates
SET timescaledb.enable_transparent_decompression to ON;
update foo set c = 40
where a = (SELECT max(a) FROM foo);
SET timescaledb.enable_transparent_decompression to OFF;
select id, schema_name, table_name, compression_state as compressed, compressed_hypertable_id from
_timescaledb_catalog.hypertable order by id;
id | schema_name | table_name | compressed | compressed_hypertable_id
----+-----------------------+--------------------------+------------+--------------------------
1 | public | foo | 1 | 2
2 | _timescaledb_internal | _compressed_hypertable_2 | 2 |
(2 rows)
select * from _timescaledb_catalog.hypertable_compression order by hypertable_id, attname;
hypertable_id | attname | compression_algorithm_id | segmentby_column_index | orderby_column_index | orderby_asc | orderby_nullsfirst
---------------+---------+--------------------------+------------------------+----------------------+-------------+--------------------
1 | a | 0 | 1 | | |
1 | b | 0 | 2 | | |
1 | c | 4 | | 1 | f | t
1 | d | 4 | | 2 | t | f
(4 rows)
select * from timescaledb_information.compression_settings ORDER BY hypertable_name;
hypertable_schema | hypertable_name | attname | segmentby_column_index | orderby_column_index | orderby_asc | orderby_nullsfirst
-------------------+-----------------+---------+------------------------+----------------------+-------------+--------------------
public | foo | a | 1 | | |
public | foo | b | 2 | | |
public | foo | c | | 1 | f | t
public | foo | d | | 2 | t | f
(4 rows)
-- TEST2 compress-chunk for the chunks created earlier --
select compress_chunk( '_timescaledb_internal._hyper_1_2_chunk');
compress_chunk
----------------------------------------
_timescaledb_internal._hyper_1_2_chunk
(1 row)
select tgname , tgtype, tgenabled , relname
from pg_trigger t, pg_class rel
where t.tgrelid = rel.oid and rel.relname like '_hyper_1_2_chunk' order by tgname;
tgname | tgtype | tgenabled | relname
--------+--------+-----------+---------
(0 rows)
\x
select * from chunk_compression_stats('foo')
order by chunk_name limit 2;
-[ RECORD 1 ]------------------+----------------------
chunk_schema | _timescaledb_internal
chunk_name | _hyper_1_1_chunk
compression_status | Uncompressed
before_compression_table_bytes |
before_compression_index_bytes |
before_compression_toast_bytes |
before_compression_total_bytes |
after_compression_table_bytes |
after_compression_index_bytes |
after_compression_toast_bytes |
after_compression_total_bytes |
node_name |
-[ RECORD 2 ]------------------+----------------------
chunk_schema | _timescaledb_internal
chunk_name | _hyper_1_2_chunk
compression_status | Compressed
before_compression_table_bytes | 8192
before_compression_index_bytes | 32768
before_compression_toast_bytes | 0
before_compression_total_bytes | 40960
after_compression_table_bytes | 8192
after_compression_index_bytes | 16384
after_compression_toast_bytes | 8192
after_compression_total_bytes | 32768
node_name |
\x
select compress_chunk( '_timescaledb_internal._hyper_1_1_chunk');
compress_chunk
----------------------------------------
_timescaledb_internal._hyper_1_1_chunk
(1 row)
\x
select * from _timescaledb_catalog.compression_chunk_size
order by chunk_id;
-[ RECORD 1 ]------------+------
chunk_id | 1
compressed_chunk_id | 6
uncompressed_heap_size | 8192
uncompressed_toast_size | 0
uncompressed_index_size | 32768
compressed_heap_size | 8192
compressed_toast_size | 8192
compressed_index_size | 16384
numrows_pre_compression | 1
numrows_post_compression | 1
-[ RECORD 2 ]------------+------
chunk_id | 2
compressed_chunk_id | 5
uncompressed_heap_size | 8192
uncompressed_toast_size | 0
uncompressed_index_size | 32768
compressed_heap_size | 8192
compressed_toast_size | 8192
compressed_index_size | 16384
numrows_pre_compression | 1
numrows_post_compression | 1
\x
select ch1.id, ch1.schema_name, ch1.table_name , ch2.table_name as compress_table
from
_timescaledb_catalog.chunk ch1, _timescaledb_catalog.chunk ch2
where ch1.compressed_chunk_id = ch2.id;
id | schema_name | table_name | compress_table
----+-----------------------+------------------+--------------------------
2 | _timescaledb_internal | _hyper_1_2_chunk | compress_hyper_2_5_chunk
1 | _timescaledb_internal | _hyper_1_1_chunk | compress_hyper_2_6_chunk
(2 rows)
\set ON_ERROR_STOP 0
--cannot recompress the chunk the second time around
select compress_chunk( '_timescaledb_internal._hyper_1_2_chunk');
ERROR: chunk "_hyper_1_2_chunk" is already compressed
--TEST2a try DML on a compressed chunk
insert into foo values( 11 , 10 , 20, 120);
ERROR: insert into a compressed chunk that has primary or unique constraint is not supported
update foo set b =20 where a = 10;
ERROR: cannot update/delete rows from chunk "_hyper_1_2_chunk" as it is compressed
delete from foo where a = 10;
ERROR: cannot update/delete rows from chunk "_hyper_1_2_chunk" as it is compressed
--TEST2b try complex DML on compressed chunk
create table foo_join ( a integer, newval integer);
select table_name from create_hypertable('foo_join', 'a', chunk_time_interval=> 10);
NOTICE: adding not-null constraint to column "a"
table_name
------------
foo_join
(1 row)
insert into foo_join select generate_series(0,40, 10), 111;
create table foo_join2 ( a integer, newval integer);
select table_name from create_hypertable('foo_join2', 'a', chunk_time_interval=> 10);
NOTICE: adding not-null constraint to column "a"
table_name
------------
foo_join2
(1 row)
insert into foo_join select generate_series(0,40, 10), 222;
update foo
set b = newval
from foo_join where foo.a = foo_join.a;
ERROR: cannot update/delete rows from chunk "_hyper_1_1_chunk" as it is compressed
update foo
set b = newval
from foo_join where foo.a = foo_join.a and foo_join.a > 10;
ERROR: cannot update/delete rows from chunk "_hyper_1_1_chunk" as it is compressed
--here the chunk gets excluded , so succeeds --
update foo
set b = newval
from foo_join where foo.a = foo_join.a and foo.a > 20;
update foo
set b = (select f1.newval from foo_join f1 left join lateral (select newval as newval2 from foo_join2 f2 where f1.a= f2.a ) subq on true limit 1);
ERROR: cannot update/delete rows from chunk "_hyper_1_1_chunk" as it is compressed
--upsert test --
insert into foo values(10, 12, 12, 12)
on conflict( a, b)
do update set b = excluded.b;
ERROR: insert with ON CONFLICT or RETURNING clause is not supported on compressed chunks
--TEST2c Do DML directly on the chunk.
insert into _timescaledb_internal._hyper_1_2_chunk values(10, 12, 12, 12);
update _timescaledb_internal._hyper_1_2_chunk
set b = 12;
ERROR: cannot update/delete rows from chunk "_hyper_1_2_chunk" as it is compressed
delete from _timescaledb_internal._hyper_1_2_chunk;
ERROR: cannot update/delete rows from chunk "_hyper_1_2_chunk" as it is compressed
--TEST2d decompress the chunk and try DML
select decompress_chunk( '_timescaledb_internal._hyper_1_2_chunk');
decompress_chunk
----------------------------------------
_timescaledb_internal._hyper_1_2_chunk
(1 row)
insert into foo values( 11 , 10 , 20, 120);
update foo set b =20 where a = 10;
ERROR: duplicate key value violates unique constraint "_hyper_1_2_chunk_foo_uniq"
select * from _timescaledb_internal._hyper_1_2_chunk order by a;
a | b | c | d
----+----+----+-----
10 | 12 | 12 | 12
10 | 10 | 20 |
11 | 10 | 20 | 120
(3 rows)
delete from foo where a = 10;
select * from _timescaledb_internal._hyper_1_2_chunk order by a;
a | b | c | d
----+----+----+-----
11 | 10 | 20 | 120
(1 row)
-- TEST3 check if compress data from views is accurate
CREATE TABLE conditions (
time TIMESTAMPTZ NOT NULL,
location TEXT NOT NULL,
location2 char(10) NOT NULL,
temperature DOUBLE PRECISION NULL,
humidity DOUBLE PRECISION NULL
);
select create_hypertable( 'conditions', 'time', chunk_time_interval=> '31days'::interval);
create_hypertable
-------------------------
(5,public,conditions,t)
(1 row)
alter table conditions set (timescaledb.compress, timescaledb.compress_segmentby = 'location', timescaledb.compress_orderby = 'time');
insert into conditions
select generate_series('2018-12-01 00:00'::timestamp, '2018-12-31 00:00'::timestamp, '1 day'), 'POR', 'klick', 55, 75;
insert into conditions
select generate_series('2018-12-01 00:00'::timestamp, '2018-12-31 00:00'::timestamp, '1 day'), 'NYC', 'klick', 55, 75;
select hypertable_id, attname, compression_algorithm_id , al.name
from _timescaledb_catalog.hypertable_compression hc,
_timescaledb_catalog.hypertable ht,
_timescaledb_catalog.compression_algorithm al
where ht.id = hc.hypertable_id and ht.table_name like 'conditions' and al.id = hc.compression_algorithm_id
ORDER BY hypertable_id, attname;
hypertable_id | attname | compression_algorithm_id | name
---------------+-------------+--------------------------+----------------------------------
5 | humidity | 3 | COMPRESSION_ALGORITHM_GORILLA
5 | location | 0 | COMPRESSION_ALGORITHM_NONE
5 | location2 | 2 | COMPRESSION_ALGORITHM_DICTIONARY
5 | temperature | 3 | COMPRESSION_ALGORITHM_GORILLA
5 | time | 4 | COMPRESSION_ALGORITHM_DELTADELTA
(5 rows)
select attname, attstorage, typname from pg_attribute at, pg_class cl , pg_type ty
where cl.oid = at.attrelid and at.attnum > 0
and cl.relname = '_compressed_hypertable_4'
and atttypid = ty.oid
order by at.attnum;
attname | attstorage | typname
---------+------------+---------
(0 rows)
SELECT ch1.schema_name|| '.' || ch1.table_name as "CHUNK_NAME", ch1.id "CHUNK_ID"
FROM _timescaledb_catalog.chunk ch1, _timescaledb_catalog.hypertable ht where ch1.hypertable_id = ht.id and ht.table_name like 'conditions'
ORDER BY ch1.id
LIMIT 1 \gset
SELECT count(*) from :CHUNK_NAME;
count
-------
42
(1 row)
SELECT count(*) as "ORIGINAL_CHUNK_COUNT" from :CHUNK_NAME \gset
select tableoid::regclass, count(*) from conditions group by tableoid order by tableoid;
tableoid | count
-----------------------------------------+-------
_timescaledb_internal._hyper_5_12_chunk | 42
_timescaledb_internal._hyper_5_13_chunk | 20
(2 rows)
select compress_chunk(ch1.schema_name|| '.' || ch1.table_name)
FROM _timescaledb_catalog.chunk ch1, _timescaledb_catalog.hypertable ht where ch1.hypertable_id = ht.id and ht.table_name like 'conditions' ORDER BY ch1.id limit 1;
compress_chunk
-----------------------------------------
_timescaledb_internal._hyper_5_12_chunk
(1 row)
--test that only one chunk was affected
--note tables with 0 rows will not show up in here.
select tableoid::regclass, count(*) from conditions group by tableoid order by tableoid;
tableoid | count
-----------------------------------------+-------
_timescaledb_internal._hyper_5_13_chunk | 20
(1 row)
select compress_chunk(ch1.schema_name|| '.' || ch1.table_name)
FROM _timescaledb_catalog.chunk ch1, _timescaledb_catalog.hypertable ht where ch1.hypertable_id = ht.id and ht.table_name like 'conditions' and ch1.compressed_chunk_id IS NULL;
compress_chunk
-----------------------------------------
_timescaledb_internal._hyper_5_13_chunk
(1 row)
select tableoid::regclass, count(*) from conditions group by tableoid order by tableoid;
tableoid | count
----------+-------
(0 rows)
select compressed.schema_name|| '.' || compressed.table_name as "COMPRESSED_CHUNK_NAME"
from _timescaledb_catalog.chunk uncompressed, _timescaledb_catalog.chunk compressed
where uncompressed.compressed_chunk_id = compressed.id AND uncompressed.id = :'CHUNK_ID' \gset
SELECT count(*) from :CHUNK_NAME;
count
-------
0
(1 row)
SELECT count(*) from :COMPRESSED_CHUNK_NAME;
count
-------
2
(1 row)
SELECT sum(_ts_meta_count) from :COMPRESSED_CHUNK_NAME;
sum
-----
42
(1 row)
SELECT location, _ts_meta_sequence_num from :COMPRESSED_CHUNK_NAME ORDER BY 1,2;
location | _ts_meta_sequence_num
----------+-----------------------
NYC | 10
POR | 10
(2 rows)
\x
SELECT chunk_id, numrows_pre_compression, numrows_post_compression
FROM _timescaledb_catalog.chunk srcch,
_timescaledb_catalog.compression_chunk_size map,
_timescaledb_catalog.hypertable srcht
WHERE map.chunk_id = srcch.id and srcht.id = srcch.hypertable_id
and srcht.table_name like 'conditions'
order by chunk_id;
-[ RECORD 1 ]------------+---
chunk_id | 12
numrows_pre_compression | 42
numrows_post_compression | 2
-[ RECORD 2 ]------------+---
chunk_id | 13
numrows_pre_compression | 20
numrows_post_compression | 2
select * from chunk_compression_stats('conditions')
order by chunk_name;
-[ RECORD 1 ]------------------+----------------------
chunk_schema | _timescaledb_internal
chunk_name | _hyper_5_12_chunk
compression_status | Compressed
before_compression_table_bytes | 8192
before_compression_index_bytes | 16384
before_compression_toast_bytes | 8192
before_compression_total_bytes | 32768
after_compression_table_bytes | 8192
after_compression_index_bytes | 16384
after_compression_toast_bytes | 8192
after_compression_total_bytes | 32768
node_name |
-[ RECORD 2 ]------------------+----------------------
chunk_schema | _timescaledb_internal
chunk_name | _hyper_5_13_chunk
compression_status | Compressed
before_compression_table_bytes | 8192
before_compression_index_bytes | 16384
before_compression_toast_bytes | 8192
before_compression_total_bytes | 32768
after_compression_table_bytes | 8192
after_compression_index_bytes | 16384
after_compression_toast_bytes | 8192
after_compression_total_bytes | 32768
node_name |
select * from hypertable_compression_stats('foo');
-[ RECORD 1 ]------------------+------
total_chunks | 4
number_compressed_chunks | 1
before_compression_table_bytes | 8192
before_compression_index_bytes | 32768
before_compression_toast_bytes | 0
before_compression_total_bytes | 40960
after_compression_table_bytes | 8192
after_compression_index_bytes | 16384
after_compression_toast_bytes | 8192
after_compression_total_bytes | 32768
node_name |
select * from hypertable_compression_stats('conditions');
-[ RECORD 1 ]------------------+------
total_chunks | 2
number_compressed_chunks | 2
before_compression_table_bytes | 16384
before_compression_index_bytes | 32768
before_compression_toast_bytes | 16384
before_compression_total_bytes | 65536
after_compression_table_bytes | 16384
after_compression_index_bytes | 32768
after_compression_toast_bytes | 16384
after_compression_total_bytes | 65536
node_name |
vacuum full foo;
vacuum full conditions;
-- After vacuum, table_bytes is 0, but any associated index/toast storage is not
-- completely reclaimed. Sets it at 8K (page size). So a chunk which has
-- been compressed still incurs an overhead of n * 8KB (for every index + toast table) storage on the original uncompressed chunk.
select pg_size_pretty(table_bytes), pg_size_pretty(index_bytes),
pg_size_pretty(toast_bytes), pg_size_pretty(total_bytes)
from hypertable_detailed_size('foo');
-[ RECORD 1 ]--+-----------
pg_size_pretty | 32 kB
pg_size_pretty | 144 kB
pg_size_pretty | 8192 bytes
pg_size_pretty | 184 kB
select pg_size_pretty(table_bytes), pg_size_pretty(index_bytes),
pg_size_pretty(toast_bytes), pg_size_pretty(total_bytes)
from hypertable_detailed_size('conditions');
-[ RECORD 1 ]--+-------
pg_size_pretty | 16 kB
pg_size_pretty | 56 kB
pg_size_pretty | 40 kB
pg_size_pretty | 112 kB
select * from timescaledb_information.hypertables
where hypertable_name like 'foo' or hypertable_name like 'conditions'
order by hypertable_name;
-[ RECORD 1 ]-------+------------------
hypertable_schema | public
hypertable_name | conditions
owner | default_perm_user
num_dimensions | 1
num_chunks | 2
compression_enabled | t
is_distributed | f
replication_factor |
data_nodes |
tablespaces |
-[ RECORD 2 ]-------+------------------
hypertable_schema | public
hypertable_name | foo
owner | default_perm_user
num_dimensions | 1
num_chunks | 4
compression_enabled | t
is_distributed | f
replication_factor |
data_nodes |
tablespaces |
\x
SELECT decompress_chunk(ch1.schema_name|| '.' || ch1.table_name) AS chunk
FROM _timescaledb_catalog.chunk ch1, _timescaledb_catalog.hypertable ht
WHERE ch1.hypertable_id = ht.id and ht.table_name LIKE 'conditions'
ORDER BY chunk;
chunk
-----------------------------------------
_timescaledb_internal._hyper_5_12_chunk
_timescaledb_internal._hyper_5_13_chunk
(2 rows)
SELECT count(*), count(*) = :'ORIGINAL_CHUNK_COUNT' from :CHUNK_NAME;
count | ?column?
-------+----------
42 | t
(1 row)
--check that the compressed chunk is dropped
\set ON_ERROR_STOP 0
SELECT count(*) from :COMPRESSED_CHUNK_NAME;
ERROR: relation "_timescaledb_internal.compress_hyper_6_14_chunk" does not exist at character 22
\set ON_ERROR_STOP 1
--size information is gone too
select count(*)
FROM _timescaledb_catalog.chunk ch1, _timescaledb_catalog.hypertable ht,
_timescaledb_catalog.compression_chunk_size map
where ch1.hypertable_id = ht.id and ht.table_name like 'conditions'
and map.chunk_id = ch1.id;
count
-------
0
(1 row)
--make sure compressed_chunk_id is reset to NULL
select ch1.compressed_chunk_id IS NULL
FROM _timescaledb_catalog.chunk ch1, _timescaledb_catalog.hypertable ht where ch1.hypertable_id = ht.id and ht.table_name like 'conditions';
?column?
----------
t
t
(2 rows)
-- test plans get invalidated when chunks get compressed
SET timescaledb.enable_transparent_decompression TO ON;
CREATE TABLE plan_inval(time timestamptz, device_id int);
SELECT create_hypertable('plan_inval','time');
NOTICE: adding not-null constraint to column "time"
create_hypertable
-------------------------
(7,public,plan_inval,t)
(1 row)
ALTER TABLE plan_inval SET (timescaledb.compress,timescaledb.compress_orderby='time desc');
-- create 2 chunks
INSERT INTO plan_inval SELECT * FROM (VALUES ('2000-01-01'::timestamptz,1), ('2000-01-07'::timestamptz,1)) v(time,device_id);
SET max_parallel_workers_per_gather to 0;
PREPARE prep_plan AS SELECT count(*) FROM plan_inval;
EXECUTE prep_plan;
count
-------
2
(1 row)
EXECUTE prep_plan;
count
-------
2
(1 row)
EXECUTE prep_plan;
count
-------
2
(1 row)
-- get name of first chunk
SELECT tableoid::regclass AS "CHUNK_NAME" FROM plan_inval ORDER BY time LIMIT 1
\gset
SELECT compress_chunk(:'CHUNK_NAME');
compress_chunk
-----------------------------------------
_timescaledb_internal._hyper_7_16_chunk
(1 row)
EXECUTE prep_plan;
count
-------
2
(1 row)
EXPLAIN (COSTS OFF) EXECUTE prep_plan;
QUERY PLAN
----------------------------------------------------------------
Aggregate
-> Append
-> Custom Scan (DecompressChunk) on _hyper_7_16_chunk
-> Seq Scan on compress_hyper_8_18_chunk
-> Seq Scan on _hyper_7_17_chunk
(5 rows)
CREATE TABLE test_collation (
time TIMESTAMPTZ NOT NULL,
device_id TEXT COLLATE "C" NULL,
device_id_2 TEXT COLLATE "POSIX" NULL,
val_1 TEXT COLLATE "C" NULL,
val_2 TEXT COLLATE "POSIX" NULL
);
--we want all the data to go into 1 chunk. so use 1 year chunk interval
select create_hypertable( 'test_collation', 'time', chunk_time_interval=> '1 day'::interval);
create_hypertable
-----------------------------
(9,public,test_collation,t)
(1 row)
\set ON_ERROR_STOP 0
--forbid setting collation in compression ORDER BY clause. (parse error is fine)
alter table test_collation set (timescaledb.compress, timescaledb.compress_segmentby='device_id, device_id_2', timescaledb.compress_orderby = 'val_1 COLLATE "POSIX", val2, time');
ERROR: unable to parse ordering option "val_1 COLLATE "POSIX", val2, time"
\set ON_ERROR_STOP 1
alter table test_collation set (timescaledb.compress, timescaledb.compress_segmentby='device_id, device_id_2', timescaledb.compress_orderby = 'val_1, val_2, time');
insert into test_collation
select generate_series('2018-01-01 00:00'::timestamp, '2018-01-10 00:00'::timestamp, '2 hour'), 'device_1', 'device_3', gen_rand_minstd(), gen_rand_minstd();
insert into test_collation
select generate_series('2018-01-01 00:00'::timestamp, '2018-01-10 00:00'::timestamp, '2 hour'), 'device_2', 'device_4', gen_rand_minstd(), gen_rand_minstd();
insert into test_collation
select generate_series('2018-01-01 00:00'::timestamp, '2018-01-10 00:00'::timestamp, '2 hour'), NULL, 'device_5', gen_rand_minstd(), gen_rand_minstd();
--compress 2 chunks
SELECT compress_chunk(ch1.schema_name|| '.' || ch1.table_name)
FROM _timescaledb_catalog.chunk ch1, _timescaledb_catalog.hypertable ht where ch1.hypertable_id = ht.id
and ht.table_name like 'test_collation' ORDER BY ch1.id LIMIT 2;
compress_chunk
-----------------------------------------
_timescaledb_internal._hyper_9_19_chunk
_timescaledb_internal._hyper_9_20_chunk
(2 rows)
--segment bys are pushed down correctly
EXPLAIN (costs off) SELECT * FROM test_collation WHERE device_id < 'a';
QUERY PLAN
----------------------------------------------------------
Append
-> Custom Scan (DecompressChunk) on _hyper_9_19_chunk
-> Seq Scan on compress_hyper_10_29_chunk
Filter: (device_id < 'a'::text)
-> Custom Scan (DecompressChunk) on _hyper_9_20_chunk
-> Seq Scan on compress_hyper_10_30_chunk
Filter: (device_id < 'a'::text)
-> Seq Scan on _hyper_9_21_chunk
Filter: (device_id < 'a'::text)
-> Seq Scan on _hyper_9_22_chunk
Filter: (device_id < 'a'::text)
-> Seq Scan on _hyper_9_23_chunk
Filter: (device_id < 'a'::text)
-> Seq Scan on _hyper_9_24_chunk
Filter: (device_id < 'a'::text)
-> Seq Scan on _hyper_9_25_chunk
Filter: (device_id < 'a'::text)
-> Seq Scan on _hyper_9_26_chunk
Filter: (device_id < 'a'::text)
-> Seq Scan on _hyper_9_27_chunk
Filter: (device_id < 'a'::text)
-> Seq Scan on _hyper_9_28_chunk
Filter: (device_id < 'a'::text)
(23 rows)
EXPLAIN (costs off) SELECT * FROM test_collation WHERE device_id < 'a' COLLATE "POSIX";
QUERY PLAN
---------------------------------------------------------------
Append
-> Custom Scan (DecompressChunk) on _hyper_9_19_chunk
-> Seq Scan on compress_hyper_10_29_chunk
Filter: (device_id < 'a'::text COLLATE "POSIX")
-> Custom Scan (DecompressChunk) on _hyper_9_20_chunk
-> Seq Scan on compress_hyper_10_30_chunk
Filter: (device_id < 'a'::text COLLATE "POSIX")
-> Seq Scan on _hyper_9_21_chunk
Filter: (device_id < 'a'::text COLLATE "POSIX")
-> Seq Scan on _hyper_9_22_chunk
Filter: (device_id < 'a'::text COLLATE "POSIX")
-> Seq Scan on _hyper_9_23_chunk
Filter: (device_id < 'a'::text COLLATE "POSIX")
-> Seq Scan on _hyper_9_24_chunk
Filter: (device_id < 'a'::text COLLATE "POSIX")
-> Seq Scan on _hyper_9_25_chunk
Filter: (device_id < 'a'::text COLLATE "POSIX")
-> Seq Scan on _hyper_9_26_chunk
Filter: (device_id < 'a'::text COLLATE "POSIX")
-> Seq Scan on _hyper_9_27_chunk
Filter: (device_id < 'a'::text COLLATE "POSIX")
-> Seq Scan on _hyper_9_28_chunk
Filter: (device_id < 'a'::text COLLATE "POSIX")
(23 rows)
\set ON_ERROR_STOP 0
EXPLAIN (costs off) SELECT * FROM test_collation WHERE device_id COLLATE "POSIX" < device_id_2 COLLATE "C";
ERROR: collation mismatch between explicit collations "POSIX" and "C" at character 96
SELECT device_id < device_id_2 FROM test_collation;
ERROR: could not determine which collation to use for string comparison
\set ON_ERROR_STOP 1
--segment meta on order bys pushdown
--should work
EXPLAIN (costs off) SELECT * FROM test_collation WHERE val_1 < 'a';
QUERY PLAN
----------------------------------------------------------
Append
-> Custom Scan (DecompressChunk) on _hyper_9_19_chunk
Filter: (val_1 < 'a'::text)
-> Seq Scan on compress_hyper_10_29_chunk
Filter: (_ts_meta_min_1 < 'a'::text)
-> Custom Scan (DecompressChunk) on _hyper_9_20_chunk
Filter: (val_1 < 'a'::text)
-> Seq Scan on compress_hyper_10_30_chunk
Filter: (_ts_meta_min_1 < 'a'::text)
-> Seq Scan on _hyper_9_21_chunk
Filter: (val_1 < 'a'::text)
-> Seq Scan on _hyper_9_22_chunk
Filter: (val_1 < 'a'::text)
-> Seq Scan on _hyper_9_23_chunk
Filter: (val_1 < 'a'::text)
-> Seq Scan on _hyper_9_24_chunk
Filter: (val_1 < 'a'::text)
-> Seq Scan on _hyper_9_25_chunk
Filter: (val_1 < 'a'::text)
-> Seq Scan on _hyper_9_26_chunk
Filter: (val_1 < 'a'::text)
-> Seq Scan on _hyper_9_27_chunk
Filter: (val_1 < 'a'::text)
-> Seq Scan on _hyper_9_28_chunk
Filter: (val_1 < 'a'::text)
(25 rows)
EXPLAIN (costs off) SELECT * FROM test_collation WHERE val_2 < 'a';
QUERY PLAN
----------------------------------------------------------
Append
-> Custom Scan (DecompressChunk) on _hyper_9_19_chunk
Filter: (val_2 < 'a'::text)
-> Seq Scan on compress_hyper_10_29_chunk
Filter: (_ts_meta_min_2 < 'a'::text)
-> Custom Scan (DecompressChunk) on _hyper_9_20_chunk
Filter: (val_2 < 'a'::text)
-> Seq Scan on compress_hyper_10_30_chunk
Filter: (_ts_meta_min_2 < 'a'::text)
-> Seq Scan on _hyper_9_21_chunk
Filter: (val_2 < 'a'::text)
-> Seq Scan on _hyper_9_22_chunk
Filter: (val_2 < 'a'::text)
-> Seq Scan on _hyper_9_23_chunk
Filter: (val_2 < 'a'::text)
-> Seq Scan on _hyper_9_24_chunk
Filter: (val_2 < 'a'::text)
-> Seq Scan on _hyper_9_25_chunk
Filter: (val_2 < 'a'::text)
-> Seq Scan on _hyper_9_26_chunk
Filter: (val_2 < 'a'::text)
-> Seq Scan on _hyper_9_27_chunk
Filter: (val_2 < 'a'::text)
-> Seq Scan on _hyper_9_28_chunk
Filter: (val_2 < 'a'::text)
(25 rows)
EXPLAIN (costs off) SELECT * FROM test_collation WHERE val_1 < 'a' COLLATE "C";
QUERY PLAN
----------------------------------------------------------------
Append
-> Custom Scan (DecompressChunk) on _hyper_9_19_chunk
Filter: (val_1 < 'a'::text COLLATE "C")
-> Seq Scan on compress_hyper_10_29_chunk
Filter: (_ts_meta_min_1 < 'a'::text COLLATE "C")
-> Custom Scan (DecompressChunk) on _hyper_9_20_chunk
Filter: (val_1 < 'a'::text COLLATE "C")
-> Seq Scan on compress_hyper_10_30_chunk
Filter: (_ts_meta_min_1 < 'a'::text COLLATE "C")
-> Seq Scan on _hyper_9_21_chunk
Filter: (val_1 < 'a'::text COLLATE "C")
-> Seq Scan on _hyper_9_22_chunk
Filter: (val_1 < 'a'::text COLLATE "C")
-> Seq Scan on _hyper_9_23_chunk
Filter: (val_1 < 'a'::text COLLATE "C")
-> Seq Scan on _hyper_9_24_chunk
Filter: (val_1 < 'a'::text COLLATE "C")
-> Seq Scan on _hyper_9_25_chunk
Filter: (val_1 < 'a'::text COLLATE "C")
-> Seq Scan on _hyper_9_26_chunk
Filter: (val_1 < 'a'::text COLLATE "C")
-> Seq Scan on _hyper_9_27_chunk
Filter: (val_1 < 'a'::text COLLATE "C")
-> Seq Scan on _hyper_9_28_chunk
Filter: (val_1 < 'a'::text COLLATE "C")
(25 rows)
EXPLAIN (costs off) SELECT * FROM test_collation WHERE val_2 < 'a' COLLATE "POSIX";
QUERY PLAN
--------------------------------------------------------------------
Append
-> Custom Scan (DecompressChunk) on _hyper_9_19_chunk
Filter: (val_2 < 'a'::text COLLATE "POSIX")
-> Seq Scan on compress_hyper_10_29_chunk
Filter: (_ts_meta_min_2 < 'a'::text COLLATE "POSIX")
-> Custom Scan (DecompressChunk) on _hyper_9_20_chunk
Filter: (val_2 < 'a'::text COLLATE "POSIX")
-> Seq Scan on compress_hyper_10_30_chunk
Filter: (_ts_meta_min_2 < 'a'::text COLLATE "POSIX")
-> Seq Scan on _hyper_9_21_chunk
Filter: (val_2 < 'a'::text COLLATE "POSIX")
-> Seq Scan on _hyper_9_22_chunk
Filter: (val_2 < 'a'::text COLLATE "POSIX")
-> Seq Scan on _hyper_9_23_chunk
Filter: (val_2 < 'a'::text COLLATE "POSIX")
-> Seq Scan on _hyper_9_24_chunk
Filter: (val_2 < 'a'::text COLLATE "POSIX")
-> Seq Scan on _hyper_9_25_chunk
Filter: (val_2 < 'a'::text COLLATE "POSIX")
-> Seq Scan on _hyper_9_26_chunk
Filter: (val_2 < 'a'::text COLLATE "POSIX")
-> Seq Scan on _hyper_9_27_chunk
Filter: (val_2 < 'a'::text COLLATE "POSIX")
-> Seq Scan on _hyper_9_28_chunk
Filter: (val_2 < 'a'::text COLLATE "POSIX")
(25 rows)
--cannot pushdown when op collation does not match column's collation since min/max used different collation than what op needs
EXPLAIN (costs off) SELECT * FROM test_collation WHERE val_1 < 'a' COLLATE "POSIX";
QUERY PLAN
----------------------------------------------------------
Append
-> Custom Scan (DecompressChunk) on _hyper_9_19_chunk
Filter: (val_1 < 'a'::text COLLATE "POSIX")
-> Seq Scan on compress_hyper_10_29_chunk
-> Custom Scan (DecompressChunk) on _hyper_9_20_chunk
Filter: (val_1 < 'a'::text COLLATE "POSIX")
-> Seq Scan on compress_hyper_10_30_chunk
-> Seq Scan on _hyper_9_21_chunk
Filter: (val_1 < 'a'::text COLLATE "POSIX")
-> Seq Scan on _hyper_9_22_chunk
Filter: (val_1 < 'a'::text COLLATE "POSIX")
-> Seq Scan on _hyper_9_23_chunk
Filter: (val_1 < 'a'::text COLLATE "POSIX")
-> Seq Scan on _hyper_9_24_chunk
Filter: (val_1 < 'a'::text COLLATE "POSIX")
-> Seq Scan on _hyper_9_25_chunk
Filter: (val_1 < 'a'::text COLLATE "POSIX")
-> Seq Scan on _hyper_9_26_chunk
Filter: (val_1 < 'a'::text COLLATE "POSIX")
-> Seq Scan on _hyper_9_27_chunk
Filter: (val_1 < 'a'::text COLLATE "POSIX")
-> Seq Scan on _hyper_9_28_chunk
Filter: (val_1 < 'a'::text COLLATE "POSIX")
(23 rows)
EXPLAIN (costs off) SELECT * FROM test_collation WHERE val_2 < 'a' COLLATE "C";
QUERY PLAN
----------------------------------------------------------
Append
-> Custom Scan (DecompressChunk) on _hyper_9_19_chunk
Filter: (val_2 < 'a'::text COLLATE "C")
-> Seq Scan on compress_hyper_10_29_chunk
-> Custom Scan (DecompressChunk) on _hyper_9_20_chunk
Filter: (val_2 < 'a'::text COLLATE "C")
-> Seq Scan on compress_hyper_10_30_chunk
-> Seq Scan on _hyper_9_21_chunk
Filter: (val_2 < 'a'::text COLLATE "C")
-> Seq Scan on _hyper_9_22_chunk
Filter: (val_2 < 'a'::text COLLATE "C")
-> Seq Scan on _hyper_9_23_chunk
Filter: (val_2 < 'a'::text COLLATE "C")
-> Seq Scan on _hyper_9_24_chunk
Filter: (val_2 < 'a'::text COLLATE "C")
-> Seq Scan on _hyper_9_25_chunk
Filter: (val_2 < 'a'::text COLLATE "C")
-> Seq Scan on _hyper_9_26_chunk
Filter: (val_2 < 'a'::text COLLATE "C")
-> Seq Scan on _hyper_9_27_chunk
Filter: (val_2 < 'a'::text COLLATE "C")
-> Seq Scan on _hyper_9_28_chunk
Filter: (val_2 < 'a'::text COLLATE "C")
(23 rows)
--test datatypes
CREATE TABLE datatype_test(
time timestamptz NOT NULL,
int2_column int2,
int4_column int4,
int8_column int8,
float4_column float4,
float8_column float8,
date_column date,
timestamp_column timestamp,
timestamptz_column timestamptz,
interval_column interval,
numeric_column numeric,
decimal_column decimal,
text_column text,
char_column char
);
SELECT create_hypertable('datatype_test','time');
WARNING: column type "timestamp without time zone" used for "timestamp_column" does not follow best practices
create_hypertable
-----------------------------
(11,public,datatype_test,t)
(1 row)
ALTER TABLE datatype_test SET (timescaledb.compress);
INSERT INTO datatype_test VALUES ('2000-01-01',2,4,8,4.0,8.0,'2000-01-01','2001-01-01 12:00','2001-01-01 6:00','1 week', 3.41, 4.2, 'text', 'x');
SELECT compress_chunk(ch1.schema_name|| '.' || ch1.table_name)
FROM _timescaledb_catalog.chunk ch1, _timescaledb_catalog.hypertable ht where ch1.hypertable_id = ht.id
and ht.table_name like 'datatype_test' ORDER BY ch1.id;
compress_chunk
------------------------------------------
_timescaledb_internal._hyper_11_31_chunk
(1 row)
SELECT
attname, alg.name
FROM _timescaledb_catalog.hypertable ht
INNER JOIN _timescaledb_catalog.hypertable_compression htc ON ht.id=htc.hypertable_id
INNER JOIN _timescaledb_catalog.compression_algorithm alg ON alg.id=htc.compression_algorithm_id
WHERE ht.table_name='datatype_test'
ORDER BY attname;
attname | name
--------------------+----------------------------------
char_column | COMPRESSION_ALGORITHM_DICTIONARY
date_column | COMPRESSION_ALGORITHM_DELTADELTA
decimal_column | COMPRESSION_ALGORITHM_ARRAY
float4_column | COMPRESSION_ALGORITHM_GORILLA
float8_column | COMPRESSION_ALGORITHM_GORILLA
int2_column | COMPRESSION_ALGORITHM_DELTADELTA
int4_column | COMPRESSION_ALGORITHM_DELTADELTA
int8_column | COMPRESSION_ALGORITHM_DELTADELTA
interval_column | COMPRESSION_ALGORITHM_DICTIONARY
numeric_column | COMPRESSION_ALGORITHM_ARRAY
text_column | COMPRESSION_ALGORITHM_DICTIONARY
time | COMPRESSION_ALGORITHM_DELTADELTA
timestamp_column | COMPRESSION_ALGORITHM_DELTADELTA
timestamptz_column | COMPRESSION_ALGORITHM_DELTADELTA
(14 rows)
--TEST try to compress a hypertable that has a continuous aggregate
CREATE TABLE metrics(time timestamptz, device_id int, v1 float, v2 float);
SELECT create_hypertable('metrics','time');
NOTICE: adding not-null constraint to column "time"
create_hypertable
-----------------------
(13,public,metrics,t)
(1 row)
INSERT INTO metrics SELECT generate_series('2000-01-01'::timestamptz,'2000-01-10','1m'),1,0.25,0.75;
-- check expressions in view definition
CREATE MATERIALIZED VIEW cagg_expr WITH (timescaledb.continuous)
AS
SELECT
time_bucket('1d', time) AS time,
'Const'::text AS Const,
4.3::numeric AS "numeric",
first(metrics,time),
CASE WHEN true THEN 'foo' ELSE 'bar' END,
COALESCE(NULL,'coalesce'),
avg(v1) + avg(v2) AS avg1,
avg(v1+v2) AS avg2,
count(*) AS cnt
FROM metrics
GROUP BY 1 WITH NO DATA;
CALL refresh_continuous_aggregate('cagg_expr', NULL, NULL);
SELECT * FROM cagg_expr ORDER BY time LIMIT 5;
time | const | numeric | first | case | coalesce | avg1 | avg2 | cnt
------------------------------+-------+---------+----------------------------------------------+------+----------+------+------+------
Fri Dec 31 16:00:00 1999 PST | Const | 4.3 | ("Sat Jan 01 00:00:00 2000 PST",1,0.25,0.75) | foo | coalesce | 1 | 1 | 960
Sat Jan 01 16:00:00 2000 PST | Const | 4.3 | ("Sat Jan 01 16:00:00 2000 PST",1,0.25,0.75) | foo | coalesce | 1 | 1 | 1440
Sun Jan 02 16:00:00 2000 PST | Const | 4.3 | ("Sun Jan 02 16:00:00 2000 PST",1,0.25,0.75) | foo | coalesce | 1 | 1 | 1440
Mon Jan 03 16:00:00 2000 PST | Const | 4.3 | ("Mon Jan 03 16:00:00 2000 PST",1,0.25,0.75) | foo | coalesce | 1 | 1 | 1440
Tue Jan 04 16:00:00 2000 PST | Const | 4.3 | ("Tue Jan 04 16:00:00 2000 PST",1,0.25,0.75) | foo | coalesce | 1 | 1 | 1440
(5 rows)
ALTER TABLE metrics set(timescaledb.compress);
-- test rescan in compress chunk dml blocker
CREATE TABLE rescan_test(id integer NOT NULL, t timestamptz NOT NULL, val double precision, PRIMARY KEY(id, t));
SELECT create_hypertable('rescan_test', 't', chunk_time_interval => interval '1 day');
create_hypertable
---------------------------
(16,public,rescan_test,t)
(1 row)
-- compression
ALTER TABLE rescan_test SET (timescaledb.compress, timescaledb.compress_segmentby = 'id');
-- INSERT dummy data
INSERT INTO rescan_test SELECT 1, time, random() FROM generate_series('2000-01-01'::timestamptz, '2000-01-05'::timestamptz, '1h'::interval) g(time);
SELECT count(*) FROM rescan_test;
count
-------
97
(1 row)
-- compress first chunk
SELECT compress_chunk(ch1.schema_name|| '.' || ch1.table_name)
FROM _timescaledb_catalog.chunk ch1, _timescaledb_catalog.hypertable ht where ch1.hypertable_id = ht.id
and ht.table_name like 'rescan_test' ORDER BY ch1.id LIMIT 1;
compress_chunk
------------------------------------------
_timescaledb_internal._hyper_16_36_chunk
(1 row)
-- count should be equal to count before compression
SELECT count(*) FROM rescan_test;
count
-------
97
(1 row)
-- single row update is fine
UPDATE rescan_test SET val = val + 1 WHERE rescan_test.id = 1 AND rescan_test.t = '2000-01-03 00:00:00+00';
-- multi row update via WHERE is fine
UPDATE rescan_test SET val = val + 1 WHERE rescan_test.id = 1 AND rescan_test.t > '2000-01-03 00:00:00+00';
-- single row update with FROM is allowed if no compressed chunks are hit
UPDATE rescan_test SET val = tmp.val
FROM (SELECT x.id, x.t, x.val FROM unnest(array[(1, '2000-01-03 00:00:00+00', 2.045)]::rescan_test[]) AS x) AS tmp
WHERE rescan_test.id = tmp.id AND rescan_test.t = tmp.t AND rescan_test.t >= '2000-01-03';
-- single row update with FROM is blocked
\set ON_ERROR_STOP 0
UPDATE rescan_test SET val = tmp.val
FROM (SELECT x.id, x.t, x.val FROM unnest(array[(1, '2000-01-03 00:00:00+00', 2.045)]::rescan_test[]) AS x) AS tmp
WHERE rescan_test.id = tmp.id AND rescan_test.t = tmp.t;
ERROR: cannot update/delete rows from chunk "_hyper_16_36_chunk" as it is compressed
-- bulk row update with FROM is blocked
UPDATE rescan_test SET val = tmp.val
FROM (SELECT x.id, x.t, x.val FROM unnest(array[(1, '2000-01-03 00:00:00+00', 2.045), (1, '2000-01-03 01:00:00+00', 8.045)]::rescan_test[]) AS x) AS tmp
WHERE rescan_test.id = tmp.id AND rescan_test.t = tmp.t;
ERROR: cannot update/delete rows from chunk "_hyper_16_36_chunk" as it is compressed
\set ON_ERROR_STOP 1
-- Test FK constraint drop and recreate during compression and decompression on a chunk
CREATE TABLE meta (device_id INT PRIMARY KEY);
CREATE TABLE hyper(
time INT NOT NULL,
device_id INT REFERENCES meta(device_id) ON DELETE CASCADE ON UPDATE CASCADE,
val INT);
SELECT * FROM create_hypertable('hyper', 'time', chunk_time_interval => 10);
hypertable_id | schema_name | table_name | created
---------------+-------------+------------+---------
18 | public | hyper | t
(1 row)
ALTER TABLE hyper SET (
timescaledb.compress,
timescaledb.compress_orderby = 'time',
timescaledb.compress_segmentby = 'device_id');
INSERT INTO meta VALUES (1), (2), (3), (4), (5);
INSERT INTO hyper VALUES (1, 1, 1), (2, 2, 1), (3, 3, 1), (10, 3, 2), (11, 4, 2), (11, 5, 2);
SELECT ch1.table_name AS "CHUNK_NAME", ch1.schema_name|| '.' || ch1.table_name AS "CHUNK_FULL_NAME"
FROM _timescaledb_catalog.chunk ch1, _timescaledb_catalog.hypertable ht
WHERE ch1.hypertable_id = ht.id AND ht.table_name LIKE 'hyper'
ORDER BY ch1.id LIMIT 1 \gset
SELECT constraint_schema, constraint_name, table_schema, table_name, constraint_type
FROM information_schema.table_constraints
WHERE table_name = :'CHUNK_NAME' AND constraint_type = 'FOREIGN KEY'
ORDER BY constraint_name;
constraint_schema | constraint_name | table_schema | table_name | constraint_type
-----------------------+---------------------------+-----------------------+--------------------+-----------------
_timescaledb_internal | 42_6_hyper_device_id_fkey | _timescaledb_internal | _hyper_18_42_chunk | FOREIGN KEY
(1 row)
SELECT compress_chunk(:'CHUNK_FULL_NAME');
compress_chunk
------------------------------------------
_timescaledb_internal._hyper_18_42_chunk
(1 row)
SELECT constraint_schema, constraint_name, table_schema, table_name, constraint_type
FROM information_schema.table_constraints
WHERE table_name = :'CHUNK_NAME' AND constraint_type = 'FOREIGN KEY'
ORDER BY constraint_name;
constraint_schema | constraint_name | table_schema | table_name | constraint_type
-------------------+-----------------+--------------+------------+-----------------
(0 rows)
-- Delete data from compressed chunk directly fails
\set ON_ERROR_STOP 0
DELETE FROM hyper WHERE device_id = 3;
ERROR: cannot update/delete rows from chunk "_hyper_18_42_chunk" as it is compressed
\set ON_ERROR_STOP 0
-- Delete data from FK-referenced table deletes data from compressed chunk
SELECT * FROM hyper ORDER BY time, device_id;
time | device_id | val
------+-----------+-----
1 | 1 | 1
2 | 2 | 1
3 | 3 | 1
10 | 3 | 2
11 | 4 | 2
11 | 5 | 2
(6 rows)
DELETE FROM meta WHERE device_id = 3;
SELECT * FROM hyper ORDER BY time, device_id;
time | device_id | val
------+-----------+-----
1 | 1 | 1
2 | 2 | 1
11 | 4 | 2
11 | 5 | 2
(4 rows)
SELECT decompress_chunk(:'CHUNK_FULL_NAME');
decompress_chunk
------------------------------------------
_timescaledb_internal._hyper_18_42_chunk
(1 row)
SELECT constraint_schema, constraint_name, table_schema, table_name, constraint_type
FROM information_schema.table_constraints
WHERE table_name = :'CHUNK_NAME' AND constraint_type = 'FOREIGN KEY'
ORDER BY constraint_name;
constraint_schema | constraint_name | table_schema | table_name | constraint_type
-----------------------+---------------------------+-----------------------+--------------------+-----------------
_timescaledb_internal | 42_9_hyper_device_id_fkey | _timescaledb_internal | _hyper_18_42_chunk | FOREIGN KEY
(1 row)
-- create hypertable with 2 chunks
CREATE TABLE ht5(time TIMESTAMPTZ NOT NULL);
SELECT create_hypertable('ht5','time');
create_hypertable
-------------------
(20,public,ht5,t)
(1 row)
INSERT INTO ht5 SELECT '2000-01-01'::TIMESTAMPTZ;
INSERT INTO ht5 SELECT '2001-01-01'::TIMESTAMPTZ;
-- compressed chunk stats should not show dropped chunks
ALTER TABLE ht5 SET (timescaledb.compress);
SELECT compress_chunk(i) FROM show_chunks('ht5') i;
compress_chunk
------------------------------------------
_timescaledb_internal._hyper_20_45_chunk
_timescaledb_internal._hyper_20_46_chunk
(2 rows)
SELECT drop_chunks('ht5', newer_than => '2000-01-01'::TIMESTAMPTZ);
drop_chunks
------------------------------------------
_timescaledb_internal._hyper_20_46_chunk
(1 row)
select chunk_name from chunk_compression_stats('ht5')
order by chunk_name;
chunk_name
--------------------
_hyper_20_45_chunk
(1 row)
-- Test enabling compression for a table with compound foreign key
-- (Issue https://github.com/timescale/timescaledb/issues/2000)
CREATE TABLE table2(col1 INT, col2 int, primary key (col1,col2));
CREATE TABLE table1(col1 INT NOT NULL, col2 INT);
ALTER TABLE table1 ADD CONSTRAINT fk_table1 FOREIGN KEY (col1,col2) REFERENCES table2(col1,col2);
SELECT create_hypertable('table1','col1', chunk_time_interval => 10);
create_hypertable
----------------------
(22,public,table1,t)
(1 row)
-- Trying to list an incomplete set of fields of the compound key (should fail with a nice message)
ALTER TABLE table1 SET (timescaledb.compress, timescaledb.compress_segmentby = 'col1');
ERROR: column "col2" must be used for segmenting
-- Listing all fields of the compound key should succeed:
ALTER TABLE table1 SET (timescaledb.compress, timescaledb.compress_segmentby = 'col1,col2');
SELECT * FROM timescaledb_information.compression_settings ORDER BY hypertable_name;
hypertable_schema | hypertable_name | attname | segmentby_column_index | orderby_column_index | orderby_asc | orderby_nullsfirst
-------------------+-----------------+-------------+------------------------+----------------------+-------------+--------------------
public | conditions | location | 1 | | |
public | conditions | time | | 1 | t | f
public | datatype_test | time | | 1 | f | t
public | foo | a | 1 | | |
public | foo | b | 2 | | |
public | foo | c | | 1 | f | t
public | foo | d | | 2 | t | f
public | ht5 | time | | 1 | f | t
public | hyper | device_id | 1 | | |
public | hyper | time | | 1 | t | f
public | metrics | time | | 1 | f | t
public | plan_inval | time | | 1 | f | t
public | rescan_test | id | 1 | | |
public | rescan_test | t | | 1 | f | t
public | table1 | col1 | 1 | | |
public | table1 | col2 | 2 | | |
public | test_collation | device_id | 1 | | |
public | test_collation | device_id_2 | 2 | | |
public | test_collation | val_1 | | 1 | t | f
public | test_collation | val_2 | | 2 | t | f
public | test_collation | time | | 3 | t | f
(21 rows)
-- test delete/update on non-compressed tables involving hypertables with compression
CREATE TABLE uncompressed_ht (
time timestamptz NOT NULL,
value double precision,
series_id integer
);
SELECT table_name FROM create_hypertable ('uncompressed_ht', 'time');
table_name
-----------------
uncompressed_ht
(1 row)
INSERT INTO uncompressed_ht
VALUES ('2020-04-20 01:01', 100, 1), ('2020-05-20 01:01', 100, 1), ('2020-04-20 01:01', 200, 2);
CREATE TABLE compressed_ht (
time timestamptz NOT NULL,
value double precision,
series_id integer
);
SELECT table_name FROM create_hypertable ('compressed_ht', 'time');
table_name
---------------
compressed_ht
(1 row)
ALTER TABLE compressed_ht SET (timescaledb.compress);
INSERT INTO compressed_ht
VALUES ('2020-04-20 01:01', 100, 1), ('2020-05-20 01:01', 100, 1);
SELECT compress_chunk (ch1.schema_name || '.' || ch1.table_name)
FROM _timescaledb_catalog.chunk ch1,
_timescaledb_catalog.hypertable ht
WHERE ch1.hypertable_id = ht.id
AND ht.table_name LIKE 'compressed_ht'
ORDER BY ch1.id;
compress_chunk
------------------------------------------
_timescaledb_internal._hyper_25_51_chunk
_timescaledb_internal._hyper_25_52_chunk
(2 rows)
BEGIN;
WITH compressed AS (
SELECT series_id
FROM compressed_ht
WHERE time >= '2020-04-17 17:14:24.161989+00'
)
DELETE FROM uncompressed_ht
WHERE series_id IN (SELECT series_id FROM compressed);
ROLLBACK;
\set ON_ERROR_STOP 0
-- test delete inside CTE is blocked
WITH compressed AS (
DELETE FROM compressed_ht RETURNING series_id
)
SELECT * FROM uncompressed_ht
WHERE series_id IN (SELECT series_id FROM compressed);
ERROR: cannot update/delete rows from chunk "_hyper_25_51_chunk" as it is compressed
-- test update inside CTE is blocked
WITH compressed AS (
UPDATE compressed_ht SET value = 0.2 RETURNING *
)
SELECT * FROM uncompressed_ht
WHERE series_id IN (SELECT series_id FROM compressed);
ERROR: cannot update/delete rows from chunk "_hyper_25_51_chunk" as it is compressed
\set ON_ERROR_STOP 1
DROP TABLE compressed_ht;
DROP TABLE uncompressed_ht;
-- Test that pg_stats and pg_class stats for uncompressed chunks are frozen at compression time
-- Note that approximate_row_count pulls from pg_class
CREATE TABLE stattest(time TIMESTAMPTZ NOT NULL, c1 int);
SELECT create_hypertable('stattest', 'time');
create_hypertable
------------------------
(27,public,stattest,t)
(1 row)
INSERT INTO stattest SELECT '2020/02/20 01:00'::TIMESTAMPTZ + ('1 hour'::interval * v), 250 * v FROM generate_series(0,25) v;
SELECT table_name INTO TEMPORARY temptable FROM _timescaledb_catalog.chunk WHERE hypertable_id = (SELECT id FROM _timescaledb_catalog.hypertable WHERE table_name = 'stattest');
\set statchunk '(select table_name from temptable)'
SELECT * FROM pg_stats WHERE tablename = :statchunk;
schemaname | tablename | attname | inherited | null_frac | avg_width | n_distinct | most_common_vals | most_common_freqs | histogram_bounds | correlation | most_common_elems | most_common_elem_freqs | elem_count_histogram
------------+-----------+---------+-----------+-----------+-----------+------------+------------------+-------------------+------------------+-------------+-------------------+------------------------+----------------------
(0 rows)
ALTER TABLE stattest SET (timescaledb.compress);
SELECT approximate_row_count('stattest');
approximate_row_count
-----------------------
0
(1 row)
SELECT compress_chunk(c) FROM show_chunks('stattest') c;
compress_chunk
------------------------------------------
_timescaledb_internal._hyper_27_55_chunk
(1 row)
SELECT approximate_row_count('stattest');
approximate_row_count
-----------------------
26
(1 row)
SELECT relpages, reltuples FROM pg_class WHERE relname = :statchunk;
relpages | reltuples
----------+-----------
1 | 26
(1 row)
SELECT histogram_bounds FROM pg_stats WHERE tablename = :statchunk AND attname = 'c1';
histogram_bounds
-------------------------------------------------------------------------------------------------------------------------------
{0,250,500,750,1000,1250,1500,1750,2000,2250,2500,2750,3000,3250,3500,3750,4000,4250,4500,4750,5000,5250,5500,5750,6000,6250}
(1 row)
SELECT compch.table_name as "STAT_COMP_CHUNK_NAME"
FROM _timescaledb_catalog.hypertable ht, _timescaledb_catalog.chunk ch
, _timescaledb_catalog.chunk compch
WHERE ht.table_name = 'stattest' AND ch.hypertable_id = ht.id
AND compch.id = ch.compressed_chunk_id AND ch.compressed_chunk_id > 0 \gset
-- reltuples is initially -1 on PG14 before VACUUM/ANALYZE was run
SELECT relpages, CASE WHEN reltuples > 0 THEN reltuples ELSE 0 END AS reltuples FROM pg_class WHERE relname = :'STAT_COMP_CHUNK_NAME';
relpages | reltuples
----------+-----------
0 | 0
(1 row)
-- Now verify stats are not changed when we analyze the hypertable
ANALYZE stattest;
SELECT histogram_bounds FROM pg_stats WHERE tablename = :statchunk AND attname = 'c1';
histogram_bounds
-------------------------------------------------------------------------------------------------------------------------------
{0,250,500,750,1000,1250,1500,1750,2000,2250,2500,2750,3000,3250,3500,3750,4000,4250,4500,4750,5000,5250,5500,5750,6000,6250}
(1 row)
-- Unfortunately, the stats on the hypertable won't find any rows to sample from the chunk
SELECT histogram_bounds FROM pg_stats WHERE tablename = 'stattest' AND attname = 'c1';
histogram_bounds
------------------
(0 rows)
SELECT relpages, reltuples FROM pg_class WHERE relname = :statchunk;
relpages | reltuples
----------+-----------
1 | 26
(1 row)
-- verify that corresponding compressed chunk's stats is updated as well.
SELECT relpages, reltuples FROM pg_class WHERE relname = :'STAT_COMP_CHUNK_NAME';
relpages | reltuples
----------+-----------
1 | 1
(1 row)
-- Verify that even a global analyze doesn't affect the chunk stats, changing message scope here
-- to hide WARNINGs for skipped tables
SET client_min_messages TO ERROR;
ANALYZE;
SET client_min_messages TO NOTICE;
SELECT histogram_bounds FROM pg_stats WHERE tablename = :statchunk AND attname = 'c1';
histogram_bounds
-------------------------------------------------------------------------------------------------------------------------------
{0,250,500,750,1000,1250,1500,1750,2000,2250,2500,2750,3000,3250,3500,3750,4000,4250,4500,4750,5000,5250,5500,5750,6000,6250}
(1 row)
SELECT relpages, reltuples FROM pg_class WHERE relname = :statchunk;
relpages | reltuples
----------+-----------
1 | 26
(1 row)
-- Verify that decompressing the chunk restores autoanalyze to the hypertable's setting
SELECT reloptions FROM pg_class WHERE relname = :statchunk;
reloptions
----------------------------
{autovacuum_enabled=false}
(1 row)
SELECT decompress_chunk(c) FROM show_chunks('stattest') c;
decompress_chunk
------------------------------------------
_timescaledb_internal._hyper_27_55_chunk
(1 row)
SELECT reloptions FROM pg_class WHERE relname = :statchunk;
reloptions
---------------------------
{autovacuum_enabled=true}
(1 row)
SELECT compress_chunk(c) FROM show_chunks('stattest') c;
compress_chunk
------------------------------------------
_timescaledb_internal._hyper_27_55_chunk
(1 row)
SELECT reloptions FROM pg_class WHERE relname = :statchunk;
reloptions
----------------------------
{autovacuum_enabled=false}
(1 row)
ALTER TABLE stattest SET (autovacuum_enabled = false);
SELECT decompress_chunk(c) FROM show_chunks('stattest') c;
decompress_chunk
------------------------------------------
_timescaledb_internal._hyper_27_55_chunk
(1 row)
SELECT reloptions FROM pg_class WHERE relname = :statchunk;
reloptions
----------------------------
{autovacuum_enabled=false}
(1 row)
DROP TABLE stattest;
--- Test that analyze on compression internal table updates stats on original chunks
CREATE TABLE stattest2(time TIMESTAMPTZ NOT NULL, c1 int, c2 int);
SELECT create_hypertable('stattest2', 'time', chunk_time_interval=>'1 day'::interval);
create_hypertable
-------------------------
(29,public,stattest2,t)
(1 row)
ALTER TABLE stattest2 SET (timescaledb.compress, timescaledb.compress_segmentby='c1');
INSERT INTO stattest2 SELECT '2020/06/20 01:00'::TIMESTAMPTZ ,1 , generate_series(1, 200, 1);
INSERT INTO stattest2 SELECT '2020/07/20 01:00'::TIMESTAMPTZ ,1 , generate_series(1, 200, 1);
SELECT compress_chunk(ch1.schema_name|| '.' || ch1.table_name)
FROM _timescaledb_catalog.chunk ch1, _timescaledb_catalog.hypertable ht
WHERE ch1.hypertable_id = ht.id and ht.table_name like 'stattest2'
ORDER BY ch1.id limit 1;
compress_chunk
------------------------------------------
_timescaledb_internal._hyper_29_58_chunk
(1 row)
-- reltuples is initially -1 on PG14 before VACUUM/ANALYZE has been run
SELECT relname, CASE WHEN reltuples > 0 THEN reltuples ELSE 0 END AS reltuples, relpages, relallvisible FROM pg_class
WHERE relname in ( SELECT ch.table_name FROM
_timescaledb_catalog.chunk ch, _timescaledb_catalog.hypertable ht
WHERE ht.table_name = 'stattest2' AND ch.hypertable_id = ht.id )
order by relname;
relname | reltuples | relpages | relallvisible
--------------------+-----------+----------+---------------
_hyper_29_58_chunk | 200 | 2 | 0
_hyper_29_59_chunk | 0 | 0 | 0
(2 rows)
\c :TEST_DBNAME :ROLE_SUPERUSER
--overwrite pg_class stats for the compressed chunk.
UPDATE pg_class
SET reltuples = 0, relpages = 0
WHERE relname in ( SELECT ch.table_name FROM
_timescaledb_catalog.chunk ch,
_timescaledb_catalog.hypertable ht
WHERE ht.table_name = 'stattest2' AND ch.hypertable_id = ht.id
AND ch.compressed_chunk_id > 0 );
\c :TEST_DBNAME :ROLE_DEFAULT_PERM_USER
-- reltuples is initially -1 on PG14 before VACUUM/ANALYZE has been run
SELECT relname, CASE WHEN reltuples > 0 THEN reltuples ELSE 0 END AS reltuples, relpages, relallvisible FROM pg_class
WHERE relname in ( SELECT ch.table_name FROM
_timescaledb_catalog.chunk ch, _timescaledb_catalog.hypertable ht
WHERE ht.table_name = 'stattest2' AND ch.hypertable_id = ht.id )
order by relname;
relname | reltuples | relpages | relallvisible
--------------------+-----------+----------+---------------
_hyper_29_58_chunk | 0 | 0 | 0
_hyper_29_59_chunk | 0 | 0 | 0
(2 rows)
SELECT '_timescaledb_internal.' || compht.table_name as "STAT_COMP_TABLE",
compht.table_name as "STAT_COMP_TABLE_NAME"
FROM _timescaledb_catalog.hypertable ht, _timescaledb_catalog.hypertable compht
WHERE ht.table_name = 'stattest2' AND ht.compressed_hypertable_id = compht.id \gset
--analyze the compressed table, will update stats for the raw table.
ANALYZE :STAT_COMP_TABLE;
-- reltuples is initially -1 on PG14 before VACUUM/ANALYZE has been run
SELECT relname, CASE WHEN reltuples > 0 THEN reltuples ELSE 0 END AS reltuples, relpages, relallvisible FROM pg_class
WHERE relname in ( SELECT ch.table_name FROM
_timescaledb_catalog.chunk ch, _timescaledb_catalog.hypertable ht
WHERE ht.table_name = 'stattest2' AND ch.hypertable_id = ht.id )
ORDER BY relname;
relname | reltuples | relpages | relallvisible
--------------------+-----------+----------+---------------
_hyper_29_58_chunk | 200 | 1 | 0
_hyper_29_59_chunk | 0 | 0 | 0
(2 rows)
SELECT relname, reltuples, relpages, relallvisible FROM pg_class
WHERE relname in ( SELECT ch.table_name FROM
_timescaledb_catalog.chunk ch, _timescaledb_catalog.hypertable ht
WHERE ht.table_name = :'STAT_COMP_TABLE_NAME' AND ch.hypertable_id = ht.id )
ORDER BY relname;
relname | reltuples | relpages | relallvisible
----------------------------+-----------+----------+---------------
compress_hyper_30_60_chunk | 1 | 1 | 0
(1 row)
--analyze on stattest2 should not overwrite
ANALYZE stattest2;
SELECT relname, reltuples, relpages, relallvisible FROM pg_class
WHERE relname in ( SELECT ch.table_name FROM
_timescaledb_catalog.chunk ch, _timescaledb_catalog.hypertable ht
WHERE ht.table_name = 'stattest2' AND ch.hypertable_id = ht.id )
ORDER BY relname;
relname | reltuples | relpages | relallvisible
--------------------+-----------+----------+---------------
_hyper_29_58_chunk | 200 | 1 | 0
_hyper_29_59_chunk | 200 | 2 | 0
(2 rows)
SELECT relname, reltuples, relpages, relallvisible FROM pg_class
WHERE relname in ( SELECT ch.table_name FROM
_timescaledb_catalog.chunk ch, _timescaledb_catalog.hypertable ht
WHERE ht.table_name = :'STAT_COMP_TABLE_NAME' AND ch.hypertable_id = ht.id )
ORDER BY relname;
relname | reltuples | relpages | relallvisible
----------------------------+-----------+----------+---------------
compress_hyper_30_60_chunk | 1 | 1 | 0
(1 row)
-- analyze on compressed hypertable should restore stats
-- Test approximate_row_count() with compressed hypertable
--
CREATE TABLE approx_count(time timestamptz not null, device int, temp float);
SELECT create_hypertable('approx_count', 'time');
create_hypertable
----------------------------
(31,public,approx_count,t)
(1 row)
INSERT INTO approx_count SELECT t, (abs(timestamp_hash(t::timestamp)) % 10) + 1, random()*80
FROM generate_series('2018-03-02 1:00'::TIMESTAMPTZ, '2018-03-04 1:00', '1 hour') t;
SELECT count(*) FROM approx_count;
count
-------
49
(1 row)
ALTER TABLE approx_count SET (timescaledb.compress, timescaledb.compress_segmentby='device', timescaledb.compress_orderby = 'time DESC');
SELECT approximate_row_count('approx_count');
approximate_row_count
-----------------------
0
(1 row)
ANALYZE approx_count;
SELECT approximate_row_count('approx_count');
approximate_row_count
-----------------------
49
(1 row)
DROP TABLE approx_count;
--TEST drop_chunks from a compressed hypertable (that has caggs defined).
-- chunk metadata is still retained. verify correct status for chunk
SELECT count(*)
FROM (SELECT compress_chunk(ch) FROM show_chunks('metrics') ch ) q;
count
-------
2
(1 row)
SELECT drop_chunks('metrics', older_than=>'1 day'::interval);
drop_chunks
------------------------------------------
_timescaledb_internal._hyper_13_33_chunk
_timescaledb_internal._hyper_13_34_chunk
(2 rows)
SELECT
c.table_name as chunk_name,
c.status as chunk_status, c.dropped, c.compressed_chunk_id as comp_id
FROM _timescaledb_catalog.hypertable h, _timescaledb_catalog.chunk c
WHERE h.id = c.hypertable_id and h.table_name = 'metrics'
ORDER BY 1;
chunk_name | chunk_status | dropped | comp_id
--------------------+--------------+---------+---------
_hyper_13_33_chunk | 0 | t |
_hyper_13_34_chunk | 0 | t |
(2 rows)
SELECT "time", cnt FROM cagg_expr ORDER BY time LIMIT 5;
time | cnt
------------------------------+------
Fri Dec 31 16:00:00 1999 PST | 960
Sat Jan 01 16:00:00 2000 PST | 1440
Sun Jan 02 16:00:00 2000 PST | 1440
Mon Jan 03 16:00:00 2000 PST | 1440
Tue Jan 04 16:00:00 2000 PST | 1440
(5 rows)
--now reload data into the dropped chunks region, then compress
-- then verify chunk status/dropped column
INSERT INTO metrics SELECT generate_series('2000-01-01'::timestamptz,'2000-01-10','1m'),1,0.25,0.75;
SELECT count(*)
FROM (SELECT compress_chunk(ch) FROM show_chunks('metrics') ch) q;
count
-------
2
(1 row)
SELECT
c.table_name as chunk_name,
c.status as chunk_status, c.dropped, c.compressed_chunk_id as comp_id
FROM _timescaledb_catalog.hypertable h, _timescaledb_catalog.chunk c
WHERE h.id = c.hypertable_id and h.table_name = 'metrics'
ORDER BY 1;
chunk_name | chunk_status | dropped | comp_id
--------------------+--------------+---------+---------
_hyper_13_33_chunk | 1 | f | 64
_hyper_13_34_chunk | 1 | f | 65
(2 rows)
SELECT count(*) FROM metrics;
count
-------
12961
(1 row)
-- test sequence number is local to segment by
CREATE TABLE local_seq(time timestamptz, device int);
SELECT table_name FROM create_hypertable('local_seq','time');
NOTICE: adding not-null constraint to column "time"
table_name
------------
local_seq
(1 row)
ALTER TABLE local_seq SET(timescaledb.compress,timescaledb.compress_segmentby='device');
INSERT INTO local_seq SELECT '2000-01-01',1 FROM generate_series(1,3000);
INSERT INTO local_seq SELECT '2000-01-01',2 FROM generate_series(1,3500);
INSERT INTO local_seq SELECT '2000-01-01',3 FROM generate_series(1,3000);
INSERT INTO local_seq SELECT '2000-01-01',4 FROM generate_series(1,3000);
INSERT INTO local_seq SELECT '2000-01-01', generate_series(5,8);
SELECT compress_chunk(c) FROM show_chunks('local_seq') c;
compress_chunk
------------------------------------------
_timescaledb_internal._hyper_33_66_chunk
(1 row)
SELECT
format('%s.%s',chunk.schema_name,chunk.table_name) AS "COMP_CHUNK"
FROM _timescaledb_catalog.hypertable ht
INNER JOIN _timescaledb_catalog.hypertable ht_comp ON ht_comp.id = ht.compressed_hypertable_id
INNER JOIN _timescaledb_catalog.chunk ON chunk.hypertable_id = ht_comp.id
WHERE ht.table_name = 'local_seq' \gset
SELECT device, _ts_meta_sequence_num, _ts_meta_count FROM :COMP_CHUNK ORDER BY 1,2;
device | _ts_meta_sequence_num | _ts_meta_count
--------+-----------------------+----------------
1 | 10 | 1000
1 | 20 | 1000
1 | 30 | 1000
2 | 10 | 1000
2 | 20 | 1000
2 | 30 | 1000
2 | 40 | 500
3 | 10 | 1000
3 | 20 | 1000
3 | 30 | 1000
4 | 10 | 1000
4 | 20 | 1000
4 | 30 | 1000
5 | 10 | 1
6 | 10 | 1
7 | 10 | 1
8 | 10 | 1
(17 rows)
Reference in New Issue View Git Blame Copy Permalink