timescaledb/tsl/test/expected/compression_insert-12.out
gayyappan 4f865f7870 Add recompress_chunk function
After inserts go into a compressed chunk, the chunk is marked as
unordered.This PR adds a new function recompress_chunk that
compresses the data and sets the status back to compressed. Further
optimizations for this function are planned but not part of this PR.

This function can be invoked by calling
SELECT recompress_chunk(<chunk_name>).

recompress_chunk function is automatically invoked by the compression
policy job, when it sees that a chunk is in unordered state.
2021-05-24 18:03:47 -04:00

544 lines
18 KiB
Plaintext

-- This file and its contents are licensed under the Timescale License.
-- Please see the included NOTICE for copyright information and
-- LICENSE-TIMESCALE for a copy of the license.
\set PREFIX 'EXPLAIN (costs off, summary off, timing off) '
CREATE TABLE test1 (timec timestamptz , i integer ,
b bigint, t text);
SELECT table_name from create_hypertable('test1', 'timec', chunk_time_interval=> INTERVAL '7 days');
NOTICE: adding not-null constraint to column "timec"
table_name
------------
test1
(1 row)
INSERT INTO test1 SELECT q, 10, 11, 'hello' FROM generate_series( '2020-01-03 10:00:00-05', '2020-01-03 12:00:00-05' , '5 min'::interval) q;
ALTER TABLE test1 set (timescaledb.compress,
timescaledb.compress_segmentby = 'b',
timescaledb.compress_orderby = 'timec DESC');
SELECT compress_chunk(c)
FROM show_chunks('test1') c;
compress_chunk
----------------------------------------
_timescaledb_internal._hyper_1_1_chunk
(1 row)
SELECT count(*) FROM test1;
count
-------
25
(1 row)
--we have 1 compressed row --
SELECT COUNT(*) from _timescaledb_internal.compress_hyper_2_2_chunk;
count
-------
1
(1 row)
-- single and multi row insert into the compressed chunk --
INSERT INTO test1 SELECT '2020-01-02 11:16:00-05' , 11, 16, 'new' ;
SELECT COUNT(*) from _timescaledb_internal.compress_hyper_2_2_chunk;
count
-------
2
(1 row)
INSERT INTO test1 SELECT '2020-01-02 11:16:00-05' , i, i +5, 'clay'
FROM (Select generate_series(10, 20, 1) i ) q;
SELECT COUNT(*) from _timescaledb_internal.compress_hyper_2_2_chunk;
count
-------
13
(1 row)
SELECT count(*) from test1;
count
-------
37
(1 row)
-- single row copy
COPY test1 FROM STDIN DELIMITER ',';
SELECT COUNT(*) from _timescaledb_internal.compress_hyper_2_2_chunk;
count
-------
14
(1 row)
-- multi row copy
COPY test1 FROM STDIN DELIMITER ',';
SELECT COUNT(*) from _timescaledb_internal.compress_hyper_2_2_chunk;
count
-------
17
(1 row)
--Verify that all the data went into the initial chunk
SELECT count(*)
FROM show_chunks('test1') c;
count
-------
1
(1 row)
SELECT * FROM test1 WHERE b = 11 order by i, timec ;
timec | i | b | t
------------------------------+----+----+-------
Fri Jan 03 07:00:00 2020 PST | 10 | 11 | hello
Fri Jan 03 07:05:00 2020 PST | 10 | 11 | hello
Fri Jan 03 07:10:00 2020 PST | 10 | 11 | hello
Fri Jan 03 07:15:00 2020 PST | 10 | 11 | hello
Fri Jan 03 07:20:00 2020 PST | 10 | 11 | hello
Fri Jan 03 07:25:00 2020 PST | 10 | 11 | hello
Fri Jan 03 07:30:00 2020 PST | 10 | 11 | hello
Fri Jan 03 07:35:00 2020 PST | 10 | 11 | hello
Fri Jan 03 07:40:00 2020 PST | 10 | 11 | hello
Fri Jan 03 07:45:00 2020 PST | 10 | 11 | hello
Fri Jan 03 07:50:00 2020 PST | 10 | 11 | hello
Fri Jan 03 07:55:00 2020 PST | 10 | 11 | hello
Fri Jan 03 08:00:00 2020 PST | 10 | 11 | hello
Fri Jan 03 08:05:00 2020 PST | 10 | 11 | hello
Fri Jan 03 08:10:00 2020 PST | 10 | 11 | hello
Fri Jan 03 08:15:00 2020 PST | 10 | 11 | hello
Fri Jan 03 08:20:00 2020 PST | 10 | 11 | hello
Fri Jan 03 08:25:00 2020 PST | 10 | 11 | hello
Fri Jan 03 08:30:00 2020 PST | 10 | 11 | hello
Fri Jan 03 08:35:00 2020 PST | 10 | 11 | hello
Fri Jan 03 08:40:00 2020 PST | 10 | 11 | hello
Fri Jan 03 08:45:00 2020 PST | 10 | 11 | hello
Fri Jan 03 08:50:00 2020 PST | 10 | 11 | hello
Fri Jan 03 08:55:00 2020 PST | 10 | 11 | hello
Fri Jan 03 09:00:00 2020 PST | 10 | 11 | hello
(25 rows)
SELECT * FROM test1 WHERE i = 11 order by 1, 2, 3, 4;
timec | i | b | t
------------------------------+----+----+-----------
Thu Jan 02 08:16:00 2020 PST | 11 | 16 | clay
Thu Jan 02 08:16:00 2020 PST | 11 | 16 | copy
Thu Jan 02 08:16:00 2020 PST | 11 | 16 | multicopy
Thu Jan 02 08:16:00 2020 PST | 11 | 16 | new
(4 rows)
-- insert nulls except for timec
INSERT INTO test1 SELECT '2020-01-02 11:46:00-05' , NULL, NULL, NULL;
SELECT count(*)
FROM show_chunks('test1') c;
count
-------
1
(1 row)
-- copy NULL
COPY test1 FROM STDIN DELIMITER ',' NULL 'NULL';
SELECT count(*)
FROM show_chunks('test1') c;
count
-------
1
(1 row)
SELECT * from test1 WHERE i is NULL;
timec | i | b | t
------------------------------+---+---+---
Thu Jan 02 08:46:00 2020 PST | | |
Thu Jan 02 08:46:00 2020 PST | | |
(2 rows)
--TEST 2 now alter the table and add a new column to it
ALTER TABLE test1 ADD COLUMN newtcol varchar(400);
--add rows with segments that overlap some of the previous ones
SELECT count(*) from _timescaledb_internal.compress_hyper_2_2_chunk;
count
-------
19
(1 row)
INSERT INTO test1 SELECT '2020-01-02 11:16:00-05' , 100, 101, 'prev101', 'this is the newtcol101';
INSERT INTO test1 SELECT '2020-01-02 11:16:00-05' , i, 16, 'prev16', 'this is the newtcol16'
FROM (Select generate_series(11, 16, 1) i ) q;
SELECT * FROM test1 WHERE b = 16 order by 1, 2, 3, 4, 5;
timec | i | b | t | newtcol
------------------------------+----+----+-----------+-----------------------
Thu Jan 02 08:16:00 2020 PST | 11 | 16 | clay |
Thu Jan 02 08:16:00 2020 PST | 11 | 16 | copy |
Thu Jan 02 08:16:00 2020 PST | 11 | 16 | multicopy |
Thu Jan 02 08:16:00 2020 PST | 11 | 16 | new |
Thu Jan 02 08:16:00 2020 PST | 11 | 16 | prev16 | this is the newtcol16
Thu Jan 02 08:16:00 2020 PST | 12 | 16 | prev16 | this is the newtcol16
Thu Jan 02 08:16:00 2020 PST | 13 | 16 | prev16 | this is the newtcol16
Thu Jan 02 08:16:00 2020 PST | 14 | 16 | prev16 | this is the newtcol16
Thu Jan 02 08:16:00 2020 PST | 15 | 16 | prev16 | this is the newtcol16
Thu Jan 02 08:16:00 2020 PST | 16 | 16 | prev16 | this is the newtcol16
(10 rows)
--number of rows in the chunk
SELECT count(*) from _timescaledb_internal.compress_hyper_2_2_chunk;
count
-------
26
(1 row)
SELECT count(*)
FROM show_chunks('test1') c;
count
-------
1
(1 row)
COPY test1 FROM STDIN DELIMITER ',';
COPY test1 FROM STDIN DELIMITER ',';
--number of rows in the chunk
SELECT count(*) from _timescaledb_internal.compress_hyper_2_2_chunk;
count
-------
30
(1 row)
SELECT count(*)
FROM show_chunks('test1') c;
count
-------
1
(1 row)
SELECT * FROM test1 WHERE newtcol IS NOT NULL ORDER BY 1,2,3;
timec | i | b | t | newtcol
------------------------------+-----+-----+---------+------------------------
Thu Jan 02 08:16:00 2020 PST | 11 | 16 | prev16 | this is the newtcol16
Thu Jan 02 08:16:00 2020 PST | 11 | 16 | prev16 | newtcol16
Thu Jan 02 08:16:00 2020 PST | 12 | 16 | prev16 | newtcol16
Thu Jan 02 08:16:00 2020 PST | 12 | 16 | prev16 | this is the newtcol16
Thu Jan 02 08:16:00 2020 PST | 13 | 16 | prev16 | this is the newtcol16
Thu Jan 02 08:16:00 2020 PST | 13 | 16 | prev16 | newtcol16
Thu Jan 02 08:16:00 2020 PST | 14 | 16 | prev16 | this is the newtcol16
Thu Jan 02 08:16:00 2020 PST | 15 | 16 | prev16 | this is the newtcol16
Thu Jan 02 08:16:00 2020 PST | 16 | 16 | prev16 | this is the newtcol16
Thu Jan 02 08:16:00 2020 PST | 100 | 101 | prev101 | newtcol101
Thu Jan 02 08:16:00 2020 PST | 100 | 101 | prev101 | this is the newtcol101
(11 rows)
DROP TABLE test1;
-- TEST 3 add tests with dropped columns on hypertable
-- also tests defaults
CREATE TABLE test2 ( itime integer, b bigint, t text);
SELECT table_name from create_hypertable('test2', 'itime', chunk_time_interval=> 10::integer);
NOTICE: adding not-null constraint to column "itime"
table_name
------------
test2
(1 row)
--create a chunk
INSERT INTO test2 SELECT t, 10, 'first'::text FROM generate_series(1, 7) t;
ALTER TABLE test2 DROP COLUMN b;
ALTER TABLE test2 ADD COLUMN c INT DEFAULT -15;
ALTER TABLE test2 ADD COLUMN d INT;
--create a new chunk
INSERT INTO test2 SELECT t, 'second'::text, 120, 1 FROM generate_series(11, 15) t;
ALTER TABLE test2 set (timescaledb.compress, timescaledb.compress_segmentby = '', timescaledb.compress_orderby = 'c, itime DESC');
SELECT count(*) from ( SELECT compress_chunk(c)
FROM show_chunks('test2') c ) q;
count
-------
2
(1 row)
--write to both old chunks and new chunks
INSERT INTO test2(itime ,t , d) SELECT 9, '9', 90 ;
INSERT INTO test2(itime ,t , d) SELECT 17, '17', 1700 ;
COPY test2(itime,t,d) FROM STDIN DELIMITER ',';
SELECT count(*) FROM show_chunks('test2') q;
count
-------
2
(1 row)
SELECT * from test2 WHERE itime >= 9 and itime <= 17
ORDER BY 1,2,3;
itime | t | c | d
-------+--------+-----+------
9 | 9 | -15 | 90
9 | 9copy | -15 | 90
11 | second | 120 | 1
12 | second | 120 | 1
13 | second | 120 | 1
14 | second | 120 | 1
15 | second | 120 | 1
17 | 17 | -15 | 1700
17 | 17copy | -15 | 1700
(9 rows)
-- now add a column to the compressed hypertable
-- we have dropped columns and newly added columns now
ALTER TABLE test2 ADD COLUMN charcol varchar(45);
INSERT INTO test2(itime ,t , d, charcol)
values (2, '2', 22, 'thisis22'), (17, '17', 1701, 'thisis1700') ;
COPY test2(itime,t,d,charcol) FROM STDIN DELIMITER ',';
SELECT * from test2 where itime = 2 or itime =17
ORDER BY 1, 2, 3, 4, 5;
itime | t | c | d | charcol
-------+--------+-----+------+------------
2 | 2 | -15 | 22 | thisis22
2 | 2copy | -15 | 22 | 22copy
2 | first | -15 | |
17 | 17 | -15 | 1700 |
17 | 17 | -15 | 1701 | thisis1700
17 | 17copy | -15 | 1700 |
17 | 17copy | -15 | 1701 | 1700copy
(7 rows)
DROP TABLE test2;
--- TEST3 tables with defaults ---
-- sequences, generated values, check constraints into compressed chunks
CREATE TABLE test2 (timec timestamptz ,
i integer CHECK ( i > 10) ,
b bigint default 20 ,
t text NOT NULL,
CONSTRAINT rowconstr CHECK ( b > i )
);
SELECT table_name from create_hypertable('test2', 'timec', chunk_time_interval=> INTERVAL '7 days');
NOTICE: adding not-null constraint to column "timec"
table_name
------------
test2
(1 row)
ALTER TABLE test2 set (timescaledb.compress,
timescaledb.compress_segmentby = 'b',
timescaledb.compress_orderby = 'timec DESC');
INSERT INTO test2 values('2020-01-02 11:16:00-05' , 100, 105, 'first' );
SELECT compress_chunk(c)
FROM show_chunks('test2') c;
compress_chunk
----------------------------------------
_timescaledb_internal._hyper_5_7_chunk
(1 row)
-- test if default value for b is used
INSERT INTO test2(timec, i, t) values('2020-01-02 10:16:00-05' , 11, 'default' );
COPY test2(timec,i,t) FROM STDIN DELIMITER ',';
SELECT b from test2 ORDER BY 1;
b
-----
20
20
105
(3 rows)
\set ON_ERROR_STOP 0
--null value for t, should fail
INSERT INTO test2 values ( '2020-01-02 01:00:00-05', 100, 200, NULL);
ERROR: null value in column "t" violates not-null constraint
COPY test2 FROM STDIN DELIMITER ',' NULL 'NULL';
ERROR: null value in column "t" violates not-null constraint
-- i=1, should fail
INSERT INTO test2 values ( '2020-01-02 01:00:00-05', 1, 10, 'null i');
ERROR: new row for relation "_hyper_5_7_chunk" violates check constraint "test2_i_check"
COPY test2 FROM STDIN DELIMITER ',';
ERROR: new row for relation "_hyper_5_7_chunk" violates check constraint "test2_i_check"
-- b < i, should fail
INSERT INTO test2 values ( '2020-01-02 01:00:00-05', 22, 1, 'null i');
ERROR: new row for relation "_hyper_5_7_chunk" violates check constraint "rowconstr"
COPY test2 FROM STDIN DELIMITER ',';
ERROR: new row for relation "_hyper_5_7_chunk" violates check constraint "rowconstr"
\set ON_ERROR_STOP 1
--verify we are still INSERTing into the compressed chunk i.e did not
--create a new chunk
SELECT count(c)
FROM show_chunks('test2') c;
count
-------
1
(1 row)
-- TEST4 with sequence .
CREATE SEQUENCE vessel_id_seq
INCREMENT 1
START 1 MINVALUE 1
MAXVALUE 9223372036854775807
CACHE 1;
CREATE TABLE vessels (timec timestamptz ,
id bigint NOT NULL DEFAULT nextval('vessel_id_seq'::regclass),
i integer CHECK ( i > 10) ,
b bigint default 20 ,
t text NOT NULL,
CONSTRAINT rowconstr CHECK ( b > i )
);
SELECT table_name from create_hypertable('vessels', 'timec', chunk_time_interval=> INTERVAL '7 days');
NOTICE: adding not-null constraint to column "timec"
table_name
------------
vessels
(1 row)
ALTER TABLE vessels set (timescaledb.compress,
timescaledb.compress_segmentby = 'b',
timescaledb.compress_orderby = 'timec DESC');
INSERT INTO vessels(timec, i, b, t) values('2020-01-02 11:16:00-05' , 100, 105, 'first' );
SELECT compress_chunk(c)
FROM show_chunks('vessels') c;
compress_chunk
----------------------------------------
_timescaledb_internal._hyper_7_9_chunk
(1 row)
-- test if default value for b and sequence value for id is used
INSERT INTO vessels(timec, i, t) values('2020-01-02 10:16:00-05' , 11, 'default' );
COPY vessels(timec,i,t )FROM STDIN DELIMITER ',';
SELECT timec, id, b from vessels order by 2, 1;
timec | id | b
------------------------------+----+-----
Thu Jan 02 08:16:00 2020 PST | 1 | 105
Thu Jan 02 07:16:00 2020 PST | 2 | 20
Thu Jan 02 07:16:00 2020 PST | 3 | 20
(3 rows)
-- TEST5 generated values
CREATE table test_gen (
id int generated by default AS IDENTITY ,
payload text
);
SELECT create_hypertable('test_gen', 'id', chunk_time_interval=>10);
create_hypertable
-----------------------
(9,public,test_gen,t)
(1 row)
ALTER TABLE test_gen set (timescaledb.compress);
INSERT into test_gen (payload) SELECT generate_series(1,15) ;
SELECT max(id) from test_gen;
max
-----
15
(1 row)
SELECT compress_chunk(c)
FROM show_chunks('test_gen') c;
compress_chunk
-----------------------------------------
_timescaledb_internal._hyper_9_11_chunk
_timescaledb_internal._hyper_9_12_chunk
(2 rows)
INSERT INTO test_gen (payload) values(17);
SELECT * from test_gen WHERE id = (Select max(id) from test_gen);
id | payload
----+---------
16 | 17
(1 row)
COPY test_gen(payload) FROM STDIN DELIMITER ',';
SELECT * from test_gen WHERE id = (Select max(id) from test_gen);
id | payload
----+---------
17 | 18
(1 row)
-- test interaction between newly inserted batches and pathkeys/ordered append
CREATE TABLE test_ordering(time int);
SELECT table_name FROM create_hypertable('test_ordering','time',chunk_time_interval:=100);
NOTICE: adding not-null constraint to column "time"
table_name
---------------
test_ordering
(1 row)
ALTER TABLE test_ordering SET (timescaledb.compress,timescaledb.compress_orderby='time desc');
INSERT INTO test_ordering VALUES (5),(4),(3);
-- should be ordered append
:PREFIX SELECT * FROM test_ordering ORDER BY 1;
QUERY PLAN
------------------------------------------------------------------------------------------------------
Custom Scan (ChunkAppend) on test_ordering
Order: test_ordering."time"
-> Index Only Scan Backward using _hyper_11_15_chunk_test_ordering_time_idx on _hyper_11_15_chunk
(3 rows)
SELECT compress_chunk(format('%I.%I',chunk_schema,chunk_name), true) FROM timescaledb_information.chunks WHERE hypertable_name = 'test_ordering';
compress_chunk
------------------------------------------
_timescaledb_internal._hyper_11_15_chunk
(1 row)
-- should be ordered append
:PREFIX SELECT * FROM test_ordering ORDER BY 1;
QUERY PLAN
-------------------------------------------------------------------------------
Custom Scan (ChunkAppend) on test_ordering
Order: test_ordering."time"
-> Custom Scan (DecompressChunk) on _hyper_11_15_chunk
-> Sort
Sort Key: compress_hyper_12_16_chunk._ts_meta_sequence_num DESC
-> Seq Scan on compress_hyper_12_16_chunk
(6 rows)
INSERT INTO test_ordering SELECT 1;
-- should not be ordered append
:PREFIX SELECT * FROM test_ordering ORDER BY 1;
QUERY PLAN
-----------------------------------------------------------
Sort
Sort Key: _hyper_11_15_chunk."time"
-> Custom Scan (DecompressChunk) on _hyper_11_15_chunk
-> Seq Scan on compress_hyper_12_16_chunk
(4 rows)
INSERT INTO test_ordering VALUES (105),(104),(103);
-- should be ordered append
:PREFIX SELECT * FROM test_ordering ORDER BY 1;
QUERY PLAN
------------------------------------------------------------------------------------------------------
Custom Scan (ChunkAppend) on test_ordering
Order: test_ordering."time"
-> Sort
Sort Key: _hyper_11_15_chunk."time"
-> Custom Scan (DecompressChunk) on _hyper_11_15_chunk
-> Seq Scan on compress_hyper_12_16_chunk
-> Index Only Scan Backward using _hyper_11_17_chunk_test_ordering_time_idx on _hyper_11_17_chunk
(7 rows)
--insert into compressed + uncompressed chunk
INSERT INTO test_ordering VALUES (21), (22),(113);
SELECT count(*) FROM test_ordering;
count
-------
10
(1 row)
INSERT INTO test_ordering VALUES (106) RETURNING *;
time
------
106
(1 row)
-- insert into compressed chunk does not support RETURNING
\set ON_ERROR_STOP 0
INSERT INTO test_ordering VALUES (23), (24),(115) RETURNING *;
ERROR: insert with ON CONFLICT and RETURNING clause is not supported on compressed chunks
\set ON_ERROR_STOP 1
SELECT compress_chunk(format('%I.%I',chunk_schema,chunk_name), true) FROM timescaledb_information.chunks WHERE hypertable_name = 'test_ordering';
NOTICE: chunk "_hyper_11_15_chunk" is already compressed
compress_chunk
------------------------------------------
_timescaledb_internal._hyper_11_15_chunk
_timescaledb_internal._hyper_11_17_chunk
(2 rows)
-- should be ordered append
:PREFIX SELECT * FROM test_ordering ORDER BY 1;
QUERY PLAN
-------------------------------------------------------------------------------
Custom Scan (ChunkAppend) on test_ordering
Order: test_ordering."time"
-> Sort
Sort Key: _hyper_11_15_chunk."time"
-> Custom Scan (DecompressChunk) on _hyper_11_15_chunk
-> Seq Scan on compress_hyper_12_16_chunk
-> Custom Scan (DecompressChunk) on _hyper_11_17_chunk
-> Sort
Sort Key: compress_hyper_12_18_chunk._ts_meta_sequence_num DESC
-> Seq Scan on compress_hyper_12_18_chunk
(10 rows)