diff --git a/src/compression_with_clause.c b/src/compression_with_clause.c index 7f357c2ca..0c39c3145 100644 --- a/src/compression_with_clause.c +++ b/src/compression_with_clause.c @@ -82,6 +82,9 @@ parse_segment_collist(char *inpstr, Hypertable *hypertable) RawStmt *raw; #endif + if(strlen(inpstr) == 0) + return NIL; + initStringInfo(&buf); /* parse the segment by list exactly how you would a group by */ @@ -171,6 +174,9 @@ parse_order_collist(char *inpstr, Hypertable *hypertable) RawStmt *raw; #endif + if(strlen(inpstr) == 0) + return NIL; + initStringInfo(&buf); /* parse the segment by list exactly how you would a order by by */ diff --git a/tsl/src/compression/compression.c b/tsl/src/compression/compression.c index a31807a59..1ed715882 100644 --- a/tsl/src/compression/compression.c +++ b/tsl/src/compression/compression.c @@ -227,6 +227,10 @@ compress_chunk(Oid in_table, Oid out_table, const ColumnCompressionInfo **column truncate_relation(in_table); + /* recreate all indexes on out rel, we already have an exvclusive lock on it + * so the strong locks taken by reindex_relation shouldn't matter. */ + reindex_relation(out_table, 0, 0); + RelationClose(out_rel); RelationClose(in_rel); } @@ -879,6 +883,10 @@ decompress_chunk(Oid in_table, Oid out_table) FreeBulkInsertState(decompressor.bistate); } + /* recreate all indexes on out rel, we already have an exvclusive lock on it + * so the strong locks taken by reindex_relation shouldn't matter. */ + reindex_relation(out_table, 0, 0); + RelationClose(out_rel); RelationClose(in_rel); } diff --git a/tsl/test/expected/compression_algos.out b/tsl/test/expected/compression_algos.out index c3f35a376..7614c3363 100644 --- a/tsl/test/expected/compression_algos.out +++ b/tsl/test/expected/compression_algos.out @@ -20,6 +20,10 @@ SELECT ts_test_compression(); (1 row) +\ir include/rand_generator.sql +-- This file and its contents are licensed under the Timescale License. +-- Please see the included NOTICE for copyright information and +-- LICENSE-TIMESCALE for a copy of the license -------------------------- -- cheap rand generator -- -------------------------- diff --git a/tsl/test/expected/compression_hypertable.out b/tsl/test/expected/compression_hypertable.out new file mode 100644 index 000000000..51210b594 --- /dev/null +++ b/tsl/test/expected/compression_hypertable.out @@ -0,0 +1,59 @@ +-- This file and its contents are licensed under the Timescale License. +-- Please see the included NOTICE for copyright information and +-- LICENSE-TIMESCALE for a copy of the license. +\ir include/rand_generator.sql +-- This file and its contents are licensed under the Timescale License. +-- Please see the included NOTICE for copyright information and +-- LICENSE-TIMESCALE for a copy of the license +-------------------------- +-- cheap rand generator -- +-------------------------- +create table rand_minstd_state(i bigint); +create function rand_minstd_advance(bigint) returns bigint +language sql immutable as +$$ + select (16807 * $1) % 2147483647 +$$; +create function gen_rand_minstd() returns bigint +language sql as +$$ + update rand_minstd_state set i = rand_minstd_advance(i) returning i +$$; +-- seed the random num generator +insert into rand_minstd_state values (321); +CREATE TABLE test1 ("Time" timestamptz, i integer, b bigint, t text); +SELECT table_name from create_hypertable('test1', 'Time', chunk_time_interval=> INTERVAL '1 day'); +NOTICE: adding not-null constraint to column "Time" + table_name +------------ + test1 +(1 row) + +INSERT INTO test1 SELECT t, gen_rand_minstd(), gen_rand_minstd(), gen_rand_minstd()::text FROM generate_series('2018-03-02 1:00'::TIMESTAMPTZ, '2018-03-28 1:00', '1 hour') t; +ALTER TABLE test1 set (timescaledb.compress, timescaledb.compress_segmentby = '', timescaledb.compress_orderby = '"Time" DESC'); +SELECT + $$ + SELECT * FROM test1 ORDER BY "Time" + $$ AS "QUERY" \gset +SELECT 'test1' AS "HYPERTABLE_NAME" \gset +\ir include/compression_test_hypertable.sql +-- This file and its contents are licensed under the Timescale License. +-- Please see the included NOTICE for copyright information and +-- LICENSE-TIMESCALE for a copy of the license. +\set ECHO errors +psql:include/compression_test_hypertable.sql:7: NOTICE: table "original_result" does not exist, skipping + count +------- + 27 +(1 row) + + count +------- + 27 +(1 row) + + ?column? | count +-----------------------------------------------------------------------+------- + Number of rows different between original and uncompressed (expect 0) | 0 +(1 row) + diff --git a/tsl/test/sql/compression_algos.sql b/tsl/test/sql/compression_algos.sql index 81616457c..4d9389ce0 100644 --- a/tsl/test/sql/compression_algos.sql +++ b/tsl/test/sql/compression_algos.sql @@ -15,26 +15,7 @@ AS :TSL_MODULE_PATHNAME LANGUAGE C VOLATILE; SELECT ts_test_compression(); --------------------------- --- cheap rand generator -- --------------------------- -create table rand_minstd_state(i bigint); - -create function rand_minstd_advance(bigint) returns bigint -language sql immutable as -$$ - select (16807 * $1) % 2147483647 -$$; - -create function gen_rand_minstd() returns bigint -language sql as -$$ - update rand_minstd_state set i = rand_minstd_advance(i) returning i -$$; - --- seed the random num generator -insert into rand_minstd_state values (321); - +\ir include/rand_generator.sql ------------------------ -- BIGINT Compression -- diff --git a/tsl/test/sql/compression_hypertable.sql b/tsl/test/sql/compression_hypertable.sql new file mode 100644 index 000000000..934a359a2 --- /dev/null +++ b/tsl/test/sql/compression_hypertable.sql @@ -0,0 +1,21 @@ +-- This file and its contents are licensed under the Timescale License. +-- Please see the included NOTICE for copyright information and +-- LICENSE-TIMESCALE for a copy of the license. + +\ir include/rand_generator.sql + +CREATE TABLE test1 ("Time" timestamptz, i integer, b bigint, t text); +SELECT table_name from create_hypertable('test1', 'Time', chunk_time_interval=> INTERVAL '1 day'); + +INSERT INTO test1 SELECT t, gen_rand_minstd(), gen_rand_minstd(), gen_rand_minstd()::text FROM generate_series('2018-03-02 1:00'::TIMESTAMPTZ, '2018-03-28 1:00', '1 hour') t; + +ALTER TABLE test1 set (timescaledb.compress, timescaledb.compress_segmentby = '', timescaledb.compress_orderby = '"Time" DESC'); + +SELECT + $$ + SELECT * FROM test1 ORDER BY "Time" + $$ AS "QUERY" \gset + +SELECT 'test1' AS "HYPERTABLE_NAME" \gset + +\ir include/compression_test_hypertable.sql diff --git a/tsl/test/sql/include/compression_test_hypertable.sql b/tsl/test/sql/include/compression_test_hypertable.sql new file mode 100644 index 000000000..df027cef9 --- /dev/null +++ b/tsl/test/sql/include/compression_test_hypertable.sql @@ -0,0 +1,39 @@ +-- This file and its contents are licensed under the Timescale License. +-- Please see the included NOTICE for copyright information and +-- LICENSE-TIMESCALE for a copy of the license. + +\set ECHO errors + +DROP TABLE IF EXISTS original_result; + + +CREATE TABLE original_result AS :QUERY; + +SELECT count(compress_chunk(chunk.schema_name|| '.' || chunk.table_name)) +FROM _timescaledb_catalog.chunk chunk +INNER JOIN _timescaledb_catalog.hypertable hypertable ON (chunk.hypertable_id = hypertable.id) +WHERE hypertable.table_name like :'HYPERTABLE_NAME' and chunk.compressed_chunk_id IS NULL; + +--TODO: run query on compressed data and compare to original result once transparent decryption is in + +SELECT count(decompress_chunk(chunk.schema_name|| '.' || chunk.table_name)) +FROM _timescaledb_catalog.chunk chunk +INNER JOIN _timescaledb_catalog.hypertable hypertable ON (chunk.hypertable_id = hypertable.id) +WHERE hypertable.table_name like :'HYPERTABLE_NAME' and chunk.compressed_chunk_id IS NOT NULL; + +--run data on data that's been compressed and decompressed, make sure it's the same. +with original AS ( + SELECT row_number() OVER() row_number, * FROM original_result +), +uncompressed AS ( + SELECT row_number() OVER() row_number, * FROM (:QUERY) as q +) +SELECT 'Number of rows different between original and uncompressed (expect 0)', count(*) +--SELECT original."Time", uncompressed."Time" +FROM original +FULL OUTER JOIN uncompressed ON (original.row_number = uncompressed.row_number) +WHERE (original.*) IS DISTINCT FROM (uncompressed.*); + +--TODO: add pg_dump, restore. + +\set ECHO all diff --git a/tsl/test/sql/include/rand_generator.sql b/tsl/test/sql/include/rand_generator.sql new file mode 100644 index 000000000..35d39653d --- /dev/null +++ b/tsl/test/sql/include/rand_generator.sql @@ -0,0 +1,23 @@ +-- This file and its contents are licensed under the Timescale License. +-- Please see the included NOTICE for copyright information and +-- LICENSE-TIMESCALE for a copy of the license + +-------------------------- +-- cheap rand generator -- +-------------------------- +create table rand_minstd_state(i bigint); + +create function rand_minstd_advance(bigint) returns bigint +language sql immutable as +$$ + select (16807 * $1) % 2147483647 +$$; + +create function gen_rand_minstd() returns bigint +language sql as +$$ + update rand_minstd_state set i = rand_minstd_advance(i) returning i +$$; + +-- seed the random num generator +insert into rand_minstd_state values (321);