Fix indexes during compression and decompression

This rebuilds indexes during compression and decompression. Previously,
indexes were not updated during these operations. We also fix
a small bug with orderby and segmentby handling of empty strings/
lists.

Finally, we add some more tests.

src/compression_with_clause.c

@@ -82,6 +82,9 @@ parse_segment_collist(char *inpstr, Hypertable *hypertable)
 	RawStmt *raw;
 #endif
 
+	if(strlen(inpstr) == 0)
+		return NIL;
+
 	initStringInfo(&buf);
 
 	/* parse the segment by list exactly how you would a group by */
@@ -171,6 +174,9 @@ parse_order_collist(char *inpstr, Hypertable *hypertable)
 	RawStmt *raw;
 #endif
 
+	if(strlen(inpstr) == 0)
+		return NIL;
+
 	initStringInfo(&buf);
 
 	/* parse the segment by list exactly how you would a order by by */

tsl/src/compression/compression.c

@@ -227,6 +227,10 @@ compress_chunk(Oid in_table, Oid out_table, const ColumnCompressionInfo **column
 
 	truncate_relation(in_table);
 
+	/* recreate all indexes on out rel, we already have an exvclusive lock on it
+	 * so the strong locks taken by reindex_relation shouldn't matter. */
+	reindex_relation(out_table, 0, 0);
+
 	RelationClose(out_rel);
 	RelationClose(in_rel);
 }
@@ -879,6 +883,10 @@ decompress_chunk(Oid in_table, Oid out_table)
 		FreeBulkInsertState(decompressor.bistate);
 	}
 
+	/* recreate all indexes on out rel, we already have an exvclusive lock on it
+	 * so the strong locks taken by reindex_relation shouldn't matter. */
+	reindex_relation(out_table, 0, 0);
+
 	RelationClose(out_rel);
 	RelationClose(in_rel);
 }

tsl/test/expected/compression_algos.out

@@ -20,6 +20,10 @@ SELECT ts_test_compression();
  
 (1 row)
 
+\ir include/rand_generator.sql
+-- This file and its contents are licensed under the Timescale License.
+-- Please see the included NOTICE for copyright information and
+-- LICENSE-TIMESCALE for a copy of the license
 --------------------------
 -- cheap rand generator --
 --------------------------

tsl/test/expected/compression_hypertable.out

@@ -0,0 +1,59 @@
+-- This file and its contents are licensed under the Timescale License.
+-- Please see the included NOTICE for copyright information and
+-- LICENSE-TIMESCALE for a copy of the license.
+\ir include/rand_generator.sql
+-- This file and its contents are licensed under the Timescale License.
+-- Please see the included NOTICE for copyright information and
+-- LICENSE-TIMESCALE for a copy of the license
+--------------------------
+-- cheap rand generator --
+--------------------------
+create table rand_minstd_state(i bigint);
+create function rand_minstd_advance(bigint) returns bigint
+language sql immutable as
+$$
+	select (16807 * $1) % 2147483647
+$$;
+create function gen_rand_minstd() returns bigint
+language sql as
+$$
+	update rand_minstd_state set i = rand_minstd_advance(i) returning i
+$$;
+-- seed the random num generator
+insert into rand_minstd_state values (321);
+CREATE TABLE test1 ("Time" timestamptz, i integer, b bigint, t text);
+SELECT table_name from create_hypertable('test1', 'Time', chunk_time_interval=> INTERVAL '1 day');
+NOTICE:  adding not-null constraint to column "Time"
+ table_name 
+------------
+ test1
+(1 row)
+
+INSERT INTO test1 SELECT t,  gen_rand_minstd(), gen_rand_minstd(), gen_rand_minstd()::text FROM generate_series('2018-03-02 1:00'::TIMESTAMPTZ, '2018-03-28 1:00', '1 hour') t;
+ALTER TABLE test1 set (timescaledb.compress, timescaledb.compress_segmentby = '', timescaledb.compress_orderby = '"Time" DESC');
+SELECT
+  $$
+  SELECT * FROM test1 ORDER BY "Time"
+  $$ AS "QUERY" \gset
+SELECT 'test1' AS "HYPERTABLE_NAME" \gset
+\ir include/compression_test_hypertable.sql
+-- This file and its contents are licensed under the Timescale License.
+-- Please see the included NOTICE for copyright information and
+-- LICENSE-TIMESCALE for a copy of the license.
+\set ECHO errors
+psql:include/compression_test_hypertable.sql:7: NOTICE:  table "original_result" does not exist, skipping
+ count 
+-------
+    27
+(1 row)
+
+ count 
+-------
+    27
+(1 row)
+
+                               ?column?                                | count 
+-----------------------------------------------------------------------+-------
+ Number of rows different between original and uncompressed (expect 0) |     0
+(1 row)
+

tsl/test/sql/compression_algos.sql

@@ -15,26 +15,7 @@ AS :TSL_MODULE_PATHNAME LANGUAGE C VOLATILE;
 
 SELECT ts_test_compression();
 
---------------------------
--- cheap rand generator --
---------------------------
-create table rand_minstd_state(i bigint);
-
-create function rand_minstd_advance(bigint) returns bigint
-language sql immutable as
-$$
-	select (16807 * $1) % 2147483647
-$$;
-
-create function gen_rand_minstd() returns bigint
-language sql as
-$$
-	update rand_minstd_state set i = rand_minstd_advance(i) returning i
-$$;
-
--- seed the random num generator
-insert into rand_minstd_state values (321);
-
+\ir include/rand_generator.sql
 
 ------------------------
 -- BIGINT Compression --

tsl/test/sql/compression_hypertable.sql

@@ -0,0 +1,21 @@
+-- This file and its contents are licensed under the Timescale License.
+-- Please see the included NOTICE for copyright information and
+-- LICENSE-TIMESCALE for a copy of the license.
+
+\ir include/rand_generator.sql
+
+CREATE TABLE test1 ("Time" timestamptz, i integer, b bigint, t text);
+SELECT table_name from create_hypertable('test1', 'Time', chunk_time_interval=> INTERVAL '1 day');
+
+INSERT INTO test1 SELECT t,  gen_rand_minstd(), gen_rand_minstd(), gen_rand_minstd()::text FROM generate_series('2018-03-02 1:00'::TIMESTAMPTZ, '2018-03-28 1:00', '1 hour') t;
+
+ALTER TABLE test1 set (timescaledb.compress, timescaledb.compress_segmentby = '', timescaledb.compress_orderby = '"Time" DESC');
+
+SELECT
+  $$
+  SELECT * FROM test1 ORDER BY "Time"
+  $$ AS "QUERY" \gset
+
+SELECT 'test1' AS "HYPERTABLE_NAME" \gset
+
+\ir include/compression_test_hypertable.sql

tsl/test/sql/include/compression_test_hypertable.sql

@@ -0,0 +1,39 @@
+-- This file and its contents are licensed under the Timescale License.
+-- Please see the included NOTICE for copyright information and
+-- LICENSE-TIMESCALE for a copy of the license.
+
+\set ECHO errors
+
+DROP TABLE IF EXISTS original_result;
+
+
+CREATE TABLE original_result AS :QUERY;
+
+SELECT count(compress_chunk(chunk.schema_name|| '.' || chunk.table_name))
+FROM _timescaledb_catalog.chunk chunk
+INNER JOIN _timescaledb_catalog.hypertable hypertable ON (chunk.hypertable_id = hypertable.id)
+WHERE hypertable.table_name like :'HYPERTABLE_NAME' and chunk.compressed_chunk_id IS NULL;
+
+--TODO: run query on compressed data and compare to original result once transparent decryption is in
+
+SELECT count(decompress_chunk(chunk.schema_name|| '.' || chunk.table_name))
+FROM _timescaledb_catalog.chunk chunk
+INNER JOIN _timescaledb_catalog.hypertable hypertable ON (chunk.hypertable_id = hypertable.id)
+WHERE hypertable.table_name like :'HYPERTABLE_NAME' and chunk.compressed_chunk_id IS NOT NULL;
+
+--run data on data that's been compressed and decompressed, make sure it's the same.
+with original AS (
+  SELECT row_number() OVER() row_number, * FROM original_result
+),
+uncompressed AS (
+  SELECT row_number() OVER() row_number, * FROM (:QUERY) as q
+)
+SELECT 'Number of rows different between original and uncompressed (expect 0)', count(*)
+--SELECT original."Time", uncompressed."Time"
+FROM original
+FULL OUTER JOIN uncompressed ON (original.row_number = uncompressed.row_number)
+WHERE (original.*) IS DISTINCT FROM (uncompressed.*);
+
+--TODO: add pg_dump, restore.
+
+\set ECHO all

tsl/test/sql/include/rand_generator.sql

@@ -0,0 +1,23 @@
+-- This file and its contents are licensed under the Timescale License.
+-- Please see the included NOTICE for copyright information and
+-- LICENSE-TIMESCALE for a copy of the license
+
+--------------------------
+-- cheap rand generator --
+--------------------------
+create table rand_minstd_state(i bigint);
+
+create function rand_minstd_advance(bigint) returns bigint
+language sql immutable as
+$$
+	select (16807 * $1) % 2147483647
+$$;
+
+create function gen_rand_minstd() returns bigint
+language sql as
+$$
+	update rand_minstd_state set i = rand_minstd_advance(i) returning i
+$$;
+
+-- seed the random num generator
+insert into rand_minstd_state values (321);