Fix DEFAULT value handling in decompress_chunk

The sql function decompress_chunk did not filled in
default values during its operation.

Fixes #5412

CHANGELOG.md

@@ -18,6 +18,7 @@ accidentally triggering the load of a previous DB version.**
 * #5410 Fix file trailer handling in the COPY fetcher
 * #5233 Out of on_proc_exit slots on guc license change
 * #5427 Handle user-defined FDW options properly
+* #5442 Decompression may have lost DEFAULT values
 
 **Thanks**
 * @nikolaps for reporting an issue with the COPY fetcher

test/sql/include/test_utils.sql

@@ -8,7 +8,7 @@ CREATE OR REPLACE FUNCTION assert_true(
  RETURNS VOID LANGUAGE PLPGSQL IMMUTABLE AS
 $BODY$
 BEGIN
-    IF !val THEN
+    IF val IS NOT TRUE THEN
         RAISE 'Assert failed';
     END IF;
 END
@@ -22,8 +22,8 @@ CREATE OR REPLACE FUNCTION assert_equal(
  RETURNS VOID LANGUAGE PLPGSQL IMMUTABLE AS
 $BODY$
 BEGIN
-    IF val1 != val2 THEN
+    IF (val1 = val2) IS NOT TRUE THEN
-        RAISE 'Assert failed';
+        RAISE 'Assert failed: % = %',val1,val2;
     END IF;
 END
 $BODY$;

tsl/src/compression/compression.c

@@ -1530,7 +1530,8 @@ static void populate_per_compressed_columns_from_data(PerCompressedColumn *per_c
 													  bool *compressed_is_nulls);
 static void row_decompressor_decompress_row(RowDecompressor *row_decompressor);
 static bool per_compressed_col_get_data(PerCompressedColumn *per_compressed_col,
-										Datum *decompressed_datums, bool *decompressed_is_nulls);
+										Datum *decompressed_datums, bool *decompressed_is_nulls,
+										TupleDesc out_desc);
 
 static RowDecompressor
 build_decompressor(Relation in_rel, Relation out_rel)
@@ -1740,7 +1741,8 @@ row_decompressor_decompress_row(RowDecompressor *decompressor)
 		{
 			bool col_is_done = per_compressed_col_get_data(&decompressor->per_compressed_cols[col],
 														   decompressor->decompressed_datums,
-														   decompressor->decompressed_is_nulls);
+														   decompressor->decompressed_is_nulls,
+														   decompressor->out_desc);
 			is_done &= col_is_done;
 		}
 
@@ -1752,6 +1754,7 @@ row_decompressor_decompress_row(RowDecompressor *decompressor)
 			HeapTuple decompressed_tuple = heap_form_tuple(decompressor->out_desc,
 														   decompressor->decompressed_datums,
 														   decompressor->decompressed_is_nulls);
+
 			heap_insert(decompressor->out_rel,
 						decompressed_tuple,
 						decompressor->mycid,
@@ -1774,7 +1777,7 @@ row_decompressor_decompress_row(RowDecompressor *decompressor)
  */
 bool
 per_compressed_col_get_data(PerCompressedColumn *per_compressed_col, Datum *decompressed_datums,
-							bool *decompressed_is_nulls)
+							bool *decompressed_is_nulls, TupleDesc out_desc)
 {
 	DecompressResult decompressed;
 	int16 decompressed_column_offset = per_compressed_col->decompressed_column_offset;
@@ -1794,7 +1797,11 @@ per_compressed_col_get_data(PerCompressedColumn *per_compressed_col, Datum *deco
 	/* compressed NULL */
 	if (per_compressed_col->is_null)
 	{
-		decompressed_is_nulls[decompressed_column_offset] = true;
+		decompressed_datums[decompressed_column_offset] =
+			getmissingattr(out_desc,
+						   decompressed_column_offset + 1,
+						   &decompressed_is_nulls[decompressed_column_offset]);
+
 		return true;
 	}
 

tsl/test/expected/compression_ddl.out

@@ -744,6 +744,33 @@ ALTER TABLE i2844 SET (timescaledb.compress = FALSE);
 -- Please see the included NOTICE for copyright information and
 -- LICENSE-TIMESCALE for a copy of the license.
 \set ECHO errors
+\ir ../../../../test/sql/include/test_utils.sql
+-- This file and its contents are licensed under the Apache License 2.0.
+-- Please see the included NOTICE for copyright information and
+-- LICENSE-APACHE for a copy of the license.
+CREATE OR REPLACE FUNCTION assert_true(
+    val boolean
+)
+ RETURNS VOID LANGUAGE PLPGSQL IMMUTABLE AS
+$BODY$
+BEGIN
+    IF val IS NOT TRUE THEN
+        RAISE 'Assert failed';
+    END IF;
+END
+$BODY$;
+CREATE OR REPLACE FUNCTION assert_equal(
+    val1 anyelement,
+    val2 anyelement
+)
+ RETURNS VOID LANGUAGE PLPGSQL IMMUTABLE AS
+$BODY$
+BEGIN
+    IF (val1 = val2) IS NOT TRUE THEN
+        RAISE 'Assert failed: % = %',val1,val2;
+    END IF;
+END
+$BODY$;
 CREATE TABLE test1 ("Time" timestamptz, intcol integer, bntcol bigint, txtcol text);
 SELECT table_name from create_hypertable('test1', 'Time', chunk_time_interval=> INTERVAL '1 day');
 psql:include/compression_alter.sql:8: NOTICE:  adding not-null constraint to column "Time"
@@ -1053,10 +1080,51 @@ SELECT * FROM test_defaults ORDER BY 1,2;
  Mon Jan 01 00:00:00 2001 PST |         1 |    | 42
 (3 rows)
 
+-- timescale/timescaledb#5412
+ALTER TABLE test_defaults ADD COLUMN c3 int NOT NULL DEFAULT 43;
+SELECT *,assert_equal(c3,43) FROM test_defaults ORDER BY 1,2;
+             time             | device_id | c1 | c2 | c3 | assert_equal 
+------------------------------+-----------+----+----+----+--------------
+ Sat Jan 01 00:00:00 2000 PST |         1 |    | 42 | 43 | 
+ Sat Jan 01 00:00:00 2000 PST |         2 |    | 42 | 43 | 
+ Mon Jan 01 00:00:00 2001 PST |         1 |    | 42 | 43 | 
+(3 rows)
+
+select decompress_chunk(show_chunks('test_defaults'),true);
+psql:include/compression_alter.sql:179: NOTICE:  chunk "_hyper_14_90_chunk" is not compressed
+             decompress_chunk             
+------------------------------------------
+ _timescaledb_internal._hyper_14_89_chunk
+ 
+(2 rows)
+
+SELECT *,assert_equal(c3,43) FROM test_defaults ORDER BY 1,2;
+             time             | device_id | c1 | c2 | c3 | assert_equal 
+------------------------------+-----------+----+----+----+--------------
+ Sat Jan 01 00:00:00 2000 PST |         1 |    | 42 | 43 | 
+ Sat Jan 01 00:00:00 2000 PST |         2 |    | 42 | 43 | 
+ Mon Jan 01 00:00:00 2001 PST |         1 |    | 42 | 43 | 
+(3 rows)
+
+select compress_chunk(show_chunks('test_defaults'));
+              compress_chunk              
+------------------------------------------
+ _timescaledb_internal._hyper_14_89_chunk
+ _timescaledb_internal._hyper_14_90_chunk
+(2 rows)
+
+SELECT *,assert_equal(c3,43) FROM test_defaults ORDER BY 1,2;
+             time             | device_id | c1 | c2 | c3 | assert_equal 
+------------------------------+-----------+----+----+----+--------------
+ Sat Jan 01 00:00:00 2000 PST |         1 |    | 42 | 43 | 
+ Sat Jan 01 00:00:00 2000 PST |         2 |    | 42 | 43 | 
+ Mon Jan 01 00:00:00 2001 PST |         1 |    | 42 | 43 | 
+(3 rows)
+
 -- test dropping columns from compressed
 CREATE TABLE test_drop(f1 text, f2 text, f3 text, time timestamptz, device int, o1 text, o2 text);
 SELECT create_hypertable('test_drop','time');
-psql:include/compression_alter.sql:178: NOTICE:  adding not-null constraint to column "time"
+psql:include/compression_alter.sql:186: NOTICE:  adding not-null constraint to column "time"
     create_hypertable    
 -------------------------
  (16,public,test_drop,t)
@@ -1066,13 +1134,13 @@ ALTER TABLE test_drop SET (timescaledb.compress,timescaledb.compress_segmentby='
 -- dropping segmentby or orderby columns will fail
 \set ON_ERROR_STOP 0
 ALTER TABLE test_drop DROP COLUMN time;
-psql:include/compression_alter.sql:183: ERROR:  cannot drop column named in partition key
+psql:include/compression_alter.sql:191: ERROR:  cannot drop column named in partition key
 ALTER TABLE test_drop DROP COLUMN o1;
-psql:include/compression_alter.sql:184: ERROR:  cannot drop orderby or segmentby column from a hypertable with compression enabled
+psql:include/compression_alter.sql:192: ERROR:  cannot drop orderby or segmentby column from a hypertable with compression enabled
 ALTER TABLE test_drop DROP COLUMN o2;
-psql:include/compression_alter.sql:185: ERROR:  cannot drop orderby or segmentby column from a hypertable with compression enabled
+psql:include/compression_alter.sql:193: ERROR:  cannot drop orderby or segmentby column from a hypertable with compression enabled
 ALTER TABLE test_drop DROP COLUMN device;
-psql:include/compression_alter.sql:186: ERROR:  cannot drop orderby or segmentby column from a hypertable with compression enabled
+psql:include/compression_alter.sql:194: ERROR:  cannot drop orderby or segmentby column from a hypertable with compression enabled
 \set ON_ERROR_STOP 1
 -- switch to WARNING only to suppress compress_chunk NOTICEs
 SET client_min_messages TO WARNING;
@@ -1089,7 +1157,7 @@ ALTER TABLE test_drop DROP COLUMN f2;
 -- test non-existant column
 \set ON_ERROR_STOP 0
 ALTER TABLE test_drop DROP COLUMN f10;
-psql:include/compression_alter.sql:200: ERROR:  column "f10" of relation "test_drop" does not exist
+psql:include/compression_alter.sql:208: ERROR:  column "f10" of relation "test_drop" does not exist
 \set ON_ERROR_STOP 1
 ALTER TABLE test_drop DROP COLUMN IF EXISTS f10;
 INSERT INTO test_drop SELECT NULL,'2001-01-01',2,'o1','o2';
@@ -1171,8 +1239,8 @@ WHERE reltablespace in
   ( SELECT oid from pg_tablespace WHERE spcname = 'tablespace2') ORDER BY 1;
                relname               
 -------------------------------------
- _hyper_18_103_chunk
+ _hyper_18_105_chunk
- _hyper_18_103_chunk_test2_timec_idx
+ _hyper_18_105_chunk_test2_timec_idx
  test2
 (3 rows)
 
@@ -1183,7 +1251,7 @@ SELECT decompress_chunk(ch) INTO decompressed_chunks FROM show_chunks('test2') c
 SELECT compress_chunk(ch) FROM show_chunks('test2') ch;
               compress_chunk               
 -------------------------------------------
- _timescaledb_internal._hyper_18_103_chunk
+ _timescaledb_internal._hyper_18_105_chunk
 (1 row)
 
 -- the chunk, compressed chunk + index + toast tables are in tablespace2 now .
@@ -1199,7 +1267,7 @@ WHERE reltablespace in
 (1 row)
 
 DROP TABLE test2 CASCADE;
-NOTICE:  drop cascades to table _timescaledb_internal.compress_hyper_19_105_chunk
+NOTICE:  drop cascades to table _timescaledb_internal.compress_hyper_19_107_chunk
 DROP TABLESPACE tablespace2;
 -- Create a table with a compressed table and then delete the
 -- compressed table and see that the drop of the hypertable does not

tsl/test/sql/include/compression_alter.sql

@@ -3,7 +3,7 @@
 -- LICENSE-TIMESCALE for a copy of the license.
 
 \ir compression_utils.sql
-
+\ir ../../../../test/sql/include/test_utils.sql
 CREATE TABLE test1 ("Time" timestamptz, intcol integer, bntcol bigint, txtcol text);
 SELECT table_name from create_hypertable('test1', 'Time', chunk_time_interval=> INTERVAL '1 day');
 
@@ -173,6 +173,14 @@ SELECT * FROM test_defaults ORDER BY 1,2;
 CALL recompress_all_chunks('test_defaults', 1, false);
 SELECT * FROM test_defaults ORDER BY 1,2;
 
+-- timescale/timescaledb#5412
+ALTER TABLE test_defaults ADD COLUMN c3 int NOT NULL DEFAULT 43;
+SELECT *,assert_equal(c3,43) FROM test_defaults ORDER BY 1,2;
+select decompress_chunk(show_chunks('test_defaults'),true);
+SELECT *,assert_equal(c3,43) FROM test_defaults ORDER BY 1,2;
+select compress_chunk(show_chunks('test_defaults'));
+SELECT *,assert_equal(c3,43) FROM test_defaults ORDER BY 1,2;
+
 -- test dropping columns from compressed
 CREATE TABLE test_drop(f1 text, f2 text, f3 text, time timestamptz, device int, o1 text, o2 text);
 SELECT create_hypertable('test_drop','time');