diff --git a/CHANGELOG.md b/CHANGELOG.md index 321eeb533..1b054e244 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,6 +9,7 @@ accidentally triggering the load of a previous DB version.** **Minor features** * #2736 Support adding columns to hypertables with compression enabled * #2926 Optimize cagg refresh for small invalidations +* #2909 Support renaming columns of compression enabled hypertables. **Bugfixes** * #2883 Fix join qual propagation for nested joins diff --git a/src/cross_module_fn.c b/src/cross_module_fn.c index a8dfbcb8e..70bc6a5fe 100644 --- a/src/cross_module_fn.c +++ b/src/cross_module_fn.c @@ -292,6 +292,7 @@ TSDLLEXPORT CrossModuleFunctions ts_cm_functions_default = { .ddl_command_end = NULL, .sql_drop = NULL, .process_altertable_cmd = NULL, + .process_rename_cmd = NULL, /* gapfill */ .gapfill_marker = error_no_default_fn_pg_community, diff --git a/src/cross_module_fn.h b/src/cross_module_fn.h index d31b60ba2..8195ed038 100644 --- a/src/cross_module_fn.h +++ b/src/cross_module_fn.h @@ -99,6 +99,7 @@ typedef struct CrossModuleFunctions bool (*process_compress_table)(AlterTableCmd *cmd, Hypertable *ht, WithClauseResult *with_clause_options); void (*process_altertable_cmd)(Hypertable *ht, const AlterTableCmd *cmd); + void (*process_rename_cmd)(Hypertable *ht, const RenameStmt *stmt); PGFunction compress_chunk; PGFunction decompress_chunk; /* The compression functions below are not installed in SQL as part of create extension; diff --git a/src/hypertable_compression.c b/src/hypertable_compression.c index 378586011..2759f4a00 100644 --- a/src/hypertable_compression.c +++ b/src/hypertable_compression.c @@ -158,3 +158,49 @@ ts_hypertable_compression_delete_by_hypertable_id(int32 htid) } return count > 0; } + +TSDLLEXPORT void +ts_hypertable_compression_rename_column(int32 htid, char *old_column_name, char *new_column_name) +{ + bool found = false; + ScanIterator iterator = + ts_scan_iterator_create(HYPERTABLE_COMPRESSION, AccessShareLock, CurrentMemoryContext); + iterator.ctx.index = + catalog_get_index(ts_catalog_get(), HYPERTABLE_COMPRESSION, HYPERTABLE_COMPRESSION_PKEY); + ts_scan_iterator_scan_key_init(&iterator, + Anum_hypertable_compression_pkey_hypertable_id, + BTEqualStrategyNumber, + F_INT4EQ, + Int32GetDatum(htid)); + + ts_scanner_foreach(&iterator) + { + bool isnull; + TupleInfo *ti = ts_scan_iterator_tuple_info(&iterator); + Datum datum = slot_getattr(ti->slot, Anum_hypertable_compression_attname, &isnull); + char *attname = NameStr(*DatumGetName(datum)); + if (strncmp(attname, old_column_name, NAMEDATALEN) == 0) + { + Datum values[Natts_hypertable_compression]; + bool isnulls[Natts_hypertable_compression]; + bool repl[Natts_hypertable_compression] = { false }; + bool should_free; + HeapTuple tuple, new_tuple; + TupleDesc tupdesc = ts_scanner_get_tupledesc(ti); + found = true; + tuple = ts_scanner_fetch_heap_tuple(ti, false, &should_free); + heap_deform_tuple(tuple, tupdesc, values, isnulls); + + values[AttrNumberGetAttrOffset(Anum_hypertable_compression_attname)] = + CStringGetDatum(new_column_name); + repl[AttrNumberGetAttrOffset(Anum_hypertable_compression_attname)] = true; + new_tuple = heap_modify_tuple(tuple, tupdesc, values, isnulls, repl); + ts_catalog_update(ti->scanrel, new_tuple); + + if (should_free) + heap_freetuple(new_tuple); + } + } + if (found == false) + elog(ERROR, "column %s not found in hypertable_compression catalog table", old_column_name); +} diff --git a/src/hypertable_compression.h b/src/hypertable_compression.h index b1634c8c4..17e12bc86 100644 --- a/src/hypertable_compression.h +++ b/src/hypertable_compression.h @@ -17,5 +17,7 @@ ts_hypertable_compression_fill_tuple_values(FormData_hypertable_compression *fd, bool *nulls); extern TSDLLEXPORT bool ts_hypertable_compression_delete_by_hypertable_id(int32 htid); +extern TSDLLEXPORT void ts_hypertable_compression_rename_column(int32 htid, char *old_column_name, + char *new_column_name); #endif diff --git a/src/process_utility.c b/src/process_utility.c index 0963e7392..8ed0e4bec 100644 --- a/src/process_utility.c +++ b/src/process_utility.c @@ -1571,10 +1571,10 @@ process_rename_column(ProcessUtilityArgs *args, Cache *hcache, Oid relid, Rename dim = ts_hyperspace_get_dimension_by_name(ht->space, DIMENSION_TYPE_ANY, stmt->subname); - if (NULL == dim) - return; - - ts_dimension_set_name(dim, stmt->newname); + if (dim) + ts_dimension_set_name(dim, stmt->newname); + if (ts_cm_functions->process_rename_cmd) + ts_cm_functions->process_rename_cmd(ht, stmt); } static void diff --git a/tsl/src/compression/compression.c b/tsl/src/compression/compression.c index 169f74e93..95af7cb3d 100644 --- a/tsl/src/compression/compression.c +++ b/tsl/src/compression/compression.c @@ -1134,6 +1134,8 @@ create_per_compressed_column(TupleDesc in_desc, TupleDesc out_desc, Oid out_reli /* find the mapping from compressed column to uncompressed column, setting * the index of columns that don't have an uncompressed version * (such as metadata) to -1 + * Assumption: column names are the same on compressed and + * uncompressed chunk. */ AttrNumber decompressed_colnum = get_attnum(out_relid, col_name); if (!AttributeNumberIsValid(decompressed_colnum)) diff --git a/tsl/src/compression/create.c b/tsl/src/compression/create.c index 320afad79..a2ab4d951 100644 --- a/tsl/src/compression/create.c +++ b/tsl/src/compression/create.c @@ -16,6 +16,7 @@ #include #include #include +#include #include #include #include @@ -950,6 +951,12 @@ add_column_to_compression_table(Hypertable *compress_ht, CompressColInfo *compre * Note: caller should check security permissions * * Return true if compression was enabled, false otherwise. + * + * Steps: + * 1. Check existing constraints on the table -> can we support them with compression? + * 2. Create internal compression table + mark hypertable as compression enabled + * 3. Add catalog entries to hypertable_compression to record compression settings. + * 4. Copy constraints to internal compression table */ bool tsl_process_compress_table(AlterTableCmd *cmd, Hypertable *ht, @@ -1068,3 +1075,31 @@ tsl_process_compress_table_add_column(Hypertable *ht, ColumnDef *orig_def) /* add catalog entries for the new column for the hypertable */ compresscolinfo_add_catalog_entries(&compress_cols, orig_htid); } + +/* Rename a column on a hypertable that has compression enabled. + * + * This function renames the existing column in the internal compression table. + * We assume that there is a 1-1 mapping between the original chunk and + * compressed chunk column names and that the names are identical. + * Also update any metadata associated with the column. + */ +void +tsl_process_compress_table_rename_column(Hypertable *ht, const RenameStmt *stmt) +{ + int32 orig_htid = ht->fd.id; + + Assert(stmt->relationType == OBJECT_TABLE && stmt->renameType == OBJECT_COLUMN); + Assert(TS_HYPERTABLE_HAS_COMPRESSION_ENABLED(ht)); + if (TS_HYPERTABLE_HAS_COMPRESSION_TABLE(ht)) + { + int32 compress_htid = ht->fd.compressed_hypertable_id; + Hypertable *compress_ht = ts_hypertable_get_by_id(compress_htid); + RenameStmt *compress_col_stmt = (RenameStmt *) copyObject(stmt); + compress_col_stmt->relation = makeRangeVar(NameStr(compress_ht->fd.schema_name), + NameStr(compress_ht->fd.table_name), + -1); + ExecRenameStmt(compress_col_stmt); + } + // update catalog entries for the renamed column for the hypertable + ts_hypertable_compression_rename_column(orig_htid, stmt->subname, stmt->newname); +} diff --git a/tsl/src/compression/create.h b/tsl/src/compression/create.h index 8b8be7285..ce426fcbb 100644 --- a/tsl/src/compression/create.h +++ b/tsl/src/compression/create.h @@ -20,6 +20,7 @@ bool tsl_process_compress_table(AlterTableCmd *cmd, Hypertable *ht, WithClauseResult *with_clause_options); void tsl_process_compress_table_add_column(Hypertable *ht, ColumnDef *orig_def); Chunk *create_compress_chunk_table(Hypertable *compress_ht, Chunk *src_chunk); +void tsl_process_compress_table_rename_column(Hypertable *ht, const RenameStmt *stmt); char *compression_column_segment_min_name(const FormData_hypertable_compression *fd); char *compression_column_segment_max_name(const FormData_hypertable_compression *fd); diff --git a/tsl/src/init.c b/tsl/src/init.c index 2f3e49842..390b930f1 100644 --- a/tsl/src/init.c +++ b/tsl/src/init.c @@ -141,6 +141,7 @@ CrossModuleFunctions tsl_cm_functions = { .array_compressor_finish = tsl_array_compressor_finish, .process_compress_table = tsl_process_compress_table, .process_altertable_cmd = tsl_process_altertable_cmd, + .process_rename_cmd = tsl_process_rename_cmd, .compress_chunk = tsl_compress_chunk, .decompress_chunk = tsl_decompress_chunk, .data_node_add = data_node_add, diff --git a/tsl/src/process_utility.c b/tsl/src/process_utility.c index ce9d348f8..6abbd9495 100644 --- a/tsl/src/process_utility.c +++ b/tsl/src/process_utility.c @@ -6,13 +6,15 @@ #include #include #include +#include #include +#include "compression/create.h" +#include "hypertable_cache.h" #include "process_utility.h" #include "remote/dist_commands.h" #include "remote/connection_cache.h" #include "remote/dist_ddl.h" -#include "compression/create.h" void tsl_ddl_command_start(ProcessUtilityArgs *args) @@ -41,6 +43,18 @@ tsl_process_altertable_cmd(Hypertable *ht, const AlterTableCmd *cmd) } } +void +tsl_process_rename_cmd(Hypertable *ht, const RenameStmt *stmt) +{ + if (stmt->renameType == OBJECT_COLUMN) + { + if (TS_HYPERTABLE_HAS_COMPRESSION_TABLE(ht) || TS_HYPERTABLE_HAS_COMPRESSION_ENABLED(ht)) + { + tsl_process_compress_table_rename_column(ht, stmt); + } + } +} + void tsl_ddl_command_end(EventTriggerData *command) { diff --git a/tsl/src/process_utility.h b/tsl/src/process_utility.h index d566933b9..3c4d5c039 100644 --- a/tsl/src/process_utility.h +++ b/tsl/src/process_utility.h @@ -15,5 +15,6 @@ extern void tsl_ddl_command_start(ProcessUtilityArgs *args); extern void tsl_ddl_command_end(EventTriggerData *command); extern void tsl_sql_drop(List *dropped_objects); extern void tsl_process_altertable_cmd(Hypertable *ht, const AlterTableCmd *cmd); +extern void tsl_process_rename_cmd(Hypertable *ht, const RenameStmt *stmt); #endif /* TIMESCALEDB_TSL_PROCESS_UTILITY_H */ diff --git a/tsl/test/expected/compression_ddl.out b/tsl/test/expected/compression_ddl.out index d0aeb11cb..05c823310 100644 --- a/tsl/test/expected/compression_ddl.out +++ b/tsl/test/expected/compression_ddl.out @@ -681,7 +681,46 @@ WHERE hypertable.table_name like 'test1' ORDER BY chunk.id ) as subq; (1 row) DROP TABLE test1; --- compression alter table tests +-- test disabling compression on hypertables with caggs and dropped chunks +-- github issue 2844 +CREATE TABLE i2844 (created_at timestamptz NOT NULL,c1 float); +SELECT create_hypertable('i2844', 'created_at', chunk_time_interval => '6 hour'::interval); + create_hypertable +-------------------- + (7,public,i2844,t) +(1 row) + +INSERT INTO i2844 SELECT generate_series('2000-01-01'::timestamptz, '2000-01-02'::timestamptz,'1h'::interval); +CREATE MATERIALIZED VIEW test_agg WITH (timescaledb.continuous) AS SELECT time_bucket('1 hour', created_at) AS bucket, AVG(c1) AS avg_c1 FROM i2844 GROUP BY bucket; +NOTICE: refreshing continuous aggregate "test_agg" +ALTER TABLE i2844 SET (timescaledb.compress); +SELECT compress_chunk(show_chunks) AS compressed_chunk FROM show_chunks('i2844'); + compressed_chunk +----------------------------------------- + _timescaledb_internal._hyper_7_62_chunk + _timescaledb_internal._hyper_7_63_chunk + _timescaledb_internal._hyper_7_64_chunk + _timescaledb_internal._hyper_7_65_chunk + _timescaledb_internal._hyper_7_66_chunk +(5 rows) + +SELECT drop_chunks('i2844', older_than => '2000-01-01 18:00'::timestamptz); + drop_chunks +----------------------------------------- + _timescaledb_internal._hyper_7_62_chunk + _timescaledb_internal._hyper_7_63_chunk + _timescaledb_internal._hyper_7_64_chunk +(3 rows) + +SELECT decompress_chunk(show_chunks, if_compressed => TRUE) AS decompressed_chunks FROM show_chunks('i2844'); + decompressed_chunks +----------------------------------------- + _timescaledb_internal._hyper_7_65_chunk + _timescaledb_internal._hyper_7_66_chunk +(2 rows) + +ALTER TABLE i2844 SET (timescaledb.compress = FALSE); +-- compression alter schema tests \ir include/compression_alter.sql -- This file and its contents are licensed under the Timescale License. -- Please see the included NOTICE for copyright information and @@ -697,6 +736,8 @@ psql:include/compression_alter.sql:6: NOTICE: adding not-null constraint to col INSERT INTO test1 SELECT t, gen_rand_minstd(), gen_rand_minstd(), gen_rand_minstd()::text FROM generate_series('2018-03-02 1:00'::TIMESTAMPTZ, '2018-03-05 1:00', '1 hour') t; +INSERT INTO test1 +SELECT '2018-03-04 2:00', 100, 200, 'hello' ; ALTER TABLE test1 set (timescaledb.compress, timescaledb.compress_segmentby = 'bntcol', timescaledb.compress_orderby = '"Time" DESC'); SELECT COUNT(*) AS count_compressed FROM @@ -719,12 +760,12 @@ SELECT * FROM _timescaledb_catalog.hypertable_compression ORDER BY attname; hypertable_id | attname | compression_algorithm_id | segmentby_column_index | orderby_column_index | orderby_asc | orderby_nullsfirst ---------------+----------+--------------------------+------------------------+----------------------+-------------+-------------------- - 7 | Time | 4 | | 1 | f | t - 7 | bntcol | 0 | 1 | | | - 7 | intcol | 4 | | | | - 7 | new_coli | 4 | | | | - 7 | new_colv | 2 | | | | - 7 | txtcol | 2 | | | | + 10 | Time | 4 | | 1 | f | t + 10 | bntcol | 0 | 1 | | | + 10 | intcol | 4 | | | | + 10 | new_coli | 4 | | | | + 10 | new_colv | 2 | | | | + 10 | txtcol | 2 | | | | (6 rows) SELECT count(*) from test1 where new_coli is not null; @@ -736,7 +777,7 @@ SELECT count(*) from test1 where new_coli is not null; SELECT count(*) from test1 where new_colv is null; count ------- - 73 + 74 (1 row) --decompress 1 chunk and query again @@ -764,7 +805,7 @@ SELECT count(*) from test1 where new_coli is not null; SELECT count(*) from test1 where new_colv is null; count ------- - 73 + 74 (1 row) --compress all chunks and query --- @@ -810,42 +851,132 @@ SELECT count(*) from test1 where new_colv = '101t'; (1 row) CREATE INDEX new_index ON test1(new_colv); --- test disabling compression on hypertables with caggs and dropped chunks --- github issue 2844 -CREATE TABLE i2844 (created_at timestamptz NOT NULL,c1 float); -SELECT create_hypertable('i2844', 'created_at', chunk_time_interval => '6 hour'::interval); - create_hypertable --------------------- - (9,public,i2844,t) +-- TEST 2: ALTER TABLE rename column +SELECT * FROM _timescaledb_catalog.hypertable_compression +WHERE attname = 'new_coli' and hypertable_id = (SELECT id from _timescaledb_catalog.hypertable + WHERE table_name = 'test1' ); + hypertable_id | attname | compression_algorithm_id | segmentby_column_index | orderby_column_index | orderby_asc | orderby_nullsfirst +---------------+----------+--------------------------+------------------------+----------------------+-------------+-------------------- + 10 | new_coli | 4 | | | | (1 row) -INSERT INTO i2844 SELECT generate_series('2000-01-01'::timestamptz, '2000-01-02'::timestamptz,'1h'::interval); -CREATE MATERIALIZED VIEW test_agg WITH (timescaledb.continuous) AS SELECT time_bucket('1 hour', created_at) AS bucket, AVG(c1) AS avg_c1 FROM i2844 GROUP BY bucket; -psql:include/compression_alter.sql:76: NOTICE: refreshing continuous aggregate "test_agg" -ALTER TABLE i2844 SET (timescaledb.compress); -SELECT compress_chunk(show_chunks) AS compressed_chunk FROM show_chunks('i2844'); - compressed_chunk ------------------------------------------ - _timescaledb_internal._hyper_9_75_chunk - _timescaledb_internal._hyper_9_76_chunk - _timescaledb_internal._hyper_9_77_chunk - _timescaledb_internal._hyper_9_78_chunk - _timescaledb_internal._hyper_9_79_chunk -(5 rows) +ALTER TABLE test1 RENAME new_coli TO coli; +SELECT * FROM _timescaledb_catalog.hypertable_compression +WHERE attname = 'coli' and hypertable_id = (SELECT id from _timescaledb_catalog.hypertable + WHERE table_name = 'test1' ); + hypertable_id | attname | compression_algorithm_id | segmentby_column_index | orderby_column_index | orderby_asc | orderby_nullsfirst +---------------+---------+--------------------------+------------------------+----------------------+-------------+-------------------- + 10 | coli | 4 | | | | +(1 row) -SELECT drop_chunks('i2844', older_than => '2000-01-01 18:00'::timestamptz); - drop_chunks ------------------------------------------ - _timescaledb_internal._hyper_9_75_chunk - _timescaledb_internal._hyper_9_76_chunk - _timescaledb_internal._hyper_9_77_chunk -(3 rows) +SELECT count(*) from test1 where coli = 100; + count +------- + 25 +(1 row) -SELECT decompress_chunk(show_chunks, if_compressed => TRUE) AS decompressed_chunks FROM show_chunks('i2844'); - decompressed_chunks ------------------------------------------ - _timescaledb_internal._hyper_9_78_chunk - _timescaledb_internal._hyper_9_79_chunk -(2 rows) +--rename segment by column name +ALTER TABLE test1 RENAME bntcol TO bigintcol ; +SELECT * FROM _timescaledb_catalog.hypertable_compression +WHERE attname = 'bigintcol' and hypertable_id = (SELECT id from _timescaledb_catalog.hypertable + WHERE table_name = 'test1' ); + hypertable_id | attname | compression_algorithm_id | segmentby_column_index | orderby_column_index | orderby_asc | orderby_nullsfirst +---------------+-----------+--------------------------+------------------------+----------------------+-------------+-------------------- + 10 | bigintcol | 0 | 1 | | | +(1 row) + +--query by segment by column name +SELECT * from test1 WHERE bigintcol = 100; + Time | intcol | bigintcol | txtcol | coli | new_colv +------+--------+-----------+--------+------+---------- +(0 rows) + +SELECT * from test1 WHERE bigintcol = 200; + Time | intcol | bigintcol | txtcol | coli | new_colv +------------------------------+--------+-----------+--------+------+---------- + Sun Mar 04 02:00:00 2018 PST | 100 | 200 | hello | | +(1 row) + +-- add a new chunk and compress +INSERT INTO test1 SELECT '2019-03-04 2:00', 99, 800, 'newchunk' ; +SELECT COUNT(*) AS count_compressed +FROM +( +SELECT compress_chunk(chunk.schema_name|| '.' || chunk.table_name) +FROM _timescaledb_catalog.chunk chunk +INNER JOIN _timescaledb_catalog.hypertable hypertable ON (chunk.hypertable_id = hypertable.id) +WHERE hypertable.table_name = 'test1' and chunk.compressed_chunk_id IS NULL ORDER BY chunk.id +) q; + count_compressed +------------------ + 1 +(1 row) + +--check if all chunks have new column names +--both counts should be equal +SELECT count(*) FROM _timescaledb_catalog.chunk +WHERE hypertable_id = ( SELECT id FROM _timescaledb_catalog.hypertable + WHERE table_name = 'test1' ); + count +------- + 7 +(1 row) + +SELECT count(*) +FROM ( SELECT attrelid::regclass, attname FROM pg_attribute + WHERE attrelid in (SELECT inhrelid::regclass from pg_inherits + where inhparent = 'test1'::regclass) + and attname = 'bigintcol' ) q; + count +------- + 7 +(1 row) + +--check count on internal compression table too i.e. all the chunks have +--the correct column name +SELECT format('%I.%I', cht.schema_name, cht.table_name) AS "COMPRESSION_TBLNM" +FROM _timescaledb_catalog.hypertable ht, _timescaledb_catalog.hypertable cht +WHERE ht.table_name = 'test1' and cht.id = ht.compressed_hypertable_id \gset +SELECT count(*) +FROM ( SELECT attrelid::regclass, attname FROM pg_attribute + WHERE attrelid in (SELECT inhrelid::regclass from pg_inherits + where inhparent = :'COMPRESSION_TBLNM'::regclass ) + and attname = 'bigintcol' ) q; + count +------- + 7 +(1 row) + +-- check column name truncation with renames +-- check if the name change is reflected for settings +ALTER TABLE test1 RENAME bigintcol TO +ccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccabdeeeeeeccccccccccccc; +psql:include/compression_alter.sql:133: NOTICE: identifier "ccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccabdeeeeeeccccccccccccc" will be truncated to "cccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccca" +SELECT * from timescaledb_information.compression_settings +WHERE hypertable_name = 'test1' and attname like 'ccc%'; + hypertable_schema | hypertable_name | attname | segmentby_column_index | orderby_column_index | orderby_asc | orderby_nullsfirst +-------------------+-----------------+-----------------------------------------------------------------+------------------------+----------------------+-------------+-------------------- + public | test1 | cccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccca | 1 | | | +(1 row) + +SELECT count(*) +FROM ( SELECT attrelid::regclass, attname FROM pg_attribute + WHERE attrelid in (SELECT inhrelid::regclass from pg_inherits + where inhparent = :'COMPRESSION_TBLNM'::regclass ) + and attname like 'ccc%a' ) q; + count +------- + 7 +(1 row) + +ALTER TABLE test1 RENAME +ccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccabdeeeeeeccccccccccccc +TO bigintcol; +psql:include/compression_alter.sql:146: NOTICE: identifier "ccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccabdeeeeeeccccccccccccc" will be truncated to "cccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccca" +SELECT * from timescaledb_information.compression_settings +WHERE hypertable_name = 'test1' and attname = 'bigintcol' ; + hypertable_schema | hypertable_name | attname | segmentby_column_index | orderby_column_index | orderby_asc | orderby_nullsfirst +-------------------+-----------------+-----------+------------------------+----------------------+-------------+-------------------- + public | test1 | bigintcol | 1 | | | +(1 row) -ALTER TABLE i2844 SET (timescaledb.compress = FALSE); diff --git a/tsl/test/expected/dist_compression.out b/tsl/test/expected/dist_compression.out index 1980b4045..31ba334ad 100644 --- a/tsl/test/expected/dist_compression.out +++ b/tsl/test/expected/dist_compression.out @@ -50,7 +50,7 @@ ORDER BY 1; compressed | 1 | (1 row) -SELECT * FROM timescaledb_information.compression_settings; +SELECT * FROM timescaledb_information.compression_settings order by attname; hypertable_schema | hypertable_name | attname | segmentby_column_index | orderby_column_index | orderby_asc | orderby_nullsfirst -------------------+-----------------+---------+------------------------+----------------------+-------------+-------------------- public | compressed | device | 1 | | | @@ -368,13 +368,13 @@ ORDER BY 1; compressed | 0 | (1 row) -SELECT * FROM timescaledb_information.compression_settings; +SELECT * FROM timescaledb_information.compression_settings order by attname; hypertable_schema | hypertable_name | attname | segmentby_column_index | orderby_column_index | orderby_asc | orderby_nullsfirst -------------------+-----------------+---------+------------------------+----------------------+-------------+-------------------- (0 rows) --Now re-enable compression -ALTER TABLE compressed SET (timescaledb.compress); +ALTER TABLE compressed SET (timescaledb.compress, timescaledb.compress_segmentby='device'); SELECT table_name, compression_state, compressed_hypertable_id FROM _timescaledb_catalog.hypertable ORDER BY 1; @@ -383,11 +383,12 @@ ORDER BY 1; compressed | 1 | (1 row) -SELECT * FROM timescaledb_information.compression_settings; +SELECT * FROM timescaledb_information.compression_settings order by attname; hypertable_schema | hypertable_name | attname | segmentby_column_index | orderby_column_index | orderby_asc | orderby_nullsfirst -------------------+-----------------+---------+------------------------+----------------------+-------------+-------------------- + public | compressed | device | 1 | | | public | compressed | time | | 1 | f | t -(1 row) +(2 rows) SELECT compress_chunk(chunk, if_not_compressed => true) FROM show_chunks('compressed') AS chunk @@ -418,7 +419,7 @@ SELECT * FROM _timescaledb_catalog.hypertable_compression ORDER BY attname; hypertable_id | attname | compression_algorithm_id | segmentby_column_index | orderby_column_index | orderby_asc | orderby_nullsfirst ---------------+----------+--------------------------+------------------------+----------------------+-------------+-------------------- - 1 | device | 4 | | | | + 1 | device | 0 | 1 | | | 1 | new_coli | 4 | | | | 1 | new_colv | 2 | | | | 1 | temp | 3 | | | | @@ -455,3 +456,56 @@ SELECT * from compressed where new_coli is not null; Thu Aug 01 00:00:00 2019 PDT | 100 | 100 | 1 | newcolv (1 row) +-- ALTER TABLE rename column does not work on distributed hypertables +\set ON_ERROR_STOP 0 +ALTER TABLE compressed RENAME new_coli TO new_intcol ; +ERROR: operation not supported on distributed hypertable +ALTER TABLE compressed RENAME device TO device_id ; +ERROR: operation not supported on distributed hypertable +\set ON_ERROR_STOP 1 +SELECT * FROM test.remote_exec( NULL, + $$ SELECT * FROM _timescaledb_catalog.hypertable_compression + WHERE attname = 'device' OR attname = 'new_coli' and + hypertable_id = (SELECT id from _timescaledb_catalog.hypertable + WHERE table_name = 'compressed' ) ORDER BY attname; $$ ); +NOTICE: [db_dist_compression_1]: SELECT * FROM _timescaledb_catalog.hypertable_compression + WHERE attname = 'device' OR attname = 'new_coli' and + hypertable_id = (SELECT id from _timescaledb_catalog.hypertable + WHERE table_name = 'compressed' ) ORDER BY attname +NOTICE: [db_dist_compression_1]: +hypertable_id|attname |compression_algorithm_id|segmentby_column_index|orderby_column_index|orderby_asc|orderby_nullsfirst +-------------+--------+------------------------+----------------------+--------------------+-----------+------------------ + 1|device | 0| 1| | | + 1|new_coli| 4| | | | +(2 rows) + + +NOTICE: [db_dist_compression_2]: SELECT * FROM _timescaledb_catalog.hypertable_compression + WHERE attname = 'device' OR attname = 'new_coli' and + hypertable_id = (SELECT id from _timescaledb_catalog.hypertable + WHERE table_name = 'compressed' ) ORDER BY attname +NOTICE: [db_dist_compression_2]: +hypertable_id|attname |compression_algorithm_id|segmentby_column_index|orderby_column_index|orderby_asc|orderby_nullsfirst +-------------+--------+------------------------+----------------------+--------------------+-----------+------------------ + 1|device | 0| 1| | | + 1|new_coli| 4| | | | +(2 rows) + + +NOTICE: [db_dist_compression_3]: SELECT * FROM _timescaledb_catalog.hypertable_compression + WHERE attname = 'device' OR attname = 'new_coli' and + hypertable_id = (SELECT id from _timescaledb_catalog.hypertable + WHERE table_name = 'compressed' ) ORDER BY attname +NOTICE: [db_dist_compression_3]: +hypertable_id|attname |compression_algorithm_id|segmentby_column_index|orderby_column_index|orderby_asc|orderby_nullsfirst +-------------+--------+------------------------+----------------------+--------------------+-----------+------------------ + 1|device | 0| 1| | | + 1|new_coli| 4| | | | +(2 rows) + + + remote_exec +------------- + +(1 row) + diff --git a/tsl/test/sql/compression_ddl.sql b/tsl/test/sql/compression_ddl.sql index b5741d1b6..328573794 100644 --- a/tsl/test/sql/compression_ddl.sql +++ b/tsl/test/sql/compression_ddl.sql @@ -446,5 +446,24 @@ WHERE hypertable.table_name like 'test1' ORDER BY chunk.id ) as subq; DROP TABLE test1; --- compression alter table tests +-- test disabling compression on hypertables with caggs and dropped chunks +-- github issue 2844 +CREATE TABLE i2844 (created_at timestamptz NOT NULL,c1 float); +SELECT create_hypertable('i2844', 'created_at', chunk_time_interval => '6 hour'::interval); +INSERT INTO i2844 SELECT generate_series('2000-01-01'::timestamptz, '2000-01-02'::timestamptz,'1h'::interval); + +CREATE MATERIALIZED VIEW test_agg WITH (timescaledb.continuous) AS SELECT time_bucket('1 hour', created_at) AS bucket, AVG(c1) AS avg_c1 FROM i2844 GROUP BY bucket; + +ALTER TABLE i2844 SET (timescaledb.compress); + +SELECT compress_chunk(show_chunks) AS compressed_chunk FROM show_chunks('i2844'); +SELECT drop_chunks('i2844', older_than => '2000-01-01 18:00'::timestamptz); +SELECT decompress_chunk(show_chunks, if_compressed => TRUE) AS decompressed_chunks FROM show_chunks('i2844'); + +ALTER TABLE i2844 SET (timescaledb.compress = FALSE); + +-- compression alter schema tests \ir include/compression_alter.sql + + + diff --git a/tsl/test/sql/dist_compression.sql b/tsl/test/sql/dist_compression.sql index 1d36f0267..9900cc9f1 100644 --- a/tsl/test/sql/dist_compression.sql +++ b/tsl/test/sql/dist_compression.sql @@ -30,7 +30,7 @@ SELECT table_name, compression_state, compressed_hypertable_id FROM _timescaledb_catalog.hypertable ORDER BY 1; -SELECT * FROM timescaledb_information.compression_settings; +SELECT * FROM timescaledb_information.compression_settings order by attname; \x SELECT * FROM _timescaledb_catalog.hypertable WHERE table_name = 'compressed'; @@ -123,14 +123,14 @@ SELECT table_name, compression_state, compressed_hypertable_id FROM _timescaledb_catalog.hypertable ORDER BY 1; -SELECT * FROM timescaledb_information.compression_settings; +SELECT * FROM timescaledb_information.compression_settings order by attname; --Now re-enable compression -ALTER TABLE compressed SET (timescaledb.compress); +ALTER TABLE compressed SET (timescaledb.compress, timescaledb.compress_segmentby='device'); SELECT table_name, compression_state, compressed_hypertable_id FROM _timescaledb_catalog.hypertable ORDER BY 1; -SELECT * FROM timescaledb_information.compression_settings; +SELECT * FROM timescaledb_information.compression_settings order by attname; SELECT compress_chunk(chunk, if_not_compressed => true) FROM show_chunks('compressed') AS chunk ORDER BY chunk @@ -164,3 +164,15 @@ WHERE hypertable.table_name like 'compressed' and chunk.compressed_chunk_id IS N AS sub; SELECT * from compressed where new_coli is not null; + +-- ALTER TABLE rename column does not work on distributed hypertables +\set ON_ERROR_STOP 0 +ALTER TABLE compressed RENAME new_coli TO new_intcol ; +ALTER TABLE compressed RENAME device TO device_id ; +\set ON_ERROR_STOP 1 + +SELECT * FROM test.remote_exec( NULL, + $$ SELECT * FROM _timescaledb_catalog.hypertable_compression + WHERE attname = 'device' OR attname = 'new_coli' and + hypertable_id = (SELECT id from _timescaledb_catalog.hypertable + WHERE table_name = 'compressed' ) ORDER BY attname; $$ ); diff --git a/tsl/test/sql/include/compression_alter.sql b/tsl/test/sql/include/compression_alter.sql index b31185ee8..51e16dae3 100644 --- a/tsl/test/sql/include/compression_alter.sql +++ b/tsl/test/sql/include/compression_alter.sql @@ -8,6 +8,8 @@ SELECT table_name from create_hypertable('test1', 'Time', chunk_time_interval=> INSERT INTO test1 SELECT t, gen_rand_minstd(), gen_rand_minstd(), gen_rand_minstd()::text FROM generate_series('2018-03-02 1:00'::TIMESTAMPTZ, '2018-03-05 1:00', '1 hour') t; +INSERT INTO test1 +SELECT '2018-03-04 2:00', 100, 200, 'hello' ; ALTER TABLE test1 set (timescaledb.compress, timescaledb.compress_segmentby = 'bntcol', timescaledb.compress_orderby = '"Time" DESC'); @@ -67,21 +69,81 @@ SELECT count(*) from test1 where new_colv = '101t'; CREATE INDEX new_index ON test1(new_colv); --- test disabling compression on hypertables with caggs and dropped chunks --- github issue 2844 -CREATE TABLE i2844 (created_at timestamptz NOT NULL,c1 float); -SELECT create_hypertable('i2844', 'created_at', chunk_time_interval => '6 hour'::interval); -INSERT INTO i2844 SELECT generate_series('2000-01-01'::timestamptz, '2000-01-02'::timestamptz,'1h'::interval); +-- TEST 2: ALTER TABLE rename column +SELECT * FROM _timescaledb_catalog.hypertable_compression +WHERE attname = 'new_coli' and hypertable_id = (SELECT id from _timescaledb_catalog.hypertable + WHERE table_name = 'test1' ); -CREATE MATERIALIZED VIEW test_agg WITH (timescaledb.continuous) AS SELECT time_bucket('1 hour', created_at) AS bucket, AVG(c1) AS avg_c1 FROM i2844 GROUP BY bucket; +ALTER TABLE test1 RENAME new_coli TO coli; +SELECT * FROM _timescaledb_catalog.hypertable_compression +WHERE attname = 'coli' and hypertable_id = (SELECT id from _timescaledb_catalog.hypertable + WHERE table_name = 'test1' ); +SELECT count(*) from test1 where coli = 100; -ALTER TABLE i2844 SET (timescaledb.compress); +--rename segment by column name +ALTER TABLE test1 RENAME bntcol TO bigintcol ; -SELECT compress_chunk(show_chunks) AS compressed_chunk FROM show_chunks('i2844'); -SELECT drop_chunks('i2844', older_than => '2000-01-01 18:00'::timestamptz); -SELECT decompress_chunk(show_chunks, if_compressed => TRUE) AS decompressed_chunks FROM show_chunks('i2844'); +SELECT * FROM _timescaledb_catalog.hypertable_compression +WHERE attname = 'bigintcol' and hypertable_id = (SELECT id from _timescaledb_catalog.hypertable + WHERE table_name = 'test1' ); -ALTER TABLE i2844 SET (timescaledb.compress = FALSE); +--query by segment by column name +SELECT * from test1 WHERE bigintcol = 100; +SELECT * from test1 WHERE bigintcol = 200; +-- add a new chunk and compress +INSERT INTO test1 SELECT '2019-03-04 2:00', 99, 800, 'newchunk' ; +SELECT COUNT(*) AS count_compressed +FROM +( +SELECT compress_chunk(chunk.schema_name|| '.' || chunk.table_name) +FROM _timescaledb_catalog.chunk chunk +INNER JOIN _timescaledb_catalog.hypertable hypertable ON (chunk.hypertable_id = hypertable.id) +WHERE hypertable.table_name = 'test1' and chunk.compressed_chunk_id IS NULL ORDER BY chunk.id +) q; +--check if all chunks have new column names +--both counts should be equal +SELECT count(*) FROM _timescaledb_catalog.chunk +WHERE hypertable_id = ( SELECT id FROM _timescaledb_catalog.hypertable + WHERE table_name = 'test1' ); + +SELECT count(*) +FROM ( SELECT attrelid::regclass, attname FROM pg_attribute + WHERE attrelid in (SELECT inhrelid::regclass from pg_inherits + where inhparent = 'test1'::regclass) + and attname = 'bigintcol' ) q; + +--check count on internal compression table too i.e. all the chunks have +--the correct column name +SELECT format('%I.%I', cht.schema_name, cht.table_name) AS "COMPRESSION_TBLNM" +FROM _timescaledb_catalog.hypertable ht, _timescaledb_catalog.hypertable cht +WHERE ht.table_name = 'test1' and cht.id = ht.compressed_hypertable_id \gset + +SELECT count(*) +FROM ( SELECT attrelid::regclass, attname FROM pg_attribute + WHERE attrelid in (SELECT inhrelid::regclass from pg_inherits + where inhparent = :'COMPRESSION_TBLNM'::regclass ) + and attname = 'bigintcol' ) q; + +-- check column name truncation with renames +-- check if the name change is reflected for settings +ALTER TABLE test1 RENAME bigintcol TO +ccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccabdeeeeeeccccccccccccc; + +SELECT * from timescaledb_information.compression_settings +WHERE hypertable_name = 'test1' and attname like 'ccc%'; + +SELECT count(*) +FROM ( SELECT attrelid::regclass, attname FROM pg_attribute + WHERE attrelid in (SELECT inhrelid::regclass from pg_inherits + where inhparent = :'COMPRESSION_TBLNM'::regclass ) + and attname like 'ccc%a' ) q; + +ALTER TABLE test1 RENAME +ccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccabdeeeeeeccccccccccccc +TO bigintcol; + +SELECT * from timescaledb_information.compression_settings +WHERE hypertable_name = 'test1' and attname = 'bigintcol' ;