postgres/timescaledb
1
0
Fork 0
mirror of https://github.com/timescale/timescaledb.git synced 2025-05-16 10:33:27 +08:00
Code Issues Packages Projects Releases Wiki Activity

Add GUC to enable/disable DML decompression

Browse Source
This commit is contained in:
Fabrízio de Royes Mello 2023-05-03 16:01:34 -03:00
parent 6782beb150
commit 3dc6824eb5
10 changed files with 276 additions and 16 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

12
src/guc.c
View File

@ -75,6 +75,7 @@ bool ts_guc_enable_qual_propagation = true;
bool ts_guc_enable_cagg_reorder_groupby = true;
bool ts_guc_enable_now_constify = true;
bool ts_guc_enable_osm_reads = true;
TSDLLEXPORT bool ts_guc_enable_dml_decompression = true;
TSDLLEXPORT bool ts_guc_enable_transparent_decompression = true;
TSDLLEXPORT bool ts_guc_enable_decompression_sorted_merge = true;
bool ts_guc_enable_per_data_node_queries = true;
@ -269,6 +270,17 @@ _guc_init(void)
NULL,
NULL);
DefineCustomBoolVariable("timescaledb.enable_dml_decompression",
"Enable DML decompression",
"Enable DML decompression when modifying compressed hypertable",
&ts_guc_enable_dml_decompression,
true,
PGC_USERSET,
0,
NULL,
NULL,
NULL);
DefineCustomBoolVariable("timescaledb.enable_transparent_decompression",
"Enable transparent decompression",
"Enable transparent decompression when querying hypertable",

1
src/guc.h
View File

@ -26,6 +26,7 @@ extern bool ts_guc_enable_constraint_exclusion;
extern bool ts_guc_enable_cagg_reorder_groupby;
extern bool ts_guc_enable_now_constify;
extern bool ts_guc_enable_osm_reads;
extern TSDLLEXPORT bool ts_guc_enable_dml_decompression;
extern TSDLLEXPORT bool ts_guc_enable_transparent_decompression;
extern TSDLLEXPORT bool ts_guc_enable_decompression_sorted_merge;
extern TSDLLEXPORT bool ts_guc_enable_per_data_node_queries;

2
src/planner/planner.c
View File

@ -1339,7 +1339,7 @@ timescaledb_get_relation_info_hook(PlannerInfo *root, Oid relation_objectid, boo
RangeTblEntry *chunk_rte = planner_rt_fetch(rel->relid, root);
Chunk *chunk = ts_chunk_get_by_relid(chunk_rte->relid, true);
if (chunk->fd.compressed_chunk_id > 0)
if (chunk->fd.compressed_chunk_id != INVALID_CHUNK_ID)
{
Relation uncompressed_chunk = table_open(relation_objectid, NoLock);

12
tsl/src/compression/compression.c
View File

@ -1957,6 +1957,12 @@ decompress_batches_for_insert(ChunkInsertState *cis, Chunk *chunk, TupleTableSlo
return;
}
if (!ts_guc_enable_dml_decompression)
ereport(ERROR,
(errcode(ERRCODE_FEATURE_NOT_SUPPORTED),
errmsg("inserting into compressed chunk with unique constraints disabled"),
errhint("Set timescaledb.enable_dml_decompression to TRUE.")));
Chunk *comp = ts_chunk_get_by_id(chunk->fd.compressed_chunk_id, true);
Relation in_rel = relation_open(comp->table_id, RowExclusiveLock);
@ -2470,6 +2476,12 @@ decompress_chunk_walker(PlanState *ps, List *relids)
current_chunk = ts_chunk_get_by_relid(rte->relid, false);
if (current_chunk && ts_chunk_is_compressed(current_chunk))
{
if (!ts_guc_enable_dml_decompression)
ereport(ERROR,
(errcode(ERRCODE_FEATURE_NOT_SUPPORTED),
errmsg("UPDATE/DELETE is disabled on compressed chunks"),
errhint("Set timescaledb.enable_dml_decompression to TRUE.")));
decompress_batches_for_update_delete(current_chunk, predicates);
}
}

4
tsl/src/nodes/compress_dml/compress_dml.c
View File

@ -84,7 +84,6 @@ compress_chunk_dml_exec(CustomScanState *node)
static void
compress_chunk_dml_end(CustomScanState *node)
{
// CompressChunkDmlState *state = (CompressChunkDmlState *) node;
PlanState *substate = linitial(node->custom_ps);
ExecEndNode(substate);
}
@ -99,7 +98,6 @@ compress_chunk_dml_path_create(Path *subpath, Oid chunk_relid)
path->cpath.path.pathtype = T_CustomScan;
path->cpath.path.parent = subpath->parent;
path->cpath.path.pathtarget = subpath->pathtarget;
// path->cpath.path.param_info = subpath->param_info;
path->cpath.methods = &compress_chunk_dml_path_methods;
path->cpath.custom_paths = list_make1(subpath);
path->chunk_relid = chunk_relid;
@ -139,6 +137,6 @@ compress_chunk_dml_state_create(CustomScan *scan)
Path *
compress_chunk_dml_generate_paths(Path *subpath, Chunk *chunk)
{
Assert(chunk->fd.compressed_chunk_id > 0);
Assert(chunk->fd.compressed_chunk_id != INVALID_CHUNK_ID);
return compress_chunk_dml_path_create(subpath, chunk->table_id);
}

4
tsl/src/planner.c
View File

@ -129,7 +129,7 @@ tsl_set_rel_pathlist_query(PlannerInfo *root, RelOptInfo *rel, Index rti, RangeT
{
Chunk *chunk = ts_chunk_get_by_relid(rte->relid, true);
if (chunk->fd.compressed_chunk_id > 0)
if (chunk->fd.compressed_chunk_id != INVALID_CHUNK_ID)
ts_decompress_chunk_generate_paths(root, rel, ht, chunk);
}
}
@ -163,7 +163,7 @@ tsl_set_rel_pathlist_dml(PlannerInfo *root, RelOptInfo *rel, Index rti, RangeTbl
{
ListCell *lc;
Chunk *chunk = ts_chunk_get_by_relid(rte->relid, true);
if (chunk->fd.compressed_chunk_id > 0)
if (chunk->fd.compressed_chunk_id != INVALID_CHUNK_ID)
{
foreach (lc, rel->pathlist)
{

22
tsl/test/expected/compression_conflicts.out
View File

@ -493,3 +493,25 @@ WHERE hypertable_name = 'compressed_ht' ORDER BY chunk_name;
9 | _hyper_9_9_chunk
(3 rows)
-- test for disabling DML decompression
SHOW timescaledb.enable_dml_decompression;
timescaledb.enable_dml_decompression
--------------------------------------
on
(1 row)
SET timescaledb.enable_dml_decompression = false;
\set ON_ERROR_STOP 0
-- Should error because we disabled the DML decompression
INSERT INTO compressed_ht VALUES ('2022-01-24 01:10:28.192199+05:30', '6', 0.876, 4.123, 'new insert row')
ON conflict(sensor_id, time)
DO UPDATE SET sensor_id = excluded.sensor_id , name = 'ON CONFLICT DO UPDATE' RETURNING *;
ERROR: inserting into compressed chunk with unique constraints disabled
INSERT INTO compressed_ht VALUES ('2022-01-24 01:10:28.192199+05:30', '6', 0.876, 4.123, 'new insert row')
ON conflict(sensor_id, time)
DO NOTHING;
ERROR: inserting into compressed chunk with unique constraints disabled
-- Even a regular insert will fail due to unique constrant checks for dml decompression
INSERT INTO compressed_ht VALUES ('2022-01-24 01:10:28.192199+05:30', '7', 0.876, 4.123, 'new insert row');
ERROR: inserting into compressed chunk with unique constraints disabled
\set ON_ERROR_STOP 1

135
tsl/test/expected/compression_update_delete.out
View File

@ -31,7 +31,7 @@ WARNING: column type "character varying" used for "name" does not follow best p
1 | public | sample_table | t
(1 row)
SELECT '2022-01-28 01:09:53.583252+05:30' as start_date \gset
\set start_date '2022-01-28 01:09:53.583252+05:30'
INSERT INTO sample_table
SELECT
time + (INTERVAL '1 minute' * random()) AS time,
@ -45,7 +45,7 @@ INSERT INTO sample_table
generate_series(1, 8, 1 ) AS g2(sensor_id)
ORDER BY
time;
SELECT '2023-03-17 17:51:11.322998+05:30' as start_date \gset
\set start_date '2023-03-17 17:51:11.322998+05:30'
-- insert into new chunks
INSERT INTO sample_table VALUES (:'start_date'::timestamptz, 12, 21.98, 33.123, 'new row1');
INSERT INTO sample_table VALUES (:'start_date'::timestamptz, 12, 17.66, 13.875, 'new row1');
@ -1274,7 +1274,7 @@ SELECT COUNT(*) FROM :COMPRESS_CHUNK_1 WHERE c4 IS NULL;
DROP TABLE sample_table;
-- test filtering with ORDER BY columns
CREATE TABLE sample_table(time timestamptz, c1 int, c2 int, c3 int, c4 int);
SELECT create_hypertable('sample_table','time');
SELECT create_hypertable('sample_table','time',chunk_time_interval=>'1 day'::interval);
NOTICE: adding not-null constraint to column "time"
create_hypertable
----------------------------
@ -1301,12 +1301,12 @@ SELECT compress_chunk(show_chunks('sample_table'));
SELECT ch1.schema_name|| '.' || ch1.table_name AS "CHUNK_1"
FROM _timescaledb_catalog.chunk ch1, _timescaledb_catalog.hypertable ht
WHERE ch1.hypertable_id = ht.id AND ch1.table_name LIKE '_hyper_%'
ORDER BY ch1.id \gset
ORDER BY ch1.id LIMIT 1 \gset
-- get FIRST compressed chunk
SELECT ch1.schema_name|| '.' || ch1.table_name AS "COMPRESS_CHUNK_1"
FROM _timescaledb_catalog.chunk ch1, _timescaledb_catalog.hypertable ht
WHERE ch1.hypertable_id = ht.id AND ch1.table_name LIKE 'compress_%'
ORDER BY ch1.id \gset
ORDER BY ch1.id LIMIT 1 \gset
-- check that you uncompress and delete only for exact SEGMENTBY value
BEGIN;
-- report 10 rows
@ -2122,3 +2122,128 @@ SELECT count(*) FROM :COMP_CHUNK_1 WHERE device_id = 2 AND _ts_meta_max_1 >= '20
(1 row)
ROLLBACK;
-- test for disabling DML decompression
SHOW timescaledb.enable_dml_decompression;
timescaledb.enable_dml_decompression
--------------------------------------
on
(1 row)
SET timescaledb.enable_dml_decompression = false;
\set ON_ERROR_STOP 0
-- should ERROR both UPDATE/DELETE statements because the DML decompression is disabled
UPDATE sample_table SET c3 = NULL WHERE c4 = 5;
ERROR: UPDATE/DELETE is disabled on compressed chunks
DELETE FROM sample_table WHERE c4 = 5;
ERROR: UPDATE/DELETE is disabled on compressed chunks
\set ON_ERROR_STOP 1
-- make sure reseting the GUC we will be able to UPDATE/DELETE compressed chunks
RESET timescaledb.enable_dml_decompression;
SHOW timescaledb.enable_dml_decompression;
timescaledb.enable_dml_decompression
--------------------------------------
on
(1 row)
BEGIN;
-- report 0 rows
SELECT count(*) FROM sample_table WHERE c4 = 5 AND c3 IS NULL;
count
-------
0
(1 row)
UPDATE sample_table SET c3 = NULL WHERE c4 = 5;
-- report 10k rows
SELECT count(*) FROM sample_table WHERE c4 = 5 AND c3 IS NULL;
count
-------
10000
(1 row)
ROLLBACK;
BEGIN;
-- report 10k rows
SELECT count(*) FROM sample_table WHERE c4 = 5;
count
-------
10000
(1 row)
DELETE FROM sample_table WHERE c4 = 5;
-- report 0 rows
SELECT count(*) FROM sample_table WHERE c4 = 5;
count
-------
0
(1 row)
ROLLBACK;
-- create new uncompressed chunk
INSERT INTO sample_table
SELECT t, 1, 1, 1, 1
FROM generate_series('2023-05-04 00:00:00-00'::timestamptz,
'2023-05-04 00:00:00-00'::timestamptz + INTERVAL '2 hours',
INTERVAL '1 hour') t;
-- check chunk compression status
SELECT chunk_name, is_compressed
FROM timescaledb_information.chunks
WHERE hypertable_name = 'sample_table'
ORDER BY chunk_name;
chunk_name | is_compressed
--------------------+---------------
_hyper_19_37_chunk | t
_hyper_19_61_chunk | f
(2 rows)
-- test for uncompressed and compressed chunks
SHOW timescaledb.enable_dml_decompression;
timescaledb.enable_dml_decompression
--------------------------------------
on
(1 row)
SET timescaledb.enable_dml_decompression = false;
BEGIN;
-- report 3 rows
SELECT count(*) FROM sample_table WHERE time >= '2023-05-04 00:00:00-00'::timestamptz;
count
-------
3
(1 row)
-- delete from uncompressed chunk should work
DELETE FROM sample_table WHERE time >= '2023-05-04 00:00:00-00'::timestamptz;
-- report 0 rows
SELECT count(*) FROM sample_table WHERE time >= '2023-05-04 00:00:00-00'::timestamptz;
count
-------
0
(1 row)
ROLLBACK;
BEGIN;
-- report 0 rows
SELECT count(*) FROM sample_table WHERE time >= '2023-05-04 00:00:00-00'::timestamptz AND c3 IS NULL;
count
-------
0
(1 row)
UPDATE sample_table SET c3 = NULL WHERE time >= '2023-05-04 00:00:00-00'::timestamptz;
-- report 3 rows
SELECT count(*) FROM sample_table WHERE time >= '2023-05-04 00:00:00-00'::timestamptz AND c3 IS NULL;
count
-------
3
(1 row)
ROLLBACK;
\set ON_ERROR_STOP 0
-- should ERROR both UPDATE/DELETE statements because the DML decompression is disabled
-- and both statements we're touching compressed and uncompressed chunks
UPDATE sample_table SET c3 = NULL WHERE time >= '2023-03-17 00:00:00-00'::timestamptz AND c3 IS NULL;
ERROR: UPDATE/DELETE is disabled on compressed chunks
DELETE FROM sample_table WHERE time >= '2023-03-17 00:00:00-00'::timestamptz;
ERROR: UPDATE/DELETE is disabled on compressed chunks
\set ON_ERROR_STOP 1

18
tsl/test/sql/compression_conflicts.sql
View File

@ -368,3 +368,21 @@ SELECT chunk_status,
chunk_name as "CHUNK_NAME"
FROM compressed_chunk_info_view
WHERE hypertable_name = 'compressed_ht' ORDER BY chunk_name;
-- test for disabling DML decompression
SHOW timescaledb.enable_dml_decompression;
SET timescaledb.enable_dml_decompression = false;
\set ON_ERROR_STOP 0
-- Should error because we disabled the DML decompression
INSERT INTO compressed_ht VALUES ('2022-01-24 01:10:28.192199+05:30', '6', 0.876, 4.123, 'new insert row')
ON conflict(sensor_id, time)
DO UPDATE SET sensor_id = excluded.sensor_id , name = 'ON CONFLICT DO UPDATE' RETURNING *;
INSERT INTO compressed_ht VALUES ('2022-01-24 01:10:28.192199+05:30', '6', 0.876, 4.123, 'new insert row')
ON conflict(sensor_id, time)
DO NOTHING;
-- Even a regular insert will fail due to unique constrant checks for dml decompression
INSERT INTO compressed_ht VALUES ('2022-01-24 01:10:28.192199+05:30', '7', 0.876, 4.123, 'new insert row');
\set ON_ERROR_STOP 1

82
tsl/test/sql/compression_update_delete.sql
View File

@ -29,7 +29,8 @@ CREATE TABLE sample_table (
SELECT * FROM create_hypertable('sample_table', 'time',
chunk_time_interval => INTERVAL '2 months');
SELECT '2022-01-28 01:09:53.583252+05:30' as start_date \gset
\set start_date '2022-01-28 01:09:53.583252+05:30'
INSERT INTO sample_table
SELECT
time + (INTERVAL '1 minute' * random()) AS time,
@ -44,7 +45,8 @@ INSERT INTO sample_table
ORDER BY
time;
SELECT '2023-03-17 17:51:11.322998+05:30' as start_date \gset
\set start_date '2023-03-17 17:51:11.322998+05:30'
-- insert into new chunks
INSERT INTO sample_table VALUES (:'start_date'::timestamptz, 12, 21.98, 33.123, 'new row1');
INSERT INTO sample_table VALUES (:'start_date'::timestamptz, 12, 17.66, 13.875, 'new row1');
@ -734,7 +736,7 @@ DROP TABLE sample_table;
-- test filtering with ORDER BY columns
CREATE TABLE sample_table(time timestamptz, c1 int, c2 int, c3 int, c4 int);
SELECT create_hypertable('sample_table','time');
SELECT create_hypertable('sample_table','time',chunk_time_interval=>'1 day'::interval);
ALTER TABLE sample_table SET (timescaledb.compress,timescaledb.compress_segmentby='c4', timescaledb.compress_orderby='c1,c2,time');
INSERT INTO sample_table
SELECT t, c1, c2, c3, c4
@ -751,13 +753,13 @@ SELECT compress_chunk(show_chunks('sample_table'));
SELECT ch1.schema_name|| '.' || ch1.table_name AS "CHUNK_1"
FROM _timescaledb_catalog.chunk ch1, _timescaledb_catalog.hypertable ht
WHERE ch1.hypertable_id = ht.id AND ch1.table_name LIKE '_hyper_%'
ORDER BY ch1.id \gset
ORDER BY ch1.id LIMIT 1 \gset
-- get FIRST compressed chunk
SELECT ch1.schema_name|| '.' || ch1.table_name AS "COMPRESS_CHUNK_1"
FROM _timescaledb_catalog.chunk ch1, _timescaledb_catalog.hypertable ht
WHERE ch1.hypertable_id = ht.id AND ch1.table_name LIKE 'compress_%'
ORDER BY ch1.id \gset
ORDER BY ch1.id LIMIT 1 \gset
-- check that you uncompress and delete only for exact SEGMENTBY value
BEGIN;
@ -1142,3 +1144,73 @@ SELECT count(*) = :UNCOMP_LEFTOVER FROM ONLY :CHUNK_1;
SELECT count(*) FROM :COMP_CHUNK_1 WHERE device_id = 2 AND _ts_meta_max_1 >= '2000-01-02'::timestamptz;
ROLLBACK;
-- test for disabling DML decompression
SHOW timescaledb.enable_dml_decompression;
SET timescaledb.enable_dml_decompression = false;
\set ON_ERROR_STOP 0
-- should ERROR both UPDATE/DELETE statements because the DML decompression is disabled
UPDATE sample_table SET c3 = NULL WHERE c4 = 5;
DELETE FROM sample_table WHERE c4 = 5;
\set ON_ERROR_STOP 1
-- make sure reseting the GUC we will be able to UPDATE/DELETE compressed chunks
RESET timescaledb.enable_dml_decompression;
SHOW timescaledb.enable_dml_decompression;
BEGIN;
-- report 0 rows
SELECT count(*) FROM sample_table WHERE c4 = 5 AND c3 IS NULL;
UPDATE sample_table SET c3 = NULL WHERE c4 = 5;
-- report 10k rows
SELECT count(*) FROM sample_table WHERE c4 = 5 AND c3 IS NULL;
ROLLBACK;
BEGIN;
-- report 10k rows
SELECT count(*) FROM sample_table WHERE c4 = 5;
DELETE FROM sample_table WHERE c4 = 5;
-- report 0 rows
SELECT count(*) FROM sample_table WHERE c4 = 5;
ROLLBACK;
-- create new uncompressed chunk
INSERT INTO sample_table
SELECT t, 1, 1, 1, 1
FROM generate_series('2023-05-04 00:00:00-00'::timestamptz,
'2023-05-04 00:00:00-00'::timestamptz + INTERVAL '2 hours',
INTERVAL '1 hour') t;
-- check chunk compression status
SELECT chunk_name, is_compressed
FROM timescaledb_information.chunks
WHERE hypertable_name = 'sample_table'
ORDER BY chunk_name;
-- test for uncompressed and compressed chunks
SHOW timescaledb.enable_dml_decompression;
SET timescaledb.enable_dml_decompression = false;
BEGIN;
-- report 3 rows
SELECT count(*) FROM sample_table WHERE time >= '2023-05-04 00:00:00-00'::timestamptz;
-- delete from uncompressed chunk should work
DELETE FROM sample_table WHERE time >= '2023-05-04 00:00:00-00'::timestamptz;
-- report 0 rows
SELECT count(*) FROM sample_table WHERE time >= '2023-05-04 00:00:00-00'::timestamptz;
ROLLBACK;
BEGIN;
-- report 0 rows
SELECT count(*) FROM sample_table WHERE time >= '2023-05-04 00:00:00-00'::timestamptz AND c3 IS NULL;
UPDATE sample_table SET c3 = NULL WHERE time >= '2023-05-04 00:00:00-00'::timestamptz;
-- report 3 rows
SELECT count(*) FROM sample_table WHERE time >= '2023-05-04 00:00:00-00'::timestamptz AND c3 IS NULL;
ROLLBACK;
\set ON_ERROR_STOP 0
-- should ERROR both UPDATE/DELETE statements because the DML decompression is disabled
-- and both statements we're touching compressed and uncompressed chunks
UPDATE sample_table SET c3 = NULL WHERE time >= '2023-03-17 00:00:00-00'::timestamptz AND c3 IS NULL;
DELETE FROM sample_table WHERE time >= '2023-03-17 00:00:00-00'::timestamptz;
\set ON_ERROR_STOP 1