From 380464df9bb35784853f186b9e268cd9d50e442b Mon Sep 17 00:00:00 2001 From: Jan Nidzwetzki Date: Fri, 28 Oct 2022 16:09:52 +0200 Subject: [PATCH] Perform frozen chunk status check via trigger The commit 9f4dcea30135d1e36d1c452d631fc8b8743b3995 introduces frozen chunks. Checking whether a chunk is frozen or not has been done so far in the query planner. If it is not possible to determine which chunks are affected by a query in the planner (e.g., due to a cast in the WHERE condition), all chunks are checked. This leads (1) to an increased planning time and (2) to the situation that a single frozen chunk could reject queries, even if the frozen chunk is not addressed by the query. --- CHANGELOG.md | 1 + sql/chunk.sql | 9 + sql/updates/latest-dev.sql | 9 + sql/updates/reverse-dev.sql | 2 + src/chunk.c | 130 +++++++- src/planner/expand_hypertable.c | 2 +- src/planner/planner.c | 30 +- src/planner/planner.h | 3 +- tsl/test/expected/chunk_utils_internal.out | 341 +++++++++++++++++++-- tsl/test/shared/expected/extension.out | 1 + tsl/test/sql/CMakeLists.txt | 8 + tsl/test/sql/chunk_utils_internal.sql | 181 ++++++++++- 12 files changed, 659 insertions(+), 58 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 5630eea2a..aec71a6a4 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -29,6 +29,7 @@ argument or resolve the type ambiguity by casting to the intended type. * #4840 Fix performance regressions in the copy code * #4823 Fix a crash that could occur when using nested user-defined functions with hypertables * #4898 Fix cagg migration failure when trying to resume +* #4906 Fix a performance regression in the query planner by speeding up frozen chunk state checks * #4910 Fix a typo in process_compressed_data_out * #4955 Fix cagg migration for hypertables using timestamp without timezone * #4968 Check for interrupts in gapfill main loop diff --git a/sql/chunk.sql b/sql/chunk.sql index 7e0286e38..36e086d1a 100644 --- a/sql/chunk.sql +++ b/sql/chunk.sql @@ -100,3 +100,12 @@ CREATE OR REPLACE FUNCTION _timescaledb_internal.attach_osm_table_chunk( hypertable REGCLASS, chunk REGCLASS) RETURNS BOOL AS '@MODULE_PATHNAME@', 'ts_chunk_attach_osm_table_chunk' LANGUAGE C VOLATILE; + +-- Trigger that blocks modifications on frozen chunks +CREATE OR REPLACE FUNCTION _timescaledb_internal.frozen_chunk_modify_blocker() RETURNS trigger + LANGUAGE plpgsql STRICT AS +$BODY$ +BEGIN + RAISE EXCEPTION 'unable to modify frozen chunk %s', TG_TABLE_NAME; +END; +$BODY$ SET search_path TO pg_catalog, pg_temp; diff --git a/sql/updates/latest-dev.sql b/sql/updates/latest-dev.sql index 62e964147..25bc52a9e 100644 --- a/sql/updates/latest-dev.sql +++ b/sql/updates/latest-dev.sql @@ -390,3 +390,12 @@ CREATE FUNCTION @extschema@.alter_data_node( available BOOLEAN = NULL ) RETURNS TABLE(node_name NAME, host TEXT, port INTEGER, database NAME, available BOOLEAN) AS '@MODULE_PATHNAME@', 'ts_data_node_alter' LANGUAGE C VOLATILE; + +-- Trigger that blocks modifications on frozen chunks +CREATE OR REPLACE FUNCTION _timescaledb_internal.frozen_chunk_modify_blocker() RETURNS trigger + LANGUAGE plpgsql STRICT AS +$BODY$ +BEGIN + RAISE EXCEPTION 'unable to modify frozen chunk %s', TG_TABLE_NAME; +END; +$BODY$ SET search_path TO pg_catalog, pg_temp; diff --git a/sql/updates/reverse-dev.sql b/sql/updates/reverse-dev.sql index 5c909d3d9..6221b9e6e 100644 --- a/sql/updates/reverse-dev.sql +++ b/sql/updates/reverse-dev.sql @@ -309,3 +309,5 @@ GRANT SELECT ON _timescaledb_catalog.dimension TO PUBLIC; -- changes related to alter_data_node() DROP INDEX _timescaledb_catalog.chunk_data_node_node_name_idx; DROP FUNCTION @extschema@.alter_data_node; + +DROP FUNCTION _timescaledb_internal.frozen_chunk_modify_blocker; diff --git a/src/chunk.c b/src/chunk.c index 51363cbe0..22157555a 100644 --- a/src/chunk.c +++ b/src/chunk.c @@ -179,6 +179,15 @@ static Chunk *chunk_resurrect(const Hypertable *ht, int chunk_id); */ #define CHUNK_STATUS_COMPRESSED_PARTIAL 8 +/* The name of the trigger function that blocks data modifications on frozen chunks */ +#define TS_FROZEN_TRIGGER_NAME_FUNCTION "frozen_chunk_modify_blocker" + +/* The name of the row freeze trigger */ +#define TS_FROZEN_TRIGGER_NAME_ROW "frozen_chunk_modify_blocker_row" + +/* The name of the statement freeze trigger */ +#define TS_FROZEN_TRIGGER_NAME_STMT "frozen_chunk_modify_blocker_stmt" + static HeapTuple chunk_formdata_make_tuple(const FormData_chunk *fd, TupleDesc desc) { @@ -3521,7 +3530,31 @@ ts_chunk_set_partial(Chunk *chunk) return ts_chunk_add_status(chunk, CHUNK_STATUS_COMPRESSED_PARTIAL); } -/*No inserts,updates and deletes are permitted on a frozen chunk. +#if PG14_GE +/* Install a new trigger of the given table OID */ +static void +install_chunk_trigger(CreateTrigStmt *stmt, Oid table_oid) +{ + ObjectAddress objaddr; + + objaddr = CreateTrigger(stmt, + NULL, + table_oid, + InvalidOid, + InvalidOid, + InvalidOid, + InvalidOid, + InvalidOid, + NULL, + false, + false); + + if (!OidIsValid(objaddr.objectId)) + elog(ERROR, "could not create internal blocker trigger on %d", table_oid); +} +#endif + +/* No inserts, updates, and deletes are permitted on a frozen chunk. * Compression policies etc do not run on a frozen chunk. * Only valid operation is dropping the chunk */ @@ -3529,6 +3562,59 @@ bool ts_chunk_set_frozen(Chunk *chunk) { #if PG14_GE + + char *relname = get_rel_name(chunk->table_id); + Oid schemaid = get_rel_namespace(chunk->table_id); + char *schema = get_namespace_name(schemaid); + + /* Create a trigger that reject data modifications on the frozen chunk. + * + * Operations like inserts and compression / decompression check the + * state of chunks directly in the code path. However, operations like + * updates and deletes are handled by PostgreSQL. For these remaining + * operations, the trigger is responsible for rejecting the operation + * on the frozen chunk. + * + * We need to create a row and a statement trigger. To block + * INSERT / UPDATE / DELETE operations, we need a row trigger. Since + * the trigger is installed on the chunk, a statement-based trigger + * will not fire for the individual chunks on these operations. + * + * A trigger that fires on TRUNCATE operations needs to be a statement + * trigger; PostgreSQL does not support row based TRUNCATE triggers. + * These triggers will also fire on the individual chunks. + */ + + /* Row based INSERT / UPDATE / DELETE trigger */ + CreateTrigStmt stmt_chunk_trigger_row = { + .type = T_CreateTrigStmt, + .row = true, + .timing = TRIGGER_TYPE_BEFORE, + .trigname = TS_FROZEN_TRIGGER_NAME_ROW, + .relation = makeRangeVar(schema, relname, -1), + .funcname = list_make2(makeString(INTERNAL_SCHEMA_NAME), + makeString(TS_FROZEN_TRIGGER_NAME_FUNCTION)), + .args = NIL, + .events = TRIGGER_TYPE_INSERT | TRIGGER_TYPE_DELETE | TRIGGER_TYPE_UPDATE, + }; + + install_chunk_trigger(&stmt_chunk_trigger_row, chunk->table_id); + + /* Statement based TRUNCATE trigger */ + CreateTrigStmt stmt_chunk_trigger_statement = { + .type = T_CreateTrigStmt, + .row = false, + .timing = TRIGGER_TYPE_BEFORE, + .trigname = TS_FROZEN_TRIGGER_NAME_STMT, + .relation = makeRangeVar(schema, relname, -1), + .funcname = list_make2(makeString(INTERNAL_SCHEMA_NAME), + makeString(TS_FROZEN_TRIGGER_NAME_FUNCTION)), + .args = NIL, + .events = TRIGGER_TYPE_TRUNCATE, + }; + + install_chunk_trigger(&stmt_chunk_trigger_statement, chunk->table_id); + return ts_chunk_add_status(chunk, CHUNK_STATUS_FROZEN); #else elog(ERROR, "freeze chunk supported only for PG14 or greater"); @@ -3540,6 +3626,48 @@ bool ts_chunk_unset_frozen(Chunk *chunk) { #if PG14_GE + Relation tgrel; + ScanKeyData skey[1]; + SysScanDesc tgscan; + HeapTuple tuple; + int deleted_triggers = 0; + + /* Search for the freeze triggers on the chunk and remove them. */ + tgrel = table_open(TriggerRelationId, RowExclusiveLock); + + ScanKeyInit(&skey[0], + Anum_pg_trigger_tgrelid, + BTEqualStrategyNumber, + F_OIDEQ, + ObjectIdGetDatum(chunk->table_id)); + + tgscan = systable_beginscan(tgrel, TriggerRelidNameIndexId, true, NULL, 1, skey); + + while (HeapTupleIsValid(tuple = systable_getnext(tgscan))) + { + Form_pg_trigger trig = (Form_pg_trigger) GETSTRUCT(tuple); + + if ((namestrcmp(&(trig->tgname), TS_FROZEN_TRIGGER_NAME_STMT) == 0) || + (namestrcmp(&(trig->tgname), TS_FROZEN_TRIGGER_NAME_ROW) == 0)) + { + ObjectAddress objaddr = { .classId = TriggerRelationId, .objectId = trig->oid }; + performDeletion(&objaddr, DROP_RESTRICT, 0); + deleted_triggers++; + } + } + + systable_endscan(tgscan); + table_close(tgrel, RowExclusiveLock); + + if (deleted_triggers != 2) + { + ereport(WARNING, + (errcode(ERRCODE_INTERNAL_ERROR), + errmsg("found only %d frozen trigger on %s (2 expected)", + deleted_triggers, + get_rel_name(chunk->table_id)))); + } + return ts_chunk_clear_status(chunk, CHUNK_STATUS_FROZEN); #else elog(ERROR, "freeze chunk supported only for PG14 or greater"); diff --git a/src/planner/expand_hypertable.c b/src/planner/expand_hypertable.c index 59731c4af..c9ac98e02 100644 --- a/src/planner/expand_hypertable.c +++ b/src/planner/expand_hypertable.c @@ -1396,7 +1396,7 @@ ts_plan_expand_hypertable_chunks(Hypertable *ht, PlannerInfo *root, RelOptInfo * * Add the information about chunks to the baserel info cache for * classify_relation(). */ - add_baserel_cache_entry_for_chunk(chunks[i]->table_id, chunks[i]->fd.status, ht); + add_baserel_cache_entry_for_chunk(chunks[i]->table_id, ht); } /* nothing to do here if we have no chunks and no data nodes */ diff --git a/src/planner/planner.c b/src/planner/planner.c index 59b3f5dda..e7b821273 100644 --- a/src/planner/planner.c +++ b/src/planner/planner.c @@ -76,7 +76,6 @@ typedef struct BaserelInfoEntry { Oid reloid; Hypertable *ht; - uint32 chunk_status; /* status of chunk, if this is a chunk */ uint32 status; /* hash status */ } BaserelInfoEntry; @@ -155,7 +154,7 @@ static struct BaserelInfo_hash *ts_baserel_info = NULL; * chunk info at the plan time chunk exclusion. */ void -add_baserel_cache_entry_for_chunk(Oid chunk_reloid, uint32 chunk_status, Hypertable *hypertable) +add_baserel_cache_entry_for_chunk(Oid chunk_reloid, Hypertable *hypertable) { Assert(hypertable != NULL); Assert(ts_baserel_info != NULL); @@ -164,15 +163,13 @@ add_baserel_cache_entry_for_chunk(Oid chunk_reloid, uint32 chunk_status, Hyperta BaserelInfoEntry *entry = BaserelInfo_insert(ts_baserel_info, chunk_reloid, &found); if (found) { - /* Already cached, check that the parameters are the same. */ + /* Already cached. */ Assert(entry->ht != NULL); - Assert(entry->chunk_status == chunk_status); return; } /* Fill the cache entry. */ entry->ht = hypertable; - entry->chunk_status = chunk_status; } static void @@ -682,9 +679,9 @@ get_or_add_baserel_from_cache(Oid chunk_reloid, Oid parent_reloid) * This reloid is not in the chunk cache, so do the full metadata * lookup. */ - int32 hypertable_id = 0; - int32 chunk_status = 0; - if (ts_chunk_get_hypertable_id_and_status_by_relid(chunk_reloid, &hypertable_id, &chunk_status)) + int32 hypertable_id = ts_chunk_get_hypertable_id_by_relid(chunk_reloid); + + if (OidIsValid(hypertable_id)) { /* * This is a chunk. Look up the hypertable for it. @@ -706,7 +703,6 @@ get_or_add_baserel_from_cache(Oid chunk_reloid, Oid parent_reloid) /* Cache the result. */ entry->ht = ht; - entry->chunk_status = chunk_status; return entry; } @@ -1187,22 +1183,6 @@ timescaledb_set_rel_pathlist(PlannerInfo *root, RelOptInfo *rel, Index rti, Rang break; case TS_REL_CHUNK_STANDALONE: case TS_REL_CHUNK_CHILD: - - if (IS_UPDL_CMD(root->parse)) - { - BaserelInfoEntry *chunk_cache_entry = - BaserelInfo_lookup(ts_baserel_info, rte->relid); - Assert(chunk_cache_entry != NULL); - int32 chunk_status = chunk_cache_entry->chunk_status; - /* throw error if chunk has invalid status for operation */ - ts_chunk_validate_chunk_status_for_operation(rte->relid, - chunk_status, - root->parse->commandType == - CMD_UPDATE ? - CHUNK_UPDATE : - CHUNK_DELETE, - true); - } /* Check for UPDATE/DELETE (DML) on compressed chunks */ if (IS_UPDL_CMD(root->parse) && dml_involves_hypertable(root, ht, rti)) { diff --git a/src/planner/planner.h b/src/planner/planner.h index 2180d6a56..f664338f6 100644 --- a/src/planner/planner.h +++ b/src/planner/planner.h @@ -106,7 +106,6 @@ extern Node *ts_constify_now(PlannerInfo *root, List *rtable, Node *node); extern void ts_planner_constraint_cleanup(PlannerInfo *root, RelOptInfo *rel); extern Node *ts_add_space_constraints(PlannerInfo *root, List *rtable, Node *node); -extern void add_baserel_cache_entry_for_chunk(Oid chunk_reloid, uint32 chunk_status, - Hypertable *hypertable); +extern void add_baserel_cache_entry_for_chunk(Oid chunk_reloid, Hypertable *hypertable); #endif /* TIMESCALEDB_PLANNER_H */ diff --git a/tsl/test/expected/chunk_utils_internal.out b/tsl/test/expected/chunk_utils_internal.out index 60f5f6b72..fb21fad69 100644 --- a/tsl/test/expected/chunk_utils_internal.out +++ b/tsl/test/expected/chunk_utils_internal.out @@ -62,12 +62,25 @@ FROM timescaledb_information.chunks WHERE hypertable_name = 'hyper1' and hypertable_schema = 'test1' ORDER BY chunk_name LIMIT 1 \gset +-- Freeze and check trigger +SELECT tgname, tgtype FROM pg_trigger WHERE tgrelid = :'CHNAME'::regclass ORDER BY tgname, tgtype; + tgname | tgtype +--------+-------- +(0 rows) + SELECT _timescaledb_internal.freeze_chunk( :'CHNAME'); freeze_chunk -------------- t (1 row) +SELECT tgname, tgtype FROM pg_trigger WHERE tgrelid = :'CHNAME'::regclass ORDER BY tgname, tgtype; + tgname | tgtype +----------------------------------+-------- + frozen_chunk_modify_blocker_row | 31 + frozen_chunk_modify_blocker_stmt | 34 +(2 rows) + SELECT * from test1.hyper1 ORDER BY 1; time | temp ------+------ @@ -77,9 +90,6 @@ SELECT * from test1.hyper1 ORDER BY 1; -- TEST updates and deletes on frozen chunk should fail \set ON_ERROR_STOP 0 -UPDATE test1.hyper1 SET temp = 40 WHERE time = 20; -UPDATE test1.hyper1 SET temp = 40 WHERE temp = 0.5; -ERROR: Update not permitted on frozen chunk "_hyper_1_1_chunk" SELECT * from test1.hyper1 ORDER BY 1; time | temp ------+------ @@ -87,9 +97,65 @@ SELECT * from test1.hyper1 ORDER BY 1; 30 | 0.5 (2 rows) +-- Value (time = 20) does not exist +UPDATE test1.hyper1 SET temp = 40 WHERE time = 20; +-- Frozen chunk is affected +UPDATE test1.hyper1 SET temp = 40 WHERE temp = 0.5; +ERROR: unable to modify frozen chunk _hyper_1_1_chunks +-- Frozen chunk is affected +UPDATE test1.hyper1 SET temp = 40 WHERE time = 10; +ERROR: unable to modify frozen chunk _hyper_1_1_chunks +-- Frozen chunk is affected +DELETE FROM test1.hyper1 WHERE time = 10; +ERROR: unable to modify frozen chunk _hyper_1_1_chunks +SELECT * from test1.hyper1 ORDER BY 1; + time | temp +------+------ + 10 | 0.5 + 30 | 0.5 +(2 rows) + +BEGIN; DELETE FROM test1.hyper1 WHERE time = 20; DELETE FROM test1.hyper1 WHERE temp = 0.5; -ERROR: Delete not permitted on frozen chunk "_hyper_1_1_chunk" +ERROR: unable to modify frozen chunk _hyper_1_1_chunks +ROLLBACK; +-- TEST update on unfrozen chunk should be possible +BEGIN; +SELECT * FROM test1.hyper1; + time | temp +------+------ + 10 | 0.5 + 30 | 0.5 +(2 rows) + +UPDATE test1.hyper1 SET temp = 40 WHERE time = 30; +SELECT * FROM test1.hyper1; + time | temp +------+------ + 10 | 0.5 + 30 | 40 +(2 rows) + +ROLLBACK; +-- Test with cast (chunk path pruning can not be done during query planning) +BEGIN; +SELECT * FROM test1.hyper1 WHERE time = 30; + time | temp +------+------ + 30 | 0.5 +(1 row) + +UPDATE test1.hyper1 SET temp = 40 WHERE time = 30::text::float; +SELECT * FROM test1.hyper1 WHERE time = 30; + time | temp +------+------ + 30 | 40 +(1 row) + +ROLLBACK; +-- TEST delete on unfrozen chunks should be possible +BEGIN; SELECT * from test1.hyper1 ORDER BY 1; time | temp ------+------ @@ -97,9 +163,42 @@ SELECT * from test1.hyper1 ORDER BY 1; 30 | 0.5 (2 rows) +DELETE FROM test1.hyper1 WHERE time = 30; +SELECT * from test1.hyper1 ORDER BY 1; + time | temp +------+------ + 10 | 0.5 +(1 row) + +ROLLBACK; +-- Test with cast +BEGIN; +SELECT * FROM test1.hyper1 WHERE time = 30; + time | temp +------+------ + 30 | 0.5 +(1 row) + +DELETE FROM test1.hyper1 WHERE time = 30::text::float; +SELECT * FROM test1.hyper1 WHERE time = 30; + time | temp +------+------ +(0 rows) + +ROLLBACK; -- TEST inserts into a frozen chunk fails INSERT INTO test1.hyper1 VALUES ( 11, 11); ERROR: Insert not permitted on frozen chunk "_hyper_1_1_chunk" +-- Test truncating table should fail +TRUNCATE test1.hyper1; +ERROR: unable to modify frozen chunk _hyper_1_1_chunks +SELECT * from test1.hyper1 ORDER BY 1; + time | temp +------+------ + 10 | 0.5 + 30 | 0.5 +(2 rows) + \set ON_ERROR_STOP 1 --insert into non-frozen chunk works INSERT INTO test1.hyper1 VALUES ( 31, 31); @@ -125,6 +224,11 @@ SELECT _timescaledb_internal.unfreeze_chunk( :'CHNAME'); t (1 row) +SELECT tgname, tgtype FROM pg_trigger WHERE tgrelid = :'CHNAME'::regclass ORDER BY tgname, tgtype; + tgname | tgtype +--------+-------- +(0 rows) + --verify status in catalog SELECT table_name, status FROM _timescaledb_catalog.chunk WHERE table_name = :'CHUNK_NAME'; @@ -133,6 +237,10 @@ FROM _timescaledb_catalog.chunk WHERE table_name = :'CHUNK_NAME'; _hyper_1_1_chunk | 0 (1 row) +-- Test update works after unfreeze +UPDATE test1.hyper1 SET temp = 40; +-- Test delete works after unfreeze +DELETE FROM test1.hyper1; --unfreezing again works SELECT _timescaledb_internal.unfreeze_chunk( :'CHNAME'); unfreeze_chunk @@ -200,10 +308,10 @@ INSERT INTO public.table_to_compress VALUES ('2020-01-01 10:00', 12, 77); ERROR: Insert not permitted on frozen chunk "_hyper_2_3_chunk" --touches all chunks UPDATE public.table_to_compress SET value = 3; -ERROR: Update not permitted on frozen chunk "_hyper_2_3_chunk" +ERROR: cannot update/delete rows from chunk "_hyper_2_3_chunk" as it is compressed --touches only frozen chunk DELETE FROM public.table_to_compress WHERE time < '2020-01-02'; -ERROR: Delete not permitted on frozen chunk "_hyper_2_3_chunk" +ERROR: cannot update/delete rows from chunk "_hyper_2_3_chunk" as it is compressed \set ON_ERROR_STOP 1 --try to refreeze SELECT _timescaledb_internal.freeze_chunk( :'CHNAME'); @@ -275,6 +383,9 @@ ERROR: compress_chunk not permitted on frozen chunk "_hyper_2_7_chunk" SELECT _timescaledb_internal.drop_chunk(:'CHNAME'); ERROR: drop_chunk not permitted on frozen chunk "_hyper_2_7_chunk" \set ON_ERROR_STOP 1 +-- Prepare table for CAGG tests +TRUNCATE test1.hyper1; +INSERT INTO test1.hyper1(time, temp) values(30, 0.5), (31, 31); --TEST drop_chunk in the presence of caggs. Does not affect cagg data CREATE OR REPLACE FUNCTION hyper_dummy_now() RETURNS BIGINT LANGUAGE SQL IMMUTABLE AS 'SELECT 100::BIGINT'; @@ -309,7 +420,7 @@ SELECT _timescaledb_internal.freeze_chunk( :'CHNAME1'); --cannot drop frozen chunk \set ON_ERROR_STOP 0 SELECT _timescaledb_internal.drop_chunk( :'CHNAME1'); -ERROR: drop_chunk not permitted on frozen chunk "_hyper_1_2_chunk" +ERROR: drop_chunk not permitted on frozen chunk "_hyper_1_8_chunk" \set ON_ERROR_STOP 1 -- unfreeze the chunk, then drop the single chunk SELECT _timescaledb_internal.unfreeze_chunk( :'CHNAME1'); @@ -401,19 +512,19 @@ SELECT _timescaledb_internal.attach_osm_table_chunk('ht_try', 'child_fdw_table') SELECT chunk_name, range_start, range_end FROM timescaledb_information.chunks WHERE hypertable_name = 'ht_try' ORDER BY 1; - chunk_name | range_start | range_end -------------------+------------------------------+------------------------------ - _hyper_5_9_chunk | Wed May 04 17:00:00 2022 PDT | Thu May 05 17:00:00 2022 PDT + chunk_name | range_start | range_end +-------------------+------------------------------+------------------------------ + _hyper_5_10_chunk | Wed May 04 17:00:00 2022 PDT | Thu May 05 17:00:00 2022 PDT (1 row) SELECT chunk_name, range_start, range_end FROM chunk_view WHERE hypertable_name = 'ht_try' ORDER BY chunk_name; - chunk_name | range_start | range_end -------------------+--------------------------------+------------------------------ - _hyper_5_9_chunk | Wed May 04 17:00:00 2022 PDT | Thu May 05 17:00:00 2022 PDT - child_fdw_table | Thu Dec 31 16:00:00 294246 PST | infinity + chunk_name | range_start | range_end +-------------------+--------------------------------+------------------------------ + _hyper_5_10_chunk | Wed May 04 17:00:00 2022 PDT | Thu May 05 17:00:00 2022 PDT + child_fdw_table | Thu Dec 31 16:00:00 294246 PST | infinity (2 rows) SELECT * FROM ht_try ORDER BY 1; @@ -425,19 +536,20 @@ SELECT * FROM ht_try ORDER BY 1; SELECT relname, relowner::regrole FROM pg_class WHERE relname in ( select chunk_name FROM chunk_view - WHERE hypertable_name = 'ht_try' ); - relname | relowner -------------------+------------- - child_fdw_table | test_role_4 - _hyper_5_9_chunk | test_role_4 + WHERE hypertable_name = 'ht_try' ) +ORDER BY relname; + relname | relowner +-------------------+------------- + _hyper_5_10_chunk | test_role_4 + child_fdw_table | test_role_4 (2 rows) SELECT inhrelid::regclass FROM pg_inherits WHERE inhparent = 'ht_try'::regclass ORDER BY 1; - inhrelid ----------------------------------------- + inhrelid +----------------------------------------- child_fdw_table - _timescaledb_internal._hyper_5_9_chunk + _timescaledb_internal._hyper_5_10_chunk (2 rows) --TEST chunk exclusion code does not filter out OSM chunk @@ -553,9 +665,9 @@ ORDER BY chunk_name LIMIT 1 \gset \set ON_ERROR_STOP 0 SELECT _timescaledb_internal.freeze_chunk( :'CHNAME3'); -ERROR: operation not supported on distributed chunk or foreign table "_dist_hyper_6_11_chunk" +ERROR: operation not supported on distributed chunk or foreign table "_dist_hyper_6_12_chunk" SELECT _timescaledb_internal.unfreeze_chunk( :'CHNAME3'); -ERROR: operation not supported on distributed chunk or foreign table "_dist_hyper_6_11_chunk" +ERROR: operation not supported on distributed chunk or foreign table "_dist_hyper_6_12_chunk" \set ON_ERROR_STOP 1 -- TEST can create OSM chunk if there are constraints on the hypertable \c :TEST_DBNAME :ROLE_4 @@ -595,7 +707,7 @@ WHERE hypertable_id IN (SELECT id from _timescaledb_catalog.hypertable ORDER BY table_name; table_name | status | osm_chunk --------------------+--------+----------- - _hyper_7_12_chunk | 0 | f + _hyper_7_13_chunk | 0 | f child_hyper_constr | 0 | t (2 rows) @@ -634,6 +746,185 @@ ORDER BY chunk_name; child_hyper_constr | Sat Jan 09 20:00:54.7758 294247 PST | infinity (1 row) +----- TESTS for copy into frozen chunk ------------ +\c :TEST_DBNAME :ROLE_DEFAULT_PERM_USER +CREATE TABLE test1.copy_test ( + "time" timestamptz NOT NULL, + "value" double precision NOT NULL +); +SELECT create_hypertable('test1.copy_test', 'time', chunk_time_interval => interval '1 day'); + create_hypertable +----------------------- + (8,test1,copy_test,t) +(1 row) + +COPY test1.copy_test FROM STDIN DELIMITER ','; +-- Freeze one of the chunks +SELECT chunk_schema || '.' || chunk_name as "COPY_CHNAME", chunk_name as "COPY_CHUNK_NAME" +FROM timescaledb_information.chunks +WHERE hypertable_name = 'copy_test' and hypertable_schema = 'test1' +ORDER BY chunk_name LIMIT 1 +\gset +SELECT _timescaledb_internal.freeze_chunk( :'COPY_CHNAME'); + freeze_chunk +-------------- + t +(1 row) + +-- Check state +SELECT table_name, status +FROM _timescaledb_catalog.chunk WHERE table_name = :'COPY_CHUNK_NAME'; + table_name | status +-------------------+-------- + _hyper_8_15_chunk | 4 +(1 row) + +\set ON_ERROR_STOP 0 +-- Copy should fail because one of che chunks is frozen +COPY test1.copy_test FROM STDIN DELIMITER ','; +ERROR: Insert not permitted on frozen chunk "_hyper_8_15_chunk" +\set ON_ERROR_STOP 1 +-- Count existing rows +SELECT COUNT(*) FROM test1.copy_test; + count +------- + 2 +(1 row) + +-- Test dump & restore +\c postgres :ROLE_SUPERUSER +\! utils/pg_dump_aux_dump.sh dump/pg_dump.sql +\c :TEST_DBNAME +-- Make sure tables was droped by pg_dump_aux_dump.sh +\set ON_ERROR_STOP 0 +SELECT * FROM test1.copy_test; +ERROR: relation "test1.copy_test" does not exist at character 15 +\set ON_ERROR_STOP 1 +SET client_min_messages = ERROR; +CREATE EXTENSION timescaledb CASCADE; +RESET client_min_messages; +--\! cp dump/pg_dump.sql /tmp/dump.sql +SELECT timescaledb_pre_restore(); + timescaledb_pre_restore +------------------------- + t +(1 row) + +\! utils/pg_dump_aux_restore.sh dump/pg_dump.sql +SELECT timescaledb_post_restore(); + timescaledb_post_restore +-------------------------- + t +(1 row) + +SELECT _timescaledb_internal.stop_background_workers(); + stop_background_workers +------------------------- + t +(1 row) + +\c :TEST_DBNAME :ROLE_DEFAULT_PERM_USER +-- Make sure the chunk is still frozen +-- Check trigger +SELECT tgname, tgtype FROM pg_trigger WHERE tgrelid = :'CHNAME'::regclass ORDER BY tgname, tgtype; + tgname | tgtype +----------------------------------+-------- + frozen_chunk_modify_blocker_row | 31 + frozen_chunk_modify_blocker_stmt | 34 +(2 rows) + +-- Check state +SELECT table_name, status +FROM _timescaledb_catalog.chunk WHERE table_name = :'COPY_CHUNK_NAME'; + table_name | status +-------------------+-------- + _hyper_8_15_chunk | 4 +(1 row) + +\set ON_ERROR_STOP 0 +-- Copy should fail because one of che chunks is frozen +COPY test1.copy_test FROM STDIN DELIMITER ','; +ERROR: Insert not permitted on frozen chunk "_hyper_8_15_chunk" +\set ON_ERROR_STOP 1 +-- Count existing rows +SELECT COUNT(*) FROM test1.copy_test; + count +------- + 2 +(1 row) + +-- Check unfreeze restored chunk +SELECT _timescaledb_internal.unfreeze_chunk( :'COPY_CHNAME'); + unfreeze_chunk +---------------- + t +(1 row) + +-- Check trigger +SELECT tgname, tgtype FROM pg_trigger WHERE tgrelid = :'CHNAME'::regclass ORDER BY tgname, tgtype; + tgname | tgtype +----------------------------------+-------- + frozen_chunk_modify_blocker_row | 31 + frozen_chunk_modify_blocker_stmt | 34 +(2 rows) + +-- Check state +SELECT table_name, status +FROM _timescaledb_catalog.chunk WHERE table_name = :'COPY_CHUNK_NAME'; + table_name | status +-------------------+-------- + _hyper_8_15_chunk | 0 +(1 row) + +-- Copy should work now +COPY test1.copy_test FROM STDIN DELIMITER ','; +-- Test that unfreeze works even if somebody has dropped one the block triggers +SELECT _timescaledb_internal.freeze_chunk( :'COPY_CHNAME'); + freeze_chunk +-------------- + t +(1 row) + +SELECT table_name, status +FROM _timescaledb_catalog.chunk WHERE table_name = :'COPY_CHUNK_NAME'; + table_name | status +-------------------+-------- + _hyper_8_15_chunk | 4 +(1 row) + +SELECT tgname, tgtype FROM pg_trigger WHERE tgrelid = :'COPY_CHNAME'::regclass ORDER BY tgname, tgtype; + tgname | tgtype +----------------------------------+-------- + frozen_chunk_modify_blocker_row | 31 + frozen_chunk_modify_blocker_stmt | 34 +(2 rows) + +DROP TRIGGER frozen_chunk_modify_blocker_row ON :COPY_CHNAME; +SELECT tgname, tgtype FROM pg_trigger WHERE tgrelid = :'COPY_CHNAME'::regclass ORDER BY tgname, tgtype; + tgname | tgtype +----------------------------------+-------- + frozen_chunk_modify_blocker_stmt | 34 +(1 row) + +SELECT _timescaledb_internal.unfreeze_chunk( :'COPY_CHNAME'); +WARNING: found only 1 frozen trigger on _hyper_8_15_chunk (2 expected) + unfreeze_chunk +---------------- + t +(1 row) + +SELECT tgname, tgtype FROM pg_trigger WHERE tgrelid = :'COPY_CHNAME'::regclass ORDER BY tgname, tgtype; + tgname | tgtype +--------+-------- +(0 rows) + +SELECT table_name, status +FROM _timescaledb_catalog.chunk WHERE table_name = :'COPY_CHUNK_NAME'; + table_name | status +-------------------+-------- + _hyper_8_15_chunk | 0 +(1 row) + -- clean up databases created \c :TEST_DBNAME :ROLE_SUPERUSER DROP DATABASE postgres_fdw_db; diff --git a/tsl/test/shared/expected/extension.out b/tsl/test/shared/expected/extension.out index 0b7a0f0bb..a0040d52f 100644 --- a/tsl/test/shared/expected/extension.out +++ b/tsl/test/shared/expected/extension.out @@ -72,6 +72,7 @@ ORDER BY pronamespace::regnamespace::text COLLATE "C", p.oid::regprocedure::text _timescaledb_internal.first_combinefunc(internal,internal) _timescaledb_internal.first_sfunc(internal,anyelement,"any") _timescaledb_internal.freeze_chunk(regclass) + _timescaledb_internal.frozen_chunk_modify_blocker() _timescaledb_internal.generate_uuid() _timescaledb_internal.get_chunk_colstats(regclass) _timescaledb_internal.get_chunk_relstats(regclass) diff --git a/tsl/test/sql/CMakeLists.txt b/tsl/test/sql/CMakeLists.txt index b3bf77052..cf766ffca 100644 --- a/tsl/test/sql/CMakeLists.txt +++ b/tsl/test/sql/CMakeLists.txt @@ -130,6 +130,14 @@ set(SOLO_TESTS reorder telemetry_stats) +# chunk_utils_internal creates a backup, drop the databases and restores the +# backup. So, it should not run in parallel with other tests. +if((${PG_VERSION_MAJOR} GREATER_EQUAL "14")) + if(CMAKE_BUILD_TYPE MATCHES Debug) + list(APPEND SOLO_TESTS chunk_utils_internal) + endif() +endif() + set(TEST_TEMPLATES compression_insert.sql.in cagg_union_view.sql.in plan_skip_scan.sql.in transparent_decompression.sql.in diff --git a/tsl/test/sql/chunk_utils_internal.sql b/tsl/test/sql/chunk_utils_internal.sql index 3e629a2bb..4e90c5e8c 100644 --- a/tsl/test/sql/chunk_utils_internal.sql +++ b/tsl/test/sql/chunk_utils_internal.sql @@ -44,8 +44,8 @@ CREATE TABLE test1.hyper1 (time bigint, temp float); SELECT create_hypertable('test1.hyper1', 'time', chunk_time_interval => 10); INSERT INTO test1.hyper1 VALUES (10, 0.5); - INSERT INTO test1.hyper1 VALUES (30, 0.5); + SELECT chunk_schema as "CHSCHEMA", chunk_name as "CHNAME", range_start_integer, range_end_integer FROM timescaledb_information.chunks @@ -61,22 +61,70 @@ WHERE hypertable_name = 'hyper1' and hypertable_schema = 'test1' ORDER BY chunk_name LIMIT 1 \gset +-- Freeze and check trigger +SELECT tgname, tgtype FROM pg_trigger WHERE tgrelid = :'CHNAME'::regclass ORDER BY tgname, tgtype; SELECT _timescaledb_internal.freeze_chunk( :'CHNAME'); +SELECT tgname, tgtype FROM pg_trigger WHERE tgrelid = :'CHNAME'::regclass ORDER BY tgname, tgtype; SELECT * from test1.hyper1 ORDER BY 1; -- TEST updates and deletes on frozen chunk should fail \set ON_ERROR_STOP 0 -UPDATE test1.hyper1 SET temp = 40 WHERE time = 20; -UPDATE test1.hyper1 SET temp = 40 WHERE temp = 0.5; + SELECT * from test1.hyper1 ORDER BY 1; +-- Value (time = 20) does not exist +UPDATE test1.hyper1 SET temp = 40 WHERE time = 20; +-- Frozen chunk is affected +UPDATE test1.hyper1 SET temp = 40 WHERE temp = 0.5; +-- Frozen chunk is affected +UPDATE test1.hyper1 SET temp = 40 WHERE time = 10; +-- Frozen chunk is affected +DELETE FROM test1.hyper1 WHERE time = 10; + +SELECT * from test1.hyper1 ORDER BY 1; + +BEGIN; DELETE FROM test1.hyper1 WHERE time = 20; DELETE FROM test1.hyper1 WHERE temp = 0.5; +ROLLBACK; + +-- TEST update on unfrozen chunk should be possible +BEGIN; +SELECT * FROM test1.hyper1; +UPDATE test1.hyper1 SET temp = 40 WHERE time = 30; +SELECT * FROM test1.hyper1; +ROLLBACK; + +-- Test with cast (chunk path pruning can not be done during query planning) +BEGIN; +SELECT * FROM test1.hyper1 WHERE time = 30; +UPDATE test1.hyper1 SET temp = 40 WHERE time = 30::text::float; +SELECT * FROM test1.hyper1 WHERE time = 30; +ROLLBACK; + +-- TEST delete on unfrozen chunks should be possible +BEGIN; SELECT * from test1.hyper1 ORDER BY 1; +DELETE FROM test1.hyper1 WHERE time = 30; +SELECT * from test1.hyper1 ORDER BY 1; +ROLLBACK; + +-- Test with cast +BEGIN; +SELECT * FROM test1.hyper1 WHERE time = 30; +DELETE FROM test1.hyper1 WHERE time = 30::text::float; +SELECT * FROM test1.hyper1 WHERE time = 30; +ROLLBACK; -- TEST inserts into a frozen chunk fails INSERT INTO test1.hyper1 VALUES ( 11, 11); + +-- Test truncating table should fail +TRUNCATE test1.hyper1; + +SELECT * from test1.hyper1 ORDER BY 1; + \set ON_ERROR_STOP 1 --insert into non-frozen chunk works @@ -89,9 +137,18 @@ FROM _timescaledb_catalog.chunk WHERE table_name = :'CHUNK_NAME'; SELECT _timescaledb_internal.unfreeze_chunk( :'CHNAME'); +SELECT tgname, tgtype FROM pg_trigger WHERE tgrelid = :'CHNAME'::regclass ORDER BY tgname, tgtype; + --verify status in catalog SELECT table_name, status FROM _timescaledb_catalog.chunk WHERE table_name = :'CHUNK_NAME'; + +-- Test update works after unfreeze +UPDATE test1.hyper1 SET temp = 40; + +-- Test delete works after unfreeze +DELETE FROM test1.hyper1; + --unfreezing again works SELECT _timescaledb_internal.unfreeze_chunk( :'CHNAME'); SELECT _timescaledb_internal.drop_chunk( :'CHNAME'); @@ -170,6 +227,10 @@ SELECT compress_chunk( :'CHNAME'); SELECT _timescaledb_internal.drop_chunk(:'CHNAME'); \set ON_ERROR_STOP 1 +-- Prepare table for CAGG tests +TRUNCATE test1.hyper1; +INSERT INTO test1.hyper1(time, temp) values(30, 0.5), (31, 31); + --TEST drop_chunk in the presence of caggs. Does not affect cagg data CREATE OR REPLACE FUNCTION hyper_dummy_now() RETURNS BIGINT LANGUAGE SQL IMMUTABLE AS 'SELECT 100::BIGINT'; @@ -268,7 +329,8 @@ SELECT * FROM ht_try ORDER BY 1; SELECT relname, relowner::regrole FROM pg_class WHERE relname in ( select chunk_name FROM chunk_view - WHERE hypertable_name = 'ht_try' ); + WHERE hypertable_name = 'ht_try' ) +ORDER BY relname; SELECT inhrelid::regclass FROM pg_inherits WHERE inhparent = 'ht_try'::regclass ORDER BY 1; @@ -394,6 +456,117 @@ FROM chunk_view WHERE hypertable_name = 'hyper_constr' ORDER BY chunk_name; +----- TESTS for copy into frozen chunk ------------ +\c :TEST_DBNAME :ROLE_DEFAULT_PERM_USER +CREATE TABLE test1.copy_test ( + "time" timestamptz NOT NULL, + "value" double precision NOT NULL +); + +SELECT create_hypertable('test1.copy_test', 'time', chunk_time_interval => interval '1 day'); + +COPY test1.copy_test FROM STDIN DELIMITER ','; +2020-01-01 01:10:00+01,1 +2021-01-01 01:10:00+01,1 +\. + +-- Freeze one of the chunks +SELECT chunk_schema || '.' || chunk_name as "COPY_CHNAME", chunk_name as "COPY_CHUNK_NAME" +FROM timescaledb_information.chunks +WHERE hypertable_name = 'copy_test' and hypertable_schema = 'test1' +ORDER BY chunk_name LIMIT 1 +\gset + +SELECT _timescaledb_internal.freeze_chunk( :'COPY_CHNAME'); + +-- Check state +SELECT table_name, status +FROM _timescaledb_catalog.chunk WHERE table_name = :'COPY_CHUNK_NAME'; + +\set ON_ERROR_STOP 0 +-- Copy should fail because one of che chunks is frozen +COPY test1.copy_test FROM STDIN DELIMITER ','; +2020-01-01 01:10:00+01,1 +2021-01-01 01:10:00+01,1 +\. +\set ON_ERROR_STOP 1 + +-- Count existing rows +SELECT COUNT(*) FROM test1.copy_test; + +-- Test dump & restore +\c postgres :ROLE_SUPERUSER +\! utils/pg_dump_aux_dump.sh dump/pg_dump.sql + +\c :TEST_DBNAME + +-- Make sure tables was droped by pg_dump_aux_dump.sh +\set ON_ERROR_STOP 0 +SELECT * FROM test1.copy_test; +\set ON_ERROR_STOP 1 + +SET client_min_messages = ERROR; +CREATE EXTENSION timescaledb CASCADE; +RESET client_min_messages; + +--\! cp dump/pg_dump.sql /tmp/dump.sql +SELECT timescaledb_pre_restore(); +\! utils/pg_dump_aux_restore.sh dump/pg_dump.sql +SELECT timescaledb_post_restore(); +SELECT _timescaledb_internal.stop_background_workers(); + +\c :TEST_DBNAME :ROLE_DEFAULT_PERM_USER + +-- Make sure the chunk is still frozen +-- Check trigger +SELECT tgname, tgtype FROM pg_trigger WHERE tgrelid = :'CHNAME'::regclass ORDER BY tgname, tgtype; + +-- Check state +SELECT table_name, status +FROM _timescaledb_catalog.chunk WHERE table_name = :'COPY_CHUNK_NAME'; + +\set ON_ERROR_STOP 0 +-- Copy should fail because one of che chunks is frozen +COPY test1.copy_test FROM STDIN DELIMITER ','; +2020-01-01 01:10:00+01,1 +2021-01-01 01:10:00+01,1 +\. +\set ON_ERROR_STOP 1 + +-- Count existing rows +SELECT COUNT(*) FROM test1.copy_test; + +-- Check unfreeze restored chunk +SELECT _timescaledb_internal.unfreeze_chunk( :'COPY_CHNAME'); + +-- Check trigger +SELECT tgname, tgtype FROM pg_trigger WHERE tgrelid = :'CHNAME'::regclass ORDER BY tgname, tgtype; + +-- Check state +SELECT table_name, status +FROM _timescaledb_catalog.chunk WHERE table_name = :'COPY_CHUNK_NAME'; + +-- Copy should work now +COPY test1.copy_test FROM STDIN DELIMITER ','; +2020-01-01 01:10:00+01,1 +2021-01-01 01:10:00+01,1 +\. + +-- Test that unfreeze works even if somebody has dropped one the block triggers +SELECT _timescaledb_internal.freeze_chunk( :'COPY_CHNAME'); + +SELECT table_name, status +FROM _timescaledb_catalog.chunk WHERE table_name = :'COPY_CHUNK_NAME'; + +SELECT tgname, tgtype FROM pg_trigger WHERE tgrelid = :'COPY_CHNAME'::regclass ORDER BY tgname, tgtype; +DROP TRIGGER frozen_chunk_modify_blocker_row ON :COPY_CHNAME; + +SELECT tgname, tgtype FROM pg_trigger WHERE tgrelid = :'COPY_CHNAME'::regclass ORDER BY tgname, tgtype; +SELECT _timescaledb_internal.unfreeze_chunk( :'COPY_CHNAME'); +SELECT tgname, tgtype FROM pg_trigger WHERE tgrelid = :'COPY_CHNAME'::regclass ORDER BY tgname, tgtype; + +SELECT table_name, status +FROM _timescaledb_catalog.chunk WHERE table_name = :'COPY_CHUNK_NAME'; -- clean up databases created \c :TEST_DBNAME :ROLE_SUPERUSER