From 422440a2de9f0d542f5cb38a46761c7a9ab030da Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Erik=20Nordstr=C3=B6m?= Date: Thu, 6 May 2021 22:38:09 +0200 Subject: [PATCH] Add GENERATED column support on distributed hypertables Add support for deparsing `GENERATED` columns when creating distributed hypertables, including support for generating values "on-the-fly" when a `RETURNING` clause is used with an `INSERT` statement. Previously, using generated columns on a hypertable led to errors or had unpredictible results. --- tsl/src/deparse.c | 9 +- tsl/src/nodes/data_node_dispatch.c | 24 ++- tsl/test/expected/dist_hypertable_am.out | 133 ------------ tsl/test/expected/dist_hypertable_pg12.out | 193 ++++++++++++++++++ tsl/test/sql/CMakeLists.txt | 2 +- ...rtable_am.sql => dist_hypertable_pg12.sql} | 38 +++- 6 files changed, 253 insertions(+), 146 deletions(-) delete mode 100644 tsl/test/expected/dist_hypertable_am.out create mode 100644 tsl/test/expected/dist_hypertable_pg12.out rename tsl/test/sql/{dist_hypertable_am.sql => dist_hypertable_pg12.sql} (61%) diff --git a/tsl/src/deparse.c b/tsl/src/deparse.c index 1643bd310..fda324f25 100644 --- a/tsl/src/deparse.c +++ b/tsl/src/deparse.c @@ -195,7 +195,14 @@ deparse_columns(StringInfo stmt, Relation rel) CStringGetTextDatum(attr_def.adbin), ObjectIdGetDatum(rel->rd_id))); - appendStringInfo(stmt, " DEFAULT %s", attr_default); +#if PG12_GE + if (attr->attgenerated == ATTRIBUTE_GENERATED_STORED) + appendStringInfo(stmt, " GENERATED ALWAYS AS %s STORED", attr_default); + else +#endif + { + appendStringInfo(stmt, " DEFAULT %s", attr_default); + } break; } } diff --git a/tsl/src/nodes/data_node_dispatch.c b/tsl/src/nodes/data_node_dispatch.c index 250e5ebd7..7b25688ac 100644 --- a/tsl/src/nodes/data_node_dispatch.c +++ b/tsl/src/nodes/data_node_dispatch.c @@ -11,6 +11,7 @@ #include #include #include +#include #include #include #include @@ -706,6 +707,19 @@ handle_read(DataNodeDispatchState *sds) ListCell *lc; bool primary_data_node = true; MemoryContext oldcontext; +#if PG12_GE + TupleDesc rri_desc = RelationGetDescr(rri->ri_RelationDesc); + + if (NULL != rri->ri_projectReturning && rri_desc->constr && + rri_desc->constr->has_generated_stored) + ExecComputeStoredGenerated(estate, + slot +#if PG13_GE + , + CMD_INSERT +#endif + ); +#endif /* PG12_GE */ Assert(NULL != cis); @@ -1063,8 +1077,14 @@ get_insert_attrs(Relation rel) { Form_pg_attribute attr = TupleDescAttr(tupdesc, i); - if (!attr->attisdropped) - attrs = lappend_int(attrs, AttrOffsetGetAttrNumber(i)); + if (attr->attisdropped +#if PG12_GE + || attr->attgenerated != '\0' +#endif + ) + continue; + + attrs = lappend_int(attrs, AttrOffsetGetAttrNumber(i)); } return attrs; diff --git a/tsl/test/expected/dist_hypertable_am.out b/tsl/test/expected/dist_hypertable_am.out deleted file mode 100644 index 03a247bd2..000000000 --- a/tsl/test/expected/dist_hypertable_am.out +++ /dev/null @@ -1,133 +0,0 @@ --- This file and its contents are licensed under the Timescale License. --- Please see the included NOTICE for copyright information and --- LICENSE-TIMESCALE for a copy of the license. --- Need to be super user to create extension and add data nodes -\c :TEST_DBNAME :ROLE_CLUSTER_SUPERUSER; -\unset ECHO -psql:include/remote_exec.sql:5: NOTICE: schema "test" already exists, skipping -\set DN_DBNAME_1 :TEST_DBNAME _1 -\set DN_DBNAME_2 :TEST_DBNAME _2 -\set DN_DBNAME_3 :TEST_DBNAME _3 --- Add data nodes using the TimescaleDB node management API -SELECT * FROM add_data_node('data_node_1', host => 'localhost', database => :'DN_DBNAME_1'); - node_name | host | port | database | node_created | database_created | extension_created --------------+-----------+-------+-------------------------+--------------+------------------+------------------- - data_node_1 | localhost | 55432 | db_dist_hypertable_am_1 | t | t | t -(1 row) - -SELECT * FROM add_data_node('data_node_2', host => 'localhost', database => :'DN_DBNAME_2'); - node_name | host | port | database | node_created | database_created | extension_created --------------+-----------+-------+-------------------------+--------------+------------------+------------------- - data_node_2 | localhost | 55432 | db_dist_hypertable_am_2 | t | t | t -(1 row) - -SELECT * FROM add_data_node('data_node_3', host => 'localhost', database => :'DN_DBNAME_3'); - node_name | host | port | database | node_created | database_created | extension_created --------------+-----------+-------+-------------------------+--------------+------------------+------------------- - data_node_3 | localhost | 55432 | db_dist_hypertable_am_3 | t | t | t -(1 row) - -GRANT USAGE ON FOREIGN SERVER data_node_1, data_node_2, data_node_3 TO PUBLIC; --- Create a new table access method by reusing heap handler -CREATE ACCESS METHOD test_am TYPE TABLE HANDLER heap_tableam_handler; -SELECT * FROM test.remote_exec('{ data_node_1, data_node_2, data_node_3 }', $$ -CREATE ACCESS METHOD test_am TYPE TABLE HANDLER heap_tableam_handler; -$$); -NOTICE: [data_node_1]: -CREATE ACCESS METHOD test_am TYPE TABLE HANDLER heap_tableam_handler -NOTICE: [data_node_2]: -CREATE ACCESS METHOD test_am TYPE TABLE HANDLER heap_tableam_handler -NOTICE: [data_node_3]: -CREATE ACCESS METHOD test_am TYPE TABLE HANDLER heap_tableam_handler - remote_exec -------------- - -(1 row) - --- Create distributed hypertable using non-default access method -CREATE TABLE disttable(time timestamptz NOT NULL, device int) USING test_am; -SELECT * FROM create_distributed_hypertable('disttable', 'time', 'device', 3); - hypertable_id | schema_name | table_name | created ----------------+-------------+------------+--------- - 1 | public | disttable | t -(1 row) - --- Make sure that distributed hypertable created on data nodes is --- using the correct table access method -SELECT * FROM test.remote_exec('{ data_node_1, data_node_2, data_node_3 }', $$ - -SELECT amname AS hypertable_amname -FROM pg_class cl, pg_am am -WHERE cl.oid = 'disttable'::regclass -AND cl.relam = am.oid; -$$); -NOTICE: [data_node_1]: - -SELECT amname AS hypertable_amname -FROM pg_class cl, pg_am am -WHERE cl.oid = 'disttable'::regclass -AND cl.relam = am.oid -NOTICE: [data_node_1]: -hypertable_amname ------------------ -test_am -(1 row) - - -NOTICE: [data_node_2]: - -SELECT amname AS hypertable_amname -FROM pg_class cl, pg_am am -WHERE cl.oid = 'disttable'::regclass -AND cl.relam = am.oid -NOTICE: [data_node_2]: -hypertable_amname ------------------ -test_am -(1 row) - - -NOTICE: [data_node_3]: - -SELECT amname AS hypertable_amname -FROM pg_class cl, pg_am am -WHERE cl.oid = 'disttable'::regclass -AND cl.relam = am.oid -NOTICE: [data_node_3]: -hypertable_amname ------------------ -test_am -(1 row) - - - remote_exec -------------- - -(1 row) - --- Check that basic operations are working as expected -INSERT INTO disttable VALUES - ('2017-01-01 06:01', 1), - ('2017-01-01 09:11', 3), - ('2017-01-01 08:01', 1), - ('2017-01-02 08:01', 2), - ('2018-07-02 08:01', 87), - ('2018-07-01 06:01', 13), - ('2018-07-01 09:11', 90), - ('2018-07-01 08:01', 29); -SELECT * FROM disttable ORDER BY time; - time | device -------------------------------+-------- - Sun Jan 01 06:01:00 2017 PST | 1 - Sun Jan 01 08:01:00 2017 PST | 1 - Sun Jan 01 09:11:00 2017 PST | 3 - Mon Jan 02 08:01:00 2017 PST | 2 - Sun Jul 01 06:01:00 2018 PDT | 13 - Sun Jul 01 08:01:00 2018 PDT | 29 - Sun Jul 01 09:11:00 2018 PDT | 90 - Mon Jul 02 08:01:00 2018 PDT | 87 -(8 rows) - -DROP DATABASE :DN_DBNAME_1; -DROP DATABASE :DN_DBNAME_2; -DROP DATABASE :DN_DBNAME_3; diff --git a/tsl/test/expected/dist_hypertable_pg12.out b/tsl/test/expected/dist_hypertable_pg12.out new file mode 100644 index 000000000..1673e3580 --- /dev/null +++ b/tsl/test/expected/dist_hypertable_pg12.out @@ -0,0 +1,193 @@ +-- This file and its contents are licensed under the Timescale License. +-- Please see the included NOTICE for copyright information and +-- LICENSE-TIMESCALE for a copy of the license. +-- Tests using features supported in PG12 and greater. +-- Need to be super user to create extension and add data nodes +\c :TEST_DBNAME :ROLE_CLUSTER_SUPERUSER; +\unset ECHO +psql:include/remote_exec.sql:5: NOTICE: schema "test" already exists, skipping +\set DN_DBNAME_1 :TEST_DBNAME _1 +\set DN_DBNAME_2 :TEST_DBNAME _2 +\set DN_DBNAME_3 :TEST_DBNAME _3 +-- Add data nodes using the TimescaleDB node management API +SELECT * FROM add_data_node('data_node_1', host => 'localhost', database => :'DN_DBNAME_1'); + node_name | host | port | database | node_created | database_created | extension_created +-------------+-----------+-------+---------------------------+--------------+------------------+------------------- + data_node_1 | localhost | 55432 | db_dist_hypertable_pg12_1 | t | t | t +(1 row) + +SELECT * FROM add_data_node('data_node_2', host => 'localhost', database => :'DN_DBNAME_2'); + node_name | host | port | database | node_created | database_created | extension_created +-------------+-----------+-------+---------------------------+--------------+------------------+------------------- + data_node_2 | localhost | 55432 | db_dist_hypertable_pg12_2 | t | t | t +(1 row) + +SELECT * FROM add_data_node('data_node_3', host => 'localhost', database => :'DN_DBNAME_3'); + node_name | host | port | database | node_created | database_created | extension_created +-------------+-----------+-------+---------------------------+--------------+------------------+------------------- + data_node_3 | localhost | 55432 | db_dist_hypertable_pg12_3 | t | t | t +(1 row) + +GRANT USAGE ON FOREIGN SERVER data_node_1, data_node_2, data_node_3 TO PUBLIC; +-- Create a new table access method by reusing heap handler +CREATE ACCESS METHOD test_am TYPE TABLE HANDLER heap_tableam_handler; +SELECT * FROM test.remote_exec('{ data_node_1, data_node_2, data_node_3 }', $$ +CREATE ACCESS METHOD test_am TYPE TABLE HANDLER heap_tableam_handler; +$$); +NOTICE: [data_node_1]: +CREATE ACCESS METHOD test_am TYPE TABLE HANDLER heap_tableam_handler +NOTICE: [data_node_2]: +CREATE ACCESS METHOD test_am TYPE TABLE HANDLER heap_tableam_handler +NOTICE: [data_node_3]: +CREATE ACCESS METHOD test_am TYPE TABLE HANDLER heap_tableam_handler + remote_exec +------------- + +(1 row) + +-- Create distributed hypertable using non-default access method +CREATE TABLE disttable(time timestamptz NOT NULL, device int, temp_c float, temp_f float GENERATED ALWAYS AS (temp_c * 9 / 5 + 32) STORED) USING test_am; +SELECT * FROM create_distributed_hypertable('disttable', 'time', 'device', 3); + hypertable_id | schema_name | table_name | created +---------------+-------------+------------+--------- + 1 | public | disttable | t +(1 row) + +-- Make sure that distributed hypertable created on data nodes is +-- using the correct table access method +SELECT * FROM test.remote_exec('{ data_node_1, data_node_2, data_node_3 }', $$ + +SELECT amname AS hypertable_amname +FROM pg_class cl, pg_am am +WHERE cl.oid = 'disttable'::regclass +AND cl.relam = am.oid; +$$); +NOTICE: [data_node_1]: + +SELECT amname AS hypertable_amname +FROM pg_class cl, pg_am am +WHERE cl.oid = 'disttable'::regclass +AND cl.relam = am.oid +NOTICE: [data_node_1]: +hypertable_amname +----------------- +test_am +(1 row) + + +NOTICE: [data_node_2]: + +SELECT amname AS hypertable_amname +FROM pg_class cl, pg_am am +WHERE cl.oid = 'disttable'::regclass +AND cl.relam = am.oid +NOTICE: [data_node_2]: +hypertable_amname +----------------- +test_am +(1 row) + + +NOTICE: [data_node_3]: + +SELECT amname AS hypertable_amname +FROM pg_class cl, pg_am am +WHERE cl.oid = 'disttable'::regclass +AND cl.relam = am.oid +NOTICE: [data_node_3]: +hypertable_amname +----------------- +test_am +(1 row) + + + remote_exec +------------- + +(1 row) + +-- Check that basic operations are working as expected +INSERT INTO disttable VALUES + ('2017-01-01 06:01', 1, -10.0), + ('2017-01-01 09:11', 3, -5.0), + ('2017-01-01 08:01', 1, 1.0), + ('2017-01-02 08:01', 2, 5.0), + ('2018-07-02 08:01', 87, 10.0), + ('2018-07-01 06:01', 13, 15.0), + ('2018-07-01 09:11', 90, 20.0), + ('2018-07-01 08:01', 29, 24.0); +SELECT * FROM disttable ORDER BY time; + time | device | temp_c | temp_f +------------------------------+--------+--------+-------- + Sun Jan 01 06:01:00 2017 PST | 1 | -10 | 14 + Sun Jan 01 08:01:00 2017 PST | 1 | 1 | 33.8 + Sun Jan 01 09:11:00 2017 PST | 3 | -5 | 23 + Mon Jan 02 08:01:00 2017 PST | 2 | 5 | 41 + Sun Jul 01 06:01:00 2018 PDT | 13 | 15 | 59 + Sun Jul 01 08:01:00 2018 PDT | 29 | 24 | 75.2 + Sun Jul 01 09:11:00 2018 PDT | 90 | 20 | 68 + Mon Jul 02 08:01:00 2018 PDT | 87 | 10 | 50 +(8 rows) + +-- Show that GENERATED columns work for INSERT with RETURNING clause +TRUNCATE disttable; +EXPLAIN VERBOSE +INSERT INTO disttable VALUES ('2017-08-01 06:01', 1, 35.0) RETURNING *; + QUERY PLAN +------------------------------------------------------------------------------------------------------------------------------------------------------------------------- + Custom Scan (HypertableInsert) (cost=0.00..0.01 rows=1 width=28) + Output: disttable."time", disttable.device, disttable.temp_c, disttable.temp_f + Insert on distributed hypertable public.disttable + Data nodes: data_node_1, data_node_2, data_node_3 + -> Insert on public.disttable (cost=0.00..0.01 rows=1 width=28) + Output: disttable."time", disttable.device, disttable.temp_c, disttable.temp_f + -> Custom Scan (DataNodeDispatch) (cost=0.00..0.01 rows=1 width=28) + Output: 'Tue Aug 01 06:01:00 2017 PDT'::timestamp with time zone, 1, '35'::double precision, NULL::double precision + Batch size: 1000 + Remote SQL: INSERT INTO public.disttable("time", device, temp_c) VALUES ($1, $2, $3), ..., ($2998, $2999, $3000) RETURNING "time", device, temp_c, temp_f + -> Custom Scan (ChunkDispatch) (cost=0.00..0.01 rows=1 width=28) + Output: 'Tue Aug 01 06:01:00 2017 PDT'::timestamp with time zone, 1, '35'::double precision, NULL::double precision + -> Result (cost=0.00..0.01 rows=1 width=28) + Output: 'Tue Aug 01 06:01:00 2017 PDT'::timestamp with time zone, 1, '35'::double precision, NULL::double precision +(14 rows) + +INSERT INTO disttable VALUES ('2017-08-01 06:01', 1, 35.0) RETURNING *; + time | device | temp_c | temp_f +------------------------------+--------+--------+-------- + Tue Aug 01 06:01:00 2017 PDT | 1 | 35 | 95 +(1 row) + +-- Same values returned with SELECT: +SELECT * FROM disttable ORDER BY 1; + time | device | temp_c | temp_f +------------------------------+--------+--------+-------- + Tue Aug 01 06:01:00 2017 PDT | 1 | 35 | 95 +(1 row) + +UPDATE disttable SET temp_c=40.0 WHERE device=1; +SELECT * FROM disttable ORDER BY 1; + time | device | temp_c | temp_f +------------------------------+--------+--------+-------- + Tue Aug 01 06:01:00 2017 PDT | 1 | 40 | 104 +(1 row) + +-- Insert another value +INSERT INTO disttable VALUES ('2017-09-01 06:01', 2, 30.0); +SELECT * FROM disttable ORDER BY 1; + time | device | temp_c | temp_f +------------------------------+--------+--------+-------- + Tue Aug 01 06:01:00 2017 PDT | 1 | 40 | 104 + Fri Sep 01 06:01:00 2017 PDT | 2 | 30 | 86 +(2 rows) + +-- Delete a value based on the generated column +DELETE FROM disttable WHERE temp_f=104; +SELECT * FROM disttable ORDER BY 1; + time | device | temp_c | temp_f +------------------------------+--------+--------+-------- + Fri Sep 01 06:01:00 2017 PDT | 2 | 30 | 86 +(1 row) + +DROP DATABASE :DN_DBNAME_1; +DROP DATABASE :DN_DBNAME_2; +DROP DATABASE :DN_DBNAME_3; diff --git a/tsl/test/sql/CMakeLists.txt b/tsl/test/sql/CMakeLists.txt index cbc97b09b..4e6fabc62 100644 --- a/tsl/test/sql/CMakeLists.txt +++ b/tsl/test/sql/CMakeLists.txt @@ -105,7 +105,7 @@ endif() if (${PG_VERSION_MAJOR} GREATER_EQUAL "12") list(APPEND TEST_FILES_DEBUG - dist_hypertable_am.sql + dist_hypertable_pg12.sql continuous_aggs_tableam.sql ) endif() diff --git a/tsl/test/sql/dist_hypertable_am.sql b/tsl/test/sql/dist_hypertable_pg12.sql similarity index 61% rename from tsl/test/sql/dist_hypertable_am.sql rename to tsl/test/sql/dist_hypertable_pg12.sql index 56ba6df95..08db0a68a 100644 --- a/tsl/test/sql/dist_hypertable_am.sql +++ b/tsl/test/sql/dist_hypertable_pg12.sql @@ -2,6 +2,7 @@ -- Please see the included NOTICE for copyright information and -- LICENSE-TIMESCALE for a copy of the license. +-- Tests using features supported in PG12 and greater. -- Need to be super user to create extension and add data nodes \c :TEST_DBNAME :ROLE_CLUSTER_SUPERUSER; @@ -29,7 +30,7 @@ CREATE ACCESS METHOD test_am TYPE TABLE HANDLER heap_tableam_handler; $$); -- Create distributed hypertable using non-default access method -CREATE TABLE disttable(time timestamptz NOT NULL, device int) USING test_am; +CREATE TABLE disttable(time timestamptz NOT NULL, device int, temp_c float, temp_f float GENERATED ALWAYS AS (temp_c * 9 / 5 + 32) STORED) USING test_am; SELECT * FROM create_distributed_hypertable('disttable', 'time', 'device', 3); -- Make sure that distributed hypertable created on data nodes is @@ -44,17 +45,36 @@ $$); -- Check that basic operations are working as expected INSERT INTO disttable VALUES - ('2017-01-01 06:01', 1), - ('2017-01-01 09:11', 3), - ('2017-01-01 08:01', 1), - ('2017-01-02 08:01', 2), - ('2018-07-02 08:01', 87), - ('2018-07-01 06:01', 13), - ('2018-07-01 09:11', 90), - ('2018-07-01 08:01', 29); + ('2017-01-01 06:01', 1, -10.0), + ('2017-01-01 09:11', 3, -5.0), + ('2017-01-01 08:01', 1, 1.0), + ('2017-01-02 08:01', 2, 5.0), + ('2018-07-02 08:01', 87, 10.0), + ('2018-07-01 06:01', 13, 15.0), + ('2018-07-01 09:11', 90, 20.0), + ('2018-07-01 08:01', 29, 24.0); SELECT * FROM disttable ORDER BY time; +-- Show that GENERATED columns work for INSERT with RETURNING clause +TRUNCATE disttable; +EXPLAIN VERBOSE +INSERT INTO disttable VALUES ('2017-08-01 06:01', 1, 35.0) RETURNING *; +INSERT INTO disttable VALUES ('2017-08-01 06:01', 1, 35.0) RETURNING *; + +-- Same values returned with SELECT: +SELECT * FROM disttable ORDER BY 1; + +UPDATE disttable SET temp_c=40.0 WHERE device=1; +SELECT * FROM disttable ORDER BY 1; + +-- Insert another value +INSERT INTO disttable VALUES ('2017-09-01 06:01', 2, 30.0); +SELECT * FROM disttable ORDER BY 1; +-- Delete a value based on the generated column +DELETE FROM disttable WHERE temp_f=104; +SELECT * FROM disttable ORDER BY 1; + DROP DATABASE :DN_DBNAME_1; DROP DATABASE :DN_DBNAME_2; DROP DATABASE :DN_DBNAME_3;