Add internal api for foreign table chunk

Add _timescaledb_internal.attach_osm_table_chunk.
This treats a pre-existing foreign table as a
hypertable chunk by adding dummy metadata to the
catalog tables.
This commit is contained in:
gayyappan 2022-04-28 17:51:27 -04:00 committed by gayyappan
parent 6c38c60b97
commit 131f58ee60
12 changed files with 284 additions and 11 deletions

View File

@ -90,3 +90,9 @@ RETURNS BOOL AS '@MODULE_PATHNAME@', 'ts_chunk_freeze_chunk' LANGUAGE C VOLATILE
CREATE OR REPLACE FUNCTION _timescaledb_internal.drop_chunk(
chunk REGCLASS)
RETURNS BOOL AS '@MODULE_PATHNAME@', 'ts_chunk_drop_single_chunk' LANGUAGE C VOLATILE;
-- internal API used by OSM extension to attach a table as a chunk of the hypertable
CREATE OR REPLACE FUNCTION _timescaledb_internal.attach_osm_table_chunk(
hypertable REGCLASS,
chunk REGCLASS)
RETURNS BOOL AS '@MODULE_PATHNAME@', 'ts_chunk_attach_osm_table_chunk' LANGUAGE C VOLATILE;

View File

@ -14,3 +14,5 @@ CREATE FUNCTION @extschema@.detach_data_node(
repartition BOOLEAN = TRUE
) RETURNS INTEGER
AS '@MODULE_PATHNAME@', 'ts_data_node_detach' LANGUAGE C VOLATILE;
DROP FUNCTION _timescaledb_internal.attach_osm_table_chunk( hypertable REGCLASS, chunk REGCLASS);

View File

@ -69,6 +69,7 @@
TS_FUNCTION_INFO_V1(ts_chunk_show_chunks);
TS_FUNCTION_INFO_V1(ts_chunk_drop_chunks);
TS_FUNCTION_INFO_V1(ts_chunk_drop_single_chunk);
TS_FUNCTION_INFO_V1(ts_chunk_attach_osm_table_chunk);
TS_FUNCTION_INFO_V1(ts_chunk_freeze_chunk);
TS_FUNCTION_INFO_V1(ts_chunks_in);
TS_FUNCTION_INFO_V1(ts_chunk_id_from_relid);
@ -170,6 +171,12 @@ static Chunk *chunk_resurrect(const Hypertable *ht, int chunk_id);
*/
#define CHUNK_STATUS_FROZEN 4
/* A OSM table is added as a chunk to the hypertable.
* This is different from a distributed hypertable chunk that
* is managed by the Timescale extension.
*/
#define CHUNK_STATUS_FOREIGN 8
static HeapTuple
chunk_formdata_make_tuple(const FormData_chunk *fd, TupleDesc desc)
{
@ -4359,3 +4366,112 @@ ts_chunk_scan_iterator_set_chunk_id(ScanIterator *it, int32 chunk_id)
F_INT4EQ,
Int32GetDatum(chunk_id));
}
#include "hypercube.h"
static Hypercube *
fill_hypercube_for_foreign_table_chunk(Hyperspace *hs)
{
Hypercube *cube = ts_hypercube_alloc(hs->num_dimensions);
Point *p = ts_point_create(hs->num_dimensions);
Assert(hs->num_dimensions == 1); // does not work with partitioned range
for (int i = 0; i < hs->num_dimensions; i++)
{
const Dimension *dim = &hs->dimensions[i];
Assert(dim->type == DIMENSION_TYPE_OPEN);
Oid dimtype = ts_dimension_get_partition_type(dim);
Datum val = Int64GetDatum(ts_time_get_min(dimtype));
p->coordinates[p->num_coords++] = ts_time_value_to_internal(val, dimtype);
cube->slices[i] = ts_dimension_calculate_default_slice(dim, p->coordinates[i]);
cube->num_slices++;
}
Assert(cube->num_slices == 1);
return cube;
}
/* adds foreign table as a chunk to the hypertable.
* creates a dummy chunk constraint for the time dimension.
* These constraints are recorded in the chunk-dimension slice metadata.
* They are NOT added as CHECK constraints on the foreign table.
*
* Does not add any inheritable constraints or indexes that are already
* defined on the hypertable.
*/
static void
add_foreign_table_as_chunk(Oid relid, Hypertable *parent_ht)
{
Hyperspace *hs = parent_ht->space;
Catalog *catalog = ts_catalog_get();
CatalogSecurityContext sec_ctx;
Chunk *chunk;
char *relschema = get_namespace_name(get_rel_namespace(relid));
char *relname = get_rel_name(relid);
Oid ht_ownerid = ts_rel_get_owner(parent_ht->main_table_relid);
if (!has_privs_of_role(GetUserId(), ht_ownerid))
ereport(ERROR,
(errcode(ERRCODE_INSUFFICIENT_PRIVILEGE),
errmsg("must be owner of hypertable \"%s\"",
get_rel_name(parent_ht->main_table_relid))));
Assert(get_rel_relkind(relid) == RELKIND_FOREIGN_TABLE);
if (hs->num_dimensions > 1)
elog(ERROR,
"cannot attach a foreign table to a hypertable that has more than 1 dimension");
/* Create a new chunk based on the hypercube */
ts_catalog_database_info_become_owner(ts_catalog_database_info_get(), &sec_ctx);
chunk = ts_chunk_create_base(ts_catalog_table_next_seq_id(catalog, CHUNK),
hs->num_dimensions,
RELKIND_RELATION);
ts_catalog_restore_user(&sec_ctx);
/* fill in the correct table_name for the chunk*/
chunk->fd.hypertable_id = hs->hypertable_id;
chunk->cube = fill_hypercube_for_foreign_table_chunk(hs);
chunk->hypertable_relid = parent_ht->main_table_relid;
chunk->constraints = ts_chunk_constraints_alloc(1, CurrentMemoryContext);
namestrcpy(&chunk->fd.schema_name, relschema);
namestrcpy(&chunk->fd.table_name, relname);
chunk->fd.status = CHUNK_STATUS_FOREIGN;
/* Insert chunk */
ts_chunk_insert_lock(chunk, RowExclusiveLock);
/* insert dimension slices if they do not exist.
* Then add dimension constraints for the chunk
*/
ts_dimension_slice_insert_multi(chunk->cube->slices, chunk->cube->num_slices);
ts_chunk_constraints_add_dimension_constraints(chunk->constraints, chunk->fd.id, chunk->cube);
/* check constraints are not automatically created for foreign tables.
* See: ts_chunk_constraints_add_dimension_constraints.
*/
ts_chunk_constraints_insert_metadata(chunk->constraints);
}
/* Internal API used by OSM extension. OSM table is a foreign table that is
* attached as a chunk of the hypertable. A chunk needs dimension constraints. We
* add dummy constraints for the OSM chunk and then attach it to the hypertable.
* OSM extension is responsible for maintaining any constraints on this table.
*/
Datum
ts_chunk_attach_osm_table_chunk(PG_FUNCTION_ARGS)
{
Oid hypertable_relid = PG_ARGISNULL(0) ? InvalidOid : PG_GETARG_OID(0);
Oid ftable_relid = PG_ARGISNULL(1) ? InvalidOid : PG_GETARG_OID(1);
bool ret = false;
Cache *hcache;
Hypertable *par_ht =
ts_hypertable_cache_get_cache_and_entry(hypertable_relid, CACHE_FLAG_MISSING_OK, &hcache);
if (par_ht == NULL)
elog(ERROR, "\"%s\" is not a hypertable", get_rel_name(hypertable_relid));
if (get_rel_relkind(ftable_relid) == RELKIND_FOREIGN_TABLE)
{
add_foreign_table_as_chunk(ftable_relid, par_ht);
ret = true;
}
ts_cache_release(hcache);
PG_RETURN_BOOL(ret);
}

View File

@ -225,6 +225,7 @@ extern ScanIterator ts_chunk_scan_iterator_create(MemoryContext result_mcxt);
extern void ts_chunk_scan_iterator_set_chunk_id(ScanIterator *it, int32 chunk_id);
extern bool ts_chunk_lock_if_exists(Oid chunk_oid, LOCKMODE chunk_lockmode);
extern int ts_chunk_oid_cmp(const void *p1, const void *p2);
extern void ts_chunk_add_foreign_table_as_chunk(Oid relid, Hypertable *parent_ht);
#define chunk_get_by_name(schema_name, table_name, fail_if_not_found) \
ts_chunk_get_by_name_with_memory_context(schema_name, \
@ -237,8 +238,7 @@ extern int ts_chunk_oid_cmp(const void *p1, const void *p2);
OidIsValid((chunk)->table_id) && OidIsValid((chunk)->hypertable_relid) && \
(chunk)->constraints && (chunk)->cube && \
(chunk)->cube->num_slices == (chunk)->constraints->num_dimension_constraints && \
((chunk)->relkind == RELKIND_RELATION || \
((chunk)->relkind == RELKIND_FOREIGN_TABLE && (chunk)->data_nodes != NIL)))
((chunk)->relkind == RELKIND_RELATION || ((chunk)->relkind == RELKIND_FOREIGN_TABLE)))
#define ASSERT_IS_VALID_CHUNK(chunk) Assert(IS_VALID_CHUNK(chunk))

View File

@ -901,8 +901,8 @@ ts_dimension_transform_value(const Dimension *dim, Oid collation, Datum value, O
return value;
}
static Point *
point_create(int16 num_dimensions)
Point *
ts_point_create(int16 num_dimensions)
{
Point *p = palloc0(POINT_SIZE(num_dimensions));
@ -915,7 +915,7 @@ point_create(int16 num_dimensions)
TSDLLEXPORT Point *
ts_hyperspace_calculate_point(const Hyperspace *hs, TupleTableSlot *slot)
{
Point *p = point_create(hs->num_dimensions);
Point *p = ts_point_create(hs->num_dimensions);
int i;
for (i = 0; i < hs->num_dimensions; i++)

View File

@ -144,6 +144,7 @@ extern TSDLLEXPORT void ts_dimension_update(const Hypertable *ht, const NameData
Oid *intervaltype, int16 *num_slices,
Oid *integer_now_func);
extern TSDLLEXPORT List *ts_dimension_get_partexprs(const Dimension *dim, Index hyper_varno);
extern TSDLLEXPORT Point *ts_point_create(int16 num_dimensions);
#define hyperspace_get_open_dimension(space, i) \
ts_hyperspace_get_dimension(space, DIMENSION_TYPE_OPEN, i)

View File

@ -3765,7 +3765,6 @@ process_altertable_end_table(Node *parsetree, CollectedCommand *cmd)
break;
}
}
ts_cache_release(hcache);
}

View File

@ -41,6 +41,8 @@ TEST_ROLE_2=${TEST_ROLE_2:-test_role_2}
TEST_ROLE_2_PASS=${TEST_ROLE_2_PASS:-pass}
TEST_ROLE_3=${TEST_ROLE_3:-test_role_3}
TEST_ROLE_3_PASS=${TEST_ROLE_3_PASS:-pass}
TEST_ROLE_4=${TEST_ROLE_4:-test_role_4}
TEST_ROLE_4_PASS=${TEST_ROLE_4_PASS:-pass}
TEST_ROLE_READ_ONLY=${TEST_ROLE_READ_ONLY:-test_role_read_only}
shift
@ -72,6 +74,7 @@ if mkdir ${TEST_OUTPUT_DIR}/.pg_init 2>/dev/null; then
GRANT CREATE ON SCHEMA public TO ${TEST_ROLE_1};
GRANT CREATE ON SCHEMA public TO ${TEST_ROLE_2};
GRANT CREATE ON SCHEMA public TO ${TEST_ROLE_3};
GRANT CREATE ON SCHEMA public TO ${TEST_ROLE_4};
END IF;
END
\$\$ LANGUAGE PLPGSQL;
@ -81,6 +84,7 @@ if mkdir ${TEST_OUTPUT_DIR}/.pg_init 2>/dev/null; then
ALTER USER ${TEST_ROLE_1} WITH CREATEDB CREATEROLE;
ALTER USER ${TEST_ROLE_2} WITH CREATEDB PASSWORD '${TEST_ROLE_2_PASS}';
ALTER USER ${TEST_ROLE_3} WITH CREATEDB PASSWORD '${TEST_ROLE_3_PASS}';
ALTER USER ${TEST_ROLE_4} WITH CREATEDB PASSWORD '${TEST_ROLE_4_PASS}';
EOF
${PSQL} "$@" -U ${USER} -d postgres -v ECHO=none -c "ALTER USER ${TEST_ROLE_SUPERUSER} WITH SUPERUSER;" >/dev/null
touch ${TEST_OUTPUT_DIR}/.pg_init/done
@ -122,9 +126,11 @@ ${PSQL} -U ${TEST_PGUSER} \
-v ROLE_1=${TEST_ROLE_1} \
-v ROLE_2=${TEST_ROLE_2} \
-v ROLE_3=${TEST_ROLE_3} \
-v ROLE_4=${TEST_ROLE_4} \
-v ROLE_READ_ONLY=${TEST_ROLE_READ_ONLY} \
-v ROLE_2_PASS=${TEST_ROLE_2_PASS} \
-v ROLE_3_PASS=${TEST_ROLE_3_PASS} \
-v ROLE_4_PASS=${TEST_ROLE_4_PASS} \
-v MODULE_PATHNAME="'timescaledb-${EXT_VERSION}'" \
-v TSL_MODULE_PATHNAME="'timescaledb-tsl-${EXT_VERSION}'" \
-v TEST_SUPPORT_FILE=${TEST_SUPPORT_FILE} \

View File

@ -6,12 +6,15 @@ set(TEST_ROLE_DEFAULT_PERM_USER_2 default_perm_user_2)
set(TEST_ROLE_1 test_role_1)
set(TEST_ROLE_2 test_role_2)
set(TEST_ROLE_3 test_role_3)
set(TEST_ROLE_4 test_role_4)
set(TEST_ROLE_READ_ONLY test_role_read_only)
# TEST_ROLE_2 has password in passfile
set(TEST_ROLE_2_PASS pass)
# TEST_ROLE_3 does not have password in passfile
set(TEST_ROLE_3_PASS pass)
# TEST_ROLE_4 does not have password in passfile
set(TEST_ROLE_4_PASS pass)
set(TEST_INPUT_DIR ${CMAKE_CURRENT_SOURCE_DIR})
set(TEST_OUTPUT_DIR ${CMAKE_CURRENT_BINARY_DIR})
@ -64,7 +67,7 @@ set(PG_REGRESS_OPTS_BASE --host=${TEST_PGHOST}
--dlpath=${PROJECT_BINARY_DIR}/src)
set(PG_REGRESS_OPTS_EXTRA
--create-role=${TEST_ROLE_SUPERUSER},${TEST_ROLE_DEFAULT_PERM_USER},${TEST_ROLE_DEFAULT_PERM_USER_2},${TEST_ROLE_CLUSTER_SUPERUSER},${TEST_ROLE_1},${TEST_ROLE_2},${TEST_ROLE_3},${TEST_ROLE_READ_ONLY}
--create-role=${TEST_ROLE_SUPERUSER},${TEST_ROLE_DEFAULT_PERM_USER},${TEST_ROLE_DEFAULT_PERM_USER_2},${TEST_ROLE_CLUSTER_SUPERUSER},${TEST_ROLE_1},${TEST_ROLE_2},${TEST_ROLE_3},${TEST_ROLE_4},${TEST_ROLE_READ_ONLY}
--dbname=${TEST_DBNAME}
--launcher=${PRIMARY_TEST_DIR}/runner.sh)
@ -112,6 +115,7 @@ if(PG_REGRESS)
TEST_ROLE_READ_ONLY=${TEST_ROLE_READ_ONLY}
TEST_ROLE_2_PASS=${TEST_ROLE_2_PASS}
TEST_ROLE_3_PASS=${TEST_ROLE_3_PASS}
TEST_ROLE_4_PASS=${TEST_ROLE_4_PASS}
TEST_DBNAME=${TEST_DBNAME}
TEST_INPUT_DIR=${TEST_INPUT_DIR}
TEST_OUTPUT_DIR=${TEST_OUTPUT_DIR}

View File

@ -2,8 +2,11 @@
-- Please see the included NOTICE for copyright information and
-- LICENSE-TIMESCALE for a copy of the license.
-- These tests work for PG14 or greater
-- Remember to corordinate any changes to freeze_chunk functionality with the Cloud
-- Storage team.
-- Remember to corordinate any changes to functionality with the Cloud
-- Storage team. Tests for the following API:
-- * freeze_chunk
-- * drop_chunk
-- * attach_foreign_table_chunk
\c :TEST_DBNAME :ROLE_SUPERUSER
CREATE SCHEMA test1;
GRANT CREATE ON SCHEMA test1 TO :ROLE_DEFAULT_PERM_USER;
@ -280,3 +283,76 @@ SELECT * FROM hyper1_cagg ORDER BY 1;
30 | 2
(1 row)
--TEST for attaching a foreign table as a chunk
--need superuser access to create foreign data server
\c :TEST_DBNAME :ROLE_SUPERUSER
CREATE DATABASE postgres_fdw_db;
GRANT ALL PRIVILEGES ON DATABASE postgres_fdw_db TO :ROLE_4;
\c postgres_fdw_db :ROLE_4
CREATE TABLE fdw_table( timec timestamptz NOT NULL , acq_id bigint, value bigint);
INSERT INTO fdw_table VALUES( '2020-01-01 01:00', 100, 1000);
--create foreign server and user mappings as superuser
\c :TEST_DBNAME :ROLE_SUPERUSER
SELECT current_setting('port') as "PORTNO" \gset
CREATE EXTENSION postgres_fdw;
CREATE SERVER s3_server FOREIGN DATA WRAPPER postgres_fdw
OPTIONS ( host 'localhost', dbname 'postgres_fdw_db', port :'PORTNO');
GRANT USAGE ON FOREIGN SERVER s3_server TO :ROLE_4;
CREATE USER MAPPING FOR :ROLE_4 SERVER s3_server
OPTIONS ( user :'ROLE_4' , password :'ROLE_4_PASS');
ALTER USER MAPPING FOR :ROLE_4 SERVER s3_server
OPTIONS (ADD password_required 'false');
-- this is a stand-in for the OSM table
CREATE FOREIGN TABLE child_fdw_table
(timec timestamptz NOT NULL, acq_id bigint, value bigint)
SERVER s3_server OPTIONS ( schema_name 'public', table_name 'fdw_table');
GRANT SELECT ON child_fdw_table TO :ROLE_4;
--now attach foreign table as a chunk of the hypertable.
\c :TEST_DBNAME :ROLE_4;
CREATE TABLE ht_try(timec timestamptz NOT NULL, acq_id bigint, value bigint);
SELECT create_hypertable('ht_try', 'timec', chunk_time_interval => interval '1 day');
create_hypertable
---------------------
(5,public,ht_try,t)
(1 row)
INSERT INTO ht_try VALUES ('2020-05-05 01:00', 222, 222);
SELECT * FROM child_fdw_table;
timec | acq_id | value
------------------------------+--------+-------
Wed Jan 01 01:00:00 2020 PST | 100 | 1000
(1 row)
SELECT _timescaledb_internal.attach_osm_table_chunk('ht_try', 'child_fdw_table');
attach_osm_table_chunk
------------------------
t
(1 row)
SELECT chunk_name, range_start, range_end
FROM timescaledb_information.chunks
WHERE hypertable_name = 'ht_try' ORDER BY 1;
chunk_name | range_start | range_end
------------------+---------------------------------+---------------------------------
_hyper_5_9_chunk | Mon May 04 17:00:00 2020 PDT | Tue May 05 17:00:00 2020 PDT
child_fdw_table | Tue Nov 23 16:00:00 4684 PST BC | Wed Nov 24 16:00:00 4684 PST BC
(2 rows)
SELECT * FROM ht_try ORDER BY 1;
timec | acq_id | value
------------------------------+--------+-------
Wed Jan 01 01:00:00 2020 PST | 100 | 1000
Tue May 05 01:00:00 2020 PDT | 222 | 222
(2 rows)
-- TEST error have to be hypertable owner to attach a chunk to it
\c :TEST_DBNAME :ROLE_DEFAULT_PERM_USER
\set ON_ERROR_STOP 0
SELECT _timescaledb_internal.attach_osm_table_chunk('ht_try', 'child_fdw_table');
ERROR: must be owner of hypertable "ht_try"
-- TEST error try to attach to non hypertable
CREATE TABLE non_ht (time bigint, temp float);
SELECT _timescaledb_internal.attach_osm_table_chunk('non_ht', 'child_fdw_table');
ERROR: "non_ht" is not a hypertable
\set ON_ERROR_STOP 1

View File

@ -19,6 +19,7 @@ FROM pg_proc p
e.oid = d.refobjid
WHERE proname <> 'get_telemetry_report'
ORDER BY pronamespace::regnamespace::text COLLATE "C", p.oid::regprocedure::text COLLATE "C";
_timescaledb_internal.attach_osm_table_chunk(regclass,regclass)
_timescaledb_internal.bookend_deserializefunc(bytea,internal)
_timescaledb_internal.bookend_finalfunc(internal,anyelement,"any")
_timescaledb_internal.bookend_serializefunc(internal)

View File

@ -3,8 +3,11 @@
-- LICENSE-TIMESCALE for a copy of the license.
-- These tests work for PG14 or greater
-- Remember to corordinate any changes to freeze_chunk functionality with the Cloud
-- Storage team.
-- Remember to corordinate any changes to functionality with the Cloud
-- Storage team. Tests for the following API:
-- * freeze_chunk
-- * drop_chunk
-- * attach_foreign_table_chunk
\c :TEST_DBNAME :ROLE_SUPERUSER
CREATE SCHEMA test1;
@ -152,3 +155,62 @@ SELECT _timescaledb_internal.freeze_chunk( :'CHNAME');
SELECT _timescaledb_internal.drop_chunk( :'CHNAME');
SELECT * from test1.hyper1 ORDER BY 1;
SELECT * FROM hyper1_cagg ORDER BY 1;
--TEST for attaching a foreign table as a chunk
--need superuser access to create foreign data server
\c :TEST_DBNAME :ROLE_SUPERUSER
CREATE DATABASE postgres_fdw_db;
GRANT ALL PRIVILEGES ON DATABASE postgres_fdw_db TO :ROLE_4;
\c postgres_fdw_db :ROLE_4
CREATE TABLE fdw_table( timec timestamptz NOT NULL , acq_id bigint, value bigint);
INSERT INTO fdw_table VALUES( '2020-01-01 01:00', 100, 1000);
--create foreign server and user mappings as superuser
\c :TEST_DBNAME :ROLE_SUPERUSER
SELECT current_setting('port') as "PORTNO" \gset
CREATE EXTENSION postgres_fdw;
CREATE SERVER s3_server FOREIGN DATA WRAPPER postgres_fdw
OPTIONS ( host 'localhost', dbname 'postgres_fdw_db', port :'PORTNO');
GRANT USAGE ON FOREIGN SERVER s3_server TO :ROLE_4;
CREATE USER MAPPING FOR :ROLE_4 SERVER s3_server
OPTIONS ( user :'ROLE_4' , password :'ROLE_4_PASS');
ALTER USER MAPPING FOR :ROLE_4 SERVER s3_server
OPTIONS (ADD password_required 'false');
-- this is a stand-in for the OSM table
CREATE FOREIGN TABLE child_fdw_table
(timec timestamptz NOT NULL, acq_id bigint, value bigint)
SERVER s3_server OPTIONS ( schema_name 'public', table_name 'fdw_table');
GRANT SELECT ON child_fdw_table TO :ROLE_4;
--now attach foreign table as a chunk of the hypertable.
\c :TEST_DBNAME :ROLE_4;
CREATE TABLE ht_try(timec timestamptz NOT NULL, acq_id bigint, value bigint);
SELECT create_hypertable('ht_try', 'timec', chunk_time_interval => interval '1 day');
INSERT INTO ht_try VALUES ('2020-05-05 01:00', 222, 222);
SELECT * FROM child_fdw_table;
SELECT _timescaledb_internal.attach_osm_table_chunk('ht_try', 'child_fdw_table');
SELECT chunk_name, range_start, range_end
FROM timescaledb_information.chunks
WHERE hypertable_name = 'ht_try' ORDER BY 1;
SELECT * FROM ht_try ORDER BY 1;
-- TEST error have to be hypertable owner to attach a chunk to it
\c :TEST_DBNAME :ROLE_DEFAULT_PERM_USER
\set ON_ERROR_STOP 0
SELECT _timescaledb_internal.attach_osm_table_chunk('ht_try', 'child_fdw_table');
-- TEST error try to attach to non hypertable
CREATE TABLE non_ht (time bigint, temp float);
SELECT _timescaledb_internal.attach_osm_table_chunk('non_ht', 'child_fdw_table');
\set ON_ERROR_STOP 1