diff --git a/tsl/src/continuous_aggs/CMakeLists.txt b/tsl/src/continuous_aggs/CMakeLists.txt index 8b75a0362..72e852123 100644 --- a/tsl/src/continuous_aggs/CMakeLists.txt +++ b/tsl/src/continuous_aggs/CMakeLists.txt @@ -1,9 +1,12 @@ set(SOURCES + ${CMAKE_CURRENT_SOURCE_DIR}/common.c + ${CMAKE_CURRENT_SOURCE_DIR}/finalize.c ${CMAKE_CURRENT_SOURCE_DIR}/create.c ${CMAKE_CURRENT_SOURCE_DIR}/insert.c ${CMAKE_CURRENT_SOURCE_DIR}/materialize.c ${CMAKE_CURRENT_SOURCE_DIR}/options.c ${CMAKE_CURRENT_SOURCE_DIR}/refresh.c + ${CMAKE_CURRENT_SOURCE_DIR}/repair.c ${CMAKE_CURRENT_SOURCE_DIR}/invalidation.c ${CMAKE_CURRENT_SOURCE_DIR}/invalidation_threshold.c) target_sources(${TSL_LIBRARY_NAME} PRIVATE ${SOURCES}) diff --git a/tsl/src/continuous_aggs/README.md b/tsl/src/continuous_aggs/README.md index 78efecdf5..c7c291f1a 100644 --- a/tsl/src/continuous_aggs/README.md +++ b/tsl/src/continuous_aggs/README.md @@ -114,3 +114,20 @@ are cut against the given refresh window, leaving only invalidation entries that are outside the refresh window. Subsequently, if the refresh window does not match any invalidations, there is nothing to refresh either. + +## Distribution of functions across files +common.c +This file contains the functions common in all scenarios of creating a continuous aggregates. + +create.c +This file contains the functions that are directly responsible for the creation of the continuous aggregates, +like creating hypertable, catalog_entry, view, etc. + +finalize.c +This file contains the specific functions for the case when continous aggregates are created in old format. + +materialize.c +This file contains the functions directly dealing with the materialization of the continuous aggregates. + +repair.c +The repair and rebuilding related functions are put together in this file diff --git a/tsl/src/continuous_aggs/common.c b/tsl/src/continuous_aggs/common.c new file mode 100644 index 000000000..fce510dfe --- /dev/null +++ b/tsl/src/continuous_aggs/common.c @@ -0,0 +1,1384 @@ +/* + * This file and its contents are licensed under the Timescale License. + * Please see the included NOTICE for copyright information and + * LICENSE-TIMESCALE for a copy of the license. + */ + +#include "common.h" + +static Const *check_time_bucket_argument(Node *arg, char *position); +static void caggtimebucketinfo_init(CAggTimebucketInfo *src, int32 hypertable_id, + Oid hypertable_oid, AttrNumber hypertable_partition_colno, + Oid hypertable_partition_coltype, + int64 hypertable_partition_col_interval, + int32 parent_mat_hypertable_id); +static void caggtimebucket_validate(CAggTimebucketInfo *tbinfo, List *groupClause, + List *targetList); +static bool cagg_agg_validate(Node *node, void *context); +static bool cagg_query_supported(const Query *query, StringInfo hint, StringInfo detail, + const bool finalized); +static Oid cagg_get_boundary_converter_funcoid(Oid typoid); +static FuncExpr *build_conversion_call(Oid type, FuncExpr *boundary); +static FuncExpr *build_boundary_call(int32 ht_id, Oid type); +static Const *cagg_boundary_make_lower_bound(Oid type); +static Node *build_union_query_quals(int32 ht_id, Oid partcoltype, Oid opno, int varno, + AttrNumber attno); +static RangeTblEntry *makeRangeTblEntry(Query *subquery, const char *aliasname); + +#define INTERNAL_TO_DATE_FUNCTION "to_date" +#define INTERNAL_TO_TSTZ_FUNCTION "to_timestamp" +#define INTERNAL_TO_TS_FUNCTION "to_timestamp_without_timezone" +#define BOUNDARY_FUNCTION "cagg_watermark" + +static Const * +check_time_bucket_argument(Node *arg, char *position) +{ + if (IsA(arg, NamedArgExpr)) + arg = (Node *) castNode(NamedArgExpr, arg)->arg; + + Node *expr = eval_const_expressions(NULL, arg); + + if (!IsA(expr, Const)) + ereport(ERROR, + (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), + errmsg("only immutable expressions allowed in time bucket function"), + errhint("Use an immutable expression as %s argument to the time bucket function.", + position))); + + return castNode(Const, expr); +} + +/* + * Initialize caggtimebucket. + */ +static void +caggtimebucketinfo_init(CAggTimebucketInfo *src, int32 hypertable_id, Oid hypertable_oid, + AttrNumber hypertable_partition_colno, Oid hypertable_partition_coltype, + int64 hypertable_partition_col_interval, int32 parent_mat_hypertable_id) +{ + src->htid = hypertable_id; + src->parent_mat_hypertable_id = parent_mat_hypertable_id; + src->htoid = hypertable_oid; + src->htpartcolno = hypertable_partition_colno; + src->htpartcoltype = hypertable_partition_coltype; + src->htpartcol_interval_len = hypertable_partition_col_interval; + src->bucket_width = 0; /* invalid value */ + src->bucket_width_type = InvalidOid; /* invalid oid */ + src->interval = NULL; /* not specified by default */ + src->timezone = NULL; /* not specified by default */ + TIMESTAMP_NOBEGIN(src->origin); /* origin is not specified by default */ +} + +/* + * Check if the supplied OID belongs to a valid bucket function + * for continuous aggregates. + */ +bool +function_allowed_in_cagg_definition(Oid funcid) +{ + FuncInfo *finfo = ts_func_cache_get_bucketing_func(funcid); + if (finfo == NULL) + return false; + + return finfo->allowed_in_cagg_definition; +} + +/* + * Return Oid for a schema-qualified relation. + */ +Oid +relation_oid(Name schema, Name name) +{ + return get_relname_relid(NameStr(*name), get_namespace_oid(NameStr(*schema), false)); +} + +/* + * When a view is created (StoreViewQuery), 2 dummy rtable entries corresponding to "old" and + * "new" are prepended to the rtable list. We remove these and adjust the varnos to recreate + * the user or direct view query. + */ +void +RemoveRangeTableEntries(Query *query) +{ + List *rtable = query->rtable; + Assert(list_length(rtable) >= 3); + rtable = list_delete_first(rtable); + query->rtable = list_delete_first(rtable); + OffsetVarNodes((Node *) query, -2, 0); + Assert(list_length(query->rtable) >= 1); +} + +/* + * Extract the final view from the UNION ALL query. + * + * q1 is the query on the materialization hypertable with the finalize call + * q2 is the query on the raw hypertable which was supplied in the inital CREATE VIEW statement + * returns q1 from: + * SELECT * from ( SELECT * from q1 where + * UNION ALL + * SELECT * from q2 where existing_qual and + * where coale_qual is: time < ----> (or >= ) + * COALESCE(_timescaledb_internal.to_timestamp(_timescaledb_internal.cagg_watermark( )), + * '-infinity'::timestamp with time zone) + * The WHERE clause of the final view is removed. + */ +Query * +destroy_union_query(Query *q) +{ + Assert(q->commandType == CMD_SELECT && + ((SetOperationStmt *) q->setOperations)->op == SETOP_UNION && + ((SetOperationStmt *) q->setOperations)->all == true); + + /* Get RTE of the left-hand side of UNION ALL. */ + RangeTblEntry *rte = linitial(q->rtable); + Assert(rte->rtekind == RTE_SUBQUERY); + + Query *query = copyObject(rte->subquery); + + /* Delete the WHERE clause from the final view. */ + query->jointree->quals = NULL; + + return query; +} + +/* + * Check if the group-by clauses has exactly 1 time_bucket(.., ) where + * is the hypertable's partitioning column and other invariants. Then fill + * the `bucket_width` and other fields of `tbinfo`. + */ +static void +caggtimebucket_validate(CAggTimebucketInfo *tbinfo, List *groupClause, List *targetList) +{ + ListCell *l; + bool found = false; + bool custom_origin = false; + Const *const_arg; + + /* Make sure tbinfo was initialized. This assumption is used below. */ + Assert(tbinfo->bucket_width == 0); + Assert(tbinfo->timezone == NULL); + Assert(TIMESTAMP_NOT_FINITE(tbinfo->origin)); + + foreach (l, groupClause) + { + SortGroupClause *sgc = (SortGroupClause *) lfirst(l); + TargetEntry *tle = get_sortgroupclause_tle(sgc, targetList); + + if (IsA(tle->expr, FuncExpr)) + { + FuncExpr *fe = ((FuncExpr *) tle->expr); + Node *width_arg; + Node *col_arg; + + if (!function_allowed_in_cagg_definition(fe->funcid)) + continue; + + /* + * Offset variants of time_bucket functions are not + * supported at the moment. + */ + if (list_length(fe->args) >= 5 || + (list_length(fe->args) == 4 && exprType(lfourth(fe->args)) == INTERVALOID)) + continue; + + if (found) + ereport(ERROR, + (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), + errmsg("continuous aggregate view cannot contain" + " multiple time bucket functions"))); + else + found = true; + + tbinfo->bucket_func = fe; + + /* Only column allowed : time_bucket('1day', ) */ + col_arg = lsecond(fe->args); + /* Could be a named argument */ + if (IsA(col_arg, NamedArgExpr)) + col_arg = (Node *) castNode(NamedArgExpr, col_arg)->arg; + + if (!(IsA(col_arg, Var)) || ((Var *) col_arg)->varattno != tbinfo->htpartcolno) + ereport(ERROR, + (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), + errmsg( + "time bucket function must reference a hypertable dimension column"))); + + if (list_length(fe->args) >= 3) + { + Const *arg = check_time_bucket_argument(lthird(fe->args), "third"); + if (exprType((Node *) arg) == TEXTOID) + { + const char *tz_name = TextDatumGetCString(arg->constvalue); + if (!ts_is_valid_timezone_name(tz_name)) + { + ereport(ERROR, + (errcode(ERRCODE_INVALID_PARAMETER_VALUE), + errmsg("invalid timezone name \"%s\"", tz_name))); + } + + tbinfo->timezone = tz_name; + tbinfo->bucket_width = BUCKET_WIDTH_VARIABLE; + } + } + + if (list_length(fe->args) >= 4) + { + /* origin */ + Const *arg = check_time_bucket_argument(lfourth(fe->args), "fourth"); + if (exprType((Node *) arg) == TEXTOID) + { + const char *tz_name = TextDatumGetCString(arg->constvalue); + if (!ts_is_valid_timezone_name(tz_name)) + { + ereport(ERROR, + (errcode(ERRCODE_INVALID_PARAMETER_VALUE), + errmsg("invalid timezone name \"%s\"", tz_name))); + } + + tbinfo->timezone = tz_name; + tbinfo->bucket_width = BUCKET_WIDTH_VARIABLE; + } + } + + /* Check for custom origin. */ + switch (exprType(col_arg)) + { + case DATEOID: + /* Origin is always 3rd arg for date variants. */ + if (list_length(fe->args) == 3) + { + Node *arg = lthird(fe->args); + custom_origin = true; + /* this function also takes care of named arguments */ + const_arg = check_time_bucket_argument(arg, "third"); + tbinfo->origin = DatumGetTimestamp( + DirectFunctionCall1(date_timestamp, const_arg->constvalue)); + } + break; + case TIMESTAMPOID: + /* Origin is always 3rd arg for timestamp variants. */ + if (list_length(fe->args) == 3) + { + Node *arg = lthird(fe->args); + custom_origin = true; + const_arg = check_time_bucket_argument(arg, "third"); + tbinfo->origin = DatumGetTimestamp(const_arg->constvalue); + } + break; + case TIMESTAMPTZOID: + /* Origin can be 3rd or 4th arg for timestamptz variants. */ + if (list_length(fe->args) >= 3 && exprType(lthird(fe->args)) == TIMESTAMPTZOID) + { + custom_origin = true; + tbinfo->origin = + DatumGetTimestampTz(castNode(Const, lthird(fe->args))->constvalue); + } + else if (list_length(fe->args) >= 4 && + exprType(lfourth(fe->args)) == TIMESTAMPTZOID) + { + custom_origin = true; + if (IsA(lfourth(fe->args), Const)) + { + tbinfo->origin = + DatumGetTimestampTz(castNode(Const, lfourth(fe->args))->constvalue); + } + /* could happen in a statement like time_bucket('1h', .., 'utc', origin => + * ...) */ + else if (IsA(lfourth(fe->args), NamedArgExpr)) + { + Const *constval = + check_time_bucket_argument(lfourth(fe->args), "fourth"); + + tbinfo->origin = DatumGetTimestampTz(constval->constvalue); + } + } + } + if (custom_origin && TIMESTAMP_NOT_FINITE(tbinfo->origin)) + { + ereport(ERROR, + (errcode(ERRCODE_INVALID_PARAMETER_VALUE), + errmsg("invalid origin value: infinity"))); + } + + /* + * We constify width expression here so any immutable expression will be allowed. + * Otherwise it would make it harder to create caggs for hypertables with e.g. int8 + * partitioning column as int constants default to int4 and so expression would + * have a cast and not be a Const. + */ + width_arg = linitial(fe->args); + + if (IsA(width_arg, NamedArgExpr)) + width_arg = (Node *) castNode(NamedArgExpr, width_arg)->arg; + + width_arg = eval_const_expressions(NULL, width_arg); + if (IsA(width_arg, Const)) + { + Const *width = castNode(Const, width_arg); + tbinfo->bucket_width_type = width->consttype; + + if (width->consttype == INTERVALOID) + { + tbinfo->interval = DatumGetIntervalP(width->constvalue); + if (tbinfo->interval->month != 0) + tbinfo->bucket_width = BUCKET_WIDTH_VARIABLE; + } + + if (tbinfo->bucket_width != BUCKET_WIDTH_VARIABLE) + { + /* The bucket size is fixed. */ + tbinfo->bucket_width = + ts_interval_value_to_internal(width->constvalue, width->consttype); + } + } + else + ereport(ERROR, + (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), + errmsg("only immutable expressions allowed in time bucket function"), + errhint("Use an immutable expression as first argument to the time bucket " + "function."))); + + if (tbinfo->interval && tbinfo->interval->month) + { + tbinfo->bucket_width = BUCKET_WIDTH_VARIABLE; + } + } + } + + if (tbinfo->bucket_width == BUCKET_WIDTH_VARIABLE) + { + /* Variable-sized buckets can be used only with intervals. */ + Assert(tbinfo->interval != NULL); + + if ((tbinfo->interval->month != 0) && + ((tbinfo->interval->day != 0) || (tbinfo->interval->time != 0))) + { + ereport(ERROR, + (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), + errmsg("invalid interval specified"), + errhint("Use either months or days and hours, but not months, days and hours " + "together"))); + } + } + + if (!found) + elog(ERROR, "continuous aggregate view must include a valid time bucket function"); +} + +static bool +cagg_agg_validate(Node *node, void *context) +{ + if (node == NULL) + return false; + + if (IsA(node, Aggref)) + { + Aggref *agg = (Aggref *) node; + HeapTuple aggtuple; + Form_pg_aggregate aggform; + if (agg->aggorder || agg->aggdistinct || agg->aggfilter) + { + ereport(ERROR, + (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), + errmsg("aggregates with FILTER / DISTINCT / ORDER BY are not supported"))); + } + /* Fetch the pg_aggregate row. */ + aggtuple = SearchSysCache1(AGGFNOID, agg->aggfnoid); + if (!HeapTupleIsValid(aggtuple)) + elog(ERROR, "cache lookup failed for aggregate %u", agg->aggfnoid); + aggform = (Form_pg_aggregate) GETSTRUCT(aggtuple); + if (aggform->aggkind != 'n') + { + ReleaseSysCache(aggtuple); + ereport(ERROR, + (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), + errmsg("ordered set/hypothetical aggregates are not supported"))); + } + if (!OidIsValid(aggform->aggcombinefn) || + (aggform->aggtranstype == INTERNALOID && !OidIsValid(aggform->aggdeserialfn))) + { + ReleaseSysCache(aggtuple); + ereport(ERROR, + (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), + errmsg("aggregates which are not parallelizable are not supported"))); + } + ReleaseSysCache(aggtuple); + + return false; + } + return expression_tree_walker(node, cagg_agg_validate, context); +} + +/* + * Check query and extract error details and error hints. + * + * Returns: + * True if the query is supported, false otherwise with hints and errors + * added. + */ +static bool +cagg_query_supported(const Query *query, StringInfo hint, StringInfo detail, const bool finalized) +{ +/* + * For now deprecate partial aggregates on release builds only. + * Once migration tests are made compatible with PG15 enable deprecation + * on debug builds as well. + */ +#ifndef DEBUG +#if PG15_GE + if (!finalized) + { + /* continuous aggregates with old format will not be allowed */ + appendStringInfoString(detail, + "Continuous Aggregates with partials is not supported anymore."); + appendStringInfoString(hint, + "Define the Continuous Aggregate with \"finalized\" parameter set " + "to true."); + return false; + } +#endif +#endif + if (!query->jointree->fromlist) + { + appendStringInfoString(hint, "FROM clause missing in the query"); + return false; + } + if (query->commandType != CMD_SELECT) + { + appendStringInfoString(hint, "Use a SELECT query in the continuous aggregate view."); + return false; + } + + if (query->hasWindowFuncs) + { + appendStringInfoString(detail, + "Window functions are not supported by continuous aggregates."); + return false; + } + + if (query->hasDistinctOn || query->distinctClause) + { + appendStringInfoString(detail, + "DISTINCT / DISTINCT ON queries are not supported by continuous " + "aggregates."); + return false; + } + + if (query->limitOffset || query->limitCount) + { + appendStringInfoString(detail, + "LIMIT and LIMIT OFFSET are not supported in queries defining " + "continuous aggregates."); + appendStringInfoString(hint, + "Use LIMIT and LIMIT OFFSET in SELECTS from the continuous " + "aggregate view instead."); + return false; + } + + if (query->sortClause && !finalized) + { + appendStringInfoString(detail, + "ORDER BY is not supported in queries defining continuous " + "aggregates."); + appendStringInfoString(hint, + "Use ORDER BY clauses in SELECTS from the continuous aggregate view " + "instead."); + return false; + } + + if (query->hasRecursive || query->hasSubLinks || query->hasTargetSRFs || query->cteList) + { + appendStringInfoString(detail, + "CTEs, subqueries and set-returning functions are not supported by " + "continuous aggregates."); + return false; + } + + if (query->hasForUpdate || query->hasModifyingCTE) + { + appendStringInfoString(detail, + "Data modification is not allowed in continuous aggregate view " + "definitions."); + return false; + } + + if (query->hasRowSecurity) + { + appendStringInfoString(detail, + "Row level security is not supported by continuous aggregate " + "views."); + return false; + } + + if (query->groupingSets) + { + appendStringInfoString(detail, + "GROUP BY GROUPING SETS, ROLLUP and CUBE are not supported by " + "continuous aggregates"); + appendStringInfoString(hint, + "Define multiple continuous aggregates with different grouping " + "levels."); + return false; + } + + if (query->setOperations) + { + appendStringInfoString(detail, + "UNION, EXCEPT & INTERSECT are not supported by continuous " + "aggregates"); + return false; + } + + if (!query->groupClause) + { + /* + * Query can have aggregate without group by , so look + * for groupClause. + */ + appendStringInfoString(hint, + "Include at least one aggregate function" + " and a GROUP BY clause with time bucket."); + return false; + } + + return true; /* Query was OK and is supported. */ +} + +static Datum +get_bucket_width_datum(CAggTimebucketInfo bucket_info) +{ + Datum width = (Datum) 0; + + switch (bucket_info.bucket_width_type) + { + case INT8OID: + case INT4OID: + case INT2OID: + width = ts_internal_to_interval_value(bucket_info.bucket_width, + bucket_info.bucket_width_type); + break; + case INTERVALOID: + width = IntervalPGetDatum(bucket_info.interval); + break; + default: + Assert(false); + } + + return width; +} + +static int64 +get_bucket_width(CAggTimebucketInfo bucket_info) +{ + int64 width = 0; + + /* Calculate the width. */ + switch (bucket_info.bucket_width_type) + { + case INT8OID: + case INT4OID: + case INT2OID: + width = bucket_info.bucket_width; + break; + case INTERVALOID: + { + /* + * epoch will treat year as 365.25 days. This leads to the unexpected + * result that year is not multiple of day or month, which is perceived + * as a bug. For that reason, we treat all months as 30 days regardless of year + */ + if (bucket_info.interval->month && !bucket_info.interval->day && + !bucket_info.interval->time) + { + bucket_info.interval->day = bucket_info.interval->month * DAYS_PER_MONTH; + bucket_info.interval->month = 0; + } + + /* Convert Interval to int64 */ + width = + ts_interval_value_to_internal(IntervalPGetDatum(bucket_info.interval), INTERVALOID); + break; + } + default: + Assert(false); + } + + return width; +} + +CAggTimebucketInfo +cagg_validate_query(const Query *query, const bool finalized, const char *cagg_schema, + const char *cagg_name) +{ + CAggTimebucketInfo bucket_info = { 0 }, bucket_info_parent; + Cache *hcache; + Hypertable *ht = NULL, *ht_parent = NULL; + RangeTblRef *rtref = NULL, *rtref_other = NULL; + RangeTblEntry *rte = NULL, *rte_other = NULL; + JoinType jointype = JOIN_FULL; + OpExpr *op = NULL; + List *fromList = NIL; + StringInfo hint = makeStringInfo(); + StringInfo detail = makeStringInfo(); + bool is_hierarchical = false; + Query *prev_query = NULL; + ContinuousAgg *cagg_parent = NULL; + Oid normal_table_id = InvalidOid; + + if (!cagg_query_supported(query, hint, detail, finalized)) + { + ereport(ERROR, + (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), + errmsg("invalid continuous aggregate query"), + hint->len > 0 ? errhint("%s", hint->data) : 0, + detail->len > 0 ? errdetail("%s", detail->data) : 0)); + } + + /* Finalized cagg doesn't have those restrictions anymore. */ + if (!finalized) + { + /* Validate aggregates allowed. */ + cagg_agg_validate((Node *) query->targetList, NULL); + cagg_agg_validate((Node *) query->havingQual, NULL); + } + /* Check if there are only two tables in the from list. */ + fromList = query->jointree->fromlist; + if (list_length(fromList) > CONTINUOUS_AGG_MAX_JOIN_RELATIONS) + { + ereport(ERROR, + (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), + errmsg("only two tables with one hypertable and one normal table" + "are allowed in continuous aggregate view"))); + } + /* Extra checks for joins in Caggs. */ + if (list_length(fromList) == CONTINUOUS_AGG_MAX_JOIN_RELATIONS || + !IsA(linitial(query->jointree->fromlist), RangeTblRef)) + { + /* Using old format caggs is not supported */ + if (!finalized) + ereport(ERROR, + (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), + errmsg("old format of continuous aggregate is not supported with joins"), + errhint("Set timescaledb.finalized to TRUE."))); + + if (list_length(fromList) == CONTINUOUS_AGG_MAX_JOIN_RELATIONS) + { + if (!IsA(linitial(fromList), RangeTblRef) || !IsA(lsecond(fromList), RangeTblRef)) + ereport(ERROR, + (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), + errmsg("invalid continuous aggregate view"), + errdetail( + "From clause can only have one hypertable and one normal table."))); + + rtref = linitial_node(RangeTblRef, query->jointree->fromlist); + rte = list_nth(query->rtable, rtref->rtindex - 1); + rtref_other = lsecond_node(RangeTblRef, query->jointree->fromlist); + rte_other = list_nth(query->rtable, rtref_other->rtindex - 1); + jointype = rte->jointype || rte_other->jointype; + + if (query->jointree->quals != NULL && IsA(query->jointree->quals, OpExpr)) + op = (OpExpr *) query->jointree->quals; + } + else + { + ListCell *l; + foreach (l, query->jointree->fromlist) + { + Node *jtnode = (Node *) lfirst(l); + JoinExpr *join = NULL; + if (IsA(jtnode, JoinExpr)) + { + join = castNode(JoinExpr, jtnode); +#if PG13_LT + if (join->usingClause != NULL) + ereport(ERROR, + (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), + errmsg("invalid continuous aggregate view"), + errdetail( + "Joins with USING clause in continuous aggregate definition" + " work for Postgres versions 13 and above."))); +#endif + jointype = join->jointype; + op = (OpExpr *) join->quals; + rte = list_nth(query->rtable, ((RangeTblRef *) join->larg)->rtindex - 1); + rte_other = list_nth(query->rtable, ((RangeTblRef *) join->rarg)->rtindex - 1); + if (rte->subquery != NULL || rte_other->subquery != NULL) + ereport(ERROR, + (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), + errmsg("invalid continuous aggregate view"), + errdetail("Sub-queries are not supported in FROM clause."))); + RangeTblEntry *jrte = rt_fetch(join->rtindex, query->rtable); + if (jrte->joinaliasvars == NIL) + ereport(ERROR, + (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), + errmsg("invalid continuous aggregate view"))); + } + } + } + + /* + * Error out if there is aynthing else than one normal table and one hypertable + * in the from clause, e.g. sub-query, lateral, two hypertables, etc. + */ + if (rte->lateral || rte_other->lateral) + ereport(ERROR, + (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), + errmsg("invalid continuous aggregate view"), + errdetail("Lateral joins are not supported in FROM clause."))); + if ((rte->relkind == RELKIND_VIEW && ts_is_hypertable(rte_other->relid)) || + (rte_other->relkind == RELKIND_VIEW && ts_is_hypertable(rte->relid))) + ereport(ERROR, + (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), + errmsg("invalid continuous aggregate view"), + errdetail("Views are not supported in FROM clause."))); + if (rte->relkind != RELKIND_VIEW && rte_other->relkind != RELKIND_VIEW && + (ts_is_hypertable(rte->relid) == ts_is_hypertable(rte_other->relid))) + ereport(ERROR, + (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), + errmsg("invalid continuous aggregate view"), + errdetail("Multiple hypertables or normal tables are not supported in FROM " + "clause."))); + + /* Only inner joins are allowed. */ + if (jointype != JOIN_INNER) + ereport(ERROR, + (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), + errmsg("only inner joins are supported in continuous aggregates"))); + + /* Only equality conditions are permitted on joins. */ + if (op && IsA(op, OpExpr) && + list_length(castNode(OpExpr, op)->args) == CONTINUOUS_AGG_MAX_JOIN_RELATIONS) + { + Oid left_type = exprType(linitial(op->args)); + Oid right_type = exprType(lsecond(op->args)); + if (!ts_is_equality_operator(op->opno, left_type, right_type)) + ereport(ERROR, + (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), + errmsg("invalid continuous aggregate view"), + errdetail( + "Only equality conditions are supported in continuous aggregates."))); + } + else + ereport(ERROR, + (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), + errmsg("invalid continuous aggregate view"), + errdetail("Unsupported expression in join clause."), + errhint("Only equality conditions are supported in continuous aggregates."))); + /* + * Record the table oid of the normal table. This is required so + * that we know which one is hypertable to carry out the related + * processing in later parts of code. + */ + if (rte->relkind == RELKIND_VIEW) + normal_table_id = rte_other->relid; + else if (rte_other->relkind == RELKIND_VIEW) + normal_table_id = rte->relid; + else + normal_table_id = ts_is_hypertable(rte->relid) ? rte_other->relid : rte->relid; + if (normal_table_id == rte->relid) + rte = rte_other; + } + else + { + /* Check if we have a hypertable in the FROM clause. */ + rtref = linitial_node(RangeTblRef, query->jointree->fromlist); + rte = list_nth(query->rtable, rtref->rtindex - 1); + } + /* FROM only sets rte->inh to false. */ + if (rte->rtekind != RTE_JOIN) + { + if ((rte->relkind != RELKIND_RELATION && rte->relkind != RELKIND_VIEW) || + rte->tablesample || rte->inh == false) + ereport(ERROR, + (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), + errmsg("invalid continuous aggregate view"))); + } + + if (rte->relkind == RELKIND_RELATION || rte->relkind == RELKIND_VIEW) + { + const Dimension *part_dimension = NULL; + int32 parent_mat_hypertable_id = INVALID_HYPERTABLE_ID; + + if (rte->relkind == RELKIND_RELATION) + ht = ts_hypertable_cache_get_cache_and_entry(rte->relid, CACHE_FLAG_NONE, &hcache); + else + { + cagg_parent = ts_continuous_agg_find_by_relid(rte->relid); + + if (!cagg_parent) + { + ereport(ERROR, + (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), + errmsg("invalid continuous aggregate query"), + errhint("Continuous aggregate needs to query hypertable or another " + "continuous aggregate."))); + } + + if (!ContinuousAggIsFinalized(cagg_parent)) + { + ereport(ERROR, + (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), + errmsg("old format of continuous aggregate is not supported"), + errhint("Run \"CALL cagg_migrate('%s.%s');\" to migrate to the new " + "format.", + NameStr(cagg_parent->data.user_view_schema), + NameStr(cagg_parent->data.user_view_name)))); + } + + parent_mat_hypertable_id = cagg_parent->data.mat_hypertable_id; + hcache = ts_hypertable_cache_pin(); + ht = ts_hypertable_cache_get_entry_by_id(hcache, cagg_parent->data.mat_hypertable_id); + + /* If parent cagg is hierarchical then we should get the matht otherwise the rawht. */ + if (ContinuousAggIsHierarchical(cagg_parent)) + ht_parent = + ts_hypertable_cache_get_entry_by_id(hcache, + cagg_parent->data.mat_hypertable_id); + else + ht_parent = + ts_hypertable_cache_get_entry_by_id(hcache, + cagg_parent->data.raw_hypertable_id); + + /* Get the querydef for the source cagg. */ + is_hierarchical = true; + prev_query = ts_continuous_agg_get_query(cagg_parent); + } + + if (TS_HYPERTABLE_IS_INTERNAL_COMPRESSION_TABLE(ht)) + ereport(ERROR, + (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), + errmsg("hypertable is an internal compressed hypertable"))); + + if (rte->relkind == RELKIND_RELATION) + { + ContinuousAggHypertableStatus status = ts_continuous_agg_hypertable_status(ht->fd.id); + + /* Prevent create a CAGG over an existing materialization hypertable. */ + if (status == HypertableIsMaterialization || + status == HypertableIsMaterializationAndRaw) + { + const ContinuousAgg *cagg = ts_continuous_agg_find_by_mat_hypertable_id(ht->fd.id); + Assert(cagg != NULL); + + ereport(ERROR, + (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), + errmsg("hypertable is a continuous aggregate materialization table"), + errdetail("Materialization hypertable \"%s.%s\".", + NameStr(ht->fd.schema_name), + NameStr(ht->fd.table_name)), + errhint("Do you want to use continuous aggregate \"%s.%s\" instead?", + NameStr(cagg->data.user_view_schema), + NameStr(cagg->data.user_view_name)))); + } + } + + /* Get primary partitioning column information. */ + part_dimension = hyperspace_get_open_dimension(ht->space, 0); + + /* + * NOTE: if we ever allow custom partitioning functions we'll need to + * change part_dimension->fd.column_type to partitioning_type + * below, along with any other fallout. + */ + if (part_dimension->partitioning != NULL) + ereport(ERROR, + (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), + errmsg("custom partitioning functions not supported" + " with continuous aggregates"))); + + if (IS_INTEGER_TYPE(ts_dimension_get_partition_type(part_dimension)) && + rte->relkind == RELKIND_RELATION) + { + const char *funcschema = NameStr(part_dimension->fd.integer_now_func_schema); + const char *funcname = NameStr(part_dimension->fd.integer_now_func); + + if (strlen(funcschema) == 0 || strlen(funcname) == 0) + ereport(ERROR, + (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), + errmsg("custom time function required on hypertable \"%s\"", + get_rel_name(ht->main_table_relid)), + errdetail("An integer-based hypertable requires a custom time function to " + "support continuous aggregates."), + errhint("Set a custom time function on the hypertable."))); + } + + caggtimebucketinfo_init(&bucket_info, + ht->fd.id, + ht->main_table_relid, + part_dimension->column_attno, + part_dimension->fd.column_type, + part_dimension->fd.interval_length, + parent_mat_hypertable_id); + + if (is_hierarchical) + { + const Dimension *part_dimension_parent = + hyperspace_get_open_dimension(ht_parent->space, 0); + + caggtimebucketinfo_init(&bucket_info_parent, + ht_parent->fd.id, + ht_parent->main_table_relid, + part_dimension_parent->column_attno, + part_dimension_parent->fd.column_type, + part_dimension_parent->fd.interval_length, + INVALID_HYPERTABLE_ID); + } + + ts_cache_release(hcache); + + /* + * We need a GROUP By clause with time_bucket on the partitioning + * column of the hypertable + */ + Assert(query->groupClause); + caggtimebucket_validate(&bucket_info, query->groupClause, query->targetList); + } + + /* Check row security settings for the table. */ + if (ts_has_row_security(rte->relid)) + ereport(ERROR, + (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), + errmsg("cannot create continuous aggregate on hypertable with row security"))); + + /* hierarchical cagg validations */ + if (is_hierarchical) + { + int64 bucket_width = 0, bucket_width_parent = 0; + bool is_greater_or_equal_than_parent = true, is_multiple_of_parent = true; + + Assert(prev_query->groupClause); + caggtimebucket_validate(&bucket_info_parent, + prev_query->groupClause, + prev_query->targetList); + + /* Cannot create cagg with fixed bucket on top of variable bucket. */ + if ((bucket_info_parent.bucket_width == BUCKET_WIDTH_VARIABLE && + bucket_info.bucket_width != BUCKET_WIDTH_VARIABLE)) + { + ereport(ERROR, + (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), + errmsg("cannot create continuous aggregate with fixed-width bucket on top of " + "one using variable-width bucket"), + errdetail("Continuous aggregate with a fixed time bucket width (e.g. 61 days) " + "cannot be created on top of one using variable time bucket width " + "(e.g. 1 month).\n" + "The variance can lead to the fixed width one not being a multiple " + "of the variable width one."))); + } + + /* Get bucket widths for validation. */ + bucket_width = get_bucket_width(bucket_info); + bucket_width_parent = get_bucket_width(bucket_info_parent); + + Assert(bucket_width != 0); + Assert(bucket_width_parent != 0); + + /* Check if the current bucket is greater or equal than the parent. */ + is_greater_or_equal_than_parent = (bucket_width >= bucket_width_parent); + + /* Check if buckets are multiple. */ + if (bucket_width_parent != 0) + { + if (bucket_width_parent > bucket_width && bucket_width != 0) + is_multiple_of_parent = ((bucket_width_parent % bucket_width) == 0); + else + is_multiple_of_parent = ((bucket_width % bucket_width_parent) == 0); + } + + /* Proceed with validation errors. */ + if (!is_greater_or_equal_than_parent || !is_multiple_of_parent) + { + Datum width, width_parent; + Oid outfuncid = InvalidOid; + bool isvarlena; + char *width_out, *width_out_parent; + char *message = NULL; + + getTypeOutputInfo(bucket_info.bucket_width_type, &outfuncid, &isvarlena); + width = get_bucket_width_datum(bucket_info); + width_out = DatumGetCString(OidFunctionCall1(outfuncid, width)); + + getTypeOutputInfo(bucket_info_parent.bucket_width_type, &outfuncid, &isvarlena); + width_parent = get_bucket_width_datum(bucket_info_parent); + width_out_parent = DatumGetCString(OidFunctionCall1(outfuncid, width_parent)); + + /* New bucket should be multiple of the parent. */ + if (!is_multiple_of_parent) + message = "multiple of"; + + /* New bucket should be greater than the parent. */ + if (!is_greater_or_equal_than_parent) + message = "greater or equal than"; + + ereport(ERROR, + (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), + errmsg("cannot create continuous aggregate with incompatible bucket width"), + errdetail("Time bucket width of \"%s.%s\" [%s] should be %s the time " + "bucket width of \"%s.%s\" [%s].", + cagg_schema, + cagg_name, + width_out, + message, + NameStr(cagg_parent->data.user_view_schema), + NameStr(cagg_parent->data.user_view_name), + width_out_parent))); + } + } + + return bucket_info; +} + +/* + * Get oid of function to convert from our internal representation + * to postgres representation. + */ +static Oid +cagg_get_boundary_converter_funcoid(Oid typoid) +{ + char *function_name; + Oid argtyp[] = { INT8OID }; + + switch (typoid) + { + case DATEOID: + function_name = INTERNAL_TO_DATE_FUNCTION; + break; + case TIMESTAMPOID: + function_name = INTERNAL_TO_TS_FUNCTION; + break; + case TIMESTAMPTZOID: + function_name = INTERNAL_TO_TSTZ_FUNCTION; + break; + default: + /* + * This should never be reached and unsupported datatypes + * should be caught at much earlier stages. + */ + ereport(ERROR, + (errcode(ERRCODE_TS_INTERNAL_ERROR), + errmsg("no converter function defined for datatype: %s", + format_type_be(typoid)))); + pg_unreachable(); + } + + List *func_name = list_make2(makeString(INTERNAL_SCHEMA_NAME), makeString(function_name)); + Oid converter_oid = LookupFuncName(func_name, lengthof(argtyp), argtyp, false); + + Assert(OidIsValid(converter_oid)); + + return converter_oid; +} + +static FuncExpr * +build_conversion_call(Oid type, FuncExpr *boundary) +{ + /* + * If the partitioning column type is not integer we need to convert + * to proper representation. + */ + switch (type) + { + case INT2OID: + case INT4OID: + { + /* Since the boundary function returns int8 we need to cast to proper type here. */ + Oid cast_oid = ts_get_cast_func(INT8OID, type); + + return makeFuncExpr(cast_oid, + type, + list_make1(boundary), + InvalidOid, + InvalidOid, + COERCE_IMPLICIT_CAST); + } + case INT8OID: + /* Nothing to do for int8. */ + return boundary; + case DATEOID: + case TIMESTAMPOID: + case TIMESTAMPTZOID: + { + /* + * date/timestamp/timestamptz need to be converted since + * we store them differently from postgres format. + */ + Oid converter_oid = cagg_get_boundary_converter_funcoid(type); + return makeFuncExpr(converter_oid, + type, + list_make1(boundary), + InvalidOid, + InvalidOid, + COERCE_EXPLICIT_CALL); + } + + default: + /* + * All valid types should be handled above, this should + * never be reached and error handling at earlier stages + * should catch this. + */ + ereport(ERROR, + (errcode(ERRCODE_TS_INTERNAL_ERROR), + errmsg("unsupported datatype for continuous aggregates: %s", + format_type_be(type)))); + pg_unreachable(); + } +} + +/* + * Build function call that returns boundary for a hypertable + * wrapped in type conversion calls when required. + */ +static FuncExpr * +build_boundary_call(int32 ht_id, Oid type) +{ + Oid argtyp[] = { INT4OID }; + FuncExpr *boundary; + + Oid boundary_func_oid = + LookupFuncName(list_make2(makeString(INTERNAL_SCHEMA_NAME), makeString(BOUNDARY_FUNCTION)), + lengthof(argtyp), + argtyp, + false); + List *func_args = + list_make1(makeConst(INT4OID, -1, InvalidOid, 4, Int32GetDatum(ht_id), false, true)); + + boundary = makeFuncExpr(boundary_func_oid, + INT8OID, + func_args, + InvalidOid, + InvalidOid, + COERCE_EXPLICIT_CALL); + + return build_conversion_call(type, boundary); +} + +/* + * Create Const of proper type for lower bound of watermark when + * watermark has not been set yet. + */ +static Const * +cagg_boundary_make_lower_bound(Oid type) +{ + Datum value; + int16 typlen; + bool typbyval; + + get_typlenbyval(type, &typlen, &typbyval); + value = ts_time_datum_get_nobegin_or_min(type); + + return makeConst(type, -1, InvalidOid, typlen, value, false, typbyval); +} + +static Node * +build_union_query_quals(int32 ht_id, Oid partcoltype, Oid opno, int varno, AttrNumber attno) +{ + Var *var = makeVar(varno, attno, partcoltype, -1, InvalidOid, InvalidOid); + FuncExpr *boundary = build_boundary_call(ht_id, partcoltype); + + CoalesceExpr *coalesce = makeNode(CoalesceExpr); + coalesce->coalescetype = partcoltype; + coalesce->coalescecollid = InvalidOid; + coalesce->args = list_make2(boundary, cagg_boundary_make_lower_bound(partcoltype)); + + return (Node *) make_opclause(opno, + BOOLOID, + false, + (Expr *) var, + (Expr *) coalesce, + InvalidOid, + InvalidOid); +} + +static RangeTblEntry * +makeRangeTblEntry(Query *query, const char *aliasname) +{ + RangeTblEntry *rte = makeNode(RangeTblEntry); + ListCell *lc; + + rte->rtekind = RTE_SUBQUERY; + rte->relid = InvalidOid; + rte->subquery = query; + rte->alias = makeAlias(aliasname, NIL); + rte->eref = copyObject(rte->alias); + + foreach (lc, query->targetList) + { + TargetEntry *tle = lfirst_node(TargetEntry, lc); + if (!tle->resjunk) + rte->eref->colnames = lappend(rte->eref->colnames, makeString(pstrdup(tle->resname))); + } + + rte->lateral = false; + rte->inh = false; /* never true for subqueries */ + rte->inFromCl = true; + + return rte; +} + +/* + * Build union query combining the materialized data with data from the raw data hypertable. + * + * q1 is the query on the materialization hypertable with the finalize call + * q2 is the query on the raw hypertable which was supplied in the inital CREATE VIEW statement + * returns a query as + * SELECT * from ( SELECT * from q1 where + * UNION ALL + * SELECT * from q2 where existing_qual and + * where coale_qual is: time < ----> (or >= ) + * COALESCE(_timescaledb_internal.to_timestamp(_timescaledb_internal.cagg_watermark( )), + * '-infinity'::timestamp with time zone) + * See build_union_quals for COALESCE clauses. + */ +Query * +build_union_query(CAggTimebucketInfo *tbinfo, int matpartcolno, Query *q1, Query *q2, + int materialize_htid) +{ + ListCell *lc1, *lc2; + List *col_types = NIL; + List *col_typmods = NIL; + List *col_collations = NIL; + List *tlist = NIL; + List *sortClause = NIL; + int varno; + Node *q2_quals = NULL; + + Assert(list_length(q1->targetList) <= list_length(q2->targetList)); + + q1 = copyObject(q1); + q2 = copyObject(q2); + + if (q1->sortClause) + sortClause = copyObject(q1->sortClause); + + TypeCacheEntry *tce = lookup_type_cache(tbinfo->htpartcoltype, TYPECACHE_LT_OPR); + + varno = list_length(q1->rtable); + q1->jointree->quals = build_union_query_quals(materialize_htid, + tbinfo->htpartcoltype, + tce->lt_opr, + varno, + matpartcolno); + /* + * If there is join in CAgg definition then adjust varno + * to get time column from the hypertable in the join. + */ + + /* + * In case of joins it is enough to check if the first node is not RangeTblRef, + * because the jointree has RangeTblRef as leaves and JoinExpr above them. + * So if JoinExpr is present, it is the first node. + * Other cases of join i.e. without explicit JOIN clause is confirmed + * by reading the length of rtable. + */ + if (list_length(q2->rtable) == CONTINUOUS_AGG_MAX_JOIN_RELATIONS || + !IsA(linitial(q2->jointree->fromlist), RangeTblRef)) + { + Oid normal_table_id = InvalidOid; + RangeTblEntry *rte = NULL; + RangeTblEntry *rte_other = NULL; + + if (list_length(q2->rtable) == CONTINUOUS_AGG_MAX_JOIN_RELATIONS) + { + RangeTblRef *rtref = linitial_node(RangeTblRef, q2->jointree->fromlist); + rte = list_nth(q2->rtable, rtref->rtindex - 1); + RangeTblRef *rtref_other = lsecond_node(RangeTblRef, q2->jointree->fromlist); + rte_other = list_nth(q2->rtable, rtref_other->rtindex - 1); + } + else if (!IsA(linitial(q2->jointree->fromlist), RangeTblRef)) + { + ListCell *l; + foreach (l, q2->jointree->fromlist) + { + Node *jtnode = (Node *) lfirst(l); + JoinExpr *join = NULL; + if (IsA(jtnode, JoinExpr)) + { + join = castNode(JoinExpr, jtnode); + rte = list_nth(q2->rtable, ((RangeTblRef *) join->larg)->rtindex - 1); + rte_other = list_nth(q2->rtable, ((RangeTblRef *) join->rarg)->rtindex - 1); + } + } + } + if (rte->relkind == RELKIND_VIEW) + normal_table_id = rte_other->relid; + else if (rte_other->relkind == RELKIND_VIEW) + normal_table_id = rte->relid; + else + normal_table_id = ts_is_hypertable(rte->relid) ? rte_other->relid : rte->relid; + if (normal_table_id == rte->relid) + varno = 2; + else + varno = 1; + } + else + varno = list_length(q2->rtable); + q2_quals = build_union_query_quals(materialize_htid, + tbinfo->htpartcoltype, + get_negator(tce->lt_opr), + varno, + tbinfo->htpartcolno); + q2->jointree->quals = make_and_qual(q2->jointree->quals, q2_quals); + + Query *query = makeNode(Query); + SetOperationStmt *setop = makeNode(SetOperationStmt); + RangeTblEntry *rte_q1 = makeRangeTblEntry(q1, "*SELECT* 1"); + RangeTblEntry *rte_q2 = makeRangeTblEntry(q2, "*SELECT* 2"); + RangeTblRef *ref_q1 = makeNode(RangeTblRef); + RangeTblRef *ref_q2 = makeNode(RangeTblRef); + + query->commandType = CMD_SELECT; + query->rtable = list_make2(rte_q1, rte_q2); + query->setOperations = (Node *) setop; + + setop->op = SETOP_UNION; + setop->all = true; + ref_q1->rtindex = 1; + ref_q2->rtindex = 2; + setop->larg = (Node *) ref_q1; + setop->rarg = (Node *) ref_q2; + + forboth (lc1, q1->targetList, lc2, q2->targetList) + { + TargetEntry *tle = lfirst_node(TargetEntry, lc1); + TargetEntry *tle2 = lfirst_node(TargetEntry, lc2); + TargetEntry *tle_union; + Var *expr; + + if (!tle->resjunk) + { + col_types = lappend_int(col_types, exprType((Node *) tle->expr)); + col_typmods = lappend_int(col_typmods, exprTypmod((Node *) tle->expr)); + col_collations = lappend_int(col_collations, exprCollation((Node *) tle->expr)); + + expr = makeVarFromTargetEntry(1, tle); + /* + * We need to use resname from q2 because that is the query from the + * initial CREATE VIEW statement so the VIEW can be updated in place. + */ + tle_union = makeTargetEntry((Expr *) copyObject(expr), + list_length(tlist) + 1, + tle2->resname, + false); + tle_union->resorigtbl = expr->varno; + tle_union->resorigcol = expr->varattno; + tle_union->ressortgroupref = tle->ressortgroupref; + + tlist = lappend(tlist, tle_union); + } + } + + query->targetList = tlist; + + if (sortClause) + { + query->sortClause = sortClause; + query->jointree = makeFromExpr(NIL, NULL); + } + + setop->colTypes = col_types; + setop->colTypmods = col_typmods; + setop->colCollations = col_collations; + + return query; +} diff --git a/tsl/src/continuous_aggs/common.h b/tsl/src/continuous_aggs/common.h new file mode 100644 index 000000000..ced99c07f --- /dev/null +++ b/tsl/src/continuous_aggs/common.h @@ -0,0 +1,135 @@ +/* + * This file and its contents are licensed under the Timescale License. + * Please see the included NOTICE for copyright information and + * LICENSE-TIMESCALE for a copy of the license. + */ +#ifndef TIMESCALEDB_TSL_CONTINUOUS_AGGS_COMMON_H +#define TIMESCALEDB_TSL_CONTINUOUS_AGGS_COMMON_H + +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "errors.h" +#include "func_cache.h" +#include "hypertable_cache.h" +#include "timezones.h" +#include "ts_catalog/catalog.h" +#include "ts_catalog/continuous_agg.h" + +#define TS_PARTIALFN "partialize_agg" +#define CONTINUOUS_AGG_MAX_JOIN_RELATIONS 2 +#define DEFAULT_MATPARTCOLUMN_NAME "time_partition_col" + +typedef struct FinalizeQueryInfo +{ + List *final_seltlist; /* select target list for finalize query */ + Node *final_havingqual; /* having qual for finalize query */ + Query *final_userquery; /* user query used to compute the finalize_query */ + bool finalized; /* finalized form? */ +} FinalizeQueryInfo; + +typedef struct MatTableColumnInfo +{ + List *matcollist; /* column defns for materialization tbl*/ + List *partial_seltlist; /* tlist entries for populating the materialization table columns */ + List *partial_grouplist; /* group clauses used for populating the materialization table */ + List *mat_groupcolname_list; /* names of columns that are populated by the group-by clause + correspond to the partial_grouplist. + time_bucket column is not included here: it is the + matpartcolname */ + int matpartcolno; /*index of partitioning column in matcollist */ + char *matpartcolname; /*name of the partition column */ +} MatTableColumnInfo; + +typedef struct CAggTimebucketInfo +{ + int32 htid; /* hypertable id */ + int32 parent_mat_hypertable_id; /* parent materialization hypertable id */ + Oid htoid; /* hypertable oid */ + AttrNumber htpartcolno; /* primary partitioning column of raw hypertable */ + /* This should also be the column used by time_bucket */ + Oid htpartcoltype; + int64 htpartcol_interval_len; /* interval length setting for primary partitioning column */ + int64 bucket_width; /* bucket_width of time_bucket, stores BUCKET_WIDTH_VARIABLE for + variable-sized buckets */ + Oid bucket_width_type; /* type of bucket_width */ + Interval *interval; /* stores the interval, NULL if not specified */ + const char *timezone; /* the name of the timezone, NULL if not specified */ + + FuncExpr *bucket_func; /* function call expr of the bucketing function */ + /* + * Custom origin value stored as UTC timestamp. + * If not specified, stores infinity. + */ + Timestamp origin; +} CAggTimebucketInfo; + +typedef struct AggPartCxt +{ + struct MatTableColumnInfo *mattblinfo; + bool added_aggref_col; + /* + * Set to true when you come across a Var + * that is not inside an Aggref node. + */ + bool var_outside_of_aggref; + Oid ignore_aggoid; + int original_query_resno; + /* + * "Original variables" are the Var nodes of the target list of the original + * CREATE MATERIALIZED VIEW query. "Mapped variables" are the Var nodes of the materialization + * table columns. The partialization query is the one that populates those columns. The + * finalization query should use the "mapped variables" to populate the user view. + */ + List *orig_vars; /* List of Var nodes that have been mapped to materialization table columns */ + List *mapped_vars; /* List of Var nodes of the corresponding materialization table columns */ + /* orig_vars and mapped_vars lists are mapped 1 to 1 */ +} AggPartCxt; + +#define CAGG_MAKEQUERY(selquery, srcquery) \ + do \ + { \ + (selquery) = makeNode(Query); \ + (selquery)->commandType = CMD_SELECT; \ + (selquery)->querySource = (srcquery)->querySource; \ + (selquery)->queryId = (srcquery)->queryId; \ + (selquery)->canSetTag = (srcquery)->canSetTag; \ + (selquery)->utilityStmt = copyObject((srcquery)->utilityStmt); \ + (selquery)->resultRelation = 0; \ + (selquery)->hasAggs = true; \ + (selquery)->hasRowSecurity = false; \ + (selquery)->rtable = NULL; \ + } while (0); + +extern CAggTimebucketInfo cagg_validate_query(const Query *query, const bool finalized, + const char *cagg_schema, const char *cagg_name); +extern Query *destroy_union_query(Query *q); +extern Oid relation_oid(Name schema, Name name); +extern void RemoveRangeTableEntries(Query *query); +extern Query *build_union_query(CAggTimebucketInfo *tbinfo, int matpartcolno, Query *q1, Query *q2, + int materialize_htid); +extern void mattablecolumninfo_init(MatTableColumnInfo *matcolinfo, List *grouplist); +extern void mattablecolumninfo_addinternal(MatTableColumnInfo *matcolinfo); +extern bool function_allowed_in_cagg_definition(Oid funcid); +#endif diff --git a/tsl/src/continuous_aggs/create.c b/tsl/src/continuous_aggs/create.c index 30dd345b7..e7dc71899 100644 --- a/tsl/src/continuous_aggs/create.c +++ b/tsl/src/continuous_aggs/create.c @@ -21,8 +21,6 @@ #include #include #include -#include -#include #include #include #include @@ -32,7 +30,6 @@ #include #include #include -#include #include #include #include @@ -47,18 +44,17 @@ #include #include #include -#include -#include #include #include #include #include -#include #include #include #include #include +#include "finalize.h" +#include "common.h" #include "create.h" #include "debug_assert.h" @@ -82,129 +78,48 @@ #include "deparse.h" #include "timezones.h" -#define FINALFN "finalize_agg" -#define PARTIALFN "partialize_agg" -#define CHUNKIDFROMRELID "chunk_id_from_relid" -#define DEFAULT_MATPARTCOLUMN_NAME "time_partition_col" +static void create_cagg_catalog_entry(int32 matht_id, int32 rawht_id, const char *user_schema, + const char *user_view, const char *partial_schema, + const char *partial_view, int64 bucket_width, + bool materialized_only, const char *direct_schema, + const char *direct_view, const bool finalized, + const int32 parent_mat_hypertable_id); +static void create_bucket_function_catalog_entry(int32 matht_id, bool experimental, + const char *name, const char *bucket_width, + const char *origin, const char *timezone); +static void cagg_create_hypertable(int32 hypertable_id, Oid mat_tbloid, const char *matpartcolname, + int64 mat_tbltimecol_interval); +static bool check_trigger_exists_hypertable(Oid relid, char *trigname); +static void cagg_add_trigger_hypertable(Oid relid, int32 hypertable_id); +static void mattablecolumninfo_add_mattable_index(MatTableColumnInfo *matcolinfo, Hypertable *ht); +static int32 mattablecolumninfo_create_materialization_table( + MatTableColumnInfo *matcolinfo, int32 hypertable_id, RangeVar *mat_rel, + CAggTimebucketInfo *bucket_info, bool create_addl_index, char *const tablespacename, + char *const table_access_method, ObjectAddress *mataddress); +static Query *mattablecolumninfo_get_partial_select_query(MatTableColumnInfo *mattblinfo, + Query *userview_query, bool finalized); +static ObjectAddress create_view_for_query(Query *selquery, RangeVar *viewrel); +static void fixup_userview_query_tlist(Query *userquery, List *tlist_aliases); +static void cagg_create(const CreateTableAsStmt *create_stmt, ViewStmt *stmt, Query *panquery, + CAggTimebucketInfo *bucket_info, WithClauseResult *with_clause_options); +void cagg_flip_realtime_view_definition(ContinuousAgg *agg, Hypertable *mat_ht); +void cagg_rename_view_columns(ContinuousAgg *agg); + #define MATPARTCOL_INTERVAL_FACTOR 10 -#define HT_DEFAULT_CHUNKFN "calculate_chunk_interval" #define CAGG_INVALIDATION_TRIGGER "continuous_agg_invalidation_trigger" -#define BOUNDARY_FUNCTION "cagg_watermark" -#define INTERNAL_TO_DATE_FUNCTION "to_date" -#define INTERNAL_TO_TSTZ_FUNCTION "to_timestamp" -#define INTERNAL_TO_TS_FUNCTION "to_timestamp_without_timezone" -#define CONTINUOUS_AGG_MAX_JOIN_RELATIONS 2 -#define PRINT_MATCOLNAME(colbuf, type, original_query_resno, colno) \ - do \ - { \ - int ret = snprintf(colbuf, NAMEDATALEN, "%s_%d_%d", type, original_query_resno, colno); \ - if (ret < 0 || ret >= NAMEDATALEN) \ - ereport(ERROR, \ - (errcode(ERRCODE_INTERNAL_ERROR), \ - errmsg("bad materialization table column name"))); \ - } while (0); - -#define PRINT_MATINTERNAL_NAME(buf, prefix, hypertable_id) \ - do \ - { \ - int ret = snprintf(buf, NAMEDATALEN, prefix, hypertable_id); \ - if (ret < 0 || ret > NAMEDATALEN) \ - { \ - ereport(ERROR, \ - (errcode(ERRCODE_INTERNAL_ERROR), \ - errmsg("bad materialization internal name"))); \ - } \ - } while (0); - -/* Note that we set rowsecurity to false here. */ -#define CAGG_MAKEQUERY(selquery, srcquery) \ - do \ - { \ - (selquery) = makeNode(Query); \ - (selquery)->commandType = CMD_SELECT; \ - (selquery)->querySource = (srcquery)->querySource; \ - (selquery)->queryId = (srcquery)->queryId; \ - (selquery)->canSetTag = (srcquery)->canSetTag; \ - (selquery)->utilityStmt = copyObject((srcquery)->utilityStmt); \ - (selquery)->resultRelation = 0; \ - (selquery)->hasAggs = true; \ - (selquery)->hasRowSecurity = false; \ - (selquery)->rtable = NULL; \ - } while (0); - -typedef struct MatTableColumnInfo +static void +makeMaterializedTableName(char *buf, const char *prefix, int hypertable_id) { - List *matcollist; /* column defns for materialization tbl*/ - List *partial_seltlist; /* tlist entries for populating the materialization table columns */ - List *partial_grouplist; /* group clauses used for populating the materialization table */ - List *mat_groupcolname_list; /* names of columns that are populated by the group-by clause - correspond to the partial_grouplist. - time_bucket column is not included here: it is the - matpartcolname */ - int matpartcolno; /*index of partitioning column in matcollist */ - char *matpartcolname; /*name of the partition column */ -} MatTableColumnInfo; - -typedef struct FinalizeQueryInfo -{ - List *final_seltlist; /* select target list for finalize query */ - Node *final_havingqual; /* having qual for finalize query */ - Query *final_userquery; /* user query used to compute the finalize_query */ - bool finalized; /* finalized form? */ -} FinalizeQueryInfo; - -typedef struct CAggTimebucketInfo -{ - int32 htid; /* hypertable id */ - int32 parent_mat_hypertable_id; /* parent materialization hypertable id */ - Oid htoid; /* hypertable oid */ - AttrNumber htpartcolno; /* primary partitioning column of raw hypertable */ - /* This should also be the column used by time_bucket */ - Oid htpartcoltype; - int64 htpartcol_interval_len; /* interval length setting for primary partitioning column */ - int64 bucket_width; /* bucket_width of time_bucket, stores BUCKET_WIDTH_VARIABLE for - variable-sized buckets */ - Oid bucket_width_type; /* type of bucket_width */ - Interval *interval; /* stores the interval, NULL if not specified */ - const char *timezone; /* the name of the timezone, NULL if not specified */ - - FuncExpr *bucket_func; /* function call expr of the bucketing function */ - /* - * Custom origin value stored as UTC timestamp. - * If not specified, stores infinity. - */ - Timestamp origin; -} CAggTimebucketInfo; - -typedef struct AggPartCxt -{ - struct MatTableColumnInfo *mattblinfo; - bool added_aggref_col; - /* - * Set to true when you come across a Var - * that is not inside an Aggref node. - */ - bool var_outside_of_aggref; - Oid ignore_aggoid; - int original_query_resno; - /* - * "Original variables" are the Var nodes of the target list of the original - * CREATE MATERIALIZED VIEW query. "Mapped variables" are the Var nodes of the materialization - * table columns. The partialization query is the one that populates those columns. The - * finalization query should use the "mapped variables" to populate the user view. - */ - List *orig_vars; /* List of Var nodes that have been mapped to materialization table columns */ - List *mapped_vars; /* List of Var nodes of the corresponding materialization table columns */ - /* orig_vars and mapped_vars lists are mapped 1 to 1 */ -} AggPartCxt; + int ret = snprintf(buf, NAMEDATALEN, prefix, hypertable_id); + if (ret < 0 || ret > NAMEDATALEN) + { + ereport(ERROR, + (errcode(ERRCODE_INTERNAL_ERROR), errmsg("bad materialization internal name"))); + } +} /* STATIC functions defined on the structs above. */ -static void mattablecolumninfo_init(MatTableColumnInfo *matcolinfo, List *grouplist); -static Var *mattablecolumninfo_addentry(MatTableColumnInfo *out, Node *input, - int original_query_resno, bool finalized, - bool *skip_adding); -static void mattablecolumninfo_addinternal(MatTableColumnInfo *matcolinfo); static int32 mattablecolumninfo_create_materialization_table( MatTableColumnInfo *matcolinfo, int32 hypertable_id, RangeVar *mat_rel, CAggTimebucketInfo *bucket_info, bool create_addl_index, char *tablespacename, @@ -212,25 +127,6 @@ static int32 mattablecolumninfo_create_materialization_table( static Query *mattablecolumninfo_get_partial_select_query(MatTableColumnInfo *mattblinfo, Query *userview_query, bool finalized); -static void caggtimebucketinfo_init(CAggTimebucketInfo *src, int32 hypertable_id, - Oid hypertable_oid, AttrNumber hypertable_partition_colno, - Oid hypertable_partition_coltype, - int64 hypertable_partition_col_interval, - int32 parent_mat_hypertable_id); -static void caggtimebucket_validate(CAggTimebucketInfo *tbinfo, List *groupClause, - List *targetList); -static void finalizequery_init(FinalizeQueryInfo *inp, Query *orig_query, - MatTableColumnInfo *mattblinfo); -static Query *finalizequery_get_select_query(FinalizeQueryInfo *inp, List *matcollist, - ObjectAddress *mattbladdress, char *relname); -static bool function_allowed_in_cagg_definition(Oid funcid); - -static Const *cagg_boundary_make_lower_bound(Oid type); -static Oid cagg_get_boundary_converter_funcoid(Oid typoid); -static Query *build_union_query(CAggTimebucketInfo *tbinfo, int matpartcolno, Query *q1, Query *q2, - int materialize_htid); -static Query *destroy_union_query(Query *q); - /* * Create a entry for the materialization table in table CONTINUOUS_AGGS. */ @@ -640,7 +536,7 @@ static Query * mattablecolumninfo_get_partial_select_query(MatTableColumnInfo *mattblinfo, Query *userview_query, bool finalized) { - Query *partial_selquery; + Query *partial_selquery = NULL; CAGG_MAKEQUERY(partial_selquery, userview_query); partial_selquery->rtable = copyObject(userview_query->rtable); @@ -713,1916 +609,6 @@ create_view_for_query(Query *selquery, RangeVar *viewrel) return address; } -/* - * Initialize caggtimebucket. - */ -static void -caggtimebucketinfo_init(CAggTimebucketInfo *src, int32 hypertable_id, Oid hypertable_oid, - AttrNumber hypertable_partition_colno, Oid hypertable_partition_coltype, - int64 hypertable_partition_col_interval, int32 parent_mat_hypertable_id) -{ - src->htid = hypertable_id; - src->parent_mat_hypertable_id = parent_mat_hypertable_id; - src->htoid = hypertable_oid; - src->htpartcolno = hypertable_partition_colno; - src->htpartcoltype = hypertable_partition_coltype; - src->htpartcol_interval_len = hypertable_partition_col_interval; - src->bucket_width = 0; /* invalid value */ - src->bucket_width_type = InvalidOid; /* invalid oid */ - src->interval = NULL; /* not specified by default */ - src->timezone = NULL; /* not specified by default */ - TIMESTAMP_NOBEGIN(src->origin); /* origin is not specified by default */ -} - -static Const * -check_time_bucket_argument(Node *arg, char *position) -{ - if (IsA(arg, NamedArgExpr)) - arg = (Node *) castNode(NamedArgExpr, arg)->arg; - - Node *expr = eval_const_expressions(NULL, arg); - - if (!IsA(expr, Const)) - ereport(ERROR, - (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), - errmsg("only immutable expressions allowed in time bucket function"), - errhint("Use an immutable expression as %s argument to the time bucket function.", - position))); - - return castNode(Const, expr); -} - -/* - * Check if the group-by clauses has exactly 1 time_bucket(.., ) where - * is the hypertable's partitioning column and other invariants. Then fill - * the `bucket_width` and other fields of `tbinfo`. - */ -static void -caggtimebucket_validate(CAggTimebucketInfo *tbinfo, List *groupClause, List *targetList) -{ - ListCell *l; - bool found = false; - bool custom_origin = false; - Const *const_arg; - - /* Make sure tbinfo was initialized. This assumption is used below. */ - Assert(tbinfo->bucket_width == 0); - Assert(tbinfo->timezone == NULL); - Assert(TIMESTAMP_NOT_FINITE(tbinfo->origin)); - - foreach (l, groupClause) - { - SortGroupClause *sgc = (SortGroupClause *) lfirst(l); - TargetEntry *tle = get_sortgroupclause_tle(sgc, targetList); - - if (IsA(tle->expr, FuncExpr)) - { - FuncExpr *fe = ((FuncExpr *) tle->expr); - Node *width_arg; - Node *col_arg; - - if (!function_allowed_in_cagg_definition(fe->funcid)) - continue; - - /* - * Offset variants of time_bucket functions are not - * supported at the moment. - */ - if (list_length(fe->args) >= 5 || - (list_length(fe->args) == 4 && exprType(lfourth(fe->args)) == INTERVALOID)) - continue; - - if (found) - ereport(ERROR, - (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), - errmsg("continuous aggregate view cannot contain" - " multiple time bucket functions"))); - else - found = true; - - tbinfo->bucket_func = fe; - - /* Only column allowed : time_bucket('1day', ) */ - col_arg = lsecond(fe->args); - /* Could be a named argument */ - if (IsA(col_arg, NamedArgExpr)) - col_arg = (Node *) castNode(NamedArgExpr, col_arg)->arg; - - if (!(IsA(col_arg, Var)) || ((Var *) col_arg)->varattno != tbinfo->htpartcolno) - ereport(ERROR, - (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), - errmsg( - "time bucket function must reference a hypertable dimension column"))); - - if (list_length(fe->args) >= 3) - { - Const *arg = check_time_bucket_argument(lthird(fe->args), "third"); - if (exprType((Node *) arg) == TEXTOID) - { - const char *tz_name = TextDatumGetCString(arg->constvalue); - if (!ts_is_valid_timezone_name(tz_name)) - { - ereport(ERROR, - (errcode(ERRCODE_INVALID_PARAMETER_VALUE), - errmsg("invalid timezone name \"%s\"", tz_name))); - } - - tbinfo->timezone = tz_name; - tbinfo->bucket_width = BUCKET_WIDTH_VARIABLE; - } - } - - if (list_length(fe->args) >= 4) - { - /* origin */ - Const *arg = check_time_bucket_argument(lfourth(fe->args), "fourth"); - if (exprType((Node *) arg) == TEXTOID) - { - const char *tz_name = TextDatumGetCString(arg->constvalue); - if (!ts_is_valid_timezone_name(tz_name)) - { - ereport(ERROR, - (errcode(ERRCODE_INVALID_PARAMETER_VALUE), - errmsg("invalid timezone name \"%s\"", tz_name))); - } - - tbinfo->timezone = tz_name; - tbinfo->bucket_width = BUCKET_WIDTH_VARIABLE; - } - } - - /* Check for custom origin. */ - switch (exprType(col_arg)) - { - case DATEOID: - /* Origin is always 3rd arg for date variants. */ - if (list_length(fe->args) == 3) - { - Node *arg = lthird(fe->args); - custom_origin = true; - /* this function also takes care of named arguments */ - const_arg = check_time_bucket_argument(arg, "third"); - tbinfo->origin = DatumGetTimestamp( - DirectFunctionCall1(date_timestamp, const_arg->constvalue)); - } - break; - case TIMESTAMPOID: - /* Origin is always 3rd arg for timestamp variants. */ - if (list_length(fe->args) == 3) - { - Node *arg = lthird(fe->args); - custom_origin = true; - const_arg = check_time_bucket_argument(arg, "third"); - tbinfo->origin = DatumGetTimestamp(const_arg->constvalue); - } - break; - case TIMESTAMPTZOID: - /* Origin can be 3rd or 4th arg for timestamptz variants. */ - if (list_length(fe->args) >= 3 && exprType(lthird(fe->args)) == TIMESTAMPTZOID) - { - custom_origin = true; - tbinfo->origin = - DatumGetTimestampTz(castNode(Const, lthird(fe->args))->constvalue); - } - else if (list_length(fe->args) >= 4 && - exprType(lfourth(fe->args)) == TIMESTAMPTZOID) - { - custom_origin = true; - if (IsA(lfourth(fe->args), Const)) - { - tbinfo->origin = - DatumGetTimestampTz(castNode(Const, lfourth(fe->args))->constvalue); - } - /* could happen in a statement like time_bucket('1h', .., 'utc', origin => - * ...) */ - else if (IsA(lfourth(fe->args), NamedArgExpr)) - { - Const *constval = - check_time_bucket_argument(lfourth(fe->args), "fourth"); - - tbinfo->origin = DatumGetTimestampTz(constval->constvalue); - } - } - } - if (custom_origin && TIMESTAMP_NOT_FINITE(tbinfo->origin)) - { - ereport(ERROR, - (errcode(ERRCODE_INVALID_PARAMETER_VALUE), - errmsg("invalid origin value: infinity"))); - } - - /* - * We constify width expression here so any immutable expression will be allowed. - * Otherwise it would make it harder to create caggs for hypertables with e.g. int8 - * partitioning column as int constants default to int4 and so expression would - * have a cast and not be a Const. - */ - width_arg = linitial(fe->args); - - if (IsA(width_arg, NamedArgExpr)) - width_arg = (Node *) castNode(NamedArgExpr, width_arg)->arg; - - width_arg = eval_const_expressions(NULL, width_arg); - if (IsA(width_arg, Const)) - { - Const *width = castNode(Const, width_arg); - tbinfo->bucket_width_type = width->consttype; - - if (width->consttype == INTERVALOID) - { - tbinfo->interval = DatumGetIntervalP(width->constvalue); - if (tbinfo->interval->month != 0) - tbinfo->bucket_width = BUCKET_WIDTH_VARIABLE; - } - - if (tbinfo->bucket_width != BUCKET_WIDTH_VARIABLE) - { - /* The bucket size is fixed. */ - tbinfo->bucket_width = - ts_interval_value_to_internal(width->constvalue, width->consttype); - } - } - else - ereport(ERROR, - (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), - errmsg("only immutable expressions allowed in time bucket function"), - errhint("Use an immutable expression as first argument to the time bucket " - "function."))); - - if (tbinfo->interval && tbinfo->interval->month) - { - tbinfo->bucket_width = BUCKET_WIDTH_VARIABLE; - } - } - } - - if (tbinfo->bucket_width == BUCKET_WIDTH_VARIABLE) - { - /* Variable-sized buckets can be used only with intervals. */ - Assert(tbinfo->interval != NULL); - - if ((tbinfo->interval->month != 0) && - ((tbinfo->interval->day != 0) || (tbinfo->interval->time != 0))) - { - ereport(ERROR, - (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), - errmsg("invalid interval specified"), - errhint("Use either months or days and hours, but not months, days and hours " - "together"))); - } - } - - if (!found) - elog(ERROR, "continuous aggregate view must include a valid time bucket function"); -} - -static bool -cagg_agg_validate(Node *node, void *context) -{ - if (node == NULL) - return false; - - if (IsA(node, Aggref)) - { - Aggref *agg = (Aggref *) node; - HeapTuple aggtuple; - Form_pg_aggregate aggform; - if (agg->aggorder || agg->aggdistinct || agg->aggfilter) - { - ereport(ERROR, - (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), - errmsg("aggregates with FILTER / DISTINCT / ORDER BY are not supported"))); - } - /* Fetch the pg_aggregate row. */ - aggtuple = SearchSysCache1(AGGFNOID, agg->aggfnoid); - if (!HeapTupleIsValid(aggtuple)) - elog(ERROR, "cache lookup failed for aggregate %u", agg->aggfnoid); - aggform = (Form_pg_aggregate) GETSTRUCT(aggtuple); - if (aggform->aggkind != 'n') - { - ReleaseSysCache(aggtuple); - ereport(ERROR, - (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), - errmsg("ordered set/hypothetical aggregates are not supported"))); - } - if (!OidIsValid(aggform->aggcombinefn) || - (aggform->aggtranstype == INTERNALOID && !OidIsValid(aggform->aggdeserialfn))) - { - ReleaseSysCache(aggtuple); - ereport(ERROR, - (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), - errmsg("aggregates which are not parallelizable are not supported"))); - } - ReleaseSysCache(aggtuple); - - return false; - } - return expression_tree_walker(node, cagg_agg_validate, context); -} - -/* - * Check query and extract error details and error hints. - * - * Returns: - * True if the query is supported, false otherwise with hints and errors - * added. - */ -static bool -cagg_query_supported(const Query *query, StringInfo hint, StringInfo detail, const bool finalized) -{ -/* - * For now deprecate partial aggregates on release builds only. - * Once migration tests are made compatible with PG15 enable deprecation - * on debug builds as well. - */ -#ifndef DEBUG -#if PG15_GE - if (!finalized) - { - /* continuous aggregates with old format will not be allowed */ - appendStringInfoString(detail, - "Continuous Aggregates with partials is not supported anymore."); - appendStringInfoString(hint, - "Define the Continuous Aggregate with \"finalized\" parameter set " - "to true."); - return false; - } -#endif -#endif - if (!query->jointree->fromlist) - { - appendStringInfoString(hint, "FROM clause missing in the query"); - return false; - } - if (query->commandType != CMD_SELECT) - { - appendStringInfoString(hint, "Use a SELECT query in the continuous aggregate view."); - return false; - } - - if (query->hasWindowFuncs) - { - appendStringInfoString(detail, - "Window functions are not supported by continuous aggregates."); - return false; - } - - if (query->hasDistinctOn || query->distinctClause) - { - appendStringInfoString(detail, - "DISTINCT / DISTINCT ON queries are not supported by continuous " - "aggregates."); - return false; - } - - if (query->limitOffset || query->limitCount) - { - appendStringInfoString(detail, - "LIMIT and LIMIT OFFSET are not supported in queries defining " - "continuous aggregates."); - appendStringInfoString(hint, - "Use LIMIT and LIMIT OFFSET in SELECTS from the continuous " - "aggregate view instead."); - return false; - } - - if (query->sortClause && !finalized) - { - appendStringInfoString(detail, - "ORDER BY is not supported in queries defining continuous " - "aggregates."); - appendStringInfoString(hint, - "Use ORDER BY clauses in SELECTS from the continuous aggregate view " - "instead."); - return false; - } - - if (query->hasRecursive || query->hasSubLinks || query->hasTargetSRFs || query->cteList) - { - appendStringInfoString(detail, - "CTEs, subqueries and set-returning functions are not supported by " - "continuous aggregates."); - return false; - } - - if (query->hasForUpdate || query->hasModifyingCTE) - { - appendStringInfoString(detail, - "Data modification is not allowed in continuous aggregate view " - "definitions."); - return false; - } - - if (query->hasRowSecurity) - { - appendStringInfoString(detail, - "Row level security is not supported by continuous aggregate " - "views."); - return false; - } - - if (query->groupingSets) - { - appendStringInfoString(detail, - "GROUP BY GROUPING SETS, ROLLUP and CUBE are not supported by " - "continuous aggregates"); - appendStringInfoString(hint, - "Define multiple continuous aggregates with different grouping " - "levels."); - return false; - } - - if (query->setOperations) - { - appendStringInfoString(detail, - "UNION, EXCEPT & INTERSECT are not supported by continuous " - "aggregates"); - return false; - } - - if (!query->groupClause) - { - /* - * Query can have aggregate without group by , so look - * for groupClause. - */ - appendStringInfoString(hint, - "Include at least one aggregate function" - " and a GROUP BY clause with time bucket."); - return false; - } - - return true; /* Query was OK and is supported. */ -} - -static inline int64 -get_bucket_width(CAggTimebucketInfo bucket_info) -{ - int64 width = 0; - - /* Calculate the width. */ - switch (bucket_info.bucket_width_type) - { - case INT8OID: - case INT4OID: - case INT2OID: - width = bucket_info.bucket_width; - break; - case INTERVALOID: - { - /* - * epoch will treat year as 365.25 days. This leads to the unexpected - * result that year is not multiple of day or month, which is perceived - * as a bug. For that reason, we treat all months as 30 days regardless of year - */ - if (bucket_info.interval->month && !bucket_info.interval->day && - !bucket_info.interval->time) - { - bucket_info.interval->day = bucket_info.interval->month * DAYS_PER_MONTH; - bucket_info.interval->month = 0; - } - - /* Convert Interval to int64 */ - width = - ts_interval_value_to_internal(IntervalPGetDatum(bucket_info.interval), INTERVALOID); - break; - } - default: - Assert(false); - } - - return width; -} - -static inline Datum -get_bucket_width_datum(CAggTimebucketInfo bucket_info) -{ - Datum width = (Datum) 0; - - switch (bucket_info.bucket_width_type) - { - case INT8OID: - case INT4OID: - case INT2OID: - width = ts_internal_to_interval_value(bucket_info.bucket_width, - bucket_info.bucket_width_type); - break; - case INTERVALOID: - width = IntervalPGetDatum(bucket_info.interval); - break; - default: - Assert(false); - } - - return width; -} - -static CAggTimebucketInfo -cagg_validate_query(const Query *query, const bool finalized, const char *cagg_schema, - const char *cagg_name) -{ - CAggTimebucketInfo bucket_info = { 0 }, bucket_info_parent; - Cache *hcache; - Hypertable *ht = NULL, *ht_parent = NULL; - RangeTblRef *rtref = NULL, *rtref_other = NULL; - RangeTblEntry *rte = NULL, *rte_other = NULL; - JoinType jointype = JOIN_FULL; - OpExpr *op = NULL; - List *fromList = NIL; - StringInfo hint = makeStringInfo(); - StringInfo detail = makeStringInfo(); - bool is_hierarchical = false; - Query *prev_query = NULL; - ContinuousAgg *cagg_parent = NULL; - Oid normal_table_id = InvalidOid; - - if (!cagg_query_supported(query, hint, detail, finalized)) - { - ereport(ERROR, - (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), - errmsg("invalid continuous aggregate query"), - hint->len > 0 ? errhint("%s", hint->data) : 0, - detail->len > 0 ? errdetail("%s", detail->data) : 0)); - } - - /* Finalized cagg doesn't have those restrictions anymore. */ - if (!finalized) - { - /* Validate aggregates allowed. */ - cagg_agg_validate((Node *) query->targetList, NULL); - cagg_agg_validate((Node *) query->havingQual, NULL); - } - /* Check if there are only two tables in the from list. */ - fromList = query->jointree->fromlist; - if (list_length(fromList) > CONTINUOUS_AGG_MAX_JOIN_RELATIONS) - { - ereport(ERROR, - (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), - errmsg("only two tables with one hypertable and one normal table" - "are allowed in continuous aggregate view"))); - } - /* Extra checks for joins in Caggs. */ - if (list_length(fromList) == CONTINUOUS_AGG_MAX_JOIN_RELATIONS || - !IsA(linitial(query->jointree->fromlist), RangeTblRef)) - { - /* Using old format caggs is not supported */ - if (!finalized) - ereport(ERROR, - (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), - errmsg("old format of continuous aggregate is not supported with joins"), - errhint("Set timescaledb.finalized to TRUE."))); - - if (list_length(fromList) == CONTINUOUS_AGG_MAX_JOIN_RELATIONS) - { - if (!IsA(linitial(fromList), RangeTblRef) || !IsA(lsecond(fromList), RangeTblRef)) - ereport(ERROR, - (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), - errmsg("invalid continuous aggregate view"), - errdetail( - "From clause can only have one hypertable and one normal table."))); - - rtref = linitial_node(RangeTblRef, query->jointree->fromlist); - rte = list_nth(query->rtable, rtref->rtindex - 1); - rtref_other = lsecond_node(RangeTblRef, query->jointree->fromlist); - rte_other = list_nth(query->rtable, rtref_other->rtindex - 1); - jointype = rte->jointype || rte_other->jointype; - - if (query->jointree->quals != NULL && IsA(query->jointree->quals, OpExpr)) - op = (OpExpr *) query->jointree->quals; - } - else - { - ListCell *l; - foreach (l, query->jointree->fromlist) - { - Node *jtnode = (Node *) lfirst(l); - JoinExpr *join = NULL; - if (IsA(jtnode, JoinExpr)) - { - join = castNode(JoinExpr, jtnode); -#if PG13_LT - if (join->usingClause != NULL) - ereport(ERROR, - (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), - errmsg("invalid continuous aggregate view"), - errdetail( - "Joins with USING clause in continuous aggregate definition" - " work for Postgres versions 13 and above."))); -#endif - jointype = join->jointype; - op = (OpExpr *) join->quals; - rte = list_nth(query->rtable, ((RangeTblRef *) join->larg)->rtindex - 1); - rte_other = list_nth(query->rtable, ((RangeTblRef *) join->rarg)->rtindex - 1); - if (rte->subquery != NULL || rte_other->subquery != NULL) - ereport(ERROR, - (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), - errmsg("invalid continuous aggregate view"), - errdetail("Sub-queries are not supported in FROM clause."))); - RangeTblEntry *jrte = rt_fetch(join->rtindex, query->rtable); - if (jrte->joinaliasvars == NIL) - ereport(ERROR, - (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), - errmsg("invalid continuous aggregate view"))); - } - } - } - - /* - * Error out if there is aynthing else than one normal table and one hypertable - * in the from clause, e.g. sub-query, lateral, two hypertables, etc. - */ - if (rte->lateral || rte_other->lateral) - ereport(ERROR, - (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), - errmsg("invalid continuous aggregate view"), - errdetail("Lateral joins are not supported in FROM clause."))); - if ((rte->relkind == RELKIND_VIEW && ts_is_hypertable(rte_other->relid)) || - (rte_other->relkind == RELKIND_VIEW && ts_is_hypertable(rte->relid))) - ereport(ERROR, - (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), - errmsg("invalid continuous aggregate view"), - errdetail("Views are not supported in FROM clause."))); - if (rte->relkind != RELKIND_VIEW && rte_other->relkind != RELKIND_VIEW && - (ts_is_hypertable(rte->relid) == ts_is_hypertable(rte_other->relid))) - ereport(ERROR, - (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), - errmsg("invalid continuous aggregate view"), - errdetail("Multiple hypertables or normal tables are not supported in FROM " - "clause."))); - - /* Only inner joins are allowed. */ - if (jointype != JOIN_INNER) - ereport(ERROR, - (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), - errmsg("only inner joins are supported in continuous aggregates"))); - - /* Only equality conditions are permitted on joins. */ - if (op && IsA(op, OpExpr) && - list_length(castNode(OpExpr, op)->args) == CONTINUOUS_AGG_MAX_JOIN_RELATIONS) - { - Oid left_type = exprType(linitial(op->args)); - Oid right_type = exprType(lsecond(op->args)); - if (!ts_is_equality_operator(op->opno, left_type, right_type)) - ereport(ERROR, - (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), - errmsg("invalid continuous aggregate view"), - errdetail( - "Only equality conditions are supported in continuous aggregates."))); - } - else - ereport(ERROR, - (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), - errmsg("invalid continuous aggregate view"), - errdetail("Unsupported expression in join clause."), - errhint("Only equality conditions are supported in continuous aggregates."))); - /* - * Record the table oid of the normal table. This is required so - * that we know which one is hypertable to carry out the related - * processing in later parts of code. - */ - if (rte->relkind == RELKIND_VIEW) - normal_table_id = rte_other->relid; - else if (rte_other->relkind == RELKIND_VIEW) - normal_table_id = rte->relid; - else - normal_table_id = ts_is_hypertable(rte->relid) ? rte_other->relid : rte->relid; - if (normal_table_id == rte->relid) - rte = rte_other; - } - else - { - /* Check if we have a hypertable in the FROM clause. */ - rtref = linitial_node(RangeTblRef, query->jointree->fromlist); - rte = list_nth(query->rtable, rtref->rtindex - 1); - } - /* FROM only sets rte->inh to false. */ - if (rte->rtekind != RTE_JOIN) - { - if ((rte->relkind != RELKIND_RELATION && rte->relkind != RELKIND_VIEW) || - rte->tablesample || rte->inh == false) - ereport(ERROR, - (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), - errmsg("invalid continuous aggregate view"))); - } - - if (rte->relkind == RELKIND_RELATION || rte->relkind == RELKIND_VIEW) - { - const Dimension *part_dimension = NULL; - int32 parent_mat_hypertable_id = INVALID_HYPERTABLE_ID; - - if (rte->relkind == RELKIND_RELATION) - ht = ts_hypertable_cache_get_cache_and_entry(rte->relid, CACHE_FLAG_NONE, &hcache); - else - { - cagg_parent = ts_continuous_agg_find_by_relid(rte->relid); - - if (!cagg_parent) - { - ereport(ERROR, - (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), - errmsg("invalid continuous aggregate query"), - errhint("Continuous aggregate needs to query hypertable or another " - "continuous aggregate."))); - } - - if (!ContinuousAggIsFinalized(cagg_parent)) - { - ereport(ERROR, - (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), - errmsg("old format of continuous aggregate is not supported"), - errhint("Run \"CALL cagg_migrate('%s.%s');\" to migrate to the new " - "format.", - NameStr(cagg_parent->data.user_view_schema), - NameStr(cagg_parent->data.user_view_name)))); - } - - parent_mat_hypertable_id = cagg_parent->data.mat_hypertable_id; - hcache = ts_hypertable_cache_pin(); - ht = ts_hypertable_cache_get_entry_by_id(hcache, cagg_parent->data.mat_hypertable_id); - - /* If parent cagg is hierarchical then we should get the matht otherwise the rawht. */ - if (ContinuousAggIsHierarchical(cagg_parent)) - ht_parent = - ts_hypertable_cache_get_entry_by_id(hcache, - cagg_parent->data.mat_hypertable_id); - else - ht_parent = - ts_hypertable_cache_get_entry_by_id(hcache, - cagg_parent->data.raw_hypertable_id); - - /* Get the querydef for the source cagg. */ - is_hierarchical = true; - prev_query = ts_continuous_agg_get_query(cagg_parent); - } - - if (TS_HYPERTABLE_IS_INTERNAL_COMPRESSION_TABLE(ht)) - ereport(ERROR, - (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), - errmsg("hypertable is an internal compressed hypertable"))); - - if (rte->relkind == RELKIND_RELATION) - { - ContinuousAggHypertableStatus status = ts_continuous_agg_hypertable_status(ht->fd.id); - - /* Prevent create a CAGG over an existing materialization hypertable. */ - if (status == HypertableIsMaterialization || - status == HypertableIsMaterializationAndRaw) - { - const ContinuousAgg *cagg = ts_continuous_agg_find_by_mat_hypertable_id(ht->fd.id); - Assert(cagg != NULL); - - ereport(ERROR, - (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), - errmsg("hypertable is a continuous aggregate materialization table"), - errdetail("Materialization hypertable \"%s.%s\".", - NameStr(ht->fd.schema_name), - NameStr(ht->fd.table_name)), - errhint("Do you want to use continuous aggregate \"%s.%s\" instead?", - NameStr(cagg->data.user_view_schema), - NameStr(cagg->data.user_view_name)))); - } - } - - /* Get primary partitioning column information. */ - part_dimension = hyperspace_get_open_dimension(ht->space, 0); - - /* - * NOTE: if we ever allow custom partitioning functions we'll need to - * change part_dimension->fd.column_type to partitioning_type - * below, along with any other fallout. - */ - if (part_dimension->partitioning != NULL) - ereport(ERROR, - (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), - errmsg("custom partitioning functions not supported" - " with continuous aggregates"))); - - if (IS_INTEGER_TYPE(ts_dimension_get_partition_type(part_dimension)) && - rte->relkind == RELKIND_RELATION) - { - const char *funcschema = NameStr(part_dimension->fd.integer_now_func_schema); - const char *funcname = NameStr(part_dimension->fd.integer_now_func); - - if (strlen(funcschema) == 0 || strlen(funcname) == 0) - ereport(ERROR, - (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), - errmsg("custom time function required on hypertable \"%s\"", - get_rel_name(ht->main_table_relid)), - errdetail("An integer-based hypertable requires a custom time function to " - "support continuous aggregates."), - errhint("Set a custom time function on the hypertable."))); - } - - caggtimebucketinfo_init(&bucket_info, - ht->fd.id, - ht->main_table_relid, - part_dimension->column_attno, - part_dimension->fd.column_type, - part_dimension->fd.interval_length, - parent_mat_hypertable_id); - - if (is_hierarchical) - { - const Dimension *part_dimension_parent = - hyperspace_get_open_dimension(ht_parent->space, 0); - - caggtimebucketinfo_init(&bucket_info_parent, - ht_parent->fd.id, - ht_parent->main_table_relid, - part_dimension_parent->column_attno, - part_dimension_parent->fd.column_type, - part_dimension_parent->fd.interval_length, - INVALID_HYPERTABLE_ID); - } - - ts_cache_release(hcache); - - /* - * We need a GROUP By clause with time_bucket on the partitioning - * column of the hypertable - */ - Assert(query->groupClause); - caggtimebucket_validate(&bucket_info, query->groupClause, query->targetList); - } - - /* Check row security settings for the table. */ - if (ts_has_row_security(rte->relid)) - ereport(ERROR, - (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), - errmsg("cannot create continuous aggregate on hypertable with row security"))); - - /* hierarchical cagg validations */ - if (is_hierarchical) - { - int64 bucket_width = 0, bucket_width_parent = 0; - bool is_greater_or_equal_than_parent = true, is_multiple_of_parent = true; - - Assert(prev_query->groupClause); - caggtimebucket_validate(&bucket_info_parent, - prev_query->groupClause, - prev_query->targetList); - - /* Cannot create cagg with fixed bucket on top of variable bucket. */ - if ((bucket_info_parent.bucket_width == BUCKET_WIDTH_VARIABLE && - bucket_info.bucket_width != BUCKET_WIDTH_VARIABLE)) - { - ereport(ERROR, - (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), - errmsg("cannot create continuous aggregate with fixed-width bucket on top of " - "one using variable-width bucket"), - errdetail("Continuous aggregate with a fixed time bucket width (e.g. 61 days) " - "cannot be created on top of one using variable time bucket width " - "(e.g. 1 month).\n" - "The variance can lead to the fixed width one not being a multiple " - "of the variable width one."))); - } - - /* Get bucket widths for validation. */ - bucket_width = get_bucket_width(bucket_info); - bucket_width_parent = get_bucket_width(bucket_info_parent); - - Assert(bucket_width != 0); - Assert(bucket_width_parent != 0); - - /* Check if the current bucket is greater or equal than the parent. */ - is_greater_or_equal_than_parent = (bucket_width >= bucket_width_parent); - - /* Check if buckets are multiple. */ - if (bucket_width_parent != 0) - { - if (bucket_width_parent > bucket_width && bucket_width != 0) - is_multiple_of_parent = ((bucket_width_parent % bucket_width) == 0); - else - is_multiple_of_parent = ((bucket_width % bucket_width_parent) == 0); - } - - /* Proceed with validation errors. */ - if (!is_greater_or_equal_than_parent || !is_multiple_of_parent) - { - Datum width, width_parent; - Oid outfuncid = InvalidOid; - bool isvarlena; - char *width_out, *width_out_parent; - char *message = NULL; - - getTypeOutputInfo(bucket_info.bucket_width_type, &outfuncid, &isvarlena); - width = get_bucket_width_datum(bucket_info); - width_out = DatumGetCString(OidFunctionCall1(outfuncid, width)); - - getTypeOutputInfo(bucket_info_parent.bucket_width_type, &outfuncid, &isvarlena); - width_parent = get_bucket_width_datum(bucket_info_parent); - width_out_parent = DatumGetCString(OidFunctionCall1(outfuncid, width_parent)); - - /* New bucket should be multiple of the parent. */ - if (!is_multiple_of_parent) - message = "multiple of"; - - /* New bucket should be greater than the parent. */ - if (!is_greater_or_equal_than_parent) - message = "greater or equal than"; - - ereport(ERROR, - (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), - errmsg("cannot create continuous aggregate with incompatible bucket width"), - errdetail("Time bucket width of \"%s.%s\" [%s] should be %s the time " - "bucket width of \"%s.%s\" [%s].", - cagg_schema, - cagg_name, - width_out, - message, - NameStr(cagg_parent->data.user_view_schema), - NameStr(cagg_parent->data.user_view_name), - width_out_parent))); - } - } - - return bucket_info; -} - -/* - * Add ts_internal_cagg_final to bytea column. - * Bytea column is the internal state for an agg. - * Pass info for the agg as "inp". - * inpcol = bytea column. - * This function returns an aggref - * ts_internal_cagg_final( Oid, Oid, bytea, NULL::output_typeid) - * the arguments are a list of targetentry. - */ -static Oid -get_finalizefnoid() -{ - Oid finalfnoid; - Oid finalfnargtypes[] = { TEXTOID, NAMEOID, NAMEOID, get_array_type(NAMEOID), - BYTEAOID, ANYELEMENTOID }; - List *funcname = list_make2(makeString(INTERNAL_SCHEMA_NAME), makeString(FINALFN)); - int nargs = sizeof(finalfnargtypes) / sizeof(finalfnargtypes[0]); - finalfnoid = LookupFuncName(funcname, nargs, finalfnargtypes, false); - return finalfnoid; -} - -/* - * Build a [N][2] array where N is number of arguments - * and the inner array is of [schema_name,type_name]. - */ -static Datum -get_input_types_array_datum(Aggref *original_aggregate) -{ - ListCell *lc; - MemoryContext builder_context = - AllocSetContextCreate(CurrentMemoryContext, "input types builder", ALLOCSET_DEFAULT_SIZES); - Oid name_array_type_oid = get_array_type(NAMEOID); - ArrayBuildStateArr *outer_builder = - initArrayResultArr(name_array_type_oid, NAMEOID, builder_context, false); - Datum result; - - foreach (lc, original_aggregate->args) - { - TargetEntry *te = lfirst(lc); - Oid type_oid = exprType((Node *) te->expr); - ArrayBuildState *schema_name_builder = initArrayResult(NAMEOID, builder_context, false); - HeapTuple tp; - Form_pg_type typtup; - char *schema_name; - Name type_name = (Name) palloc0(NAMEDATALEN); - Datum schema_datum; - Datum type_name_datum; - Datum inner_array_datum; - - tp = SearchSysCache1(TYPEOID, ObjectIdGetDatum(type_oid)); - if (!HeapTupleIsValid(tp)) - elog(ERROR, "cache lookup failed for type %u", type_oid); - - typtup = (Form_pg_type) GETSTRUCT(tp); - namestrcpy(type_name, NameStr(typtup->typname)); - schema_name = get_namespace_name(typtup->typnamespace); - ReleaseSysCache(tp); - - type_name_datum = NameGetDatum(type_name); - /* Using name in because creating from a char * (that may be null or too long). */ - schema_datum = DirectFunctionCall1(namein, CStringGetDatum(schema_name)); - - accumArrayResult(schema_name_builder, schema_datum, false, NAMEOID, builder_context); - accumArrayResult(schema_name_builder, type_name_datum, false, NAMEOID, builder_context); - - inner_array_datum = makeArrayResult(schema_name_builder, CurrentMemoryContext); - - accumArrayResultArr(outer_builder, - inner_array_datum, - false, - name_array_type_oid, - builder_context); - } - result = makeArrayResultArr(outer_builder, CurrentMemoryContext, false); - - MemoryContextDelete(builder_context); - return result; -} - -/* - * Creates an aggref of the form: - * finalize-agg( - * "sum(int)" TEXT, - * collation_schema_name NAME, collation_name NAME, - * input_types_array NAME[N][2], - * BYTEA, - * null:: - * ) - * here sum(int) is the input aggregate "inp" in the parameter-list. - */ -static Aggref * -get_finalize_aggref(Aggref *inp, Var *partial_state_var) -{ - Aggref *aggref; - TargetEntry *te; - char *aggregate_signature; - Const *aggregate_signature_const, *collation_schema_const, *collation_name_const, - *input_types_const, *return_type_const; - Oid name_array_type_oid = get_array_type(NAMEOID); - Var *partial_bytea_var; - List *tlist = NIL; - int tlist_attno = 1; - List *argtypes = NIL; - char *collation_name = NULL, *collation_schema_name = NULL; - Datum collation_name_datum = (Datum) 0; - Datum collation_schema_datum = (Datum) 0; - Oid finalfnoid = get_finalizefnoid(); - - argtypes = list_make5_oid(TEXTOID, NAMEOID, NAMEOID, name_array_type_oid, BYTEAOID); - argtypes = lappend_oid(argtypes, inp->aggtype); - - aggref = makeNode(Aggref); - aggref->aggfnoid = finalfnoid; - aggref->aggtype = inp->aggtype; - aggref->aggcollid = inp->aggcollid; - aggref->inputcollid = inp->inputcollid; - aggref->aggtranstype = InvalidOid; /* will be set by planner */ - aggref->aggargtypes = argtypes; - aggref->aggdirectargs = NULL; /*relevant for hypothetical set aggs*/ - aggref->aggorder = NULL; - aggref->aggdistinct = NULL; - aggref->aggfilter = NULL; - aggref->aggstar = false; - aggref->aggvariadic = false; - aggref->aggkind = AGGKIND_NORMAL; - aggref->aggsplit = AGGSPLIT_SIMPLE; - aggref->location = -1; - /* Construct the arguments. */ - aggregate_signature = format_procedure_qualified(inp->aggfnoid); - aggregate_signature_const = makeConst(TEXTOID, - -1, - DEFAULT_COLLATION_OID, - -1, - CStringGetTextDatum(aggregate_signature), - false, - false /* passbyval */ - ); - te = makeTargetEntry((Expr *) aggregate_signature_const, tlist_attno++, NULL, false); - tlist = lappend(tlist, te); - - if (OidIsValid(inp->inputcollid)) - { - /* Similar to generate_collation_name. */ - HeapTuple tp; - Form_pg_collation colltup; - tp = SearchSysCache1(COLLOID, ObjectIdGetDatum(inp->inputcollid)); - if (!HeapTupleIsValid(tp)) - elog(ERROR, "cache lookup failed for collation %u", inp->inputcollid); - colltup = (Form_pg_collation) GETSTRUCT(tp); - collation_name = pstrdup(NameStr(colltup->collname)); - collation_name_datum = DirectFunctionCall1(namein, CStringGetDatum(collation_name)); - - collation_schema_name = get_namespace_name(colltup->collnamespace); - if (collation_schema_name != NULL) - collation_schema_datum = - DirectFunctionCall1(namein, CStringGetDatum(collation_schema_name)); - ReleaseSysCache(tp); - } - collation_schema_const = makeConst(NAMEOID, - -1, - InvalidOid, - NAMEDATALEN, - collation_schema_datum, - (collation_schema_name == NULL) ? true : false, - false /* passbyval */ - ); - te = makeTargetEntry((Expr *) collation_schema_const, tlist_attno++, NULL, false); - tlist = lappend(tlist, te); - - collation_name_const = makeConst(NAMEOID, - -1, - InvalidOid, - NAMEDATALEN, - collation_name_datum, - (collation_name == NULL) ? true : false, - false /* passbyval */ - ); - te = makeTargetEntry((Expr *) collation_name_const, tlist_attno++, NULL, false); - tlist = lappend(tlist, te); - - input_types_const = makeConst(get_array_type(NAMEOID), - -1, - InvalidOid, - -1, - get_input_types_array_datum(inp), - false, - false /* passbyval */ - ); - te = makeTargetEntry((Expr *) input_types_const, tlist_attno++, NULL, false); - tlist = lappend(tlist, te); - - partial_bytea_var = copyObject(partial_state_var); - te = makeTargetEntry((Expr *) partial_bytea_var, tlist_attno++, NULL, false); - tlist = lappend(tlist, te); - - return_type_const = makeNullConst(inp->aggtype, -1, inp->aggcollid); - te = makeTargetEntry((Expr *) return_type_const, tlist_attno++, NULL, false); - tlist = lappend(tlist, te); - - Assert(tlist_attno == 7); - aggref->args = tlist; - return aggref; -} - -/* - * Creates a partialize expr for the passed in agg: - * partialize_agg(agg). - */ -static FuncExpr * -get_partialize_funcexpr(Aggref *agg) -{ - FuncExpr *partialize_fnexpr; - Oid partfnoid, partargtype; - partargtype = ANYELEMENTOID; - partfnoid = LookupFuncName(list_make2(makeString(INTERNAL_SCHEMA_NAME), makeString(PARTIALFN)), - 1, - &partargtype, - false); - partialize_fnexpr = makeFuncExpr(partfnoid, - BYTEAOID, - list_make1(agg), /*args*/ - InvalidOid, - InvalidOid, - COERCE_EXPLICIT_CALL); - return partialize_fnexpr; -} - -/* - * Check if the supplied OID belongs to a valid bucket function - * for continuous aggregates. - */ -static bool -function_allowed_in_cagg_definition(Oid funcid) -{ - FuncInfo *finfo = ts_func_cache_get_bucketing_func(funcid); - if (finfo == NULL) - return false; - - return finfo->allowed_in_cagg_definition; -} - -/* - * Initialize MatTableColumnInfo. - */ -static void -mattablecolumninfo_init(MatTableColumnInfo *matcolinfo, List *grouplist) -{ - matcolinfo->matcollist = NIL; - matcolinfo->partial_seltlist = NIL; - matcolinfo->partial_grouplist = grouplist; - matcolinfo->mat_groupcolname_list = NIL; - matcolinfo->matpartcolno = -1; - matcolinfo->matpartcolname = NULL; -} - -/* - * Add Information required to create and populate the materialization table columns - * a) create a columndef for the materialization table - * b) create the corresponding expr to populate the column of the materialization table (e..g for a - * column that is an aggref, we create a partialize_agg expr to populate the column Returns: the - * Var corresponding to the newly created column of the materialization table - * - * Notes: make sure the materialization table columns do not save - * values computed by mutable function. - * - * Notes on TargetEntry fields: - * - (resname != NULL) means it's projected in our case - * - (ressortgroupref > 0) means part of GROUP BY, which can be projected or not, depending of the - * value of the resjunk - * - (resjunk == true) applies for GROUP BY columns that are not projected - * - */ -static Var * -mattablecolumninfo_addentry(MatTableColumnInfo *out, Node *input, int original_query_resno, - bool finalized, bool *skip_adding) -{ - int matcolno = list_length(out->matcollist) + 1; - char colbuf[NAMEDATALEN]; - char *colname; - TargetEntry *part_te = NULL; - ColumnDef *col; - Var *var; - Oid coltype, colcollation; - int32 coltypmod; - - *skip_adding = false; - - if (contain_mutable_functions(input)) - { - ereport(ERROR, - (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), - errmsg("only immutable functions supported in continuous aggregate view"), - errhint("Make sure all functions in the continuous aggregate definition" - " have IMMUTABLE volatility. Note that functions or expressions" - " may be IMMUTABLE for one data type, but STABLE or VOLATILE for" - " another."))); - } - - switch (nodeTag(input)) - { - case T_Aggref: - { - FuncExpr *fexpr = get_partialize_funcexpr((Aggref *) input); - PRINT_MATCOLNAME(colbuf, "agg", original_query_resno, matcolno); - colname = colbuf; - coltype = BYTEAOID; - coltypmod = -1; - colcollation = InvalidOid; - col = makeColumnDef(colname, coltype, coltypmod, colcollation); - part_te = makeTargetEntry((Expr *) fexpr, matcolno, pstrdup(colname), false); - } - break; - - case T_TargetEntry: - { - TargetEntry *tle = (TargetEntry *) input; - bool timebkt_chk = false; - - if (IsA(tle->expr, FuncExpr)) - timebkt_chk = function_allowed_in_cagg_definition(((FuncExpr *) tle->expr)->funcid); - - if (tle->resname) - colname = pstrdup(tle->resname); - else - { - if (timebkt_chk) - colname = DEFAULT_MATPARTCOLUMN_NAME; - else - { - PRINT_MATCOLNAME(colbuf, "grp", original_query_resno, matcolno); - colname = colbuf; - - /* For finalized form we skip adding extra group by columns. */ - *skip_adding = finalized; - } - } - - if (timebkt_chk) - { - tle->resname = pstrdup(colname); - out->matpartcolno = matcolno; - out->matpartcolname = pstrdup(colname); - } - else - { - /* - * Add indexes only for columns that are part of the GROUP BY clause - * and for finals form. - * We skip adding it because we'll not add the extra group by columns - * to the materialization hypertable anymore. - */ - if (!*skip_adding && tle->ressortgroupref > 0) - out->mat_groupcolname_list = - lappend(out->mat_groupcolname_list, pstrdup(colname)); - } - - coltype = exprType((Node *) tle->expr); - coltypmod = exprTypmod((Node *) tle->expr); - colcollation = exprCollation((Node *) tle->expr); - col = makeColumnDef(colname, coltype, coltypmod, colcollation); - part_te = (TargetEntry *) copyObject(input); - - /* Keep original resjunk if finalized or not time bucket. */ - if (!finalized || timebkt_chk) - { - /* - * Need to project all the partial entries so that - * materialization table is filled. - */ - part_te->resjunk = false; - } - - part_te->resno = matcolno; - - if (timebkt_chk) - { - col->is_not_null = true; - } - - if (part_te->resname == NULL) - { - part_te->resname = pstrdup(colname); - } - } - break; - - case T_Var: - { - PRINT_MATCOLNAME(colbuf, "var", original_query_resno, matcolno); - colname = colbuf; - - coltype = exprType(input); - coltypmod = exprTypmod(input); - colcollation = exprCollation(input); - col = makeColumnDef(colname, coltype, coltypmod, colcollation); - part_te = makeTargetEntry((Expr *) input, matcolno, pstrdup(colname), false); - - /* Need to project all the partial entries so that materialization table is filled. */ - part_te->resjunk = false; - part_te->resno = matcolno; - } - break; - - default: - elog(ERROR, "invalid node type %d", nodeTag(input)); - break; - } - Assert((!finalized && list_length(out->matcollist) == list_length(out->partial_seltlist)) || - (finalized && list_length(out->matcollist) <= list_length(out->partial_seltlist))); - Assert(col != NULL); - Assert(part_te != NULL); - - if (!*skip_adding) - { - out->matcollist = lappend(out->matcollist, col); - } - - out->partial_seltlist = lappend(out->partial_seltlist, part_te); - - var = makeVar(1, matcolno, coltype, coltypmod, colcollation, 0); - return var; -} - -/* - * Add internal columns for the materialization table. - */ -static void -mattablecolumninfo_addinternal(MatTableColumnInfo *matcolinfo) -{ - Index maxRef; - int colno = list_length(matcolinfo->partial_seltlist) + 1; - ColumnDef *col; - Var *chunkfn_arg1; - FuncExpr *chunk_fnexpr; - Oid chunkfnoid; - Oid argtype[] = { OIDOID }; - Oid rettype = INT4OID; - TargetEntry *chunk_te; - Oid sortop, eqop; - bool hashable; - ListCell *lc; - SortGroupClause *grpcl; - - /* Add a chunk_id column for materialization table */ - Node *vexpr = (Node *) makeVar(1, colno, INT4OID, -1, InvalidOid, 0); - col = makeColumnDef(CONTINUOUS_AGG_CHUNK_ID_COL_NAME, - exprType(vexpr), - exprTypmod(vexpr), - exprCollation(vexpr)); - matcolinfo->matcollist = lappend(matcolinfo->matcollist, col); - - /* - * Need to add an entry to the target list for computing chunk_id column - * : chunk_for_tuple( htid, table.*). - */ - chunkfnoid = - LookupFuncName(list_make2(makeString(INTERNAL_SCHEMA_NAME), makeString(CHUNKIDFROMRELID)), - sizeof(argtype) / sizeof(argtype[0]), - argtype, - false); - chunkfn_arg1 = makeVar(1, TableOidAttributeNumber, OIDOID, -1, 0, 0); - - chunk_fnexpr = makeFuncExpr(chunkfnoid, - rettype, - list_make1(chunkfn_arg1), - InvalidOid, - InvalidOid, - COERCE_EXPLICIT_CALL); - chunk_te = makeTargetEntry((Expr *) chunk_fnexpr, - colno, - pstrdup(CONTINUOUS_AGG_CHUNK_ID_COL_NAME), - false); - matcolinfo->partial_seltlist = lappend(matcolinfo->partial_seltlist, chunk_te); - /* Any internal column needs to be added to the group-by clause as well. */ - maxRef = 0; - foreach (lc, matcolinfo->partial_seltlist) - { - Index ref = ((TargetEntry *) lfirst(lc))->ressortgroupref; - - if (ref > maxRef) - maxRef = ref; - } - chunk_te->ressortgroupref = - maxRef + 1; /* used by sortgroupclause to identify the targetentry */ - grpcl = makeNode(SortGroupClause); - get_sort_group_operators(exprType((Node *) chunk_te->expr), - false, - true, - false, - &sortop, - &eqop, - NULL, - &hashable); - grpcl->tleSortGroupRef = chunk_te->ressortgroupref; - grpcl->eqop = eqop; - grpcl->sortop = sortop; - grpcl->nulls_first = false; - grpcl->hashable = hashable; - - matcolinfo->partial_grouplist = lappend(matcolinfo->partial_grouplist, grpcl); -} - -static Aggref * -add_partialize_column(Aggref *agg_to_partialize, AggPartCxt *cxt) -{ - Aggref *newagg; - Var *var; - bool skip_adding; - - /* - * Step 1: create partialize( aggref) column - * for materialization table. - */ - var = mattablecolumninfo_addentry(cxt->mattblinfo, - (Node *) agg_to_partialize, - cxt->original_query_resno, - false, - &skip_adding); - cxt->added_aggref_col = true; - /* - * Step 2: create finalize_agg expr using var - * for the column added to the materialization table. - */ - - /* This is a var for the column we created. */ - newagg = get_finalize_aggref(agg_to_partialize, var); - return newagg; -} - -static void -set_var_mapping(Var *orig_var, Var *mapped_var, AggPartCxt *cxt) -{ - cxt->orig_vars = lappend(cxt->orig_vars, orig_var); - cxt->mapped_vars = lappend(cxt->mapped_vars, mapped_var); -} - -/* - * Checks whether var has already been mapped and returns the - * corresponding column of the materialization table. - */ -static Var * -var_already_mapped(Var *var, AggPartCxt *cxt) -{ - ListCell *lc_old, *lc_new; - - forboth (lc_old, cxt->orig_vars, lc_new, cxt->mapped_vars) - { - Var *orig_var = (Var *) lfirst_node(Var, lc_old); - Var *mapped_var = (Var *) lfirst_node(Var, lc_new); - - /* There should be no subqueries so varlevelsup should not be a problem here. */ - if (var->varno == orig_var->varno && var->varattno == orig_var->varattno) - return mapped_var; - } - return NULL; -} - -static Node * -add_var_mutator(Node *node, AggPartCxt *cxt) -{ - if (node == NULL) - return NULL; - if (IsA(node, Aggref)) - { - return node; /* don't process this further */ - } - if (IsA(node, Var)) - { - Var *orig_var, *mapped_var; - bool skip_adding = false; - - mapped_var = var_already_mapped((Var *) node, cxt); - /* Avoid duplicating columns in the materialization table. */ - if (mapped_var) - /* - * There should be no subquery so mapped_var->varlevelsup - * should not be a problem here. - */ - return (Node *) copyObject(mapped_var); - - orig_var = (Var *) node; - mapped_var = mattablecolumninfo_addentry(cxt->mattblinfo, - node, - cxt->original_query_resno, - false, - &skip_adding); - set_var_mapping(orig_var, mapped_var, cxt); - return (Node *) mapped_var; - } - return expression_tree_mutator(node, add_var_mutator, cxt); -} - -static Node * -add_aggregate_partialize_mutator(Node *node, AggPartCxt *cxt) -{ - if (node == NULL) - return NULL; - /* - * Modify the aggref and create a partialize(aggref) expr - * for the materialization. - * Add a corresponding columndef for the mat table. - * Replace the aggref with the ts_internal_cagg_final fn. - * using a Var for the corresponding column in the mat table. - * All new Vars have varno = 1 (for RTE 1). - */ - if (IsA(node, Aggref)) - { - if (cxt->ignore_aggoid == ((Aggref *) node)->aggfnoid) - return node; /* don't process this further */ - - Aggref *newagg = add_partialize_column((Aggref *) node, cxt); - return (Node *) newagg; - } - if (IsA(node, Var)) - { - cxt->var_outside_of_aggref = true; - } - return expression_tree_mutator(node, add_aggregate_partialize_mutator, cxt); -} - -typedef struct Cagg_havingcxt -{ - List *origq_tlist; - List *finalizeq_tlist; - AggPartCxt agg_cxt; -} cagg_havingcxt; - -/* - * This function modifies the passed in havingQual by mapping exprs to - * columns in materialization table or finalized aggregate form. - * Note that HAVING clause can contain only exprs from group-by or aggregates - * and GROUP BY clauses cannot be aggregates. - * (By the time we process havingQuals, all the group by exprs have been - * processed and have associated columns in the materialization hypertable). - * Example, if the original query has - * GROUP BY colA + colB, colC - * HAVING colA + colB + sum(colD) > 10 OR count(colE) = 10 - * - * The transformed havingqual would be - * HAVING matCol3 + finalize_agg( sum(matCol4) > 10 - * OR finalize_agg( count(matCol5)) = 10 - * - * - * Note: GROUP BY exprs always appear in the query's targetlist. - * Some of the aggregates from the havingQual might also already appear in the targetlist. - * We replace all existing entries with their corresponding entry from the modified targetlist. - * If an aggregate (in the havingqual) does not exist in the TL, we create a - * materialization table column for it and use the finalize(column) form in the - * transformed havingQual. - */ -static Node * -create_replace_having_qual_mutator(Node *node, cagg_havingcxt *cxt) -{ - if (node == NULL) - return NULL; - /* - * See if we already have a column in materialization hypertable for this - * expr. We do this by checking the existing targetlist - * entries for the query. - */ - ListCell *lc, *lc2; - List *origtlist = cxt->origq_tlist; - List *modtlist = cxt->finalizeq_tlist; - forboth (lc, origtlist, lc2, modtlist) - { - TargetEntry *te = (TargetEntry *) lfirst(lc); - TargetEntry *modte = (TargetEntry *) lfirst(lc2); - if (equal(node, te->expr)) - { - return (Node *) modte->expr; - } - } - /* - * Didn't find a match in targetlist. If it is an aggregate, - * create a partialize column for it in materialization hypertable - * and return corresponding finalize expr. - */ - if (IsA(node, Aggref)) - { - AggPartCxt *agg_cxt = &(cxt->agg_cxt); - agg_cxt->added_aggref_col = false; - Aggref *newagg = add_partialize_column((Aggref *) node, agg_cxt); - Assert(agg_cxt->added_aggref_col == true); - return (Node *) newagg; - } - return expression_tree_mutator(node, create_replace_having_qual_mutator, cxt); -} - -static Node * -finalizequery_create_havingqual(FinalizeQueryInfo *inp, MatTableColumnInfo *mattblinfo) -{ - Query *orig_query = inp->final_userquery; - if (orig_query->havingQual == NULL) - return NULL; - Node *havingQual = copyObject(orig_query->havingQual); - Assert(inp->final_seltlist != NULL); - cagg_havingcxt hcxt = { .origq_tlist = orig_query->targetList, - .finalizeq_tlist = inp->final_seltlist, - .agg_cxt.mattblinfo = mattblinfo, - .agg_cxt.original_query_resno = 0, - .agg_cxt.ignore_aggoid = get_finalizefnoid(), - .agg_cxt.added_aggref_col = false, - .agg_cxt.var_outside_of_aggref = false, - .agg_cxt.orig_vars = NIL, - .agg_cxt.mapped_vars = NIL }; - return create_replace_having_qual_mutator(havingQual, &hcxt); -} - -/* - * Init the finalize query data structure. - * Parameters: - * orig_query - the original query from user view that is being used as template for the finalize - * query tlist_aliases - aliases for the view select list materialization table columns are created - * . This will be returned in the mattblinfo - * - * DO NOT modify orig_query. Make a copy if needed. - * SIDE_EFFECT: the data structure in mattblinfo is modified as a side effect by adding new - * materialize table columns and partialize exprs. - */ -static void -finalizequery_init(FinalizeQueryInfo *inp, Query *orig_query, MatTableColumnInfo *mattblinfo) -{ - AggPartCxt cxt; - ListCell *lc; - int resno = 1; - - inp->final_userquery = copyObject(orig_query); - inp->final_seltlist = NIL; - inp->final_havingqual = NULL; - - /* Set up the final_seltlist and final_havingqual entries */ - cxt.mattblinfo = mattblinfo; - cxt.ignore_aggoid = InvalidOid; - - /* Set up the left over variable mapping lists */ - cxt.orig_vars = NIL; - cxt.mapped_vars = NIL; - - /* - * We want all the entries in the targetlist (resjunk or not) - * in the materialization table definition so we include group-by/having clause etc. - * We have to do 3 things here: - * 1) create a column for mat table - * 2) partialize_expr to populate it, and - * 3) modify the target entry to be a finalize_expr - * that selects from the materialization table. - */ - foreach (lc, orig_query->targetList) - { - TargetEntry *tle = (TargetEntry *) lfirst(lc); - TargetEntry *modte = copyObject(tle); - cxt.added_aggref_col = false; - cxt.var_outside_of_aggref = false; - cxt.original_query_resno = resno; - - if (!inp->finalized) - { - /* - * If tle has aggrefs, get the corresponding - * finalize_agg expression and save it in modte. - * Also add correspong materialization table column info - * for the aggrefs in tle. - */ - modte = (TargetEntry *) expression_tree_mutator((Node *) modte, - add_aggregate_partialize_mutator, - &cxt); - } - - /* - * We need columns for non-aggregate targets. - * If it is not a resjunk OR appears in the grouping clause. - */ - if (cxt.added_aggref_col == false && (tle->resjunk == false || tle->ressortgroupref > 0)) - { - Var *var; - bool skip_adding = false; - var = mattablecolumninfo_addentry(cxt.mattblinfo, - (Node *) tle, - cxt.original_query_resno, - inp->finalized, - &skip_adding); - - /* Skip adding this column for finalized form. */ - if (skip_adding) - { - continue; - } - - /* Fix the expression for the target entry. */ - modte->expr = (Expr *) var; - } - /* Check for left over variables (Var) of targets that contain Aggref. */ - if (cxt.added_aggref_col && cxt.var_outside_of_aggref && !inp->finalized) - { - modte = (TargetEntry *) expression_tree_mutator((Node *) modte, add_var_mutator, &cxt); - } - /* - * Construct the targetlist for the query on the - * materialization table. The TL maps 1-1 with the original query: - * e.g select a, min(b)+max(d) from foo group by a,timebucket(a); - * becomes - * select , - * ts_internal_cagg_final(..b-col ) + ts_internal_cagg_final(..d-col) - * from mattbl - * group by a-col, timebucket(a-col) - */ - - /* - * We copy the modte target entries, resnos should be the same for - * final_selquery and origquery. So tleSortGroupReffor the targetentry - * can be reused, only table info needs to be modified. - */ - Assert((!inp->finalized && modte->resno == resno) || - (inp->finalized && modte->resno >= resno)); - resno++; - if (IsA(modte->expr, Var)) - { - modte->resorigcol = ((Var *) modte->expr)->varattno; - } - inp->final_seltlist = lappend(inp->final_seltlist, modte); - } - /* - * All grouping clause elements are in targetlist already. - * So let's check the having clause. - */ - if (!inp->finalized) - inp->final_havingqual = finalizequery_create_havingqual(inp, mattblinfo); -} - -/* - * Create select query with the finalize aggregates - * for the materialization table. - * matcollist - column list for mat table - * mattbladdress - materialization table ObjectAddress - * This is the function responsible for creating the final - * structures for selecting from the materialized hypertable - * created for the Cagg which is - * select * from _timescaldeb_internal._materialized_hypertable_ - */ -static Query * -finalizequery_get_select_query(FinalizeQueryInfo *inp, List *matcollist, - ObjectAddress *mattbladdress, char *relname) -{ - Query *final_selquery = NULL; - ListCell *lc; - FromExpr *fromexpr; - RangeTblEntry *rte; - - /* - * For initial cagg creation rtable will have only 1 entry, - * for alter table rtable will have multiple entries with our - * RangeTblEntry as last member. - * For cagg with joins, we need to create a new RTE and jointree - * which contains the information of the materialised hypertable - * that is created for this cagg. - */ - if (list_length(inp->final_userquery->jointree->fromlist) >= - CONTINUOUS_AGG_MAX_JOIN_RELATIONS || - !IsA(linitial(inp->final_userquery->jointree->fromlist), RangeTblRef)) - { - rte = makeNode(RangeTblEntry); - rte->alias = makeAlias(relname, NIL); - rte->inFromCl = true; - rte->inh = true; - rte->rellockmode = 1; - rte->eref = copyObject(rte->alias); - ListCell *l; - foreach (l, inp->final_userquery->jointree->fromlist) - { - /* - * In case of joins, update the rte with all the join related struct. - */ - Node *jtnode = (Node *) lfirst(l); - JoinExpr *join = NULL; - if (IsA(jtnode, JoinExpr)) - { - join = castNode(JoinExpr, jtnode); - RangeTblEntry *jrte = rt_fetch(join->rtindex, inp->final_userquery->rtable); - rte->joinaliasvars = jrte->joinaliasvars; - rte->jointype = jrte->jointype; -#if PG13_GE - rte->joinleftcols = jrte->joinleftcols; - rte->joinrightcols = jrte->joinrightcols; - rte->joinmergedcols = jrte->joinmergedcols; -#endif -#if PG14_GE - rte->join_using_alias = jrte->join_using_alias; -#endif - rte->selectedCols = jrte->selectedCols; - } - } - } - else - { - rte = llast_node(RangeTblEntry, inp->final_userquery->rtable); - rte->eref->colnames = NIL; - rte->selectedCols = NULL; - } - if (rte->eref->colnames == NIL) - { - /* - * We only need to do this for the case when there is no Join node in the query. - * In the case of join, rte->eref is already populated by jrte->eref and hence the - * relevant info, so need not to do this. - */ - - /* Aliases for column names for the materialization table. */ - foreach (lc, matcollist) - { - ColumnDef *cdef = lfirst_node(ColumnDef, lc); - rte->eref->colnames = lappend(rte->eref->colnames, makeString(cdef->colname)); - rte->selectedCols = bms_add_member(rte->selectedCols, - list_length(rte->eref->colnames) - - FirstLowInvalidHeapAttributeNumber); - } - } - rte->relid = mattbladdress->objectId; - rte->rtekind = RTE_RELATION; - rte->relkind = RELKIND_RELATION; - rte->tablesample = NULL; - rte->requiredPerms |= ACL_SELECT; - rte->insertedCols = NULL; - rte->updatedCols = NULL; - - /* 2. Fixup targetlist with the correct rel information. */ - foreach (lc, inp->final_seltlist) - { - TargetEntry *tle = (TargetEntry *) lfirst(lc); - /* - * In case when this is a cagg wth joins, the Var from the normal table - * already has resorigtbl populated and we need to use that to resolve - * the Var. Hence only modify the tle when resorigtbl is unset - * which means it is Var of the Hypertable - */ - if (IsA(tle->expr, Var) && !OidIsValid(tle->resorigtbl)) - { - tle->resorigtbl = rte->relid; - tle->resorigcol = ((Var *) tle->expr)->varattno; - } - } - - CAGG_MAKEQUERY(final_selquery, inp->final_userquery); - final_selquery->hasAggs = !inp->finalized; - if (list_length(inp->final_userquery->jointree->fromlist) >= - CONTINUOUS_AGG_MAX_JOIN_RELATIONS || - !IsA(linitial(inp->final_userquery->jointree->fromlist), RangeTblRef)) - { - RangeTblRef *rtr; - final_selquery->rtable = list_make1(rte); - rtr = makeNode(RangeTblRef); - rtr->rtindex = 1; - fromexpr = makeFromExpr(list_make1(rtr), NULL); - } - else - { - final_selquery->rtable = inp->final_userquery->rtable; - fromexpr = inp->final_userquery->jointree; - fromexpr->quals = NULL; - } - - /* - * Fixup from list. No quals on original table should be - * present here - they should be on the query that populates - * the mattable (partial_selquery). For the Cagg with join, - * we can not copy the fromlist from inp->final_userquery as - * it has two tables in this case. - */ - Assert(list_length(inp->final_userquery->jointree->fromlist) <= - CONTINUOUS_AGG_MAX_JOIN_RELATIONS); - - final_selquery->jointree = fromexpr; - final_selquery->targetList = inp->final_seltlist; - final_selquery->sortClause = inp->final_userquery->sortClause; - - if (!inp->finalized) - { - final_selquery->groupClause = inp->final_userquery->groupClause; - /* Copy the having clause too */ - final_selquery->havingQual = inp->final_havingqual; - } - - return final_selquery; -} - /* * Assign aliases to the targetlist in the query according to the * column_names provided in the CREATE VIEW statement. @@ -2756,7 +742,7 @@ cagg_create(const CreateTableAsStmt *create_stmt, ViewStmt *stmt, Query *panquer ts_catalog_database_info_become_owner(ts_catalog_database_info_get(), &sec_ctx); materialize_hypertable_id = ts_catalog_table_next_seq_id(ts_catalog_get(), HYPERTABLE); ts_catalog_restore_user(&sec_ctx); - PRINT_MATINTERNAL_NAME(relnamebuf, "_materialized_hypertable_%d", materialize_hypertable_id); + makeMaterializedTableName(relnamebuf, "_materialized_hypertable_%d", materialize_hypertable_id); mat_rel = makeRangeVar(pstrdup(INTERNAL_SCHEMA_NAME), pstrdup(relnamebuf), -1); is_create_mattbl_index = DatumGetBool(with_clause_options[ContinuousViewOptionCreateGroupIndex].parsed); @@ -2793,7 +779,7 @@ cagg_create(const CreateTableAsStmt *create_stmt, ViewStmt *stmt, Query *panquer partial_selquery = mattablecolumninfo_get_partial_select_query(&mattblinfo, panquery, finalqinfo.finalized); - PRINT_MATINTERNAL_NAME(relnamebuf, "_partial_view_%d", materialize_hypertable_id); + makeMaterializedTableName(relnamebuf, "_partial_view_%d", materialize_hypertable_id); part_rel = makeRangeVar(pstrdup(INTERNAL_SCHEMA_NAME), pstrdup(relnamebuf), -1); create_view_for_query(partial_selquery, part_rel); @@ -2807,7 +793,7 @@ cagg_create(const CreateTableAsStmt *create_stmt, ViewStmt *stmt, Query *panquer * having to replicate the PG source code for make_viewdef. */ orig_userview_query = copyObject(panquery); - PRINT_MATINTERNAL_NAME(relnamebuf, "_direct_view_%d", materialize_hypertable_id); + makeMaterializedTableName(relnamebuf, "_direct_view_%d", materialize_hypertable_id); dum_rel = makeRangeVar(pstrdup(INTERNAL_SCHEMA_NAME), pstrdup(relnamebuf), -1); create_view_for_query(orig_userview_query, dum_rel); /* Step 4: Add catalog table entry for the objects we just created. */ @@ -2981,259 +967,6 @@ tsl_process_continuous_agg_viewstmt(Node *node, const char *query_string, void * return DDL_DONE; } -/* - * When a view is created (StoreViewQuery), 2 dummy rtable entries corresponding to "old" and - * "new" are prepended to the rtable list. We remove these and adjust the varnos to recreate - * the user or direct view query. - */ -static void -remove_old_and_new_rte_from_query(Query *query) -{ - List *rtable = query->rtable; - Assert(list_length(rtable) >= 3); - rtable = list_delete_first(rtable); - query->rtable = list_delete_first(rtable); - OffsetVarNodes((Node *) query, -2, 0); - Assert(list_length(query->rtable) >= 1); -} - -/* - * Test the view definition of an existing continuous aggregate - * for errors and attempt to rebuild it if required. - */ -static void -cagg_rebuild_view_definition(ContinuousAgg *agg, Hypertable *mat_ht, bool force_rebuild) -{ - bool test_failed = false; - char *relname = agg->data.user_view_name.data; - char *schema = agg->data.user_view_schema.data; - ListCell *lc1, *lc2; - int sec_ctx; - Oid uid, saved_uid; - - /* Cagg view created by the user. */ - Oid user_view_oid = ts_get_relation_relid(NameStr(agg->data.user_view_schema), - NameStr(agg->data.user_view_name), - false); - Relation user_view_rel = relation_open(user_view_oid, AccessShareLock); - Query *user_query = get_view_query(user_view_rel); - - bool finalized = ContinuousAggIsFinalized(agg); - bool rebuild_cagg_with_joins = false; - - /* Extract final query from user view query. */ - Query *final_query = copyObject(user_query); - remove_old_and_new_rte_from_query(final_query); - - if (finalized && !force_rebuild) - { - /* This continuous aggregate does not have partials, do not check for defects. */ - elog(DEBUG1, - "[cagg_rebuild_view_definition] %s.%s does not have partials, do not check for " - "defects!", - NameStr(agg->data.user_view_schema), - NameStr(agg->data.user_view_name) - - ); - relation_close(user_view_rel, NoLock); - return; - } - - if (!agg->data.materialized_only) - { - final_query = destroy_union_query(final_query); - } - FinalizeQueryInfo fqi; - MatTableColumnInfo mattblinfo; - ObjectAddress mataddress = { - .classId = RelationRelationId, - .objectId = mat_ht->main_table_relid, - }; - - Oid direct_view_oid = ts_get_relation_relid(NameStr(agg->data.direct_view_schema), - NameStr(agg->data.direct_view_name), - false); - Relation direct_view_rel = relation_open(direct_view_oid, AccessShareLock); - Query *direct_query = copyObject(get_view_query(direct_view_rel)); - remove_old_and_new_rte_from_query(direct_query); - - /* - * If there is a join in CAggs then rebuild it definitley, - * because v2.10.0 has created the definition with missing structs. - * - * Removed the check for direct_query->jointree != NULL because - * we don't allow queries without FROM clause in Continuous Aggregate - * definition. - * - * Per coverityscan: - * https://scan4.scan.coverity.com/reports.htm#v54116/p12995/fileInstanceId=131745632&defectInstanceId=14569562&mergedDefectId=384045 - * - */ - if (force_rebuild) - { - ListCell *l; - foreach (l, direct_query->jointree->fromlist) - { - Node *jtnode = (Node *) lfirst(l); - if (IsA(jtnode, JoinExpr)) - rebuild_cagg_with_joins = true; - } - } - - if (!rebuild_cagg_with_joins && finalized) - { - /* There's nothing to fix, so no need to rebuild */ - elog(DEBUG1, - "[cagg_rebuild_view_definition] %s.%s does not have JOINS, so no need to rebuild the " - "definition!", - NameStr(agg->data.user_view_schema), - NameStr(agg->data.user_view_name) - - ); - relation_close(user_view_rel, NoLock); - relation_close(direct_view_rel, NoLock); - return; - } - else - elog(DEBUG1, - "[cagg_rebuild_view_definition] %s.%s has been rebuilt!", - NameStr(agg->data.user_view_schema), - NameStr(agg->data.user_view_name)); - - CAggTimebucketInfo timebucket_exprinfo = - cagg_validate_query(direct_query, - finalized, - NameStr(agg->data.user_view_schema), - NameStr(agg->data.user_view_name)); - - mattablecolumninfo_init(&mattblinfo, copyObject(direct_query->groupClause)); - fqi.finalized = finalized; - finalizequery_init(&fqi, direct_query, &mattblinfo); - - /* - * Add any internal columns needed for materialization based - * on the user query's table. - */ - if (!finalized) - mattablecolumninfo_addinternal(&mattblinfo); - - Query *view_query = NULL; - if (rebuild_cagg_with_joins) - { - view_query = finalizequery_get_select_query(&fqi, - mattblinfo.matcollist, - &mataddress, - NameStr(mat_ht->fd.table_name)); - } - else - view_query = - finalizequery_get_select_query(&fqi, mattblinfo.matcollist, &mataddress, relname); - - if (!agg->data.materialized_only) - { - view_query = build_union_query(&timebucket_exprinfo, - mattblinfo.matpartcolno, - view_query, - direct_query, - mat_ht->fd.id); - } - - if (list_length(mattblinfo.matcollist) != ts_get_relnatts(mat_ht->main_table_relid)) - /* - * There is a mismatch of columns between the current version's finalization view - * building logic and the existing schema of the materialization table. As of version - * 2.7.0 this only happens due to buggy view generation in previous versions. Do not - * rebuild those views since the materialization table can not be queried correctly. - */ - test_failed = true; - /* - * When calling StoreViewQuery the target list names of the query have to - * match the view's tuple descriptor attribute names. But if a column of the continuous - * aggregate has been renamed, the query tree will not have the correct - * names in the target list, which will error out when calling - * StoreViewQuery. For that reason, we fetch the name from the user view - * relation and update the resource name in the query target list to match - * the name in the user view. - */ - TupleDesc desc = RelationGetDescr(user_view_rel); - int i = 0; - forboth (lc1, view_query->targetList, lc2, user_query->targetList) - { - TargetEntry *view_tle, *user_tle; - FormData_pg_attribute *attr = TupleDescAttr(desc, i); - view_tle = lfirst_node(TargetEntry, lc1); - user_tle = lfirst_node(TargetEntry, lc2); - if (view_tle->resjunk && user_tle->resjunk) - break; - else if (view_tle->resjunk || user_tle->resjunk) - { - /* - * This should never happen but if it ever does it's safer to - * error here instead of creating broken view definitions. - */ - test_failed = true; - break; - } - view_tle->resname = user_tle->resname = NameStr(attr->attname); - ++i; - } - - if (test_failed) - { - ereport(WARNING, - (errmsg("Inconsistent view definitions for continuous aggregate view " - "\"%s.%s\"", - schema, - relname), - errdetail("Continuous aggregate data possibly corrupted."), - errhint("You may need to recreate the continuous aggregate with CREATE " - "MATERIALIZED VIEW."))); - } - else - { - SWITCH_TO_TS_USER(NameStr(agg->data.user_view_schema), uid, saved_uid, sec_ctx); - StoreViewQuery(user_view_oid, view_query, true); - CommandCounterIncrement(); - RESTORE_USER(uid, saved_uid, sec_ctx); - } - /* - * Keep locks until end of transaction and do not close the relation - * before the call to StoreViewQuery since it can otherwise release the - * memory for attr->attname, causing a segfault. - */ - relation_close(direct_view_rel, NoLock); - relation_close(user_view_rel, NoLock); -} - -Datum -tsl_cagg_try_repair(PG_FUNCTION_ARGS) -{ - Oid relid = PG_ARGISNULL(0) ? InvalidOid : PG_GETARG_OID(0); - char relkind = get_rel_relkind(relid); - bool force_rebuild = PG_ARGISNULL(0) ? false : PG_GETARG_BOOL(1); - ContinuousAgg *cagg = NULL; - - if (RELKIND_VIEW == relkind) - cagg = ts_continuous_agg_find_by_relid(relid); - - if (RELKIND_VIEW != relkind || !cagg) - { - ereport(WARNING, - (errmsg("invalid OID \"%u\" for continuous aggregate view", relid), - errdetail("Check for database corruption."))); - PG_RETURN_VOID(); - } - - Cache *hcache = ts_hypertable_cache_pin(); - - Hypertable *mat_ht = ts_hypertable_cache_get_entry_by_id(hcache, cagg->data.mat_hypertable_id); - Assert(mat_ht != NULL); - cagg_rebuild_view_definition(cagg, mat_ht, force_rebuild); - - ts_cache_release(hcache); - - PG_RETURN_VOID(); -} /* * Flip the view definition of an existing continuous aggregate from * real-time to materialized-only or vice versa depending on the current state. @@ -3253,7 +986,7 @@ cagg_flip_realtime_view_definition(ContinuousAgg *agg, Hypertable *mat_ht) Query *user_query = copyObject(get_view_query(user_view_rel)); /* Keep lock until end of transaction. */ relation_close(user_view_rel, NoLock); - remove_old_and_new_rte_from_query(user_query); + RemoveRangeTableEntries(user_query); /* Direct view query of the original user view definition at CAGG creation. */ Oid direct_view_oid = ts_get_relation_relid(NameStr(agg->data.direct_view_schema), @@ -3263,7 +996,7 @@ cagg_flip_realtime_view_definition(ContinuousAgg *agg, Hypertable *mat_ht) Query *direct_query = copyObject(get_view_query(direct_view_rel)); /* Keep lock until end of transaction. */ relation_close(direct_view_rel, NoLock); - remove_old_and_new_rte_from_query(direct_query); + RemoveRangeTableEntries(direct_query); CAggTimebucketInfo timebucket_exprinfo = cagg_validate_query(direct_query, @@ -3306,7 +1039,7 @@ cagg_rename_view_columns(ContinuousAgg *agg) false); Relation user_view_rel = relation_open(user_view_oid, AccessShareLock); Query *user_query = copyObject(get_view_query(user_view_rel)); - remove_old_and_new_rte_from_query(user_query); + RemoveRangeTableEntries(user_query); /* * When calling StoreViewQuery the target list names of the query have to @@ -3342,391 +1075,3 @@ cagg_rename_view_columns(ContinuousAgg *agg) */ relation_close(user_view_rel, NoLock); } - -/* - * Create Const of proper type for lower bound of watermark when - * watermark has not been set yet. - */ -static Const * -cagg_boundary_make_lower_bound(Oid type) -{ - Datum value; - int16 typlen; - bool typbyval; - - get_typlenbyval(type, &typlen, &typbyval); - value = ts_time_datum_get_nobegin_or_min(type); - - return makeConst(type, -1, InvalidOid, typlen, value, false, typbyval); -} - -/* - * Get oid of function to convert from our internal representation - * to postgres representation. - */ -static Oid -cagg_get_boundary_converter_funcoid(Oid typoid) -{ - char *function_name; - Oid argtyp[] = { INT8OID }; - - switch (typoid) - { - case DATEOID: - function_name = INTERNAL_TO_DATE_FUNCTION; - break; - case TIMESTAMPOID: - function_name = INTERNAL_TO_TS_FUNCTION; - break; - case TIMESTAMPTZOID: - function_name = INTERNAL_TO_TSTZ_FUNCTION; - break; - default: - /* - * This should never be reached and unsupported datatypes - * should be caught at much earlier stages. - */ - ereport(ERROR, - (errcode(ERRCODE_TS_INTERNAL_ERROR), - errmsg("no converter function defined for datatype: %s", - format_type_be(typoid)))); - pg_unreachable(); - } - - List *func_name = list_make2(makeString(INTERNAL_SCHEMA_NAME), makeString(function_name)); - Oid converter_oid = LookupFuncName(func_name, lengthof(argtyp), argtyp, false); - - Assert(OidIsValid(converter_oid)); - - return converter_oid; -} - -static FuncExpr * -build_conversion_call(Oid type, FuncExpr *boundary) -{ - /* - * If the partitioning column type is not integer we need to convert - * to proper representation. - */ - switch (type) - { - case INT2OID: - case INT4OID: - { - /* Since the boundary function returns int8 we need to cast to proper type here. */ - Oid cast_oid = ts_get_cast_func(INT8OID, type); - - return makeFuncExpr(cast_oid, - type, - list_make1(boundary), - InvalidOid, - InvalidOid, - COERCE_IMPLICIT_CAST); - } - case INT8OID: - /* Nothing to do for int8. */ - return boundary; - case DATEOID: - case TIMESTAMPOID: - case TIMESTAMPTZOID: - { - /* - * date/timestamp/timestamptz need to be converted since - * we store them differently from postgres format. - */ - Oid converter_oid = cagg_get_boundary_converter_funcoid(type); - return makeFuncExpr(converter_oid, - type, - list_make1(boundary), - InvalidOid, - InvalidOid, - COERCE_EXPLICIT_CALL); - } - - default: - /* - * All valid types should be handled above, this should - * never be reached and error handling at earlier stages - * should catch this. - */ - ereport(ERROR, - (errcode(ERRCODE_TS_INTERNAL_ERROR), - errmsg("unsupported datatype for continuous aggregates: %s", - format_type_be(type)))); - pg_unreachable(); - } -} - -/* - * Build function call that returns boundary for a hypertable - * wrapped in type conversion calls when required. - */ -static FuncExpr * -build_boundary_call(int32 ht_id, Oid type) -{ - Oid argtyp[] = { INT4OID }; - FuncExpr *boundary; - - Oid boundary_func_oid = - LookupFuncName(list_make2(makeString(INTERNAL_SCHEMA_NAME), makeString(BOUNDARY_FUNCTION)), - lengthof(argtyp), - argtyp, - false); - List *func_args = - list_make1(makeConst(INT4OID, -1, InvalidOid, 4, Int32GetDatum(ht_id), false, true)); - - boundary = makeFuncExpr(boundary_func_oid, - INT8OID, - func_args, - InvalidOid, - InvalidOid, - COERCE_EXPLICIT_CALL); - - return build_conversion_call(type, boundary); -} - -static Node * -build_union_query_quals(int32 ht_id, Oid partcoltype, Oid opno, int varno, AttrNumber attno) -{ - Var *var = makeVar(varno, attno, partcoltype, -1, InvalidOid, InvalidOid); - FuncExpr *boundary = build_boundary_call(ht_id, partcoltype); - - CoalesceExpr *coalesce = makeNode(CoalesceExpr); - coalesce->coalescetype = partcoltype; - coalesce->coalescecollid = InvalidOid; - coalesce->args = list_make2(boundary, cagg_boundary_make_lower_bound(partcoltype)); - - return (Node *) make_opclause(opno, - BOOLOID, - false, - (Expr *) var, - (Expr *) coalesce, - InvalidOid, - InvalidOid); -} - -static RangeTblEntry * -make_subquery_rte(Query *subquery, const char *aliasname) -{ - RangeTblEntry *rte = makeNode(RangeTblEntry); - ListCell *lc; - - rte->rtekind = RTE_SUBQUERY; - rte->relid = InvalidOid; - rte->subquery = subquery; - rte->alias = makeAlias(aliasname, NIL); - rte->eref = copyObject(rte->alias); - - foreach (lc, subquery->targetList) - { - TargetEntry *tle = lfirst_node(TargetEntry, lc); - if (!tle->resjunk) - rte->eref->colnames = lappend(rte->eref->colnames, makeString(pstrdup(tle->resname))); - } - - rte->lateral = false; - rte->inh = false; /* never true for subqueries */ - rte->inFromCl = true; - - return rte; -} - -/* - * Build union query combining the materialized data with data from the raw data hypertable. - * - * q1 is the query on the materialization hypertable with the finalize call - * q2 is the query on the raw hypertable which was supplied in the inital CREATE VIEW statement - * returns a query as - * SELECT * from ( SELECT * from q1 where - * UNION ALL - * SELECT * from q2 where existing_qual and - * where coale_qual is: time < ----> (or >= ) - * COALESCE(_timescaledb_internal.to_timestamp(_timescaledb_internal.cagg_watermark( )), - * '-infinity'::timestamp with time zone) - * See build_union_quals for COALESCE clauses. - */ -static Query * -build_union_query(CAggTimebucketInfo *tbinfo, int matpartcolno, Query *q1, Query *q2, - int materialize_htid) -{ - ListCell *lc1, *lc2; - List *col_types = NIL; - List *col_typmods = NIL; - List *col_collations = NIL; - List *tlist = NIL; - List *sortClause = NIL; - int varno; - Node *q2_quals = NULL; - - Assert(list_length(q1->targetList) <= list_length(q2->targetList)); - - q1 = copyObject(q1); - q2 = copyObject(q2); - - if (q1->sortClause) - sortClause = copyObject(q1->sortClause); - - TypeCacheEntry *tce = lookup_type_cache(tbinfo->htpartcoltype, TYPECACHE_LT_OPR); - - varno = list_length(q1->rtable); - q1->jointree->quals = build_union_query_quals(materialize_htid, - tbinfo->htpartcoltype, - tce->lt_opr, - varno, - matpartcolno); - /* - * If there is join in CAgg definition then adjust varno - * to get time column from the hypertable in the join. - */ - - /* - * In case of joins it is enough to check if the first node is not RangeTblRef, - * because the jointree has RangeTblRef as leaves and JoinExpr above them. - * So if JoinExpr is present, it is the first node. - * Other cases of join i.e. without explicit JOIN clause is confirmed - * by reading the length of rtable. - */ - if (list_length(q2->rtable) == CONTINUOUS_AGG_MAX_JOIN_RELATIONS || - !IsA(linitial(q2->jointree->fromlist), RangeTblRef)) - { - Oid normal_table_id = InvalidOid; - RangeTblEntry *rte = NULL; - RangeTblEntry *rte_other = NULL; - - if (list_length(q2->rtable) == CONTINUOUS_AGG_MAX_JOIN_RELATIONS) - { - RangeTblRef *rtref = linitial_node(RangeTblRef, q2->jointree->fromlist); - rte = list_nth(q2->rtable, rtref->rtindex - 1); - RangeTblRef *rtref_other = lsecond_node(RangeTblRef, q2->jointree->fromlist); - rte_other = list_nth(q2->rtable, rtref_other->rtindex - 1); - } - else if (!IsA(linitial(q2->jointree->fromlist), RangeTblRef)) - { - ListCell *l; - foreach (l, q2->jointree->fromlist) - { - Node *jtnode = (Node *) lfirst(l); - JoinExpr *join = NULL; - if (IsA(jtnode, JoinExpr)) - { - join = castNode(JoinExpr, jtnode); - rte = list_nth(q2->rtable, ((RangeTblRef *) join->larg)->rtindex - 1); - rte_other = list_nth(q2->rtable, ((RangeTblRef *) join->rarg)->rtindex - 1); - } - } - } - if (rte->relkind == RELKIND_VIEW) - normal_table_id = rte_other->relid; - else if (rte_other->relkind == RELKIND_VIEW) - normal_table_id = rte->relid; - else - normal_table_id = ts_is_hypertable(rte->relid) ? rte_other->relid : rte->relid; - if (normal_table_id == rte->relid) - varno = 2; - else - varno = 1; - } - else - varno = list_length(q2->rtable); - q2_quals = build_union_query_quals(materialize_htid, - tbinfo->htpartcoltype, - get_negator(tce->lt_opr), - varno, - tbinfo->htpartcolno); - q2->jointree->quals = make_and_qual(q2->jointree->quals, q2_quals); - - Query *query = makeNode(Query); - SetOperationStmt *setop = makeNode(SetOperationStmt); - RangeTblEntry *rte_q1 = make_subquery_rte(q1, "*SELECT* 1"); - RangeTblEntry *rte_q2 = make_subquery_rte(q2, "*SELECT* 2"); - RangeTblRef *ref_q1 = makeNode(RangeTblRef); - RangeTblRef *ref_q2 = makeNode(RangeTblRef); - - query->commandType = CMD_SELECT; - query->rtable = list_make2(rte_q1, rte_q2); - query->setOperations = (Node *) setop; - - setop->op = SETOP_UNION; - setop->all = true; - ref_q1->rtindex = 1; - ref_q2->rtindex = 2; - setop->larg = (Node *) ref_q1; - setop->rarg = (Node *) ref_q2; - - forboth (lc1, q1->targetList, lc2, q2->targetList) - { - TargetEntry *tle = lfirst_node(TargetEntry, lc1); - TargetEntry *tle2 = lfirst_node(TargetEntry, lc2); - TargetEntry *tle_union; - Var *expr; - - if (!tle->resjunk) - { - col_types = lappend_int(col_types, exprType((Node *) tle->expr)); - col_typmods = lappend_int(col_typmods, exprTypmod((Node *) tle->expr)); - col_collations = lappend_int(col_collations, exprCollation((Node *) tle->expr)); - - expr = makeVarFromTargetEntry(1, tle); - /* - * We need to use resname from q2 because that is the query from the - * initial CREATE VIEW statement so the VIEW can be updated in place. - */ - tle_union = makeTargetEntry((Expr *) copyObject(expr), - list_length(tlist) + 1, - tle2->resname, - false); - tle_union->resorigtbl = expr->varno; - tle_union->resorigcol = expr->varattno; - tle_union->ressortgroupref = tle->ressortgroupref; - - tlist = lappend(tlist, tle_union); - } - } - - query->targetList = tlist; - - if (sortClause) - { - query->sortClause = sortClause; - query->jointree = makeFromExpr(NIL, NULL); - } - - setop->colTypes = col_types; - setop->colTypmods = col_typmods; - setop->colCollations = col_collations; - - return query; -} - -/* - * Extract the final view from the UNION ALL query. - * - * q1 is the query on the materialization hypertable with the finalize call - * q2 is the query on the raw hypertable which was supplied in the inital CREATE VIEW statement - * returns q1 from: - * SELECT * from ( SELECT * from q1 where - * UNION ALL - * SELECT * from q2 where existing_qual and - * where coale_qual is: time < ----> (or >= ) - * COALESCE(_timescaledb_internal.to_timestamp(_timescaledb_internal.cagg_watermark( )), - * '-infinity'::timestamp with time zone) - * The WHERE clause of the final view is removed. - */ -static Query * -destroy_union_query(Query *q) -{ - Assert(q->commandType == CMD_SELECT && - ((SetOperationStmt *) q->setOperations)->op == SETOP_UNION && - ((SetOperationStmt *) q->setOperations)->all == true); - - /* Get RTE of the left-hand side of UNION ALL. */ - RangeTblEntry *rte = linitial(q->rtable); - Assert(rte->rtekind == RTE_SUBQUERY); - - Query *query = copyObject(rte->subquery); - - /* Delete the WHERE clause from the final view. */ - query->jointree->quals = NULL; - - return query; -} diff --git a/tsl/src/continuous_aggs/create.h b/tsl/src/continuous_aggs/create.h index b22bdc8eb..661ff0afc 100644 --- a/tsl/src/continuous_aggs/create.h +++ b/tsl/src/continuous_aggs/create.h @@ -11,14 +11,10 @@ #include "with_clause_parser.h" #include "ts_catalog/continuous_agg.h" -#define CONTINUOUS_AGG_CHUNK_ID_COL_NAME "chunk_id" - DDLResult tsl_process_continuous_agg_viewstmt(Node *node, const char *query_string, void *pstmt, WithClauseResult *with_clause_options); extern void cagg_flip_realtime_view_definition(ContinuousAgg *agg, Hypertable *mat_ht); extern void cagg_rename_view_columns(ContinuousAgg *agg); -extern Datum tsl_cagg_try_repair(PG_FUNCTION_ARGS); - #endif /* TIMESCALEDB_TSL_CONTINUOUS_AGGS_CAGG_CREATE_H */ diff --git a/tsl/src/continuous_aggs/finalize.c b/tsl/src/continuous_aggs/finalize.c new file mode 100644 index 000000000..d3a9665e5 --- /dev/null +++ b/tsl/src/continuous_aggs/finalize.c @@ -0,0 +1,914 @@ +/* + * This file and its contents are licensed under the Timescale License. + * Please see the included NOTICE for copyright information and + * LICENSE-TIMESCALE for a copy of the license. + */ + +/* + * This file contains the code related to the *NOT* finalized version of + * Continuous Aggregates (with partials) + */ +#include "finalize.h" +#include "create.h" +#include "common.h" + +typedef struct CAggHavingCxt +{ + List *origq_tlist; + List *finalizeq_tlist; + AggPartCxt agg_cxt; +} CAggHavingCxt; + +/* Static function prototypes */ +static Datum get_input_types_array_datum(Aggref *original_aggregate); +static Aggref *add_partialize_column(Aggref *agg_to_partialize, AggPartCxt *cxt); +static void set_var_mapping(Var *orig_var, Var *mapped_var, AggPartCxt *cxt); +static Var *var_already_mapped(Var *var, AggPartCxt *cxt); +static Node *create_replace_having_qual_mutator(Node *node, CAggHavingCxt *cxt); +static Node *finalizequery_create_havingqual(FinalizeQueryInfo *inp, + MatTableColumnInfo *mattblinfo); +static Var *mattablecolumninfo_addentry(MatTableColumnInfo *out, Node *input, + int original_query_resno, bool finalized, + bool *skip_adding); +static FuncExpr *get_partialize_funcexpr(Aggref *agg); +static inline void makeMaterializeColumnName(char *colbuf, const char *type, + int original_query_resno, int colno); + +static inline void +makeMaterializeColumnName(char *colbuf, const char *type, int original_query_resno, int colno) +{ + int ret = snprintf(colbuf, NAMEDATALEN, "%s_%d_%d", type, original_query_resno, colno); + if (ret < 0 || ret >= NAMEDATALEN) + ereport(ERROR, + (errcode(ERRCODE_INTERNAL_ERROR), errmsg("bad materialization table column name"))); +} + +/* + * Creates a partialize expr for the passed in agg: + * partialize_agg(agg). + */ +static FuncExpr * +get_partialize_funcexpr(Aggref *agg) +{ + FuncExpr *partialize_fnexpr; + Oid partfnoid, partargtype; + partargtype = ANYELEMENTOID; + partfnoid = + LookupFuncName(list_make2(makeString(INTERNAL_SCHEMA_NAME), makeString(TS_PARTIALFN)), + 1, + &partargtype, + false); + partialize_fnexpr = makeFuncExpr(partfnoid, + BYTEAOID, + list_make1(agg), /*args*/ + InvalidOid, + InvalidOid, + COERCE_EXPLICIT_CALL); + return partialize_fnexpr; +} + +/* + * Build a [N][2] array where N is number of arguments + * and the inner array is of [schema_name,type_name]. + */ +static Datum +get_input_types_array_datum(Aggref *original_aggregate) +{ + ListCell *lc; + MemoryContext builder_context = + AllocSetContextCreate(CurrentMemoryContext, "input types builder", ALLOCSET_DEFAULT_SIZES); + Oid name_array_type_oid = get_array_type(NAMEOID); + ArrayBuildStateArr *outer_builder = + initArrayResultArr(name_array_type_oid, NAMEOID, builder_context, false); + Datum result; + + foreach (lc, original_aggregate->args) + { + TargetEntry *te = lfirst(lc); + Oid type_oid = exprType((Node *) te->expr); + ArrayBuildState *schema_name_builder = initArrayResult(NAMEOID, builder_context, false); + HeapTuple tp; + Form_pg_type typtup; + char *schema_name; + Name type_name = (Name) palloc0(NAMEDATALEN); + Datum schema_datum; + Datum type_name_datum; + Datum inner_array_datum; + + tp = SearchSysCache1(TYPEOID, ObjectIdGetDatum(type_oid)); + if (!HeapTupleIsValid(tp)) + elog(ERROR, "cache lookup failed for type %u", type_oid); + + typtup = (Form_pg_type) GETSTRUCT(tp); + namestrcpy(type_name, NameStr(typtup->typname)); + schema_name = get_namespace_name(typtup->typnamespace); + ReleaseSysCache(tp); + + type_name_datum = NameGetDatum(type_name); + /* Using name in because creating from a char * (that may be null or too long). */ + schema_datum = DirectFunctionCall1(namein, CStringGetDatum(schema_name)); + + accumArrayResult(schema_name_builder, schema_datum, false, NAMEOID, builder_context); + accumArrayResult(schema_name_builder, type_name_datum, false, NAMEOID, builder_context); + + inner_array_datum = makeArrayResult(schema_name_builder, CurrentMemoryContext); + + accumArrayResultArr(outer_builder, + inner_array_datum, + false, + name_array_type_oid, + builder_context); + } + result = makeArrayResultArr(outer_builder, CurrentMemoryContext, false); + + MemoryContextDelete(builder_context); + return result; +} + +static Aggref * +add_partialize_column(Aggref *agg_to_partialize, AggPartCxt *cxt) +{ + Aggref *newagg; + Var *var; + bool skip_adding; + + /* + * Step 1: create partialize( aggref) column + * for materialization table. + */ + var = mattablecolumninfo_addentry(cxt->mattblinfo, + (Node *) agg_to_partialize, + cxt->original_query_resno, + false, + &skip_adding); + cxt->added_aggref_col = true; + /* + * Step 2: create finalize_agg expr using var + * for the column added to the materialization table. + */ + + /* This is a var for the column we created. */ + newagg = get_finalize_aggref(agg_to_partialize, var); + return newagg; +} + +static void +set_var_mapping(Var *orig_var, Var *mapped_var, AggPartCxt *cxt) +{ + cxt->orig_vars = lappend(cxt->orig_vars, orig_var); + cxt->mapped_vars = lappend(cxt->mapped_vars, mapped_var); +} + +/* + * Checks whether var has already been mapped and returns the + * corresponding column of the materialization table. + */ +static Var * +var_already_mapped(Var *var, AggPartCxt *cxt) +{ + ListCell *lc_old, *lc_new; + + forboth (lc_old, cxt->orig_vars, lc_new, cxt->mapped_vars) + { + Var *orig_var = (Var *) lfirst_node(Var, lc_old); + Var *mapped_var = (Var *) lfirst_node(Var, lc_new); + + /* There should be no subqueries so varlevelsup should not be a problem here. */ + if (var->varno == orig_var->varno && var->varattno == orig_var->varattno) + return mapped_var; + } + return NULL; +} + +/* + * Add ts_internal_cagg_final to bytea column. + * bytea column is the internal state for an agg. Pass info for the agg as "inp". + * inpcol = bytea column. + * This function returns an aggref + * ts_internal_cagg_final( Oid, Oid, bytea, NULL::output_typeid) + * the arguments are a list of targetentry + */ +Oid +get_finalize_function_oid(void) +{ + Oid finalfnoid; + Oid finalfnargtypes[] = { TEXTOID, NAMEOID, NAMEOID, get_array_type(NAMEOID), + BYTEAOID, ANYELEMENTOID }; + List *funcname = list_make2(makeString(INTERNAL_SCHEMA_NAME), makeString(FINALFN)); + int nargs = sizeof(finalfnargtypes) / sizeof(finalfnargtypes[0]); + finalfnoid = LookupFuncName(funcname, nargs, finalfnargtypes, false); + return finalfnoid; +} + +/* + * Creates an aggref of the form: + * finalize-agg( + * "sum(int)" TEXT, + * collation_schema_name NAME, collation_name NAME, + * input_types_array NAME[N][2], + * BYTEA, + * null:: + * ) + * here sum(int) is the input aggregate "inp" in the parameter-list. + */ +Aggref * +get_finalize_aggref(Aggref *inp, Var *partial_state_var) +{ + Aggref *aggref; + TargetEntry *te; + char *aggregate_signature; + Const *aggregate_signature_const, *collation_schema_const, *collation_name_const, + *input_types_const, *return_type_const; + Oid name_array_type_oid = get_array_type(NAMEOID); + Var *partial_bytea_var; + List *tlist = NIL; + int tlist_attno = 1; + List *argtypes = NIL; + char *collation_name = NULL, *collation_schema_name = NULL; + Datum collation_name_datum = (Datum) 0; + Datum collation_schema_datum = (Datum) 0; + Oid finalfnoid = get_finalize_function_oid(); + + argtypes = list_make5_oid(TEXTOID, NAMEOID, NAMEOID, name_array_type_oid, BYTEAOID); + argtypes = lappend_oid(argtypes, inp->aggtype); + + aggref = makeNode(Aggref); + aggref->aggfnoid = finalfnoid; + aggref->aggtype = inp->aggtype; + aggref->aggcollid = inp->aggcollid; + aggref->inputcollid = inp->inputcollid; + aggref->aggtranstype = InvalidOid; /* will be set by planner */ + aggref->aggargtypes = argtypes; + aggref->aggdirectargs = NULL; /*relevant for hypothetical set aggs*/ + aggref->aggorder = NULL; + aggref->aggdistinct = NULL; + aggref->aggfilter = NULL; + aggref->aggstar = false; + aggref->aggvariadic = false; + aggref->aggkind = AGGKIND_NORMAL; + aggref->aggsplit = AGGSPLIT_SIMPLE; + aggref->location = -1; + /* Construct the arguments. */ + aggregate_signature = format_procedure_qualified(inp->aggfnoid); + aggregate_signature_const = makeConst(TEXTOID, + -1, + DEFAULT_COLLATION_OID, + -1, + CStringGetTextDatum(aggregate_signature), + false, + false /* passbyval */ + ); + te = makeTargetEntry((Expr *) aggregate_signature_const, tlist_attno++, NULL, false); + tlist = lappend(tlist, te); + + if (OidIsValid(inp->inputcollid)) + { + /* Similar to generate_collation_name. */ + HeapTuple tp; + Form_pg_collation colltup; + tp = SearchSysCache1(COLLOID, ObjectIdGetDatum(inp->inputcollid)); + if (!HeapTupleIsValid(tp)) + elog(ERROR, "cache lookup failed for collation %u", inp->inputcollid); + colltup = (Form_pg_collation) GETSTRUCT(tp); + collation_name = pstrdup(NameStr(colltup->collname)); + collation_name_datum = DirectFunctionCall1(namein, CStringGetDatum(collation_name)); + + collation_schema_name = get_namespace_name(colltup->collnamespace); + if (collation_schema_name != NULL) + collation_schema_datum = + DirectFunctionCall1(namein, CStringGetDatum(collation_schema_name)); + ReleaseSysCache(tp); + } + collation_schema_const = makeConst(NAMEOID, + -1, + InvalidOid, + NAMEDATALEN, + collation_schema_datum, + (collation_schema_name == NULL) ? true : false, + false /* passbyval */ + ); + te = makeTargetEntry((Expr *) collation_schema_const, tlist_attno++, NULL, false); + tlist = lappend(tlist, te); + + collation_name_const = makeConst(NAMEOID, + -1, + InvalidOid, + NAMEDATALEN, + collation_name_datum, + (collation_name == NULL) ? true : false, + false /* passbyval */ + ); + te = makeTargetEntry((Expr *) collation_name_const, tlist_attno++, NULL, false); + tlist = lappend(tlist, te); + + input_types_const = makeConst(get_array_type(NAMEOID), + -1, + InvalidOid, + -1, + get_input_types_array_datum(inp), + false, + false /* passbyval */ + ); + te = makeTargetEntry((Expr *) input_types_const, tlist_attno++, NULL, false); + tlist = lappend(tlist, te); + + partial_bytea_var = copyObject(partial_state_var); + te = makeTargetEntry((Expr *) partial_bytea_var, tlist_attno++, NULL, false); + tlist = lappend(tlist, te); + + return_type_const = makeNullConst(inp->aggtype, -1, inp->aggcollid); + te = makeTargetEntry((Expr *) return_type_const, tlist_attno++, NULL, false); + tlist = lappend(tlist, te); + + Assert(tlist_attno == 7); + aggref->args = tlist; + return aggref; +} + +Node * +add_var_mutator(Node *node, AggPartCxt *cxt) +{ + if (node == NULL) + return NULL; + if (IsA(node, Aggref)) + { + return node; /* don't process this further */ + } + if (IsA(node, Var)) + { + Var *orig_var, *mapped_var; + bool skip_adding = false; + + mapped_var = var_already_mapped((Var *) node, cxt); + /* Avoid duplicating columns in the materialization table. */ + if (mapped_var) + /* + * There should be no subquery so mapped_var->varlevelsup + * should not be a problem here. + */ + return (Node *) copyObject(mapped_var); + + orig_var = (Var *) node; + mapped_var = mattablecolumninfo_addentry(cxt->mattblinfo, + node, + cxt->original_query_resno, + false, + &skip_adding); + set_var_mapping(orig_var, mapped_var, cxt); + return (Node *) mapped_var; + } + return expression_tree_mutator(node, add_var_mutator, cxt); +} + +/* + * This function modifies the passed in havingQual by mapping exprs to + * columns in materialization table or finalized aggregate form. + * Note that HAVING clause can contain only exprs from group-by or aggregates + * and GROUP BY clauses cannot be aggregates. + * (By the time we process havingQuals, all the group by exprs have been + * processed and have associated columns in the materialization hypertable). + * Example, if the original query has + * GROUP BY colA + colB, colC + * HAVING colA + colB + sum(colD) > 10 OR count(colE) = 10 + * + * The transformed havingqual would be + * HAVING matCol3 + finalize_agg( sum(matCol4) > 10 + * OR finalize_agg( count(matCol5)) = 10 + * + * + * Note: GROUP BY exprs always appear in the query's targetlist. + * Some of the aggregates from the havingQual might also already appear in the targetlist. + * We replace all existing entries with their corresponding entry from the modified targetlist. + * If an aggregate (in the havingqual) does not exist in the TL, we create a + * materialization table column for it and use the finalize(column) form in the + * transformed havingQual. + */ +static Node * +create_replace_having_qual_mutator(Node *node, CAggHavingCxt *cxt) +{ + if (node == NULL) + return NULL; + /* + * See if we already have a column in materialization hypertable for this + * expr. We do this by checking the existing targetlist + * entries for the query. + */ + ListCell *lc, *lc2; + List *origtlist = cxt->origq_tlist; + List *modtlist = cxt->finalizeq_tlist; + forboth (lc, origtlist, lc2, modtlist) + { + TargetEntry *te = (TargetEntry *) lfirst(lc); + TargetEntry *modte = (TargetEntry *) lfirst(lc2); + if (equal(node, te->expr)) + { + return (Node *) modte->expr; + } + } + /* + * Didn't find a match in targetlist. If it is an aggregate, + * create a partialize column for it in materialization hypertable + * and return corresponding finalize expr. + */ + if (IsA(node, Aggref)) + { + AggPartCxt *agg_cxt = &(cxt->agg_cxt); + agg_cxt->added_aggref_col = false; + Aggref *newagg = add_partialize_column((Aggref *) node, agg_cxt); + Assert(agg_cxt->added_aggref_col == true); + return (Node *) newagg; + } + return expression_tree_mutator(node, create_replace_having_qual_mutator, cxt); +} + +static Node * +finalizequery_create_havingqual(FinalizeQueryInfo *inp, MatTableColumnInfo *mattblinfo) +{ + Query *orig_query = inp->final_userquery; + if (orig_query->havingQual == NULL) + return NULL; + Node *havingQual = copyObject(orig_query->havingQual); + Assert(inp->final_seltlist != NULL); + CAggHavingCxt hcxt = { .origq_tlist = orig_query->targetList, + .finalizeq_tlist = inp->final_seltlist, + .agg_cxt.mattblinfo = mattblinfo, + .agg_cxt.original_query_resno = 0, + .agg_cxt.ignore_aggoid = get_finalize_function_oid(), + .agg_cxt.added_aggref_col = false, + .agg_cxt.var_outside_of_aggref = false, + .agg_cxt.orig_vars = NIL, + .agg_cxt.mapped_vars = NIL }; + return create_replace_having_qual_mutator(havingQual, &hcxt); +} + +Node * +add_aggregate_partialize_mutator(Node *node, AggPartCxt *cxt) +{ + if (node == NULL) + return NULL; + /* + * Modify the aggref and create a partialize(aggref) expr + * for the materialization. + * Add a corresponding columndef for the mat table. + * Replace the aggref with the ts_internal_cagg_final fn. + * using a Var for the corresponding column in the mat table. + * All new Vars have varno = 1 (for RTE 1). + */ + if (IsA(node, Aggref)) + { + if (cxt->ignore_aggoid == ((Aggref *) node)->aggfnoid) + return node; /* don't process this further */ + + Aggref *newagg = add_partialize_column((Aggref *) node, cxt); + return (Node *) newagg; + } + if (IsA(node, Var)) + { + cxt->var_outside_of_aggref = true; + } + return expression_tree_mutator(node, add_aggregate_partialize_mutator, cxt); +} + +/* + * Init the finalize query data structure. + * Parameters: + * orig_query - the original query from user view that is being used as template for the finalize + * query tlist_aliases - aliases for the view select list materialization table columns are created + * . This will be returned in the mattblinfo + * + * DO NOT modify orig_query. Make a copy if needed. + * SIDE_EFFECT: the data structure in mattblinfo is modified as a side effect by adding new + * materialize table columns and partialize exprs. + */ +void +finalizequery_init(FinalizeQueryInfo *inp, Query *orig_query, MatTableColumnInfo *mattblinfo) +{ + AggPartCxt cxt; + ListCell *lc; + int resno = 1; + + inp->final_userquery = copyObject(orig_query); + inp->final_seltlist = NIL; + inp->final_havingqual = NULL; + + /* Set up the final_seltlist and final_havingqual entries */ + cxt.mattblinfo = mattblinfo; + cxt.ignore_aggoid = InvalidOid; + + /* Set up the left over variable mapping lists */ + cxt.orig_vars = NIL; + cxt.mapped_vars = NIL; + + /* + * We want all the entries in the targetlist (resjunk or not) + * in the materialization table definition so we include group-by/having clause etc. + * We have to do 3 things here: + * 1) create a column for mat table + * 2) partialize_expr to populate it, and + * 3) modify the target entry to be a finalize_expr + * that selects from the materialization table. + */ + foreach (lc, orig_query->targetList) + { + TargetEntry *tle = (TargetEntry *) lfirst(lc); + TargetEntry *modte = copyObject(tle); + cxt.added_aggref_col = false; + cxt.var_outside_of_aggref = false; + cxt.original_query_resno = resno; + + if (!inp->finalized) + { + /* + * If tle has aggrefs, get the corresponding + * finalize_agg expression and save it in modte. + * Also add correspong materialization table column info + * for the aggrefs in tle. + */ + modte = (TargetEntry *) expression_tree_mutator((Node *) modte, + add_aggregate_partialize_mutator, + &cxt); + } + + /* + * We need columns for non-aggregate targets. + * If it is not a resjunk OR appears in the grouping clause. + */ + if (cxt.added_aggref_col == false && (tle->resjunk == false || tle->ressortgroupref > 0)) + { + Var *var; + bool skip_adding = false; + var = mattablecolumninfo_addentry(cxt.mattblinfo, + (Node *) tle, + cxt.original_query_resno, + inp->finalized, + &skip_adding); + + /* Skip adding this column for finalized form. */ + if (skip_adding) + { + continue; + } + + /* Fix the expression for the target entry. */ + modte->expr = (Expr *) var; + } + /* Check for left over variables (Var) of targets that contain Aggref. */ + if (cxt.added_aggref_col && cxt.var_outside_of_aggref && !inp->finalized) + { + modte = (TargetEntry *) expression_tree_mutator((Node *) modte, add_var_mutator, &cxt); + } + /* + * Construct the targetlist for the query on the + * materialization table. The TL maps 1-1 with the original query: + * e.g select a, min(b)+max(d) from foo group by a,timebucket(a); + * becomes + * select , + * ts_internal_cagg_final(..b-col ) + ts_internal_cagg_final(..d-col) + * from mattbl + * group by a-col, timebucket(a-col) + */ + + /* + * We copy the modte target entries, resnos should be the same for + * final_selquery and origquery. So tleSortGroupReffor the targetentry + * can be reused, only table info needs to be modified. + */ + Assert((!inp->finalized && modte->resno == resno) || + (inp->finalized && modte->resno >= resno)); + resno++; + if (IsA(modte->expr, Var)) + { + modte->resorigcol = ((Var *) modte->expr)->varattno; + } + inp->final_seltlist = lappend(inp->final_seltlist, modte); + } + /* + * All grouping clause elements are in targetlist already. + * So let's check the having clause. + */ + if (!inp->finalized) + inp->final_havingqual = finalizequery_create_havingqual(inp, mattblinfo); +} + +/* + * Create select query with the finalize aggregates + * for the materialization table. + * matcollist - column list for mat table + * mattbladdress - materialization table ObjectAddress + * This is the function responsible for creating the final + * structures for selecting from the materialized hypertable + * created for the Cagg which is + * select * from _timescaldeb_internal._materialized_hypertable_ + */ +Query * +finalizequery_get_select_query(FinalizeQueryInfo *inp, List *matcollist, + ObjectAddress *mattbladdress, char *relname) +{ + Query *final_selquery = NULL; + ListCell *lc; + FromExpr *fromexpr; + RangeTblEntry *rte; + + /* + * For initial cagg creation rtable will have only 1 entry, + * for alter table rtable will have multiple entries with our + * RangeTblEntry as last member. + * For cagg with joins, we need to create a new RTE and jointree + * which contains the information of the materialised hypertable + * that is created for this cagg. + */ + if (list_length(inp->final_userquery->jointree->fromlist) >= + CONTINUOUS_AGG_MAX_JOIN_RELATIONS || + !IsA(linitial(inp->final_userquery->jointree->fromlist), RangeTblRef)) + { + rte = makeNode(RangeTblEntry); + rte->alias = makeAlias(relname, NIL); + rte->inFromCl = true; + rte->inh = true; + rte->rellockmode = 1; + rte->eref = copyObject(rte->alias); + ListCell *l; + foreach (l, inp->final_userquery->jointree->fromlist) + { + /* + * In case of joins, update the rte with all the join related struct. + */ + Node *jtnode = (Node *) lfirst(l); + JoinExpr *join = NULL; + if (IsA(jtnode, JoinExpr)) + { + join = castNode(JoinExpr, jtnode); + RangeTblEntry *jrte = rt_fetch(join->rtindex, inp->final_userquery->rtable); + rte->joinaliasvars = jrte->joinaliasvars; + rte->jointype = jrte->jointype; +#if PG13_GE + rte->joinleftcols = jrte->joinleftcols; + rte->joinrightcols = jrte->joinrightcols; + rte->joinmergedcols = jrte->joinmergedcols; +#endif +#if PG14_GE + rte->join_using_alias = jrte->join_using_alias; +#endif + rte->selectedCols = jrte->selectedCols; + } + } + } + else + { + rte = llast_node(RangeTblEntry, inp->final_userquery->rtable); + rte->eref->colnames = NIL; + rte->selectedCols = NULL; + } + if (rte->eref->colnames == NIL) + { + /* + * We only need to do this for the case when there is no Join node in the query. + * In the case of join, rte->eref is already populated by jrte->eref and hence the + * relevant info, so need not to do this. + */ + + /* Aliases for column names for the materialization table. */ + foreach (lc, matcollist) + { + ColumnDef *cdef = lfirst_node(ColumnDef, lc); + rte->eref->colnames = lappend(rte->eref->colnames, makeString(cdef->colname)); + rte->selectedCols = bms_add_member(rte->selectedCols, + list_length(rte->eref->colnames) - + FirstLowInvalidHeapAttributeNumber); + } + } + rte->relid = mattbladdress->objectId; + rte->rtekind = RTE_RELATION; + rte->relkind = RELKIND_RELATION; + rte->tablesample = NULL; + rte->requiredPerms |= ACL_SELECT; + rte->insertedCols = NULL; + rte->updatedCols = NULL; + + /* 2. Fixup targetlist with the correct rel information. */ + foreach (lc, inp->final_seltlist) + { + TargetEntry *tle = (TargetEntry *) lfirst(lc); + /* + * In case when this is a cagg wth joins, the Var from the normal table + * already has resorigtbl populated and we need to use that to resolve + * the Var. Hence only modify the tle when resorigtbl is unset + * which means it is Var of the Hypertable + */ + if (IsA(tle->expr, Var) && !OidIsValid(tle->resorigtbl)) + { + tle->resorigtbl = rte->relid; + tle->resorigcol = ((Var *) tle->expr)->varattno; + } + } + + CAGG_MAKEQUERY(final_selquery, inp->final_userquery); + final_selquery->hasAggs = !inp->finalized; + if (list_length(inp->final_userquery->jointree->fromlist) >= + CONTINUOUS_AGG_MAX_JOIN_RELATIONS || + !IsA(linitial(inp->final_userquery->jointree->fromlist), RangeTblRef)) + { + RangeTblRef *rtr; + final_selquery->rtable = list_make1(rte); + rtr = makeNode(RangeTblRef); + rtr->rtindex = 1; + fromexpr = makeFromExpr(list_make1(rtr), NULL); + } + else + { + final_selquery->rtable = inp->final_userquery->rtable; + fromexpr = inp->final_userquery->jointree; + fromexpr->quals = NULL; + } + + /* + * Fixup from list. No quals on original table should be + * present here - they should be on the query that populates + * the mattable (partial_selquery). For the Cagg with join, + * we can not copy the fromlist from inp->final_userquery as + * it has two tables in this case. + */ + Assert(list_length(inp->final_userquery->jointree->fromlist) <= + CONTINUOUS_AGG_MAX_JOIN_RELATIONS); + + final_selquery->jointree = fromexpr; + final_selquery->targetList = inp->final_seltlist; + final_selquery->sortClause = inp->final_userquery->sortClause; + + if (!inp->finalized) + { + final_selquery->groupClause = inp->final_userquery->groupClause; + /* Copy the having clause too */ + final_selquery->havingQual = inp->final_havingqual; + } + + return final_selquery; +} + +/* + * Add Information required to create and populate the materialization table columns + * a) create a columndef for the materialization table + * b) create the corresponding expr to populate the column of the materialization table (e..g for a + * column that is an aggref, we create a partialize_agg expr to populate the column Returns: the + * Var corresponding to the newly created column of the materialization table + * + * Notes: make sure the materialization table columns do not save + * values computed by mutable function. + * + * Notes on TargetEntry fields: + * - (resname != NULL) means it's projected in our case + * - (ressortgroupref > 0) means part of GROUP BY, which can be projected or not, depending of the + * value of the resjunk + * - (resjunk == true) applies for GROUP BY columns that are not projected + * + */ +static Var * +mattablecolumninfo_addentry(MatTableColumnInfo *out, Node *input, int original_query_resno, + bool finalized, bool *skip_adding) +{ + int matcolno = list_length(out->matcollist) + 1; + char colbuf[NAMEDATALEN]; + char *colname; + TargetEntry *part_te = NULL; + ColumnDef *col; + Var *var; + Oid coltype, colcollation; + int32 coltypmod; + + *skip_adding = false; + + if (contain_mutable_functions(input)) + { + ereport(ERROR, + (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), + errmsg("only immutable functions supported in continuous aggregate view"), + errhint("Make sure all functions in the continuous aggregate definition" + " have IMMUTABLE volatility. Note that functions or expressions" + " may be IMMUTABLE for one data type, but STABLE or VOLATILE for" + " another."))); + } + + switch (nodeTag(input)) + { + case T_Aggref: + { + FuncExpr *fexpr = get_partialize_funcexpr((Aggref *) input); + makeMaterializeColumnName(colbuf, "agg", original_query_resno, matcolno); + colname = colbuf; + coltype = BYTEAOID; + coltypmod = -1; + colcollation = InvalidOid; + col = makeColumnDef(colname, coltype, coltypmod, colcollation); + part_te = makeTargetEntry((Expr *) fexpr, matcolno, pstrdup(colname), false); + } + break; + + case T_TargetEntry: + { + TargetEntry *tle = (TargetEntry *) input; + bool timebkt_chk = false; + + if (IsA(tle->expr, FuncExpr)) + timebkt_chk = function_allowed_in_cagg_definition(((FuncExpr *) tle->expr)->funcid); + + if (tle->resname) + colname = pstrdup(tle->resname); + else + { + if (timebkt_chk) + colname = DEFAULT_MATPARTCOLUMN_NAME; + else + { + makeMaterializeColumnName(colbuf, "grp", original_query_resno, matcolno); + colname = colbuf; + + /* For finalized form we skip adding extra group by columns. */ + *skip_adding = finalized; + } + } + + if (timebkt_chk) + { + tle->resname = pstrdup(colname); + out->matpartcolno = matcolno; + out->matpartcolname = pstrdup(colname); + } + else + { + /* + * Add indexes only for columns that are part of the GROUP BY clause + * and for finals form. + * We skip adding it because we'll not add the extra group by columns + * to the materialization hypertable anymore. + */ + if (!*skip_adding && tle->ressortgroupref > 0) + out->mat_groupcolname_list = + lappend(out->mat_groupcolname_list, pstrdup(colname)); + } + + coltype = exprType((Node *) tle->expr); + coltypmod = exprTypmod((Node *) tle->expr); + colcollation = exprCollation((Node *) tle->expr); + col = makeColumnDef(colname, coltype, coltypmod, colcollation); + part_te = (TargetEntry *) copyObject(input); + + /* Keep original resjunk if finalized or not time bucket. */ + if (!finalized || timebkt_chk) + { + /* + * Need to project all the partial entries so that + * materialization table is filled. + */ + part_te->resjunk = false; + } + + part_te->resno = matcolno; + + if (timebkt_chk) + { + col->is_not_null = true; + } + + if (part_te->resname == NULL) + { + part_te->resname = pstrdup(colname); + } + } + break; + + case T_Var: + { + makeMaterializeColumnName(colbuf, "var", original_query_resno, matcolno); + colname = colbuf; + + coltype = exprType(input); + coltypmod = exprTypmod(input); + colcollation = exprCollation(input); + col = makeColumnDef(colname, coltype, coltypmod, colcollation); + part_te = makeTargetEntry((Expr *) input, matcolno, pstrdup(colname), false); + + /* Need to project all the partial entries so that materialization table is filled. */ + part_te->resjunk = false; + part_te->resno = matcolno; + } + break; + + default: + elog(ERROR, "invalid node type %d", nodeTag(input)); + break; + } + Assert((!finalized && list_length(out->matcollist) == list_length(out->partial_seltlist)) || + (finalized && list_length(out->matcollist) <= list_length(out->partial_seltlist))); + Assert(col != NULL); + Assert(part_te != NULL); + + if (!*skip_adding) + { + out->matcollist = lappend(out->matcollist, col); + } + + out->partial_seltlist = lappend(out->partial_seltlist, part_te); + + var = makeVar(1, matcolno, coltype, coltypmod, colcollation, 0); + return var; +} diff --git a/tsl/src/continuous_aggs/finalize.h b/tsl/src/continuous_aggs/finalize.h new file mode 100644 index 000000000..5402c72d5 --- /dev/null +++ b/tsl/src/continuous_aggs/finalize.h @@ -0,0 +1,39 @@ +/* + * This file and its contents are licensed under the Timescale License. + * Please see the included NOTICE for copyright information and + * LICENSE-TIMESCALE for a copy of the license. + */ +#ifndef TIMESCALEDB_TSL_CONTINUOUS_AGGS_FINALIZE_H +#define TIMESCALEDB_TSL_CONTINUOUS_AGGS_FINALIZE_H + +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "ts_catalog/catalog.h" +#include "common.h" + +#define FINALFN "finalize_agg" + +extern Oid get_finalize_function_oid(void); +extern Aggref *get_finalize_aggref(Aggref *inp, Var *partial_state_var); +extern Node *add_aggregate_partialize_mutator(Node *node, AggPartCxt *cxt); +extern Node *add_var_mutator(Node *node, AggPartCxt *cxt); +extern Node *finalize_query_create_having_qual(FinalizeQueryInfo *inp, + MatTableColumnInfo *mattblinfo); +extern Query *finalize_query_get_select_query(FinalizeQueryInfo *inp, List *matcollist, + ObjectAddress *mattbladdress); +extern void finalizequery_init(FinalizeQueryInfo *inp, Query *orig_query, + MatTableColumnInfo *mattblinfo); +extern Query *finalizequery_get_select_query(FinalizeQueryInfo *inp, List *matcollist, + ObjectAddress *mattbladdress, char *relname); +#endif /* TIMESCALEDB_TSL_CONTINUOUS_AGGS_FINALIZE_H */ diff --git a/tsl/src/continuous_aggs/insert.c b/tsl/src/continuous_aggs/insert.c index 81f22dccc..3087f0b04 100644 --- a/tsl/src/continuous_aggs/insert.c +++ b/tsl/src/continuous_aggs/insert.c @@ -84,6 +84,18 @@ static MemoryContext continuous_aggs_trigger_mctx = NULL; void _continuous_aggs_cache_inval_init(void); void _continuous_aggs_cache_inval_fini(void); +static int64 tuple_get_time(Dimension *d, HeapTuple tuple, AttrNumber col, TupleDesc tupdesc); +static inline void cache_inval_entry_init(ContinuousAggsCacheInvalEntry *cache_entry, + int32 hypertable_id, int32 entry_id); +static inline void cache_entry_switch_to_chunk(ContinuousAggsCacheInvalEntry *cache_entry, + Oid chunk_id, Relation chunk_relation); +static inline void update_cache_entry(ContinuousAggsCacheInvalEntry *cache_entry, int64 timeval); +static void cache_inval_entry_write(ContinuousAggsCacheInvalEntry *entry); +static void cache_inval_cleanup(void); +static void cache_inval_htab_write(void); +static void continuous_agg_xact_invalidation_callback(XactEvent event, void *arg); +static ScanTupleResult invalidation_tuple_found(TupleInfo *ti, void *min); + static void cache_inval_init() { diff --git a/tsl/src/continuous_aggs/invalidation.c b/tsl/src/continuous_aggs/invalidation.c index 57a7ba53b..82505937c 100644 --- a/tsl/src/continuous_aggs/invalidation.c +++ b/tsl/src/continuous_aggs/invalidation.c @@ -111,6 +111,48 @@ typedef enum LogType LOG_CAGG, } LogType; +static Relation open_invalidation_log(LogType type, LOCKMODE lockmode); +static void hypertable_invalidation_scan_init(ScanIterator *iterator, int32 hyper_id, + LOCKMODE lockmode); +static HeapTuple create_invalidation_tup(const TupleDesc tupdesc, int32 cagg_hyper_id, int64 start, + int64 end); +static bool save_invalidation_for_refresh(const CaggInvalidationState *state, + const Invalidation *invalidation); +static void set_remainder_after_cut(Invalidation *remainder, int32 hyper_id, + int64 lowest_modified_value, int64 greatest_modified_value); +static void invalidation_entry_reset(Invalidation *entry); +static void +invalidation_expand_to_bucket_boundaries(Invalidation *inv, Oid time_type_oid, int64 bucket_width, + const ContinuousAggsBucketFunction *bucket_function); +static void +invalidation_entry_set_from_hyper_invalidation(Invalidation *entry, const TupleInfo *ti, + int32 hyper_id, Oid dimtype, int64 bucket_width, + const ContinuousAggsBucketFunction *bucket_function); +static void +invalidation_entry_set_from_cagg_invalidation(Invalidation *entry, const TupleInfo *ti, Oid dimtype, + int64 bucket_width, + const ContinuousAggsBucketFunction *bucket_function); +static bool invalidations_can_be_merged(const Invalidation *a, const Invalidation *b); +static bool invalidation_entry_try_merge(Invalidation *entry, const Invalidation *newentry); +static void cut_and_insert_new_cagg_invalidation(const CaggInvalidationState *state, + const Invalidation *entry, int32 cagg_hyper_id); +static void move_invalidations_from_hyper_to_cagg_log(const CaggInvalidationState *state); +static void cagg_invalidations_scan_by_hypertable_init(ScanIterator *iterator, int32 cagg_hyper_id, + LOCKMODE lockmode); +static Invalidation cut_cagg_invalidation(const CaggInvalidationState *state, + const InternalTimeRange *refresh_window, + const Invalidation *entry); +static Invalidation cut_cagg_invalidation_and_compute_remainder( + const CaggInvalidationState *state, const InternalTimeRange *refresh_window, + const Invalidation *mergedentry, const Invalidation *current_remainder); +static void clear_cagg_invalidations_for_refresh(const CaggInvalidationState *state, + const InternalTimeRange *refresh_window); +static void invalidation_state_init(CaggInvalidationState *state, int32 mat_hypertable_id, + int32 raw_hypertable_id, Oid dimtype, + const CaggsInfo *all_caggs); +static void invalidation_state_cleanup(const CaggInvalidationState *state); +static ArrayType *bucket_functions_default_argument(int ndim); + static Relation open_invalidation_log(LogType type, LOCKMODE lockmode) { diff --git a/tsl/src/continuous_aggs/invalidation_threshold.c b/tsl/src/continuous_aggs/invalidation_threshold.c index f3c845bda..964b96316 100644 --- a/tsl/src/continuous_aggs/invalidation_threshold.c +++ b/tsl/src/continuous_aggs/invalidation_threshold.c @@ -70,6 +70,9 @@ typedef struct InvalidationThresholdData bool was_updated; } InvalidationThresholdData; +static ScanTupleResult scan_update_invalidation_threshold(TupleInfo *ti, void *data); +static ScanTupleResult invalidation_threshold_tuple_found(TupleInfo *ti, void *data); + static ScanTupleResult scan_update_invalidation_threshold(TupleInfo *ti, void *data) { diff --git a/tsl/src/continuous_aggs/materialize.c b/tsl/src/continuous_aggs/materialize.c index bcccd8645..648926123 100644 --- a/tsl/src/continuous_aggs/materialize.c +++ b/tsl/src/continuous_aggs/materialize.c @@ -23,6 +23,9 @@ #include "materialize.h" +#define CHUNKIDFROMRELID "chunk_id_from_relid" +#define CONTINUOUS_AGG_CHUNK_ID_COL_NAME "chunk_id" + static bool ranges_overlap(InternalTimeRange invalidation_range, InternalTimeRange new_materialization_range); static TimeRange internal_time_range_to_time_range(InternalTimeRange internal); @@ -349,3 +352,95 @@ spi_insert_materializations(Hypertable *mat_ht, SchemaAndName partial_view, } } } +/* + * Initialize MatTableColumnInfo. + */ +void +mattablecolumninfo_init(MatTableColumnInfo *matcolinfo, List *grouplist) +{ + matcolinfo->matcollist = NIL; + matcolinfo->partial_seltlist = NIL; + matcolinfo->partial_grouplist = grouplist; + matcolinfo->mat_groupcolname_list = NIL; + matcolinfo->matpartcolno = -1; + matcolinfo->matpartcolname = NULL; +} + +/* + * Add internal columns for the materialization table. + */ +void +mattablecolumninfo_addinternal(MatTableColumnInfo *matcolinfo) +{ + Index maxRef; + int colno = list_length(matcolinfo->partial_seltlist) + 1; + ColumnDef *col; + Var *chunkfn_arg1; + FuncExpr *chunk_fnexpr; + Oid chunkfnoid; + Oid argtype[] = { OIDOID }; + Oid rettype = INT4OID; + TargetEntry *chunk_te; + Oid sortop, eqop; + bool hashable; + ListCell *lc; + SortGroupClause *grpcl; + + /* Add a chunk_id column for materialization table */ + Node *vexpr = (Node *) makeVar(1, colno, INT4OID, -1, InvalidOid, 0); + col = makeColumnDef(CONTINUOUS_AGG_CHUNK_ID_COL_NAME, + exprType(vexpr), + exprTypmod(vexpr), + exprCollation(vexpr)); + matcolinfo->matcollist = lappend(matcolinfo->matcollist, col); + + /* + * Need to add an entry to the target list for computing chunk_id column + * : chunk_for_tuple( htid, table.*). + */ + chunkfnoid = + LookupFuncName(list_make2(makeString(INTERNAL_SCHEMA_NAME), makeString(CHUNKIDFROMRELID)), + sizeof(argtype) / sizeof(argtype[0]), + argtype, + false); + chunkfn_arg1 = makeVar(1, TableOidAttributeNumber, OIDOID, -1, 0, 0); + + chunk_fnexpr = makeFuncExpr(chunkfnoid, + rettype, + list_make1(chunkfn_arg1), + InvalidOid, + InvalidOid, + COERCE_EXPLICIT_CALL); + chunk_te = makeTargetEntry((Expr *) chunk_fnexpr, + colno, + pstrdup(CONTINUOUS_AGG_CHUNK_ID_COL_NAME), + false); + matcolinfo->partial_seltlist = lappend(matcolinfo->partial_seltlist, chunk_te); + /* Any internal column needs to be added to the group-by clause as well. */ + maxRef = 0; + foreach (lc, matcolinfo->partial_seltlist) + { + Index ref = ((TargetEntry *) lfirst(lc))->ressortgroupref; + + if (ref > maxRef) + maxRef = ref; + } + chunk_te->ressortgroupref = + maxRef + 1; /* used by sortgroupclause to identify the targetentry */ + grpcl = makeNode(SortGroupClause); + get_sort_group_operators(exprType((Node *) chunk_te->expr), + false, + true, + false, + &sortop, + &eqop, + NULL, + &hashable); + grpcl->tleSortGroupRef = chunk_te->ressortgroupref; + grpcl->eqop = eqop; + grpcl->sortop = sortop; + grpcl->nulls_first = false; + grpcl->hashable = hashable; + + matcolinfo->partial_grouplist = lappend(matcolinfo->partial_grouplist, grpcl); +} diff --git a/tsl/src/continuous_aggs/materialize.h b/tsl/src/continuous_aggs/materialize.h index 5eb504334..3c00b6d34 100644 --- a/tsl/src/continuous_aggs/materialize.h +++ b/tsl/src/continuous_aggs/materialize.h @@ -10,6 +10,7 @@ #include #include #include "ts_catalog/continuous_agg.h" +#include "common.h" typedef struct SchemaAndName { @@ -40,5 +41,4 @@ void continuous_agg_update_materialization(Hypertable *mat_ht, SchemaAndName par const NameData *time_column_name, InternalTimeRange new_materialization_range, InternalTimeRange invalidation_range, int32 chunk_id); - #endif /* TIMESCALEDB_TSL_CONTINUOUS_AGGS_MATERIALIZE_H */ diff --git a/tsl/src/continuous_aggs/options.c b/tsl/src/continuous_aggs/options.c index 098ec8bde..a86e55b21 100644 --- a/tsl/src/continuous_aggs/options.c +++ b/tsl/src/continuous_aggs/options.c @@ -23,8 +23,13 @@ #include "hypertable_cache.h" #include "scan_iterator.h" +static void cagg_update_materialized_only(ContinuousAgg *agg, bool materialized_only); +static List *cagg_find_groupingcols(ContinuousAgg *agg, Hypertable *mat_ht); +static List *cagg_get_compression_params(ContinuousAgg *agg, Hypertable *mat_ht); +static void cagg_alter_compression(ContinuousAgg *agg, Hypertable *mat_ht, List *compress_defelems); + static void -update_materialized_only(ContinuousAgg *agg, bool materialized_only) +cagg_update_materialized_only(ContinuousAgg *agg, bool materialized_only) { ScanIterator iterator = ts_scan_iterator_create(CONTINUOUS_AGG, RowExclusiveLock, CurrentMemoryContext); @@ -255,7 +260,7 @@ continuous_agg_update_options(ContinuousAgg *agg, WithClauseResult *with_clause_ Assert(mat_ht != NULL); cagg_flip_realtime_view_definition(agg, mat_ht); - update_materialized_only(agg, materialized_only); + cagg_update_materialized_only(agg, materialized_only); ts_cache_release(hcache); } List *compression_options = ts_continuous_agg_get_compression_defelems(with_clause_options); diff --git a/tsl/src/continuous_aggs/refresh.c b/tsl/src/continuous_aggs/refresh.c index 0a44e8e00..3faf08e90 100644 --- a/tsl/src/continuous_aggs/refresh.c +++ b/tsl/src/continuous_aggs/refresh.c @@ -39,6 +39,43 @@ typedef struct CaggRefreshState SchemaAndName partial_view; } CaggRefreshState; +static Hypertable *cagg_get_hypertable_or_fail(int32 hypertable_id); +static InternalTimeRange get_largest_bucketed_window(Oid timetype, int64 bucket_width); +static InternalTimeRange +compute_inscribed_bucketed_refresh_window(const InternalTimeRange *const refresh_window, + const int64 bucket_width); +static InternalTimeRange +compute_circumscribed_bucketed_refresh_window(const InternalTimeRange *const refresh_window, + const int64 bucket_width, + const ContinuousAggsBucketFunction *bucket_function); +static void continuous_agg_refresh_init(CaggRefreshState *refresh, const ContinuousAgg *cagg, + const InternalTimeRange *refresh_window); +static void continuous_agg_refresh_execute(const CaggRefreshState *refresh, + const InternalTimeRange *bucketed_refresh_window, + const int32 chunk_id); +static void log_refresh_window(int elevel, const ContinuousAgg *cagg, + const InternalTimeRange *refresh_window, const char *msg); +static long materialization_per_refresh_window(void); +static void continuous_agg_refresh_execute_wrapper(const InternalTimeRange *bucketed_refresh_window, + const long iteration, void *arg1_refresh, + void *arg2_chunk_id); +static void update_merged_refresh_window(const InternalTimeRange *bucketed_refresh_window, + const long iteration, void *arg1_merged_refresh_window, + void *arg2); +static void continuous_agg_refresh_with_window(const ContinuousAgg *cagg, + const InternalTimeRange *refresh_window, + const InvalidationStore *invalidations, + const int64 bucket_width, int32 chunk_id, + const bool is_raw_ht_distributed, + const bool do_merged_refresh, + const InternalTimeRange merged_refresh_window); +static ContinuousAgg *get_cagg_by_relid(const Oid cagg_relid); +static void emit_up_to_date_notice(const ContinuousAgg *cagg, const CaggRefreshCallContext callctx); +static bool process_cagg_invalidations_and_refresh(const ContinuousAgg *cagg, + const InternalTimeRange *refresh_window, + const CaggRefreshCallContext callctx, + int32 chunk_id); + static Hypertable * cagg_get_hypertable_or_fail(int32 hypertable_id) { diff --git a/tsl/src/continuous_aggs/repair.c b/tsl/src/continuous_aggs/repair.c new file mode 100644 index 000000000..87e2a129e --- /dev/null +++ b/tsl/src/continuous_aggs/repair.c @@ -0,0 +1,244 @@ +/* + * This file and its contents are licensed under the Timescale License. + * Please see the included NOTICE for copyright information and + * LICENSE-TIMESCALE for a copy of the license. + */ + +#include "repair.h" + +static void cagg_rebuild_view_definition(ContinuousAgg *agg, Hypertable *mat_ht, + bool force_rebuild); + +/* + * Test the view definition of an existing continuous aggregate + * for errors and attempt to rebuild it if required. + */ +static void +cagg_rebuild_view_definition(ContinuousAgg *agg, Hypertable *mat_ht, bool force_rebuild) +{ + bool test_failed = false; + char *relname = agg->data.user_view_name.data; + char *schema = agg->data.user_view_schema.data; + ListCell *lc1, *lc2; + int sec_ctx; + Oid uid, saved_uid; + + /* Cagg view created by the user. */ + Oid user_view_oid = relation_oid(&agg->data.user_view_schema, &agg->data.user_view_name); + Relation user_view_rel = relation_open(user_view_oid, AccessShareLock); + Query *user_query = get_view_query(user_view_rel); + + bool finalized = ContinuousAggIsFinalized(agg); + bool rebuild_cagg_with_joins = false; + + /* Extract final query from user view query. */ + Query *final_query = copyObject(user_query); + RemoveRangeTableEntries(final_query); + + if (finalized && !force_rebuild) + { + /* This continuous aggregate does not have partials, do not check for defects. */ + elog(DEBUG1, + "[cagg_rebuild_view_definition] %s.%s does not have partials, do not check for " + "defects!", + NameStr(agg->data.user_view_schema), + NameStr(agg->data.user_view_name) + + ); + relation_close(user_view_rel, NoLock); + return; + } + + if (!agg->data.materialized_only) + { + final_query = destroy_union_query(final_query); + } + FinalizeQueryInfo fqi; + MatTableColumnInfo mattblinfo; + ObjectAddress mataddress = { + .classId = RelationRelationId, + .objectId = mat_ht->main_table_relid, + }; + + Oid direct_view_oid = relation_oid(&agg->data.direct_view_schema, &agg->data.direct_view_name); + Relation direct_view_rel = relation_open(direct_view_oid, AccessShareLock); + Query *direct_query = copyObject(get_view_query(direct_view_rel)); + RemoveRangeTableEntries(direct_query); + + /* + * If there is a join in CAggs then rebuild it definitley, + * because v2.10.0 has created the definition with missing structs. + * + * Removed the check for direct_query->jointree != NULL because + * we don't allow queries without FROM clause in Continuous Aggregate + * definition. + * + * Per coverityscan: + * https://scan4.scan.coverity.com/reports.htm#v54116/p12995/fileInstanceId=131745632&defectInstanceId=14569562&mergedDefectId=384045 + * + */ + if (force_rebuild) + { + ListCell *l; + foreach (l, direct_query->jointree->fromlist) + { + Node *jtnode = (Node *) lfirst(l); + if (IsA(jtnode, JoinExpr)) + rebuild_cagg_with_joins = true; + } + } + + if (!rebuild_cagg_with_joins && finalized) + { + /* There's nothing to fix, so no need to rebuild */ + elog(DEBUG1, + "[cagg_rebuild_view_definition] %s.%s does not have JOINS, so no need to rebuild the " + "definition!", + NameStr(agg->data.user_view_schema), + NameStr(agg->data.user_view_name) + + ); + relation_close(user_view_rel, NoLock); + relation_close(direct_view_rel, NoLock); + return; + } + else + elog(DEBUG1, + "[cagg_rebuild_view_definition] %s.%s has been rebuilt!", + NameStr(agg->data.user_view_schema), + NameStr(agg->data.user_view_name)); + + CAggTimebucketInfo timebucket_exprinfo = + cagg_validate_query(direct_query, + finalized, + NameStr(agg->data.user_view_schema), + NameStr(agg->data.user_view_name)); + + mattablecolumninfo_init(&mattblinfo, copyObject(direct_query->groupClause)); + fqi.finalized = finalized; + finalizequery_init(&fqi, direct_query, &mattblinfo); + + /* + * Add any internal columns needed for materialization based + * on the user query's table. + */ + if (!finalized) + mattablecolumninfo_addinternal(&mattblinfo); + + Query *view_query = NULL; + if (rebuild_cagg_with_joins) + { + view_query = finalizequery_get_select_query(&fqi, + mattblinfo.matcollist, + &mataddress, + NameStr(mat_ht->fd.table_name)); + } + else + view_query = + finalizequery_get_select_query(&fqi, mattblinfo.matcollist, &mataddress, relname); + + if (!agg->data.materialized_only) + { + view_query = build_union_query(&timebucket_exprinfo, + mattblinfo.matpartcolno, + view_query, + direct_query, + mat_ht->fd.id); + } + + if (list_length(mattblinfo.matcollist) != ts_get_relnatts(mat_ht->main_table_relid)) + /* + * There is a mismatch of columns between the current version's finalization view + * building logic and the existing schema of the materialization table. As of version + * 2.7.0 this only happens due to buggy view generation in previous versions. Do not + * rebuild those views since the materialization table can not be queried correctly. + */ + test_failed = true; + /* + * When calling StoreViewQuery the target list names of the query have to + * match the view's tuple descriptor attribute names. But if a column of the continuous + * aggregate has been renamed, the query tree will not have the correct + * names in the target list, which will error out when calling + * StoreViewQuery. For that reason, we fetch the name from the user view + * relation and update the resource name in the query target list to match + * the name in the user view. + */ + TupleDesc desc = RelationGetDescr(user_view_rel); + int i = 0; + forboth (lc1, view_query->targetList, lc2, user_query->targetList) + { + TargetEntry *view_tle, *user_tle; + FormData_pg_attribute *attr = TupleDescAttr(desc, i); + view_tle = lfirst_node(TargetEntry, lc1); + user_tle = lfirst_node(TargetEntry, lc2); + if (view_tle->resjunk && user_tle->resjunk) + break; + else if (view_tle->resjunk || user_tle->resjunk) + { + /* + * This should never happen but if it ever does it's safer to + * error here instead of creating broken view definitions. + */ + test_failed = true; + break; + } + view_tle->resname = user_tle->resname = NameStr(attr->attname); + ++i; + } + + if (test_failed) + { + ereport(WARNING, + (errmsg("Inconsistent view definitions for continuous aggregate view " + "\"%s.%s\"", + schema, + relname), + errdetail("Continuous aggregate data possibly corrupted."), + errhint("You may need to recreate the continuous aggregate with CREATE " + "MATERIALIZED VIEW."))); + } + else + { + SWITCH_TO_TS_USER(NameStr(agg->data.user_view_schema), uid, saved_uid, sec_ctx); + StoreViewQuery(user_view_oid, view_query, true); + CommandCounterIncrement(); + RESTORE_USER(uid, saved_uid, sec_ctx); + } + /* + * Keep locks until end of transaction and do not close the relation + * before the call to StoreViewQuery since it can otherwise release the + * memory for attr->attname, causing a segfault. + */ + relation_close(direct_view_rel, NoLock); + relation_close(user_view_rel, NoLock); +} + +Datum +tsl_cagg_try_repair(PG_FUNCTION_ARGS) +{ + Oid relid = PG_ARGISNULL(0) ? InvalidOid : PG_GETARG_OID(0); + char relkind = get_rel_relkind(relid); + bool force_rebuild = PG_ARGISNULL(0) ? false : PG_GETARG_BOOL(1); + ContinuousAgg *cagg = NULL; + + if (RELKIND_VIEW == relkind) + cagg = ts_continuous_agg_find_by_relid(relid); + + if (RELKIND_VIEW != relkind || !cagg) + { + ereport(WARNING, + (errmsg("invalid OID \"%u\" for continuous aggregate view", relid), + errdetail("Check for database corruption."))); + PG_RETURN_VOID(); + } + + Cache *hcache = ts_hypertable_cache_pin(); + + Hypertable *mat_ht = ts_hypertable_cache_get_entry_by_id(hcache, cagg->data.mat_hypertable_id); + Assert(mat_ht != NULL); + cagg_rebuild_view_definition(cagg, mat_ht, force_rebuild); + + ts_cache_release(hcache); + + PG_RETURN_VOID(); +} diff --git a/tsl/src/continuous_aggs/repair.h b/tsl/src/continuous_aggs/repair.h new file mode 100644 index 000000000..4f6a5cb7b --- /dev/null +++ b/tsl/src/continuous_aggs/repair.h @@ -0,0 +1,19 @@ +/* + * This file and its contents are licensed under the Timescale License. + * Please see the included NOTICE for copyright information and + * LICENSE-TIMESCALE for a copy of the license. + */ +#ifndef TIMESCALEDB_TSL_CONTINUOUS_AGGS_CAGG_REPAIR_H +#define TIMESCALEDB_TSL_CONTINUOUS_AGGS_CAGG_REPAIR_H +#include + +#include + +#include "continuous_aggs/common.h" +#include "continuous_aggs/finalize.h" +#include "continuous_aggs/materialize.h" +#include "ts_catalog/continuous_agg.h" + +extern Datum tsl_cagg_try_repair(PG_FUNCTION_ARGS); + +#endif diff --git a/tsl/src/init.c b/tsl/src/init.c index f96812e0d..2366f98ab 100644 --- a/tsl/src/init.c +++ b/tsl/src/init.c @@ -30,6 +30,7 @@ #include "continuous_aggs/options.h" #include "continuous_aggs/refresh.h" #include "continuous_aggs/invalidation.h" +#include "continuous_aggs/repair.h" #include "cross_module_fn.h" #include "nodes/data_node_dispatch.h" #include "data_node.h"