From 859b97f01cafc840902a9d76e8502076261b0f98 Mon Sep 17 00:00:00 2001 From: Sven Klemm Date: Tue, 26 Mar 2019 05:28:01 +0100 Subject: [PATCH] Fix gapfill with prepared statements The start and finish arguments for time_bucket_gapfill are removed from the final execution. This adjustment used to happen in executor but this patch changes this behaviour and removes them in the planning phase and pass the original arguments to time_bucket_gapfill in custom_private so in execution phase time_bucket_gapfill arguments need not be adjusted. --- CHANGELOG.md | 1 + tsl/src/gapfill/exec.c | 23 +- tsl/src/gapfill/planner.c | 6 +- tsl/test/expected/gapfill.out | 331 +++++++++++++++++++++++++ tsl/test/expected/plan_gapfill-10.out | 40 +-- tsl/test/expected/plan_gapfill-11.out | 40 +-- tsl/test/expected/plan_gapfill-9.6.out | 32 +-- tsl/test/sql/gapfill.sql | 80 ++++++ 8 files changed, 483 insertions(+), 70 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index d943b9054..beecab26a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,7 @@ accidentally triggering the load of a previous DB version.** ## 1.3.0 (unreleased) **Minor Features** +* #1112 Add support for window functions to gapfill * #1062 Make constraint aware append parallel safe * #1005 Enable creating indexes with one transaction per chunk * #1007 Remove parent oid from find_children_oids result diff --git a/tsl/src/gapfill/exec.c b/tsl/src/gapfill/exec.c index 05910e927..d2bd9e875 100644 --- a/tsl/src/gapfill/exec.c +++ b/tsl/src/gapfill/exec.c @@ -500,6 +500,7 @@ gapfill_begin(CustomScanState *node, EState *estate, int eflags) * extract arguments and to align gapfill_start */ FuncExpr *func = linitial(cscan->custom_private); + List *args = lfourth(cscan->custom_private); TupleDesc tupledesc = state->csstate.ss.ps.ps_ResultTupleSlot->tts_tupleDescriptor; List *targetlist = copyObject(state->csstate.ss.ps.plan->targetlist); Node *entry; @@ -513,27 +514,26 @@ gapfill_begin(CustomScanState *node, EState *estate, int eflags) state->scanslot = MakeSingleTupleTableSlot(tupledesc); /* bucket_width */ - if (!is_simple_expr(linitial(func->args))) + if (!is_simple_expr(linitial(args))) ereport(ERROR, (errcode(ERRCODE_INVALID_PARAMETER_VALUE), errmsg("invalid time_bucket_gapfill argument: bucket_width must be a simple " "expression"))); - arg_value = gapfill_exec_expr(state, linitial(func->args), &isnull); + arg_value = gapfill_exec_expr(state, linitial(args), &isnull); if (isnull) ereport(ERROR, (errcode(ERRCODE_INVALID_PARAMETER_VALUE), errmsg("invalid time_bucket_gapfill argument: bucket_width cannot be NULL"))); - state->gapfill_period = gapfill_period_get_internal(func->funcresulttype, - exprType(linitial(func->args)), - arg_value); + state->gapfill_period = + gapfill_period_get_internal(func->funcresulttype, exprType(linitial(args)), arg_value); /* * check if gapfill start was left out so we have to infer from WHERE * clause */ - if (is_const_null(lthird(func->args))) + if (is_const_null(lthird(args))) { int64 start = infer_gapfill_boundary(state, GAPFILL_START); Const *expr = make_const_value_for_gapfill_internal(state->gapfill_typid, start); @@ -546,21 +546,21 @@ gapfill_begin(CustomScanState *node, EState *estate, int eflags) * pass gapfill start through time_bucket so it is aligned with bucket * start */ - state->gapfill_start = align_with_time_bucket(state, lthird(func->args)); + state->gapfill_start = align_with_time_bucket(state, lthird(args)); } state->next_timestamp = state->gapfill_start; /* gap fill end */ - if (is_const_null(lfourth(func->args))) + if (is_const_null(lfourth(args))) state->gapfill_end = infer_gapfill_boundary(state, GAPFILL_END); else { - if (!is_simple_expr(lfourth(func->args))) + if (!is_simple_expr(lfourth(args))) ereport(ERROR, (errcode(ERRCODE_INVALID_PARAMETER_VALUE), errmsg("invalid time_bucket_gapfill argument: finish must be a simple " "expression"))); - arg_value = gapfill_exec_expr(state, lfourth(func->args), &isnull); + arg_value = gapfill_exec_expr(state, lfourth(args), &isnull); /* * the default value for finish is NULL but this is checked above, @@ -576,9 +576,6 @@ gapfill_begin(CustomScanState *node, EState *estate, int eflags) state->gapfill_end = gapfill_datum_get_internal(arg_value, func->funcresulttype); } - /* remove start and end argument from time_bucket call */ - func->args = list_make2(linitial(func->args), lsecond(func->args)); - gapfill_state_initialize_columns(state); /* diff --git a/tsl/src/gapfill/planner.c b/tsl/src/gapfill/planner.c index 0ab0be5da..86cf78edf 100644 --- a/tsl/src/gapfill/planner.c +++ b/tsl/src/gapfill/planner.c @@ -166,6 +166,7 @@ gapfill_plan_create(PlannerInfo *root, RelOptInfo *rel, struct CustomPath *path, { GapFillPath *gfpath = (GapFillPath *) path; CustomScan *cscan = makeNode(CustomScan); + List *args = list_copy(gfpath->func->args); cscan->scan.scanrelid = 0; cscan->scan.plan.targetlist = tlist; @@ -175,7 +176,10 @@ gapfill_plan_create(PlannerInfo *root, RelOptInfo *rel, struct CustomPath *path, cscan->methods = &gapfill_plan_methods; cscan->custom_private = - list_make3(gfpath->func, root->parse->groupClause, root->parse->jointree); + list_make4(gfpath->func, root->parse->groupClause, root->parse->jointree, args); + + /* remove start and end argument from time_bucket call */ + gfpath->func->args = list_make2(linitial(gfpath->func->args), lsecond(gfpath->func->args)); return &cscan->scan.plan; } diff --git a/tsl/test/expected/gapfill.out b/tsl/test/expected/gapfill.out index 3ad8f26e1..95c6bdc5f 100644 --- a/tsl/test/expected/gapfill.out +++ b/tsl/test/expected/gapfill.out @@ -2116,3 +2116,334 @@ GROUP BY 3,4; 5 | 2 | red | 4 (10 rows) +-- test prepared statement +PREPARE prep_gapfill AS +SELECT + time_bucket_gapfill(1,time,0,5) as time, + locf(min(value)) +FROM (VALUES (1,1),(2,2)) v(time,value) +GROUP BY 1; +-- execute 10 times to make sure turning it into generic plan works +EXECUTE prep_gapfill; + time | locf +------+------ + 0 | + 1 | 1 + 2 | 2 + 3 | 2 + 4 | 2 +(5 rows) + +EXECUTE prep_gapfill; + time | locf +------+------ + 0 | + 1 | 1 + 2 | 2 + 3 | 2 + 4 | 2 +(5 rows) + +EXECUTE prep_gapfill; + time | locf +------+------ + 0 | + 1 | 1 + 2 | 2 + 3 | 2 + 4 | 2 +(5 rows) + +EXECUTE prep_gapfill; + time | locf +------+------ + 0 | + 1 | 1 + 2 | 2 + 3 | 2 + 4 | 2 +(5 rows) + +EXECUTE prep_gapfill; + time | locf +------+------ + 0 | + 1 | 1 + 2 | 2 + 3 | 2 + 4 | 2 +(5 rows) + +EXECUTE prep_gapfill; + time | locf +------+------ + 0 | + 1 | 1 + 2 | 2 + 3 | 2 + 4 | 2 +(5 rows) + +EXECUTE prep_gapfill; + time | locf +------+------ + 0 | + 1 | 1 + 2 | 2 + 3 | 2 + 4 | 2 +(5 rows) + +EXECUTE prep_gapfill; + time | locf +------+------ + 0 | + 1 | 1 + 2 | 2 + 3 | 2 + 4 | 2 +(5 rows) + +EXECUTE prep_gapfill; + time | locf +------+------ + 0 | + 1 | 1 + 2 | 2 + 3 | 2 + 4 | 2 +(5 rows) + +EXECUTE prep_gapfill; + time | locf +------+------ + 0 | + 1 | 1 + 2 | 2 + 3 | 2 + 4 | 2 +(5 rows) + +DEALLOCATE prep_gapfill; +-- test prepared statement with locf with lookup query +PREPARE prep_gapfill AS +SELECT + time_bucket_gapfill(5,time,0,11) AS time, + device_id, + sensor_id, + locf(min(value)::int,(SELECT 1/(SELECT 0) FROM metrics_int m2 WHERE m2.device_id=m1.device_id AND m2.sensor_id=m1.sensor_id ORDER BY time DESC LIMIT 1)) +FROM metrics_int m1 +WHERE time >= 0 AND time < 5 +GROUP BY 1,2,3; +-- execute 10 times to make sure turning it into generic plan works +EXECUTE prep_gapfill; + time | device_id | sensor_id | locf +------+-----------+-----------+------ + 0 | 1 | 1 | 5 + 5 | 1 | 1 | 5 + 10 | 1 | 1 | 5 +(3 rows) + +EXECUTE prep_gapfill; + time | device_id | sensor_id | locf +------+-----------+-----------+------ + 0 | 1 | 1 | 5 + 5 | 1 | 1 | 5 + 10 | 1 | 1 | 5 +(3 rows) + +EXECUTE prep_gapfill; + time | device_id | sensor_id | locf +------+-----------+-----------+------ + 0 | 1 | 1 | 5 + 5 | 1 | 1 | 5 + 10 | 1 | 1 | 5 +(3 rows) + +EXECUTE prep_gapfill; + time | device_id | sensor_id | locf +------+-----------+-----------+------ + 0 | 1 | 1 | 5 + 5 | 1 | 1 | 5 + 10 | 1 | 1 | 5 +(3 rows) + +EXECUTE prep_gapfill; + time | device_id | sensor_id | locf +------+-----------+-----------+------ + 0 | 1 | 1 | 5 + 5 | 1 | 1 | 5 + 10 | 1 | 1 | 5 +(3 rows) + +EXECUTE prep_gapfill; + time | device_id | sensor_id | locf +------+-----------+-----------+------ + 0 | 1 | 1 | 5 + 5 | 1 | 1 | 5 + 10 | 1 | 1 | 5 +(3 rows) + +EXECUTE prep_gapfill; + time | device_id | sensor_id | locf +------+-----------+-----------+------ + 0 | 1 | 1 | 5 + 5 | 1 | 1 | 5 + 10 | 1 | 1 | 5 +(3 rows) + +EXECUTE prep_gapfill; + time | device_id | sensor_id | locf +------+-----------+-----------+------ + 0 | 1 | 1 | 5 + 5 | 1 | 1 | 5 + 10 | 1 | 1 | 5 +(3 rows) + +EXECUTE prep_gapfill; + time | device_id | sensor_id | locf +------+-----------+-----------+------ + 0 | 1 | 1 | 5 + 5 | 1 | 1 | 5 + 10 | 1 | 1 | 5 +(3 rows) + +EXECUTE prep_gapfill; + time | device_id | sensor_id | locf +------+-----------+-----------+------ + 0 | 1 | 1 | 5 + 5 | 1 | 1 | 5 + 10 | 1 | 1 | 5 +(3 rows) + +DEALLOCATE prep_gapfill; +-- test prepared statement with interpolate with lookup query +PREPARE prep_gapfill AS +SELECT + time_bucket_gapfill(5,time,0,11) AS time, + device_id, + sensor_id, + interpolate( + min(value), + (SELECT (time,value) FROM metrics_int m2 + WHERE time<0 AND m2.device_id=m1.device_id AND m2.sensor_id=m1.sensor_id + ORDER BY time DESC LIMIT 1), + (SELECT (time,value) FROM metrics_int m2 + WHERE time>10 AND m2.device_id=m1.device_id AND m2.sensor_id=m1.sensor_id + ORDER BY time LIMIT 1) + ) +FROM metrics_int m1 +WHERE time >= 0 AND time < 10 +GROUP BY 1,2,3 ORDER BY 2,3,1; +-- execute 10 times to make sure turning it into generic plan works +EXECUTE prep_gapfill; + time | device_id | sensor_id | interpolate +------+-----------+-----------+------------------ + 0 | 1 | 1 | 5 + 5 | 1 | 1 | 4.75 + 10 | 1 | 1 | 4.5 + 0 | 1 | 2 | 4.76190476190476 + 5 | 1 | 2 | 10 + 10 | 1 | 2 | 4.21052631578947 +(6 rows) + +EXECUTE prep_gapfill; + time | device_id | sensor_id | interpolate +------+-----------+-----------+------------------ + 0 | 1 | 1 | 5 + 5 | 1 | 1 | 4.75 + 10 | 1 | 1 | 4.5 + 0 | 1 | 2 | 4.76190476190476 + 5 | 1 | 2 | 10 + 10 | 1 | 2 | 4.21052631578947 +(6 rows) + +EXECUTE prep_gapfill; + time | device_id | sensor_id | interpolate +------+-----------+-----------+------------------ + 0 | 1 | 1 | 5 + 5 | 1 | 1 | 4.75 + 10 | 1 | 1 | 4.5 + 0 | 1 | 2 | 4.76190476190476 + 5 | 1 | 2 | 10 + 10 | 1 | 2 | 4.21052631578947 +(6 rows) + +EXECUTE prep_gapfill; + time | device_id | sensor_id | interpolate +------+-----------+-----------+------------------ + 0 | 1 | 1 | 5 + 5 | 1 | 1 | 4.75 + 10 | 1 | 1 | 4.5 + 0 | 1 | 2 | 4.76190476190476 + 5 | 1 | 2 | 10 + 10 | 1 | 2 | 4.21052631578947 +(6 rows) + +EXECUTE prep_gapfill; + time | device_id | sensor_id | interpolate +------+-----------+-----------+------------------ + 0 | 1 | 1 | 5 + 5 | 1 | 1 | 4.75 + 10 | 1 | 1 | 4.5 + 0 | 1 | 2 | 4.76190476190476 + 5 | 1 | 2 | 10 + 10 | 1 | 2 | 4.21052631578947 +(6 rows) + +EXECUTE prep_gapfill; + time | device_id | sensor_id | interpolate +------+-----------+-----------+------------------ + 0 | 1 | 1 | 5 + 5 | 1 | 1 | 4.75 + 10 | 1 | 1 | 4.5 + 0 | 1 | 2 | 4.76190476190476 + 5 | 1 | 2 | 10 + 10 | 1 | 2 | 4.21052631578947 +(6 rows) + +EXECUTE prep_gapfill; + time | device_id | sensor_id | interpolate +------+-----------+-----------+------------------ + 0 | 1 | 1 | 5 + 5 | 1 | 1 | 4.75 + 10 | 1 | 1 | 4.5 + 0 | 1 | 2 | 4.76190476190476 + 5 | 1 | 2 | 10 + 10 | 1 | 2 | 4.21052631578947 +(6 rows) + +EXECUTE prep_gapfill; + time | device_id | sensor_id | interpolate +------+-----------+-----------+------------------ + 0 | 1 | 1 | 5 + 5 | 1 | 1 | 4.75 + 10 | 1 | 1 | 4.5 + 0 | 1 | 2 | 4.76190476190476 + 5 | 1 | 2 | 10 + 10 | 1 | 2 | 4.21052631578947 +(6 rows) + +EXECUTE prep_gapfill; + time | device_id | sensor_id | interpolate +------+-----------+-----------+------------------ + 0 | 1 | 1 | 5 + 5 | 1 | 1 | 4.75 + 10 | 1 | 1 | 4.5 + 0 | 1 | 2 | 4.76190476190476 + 5 | 1 | 2 | 10 + 10 | 1 | 2 | 4.21052631578947 +(6 rows) + +EXECUTE prep_gapfill; + time | device_id | sensor_id | interpolate +------+-----------+-----------+------------------ + 0 | 1 | 1 | 5 + 5 | 1 | 1 | 4.75 + 10 | 1 | 1 | 4.5 + 0 | 1 | 2 | 4.76190476190476 + 5 | 1 | 2 | 10 + 10 | 1 | 2 | 4.21052631578947 +(6 rows) + +DEALLOCATE prep_gapfill; diff --git a/tsl/test/expected/plan_gapfill-10.out b/tsl/test/expected/plan_gapfill-10.out index 2b6200495..0ea443852 100644 --- a/tsl/test/expected/plan_gapfill-10.out +++ b/tsl/test/expected/plan_gapfill-10.out @@ -114,17 +114,17 @@ SELECT FROM gapfill_plan_test GROUP BY 1 ORDER BY 1; - QUERY PLAN ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ + QUERY PLAN +--------------------------------------------------------------------------------------------------------- Custom Scan (GapFill) -> Finalize GroupAggregate - Group Key: (time_bucket_gapfill('@ 5 mins'::interval, _hyper_1_1_chunk."time", 'Wed Dec 31 16:00:00 1969 PST'::timestamp with time zone, 'Wed Dec 31 16:00:00 1969 PST'::timestamp with time zone)) + Group Key: (time_bucket_gapfill('@ 5 mins'::interval, _hyper_1_1_chunk."time")) -> Sort - Sort Key: (time_bucket_gapfill('@ 5 mins'::interval, _hyper_1_1_chunk."time", 'Wed Dec 31 16:00:00 1969 PST'::timestamp with time zone, 'Wed Dec 31 16:00:00 1969 PST'::timestamp with time zone)) + Sort Key: (time_bucket_gapfill('@ 5 mins'::interval, _hyper_1_1_chunk."time")) -> Gather Workers Planned: 1 -> Partial HashAggregate - Group Key: time_bucket_gapfill('@ 5 mins'::interval, _hyper_1_1_chunk."time", 'Wed Dec 31 16:00:00 1969 PST'::timestamp with time zone, 'Wed Dec 31 16:00:00 1969 PST'::timestamp with time zone) + Group Key: time_bucket_gapfill('@ 5 mins'::interval, _hyper_1_1_chunk."time") -> Result -> Append -> Parallel Seq Scan on _hyper_1_1_chunk @@ -141,17 +141,17 @@ SELECT FROM gapfill_plan_test GROUP BY 1 ORDER BY 1; - QUERY PLAN ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ + QUERY PLAN +--------------------------------------------------------------------------------------------------------- Custom Scan (GapFill) -> Finalize GroupAggregate - Group Key: (time_bucket_gapfill('@ 5 mins'::interval, _hyper_1_1_chunk."time", 'Wed Dec 31 16:00:00 1969 PST'::timestamp with time zone, 'Wed Dec 31 16:00:00 1969 PST'::timestamp with time zone)) + Group Key: (time_bucket_gapfill('@ 5 mins'::interval, _hyper_1_1_chunk."time")) -> Sort - Sort Key: (time_bucket_gapfill('@ 5 mins'::interval, _hyper_1_1_chunk."time", 'Wed Dec 31 16:00:00 1969 PST'::timestamp with time zone, 'Wed Dec 31 16:00:00 1969 PST'::timestamp with time zone)) + Sort Key: (time_bucket_gapfill('@ 5 mins'::interval, _hyper_1_1_chunk."time")) -> Gather Workers Planned: 1 -> Partial HashAggregate - Group Key: time_bucket_gapfill('@ 5 mins'::interval, _hyper_1_1_chunk."time", 'Wed Dec 31 16:00:00 1969 PST'::timestamp with time zone, 'Wed Dec 31 16:00:00 1969 PST'::timestamp with time zone) + Group Key: time_bucket_gapfill('@ 5 mins'::interval, _hyper_1_1_chunk."time") -> Result -> Append -> Parallel Seq Scan on _hyper_1_1_chunk @@ -168,17 +168,17 @@ SELECT FROM gapfill_plan_test GROUP BY 1 ORDER BY 1; - QUERY PLAN ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ + QUERY PLAN +--------------------------------------------------------------------------------------------------------- Custom Scan (GapFill) -> Finalize GroupAggregate - Group Key: (time_bucket_gapfill('@ 5 mins'::interval, _hyper_1_1_chunk."time", 'Wed Dec 31 16:00:00 1969 PST'::timestamp with time zone, 'Wed Dec 31 16:00:00 1969 PST'::timestamp with time zone)) + Group Key: (time_bucket_gapfill('@ 5 mins'::interval, _hyper_1_1_chunk."time")) -> Sort - Sort Key: (time_bucket_gapfill('@ 5 mins'::interval, _hyper_1_1_chunk."time", 'Wed Dec 31 16:00:00 1969 PST'::timestamp with time zone, 'Wed Dec 31 16:00:00 1969 PST'::timestamp with time zone)) + Sort Key: (time_bucket_gapfill('@ 5 mins'::interval, _hyper_1_1_chunk."time")) -> Gather Workers Planned: 1 -> Partial HashAggregate - Group Key: time_bucket_gapfill('@ 5 mins'::interval, _hyper_1_1_chunk."time", 'Wed Dec 31 16:00:00 1969 PST'::timestamp with time zone, 'Wed Dec 31 16:00:00 1969 PST'::timestamp with time zone) + Group Key: time_bucket_gapfill('@ 5 mins'::interval, _hyper_1_1_chunk."time") -> Result -> Append -> Parallel Seq Scan on _hyper_1_1_chunk @@ -197,20 +197,20 @@ FROM gapfill_plan_test GROUP BY 1 ORDER BY 2 LIMIT 1; - QUERY PLAN ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ + QUERY PLAN +--------------------------------------------------------------------------------------------------------------------- Limit -> Sort Sort Key: (interpolate(avg(value), NULL::record, NULL::record)) -> Custom Scan (GapFill) -> Finalize GroupAggregate - Group Key: (time_bucket_gapfill('@ 5 mins'::interval, _hyper_1_1_chunk."time", 'Wed Dec 31 16:00:00 1969 PST'::timestamp with time zone, 'Wed Dec 31 16:00:00 1969 PST'::timestamp with time zone)) + Group Key: (time_bucket_gapfill('@ 5 mins'::interval, _hyper_1_1_chunk."time")) -> Sort - Sort Key: (time_bucket_gapfill('@ 5 mins'::interval, _hyper_1_1_chunk."time", 'Wed Dec 31 16:00:00 1969 PST'::timestamp with time zone, 'Wed Dec 31 16:00:00 1969 PST'::timestamp with time zone)) + Sort Key: (time_bucket_gapfill('@ 5 mins'::interval, _hyper_1_1_chunk."time")) -> Gather Workers Planned: 1 -> Partial HashAggregate - Group Key: time_bucket_gapfill('@ 5 mins'::interval, _hyper_1_1_chunk."time", 'Wed Dec 31 16:00:00 1969 PST'::timestamp with time zone, 'Wed Dec 31 16:00:00 1969 PST'::timestamp with time zone) + Group Key: time_bucket_gapfill('@ 5 mins'::interval, _hyper_1_1_chunk."time") -> Result -> Append -> Parallel Seq Scan on _hyper_1_1_chunk diff --git a/tsl/test/expected/plan_gapfill-11.out b/tsl/test/expected/plan_gapfill-11.out index d0160f8cf..6fd79e851 100644 --- a/tsl/test/expected/plan_gapfill-11.out +++ b/tsl/test/expected/plan_gapfill-11.out @@ -114,17 +114,17 @@ SELECT FROM gapfill_plan_test GROUP BY 1 ORDER BY 1; - QUERY PLAN ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ + QUERY PLAN +--------------------------------------------------------------------------------------------------------- Custom Scan (GapFill) -> Finalize GroupAggregate - Group Key: (time_bucket_gapfill('@ 5 mins'::interval, _hyper_1_2_chunk."time", 'Wed Dec 31 16:00:00 1969 PST'::timestamp with time zone, 'Wed Dec 31 16:00:00 1969 PST'::timestamp with time zone)) + Group Key: (time_bucket_gapfill('@ 5 mins'::interval, _hyper_1_2_chunk."time")) -> Gather Merge Workers Planned: 2 -> Sort - Sort Key: (time_bucket_gapfill('@ 5 mins'::interval, _hyper_1_2_chunk."time", 'Wed Dec 31 16:00:00 1969 PST'::timestamp with time zone, 'Wed Dec 31 16:00:00 1969 PST'::timestamp with time zone)) + Sort Key: (time_bucket_gapfill('@ 5 mins'::interval, _hyper_1_2_chunk."time")) -> Partial HashAggregate - Group Key: time_bucket_gapfill('@ 5 mins'::interval, _hyper_1_2_chunk."time", 'Wed Dec 31 16:00:00 1969 PST'::timestamp with time zone, 'Wed Dec 31 16:00:00 1969 PST'::timestamp with time zone) + Group Key: time_bucket_gapfill('@ 5 mins'::interval, _hyper_1_2_chunk."time") -> Result -> Parallel Append -> Parallel Seq Scan on _hyper_1_2_chunk @@ -141,17 +141,17 @@ SELECT FROM gapfill_plan_test GROUP BY 1 ORDER BY 1; - QUERY PLAN ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ + QUERY PLAN +--------------------------------------------------------------------------------------------------------- Custom Scan (GapFill) -> Finalize GroupAggregate - Group Key: (time_bucket_gapfill('@ 5 mins'::interval, _hyper_1_2_chunk."time", 'Wed Dec 31 16:00:00 1969 PST'::timestamp with time zone, 'Wed Dec 31 16:00:00 1969 PST'::timestamp with time zone)) + Group Key: (time_bucket_gapfill('@ 5 mins'::interval, _hyper_1_2_chunk."time")) -> Gather Merge Workers Planned: 2 -> Sort - Sort Key: (time_bucket_gapfill('@ 5 mins'::interval, _hyper_1_2_chunk."time", 'Wed Dec 31 16:00:00 1969 PST'::timestamp with time zone, 'Wed Dec 31 16:00:00 1969 PST'::timestamp with time zone)) + Sort Key: (time_bucket_gapfill('@ 5 mins'::interval, _hyper_1_2_chunk."time")) -> Partial HashAggregate - Group Key: time_bucket_gapfill('@ 5 mins'::interval, _hyper_1_2_chunk."time", 'Wed Dec 31 16:00:00 1969 PST'::timestamp with time zone, 'Wed Dec 31 16:00:00 1969 PST'::timestamp with time zone) + Group Key: time_bucket_gapfill('@ 5 mins'::interval, _hyper_1_2_chunk."time") -> Result -> Parallel Append -> Parallel Seq Scan on _hyper_1_2_chunk @@ -168,17 +168,17 @@ SELECT FROM gapfill_plan_test GROUP BY 1 ORDER BY 1; - QUERY PLAN ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ + QUERY PLAN +--------------------------------------------------------------------------------------------------------- Custom Scan (GapFill) -> Finalize GroupAggregate - Group Key: (time_bucket_gapfill('@ 5 mins'::interval, _hyper_1_2_chunk."time", 'Wed Dec 31 16:00:00 1969 PST'::timestamp with time zone, 'Wed Dec 31 16:00:00 1969 PST'::timestamp with time zone)) + Group Key: (time_bucket_gapfill('@ 5 mins'::interval, _hyper_1_2_chunk."time")) -> Gather Merge Workers Planned: 2 -> Sort - Sort Key: (time_bucket_gapfill('@ 5 mins'::interval, _hyper_1_2_chunk."time", 'Wed Dec 31 16:00:00 1969 PST'::timestamp with time zone, 'Wed Dec 31 16:00:00 1969 PST'::timestamp with time zone)) + Sort Key: (time_bucket_gapfill('@ 5 mins'::interval, _hyper_1_2_chunk."time")) -> Partial HashAggregate - Group Key: time_bucket_gapfill('@ 5 mins'::interval, _hyper_1_2_chunk."time", 'Wed Dec 31 16:00:00 1969 PST'::timestamp with time zone, 'Wed Dec 31 16:00:00 1969 PST'::timestamp with time zone) + Group Key: time_bucket_gapfill('@ 5 mins'::interval, _hyper_1_2_chunk."time") -> Result -> Parallel Append -> Parallel Seq Scan on _hyper_1_2_chunk @@ -197,20 +197,20 @@ FROM gapfill_plan_test GROUP BY 1 ORDER BY 2 LIMIT 1; - QUERY PLAN ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ + QUERY PLAN +--------------------------------------------------------------------------------------------------------------------- Limit -> Sort Sort Key: (interpolate(avg(value), NULL::record, NULL::record)) -> Custom Scan (GapFill) -> Finalize GroupAggregate - Group Key: (time_bucket_gapfill('@ 5 mins'::interval, _hyper_1_2_chunk."time", 'Wed Dec 31 16:00:00 1969 PST'::timestamp with time zone, 'Wed Dec 31 16:00:00 1969 PST'::timestamp with time zone)) + Group Key: (time_bucket_gapfill('@ 5 mins'::interval, _hyper_1_2_chunk."time")) -> Gather Merge Workers Planned: 2 -> Sort - Sort Key: (time_bucket_gapfill('@ 5 mins'::interval, _hyper_1_2_chunk."time", 'Wed Dec 31 16:00:00 1969 PST'::timestamp with time zone, 'Wed Dec 31 16:00:00 1969 PST'::timestamp with time zone)) + Sort Key: (time_bucket_gapfill('@ 5 mins'::interval, _hyper_1_2_chunk."time")) -> Partial HashAggregate - Group Key: time_bucket_gapfill('@ 5 mins'::interval, _hyper_1_2_chunk."time", 'Wed Dec 31 16:00:00 1969 PST'::timestamp with time zone, 'Wed Dec 31 16:00:00 1969 PST'::timestamp with time zone) + Group Key: time_bucket_gapfill('@ 5 mins'::interval, _hyper_1_2_chunk."time") -> Result -> Parallel Append -> Parallel Seq Scan on _hyper_1_2_chunk diff --git a/tsl/test/expected/plan_gapfill-9.6.out b/tsl/test/expected/plan_gapfill-9.6.out index 1ee35d2be..ad1ed9d69 100644 --- a/tsl/test/expected/plan_gapfill-9.6.out +++ b/tsl/test/expected/plan_gapfill-9.6.out @@ -114,13 +114,13 @@ SELECT FROM gapfill_plan_test GROUP BY 1 ORDER BY 1; - QUERY PLAN ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ + QUERY PLAN +--------------------------------------------------------------------------------------------- Custom Scan (GapFill) -> Sort - Sort Key: (time_bucket_gapfill('@ 5 mins'::interval, _hyper_1_1_chunk."time", 'Wed Dec 31 16:00:00 1969 PST'::timestamp with time zone, 'Wed Dec 31 16:00:00 1969 PST'::timestamp with time zone)) + Sort Key: (time_bucket_gapfill('@ 5 mins'::interval, _hyper_1_1_chunk."time")) -> HashAggregate - Group Key: time_bucket_gapfill('@ 5 mins'::interval, _hyper_1_1_chunk."time", 'Wed Dec 31 16:00:00 1969 PST'::timestamp with time zone, 'Wed Dec 31 16:00:00 1969 PST'::timestamp with time zone) + Group Key: time_bucket_gapfill('@ 5 mins'::interval, _hyper_1_1_chunk."time") -> Result -> Append -> Seq Scan on _hyper_1_1_chunk @@ -137,13 +137,13 @@ SELECT FROM gapfill_plan_test GROUP BY 1 ORDER BY 1; - QUERY PLAN ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ + QUERY PLAN +--------------------------------------------------------------------------------------------- Custom Scan (GapFill) -> Sort - Sort Key: (time_bucket_gapfill('@ 5 mins'::interval, _hyper_1_1_chunk."time", 'Wed Dec 31 16:00:00 1969 PST'::timestamp with time zone, 'Wed Dec 31 16:00:00 1969 PST'::timestamp with time zone)) + Sort Key: (time_bucket_gapfill('@ 5 mins'::interval, _hyper_1_1_chunk."time")) -> HashAggregate - Group Key: time_bucket_gapfill('@ 5 mins'::interval, _hyper_1_1_chunk."time", 'Wed Dec 31 16:00:00 1969 PST'::timestamp with time zone, 'Wed Dec 31 16:00:00 1969 PST'::timestamp with time zone) + Group Key: time_bucket_gapfill('@ 5 mins'::interval, _hyper_1_1_chunk."time") -> Result -> Append -> Seq Scan on _hyper_1_1_chunk @@ -160,13 +160,13 @@ SELECT FROM gapfill_plan_test GROUP BY 1 ORDER BY 1; - QUERY PLAN ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ + QUERY PLAN +--------------------------------------------------------------------------------------------- Custom Scan (GapFill) -> Sort - Sort Key: (time_bucket_gapfill('@ 5 mins'::interval, _hyper_1_1_chunk."time", 'Wed Dec 31 16:00:00 1969 PST'::timestamp with time zone, 'Wed Dec 31 16:00:00 1969 PST'::timestamp with time zone)) + Sort Key: (time_bucket_gapfill('@ 5 mins'::interval, _hyper_1_1_chunk."time")) -> HashAggregate - Group Key: time_bucket_gapfill('@ 5 mins'::interval, _hyper_1_1_chunk."time", 'Wed Dec 31 16:00:00 1969 PST'::timestamp with time zone, 'Wed Dec 31 16:00:00 1969 PST'::timestamp with time zone) + Group Key: time_bucket_gapfill('@ 5 mins'::interval, _hyper_1_1_chunk."time") -> Result -> Append -> Seq Scan on _hyper_1_1_chunk @@ -185,16 +185,16 @@ FROM gapfill_plan_test GROUP BY 1 ORDER BY 2 LIMIT 1; - QUERY PLAN ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ + QUERY PLAN +--------------------------------------------------------------------------------------------------------- Limit -> Sort Sort Key: (interpolate(avg(value), NULL::record, NULL::record)) -> Custom Scan (GapFill) -> Sort - Sort Key: (time_bucket_gapfill('@ 5 mins'::interval, _hyper_1_1_chunk."time", 'Wed Dec 31 16:00:00 1969 PST'::timestamp with time zone, 'Wed Dec 31 16:00:00 1969 PST'::timestamp with time zone)) + Sort Key: (time_bucket_gapfill('@ 5 mins'::interval, _hyper_1_1_chunk."time")) -> HashAggregate - Group Key: time_bucket_gapfill('@ 5 mins'::interval, _hyper_1_1_chunk."time", 'Wed Dec 31 16:00:00 1969 PST'::timestamp with time zone, 'Wed Dec 31 16:00:00 1969 PST'::timestamp with time zone) + Group Key: time_bucket_gapfill('@ 5 mins'::interval, _hyper_1_1_chunk."time") -> Result -> Append -> Seq Scan on _hyper_1_1_chunk diff --git a/tsl/test/sql/gapfill.sql b/tsl/test/sql/gapfill.sql index 209755660..29fea6c7e 100644 --- a/tsl/test/sql/gapfill.sql +++ b/tsl/test/sql/gapfill.sql @@ -1258,3 +1258,83 @@ SELECT FROM (VALUES (1,'blue',1),(2,'red',2)) v(time,color,value) GROUP BY 3,4; +-- test prepared statement +PREPARE prep_gapfill AS +SELECT + time_bucket_gapfill(1,time,0,5) as time, + locf(min(value)) +FROM (VALUES (1,1),(2,2)) v(time,value) +GROUP BY 1; + +-- execute 10 times to make sure turning it into generic plan works +EXECUTE prep_gapfill; +EXECUTE prep_gapfill; +EXECUTE prep_gapfill; +EXECUTE prep_gapfill; +EXECUTE prep_gapfill; +EXECUTE prep_gapfill; +EXECUTE prep_gapfill; +EXECUTE prep_gapfill; +EXECUTE prep_gapfill; +EXECUTE prep_gapfill; + +DEALLOCATE prep_gapfill; + +-- test prepared statement with locf with lookup query +PREPARE prep_gapfill AS +SELECT + time_bucket_gapfill(5,time,0,11) AS time, + device_id, + sensor_id, + locf(min(value)::int,(SELECT 1/(SELECT 0) FROM metrics_int m2 WHERE m2.device_id=m1.device_id AND m2.sensor_id=m1.sensor_id ORDER BY time DESC LIMIT 1)) +FROM metrics_int m1 +WHERE time >= 0 AND time < 5 +GROUP BY 1,2,3; + +-- execute 10 times to make sure turning it into generic plan works +EXECUTE prep_gapfill; +EXECUTE prep_gapfill; +EXECUTE prep_gapfill; +EXECUTE prep_gapfill; +EXECUTE prep_gapfill; +EXECUTE prep_gapfill; +EXECUTE prep_gapfill; +EXECUTE prep_gapfill; +EXECUTE prep_gapfill; +EXECUTE prep_gapfill; + +DEALLOCATE prep_gapfill; + +-- test prepared statement with interpolate with lookup query +PREPARE prep_gapfill AS +SELECT + time_bucket_gapfill(5,time,0,11) AS time, + device_id, + sensor_id, + interpolate( + min(value), + (SELECT (time,value) FROM metrics_int m2 + WHERE time<0 AND m2.device_id=m1.device_id AND m2.sensor_id=m1.sensor_id + ORDER BY time DESC LIMIT 1), + (SELECT (time,value) FROM metrics_int m2 + WHERE time>10 AND m2.device_id=m1.device_id AND m2.sensor_id=m1.sensor_id + ORDER BY time LIMIT 1) + ) +FROM metrics_int m1 +WHERE time >= 0 AND time < 10 +GROUP BY 1,2,3 ORDER BY 2,3,1; + +-- execute 10 times to make sure turning it into generic plan works +EXECUTE prep_gapfill; +EXECUTE prep_gapfill; +EXECUTE prep_gapfill; +EXECUTE prep_gapfill; +EXECUTE prep_gapfill; +EXECUTE prep_gapfill; +EXECUTE prep_gapfill; +EXECUTE prep_gapfill; +EXECUTE prep_gapfill; +EXECUTE prep_gapfill; + +DEALLOCATE prep_gapfill; +