postgres/timescaledb
1
0
Fork 0
mirror of https://github.com/timescale/timescaledb.git synced 2025-05-15 18:13:18 +08:00
Code Issues Packages Projects Releases Wiki Activity

Optimize index path generation for ordered append

Browse Source 
When creating index paths for compressed chunks, root->eq_classes
can contain a lot of entries from other relations which slow down
plan time. By filtering them to include only ECs which are from
that relation, we can improve plan times significantly when
lots of chunks are involved in the query.
This commit is contained in:
Ante Kresic 2024-07-16 09:50:15 +02:00 committed by Ante Kresic
parent 11af76179d
commit 377cc15c70
5 changed files with 391 additions and 0 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

27
tsl/src/nodes/decompress_chunk/decompress_chunk.c
View File

@ -139,6 +139,8 @@ append_ec_for_seqnum(PlannerInfo *root, CompressionInfo *info, SortInfo *sort_in
newec->ec_max_security = 0;
newec->ec_merged = NULL;
info->compressed_rel->eclass_indexes =
bms_add_member(info->compressed_rel->eclass_indexes, list_length(root->eq_classes));
root->eq_classes = lappend(root->eq_classes, newec);
MemoryContextSwitchTo(oldcontext);
@ -1839,11 +1841,36 @@ create_compressed_scan_paths(PlannerInfo *root, RelOptInfo *compressed_rel, Comp
* decompression
*/
List *orig_pathkeys = root->query_pathkeys;
List *orig_eq_classes = root->eq_classes;
Bitmapset *orig_eclass_indexes = info->compressed_rel->eclass_indexes;
build_compressed_scan_pathkeys(sort_info, root, root->query_pathkeys, info);
root->query_pathkeys = sort_info->required_compressed_pathkeys;
/* We can optimize iterating over EquivalenceClasses by reducing them to
* the subset which are from the compressed chunk. This only works if we don't
* have joins based on equivalence classes involved since those
* use eclass_indexes which is not valid with this optimization.
*
* Clauseless joins work fine since they don't rely on eclass_indexes.
*/
if (!info->chunk_rel->has_eclass_joins)
{
int i = -1;
List *required_eq_classes = NIL;
while ((i = bms_next_member(info->compressed_rel->eclass_indexes, i)) >= 0)
{
EquivalenceClass *cur_ec = (EquivalenceClass *) list_nth(root->eq_classes, i);
required_eq_classes = lappend(required_eq_classes, cur_ec);
}
root->eq_classes = required_eq_classes;
info->compressed_rel->eclass_indexes = NULL;
}
check_index_predicates(root, compressed_rel);
create_index_paths(root, compressed_rel);
root->query_pathkeys = orig_pathkeys;
root->eq_classes = orig_eq_classes;
info->compressed_rel->eclass_indexes = orig_eclass_indexes;
}
else
{

115
tsl/test/expected/transparent_decompression-14.out
View File

@ -1799,6 +1799,64 @@ ORDER BY q1.time;
Rows Removed by Filter: 4
(41 rows)
:PREFIX WITH q1 AS (
SELECT time,
v1
FROM :TEST_TABLE
WHERE device_id = 1
ORDER BY time
LIMIT 5
),
q2 AS (
SELECT time,
v2
FROM :TEST_TABLE
WHERE device_id = 2
ORDER BY time
LIMIT 5
)
SELECT *
FROM q1, q2;
QUERY PLAN
---------------------------------------------------------------------------------------------------------------------------------------------
Nested Loop (actual rows=25 loops=1)
-> Limit (actual rows=5 loops=1)
-> Custom Scan (ChunkAppend) on metrics (actual rows=5 loops=1)
Order: metrics."time"
-> Sort (actual rows=5 loops=1)
Sort Key: _hyper_1_1_chunk."time"
Sort Method: top-N heapsort
-> Custom Scan (DecompressChunk) on _hyper_1_1_chunk (actual rows=720 loops=1)
-> Seq Scan on compress_hyper_5_15_chunk (actual rows=1 loops=1)
Filter: (device_id = 1)
Rows Removed by Filter: 4
-> Index Scan Backward using _hyper_1_2_chunk_metrics_time_idx on _hyper_1_2_chunk (never executed)
Filter: (device_id = 1)
-> Sort (never executed)
Sort Key: _hyper_1_3_chunk."time"
-> Custom Scan (DecompressChunk) on _hyper_1_3_chunk (never executed)
-> Seq Scan on compress_hyper_5_16_chunk (never executed)
Filter: (device_id = 1)
-> Materialize (actual rows=5 loops=5)
-> Limit (actual rows=5 loops=1)
-> Custom Scan (ChunkAppend) on metrics metrics_1 (actual rows=5 loops=1)
Order: metrics_1."time"
-> Sort (actual rows=5 loops=1)
Sort Key: _hyper_1_1_chunk_1."time"
Sort Method: top-N heapsort
-> Custom Scan (DecompressChunk) on _hyper_1_1_chunk _hyper_1_1_chunk_1 (actual rows=720 loops=1)
-> Seq Scan on compress_hyper_5_15_chunk compress_hyper_5_15_chunk_1 (actual rows=1 loops=1)
Filter: (device_id = 2)
Rows Removed by Filter: 4
-> Index Scan Backward using _hyper_1_2_chunk_metrics_time_idx on _hyper_1_2_chunk _hyper_1_2_chunk_1 (never executed)
Filter: (device_id = 2)
-> Sort (never executed)
Sort Key: _hyper_1_3_chunk_1."time"
-> Custom Scan (DecompressChunk) on _hyper_1_3_chunk _hyper_1_3_chunk_1 (never executed)
-> Seq Scan on compress_hyper_5_16_chunk compress_hyper_5_16_chunk_1 (never executed)
Filter: (device_id = 2)
(36 rows)
-- test prepared statement
PREPARE prep AS
SELECT count(time)
@ -5818,6 +5876,63 @@ ORDER BY q1.time;
Rows Removed by Filter: 2
(37 rows)
:PREFIX WITH q1 AS (
SELECT time,
v1
FROM :TEST_TABLE
WHERE device_id = 1
ORDER BY time
LIMIT 5
),
q2 AS (
SELECT time,
v2
FROM :TEST_TABLE
WHERE device_id = 2
ORDER BY time
LIMIT 5
)
SELECT *
FROM q1, q2;
QUERY PLAN
------------------------------------------------------------------------------------------------------------------------------------------
Nested Loop (actual rows=25 loops=1)
-> Limit (actual rows=5 loops=1)
-> Custom Scan (ChunkAppend) on metrics_space (actual rows=5 loops=1)
Order: metrics_space."time"
-> Sort (actual rows=5 loops=1)
Sort Key: _hyper_2_4_chunk."time"
Sort Method: top-N heapsort
-> Custom Scan (DecompressChunk) on _hyper_2_4_chunk (actual rows=720 loops=1)
-> Seq Scan on compress_hyper_6_17_chunk (actual rows=1 loops=1)
Filter: (device_id = 1)
-> Index Scan Backward using _hyper_2_7_chunk_metrics_space_time_idx on _hyper_2_7_chunk (never executed)
Filter: (device_id = 1)
-> Sort (never executed)
Sort Key: _hyper_2_10_chunk."time"
-> Custom Scan (DecompressChunk) on _hyper_2_10_chunk (never executed)
-> Seq Scan on compress_hyper_6_20_chunk (never executed)
Filter: (device_id = 1)
-> Materialize (actual rows=5 loops=5)
-> Limit (actual rows=5 loops=1)
-> Custom Scan (ChunkAppend) on metrics_space metrics_space_1 (actual rows=5 loops=1)
Order: metrics_space_1."time"
-> Sort (actual rows=5 loops=1)
Sort Key: _hyper_2_5_chunk."time"
Sort Method: top-N heapsort
-> Custom Scan (DecompressChunk) on _hyper_2_5_chunk (actual rows=720 loops=1)
-> Seq Scan on compress_hyper_6_18_chunk (actual rows=1 loops=1)
Filter: (device_id = 2)
Rows Removed by Filter: 2
-> Index Scan Backward using _hyper_2_8_chunk_metrics_space_device_id_time_idx on _hyper_2_8_chunk (never executed)
Index Cond: (device_id = 2)
-> Sort (never executed)
Sort Key: _hyper_2_11_chunk."time"
-> Custom Scan (DecompressChunk) on _hyper_2_11_chunk (never executed)
-> Seq Scan on compress_hyper_6_21_chunk (never executed)
Filter: (device_id = 2)
(35 rows)
-- test prepared statement
PREPARE prep AS
SELECT count(time)

115
tsl/test/expected/transparent_decompression-15.out
View File

@ -1799,6 +1799,64 @@ ORDER BY q1.time;
Rows Removed by Filter: 4
(41 rows)
:PREFIX WITH q1 AS (
SELECT time,
v1
FROM :TEST_TABLE
WHERE device_id = 1
ORDER BY time
LIMIT 5
),
q2 AS (
SELECT time,
v2
FROM :TEST_TABLE
WHERE device_id = 2
ORDER BY time
LIMIT 5
)
SELECT *
FROM q1, q2;
QUERY PLAN
---------------------------------------------------------------------------------------------------------------------------------------------
Nested Loop (actual rows=25 loops=1)
-> Limit (actual rows=5 loops=1)
-> Custom Scan (ChunkAppend) on metrics (actual rows=5 loops=1)
Order: metrics."time"
-> Sort (actual rows=5 loops=1)
Sort Key: _hyper_1_1_chunk."time"
Sort Method: top-N heapsort
-> Custom Scan (DecompressChunk) on _hyper_1_1_chunk (actual rows=720 loops=1)
-> Seq Scan on compress_hyper_5_15_chunk (actual rows=1 loops=1)
Filter: (device_id = 1)
Rows Removed by Filter: 4
-> Index Scan Backward using _hyper_1_2_chunk_metrics_time_idx on _hyper_1_2_chunk (never executed)
Filter: (device_id = 1)
-> Sort (never executed)
Sort Key: _hyper_1_3_chunk."time"
-> Custom Scan (DecompressChunk) on _hyper_1_3_chunk (never executed)
-> Seq Scan on compress_hyper_5_16_chunk (never executed)
Filter: (device_id = 1)
-> Materialize (actual rows=5 loops=5)
-> Limit (actual rows=5 loops=1)
-> Custom Scan (ChunkAppend) on metrics metrics_1 (actual rows=5 loops=1)
Order: metrics_1."time"
-> Sort (actual rows=5 loops=1)
Sort Key: _hyper_1_1_chunk_1."time"
Sort Method: top-N heapsort
-> Custom Scan (DecompressChunk) on _hyper_1_1_chunk _hyper_1_1_chunk_1 (actual rows=720 loops=1)
-> Seq Scan on compress_hyper_5_15_chunk compress_hyper_5_15_chunk_1 (actual rows=1 loops=1)
Filter: (device_id = 2)
Rows Removed by Filter: 4
-> Index Scan Backward using _hyper_1_2_chunk_metrics_time_idx on _hyper_1_2_chunk _hyper_1_2_chunk_1 (never executed)
Filter: (device_id = 2)
-> Sort (never executed)
Sort Key: _hyper_1_3_chunk_1."time"
-> Custom Scan (DecompressChunk) on _hyper_1_3_chunk _hyper_1_3_chunk_1 (never executed)
-> Seq Scan on compress_hyper_5_16_chunk compress_hyper_5_16_chunk_1 (never executed)
Filter: (device_id = 2)
(36 rows)
-- test prepared statement
PREPARE prep AS
SELECT count(time)
@ -5818,6 +5876,63 @@ ORDER BY q1.time;
Rows Removed by Filter: 2
(37 rows)
:PREFIX WITH q1 AS (
SELECT time,
v1
FROM :TEST_TABLE
WHERE device_id = 1
ORDER BY time
LIMIT 5
),
q2 AS (
SELECT time,
v2
FROM :TEST_TABLE
WHERE device_id = 2
ORDER BY time
LIMIT 5
)
SELECT *
FROM q1, q2;
QUERY PLAN
------------------------------------------------------------------------------------------------------------------------------------------
Nested Loop (actual rows=25 loops=1)
-> Limit (actual rows=5 loops=1)
-> Custom Scan (ChunkAppend) on metrics_space (actual rows=5 loops=1)
Order: metrics_space."time"
-> Sort (actual rows=5 loops=1)
Sort Key: _hyper_2_4_chunk."time"
Sort Method: top-N heapsort
-> Custom Scan (DecompressChunk) on _hyper_2_4_chunk (actual rows=720 loops=1)
-> Seq Scan on compress_hyper_6_17_chunk (actual rows=1 loops=1)
Filter: (device_id = 1)
-> Index Scan Backward using _hyper_2_7_chunk_metrics_space_time_idx on _hyper_2_7_chunk (never executed)
Filter: (device_id = 1)
-> Sort (never executed)
Sort Key: _hyper_2_10_chunk."time"
-> Custom Scan (DecompressChunk) on _hyper_2_10_chunk (never executed)
-> Seq Scan on compress_hyper_6_20_chunk (never executed)
Filter: (device_id = 1)
-> Materialize (actual rows=5 loops=5)
-> Limit (actual rows=5 loops=1)
-> Custom Scan (ChunkAppend) on metrics_space metrics_space_1 (actual rows=5 loops=1)
Order: metrics_space_1."time"
-> Sort (actual rows=5 loops=1)
Sort Key: _hyper_2_5_chunk."time"
Sort Method: top-N heapsort
-> Custom Scan (DecompressChunk) on _hyper_2_5_chunk (actual rows=720 loops=1)
-> Seq Scan on compress_hyper_6_18_chunk (actual rows=1 loops=1)
Filter: (device_id = 2)
Rows Removed by Filter: 2
-> Index Scan Backward using _hyper_2_8_chunk_metrics_space_device_id_time_idx on _hyper_2_8_chunk (never executed)
Index Cond: (device_id = 2)
-> Sort (never executed)
Sort Key: _hyper_2_11_chunk."time"
-> Custom Scan (DecompressChunk) on _hyper_2_11_chunk (never executed)
-> Seq Scan on compress_hyper_6_21_chunk (never executed)
Filter: (device_id = 2)
(35 rows)
-- test prepared statement
PREPARE prep AS
SELECT count(time)

115
tsl/test/expected/transparent_decompression-16.out
View File

@ -1799,6 +1799,64 @@ ORDER BY q1.time;
Rows Removed by Filter: 4
(41 rows)
:PREFIX WITH q1 AS (
SELECT time,
v1
FROM :TEST_TABLE
WHERE device_id = 1
ORDER BY time
LIMIT 5
),
q2 AS (
SELECT time,
v2
FROM :TEST_TABLE
WHERE device_id = 2
ORDER BY time
LIMIT 5
)
SELECT *
FROM q1, q2;
QUERY PLAN
---------------------------------------------------------------------------------------------------------------------------------------------
Nested Loop (actual rows=25 loops=1)
-> Limit (actual rows=5 loops=1)
-> Custom Scan (ChunkAppend) on metrics (actual rows=5 loops=1)
Order: metrics."time"
-> Sort (actual rows=5 loops=1)
Sort Key: _hyper_1_1_chunk."time"
Sort Method: top-N heapsort
-> Custom Scan (DecompressChunk) on _hyper_1_1_chunk (actual rows=720 loops=1)
-> Seq Scan on compress_hyper_5_15_chunk (actual rows=1 loops=1)
Filter: (device_id = 1)
Rows Removed by Filter: 4
-> Index Scan Backward using _hyper_1_2_chunk_metrics_time_idx on _hyper_1_2_chunk (never executed)
Filter: (device_id = 1)
-> Sort (never executed)
Sort Key: _hyper_1_3_chunk."time"
-> Custom Scan (DecompressChunk) on _hyper_1_3_chunk (never executed)
-> Seq Scan on compress_hyper_5_16_chunk (never executed)
Filter: (device_id = 1)
-> Materialize (actual rows=5 loops=5)
-> Limit (actual rows=5 loops=1)
-> Custom Scan (ChunkAppend) on metrics metrics_1 (actual rows=5 loops=1)
Order: metrics_1."time"
-> Sort (actual rows=5 loops=1)
Sort Key: _hyper_1_1_chunk_1."time"
Sort Method: top-N heapsort
-> Custom Scan (DecompressChunk) on _hyper_1_1_chunk _hyper_1_1_chunk_1 (actual rows=720 loops=1)
-> Seq Scan on compress_hyper_5_15_chunk compress_hyper_5_15_chunk_1 (actual rows=1 loops=1)
Filter: (device_id = 2)
Rows Removed by Filter: 4
-> Index Scan Backward using _hyper_1_2_chunk_metrics_time_idx on _hyper_1_2_chunk _hyper_1_2_chunk_1 (never executed)
Filter: (device_id = 2)
-> Sort (never executed)
Sort Key: _hyper_1_3_chunk_1."time"
-> Custom Scan (DecompressChunk) on _hyper_1_3_chunk _hyper_1_3_chunk_1 (never executed)
-> Seq Scan on compress_hyper_5_16_chunk compress_hyper_5_16_chunk_1 (never executed)
Filter: (device_id = 2)
(36 rows)
-- test prepared statement
PREPARE prep AS
SELECT count(time)
@ -5818,6 +5876,63 @@ ORDER BY q1.time;
Rows Removed by Filter: 2
(37 rows)
:PREFIX WITH q1 AS (
SELECT time,
v1
FROM :TEST_TABLE
WHERE device_id = 1
ORDER BY time
LIMIT 5
),
q2 AS (
SELECT time,
v2
FROM :TEST_TABLE
WHERE device_id = 2
ORDER BY time
LIMIT 5
)
SELECT *
FROM q1, q2;
QUERY PLAN
------------------------------------------------------------------------------------------------------------------------------------------
Nested Loop (actual rows=25 loops=1)
-> Limit (actual rows=5 loops=1)
-> Custom Scan (ChunkAppend) on metrics_space (actual rows=5 loops=1)
Order: metrics_space."time"
-> Sort (actual rows=5 loops=1)
Sort Key: _hyper_2_4_chunk."time"
Sort Method: top-N heapsort
-> Custom Scan (DecompressChunk) on _hyper_2_4_chunk (actual rows=720 loops=1)
-> Seq Scan on compress_hyper_6_17_chunk (actual rows=1 loops=1)
Filter: (device_id = 1)
-> Index Scan Backward using _hyper_2_7_chunk_metrics_space_time_idx on _hyper_2_7_chunk (never executed)
Filter: (device_id = 1)
-> Sort (never executed)
Sort Key: _hyper_2_10_chunk."time"
-> Custom Scan (DecompressChunk) on _hyper_2_10_chunk (never executed)
-> Seq Scan on compress_hyper_6_20_chunk (never executed)
Filter: (device_id = 1)
-> Materialize (actual rows=5 loops=5)
-> Limit (actual rows=5 loops=1)
-> Custom Scan (ChunkAppend) on metrics_space metrics_space_1 (actual rows=5 loops=1)
Order: metrics_space_1."time"
-> Sort (actual rows=5 loops=1)
Sort Key: _hyper_2_5_chunk."time"
Sort Method: top-N heapsort
-> Custom Scan (DecompressChunk) on _hyper_2_5_chunk (actual rows=720 loops=1)
-> Seq Scan on compress_hyper_6_18_chunk (actual rows=1 loops=1)
Filter: (device_id = 2)
Rows Removed by Filter: 2
-> Index Scan Backward using _hyper_2_8_chunk_metrics_space_device_id_time_idx on _hyper_2_8_chunk (never executed)
Index Cond: (device_id = 2)
-> Sort (never executed)
Sort Key: _hyper_2_11_chunk."time"
-> Custom Scan (DecompressChunk) on _hyper_2_11_chunk (never executed)
-> Seq Scan on compress_hyper_6_21_chunk (never executed)
Filter: (device_id = 2)
(35 rows)
-- test prepared statement
PREPARE prep AS
SELECT count(time)

19
tsl/test/sql/include/transparent_decompression_query.sql
View File

@ -506,6 +506,25 @@ FROM q1
INNER JOIN q2 ON q1.time = q2.time
ORDER BY q1.time;
:PREFIX WITH q1 AS (
SELECT time,
v1
FROM :TEST_TABLE
WHERE device_id = 1
ORDER BY time
LIMIT 5
),
q2 AS (
SELECT time,
v2
FROM :TEST_TABLE
WHERE device_id = 2
ORDER BY time
LIMIT 5
)
SELECT *
FROM q1, q2;
-- test prepared statement
PREPARE prep AS
SELECT count(time)