From a76f76f4ee6ee1d5437fe31cbd551e0d6f51393f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fabr=C3=ADzio=20de=20Royes=20Mello?= Date: Tue, 4 Oct 2022 14:20:23 -0300 Subject: [PATCH] Improve size utils functions and views performance Changed queries to use LATERAL join on size functions and views instead of CTEs and it eliminate a lot of unnecessary projections and give a chance for the planner to push-down predicates. Closes #4775 --- sql/size_utils.sql | 82 +++++++++------------------ sql/updates/latest-dev.sql | 109 ++++++++++++++++++++++++++++++++++-- sql/updates/reverse-dev.sql | 72 +++++++++++++++++++++++- 3 files changed, 202 insertions(+), 61 deletions(-) diff --git a/sql/size_utils.sql b/sql/size_utils.sql index 8aa2a6419..c5bac0730 100644 --- a/sql/size_utils.sql +++ b/sql/size_utils.sql @@ -10,47 +10,13 @@ RETURNS TABLE (total_size BIGINT, heap_size BIGINT, index_size BIGINT, toast_siz AS '@MODULE_PATHNAME@', 'ts_relation_size' LANGUAGE C VOLATILE; CREATE OR REPLACE VIEW _timescaledb_internal.hypertable_chunk_local_size AS -WITH chunks AS ( - SELECT - h.schema_name AS hypertable_schema, - h.table_name AS hypertable_name, - h.id AS hypertable_id, - c.id AS chunk_id, - c.schema_name AS chunk_schema, - c.table_name AS chunk_name, - format('%I.%I', c.schema_name, c.table_name)::regclass AS relid, - CASE WHEN comp.schema_name IS NOT NULL AND comp.table_name IS NOT NULL THEN - format('%I.%I', comp.schema_name, comp.table_name)::regclass - ELSE - NULL::regclass - END AS relidcomp, - c.compressed_chunk_id - FROM - _timescaledb_catalog.hypertable h - JOIN _timescaledb_catalog.chunk c ON h.id = c.hypertable_id - AND c.dropped IS FALSE - LEFT JOIN _timescaledb_catalog.chunk comp ON comp.id = c.compressed_chunk_id -), -sizes AS ( - SELECT - ch.hypertable_schema, - ch.hypertable_name, - ch.hypertable_id, - ch.chunk_id, - ch.chunk_schema, - ch.chunk_name, - _timescaledb_internal.relation_size(ch.relid) AS relsize, - _timescaledb_internal.relation_size(ch.relidcomp) AS relcompsize - FROM - chunks ch -) SELECT - hypertable_schema, - hypertable_name, - hypertable_id, - chunk_id, - chunk_schema, - chunk_name, + h.schema_name AS hypertable_schema, + h.table_name AS hypertable_name, + h.id AS hypertable_id, + c.id AS chunk_id, + c.schema_name AS chunk_schema, + c.table_name AS chunk_name, COALESCE((relsize).total_size, 0) AS total_bytes, COALESCE((relsize).heap_size, 0) AS heap_bytes, COALESCE((relsize).index_size, 0) AS index_bytes, @@ -60,7 +26,19 @@ SELECT COALESCE((relcompsize).index_size, 0) AS compressed_index_size, COALESCE((relcompsize).toast_size, 0) AS compressed_toast_size FROM - sizes; + _timescaledb_catalog.hypertable h + JOIN _timescaledb_catalog.chunk c ON h.id = c.hypertable_id + AND c.dropped IS FALSE + JOIN LATERAL _timescaledb_internal.relation_size( + format('%I.%I'::text, c.schema_name, c.table_name)::regclass) AS relsize ON TRUE + LEFT JOIN _timescaledb_catalog.chunk comp ON comp.id = c.compressed_chunk_id + LEFT JOIN LATERAL _timescaledb_internal.relation_size( + CASE WHEN comp.schema_name IS NOT NULL AND comp.table_name IS NOT NULL THEN + format('%I.%I', comp.schema_name, comp.table_name)::regclass + ELSE + NULL::regclass + END + ) AS relcompsize ON TRUE; GRANT SELECT ON _timescaledb_internal.hypertable_chunk_local_size TO PUBLIC; @@ -102,18 +80,7 @@ RETURNS TABLE ( LANGUAGE SQL VOLATILE STRICT AS $BODY$ /* get the main hypertable id and sizes */ - WITH _hypertable AS ( - SELECT - id, - _timescaledb_internal.relation_size(format('%I.%I', schema_name, table_name)::regclass) AS relsize - FROM - _timescaledb_catalog.hypertable - WHERE - schema_name = schema_name_in - AND table_name = table_name_in - ), - /* project the size of the parent hypertable */ - _hypertable_sizes AS ( + WITH _hypertable_sizes AS ( SELECT id, COALESCE((relsize).total_size, 0) AS total_bytes, @@ -125,7 +92,12 @@ $BODY$ 0::BIGINT AS compressed_toast_size, 0::BIGINT AS compressed_heap_size FROM - _hypertable + _timescaledb_catalog.hypertable + JOIN LATERAL _timescaledb_internal.relation_size( + format('%I.%I', schema_name, table_name)::regclass) AS relsize ON TRUE + WHERE + schema_name = schema_name_in + AND table_name = table_name_in ), /* calculate the size of the hypertable chunks */ _chunk_sizes AS ( @@ -142,6 +114,8 @@ $BODY$ FROM _timescaledb_internal.hypertable_chunk_local_size ch JOIN _hypertable_sizes ht ON ht.id = ch.hypertable_id + WHERE hypertable_schema = schema_name_in + AND hypertable_name = table_name_in ) /* calculate the SUM of the hypertable and chunk sizes */ SELECT diff --git a/sql/updates/latest-dev.sql b/sql/updates/latest-dev.sql index c90080094..84740fab7 100644 --- a/sql/updates/latest-dev.sql +++ b/sql/updates/latest-dev.sql @@ -7,7 +7,7 @@ ALTER TABLE _timescaledb_catalog.compression_chunk_size DROP CONSTRAINT compress ALTER TABLE _timescaledb_catalog.compression_chunk_size ADD CONSTRAINT compression_chunk_size_pkey PRIMARY KEY(chunk_id); CREATE TABLE _timescaledb_internal.job_errors ( - job_id integer not null, + job_id integer not null, pid integer, start_time timestamptz, finish_time timestamptz, @@ -16,7 +16,7 @@ CREATE TABLE _timescaledb_internal.job_errors ( SELECT pg_catalog.pg_extension_config_dump('_timescaledb_internal.job_errors', ''); -CREATE VIEW timescaledb_information.job_errors AS +CREATE VIEW timescaledb_information.job_errors AS SELECT job_id, error_data ->> 'proc_schema' as proc_schema, @@ -25,8 +25,8 @@ SELECT start_time, finish_time, error_data ->> 'sqlerrcode' AS sqlerrcode, - CASE WHEN error_data ->>'message' IS NOT NULL THEN - CASE WHEN error_data ->>'detail' IS NOT NULL THEN + CASE WHEN error_data ->>'message' IS NOT NULL THEN + CASE WHEN error_data ->>'detail' IS NOT NULL THEN CASE WHEN error_data ->>'hint' IS NOT NULL THEN concat(error_data ->>'message', '. ', error_data ->>'detail', '. ', error_data->>'hint') ELSE concat(error_data ->>'message', ' ', error_data ->>'detail') END @@ -45,7 +45,106 @@ FROM ALTER TABLE _timescaledb_internal.bgw_job_stat ADD COLUMN flags integer; UPDATE _timescaledb_internal.bgw_job_stat SET flags = 0; -ALTER TABLE _timescaledb_internal.bgw_job_stat +ALTER TABLE _timescaledb_internal.bgw_job_stat ALTER COLUMN flags SET NOT NULL, ALTER COLUMN flags SET DEFAULT 0; +DROP FUNCTION IF EXISTS _timescaledb_internal.hypertable_local_size(name, name); +DROP VIEW IF EXISTS _timescaledb_internal.hypertable_chunk_local_size; + +CREATE VIEW _timescaledb_internal.hypertable_chunk_local_size AS +SELECT + h.schema_name AS hypertable_schema, + h.table_name AS hypertable_name, + h.id AS hypertable_id, + c.id AS chunk_id, + c.schema_name AS chunk_schema, + c.table_name AS chunk_name, + COALESCE((relsize).total_size, 0) AS total_bytes, + COALESCE((relsize).heap_size, 0) AS heap_bytes, + COALESCE((relsize).index_size, 0) AS index_bytes, + COALESCE((relsize).toast_size, 0) AS toast_bytes, + COALESCE((relcompsize).total_size, 0) AS compressed_total_size, + COALESCE((relcompsize).heap_size, 0) AS compressed_heap_size, + COALESCE((relcompsize).index_size, 0) AS compressed_index_size, + COALESCE((relcompsize).toast_size, 0) AS compressed_toast_size +FROM + _timescaledb_catalog.hypertable h + JOIN _timescaledb_catalog.chunk c ON h.id = c.hypertable_id + AND c.dropped IS FALSE + JOIN LATERAL _timescaledb_internal.relation_size( + format('%I.%I'::text, c.schema_name, c.table_name)::regclass) AS relsize ON TRUE + LEFT JOIN _timescaledb_catalog.chunk comp ON comp.id = c.compressed_chunk_id + LEFT JOIN LATERAL _timescaledb_internal.relation_size( + CASE WHEN comp.schema_name IS NOT NULL AND comp.table_name IS NOT NULL THEN + format('%I.%I', comp.schema_name, comp.table_name)::regclass + ELSE + NULL::regclass + END + ) AS relcompsize ON TRUE; + +CREATE FUNCTION _timescaledb_internal.hypertable_local_size( + schema_name_in name, + table_name_in name) +RETURNS TABLE ( + table_bytes BIGINT, + index_bytes BIGINT, + toast_bytes BIGINT, + total_bytes BIGINT) +LANGUAGE SQL VOLATILE STRICT AS +$BODY$ + /* get the main hypertable id and sizes */ + WITH _hypertable AS ( + SELECT + id, + _timescaledb_internal.relation_size(format('%I.%I', schema_name, table_name)::regclass) AS relsize + FROM + _timescaledb_catalog.hypertable + WHERE + schema_name = schema_name_in + AND table_name = table_name_in + ), + /* project the size of the parent hypertable */ + _hypertable_sizes AS ( + SELECT + id, + COALESCE((relsize).total_size, 0) AS total_bytes, + COALESCE((relsize).heap_size, 0) AS heap_bytes, + COALESCE((relsize).index_size, 0) AS index_bytes, + COALESCE((relsize).toast_size, 0) AS toast_bytes, + 0::BIGINT AS compressed_total_size, + 0::BIGINT AS compressed_index_size, + 0::BIGINT AS compressed_toast_size, + 0::BIGINT AS compressed_heap_size + FROM + _hypertable + ), + /* calculate the size of the hypertable chunks */ + _chunk_sizes AS ( + SELECT + chunk_id, + COALESCE(ch.total_bytes, 0) AS total_bytes, + COALESCE(ch.heap_bytes, 0) AS heap_bytes, + COALESCE(ch.index_bytes, 0) AS index_bytes, + COALESCE(ch.toast_bytes, 0) AS toast_bytes, + COALESCE(ch.compressed_total_size, 0) AS compressed_total_size, + COALESCE(ch.compressed_index_size, 0) AS compressed_index_size, + COALESCE(ch.compressed_toast_size, 0) AS compressed_toast_size, + COALESCE(ch.compressed_heap_size, 0) AS compressed_heap_size + FROM + _timescaledb_internal.hypertable_chunk_local_size ch + JOIN _hypertable_sizes ht ON ht.id = ch.hypertable_id + WHERE hypertable_schema = schema_name_in + AND hypertable_name = table_name_in + ) + /* calculate the SUM of the hypertable and chunk sizes */ + SELECT + (SUM(heap_bytes) + SUM(compressed_heap_size))::BIGINT AS heap_bytes, + (SUM(index_bytes) + SUM(compressed_index_size))::BIGINT AS index_bytes, + (SUM(toast_bytes) + SUM(compressed_toast_size))::BIGINT AS toast_bytes, + (SUM(total_bytes) + SUM(compressed_total_size))::BIGINT AS total_bytes + FROM + (SELECT * FROM _hypertable_sizes + UNION ALL + SELECT * FROM _chunk_sizes) AS sizes; +$BODY$ SET search_path TO pg_catalog, pg_temp; diff --git a/sql/updates/reverse-dev.sql b/sql/updates/reverse-dev.sql index c7e152c0b..13f8a455a 100644 --- a/sql/updates/reverse-dev.sql +++ b/sql/updates/reverse-dev.sql @@ -188,7 +188,7 @@ DROP VIEW IF EXISTS timescaledb_information.jobs; ALTER TABLE _timescaledb_internal.bgw_job_stat DROP COLUMN flags; --- need to recreate the bgw_job_stats table because dropping the column +-- need to recreate the bgw_job_stats table because dropping the column -- will not remove it from the pg_attribute table CREATE TABLE _timescaledb_internal.bgw_job_stat_tmp ( @@ -223,7 +223,75 @@ ALTER TABLE _timescaledb_internal.bgw_job_stat_tmp RENAME TO bgw_job_stat; ALTER TABLE _timescaledb_internal.bgw_job_stat ADD CONSTRAINT bgw_job_stat_pkey PRIMARY KEY (job_id), - ADD CONSTRAINT bgw_job_stat_job_id_fkey FOREIGN KEY (job_id) + ADD CONSTRAINT bgw_job_stat_job_id_fkey FOREIGN KEY (job_id) REFERENCES _timescaledb_config.bgw_job (id) ON DELETE CASCADE; GRANT SELECT ON TABLE _timescaledb_internal.bgw_job_stat TO PUBLIC; + +DROP VIEW _timescaledb_internal.hypertable_chunk_local_size; +DROP FUNCTION _timescaledb_internal.hypertable_local_size(name, name); + +CREATE FUNCTION _timescaledb_internal.hypertable_local_size( + schema_name_in name, + table_name_in name) +RETURNS TABLE ( + table_bytes BIGINT, + index_bytes BIGINT, + toast_bytes BIGINT, + total_bytes BIGINT) +LANGUAGE SQL VOLATILE STRICT AS +$BODY$ + /* get the main hypertable id and sizes */ + WITH _hypertable AS ( + SELECT + id, + _timescaledb_internal.relation_size(format('%I.%I', schema_name, table_name)::regclass) AS relsize + FROM + _timescaledb_catalog.hypertable + WHERE + schema_name = schema_name_in + AND table_name = table_name_in + ), + /* project the size of the parent hypertable */ + _hypertable_sizes AS ( + SELECT + id, + COALESCE((relsize).total_size, 0) AS total_bytes, + COALESCE((relsize).heap_size, 0) AS heap_bytes, + COALESCE((relsize).index_size, 0) AS index_bytes, + COALESCE((relsize).toast_size, 0) AS toast_bytes, + 0::BIGINT AS compressed_total_size, + 0::BIGINT AS compressed_index_size, + 0::BIGINT AS compressed_toast_size, + 0::BIGINT AS compressed_heap_size + FROM + _hypertable + ), + /* calculate the size of the hypertable chunks */ + _chunk_sizes AS ( + SELECT + chunk_id, + COALESCE(ch.total_bytes, 0) AS total_bytes, + COALESCE(ch.heap_bytes, 0) AS heap_bytes, + COALESCE(ch.index_bytes, 0) AS index_bytes, + COALESCE(ch.toast_bytes, 0) AS toast_bytes, + COALESCE(ch.compressed_total_size, 0) AS compressed_total_size, + COALESCE(ch.compressed_index_size, 0) AS compressed_index_size, + COALESCE(ch.compressed_toast_size, 0) AS compressed_toast_size, + COALESCE(ch.compressed_heap_size, 0) AS compressed_heap_size + FROM + _timescaledb_internal.hypertable_chunk_local_size ch + JOIN _hypertable_sizes ht ON ht.id = ch.hypertable_id + ) + /* calculate the SUM of the hypertable and chunk sizes */ + SELECT + (SUM(heap_bytes) + SUM(compressed_heap_size))::BIGINT AS heap_bytes, + (SUM(index_bytes) + SUM(compressed_index_size))::BIGINT AS index_bytes, + (SUM(toast_bytes) + SUM(compressed_toast_size))::BIGINT AS toast_bytes, + (SUM(total_bytes) + SUM(compressed_total_size))::BIGINT AS total_bytes + FROM + (SELECT * FROM _hypertable_sizes + UNION ALL + SELECT * FROM _chunk_sizes) AS sizes; +$BODY$ SET search_path TO pg_catalog, pg_temp; +