timescaledb/sql/updates/post-update.sql
Fabrízio de Royes Mello 38fcd1b76b Improve Realtime Continuous Aggregate performance
When calling the `cagg_watermark` function to get the watermark of a
Continuous Aggregate we execute a `SELECT MAX(time_dimension)` query
in the underlying materialization hypertable.

The problem is that a `SELECT MAX(time_dimention)` query can be
expensive because it will scan all hypertable chunks increasing the
planning time for a Realtime Continuous Aggregates.

Improved it by creating a new catalog table to serve as a cache table
to store the current Continous Aggregate watermark in the following
situations:
- Create CAgg: store the minimum value of hypertable time dimension
  data type;
- Refresh CAgg: store the last value of the time dimension materialized
  in the underlying materialization hypertable (or the minimum value of
  materialization hypertable time dimension data type if there's no
  data materialized);
- Drop CAgg Chunks: the same as refresh cagg.

Closes #4699, #5307
2023-03-22 16:35:23 -03:00

169 lines
7.1 KiB
SQL

-- needed post 1.7.0 to fixup continuous aggregates created in 1.7.0 ---
DO $$
DECLARE
vname regclass;
materialized_only bool;
ts_version TEXT;
BEGIN
SELECT extversion INTO ts_version FROM pg_extension WHERE extname = 'timescaledb';
IF ts_version >= '2.7.0' THEN
CREATE PROCEDURE _timescaledb_internal.post_update_cagg_try_repair(
cagg_view REGCLASS
) AS '@MODULE_PATHNAME@', 'ts_cagg_try_repair' LANGUAGE C;
END IF;
FOR vname, materialized_only IN select format('%I.%I', cagg.user_view_schema, cagg.user_view_name)::regclass, cagg.materialized_only from _timescaledb_catalog.continuous_agg cagg
LOOP
-- the cast from oid to text returns
-- quote_qualified_identifier (see regclassout).
--
-- We use the if statement to handle pre-2.0 as well as
-- post-2.0. This could be turned into a procedure if we want
-- to have something more generic, but right now it is just
-- this case.
IF ts_version < '2.0.0' THEN
EXECUTE format('ALTER VIEW %s SET (timescaledb.materialized_only=%L) ', vname::text, materialized_only);
ELSIF ts_version < '2.7.0' THEN
EXECUTE format('ALTER MATERIALIZED VIEW %s SET (timescaledb.materialized_only=%L) ', vname::text, materialized_only);
ELSE
SET log_error_verbosity TO VERBOSE;
CALL _timescaledb_internal.post_update_cagg_try_repair(vname);
END IF;
END LOOP;
IF ts_version >= '2.7.0' THEN
DROP PROCEDURE IF EXISTS _timescaledb_internal.post_update_cagg_try_repair;
END IF;
EXCEPTION WHEN OTHERS THEN RAISE;
END
$$;
-- can only be dropped after views have been rebuilt
DROP FUNCTION IF EXISTS _timescaledb_internal.cagg_watermark(oid);
-- For objects that are newly created, we need to set the initprivs to
-- the initprivs for some table that was created in the installation
-- of the TimescaleDB extension and not as part of any update.
--
-- We chose the "chunk" catalog table for this since that is created
-- in the first version of TimescaleDB and should have the correct
-- initprivs, but we could use any other table that existed in the
-- first installation.
INSERT INTO _timescaledb_internal.saved_privs
SELECT nspname, relname, relacl,
(SELECT tmpini FROM _timescaledb_internal.saved_privs
WHERE tmpnsp = '_timescaledb_catalog' AND tmpname = 'chunk')
FROM pg_class JOIN pg_namespace ns ON ns.oid = relnamespace
LEFT JOIN _timescaledb_internal.saved_privs ON tmpnsp = nspname AND tmpname = relname
WHERE relkind IN ('r', 'v') AND nspname IN ('_timescaledb_catalog', '_timescaledb_config')
OR nspname = '_timescaledb_internal'
AND relname IN ('hypertable_chunk_local_size', 'compressed_chunk_stats',
'bgw_job_stat', 'bgw_policy_chunk_stats', 'job_errors')
ON CONFLICT DO NOTHING;
-- The above is good enough for tables and views. However sequences need to
-- use the "chunk_id_seq" catalog sequence as a template
INSERT INTO _timescaledb_internal.saved_privs
SELECT nspname, relname, relacl,
(SELECT tmpini FROM _timescaledb_internal.saved_privs
WHERE tmpnsp = '_timescaledb_catalog' AND tmpname = 'chunk_id_seq')
FROM pg_class JOIN pg_namespace ns ON ns.oid = relnamespace
LEFT JOIN _timescaledb_internal.saved_privs ON tmpnsp = nspname AND tmpname = relname
WHERE relkind IN ('S') AND nspname IN ('_timescaledb_catalog', '_timescaledb_config')
OR nspname = '_timescaledb_internal'
AND relname IN ('hypertable_chunk_local_size', 'compressed_chunk_stats',
'bgw_job_stat', 'bgw_policy_chunk_stats')
ON CONFLICT DO NOTHING;
-- We can now copy back saved initprivs.
WITH to_update AS (
SELECT objoid, tmpini
FROM pg_class cl JOIN pg_namespace ns ON ns.oid = relnamespace
JOIN pg_init_privs ip ON ip.objoid = cl.oid AND ip.objsubid = 0
JOIN _timescaledb_internal.saved_privs ON tmpnsp = nspname AND tmpname = relname)
UPDATE pg_init_privs
SET initprivs = tmpini
FROM to_update
WHERE to_update.objoid = pg_init_privs.objoid
AND classoid = 'pg_class'::regclass
AND objsubid = 0;
-- Can only restore permissions on views after they have been rebuilt,
-- so we restore for all types of objects here.
WITH to_update AS (
SELECT cl.oid, tmpacl
FROM pg_class cl JOIN pg_namespace ns ON ns.oid = relnamespace
JOIN _timescaledb_internal.saved_privs ON tmpnsp = nspname AND tmpname = relname)
UPDATE pg_class cl SET relacl = tmpacl
FROM to_update WHERE cl.oid = to_update.oid;
DROP TABLE _timescaledb_internal.saved_privs;
-- warn about partial storage format change for numeric
DO $$
DECLARE
cagg_name text;
cagg_column text;
cnt int := 0;
BEGIN
IF current_setting('server_version_num')::int < 140000 THEN
FOR cagg_name, cagg_column IN
SELECT
attrelid::regclass::text,
att.attname
FROM _timescaledb_catalog.continuous_agg cagg
INNER JOIN pg_attribute att ON (
att.attrelid = format('%I.%I',cagg.user_view_schema,cagg.user_view_name)::regclass AND
atttypid = 'numeric'::regtype)
WHERE cagg.finalized = false
LOOP
RAISE WARNING 'Continuous Aggregate: % column: %', cagg_name, cagg_column;
cnt := cnt + 1;
END LOOP;
IF cnt > 0 THEN
RAISE WARNING 'The aggregation state format for numeric changed between PG13 and PG14. You should upgrade the above mentioned caggs to the new format before upgrading to PG14';
END IF;
END IF;
END $$;
-- Create dimension partition information for existing space-partitioned hypertables
CREATE FUNCTION _timescaledb_internal.update_dimension_partition(hypertable REGCLASS) RETURNS VOID AS '@MODULE_PATHNAME@', 'ts_dimension_partition_update' LANGUAGE C VOLATILE;
SELECT _timescaledb_internal.update_dimension_partition(format('%I.%I', h.schema_name, h.table_name))
FROM _timescaledb_catalog.hypertable h
INNER JOIN _timescaledb_catalog.dimension d ON (d.hypertable_id = h.id)
WHERE d.interval_length IS NULL;
DROP FUNCTION _timescaledb_internal.update_dimension_partition;
-- Report warning when partial aggregates are used
DO $$
DECLARE
cagg_name text;
BEGIN
FOR cagg_name IN
SELECT
format('%I.%I', user_view_schema, user_view_name)
FROM _timescaledb_catalog.continuous_agg
WHERE finalized IS FALSE
ORDER BY 1
LOOP
RAISE WARNING 'Continuous Aggregate: % with old format will not be supported with PG15. You should upgrade to the new format', cagg_name;
END LOOP;
END $$;
-- Create watermark record when required
DO
$$
DECLARE
ts_version TEXT;
BEGIN
SELECT extversion INTO ts_version FROM pg_extension WHERE extname = 'timescaledb';
IF ts_version >= '2.11.0' THEN
INSERT INTO _timescaledb_catalog.continuous_aggs_watermark (mat_hypertable_id, watermark)
SELECT a.mat_hypertable_id, _timescaledb_internal.cagg_watermark_materialized(a.mat_hypertable_id)
FROM _timescaledb_catalog.continuous_agg a
LEFT JOIN _timescaledb_catalog.continuous_aggs_watermark b ON b.mat_hypertable_id = a.mat_hypertable_id
WHERE b.mat_hypertable_id IS NULL
ORDER BY 1;
END IF;
END;
$$;