diff --git a/sql/functions_load_order.txt b/sql/functions_load_order.txt index 4c5c5c804..da0f34cb1 100644 --- a/sql/functions_load_order.txt +++ b/sql/functions_load_order.txt @@ -19,3 +19,4 @@ sql/bookend.sql sql/time_bucket.sql sql/version.sql sql/cache_functions.sql +sql/size_utils.sql diff --git a/sql/size_utils.sql b/sql/size_utils.sql new file mode 100644 index 000000000..b5a9e0ccd --- /dev/null +++ b/sql/size_utils.sql @@ -0,0 +1,229 @@ + +-- This file contains utility functions to get the relation size +-- of hypertables, chunks, and indexes on hypertables. + +-- Get relation size of hypertable +-- like pg_relation_size(hypertable) +-- (https://www.postgresql.org/docs/9.6/static/functions-admin.html#FUNCTIONS-ADMIN-DBSIZE) +-- +-- main_table - hypertable to get size of +-- +-- Returns: +-- table_bytes - Disk space used by main_table (like pg_relation_size(main_table)) +-- index_bytes - Disc space used by indexes +-- toast_bytes - Disc space of toast tables +-- total_bytes - Total disk space used by the specified table, including all indexes and TOAST data +-- table_size - Pretty output of table_bytes +-- index_bytes - Pretty output of index_bytes +-- toast_bytes - Pretty output of toast_bytes +-- total_size - Pretty output of total_bytes + +CREATE OR REPLACE FUNCTION hypertable_relation_size( + main_table REGCLASS +) +RETURNS TABLE (table_bytes BIGINT, + index_bytes BIGINT, + toast_bytes BIGINT, + total_bytes BIGINT, + table_size TEXT, + index_size TEXT, + toast_size TEXT, + total_size TEXT) LANGUAGE PLPGSQL VOLATILE + SECURITY DEFINER SET search_path = '' + AS +$BODY$ +DECLARE + table_name NAME; + schema_name NAME; +BEGIN + SELECT relname, nspname + INTO STRICT table_name, schema_name + FROM pg_class c + INNER JOIN pg_namespace n ON (n.OID = c.relnamespace) + WHERE c.OID = main_table; + + RETURN QUERY EXECUTE format( + $$ + SELECT table_bytes, + index_bytes, + toast_bytes, + total_bytes, + pg_size_pretty(table_bytes) as table, + pg_size_pretty(index_bytes) as index, + pg_size_pretty(toast_bytes) as toast, + pg_size_pretty(total_bytes) as total + FROM ( + SELECT *, total_bytes-index_bytes-COALESCE(toast_bytes,0) AS table_bytes FROM ( + SELECT + sum(pg_total_relation_size('"' || c.schema_name || '"."' || c.table_name || '"'))::bigint as total_bytes, + sum(pg_indexes_size('"' || c.schema_name || '"."' || c.table_name || '"'))::bigint AS index_bytes, + sum(pg_total_relation_size(reltoastrelid))::bigint AS toast_bytes + FROM + _timescaledb_catalog.hypertable h, + _timescaledb_catalog.chunk c, + pg_class pgc, + pg_namespace pns + WHERE h.schema_name = %L + AND h.table_name = %L + AND c.hypertable_id = h.id + AND pgc.relname = h.table_name + AND pns.oid = pgc.relnamespace + AND pns.nspname = h.schema_name + AND relkind = 'r' + ) sub1 + ) sub2; + $$, + schema_name, table_name); + +END; +$BODY$; + +-- Get relation size of the chunks of an hypertable +-- like pg_relation_size +-- (https://www.postgresql.org/docs/9.6/static/functions-admin.html#FUNCTIONS-ADMIN-DBSIZE) +-- +-- main_table - hypertable to get size of +-- +-- Returns: +-- chunk_id - timescaledb id of a chunk +-- chunk_table - table used for the chunk +-- table_bytes - Disk space used by main_table +-- index_bytes - Disk space used by indexes +-- toast_bytes - Disc space of toast tables +-- total_bytes - Disk space used in total +-- table_size - Pretty output of table_bytes +-- index_size - Pretty output of index_bytes +-- toast_size - Pretty output of toast_bytes +-- total_size - Pretty output of total_bytes + +CREATE OR REPLACE FUNCTION chunk_relation_size( + main_table REGCLASS +) +RETURNS TABLE (chunk_id INT, + chunk_table TEXT, + dimensions NAME[], + ranges int8range[], + table_bytes BIGINT, + index_bytes BIGINT, + toast_bytes BIGINT, + total_bytes BIGINT, + table_size TEXT, + index_size TEXT, + toast_size TEXT, + total_size TEXT) + LANGUAGE PLPGSQL VOLATILE + SECURITY DEFINER SET search_path = '' + AS +$BODY$ +DECLARE + table_name NAME; + schema_name NAME; +BEGIN + SELECT relname, nspname + INTO STRICT table_name, schema_name + FROM pg_class c + INNER JOIN pg_namespace n ON (n.OID = c.relnamespace) + WHERE c.OID = main_table; + + RETURN QUERY EXECUTE format( + $$ + + SELECT chunk_id, + chunk_table, + dimensions, + ranges, + table_bytes, + index_bytes, + toast_bytes, + total_bytes, + pg_size_pretty(table_bytes) AS table, + pg_size_pretty(index_bytes) AS index, + pg_size_pretty(toast_bytes) AS toast, + pg_size_pretty(total_bytes) AS total + FROM ( + SELECT *, + total_bytes-index_bytes-COALESCE(toast_bytes,0) AS table_bytes + FROM ( + SELECT c.id as chunk_id, + '"' || c.schema_name || '"."' || c.table_name || '"' as chunk_table, + pg_total_relation_size('"' || c.schema_name || '"."' || c.table_name || '"') AS total_bytes, + pg_indexes_size('"' || c.schema_name || '"."' || c.table_name || '"') AS index_bytes, + pg_total_relation_size(reltoastrelid) AS toast_bytes, + array_agg(d.column_name) as dimensions, + array_agg(int8range(range_start, range_end)) as ranges + FROM + _timescaledb_catalog.hypertable h, + _timescaledb_catalog.chunk c, + _timescaledb_catalog.chunk_constraint cc, + _timescaledb_catalog.dimension d, + _timescaledb_catalog.dimension_slice ds, + pg_class pgc, + pg_namespace pns + WHERE h.schema_name = %L + AND h.table_name = %L + AND pgc.relname = h.table_name + AND pns.oid = pgc.relnamespace + AND pns.nspname = h.schema_name + AND relkind = 'r' + AND c.hypertable_id = h.id + AND c.id = cc.chunk_id + AND cc.dimension_slice_id = ds.id + AND ds.dimension_id = d.id + + GROUP BY c.id, pgc.reltoastrelid, pgc.oid ORDER BY c.id + ) sub1 + ) sub2; + $$, + schema_name, table_name); + +END; +$BODY$; + +-- Get sizes of indexes on a hypertable +-- +-- main_table - hypertable to get index sizes of +-- +-- Returns: +-- index_name - index on hyper table +-- total_bytes - size of index on disk +-- total_size - pretty output of total_bytes + +CREATE OR REPLACE FUNCTION indexes_relation_size( + main_table REGCLASS +) +RETURNS TABLE (index_name TEXT, + total_bytes BIGINT, + total_size TEXT) LANGUAGE PLPGSQL VOLATILE + SECURITY DEFINER SET search_path = '' + AS +$BODY$ +DECLARE + table_name NAME; + schema_name NAME; +BEGIN + SELECT relname, nspname + INTO STRICT table_name, schema_name + FROM pg_class c + INNER JOIN pg_namespace n ON (n.OID = c.relnamespace) + WHERE c.OID = main_table; + + RETURN QUERY EXECUTE format( + $$ + SELECT hi.main_schema_name || '.' || hi.main_index_name, + sum(pg_relation_size('"' || ci.schema_name || '"."' || ci.index_name || '"'))::bigint, + pg_size_pretty(sum(pg_relation_size('"' || ci.schema_name || '"."' || ci.index_name || '"'))) + FROM + _timescaledb_catalog.hypertable h, + _timescaledb_catalog.hypertable_index hi, + _timescaledb_catalog.chunk_index ci + WHERE h.id = hi.hypertable_id + AND h.schema_name = %L + AND h.table_name = %L + AND ci.main_index_name = hi.main_index_name + AND ci.main_schema_name = hi.main_schema_name + GROUP BY hi.main_schema_name || '.' || hi.main_index_name; + $$, + schema_name, table_name); + +END; +$BODY$; diff --git a/test/expected/extension.out b/test/expected/extension.out index 39a96089a..1aba12fd9 100644 --- a/test/expected/extension.out +++ b/test/expected/extension.out @@ -16,16 +16,19 @@ WHERE OID IN ( deptype = 'e' and classid = 'pg_catalog.pg_proc'::regclass ) AND pronamespace = 'public'::regnamespace ORDER BY proname; - proname -------------------------- + proname +-------------------------- add_dimension attach_tablespace + chunk_relation_size create_hypertable drop_chunks first + hypertable_relation_size + indexes_relation_size last restore_timescaledb set_chunk_time_interval time_bucket -(9 rows) +(12 rows) diff --git a/test/expected/pg_dump.out b/test/expected/pg_dump.out index e3941b88a..ee045531a 100644 --- a/test/expected/pg_dump.out +++ b/test/expected/pg_dump.out @@ -42,7 +42,7 @@ SELECT count(*) AND refobjid = (SELECT oid FROM pg_extension WHERE extname = 'timescaledb'); count ------- - 112 + 115 (1 row) \c postgres @@ -66,7 +66,7 @@ SELECT count(*) AND refobjid = (SELECT oid FROM pg_extension WHERE extname = 'timescaledb'); count ------- - 112 + 115 (1 row) \c single diff --git a/test/expected/size_utils.out b/test/expected/size_utils.out new file mode 100644 index 000000000..408898ff4 --- /dev/null +++ b/test/expected/size_utils.out @@ -0,0 +1,73 @@ +\ir include/insert_two_partitions.sql +\ir create_single_db.sql +SET client_min_messages = WARNING; +DROP DATABASE IF EXISTS single; +SET client_min_messages = NOTICE; +CREATE DATABASE single; +\c single +CREATE EXTENSION IF NOT EXISTS timescaledb; +\c single +CREATE TABLE PUBLIC."two_Partitions" ( + "timeCustom" BIGINT NOT NULL, + device_id TEXT NOT NULL, + series_0 DOUBLE PRECISION NULL, + series_1 DOUBLE PRECISION NULL, + series_2 DOUBLE PRECISION NULL, + series_bool BOOLEAN NULL +); +CREATE INDEX ON PUBLIC."two_Partitions" (device_id, "timeCustom" DESC NULLS LAST) WHERE device_id IS NOT NULL; +CREATE INDEX ON PUBLIC."two_Partitions" ("timeCustom" DESC NULLS LAST, series_0) WHERE series_0 IS NOT NULL; +CREATE INDEX ON PUBLIC."two_Partitions" ("timeCustom" DESC NULLS LAST, series_1) WHERE series_1 IS NOT NULL; +CREATE INDEX ON PUBLIC."two_Partitions" ("timeCustom" DESC NULLS LAST, series_2) WHERE series_2 IS NOT NULL; +CREATE INDEX ON PUBLIC."two_Partitions" ("timeCustom" DESC NULLS LAST, series_bool) WHERE series_bool IS NOT NULL; +CREATE INDEX ON PUBLIC."two_Partitions" ("timeCustom" DESC NULLS LAST, device_id); +SELECT * FROM create_hypertable('"public"."two_Partitions"'::regclass, 'timeCustom'::name, 'device_id'::name, associated_schema_name=>'_timescaledb_internal'::text, number_partitions => 2); + create_hypertable +------------------- + +(1 row) + +\set QUIET off +BEGIN; +BEGIN +\COPY public."two_Partitions" FROM 'data/ds1_dev1_1.tsv' NULL AS ''; +COPY 7 +COMMIT; +COMMIT +INSERT INTO public."two_Partitions"("timeCustom", device_id, series_0, series_1) VALUES +(1257987600000000000, 'dev1', 1.5, 1), +(1257987600000000000, 'dev1', 1.5, 2), +(1257894000000000000, 'dev2', 1.5, 1), +(1257894002000000000, 'dev1', 2.5, 3); +INSERT 0 4 +INSERT INTO "two_Partitions"("timeCustom", device_id, series_0, series_1) VALUES +(1257894000000000000, 'dev2', 1.5, 2); +INSERT 0 1 +\set QUIET on +SELECT * FROM hypertable_relation_size('"public"."two_Partitions"'); + table_bytes | index_bytes | toast_bytes | total_bytes | table_size | index_size | toast_size | total_size +-------------+-------------+-------------+-------------+------------+------------+------------+------------ + 32768 | 417792 | 32768 | 483328 | 32 kB | 408 kB | 32 kB | 472 kB +(1 row) + +SELECT * FROM chunk_relation_size('"public"."two_Partitions"'); + chunk_id | chunk_table | dimensions | ranges | table_bytes | index_bytes | toast_bytes | total_bytes | table_size | index_size | toast_size | total_size +----------+--------------------------------------------+------------------------+-------------------------------------------------------------------------+-------------+-------------+-------------+-------------+------------+------------+------------+------------ + 1 | "_timescaledb_internal"."_hyper_1_1_chunk" | {timeCustom,device_id} | {"[1257892416000000000,1257895008000000000)","[1073741823,2147483647)"} | 8192 | 114688 | 8192 | 131072 | 8192 bytes | 112 kB | 8192 bytes | 128 kB + 2 | "_timescaledb_internal"."_hyper_1_2_chunk" | {device_id,timeCustom} | {"[1073741823,2147483647)","[1257897600000000000,1257900192000000000)"} | 8192 | 106496 | 8192 | 122880 | 8192 bytes | 104 kB | 8192 bytes | 120 kB + 3 | "_timescaledb_internal"."_hyper_1_3_chunk" | {device_id,timeCustom} | {"[1073741823,2147483647)","[1257985728000000000,1257988320000000000)"} | 8192 | 98304 | 8192 | 114688 | 8192 bytes | 96 kB | 8192 bytes | 112 kB + 4 | "_timescaledb_internal"."_hyper_1_4_chunk" | {timeCustom,device_id} | {"[1257892416000000000,1257895008000000000)","[0,1073741823)"} | 8192 | 98304 | 8192 | 114688 | 8192 bytes | 96 kB | 8192 bytes | 112 kB +(4 rows) + +SELECT * FROM indexes_relation_size('"public"."two_Partitions"'); + index_name | total_bytes | total_size +--------------------------------------------------+-------------+------------ + public.two_Partitions_device_id_timeCustom_idx | 65536 | 64 kB + public.two_Partitions_timeCustom_device_id_idx | 65536 | 64 kB + public.two_Partitions_timeCustom_idx | 65536 | 64 kB + public.two_Partitions_timeCustom_series_0_idx | 65536 | 64 kB + public.two_Partitions_timeCustom_series_1_idx | 65536 | 64 kB + public.two_Partitions_timeCustom_series_2_idx | 40960 | 40 kB + public.two_Partitions_timeCustom_series_bool_idx | 49152 | 48 kB +(7 rows) + diff --git a/test/sql/size_utils.sql b/test/sql/size_utils.sql new file mode 100644 index 000000000..6bcd8b2fa --- /dev/null +++ b/test/sql/size_utils.sql @@ -0,0 +1,6 @@ +\ir include/insert_two_partitions.sql + +SELECT * FROM hypertable_relation_size('"public"."two_Partitions"'); +SELECT * FROM chunk_relation_size('"public"."two_Partitions"'); +SELECT * FROM indexes_relation_size('"public"."two_Partitions"'); +