mirror of
https://github.com/timescale/timescaledb.git
synced 2025-05-23 22:41:34 +08:00
906bd38573
When a job finishes execution, either because of an error or a success, this commit will print out the execution time of the job in the log together with a message about what job that finished. For continuous aggregate refreshes, the number of rows deleted from and inserted into the materialization table will be printed.
364 lines
14 KiB
SQL
364 lines
14 KiB
SQL
-- This file and its contents are licensed under the Timescale License.
|
|
-- Please see the included NOTICE for copyright information and
|
|
-- LICENSE-TIMESCALE for a copy of the license.
|
|
|
|
--
|
|
-- Setup
|
|
--
|
|
\c :TEST_DBNAME :ROLE_SUPERUSER
|
|
CREATE OR REPLACE FUNCTION ts_bgw_db_scheduler_test_run_and_wait_for_scheduler_finish(timeout INT = -1) RETURNS VOID
|
|
AS :MODULE_PATHNAME LANGUAGE C VOLATILE;
|
|
|
|
CREATE OR REPLACE FUNCTION ts_bgw_db_scheduler_test_run(timeout INT = -1, mock_start_time INT = 0) RETURNS VOID
|
|
AS :MODULE_PATHNAME LANGUAGE C VOLATILE;
|
|
|
|
CREATE OR REPLACE FUNCTION ts_bgw_db_scheduler_test_wait_for_scheduler_finish() RETURNS VOID
|
|
AS :MODULE_PATHNAME LANGUAGE C VOLATILE;
|
|
|
|
CREATE OR REPLACE FUNCTION ts_bgw_params_create() RETURNS VOID
|
|
AS :MODULE_PATHNAME LANGUAGE C VOLATILE;
|
|
|
|
CREATE OR REPLACE FUNCTION ts_bgw_params_destroy() RETURNS VOID
|
|
AS :MODULE_PATHNAME LANGUAGE C VOLATILE;
|
|
|
|
CREATE OR REPLACE FUNCTION ts_bgw_params_reset_time(set_time BIGINT = 0, wait BOOLEAN = false) RETURNS VOID
|
|
AS :MODULE_PATHNAME LANGUAGE C VOLATILE;
|
|
|
|
\set WAIT_ON_JOB 0
|
|
\set IMMEDIATELY_SET_UNTIL 1
|
|
\set WAIT_FOR_OTHER_TO_ADVANCE 2
|
|
|
|
-- Remove any default jobs, e.g., telemetry
|
|
DELETE FROM _timescaledb_config.bgw_job;
|
|
TRUNCATE _timescaledb_internal.bgw_job_stat;
|
|
|
|
\c :TEST_DBNAME :ROLE_DEFAULT_PERM_USER
|
|
|
|
CREATE TABLE public.bgw_log(
|
|
msg_no INT,
|
|
mock_time BIGINT,
|
|
application_name TEXT,
|
|
msg TEXT
|
|
);
|
|
|
|
CREATE VIEW sorted_bgw_log AS
|
|
SELECT msg_no,
|
|
mock_time,
|
|
application_name,
|
|
regexp_replace(CASE WHEN length(msg) > 80 THEN substring(msg, 1, 80) || '...' ELSE msg END,
|
|
'(execution time) [0-9]+(\.[0-9]+)?', '\1 (RANDOM)', 'g') AS msg
|
|
FROM bgw_log
|
|
ORDER BY mock_time,
|
|
application_name COLLATE "C",
|
|
msg_no;
|
|
CREATE TABLE public.bgw_dsm_handle_store(
|
|
handle BIGINT
|
|
);
|
|
|
|
INSERT INTO public.bgw_dsm_handle_store VALUES (0);
|
|
SELECT ts_bgw_params_create();
|
|
|
|
SELECT * FROM _timescaledb_config.bgw_job;
|
|
SELECT * FROM timescaledb_information.job_stats;
|
|
|
|
\c :TEST_DBNAME :ROLE_DEFAULT_PERM_USER
|
|
|
|
------------------------------
|
|
-- test reorder policy runs --
|
|
------------------------------
|
|
|
|
CREATE TABLE test_reorder_table(time int, chunk_id int);
|
|
SELECT create_hypertable('test_reorder_table', 'time', chunk_time_interval => 1);
|
|
|
|
-- These inserts should create 5 different chunks
|
|
INSERT INTO test_reorder_table VALUES (1, 1);
|
|
INSERT INTO test_reorder_table VALUES (2, 2);
|
|
INSERT INTO test_reorder_table VALUES (3, 3);
|
|
INSERT INTO test_reorder_table VALUES (4, 4);
|
|
INSERT INTO test_reorder_table VALUES (5, 5);
|
|
|
|
SELECT COUNT(*) FROM _timescaledb_catalog.chunk as c, _timescaledb_catalog.hypertable as ht where c.hypertable_id = ht.id and ht.table_name='test_reorder_table';
|
|
|
|
SELECT count(*) FROM _timescaledb_config.bgw_job WHERE proc_schema = '_timescaledb_internal' AND proc_name = 'policy_reorder';
|
|
select add_reorder_policy('test_reorder_table', 'test_reorder_table_time_idx') as reorder_job_id \gset
|
|
SELECT count(*) FROM _timescaledb_config.bgw_job WHERE proc_schema = '_timescaledb_internal' AND proc_name = 'policy_reorder';
|
|
|
|
-- job was created
|
|
SELECT * FROM timescaledb_information.jobs WHERE job_id=:reorder_job_id;
|
|
|
|
-- no stats
|
|
SELECT job_id, next_start, last_finish as until_next, last_run_success, total_runs, total_successes, total_failures, total_crashes
|
|
FROM _timescaledb_internal.bgw_job_stat
|
|
ORDER BY job_id;
|
|
|
|
-- nothing clustered
|
|
SELECT indexrelid::regclass, indisclustered
|
|
FROM pg_index
|
|
WHERE indisclustered = true ORDER BY 1;
|
|
|
|
-- run first time
|
|
SELECT ts_bgw_db_scheduler_test_run_and_wait_for_scheduler_finish(25);
|
|
|
|
SELECT * FROM sorted_bgw_log;
|
|
|
|
SELECT * FROM timescaledb_information.jobs WHERE job_id=:reorder_job_id;
|
|
|
|
-- job ran once, successfully
|
|
SELECT job_id, next_start, last_finish as until_next, last_run_success, total_runs, total_successes, total_failures, total_crashes
|
|
FROM _timescaledb_internal.bgw_job_stat
|
|
where job_id=:reorder_job_id;
|
|
|
|
-- first chunk reordered
|
|
SELECT indexrelid::regclass, indisclustered
|
|
FROM pg_index
|
|
WHERE indisclustered = true ORDER BY 1;
|
|
|
|
-- second call to scheduler should immediately run reorder again, due to catchup
|
|
SELECT ts_bgw_db_scheduler_test_run_and_wait_for_scheduler_finish(25);
|
|
|
|
SELECT * FROM sorted_bgw_log;
|
|
|
|
SELECT * FROM timescaledb_information.jobs WHERE job_id=:reorder_job_id;
|
|
|
|
-- two runs
|
|
SELECT job_id, next_start, last_finish as until_next, last_run_success, total_runs, total_successes, total_failures, total_crashes
|
|
FROM _timescaledb_internal.bgw_job_stat
|
|
where job_id=:reorder_job_id;
|
|
|
|
-- two chunks clustered
|
|
SELECT indexrelid::regclass, indisclustered
|
|
FROM pg_index
|
|
WHERE indisclustered = true ORDER BY 1;
|
|
|
|
-- third call to scheduler should immediately run reorder again, due to catchup
|
|
SELECT ts_bgw_db_scheduler_test_run_and_wait_for_scheduler_finish(50);
|
|
|
|
-- job info is gone
|
|
|
|
SELECT * FROM sorted_bgw_log;
|
|
|
|
SELECT * FROM timescaledb_information.jobs WHERE job_id=:reorder_job_id;
|
|
|
|
SELECT *
|
|
FROM _timescaledb_internal.bgw_job_stat
|
|
where job_id=:reorder_job_id;
|
|
|
|
-- three chunks clustered
|
|
SELECT indexrelid::regclass, indisclustered
|
|
FROM pg_index
|
|
WHERE indisclustered = true ORDER BY 1;
|
|
|
|
|
|
-- running is a nop
|
|
SELECT ts_bgw_db_scheduler_test_run_and_wait_for_scheduler_finish(100);
|
|
|
|
SELECT * FROM sorted_bgw_log;
|
|
|
|
SELECT * FROM timescaledb_information.jobs WHERE job_id=:reorder_job_id;
|
|
|
|
SELECT *
|
|
FROM _timescaledb_internal.bgw_job_stat
|
|
where job_id=:reorder_job_id;
|
|
|
|
-- still have 3 chunks clustered
|
|
SELECT indexrelid::regclass, indisclustered
|
|
FROM pg_index
|
|
WHERE indisclustered = true ORDER BY 1;
|
|
|
|
--check that views work correctly
|
|
SELECT * FROM timescaledb_information.job_stats;
|
|
|
|
-- test deleting the policy
|
|
SELECT remove_reorder_policy('test_reorder_table');
|
|
|
|
SELECT * FROM timescaledb_information.jobs WHERE job_id=:reorder_job_id;
|
|
|
|
SELECT job_id, next_start, last_finish as until_next, last_run_success, total_runs, total_successes, total_failures, total_crashes
|
|
FROM _timescaledb_internal.bgw_job_stat
|
|
where job_id=:reorder_job_id;
|
|
|
|
SELECT ts_bgw_db_scheduler_test_run_and_wait_for_scheduler_finish(125);
|
|
|
|
SELECT * FROM sorted_bgw_log;
|
|
|
|
SELECT * FROM timescaledb_information.jobs WHERE job_id=:reorder_job_id;
|
|
|
|
-- still only 3 chunks clustered
|
|
SELECT indexrelid::regclass, indisclustered
|
|
FROM pg_index
|
|
WHERE indisclustered = true ORDER BY 1;
|
|
|
|
|
|
|
|
\c :TEST_DBNAME :ROLE_SUPERUSER
|
|
TRUNCATE bgw_log;
|
|
TRUNCATE _timescaledb_internal.bgw_job_stat;
|
|
DELETE FROM _timescaledb_config.bgw_job;
|
|
SELECT ts_bgw_params_reset_time();
|
|
\c :TEST_DBNAME :ROLE_DEFAULT_PERM_USER
|
|
|
|
-----------------------------------
|
|
-- test drop chunnks policy runs --
|
|
-----------------------------------
|
|
|
|
|
|
CREATE TABLE test_drop_chunks_table(time timestamptz, drop_order int);
|
|
SELECT create_hypertable('test_drop_chunks_table', 'time', chunk_time_interval => INTERVAL '1 week');
|
|
|
|
-- These inserts should create 5 different chunks
|
|
INSERT INTO test_drop_chunks_table VALUES (now() - INTERVAL '2 month', 4);
|
|
INSERT INTO test_drop_chunks_table VALUES (now(), 5);
|
|
INSERT INTO test_drop_chunks_table VALUES (now() - INTERVAL '6 months', 2);
|
|
INSERT INTO test_drop_chunks_table VALUES (now() - INTERVAL '4 months', 3);
|
|
INSERT INTO test_drop_chunks_table VALUES (now() - INTERVAL '8 months', 1);
|
|
|
|
|
|
SELECT show_chunks('test_drop_chunks_table');
|
|
SELECT COUNT(*) FROM _timescaledb_catalog.chunk as c, _timescaledb_catalog.hypertable as ht where c.hypertable_id = ht.id and ht.table_name='test_drop_chunks_table';
|
|
|
|
SELECT count(*) FROM _timescaledb_config.bgw_job WHERE proc_schema = '_timescaledb_internal' AND proc_name = 'policy_retention';
|
|
SELECT add_retention_policy('test_drop_chunks_table', INTERVAL '4 months') as drop_chunks_job_id \gset
|
|
SELECT count(*) FROM _timescaledb_config.bgw_job WHERE proc_schema = '_timescaledb_internal' AND proc_name = 'policy_retention';
|
|
|
|
SELECT alter_job(:drop_chunks_job_id, schedule_interval => INTERVAL '1 second');
|
|
|
|
SELECT * FROM timescaledb_information.jobs WHERE job_id=:drop_chunks_job_id;
|
|
|
|
-- no stats
|
|
SELECT job_id, next_start, last_finish as until_next, last_run_success, total_runs, total_successes, total_failures, total_crashes
|
|
FROM _timescaledb_internal.bgw_job_stat
|
|
WHERE job_id=:drop_chunks_job_id;
|
|
|
|
-- all chunks are there
|
|
SELECT show_chunks('test_drop_chunks_table');
|
|
|
|
-- run first time
|
|
SELECT ts_bgw_db_scheduler_test_run_and_wait_for_scheduler_finish(25);
|
|
|
|
SELECT * FROM sorted_bgw_log;
|
|
|
|
SELECT * FROM timescaledb_information.jobs WHERE job_id=:drop_chunks_job_id;
|
|
|
|
-- job ran once, successfully
|
|
SELECT job_id, time_bucket('1m',next_start) AS next_start, time_bucket('1m',last_finish) as until_next, last_run_success, total_runs, total_successes, total_failures, total_crashes
|
|
FROM _timescaledb_internal.bgw_job_stat
|
|
where job_id=:drop_chunks_job_id;
|
|
|
|
-- chunks 8 and 10 dropped
|
|
SELECT show_chunks('test_drop_chunks_table');
|
|
|
|
-- job doesn't run again immediately
|
|
SELECT ts_bgw_db_scheduler_test_run_and_wait_for_scheduler_finish(25);
|
|
|
|
SELECT * FROM sorted_bgw_log;
|
|
|
|
SELECT * FROM timescaledb_information.jobs WHERE job_id=:drop_chunks_job_id;
|
|
|
|
-- still only 1 run
|
|
SELECT job_id, time_bucket('1m',next_start) AS next_start, time_bucket('1m',last_finish) as until_next, last_run_success, total_runs, total_successes, total_failures, total_crashes
|
|
FROM _timescaledb_internal.bgw_job_stat
|
|
where job_id=:drop_chunks_job_id;
|
|
|
|
-- same chunks
|
|
SELECT show_chunks('test_drop_chunks_table');
|
|
|
|
-- a new chunk older than the drop date will be dropped
|
|
INSERT INTO test_drop_chunks_table VALUES (now() - INTERVAL '12 months', 0);
|
|
|
|
SELECT show_chunks('test_drop_chunks_table');
|
|
|
|
SELECT ts_bgw_db_scheduler_test_run_and_wait_for_scheduler_finish(10000);
|
|
|
|
SELECT * FROM sorted_bgw_log;
|
|
|
|
SELECT * FROM timescaledb_information.jobs WHERE job_id=:drop_chunks_job_id;
|
|
|
|
-- 2 runs
|
|
SELECT job_id, time_bucket('1m',next_start) AS next_start, time_bucket('1m',last_finish) as until_next, last_run_success, total_runs, total_successes, total_failures, total_crashes
|
|
FROM _timescaledb_internal.bgw_job_stat
|
|
where job_id=:drop_chunks_job_id;
|
|
|
|
SELECT show_chunks('test_drop_chunks_table');
|
|
|
|
--test that views work
|
|
SELECT * FROM timescaledb_information.job_stats;
|
|
|
|
|
|
-- Test that add_retention_policy also works with timestamp (without time zone) and date types
|
|
-- and that the policy execution is being logged
|
|
|
|
-- Test for date
|
|
CREATE TABLE test_drop_chunks_table_date(time date, drop_order int);
|
|
SELECT create_hypertable('test_drop_chunks_table_date', 'time', chunk_time_interval => INTERVAL '1 week');
|
|
|
|
INSERT INTO test_drop_chunks_table_date VALUES (now() - INTERVAL '2 month', 4);
|
|
INSERT INTO test_drop_chunks_table_date VALUES (now(), 5);
|
|
INSERT INTO test_drop_chunks_table_date VALUES (now() - INTERVAL '6 months', 2);
|
|
INSERT INTO test_drop_chunks_table_date VALUES (now() - INTERVAL '4 months', 3);
|
|
INSERT INTO test_drop_chunks_table_date VALUES (now() - INTERVAL '8 months', 1);
|
|
|
|
-- Clear the job stats and reset timer, this will also clear the bgw_log
|
|
\c :TEST_DBNAME :ROLE_SUPERUSER
|
|
TRUNCATE bgw_log;
|
|
TRUNCATE _timescaledb_internal.bgw_job_stat;
|
|
DELETE FROM _timescaledb_config.bgw_job;
|
|
SELECT ts_bgw_params_reset_time();
|
|
\c :TEST_DBNAME :ROLE_DEFAULT_PERM_USER
|
|
|
|
|
|
-- Test for timestamp
|
|
CREATE TABLE test_drop_chunks_table_tsntz(time date, drop_order int);
|
|
SELECT create_hypertable('test_drop_chunks_table_tsntz', 'time', chunk_time_interval => INTERVAL '1 week');
|
|
|
|
INSERT INTO test_drop_chunks_table_tsntz VALUES (now() - INTERVAL '2 month', 4);
|
|
INSERT INTO test_drop_chunks_table_tsntz VALUES (now(), 5);
|
|
INSERT INTO test_drop_chunks_table_tsntz VALUES (now() - INTERVAL '6 months', 2);
|
|
INSERT INTO test_drop_chunks_table_tsntz VALUES (now() - INTERVAL '4 months', 3);
|
|
INSERT INTO test_drop_chunks_table_tsntz VALUES (now() - INTERVAL '8 months', 1);
|
|
|
|
-- Add retention policies for both tables
|
|
SELECT add_retention_policy('test_drop_chunks_table_date', INTERVAL '4 months') as drop_chunks_date_job_id \gset
|
|
SELECT add_retention_policy('test_drop_chunks_table_tsntz', INTERVAL '4 months') as drop_chunks_tsntz_job_id \gset
|
|
|
|
-- Test that retention policy is being logged
|
|
SELECT alter_job(id,config:=jsonb_set(config,'{verbose_log}', 'true'))
|
|
FROM _timescaledb_config.bgw_job WHERE id = :drop_chunks_date_job_id;
|
|
|
|
SELECT alter_job(id,config:=jsonb_set(config,'{verbose_log}', 'true'))
|
|
FROM _timescaledb_config.bgw_job WHERE id = :drop_chunks_tsntz_job_id;
|
|
|
|
CALL run_job(:drop_chunks_date_job_id);
|
|
SELECT ts_bgw_db_scheduler_test_run_and_wait_for_scheduler_finish(1000);
|
|
|
|
CALL run_job(:drop_chunks_tsntz_job_id);
|
|
SELECT ts_bgw_db_scheduler_test_run_and_wait_for_scheduler_finish(1000);
|
|
|
|
SELECT * FROM sorted_bgw_log;
|
|
|
|
-- test the schedule_interval parameter for policies
|
|
CREATE TABLE test_schedint(time timestamptz, a int, b int);
|
|
select create_hypertable('test_schedint', 'time');
|
|
insert into test_schedint values (now(), 1, 2), (now() + interval '2 seconds', 2, 3);
|
|
|
|
-- test the retention policy
|
|
select add_retention_policy('test_schedint', interval '2 months', schedule_interval => '30 seconds') as polret_schedint \gset
|
|
-- wait for a bit more than "schedule_interval" seconds, then verify the policy has run twice
|
|
select ts_bgw_db_scheduler_test_run_and_wait_for_scheduler_finish(1000);
|
|
select total_runs, total_successes, total_failures from timescaledb_information.job_stats where job_id = :polret_schedint;
|
|
select ts_bgw_db_scheduler_test_run_and_wait_for_scheduler_finish(30000);
|
|
select total_runs, total_successes, total_failures from timescaledb_information.job_stats where job_id = :polret_schedint;
|
|
-- if we wait another 30s, we should see 3 runs of the job
|
|
select ts_bgw_db_scheduler_test_run_and_wait_for_scheduler_finish(30000);
|
|
select total_runs, total_successes, total_failures from timescaledb_information.job_stats where job_id = :polret_schedint;
|
|
|
|
-- test the compression policy
|
|
alter table test_schedint set (timescaledb.compress);
|
|
select add_compression_policy('test_schedint', interval '3 weeks', schedule_interval => '40 seconds') as polcomp_schedint \gset
|
|
select ts_bgw_db_scheduler_test_run_and_wait_for_scheduler_finish(1000);
|
|
select total_runs, total_successes, total_failures from timescaledb_information.job_stats where job_id = :polcomp_schedint;
|
|
select ts_bgw_db_scheduler_test_run_and_wait_for_scheduler_finish(40000);
|
|
select total_runs, total_successes, total_failures from timescaledb_information.job_stats where job_id = :polcomp_schedint;
|
|
-- if we wait another 40s, we should see 3 runs of the job
|
|
select ts_bgw_db_scheduler_test_run_and_wait_for_scheduler_finish(40000);
|
|
select total_runs, total_successes, total_failures from timescaledb_information.job_stats where job_id = :polcomp_schedint;
|