postgres/timescaledb
1
0
Fork 0
mirror of https://github.com/timescale/timescaledb.git synced 2025-05-15 18:13:18 +08:00
Code Issues Packages Projects Releases Wiki Activity

Use segmentwise recompression in compress policy

Browse Source 
Change compression policy to use segmentwise
recompression when possible to increase performance.
Segmentwise recompression decompresses rows into memory,
thus reducing IO load when recompressing, making it
much faster for bigger chunks.
This commit is contained in:
Ante Kresic 2023-11-22 16:09:29 +01:00 committed by Ante Kresic
parent 8b3227a01e
commit 8745063bd4
11 changed files with 103 additions and 83 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
.unreleased/fix_6343 Normal file
View File

@ -0,0 +1,2 @@
Fixes: #6343 Enable segmentwise recompression in compression policy
Thanks: @fetchezar for reporting the issue

25
sql/policy_internal.sql
View File

@ -123,26 +123,19 @@ BEGIN
)
) AND recompress_enabled IS TRUE THEN
BEGIN
PERFORM @extschema@.decompress_chunk(chunk_rec.oid, if_compressed => true);
-- first check if there's an index. Might have to use a heuristic to determine if index usage would be efficient,
-- or if we'd better fall back to decompressing & recompressing entire chunk
IF _timescaledb_functions.get_compressed_chunk_index_for_recompression(chunk_rec.oid) IS NOT NULL THEN
PERFORM _timescaledb_functions.recompress_chunk_segmentwise(chunk_rec.oid);
ELSE
PERFORM @extschema@.decompress_chunk(chunk_rec.oid, if_compressed => true);
PERFORM @extschema@.compress_chunk(chunk_rec.oid);
END IF;
EXCEPTION WHEN OTHERS THEN
GET STACKED DIAGNOSTICS
_message = MESSAGE_TEXT,
_detail = PG_EXCEPTION_DETAIL;
RAISE WARNING 'decompressing chunk "%" failed when compression policy is executed', chunk_rec.oid::regclass::text
USING DETAIL = format('Message: (%s), Detail: (%s).', _message, _detail),
ERRCODE = sqlstate;
END;
-- SET LOCAL is only active until end of transaction.
-- While we could use SET at the start of the function we do not
-- want to bleed out search_path to caller, so we do SET LOCAL
-- again after COMMIT
BEGIN
PERFORM @extschema@.compress_chunk(chunk_rec.oid);
EXCEPTION WHEN OTHERS THEN
GET STACKED DIAGNOSTICS
_message = MESSAGE_TEXT,
_detail = PG_EXCEPTION_DETAIL;
RAISE WARNING 'compressing chunk "%" failed when compression policy is executed', chunk_rec.oid::regclass::text
RAISE WARNING 'recompressing chunk "%" failed when compression policy is executed', chunk_rec.oid::regclass::text
USING DETAIL = format('Message: (%s), Detail: (%s).', _message, _detail),
ERRCODE = sqlstate;
END;

2
tsl/test/expected/bgw_custom-13.out
View File

@ -572,7 +572,7 @@ FROM _timescaledb_config.bgw_job WHERE id = :job_id_5;
--verify that job is dropped when cagg is dropped
DROP MATERIALIZED VIEW conditions_summary_daily;
NOTICE: drop cascades to table _timescaledb_internal._hyper_3_10_chunk
NOTICE: drop cascades to table _timescaledb_internal._hyper_3_7_chunk
SELECT id, proc_name, hypertable_id
FROM _timescaledb_config.bgw_job WHERE id = :job_id_5;
id | proc_name | hypertable_id

2
tsl/test/expected/bgw_custom-14.out
View File

@ -572,7 +572,7 @@ FROM _timescaledb_config.bgw_job WHERE id = :job_id_5;
--verify that job is dropped when cagg is dropped
DROP MATERIALIZED VIEW conditions_summary_daily;
NOTICE: drop cascades to table _timescaledb_internal._hyper_3_10_chunk
NOTICE: drop cascades to table _timescaledb_internal._hyper_3_7_chunk
SELECT id, proc_name, hypertable_id
FROM _timescaledb_config.bgw_job WHERE id = :job_id_5;
id | proc_name | hypertable_id

2
tsl/test/expected/bgw_custom-15.out
View File

@ -572,7 +572,7 @@ FROM _timescaledb_config.bgw_job WHERE id = :job_id_5;
--verify that job is dropped when cagg is dropped
DROP MATERIALIZED VIEW conditions_summary_daily;
NOTICE: drop cascades to table _timescaledb_internal._hyper_3_10_chunk
NOTICE: drop cascades to table _timescaledb_internal._hyper_3_7_chunk
SELECT id, proc_name, hypertable_id
FROM _timescaledb_config.bgw_job WHERE id = :job_id_5;
id | proc_name | hypertable_id

2
tsl/test/expected/bgw_custom-16.out
View File

@ -572,7 +572,7 @@ FROM _timescaledb_config.bgw_job WHERE id = :job_id_5;
--verify that job is dropped when cagg is dropped
DROP MATERIALIZED VIEW conditions_summary_daily;
NOTICE: drop cascades to table _timescaledb_internal._hyper_3_10_chunk
NOTICE: drop cascades to table _timescaledb_internal._hyper_3_7_chunk
SELECT id, proc_name, hypertable_id
FROM _timescaledb_config.bgw_job WHERE id = :job_id_5;
id | proc_name | hypertable_id

35
tsl/test/expected/compression_bgw-13.out
View File

@ -474,15 +474,16 @@ ORDER BY 1, 2;
Thu Apr 02 17:00:00 2020 PDT | 13 | 1
(4 rows)
--chunk status should be unordered for the previously compressed chunk
--chunk status should be partially compressed for the previously compressed chunk
SELECT chunk_status,
chunk_name as "CHUNK_NAME"
chunk_name as "CHUNK_NAME",
compressed_chunk_name as "COMPRESSED_CHUNK_NAME"
FROM compressed_chunk_info_view
WHERE hypertable_name = 'test2' ORDER BY chunk_name;
chunk_status | CHUNK_NAME
--------------+--------------------
9 | _hyper_14_62_chunk
0 | _hyper_14_64_chunk
chunk_status | CHUNK_NAME | COMPRESSED_CHUNK_NAME
--------------+--------------------+----------------------------
9 | _hyper_14_62_chunk | compress_hyper_15_63_chunk
0 | _hyper_14_64_chunk |
(2 rows)
SELECT add_compression_policy AS job_id
@ -490,23 +491,27 @@ SELECT add_compression_policy AS job_id
CALL run_job(:job_id);
CALL run_job(:job_id);
-- status should be compressed ---
-- compressed chunk name should not change for
-- the partially compressed chunk indicating
-- it was done segmentwise
SELECT chunk_status,
chunk_name as "CHUNK_NAME"
chunk_name as "CHUNK_NAME",
compressed_chunk_name as "COMPRESSED_CHUNK_NAME"
FROM compressed_chunk_info_view
WHERE hypertable_name = 'test2' ORDER BY chunk_name;
chunk_status | CHUNK_NAME
--------------+--------------------
1 | _hyper_14_62_chunk
1 | _hyper_14_64_chunk
chunk_status | CHUNK_NAME | COMPRESSED_CHUNK_NAME
--------------+--------------------+----------------------------
1 | _hyper_14_62_chunk | compress_hyper_15_63_chunk
1 | _hyper_14_64_chunk | compress_hyper_15_65_chunk
(2 rows)
\set ON_ERROR_STOP 0
-- call recompress_chunk when status is not unordered
CALL recompress_chunk(:'CHUNK_NAME'::regclass, true);
psql:include/recompress_basic.sql:110: NOTICE: nothing to recompress in chunk "_hyper_14_62_chunk"
psql:include/recompress_basic.sql:115: NOTICE: nothing to recompress in chunk "_hyper_14_62_chunk"
-- This will succeed and compress the chunk for the test below.
CALL recompress_chunk(:'CHUNK_NAME'::regclass, false);
psql:include/recompress_basic.sql:113: ERROR: nothing to recompress in chunk "_hyper_14_62_chunk"
psql:include/recompress_basic.sql:118: ERROR: nothing to recompress in chunk "_hyper_14_62_chunk"
--now decompress it , then try and recompress
SELECT decompress_chunk(:'CHUNK_NAME'::regclass);
decompress_chunk
@ -515,7 +520,7 @@ SELECT decompress_chunk(:'CHUNK_NAME'::regclass);
(1 row)
CALL recompress_chunk(:'CHUNK_NAME'::regclass);
psql:include/recompress_basic.sql:117: ERROR: call compress_chunk instead of recompress_chunk
psql:include/recompress_basic.sql:122: ERROR: call compress_chunk instead of recompress_chunk
\set ON_ERROR_STOP 1
-- test recompress policy
CREATE TABLE metrics(time timestamptz NOT NULL);
@ -620,7 +625,7 @@ SELECT chunk_status FROM compressed_chunk_info_view WHERE hypertable_name = 'met
---- nothing to do yet
CALL run_job(:JOB_RECOMPRESS);
psql:include/recompress_basic.sql:189: NOTICE: no chunks for hypertable "public.metrics" that satisfy recompress chunk policy
psql:include/recompress_basic.sql:194: NOTICE: no chunks for hypertable "public.metrics" that satisfy recompress chunk policy
---- status should be 1
SELECT chunk_status FROM compressed_chunk_info_view WHERE hypertable_name = 'metrics';
chunk_status

35
tsl/test/expected/compression_bgw-14.out
View File

@ -474,15 +474,16 @@ ORDER BY 1, 2;
Thu Apr 02 17:00:00 2020 PDT | 13 | 1
(4 rows)
--chunk status should be unordered for the previously compressed chunk
--chunk status should be partially compressed for the previously compressed chunk
SELECT chunk_status,
chunk_name as "CHUNK_NAME"
chunk_name as "CHUNK_NAME",
compressed_chunk_name as "COMPRESSED_CHUNK_NAME"
FROM compressed_chunk_info_view
WHERE hypertable_name = 'test2' ORDER BY chunk_name;
chunk_status | CHUNK_NAME
--------------+--------------------
9 | _hyper_14_62_chunk
0 | _hyper_14_64_chunk
chunk_status | CHUNK_NAME | COMPRESSED_CHUNK_NAME
--------------+--------------------+----------------------------
9 | _hyper_14_62_chunk | compress_hyper_15_63_chunk
0 | _hyper_14_64_chunk |
(2 rows)
SELECT add_compression_policy AS job_id
@ -490,23 +491,27 @@ SELECT add_compression_policy AS job_id
CALL run_job(:job_id);
CALL run_job(:job_id);
-- status should be compressed ---
-- compressed chunk name should not change for
-- the partially compressed chunk indicating
-- it was done segmentwise
SELECT chunk_status,
chunk_name as "CHUNK_NAME"
chunk_name as "CHUNK_NAME",
compressed_chunk_name as "COMPRESSED_CHUNK_NAME"
FROM compressed_chunk_info_view
WHERE hypertable_name = 'test2' ORDER BY chunk_name;
chunk_status | CHUNK_NAME
--------------+--------------------
1 | _hyper_14_62_chunk
1 | _hyper_14_64_chunk
chunk_status | CHUNK_NAME | COMPRESSED_CHUNK_NAME
--------------+--------------------+----------------------------
1 | _hyper_14_62_chunk | compress_hyper_15_63_chunk
1 | _hyper_14_64_chunk | compress_hyper_15_65_chunk
(2 rows)
\set ON_ERROR_STOP 0
-- call recompress_chunk when status is not unordered
CALL recompress_chunk(:'CHUNK_NAME'::regclass, true);
psql:include/recompress_basic.sql:110: NOTICE: nothing to recompress in chunk "_hyper_14_62_chunk"
psql:include/recompress_basic.sql:115: NOTICE: nothing to recompress in chunk "_hyper_14_62_chunk"
-- This will succeed and compress the chunk for the test below.
CALL recompress_chunk(:'CHUNK_NAME'::regclass, false);
psql:include/recompress_basic.sql:113: ERROR: nothing to recompress in chunk "_hyper_14_62_chunk"
psql:include/recompress_basic.sql:118: ERROR: nothing to recompress in chunk "_hyper_14_62_chunk"
--now decompress it , then try and recompress
SELECT decompress_chunk(:'CHUNK_NAME'::regclass);
decompress_chunk
@ -515,7 +520,7 @@ SELECT decompress_chunk(:'CHUNK_NAME'::regclass);
(1 row)
CALL recompress_chunk(:'CHUNK_NAME'::regclass);
psql:include/recompress_basic.sql:117: ERROR: call compress_chunk instead of recompress_chunk
psql:include/recompress_basic.sql:122: ERROR: call compress_chunk instead of recompress_chunk
\set ON_ERROR_STOP 1
-- test recompress policy
CREATE TABLE metrics(time timestamptz NOT NULL);
@ -620,7 +625,7 @@ SELECT chunk_status FROM compressed_chunk_info_view WHERE hypertable_name = 'met
---- nothing to do yet
CALL run_job(:JOB_RECOMPRESS);
psql:include/recompress_basic.sql:189: NOTICE: no chunks for hypertable "public.metrics" that satisfy recompress chunk policy
psql:include/recompress_basic.sql:194: NOTICE: no chunks for hypertable "public.metrics" that satisfy recompress chunk policy
---- status should be 1
SELECT chunk_status FROM compressed_chunk_info_view WHERE hypertable_name = 'metrics';
chunk_status

35
tsl/test/expected/compression_bgw-15.out
View File

@ -474,15 +474,16 @@ ORDER BY 1, 2;
Thu Apr 02 17:00:00 2020 PDT | 13 | 1
(4 rows)
--chunk status should be unordered for the previously compressed chunk
--chunk status should be partially compressed for the previously compressed chunk
SELECT chunk_status,
chunk_name as "CHUNK_NAME"
chunk_name as "CHUNK_NAME",
compressed_chunk_name as "COMPRESSED_CHUNK_NAME"
FROM compressed_chunk_info_view
WHERE hypertable_name = 'test2' ORDER BY chunk_name;
chunk_status | CHUNK_NAME
--------------+--------------------
9 | _hyper_14_62_chunk
0 | _hyper_14_64_chunk
chunk_status | CHUNK_NAME | COMPRESSED_CHUNK_NAME
--------------+--------------------+----------------------------
9 | _hyper_14_62_chunk | compress_hyper_15_63_chunk
0 | _hyper_14_64_chunk |
(2 rows)
SELECT add_compression_policy AS job_id
@ -490,23 +491,27 @@ SELECT add_compression_policy AS job_id
CALL run_job(:job_id);
CALL run_job(:job_id);
-- status should be compressed ---
-- compressed chunk name should not change for
-- the partially compressed chunk indicating
-- it was done segmentwise
SELECT chunk_status,
chunk_name as "CHUNK_NAME"
chunk_name as "CHUNK_NAME",
compressed_chunk_name as "COMPRESSED_CHUNK_NAME"
FROM compressed_chunk_info_view
WHERE hypertable_name = 'test2' ORDER BY chunk_name;
chunk_status | CHUNK_NAME
--------------+--------------------
1 | _hyper_14_62_chunk
1 | _hyper_14_64_chunk
chunk_status | CHUNK_NAME | COMPRESSED_CHUNK_NAME
--------------+--------------------+----------------------------
1 | _hyper_14_62_chunk | compress_hyper_15_63_chunk
1 | _hyper_14_64_chunk | compress_hyper_15_65_chunk
(2 rows)
\set ON_ERROR_STOP 0
-- call recompress_chunk when status is not unordered
CALL recompress_chunk(:'CHUNK_NAME'::regclass, true);
psql:include/recompress_basic.sql:110: NOTICE: nothing to recompress in chunk "_hyper_14_62_chunk"
psql:include/recompress_basic.sql:115: NOTICE: nothing to recompress in chunk "_hyper_14_62_chunk"
-- This will succeed and compress the chunk for the test below.
CALL recompress_chunk(:'CHUNK_NAME'::regclass, false);
psql:include/recompress_basic.sql:113: ERROR: nothing to recompress in chunk "_hyper_14_62_chunk"
psql:include/recompress_basic.sql:118: ERROR: nothing to recompress in chunk "_hyper_14_62_chunk"
--now decompress it , then try and recompress
SELECT decompress_chunk(:'CHUNK_NAME'::regclass);
decompress_chunk
@ -515,7 +520,7 @@ SELECT decompress_chunk(:'CHUNK_NAME'::regclass);
(1 row)
CALL recompress_chunk(:'CHUNK_NAME'::regclass);
psql:include/recompress_basic.sql:117: ERROR: call compress_chunk instead of recompress_chunk
psql:include/recompress_basic.sql:122: ERROR: call compress_chunk instead of recompress_chunk
\set ON_ERROR_STOP 1
-- test recompress policy
CREATE TABLE metrics(time timestamptz NOT NULL);
@ -620,7 +625,7 @@ SELECT chunk_status FROM compressed_chunk_info_view WHERE hypertable_name = 'met
---- nothing to do yet
CALL run_job(:JOB_RECOMPRESS);
psql:include/recompress_basic.sql:189: NOTICE: no chunks for hypertable "public.metrics" that satisfy recompress chunk policy
psql:include/recompress_basic.sql:194: NOTICE: no chunks for hypertable "public.metrics" that satisfy recompress chunk policy
---- status should be 1
SELECT chunk_status FROM compressed_chunk_info_view WHERE hypertable_name = 'metrics';
chunk_status

35
tsl/test/expected/compression_bgw-16.out
View File

@ -474,15 +474,16 @@ ORDER BY 1, 2;
Thu Apr 02 17:00:00 2020 PDT | 13 | 1
(4 rows)
--chunk status should be unordered for the previously compressed chunk
--chunk status should be partially compressed for the previously compressed chunk
SELECT chunk_status,
chunk_name as "CHUNK_NAME"
chunk_name as "CHUNK_NAME",
compressed_chunk_name as "COMPRESSED_CHUNK_NAME"
FROM compressed_chunk_info_view
WHERE hypertable_name = 'test2' ORDER BY chunk_name;
chunk_status | CHUNK_NAME
--------------+--------------------
9 | _hyper_14_62_chunk
0 | _hyper_14_64_chunk
chunk_status | CHUNK_NAME | COMPRESSED_CHUNK_NAME
--------------+--------------------+----------------------------
9 | _hyper_14_62_chunk | compress_hyper_15_63_chunk
0 | _hyper_14_64_chunk |
(2 rows)
SELECT add_compression_policy AS job_id
@ -490,23 +491,27 @@ SELECT add_compression_policy AS job_id
CALL run_job(:job_id);
CALL run_job(:job_id);
-- status should be compressed ---
-- compressed chunk name should not change for
-- the partially compressed chunk indicating
-- it was done segmentwise
SELECT chunk_status,
chunk_name as "CHUNK_NAME"
chunk_name as "CHUNK_NAME",
compressed_chunk_name as "COMPRESSED_CHUNK_NAME"
FROM compressed_chunk_info_view
WHERE hypertable_name = 'test2' ORDER BY chunk_name;
chunk_status | CHUNK_NAME
--------------+--------------------
1 | _hyper_14_62_chunk
1 | _hyper_14_64_chunk
chunk_status | CHUNK_NAME | COMPRESSED_CHUNK_NAME
--------------+--------------------+----------------------------
1 | _hyper_14_62_chunk | compress_hyper_15_63_chunk
1 | _hyper_14_64_chunk | compress_hyper_15_65_chunk
(2 rows)
\set ON_ERROR_STOP 0
-- call recompress_chunk when status is not unordered
CALL recompress_chunk(:'CHUNK_NAME'::regclass, true);
psql:include/recompress_basic.sql:110: NOTICE: nothing to recompress in chunk "_hyper_14_62_chunk"
psql:include/recompress_basic.sql:115: NOTICE: nothing to recompress in chunk "_hyper_14_62_chunk"
-- This will succeed and compress the chunk for the test below.
CALL recompress_chunk(:'CHUNK_NAME'::regclass, false);
psql:include/recompress_basic.sql:113: ERROR: nothing to recompress in chunk "_hyper_14_62_chunk"
psql:include/recompress_basic.sql:118: ERROR: nothing to recompress in chunk "_hyper_14_62_chunk"
--now decompress it , then try and recompress
SELECT decompress_chunk(:'CHUNK_NAME'::regclass);
decompress_chunk
@ -515,7 +520,7 @@ SELECT decompress_chunk(:'CHUNK_NAME'::regclass);
(1 row)
CALL recompress_chunk(:'CHUNK_NAME'::regclass);
psql:include/recompress_basic.sql:117: ERROR: call compress_chunk instead of recompress_chunk
psql:include/recompress_basic.sql:122: ERROR: call compress_chunk instead of recompress_chunk
\set ON_ERROR_STOP 1
-- test recompress policy
CREATE TABLE metrics(time timestamptz NOT NULL);
@ -620,7 +625,7 @@ SELECT chunk_status FROM compressed_chunk_info_view WHERE hypertable_name = 'met
---- nothing to do yet
CALL run_job(:JOB_RECOMPRESS);
psql:include/recompress_basic.sql:189: NOTICE: no chunks for hypertable "public.metrics" that satisfy recompress chunk policy
psql:include/recompress_basic.sql:194: NOTICE: no chunks for hypertable "public.metrics" that satisfy recompress chunk policy
---- status should be 1
SELECT chunk_status FROM compressed_chunk_info_view WHERE hypertable_name = 'metrics';
chunk_status

11
tsl/test/sql/include/recompress_basic.sql
View File

@ -88,9 +88,10 @@ FROM test2
GROUP BY time_bucket(INTERVAL '2 hour', timec), b
ORDER BY 1, 2;
--chunk status should be unordered for the previously compressed chunk
--chunk status should be partially compressed for the previously compressed chunk
SELECT chunk_status,
chunk_name as "CHUNK_NAME"
chunk_name as "CHUNK_NAME",
compressed_chunk_name as "COMPRESSED_CHUNK_NAME"
FROM compressed_chunk_info_view
WHERE hypertable_name = 'test2' ORDER BY chunk_name;
@ -100,8 +101,12 @@ CALL run_job(:job_id);
CALL run_job(:job_id);
-- status should be compressed ---
-- compressed chunk name should not change for
-- the partially compressed chunk indicating
-- it was done segmentwise
SELECT chunk_status,
chunk_name as "CHUNK_NAME"
chunk_name as "CHUNK_NAME",
compressed_chunk_name as "COMPRESSED_CHUNK_NAME"
FROM compressed_chunk_info_view
WHERE hypertable_name = 'test2' ORDER BY chunk_name;