postgres/timescaledb
1
0
Fork 0
mirror of https://github.com/timescale/timescaledb.git synced 2025-05-26 08:41:09 +08:00
Code Issues Packages Projects Releases Wiki Activity

Fix removal of metadata function and update script

Browse Source 
Changing the code to remove the assumption of 1:1
mapping between chunks and chunk constraints. Including
a check if a chunk constraint is shared with another chunk.
In that case, skip deleting the dimension slice.
This commit is contained in:
Ante Kresic 2024-06-04 12:27:32 +02:00 committed by Ante Kresic
parent 577b923822
commit 5c2c80f845
5 changed files with 208 additions and 0 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
.unreleased/pr_6996 Normal file
View File

@ -0,0 +1,2 @@
Fixes: #6976 Fix removal of metadata function and update script
Thanks: @gugu for reporting the issue with catalog corruption due to update

5
sql/maintenance_utils.sql
View File

@ -144,6 +144,11 @@ BEGIN
USING _timescaledb_catalog.chunk_constraint USING _timescaledb_catalog.chunk_constraint
WHERE dimension_slice.id = chunk_constraint.dimension_slice_id WHERE dimension_slice.id = chunk_constraint.dimension_slice_id
AND chunk_constraint.chunk_id = _chunk_id AND chunk_constraint.chunk_id = _chunk_id
AND NOT EXISTS (
SELECT FROM _timescaledb_catalog.chunk_constraint cc
WHERE cc.chunk_id <> _chunk_id
AND cc.dimension_slice_id = dimension_slice.id
)
RETURNING _timescaledb_catalog.dimension_slice.id RETURNING _timescaledb_catalog.dimension_slice.id
) )
DELETE FROM _timescaledb_catalog.chunk_constraint DELETE FROM _timescaledb_catalog.chunk_constraint

5
sql/updates/2.14.2--2.15.0.sql
View File

@ -49,6 +49,11 @@ BEGIN
USING _timescaledb_catalog.chunk_constraint USING _timescaledb_catalog.chunk_constraint
WHERE dimension_slice.id = chunk_constraint.dimension_slice_id WHERE dimension_slice.id = chunk_constraint.dimension_slice_id
AND chunk_constraint.chunk_id = _chunk_id AND chunk_constraint.chunk_id = _chunk_id
AND NOT EXISTS (
SELECT FROM _timescaledb_catalog.chunk_constraint cc
WHERE cc.chunk_id <> _chunk_id
AND cc.dimension_slice_id = dimension_slice.id
)
RETURNING _timescaledb_catalog.dimension_slice.id RETURNING _timescaledb_catalog.dimension_slice.id
) )
DELETE FROM _timescaledb_catalog.chunk_constraint DELETE FROM _timescaledb_catalog.chunk_constraint

113
tsl/test/expected/cagg_migrate.out
View File

@ -2955,3 +2955,116 @@ DROP MATERIALIZED VIEW conditions_summary_daily;
DROP MATERIALIZED VIEW conditions_summary_weekly; DROP MATERIALIZED VIEW conditions_summary_weekly;
DROP TABLE conditions CASCADE; DROP TABLE conditions CASCADE;
psql:include/cagg_migrate_custom_timezone.sql:23: NOTICE: drop cascades to 3 other objects psql:include/cagg_migrate_custom_timezone.sql:23: NOTICE: drop cascades to 3 other objects
-- #########################################################
-- Issue 6976 - space partitioning should not cause catalog corruption
-- #########################################################
CREATE TABLE space_partitioning (
time timestamptz,
device_id integer,
value float
);
-- Updating sequence numbers so creating a hypertable doesn't mess with
-- data imports used by migration tests
SELECT setval('_timescaledb_catalog.hypertable_id_seq', 999, true);
setval
--------
999
(1 row)
SELECT setval('_timescaledb_catalog.chunk_id_seq', 999, true);
setval
--------
999
(1 row)
SELECT setval('_timescaledb_catalog.dimension_id_seq', 999, true);
setval
--------
999
(1 row)
SELECT setval('_timescaledb_catalog.dimension_slice_id_seq', 999, true);
setval
--------
999
(1 row)
SELECT create_hypertable('space_partitioning', 'time', chunk_time_interval=>'1 hour'::interval);
NOTICE: adding not-null constraint to column "time"
create_hypertable
------------------------------------
(1000,public,space_partitioning,t)
(1 row)
SELECT add_dimension('space_partitioning', 'device_id', 3);
add_dimension
----------------------------------------------
(1001,public,space_partitioning,device_id,t)
(1 row)
INSERT INTO space_partitioning SELECT t, 1, 1.0 FROM generate_series('2024-01-01'::timestamptz, '2024-02-01'::timestamptz, '10 minutes'::interval) t;
INSERT INTO space_partitioning SELECT t, 1000, 1.0 FROM generate_series('2024-01-01'::timestamptz, '2024-02-01'::timestamptz, '10 minutes'::interval) t;
CREATE MATERIALIZED VIEW space_partitioning_summary
WITH (timescaledb.continuous, timescaledb.materialized_only=false) AS
SELECT
time_bucket(INTERVAL '1 week', "time") AS bucket,
device_id,
MIN(value),
MAX(value),
SUM(value)
FROM
space_partitioning
GROUP BY
1, 2
WITH NO DATA;
-- setting up the state so that remove_dropped_chunk_metadata
-- would run on the hypertable and trigger the catalog corruption
UPDATE _timescaledb_catalog.chunk
SET dropped = TRUE
FROM _timescaledb_catalog.hypertable
WHERE chunk.hypertable_id = hypertable.id
AND hypertable.table_name = 'space_partitioning'
AND chunk.id = 1000;
UPDATE _timescaledb_catalog.continuous_agg
SET finalized = true
FROM _timescaledb_catalog.hypertable
WHERE continuous_agg.raw_hypertable_id = hypertable.id
AND hypertable.table_name = 'space_partitioning';
SET timescaledb.restoring TO ON;
DROP TABLE _timescaledb_internal._hyper_1000_1000_chunk;
SET timescaledb.restoring TO OFF;
SELECT _timescaledb_functions.remove_dropped_chunk_metadata(id)
FROM _timescaledb_catalog.hypertable
WHERE table_name = 'space_partitioning';
INFO: Removing metadata of chunk 1000 from hypertable 1000
remove_dropped_chunk_metadata
-------------------------------
1
(1 row)
-- check every chunk has as many chunk constraints as
-- there are dimensions, should return empty result
-- this ensures we have avoided catalog corruption
WITH dimension_count as (
SELECT ht.id, count(*)
FROM _timescaledb_catalog.hypertable ht
INNER JOIN _timescaledb_catalog.dimension d
ON d.hypertable_id = ht.id
WHERE table_name = 'space_partitioning'
GROUP BY 1),
chunk_constraint_count AS (
SELECT c.hypertable_id, cc.chunk_id, count(*)
FROM _timescaledb_catalog.chunk_constraint cc
INNER JOIN _timescaledb_catalog.chunk c
ON cc.chunk_id = c.id
GROUP BY 1, 2
)
SELECT *
FROM dimension_count dc
INNER JOIN chunk_constraint_count ccc
ON ccc.hypertable_id = dc.id
WHERE dc.count != ccc.count;
id | count | hypertable_id | chunk_id | count
----+-------+---------------+----------+-------
(0 rows)

83
tsl/test/sql/cagg_migrate.sql
View File

@ -38,3 +38,86 @@ SET timezone = 'Europe/Budapest';
-- Test with timestamptz -- Test with timestamptz
\set TIME_DIMENSION_DATATYPE TIMESTAMPTZ \set TIME_DIMENSION_DATATYPE TIMESTAMPTZ
\ir include/cagg_migrate_custom_timezone.sql \ir include/cagg_migrate_custom_timezone.sql
-- #########################################################
-- Issue 6976 - space partitioning should not cause catalog corruption
-- #########################################################
CREATE TABLE space_partitioning (
time timestamptz,
device_id integer,
value float
);
-- Updating sequence numbers so creating a hypertable doesn't mess with
-- data imports used by migration tests
SELECT setval('_timescaledb_catalog.hypertable_id_seq', 999, true);
SELECT setval('_timescaledb_catalog.chunk_id_seq', 999, true);
SELECT setval('_timescaledb_catalog.dimension_id_seq', 999, true);
SELECT setval('_timescaledb_catalog.dimension_slice_id_seq', 999, true);
SELECT create_hypertable('space_partitioning', 'time', chunk_time_interval=>'1 hour'::interval);
SELECT add_dimension('space_partitioning', 'device_id', 3);
INSERT INTO space_partitioning SELECT t, 1, 1.0 FROM generate_series('2024-01-01'::timestamptz, '2024-02-01'::timestamptz, '10 minutes'::interval) t;
INSERT INTO space_partitioning SELECT t, 1000, 1.0 FROM generate_series('2024-01-01'::timestamptz, '2024-02-01'::timestamptz, '10 minutes'::interval) t;
CREATE MATERIALIZED VIEW space_partitioning_summary
WITH (timescaledb.continuous, timescaledb.materialized_only=false) AS
SELECT
time_bucket(INTERVAL '1 week', "time") AS bucket,
device_id,
MIN(value),
MAX(value),
SUM(value)
FROM
space_partitioning
GROUP BY
1, 2
WITH NO DATA;
-- setting up the state so that remove_dropped_chunk_metadata
-- would run on the hypertable and trigger the catalog corruption
UPDATE _timescaledb_catalog.chunk
SET dropped = TRUE
FROM _timescaledb_catalog.hypertable
WHERE chunk.hypertable_id = hypertable.id
AND hypertable.table_name = 'space_partitioning'
AND chunk.id = 1000;
UPDATE _timescaledb_catalog.continuous_agg
SET finalized = true
FROM _timescaledb_catalog.hypertable
WHERE continuous_agg.raw_hypertable_id = hypertable.id
AND hypertable.table_name = 'space_partitioning';
SET timescaledb.restoring TO ON;
DROP TABLE _timescaledb_internal._hyper_1000_1000_chunk;
SET timescaledb.restoring TO OFF;
SELECT _timescaledb_functions.remove_dropped_chunk_metadata(id)
FROM _timescaledb_catalog.hypertable
WHERE table_name = 'space_partitioning';
-- check every chunk has as many chunk constraints as
-- there are dimensions, should return empty result
-- this ensures we have avoided catalog corruption
WITH dimension_count as (
SELECT ht.id, count(*)
FROM _timescaledb_catalog.hypertable ht
INNER JOIN _timescaledb_catalog.dimension d
ON d.hypertable_id = ht.id
WHERE table_name = 'space_partitioning'
GROUP BY 1),
chunk_constraint_count AS (
SELECT c.hypertable_id, cc.chunk_id, count(*)
FROM _timescaledb_catalog.chunk_constraint cc
INNER JOIN _timescaledb_catalog.chunk c
ON cc.chunk_id = c.id
GROUP BY 1, 2
)
SELECT *
FROM dimension_count dc
INNER JOIN chunk_constraint_count ccc
ON ccc.hypertable_id = dc.id
WHERE dc.count != ccc.count;