mirror of
https://github.com/timescale/timescaledb.git
synced 2025-05-23 06:22:03 +08:00
Fix removal of metadata function and update script
Changing the code to remove the assumption of 1:1 mapping between chunks and chunk constraints. Including a check if a chunk constraint is shared with another chunk. In that case, skip deleting the dimension slice.
This commit is contained in:
Ante Kresic
Ante Kresic
parent
577b923822
commit
5c2c80f845
.unreleased
sql
tsl/test
2
.unreleased/pr_6996
Normal file
2
.unreleased/pr_6996
Normal file
@ -0,0 +1,2 @@
|
||||
Fixes: #6976 Fix removal of metadata function and update script
|
||||
Thanks: @gugu for reporting the issue with catalog corruption due to update
|
||||
@ -144,6 +144,11 @@ BEGIN
|
||||
USING _timescaledb_catalog.chunk_constraint
|
||||
WHERE dimension_slice.id = chunk_constraint.dimension_slice_id
|
||||
AND chunk_constraint.chunk_id = _chunk_id
|
||||
AND NOT EXISTS (
|
||||
SELECT FROM _timescaledb_catalog.chunk_constraint cc
|
||||
WHERE cc.chunk_id <> _chunk_id
|
||||
AND cc.dimension_slice_id = dimension_slice.id
|
||||
)
|
||||
RETURNING _timescaledb_catalog.dimension_slice.id
|
||||
)
|
||||
DELETE FROM _timescaledb_catalog.chunk_constraint
|
||||
|
||||
@ -49,6 +49,11 @@ BEGIN
|
||||
USING _timescaledb_catalog.chunk_constraint
|
||||
WHERE dimension_slice.id = chunk_constraint.dimension_slice_id
|
||||
AND chunk_constraint.chunk_id = _chunk_id
|
||||
AND NOT EXISTS (
|
||||
SELECT FROM _timescaledb_catalog.chunk_constraint cc
|
||||
WHERE cc.chunk_id <> _chunk_id
|
||||
AND cc.dimension_slice_id = dimension_slice.id
|
||||
)
|
||||
RETURNING _timescaledb_catalog.dimension_slice.id
|
||||
)
|
||||
DELETE FROM _timescaledb_catalog.chunk_constraint
|
||||
|
||||
@ -2955,3 +2955,116 @@ DROP MATERIALIZED VIEW conditions_summary_daily;
|
||||
DROP MATERIALIZED VIEW conditions_summary_weekly;
|
||||
DROP TABLE conditions CASCADE;
|
||||
psql:include/cagg_migrate_custom_timezone.sql:23: NOTICE: drop cascades to 3 other objects
|
||||
-- #########################################################
|
||||
-- Issue 6976 - space partitioning should not cause catalog corruption
|
||||
-- #########################################################
|
||||
CREATE TABLE space_partitioning (
|
||||
time timestamptz,
|
||||
device_id integer,
|
||||
value float
|
||||
);
|
||||
-- Updating sequence numbers so creating a hypertable doesn't mess with
|
||||
-- data imports used by migration tests
|
||||
SELECT setval('_timescaledb_catalog.hypertable_id_seq', 999, true);
|
||||
setval
|
||||
--------
|
||||
999
|
||||
(1 row)
|
||||
|
||||
SELECT setval('_timescaledb_catalog.chunk_id_seq', 999, true);
|
||||
setval
|
||||
--------
|
||||
999
|
||||
(1 row)
|
||||
|
||||
SELECT setval('_timescaledb_catalog.dimension_id_seq', 999, true);
|
||||
setval
|
||||
--------
|
||||
999
|
||||
(1 row)
|
||||
|
||||
SELECT setval('_timescaledb_catalog.dimension_slice_id_seq', 999, true);
|
||||
setval
|
||||
--------
|
||||
999
|
||||
(1 row)
|
||||
|
||||
SELECT create_hypertable('space_partitioning', 'time', chunk_time_interval=>'1 hour'::interval);
|
||||
NOTICE: adding not-null constraint to column "time"
|
||||
create_hypertable
|
||||
------------------------------------
|
||||
(1000,public,space_partitioning,t)
|
||||
(1 row)
|
||||
|
||||
SELECT add_dimension('space_partitioning', 'device_id', 3);
|
||||
add_dimension
|
||||
----------------------------------------------
|
||||
(1001,public,space_partitioning,device_id,t)
|
||||
(1 row)
|
||||
|
||||
INSERT INTO space_partitioning SELECT t, 1, 1.0 FROM generate_series('2024-01-01'::timestamptz, '2024-02-01'::timestamptz, '10 minutes'::interval) t;
|
||||
INSERT INTO space_partitioning SELECT t, 1000, 1.0 FROM generate_series('2024-01-01'::timestamptz, '2024-02-01'::timestamptz, '10 minutes'::interval) t;
|
||||
CREATE MATERIALIZED VIEW space_partitioning_summary
|
||||
WITH (timescaledb.continuous, timescaledb.materialized_only=false) AS
|
||||
SELECT
|
||||
time_bucket(INTERVAL '1 week', "time") AS bucket,
|
||||
device_id,
|
||||
MIN(value),
|
||||
MAX(value),
|
||||
SUM(value)
|
||||
FROM
|
||||
space_partitioning
|
||||
GROUP BY
|
||||
1, 2
|
||||
WITH NO DATA;
|
||||
-- setting up the state so that remove_dropped_chunk_metadata
|
||||
-- would run on the hypertable and trigger the catalog corruption
|
||||
UPDATE _timescaledb_catalog.chunk
|
||||
SET dropped = TRUE
|
||||
FROM _timescaledb_catalog.hypertable
|
||||
WHERE chunk.hypertable_id = hypertable.id
|
||||
AND hypertable.table_name = 'space_partitioning'
|
||||
AND chunk.id = 1000;
|
||||
UPDATE _timescaledb_catalog.continuous_agg
|
||||
SET finalized = true
|
||||
FROM _timescaledb_catalog.hypertable
|
||||
WHERE continuous_agg.raw_hypertable_id = hypertable.id
|
||||
AND hypertable.table_name = 'space_partitioning';
|
||||
SET timescaledb.restoring TO ON;
|
||||
DROP TABLE _timescaledb_internal._hyper_1000_1000_chunk;
|
||||
SET timescaledb.restoring TO OFF;
|
||||
SELECT _timescaledb_functions.remove_dropped_chunk_metadata(id)
|
||||
FROM _timescaledb_catalog.hypertable
|
||||
WHERE table_name = 'space_partitioning';
|
||||
INFO: Removing metadata of chunk 1000 from hypertable 1000
|
||||
remove_dropped_chunk_metadata
|
||||
-------------------------------
|
||||
1
|
||||
(1 row)
|
||||
|
||||
-- check every chunk has as many chunk constraints as
|
||||
-- there are dimensions, should return empty result
|
||||
-- this ensures we have avoided catalog corruption
|
||||
WITH dimension_count as (
|
||||
SELECT ht.id, count(*)
|
||||
FROM _timescaledb_catalog.hypertable ht
|
||||
INNER JOIN _timescaledb_catalog.dimension d
|
||||
ON d.hypertable_id = ht.id
|
||||
WHERE table_name = 'space_partitioning'
|
||||
GROUP BY 1),
|
||||
chunk_constraint_count AS (
|
||||
SELECT c.hypertable_id, cc.chunk_id, count(*)
|
||||
FROM _timescaledb_catalog.chunk_constraint cc
|
||||
INNER JOIN _timescaledb_catalog.chunk c
|
||||
ON cc.chunk_id = c.id
|
||||
GROUP BY 1, 2
|
||||
)
|
||||
SELECT *
|
||||
FROM dimension_count dc
|
||||
INNER JOIN chunk_constraint_count ccc
|
||||
ON ccc.hypertable_id = dc.id
|
||||
WHERE dc.count != ccc.count;
|
||||
id | count | hypertable_id | chunk_id | count
|
||||
----+-------+---------------+----------+-------
|
||||
(0 rows)
|
||||
|
||||
|
||||
@ -38,3 +38,86 @@ SET timezone = 'Europe/Budapest';
|
||||
-- Test with timestamptz
|
||||
\set TIME_DIMENSION_DATATYPE TIMESTAMPTZ
|
||||
\ir include/cagg_migrate_custom_timezone.sql
|
||||
|
||||
-- #########################################################
|
||||
-- Issue 6976 - space partitioning should not cause catalog corruption
|
||||
-- #########################################################
|
||||
|
||||
CREATE TABLE space_partitioning (
|
||||
time timestamptz,
|
||||
device_id integer,
|
||||
value float
|
||||
);
|
||||
|
||||
-- Updating sequence numbers so creating a hypertable doesn't mess with
|
||||
-- data imports used by migration tests
|
||||
SELECT setval('_timescaledb_catalog.hypertable_id_seq', 999, true);
|
||||
SELECT setval('_timescaledb_catalog.chunk_id_seq', 999, true);
|
||||
SELECT setval('_timescaledb_catalog.dimension_id_seq', 999, true);
|
||||
SELECT setval('_timescaledb_catalog.dimension_slice_id_seq', 999, true);
|
||||
|
||||
SELECT create_hypertable('space_partitioning', 'time', chunk_time_interval=>'1 hour'::interval);
|
||||
SELECT add_dimension('space_partitioning', 'device_id', 3);
|
||||
|
||||
INSERT INTO space_partitioning SELECT t, 1, 1.0 FROM generate_series('2024-01-01'::timestamptz, '2024-02-01'::timestamptz, '10 minutes'::interval) t;
|
||||
INSERT INTO space_partitioning SELECT t, 1000, 1.0 FROM generate_series('2024-01-01'::timestamptz, '2024-02-01'::timestamptz, '10 minutes'::interval) t;
|
||||
|
||||
CREATE MATERIALIZED VIEW space_partitioning_summary
|
||||
WITH (timescaledb.continuous, timescaledb.materialized_only=false) AS
|
||||
SELECT
|
||||
time_bucket(INTERVAL '1 week', "time") AS bucket,
|
||||
device_id,
|
||||
MIN(value),
|
||||
MAX(value),
|
||||
SUM(value)
|
||||
FROM
|
||||
space_partitioning
|
||||
GROUP BY
|
||||
1, 2
|
||||
WITH NO DATA;
|
||||
|
||||
-- setting up the state so that remove_dropped_chunk_metadata
|
||||
-- would run on the hypertable and trigger the catalog corruption
|
||||
UPDATE _timescaledb_catalog.chunk
|
||||
SET dropped = TRUE
|
||||
FROM _timescaledb_catalog.hypertable
|
||||
WHERE chunk.hypertable_id = hypertable.id
|
||||
AND hypertable.table_name = 'space_partitioning'
|
||||
AND chunk.id = 1000;
|
||||
UPDATE _timescaledb_catalog.continuous_agg
|
||||
SET finalized = true
|
||||
FROM _timescaledb_catalog.hypertable
|
||||
WHERE continuous_agg.raw_hypertable_id = hypertable.id
|
||||
AND hypertable.table_name = 'space_partitioning';
|
||||
SET timescaledb.restoring TO ON;
|
||||
DROP TABLE _timescaledb_internal._hyper_1000_1000_chunk;
|
||||
SET timescaledb.restoring TO OFF;
|
||||
|
||||
SELECT _timescaledb_functions.remove_dropped_chunk_metadata(id)
|
||||
FROM _timescaledb_catalog.hypertable
|
||||
WHERE table_name = 'space_partitioning';
|
||||
|
||||
|
||||
-- check every chunk has as many chunk constraints as
|
||||
-- there are dimensions, should return empty result
|
||||
-- this ensures we have avoided catalog corruption
|
||||
WITH dimension_count as (
|
||||
SELECT ht.id, count(*)
|
||||
FROM _timescaledb_catalog.hypertable ht
|
||||
INNER JOIN _timescaledb_catalog.dimension d
|
||||
ON d.hypertable_id = ht.id
|
||||
WHERE table_name = 'space_partitioning'
|
||||
GROUP BY 1),
|
||||
chunk_constraint_count AS (
|
||||
SELECT c.hypertable_id, cc.chunk_id, count(*)
|
||||
FROM _timescaledb_catalog.chunk_constraint cc
|
||||
INNER JOIN _timescaledb_catalog.chunk c
|
||||
ON cc.chunk_id = c.id
|
||||
GROUP BY 1, 2
|
||||
)
|
||||
SELECT *
|
||||
FROM dimension_count dc
|
||||
INNER JOIN chunk_constraint_count ccc
|
||||
ON ccc.hypertable_id = dc.id
|
||||
WHERE dc.count != ccc.count;
|
||||
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user