postgres/timescaledb
1
0
Fork 0
mirror of https://github.com/timescale/timescaledb.git synced 2025-05-16 02:23:49 +08:00
Code Issues Packages Projects Releases Wiki Activity

Record chunk sizes after compression

Browse Source 
Compute chunk size before/after compressing a chunk and record in
catalog table.
This commit is contained in:
gayyappan 2019-08-12 15:31:30 -04:00 committed by Matvey Arye
parent 44941f7bd2
commit 1f4689eca9
8 changed files with 196 additions and 1 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

14
sql/pre_install/tables.sql
View File

@ -318,6 +318,20 @@ CREATE TABLE IF NOT EXISTS _timescaledb_catalog.hypertable_compression (
SELECT pg_catalog.pg_extension_config_dump('_timescaledb_catalog.hypertable_compression', '');
CREATE TABLE IF NOT EXISTS _timescaledb_catalog.compression_chunk_size (
chunk_id INTEGER REFERENCES _timescaledb_catalog.chunk(id) ON DELETE CASCADE,
compressed_chunk_id INTEGER REFERENCES _timescaledb_catalog.chunk(id) ON DELETE CASCADE,
uncompressed_heap_size BIGINT NOT NULL,
uncompressed_toast_size BIGINT NOT NULL,
uncompressed_index_size BIGINT NOT NULL,
compressed_heap_size BIGINT NOT NULL,
compressed_toast_size BIGINT NOT NULL,
compressed_index_size BIGINT NOT NULL,
PRIMARY KEY( chunk_id, compressed_chunk_id)
);
SELECT pg_catalog.pg_extension_config_dump('_timescaledb_catalog.compression_chunk_size', '');
CREATE TABLE IF NOT EXISTS _timescaledb_config.bgw_compress_chunks_policy(
hypertable_id INTEGER REFERENCES _timescaledb_catalog.hypertable(id) ON DELETE CASCADE,
older_than BIGINT NOT NULL,

15
sql/updates/latest-dev.sql
View File

@ -142,6 +142,20 @@ CREATE TABLE IF NOT EXISTS _timescaledb_catalog.hypertable_compression (
SELECT pg_catalog.pg_extension_config_dump('_timescaledb_catalog.hypertable_compression', '');
CREATE TABLE IF NOT EXISTS _timescaledb_catalog.compression_chunk_size (
chunk_id INTEGER REFERENCES _timescaledb_catalog.chunk(id) ON DELETE CASCADE,
compressed_chunk_id INTEGER REFERENCES _timescaledb_catalog.chunk(id) ON DELETE CASCADE,
uncompressed_heap_size BIGINT NOT NULL,
uncompressed_toast_size BIGINT NOT NULL,
uncompressed_index_size BIGINT NOT NULL,
compressed_heap_size BIGINT NOT NULL,
compressed_toast_size BIGINT NOT NULL,
compressed_index_size BIGINT NOT NULL,
PRIMARY KEY( chunk_id, compressed_chunk_id)
);
SELECT pg_catalog.pg_extension_config_dump('_timescaledb_catalog.compression_chunk_size', '');
CREATE TABLE IF NOT EXISTS _timescaledb_config.bgw_compress_chunks_policy(
hypertable_id INTEGER REFERENCES _timescaledb_catalog.hypertable(id) ON DELETE CASCADE,
older_than BIGINT NOT NULL,
@ -154,6 +168,7 @@ SELECT pg_catalog.pg_extension_config_dump('_timescaledb_config.bgw_compress_chu
GRANT SELECT ON _timescaledb_catalog.compression_algorithm TO PUBLIC;
GRANT SELECT ON _timescaledb_catalog.hypertable_compression TO PUBLIC;
GRANT SELECT ON _timescaledb_catalog.compression_chunk_size TO PUBLIC;
GRANT SELECT ON _timescaledb_config.bgw_compress_chunks_policy TO PUBLIC;
CREATE TYPE _timescaledb_internal.compressed_data;

11
src/catalog.c
View File

@ -102,6 +102,10 @@ static const TableInfoDef catalog_table_names[_MAX_CATALOG_TABLES + 1] = {
.schema_name = CATALOG_SCHEMA_NAME,
.table_name = HYPERTABLE_COMPRESSION_TABLE_NAME,
},
[COMPRESSION_CHUNK_SIZE] = {
.schema_name = CATALOG_SCHEMA_NAME,
.table_name = COMPRESSION_CHUNK_SIZE_TABLE_NAME,
},
[BGW_COMPRESS_CHUNKS_POLICY] = {
.schema_name = CONFIG_SCHEMA_NAME,
.table_name = BGW_COMPRESS_CHUNKS_POLICY_TABLE_NAME,
@ -240,6 +244,12 @@ static const TableIndexDef catalog_table_index_definitions[_MAX_CATALOG_TABLES]
[HYPERTABLE_COMPRESSION_PKEY] = "hypertable_compression_pkey",
},
},
[COMPRESSION_CHUNK_SIZE] = {
.length = _MAX_COMPRESSION_CHUNK_SIZE_INDEX,
.names = (char *[]) {
[COMPRESSION_CHUNK_SIZE_PKEY] = "compression_chunk_size_pkey",
},
},
[BGW_COMPRESS_CHUNKS_POLICY] = {
.length = _MAX_BGW_COMPRESS_CHUNKS_POLICY_INDEX,
.names = (char *[]) {
@ -265,6 +275,7 @@ static const char *catalog_table_serial_id_names[_MAX_CATALOG_TABLES] = {
[CONTINUOUS_AGGS_INVALIDATION_THRESHOLD] = NULL,
[CONTINUOUS_AGGS_MATERIALIZATION_INVALIDATION_LOG] = NULL,
[HYPERTABLE_COMPRESSION] = NULL,
[COMPRESSION_CHUNK_SIZE] = NULL,
[BGW_COMPRESS_CHUNKS_POLICY] = NULL,
};

45
src/catalog.h
View File

@ -53,6 +53,7 @@ typedef enum CatalogTable
CONTINUOUS_AGGS_INVALIDATION_THRESHOLD,
CONTINUOUS_AGGS_MATERIALIZATION_INVALIDATION_LOG,
HYPERTABLE_COMPRESSION,
COMPRESSION_CHUNK_SIZE,
BGW_COMPRESS_CHUNKS_POLICY,
_MAX_CATALOG_TABLES,
} CatalogTable;
@ -1087,6 +1088,50 @@ typedef enum Anum_hypertable_compression_pkey
#define Natts_hypertable_compression_pkey (_Anum_hypertable_compression_pkey_max - 1)
#define COMPRESSION_CHUNK_SIZE_TABLE_NAME "compression_chunk_size"
typedef enum Anum_compression_chunk_size
{
Anum_compression_chunk_size_chunk_id = 1,
Anum_compression_chunk_size_compressed_chunk_id,
Anum_compression_chunk_size_uncompressed_heap_size,
Anum_compression_chunk_size_uncompressed_toast_size,
Anum_compression_chunk_size_uncompressed_index_size,
Anum_compression_chunk_size_compressed_heap_size,
Anum_compression_chunk_size_compressed_toast_size,
Anum_compression_chunk_size_compressed_index_size,
_Anum_compression_chunk_size_max,
} Anum_compression_chunk_size;
#define Natts_compression_chunk_size (_Anum_compression_chunk_size_max - 1)
typedef struct FormData_compression_chunk_size
{
int32 chunk_id;
int32 compressed_chunk_id;
int64 uncompressed_heap_size;
int64 uncompressed_toast_size;
int64 uncompressed_index_size;
int64 compressed_heap_size;
int64 compressed_toast_size;
int64 compressed_index_size;
} FormData_compression_chunk_size;
typedef FormData_compression_chunk_size *Form_compression_chunk_size;
enum
{
COMPRESSION_CHUNK_SIZE_PKEY = 0,
_MAX_COMPRESSION_CHUNK_SIZE_INDEX,
};
typedef enum Anum_compression_chunk_size_pkey
{
Anum_compression_chunk_size_pkey_chunk_id = 1,
Anum_compression_chunk_size_pkey_compressed_chunk_id,
_Anum_compression_chunk_size_pkey_max,
} Anum_compression_chunk_size_pkey;
#define Natts_compression_chunk_size_pkey (_Anum_compression_chunk_size_pkey_max - 1)
#define BGW_COMPRESS_CHUNKS_POLICY_TABLE_NAME "bgw_compress_chunks_policy"
typedef enum Anum_bgw_compress_chunks_policy
{

3
test/expected/drop_rename_hypertable.out
View File

@ -198,6 +198,7 @@ SELECT * FROM _timescaledb_catalog.hypertable;
_timescaledb_catalog | chunk_constraint | table | super_user
_timescaledb_catalog | chunk_index | table | super_user
_timescaledb_catalog | compression_algorithm | table | super_user
_timescaledb_catalog | compression_chunk_size | table | super_user
_timescaledb_catalog | continuous_agg | table | super_user
_timescaledb_catalog | continuous_aggs_completed_threshold | table | super_user
_timescaledb_catalog | continuous_aggs_hypertable_invalidation_log | table | super_user
@ -209,7 +210,7 @@ SELECT * FROM _timescaledb_catalog.hypertable;
_timescaledb_catalog | hypertable_compression | table | super_user
_timescaledb_catalog | metadata | table | super_user
_timescaledb_catalog | tablespace | table | super_user
(15 rows)
(16 rows)
\dt "_timescaledb_internal".*
List of relations

79
tsl/src/compression/compress_utils.c
View File

@ -11,6 +11,8 @@
#include <miscadmin.h>
#include <storage/lmgr.h>
#include <utils/elog.h>
#include <utils/fmgrprotos.h>
#include <utils/builtins.h>
#include "chunk.h"
#include "errors.h"
@ -28,6 +30,76 @@ typedef struct CompressChunkCxt
Hypertable *compress_ht; /*compressed table for srcht */
} CompressChunkCxt;
typedef struct
{
int64 heap_size;
int64 toast_size;
int64 index_size;
} ChunkSize;
static ChunkSize
compute_chunk_size(Oid chunk_relid)
{
int64 tot_size;
int i = 0;
ChunkSize ret;
Datum relid = ObjectIdGetDatum(chunk_relid);
char *filtyp[] = { "main", "init", "fsm", "vm" };
/* for heap get size from fsm, vm, init and main as this is included in
* pg_table_size calculation
*/
ret.heap_size = 0;
for (i = 0; i < 4; i++)
{
ret.heap_size += DatumGetInt64(
DirectFunctionCall2(pg_relation_size, relid, CStringGetTextDatum(filtyp[i])));
}
ret.index_size = DatumGetInt64(DirectFunctionCall1(pg_indexes_size, relid));
tot_size = DatumGetInt64(DirectFunctionCall1(pg_table_size, relid));
ret.toast_size = tot_size - ret.heap_size;
return ret;
}
static void
compression_chunk_size_catalog_insert(int32 src_chunk_id, ChunkSize *src_size,
int32 compress_chunk_id, ChunkSize *compress_size)
{
Catalog *catalog = ts_catalog_get();
Relation rel;
TupleDesc desc;
CatalogSecurityContext sec_ctx;
Datum values[Natts_compression_chunk_size];
bool nulls[Natts_compression_chunk_size] = { false };
rel = heap_open(catalog_get_table_id(catalog, COMPRESSION_CHUNK_SIZE), RowExclusiveLock);
desc = RelationGetDescr(rel);
memset(values, 0, sizeof(values));
values[AttrNumberGetAttrOffset(Anum_compression_chunk_size_chunk_id)] =
Int32GetDatum(src_chunk_id);
values[AttrNumberGetAttrOffset(Anum_compression_chunk_size_compressed_chunk_id)] =
Int32GetDatum(compress_chunk_id);
values[AttrNumberGetAttrOffset(Anum_compression_chunk_size_uncompressed_heap_size)] =
Int64GetDatum(src_size->heap_size);
values[AttrNumberGetAttrOffset(Anum_compression_chunk_size_uncompressed_toast_size)] =
Int64GetDatum(src_size->toast_size);
values[AttrNumberGetAttrOffset(Anum_compression_chunk_size_uncompressed_index_size)] =
Int64GetDatum(src_size->index_size);
values[AttrNumberGetAttrOffset(Anum_compression_chunk_size_compressed_heap_size)] =
Int64GetDatum(compress_size->heap_size);
values[AttrNumberGetAttrOffset(Anum_compression_chunk_size_compressed_toast_size)] =
Int64GetDatum(compress_size->toast_size);
values[AttrNumberGetAttrOffset(Anum_compression_chunk_size_compressed_index_size)] =
Int64GetDatum(compress_size->index_size);
ts_catalog_database_info_become_owner(ts_catalog_database_info_get(), &sec_ctx);
ts_catalog_insert_values(rel, desc, values, nulls);
ts_catalog_restore_user(&sec_ctx);
heap_close(rel, RowExclusiveLock);
}
static void
compresschunkcxt_init(CompressChunkCxt *cxt, Cache *hcache, Oid hypertable_relid, Oid chunk_relid)
{
@ -74,6 +146,7 @@ compress_chunk_impl(Oid hypertable_relid, Oid chunk_relid)
List *htcols_list = NIL;
const ColumnCompressionInfo **colinfo_array;
int i = 0, htcols_listlen;
ChunkSize before_size, after_size;
hcache = ts_hypertable_cache_pin();
compresschunkcxt_init(&cxt, hcache, hypertable_relid, chunk_relid);
@ -95,10 +168,16 @@ compress_chunk_impl(Oid hypertable_relid, Oid chunk_relid)
FormData_hypertable_compression *fd = (FormData_hypertable_compression *) lfirst(lc);
colinfo_array[i++] = fd;
}
before_size = compute_chunk_size(cxt.srcht_chunk->table_id);
compress_chunk(cxt.srcht_chunk->table_id,
compress_ht_chunk->table_id,
colinfo_array,
htcols_listlen);
after_size = compute_chunk_size(compress_ht_chunk->table_id);
compression_chunk_size_catalog_insert(cxt.srcht_chunk->fd.id,
&before_size,
compress_ht_chunk->fd.id,
&after_size);
ts_chunk_set_compressed_chunk(cxt.srcht_chunk, compress_ht_chunk->fd.id, false);
ts_cache_release(hcache);
}

24
tsl/test/expected/compression.out
View File

@ -33,6 +33,7 @@ select * from _timescaledb_catalog.hypertable_compression order by hypertable_id
1 | d | 4 | | 2 | t | f
(4 rows)
-- TEST2 compress-chunk for the chunks created earlier --
select compress_chunk( '_timescaledb_internal._hyper_1_2_chunk');
compress_chunk
----------------
@ -45,6 +46,29 @@ select compress_chunk( '_timescaledb_internal._hyper_1_1_chunk');
(1 row)
\x
select * from _timescaledb_catalog.compression_chunk_size
order by chunk_id;
-[ RECORD 1 ]-----------+------
chunk_id | 1
compressed_chunk_id | 6
uncompressed_heap_size | 8192
uncompressed_toast_size | 0
uncompressed_index_size | 16384
compressed_heap_size | 8192
compressed_toast_size | 8192
compressed_index_size | 0
-[ RECORD 2 ]-----------+------
chunk_id | 2
compressed_chunk_id | 5
uncompressed_heap_size | 8192
uncompressed_toast_size | 0
uncompressed_index_size | 16384
compressed_heap_size | 8192
compressed_toast_size | 8192
compressed_index_size | 0
\x
select ch1.id, ch1.schema_name, ch1.table_name , ch2.table_name as compress_table
from
_timescaledb_catalog.chunk ch1, _timescaledb_catalog.chunk ch2

6
tsl/test/sql/compression.sql
View File

@ -18,9 +18,14 @@ select id, schema_name, table_name, compressed, compressed_hypertable_id from
_timescaledb_catalog.hypertable order by id;
select * from _timescaledb_catalog.hypertable_compression order by hypertable_id, attname;
-- TEST2 compress-chunk for the chunks created earlier --
select compress_chunk( '_timescaledb_internal._hyper_1_2_chunk');
select compress_chunk( '_timescaledb_internal._hyper_1_1_chunk');
\x
select * from _timescaledb_catalog.compression_chunk_size
order by chunk_id;
\x
select ch1.id, ch1.schema_name, ch1.table_name , ch2.table_name as compress_table
from
_timescaledb_catalog.chunk ch1, _timescaledb_catalog.chunk ch2
@ -29,3 +34,4 @@ where ch1.compressed_chunk_id = ch2.id;
\set ON_ERROR_STOP 0
--cannot recompress the chunk the second time around
select compress_chunk( '_timescaledb_internal._hyper_1_2_chunk');