timescaledb/tsl/src/compression/compression.c
gayyappan 5be6a3e4e9 Support column rename for hypertables with compression enabled
ALTER TABLE <hypertable> RENAME <column_name> TO <new_column_name>
is now supported for hypertables that have compression enabled.

Note: Column renaming is not supported for distributed hypertables.
So this will not work on distributed hypertables that have
compression enabled.
2021-02-19 10:21:50 -05:00

1515 lines
48 KiB
C

/*
* This file and its contents are licensed under the Timescale License.
* Please see the included NOTICE for copyright information and
* LICENSE-TIMESCALE for a copy of the license.
*/
#include "compression/compression.h"
#include <access/heapam.h>
#include <access/htup_details.h>
#include <access/multixact.h>
#include <access/xact.h>
#include <catalog/namespace.h>
#include <catalog/pg_attribute.h>
#include <catalog/pg_type.h>
#include <catalog/index.h>
#include <catalog/heap.h>
#include <common/base64.h>
#include <executor/tuptable.h>
#include <funcapi.h>
#include <libpq/pqformat.h>
#include <miscadmin.h>
#include <storage/predicate.h>
#include <utils/builtins.h>
#include <utils/datum.h>
#include <utils/lsyscache.h>
#include <utils/memutils.h>
#include <utils/rel.h>
#include <utils/snapmgr.h>
#include <utils/syscache.h>
#include <utils/tuplesort.h>
#include <utils/typcache.h>
#include <catalog.h>
#include <utils.h>
#include "compat.h"
#include "array.h"
#include "chunk.h"
#include "deltadelta.h"
#include "dictionary.h"
#include "gorilla.h"
#include "compression_chunk_size.h"
#include "create.h"
#include "custom_type_cache.h"
#include "segment_meta.h"
#define MAX_ROWS_PER_COMPRESSION 1000
/* gap in sequence id between rows, potential for adding rows in gap later */
#define SEQUENCE_NUM_GAP 10
#define COMPRESSIONCOL_IS_SEGMENT_BY(col) (col->segmentby_column_index > 0)
#define COMPRESSIONCOL_IS_ORDER_BY(col) (col->orderby_column_index > 0)
static const CompressionAlgorithmDefinition definitions[_END_COMPRESSION_ALGORITHMS] = {
[COMPRESSION_ALGORITHM_ARRAY] = ARRAY_ALGORITHM_DEFINITION,
[COMPRESSION_ALGORITHM_DICTIONARY] = DICTIONARY_ALGORITHM_DEFINITION,
[COMPRESSION_ALGORITHM_GORILLA] = GORILLA_ALGORITHM_DEFINITION,
[COMPRESSION_ALGORITHM_DELTADELTA] = DELTA_DELTA_ALGORITHM_DEFINITION,
};
static Compressor *
compressor_for_algorithm_and_type(CompressionAlgorithms algorithm, Oid type)
{
if (algorithm >= _END_COMPRESSION_ALGORITHMS)
elog(ERROR, "invalid compression algorithm %d", algorithm);
return definitions[algorithm].compressor_for_type(type);
}
DecompressionIterator *(*tsl_get_decompression_iterator_init(CompressionAlgorithms algorithm,
bool reverse))(Datum, Oid)
{
if (algorithm >= _END_COMPRESSION_ALGORITHMS)
elog(ERROR, "invalid compression algorithm %d", algorithm);
if (reverse)
return definitions[algorithm].iterator_init_reverse;
else
return definitions[algorithm].iterator_init_forward;
}
typedef struct SegmentInfo
{
Datum val;
FmgrInfo eq_fn;
FunctionCallInfo eq_fcinfo;
int16 typlen;
bool is_null;
bool typ_by_val;
} SegmentInfo;
typedef struct PerColumn
{
/* the compressor to use for regular columns, NULL for segmenters */
Compressor *compressor;
/*
* Information on the metadata we'll store for this column (currently only min/max).
* Only used for order-by columns right now, will be {-1, NULL} for others.
*/
int16 min_metadata_attr_offset;
int16 max_metadata_attr_offset;
SegmentMetaMinMaxBuilder *min_max_metadata_builder;
/* segment info; only used if compressor is NULL */
SegmentInfo *segment_info;
} PerColumn;
typedef struct RowCompressor
{
/* memory context reset per-row is stored */
MemoryContext per_row_ctx;
/* the table we're writing the compressed data to */
Relation compressed_table;
BulkInsertState bistate;
/* in theory we could have more input columns than outputted ones, so we
store the number of inputs/compressors seperately*/
int n_input_columns;
/* info about each column */
struct PerColumn *per_column;
/* the order of columns in the compressed data need not match the order in the
* uncompressed. This array maps each attribute offset in the uncompressed
* data to the corresponding one in the compressed
*/
int16 *uncompressed_col_to_compressed_col;
int16 count_metadata_column_offset;
int16 sequence_num_metadata_column_offset;
/* the number of uncompressed rows compressed into the current compressed row */
uint32 rows_compressed_into_current_value;
/* a unique monotonically increasing (according to order by) id for each compressed row */
int32 sequence_num;
/* cached arrays used to build the HeapTuple */
Datum *compressed_values;
bool *compressed_is_null;
int64 rowcnt_pre_compression;
int64 num_compressed_rows;
} RowCompressor;
static int16 *compress_chunk_populate_keys(Oid in_table, const ColumnCompressionInfo **columns,
int n_columns, int *n_keys_out,
const ColumnCompressionInfo ***keys_out);
static Tuplesortstate *compress_chunk_sort_relation(Relation in_rel, int n_keys,
const ColumnCompressionInfo **keys);
static void row_compressor_init(RowCompressor *row_compressor, TupleDesc uncompressed_tuple_desc,
Relation compressed_table, int num_compression_infos,
const ColumnCompressionInfo **column_compression_info,
int16 *column_offsets, int16 num_compressed_columns);
static void row_compressor_append_sorted_rows(RowCompressor *row_compressor,
Tuplesortstate *sorted_rel, TupleDesc sorted_desc);
static void row_compressor_finish(RowCompressor *row_compressor);
/********************
** compress_chunk **
********************/
static void
capture_pgclass_stats(Oid table_oid, int *out_pages, int *out_visible, float *out_tuples)
{
Relation pg_class = table_open(RelationRelationId, RowExclusiveLock);
HeapTuple tuple = SearchSysCacheCopy1(RELOID, ObjectIdGetDatum(table_oid));
Form_pg_class classform;
if (!HeapTupleIsValid(tuple))
elog(ERROR, "could not find tuple for relation %u", table_oid);
classform = (Form_pg_class) GETSTRUCT(tuple);
*out_pages = classform->relpages;
*out_visible = classform->relallvisible;
*out_tuples = classform->reltuples;
heap_freetuple(tuple);
table_close(pg_class, RowExclusiveLock);
}
static void
restore_pgclass_stats(Oid table_oid, int pages, int visible, float tuples)
{
Relation pg_class;
HeapTuple tuple;
Form_pg_class classform;
pg_class = table_open(RelationRelationId, RowExclusiveLock);
tuple = SearchSysCacheCopy1(RELOID, ObjectIdGetDatum(table_oid));
if (!HeapTupleIsValid(tuple))
elog(ERROR, "could not find tuple for relation %u", table_oid);
classform = (Form_pg_class) GETSTRUCT(tuple);
classform->relpages = pages;
classform->relallvisible = visible;
classform->reltuples = tuples;
CatalogTupleUpdate(pg_class, &tuple->t_self, tuple);
heap_freetuple(tuple);
table_close(pg_class, RowExclusiveLock);
}
/* Truncate the relation WITHOUT applying triggers. This is the
* main difference with ExecuteTruncate. Triggers aren't applied
* because the data remains, just in compressed form. Also don't
* restart sequences. Use the transactional branch through ExecuteTruncate.
*/
static void
truncate_relation(Oid table_oid)
{
List *fks = heap_truncate_find_FKs(list_make1_oid(table_oid));
/* Take an access exclusive lock now. Note that this may very well
* be a lock upgrade. */
Relation rel = table_open(table_oid, AccessExclusiveLock);
#if PG12_LT
MultiXactId minmulti;
#endif
Oid toast_relid;
int pages, visible;
float tuples;
/* Chunks should never have fks into them, but double check */
if (fks != NIL)
elog(ERROR, "found a FK into a chunk while truncating");
CheckTableForSerializableConflictIn(rel);
#if PG12_LT
minmulti = GetOldestMultiXactId();
#endif
capture_pgclass_stats(table_oid, &pages, &visible, &tuples);
RelationSetNewRelfilenode(rel,
rel->rd_rel->relpersistence
#if PG12_LT
,
RecentXmin,
minmulti
#endif
);
toast_relid = rel->rd_rel->reltoastrelid;
table_close(rel, NoLock);
if (OidIsValid(toast_relid))
{
rel = table_open(toast_relid, AccessExclusiveLock);
RelationSetNewRelfilenode(rel,
rel->rd_rel->relpersistence
#if PG12_LT
,
RecentXmin,
minmulti
#endif
);
Assert(rel->rd_rel->relpersistence != RELPERSISTENCE_UNLOGGED);
table_close(rel, NoLock);
}
reindex_relation(table_oid, REINDEX_REL_PROCESS_TOAST, 0);
rel = table_open(table_oid, AccessExclusiveLock);
restore_pgclass_stats(table_oid, pages, visible, tuples);
CommandCounterIncrement();
table_close(rel, NoLock);
}
CompressionStats
compress_chunk(Oid in_table, Oid out_table, const ColumnCompressionInfo **column_compression_info,
int num_compression_infos)
{
int n_keys;
const ColumnCompressionInfo **keys;
CompressionStats cstat;
/* We want to prevent other compressors from compressing this table,
* and we want to prevent INSERTs or UPDATEs which could mess up our compression.
* We may as well allow readers to keep reading the uncompressed data while
* we are compressing, so we only take an ExclusiveLock instead of AccessExclusive.
*/
Relation in_rel = table_open(in_table, ExclusiveLock);
/* We are _just_ INSERTing into the out_table so in principle we could take
* a RowExclusive lock, and let other operations read and write this table
* as we work. However, we currently compress each table as a oneshot, so
* we're taking the stricter lock to prevent accidents.
*/
Relation out_rel = relation_open(out_table, ExclusiveLock);
int16 *in_column_offsets = compress_chunk_populate_keys(in_table,
column_compression_info,
num_compression_infos,
&n_keys,
&keys);
TupleDesc in_desc = RelationGetDescr(in_rel);
TupleDesc out_desc = RelationGetDescr(out_rel);
Tuplesortstate *sorted_rel = compress_chunk_sort_relation(in_rel, n_keys, keys);
RowCompressor row_compressor;
Assert(num_compression_infos <= in_desc->natts);
Assert(num_compression_infos <= out_desc->natts);
row_compressor_init(&row_compressor,
in_desc,
out_rel,
num_compression_infos,
column_compression_info,
in_column_offsets,
out_desc->natts);
row_compressor_append_sorted_rows(&row_compressor, sorted_rel, in_desc);
row_compressor_finish(&row_compressor);
tuplesort_end(sorted_rel);
truncate_relation(in_table);
/* Recreate all indexes on out rel, we already have an exclusive lock on it,
* so the strong locks taken by reindex_relation shouldn't matter. */
reindex_relation(out_table, 0, 0);
table_close(out_rel, NoLock);
table_close(in_rel, NoLock);
cstat.rowcnt_pre_compression = row_compressor.rowcnt_pre_compression;
cstat.rowcnt_post_compression = row_compressor.num_compressed_rows;
return cstat;
}
static int16 *
compress_chunk_populate_keys(Oid in_table, const ColumnCompressionInfo **columns, int n_columns,
int *n_keys_out, const ColumnCompressionInfo ***keys_out)
{
int16 *column_offsets = palloc(sizeof(*column_offsets) * n_columns);
int i;
int n_segment_keys = 0;
*n_keys_out = 0;
for (i = 0; i < n_columns; i++)
{
if (COMPRESSIONCOL_IS_SEGMENT_BY(columns[i]))
n_segment_keys += 1;
if (COMPRESSIONCOL_IS_SEGMENT_BY(columns[i]) || COMPRESSIONCOL_IS_ORDER_BY(columns[i]))
*n_keys_out += 1;
}
if (*n_keys_out == 0)
elog(ERROR, "compression should be configured with an orderby or segment by");
*keys_out = palloc(sizeof(**keys_out) * *n_keys_out);
for (i = 0; i < n_columns; i++)
{
const ColumnCompressionInfo *column = columns[i];
/* valid values for segmentby_columnn_index and orderby_column_index
are > 0 */
int16 segment_offset = column->segmentby_column_index - 1;
int16 orderby_offset = column->orderby_column_index - 1;
AttrNumber compressed_att;
if (COMPRESSIONCOL_IS_SEGMENT_BY(column))
(*keys_out)[segment_offset] = column;
else if (COMPRESSIONCOL_IS_ORDER_BY(column))
(*keys_out)[n_segment_keys + orderby_offset] = column;
compressed_att = get_attnum(in_table, NameStr(column->attname));
if (!AttributeNumberIsValid(compressed_att))
elog(ERROR, "could not find compressed column for \"%s\"", NameStr(column->attname));
column_offsets[i] = AttrNumberGetAttrOffset(compressed_att);
}
return column_offsets;
}
static void compress_chunk_populate_sort_info_for_column(Oid table,
const ColumnCompressionInfo *column,
AttrNumber *att_nums, Oid *sort_operator,
Oid *collation, bool *nulls_first);
static Tuplesortstate *
compress_chunk_sort_relation(Relation in_rel, int n_keys, const ColumnCompressionInfo **keys)
{
TupleDesc tupDesc = RelationGetDescr(in_rel);
Tuplesortstate *tuplesortstate;
HeapTuple tuple;
TableScanDesc heapScan;
TupleTableSlot *heap_tuple_slot = MakeTupleTableSlotCompat(tupDesc, TTSOpsHeapTupleP);
AttrNumber *sort_keys = palloc(sizeof(*sort_keys) * n_keys);
Oid *sort_operators = palloc(sizeof(*sort_operators) * n_keys);
Oid *sort_collations = palloc(sizeof(*sort_collations) * n_keys);
bool *nulls_first = palloc(sizeof(*nulls_first) * n_keys);
int n;
for (n = 0; n < n_keys; n++)
compress_chunk_populate_sort_info_for_column(RelationGetRelid(in_rel),
keys[n],
&sort_keys[n],
&sort_operators[n],
&sort_collations[n],
&nulls_first[n]);
tuplesortstate = tuplesort_begin_heap(tupDesc,
n_keys,
sort_keys,
sort_operators,
sort_collations,
nulls_first,
work_mem,
NULL,
false /*=randomAccess*/);
heapScan = table_beginscan(in_rel, GetLatestSnapshot(), 0, (ScanKey) NULL);
for (tuple = heap_getnext(heapScan, ForwardScanDirection); tuple != NULL;
tuple = heap_getnext(heapScan, ForwardScanDirection))
{
if (HeapTupleIsValid(tuple))
{
/* This may not be the most efficient way to do things.
* Since we use begin_heap() the tuplestore expects tupleslots,
* so ISTM that the options are this or maybe putdatum().
*/
#if PG12_LT
ExecStoreTuple(tuple, heap_tuple_slot, InvalidBuffer, false);
#else
ExecStoreHeapTuple(tuple, heap_tuple_slot, false);
#endif
tuplesort_puttupleslot(tuplesortstate, heap_tuple_slot);
}
}
heap_endscan(heapScan);
ExecDropSingleTupleTableSlot(heap_tuple_slot);
tuplesort_performsort(tuplesortstate);
return tuplesortstate;
}
static void
compress_chunk_populate_sort_info_for_column(Oid table, const ColumnCompressionInfo *column,
AttrNumber *att_nums, Oid *sort_operator,
Oid *collation, bool *nulls_first)
{
HeapTuple tp;
Form_pg_attribute att_tup;
TypeCacheEntry *tentry;
tp = SearchSysCacheAttName(table, NameStr(column->attname));
if (!HeapTupleIsValid(tp))
elog(ERROR, "table %d does not have column \"%s\"", table, NameStr(column->attname));
att_tup = (Form_pg_attribute) GETSTRUCT(tp);
/* Other valdation checks beyond just existence of a valid comparison operator could be useful
*/
*att_nums = att_tup->attnum;
*collation = att_tup->attcollation;
*nulls_first = (!(COMPRESSIONCOL_IS_SEGMENT_BY(column))) && column->orderby_nullsfirst;
tentry = lookup_type_cache(att_tup->atttypid, TYPECACHE_LT_OPR | TYPECACHE_GT_OPR);
if (COMPRESSIONCOL_IS_SEGMENT_BY(column) || column->orderby_asc)
*sort_operator = tentry->lt_opr;
else
*sort_operator = tentry->gt_opr;
if (!OidIsValid(*sort_operator))
elog(ERROR,
"no valid sort operator for column \"%s\" of type \"%s\"",
NameStr(column->attname),
format_type_be(att_tup->atttypid));
ReleaseSysCache(tp);
}
/********************
** row_compressor **
********************/
static void row_compressor_update_group(RowCompressor *row_compressor, TupleTableSlot *row);
static bool row_compressor_new_row_is_in_new_group(RowCompressor *row_compressor,
TupleTableSlot *row);
static void row_compressor_append_row(RowCompressor *row_compressor, TupleTableSlot *row);
static void row_compressor_flush(RowCompressor *row_compressor, CommandId mycid,
bool changed_groups);
static SegmentInfo *segment_info_new(Form_pg_attribute column_attr);
static void segment_info_update(SegmentInfo *segment_info, Datum val, bool is_null);
static bool segment_info_datum_is_in_group(SegmentInfo *segment_info, Datum datum, bool is_null);
/* num_compression_infos is the number of columns we will write to in the compressed table */
static void
row_compressor_init(RowCompressor *row_compressor, TupleDesc uncompressed_tuple_desc,
Relation compressed_table, int num_compression_infos,
const ColumnCompressionInfo **column_compression_info, int16 *in_column_offsets,
int16 num_columns_in_compressed_table)
{
TupleDesc out_desc = RelationGetDescr(compressed_table);
int col;
Name count_metadata_name = DatumGetName(
DirectFunctionCall1(namein, CStringGetDatum(COMPRESSION_COLUMN_METADATA_COUNT_NAME)));
Name sequence_num_metadata_name = DatumGetName(
DirectFunctionCall1(namein,
CStringGetDatum(COMPRESSION_COLUMN_METADATA_SEQUENCE_NUM_NAME)));
AttrNumber count_metadata_column_num =
get_attnum(compressed_table->rd_id, NameStr(*count_metadata_name));
AttrNumber sequence_num_column_num =
get_attnum(compressed_table->rd_id, NameStr(*sequence_num_metadata_name));
Oid compressed_data_type_oid = ts_custom_type_cache_get(CUSTOM_TYPE_COMPRESSED_DATA)->type_oid;
if (count_metadata_column_num == InvalidAttrNumber)
elog(ERROR,
"missing metadata column '%s' in compressed table",
COMPRESSION_COLUMN_METADATA_COUNT_NAME);
if (sequence_num_column_num == InvalidAttrNumber)
elog(ERROR,
"missing metadata column '%s' in compressed table",
COMPRESSION_COLUMN_METADATA_SEQUENCE_NUM_NAME);
*row_compressor = (RowCompressor){
.per_row_ctx = AllocSetContextCreate(CurrentMemoryContext,
"compress chunk per-row",
ALLOCSET_DEFAULT_SIZES),
.compressed_table = compressed_table,
.bistate = GetBulkInsertState(),
.n_input_columns = uncompressed_tuple_desc->natts,
.per_column = palloc0(sizeof(PerColumn) * uncompressed_tuple_desc->natts),
.uncompressed_col_to_compressed_col =
palloc0(sizeof(*row_compressor->uncompressed_col_to_compressed_col) *
uncompressed_tuple_desc->natts),
.count_metadata_column_offset = AttrNumberGetAttrOffset(count_metadata_column_num),
.sequence_num_metadata_column_offset = AttrNumberGetAttrOffset(sequence_num_column_num),
.compressed_values = palloc(sizeof(Datum) * num_columns_in_compressed_table),
.compressed_is_null = palloc(sizeof(bool) * num_columns_in_compressed_table),
.rows_compressed_into_current_value = 0,
.rowcnt_pre_compression = 0,
.num_compressed_rows = 0,
.sequence_num = SEQUENCE_NUM_GAP,
};
memset(row_compressor->compressed_is_null, 1, sizeof(bool) * num_columns_in_compressed_table);
for (col = 0; col < num_compression_infos; col++)
{
const ColumnCompressionInfo *compression_info = column_compression_info[col];
/* we want row_compressor.per_column to be in the same order as the underlying table */
int16 in_column_offset = in_column_offsets[col];
PerColumn *column = &row_compressor->per_column[in_column_offset];
Form_pg_attribute column_attr = TupleDescAttr(uncompressed_tuple_desc, in_column_offset);
AttrNumber compressed_colnum =
get_attnum(compressed_table->rd_id, NameStr(compression_info->attname));
Form_pg_attribute compressed_column_attr =
TupleDescAttr(out_desc, AttrNumberGetAttrOffset(compressed_colnum));
row_compressor->uncompressed_col_to_compressed_col[in_column_offset] =
AttrNumberGetAttrOffset(compressed_colnum);
Assert(AttrNumberGetAttrOffset(compressed_colnum) < num_columns_in_compressed_table);
if (!COMPRESSIONCOL_IS_SEGMENT_BY(compression_info))
{
int16 segment_min_attr_offset = -1;
int16 segment_max_attr_offset = -1;
SegmentMetaMinMaxBuilder *segment_min_max_builder = NULL;
if (compressed_column_attr->atttypid != compressed_data_type_oid)
elog(ERROR,
"expected column '%s' to be a compressed data type",
compression_info->attname.data);
if (compression_info->orderby_column_index > 0)
{
char *segment_min_col_name = compression_column_segment_min_name(compression_info);
char *segment_max_col_name = compression_column_segment_max_name(compression_info);
AttrNumber segment_min_attr_number =
get_attnum(compressed_table->rd_id, segment_min_col_name);
AttrNumber segment_max_attr_number =
get_attnum(compressed_table->rd_id, segment_max_col_name);
if (segment_min_attr_number == InvalidAttrNumber)
elog(ERROR, "couldn't find metadata column \"%s\"", segment_min_col_name);
if (segment_max_attr_number == InvalidAttrNumber)
elog(ERROR, "couldn't find metadata column \"%s\"", segment_max_col_name);
segment_min_attr_offset = AttrNumberGetAttrOffset(segment_min_attr_number);
segment_max_attr_offset = AttrNumberGetAttrOffset(segment_max_attr_number);
segment_min_max_builder =
segment_meta_min_max_builder_create(column_attr->atttypid,
column_attr->attcollation);
}
*column = (PerColumn){
.compressor = compressor_for_algorithm_and_type(compression_info->algo_id,
column_attr->atttypid),
.min_metadata_attr_offset = segment_min_attr_offset,
.max_metadata_attr_offset = segment_max_attr_offset,
.min_max_metadata_builder = segment_min_max_builder,
};
}
else
{
if (column_attr->atttypid != compressed_column_attr->atttypid)
elog(ERROR,
"expected segment by column \"%s\" to be same type as uncompressed column",
compression_info->attname.data);
*column = (PerColumn){
.segment_info = segment_info_new(column_attr),
.min_metadata_attr_offset = -1,
.max_metadata_attr_offset = -1,
};
}
}
}
static void
row_compressor_append_sorted_rows(RowCompressor *row_compressor, Tuplesortstate *sorted_rel,
TupleDesc sorted_desc)
{
CommandId mycid = GetCurrentCommandId(true);
TupleTableSlot *slot = MakeTupleTableSlotCompat(sorted_desc, TTSOpsMinimalTupleP);
bool got_tuple;
bool first_iteration = true;
for (got_tuple = tuplesort_gettupleslot(sorted_rel,
true /*=forward*/,
false /*=copy*/,
slot,
NULL /*=abbrev*/);
got_tuple;
got_tuple = tuplesort_gettupleslot(sorted_rel,
true /*=forward*/,
false /*=copy*/,
slot,
NULL /*=abbrev*/))
{
bool changed_groups, compressed_row_is_full;
MemoryContext old_ctx;
slot_getallattrs(slot);
old_ctx = MemoryContextSwitchTo(row_compressor->per_row_ctx);
/* first time through */
if (first_iteration)
{
row_compressor_update_group(row_compressor, slot);
first_iteration = false;
}
changed_groups = row_compressor_new_row_is_in_new_group(row_compressor, slot);
compressed_row_is_full =
row_compressor->rows_compressed_into_current_value >= MAX_ROWS_PER_COMPRESSION;
if (compressed_row_is_full || changed_groups)
{
if (row_compressor->rows_compressed_into_current_value > 0)
row_compressor_flush(row_compressor, mycid, changed_groups);
if (changed_groups)
row_compressor_update_group(row_compressor, slot);
}
row_compressor_append_row(row_compressor, slot);
MemoryContextSwitchTo(old_ctx);
ExecClearTuple(slot);
}
if (row_compressor->rows_compressed_into_current_value > 0)
row_compressor_flush(row_compressor, mycid, true);
ExecDropSingleTupleTableSlot(slot);
}
static void
row_compressor_update_group(RowCompressor *row_compressor, TupleTableSlot *row)
{
int col;
Assert(row_compressor->rows_compressed_into_current_value == 0);
Assert(row_compressor->n_input_columns <= row->tts_nvalid);
for (col = 0; col < row_compressor->n_input_columns; col++)
{
PerColumn *column = &row_compressor->per_column[col];
Datum val;
bool is_null;
if (column->segment_info == NULL)
continue;
Assert(column->compressor == NULL);
MemoryContextSwitchTo(row_compressor->per_row_ctx->parent);
/* Performance Improvment: We should just use array access here; everything is guaranteed to
be fetched */
val = slot_getattr(row, AttrOffsetGetAttrNumber(col), &is_null);
segment_info_update(column->segment_info, val, is_null);
MemoryContextSwitchTo(row_compressor->per_row_ctx);
}
}
static bool
row_compressor_new_row_is_in_new_group(RowCompressor *row_compressor, TupleTableSlot *row)
{
int col;
for (col = 0; col < row_compressor->n_input_columns; col++)
{
PerColumn *column = &row_compressor->per_column[col];
Datum datum = CharGetDatum(0);
bool is_null;
if (column->segment_info == NULL)
continue;
Assert(column->compressor == NULL);
datum = slot_getattr(row, AttrOffsetGetAttrNumber(col), &is_null);
if (!segment_info_datum_is_in_group(column->segment_info, datum, is_null))
return true;
}
return false;
}
static void
row_compressor_append_row(RowCompressor *row_compressor, TupleTableSlot *row)
{
int col;
for (col = 0; col < row_compressor->n_input_columns; col++)
{
Compressor *compressor = row_compressor->per_column[col].compressor;
bool is_null;
Datum val;
/* if there is no compressor, this must be a segmenter, so just skip */
if (compressor == NULL)
continue;
/* Performance Improvement: Since we call getallatts at the beginning, slot_getattr is
* useless overhead here, and we should just access the array directly.
*/
val = slot_getattr(row, AttrOffsetGetAttrNumber(col), &is_null);
if (is_null)
{
compressor->append_null(compressor);
if (row_compressor->per_column[col].min_max_metadata_builder != NULL)
segment_meta_min_max_builder_update_null(
row_compressor->per_column[col].min_max_metadata_builder);
}
else
{
compressor->append_val(compressor, val);
if (row_compressor->per_column[col].min_max_metadata_builder != NULL)
segment_meta_min_max_builder_update_val(row_compressor->per_column[col]
.min_max_metadata_builder,
val);
}
}
row_compressor->rows_compressed_into_current_value += 1;
}
static void
row_compressor_flush(RowCompressor *row_compressor, CommandId mycid, bool changed_groups)
{
int16 col;
HeapTuple compressed_tuple;
for (col = 0; col < row_compressor->n_input_columns; col++)
{
PerColumn *column = &row_compressor->per_column[col];
Compressor *compressor;
int16 compressed_col;
if (column->compressor == NULL && column->segment_info == NULL)
continue;
compressor = column->compressor;
compressed_col = row_compressor->uncompressed_col_to_compressed_col[col];
Assert(compressed_col >= 0);
if (compressor != NULL)
{
void *compressed_data;
Assert(column->segment_info == NULL);
compressed_data = compressor->finish(compressor);
/* non-segment columns are NULL iff all the values are NULL */
row_compressor->compressed_is_null[compressed_col] = compressed_data == NULL;
if (compressed_data != NULL)
row_compressor->compressed_values[compressed_col] =
PointerGetDatum(compressed_data);
if (column->min_max_metadata_builder != NULL)
{
Assert(column->min_metadata_attr_offset >= 0);
Assert(column->max_metadata_attr_offset >= 0);
if (!segment_meta_min_max_builder_empty(column->min_max_metadata_builder))
{
Assert(compressed_data != NULL);
row_compressor->compressed_is_null[column->min_metadata_attr_offset] = false;
row_compressor->compressed_is_null[column->max_metadata_attr_offset] = false;
row_compressor->compressed_values[column->min_metadata_attr_offset] =
segment_meta_min_max_builder_min(column->min_max_metadata_builder);
row_compressor->compressed_values[column->max_metadata_attr_offset] =
segment_meta_min_max_builder_max(column->min_max_metadata_builder);
}
else
{
Assert(compressed_data == NULL);
row_compressor->compressed_is_null[column->min_metadata_attr_offset] = true;
row_compressor->compressed_is_null[column->max_metadata_attr_offset] = true;
}
}
}
else if (column->segment_info != NULL)
{
row_compressor->compressed_values[compressed_col] = column->segment_info->val;
row_compressor->compressed_is_null[compressed_col] = column->segment_info->is_null;
}
}
row_compressor->compressed_values[row_compressor->count_metadata_column_offset] =
Int32GetDatum(row_compressor->rows_compressed_into_current_value);
row_compressor->compressed_is_null[row_compressor->count_metadata_column_offset] = false;
row_compressor->compressed_values[row_compressor->sequence_num_metadata_column_offset] =
Int32GetDatum(row_compressor->sequence_num);
row_compressor->compressed_is_null[row_compressor->sequence_num_metadata_column_offset] = false;
/* overflow could happen only if chunk has more than 200B rows */
if (row_compressor->sequence_num > PG_INT32_MAX - SEQUENCE_NUM_GAP)
elog(ERROR, "sequence id overflow");
row_compressor->sequence_num += SEQUENCE_NUM_GAP;
compressed_tuple = heap_form_tuple(RelationGetDescr(row_compressor->compressed_table),
row_compressor->compressed_values,
row_compressor->compressed_is_null);
heap_insert(row_compressor->compressed_table,
compressed_tuple,
mycid,
0 /*=options*/,
row_compressor->bistate);
/* free the compressed values now that we're done with them (the old compressor is freed in
* finish()) */
for (col = 0; col < row_compressor->n_input_columns; col++)
{
PerColumn *column = &row_compressor->per_column[col];
int16 compressed_col;
if (column->compressor == NULL && column->segment_info == NULL)
continue;
compressed_col = row_compressor->uncompressed_col_to_compressed_col[col];
Assert(compressed_col >= 0);
if (row_compressor->compressed_is_null[compressed_col])
continue;
/* don't free the segment-bys if we've overflowed the row, we still need them */
if (column->segment_info != NULL && !changed_groups)
continue;
if (column->compressor != NULL || !column->segment_info->typ_by_val)
pfree(DatumGetPointer(row_compressor->compressed_values[compressed_col]));
if (column->min_max_metadata_builder != NULL)
{
/* segment_meta_min_max_builder_reset will free the values, so clear here */
if (!row_compressor->compressed_is_null[column->min_metadata_attr_offset])
{
row_compressor->compressed_values[column->min_metadata_attr_offset] = 0;
row_compressor->compressed_is_null[column->min_metadata_attr_offset] = true;
}
if (!row_compressor->compressed_is_null[column->max_metadata_attr_offset])
{
row_compressor->compressed_values[column->max_metadata_attr_offset] = 0;
row_compressor->compressed_is_null[column->max_metadata_attr_offset] = true;
}
segment_meta_min_max_builder_reset(column->min_max_metadata_builder);
}
row_compressor->compressed_values[compressed_col] = 0;
row_compressor->compressed_is_null[compressed_col] = true;
}
row_compressor->rowcnt_pre_compression += row_compressor->rows_compressed_into_current_value;
row_compressor->num_compressed_rows++;
row_compressor->rows_compressed_into_current_value = 0;
MemoryContextReset(row_compressor->per_row_ctx);
}
static void
row_compressor_finish(RowCompressor *row_compressor)
{
FreeBulkInsertState(row_compressor->bistate);
}
/******************
** segment_info **
******************/
static SegmentInfo *
segment_info_new(Form_pg_attribute column_attr)
{
Oid eq_fn_oid =
lookup_type_cache(column_attr->atttypid, TYPECACHE_EQ_OPR_FINFO)->eq_opr_finfo.fn_oid;
SegmentInfo *segment_info = palloc(sizeof(*segment_info));
*segment_info = (SegmentInfo){
.typlen = column_attr->attlen,
.typ_by_val = column_attr->attbyval,
};
if (!OidIsValid(eq_fn_oid))
elog(ERROR, "no equality function for column \"%s\"", NameStr(column_attr->attname));
fmgr_info_cxt(eq_fn_oid, &segment_info->eq_fn, CurrentMemoryContext);
segment_info->eq_fcinfo = HEAP_FCINFO(2);
InitFunctionCallInfoData(*segment_info->eq_fcinfo,
&segment_info->eq_fn /*=Flinfo*/,
2 /*=Nargs*/,
column_attr->attcollation /*=Collation*/,
NULL, /*=Context*/
NULL /*=ResultInfo*/
);
return segment_info;
}
static void
segment_info_update(SegmentInfo *segment_info, Datum val, bool is_null)
{
segment_info->is_null = is_null;
if (is_null)
segment_info->val = 0;
else
segment_info->val = datumCopy(val, segment_info->typ_by_val, segment_info->typlen);
}
static bool
segment_info_datum_is_in_group(SegmentInfo *segment_info, Datum datum, bool is_null)
{
Datum data_is_eq;
FunctionCallInfo eq_fcinfo;
/* if one of the datums is null and the other isn't, we must be in a new group */
if (segment_info->is_null != is_null)
return false;
/* they're both null */
if (segment_info->is_null)
return true;
/* neither is null, call the eq function */
eq_fcinfo = segment_info->eq_fcinfo;
FC_SET_ARG(eq_fcinfo, 0, segment_info->val);
FC_SET_ARG(eq_fcinfo, 1, datum);
data_is_eq = FunctionCallInvoke(eq_fcinfo);
if (eq_fcinfo->isnull)
return false;
return DatumGetBool(data_is_eq);
}
/**********************
** decompress_chunk **
**********************/
typedef struct PerCompressedColumn
{
Oid decompressed_type;
/* the compressor to use for compressed columns, always NULL for segmenters
* only use if is_compressed
*/
DecompressionIterator *iterator;
/* segment info; only used if !is_compressed */
Datum val;
/* is this a compressed column or a segment-by column */
bool is_compressed;
/* the value stored in the compressed table was NULL */
bool is_null;
/* the index in the decompressed table of the data -1,
* if the data is metadata not found in the decompressed table
*/
int16 decompressed_column_offset;
} PerCompressedColumn;
typedef struct RowDecompressor
{
PerCompressedColumn *per_compressed_cols;
int16 num_compressed_columns;
TupleDesc out_desc;
Relation out_rel;
CommandId mycid;
BulkInsertState bistate;
/* cache memory used to store the decompressed datums/is_null for form_tuple */
Datum *decompressed_datums;
bool *decompressed_is_nulls;
} RowDecompressor;
static PerCompressedColumn *create_per_compressed_column(TupleDesc in_desc, TupleDesc out_desc,
Oid out_relid,
Oid compressed_data_type_oid);
static void populate_per_compressed_columns_from_data(PerCompressedColumn *per_compressed_cols,
int16 num_cols, Datum *compressed_datums,
bool *compressed_is_nulls);
static void row_decompressor_decompress_row(RowDecompressor *row_decompressor);
static bool per_compressed_col_get_data(PerCompressedColumn *per_compressed_col,
Datum *decompressed_datums, bool *decompressed_is_nulls);
void
decompress_chunk(Oid in_table, Oid out_table)
{
/* these locks are taken in the order uncompressed table then compressed table
* for consistency with compress_chunk
*/
/* we are _just_ INSERTing into the out_table so in principle we could take
* a RowExclusive lock, and let other operations read and write this table
* as we work. However, we currently compress each table as a oneshot, so
* we're taking the stricter lock to prevent accidents.
*/
Relation out_rel = table_open(out_table, ExclusiveLock);
/*We want to prevent other decompressors from decompressing this table,
* and we want to prevent INSERTs or UPDATEs which could mess up our decompression.
* We may as well allow readers to keep reading the compressed data while
* we are compressing, so we only take an ExclusiveLock instead of AccessExclusive.
*/
Relation in_rel = relation_open(in_table, ExclusiveLock);
TupleDesc in_desc = RelationGetDescr(in_rel);
TupleDesc out_desc = RelationGetDescr(out_rel);
Oid compressed_data_type_oid = ts_custom_type_cache_get(CUSTOM_TYPE_COMPRESSED_DATA)->type_oid;
Assert(OidIsValid(compressed_data_type_oid));
{
RowDecompressor decompressor = {
.per_compressed_cols = create_per_compressed_column(in_desc,
out_desc,
out_table,
compressed_data_type_oid),
.num_compressed_columns = in_desc->natts,
.out_desc = out_desc,
.out_rel = out_rel,
.mycid = GetCurrentCommandId(true),
.bistate = GetBulkInsertState(),
/* cache memory used to store the decompressed datums/is_null for form_tuple */
.decompressed_datums = palloc(sizeof(Datum) * out_desc->natts),
.decompressed_is_nulls = palloc(sizeof(bool) * out_desc->natts),
};
/*
* We need to make sure decompressed_is_nulls is in a defined state. While this
* will get written for normal columns it will not get written for dropped columns
* since dropped columns don't exist in the compressed chunk so we initiallize
* with true here.
*/
memset(decompressor.decompressed_is_nulls, true, out_desc->natts);
Datum *compressed_datums = palloc(sizeof(*compressed_datums) * in_desc->natts);
bool *compressed_is_nulls = palloc(sizeof(*compressed_is_nulls) * in_desc->natts);
HeapTuple compressed_tuple;
TableScanDesc heapScan = table_beginscan(in_rel, GetLatestSnapshot(), 0, (ScanKey) NULL);
MemoryContext per_compressed_row_ctx =
AllocSetContextCreate(CurrentMemoryContext,
"decompress chunk per-compressed row",
ALLOCSET_DEFAULT_SIZES);
for (compressed_tuple = heap_getnext(heapScan, ForwardScanDirection);
compressed_tuple != NULL;
compressed_tuple = heap_getnext(heapScan, ForwardScanDirection))
{
MemoryContext old_ctx;
Assert(HeapTupleIsValid(compressed_tuple));
old_ctx = MemoryContextSwitchTo(per_compressed_row_ctx);
heap_deform_tuple(compressed_tuple, in_desc, compressed_datums, compressed_is_nulls);
populate_per_compressed_columns_from_data(decompressor.per_compressed_cols,
in_desc->natts,
compressed_datums,
compressed_is_nulls);
row_decompressor_decompress_row(&decompressor);
MemoryContextSwitchTo(old_ctx);
MemoryContextReset(per_compressed_row_ctx);
}
heap_endscan(heapScan);
FreeBulkInsertState(decompressor.bistate);
}
/* Recreate all indexes on out rel, we already have an exclusive lock on it,
* so the strong locks taken by reindex_relation shouldn't matter. */
reindex_relation(out_table, 0, 0);
table_close(out_rel, NoLock);
table_close(in_rel, NoLock);
}
static PerCompressedColumn *
create_per_compressed_column(TupleDesc in_desc, TupleDesc out_desc, Oid out_relid,
Oid compressed_data_type_oid)
{
PerCompressedColumn *per_compressed_cols =
palloc(sizeof(*per_compressed_cols) * in_desc->natts);
Assert(OidIsValid(compressed_data_type_oid));
for (int16 col = 0; col < in_desc->natts; col++)
{
Oid decompressed_type;
bool is_compressed;
int16 decompressed_column_offset;
PerCompressedColumn *per_compressed_col = &per_compressed_cols[col];
Form_pg_attribute compressed_attr = TupleDescAttr(in_desc, col);
char *col_name = NameStr(compressed_attr->attname);
/* find the mapping from compressed column to uncompressed column, setting
* the index of columns that don't have an uncompressed version
* (such as metadata) to -1
* Assumption: column names are the same on compressed and
* uncompressed chunk.
*/
AttrNumber decompressed_colnum = get_attnum(out_relid, col_name);
if (!AttributeNumberIsValid(decompressed_colnum))
{
*per_compressed_col = (PerCompressedColumn){
.decompressed_column_offset = -1,
.is_null = true,
};
continue;
}
decompressed_column_offset = AttrNumberGetAttrOffset(decompressed_colnum);
decompressed_type = TupleDescAttr(out_desc, decompressed_column_offset)->atttypid;
/* determine if the data is compressed or not */
is_compressed = compressed_attr->atttypid == compressed_data_type_oid;
if (!is_compressed && compressed_attr->atttypid != decompressed_type)
elog(ERROR,
"compressed table type '%s' does not match decompressed table type '%s' for "
"segment-by column \"%s\"",
format_type_be(compressed_attr->atttypid),
format_type_be(decompressed_type),
col_name);
*per_compressed_col = (PerCompressedColumn){
.decompressed_column_offset = decompressed_column_offset,
.is_null = true,
.is_compressed = is_compressed,
.decompressed_type = decompressed_type,
};
}
return per_compressed_cols;
}
static void
populate_per_compressed_columns_from_data(PerCompressedColumn *per_compressed_cols, int16 num_cols,
Datum *compressed_datums, bool *compressed_is_nulls)
{
for (int16 col = 0; col < num_cols; col++)
{
PerCompressedColumn *per_col = &per_compressed_cols[col];
if (per_col->decompressed_column_offset < 0)
continue;
per_col->is_null = compressed_is_nulls[col];
if (per_col->is_null)
{
per_col->is_null = true;
per_col->iterator = NULL;
per_col->val = 0;
continue;
}
if (per_col->is_compressed)
{
char *data = (char *) PG_DETOAST_DATUM(compressed_datums[col]);
CompressedDataHeader *header = (CompressedDataHeader *) data;
per_col->iterator =
definitions[header->compression_algorithm]
.iterator_init_forward(PointerGetDatum(data), per_col->decompressed_type);
}
else
per_col->val = compressed_datums[col];
}
}
static void
row_decompressor_decompress_row(RowDecompressor *row_decompressor)
{
/* each compressed row decompresses to at least one row,
* even if all the data is NULL
*/
bool wrote_data = false;
bool is_done = false;
do
{
/* we're done if all the decompressors return NULL */
is_done = true;
for (int16 col = 0; col < row_decompressor->num_compressed_columns; col++)
{
bool col_is_done =
per_compressed_col_get_data(&row_decompressor->per_compressed_cols[col],
row_decompressor->decompressed_datums,
row_decompressor->decompressed_is_nulls);
is_done &= col_is_done;
}
/* if we're not done we have data to write. even if we're done, each
* compressed should decompress to at least one row, so we should write that
*/
if (!is_done || !wrote_data)
{
HeapTuple decompressed_tuple = heap_form_tuple(row_decompressor->out_desc,
row_decompressor->decompressed_datums,
row_decompressor->decompressed_is_nulls);
heap_insert(row_decompressor->out_rel,
decompressed_tuple,
row_decompressor->mycid,
0 /*=options*/,
row_decompressor->bistate);
heap_freetuple(decompressed_tuple);
wrote_data = true;
}
} while (!is_done);
}
/* populate the relevent index in an array from a per_compressed_col.
* returns if decompression is done for this column
*/
bool
per_compressed_col_get_data(PerCompressedColumn *per_compressed_col, Datum *decompressed_datums,
bool *decompressed_is_nulls)
{
DecompressResult decompressed;
int16 decompressed_column_offset = per_compressed_col->decompressed_column_offset;
/* skip metadata columns */
if (decompressed_column_offset < 0)
return true;
/* segment-bys */
if (!per_compressed_col->is_compressed)
{
decompressed_datums[decompressed_column_offset] = per_compressed_col->val;
decompressed_is_nulls[decompressed_column_offset] = per_compressed_col->is_null;
return true;
}
/* compressed NULL */
if (per_compressed_col->is_null)
{
decompressed_is_nulls[decompressed_column_offset] = true;
return true;
}
/* other compressed data */
if (per_compressed_col->iterator == NULL)
elog(ERROR, "tried to decompress more data than was compressed in column");
decompressed = per_compressed_col->iterator->try_next(per_compressed_col->iterator);
if (decompressed.is_done)
{
/* We want a way to free the decompression iterator's data to avoid OOM issues */
per_compressed_col->iterator = NULL;
decompressed_is_nulls[decompressed_column_offset] = true;
return true;
}
decompressed_is_nulls[decompressed_column_offset] = decompressed.is_null;
if (decompressed.is_null)
decompressed_datums[decompressed_column_offset] = 0;
else
decompressed_datums[decompressed_column_offset] = decompressed.val;
return false;
}
/********************/
/*** SQL Bindings ***/
/********************/
Datum
tsl_compressed_data_decompress_forward(PG_FUNCTION_ARGS)
{
Datum compressed;
CompressedDataHeader *header;
FuncCallContext *funcctx;
MemoryContext oldcontext;
DecompressionIterator *iter;
DecompressResult res;
if (PG_ARGISNULL(0))
PG_RETURN_NULL();
compressed = PG_GETARG_DATUM(0);
header = (CompressedDataHeader *) PG_DETOAST_DATUM(compressed);
if (SRF_IS_FIRSTCALL())
{
funcctx = SRF_FIRSTCALL_INIT();
oldcontext = MemoryContextSwitchTo(funcctx->multi_call_memory_ctx);
iter =
definitions[header->compression_algorithm]
.iterator_init_forward(PG_GETARG_DATUM(0), get_fn_expr_argtype(fcinfo->flinfo, 1));
funcctx->user_fctx = iter;
MemoryContextSwitchTo(oldcontext);
}
funcctx = SRF_PERCALL_SETUP();
iter = funcctx->user_fctx;
res = iter->try_next(iter);
if (res.is_done)
SRF_RETURN_DONE(funcctx);
if (res.is_null)
SRF_RETURN_NEXT_NULL(funcctx);
SRF_RETURN_NEXT(funcctx, res.val);
}
Datum
tsl_compressed_data_decompress_reverse(PG_FUNCTION_ARGS)
{
Datum compressed;
CompressedDataHeader *header;
FuncCallContext *funcctx;
MemoryContext oldcontext;
DecompressionIterator *iter;
DecompressResult res;
if (PG_ARGISNULL(0))
PG_RETURN_NULL();
compressed = PG_GETARG_DATUM(0);
header = (CompressedDataHeader *) PG_DETOAST_DATUM(compressed);
if (SRF_IS_FIRSTCALL())
{
funcctx = SRF_FIRSTCALL_INIT();
oldcontext = MemoryContextSwitchTo(funcctx->multi_call_memory_ctx);
iter =
definitions[header->compression_algorithm]
.iterator_init_reverse(PG_GETARG_DATUM(0), get_fn_expr_argtype(fcinfo->flinfo, 1));
funcctx->user_fctx = iter;
MemoryContextSwitchTo(oldcontext);
}
funcctx = SRF_PERCALL_SETUP();
iter = funcctx->user_fctx;
res = iter->try_next(iter);
if (res.is_done)
SRF_RETURN_DONE(funcctx);
if (res.is_null)
SRF_RETURN_NEXT_NULL(funcctx);
SRF_RETURN_NEXT(funcctx, res.val);
;
}
Datum
tsl_compressed_data_send(PG_FUNCTION_ARGS)
{
CompressedDataHeader *header = (CompressedDataHeader *) PG_DETOAST_DATUM(PG_GETARG_DATUM(0));
StringInfoData buf;
pq_begintypsend(&buf);
pq_sendbyte(&buf, header->compression_algorithm);
definitions[header->compression_algorithm].compressed_data_send(header, &buf);
PG_RETURN_BYTEA_P(pq_endtypsend(&buf));
}
Datum
tsl_compressed_data_recv(PG_FUNCTION_ARGS)
{
StringInfo buf = (StringInfo) PG_GETARG_POINTER(0);
CompressedDataHeader header = { { 0 } };
header.compression_algorithm = pq_getmsgbyte(buf);
return definitions[header.compression_algorithm].compressed_data_recv(buf);
}
extern Datum
tsl_compressed_data_in(PG_FUNCTION_ARGS)
{
const char *input = PG_GETARG_CSTRING(0);
size_t input_len = strlen(input);
int decoded_len;
char *decoded;
StringInfoData data;
Datum result;
if (input_len > PG_INT32_MAX)
elog(ERROR, "input too long");
decoded_len = pg_b64_dec_len(input_len);
decoded = palloc(decoded_len + 1);
decoded_len = pg_b64_decode_compat(input, input_len, decoded, decoded_len);
if (decoded_len < 0)
elog(ERROR, "could not decode base64-encoded compressed data");
decoded[decoded_len] = '\0';
data = (StringInfoData){
.data = decoded,
.len = decoded_len,
.maxlen = decoded_len,
};
result = DirectFunctionCall1(tsl_compressed_data_recv, PointerGetDatum(&data));
PG_RETURN_DATUM(result);
}
extern Datum
tsl_compressed_data_out(PG_FUNCTION_ARGS)
{
Datum bytes_data = DirectFunctionCall1(tsl_compressed_data_send, PG_GETARG_DATUM(0));
bytea *bytes = DatumGetByteaP(bytes_data);
int raw_len = VARSIZE_ANY_EXHDR(bytes);
const char *raw_data = VARDATA(bytes);
int encoded_len = pg_b64_enc_len(raw_len);
char *encoded = palloc(encoded_len + 1);
encoded_len = pg_b64_encode_compat(raw_data, raw_len, encoded, encoded_len);
if (encoded_len < 0)
elog(ERROR, "could not base64-encode compressed data");
encoded[encoded_len] = '\0';
PG_RETURN_CSTRING(encoded);
}
extern CompressionStorage
compression_get_toast_storage(CompressionAlgorithms algorithm)
{
if (algorithm == _INVALID_COMPRESSION_ALGORITHM || algorithm >= _END_COMPRESSION_ALGORITHMS)
elog(ERROR, "invalid compression algorithm %d", algorithm);
return definitions[algorithm].compressed_data_storage;
}
/* Get relstats from compressed chunk and insert into relstats for the
* corresponding chunk (that held the uncompressed data) from raw hypertable
*/
extern void
update_compressed_chunk_relstats(Oid uncompressed_relid, Oid compressed_relid)
{
double rowcnt;
int comp_pages, uncomp_pages, comp_visible, uncomp_visible;
float comp_tuples, uncomp_tuples, out_tuples;
Chunk *uncompressed_chunk = ts_chunk_get_by_relid(uncompressed_relid, true);
Chunk *compressed_chunk = ts_chunk_get_by_relid(compressed_relid, true);
if (uncompressed_chunk->table_id != uncompressed_relid ||
uncompressed_chunk->fd.compressed_chunk_id != compressed_chunk->fd.id ||
compressed_chunk->table_id != compressed_relid)
{
ereport(ERROR,
(errcode(ERRCODE_INTERNAL_ERROR),
errmsg("mismatched chunks for relstats update %d %d",
uncompressed_relid,
compressed_relid)));
}
capture_pgclass_stats(uncompressed_relid, &uncomp_pages, &uncomp_visible, &uncomp_tuples);
/* Before compressing a chunk in 2.0, we save its stats. Prior
* releases do not support this. So the stats on uncompressed relid
* could be invalid. In this case, do the best that we can.
*/
if (uncomp_tuples == 0)
{
/* we need page info from compressed relid */
capture_pgclass_stats(compressed_relid, &comp_pages, &comp_visible, &comp_tuples);
rowcnt = (double) ts_compression_chunk_size_row_count(uncompressed_chunk->fd.id);
if (rowcnt > 0)
out_tuples = (float4) rowcnt;
else
out_tuples = (float4) comp_tuples;
restore_pgclass_stats(uncompressed_relid, comp_pages, comp_visible, out_tuples);
CommandCounterIncrement();
}
}