postgres/timescaledb
1
0
Fork 0
mirror of https://github.com/timescale/timescaledb.git synced 2025-05-14 01:10:37 +08:00
Code Issues Packages Projects Releases Wiki Activity

Avoid excessive reallocation in row compressors (#6638)

Browse Source 
Memory operations can add up to tens of percents of the total
compression CPU load. To reduce the need for them, reserve for the
expected array sizes when initializing the compressor.
This commit is contained in:
Alexander Kuzmenkov 2024-02-14 17:02:16 +01:00 committed by GitHub
parent 87430168b5
commit e44e0ad10e
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
6 changed files with 35 additions and 10 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
src/adts/bit_array.h
View File

@ -25,7 +25,7 @@ typedef struct BitArray BitArray;
typedef struct BitArrayIterator BitArrayIterator;
/* Main Interface */
static void bit_array_init(BitArray *array);
static void bit_array_init(BitArray *array, int expected_bits);
/* Append num_bits to the array */
static void bit_array_append(BitArray *array, uint8 num_bits, uint64 bits);

4
src/adts/bit_array_impl.h
View File

@ -45,12 +45,12 @@ static inline void bit_array_wrap_internal(BitArray *array, uint32 num_buckets,
************************/
static inline void
bit_array_init(BitArray *array)
bit_array_init(BitArray *array, int expected_bits)
{
*array = (BitArray){
.bits_used_in_last_bucket = 0,
};
uint64_vec_init(&array->buckets, CurrentMemoryContext, 0);
uint64_vec_init(&array->buckets, CurrentMemoryContext, expected_bits / 64);
}
/* This initializes the bit array by wrapping buckets. Note, that the bit array will

7
src/adts/vec.h
View File

@ -110,8 +110,11 @@ VEC_RESERVE(VEC_TYPE *vec, uint32 additional)
if (num_new_elements == 0 || vec->num_elements + num_new_elements <= vec->max_elements)
return;
if (num_new_elements < vec->num_elements / 2)
num_new_elements = vec->num_elements / 2;
if (num_new_elements < vec->num_elements)
{
/* Follow the usual doubling progression of allocation sizes. */
num_new_elements = vec->num_elements;
}
num_elements = vec->num_elements + num_new_elements;
Assert(num_elements > vec->num_elements);

2
test/src/adt_tests.c
View File

@ -105,7 +105,7 @@ bit_array_test(void)
BitArray bits;
BitArrayIterator iter;
int i;
bit_array_init(&bits);
bit_array_init(&bits, 0);
for (i = 0; i < 65; i++)
bit_array_append(&bits, i, i);

14
tsl/src/compression/gorilla.c
View File

@ -254,9 +254,19 @@ gorilla_compressor_alloc(void)
GorillaCompressor *compressor = palloc(sizeof(*compressor));
simple8brle_compressor_init(&compressor->tag0s);
simple8brle_compressor_init(&compressor->tag1s);
bit_array_init(&compressor->leading_zeros);
/*
* The number of leading zeros takes about 5 bits to encode, and changes
* maybe every 100 rows, so use this as a conservative estimate.
*/
bit_array_init(&compressor->leading_zeros,
/* expected_bits = */ (GLOBAL_MAX_ROWS_PER_COMPRESSION * 5) / 100);
simple8brle_compressor_init(&compressor->bits_used_per_xor);
bit_array_init(&compressor->xors);
/*
* We typically see about 12 bits or 4 decimal digits per row for the "xors"
* part in gorilla compression.
*/
bit_array_init(&compressor->xors,
/* expected_bits = */ GLOBAL_MAX_ROWS_PER_COMPRESSION * 12);
simple8brle_compressor_init(&compressor->nulls);
compressor->has_nulls = false;
compressor->prev_leading_zeroes = 0;

16
tsl/src/compression/simple8b_rle.h
View File

@ -304,8 +304,20 @@ simple8brle_compressor_init(Simple8bRleCompressor *compressor)
.num_elements = 0,
.num_uncompressed_elements = 0,
};
uint64_vec_init(&compressor->compressed_data, CurrentMemoryContext, 0);
bit_array_init(&compressor->selectors);
/*
* It is good to have some estimate of the resulting size of compressed
* data, because it helps to allocate memory in advance to avoid frequent
* reallocations. Here we use a completely arbitrary but pretty realistic
* ratio of 10.
*/
const int expected_compression_ratio = 10;
uint64_vec_init(&compressor->compressed_data,
CurrentMemoryContext,
GLOBAL_MAX_ROWS_PER_COMPRESSION / expected_compression_ratio);
bit_array_init(&compressor->selectors,
/* expected_bits = */ (GLOBAL_MAX_ROWS_PER_COMPRESSION *
SIMPLE8B_BITS_PER_SELECTOR) /
expected_compression_ratio);
}
static void