From 2e7b32cd9156a43741ec39983171a77c70aa7f21 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Erik=20Nordstr=C3=B6m?= Date: Mon, 6 Aug 2018 21:09:29 +0200 Subject: [PATCH] Add WARNING when doing min-max heap scan for adaptive chunking Adaptive chunking uses the min and max value of previous chunks to estimate their "fill factor". Ideally, min and max should be retreived using an index, but if no index exists we fall back to a heap scan. A heap scan can be very expensive, so we now raise a WARNING if no index exists. This change also renames set_adaptive_chunk_sizing() to simply set_adaptive_chunking(). --- sql/ddl_api.sql | 4 +- src/chunk.c | 41 +++++- src/chunk_adaptive.c | 227 +++++++++++++++++++++++-------- src/chunk_adaptive.h | 7 +- src/hypertable.c | 10 +- test/expected/chunk_adaptive.out | 60 ++++++-- test/expected/extension.out | 2 +- test/expected/pg_dump.out | 3 +- test/sql/chunk_adaptive.sql | 26 ++-- test/sql/pg_dump.sql | 2 +- 10 files changed, 288 insertions(+), 94 deletions(-) diff --git a/sql/ddl_api.sql b/sql/ddl_api.sql index 850d3ced7..0e03c219a 100644 --- a/sql/ddl_api.sql +++ b/sql/ddl_api.sql @@ -32,12 +32,12 @@ CREATE OR REPLACE FUNCTION create_hypertable( ) RETURNS VOID AS '@MODULE_PATHNAME@', 'hypertable_create' LANGUAGE C VOLATILE; -- Set adaptive chunking. To disable, set chunk_target_size => 'off'. -CREATE OR REPLACE FUNCTION set_adaptive_chunk_sizing( +CREATE OR REPLACE FUNCTION set_adaptive_chunking( hypertable REGCLASS, chunk_target_size TEXT, INOUT chunk_sizing_func REGPROC = '_timescaledb_internal.calculate_chunk_interval'::regproc, OUT chunk_target_size BIGINT -) RETURNS RECORD AS '@MODULE_PATHNAME@', 'chunk_adaptive_set_chunk_sizing' LANGUAGE C VOLATILE; +) RETURNS RECORD AS '@MODULE_PATHNAME@', 'chunk_adaptive_set' LANGUAGE C VOLATILE; -- Update chunk_time_interval for a hypertable. -- diff --git a/src/chunk.c b/src/chunk.c index f3ad61bd0..de72df711 100644 --- a/src/chunk.c +++ b/src/chunk.c @@ -1055,15 +1055,33 @@ chunk_scan_internal(int indexid, return scanner_scan(&ctx); } -#define DEFAULT_CHUNKS_PER_INTERVAL 4 - /* * Get a window of chunks that "preceed" the given dimensional point. * * For instance, if the dimension is "time", then given a point in time the * function returns the recent chunks that come before the chunk that includes * that point. The count parameter determines the number or slices the window - * should include in the given dimension. + * should include in the given dimension. Note, that with multi-dimensional + * partitioning, there might be multiple chunks in each dimensional slice that + * all preceed the given point. For instance, the example below shows two + * different situations that each go "back" two slices (count = 2) in the + * x-dimension, but returns two vs. eight chunks due to different + * partitioning. + * + * '_____________ + * '| | | * | + * '|___|___|___| + * ' + * ' + * '____ ________ + * '| | | * | + * '|___|___|___| + * '| | | | + * '|___|___|___| + * '| | | | + * '|___|___|___| + * '| | | | + * '|___|___|___| * * Note that the returned chunks will be allocated on the given memory * context, inlcuding the list itself. So, beware of not leaking the list if @@ -1073,17 +1091,30 @@ List * chunk_get_window(int32 dimension_id, int64 point, int count, MemoryContext mctx) { List *chunks = NIL; - DimensionVec *dimvec = dimension_slice_scan_by_dimension_before_point(dimension_id, point, count, BackwardScanDirection, mctx); + DimensionVec *dimvec; int i; + /* Scan for "count" slices that preceeds the point in the given dimension */ + dimvec = dimension_slice_scan_by_dimension_before_point(dimension_id, + point, + count, + BackwardScanDirection, + mctx); + + /* + * For each slice, join with any constraints that reference the slice. + * There might be multiple constraints for each slice in case of + * multi-dimensional partitioning. + */ for (i = 0; i < dimvec->num_slices; i++) { DimensionSlice *slice = dimvec->slices[i]; - ChunkConstraints *ccs = chunk_constraints_alloc(DEFAULT_CHUNKS_PER_INTERVAL, mctx); + ChunkConstraints *ccs = chunk_constraints_alloc(1, mctx); int j; chunk_constraint_scan_by_dimension_slice_id(slice->fd.id, ccs, mctx); + /* For each constraint, find the corresponding chunk */ for (j = 0; j < ccs->num_constraints; j++) { ChunkConstraint *cc = &ccs->constraints[j]; diff --git a/src/chunk_adaptive.c b/src/chunk_adaptive.c index 4db3177b2..45c6f2b17 100644 --- a/src/chunk_adaptive.c +++ b/src/chunk_adaptive.c @@ -105,7 +105,9 @@ set_effective_memory_cache_size(PG_FUNCTION_ARGS) * of the system, while a common recommended setting for shared_buffers is 1/4 * of system memory. In case shared_buffers is set higher than * effective_cache_size, we use the max of the two (a larger shared_buffers is a - * strange setting though). Ultimately we are limited by system memory. + * strange setting though). Ultimately we are limited by system memory. Thus, + * this functions returns a value effective_memory_cache which is: + * shared_buffers >= effective_memory_cache <= system_mem / 2. * * Note that this relies on the user setting a good value for * effective_cache_size, or otherwise our estimate will be off. Alternatively, @@ -160,9 +162,9 @@ estimate_effective_memory_cache_size(void) return memory_bytes; } -/* The default concurrency factor, i.e., the number of chunks we expect to fit - * in memory at the same time */ -#define DEFAULT_CONCURRENT_CHUNK_USAGE 4 +/* The default the number of chunks we expect to be able to have in cache + * memory at the same time */ +#define DEFAULT_NUM_CHUNKS_TO_FIT_IN_CACHE_MEM 4 static inline int64 calculate_initial_chunk_target_size(void) @@ -175,23 +177,28 @@ calculate_initial_chunk_target_size(void) * hypertables in all schemas and databases, and might not be a good * estimate in case of many "old" (unused) hypertables. */ - return estimate_effective_memory_cache_size() / DEFAULT_CONCURRENT_CHUNK_USAGE; + return estimate_effective_memory_cache_size() / DEFAULT_NUM_CHUNKS_TO_FIT_IN_CACHE_MEM; } +typedef enum MinMaxResult +{ + MINMAX_NO_INDEX, + MINMAX_NO_TUPLES, + MINMAX_FOUND, +} MinMaxResult; + /* * Use a heap scan to find the min and max of a given column of a chunk. This * could be a rather costly operation. Should figure out how to keep min-max * stats cached. - * - * Returns true iff min and max is found. */ -static bool +static MinMaxResult minmax_heapscan(Relation rel, Oid atttype, AttrNumber attnum, Datum minmax[2]) { HeapScanDesc scan; HeapTuple tuple; TypeCacheEntry *tce; - bool minmaxnull[2] = {true}; + bool nulls[2] = {true}; /* Lookup the tuple comparison function from the type cache */ tce = lookup_type_cache(atttype, TYPECACHE_CMP_PROC | TYPECACHE_CMP_PROC_FINFO); @@ -206,53 +213,106 @@ minmax_heapscan(Relation rel, Oid atttype, AttrNumber attnum, Datum minmax[2]) bool isnull; Datum value = heap_getattr(tuple, attnum, RelationGetDescr(rel), &isnull); + if (isnull) + continue; + /* Check for new min */ - if (minmaxnull[0] || DatumGetInt32(FunctionCall2(&tce->cmp_proc_finfo, value, minmax[0])) < 0) + if (nulls[0] || DatumGetInt32(FunctionCall2(&tce->cmp_proc_finfo, value, minmax[0])) < 0) { - minmaxnull[0] = false; + nulls[0] = false; minmax[0] = value; } /* Check for new max */ - if (minmaxnull[1] || DatumGetInt32(FunctionCall2(&tce->cmp_proc_finfo, value, minmax[1])) > 0) + if (nulls[1] || DatumGetInt32(FunctionCall2(&tce->cmp_proc_finfo, value, minmax[1])) > 0) { - minmaxnull[1] = false; + nulls[1] = false; minmax[1] = value; } } heap_endscan(scan); - return !minmaxnull[0] && !minmaxnull[1]; + return (nulls[0] || nulls[1]) ? MINMAX_NO_TUPLES : MINMAX_FOUND; } /* * Use an index scan to find the min and max of a given column of a chunk. - * - * Returns true iff min and max is found. */ -static bool +static MinMaxResult minmax_indexscan(Relation rel, Relation idxrel, AttrNumber attnum, Datum minmax[2]) { IndexScanDesc scan = index_beginscan(rel, idxrel, GetTransactionSnapshot(), 0, 0); HeapTuple tuple; bool isnull; + bool nulls[2] = {true}; int n = 0; + nulls[0] = nulls[1] = true; + tuple = index_getnext(scan, BackwardScanDirection); if (HeapTupleIsValid(tuple)) - minmax[n++] = heap_getattr(tuple, attnum, RelationGetDescr(rel), &isnull); + { + minmax[n] = heap_getattr(tuple, attnum, RelationGetDescr(rel), &isnull); + nulls[n++] = false; + } index_rescan(scan, NULL, 0, NULL, 0); tuple = index_getnext(scan, ForwardScanDirection); if (HeapTupleIsValid(tuple)) - minmax[n++] = heap_getattr(tuple, attnum, RelationGetDescr(rel), &isnull); + { + minmax[n] = heap_getattr(tuple, attnum, RelationGetDescr(rel), &isnull); + nulls[n++] = false; + } index_endscan(scan); - return n == 2; + return (nulls[0] || nulls[1]) ? MINMAX_NO_TUPLES : MINMAX_FOUND; +} + +/* + * Do a scan for min and max using and index on the given column. + */ +static MinMaxResult +relation_minmax_indexscan(Relation rel, + Oid atttype, + AttrNumber attnum, + Datum minmax[2]) +{ + List *indexlist = RelationGetIndexList(rel); + ListCell *lc; + MinMaxResult res = MINMAX_NO_INDEX; + + foreach(lc, indexlist) + { + Relation idxrel; + + idxrel = index_open(lfirst_oid(lc), AccessShareLock); + + if (idxrel->rd_att->attrs[0]->attnum == attnum) + res = minmax_indexscan(rel, idxrel, attnum, minmax); + + index_close(idxrel, AccessShareLock); + + if (res == MINMAX_FOUND) + break; + } + + return res; +} + +static bool +table_has_minmax_index(Oid relid, Oid atttype, AttrNumber attnum) +{ + Datum minmax[2]; + Relation rel = heap_open(relid, AccessShareLock); + MinMaxResult res = relation_minmax_indexscan(rel, atttype, attnum, minmax); + + heap_close(rel, AccessShareLock); + + return res != MINMAX_NO_INDEX; } /* @@ -264,31 +324,29 @@ static bool chunk_get_minmax(Oid relid, Oid atttype, AttrNumber attnum, Datum minmax[2]) { Relation rel = heap_open(relid, AccessShareLock); - List *indexlist = RelationGetIndexList(rel); - ListCell *lc; - bool found = false; + MinMaxResult res = relation_minmax_indexscan(rel, atttype, attnum, minmax); - foreach(lc, indexlist) + if (res == MINMAX_NO_INDEX) { - Relation idxrel; + ereport(WARNING, + (errmsg("no index on \"%s\" found for adaptive chunking on chunk \"%s\"", + get_attname(relid, attnum), get_rel_name(relid)), + errdetail("Adaptive chunking works best with an index on the dimension being adapted."))); - idxrel = index_open(lfirst_oid(lc), AccessShareLock); - - if (idxrel->rd_att->attrs[0]->attnum == attnum) - found = minmax_indexscan(rel, idxrel, attnum, minmax); - - index_close(idxrel, AccessShareLock); - - if (found) - break; + res = minmax_heapscan(rel, atttype, attnum, minmax); } - if (!found) - found = minmax_heapscan(rel, atttype, attnum, minmax); - heap_close(rel, AccessShareLock); - return found; + return res == MINMAX_FOUND; +} + +static AttrNumber +chunk_get_attno(Oid hypertable_relid, Oid chunk_relid, AttrNumber hypertable_attnum) +{ + const char *attname = get_attname(hypertable_relid, hypertable_attnum); + + return get_attnum(chunk_relid, attname); } #define CHUNK_SIZING_FUNC_NARGS 3 @@ -427,6 +485,7 @@ calculate_chunk_interval(PG_FUNCTION_ARGS) int64 chunk_size, slice_interval; Datum minmax[2]; + AttrNumber attno = chunk_get_attno(ht->main_table_relid, chunk->table_id, dim->column_attno); Assert(NULL != slice); @@ -435,7 +494,8 @@ calculate_chunk_interval(PG_FUNCTION_ARGS) slice_interval = slice->fd.range_end - slice->fd.range_start; - if (chunk_get_minmax(chunk->table_id, dim->fd.column_type, dim->column_attno, minmax)) + + if (chunk_get_minmax(chunk->table_id, dim->fd.column_type, attno, minmax)) { int64 min = time_value_to_internal(minmax[0], dim->fd.column_type, false); int64 max = time_value_to_internal(minmax[1], dim->fd.column_type, false); @@ -559,7 +619,7 @@ chunk_sizing_func_validate(regproc func, ChunkSizingInfo *info) ReleaseSysCache(tuple); ereport(ERROR, (errcode(ERRCODE_INVALID_FUNCTION_DEFINITION), - errmsg("invalid number of function arguments"), + errmsg("invalid function signature"), errhint("A chunk sizing function's signature should be (int, bigint, bigint) -> bigint"))); } @@ -583,37 +643,85 @@ chunk_target_size_in_bytes(const text *target_size_text) pg_strcasecmp(target_size, "disable") == 0) return 0; - if (pg_strcasecmp(target_size, "estimate") != 0) + if (pg_strcasecmp(target_size, "estimate") == 0) + target_size_bytes = calculate_initial_chunk_target_size(); + else target_size_bytes = convert_text_memory_amount_to_bytes(target_size); + /* Disable if target size is zero or less */ if (target_size_bytes <= 0) - target_size_bytes = calculate_initial_chunk_target_size(); + target_size_bytes = 0; return target_size_bytes; } +#define MB (1024*1024) + void -chunk_adaptive_validate_sizing_info(ChunkSizingInfo *info) +chunk_adaptive_sizing_info_validate(ChunkSizingInfo *info) { + AttrNumber attnum; + Oid atttype; + + if (!OidIsValid(info->table_relid)) + ereport(ERROR, + (errcode(ERRCODE_UNDEFINED_TABLE), + errmsg("table does not exist"))); + + if (NULL == info->colname) + ereport(ERROR, + (errcode(ERRCODE_IO_DIMENSION_NOT_EXIST), + errmsg("no open dimension found for adaptive chunking"))); + + attnum = get_attnum(info->table_relid, info->colname); + atttype = get_atttype(info->table_relid, attnum); + + if (!OidIsValid(atttype)) + ereport(ERROR, + (errcode(ERRCODE_IO_DIMENSION_NOT_EXIST), + errmsg("no open dimension found for adaptive chunking"))); + chunk_sizing_func_validate(info->func, info); - if (NULL != info->target_size) - info->target_size_bytes = chunk_target_size_in_bytes(info->target_size); - else + if (NULL == info->target_size) info->target_size_bytes = 0; + else + info->target_size_bytes = chunk_target_size_in_bytes(info->target_size); + + /* Don't validate further if disabled */ + if (info->target_size_bytes <= 0 || !OidIsValid(info->func)) + return; + + /* Warn of small target sizes */ + if (info->target_size_bytes > 0 && + info->target_size_bytes < (10 * MB)) + elog(WARNING, "target chunk size for adaptive chunking is less than 10 MB"); + + if (info->check_for_index && + !table_has_minmax_index(info->table_relid, atttype, attnum)) + ereport(WARNING, + (errmsg("no index on \"%s\" found for adaptive chunking on hypertable \"%s\"", + info->colname, get_rel_name(info->table_relid)), + errdetail("Adaptive chunking works best with an index on the dimension being adapted."))); } -TS_FUNCTION_INFO_V1(chunk_adaptive_set_chunk_sizing); +TS_FUNCTION_INFO_V1(chunk_adaptive_set); +/* + * Change the settings for adaptive chunking. + */ Datum -chunk_adaptive_set_chunk_sizing(PG_FUNCTION_ARGS) +chunk_adaptive_set(PG_FUNCTION_ARGS) { - Oid relid = PG_GETARG_OID(0); ChunkSizingInfo info = { - .func = PG_ARGISNULL(2) ? InvalidOid : PG_GETARG_OID(2), + .table_relid = PG_GETARG_OID(0), .target_size = PG_ARGISNULL(1) ? NULL : PG_GETARG_TEXT_P(1), + .func = PG_ARGISNULL(2) ? InvalidOid : PG_GETARG_OID(2), + .colname = NULL, + .check_for_index = true, }; Hypertable *ht; + Dimension *dim; Cache *hcache; HeapTuple tuple; TupleDesc tupdesc; @@ -621,24 +729,31 @@ chunk_adaptive_set_chunk_sizing(PG_FUNCTION_ARGS) Datum values[2]; bool nulls[2] = {false, false}; - if (!OidIsValid(relid)) + if (!OidIsValid(info.table_relid)) ereport(ERROR, (errcode(ERRCODE_UNDEFINED_TABLE), errmsg("table does not exist"))); hcache = hypertable_cache_pin(); - ht = hypertable_cache_get_entry(hcache, relid); + ht = hypertable_cache_get_entry(hcache, info.table_relid); if (NULL == ht) ereport(ERROR, (errcode(ERRCODE_IO_HYPERTABLE_NOT_EXIST), errmsg("table \"%s\" is not a hypertable", - get_rel_name(relid)))); + get_rel_name(info.table_relid)))); - chunk_adaptive_validate_sizing_info(&info); + /* Get the first open dimension that we will adapt on */ + dim = hyperspace_get_dimension(ht->space, DIMENSION_TYPE_OPEN, 0); - if (NULL != info.target_size) - info.target_size_bytes = chunk_target_size_in_bytes(info.target_size); + if (NULL == dim) + ereport(ERROR, + (errcode(ERRCODE_IO_DIMENSION_NOT_EXIST), + errmsg("no open dimension found for adaptive chunking"))); + + info.colname = NameStr(dim->fd.column_name); + + chunk_adaptive_sizing_info_validate(&info); if (get_call_result_type(fcinfo, NULL, &tupdesc) != TYPEFUNC_COMPOSITE) elog(ERROR, "function returning record called in context that cannot accept type record"); diff --git a/src/chunk_adaptive.h b/src/chunk_adaptive.h index 71282b930..0b4fbb067 100644 --- a/src/chunk_adaptive.h +++ b/src/chunk_adaptive.h @@ -5,9 +5,14 @@ typedef struct ChunkSizingInfo { + Oid table_relid; /* Set manually */ Oid func; text *target_size; + const char *colname; /* The column of the dimension we are adapting + * on */ + bool check_for_index; /* Set if we should check for an index on + * the dimension we are adapting on */ /* Validated info */ NameData func_name; @@ -15,6 +20,6 @@ typedef struct ChunkSizingInfo int64 target_size_bytes; } ChunkSizingInfo; -void chunk_adaptive_validate_sizing_info(ChunkSizingInfo *info); +void chunk_adaptive_sizing_info_validate(ChunkSizingInfo *info); #endif /* TIMESCALEDB_CHUNK_ADAPTIVE_H */ diff --git a/src/hypertable.c b/src/hypertable.c index 20f25ee78..8fc1db1f1 100644 --- a/src/hypertable.c +++ b/src/hypertable.c @@ -224,11 +224,14 @@ hypertable_tuple_update(TupleInfo *ti, void *data) if (OidIsValid(ht->chunk_sizing_func)) { + Dimension *dim = hyperspace_get_dimension(ht->space, DIMENSION_TYPE_OPEN, 0); ChunkSizingInfo info = { + .table_relid = ht->main_table_relid, + .colname = dim == NULL ? NULL : NameStr(dim->fd.column_name), .func = ht->chunk_sizing_func, }; - chunk_adaptive_validate_sizing_info(&info); + chunk_adaptive_sizing_info_validate(&info); namestrcpy(&ht->fd.chunk_sizing_func_schema, NameStr(info.func_schema)); namestrcpy(&ht->fd.chunk_sizing_func_name, NameStr(info.func_name)); @@ -1210,8 +1213,11 @@ hypertable_create(PG_FUNCTION_ARGS) .partitioning_func = PG_ARGISNULL(9) ? InvalidOid : PG_GETARG_OID(9), }; ChunkSizingInfo chunk_sizing_info = { + .table_relid = table_relid, .target_size = PG_ARGISNULL(11) ? NULL : PG_GETARG_TEXT_P(11), .func = PG_ARGISNULL(12) ? InvalidOid : PG_GETARG_OID(12), + .colname = PG_ARGISNULL(1) ? NULL : PG_GETARG_CSTRING(1), + .check_for_index = !create_default_indexes, }; Cache *hcache; Hypertable *ht; @@ -1370,7 +1376,7 @@ hypertable_create(PG_FUNCTION_ARGS) /* Validate and set chunk sizing information */ if (OidIsValid(chunk_sizing_info.func)) - chunk_adaptive_validate_sizing_info(&chunk_sizing_info); + chunk_adaptive_sizing_info_validate(&chunk_sizing_info); hypertable_insert(&schema_name, &table_name, diff --git a/test/expected/chunk_adaptive.out b/test/expected/chunk_adaptive.out index f81f1bae8..f264b5189 100644 --- a/test/expected/chunk_adaptive.out +++ b/test/expected/chunk_adaptive.out @@ -36,12 +36,13 @@ CREATE TABLE test_adaptive(time timestamptz, temp float, location int); SELECT create_hypertable('test_adaptive', 'time', chunk_target_size => '1MB', chunk_sizing_func => 'bad_calculate_chunk_interval'); -ERROR: invalid number of function arguments +ERROR: invalid function signature \set ON_ERROR_STOP 1 -- Setting sizing func with correct signature should work SELECT create_hypertable('test_adaptive', 'time', chunk_target_size => '1MB', chunk_sizing_func => 'calculate_chunk_interval'); +WARNING: target chunk size for adaptive chunking is less than 10 MB NOTICE: adding not-null constraint to column "time" create_hypertable ------------------- @@ -54,6 +55,7 @@ CREATE TABLE test_adaptive(time timestamptz, temp float, location int); SELECT create_hypertable('test_adaptive', 'time', chunk_target_size => '1MB', create_default_indexes => true); +WARNING: target chunk size for adaptive chunking is less than 10 MB NOTICE: adding not-null constraint to column "time" create_hypertable ------------------- @@ -68,7 +70,8 @@ FROM _timescaledb_catalog.hypertable; (1 row) -- Change the target size -SELECT * FROM set_adaptive_chunk_sizing('test_adaptive', '2MB'); +SELECT * FROM set_adaptive_chunking('test_adaptive', '2MB'); +WARNING: target chunk size for adaptive chunking is less than 10 MB chunk_sizing_func | chunk_target_size ------------------------------------------------+------------------- _timescaledb_internal.calculate_chunk_interval | 2097152 @@ -83,11 +86,11 @@ FROM _timescaledb_catalog.hypertable; \set ON_ERROR_STOP 0 -- Setting NULL func should fail -SELECT * FROM set_adaptive_chunk_sizing('test_adaptive', '1MB', NULL); +SELECT * FROM set_adaptive_chunking('test_adaptive', '1MB', NULL); ERROR: invalid chunk sizing function \set ON_ERROR_STOP 1 -- Setting NULL size disables adaptive chunking -SELECT * FROM set_adaptive_chunk_sizing('test_adaptive', NULL); +SELECT * FROM set_adaptive_chunking('test_adaptive', NULL); chunk_sizing_func | chunk_target_size ------------------------------------------------+------------------- _timescaledb_internal.calculate_chunk_interval | 0 @@ -100,14 +103,15 @@ FROM _timescaledb_catalog.hypertable; test_adaptive | _timescaledb_internal | calculate_chunk_interval | 0 (1 row) -SELECT * FROM set_adaptive_chunk_sizing('test_adaptive', '1MB'); +SELECT * FROM set_adaptive_chunking('test_adaptive', '1MB'); +WARNING: target chunk size for adaptive chunking is less than 10 MB chunk_sizing_func | chunk_target_size ------------------------------------------------+------------------- _timescaledb_internal.calculate_chunk_interval | 1048576 (1 row) -- Setting size to 'off' should also disable -SELECT * FROM set_adaptive_chunk_sizing('test_adaptive', 'off'); +SELECT * FROM set_adaptive_chunking('test_adaptive', 'off'); chunk_sizing_func | chunk_target_size ------------------------------------------------+------------------- _timescaledb_internal.calculate_chunk_interval | 0 @@ -120,28 +124,36 @@ FROM _timescaledb_catalog.hypertable; test_adaptive | _timescaledb_internal | calculate_chunk_interval | 0 (1 row) --- Setting 0 size should do an estimate. -SELECT * FROM set_adaptive_chunk_sizing('test_adaptive', '0MB'); +-- Setting 0 size should also disable +SELECT * FROM set_adaptive_chunking('test_adaptive', '0MB'); chunk_sizing_func | chunk_target_size ------------------------------------------------+------------------- - _timescaledb_internal.calculate_chunk_interval | 536870912 + _timescaledb_internal.calculate_chunk_interval | 0 (1 row) SELECT table_name, chunk_sizing_func_schema, chunk_sizing_func_name, chunk_target_size FROM _timescaledb_catalog.hypertable; table_name | chunk_sizing_func_schema | chunk_sizing_func_name | chunk_target_size ---------------+--------------------------+--------------------------+------------------- - test_adaptive | _timescaledb_internal | calculate_chunk_interval | 536870912 + test_adaptive | _timescaledb_internal | calculate_chunk_interval | 0 (1 row) -SELECT * FROM set_adaptive_chunk_sizing('test_adaptive', '1MB'); +SELECT * FROM set_adaptive_chunking('test_adaptive', '1MB'); +WARNING: target chunk size for adaptive chunking is less than 10 MB chunk_sizing_func | chunk_target_size ------------------------------------------------+------------------- _timescaledb_internal.calculate_chunk_interval | 1048576 (1 row) +-- No warning about small target size if > 10MB +SELECT * FROM set_adaptive_chunking('test_adaptive', '11MB'); + chunk_sizing_func | chunk_target_size +------------------------------------------------+------------------- + _timescaledb_internal.calculate_chunk_interval | 11534336 +(1 row) + -- Setting size to 'estimate' should also estimate size -SELECT * FROM set_adaptive_chunk_sizing('test_adaptive', 'estimate'); +SELECT * FROM set_adaptive_chunking('test_adaptive', 'estimate'); chunk_sizing_func | chunk_target_size ------------------------------------------------+------------------- _timescaledb_internal.calculate_chunk_interval | 536870912 @@ -161,7 +173,7 @@ SELECT * FROM test.set_effective_memory_cache_size('512MB'); 536870912 (1 row) -SELECT * FROM set_adaptive_chunk_sizing('test_adaptive', 'estimate'); +SELECT * FROM set_adaptive_chunking('test_adaptive', 'estimate'); chunk_sizing_func | chunk_target_size ------------------------------------------------+------------------- _timescaledb_internal.calculate_chunk_interval | 134217728 @@ -182,7 +194,8 @@ SELECT * FROM test.set_effective_memory_cache_size('2GB'); (1 row) -- Set a reasonable test value -SELECT * FROM set_adaptive_chunk_sizing('test_adaptive', '1MB'); +SELECT * FROM set_adaptive_chunking('test_adaptive', '1MB'); +WARNING: target chunk size for adaptive chunking is less than 10 MB chunk_sizing_func | chunk_target_size ------------------------------------------------+------------------- _timescaledb_internal.calculate_chunk_interval | 1048576 @@ -223,9 +236,12 @@ SELECT * FROM chunk_relation_size('test_adaptive'); -- both the calculation of fill-factor of the chunk and its size CREATE TABLE test_adaptive_no_index(time timestamptz, temp float, location int); -- Size but no explicit func should use default func +-- No default indexes should warn and use heap scan for min and max SELECT create_hypertable('test_adaptive_no_index', 'time', chunk_target_size => '1MB', create_default_indexes => false); +WARNING: target chunk size for adaptive chunking is less than 10 MB +WARNING: no index on "time" found for adaptive chunking on hypertable "test_adaptive_no_index" NOTICE: adding not-null constraint to column "time" create_hypertable ------------------- @@ -244,6 +260,21 @@ SELECT time, random() * 35, _timescaledb_internal.get_partition_hash(time) FROM generate_series('2017-03-07T18:18:03+00'::timestamptz - interval '175 days', '2017-03-07T18:18:03+00'::timestamptz, '2 minutes') as time; +WARNING: no index on "time" found for adaptive chunking on chunk "_hyper_3_12_chunk" +WARNING: no index on "time" found for adaptive chunking on chunk "_hyper_3_12_chunk" +WARNING: no index on "time" found for adaptive chunking on chunk "_hyper_3_13_chunk" +WARNING: no index on "time" found for adaptive chunking on chunk "_hyper_3_12_chunk" +WARNING: no index on "time" found for adaptive chunking on chunk "_hyper_3_13_chunk" +WARNING: no index on "time" found for adaptive chunking on chunk "_hyper_3_14_chunk" +WARNING: no index on "time" found for adaptive chunking on chunk "_hyper_3_13_chunk" +WARNING: no index on "time" found for adaptive chunking on chunk "_hyper_3_14_chunk" +WARNING: no index on "time" found for adaptive chunking on chunk "_hyper_3_15_chunk" +WARNING: no index on "time" found for adaptive chunking on chunk "_hyper_3_14_chunk" +WARNING: no index on "time" found for adaptive chunking on chunk "_hyper_3_15_chunk" +WARNING: no index on "time" found for adaptive chunking on chunk "_hyper_3_16_chunk" +WARNING: no index on "time" found for adaptive chunking on chunk "_hyper_3_15_chunk" +WARNING: no index on "time" found for adaptive chunking on chunk "_hyper_3_16_chunk" +WARNING: no index on "time" found for adaptive chunking on chunk "_hyper_3_17_chunk" SELECT * FROM chunk_relation_size('test_adaptive_no_index'); chunk_id | chunk_table | partitioning_columns | partitioning_column_types | partitioning_hash_functions | ranges | table_bytes | index_bytes | toast_bytes | total_bytes ----------+-----------------------------------------+----------------------+------------------------------+-----------------------------+-----------------------------------------+-------------+-------------+-------------+------------- @@ -263,6 +294,7 @@ CREATE TABLE test_adaptive_space(time timestamptz, temp float, location int); SELECT create_hypertable('test_adaptive_space', 'time', 'location', 2, chunk_target_size => '1MB', create_default_indexes => true); +WARNING: target chunk size for adaptive chunking is less than 10 MB NOTICE: adding not-null constraint to column "time" create_hypertable ------------------- diff --git a/test/expected/extension.out b/test/expected/extension.out index a73abf63c..b410448e3 100644 --- a/test/expected/extension.out +++ b/test/expected/extension.out @@ -27,7 +27,7 @@ ORDER BY proname; indexes_relation_size indexes_relation_size_pretty last - set_adaptive_chunk_sizing + set_adaptive_chunking set_chunk_time_interval set_number_partitions show_tablespaces diff --git a/test/expected/pg_dump.out b/test/expected/pg_dump.out index fb3c2bd50..0206a3173 100644 --- a/test/expected/pg_dump.out +++ b/test/expected/pg_dump.out @@ -58,7 +58,8 @@ BEGIN RETURN -1; END $BODY$; -SELECT * FROM set_adaptive_chunk_sizing('"test_schema"."two_Partitions"', '1 MB', 'custom_calculate_chunk_interval'); +SELECT * FROM set_adaptive_chunking('"test_schema"."two_Partitions"', '1 MB', 'custom_calculate_chunk_interval'); +WARNING: target chunk size for adaptive chunking is less than 10 MB chunk_sizing_func | chunk_target_size ---------------------------------+------------------- custom_calculate_chunk_interval | 1048576 diff --git a/test/sql/chunk_adaptive.sql b/test/sql/chunk_adaptive.sql index 320e0e70c..1d98c676d 100644 --- a/test/sql/chunk_adaptive.sql +++ b/test/sql/chunk_adaptive.sql @@ -54,42 +54,45 @@ SELECT table_name, chunk_sizing_func_schema, chunk_sizing_func_name, chunk_targe FROM _timescaledb_catalog.hypertable; -- Change the target size -SELECT * FROM set_adaptive_chunk_sizing('test_adaptive', '2MB'); +SELECT * FROM set_adaptive_chunking('test_adaptive', '2MB'); SELECT table_name, chunk_sizing_func_schema, chunk_sizing_func_name, chunk_target_size FROM _timescaledb_catalog.hypertable; \set ON_ERROR_STOP 0 -- Setting NULL func should fail -SELECT * FROM set_adaptive_chunk_sizing('test_adaptive', '1MB', NULL); +SELECT * FROM set_adaptive_chunking('test_adaptive', '1MB', NULL); \set ON_ERROR_STOP 1 -- Setting NULL size disables adaptive chunking -SELECT * FROM set_adaptive_chunk_sizing('test_adaptive', NULL); +SELECT * FROM set_adaptive_chunking('test_adaptive', NULL); SELECT table_name, chunk_sizing_func_schema, chunk_sizing_func_name, chunk_target_size FROM _timescaledb_catalog.hypertable; -SELECT * FROM set_adaptive_chunk_sizing('test_adaptive', '1MB'); +SELECT * FROM set_adaptive_chunking('test_adaptive', '1MB'); -- Setting size to 'off' should also disable -SELECT * FROM set_adaptive_chunk_sizing('test_adaptive', 'off'); +SELECT * FROM set_adaptive_chunking('test_adaptive', 'off'); SELECT table_name, chunk_sizing_func_schema, chunk_sizing_func_name, chunk_target_size FROM _timescaledb_catalog.hypertable; --- Setting 0 size should do an estimate. -SELECT * FROM set_adaptive_chunk_sizing('test_adaptive', '0MB'); +-- Setting 0 size should also disable +SELECT * FROM set_adaptive_chunking('test_adaptive', '0MB'); SELECT table_name, chunk_sizing_func_schema, chunk_sizing_func_name, chunk_target_size FROM _timescaledb_catalog.hypertable; -SELECT * FROM set_adaptive_chunk_sizing('test_adaptive', '1MB'); +SELECT * FROM set_adaptive_chunking('test_adaptive', '1MB'); + +-- No warning about small target size if > 10MB +SELECT * FROM set_adaptive_chunking('test_adaptive', '11MB'); -- Setting size to 'estimate' should also estimate size -SELECT * FROM set_adaptive_chunk_sizing('test_adaptive', 'estimate'); +SELECT * FROM set_adaptive_chunking('test_adaptive', 'estimate'); SELECT table_name, chunk_sizing_func_schema, chunk_sizing_func_name, chunk_target_size FROM _timescaledb_catalog.hypertable; -- Use a lower memory setting to test that the calculated chunk_target_size is reduced SELECT * FROM test.set_effective_memory_cache_size('512MB'); -SELECT * FROM set_adaptive_chunk_sizing('test_adaptive', 'estimate'); +SELECT * FROM set_adaptive_chunking('test_adaptive', 'estimate'); SELECT table_name, chunk_sizing_func_schema, chunk_sizing_func_name, chunk_target_size FROM _timescaledb_catalog.hypertable; @@ -97,7 +100,7 @@ FROM _timescaledb_catalog.hypertable; SELECT * FROM test.set_effective_memory_cache_size('2GB'); -- Set a reasonable test value -SELECT * FROM set_adaptive_chunk_sizing('test_adaptive', '1MB'); +SELECT * FROM set_adaptive_chunking('test_adaptive', '1MB'); -- Show the interval length before and after adaptation SELECT id, hypertable_id, interval_length FROM _timescaledb_catalog.dimension; @@ -118,6 +121,7 @@ SELECT * FROM chunk_relation_size('test_adaptive'); CREATE TABLE test_adaptive_no_index(time timestamptz, temp float, location int); -- Size but no explicit func should use default func +-- No default indexes should warn and use heap scan for min and max SELECT create_hypertable('test_adaptive_no_index', 'time', chunk_target_size => '1MB', create_default_indexes => false); diff --git a/test/sql/pg_dump.sql b/test/sql/pg_dump.sql index 1f15ff3e7..d16f20941 100644 --- a/test/sql/pg_dump.sql +++ b/test/sql/pg_dump.sql @@ -34,7 +34,7 @@ BEGIN END $BODY$; -SELECT * FROM set_adaptive_chunk_sizing('"test_schema"."two_Partitions"', '1 MB', 'custom_calculate_chunk_interval'); +SELECT * FROM set_adaptive_chunking('"test_schema"."two_Partitions"', '1 MB', 'custom_calculate_chunk_interval'); -- Chunk sizing func set SELECT * FROM _timescaledb_catalog.hypertable;