diff --git a/sql/ddl_api.sql b/sql/ddl_api.sql index 850d3ced7..0e03c219a 100644 --- a/sql/ddl_api.sql +++ b/sql/ddl_api.sql @@ -32,12 +32,12 @@ CREATE OR REPLACE FUNCTION create_hypertable( ) RETURNS VOID AS '@MODULE_PATHNAME@', 'hypertable_create' LANGUAGE C VOLATILE; -- Set adaptive chunking. To disable, set chunk_target_size => 'off'. -CREATE OR REPLACE FUNCTION set_adaptive_chunk_sizing( +CREATE OR REPLACE FUNCTION set_adaptive_chunking( hypertable REGCLASS, chunk_target_size TEXT, INOUT chunk_sizing_func REGPROC = '_timescaledb_internal.calculate_chunk_interval'::regproc, OUT chunk_target_size BIGINT -) RETURNS RECORD AS '@MODULE_PATHNAME@', 'chunk_adaptive_set_chunk_sizing' LANGUAGE C VOLATILE; +) RETURNS RECORD AS '@MODULE_PATHNAME@', 'chunk_adaptive_set' LANGUAGE C VOLATILE; -- Update chunk_time_interval for a hypertable. -- diff --git a/src/chunk.c b/src/chunk.c index f3ad61bd0..de72df711 100644 --- a/src/chunk.c +++ b/src/chunk.c @@ -1055,15 +1055,33 @@ chunk_scan_internal(int indexid, return scanner_scan(&ctx); } -#define DEFAULT_CHUNKS_PER_INTERVAL 4 - /* * Get a window of chunks that "preceed" the given dimensional point. * * For instance, if the dimension is "time", then given a point in time the * function returns the recent chunks that come before the chunk that includes * that point. The count parameter determines the number or slices the window - * should include in the given dimension. + * should include in the given dimension. Note, that with multi-dimensional + * partitioning, there might be multiple chunks in each dimensional slice that + * all preceed the given point. For instance, the example below shows two + * different situations that each go "back" two slices (count = 2) in the + * x-dimension, but returns two vs. eight chunks due to different + * partitioning. + * + * '_____________ + * '| | | * | + * '|___|___|___| + * ' + * ' + * '____ ________ + * '| | | * | + * '|___|___|___| + * '| | | | + * '|___|___|___| + * '| | | | + * '|___|___|___| + * '| | | | + * '|___|___|___| * * Note that the returned chunks will be allocated on the given memory * context, inlcuding the list itself. So, beware of not leaking the list if @@ -1073,17 +1091,30 @@ List * chunk_get_window(int32 dimension_id, int64 point, int count, MemoryContext mctx) { List *chunks = NIL; - DimensionVec *dimvec = dimension_slice_scan_by_dimension_before_point(dimension_id, point, count, BackwardScanDirection, mctx); + DimensionVec *dimvec; int i; + /* Scan for "count" slices that preceeds the point in the given dimension */ + dimvec = dimension_slice_scan_by_dimension_before_point(dimension_id, + point, + count, + BackwardScanDirection, + mctx); + + /* + * For each slice, join with any constraints that reference the slice. + * There might be multiple constraints for each slice in case of + * multi-dimensional partitioning. + */ for (i = 0; i < dimvec->num_slices; i++) { DimensionSlice *slice = dimvec->slices[i]; - ChunkConstraints *ccs = chunk_constraints_alloc(DEFAULT_CHUNKS_PER_INTERVAL, mctx); + ChunkConstraints *ccs = chunk_constraints_alloc(1, mctx); int j; chunk_constraint_scan_by_dimension_slice_id(slice->fd.id, ccs, mctx); + /* For each constraint, find the corresponding chunk */ for (j = 0; j < ccs->num_constraints; j++) { ChunkConstraint *cc = &ccs->constraints[j]; diff --git a/src/chunk_adaptive.c b/src/chunk_adaptive.c index 4db3177b2..45c6f2b17 100644 --- a/src/chunk_adaptive.c +++ b/src/chunk_adaptive.c @@ -105,7 +105,9 @@ set_effective_memory_cache_size(PG_FUNCTION_ARGS) * of the system, while a common recommended setting for shared_buffers is 1/4 * of system memory. In case shared_buffers is set higher than * effective_cache_size, we use the max of the two (a larger shared_buffers is a - * strange setting though). Ultimately we are limited by system memory. + * strange setting though). Ultimately we are limited by system memory. Thus, + * this functions returns a value effective_memory_cache which is: + * shared_buffers >= effective_memory_cache <= system_mem / 2. * * Note that this relies on the user setting a good value for * effective_cache_size, or otherwise our estimate will be off. Alternatively, @@ -160,9 +162,9 @@ estimate_effective_memory_cache_size(void) return memory_bytes; } -/* The default concurrency factor, i.e., the number of chunks we expect to fit - * in memory at the same time */ -#define DEFAULT_CONCURRENT_CHUNK_USAGE 4 +/* The default the number of chunks we expect to be able to have in cache + * memory at the same time */ +#define DEFAULT_NUM_CHUNKS_TO_FIT_IN_CACHE_MEM 4 static inline int64 calculate_initial_chunk_target_size(void) @@ -175,23 +177,28 @@ calculate_initial_chunk_target_size(void) * hypertables in all schemas and databases, and might not be a good * estimate in case of many "old" (unused) hypertables. */ - return estimate_effective_memory_cache_size() / DEFAULT_CONCURRENT_CHUNK_USAGE; + return estimate_effective_memory_cache_size() / DEFAULT_NUM_CHUNKS_TO_FIT_IN_CACHE_MEM; } +typedef enum MinMaxResult +{ + MINMAX_NO_INDEX, + MINMAX_NO_TUPLES, + MINMAX_FOUND, +} MinMaxResult; + /* * Use a heap scan to find the min and max of a given column of a chunk. This * could be a rather costly operation. Should figure out how to keep min-max * stats cached. - * - * Returns true iff min and max is found. */ -static bool +static MinMaxResult minmax_heapscan(Relation rel, Oid atttype, AttrNumber attnum, Datum minmax[2]) { HeapScanDesc scan; HeapTuple tuple; TypeCacheEntry *tce; - bool minmaxnull[2] = {true}; + bool nulls[2] = {true}; /* Lookup the tuple comparison function from the type cache */ tce = lookup_type_cache(atttype, TYPECACHE_CMP_PROC | TYPECACHE_CMP_PROC_FINFO); @@ -206,53 +213,106 @@ minmax_heapscan(Relation rel, Oid atttype, AttrNumber attnum, Datum minmax[2]) bool isnull; Datum value = heap_getattr(tuple, attnum, RelationGetDescr(rel), &isnull); + if (isnull) + continue; + /* Check for new min */ - if (minmaxnull[0] || DatumGetInt32(FunctionCall2(&tce->cmp_proc_finfo, value, minmax[0])) < 0) + if (nulls[0] || DatumGetInt32(FunctionCall2(&tce->cmp_proc_finfo, value, minmax[0])) < 0) { - minmaxnull[0] = false; + nulls[0] = false; minmax[0] = value; } /* Check for new max */ - if (minmaxnull[1] || DatumGetInt32(FunctionCall2(&tce->cmp_proc_finfo, value, minmax[1])) > 0) + if (nulls[1] || DatumGetInt32(FunctionCall2(&tce->cmp_proc_finfo, value, minmax[1])) > 0) { - minmaxnull[1] = false; + nulls[1] = false; minmax[1] = value; } } heap_endscan(scan); - return !minmaxnull[0] && !minmaxnull[1]; + return (nulls[0] || nulls[1]) ? MINMAX_NO_TUPLES : MINMAX_FOUND; } /* * Use an index scan to find the min and max of a given column of a chunk. - * - * Returns true iff min and max is found. */ -static bool +static MinMaxResult minmax_indexscan(Relation rel, Relation idxrel, AttrNumber attnum, Datum minmax[2]) { IndexScanDesc scan = index_beginscan(rel, idxrel, GetTransactionSnapshot(), 0, 0); HeapTuple tuple; bool isnull; + bool nulls[2] = {true}; int n = 0; + nulls[0] = nulls[1] = true; + tuple = index_getnext(scan, BackwardScanDirection); if (HeapTupleIsValid(tuple)) - minmax[n++] = heap_getattr(tuple, attnum, RelationGetDescr(rel), &isnull); + { + minmax[n] = heap_getattr(tuple, attnum, RelationGetDescr(rel), &isnull); + nulls[n++] = false; + } index_rescan(scan, NULL, 0, NULL, 0); tuple = index_getnext(scan, ForwardScanDirection); if (HeapTupleIsValid(tuple)) - minmax[n++] = heap_getattr(tuple, attnum, RelationGetDescr(rel), &isnull); + { + minmax[n] = heap_getattr(tuple, attnum, RelationGetDescr(rel), &isnull); + nulls[n++] = false; + } index_endscan(scan); - return n == 2; + return (nulls[0] || nulls[1]) ? MINMAX_NO_TUPLES : MINMAX_FOUND; +} + +/* + * Do a scan for min and max using and index on the given column. + */ +static MinMaxResult +relation_minmax_indexscan(Relation rel, + Oid atttype, + AttrNumber attnum, + Datum minmax[2]) +{ + List *indexlist = RelationGetIndexList(rel); + ListCell *lc; + MinMaxResult res = MINMAX_NO_INDEX; + + foreach(lc, indexlist) + { + Relation idxrel; + + idxrel = index_open(lfirst_oid(lc), AccessShareLock); + + if (idxrel->rd_att->attrs[0]->attnum == attnum) + res = minmax_indexscan(rel, idxrel, attnum, minmax); + + index_close(idxrel, AccessShareLock); + + if (res == MINMAX_FOUND) + break; + } + + return res; +} + +static bool +table_has_minmax_index(Oid relid, Oid atttype, AttrNumber attnum) +{ + Datum minmax[2]; + Relation rel = heap_open(relid, AccessShareLock); + MinMaxResult res = relation_minmax_indexscan(rel, atttype, attnum, minmax); + + heap_close(rel, AccessShareLock); + + return res != MINMAX_NO_INDEX; } /* @@ -264,31 +324,29 @@ static bool chunk_get_minmax(Oid relid, Oid atttype, AttrNumber attnum, Datum minmax[2]) { Relation rel = heap_open(relid, AccessShareLock); - List *indexlist = RelationGetIndexList(rel); - ListCell *lc; - bool found = false; + MinMaxResult res = relation_minmax_indexscan(rel, atttype, attnum, minmax); - foreach(lc, indexlist) + if (res == MINMAX_NO_INDEX) { - Relation idxrel; + ereport(WARNING, + (errmsg("no index on \"%s\" found for adaptive chunking on chunk \"%s\"", + get_attname(relid, attnum), get_rel_name(relid)), + errdetail("Adaptive chunking works best with an index on the dimension being adapted."))); - idxrel = index_open(lfirst_oid(lc), AccessShareLock); - - if (idxrel->rd_att->attrs[0]->attnum == attnum) - found = minmax_indexscan(rel, idxrel, attnum, minmax); - - index_close(idxrel, AccessShareLock); - - if (found) - break; + res = minmax_heapscan(rel, atttype, attnum, minmax); } - if (!found) - found = minmax_heapscan(rel, atttype, attnum, minmax); - heap_close(rel, AccessShareLock); - return found; + return res == MINMAX_FOUND; +} + +static AttrNumber +chunk_get_attno(Oid hypertable_relid, Oid chunk_relid, AttrNumber hypertable_attnum) +{ + const char *attname = get_attname(hypertable_relid, hypertable_attnum); + + return get_attnum(chunk_relid, attname); } #define CHUNK_SIZING_FUNC_NARGS 3 @@ -427,6 +485,7 @@ calculate_chunk_interval(PG_FUNCTION_ARGS) int64 chunk_size, slice_interval; Datum minmax[2]; + AttrNumber attno = chunk_get_attno(ht->main_table_relid, chunk->table_id, dim->column_attno); Assert(NULL != slice); @@ -435,7 +494,8 @@ calculate_chunk_interval(PG_FUNCTION_ARGS) slice_interval = slice->fd.range_end - slice->fd.range_start; - if (chunk_get_minmax(chunk->table_id, dim->fd.column_type, dim->column_attno, minmax)) + + if (chunk_get_minmax(chunk->table_id, dim->fd.column_type, attno, minmax)) { int64 min = time_value_to_internal(minmax[0], dim->fd.column_type, false); int64 max = time_value_to_internal(minmax[1], dim->fd.column_type, false); @@ -559,7 +619,7 @@ chunk_sizing_func_validate(regproc func, ChunkSizingInfo *info) ReleaseSysCache(tuple); ereport(ERROR, (errcode(ERRCODE_INVALID_FUNCTION_DEFINITION), - errmsg("invalid number of function arguments"), + errmsg("invalid function signature"), errhint("A chunk sizing function's signature should be (int, bigint, bigint) -> bigint"))); } @@ -583,37 +643,85 @@ chunk_target_size_in_bytes(const text *target_size_text) pg_strcasecmp(target_size, "disable") == 0) return 0; - if (pg_strcasecmp(target_size, "estimate") != 0) + if (pg_strcasecmp(target_size, "estimate") == 0) + target_size_bytes = calculate_initial_chunk_target_size(); + else target_size_bytes = convert_text_memory_amount_to_bytes(target_size); + /* Disable if target size is zero or less */ if (target_size_bytes <= 0) - target_size_bytes = calculate_initial_chunk_target_size(); + target_size_bytes = 0; return target_size_bytes; } +#define MB (1024*1024) + void -chunk_adaptive_validate_sizing_info(ChunkSizingInfo *info) +chunk_adaptive_sizing_info_validate(ChunkSizingInfo *info) { + AttrNumber attnum; + Oid atttype; + + if (!OidIsValid(info->table_relid)) + ereport(ERROR, + (errcode(ERRCODE_UNDEFINED_TABLE), + errmsg("table does not exist"))); + + if (NULL == info->colname) + ereport(ERROR, + (errcode(ERRCODE_IO_DIMENSION_NOT_EXIST), + errmsg("no open dimension found for adaptive chunking"))); + + attnum = get_attnum(info->table_relid, info->colname); + atttype = get_atttype(info->table_relid, attnum); + + if (!OidIsValid(atttype)) + ereport(ERROR, + (errcode(ERRCODE_IO_DIMENSION_NOT_EXIST), + errmsg("no open dimension found for adaptive chunking"))); + chunk_sizing_func_validate(info->func, info); - if (NULL != info->target_size) - info->target_size_bytes = chunk_target_size_in_bytes(info->target_size); - else + if (NULL == info->target_size) info->target_size_bytes = 0; + else + info->target_size_bytes = chunk_target_size_in_bytes(info->target_size); + + /* Don't validate further if disabled */ + if (info->target_size_bytes <= 0 || !OidIsValid(info->func)) + return; + + /* Warn of small target sizes */ + if (info->target_size_bytes > 0 && + info->target_size_bytes < (10 * MB)) + elog(WARNING, "target chunk size for adaptive chunking is less than 10 MB"); + + if (info->check_for_index && + !table_has_minmax_index(info->table_relid, atttype, attnum)) + ereport(WARNING, + (errmsg("no index on \"%s\" found for adaptive chunking on hypertable \"%s\"", + info->colname, get_rel_name(info->table_relid)), + errdetail("Adaptive chunking works best with an index on the dimension being adapted."))); } -TS_FUNCTION_INFO_V1(chunk_adaptive_set_chunk_sizing); +TS_FUNCTION_INFO_V1(chunk_adaptive_set); +/* + * Change the settings for adaptive chunking. + */ Datum -chunk_adaptive_set_chunk_sizing(PG_FUNCTION_ARGS) +chunk_adaptive_set(PG_FUNCTION_ARGS) { - Oid relid = PG_GETARG_OID(0); ChunkSizingInfo info = { - .func = PG_ARGISNULL(2) ? InvalidOid : PG_GETARG_OID(2), + .table_relid = PG_GETARG_OID(0), .target_size = PG_ARGISNULL(1) ? NULL : PG_GETARG_TEXT_P(1), + .func = PG_ARGISNULL(2) ? InvalidOid : PG_GETARG_OID(2), + .colname = NULL, + .check_for_index = true, }; Hypertable *ht; + Dimension *dim; Cache *hcache; HeapTuple tuple; TupleDesc tupdesc; @@ -621,24 +729,31 @@ chunk_adaptive_set_chunk_sizing(PG_FUNCTION_ARGS) Datum values[2]; bool nulls[2] = {false, false}; - if (!OidIsValid(relid)) + if (!OidIsValid(info.table_relid)) ereport(ERROR, (errcode(ERRCODE_UNDEFINED_TABLE), errmsg("table does not exist"))); hcache = hypertable_cache_pin(); - ht = hypertable_cache_get_entry(hcache, relid); + ht = hypertable_cache_get_entry(hcache, info.table_relid); if (NULL == ht) ereport(ERROR, (errcode(ERRCODE_IO_HYPERTABLE_NOT_EXIST), errmsg("table \"%s\" is not a hypertable", - get_rel_name(relid)))); + get_rel_name(info.table_relid)))); - chunk_adaptive_validate_sizing_info(&info); + /* Get the first open dimension that we will adapt on */ + dim = hyperspace_get_dimension(ht->space, DIMENSION_TYPE_OPEN, 0); - if (NULL != info.target_size) - info.target_size_bytes = chunk_target_size_in_bytes(info.target_size); + if (NULL == dim) + ereport(ERROR, + (errcode(ERRCODE_IO_DIMENSION_NOT_EXIST), + errmsg("no open dimension found for adaptive chunking"))); + + info.colname = NameStr(dim->fd.column_name); + + chunk_adaptive_sizing_info_validate(&info); if (get_call_result_type(fcinfo, NULL, &tupdesc) != TYPEFUNC_COMPOSITE) elog(ERROR, "function returning record called in context that cannot accept type record"); diff --git a/src/chunk_adaptive.h b/src/chunk_adaptive.h index 71282b930..0b4fbb067 100644 --- a/src/chunk_adaptive.h +++ b/src/chunk_adaptive.h @@ -5,9 +5,14 @@ typedef struct ChunkSizingInfo { + Oid table_relid; /* Set manually */ Oid func; text *target_size; + const char *colname; /* The column of the dimension we are adapting + * on */ + bool check_for_index; /* Set if we should check for an index on + * the dimension we are adapting on */ /* Validated info */ NameData func_name; @@ -15,6 +20,6 @@ typedef struct ChunkSizingInfo int64 target_size_bytes; } ChunkSizingInfo; -void chunk_adaptive_validate_sizing_info(ChunkSizingInfo *info); +void chunk_adaptive_sizing_info_validate(ChunkSizingInfo *info); #endif /* TIMESCALEDB_CHUNK_ADAPTIVE_H */ diff --git a/src/hypertable.c b/src/hypertable.c index 20f25ee78..8fc1db1f1 100644 --- a/src/hypertable.c +++ b/src/hypertable.c @@ -224,11 +224,14 @@ hypertable_tuple_update(TupleInfo *ti, void *data) if (OidIsValid(ht->chunk_sizing_func)) { + Dimension *dim = hyperspace_get_dimension(ht->space, DIMENSION_TYPE_OPEN, 0); ChunkSizingInfo info = { + .table_relid = ht->main_table_relid, + .colname = dim == NULL ? NULL : NameStr(dim->fd.column_name), .func = ht->chunk_sizing_func, }; - chunk_adaptive_validate_sizing_info(&info); + chunk_adaptive_sizing_info_validate(&info); namestrcpy(&ht->fd.chunk_sizing_func_schema, NameStr(info.func_schema)); namestrcpy(&ht->fd.chunk_sizing_func_name, NameStr(info.func_name)); @@ -1210,8 +1213,11 @@ hypertable_create(PG_FUNCTION_ARGS) .partitioning_func = PG_ARGISNULL(9) ? InvalidOid : PG_GETARG_OID(9), }; ChunkSizingInfo chunk_sizing_info = { + .table_relid = table_relid, .target_size = PG_ARGISNULL(11) ? NULL : PG_GETARG_TEXT_P(11), .func = PG_ARGISNULL(12) ? InvalidOid : PG_GETARG_OID(12), + .colname = PG_ARGISNULL(1) ? NULL : PG_GETARG_CSTRING(1), + .check_for_index = !create_default_indexes, }; Cache *hcache; Hypertable *ht; @@ -1370,7 +1376,7 @@ hypertable_create(PG_FUNCTION_ARGS) /* Validate and set chunk sizing information */ if (OidIsValid(chunk_sizing_info.func)) - chunk_adaptive_validate_sizing_info(&chunk_sizing_info); + chunk_adaptive_sizing_info_validate(&chunk_sizing_info); hypertable_insert(&schema_name, &table_name, diff --git a/test/expected/chunk_adaptive.out b/test/expected/chunk_adaptive.out index f81f1bae8..f264b5189 100644 --- a/test/expected/chunk_adaptive.out +++ b/test/expected/chunk_adaptive.out @@ -36,12 +36,13 @@ CREATE TABLE test_adaptive(time timestamptz, temp float, location int); SELECT create_hypertable('test_adaptive', 'time', chunk_target_size => '1MB', chunk_sizing_func => 'bad_calculate_chunk_interval'); -ERROR: invalid number of function arguments +ERROR: invalid function signature \set ON_ERROR_STOP 1 -- Setting sizing func with correct signature should work SELECT create_hypertable('test_adaptive', 'time', chunk_target_size => '1MB', chunk_sizing_func => 'calculate_chunk_interval'); +WARNING: target chunk size for adaptive chunking is less than 10 MB NOTICE: adding not-null constraint to column "time" create_hypertable ------------------- @@ -54,6 +55,7 @@ CREATE TABLE test_adaptive(time timestamptz, temp float, location int); SELECT create_hypertable('test_adaptive', 'time', chunk_target_size => '1MB', create_default_indexes => true); +WARNING: target chunk size for adaptive chunking is less than 10 MB NOTICE: adding not-null constraint to column "time" create_hypertable ------------------- @@ -68,7 +70,8 @@ FROM _timescaledb_catalog.hypertable; (1 row) -- Change the target size -SELECT * FROM set_adaptive_chunk_sizing('test_adaptive', '2MB'); +SELECT * FROM set_adaptive_chunking('test_adaptive', '2MB'); +WARNING: target chunk size for adaptive chunking is less than 10 MB chunk_sizing_func | chunk_target_size ------------------------------------------------+------------------- _timescaledb_internal.calculate_chunk_interval | 2097152 @@ -83,11 +86,11 @@ FROM _timescaledb_catalog.hypertable; \set ON_ERROR_STOP 0 -- Setting NULL func should fail -SELECT * FROM set_adaptive_chunk_sizing('test_adaptive', '1MB', NULL); +SELECT * FROM set_adaptive_chunking('test_adaptive', '1MB', NULL); ERROR: invalid chunk sizing function \set ON_ERROR_STOP 1 -- Setting NULL size disables adaptive chunking -SELECT * FROM set_adaptive_chunk_sizing('test_adaptive', NULL); +SELECT * FROM set_adaptive_chunking('test_adaptive', NULL); chunk_sizing_func | chunk_target_size ------------------------------------------------+------------------- _timescaledb_internal.calculate_chunk_interval | 0 @@ -100,14 +103,15 @@ FROM _timescaledb_catalog.hypertable; test_adaptive | _timescaledb_internal | calculate_chunk_interval | 0 (1 row) -SELECT * FROM set_adaptive_chunk_sizing('test_adaptive', '1MB'); +SELECT * FROM set_adaptive_chunking('test_adaptive', '1MB'); +WARNING: target chunk size for adaptive chunking is less than 10 MB chunk_sizing_func | chunk_target_size ------------------------------------------------+------------------- _timescaledb_internal.calculate_chunk_interval | 1048576 (1 row) -- Setting size to 'off' should also disable -SELECT * FROM set_adaptive_chunk_sizing('test_adaptive', 'off'); +SELECT * FROM set_adaptive_chunking('test_adaptive', 'off'); chunk_sizing_func | chunk_target_size ------------------------------------------------+------------------- _timescaledb_internal.calculate_chunk_interval | 0 @@ -120,28 +124,36 @@ FROM _timescaledb_catalog.hypertable; test_adaptive | _timescaledb_internal | calculate_chunk_interval | 0 (1 row) --- Setting 0 size should do an estimate. -SELECT * FROM set_adaptive_chunk_sizing('test_adaptive', '0MB'); +-- Setting 0 size should also disable +SELECT * FROM set_adaptive_chunking('test_adaptive', '0MB'); chunk_sizing_func | chunk_target_size ------------------------------------------------+------------------- - _timescaledb_internal.calculate_chunk_interval | 536870912 + _timescaledb_internal.calculate_chunk_interval | 0 (1 row) SELECT table_name, chunk_sizing_func_schema, chunk_sizing_func_name, chunk_target_size FROM _timescaledb_catalog.hypertable; table_name | chunk_sizing_func_schema | chunk_sizing_func_name | chunk_target_size ---------------+--------------------------+--------------------------+------------------- - test_adaptive | _timescaledb_internal | calculate_chunk_interval | 536870912 + test_adaptive | _timescaledb_internal | calculate_chunk_interval | 0 (1 row) -SELECT * FROM set_adaptive_chunk_sizing('test_adaptive', '1MB'); +SELECT * FROM set_adaptive_chunking('test_adaptive', '1MB'); +WARNING: target chunk size for adaptive chunking is less than 10 MB chunk_sizing_func | chunk_target_size ------------------------------------------------+------------------- _timescaledb_internal.calculate_chunk_interval | 1048576 (1 row) +-- No warning about small target size if > 10MB +SELECT * FROM set_adaptive_chunking('test_adaptive', '11MB'); + chunk_sizing_func | chunk_target_size +------------------------------------------------+------------------- + _timescaledb_internal.calculate_chunk_interval | 11534336 +(1 row) + -- Setting size to 'estimate' should also estimate size -SELECT * FROM set_adaptive_chunk_sizing('test_adaptive', 'estimate'); +SELECT * FROM set_adaptive_chunking('test_adaptive', 'estimate'); chunk_sizing_func | chunk_target_size ------------------------------------------------+------------------- _timescaledb_internal.calculate_chunk_interval | 536870912 @@ -161,7 +173,7 @@ SELECT * FROM test.set_effective_memory_cache_size('512MB'); 536870912 (1 row) -SELECT * FROM set_adaptive_chunk_sizing('test_adaptive', 'estimate'); +SELECT * FROM set_adaptive_chunking('test_adaptive', 'estimate'); chunk_sizing_func | chunk_target_size ------------------------------------------------+------------------- _timescaledb_internal.calculate_chunk_interval | 134217728 @@ -182,7 +194,8 @@ SELECT * FROM test.set_effective_memory_cache_size('2GB'); (1 row) -- Set a reasonable test value -SELECT * FROM set_adaptive_chunk_sizing('test_adaptive', '1MB'); +SELECT * FROM set_adaptive_chunking('test_adaptive', '1MB'); +WARNING: target chunk size for adaptive chunking is less than 10 MB chunk_sizing_func | chunk_target_size ------------------------------------------------+------------------- _timescaledb_internal.calculate_chunk_interval | 1048576 @@ -223,9 +236,12 @@ SELECT * FROM chunk_relation_size('test_adaptive'); -- both the calculation of fill-factor of the chunk and its size CREATE TABLE test_adaptive_no_index(time timestamptz, temp float, location int); -- Size but no explicit func should use default func +-- No default indexes should warn and use heap scan for min and max SELECT create_hypertable('test_adaptive_no_index', 'time', chunk_target_size => '1MB', create_default_indexes => false); +WARNING: target chunk size for adaptive chunking is less than 10 MB +WARNING: no index on "time" found for adaptive chunking on hypertable "test_adaptive_no_index" NOTICE: adding not-null constraint to column "time" create_hypertable ------------------- @@ -244,6 +260,21 @@ SELECT time, random() * 35, _timescaledb_internal.get_partition_hash(time) FROM generate_series('2017-03-07T18:18:03+00'::timestamptz - interval '175 days', '2017-03-07T18:18:03+00'::timestamptz, '2 minutes') as time; +WARNING: no index on "time" found for adaptive chunking on chunk "_hyper_3_12_chunk" +WARNING: no index on "time" found for adaptive chunking on chunk "_hyper_3_12_chunk" +WARNING: no index on "time" found for adaptive chunking on chunk "_hyper_3_13_chunk" +WARNING: no index on "time" found for adaptive chunking on chunk "_hyper_3_12_chunk" +WARNING: no index on "time" found for adaptive chunking on chunk "_hyper_3_13_chunk" +WARNING: no index on "time" found for adaptive chunking on chunk "_hyper_3_14_chunk" +WARNING: no index on "time" found for adaptive chunking on chunk "_hyper_3_13_chunk" +WARNING: no index on "time" found for adaptive chunking on chunk "_hyper_3_14_chunk" +WARNING: no index on "time" found for adaptive chunking on chunk "_hyper_3_15_chunk" +WARNING: no index on "time" found for adaptive chunking on chunk "_hyper_3_14_chunk" +WARNING: no index on "time" found for adaptive chunking on chunk "_hyper_3_15_chunk" +WARNING: no index on "time" found for adaptive chunking on chunk "_hyper_3_16_chunk" +WARNING: no index on "time" found for adaptive chunking on chunk "_hyper_3_15_chunk" +WARNING: no index on "time" found for adaptive chunking on chunk "_hyper_3_16_chunk" +WARNING: no index on "time" found for adaptive chunking on chunk "_hyper_3_17_chunk" SELECT * FROM chunk_relation_size('test_adaptive_no_index'); chunk_id | chunk_table | partitioning_columns | partitioning_column_types | partitioning_hash_functions | ranges | table_bytes | index_bytes | toast_bytes | total_bytes ----------+-----------------------------------------+----------------------+------------------------------+-----------------------------+-----------------------------------------+-------------+-------------+-------------+------------- @@ -263,6 +294,7 @@ CREATE TABLE test_adaptive_space(time timestamptz, temp float, location int); SELECT create_hypertable('test_adaptive_space', 'time', 'location', 2, chunk_target_size => '1MB', create_default_indexes => true); +WARNING: target chunk size for adaptive chunking is less than 10 MB NOTICE: adding not-null constraint to column "time" create_hypertable ------------------- diff --git a/test/expected/extension.out b/test/expected/extension.out index a73abf63c..b410448e3 100644 --- a/test/expected/extension.out +++ b/test/expected/extension.out @@ -27,7 +27,7 @@ ORDER BY proname; indexes_relation_size indexes_relation_size_pretty last - set_adaptive_chunk_sizing + set_adaptive_chunking set_chunk_time_interval set_number_partitions show_tablespaces diff --git a/test/expected/pg_dump.out b/test/expected/pg_dump.out index fb3c2bd50..0206a3173 100644 --- a/test/expected/pg_dump.out +++ b/test/expected/pg_dump.out @@ -58,7 +58,8 @@ BEGIN RETURN -1; END $BODY$; -SELECT * FROM set_adaptive_chunk_sizing('"test_schema"."two_Partitions"', '1 MB', 'custom_calculate_chunk_interval'); +SELECT * FROM set_adaptive_chunking('"test_schema"."two_Partitions"', '1 MB', 'custom_calculate_chunk_interval'); +WARNING: target chunk size for adaptive chunking is less than 10 MB chunk_sizing_func | chunk_target_size ---------------------------------+------------------- custom_calculate_chunk_interval | 1048576 diff --git a/test/sql/chunk_adaptive.sql b/test/sql/chunk_adaptive.sql index 320e0e70c..1d98c676d 100644 --- a/test/sql/chunk_adaptive.sql +++ b/test/sql/chunk_adaptive.sql @@ -54,42 +54,45 @@ SELECT table_name, chunk_sizing_func_schema, chunk_sizing_func_name, chunk_targe FROM _timescaledb_catalog.hypertable; -- Change the target size -SELECT * FROM set_adaptive_chunk_sizing('test_adaptive', '2MB'); +SELECT * FROM set_adaptive_chunking('test_adaptive', '2MB'); SELECT table_name, chunk_sizing_func_schema, chunk_sizing_func_name, chunk_target_size FROM _timescaledb_catalog.hypertable; \set ON_ERROR_STOP 0 -- Setting NULL func should fail -SELECT * FROM set_adaptive_chunk_sizing('test_adaptive', '1MB', NULL); +SELECT * FROM set_adaptive_chunking('test_adaptive', '1MB', NULL); \set ON_ERROR_STOP 1 -- Setting NULL size disables adaptive chunking -SELECT * FROM set_adaptive_chunk_sizing('test_adaptive', NULL); +SELECT * FROM set_adaptive_chunking('test_adaptive', NULL); SELECT table_name, chunk_sizing_func_schema, chunk_sizing_func_name, chunk_target_size FROM _timescaledb_catalog.hypertable; -SELECT * FROM set_adaptive_chunk_sizing('test_adaptive', '1MB'); +SELECT * FROM set_adaptive_chunking('test_adaptive', '1MB'); -- Setting size to 'off' should also disable -SELECT * FROM set_adaptive_chunk_sizing('test_adaptive', 'off'); +SELECT * FROM set_adaptive_chunking('test_adaptive', 'off'); SELECT table_name, chunk_sizing_func_schema, chunk_sizing_func_name, chunk_target_size FROM _timescaledb_catalog.hypertable; --- Setting 0 size should do an estimate. -SELECT * FROM set_adaptive_chunk_sizing('test_adaptive', '0MB'); +-- Setting 0 size should also disable +SELECT * FROM set_adaptive_chunking('test_adaptive', '0MB'); SELECT table_name, chunk_sizing_func_schema, chunk_sizing_func_name, chunk_target_size FROM _timescaledb_catalog.hypertable; -SELECT * FROM set_adaptive_chunk_sizing('test_adaptive', '1MB'); +SELECT * FROM set_adaptive_chunking('test_adaptive', '1MB'); + +-- No warning about small target size if > 10MB +SELECT * FROM set_adaptive_chunking('test_adaptive', '11MB'); -- Setting size to 'estimate' should also estimate size -SELECT * FROM set_adaptive_chunk_sizing('test_adaptive', 'estimate'); +SELECT * FROM set_adaptive_chunking('test_adaptive', 'estimate'); SELECT table_name, chunk_sizing_func_schema, chunk_sizing_func_name, chunk_target_size FROM _timescaledb_catalog.hypertable; -- Use a lower memory setting to test that the calculated chunk_target_size is reduced SELECT * FROM test.set_effective_memory_cache_size('512MB'); -SELECT * FROM set_adaptive_chunk_sizing('test_adaptive', 'estimate'); +SELECT * FROM set_adaptive_chunking('test_adaptive', 'estimate'); SELECT table_name, chunk_sizing_func_schema, chunk_sizing_func_name, chunk_target_size FROM _timescaledb_catalog.hypertable; @@ -97,7 +100,7 @@ FROM _timescaledb_catalog.hypertable; SELECT * FROM test.set_effective_memory_cache_size('2GB'); -- Set a reasonable test value -SELECT * FROM set_adaptive_chunk_sizing('test_adaptive', '1MB'); +SELECT * FROM set_adaptive_chunking('test_adaptive', '1MB'); -- Show the interval length before and after adaptation SELECT id, hypertable_id, interval_length FROM _timescaledb_catalog.dimension; @@ -118,6 +121,7 @@ SELECT * FROM chunk_relation_size('test_adaptive'); CREATE TABLE test_adaptive_no_index(time timestamptz, temp float, location int); -- Size but no explicit func should use default func +-- No default indexes should warn and use heap scan for min and max SELECT create_hypertable('test_adaptive_no_index', 'time', chunk_target_size => '1MB', create_default_indexes => false); diff --git a/test/sql/pg_dump.sql b/test/sql/pg_dump.sql index 1f15ff3e7..d16f20941 100644 --- a/test/sql/pg_dump.sql +++ b/test/sql/pg_dump.sql @@ -34,7 +34,7 @@ BEGIN END $BODY$; -SELECT * FROM set_adaptive_chunk_sizing('"test_schema"."two_Partitions"', '1 MB', 'custom_calculate_chunk_interval'); +SELECT * FROM set_adaptive_chunking('"test_schema"."two_Partitions"', '1 MB', 'custom_calculate_chunk_interval'); -- Chunk sizing func set SELECT * FROM _timescaledb_catalog.hypertable;