Fix ANALYZE on replicated distributed hypertable

With replicated chunks, the function to import column stats would
experience errors when updating `pg_statistics`, since it tried to
write identical stats from several replica chunks.

This change fixes this issue by filtering duplicate stats rows
received from data nodes.

In the future, this could be improved by only requesting stats from
"primary" chunks on each data node, thus avoiding duplicates without
having to filter the result. However, this would complicate the
function interface as it would require sending a list of chunks
instead of just getting the stats for all chunks in a hypertable.

tsl/src/chunk_api.c

@@ -958,8 +958,8 @@ chunk_update_colstats(Chunk *chunk, int16 attnum, float nullfract, int32 width,
 	if (HeapTupleIsValid(oldtup))
 	{
 		stup = heap_modify_tuple(oldtup, RelationGetDescr(sd), values, nulls, replaces);
+		CatalogTupleUpdate(sd, &oldtup->t_self, stup);
 		ReleaseSysCache(oldtup);
-		CatalogTupleUpdate(sd, &stup->t_self, stup);
 	}
 	else
 	{
@@ -970,13 +970,78 @@ chunk_update_colstats(Chunk *chunk, int16 attnum, float nullfract, int32 width,
 	heap_freetuple(stup);
 
 	relation_close(sd, RowExclusiveLock);
-
 	relation_close(rel, ShareUpdateExclusiveLock);
 }
 
+/*
+ * StatsProcessContext filters out duplicate stats from replica chunks.
+ *
+ * When processing chunk stats from data nodes, we might receive the same
+ * stats from multiple data nodes when native replication is enabled. With the
+ * StatsProcessContext we can filter out the duplicates, and ensure we only
+ * add the stats once. Without the filtering, we will get errors (e.g., unique
+ * violations).
+ *
+ * We could elide the filtering if we requested stats only for the chunks that
+ * are "primary", but that requires the ability to specify the specific remote
+ * chunks to retrieve stats for rather than specifying "all chunks" for the
+ * given hypertable.
+ */
+typedef struct ChunkAttKey
+{
+	Oid chunk_relid;
+	Index attnum;
+} ChunkAttKey;
+
+typedef struct StatsProcessContext
+{
+	HTAB *htab;
+} StatsProcessContext;
+
 static void
-chunk_process_remote_colstats_row(TupleFactory *tf, TupleDesc tupdesc, PGresult *res, int row,
+stats_process_context_init(StatsProcessContext *ctx, long nstats)
-								  const char *node_name)
+{
+	HASHCTL ctl;
+
+	MemSet(&ctl, 0, sizeof(ctl));
+	ctl.keysize = sizeof(ChunkAttKey);
+	ctl.entrysize = sizeof(ChunkAttKey);
+
+	ctl.hcxt = CurrentMemoryContext;
+	ctx->htab =
+		hash_create("StatsProcessContext", nstats, &ctl, HASH_ELEM | HASH_BLOBS | HASH_CONTEXT);
+}
+
+static bool
+stats_process_context_add_chunk_attributed(StatsProcessContext *ctx, Oid relid, Index attnum)
+{
+	ChunkAttKey key = {
+		.chunk_relid = relid,
+		.attnum = attnum,
+	};
+	ChunkAttKey *entry;
+	bool found;
+
+	entry = hash_search(ctx->htab, &key, HASH_ENTER, &found);
+
+	if (!found)
+	{
+		entry->chunk_relid = relid;
+		entry->attnum = attnum;
+	}
+
+	return found;
+}
+
+static void
+stats_process_context_finish(StatsProcessContext *ctx)
+{
+	hash_destroy(ctx->htab);
+}
+
+static void
+chunk_process_remote_colstats_row(StatsProcessContext *ctx, TupleFactory *tf, TupleDesc tupdesc,
+								  PGresult *res, int row, const char *node_name)
 {
 	Datum values[_Anum_chunk_colstats_max];
 	bool nulls[_Anum_chunk_colstats_max] = { false };
@@ -1017,6 +1082,12 @@ chunk_process_remote_colstats_row(TupleFactory *tf, TupleDesc tupdesc, PGresult
 	slot_kinds = (int *) ARR_DATA_PTR(kind_array);
 	os_idx = 1;
 	vt_idx = 1;
+
+	/* Filter out chunk cols we've already added. This happens when there are
+	 * replica chunks */
+	if (stats_process_context_add_chunk_attributed(ctx, chunk->table_id, col_id))
+		return;
+
 	for (i = 0; i < STATISTIC_NUM_SLOTS; ++i)
 	{
 		Datum strings[STRINGS_PER_OP_OID];
@@ -1173,10 +1244,13 @@ chunk_process_remote_relstats_row(TupleFactory *tf, TupleDesc tupdesc, PGresult
 static void
 fetch_remote_chunk_stats(Hypertable *ht, FunctionCallInfo fcinfo, bool col_stats)
 {
+	StatsProcessContext statsctx;
 	DistCmdResult *cmdres;
 	TupleDesc tupdesc;
 	TupleFactory *tf;
 	Size i;
+	long num_rows;
+	long num_stats;
 
 	Assert(hypertable_is_distributed(ht));
 
@@ -1187,9 +1261,17 @@ fetch_remote_chunk_stats(Hypertable *ht, FunctionCallInfo fcinfo, bool col_stats
 						"that cannot accept type record")));
 
 	cmdres = ts_dist_cmd_invoke_func_call_on_all_data_nodes(fcinfo);
+
 	/* Expect TEXT response format since dist command API currently defaults
 	 * to requesting TEXT */
 	tf = tuplefactory_create_for_tupdesc(tupdesc, true);
+	num_rows = ts_dist_cmd_total_row_count(cmdres);
+	/* Estimate the number of non-duplicate stats to use for initial size of
+	 * StatsProcessContext. Use slightly bigger than strictly necessary to
+	 * avoid a resize. */
+	num_stats = (5 * num_rows) / (ht->fd.replication_factor * 4);
+
+	stats_process_context_init(&statsctx, num_stats);
 
 	for (i = 0; /* exit when res == NULL below */; i++)
 	{
@@ -1204,12 +1286,17 @@ fetch_remote_chunk_stats(Hypertable *ht, FunctionCallInfo fcinfo, bool col_stats
 
 		if (col_stats)
 			for (row = 0; row < PQntuples(res); row++)
-				chunk_process_remote_colstats_row(tf, tupdesc, res, row, node_name);
+				chunk_process_remote_colstats_row(&statsctx, tf, tupdesc, res, row, node_name);
 		else
 			for (row = 0; row < PQntuples(res); row++)
 				chunk_process_remote_relstats_row(tf, tupdesc, res, row, node_name);
+
+		/* Early cleanup of PGresult protects against ballooning memory usage
+		 * when there are a lot of rows */
+		ts_dist_cmd_clear_result_by_index(cmdres, i);
 	}
 
+	stats_process_context_finish(&statsctx);
 	ts_dist_cmd_close_response(cmdres);
 }
 

tsl/src/remote/dist_commands.c

@@ -254,6 +254,22 @@ ts_dist_cmd_response_count(DistCmdResult *result)
 	return result->num_responses;
 }
 
+long
+ts_dist_cmd_total_row_count(DistCmdResult *result)
+{
+	int i;
+	long num_rows = 0;
+
+	for (i = 0; i < result->num_responses; ++i)
+	{
+		DistCmdResponse *resp = &result->responses[i];
+
+		num_rows += PQntuples(async_response_result_get_pg_result(resp->result));
+	}
+
+	return num_rows;
+}
+
 /*
  * Convert an expected scalar return value.
  *
@@ -305,18 +321,36 @@ ts_dist_cmd_get_single_scalar_result_by_index(DistCmdResult *result, Size index,
 	return OidInputFunctionCall(typinfunc, PQgetvalue(pgres, 0, 0), typioparam, -1);
 }
 
+void
+ts_dist_cmd_clear_result_by_index(DistCmdResult *response, Size index)
+{
+	DistCmdResponse *resp;
+
+	if (index >= response->num_responses)
+		elog(ERROR, "no response for index %zu", index);
+
+	resp = &response->responses[index];
+
+	if (resp->result != NULL)
+	{
+		async_response_result_close(resp->result);
+		resp->result = NULL;
+	}
+
+	if (resp->data_node != NULL)
+	{
+		pfree((char *) resp->data_node);
+		resp->data_node = NULL;
+	}
+}
+
 void
 ts_dist_cmd_close_response(DistCmdResult *response)
 {
-	int i;
+	Size i;
 
 	for (i = 0; i < response->num_responses; ++i)
-	{
+		ts_dist_cmd_clear_result_by_index(response, i);
-		DistCmdResponse *resp = &response->responses[i];
-
-		async_response_result_close(resp->result);
-		pfree((char *) resp->data_node);
-	}
 
 	pfree(response);
 }

tsl/src/remote/dist_commands.h

@@ -29,8 +29,9 @@ extern PGresult *ts_dist_cmd_get_result_by_node_name(DistCmdResult *response,
 													 const char *node_name);
 extern PGresult *ts_dist_cmd_get_result_by_index(DistCmdResult *response, Size index,
 												 const char **node_name);
+extern void ts_dist_cmd_clear_result_by_index(DistCmdResult *response, Size index);
 extern Size ts_dist_cmd_response_count(DistCmdResult *result);
-
+extern long ts_dist_cmd_total_row_count(DistCmdResult *result);
 extern void ts_dist_cmd_close_response(DistCmdResult *response);
 
 #define ts_dist_cmd_run_on_data_nodes(command, nodes)                                              \

tsl/test/expected/chunk_api.out

@@ -26,7 +26,8 @@ NOTICE:  adding not-null constraint to column "time"
 
 INSERT INTO chunkapi VALUES ('2018-01-01 05:00:00-8', 1, 23.4);
 SELECT (_timescaledb_internal.show_chunk(show_chunks)).*
-FROM show_chunks('chunkapi');
+FROM show_chunks('chunkapi')
+ORDER BY chunk_id;
  chunk_id | hypertable_id |      schema_name      |    table_name    | relkind |                                            slices                                            
 ----------+---------------+-----------------------+------------------+---------+----------------------------------------------------------------------------------------------
         1 |             1 | _timescaledb_internal | _hyper_1_1_chunk | r       | {"time": [1514419200000000, 1515024000000000], "device": [-9223372036854775808, 1073741823]}
@@ -91,7 +92,8 @@ SELECT * FROM _timescaledb_internal.create_chunk('chunkapi',' {"time": [15150240
 (1 row)
 
 SELECT (_timescaledb_internal.show_chunk(show_chunks)).*
-FROM show_chunks('chunkapi');
+FROM show_chunks('chunkapi')
+ORDER BY chunk_id;
  chunk_id | hypertable_id |      schema_name      |     table_name      | relkind |                                            slices                                            
 ----------+---------------+-----------------------+---------------------+---------+----------------------------------------------------------------------------------------------
         1 |             1 | _timescaledb_internal | _hyper_1_1_chunk    | r       | {"time": [1514419200000000, 1515024000000000], "device": [-9223372036854775808, 1073741823]}
@@ -120,11 +122,12 @@ SELECT setseed(1);
  
 (1 row)
 
--- Test getting relation stats for chunk.  First get stats
+-- Test getting relation stats for chunks.  First get stats
 -- chunk-by-chunk. Note that the table isn't ANALYZED, so no stats
 -- present yet.
 SELECT (_timescaledb_internal.get_chunk_relstats(show_chunks)).*
-FROM show_chunks('chunkapi');
+FROM show_chunks('chunkapi')
+ORDER BY chunk_id;
  chunk_id | hypertable_id | num_pages | num_tuples | num_allvisible 
 ----------+---------------+-----------+------------+----------------
         1 |             1 |         0 |          0 |              0
@@ -132,7 +135,8 @@ FROM show_chunks('chunkapi');
 (2 rows)
 
 SELECT (_timescaledb_internal.get_chunk_colstats(show_chunks)).*
-FROM show_chunks('chunkapi');
+FROM show_chunks('chunkapi')
+ORDER BY chunk_id;
  chunk_id | hypertable_id | att_num | nullfrac | width | distinctval | slotkind | slotopstrings | slot1numbers | slot2numbers | slot3numbers | slot4numbers | slot5numbers | slotvaluetypetrings | slot1values | slot2values | slot3values | slot4values | slot5values 
 ----------+---------------+---------+----------+-------+-------------+----------+---------------+--------------+--------------+--------------+--------------+--------------+---------------------+-------------+-------------+-------------+-------------+-------------
 (0 rows)
@@ -151,16 +155,20 @@ SELECT * FROM _timescaledb_internal.get_chunk_colstats('chunkapi');
 (0 rows)
 
 SELECT relname, reltuples, relpages, relallvisible FROM pg_class WHERE relname IN
-(SELECT (_timescaledb_internal.show_chunk(show_chunks)).table_name FROM show_chunks('chunkapi'));
+(SELECT (_timescaledb_internal.show_chunk(show_chunks)).table_name
+ FROM show_chunks('chunkapi'))
+ORDER BY relname;
        relname       | reltuples | relpages | relallvisible 
 ---------------------+-----------+----------+---------------
- _hyper_1_1_chunk    |         0 |        0 |             0
  My_chunk_Table_name |         0 |        0 |             0
+ _hyper_1_1_chunk    |         0 |        0 |             0
 (2 rows)
 
 SELECT tablename, attname, inherited, null_frac, avg_width, n_distinct
 FROM pg_stats WHERE tablename IN
-(SELECT (_timescaledb_internal.show_chunk(show_chunks)).table_name FROM show_chunks('chunkapi'));
+(SELECT (_timescaledb_internal.show_chunk(show_chunks)).table_name
+ FROM show_chunks('chunkapi'))
+ORDER BY tablename, attname;
  tablename | attname | inherited | null_frac | avg_width | n_distinct 
 -----------+---------+-----------+-----------+-----------+------------
 (0 rows)
@@ -183,20 +191,24 @@ SELECT * FROM _timescaledb_internal.get_chunk_colstats('chunkapi');
 (3 rows)
 
 SELECT relname, reltuples, relpages, relallvisible FROM pg_class WHERE relname IN
-(SELECT (_timescaledb_internal.show_chunk(show_chunks)).table_name FROM show_chunks('chunkapi'));
+(SELECT (_timescaledb_internal.show_chunk(show_chunks)).table_name
+ FROM show_chunks('chunkapi'))
+ORDER BY relname;
        relname       | reltuples | relpages | relallvisible 
 ---------------------+-----------+----------+---------------
- _hyper_1_1_chunk    |         1 |        1 |             0
  My_chunk_Table_name |         0 |        0 |             0
+ _hyper_1_1_chunk    |         1 |        1 |             0
 (2 rows)
 
 SELECT tablename, attname, inherited, null_frac, avg_width, n_distinct
 FROM pg_stats WHERE tablename IN
-(SELECT (_timescaledb_internal.show_chunk(show_chunks)).table_name FROM show_chunks('chunkapi'));
+(SELECT (_timescaledb_internal.show_chunk(show_chunks)).table_name
+FROM show_chunks('chunkapi'))
+ORDER BY tablename, attname;
     tablename     | attname | inherited | null_frac | avg_width | n_distinct 
 ------------------+---------+-----------+-----------+-----------+------------
- _hyper_1_1_chunk | temp    | f         |         0 |         8 |         -1
  _hyper_1_1_chunk | device  | f         |         0 |         4 |         -1
+ _hyper_1_1_chunk | temp    | f         |         0 |         8 |         -1
  _hyper_1_1_chunk | time    | f         |         0 |         8 |         -1
 (3 rows)
 
@@ -256,7 +268,9 @@ SELECT * FROM _timescaledb_internal.get_chunk_colstats('disttable');
 (0 rows)
 
 SELECT relname, reltuples, relpages, relallvisible FROM pg_class WHERE relname IN
-(SELECT (_timescaledb_internal.show_chunk(show_chunks)).table_name FROM show_chunks('disttable'));
+(SELECT (_timescaledb_internal.show_chunk(show_chunks)).table_name
+ FROM show_chunks('disttable'))
+ORDER BY relname;
         relname        | reltuples | relpages | relallvisible 
 -----------------------+-----------+----------+---------------
  _dist_hyper_2_3_chunk |         0 |        0 |             0
@@ -264,7 +278,8 @@ SELECT relname, reltuples, relpages, relallvisible FROM pg_class WHERE relname I
 (2 rows)
 
 SELECT * FROM pg_stats WHERE tablename IN
-(SELECT (_timescaledb_internal.show_chunk(show_chunks)).table_name FROM show_chunks('disttable'))
+(SELECT (_timescaledb_internal.show_chunk(show_chunks)).table_name
+ FROM show_chunks('disttable'))
 ORDER BY 1,2,3;
  schemaname | tablename | attname | inherited | null_frac | avg_width | n_distinct | most_common_vals | most_common_freqs | histogram_bounds | correlation | most_common_elems | most_common_elem_freqs | elem_count_histogram 
 ------------+-----------+---------+-----------+-----------+-----------+------------+------------------+-------------------+------------------+-------------+-------------------+------------------------+----------------------
@@ -279,7 +294,8 @@ SELECT * FROM distributed_exec('ANALYZE disttable', '{ "data_node_1" }');
 
 -- Stats should now be refreshed after running get_chunk_{col,rel}stats
 SELECT relname, reltuples, relpages, relallvisible FROM pg_class WHERE relname IN
-(SELECT (_timescaledb_internal.show_chunk(show_chunks)).table_name FROM show_chunks('disttable'));
+(SELECT (_timescaledb_internal.show_chunk(show_chunks)).table_name
+ FROM show_chunks('disttable'));
         relname        | reltuples | relpages | relallvisible 
 -----------------------+-----------+----------+---------------
  _dist_hyper_2_3_chunk |         0 |        0 |             0
@@ -287,7 +303,8 @@ SELECT relname, reltuples, relpages, relallvisible FROM pg_class WHERE relname I
 (2 rows)
 
 SELECT * FROM pg_stats WHERE tablename IN
-(SELECT (_timescaledb_internal.show_chunk(show_chunks)).table_name FROM show_chunks('disttable'))
+(SELECT (_timescaledb_internal.show_chunk(show_chunks)).table_name
+ FROM show_chunks('disttable'))
 ORDER BY 1,2,3;
  schemaname | tablename | attname | inherited | null_frac | avg_width | n_distinct | most_common_vals | most_common_freqs | histogram_bounds | correlation | most_common_elems | most_common_elem_freqs | elem_count_histogram 
 ------------+-----------+---------+-----------+-----------+-----------+------------+------------------+-------------------+------------------+-------------+-------------------+------------------------+----------------------
@@ -310,7 +327,9 @@ SELECT * FROM _timescaledb_internal.get_chunk_colstats('disttable');
 (4 rows)
 
 SELECT relname, reltuples, relpages, relallvisible FROM pg_class WHERE relname IN
-(SELECT (_timescaledb_internal.show_chunk(show_chunks)).table_name FROM show_chunks('disttable'));
+(SELECT (_timescaledb_internal.show_chunk(show_chunks)).table_name
+ FROM show_chunks('disttable'))
+ORDER BY relname;
         relname        | reltuples | relpages | relallvisible 
 -----------------------+-----------+----------+---------------
  _dist_hyper_2_3_chunk |         2 |        1 |             0
@@ -318,7 +337,8 @@ SELECT relname, reltuples, relpages, relallvisible FROM pg_class WHERE relname I
 (2 rows)
 
 SELECT * FROM pg_stats WHERE tablename IN
-(SELECT (_timescaledb_internal.show_chunk(show_chunks)).table_name FROM show_chunks('disttable'))
+(SELECT (_timescaledb_internal.show_chunk(show_chunks)).table_name
+ FROM show_chunks('disttable'))
 ORDER BY 1,2,3;
       schemaname       |       tablename       | attname | inherited | null_frac | avg_width | n_distinct | most_common_vals | most_common_freqs |                        histogram_bounds                         | correlation | most_common_elems | most_common_elem_freqs | elem_count_histogram 
 -----------------------+-----------------------+---------+-----------+-----------+-----------+------------+------------------+-------------------+-----------------------------------------------------------------+-------------+-------------------+------------------------+----------------------
@@ -336,13 +356,8 @@ SELECT * FROM _timescaledb_internal.get_chunk_colstats('disttable');
 (0 rows)
 
 SET ROLE :ROLE_1;
--- Run ANALYZE again, but on both nodes
+-- Run ANALYZE again, but on both nodes.
-SELECT * FROM distributed_exec('ANALYZE disttable');
+ANALYZE disttable;
- distributed_exec 
-------------------
- 
-(1 row)
-
 -- Now expect stats from all data node chunks
 SELECT * FROM _timescaledb_internal.get_chunk_relstats('disttable');
  chunk_id | hypertable_id | num_pages | num_tuples | num_allvisible 
@@ -364,16 +379,34 @@ SELECT * FROM _timescaledb_internal.get_chunk_colstats('disttable');
         4 |             2 |       4 |        1 |     0 |           0 | {0,0,0,0,0} | {}                                                                                                                      |              |              |              |              |              | {}                       |                                                                 |             |             |             | 
 (8 rows)
 
+-- Test ANALYZE with a replica chunk. We'd like to ensure the
+-- stats-fetching functions handle duplicate stats from different (but
+-- identical) replica chunks.
+SELECT set_replication_factor('disttable', 2);
+WARNING:  hypertable "disttable" is under-replicated
+ set_replication_factor 
+------------------------
+ 
+(1 row)
+
+INSERT INTO disttable VALUES ('2019-01-01 05:00:00-8', 1, 23.4, 'green');
+-- Run twice to test that stats-fetching functions handle replica chunks.
+ANALYZE disttable;
+ANALYZE disttable;
 SELECT relname, reltuples, relpages, relallvisible FROM pg_class WHERE relname IN
-(SELECT (_timescaledb_internal.show_chunk(show_chunks)).table_name FROM show_chunks('disttable'));
+(SELECT (_timescaledb_internal.show_chunk(show_chunks)).table_name
+ FROM show_chunks('disttable'))
+ORDER BY relname;
         relname        | reltuples | relpages | relallvisible 
 -----------------------+-----------+----------+---------------
  _dist_hyper_2_3_chunk |         2 |        1 |             0
  _dist_hyper_2_4_chunk |         1 |        1 |             0
-(2 rows)
+ _dist_hyper_2_5_chunk |         1 |        1 |             0
+(3 rows)
 
 SELECT * FROM pg_stats WHERE tablename IN
-(SELECT (_timescaledb_internal.show_chunk(show_chunks)).table_name FROM show_chunks('disttable'))
+(SELECT (_timescaledb_internal.show_chunk(show_chunks)).table_name
+ FROM show_chunks('disttable'))
 ORDER BY 1,2,3;
       schemaname       |       tablename       | attname | inherited | null_frac | avg_width | n_distinct | most_common_vals | most_common_freqs |                        histogram_bounds                         | correlation | most_common_elems | most_common_elem_freqs | elem_count_histogram 
 -----------------------+-----------------------+---------+-----------+-----------+-----------+------------+------------------+-------------------+-----------------------------------------------------------------+-------------+-------------------+------------------------+----------------------
@@ -385,7 +418,11 @@ ORDER BY 1,2,3;
  _timescaledb_internal | _dist_hyper_2_4_chunk | device  | f         |         0 |         4 |         -1 |                  |                   |                                                                 |             |                   |                        | 
  _timescaledb_internal | _dist_hyper_2_4_chunk | temp    | f         |         0 |         8 |         -1 |                  |                   |                                                                 |             |                   |                        | 
  _timescaledb_internal | _dist_hyper_2_4_chunk | time    | f         |         0 |         8 |         -1 |                  |                   |                                                                 |             |                   |                        | 
-(8 rows)
+ _timescaledb_internal | _dist_hyper_2_5_chunk | color   | f         |         0 |         6 |         -1 |                  |                   |                                                                 |             |                   |                        | 
+ _timescaledb_internal | _dist_hyper_2_5_chunk | device  | f         |         0 |         4 |         -1 |                  |                   |                                                                 |             |                   |                        | 
+ _timescaledb_internal | _dist_hyper_2_5_chunk | temp    | f         |         0 |         8 |         -1 |                  |                   |                                                                 |             |                   |                        | 
+ _timescaledb_internal | _dist_hyper_2_5_chunk | time    | f         |         0 |         8 |         -1 |                  |                   |                                                                 |             |                   |                        | 
+(12 rows)
 
 -- Check underlying pg_statistics table (looking at all columns except
 -- starelid, which changes depending on how many tests are run before
@@ -406,7 +443,11 @@ ORDER BY ch, staattnum;
  _timescaledb_internal._dist_hyper_2_4_chunk |         2 | f          |           0 |        4 |          -1 |        0 |        0 |        0 |        0 |        0 |      0 |      0 |      0 |      0 |      0 |             |             |             |             |             |                                                                 |            |            |            | 
  _timescaledb_internal._dist_hyper_2_4_chunk |         3 | f          |           0 |        8 |          -1 |        0 |        0 |        0 |        0 |        0 |      0 |      0 |      0 |      0 |      0 |             |             |             |             |             |                                                                 |            |            |            | 
  _timescaledb_internal._dist_hyper_2_4_chunk |         4 | f          |           1 |        0 |           0 |        0 |        0 |        0 |        0 |        0 |      0 |      0 |      0 |      0 |      0 |             |             |             |             |             |                                                                 |            |            |            | 
-(8 rows)
+ _timescaledb_internal._dist_hyper_2_5_chunk |         1 | f          |           0 |        8 |          -1 |        0 |        0 |        0 |        0 |        0 |      0 |      0 |      0 |      0 |      0 |             |             |             |             |             |                                                                 |            |            |            | 
+ _timescaledb_internal._dist_hyper_2_5_chunk |         2 | f          |           0 |        4 |          -1 |        0 |        0 |        0 |        0 |        0 |      0 |      0 |      0 |      0 |      0 |             |             |             |             |             |                                                                 |            |            |            | 
+ _timescaledb_internal._dist_hyper_2_5_chunk |         3 | f          |           0 |        8 |          -1 |        0 |        0 |        0 |        0 |        0 |      0 |      0 |      0 |      0 |      0 |             |             |             |             |             |                                                                 |            |            |            | 
+ _timescaledb_internal._dist_hyper_2_5_chunk |         4 | f          |           0 |        6 |          -1 |        0 |        0 |        0 |        0 |        0 |      0 |      0 |      0 |      0 |      0 |             |             |             |             |             |                                                                 |            |            |            | 
+(12 rows)
 
 SELECT test.remote_exec(NULL, $$
 SELECT ch, staattnum, stainherit, stanullfrac, stawidth, stadistinct, stakind1, stakind2, stakind3, stakind4, stakind5, staop1, staop2, staop3, staop4, staop5,
@@ -428,7 +469,11 @@ _timescaledb_internal._dist_hyper_2_3_chunk|        1|f         |          0|
 _timescaledb_internal._dist_hyper_2_3_chunk|        2|f         |          0|       4|       -0.5|       1|       3|       0|       0|       0|    96|    97|     0|     0|     0|{1}        |{1}        |           |           |           |{1}                                                            |          |          |          |          
 _timescaledb_internal._dist_hyper_2_3_chunk|        3|f         |          0|       8|         -1|       2|       3|       0|       0|       0|   672|   672|     0|     0|     0|           |{-1}       |           |           |           |{21.1,23.4}                                                    |          |          |          |          
 _timescaledb_internal._dist_hyper_2_3_chunk|        4|f         |          0|       6|       -0.5|       1|       3|       0|       0|       0|    98|   664|     0|     0|     0|{1}        |{1}        |           |           |           |{green}                                                        |          |          |          |          
-(4 rows)
+_timescaledb_internal._dist_hyper_2_5_chunk|        1|f         |          0|       8|         -1|       0|       0|       0|       0|       0|     0|     0|     0|     0|     0|           |           |           |           |           |                                                               |          |          |          |          
+_timescaledb_internal._dist_hyper_2_5_chunk|        2|f         |          0|       4|         -1|       0|       0|       0|       0|       0|     0|     0|     0|     0|     0|           |           |           |           |           |                                                               |          |          |          |          
+_timescaledb_internal._dist_hyper_2_5_chunk|        3|f         |          0|       8|         -1|       0|       0|       0|       0|       0|     0|     0|     0|     0|     0|           |           |           |           |           |                                                               |          |          |          |          
+_timescaledb_internal._dist_hyper_2_5_chunk|        4|f         |          0|       6|         -1|       0|       0|       0|       0|       0|     0|     0|     0|     0|     0|           |           |           |           |           |                                                               |          |          |          |          
+(8 rows)
 
 
 NOTICE:  [data_node_2]: 
@@ -444,7 +489,11 @@ _timescaledb_internal._dist_hyper_2_4_chunk|        1|f         |          0|
 _timescaledb_internal._dist_hyper_2_4_chunk|        2|f         |          0|       4|         -1|       0|       0|       0|       0|       0|     0|     0|     0|     0|     0|           |           |           |           |           |          |          |          |          |          
 _timescaledb_internal._dist_hyper_2_4_chunk|        3|f         |          0|       8|         -1|       0|       0|       0|       0|       0|     0|     0|     0|     0|     0|           |           |           |           |           |          |          |          |          |          
 _timescaledb_internal._dist_hyper_2_4_chunk|        4|f         |          1|       0|          0|       0|       0|       0|       0|       0|     0|     0|     0|     0|     0|           |           |           |           |           |          |          |          |          |          
-(4 rows)
+_timescaledb_internal._dist_hyper_2_5_chunk|        1|f         |          0|       8|         -1|       0|       0|       0|       0|       0|     0|     0|     0|     0|     0|           |           |           |           |           |          |          |          |          |          
+_timescaledb_internal._dist_hyper_2_5_chunk|        2|f         |          0|       4|         -1|       0|       0|       0|       0|       0|     0|     0|     0|     0|     0|           |           |           |           |           |          |          |          |          |          
+_timescaledb_internal._dist_hyper_2_5_chunk|        3|f         |          0|       8|         -1|       0|       0|       0|       0|       0|     0|     0|     0|     0|     0|           |           |           |           |           |          |          |          |          |          
+_timescaledb_internal._dist_hyper_2_5_chunk|        4|f         |          0|       6|         -1|       0|       0|       0|       0|       0|     0|     0|     0|     0|     0|           |           |           |           |           |          |          |          |          |          
+(8 rows)
 
 
  remote_exec 
@@ -458,6 +507,7 @@ RESET ROLE;
 TRUNCATE disttable;
 TRUNCATE costtable;
 SELECT * FROM delete_data_node('data_node_1', force => true);
+WARNING:  new data for hypertable "disttable" will be under-replicated due to deleting data node "data_node_1"
 NOTICE:  the number of partitions in dimension "device" was decreased to 1
  delete_data_node 
 ------------------

tsl/test/sql/chunk_api.sql

@@ -15,7 +15,8 @@ SELECT * FROM create_hypertable('chunkapi', 'time', 'device', 2);
 INSERT INTO chunkapi VALUES ('2018-01-01 05:00:00-8', 1, 23.4);
 
 SELECT (_timescaledb_internal.show_chunk(show_chunks)).*
-FROM show_chunks('chunkapi');
+FROM show_chunks('chunkapi')
+ORDER BY chunk_id;
 
 -- Creating a chunk with the constraints of an existing chunk should
 -- return the existing chunk
@@ -51,7 +52,8 @@ SET ROLE :ROLE_DEFAULT_PERM_USER;
 SELECT * FROM _timescaledb_internal.create_chunk('chunkapi',' {"time": [1515024000000000, 1519024000000000], "device": [-9223372036854775808, 1073741823]}', 'ChunkSchema', 'My_chunk_Table_name');
 
 SELECT (_timescaledb_internal.show_chunk(show_chunks)).*
-FROM show_chunks('chunkapi');
+FROM show_chunks('chunkapi')
+ORDER BY chunk_id;
 
 -- Show the new chunks
 \dt public.*
@@ -60,24 +62,30 @@ FROM show_chunks('chunkapi');
 -- Make ANALYZE deterministic
 SELECT setseed(1);
 
-
+-- Test getting relation stats for chunks.  First get stats
--- Test getting relation stats for chunk.  First get stats
 -- chunk-by-chunk. Note that the table isn't ANALYZED, so no stats
 -- present yet.
 SELECT (_timescaledb_internal.get_chunk_relstats(show_chunks)).*
-FROM show_chunks('chunkapi');
+FROM show_chunks('chunkapi')
+ORDER BY chunk_id;
 SELECT (_timescaledb_internal.get_chunk_colstats(show_chunks)).*
-FROM show_chunks('chunkapi');
+FROM show_chunks('chunkapi')
+ORDER BY chunk_id;
 
 -- Get the same stats but by giving the hypertable as input
 SELECT * FROM _timescaledb_internal.get_chunk_relstats('chunkapi');
 SELECT * FROM _timescaledb_internal.get_chunk_colstats('chunkapi');
 
 SELECT relname, reltuples, relpages, relallvisible FROM pg_class WHERE relname IN
-(SELECT (_timescaledb_internal.show_chunk(show_chunks)).table_name FROM show_chunks('chunkapi'));
+(SELECT (_timescaledb_internal.show_chunk(show_chunks)).table_name
+ FROM show_chunks('chunkapi'))
+ORDER BY relname;
+
 SELECT tablename, attname, inherited, null_frac, avg_width, n_distinct
 FROM pg_stats WHERE tablename IN
-(SELECT (_timescaledb_internal.show_chunk(show_chunks)).table_name FROM show_chunks('chunkapi'));
+(SELECT (_timescaledb_internal.show_chunk(show_chunks)).table_name
+ FROM show_chunks('chunkapi'))
+ORDER BY tablename, attname;
 
 -- Show stats after analyze
 ANALYZE chunkapi;
@@ -85,10 +93,15 @@ SELECT * FROM _timescaledb_internal.get_chunk_relstats('chunkapi');
 SELECT * FROM _timescaledb_internal.get_chunk_colstats('chunkapi');
 
 SELECT relname, reltuples, relpages, relallvisible FROM pg_class WHERE relname IN
-(SELECT (_timescaledb_internal.show_chunk(show_chunks)).table_name FROM show_chunks('chunkapi'));
+(SELECT (_timescaledb_internal.show_chunk(show_chunks)).table_name
+ FROM show_chunks('chunkapi'))
+ORDER BY relname;
+
 SELECT tablename, attname, inherited, null_frac, avg_width, n_distinct
 FROM pg_stats WHERE tablename IN
-(SELECT (_timescaledb_internal.show_chunk(show_chunks)).table_name FROM show_chunks('chunkapi'));
+(SELECT (_timescaledb_internal.show_chunk(show_chunks)).table_name
+FROM show_chunks('chunkapi'))
+ORDER BY tablename, attname;
 
 -- Test getting chunk stats on a distribute hypertable
 SET ROLE :ROLE_CLUSTER_SUPERUSER;
@@ -122,9 +135,12 @@ SELECT * FROM _timescaledb_internal.get_chunk_relstats('disttable');
 SELECT * FROM _timescaledb_internal.get_chunk_colstats('disttable');
 
 SELECT relname, reltuples, relpages, relallvisible FROM pg_class WHERE relname IN
-(SELECT (_timescaledb_internal.show_chunk(show_chunks)).table_name FROM show_chunks('disttable'));
+(SELECT (_timescaledb_internal.show_chunk(show_chunks)).table_name
+ FROM show_chunks('disttable'))
+ORDER BY relname;
 SELECT * FROM pg_stats WHERE tablename IN
-(SELECT (_timescaledb_internal.show_chunk(show_chunks)).table_name FROM show_chunks('disttable'))
+(SELECT (_timescaledb_internal.show_chunk(show_chunks)).table_name
+ FROM show_chunks('disttable'))
 ORDER BY 1,2,3;
 
 -- Run ANALYZE on data node 1
@@ -132,18 +148,24 @@ SELECT * FROM distributed_exec('ANALYZE disttable', '{ "data_node_1" }');
 
 -- Stats should now be refreshed after running get_chunk_{col,rel}stats
 SELECT relname, reltuples, relpages, relallvisible FROM pg_class WHERE relname IN
-(SELECT (_timescaledb_internal.show_chunk(show_chunks)).table_name FROM show_chunks('disttable'));
+(SELECT (_timescaledb_internal.show_chunk(show_chunks)).table_name
+ FROM show_chunks('disttable'));
 SELECT * FROM pg_stats WHERE tablename IN
-(SELECT (_timescaledb_internal.show_chunk(show_chunks)).table_name FROM show_chunks('disttable'))
+(SELECT (_timescaledb_internal.show_chunk(show_chunks)).table_name
+ FROM show_chunks('disttable'))
 ORDER BY 1,2,3;
 
 SELECT * FROM _timescaledb_internal.get_chunk_relstats('disttable');
 SELECT * FROM _timescaledb_internal.get_chunk_colstats('disttable');
 
 SELECT relname, reltuples, relpages, relallvisible FROM pg_class WHERE relname IN
-(SELECT (_timescaledb_internal.show_chunk(show_chunks)).table_name FROM show_chunks('disttable'));
+(SELECT (_timescaledb_internal.show_chunk(show_chunks)).table_name
+ FROM show_chunks('disttable'))
+ORDER BY relname;
+
 SELECT * FROM pg_stats WHERE tablename IN
-(SELECT (_timescaledb_internal.show_chunk(show_chunks)).table_name FROM show_chunks('disttable'))
+(SELECT (_timescaledb_internal.show_chunk(show_chunks)).table_name
+ FROM show_chunks('disttable'))
 ORDER BY 1,2,3;
 
 -- Test that user without table permissions can't get column stats
@@ -151,17 +173,29 @@ SET ROLE :ROLE_DEFAULT_PERM_USER;
 SELECT * FROM _timescaledb_internal.get_chunk_colstats('disttable');
 SET ROLE :ROLE_1;
 
--- Run ANALYZE again, but on both nodes
+-- Run ANALYZE again, but on both nodes.
-SELECT * FROM distributed_exec('ANALYZE disttable');
+ANALYZE disttable;
 
 -- Now expect stats from all data node chunks
 SELECT * FROM _timescaledb_internal.get_chunk_relstats('disttable');
 SELECT * FROM _timescaledb_internal.get_chunk_colstats('disttable');
 
+-- Test ANALYZE with a replica chunk. We'd like to ensure the
+-- stats-fetching functions handle duplicate stats from different (but
+-- identical) replica chunks.
+SELECT set_replication_factor('disttable', 2);
+INSERT INTO disttable VALUES ('2019-01-01 05:00:00-8', 1, 23.4, 'green');
+-- Run twice to test that stats-fetching functions handle replica chunks.
+ANALYZE disttable;
+ANALYZE disttable;
+
 SELECT relname, reltuples, relpages, relallvisible FROM pg_class WHERE relname IN
-(SELECT (_timescaledb_internal.show_chunk(show_chunks)).table_name FROM show_chunks('disttable'));
+(SELECT (_timescaledb_internal.show_chunk(show_chunks)).table_name
+ FROM show_chunks('disttable'))
+ORDER BY relname;
 SELECT * FROM pg_stats WHERE tablename IN
-(SELECT (_timescaledb_internal.show_chunk(show_chunks)).table_name FROM show_chunks('disttable'))
+(SELECT (_timescaledb_internal.show_chunk(show_chunks)).table_name
+ FROM show_chunks('disttable'))
 ORDER BY 1,2,3;
 
 -- Check underlying pg_statistics table (looking at all columns except