Support sending telemetry event reports

Add table `_timescaledb_catalog.telemetry_event` table containing
events that should be sent out with telemetry reports. The table will
be truncated after reporting being generated.

CHANGELOG.md

@@ -19,6 +19,7 @@ accidentally triggering the load of a previous DB version.**
 * #5510 Propagate vacuum/analyze to compressed chunks
 * #5584 Reduce decompression during constraint checking
 * #5530 Optimize compressed chunk resorting
+* #5639 Support sending telemetry event reports
 
 **Bugfixes**
 * #5396 Fix SEGMENTBY columns predicates to be pushed down

sql/pre_install/tables.sql

@@ -347,6 +347,15 @@ CREATE TABLE _timescaledb_catalog.metadata (
 SELECT pg_catalog.pg_extension_config_dump('_timescaledb_catalog.metadata', $$
   WHERE KEY = 'exported_uuid' $$);
 
+-- Log with events that will be sent out with the telemetry. The log
+-- will be flushed after it has been sent out. We do not save it to
+-- backups since it should not contain important data.
+CREATE TABLE _timescaledb_catalog.telemetry_event (
+       created timestamptz NOT NULL DEFAULT current_timestamp,
+       tag name NOT NULL,
+       body jsonb NOT NULL
+);
+
 CREATE TABLE _timescaledb_catalog.continuous_agg (
   mat_hypertable_id integer NOT NULL,
   raw_hypertable_id integer NOT NULL,

sql/updates/latest-dev.sql

@@ -59,3 +59,14 @@ ALTER TABLE _timescaledb_config.bgw_job
 ALTER TABLE _timescaledb_catalog.continuous_agg_migrate_plan
   ADD COLUMN user_view_definition TEXT,
   DROP CONSTRAINT continuous_agg_migrate_plan_mat_hypertable_id_fkey;
+
+-- Log with events that will be sent out with the telemetry. The log
+-- will be flushed after it has been sent out. We do not save it to
+-- backups since it should not contain important data.
+CREATE TABLE _timescaledb_catalog.telemetry_event (
+       created timestamptz NOT NULL DEFAULT current_timestamp,
+       tag name NOT NULL,
+       body jsonb NOT NULL
+);
+
+GRANT SELECT ON _timescaledb_catalog.telemetry_event TO PUBLIC;

sql/updates/reverse-dev.sql

@@ -226,3 +226,7 @@ ALTER TABLE _timescaledb_catalog.continuous_agg_migrate_plan_step
 SELECT pg_catalog.pg_extension_config_dump('_timescaledb_catalog.continuous_agg_migrate_plan', '');
 
 GRANT SELECT ON TABLE _timescaledb_catalog.continuous_agg_migrate_plan TO PUBLIC;
+
+ALTER EXTENSION timescaledb DROP TABLE _timescaledb_catalog.telemetry_event;
+
+DROP TABLE IF EXISTS _timescaledb_catalog.telemetry_event;

sql/with_telemetry.sql

@@ -2,8 +2,8 @@
 -- Please see the included NOTICE for copyright information and
 -- LICENSE-APACHE for a copy of the license.
 
-CREATE OR REPLACE FUNCTION @extschema@.get_telemetry_report() RETURNS jsonb
-    AS '@MODULE_PATHNAME@', 'ts_telemetry_get_report_jsonb'
+CREATE OR REPLACE FUNCTION @extschema@.get_telemetry_report()
+       RETURNS jsonb AS '@MODULE_PATHNAME@', 'ts_telemetry_get_report_jsonb'
        LANGUAGE C STABLE PARALLEL SAFE;
 
 INSERT INTO _timescaledb_config.bgw_job (id, application_name, schedule_interval, max_runtime, max_retries, retry_period, proc_schema, proc_name, owner, scheduled, fixed_schedule) VALUES

src/telemetry/telemetry.c

@@ -74,6 +74,7 @@
 #define REQ_NUM_USER_DEFINED_ACTIONS "num_user_defined_actions"
 #define REQ_RELATED_EXTENSIONS "related_extensions"
 #define REQ_METADATA "db_metadata"
+#define REQ_TELEMETRY_EVENT "db_telemetry_events"
 #define REQ_LICENSE_EDITION_APACHE "apache_only"
 #define REQ_LICENSE_EDITION_COMMUNITY "community"
 #define REQ_TS_LAST_TUNE_TIME "last_tuned_time"
@@ -956,6 +957,13 @@ build_telemetry_report()
 	ts_telemetry_metadata_add_values(parse_state);
 	pushJsonbValue(&parse_state, WJB_END_OBJECT, NULL);
 
+	/* Add telemetry events */
+	key.type = jbvString;
+	key.val.string.val = REQ_TELEMETRY_EVENT;
+	key.val.string.len = strlen(REQ_TELEMETRY_EVENT);
+	pushJsonbValue(&parse_state, WJB_KEY, &key);
+	ts_telemetry_events_add(parse_state);
+
 	/* Add function call telemetry */
 	key.type = jbvString;
 	key.val.string.val = REQ_FUNCTIONS_USED;
@@ -1101,6 +1109,7 @@ ts_telemetry_main(const char *host, const char *path, const char *service)
 	}
 
 	ts_function_telemetry_reset_counts();
+	ts_telemetry_event_truncate();
 
 	/*
 	 * Do the version-check. Response is the body of a well-formed HTTP

src/telemetry/telemetry_metadata.c

@@ -6,7 +6,9 @@
 #include <postgres.h>
 #include <catalog/pg_type.h>
 #include <utils/builtins.h>
+#include <utils/jsonb.h>
 #include <utils/timestamp.h>
+#include <commands/tablecmds.h>
 
 #include "ts_catalog/catalog.h"
 #include "ts_catalog/metadata.h"
@@ -15,6 +17,72 @@
 #include "scan_iterator.h"
 #include "jsonb_utils.h"
 
+#if PG14_LT
+/* Copied from jsonb_util.c */
+static void
+JsonbToJsonbValue(Jsonb *jsonb, JsonbValue *val)
+{
+	val->type = jbvBinary;
+	val->val.binary.data = &jsonb->root;
+	val->val.binary.len = VARSIZE(jsonb) - VARHDRSZ;
+}
+#endif
+
+void
+ts_telemetry_event_truncate(void)
+{
+	RangeVar rv = {
+		.schemaname = CATALOG_SCHEMA_NAME,
+		.relname = TELEMETRY_EVENT_TABLE_NAME,
+	};
+	ExecuteTruncate(&(TruncateStmt){
+		.type = T_TruncateStmt,
+		.relations = list_make1(&rv),
+		.behavior = DROP_RESTRICT,
+	});
+}
+
+void
+ts_telemetry_events_add(JsonbParseState *state)
+{
+	ScanIterator iterator =
+		ts_scan_iterator_create(TELEMETRY_EVENT, AccessShareLock, CurrentMemoryContext);
+	pushJsonbValue(&state, WJB_BEGIN_ARRAY, NULL);
+	ts_scanner_foreach(&iterator)
+	{
+		TupleInfo *ti = iterator.tinfo;
+		TupleDesc tupdesc = ti->slot->tts_tupleDescriptor;
+		bool created_isnull, tag_isnull, value_isnull;
+		Datum created = slot_getattr(ti->slot, Anum_telemetry_event_created, &created_isnull);
+		Datum tag = slot_getattr(ti->slot, Anum_telemetry_event_tag, &tag_isnull);
+		Datum body = slot_getattr(ti->slot, Anum_telemetry_event_body, &value_isnull);
+
+		pushJsonbValue(&state, WJB_BEGIN_OBJECT, NULL);
+		if (!created_isnull)
+			ts_jsonb_add_str(state,
+							 NameStr(
+								 TupleDescAttr(tupdesc, Anum_telemetry_event_created - 1)->attname),
+							 DatumGetCString(DirectFunctionCall1(timestamptz_out, created)));
+
+		if (!tag_isnull)
+			ts_jsonb_add_str(state,
+							 NameStr(TupleDescAttr(tupdesc, Anum_telemetry_event_tag - 1)->attname),
+							 NameStr(*DatumGetName(tag)));
+
+		if (!value_isnull)
+		{
+			JsonbValue jsonb_value;
+			JsonbToJsonbValue(DatumGetJsonbP(body), &jsonb_value);
+			ts_jsonb_add_value(state,
+							   NameStr(
+								   TupleDescAttr(tupdesc, Anum_telemetry_event_body - 1)->attname),
+							   &jsonb_value);
+		}
+		pushJsonbValue(&state, WJB_END_OBJECT, NULL);
+	}
+	pushJsonbValue(&state, WJB_END_ARRAY, NULL);
+}
+
 /*
  * add all entries from _timescaledb_catalog.metadata
  */

src/telemetry/telemetry_metadata.h

@@ -12,5 +12,7 @@
 #include <export.h>
 
 extern void ts_telemetry_metadata_add_values(JsonbParseState *state);
+extern void ts_telemetry_events_add(JsonbParseState *state);
+extern void ts_telemetry_event_truncate(void);
 
 #endif /* TIMESCALEDB_TELEMETRY_TELEMETRY_METADATA_H */

src/ts_catalog/catalog.c

@@ -124,6 +124,10 @@ static const TableInfoDef catalog_table_names[_MAX_CATALOG_TABLES + 1] = {
 		.schema_name = CATALOG_SCHEMA_NAME,
 		.table_name = CONTINUOUS_AGGS_WATERMARK_TABLE_NAME,
 	},
+	[TELEMETRY_EVENT] = {
+		.schema_name = CATALOG_SCHEMA_NAME,
+		.table_name = TELEMETRY_EVENT_TABLE_NAME,
+	},
 	[_MAX_CATALOG_TABLES] = {
 		.schema_name = "invalid schema",
 		.table_name = "invalid table",

src/ts_catalog/catalog.h

@@ -58,6 +58,7 @@ typedef enum CatalogTable
 	CONTINUOUS_AGGS_BUCKET_FUNCTION,
 	JOB_ERRORS,
 	CONTINUOUS_AGGS_WATERMARK,
+	TELEMETRY_EVENT,
 	/* Don't forget updating catalog.c when adding new tables! */
 	_MAX_CATALOG_TABLES,
 } CatalogTable;
@@ -886,6 +887,22 @@ enum
 	_MAX_METADATA_INDEX,
 };
 
+/*
+ * telemetry_event table definition
+ */
+
+#define TELEMETRY_EVENT_TABLE_NAME "telemetry_event"
+
+enum Anum_telemetry_event
+{
+	Anum_telemetry_event_created = 1,
+	Anum_telemetry_event_tag,
+	Anum_telemetry_event_body,
+	_Anum_telemetry_event_max,
+};
+
+#define Natts_telemetry_event_max (_Anum_telemetry_event_max - 1)
+
 /****** BGW_POLICY_CHUNK_STATS TABLE definitions */
 #define BGW_POLICY_CHUNK_STATS_TABLE_NAME "bgw_policy_chunk_stats"
 

test/expected/drop_rename_hypertable.out

@@ -218,7 +218,8 @@ SELECT * FROM _timescaledb_catalog.hypertable;
  _timescaledb_catalog | metadata                                         | table | super_user
  _timescaledb_catalog | remote_txn                                       | table | super_user
  _timescaledb_catalog | tablespace                                       | table | super_user
-(24 rows)
+ _timescaledb_catalog | telemetry_event                                  | table | super_user
+(25 rows)
 
 \dt "_timescaledb_internal".*
                           List of relations

test/expected/pg_dump.out

@@ -554,6 +554,7 @@ WHERE   refclassid = 'pg_catalog.pg_extension'::pg_catalog.regclass AND
  _timescaledb_catalog.chunk_copy_operation_id_seq
  _timescaledb_catalog.compression_algorithm
  _timescaledb_catalog.tablespace_id_seq
+ _timescaledb_catalog.telemetry_event
  _timescaledb_internal.bgw_job_stat
  _timescaledb_internal.bgw_policy_chunk_stats
  _timescaledb_internal.compressed_chunk_stats
@@ -569,7 +570,7 @@ WHERE   refclassid = 'pg_catalog.pg_extension'::pg_catalog.regclass AND
  timescaledb_information.job_errors
  timescaledb_information.job_stats
  timescaledb_information.jobs
-(19 rows)
+(20 rows)
 
 -- Make sure we can't run our restoring functions as a normal perm user as that would disable functionality for the whole db
 \c :TEST_DBNAME :ROLE_DEFAULT_PERM_USER

test/expected/telemetry.out

@@ -389,6 +389,7 @@ WHERE key != 'os_name_pretty';
  last_tuned_version
  postgresql_version
  related_extensions
+ db_telemetry_events
  timescaledb_version
  errors_by_sqlerrcode
  num_reorder_policies
@@ -402,7 +403,7 @@ WHERE key != 'os_name_pretty';
  num_compression_policies_fixed
  num_user_defined_actions_fixed
  num_continuous_aggs_policies_fixed
-(36 rows)
+(37 rows)
 
 CREATE MATERIALIZED VIEW telemetry_report AS
 SELECT t FROM get_telemetry_report() t;
@@ -597,3 +598,16 @@ SELECT key from _timescaledb_catalog.metadata;
 -- test that the telemetry gathering code doesn't break nonexistent statements
 EXECUTE noexistent_statement;
 ERROR:  prepared statement "noexistent_statement" does not exist
+\c :TEST_DBNAME :ROLE_SUPERUSER
+-- Insert some data into the telemetry event table
+INSERT INTO _timescaledb_catalog.telemetry_event(tag, body) VALUES
+    ('ummagumma', '{"title": "Careful with that Axe Eugene!"}'),
+    ('kaboom', '{"title": "Where is that kaboom?"}');
+-- Check that it is present in the telemetry report
+SELECT * FROM jsonb_to_recordset(get_telemetry_report()->'db_telemetry_events') AS x(tag name, body text);
+    tag    |                    body                    
+-----------+--------------------------------------------
+ ummagumma | {"title": "Careful with that Axe Eugene!"}
+ kaboom    | {"title": "Where is that kaboom?"}
+(2 rows)
+

test/sql/telemetry.sql

@@ -264,3 +264,12 @@ SELECT key from _timescaledb_catalog.metadata;
 \set ON_ERROR_STOP 0
 -- test that the telemetry gathering code doesn't break nonexistent statements
 EXECUTE noexistent_statement;
+
+\c :TEST_DBNAME :ROLE_SUPERUSER
+-- Insert some data into the telemetry event table
+INSERT INTO _timescaledb_catalog.telemetry_event(tag, body) VALUES
+    ('ummagumma', '{"title": "Careful with that Axe Eugene!"}'),
+    ('kaboom', '{"title": "Where is that kaboom?"}');
+
+-- Check that it is present in the telemetry report
+SELECT * FROM jsonb_to_recordset(get_telemetry_report()->'db_telemetry_events') AS x(tag name, body text);