diff --git a/CHANGELOG.md b/CHANGELOG.md index 0aae107c2..ea9479875 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -19,6 +19,7 @@ accidentally triggering the load of a previous DB version.** * #5510 Propagate vacuum/analyze to compressed chunks * #5584 Reduce decompression during constraint checking * #5530 Optimize compressed chunk resorting +* #5639 Support sending telemetry event reports **Bugfixes** * #5396 Fix SEGMENTBY columns predicates to be pushed down diff --git a/sql/pre_install/tables.sql b/sql/pre_install/tables.sql index 8c5fbb0d1..6826dc5a5 100644 --- a/sql/pre_install/tables.sql +++ b/sql/pre_install/tables.sql @@ -347,6 +347,15 @@ CREATE TABLE _timescaledb_catalog.metadata ( SELECT pg_catalog.pg_extension_config_dump('_timescaledb_catalog.metadata', $$ WHERE KEY = 'exported_uuid' $$); +-- Log with events that will be sent out with the telemetry. The log +-- will be flushed after it has been sent out. We do not save it to +-- backups since it should not contain important data. +CREATE TABLE _timescaledb_catalog.telemetry_event ( + created timestamptz NOT NULL DEFAULT current_timestamp, + tag name NOT NULL, + body jsonb NOT NULL +); + CREATE TABLE _timescaledb_catalog.continuous_agg ( mat_hypertable_id integer NOT NULL, raw_hypertable_id integer NOT NULL, diff --git a/sql/updates/latest-dev.sql b/sql/updates/latest-dev.sql index 6abc50efb..fa985e35c 100644 --- a/sql/updates/latest-dev.sql +++ b/sql/updates/latest-dev.sql @@ -59,3 +59,14 @@ ALTER TABLE _timescaledb_config.bgw_job ALTER TABLE _timescaledb_catalog.continuous_agg_migrate_plan ADD COLUMN user_view_definition TEXT, DROP CONSTRAINT continuous_agg_migrate_plan_mat_hypertable_id_fkey; + +-- Log with events that will be sent out with the telemetry. The log +-- will be flushed after it has been sent out. We do not save it to +-- backups since it should not contain important data. +CREATE TABLE _timescaledb_catalog.telemetry_event ( + created timestamptz NOT NULL DEFAULT current_timestamp, + tag name NOT NULL, + body jsonb NOT NULL +); + +GRANT SELECT ON _timescaledb_catalog.telemetry_event TO PUBLIC; diff --git a/sql/updates/reverse-dev.sql b/sql/updates/reverse-dev.sql index 7afc0bfdf..0bb880523 100644 --- a/sql/updates/reverse-dev.sql +++ b/sql/updates/reverse-dev.sql @@ -226,3 +226,7 @@ ALTER TABLE _timescaledb_catalog.continuous_agg_migrate_plan_step SELECT pg_catalog.pg_extension_config_dump('_timescaledb_catalog.continuous_agg_migrate_plan', ''); GRANT SELECT ON TABLE _timescaledb_catalog.continuous_agg_migrate_plan TO PUBLIC; + +ALTER EXTENSION timescaledb DROP TABLE _timescaledb_catalog.telemetry_event; + +DROP TABLE IF EXISTS _timescaledb_catalog.telemetry_event; diff --git a/sql/with_telemetry.sql b/sql/with_telemetry.sql index 22266023d..58676935f 100644 --- a/sql/with_telemetry.sql +++ b/sql/with_telemetry.sql @@ -2,9 +2,9 @@ -- Please see the included NOTICE for copyright information and -- LICENSE-APACHE for a copy of the license. -CREATE OR REPLACE FUNCTION @extschema@.get_telemetry_report() RETURNS jsonb - AS '@MODULE_PATHNAME@', 'ts_telemetry_get_report_jsonb' - LANGUAGE C STABLE PARALLEL SAFE; +CREATE OR REPLACE FUNCTION @extschema@.get_telemetry_report() + RETURNS jsonb AS '@MODULE_PATHNAME@', 'ts_telemetry_get_report_jsonb' + LANGUAGE C STABLE PARALLEL SAFE; INSERT INTO _timescaledb_config.bgw_job (id, application_name, schedule_interval, max_runtime, max_retries, retry_period, proc_schema, proc_name, owner, scheduled, fixed_schedule) VALUES (1, 'Telemetry Reporter [1]', INTERVAL '24h', INTERVAL '100s', -1, INTERVAL '1h', '_timescaledb_internal', 'policy_telemetry', pg_catalog.quote_ident(current_role)::regrole, true, false) diff --git a/src/telemetry/telemetry.c b/src/telemetry/telemetry.c index f0f3f16a0..f76d191d1 100644 --- a/src/telemetry/telemetry.c +++ b/src/telemetry/telemetry.c @@ -74,6 +74,7 @@ #define REQ_NUM_USER_DEFINED_ACTIONS "num_user_defined_actions" #define REQ_RELATED_EXTENSIONS "related_extensions" #define REQ_METADATA "db_metadata" +#define REQ_TELEMETRY_EVENT "db_telemetry_events" #define REQ_LICENSE_EDITION_APACHE "apache_only" #define REQ_LICENSE_EDITION_COMMUNITY "community" #define REQ_TS_LAST_TUNE_TIME "last_tuned_time" @@ -956,6 +957,13 @@ build_telemetry_report() ts_telemetry_metadata_add_values(parse_state); pushJsonbValue(&parse_state, WJB_END_OBJECT, NULL); + /* Add telemetry events */ + key.type = jbvString; + key.val.string.val = REQ_TELEMETRY_EVENT; + key.val.string.len = strlen(REQ_TELEMETRY_EVENT); + pushJsonbValue(&parse_state, WJB_KEY, &key); + ts_telemetry_events_add(parse_state); + /* Add function call telemetry */ key.type = jbvString; key.val.string.val = REQ_FUNCTIONS_USED; @@ -1101,6 +1109,7 @@ ts_telemetry_main(const char *host, const char *path, const char *service) } ts_function_telemetry_reset_counts(); + ts_telemetry_event_truncate(); /* * Do the version-check. Response is the body of a well-formed HTTP diff --git a/src/telemetry/telemetry_metadata.c b/src/telemetry/telemetry_metadata.c index 49cdefc69..348a2c534 100644 --- a/src/telemetry/telemetry_metadata.c +++ b/src/telemetry/telemetry_metadata.c @@ -6,7 +6,9 @@ #include #include #include +#include #include +#include #include "ts_catalog/catalog.h" #include "ts_catalog/metadata.h" @@ -15,6 +17,72 @@ #include "scan_iterator.h" #include "jsonb_utils.h" +#if PG14_LT +/* Copied from jsonb_util.c */ +static void +JsonbToJsonbValue(Jsonb *jsonb, JsonbValue *val) +{ + val->type = jbvBinary; + val->val.binary.data = &jsonb->root; + val->val.binary.len = VARSIZE(jsonb) - VARHDRSZ; +} +#endif + +void +ts_telemetry_event_truncate(void) +{ + RangeVar rv = { + .schemaname = CATALOG_SCHEMA_NAME, + .relname = TELEMETRY_EVENT_TABLE_NAME, + }; + ExecuteTruncate(&(TruncateStmt){ + .type = T_TruncateStmt, + .relations = list_make1(&rv), + .behavior = DROP_RESTRICT, + }); +} + +void +ts_telemetry_events_add(JsonbParseState *state) +{ + ScanIterator iterator = + ts_scan_iterator_create(TELEMETRY_EVENT, AccessShareLock, CurrentMemoryContext); + pushJsonbValue(&state, WJB_BEGIN_ARRAY, NULL); + ts_scanner_foreach(&iterator) + { + TupleInfo *ti = iterator.tinfo; + TupleDesc tupdesc = ti->slot->tts_tupleDescriptor; + bool created_isnull, tag_isnull, value_isnull; + Datum created = slot_getattr(ti->slot, Anum_telemetry_event_created, &created_isnull); + Datum tag = slot_getattr(ti->slot, Anum_telemetry_event_tag, &tag_isnull); + Datum body = slot_getattr(ti->slot, Anum_telemetry_event_body, &value_isnull); + + pushJsonbValue(&state, WJB_BEGIN_OBJECT, NULL); + if (!created_isnull) + ts_jsonb_add_str(state, + NameStr( + TupleDescAttr(tupdesc, Anum_telemetry_event_created - 1)->attname), + DatumGetCString(DirectFunctionCall1(timestamptz_out, created))); + + if (!tag_isnull) + ts_jsonb_add_str(state, + NameStr(TupleDescAttr(tupdesc, Anum_telemetry_event_tag - 1)->attname), + NameStr(*DatumGetName(tag))); + + if (!value_isnull) + { + JsonbValue jsonb_value; + JsonbToJsonbValue(DatumGetJsonbP(body), &jsonb_value); + ts_jsonb_add_value(state, + NameStr( + TupleDescAttr(tupdesc, Anum_telemetry_event_body - 1)->attname), + &jsonb_value); + } + pushJsonbValue(&state, WJB_END_OBJECT, NULL); + } + pushJsonbValue(&state, WJB_END_ARRAY, NULL); +} + /* * add all entries from _timescaledb_catalog.metadata */ diff --git a/src/telemetry/telemetry_metadata.h b/src/telemetry/telemetry_metadata.h index 7f6d49fe1..d9e59d7e8 100644 --- a/src/telemetry/telemetry_metadata.h +++ b/src/telemetry/telemetry_metadata.h @@ -12,5 +12,7 @@ #include extern void ts_telemetry_metadata_add_values(JsonbParseState *state); +extern void ts_telemetry_events_add(JsonbParseState *state); +extern void ts_telemetry_event_truncate(void); #endif /* TIMESCALEDB_TELEMETRY_TELEMETRY_METADATA_H */ diff --git a/src/ts_catalog/catalog.c b/src/ts_catalog/catalog.c index 2a8f98c86..03edc7608 100644 --- a/src/ts_catalog/catalog.c +++ b/src/ts_catalog/catalog.c @@ -124,6 +124,10 @@ static const TableInfoDef catalog_table_names[_MAX_CATALOG_TABLES + 1] = { .schema_name = CATALOG_SCHEMA_NAME, .table_name = CONTINUOUS_AGGS_WATERMARK_TABLE_NAME, }, + [TELEMETRY_EVENT] = { + .schema_name = CATALOG_SCHEMA_NAME, + .table_name = TELEMETRY_EVENT_TABLE_NAME, + }, [_MAX_CATALOG_TABLES] = { .schema_name = "invalid schema", .table_name = "invalid table", diff --git a/src/ts_catalog/catalog.h b/src/ts_catalog/catalog.h index 327352cb5..a4b4b020b 100644 --- a/src/ts_catalog/catalog.h +++ b/src/ts_catalog/catalog.h @@ -58,6 +58,7 @@ typedef enum CatalogTable CONTINUOUS_AGGS_BUCKET_FUNCTION, JOB_ERRORS, CONTINUOUS_AGGS_WATERMARK, + TELEMETRY_EVENT, /* Don't forget updating catalog.c when adding new tables! */ _MAX_CATALOG_TABLES, } CatalogTable; @@ -886,6 +887,22 @@ enum _MAX_METADATA_INDEX, }; +/* + * telemetry_event table definition + */ + +#define TELEMETRY_EVENT_TABLE_NAME "telemetry_event" + +enum Anum_telemetry_event +{ + Anum_telemetry_event_created = 1, + Anum_telemetry_event_tag, + Anum_telemetry_event_body, + _Anum_telemetry_event_max, +}; + +#define Natts_telemetry_event_max (_Anum_telemetry_event_max - 1) + /****** BGW_POLICY_CHUNK_STATS TABLE definitions */ #define BGW_POLICY_CHUNK_STATS_TABLE_NAME "bgw_policy_chunk_stats" diff --git a/test/expected/drop_rename_hypertable.out b/test/expected/drop_rename_hypertable.out index b15cbb8e0..faae78d70 100644 --- a/test/expected/drop_rename_hypertable.out +++ b/test/expected/drop_rename_hypertable.out @@ -218,7 +218,8 @@ SELECT * FROM _timescaledb_catalog.hypertable; _timescaledb_catalog | metadata | table | super_user _timescaledb_catalog | remote_txn | table | super_user _timescaledb_catalog | tablespace | table | super_user -(24 rows) + _timescaledb_catalog | telemetry_event | table | super_user +(25 rows) \dt "_timescaledb_internal".* List of relations diff --git a/test/expected/pg_dump.out b/test/expected/pg_dump.out index 90d0da2f2..5952c264f 100644 --- a/test/expected/pg_dump.out +++ b/test/expected/pg_dump.out @@ -554,6 +554,7 @@ WHERE refclassid = 'pg_catalog.pg_extension'::pg_catalog.regclass AND _timescaledb_catalog.chunk_copy_operation_id_seq _timescaledb_catalog.compression_algorithm _timescaledb_catalog.tablespace_id_seq + _timescaledb_catalog.telemetry_event _timescaledb_internal.bgw_job_stat _timescaledb_internal.bgw_policy_chunk_stats _timescaledb_internal.compressed_chunk_stats @@ -569,7 +570,7 @@ WHERE refclassid = 'pg_catalog.pg_extension'::pg_catalog.regclass AND timescaledb_information.job_errors timescaledb_information.job_stats timescaledb_information.jobs -(19 rows) +(20 rows) -- Make sure we can't run our restoring functions as a normal perm user as that would disable functionality for the whole db \c :TEST_DBNAME :ROLE_DEFAULT_PERM_USER diff --git a/test/expected/telemetry.out b/test/expected/telemetry.out index 91afcce26..911fe19c4 100644 --- a/test/expected/telemetry.out +++ b/test/expected/telemetry.out @@ -389,6 +389,7 @@ WHERE key != 'os_name_pretty'; last_tuned_version postgresql_version related_extensions + db_telemetry_events timescaledb_version errors_by_sqlerrcode num_reorder_policies @@ -402,7 +403,7 @@ WHERE key != 'os_name_pretty'; num_compression_policies_fixed num_user_defined_actions_fixed num_continuous_aggs_policies_fixed -(36 rows) +(37 rows) CREATE MATERIALIZED VIEW telemetry_report AS SELECT t FROM get_telemetry_report() t; @@ -597,3 +598,16 @@ SELECT key from _timescaledb_catalog.metadata; -- test that the telemetry gathering code doesn't break nonexistent statements EXECUTE noexistent_statement; ERROR: prepared statement "noexistent_statement" does not exist +\c :TEST_DBNAME :ROLE_SUPERUSER +-- Insert some data into the telemetry event table +INSERT INTO _timescaledb_catalog.telemetry_event(tag, body) VALUES + ('ummagumma', '{"title": "Careful with that Axe Eugene!"}'), + ('kaboom', '{"title": "Where is that kaboom?"}'); +-- Check that it is present in the telemetry report +SELECT * FROM jsonb_to_recordset(get_telemetry_report()->'db_telemetry_events') AS x(tag name, body text); + tag | body +-----------+-------------------------------------------- + ummagumma | {"title": "Careful with that Axe Eugene!"} + kaboom | {"title": "Where is that kaboom?"} +(2 rows) + diff --git a/test/sql/telemetry.sql b/test/sql/telemetry.sql index e901bea74..32a799880 100644 --- a/test/sql/telemetry.sql +++ b/test/sql/telemetry.sql @@ -264,3 +264,12 @@ SELECT key from _timescaledb_catalog.metadata; \set ON_ERROR_STOP 0 -- test that the telemetry gathering code doesn't break nonexistent statements EXECUTE noexistent_statement; + +\c :TEST_DBNAME :ROLE_SUPERUSER +-- Insert some data into the telemetry event table +INSERT INTO _timescaledb_catalog.telemetry_event(tag, body) VALUES + ('ummagumma', '{"title": "Careful with that Axe Eugene!"}'), + ('kaboom', '{"title": "Where is that kaboom?"}'); + +-- Check that it is present in the telemetry report +SELECT * FROM jsonb_to_recordset(get_telemetry_report()->'db_telemetry_events') AS x(tag name, body text);