From 058eb9db521dfa623b3f21a110f79cafcb7323cb Mon Sep 17 00:00:00 2001 From: Krunal Gandhi Date: Wed, 24 Jul 2024 05:14:07 +0000 Subject: [PATCH] handle userid with '_' (#1853) * fix endpoints and unused vars * add response on failure of writing events to db * update response code * handle userid with '_' & dont process events with log_to_store disabled * use '%' as separator instead of '_' * optimize '%' removal from userid --- include/analytics_manager.h | 2 +- src/analytics_manager.cpp | 33 ++++++++++++---- test/analytics_manager_test.cpp | 69 +++++++++++++++++++++++++-------- 3 files changed, 80 insertions(+), 24 deletions(-) diff --git a/include/analytics_manager.h b/include/analytics_manager.h index 099176e9..82e4e9ae 100644 --- a/include/analytics_manager.h +++ b/include/analytics_manager.h @@ -219,7 +219,7 @@ public: bool write_to_db(const nlohmann::json& payload); - void get_last_N_events(const std::string& userid, uint32_t N, std::vector& values); + void get_last_N_events(const std::string& userid, const std::string& event_type, uint32_t N, std::vector& values); #ifdef TEST_BUILD std::unordered_map> get_log_events() { diff --git a/src/analytics_manager.cpp b/src/analytics_manager.cpp index b6d968fa..48823188 100644 --- a/src/analytics_manager.cpp +++ b/src/analytics_manager.cpp @@ -461,9 +461,15 @@ Option AnalyticsManager::add_event(const std::string& client_ip, const std doc_id = event_json["doc_id"].get(); } - event_t event(query, event_type, now_ts_useconds, user_id, doc_id, - event_name, event_collection_map[event_name].log_to_store, custom_data); - events_vec.emplace_back(event); + if(event_collection_map_it->second.log_to_store) { + //only store events if log_to_store is specified in rule + //remove any '%' found in userid + user_id.erase(std::remove(user_id.begin(), user_id.end(), '%'), user_id.end()); + + event_t event(query, event_type, now_ts_useconds, user_id, doc_id, + event_name, event_collection_map[event_name].log_to_store, custom_data); + events_vec.emplace_back(event); + } if (!counter_events.empty()) { auto counter_events_it = counter_events.find(query_collection); @@ -774,8 +780,11 @@ void counter_event_t::serialize_as_docs(std::string &docs) { bool AnalyticsManager::write_to_db(const nlohmann::json& payload) { if(analytics_store) { for(const auto& event: payload) { - std::string key = event["user_id"].get() + "_" + event["type"].get() - + "_" + StringUtils::serialize_uint64_t(event["timestamp"].get()); + std::string userid = event["user_id"].get(); + std::string event_type = event["type"].get(); + std::string ts = StringUtils::serialize_uint64_t(event["timestamp"].get()); + + std::string key = userid+ "%" + event_type+ "%" + ts; bool inserted = analytics_store->insert(key, event.dump()); if(!inserted) { @@ -791,8 +800,18 @@ bool AnalyticsManager::write_to_db(const nlohmann::json& payload) { return true; } -void AnalyticsManager::get_last_N_events(const std::string& prefix, uint32_t N, std::vector& values) { - const std::string userid_prefix = prefix + "_"; +void AnalyticsManager::get_last_N_events(const std::string& userid, const std::string& event_type, uint32_t N, + std::vector& values) { + std::string user_id = userid; + + //erase any '%' in userid + user_id.erase(std::remove(user_id.begin(), user_id.end(), '%'), user_id.end()); + + auto userid_prefix = user_id + "%"; + if(event_type != "*") { + userid_prefix += event_type; + } + analytics_store->get_last_N_values(userid_prefix, N, values); } diff --git a/test/analytics_manager_test.cpp b/test/analytics_manager_test.cpp index c8d552cc..65d03184 100644 --- a/test/analytics_manager_test.cpp +++ b/test/analytics_manager_test.cpp @@ -535,7 +535,7 @@ TEST_F(AnalyticsManagerTest, EventsPersist) { ASSERT_TRUE(analyticsManager.write_to_db(payload)); std::vector values; - analyticsManager.get_last_N_events("13", 5, values); + analyticsManager.get_last_N_events("13", "*", 5, values); ASSERT_EQ(1, values.size()); auto parsed_json = nlohmann::json::parse(values[0]); @@ -574,7 +574,7 @@ TEST_F(AnalyticsManagerTest, EventsPersist) { ASSERT_TRUE(analyticsManager.write_to_db(payload)); values.clear(); - analyticsManager.get_last_N_events("13", 5, values); + analyticsManager.get_last_N_events("13", "*", 5, values); ASSERT_EQ(2, values.size()); parsed_json = nlohmann::json::parse(values[0]); @@ -1298,7 +1298,7 @@ TEST_F(AnalyticsManagerTest, PopularityScoreValidation) { ASSERT_TRUE(analyticsManager.write_to_db(payload)); std::vector values; - analyticsManager.get_last_N_events("11", 5, values); + analyticsManager.get_last_N_events("11", "*", 5, values); ASSERT_EQ(1, values.size()); auto parsed_json = nlohmann::json::parse(values[0]); @@ -1376,7 +1376,7 @@ TEST_F(AnalyticsManagerTest, PopularityScoreValidation) { ASSERT_TRUE(analyticsManager.write_to_db(payload)); values.clear(); - analyticsManager.get_last_N_events("11", 5, values); + analyticsManager.get_last_N_events("11", "*", 5, values); ASSERT_EQ(2, values.size()); parsed_json = nlohmann::json::parse(values[0]); @@ -1394,7 +1394,7 @@ TEST_F(AnalyticsManagerTest, PopularityScoreValidation) { ASSERT_EQ("shorts", parsed_json["query"]); values.clear(); - analyticsManager.get_last_N_events("13", 5, values); + analyticsManager.get_last_N_events("13", "*", 5, values); ASSERT_EQ(1, values.size()); parsed_json = nlohmann::json::parse(values[0]); @@ -1505,7 +1505,7 @@ TEST_F(AnalyticsManagerTest, AnalyticsStoreTTL) { ASSERT_TRUE(analyticsManager.write_to_db(payload)); //try fetching from db - const std::string prefix_start = "13_"; + const std::string prefix_start = "13%"; const std::string prefix_end = "13`"; std::vector events; @@ -1599,7 +1599,7 @@ TEST_F(AnalyticsManagerTest, AnalyticsStoreGetLastN) { //basic test std::vector values; - analyticsManager.get_last_N_events("13", 5, values); + analyticsManager.get_last_N_events("13", "*", 5, values); ASSERT_EQ(5, values.size()); nlohmann::json parsed_json; @@ -1611,7 +1611,7 @@ TEST_F(AnalyticsManagerTest, AnalyticsStoreGetLastN) { //fetch events for middle user values.clear(); - analyticsManager.get_last_N_events("14", 5, values); + analyticsManager.get_last_N_events("14", "*", 5, values); ASSERT_EQ(5, values.size()); start_index = 6; @@ -1622,7 +1622,7 @@ TEST_F(AnalyticsManagerTest, AnalyticsStoreGetLastN) { //fetch more events than stored in db values.clear(); - analyticsManager.get_last_N_events("15", 8, values); + analyticsManager.get_last_N_events("15", "*", 8, values); ASSERT_EQ(5, values.size()); start_index = 4; @@ -1634,7 +1634,7 @@ TEST_F(AnalyticsManagerTest, AnalyticsStoreGetLastN) { //fetch events for non-existing user values.clear(); - analyticsManager.get_last_N_events("16", 8, values); + analyticsManager.get_last_N_events("16", "*", 8, values); ASSERT_EQ(0, values.size()); //get specific event type or user @@ -1666,7 +1666,7 @@ TEST_F(AnalyticsManagerTest, AnalyticsStoreGetLastN) { //get last 5 visit events for user_id 14 values.clear(); - analyticsManager.get_last_N_events("14_visit", 5, values); + analyticsManager.get_last_N_events("14", "visit", 5, values); ASSERT_EQ(5, values.size()); for(int i = 0; i < 5; ++i) { parsed_json = nlohmann::json::parse(values[i]); @@ -1676,7 +1676,7 @@ TEST_F(AnalyticsManagerTest, AnalyticsStoreGetLastN) { //get last 5 click events for user_id 14 values.clear(); - analyticsManager.get_last_N_events("14_click", 5, values); + analyticsManager.get_last_N_events("14", "click", 5, values); ASSERT_EQ(5, values.size()); for(int i = 0; i < 5; ++i) { parsed_json = nlohmann::json::parse(values[i]); @@ -1709,11 +1709,48 @@ TEST_F(AnalyticsManagerTest, AnalyticsStoreGetLastN) { ASSERT_TRUE(analyticsManager.write_to_db(payload)); values.clear(); - analyticsManager.get_last_N_events("14_click", 10, values); - for(auto value : values) { - parsed_json = nlohmann::json::parse(value); - } + analyticsManager.get_last_N_events("14", "click", 10, values); ASSERT_EQ(10, values.size()); + for(int i = 0; i < 10; ++i) { + parsed_json = nlohmann::json::parse(values[i]); + ASSERT_EQ("AB", parsed_json["name"]); + ASSERT_EQ(std::to_string(9-i), parsed_json["doc_id"]); + } + + + //try adding userid with _ + event1["name"] = "AB"; + event1["type"] = "click"; + event1["data"]["user_id"] = "14_U1"; + + for(auto i = 0; i < 5; i++) { + event1["data"]["doc_id"] = std::to_string(i); + req->body = event1.dump(); + ASSERT_TRUE(post_create_event(req, res)); + } + + payload.clear(); + event_data.clear(); + collection_events_map = analyticsManager.get_log_events(); + for (auto &events_collection_it: collection_events_map) { + const auto& collection = events_collection_it.first; + + for(const auto& event: events_collection_it.second) { + event.to_json(event_data, collection); + payload.push_back(event_data); + } + } + + ASSERT_TRUE(analyticsManager.write_to_db(payload)); + + values.clear(); + analyticsManager.get_last_N_events("14_U1", "click", 10, values); + ASSERT_EQ(5, values.size()); + for(int i = 0; i < 5; ++i) { + parsed_json = nlohmann::json::parse(values[i]); + ASSERT_EQ("AB", parsed_json["name"]); + ASSERT_EQ(std::to_string(4-i), parsed_json["doc_id"]); + } } TEST_F(AnalyticsManagerTest, AnalyticsWithAliases) {