postgres/typesense
1
0
Fork 0
mirror of https://github.com/typesense/typesense.git synced 2025-05-18 20:52:50 +08:00
Code Issues Actions 1 Packages Projects Releases Wiki Activity

handle userid with '_' (#1853)

Browse Source 
* fix endpoints and unused vars

* add response on failure of writing events to db

* update response code

* handle userid with '_' & dont process events with log_to_store disabled

* use '%' as separator instead of '_'

* optimize '%' removal from userid
This commit is contained in:
Krunal Gandhi 2024-07-24 05:14:07 +00:00 committed by GitHub
parent cc1588ecfa
commit 058eb9db52
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
3 changed files with 80 additions and 24 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
include/analytics_manager.h
View File

@ -219,7 +219,7 @@ public:
bool write_to_db(const nlohmann::json& payload);
void get_last_N_events(const std::string& userid, uint32_t N, std::vector<std::string>& values);
void get_last_N_events(const std::string& userid, const std::string& event_type, uint32_t N, std::vector<std::string>& values);
#ifdef TEST_BUILD
std::unordered_map<std::string, std::vector<event_t>> get_log_events() {

33
src/analytics_manager.cpp
View File

@ -461,9 +461,15 @@ Option<bool> AnalyticsManager::add_event(const std::string& client_ip, const std
doc_id = event_json["doc_id"].get<std::string>();
}
event_t event(query, event_type, now_ts_useconds, user_id, doc_id,
event_name, event_collection_map[event_name].log_to_store, custom_data);
events_vec.emplace_back(event);
if(event_collection_map_it->second.log_to_store) {
//only store events if log_to_store is specified in rule
//remove any '%' found in userid
user_id.erase(std::remove(user_id.begin(), user_id.end(), '%'), user_id.end());
event_t event(query, event_type, now_ts_useconds, user_id, doc_id,
event_name, event_collection_map[event_name].log_to_store, custom_data);
events_vec.emplace_back(event);
}
if (!counter_events.empty()) {
auto counter_events_it = counter_events.find(query_collection);
@ -774,8 +780,11 @@ void counter_event_t::serialize_as_docs(std::string &docs) {
bool AnalyticsManager::write_to_db(const nlohmann::json& payload) {
if(analytics_store) {
for(const auto& event: payload) {
std::string key = event["user_id"].get<std::string>() + "_" + event["type"].get<std::string>()
+ "_" + StringUtils::serialize_uint64_t(event["timestamp"].get<uint64_t>());
std::string userid = event["user_id"].get<std::string>();
std::string event_type = event["type"].get<std::string>();
std::string ts = StringUtils::serialize_uint64_t(event["timestamp"].get<uint64_t>());
std::string key = userid+ "%" + event_type+ "%" + ts;
bool inserted = analytics_store->insert(key, event.dump());
if(!inserted) {
@ -791,8 +800,18 @@ bool AnalyticsManager::write_to_db(const nlohmann::json& payload) {
return true;
}
void AnalyticsManager::get_last_N_events(const std::string& prefix, uint32_t N, std::vector<std::string>& values) {
const std::string userid_prefix = prefix + "_";
void AnalyticsManager::get_last_N_events(const std::string& userid, const std::string& event_type, uint32_t N,
std::vector<std::string>& values) {
std::string user_id = userid;
//erase any '%' in userid
user_id.erase(std::remove(user_id.begin(), user_id.end(), '%'), user_id.end());
auto userid_prefix = user_id + "%";
if(event_type != "*") {
userid_prefix += event_type;
}
analytics_store->get_last_N_values(userid_prefix, N, values);
}

69
test/analytics_manager_test.cpp
View File

@ -535,7 +535,7 @@ TEST_F(AnalyticsManagerTest, EventsPersist) {
ASSERT_TRUE(analyticsManager.write_to_db(payload));
std::vector<std::string> values;
analyticsManager.get_last_N_events("13", 5, values);
analyticsManager.get_last_N_events("13", "*", 5, values);
ASSERT_EQ(1, values.size());
auto parsed_json = nlohmann::json::parse(values[0]);
@ -574,7 +574,7 @@ TEST_F(AnalyticsManagerTest, EventsPersist) {
ASSERT_TRUE(analyticsManager.write_to_db(payload));
values.clear();
analyticsManager.get_last_N_events("13", 5, values);
analyticsManager.get_last_N_events("13", "*", 5, values);
ASSERT_EQ(2, values.size());
parsed_json = nlohmann::json::parse(values[0]);
@ -1298,7 +1298,7 @@ TEST_F(AnalyticsManagerTest, PopularityScoreValidation) {
ASSERT_TRUE(analyticsManager.write_to_db(payload));
std::vector<std::string> values;
analyticsManager.get_last_N_events("11", 5, values);
analyticsManager.get_last_N_events("11", "*", 5, values);
ASSERT_EQ(1, values.size());
auto parsed_json = nlohmann::json::parse(values[0]);
@ -1376,7 +1376,7 @@ TEST_F(AnalyticsManagerTest, PopularityScoreValidation) {
ASSERT_TRUE(analyticsManager.write_to_db(payload));
values.clear();
analyticsManager.get_last_N_events("11", 5, values);
analyticsManager.get_last_N_events("11", "*", 5, values);
ASSERT_EQ(2, values.size());
parsed_json = nlohmann::json::parse(values[0]);
@ -1394,7 +1394,7 @@ TEST_F(AnalyticsManagerTest, PopularityScoreValidation) {
ASSERT_EQ("shorts", parsed_json["query"]);
values.clear();
analyticsManager.get_last_N_events("13", 5, values);
analyticsManager.get_last_N_events("13", "*", 5, values);
ASSERT_EQ(1, values.size());
parsed_json = nlohmann::json::parse(values[0]);
@ -1505,7 +1505,7 @@ TEST_F(AnalyticsManagerTest, AnalyticsStoreTTL) {
ASSERT_TRUE(analyticsManager.write_to_db(payload));
//try fetching from db
const std::string prefix_start = "13_";
const std::string prefix_start = "13%";
const std::string prefix_end = "13`";
std::vector<std::string> events;
@ -1599,7 +1599,7 @@ TEST_F(AnalyticsManagerTest, AnalyticsStoreGetLastN) {
//basic test
std::vector<std::string> values;
analyticsManager.get_last_N_events("13", 5, values);
analyticsManager.get_last_N_events("13", "*", 5, values);
ASSERT_EQ(5, values.size());
nlohmann::json parsed_json;
@ -1611,7 +1611,7 @@ TEST_F(AnalyticsManagerTest, AnalyticsStoreGetLastN) {
//fetch events for middle user
values.clear();
analyticsManager.get_last_N_events("14", 5, values);
analyticsManager.get_last_N_events("14", "*", 5, values);
ASSERT_EQ(5, values.size());
start_index = 6;
@ -1622,7 +1622,7 @@ TEST_F(AnalyticsManagerTest, AnalyticsStoreGetLastN) {
//fetch more events than stored in db
values.clear();
analyticsManager.get_last_N_events("15", 8, values);
analyticsManager.get_last_N_events("15", "*", 8, values);
ASSERT_EQ(5, values.size());
start_index = 4;
@ -1634,7 +1634,7 @@ TEST_F(AnalyticsManagerTest, AnalyticsStoreGetLastN) {
//fetch events for non-existing user
values.clear();
analyticsManager.get_last_N_events("16", 8, values);
analyticsManager.get_last_N_events("16", "*", 8, values);
ASSERT_EQ(0, values.size());
//get specific event type or user
@ -1666,7 +1666,7 @@ TEST_F(AnalyticsManagerTest, AnalyticsStoreGetLastN) {
//get last 5 visit events for user_id 14
values.clear();
analyticsManager.get_last_N_events("14_visit", 5, values);
analyticsManager.get_last_N_events("14", "visit", 5, values);
ASSERT_EQ(5, values.size());
for(int i = 0; i < 5; ++i) {
parsed_json = nlohmann::json::parse(values[i]);
@ -1676,7 +1676,7 @@ TEST_F(AnalyticsManagerTest, AnalyticsStoreGetLastN) {
//get last 5 click events for user_id 14
values.clear();
analyticsManager.get_last_N_events("14_click", 5, values);
analyticsManager.get_last_N_events("14", "click", 5, values);
ASSERT_EQ(5, values.size());
for(int i = 0; i < 5; ++i) {
parsed_json = nlohmann::json::parse(values[i]);
@ -1709,11 +1709,48 @@ TEST_F(AnalyticsManagerTest, AnalyticsStoreGetLastN) {
ASSERT_TRUE(analyticsManager.write_to_db(payload));
values.clear();
analyticsManager.get_last_N_events("14_click", 10, values);
for(auto value : values) {
parsed_json = nlohmann::json::parse(value);
}
analyticsManager.get_last_N_events("14", "click", 10, values);
ASSERT_EQ(10, values.size());
for(int i = 0; i < 10; ++i) {
parsed_json = nlohmann::json::parse(values[i]);
ASSERT_EQ("AB", parsed_json["name"]);
ASSERT_EQ(std::to_string(9-i), parsed_json["doc_id"]);
}
//try adding userid with _
event1["name"] = "AB";
event1["type"] = "click";
event1["data"]["user_id"] = "14_U1";
for(auto i = 0; i < 5; i++) {
event1["data"]["doc_id"] = std::to_string(i);
req->body = event1.dump();
ASSERT_TRUE(post_create_event(req, res));
}
payload.clear();
event_data.clear();
collection_events_map = analyticsManager.get_log_events();
for (auto &events_collection_it: collection_events_map) {
const auto& collection = events_collection_it.first;
for(const auto& event: events_collection_it.second) {
event.to_json(event_data, collection);
payload.push_back(event_data);
}
}
ASSERT_TRUE(analyticsManager.write_to_db(payload));
values.clear();
analyticsManager.get_last_N_events("14_U1", "click", 10, values);
ASSERT_EQ(5, values.size());
for(int i = 0; i < 5; ++i) {
parsed_json = nlohmann::json::parse(values[i]);
ASSERT_EQ("AB", parsed_json["name"]);
ASSERT_EQ(std::to_string(4-i), parsed_json["doc_id"]);
}
}
TEST_F(AnalyticsManagerTest, AnalyticsWithAliases) {