diff --git a/include/collection.h b/include/collection.h index 3a120807..4a4ff158 100644 --- a/include/collection.h +++ b/include/collection.h @@ -626,7 +626,7 @@ public: // synonym operations - spp::sparse_hash_map get_synonyms(); + Option> get_synonyms(uint32_t limit=0, uint32_t offset=0); bool get_synonym(const std::string& id, synonym_t& synonym); diff --git a/include/synonym_index.h b/include/synonym_index.h index 5035a3f9..76facae0 100644 --- a/include/synonym_index.h +++ b/include/synonym_index.h @@ -70,7 +70,7 @@ public: void synonym_reduction(const std::vector& tokens, std::vector>& results) const; - spp::sparse_hash_map get_synonyms(); + Option> get_synonyms(uint32_t limit=0, uint32_t offset=0); bool get_synonym(const std::string& id, synonym_t& synonym); diff --git a/src/collection.cpp b/src/collection.cpp index a8f254c5..a962007a 100644 --- a/src/collection.cpp +++ b/src/collection.cpp @@ -4750,9 +4750,14 @@ Option> Collection::get_overrides(uint32_t li return Option>(overrides_map); } -spp::sparse_hash_map Collection::get_synonyms() { +Option> Collection::get_synonyms(uint32_t limit, uint32_t offset) { std::shared_lock lock(mutex); - return synonym_index->get_synonyms(); + auto synonyms_op =synonym_index->get_synonyms(limit, offset); + if(!synonyms_op.ok()) { + return Option>(synonyms_op.code(), synonyms_op.error()); + } + + return synonyms_op; } SynonymIndex* Collection::get_synonym_index() { diff --git a/src/collection_manager.cpp b/src/collection_manager.cpp index 825a7295..a38841b3 100644 --- a/src/collection_manager.cpp +++ b/src/collection_manager.cpp @@ -2345,9 +2345,9 @@ Option CollectionManager::clone_collection(const string& existing_n Collection* new_coll = coll_create_op.get(); // copy synonyms - auto synonyms = existing_coll->get_synonyms(); + auto synonyms = existing_coll->get_synonyms().get(); for(const auto& synonym: synonyms) { - new_coll->get_synonym_index()->add_synonym(new_name, synonym.second); + new_coll->get_synonym_index()->add_synonym(new_name, *synonym.second); } // copy overrides diff --git a/src/core_api.cpp b/src/core_api.cpp index f9c88b0b..04167bea 100644 --- a/src/core_api.cpp +++ b/src/core_api.cpp @@ -2047,13 +2047,37 @@ bool get_synonyms(const std::shared_ptr& req, const std::shared_ptrparams.count("offset") != 0) { + const auto &offset_str = req->params["offset"]; + if(!StringUtils::is_uint32_t(offset_str)) { + res->set(400, "Offset param should be unsigned integer."); + return false; + } + offset = std::stoi(offset_str); + } + + if(req->params.count("limit") != 0) { + const auto &limit_str = req->params["limit"]; + if(!StringUtils::is_uint32_t(limit_str)) { + res->set(400, "Limit param should be unsigned integer."); + return false; + } + limit = std::stoi(limit_str); + } + nlohmann::json res_json; res_json["synonyms"] = nlohmann::json::array(); - const auto& synonyms = collection->get_synonyms(); + auto synonyms_op = collection->get_synonyms(limit, offset); + if(!synonyms_op.ok()) { + res->set(synonyms_op.code(), synonyms_op.error()); + return false; + } + + const auto synonyms = synonyms_op.get(); for(const auto & kv: synonyms) { - nlohmann::json synonym = kv.second.to_view_json(); - res_json["synonyms"].push_back(synonym); + res_json["synonyms"].push_back(kv.second->to_view_json()); } res->set_200(res_json.dump()); diff --git a/src/synonym_index.cpp b/src/synonym_index.cpp index 84802067..05ff068c 100644 --- a/src/synonym_index.cpp +++ b/src/synonym_index.cpp @@ -179,9 +179,32 @@ Option SynonymIndex::remove_synonym(const std::string & collection_name, c return Option(404, "Could not find that `id`."); } -spp::sparse_hash_map SynonymIndex::get_synonyms() { +Option> SynonymIndex::get_synonyms(uint32_t limit, uint32_t offset) { std::shared_lock lock(mutex); - return synonym_definitions; + spp::sparse_hash_map synonyms_map; + + auto synonym_it = synonym_definitions.begin(); + + if(offset > 0) { + if(offset >= synonym_definitions.size()) { + return Option>(400, "Invalid offset param."); + } + + std::advance(synonym_it, offset); + } + + auto synonym_end = synonym_definitions.end(); + + if(limit > 0 && (offset + limit < synonym_definitions.size())) { + synonym_end = synonym_it; + std::advance(synonym_end, limit); + } + + for (synonym_it; synonym_it != synonym_end; ++synonym_it) { + synonyms_map[synonym_it->first] = &synonym_it->second; + } + + return Option>(synonyms_map); } std::string SynonymIndex::get_synonym_key(const std::string & collection_name, const std::string & synonym_id) { diff --git a/test/collection_manager_test.cpp b/test/collection_manager_test.cpp index 76d90215..5c1f5c51 100644 --- a/test/collection_manager_test.cpp +++ b/test/collection_manager_test.cpp @@ -525,16 +525,16 @@ TEST_F(CollectionManagerTest, RestoreRecordsOnRestart) { ASSERT_STREQ("exclude-rule", collection1->get_overrides().get()["exclude-rule"]->id.c_str()); ASSERT_STREQ("include-rule", collection1->get_overrides().get()["include-rule"]->id.c_str()); - const auto& synonyms = collection1->get_synonyms(); + const auto& synonyms = collection1->get_synonyms().get(); ASSERT_EQ(2, synonyms.size()); - ASSERT_STREQ("id1", synonyms.at("id1").id.c_str()); - ASSERT_EQ(2, synonyms.at("id1").root.size()); - ASSERT_EQ(1, synonyms.at("id1").synonyms.size()); + ASSERT_STREQ("id1", synonyms.at("id1")->id.c_str()); + ASSERT_EQ(2, synonyms.at("id1")->root.size()); + ASSERT_EQ(1, synonyms.at("id1")->synonyms.size()); - ASSERT_STREQ("id3", synonyms.at("id3").id.c_str()); - ASSERT_EQ(0, synonyms.at("id3").root.size()); - ASSERT_EQ(2, synonyms.at("id3").synonyms.size()); + ASSERT_STREQ("id3", synonyms.at("id3")->id.c_str()); + ASSERT_EQ(0, synonyms.at("id3")->root.size()); + ASSERT_EQ(2, synonyms.at("id3")->synonyms.size()); std::vector expected_symbols = {'+'}; std::vector expected_separators = {'-'}; @@ -1408,7 +1408,7 @@ TEST_F(CollectionManagerTest, CloneCollection) { ASSERT_FALSE(coll2 == nullptr); ASSERT_EQ("coll2", coll2->get_name()); ASSERT_EQ(1, coll2->get_fields().size()); - ASSERT_EQ(1, coll2->get_synonyms().size()); + ASSERT_EQ(1, coll2->get_synonyms().get().size()); ASSERT_EQ(1, coll2->get_overrides().get().size()); ASSERT_EQ("", coll2->get_fallback_field_type()); diff --git a/test/collection_synonyms_test.cpp b/test/collection_synonyms_test.cpp index 6a0b9707..449245de 100644 --- a/test/collection_synonyms_test.cpp +++ b/test/collection_synonyms_test.cpp @@ -700,11 +700,11 @@ TEST_F(CollectionSynonymsTest, DeleteAndUpsertDuplicationOfSynonms) { coll_mul_fields->add_synonym(R"({"id": "ipod-synonyms", "root": "ipod", "synonyms": ["i pod", "ipod"]})"_json); coll_mul_fields->add_synonym(R"({"id": "samsung-synonyms", "root": "s3", "synonyms": ["s3 phone", "samsung"]})"_json); - ASSERT_EQ(2, coll_mul_fields->get_synonyms().size()); + ASSERT_EQ(2, coll_mul_fields->get_synonyms().get().size()); coll_mul_fields->remove_synonym("ipod-synonyms"); - ASSERT_EQ(1, coll_mul_fields->get_synonyms().size()); - ASSERT_STREQ("samsung-synonyms", coll_mul_fields->get_synonyms()["samsung-synonyms"].id.c_str()); + ASSERT_EQ(1, coll_mul_fields->get_synonyms().get().size()); + ASSERT_STREQ("samsung-synonyms", coll_mul_fields->get_synonyms().get()["samsung-synonyms"]->id.c_str()); // try to upsert synonym with same ID @@ -712,7 +712,7 @@ TEST_F(CollectionSynonymsTest, DeleteAndUpsertDuplicationOfSynonms) { "synonyms": ["s3 phone", "samsung"]})"_json); ASSERT_TRUE(upsert_op.ok()); - ASSERT_EQ(1, coll_mul_fields->get_synonyms().size()); + ASSERT_EQ(1, coll_mul_fields->get_synonyms().get().size()); synonym_t synonym2_updated; coll_mul_fields->get_synonym("samsung-synonyms", synonym2_updated); @@ -721,7 +721,7 @@ TEST_F(CollectionSynonymsTest, DeleteAndUpsertDuplicationOfSynonms) { ASSERT_EQ("smartphone", synonym2_updated.root[1]); coll_mul_fields->remove_synonym("samsung-synonyms"); - ASSERT_EQ(0, coll_mul_fields->get_synonyms().size()); + ASSERT_EQ(0, coll_mul_fields->get_synonyms().get().size()); } TEST_F(CollectionSynonymsTest, SynonymJsonSerialization) { @@ -1066,4 +1066,97 @@ TEST_F(CollectionSynonymsTest, MultipleSynonymSubstitution) { res = coll2->search("suit man", {"title", "gender"}, "", {}, {}, {0}, 10, 1, FREQUENCY, {true}, 0).get(); ASSERT_EQ(1, res["hits"].size()); +} + +TEST_F(CollectionSynonymsTest, SynonymsPagination) { + Collection *coll3; + + std::vector fields = {field("title", field_types::STRING, false), + field("points", field_types::INT32, false)}; + + coll3 = collectionManager.get_collection("coll3").get(); + if (coll3 == nullptr) { + coll3 = collectionManager.create_collection("coll3", 1, fields, "points").get(); + } + + for (int i = 0; i < 5; ++i) { + nlohmann::json synonym_json = R"( + { + "id": "foobar", + "synonyms": ["blazer", "suit"] + })"_json; + + synonym_json["id"] = synonym_json["id"].get() + std::to_string(i + 1); + + coll3->add_synonym(synonym_json); + } + + uint32_t limit = 0, offset = 0; + + //limit collections by 2 + limit = 2; + auto synonym_op = coll3->get_synonyms(limit); + auto synonym_map = synonym_op.get(); + auto it = synonym_map.begin(); + ASSERT_EQ(2, synonym_map.size()); + ASSERT_EQ("foobar4", it->second->id); it++; + ASSERT_EQ("foobar5", it->second->id); + + //get 2 collection from offset 3 + offset = 3; + synonym_op = coll3->get_synonyms(limit, offset); + synonym_map = synonym_op.get(); + it = synonym_map.begin(); + ASSERT_EQ(2, synonym_map.size()); + ASSERT_EQ("foobar3", it->second->id); it++; + ASSERT_EQ("foobar2", it->second->id); + + //get all collection except first + offset = 1; + limit = 0; + synonym_op = coll3->get_synonyms(limit, offset); + synonym_map = synonym_op.get(); + it = synonym_map.begin(); + ASSERT_EQ(4, synonym_map.size()); + ASSERT_EQ("foobar5", it->second->id); it++; + ASSERT_EQ("foobar1", it->second->id); it++; + ASSERT_EQ("foobar3", it->second->id); it++; + ASSERT_EQ("foobar2", it->second->id); it++; + + //get last collection + offset = 4, limit = 1; + synonym_op = coll3->get_synonyms(limit, offset); + synonym_map = synonym_op.get(); + it = synonym_map.begin(); + ASSERT_EQ(1, synonym_map.size()); + ASSERT_EQ("foobar2", it->second->id); + + //if limit is greater than number of collection then return all from offset + offset = 0; + limit = 8; + synonym_op = coll3->get_synonyms(limit, offset); + synonym_map = synonym_op.get(); + it = synonym_map.begin(); + ASSERT_EQ(5, synonym_map.size()); + ASSERT_EQ("foobar4", it->second->id); it++; + ASSERT_EQ("foobar5", it->second->id); it++; + ASSERT_EQ("foobar1", it->second->id); it++; + ASSERT_EQ("foobar3", it->second->id); it++; + ASSERT_EQ("foobar2", it->second->id); it++; + + offset = 3; + limit = 4; + synonym_op = coll3->get_synonyms(limit, offset); + synonym_map = synonym_op.get(); + it = synonym_map.begin(); + ASSERT_EQ(2, synonym_map.size()); + ASSERT_EQ("foobar3", it->second->id); it++; + ASSERT_EQ("foobar2", it->second->id); + + //invalid offset + offset = 6; + limit = 0; + synonym_op = coll3->get_synonyms(limit, offset); + ASSERT_FALSE(synonym_op.ok()); + ASSERT_EQ("Invalid offset param.", synonym_op.error()); } \ No newline at end of file diff --git a/test/core_api_utils_test.cpp b/test/core_api_utils_test.cpp index c88a1dd9..87f884e2 100644 --- a/test/core_api_utils_test.cpp +++ b/test/core_api_utils_test.cpp @@ -1694,6 +1694,64 @@ TEST_F(CoreAPIUtilsTest, OverridesPagination) { ASSERT_EQ(400, resp->status_code); ASSERT_EQ("{\"message\": \"Offset param should be unsigned integer.\"}", resp->body); + //invalid limit string + req->params["offset"] = "0"; + req->params["limit"] = "-1"; + get_collections(req, resp); + ASSERT_EQ(400, resp->status_code); + ASSERT_EQ("{\"message\": \"Limit param should be unsigned integer.\"}", resp->body); +} + +TEST_F(CoreAPIUtilsTest, SynonymsPagination) { + Collection *coll3; + + std::vector fields = {field("title", field_types::STRING, false), + field("points", field_types::INT32, false)}; + + coll3 = collectionManager.get_collection("coll3").get(); + if (coll3 == nullptr) { + coll3 = collectionManager.create_collection("coll3", 1, fields, "points").get(); + } + + for (int i = 0; i < 5; ++i) { + nlohmann::json synonym_json = R"( + { + "id": "foobar", + "synonyms": ["blazer", "suit"] + })"_json; + + synonym_json["id"] = synonym_json["id"].get() + std::to_string(i + 1); + + coll3->add_synonym(synonym_json); + } + + auto req = std::make_shared(); + auto resp = std::make_shared(nullptr); + + req->params["collection"] = "coll3"; + req->params["offset"] = "0"; + req->params["limit"] = "1"; + + get_synonyms(req, resp); + + nlohmann::json expected_json = R"({ + "synonyms":[ + { + "id":"foobar4", + "root":"", + "synonyms":["blazer","suit"] + }] + })"_json; + + ASSERT_EQ(expected_json.dump(), resp->body); + + //invalid offset string + req->params["offset"] = "0a"; + get_collections(req, resp); + + ASSERT_EQ(400, resp->status_code); + ASSERT_EQ("{\"message\": \"Offset param should be unsigned integer.\"}", resp->body); + //invalid limit string req->params["offset"] = "0"; req->params["limit"] = "-1";