From 9867ca5e294c9a7301eb135e725988bcaf4665c8 Mon Sep 17 00:00:00 2001 From: kishorenc Date: Wed, 10 Jun 2020 21:59:56 +0530 Subject: [PATCH 1/3] Handle deletion of records with optional fields. --- src/index.cpp | 4 ++++ test/collection_test.cpp | 4 ++++ 2 files changed, 8 insertions(+) diff --git a/src/index.cpp b/src/index.cpp index 71dd092b..590793c6 100644 --- a/src/index.cpp +++ b/src/index.cpp @@ -1706,6 +1706,10 @@ void Index::remove_and_shift_offset_index(sorted_array &offset_index, const uint Option Index::remove(const uint32_t seq_id, nlohmann::json & document) { for(auto & name_field: search_schema) { + if(name_field.second.optional && document.count(name_field.first) == 0) { + continue; + } + // Go through all the field names and find the keys+values so that they can be removed from in-memory index std::vector tokens; if(name_field.second.type == field_types::STRING) { diff --git a/test/collection_test.cpp b/test/collection_test.cpp index 6066da0a..cac9d715 100644 --- a/test/collection_test.cpp +++ b/test/collection_test.cpp @@ -2251,6 +2251,10 @@ TEST_F(CollectionTest, OptionalFields) { auto res_op = coll1->search("*", {"title"}, "", {}, sort_fields, 0, 10, 1, FREQUENCY, false); ASSERT_FALSE(res_op.ok()); ASSERT_STREQ("Cannot sort by `average` as it is defined as an optional field.", res_op.error().c_str()); + + // try deleting a record having optional field + Option remove_op = coll1->remove("1"); + ASSERT_TRUE(remove_op.ok()); // default sorting field should not be declared optional fields = { From 81dfe760094b6b7c3a960d4dc7240321972e670b Mon Sep 17 00:00:00 2001 From: kishorenc Date: Fri, 12 Jun 2020 20:40:15 +0530 Subject: [PATCH 2/3] Float array field should accept integer values. --- src/index.cpp | 3 ++- test/collection_test.cpp | 3 ++- test/numeric_array_documents.jsonl | 2 +- 3 files changed, 5 insertions(+), 3 deletions(-) diff --git a/src/index.cpp b/src/index.cpp index 590793c6..7e6f9ba1 100644 --- a/src/index.cpp +++ b/src/index.cpp @@ -295,7 +295,8 @@ Option Index::validate_index_in_memory(const nlohmann::json &document, return Option<>(400, "Field `" + field_name + "` must be a float array."); } - if(document[field_name].size() > 0 && !document[field_name][0].is_number_float()) { + if(document[field_name].size() > 0 && !document[field_name][0].is_number()) { + // allows integer to be passed to a float array field return Option<>(400, "Field `" + field_name + "` must be a float array."); } } else if(field_pair.second.type == field_types::BOOL_ARRAY) { diff --git a/test/collection_test.cpp b/test/collection_test.cpp index cac9d715..358e1a57 100644 --- a/test/collection_test.cpp +++ b/test/collection_test.cpp @@ -1122,7 +1122,8 @@ TEST_F(CollectionTest, FilterOnFloatFields) { std::string json_line; while (std::getline(infile, json_line)) { - coll_array_fields->add(json_line); + auto add_op = coll_array_fields->add(json_line); + ASSERT_TRUE(add_op.ok()); } infile.close(); diff --git a/test/numeric_array_documents.jsonl b/test/numeric_array_documents.jsonl index ae659381..6df3acd1 100644 --- a/test/numeric_array_documents.jsonl +++ b/test/numeric_array_documents.jsonl @@ -1,5 +1,5 @@ {"name": "Jeremy Howard", "top_3": [1.09, 1.88, 0.001], "rating": 1.09, "age": 24, "years": [2014, 2015, 2016], "timestamps": [1390354022, 1421890022, 1453426022], "tags": ["gold", "silver"]} {"name": "Jeremy Howard", "top_3": [9.999, 8.89, 7.713], "rating": 9.999, "age": 44, "years": [2015, 2016], "timestamps": [1421890022, 1453426022], "tags": ["FINE PLATINUM"]} {"name": "Jeremy Howard", "top_3": [7.812, 7.770, 6.66], "rating": 7.812, "age": 21, "years": [2016], "timestamps": [1453426022], "tags": ["bronze", "gold"]} -{"name": "Jeremy Howard", "top_3": [0.0, 0.0, 0.0], "rating": 0.0, "age": 63, "years": [1981, 1985], "timestamps": [348974822, 475205222], "tags": ["silver"]} +{"name": "Jeremy Howard", "top_3": [0, 0.0, 0.0], "rating": 0.0, "age": 63, "years": [1981, 1985], "timestamps": [348974822, 475205222], "tags": ["silver"]} {"name": "Jeremy Howard", "top_3": [5.5, 5.431, 1.001], "rating": 5.5, "age": 32, "years": [1999, 2000, 2001, 2002], "timestamps": [916968422, 948504422, 980126822, 1011662822], "tags": ["silver", "gold", "bronze"]} \ No newline at end of file From ae66d6a8d0f92db30ff08a01e5e5cb90d5f966a8 Mon Sep 17 00:00:00 2001 From: kishorenc Date: Fri, 12 Jun 2020 20:57:26 +0530 Subject: [PATCH 3/3] Schema response should contain optional field value. --- include/collection.h | 2 ++ src/collection.cpp | 23 +++++++++++++++++++++++ src/core_api.cpp | 31 ++++--------------------------- test/collection_test.cpp | 13 +++++++++++++ 4 files changed, 42 insertions(+), 27 deletions(-) diff --git a/include/collection.h b/include/collection.h index c1ebc55e..c821af1f 100644 --- a/include/collection.h +++ b/include/collection.h @@ -217,6 +217,8 @@ public: Option to_doc(const std::string & json_str, nlohmann::json & document); + nlohmann::json get_summary_json(); + Option add(const std::string & json_str); Option add_many(const std::string & json_str); diff --git a/src/collection.cpp b/src/collection.cpp index 2e7b30f2..c3342fec 100644 --- a/src/collection.cpp +++ b/src/collection.cpp @@ -127,6 +127,29 @@ Option Collection::to_doc(const std::string & json_str, nlohmann::json return Option(seq_id); } +nlohmann::json Collection::get_summary_json() { + nlohmann::json json_response; + + json_response["name"] = name; + json_response["num_documents"] = num_documents; + json_response["created_at"] = created_at; + + nlohmann::json fields_arr; + + for(const field & coll_field: fields) { + nlohmann::json field_json; + field_json[fields::name] = coll_field.name; + field_json[fields::type] = coll_field.type; + field_json[fields::facet] = coll_field.facet; + field_json[fields::optional] = coll_field.optional; + fields_arr.push_back(field_json); + } + + json_response["fields"] = fields_arr; + json_response["default_sorting_field"] = default_sorting_field; + return json_response; +} + Option Collection::add(const std::string & json_str) { nlohmann::json document; Option doc_seq_id_op = to_doc(json_str, document); diff --git a/src/core_api.cpp b/src/core_api.cpp index 56efeec7..4e47aeda 100644 --- a/src/core_api.cpp +++ b/src/core_api.cpp @@ -9,29 +9,6 @@ #include "system_metrics.h" #include "logger.h" -nlohmann::json collection_summary_json(Collection *collection) { - nlohmann::json json_response; - - json_response["name"] = collection->get_name(); - json_response["num_documents"] = collection->get_num_documents(); - json_response["created_at"] = collection->get_created_at(); - - const std::vector & coll_fields = collection->get_fields(); - nlohmann::json fields_arr; - - for(const field & coll_field: coll_fields) { - nlohmann::json field_json; - field_json[fields::name] = coll_field.name; - field_json[fields::type] = coll_field.type; - field_json[fields::facet] = coll_field.facet; - fields_arr.push_back(field_json); - } - - json_response["fields"] = fields_arr; - json_response["default_sorting_field"] = collection->get_default_sorting_field(); - return json_response; -} - bool handle_authentication(http_req& req, const route_path& rpath, const std::string& auth_key) { CollectionManager & collectionManager = CollectionManager::get_instance(); @@ -55,7 +32,7 @@ bool get_collections(http_req & req, http_res & res) { nlohmann::json json_response = nlohmann::json::array(); for(Collection* collection: collections) { - nlohmann::json collection_json = collection_summary_json(collection); + nlohmann::json collection_json = collection->get_summary_json(); json_response.push_back(collection_json); } @@ -150,7 +127,7 @@ bool post_create_collection(http_req & req, http_res & res) { collectionManager.create_collection(req_json["name"], fields, default_sorting_field); if(collection_op.ok()) { - nlohmann::json json_response = collection_summary_json(collection_op.get()); + nlohmann::json json_response = collection_op.get()->get_summary_json(); res.set_201(json_response.dump()); return true; } @@ -169,7 +146,7 @@ bool del_drop_collection(http_req & req, http_res & res) { return false; } - nlohmann::json collection_json = collection_summary_json(collection); + nlohmann::json collection_json = collection->get_summary_json(); Option drop_result = collectionManager.drop_collection(req.params["collection"]); if(!drop_result.ok()) { @@ -497,7 +474,7 @@ bool get_collection_summary(http_req & req, http_res & res) { return false; } - nlohmann::json json_response = collection_summary_json(collection); + nlohmann::json json_response = collection->get_summary_json(); res.set_200(json_response.dump()); return true; diff --git a/test/collection_test.cpp b/test/collection_test.cpp index 358e1a57..dfcfac39 100644 --- a/test/collection_test.cpp +++ b/test/collection_test.cpp @@ -2257,6 +2257,19 @@ TEST_F(CollectionTest, OptionalFields) { Option remove_op = coll1->remove("1"); ASSERT_TRUE(remove_op.ok()); + // try fetching the schema (should contain optional field) + nlohmann::json coll_summary = coll1->get_summary_json(); + LOG(INFO) << coll_summary; + ASSERT_STREQ("title", coll_summary["fields"][0]["name"].get().c_str()); + ASSERT_STREQ("string", coll_summary["fields"][0]["type"].get().c_str()); + ASSERT_FALSE(coll_summary["fields"][0]["facet"].get()); + ASSERT_FALSE(coll_summary["fields"][0]["optional"].get()); + + ASSERT_STREQ("description", coll_summary["fields"][1]["name"].get().c_str()); + ASSERT_STREQ("string", coll_summary["fields"][1]["type"].get().c_str()); + ASSERT_TRUE(coll_summary["fields"][1]["facet"].get()); + ASSERT_TRUE(coll_summary["fields"][1]["optional"].get()); + // default sorting field should not be declared optional fields = { field("title", field_types::STRING, false),