diff --git a/include/collection.h b/include/collection.h index 8501c3d1..8ac8bc55 100644 --- a/include/collection.h +++ b/include/collection.h @@ -467,7 +467,8 @@ public: static Option include_references(nlohmann::json& doc, const uint32_t& seq_id, Collection *const collection, const std::map& reference_filter_results, - const std::vector& ref_include_exclude_fields_vec); + const std::vector& ref_include_exclude_fields_vec, + const nlohmann::json& original_doc); Option prune_doc_with_lock(nlohmann::json& doc, const tsl::htrie_set& include_names, const tsl::htrie_set& exclude_names, diff --git a/src/collection.cpp b/src/collection.cpp index 7e36814a..53447b38 100644 --- a/src/collection.cpp +++ b/src/collection.cpp @@ -5448,6 +5448,11 @@ Option Collection::prune_ref_doc(nlohmann::json& doc, const tsl::htrie_set& ref_exclude_fields_full, const bool& is_reference_array, const ref_include_exclude_fields& ref_include_exclude) { + nlohmann::json original_doc; + if (!ref_include_exclude.nested_join_includes.empty()) { + original_doc = doc; + } + auto const& ref_collection_name = ref_include_exclude.collection_name; auto& cm = CollectionManager::get_instance(); auto ref_collection = cm.get_collection(ref_collection_name); @@ -5503,7 +5508,7 @@ Option Collection::prune_ref_doc(nlohmann::json& doc, ref_collection.get(), references.coll_to_references == nullptr ? refs : references.coll_to_references[0], - ref_include_exclude.nested_join_includes); + ref_include_exclude.nested_join_includes, original_doc); if (!nested_include_exclude_op.ok()) { return nested_include_exclude_op; } @@ -5568,7 +5573,7 @@ Option Collection::prune_ref_doc(nlohmann::json& doc, ref_collection.get(), references.coll_to_references == nullptr ? refs : references.coll_to_references[i], - ref_include_exclude.nested_join_includes); + ref_include_exclude.nested_join_includes, original_doc); if (!nested_include_exclude_op.ok()) { return nested_include_exclude_op; } @@ -5580,7 +5585,8 @@ Option Collection::prune_ref_doc(nlohmann::json& doc, Option Collection::include_references(nlohmann::json& doc, const uint32_t& seq_id, Collection *const collection, const std::map& reference_filter_results, - const std::vector& ref_include_exclude_fields_vec) { + const std::vector& ref_include_exclude_fields_vec, + const nlohmann::json& original_doc) { for (auto const& ref_include_exclude: ref_include_exclude_fields_vec) { auto ref_collection_name = ref_include_exclude.collection_name; @@ -5656,13 +5662,26 @@ Option Collection::include_references(nlohmann::json& doc, const uint32_t& if (collection->object_reference_helper_fields.count(field_name) != 0) { std::vector keys; StringUtils::split(field_name, keys, "."); - if (!doc.contains(keys[0])) { - return Option(400, "Could not find `" + keys[0] + - "` in the document to include the referenced document."); + auto const& key = keys[0]; + + if (!doc.contains(key)) { + if (!original_doc.contains(key)) { + return Option(400, "Could not find `" + key + + "` key in the document to include the referenced document."); + } + + // The key is excluded from the doc by the query, inserting empty object(s) so referenced doc can be + // included in it. + if (original_doc[key].is_array()) { + doc[key] = nlohmann::json::array(); + doc[key].insert(doc[key].begin(), original_doc[key].size(), nlohmann::json::object()); + } else { + doc[key] = nlohmann::json::object(); + } } - if (doc[keys[0]].is_array()) { - for (uint32_t i = 0; i < doc[keys[0]].size(); i++) { + if (doc[key].is_array()) { + for (uint32_t i = 0; i < doc[key].size(); i++) { uint32_t ref_doc_id; auto op = collection->get_object_array_related_id(field_name, seq_id, i, ref_doc_id); if (!op.ok()) { @@ -5674,7 +5693,7 @@ Option Collection::include_references(nlohmann::json& doc, const uint32_t& } reference_filter_result_t result(1, new uint32_t[1]{ref_doc_id}); - prune_doc_op = prune_ref_doc(doc[keys[0]][i], result, + prune_doc_op = prune_ref_doc(doc[key][i], result, ref_include_fields_full, ref_exclude_fields_full, false, ref_include_exclude); if (!prune_doc_op.ok()) { @@ -5688,7 +5707,7 @@ Option Collection::include_references(nlohmann::json& doc, const uint32_t& continue; } reference_filter_result_t result(ids.size(), &ids[0]); - prune_doc_op = prune_ref_doc(doc[keys[0]], result, ref_include_fields_full, ref_exclude_fields_full, + prune_doc_op = prune_ref_doc(doc[key], result, ref_include_fields_full, ref_exclude_fields_full, collection->search_schema.at(field_name).is_array(), ref_include_exclude); result.docs = nullptr; } @@ -5764,6 +5783,11 @@ Option Collection::prune_doc(nlohmann::json& doc, const std::map& reference_filter_results, Collection *const collection, const uint32_t& seq_id, const std::vector& ref_include_exclude_fields_vec) { + nlohmann::json original_doc; + if (!ref_include_exclude_fields_vec.empty()) { + original_doc = doc; + } + // doc can only be an object auto it = doc.begin(); while(it != doc.end()) { @@ -5839,7 +5863,8 @@ Option Collection::prune_doc(nlohmann::json& doc, it++; } - return include_references(doc, seq_id, collection, reference_filter_results, ref_include_exclude_fields_vec); + return include_references(doc, seq_id, collection, reference_filter_results, ref_include_exclude_fields_vec, + original_doc); } Option Collection::validate_alter_payload(nlohmann::json& schema_changes, diff --git a/test/collection_join_test.cpp b/test/collection_join_test.cpp index d2a98e4e..827a07fc 100644 --- a/test/collection_join_test.cpp +++ b/test/collection_join_test.cpp @@ -3913,8 +3913,8 @@ TEST_F(CollectionJoinTest, FilterByObjectReferenceField) { auto now_ts = std::chrono::duration_cast( std::chrono::system_clock::now().time_since_epoch()).count(); - auto search_op_bool = collectionManager.do_search(req_params, embedded_params, json_res, now_ts); - ASSERT_TRUE(search_op_bool.ok()); + auto search_op = collectionManager.do_search(req_params, embedded_params, json_res, now_ts); + ASSERT_TRUE(search_op.ok()); auto res_obj = nlohmann::json::parse(json_res); ASSERT_EQ(2, res_obj["found"].get()); @@ -3931,14 +3931,39 @@ TEST_F(CollectionJoinTest, FilterByObjectReferenceField) { ASSERT_EQ("a", res_obj["hits"][1]["document"]["coll_id"]); ASSERT_EQ(0, res_obj["hits"][1]["document"]["object"].size()); + req_params = { + {"collection", "coll1"}, + {"q", "*"}, + {"include_fields", "$Products(product_id)"}, + {"exclude_fields", "object"} + }; + search_op = collectionManager.do_search(req_params, embedded_params, json_res, now_ts); + ASSERT_TRUE(search_op.ok()); + + res_obj = nlohmann::json::parse(json_res); + ASSERT_EQ(2, res_obj["found"].get()); + ASSERT_EQ(2, res_obj["hits"].size()); + ASSERT_EQ(3, res_obj["hits"][0]["document"].size()); + ASSERT_EQ("1", res_obj["hits"][0]["document"]["id"]); + ASSERT_EQ(1, res_obj["hits"][0]["document"].count("object")); + ASSERT_EQ(1, res_obj["hits"][0]["document"]["object"].size()); + ASSERT_EQ(1, res_obj["hits"][0]["document"]["object"].count("Products")); + ASSERT_EQ(1, res_obj["hits"][0]["document"]["object"]["Products"].size()); + ASSERT_EQ(1, res_obj["hits"][0]["document"]["object"]["Products"].count("product_id")); + ASSERT_EQ("product_c", res_obj["hits"][0]["document"]["object"]["Products"]["product_id"]); + ASSERT_EQ(3, res_obj["hits"][1]["document"].size()); + ASSERT_EQ("0", res_obj["hits"][1]["document"]["id"]); + ASSERT_EQ(1, res_obj["hits"][1]["document"].count("object")); + ASSERT_EQ(0, res_obj["hits"][1]["document"]["object"].size()); + req_params = { {"collection", "Products"}, {"q", "*"}, {"filter_by", "$coll1(id: *)"}, {"include_fields", "$coll1(coll_id)"} }; - search_op_bool = collectionManager.do_search(req_params, embedded_params, json_res, now_ts); - ASSERT_TRUE(search_op_bool.ok()); + search_op = collectionManager.do_search(req_params, embedded_params, json_res, now_ts); + ASSERT_TRUE(search_op.ok()); res_obj = nlohmann::json::parse(json_res); ASSERT_EQ(1, res_obj["found"].get()); @@ -3988,8 +4013,8 @@ TEST_F(CollectionJoinTest, FilterByObjectReferenceField) { {"q", "*"}, {"include_fields", "$Products(product_id)"} }; - search_op_bool = collectionManager.do_search(req_params, embedded_params, json_res, now_ts); - ASSERT_TRUE(search_op_bool.ok()); + search_op = collectionManager.do_search(req_params, embedded_params, json_res, now_ts); + ASSERT_TRUE(search_op.ok()); res_obj = nlohmann::json::parse(json_res); ASSERT_EQ(2, res_obj["found"].get()); @@ -4009,14 +4034,41 @@ TEST_F(CollectionJoinTest, FilterByObjectReferenceField) { ASSERT_EQ("a", res_obj["hits"][1]["document"]["coll_id"]); ASSERT_EQ(0, res_obj["hits"][1]["document"]["object"].size()); + req_params = { + {"collection", "coll2"}, + {"q", "*"}, + {"include_fields", "$Products(product_id)"}, + {"exclude_fields", "object"} + }; + search_op = collectionManager.do_search(req_params, embedded_params, json_res, now_ts); + ASSERT_TRUE(search_op.ok()); + + res_obj = nlohmann::json::parse(json_res); + ASSERT_EQ(2, res_obj["found"].get()); + ASSERT_EQ(2, res_obj["hits"].size()); + ASSERT_EQ(3, res_obj["hits"][0]["document"].size()); + ASSERT_EQ("1", res_obj["hits"][0]["document"]["id"]); + ASSERT_EQ(1, res_obj["hits"][0]["document"].count("object")); + ASSERT_EQ(1, res_obj["hits"][0]["document"]["object"].size()); + ASSERT_EQ(1, res_obj["hits"][0]["document"]["object"].count("Products")); + ASSERT_EQ(2, res_obj["hits"][0]["document"]["object"]["Products"].size()); + ASSERT_EQ(1, res_obj["hits"][0]["document"]["object"]["Products"][0].count("product_id")); + ASSERT_EQ("product_a", res_obj["hits"][0]["document"]["object"]["Products"][0]["product_id"]); + ASSERT_EQ(1, res_obj["hits"][0]["document"]["object"]["Products"][1].count("product_id")); + ASSERT_EQ("product_b", res_obj["hits"][0]["document"]["object"]["Products"][1]["product_id"]); + ASSERT_EQ(3, res_obj["hits"][1]["document"].size()); + ASSERT_EQ("0", res_obj["hits"][1]["document"]["id"]); + ASSERT_EQ(1, res_obj["hits"][1]["document"].count("object")); + ASSERT_EQ(0, res_obj["hits"][1]["document"]["object"].size()); + req_params = { {"collection", "Products"}, {"q", "*"}, {"filter_by", "$coll2(id: *)"}, {"include_fields", "$coll2(coll_id)"} }; - search_op_bool = collectionManager.do_search(req_params, embedded_params, json_res, now_ts); - ASSERT_TRUE(search_op_bool.ok()); + search_op = collectionManager.do_search(req_params, embedded_params, json_res, now_ts); + ASSERT_TRUE(search_op.ok()); res_obj = nlohmann::json::parse(json_res); ASSERT_EQ(2, res_obj["found"].get()); @@ -4071,8 +4123,8 @@ TEST_F(CollectionJoinTest, FilterByObjectReferenceField) { {"q", "*"}, {"include_fields", "$Products(product_id)"} }; - search_op_bool = collectionManager.do_search(req_params, embedded_params, json_res, now_ts); - ASSERT_TRUE(search_op_bool.ok()); + search_op = collectionManager.do_search(req_params, embedded_params, json_res, now_ts); + ASSERT_TRUE(search_op.ok()); res_obj = nlohmann::json::parse(json_res); ASSERT_EQ(2, res_obj["found"].get()); @@ -4098,8 +4150,8 @@ TEST_F(CollectionJoinTest, FilterByObjectReferenceField) { {"filter_by", "$coll3(id: *)"}, {"include_fields", "$coll3(coll_id)"} }; - search_op_bool = collectionManager.do_search(req_params, embedded_params, json_res, now_ts); - ASSERT_TRUE(search_op_bool.ok()); + search_op = collectionManager.do_search(req_params, embedded_params, json_res, now_ts); + ASSERT_TRUE(search_op.ok()); res_obj = nlohmann::json::parse(json_res); ASSERT_EQ(2, res_obj["found"].get()); @@ -4206,8 +4258,8 @@ TEST_F(CollectionJoinTest, FilterByObjectReferenceField) { {"q", "*"}, {"include_fields", "$Portions(*, strategy:merge)"} }; - search_op_bool = collectionManager.do_search(req_params, embedded_params, json_res, now_ts); - ASSERT_TRUE(search_op_bool.ok()); + search_op = collectionManager.do_search(req_params, embedded_params, json_res, now_ts); + ASSERT_TRUE(search_op.ok()); res_obj = nlohmann::json::parse(json_res); ASSERT_EQ(2, res_obj["found"].get()); @@ -4245,6 +4297,45 @@ TEST_F(CollectionJoinTest, FilterByObjectReferenceField) { ASSERT_EQ("g", res_obj["hits"][1]["document"]["portions"][0].at("unit")); ASSERT_EQ(10 , res_obj["hits"][1]["document"]["portions"][0].at("count")); + req_params = { + {"collection", "Foods"}, + {"q", "*"}, + {"include_fields", "$Portions(*, strategy:merge)"}, + {"exclude_fields", "portions"} + }; + search_op = collectionManager.do_search(req_params, embedded_params, json_res, now_ts); + ASSERT_TRUE(search_op.ok()); + + res_obj = nlohmann::json::parse(json_res); + ASSERT_EQ(2, res_obj["found"].get()); + ASSERT_EQ(2, res_obj["hits"].size()); + ASSERT_EQ(3, res_obj["hits"][0]["document"].size()); + + ASSERT_EQ("1", res_obj["hits"][0]["document"]["id"]); + ASSERT_EQ(1, res_obj["hits"][0]["document"].count("portions")); + ASSERT_EQ(3, res_obj["hits"][0]["document"]["portions"].size()); + + ASSERT_EQ(4, res_obj["hits"][0]["document"]["portions"][0].size()); + ASSERT_EQ("portion_b", res_obj["hits"][0]["document"]["portions"][0].at("portion_id")); + ASSERT_EQ(1 , res_obj["hits"][0]["document"]["portions"][0].at("quantity")); + ASSERT_EQ("lt", res_obj["hits"][0]["document"]["portions"][0].at("unit")); + + ASSERT_EQ(0, res_obj["hits"][0]["document"]["portions"][1].size()); + + ASSERT_EQ(4, res_obj["hits"][0]["document"]["portions"][2].size()); + ASSERT_EQ("portion_c", res_obj["hits"][0]["document"]["portions"][2].at("portion_id")); + ASSERT_EQ(500 , res_obj["hits"][0]["document"]["portions"][2].at("quantity")); + ASSERT_EQ("ml", res_obj["hits"][0]["document"]["portions"][2].at("unit")); + + ASSERT_EQ("0", res_obj["hits"][1]["document"]["id"]); + ASSERT_EQ(1, res_obj["hits"][1]["document"].count("portions")); + ASSERT_EQ(1, res_obj["hits"][1]["document"]["portions"].size()); + + ASSERT_EQ(4, res_obj["hits"][1]["document"]["portions"][0].size()); + ASSERT_EQ("portion_a", res_obj["hits"][1]["document"]["portions"][0].at("portion_id")); + ASSERT_EQ(500 , res_obj["hits"][1]["document"]["portions"][0].at("quantity")); + ASSERT_EQ("g", res_obj["hits"][1]["document"]["portions"][0].at("unit")); + // recreate collection manager to ensure that it initializes `object_reference_helper_fields` correctly. collectionManager.dispose(); delete store; @@ -4263,8 +4354,8 @@ TEST_F(CollectionJoinTest, FilterByObjectReferenceField) { {"q", "*"}, {"include_fields", "$Portions(*, strategy:merge)"} }; - search_op_bool = collectionManager.do_search(req_params, embedded_params, json_res, now_ts); - ASSERT_TRUE(search_op_bool.ok()); + search_op = collectionManager.do_search(req_params, embedded_params, json_res, now_ts); + ASSERT_TRUE(search_op.ok()); res_obj = nlohmann::json::parse(json_res); ASSERT_EQ(2, res_obj["found"].get()); @@ -4321,8 +4412,8 @@ TEST_F(CollectionJoinTest, FilterByObjectReferenceField) { {"q", "*"}, {"include_fields", "$Portions(*, strategy:merge)"} }; - search_op_bool = collectionManager.do_search(req_params, embedded_params, json_res, now_ts); - ASSERT_TRUE(search_op_bool.ok()); + search_op = collectionManager.do_search(req_params, embedded_params, json_res, now_ts); + ASSERT_TRUE(search_op.ok()); res_obj = nlohmann::json::parse(json_res); ASSERT_EQ(2, res_obj["found"].get());