diff --git a/include/collection.h b/include/collection.h index 030cd319..33a73d3c 100644 --- a/include/collection.h +++ b/include/collection.h @@ -267,7 +267,8 @@ private: std::vector& processed_search_fields, bool extract_only_string_fields, bool enable_nested_fields, - const bool handle_wildcard = true); + const bool handle_wildcard = true, + const bool& include_id = false); bool is_nested_array(const nlohmann::json& obj, std::vector path_parts, size_t part_i) const; @@ -399,12 +400,14 @@ public: static void remove_flat_fields(nlohmann::json& document); static Option add_reference_fields(nlohmann::json& doc, + const std::string& ref_collection_name, Collection *const ref_collection, const std::string& alias, const reference_filter_result_t& references, const tsl::htrie_set& ref_include_fields_full, const tsl::htrie_set& ref_exclude_fields_full, - const std::string& error_prefix, const bool& is_reference_array); + const std::string& error_prefix, const bool& is_reference_array, + const bool& nest_ref_doc); static Option prune_doc(nlohmann::json& doc, const tsl::htrie_set& include_names, const tsl::htrie_set& exclude_names, const std::string& parent_name = "", diff --git a/include/field.h b/include/field.h index 0e55142f..274f59cd 100644 --- a/include/field.h +++ b/include/field.h @@ -496,9 +496,16 @@ namespace sort_field_const { static const std::string vector_query = "_vector_query"; } +namespace ref_include { + static const std::string merge = "merge"; + static const std::string nest = "nest"; +} + struct ref_include_fields { - std::string expression; + std::string collection_name; + std::string fields; std::string alias; + bool nest_ref_doc = false; }; struct hnsw_index_t; diff --git a/src/collection.cpp b/src/collection.cpp index d0956868..fecb2898 100644 --- a/src/collection.cpp +++ b/src/collection.cpp @@ -1278,7 +1278,8 @@ Option Collection::extract_field_name(const std::string& field_name, std::vector& processed_search_fields, const bool extract_only_string_fields, const bool enable_nested_fields, - const bool handle_wildcard) { + const bool handle_wildcard, + const bool& include_id) { // Reference to other collection if (field_name[0] == '$') { processed_search_fields.push_back(field_name); @@ -1294,6 +1295,12 @@ Option Collection::extract_field_name(const std::string& field_name, if (is_wildcard && !handle_wildcard) { return Option(400, "Pattern `" + field_name + "` is not allowed."); } + + if (is_wildcard && include_id && field_name.size() < 4 && + (field_name == "*" || field_name == "i*" || field_name == "id*")) { + processed_search_fields.emplace_back("id"); + } + // If wildcard, remove * auto prefix_it = search_schema.equal_prefix_range(field_name.substr(0, field_name.size() - is_wildcard)); bool field_found = false; @@ -4440,12 +4447,14 @@ void Collection::remove_flat_fields(nlohmann::json& document) { } Option Collection::add_reference_fields(nlohmann::json& doc, + const std::string& ref_collection_name, Collection *const ref_collection, const std::string& alias, const reference_filter_result_t& references, const tsl::htrie_set& ref_include_fields_full, const tsl::htrie_set& ref_exclude_fields_full, - const std::string& error_prefix, const bool& is_reference_array) { + const std::string& error_prefix, const bool& is_reference_array, + const bool& nest_ref_doc) { // One-to-one relation. if (!is_reference_array && references.count == 1) { auto ref_doc_seq_id = references.docs[0]; @@ -4463,15 +4472,23 @@ Option Collection::add_reference_fields(nlohmann::json& doc, return Option(prune_op.code(), error_prefix + prune_op.error()); } - if (!alias.empty()) { - auto temp_doc = ref_doc; - ref_doc.clear(); - for (const auto &item: temp_doc.items()) { - ref_doc[alias + item.key()] = item.value(); - } + if (ref_doc.empty()) { + return Option(true); } - doc.update(ref_doc); + if (nest_ref_doc) { + auto key = alias.empty() ? ref_collection_name : alias; + doc[key] = ref_doc; + } else { + if (!alias.empty()) { + auto temp_doc = ref_doc; + ref_doc.clear(); + for (const auto &item: temp_doc.items()) { + ref_doc[alias + item.key()] = item.value(); + } + } + doc.update(ref_doc); + } return Option(true); } @@ -4492,17 +4509,33 @@ Option Collection::add_reference_fields(nlohmann::json& doc, return Option(prune_op.code(), error_prefix + prune_op.error()); } - if (!alias.empty()) { - auto temp_doc = ref_doc; - ref_doc.clear(); - for (const auto &item: temp_doc.items()) { - ref_doc[alias + item.key()] = item.value(); - } + if (ref_doc.empty()) { + continue; } - for (auto ref_doc_it = ref_doc.begin(); ref_doc_it != ref_doc.end(); ref_doc_it++) { - // Add the values of ref_doc as JSON array into doc. - doc[ref_doc_it.key()] += ref_doc_it.value(); + if (nest_ref_doc) { + auto key = alias.empty() ? ref_collection_name : alias; + if (doc.contains(key) && !doc[key].is_array()) { + return Option(400, "Could not include the reference document of `" + ref_collection_name + + "` collection. Expected `" + key + "` to be an array. Try " + + (alias.empty() ? "adding an" : "renaming the") + " alias."); + } + + doc[key] += ref_doc; + } else { + for (auto ref_doc_it = ref_doc.begin(); ref_doc_it != ref_doc.end(); ref_doc_it++) { + auto const& ref_doc_key = ref_doc_it.key(); + auto const& doc_key = alias + ref_doc_key; + if (doc.contains(doc_key) && !doc[doc_key].is_array()) { + return Option(400, "Could not include the value of `" + ref_doc_key + + "` key of the reference document of `" + ref_collection_name + + "` collection. Expected `" + doc_key + "` to be an array. Try " + + (alias.empty() ? "adding an" : "renaming the") + " alias."); + } + + // Add the values of ref_doc as JSON array into doc. + doc[doc_key] += ref_doc_it.value(); + } } } @@ -4592,11 +4625,7 @@ Option Collection::prune_doc(nlohmann::json& doc, } for (auto const& ref_include: ref_includes) { - auto const& ref = ref_include.expression; - size_t parenthesis_index = ref.find('('); - - auto ref_collection_name = ref.substr(1, parenthesis_index - 1); - auto reference_fields = ref.substr(parenthesis_index + 1, ref.size() - parenthesis_index - 2); + auto const& ref_collection_name = ref_include.collection_name; auto& cm = CollectionManager::get_instance(); auto ref_collection = cm.get_collection(ref_collection_name); @@ -4631,12 +4660,12 @@ Option Collection::prune_doc(nlohmann::json& doc, } std::vector ref_include_fields_vec, ref_exclude_fields_vec; - StringUtils::split(reference_fields, ref_include_fields_vec, ","); + StringUtils::split(ref_include.fields, ref_include_fields_vec, ","); auto exclude_reference_it = exclude_names.equal_prefix_range("$" + ref_collection_name); if (exclude_reference_it.first != exclude_reference_it.second) { auto ref_exclude = exclude_reference_it.first.key(); - parenthesis_index = ref_exclude.find('('); - reference_fields = ref_exclude.substr(parenthesis_index + 1, ref_exclude.size() - parenthesis_index - 2); + auto parenthesis_index = ref_exclude.find('('); + auto reference_fields = ref_exclude.substr(parenthesis_index + 1, ref_exclude.size() - parenthesis_index - 2); StringUtils::split(reference_fields, ref_exclude_fields_vec, ","); } @@ -4664,10 +4693,12 @@ Option Collection::prune_doc(nlohmann::json& doc, if (ref_collection->search_schema.count(field_name) == 0) { continue; } - add_reference_fields_op = add_reference_fields(doc, ref_collection.get(), ref_include.alias, + add_reference_fields_op = add_reference_fields(doc, ref_include.collection_name, + ref_collection.get(), ref_include.alias, reference_filter_results.at(ref_collection_name), ref_include_fields_full, ref_exclude_fields_full, error_prefix, - ref_collection->get_schema().at(field_name).is_array()); + ref_collection->get_schema().at(field_name).is_array(), + ref_include.nest_ref_doc); } else if (doc_has_reference) { auto get_reference_field_op = ref_collection->get_referenced_in_field_with_lock(collection->name); if (!get_reference_field_op.ok()) { @@ -4687,9 +4718,11 @@ Option Collection::prune_doc(nlohmann::json& doc, result.count = ids.size(); result.docs = &ids[0]; - add_reference_fields_op = add_reference_fields(doc, ref_collection.get(), ref_include.alias, result, + add_reference_fields_op = add_reference_fields(doc, ref_include.collection_name, + ref_collection.get(), ref_include.alias, result, ref_include_fields_full, ref_exclude_fields_full, error_prefix, - collection->search_schema.at(field_name).is_array()); + collection->search_schema.at(field_name).is_array(), + ref_include.nest_ref_doc); result.docs = nullptr; } else if (joined_coll_has_reference) { auto joined_collection = cm.get_collection(joined_coll_having_reference); @@ -4720,9 +4753,11 @@ Option Collection::prune_doc(nlohmann::json& doc, reference_filter_result_t result; result.count = ids.size(); result.docs = &ids[0]; - add_reference_fields_op = add_reference_fields(doc, ref_collection.get(), ref_include.alias, result, + add_reference_fields_op = add_reference_fields(doc, ref_include.collection_name, + ref_collection.get(), ref_include.alias, result, ref_include_fields_full, ref_exclude_fields_full, error_prefix, - joined_collection->get_schema().at(reference_field_name).is_array()); + joined_collection->get_schema().at(reference_field_name).is_array(), + ref_include.nest_ref_doc); result.docs = nullptr; } @@ -5580,7 +5615,7 @@ Option Collection::populate_include_exclude_fields(const spp::sparse_hash_ std::vector exclude_fields_vec; for(auto& f_name: include_fields) { - auto field_op = extract_field_name(f_name, search_schema, include_fields_vec, false, enable_nested_fields); + auto field_op = extract_field_name(f_name, search_schema, include_fields_vec, false, enable_nested_fields, true, true); if(!field_op.ok()) { if(field_op.code() == 404) { // field need not be part of schema to be included (could be a stored value in the doc) @@ -5597,7 +5632,7 @@ Option Collection::populate_include_exclude_fields(const spp::sparse_hash_ continue; } - auto field_op = extract_field_name(f_name, search_schema, exclude_fields_vec, false, enable_nested_fields); + auto field_op = extract_field_name(f_name, search_schema, exclude_fields_vec, false, enable_nested_fields, true, true); if(!field_op.ok()) { if(field_op.code() == 404) { // field need not be part of schema to be excluded (could be a stored value in the doc) diff --git a/src/collection_manager.cpp b/src/collection_manager.cpp index 7c44ac37..8cec4c92 100644 --- a/src/collection_manager.cpp +++ b/src/collection_manager.cpp @@ -967,14 +967,30 @@ void initialize_ref_include_fields_vec(const std::string& filter_query, std::vec continue; } + // Format: $ref_collection_name(field_1, field_2: include_strategy) as ref_alias auto as_pos = include_field_exp.find(" as "); - auto ref_include = include_field_exp.substr(0, as_pos), - alias = (as_pos == std::string::npos) ? "" : + auto ref_include = include_field_exp.substr(0, as_pos); + auto alias = (as_pos == std::string::npos) ? "" : include_field_exp.substr(as_pos + 4, include_field_exp.size() - (as_pos + 4)); - // For an alias `foo`, we need append `foo.` to all the top level keys of reference doc. - ref_include_fields_vec.emplace_back(ref_include_fields{ref_include, alias.empty() ? alias : - StringUtils::trim(alias) + "."}); + auto parenthesis_index = ref_include.find('('); + auto ref_collection_name = ref_include.substr(1, parenthesis_index - 1); + auto ref_fields = ref_include.substr(parenthesis_index + 1, ref_include.size() - parenthesis_index - 2); + + auto nest_ref_doc = true; + auto colon_pos = ref_fields.find(':'); + if (colon_pos != std::string::npos) { + auto include_strategy = ref_fields.substr(colon_pos + 1, ref_fields.size() - colon_pos - 1); + StringUtils::trim(include_strategy); + nest_ref_doc = include_strategy == ref_include::nest; + ref_fields = ref_fields.substr(0, colon_pos); + } + + // For an alias `foo`, + // In case of "merge" reference doc, we need append `foo.` to all the top level keys of reference doc. + // In case of "nest" reference doc, `foo` becomes the key with reference doc as value. + auto ref_alias = !alias.empty() ? (StringUtils::trim(alias) + (nest_ref_doc ? "" : ".")) : ""; + ref_include_fields_vec.emplace_back(ref_include_fields{ref_collection_name, ref_fields, ref_alias, nest_ref_doc}); auto open_paren_pos = include_field_exp.find('('); if (open_paren_pos == std::string::npos) { @@ -993,7 +1009,7 @@ void initialize_ref_include_fields_vec(const std::string& filter_query, std::vec // Get all the fields of the referenced collection in the filter but not mentioned in include_fields. for (const auto &reference_collection_name: reference_collection_names) { - ref_include_fields_vec.emplace_back(ref_include_fields{"$" + reference_collection_name + "(*)", ""}); + ref_include_fields_vec.emplace_back(ref_include_fields{reference_collection_name, "", "", true}); } // Since no field of the collection is mentioned in include_fields, get all the fields. diff --git a/test/collection_join_test.cpp b/test/collection_join_test.cpp index 0dedeea4..daee7be7 100644 --- a/test/collection_join_test.cpp +++ b/test/collection_join_test.cpp @@ -915,6 +915,7 @@ TEST_F(CollectionJoinTest, FilterByReference_SingleMatch) { {"q", "Dan"}, {"query_by", "customer_name"}, {"filter_by", "$Products(rating:>3)"}, + {"include_fields", "$Products(*:merge)"}, }; search_op_bool = collectionManager.do_search(req_params, embedded_params, json_res, now_ts); @@ -930,6 +931,7 @@ TEST_F(CollectionJoinTest, FilterByReference_SingleMatch) { {"q", "Dan"}, {"query_by", "customer_name"}, {"filter_by", "$Products(id:*) && product_price:>100"}, + {"include_fields", "$Products(*:merge)"}, }; search_op_bool = collectionManager.do_search(req_params, embedded_params, json_res, now_ts); @@ -1678,24 +1680,55 @@ TEST_F(CollectionJoinTest, IncludeExcludeFieldsByReference) { ASSERT_EQ(1, res_obj["found"].get()); ASSERT_EQ(1, res_obj["hits"].size()); // No fields are mentioned in `include_fields`, should include all fields of Products and Customers by default. - ASSERT_EQ(10, res_obj["hits"][0]["document"].size()); + ASSERT_EQ(7, res_obj["hits"][0]["document"].size()); ASSERT_EQ(1, res_obj["hits"][0]["document"].count("id")); ASSERT_EQ(1, res_obj["hits"][0]["document"].count("product_id")); ASSERT_EQ(1, res_obj["hits"][0]["document"].count("product_name")); ASSERT_EQ(1, res_obj["hits"][0]["document"].count("product_description")); ASSERT_EQ(1, res_obj["hits"][0]["document"].count("embedding")); ASSERT_EQ(1, res_obj["hits"][0]["document"].count("rating")); - ASSERT_EQ(1, res_obj["hits"][0]["document"].count("customer_id")); - ASSERT_EQ(1, res_obj["hits"][0]["document"].count("customer_name")); - ASSERT_EQ(1, res_obj["hits"][0]["document"].count("product_price")); - ASSERT_EQ(1, res_obj["hits"][0]["document"].count("product_id_sequence_id")); + // Default strategy of reference includes is nest. No alias was provided, collection name becomes the field name. + ASSERT_EQ(6, res_obj["hits"][0]["document"]["Customers"].size()); + ASSERT_EQ(1, res_obj["hits"][0]["document"]["Customers"].count("customer_id")); + ASSERT_EQ(1, res_obj["hits"][0]["document"]["Customers"].count("customer_name")); + ASSERT_EQ(1, res_obj["hits"][0]["document"]["Customers"].count("id")); + ASSERT_EQ(1, res_obj["hits"][0]["document"]["Customers"].count("product_id")); + ASSERT_EQ(1, res_obj["hits"][0]["document"]["Customers"].count("product_id_sequence_id")); + ASSERT_EQ(1, res_obj["hits"][0]["document"]["Customers"].count("product_price")); + + req_params = { + {"collection", "Products"}, + {"q", "*"}, + {"query_by", "product_name"}, + {"filter_by", "$Customers(customer_id:=customer_a && product_price:<100)"}, + {"include_fields", "*, $Customers(*:merge) as Customers"} + }; + search_op = collectionManager.do_search(req_params, embedded_params, json_res, now_ts); + ASSERT_TRUE(search_op.ok()); + + res_obj = nlohmann::json::parse(json_res); + ASSERT_EQ(1, res_obj["found"].get()); + ASSERT_EQ(1, res_obj["hits"].size()); + ASSERT_EQ(12, res_obj["hits"][0]["document"].size()); + ASSERT_EQ(1, res_obj["hits"][0]["document"].count("id")); + ASSERT_EQ(1, res_obj["hits"][0]["document"].count("product_id")); + ASSERT_EQ(1, res_obj["hits"][0]["document"].count("product_name")); + ASSERT_EQ(1, res_obj["hits"][0]["document"].count("product_description")); + ASSERT_EQ(1, res_obj["hits"][0]["document"].count("embedding")); + ASSERT_EQ(1, res_obj["hits"][0]["document"].count("rating")); + ASSERT_EQ(1, res_obj["hits"][0]["document"].count("Customers.customer_id")); + ASSERT_EQ(1, res_obj["hits"][0]["document"].count("Customers.customer_name")); + ASSERT_EQ(1, res_obj["hits"][0]["document"].count("Customers.id")); + ASSERT_EQ(1, res_obj["hits"][0]["document"].count("Customers.product_id")); + ASSERT_EQ(1, res_obj["hits"][0]["document"].count("Customers.product_id_sequence_id")); + ASSERT_EQ(1, res_obj["hits"][0]["document"].count("Customers.product_price")); req_params = { {"collection", "Products"}, {"q", "*"}, {"query_by", "product_name"}, {"filter_by", "$Customers(customer_id:=customer_a && product_price:<100)"}, - {"include_fields", "$Customers(bar)"} + {"include_fields", "$Customers(bar:merge)"} }; search_op = collectionManager.do_search(req_params, embedded_params, json_res, now_ts); ASSERT_TRUE(search_op.ok()); @@ -1717,7 +1750,7 @@ TEST_F(CollectionJoinTest, IncludeExcludeFieldsByReference) { {"q", "*"}, {"query_by", "product_name"}, {"filter_by", "$Customers(customer_id:=customer_a && product_price:<100)"}, - {"include_fields", "$Customers(product_price)"} + {"include_fields", "$Customers(product_price:merge)"} }; search_op = collectionManager.do_search(req_params, embedded_params, json_res, now_ts); ASSERT_TRUE(search_op.ok()); @@ -1734,7 +1767,7 @@ TEST_F(CollectionJoinTest, IncludeExcludeFieldsByReference) { {"q", "*"}, {"query_by", "product_name"}, {"filter_by", "$Customers(customer_id:=customer_a && product_price:<100)"}, - {"include_fields", "$Customers(product_price, customer_id)"} + {"include_fields", "$Customers(product_price, customer_id:merge)"} }; search_op = collectionManager.do_search(req_params, embedded_params, json_res, now_ts); ASSERT_TRUE(search_op.ok()); @@ -1753,7 +1786,7 @@ TEST_F(CollectionJoinTest, IncludeExcludeFieldsByReference) { {"q", "*"}, {"query_by", "product_name"}, {"filter_by", "$Customers(customer_id:=customer_a && product_price:<100)"}, - {"include_fields", "*, $Customers(product_price, customer_id)"} + {"include_fields", "*, $Customers(product_price, customer_id:merge)"} }; search_op = collectionManager.do_search(req_params, embedded_params, json_res, now_ts); ASSERT_TRUE(search_op.ok()); @@ -1769,7 +1802,7 @@ TEST_F(CollectionJoinTest, IncludeExcludeFieldsByReference) { {"q", "*"}, {"query_by", "product_name"}, {"filter_by", "$Customers(customer_id:=customer_a && product_price:<100)"}, - {"include_fields", "$Customers(product*)"} + {"include_fields", "$Customers(product*:merge)"} }; search_op = collectionManager.do_search(req_params, embedded_params, json_res, now_ts); ASSERT_TRUE(search_op.ok()); @@ -1787,7 +1820,7 @@ TEST_F(CollectionJoinTest, IncludeExcludeFieldsByReference) { {"q", "s"}, {"query_by", "product_name"}, {"filter_by", "$Customers(customer_id:=customer_a && product_price:<100)"}, - {"include_fields", "$Customers(product*)"}, + {"include_fields", "$Customers(product*:merge)"}, {"exclude_fields", "$Customers(product_id_sequence_id)"} }; search_op = collectionManager.do_search(req_params, embedded_params, json_res, now_ts); @@ -1848,7 +1881,7 @@ TEST_F(CollectionJoinTest, IncludeExcludeFieldsByReference) { {"q", "*"}, {"query_by", "product_name"}, {"filter_by", "product_name:soap && $Customers(product_price:>100)"}, - {"include_fields", "product_name, $Customers(product_price)"}, + {"include_fields", "product_name, $Customers(product_price:merge)"}, {"exclude_fields", ""} }; search_op = collectionManager.do_search(req_params, embedded_params, json_res, now_ts); @@ -1869,7 +1902,7 @@ TEST_F(CollectionJoinTest, IncludeExcludeFieldsByReference) { {"q", "soap"}, {"query_by", "product_name"}, {"filter_by", "$Customers(product_price: >0)"}, - {"include_fields", "product_name, $Customers(customer_name, product_price)"}, + {"include_fields", "product_name, $Customers(customer_name, product_price:merge)"}, {"exclude_fields", ""} }; search_op = collectionManager.do_search(req_params, embedded_params, json_res, now_ts); @@ -1894,7 +1927,7 @@ TEST_F(CollectionJoinTest, IncludeExcludeFieldsByReference) { {"q", "natural products"}, {"query_by", "embedding"}, {"filter_by", "$Customers(customer_id:=customer_a && product_price:<100)"}, - {"include_fields", "product_name, $Customers(product_price)"}, + {"include_fields", "product_name, $Customers(product_price:merge)"}, {"exclude_fields", ""} }; search_op = collectionManager.do_search(req_params, embedded_params, json_res, now_ts); @@ -1924,7 +1957,7 @@ TEST_F(CollectionJoinTest, IncludeExcludeFieldsByReference) { {"q", "*"}, {"vector_query", "embedding:(" + vec_string + ", flat_search_cutoff: 0)"}, {"filter_by", "$Customers(customer_id:=customer_a && product_price:<100)"}, - {"include_fields", "product_name, $Customers(product_price)"}, + {"include_fields", "product_name, $Customers(product_price : merge)"}, {"exclude_fields", ""} }; search_op = collectionManager.do_search(req_params, embedded_params, json_res, now_ts); @@ -1944,7 +1977,7 @@ TEST_F(CollectionJoinTest, IncludeExcludeFieldsByReference) { {"q", "soap"}, {"query_by", "product_name, embedding"}, {"filter_by", "$Customers(customer_id:=customer_a && product_price:<100)"}, - {"include_fields", "product_name, $Customers(product_price)"}, + {"include_fields", "product_name, $Customers(product_price: merge)"}, {"exclude_fields", ""} }; search_op = collectionManager.do_search(req_params, embedded_params, json_res, now_ts); @@ -1966,7 +1999,7 @@ TEST_F(CollectionJoinTest, IncludeExcludeFieldsByReference) { {"q", "natural products"}, {"query_by", "product_name, embedding"}, {"filter_by", "$Customers(customer_id:=customer_a && product_price:<100)"}, - {"include_fields", "product_name, $Customers(product_price)"}, + {"include_fields", "product_name, $Customers(product_price :merge)"}, {"exclude_fields", ""} }; search_op = collectionManager.do_search(req_params, embedded_params, json_res, now_ts); @@ -1989,7 +2022,7 @@ TEST_F(CollectionJoinTest, IncludeExcludeFieldsByReference) { {"query_by", "product_name"}, {"infix", "always"}, {"filter_by", "$Customers(customer_id:=customer_a && product_price:<100)"}, - {"include_fields", "product_name, $Customers(product_price)"}, + {"include_fields", "product_name, $Customers(product_price:merge)"}, {"exclude_fields", ""} }; search_op = collectionManager.do_search(req_params, embedded_params, json_res, now_ts); @@ -2009,7 +2042,7 @@ TEST_F(CollectionJoinTest, IncludeExcludeFieldsByReference) { {"q", "Dan"}, {"query_by", "customer_name"}, {"filter_by", "$Products(rating:>3)"}, - {"include_fields", "$Products(product_name), product_price"} + {"include_fields", "$Products(product_name:merge), product_price"} }; search_op = collectionManager.do_search(req_params, embedded_params, json_res, now_ts); @@ -2030,7 +2063,7 @@ TEST_F(CollectionJoinTest, IncludeExcludeFieldsByReference) { {"q", "Joe"}, {"query_by", "customer_name"}, {"filter_by", "product_price:<100"}, - {"include_fields", "$Products(product_name), product_price"} + {"include_fields", "$Products(product_name: merge), product_price"} }; search_op = collectionManager.do_search(req_params, embedded_params, json_res, now_ts); ASSERT_TRUE(search_op.ok()); @@ -2045,12 +2078,37 @@ TEST_F(CollectionJoinTest, IncludeExcludeFieldsByReference) { ASSERT_EQ(73.5, res_obj["hits"][0]["document"].at("product_price")); // Add alias using `as` + req_params = { + {"collection", "Products"}, + {"q", "soap"}, + {"query_by", "product_name"}, + {"filter_by", "$Customers(id:*)"}, + {"include_fields", "id, $Customers(id :merge)"} + }; + search_op = collectionManager.do_search(req_params, embedded_params, json_res, now_ts); + ASSERT_FALSE(search_op.ok()); + ASSERT_EQ("Could not include the value of `id` key of the reference document of `Customers` collection." + " Expected `id` to be an array. Try adding an alias.", search_op.error()); + + req_params = { + {"collection", "Products"}, + {"q", "soap"}, + {"query_by", "product_name"}, + {"filter_by", "$Customers(id:*)"}, + {"include_fields", "id, $Customers(id :nest) as id"} + }; + search_op = collectionManager.do_search(req_params, embedded_params, json_res, now_ts); + ASSERT_FALSE(search_op.ok()); + ASSERT_EQ("Could not include the reference document of `Customers` collection." + " Expected `id` to be an array. Try renaming the alias.", search_op.error()); + req_params = { {"collection", "Customers"}, {"q", "Joe"}, {"query_by", "customer_name"}, {"filter_by", "product_price:<100"}, - {"include_fields", "$Products(product_name) as p, product_price"} + // With merge, alias is prepended + {"include_fields", "$Products(product_name:merge) as prod, product_price"} }; search_op = collectionManager.do_search(req_params, embedded_params, json_res, now_ts); ASSERT_TRUE(search_op.ok()); @@ -2059,11 +2117,58 @@ TEST_F(CollectionJoinTest, IncludeExcludeFieldsByReference) { ASSERT_EQ(1, res_obj["found"].get()); ASSERT_EQ(1, res_obj["hits"].size()); ASSERT_EQ(2, res_obj["hits"][0]["document"].size()); - ASSERT_EQ(1, res_obj["hits"][0]["document"].count("p.product_name")); - ASSERT_EQ("soap", res_obj["hits"][0]["document"].at("p.product_name")); + ASSERT_EQ(1, res_obj["hits"][0]["document"].count("prod.product_name")); + ASSERT_EQ("soap", res_obj["hits"][0]["document"].at("prod.product_name")); ASSERT_EQ(1, res_obj["hits"][0]["document"].count("product_price")); ASSERT_EQ(73.5, res_obj["hits"][0]["document"].at("product_price")); + req_params = { + {"collection", "Customers"}, + {"q", "Joe"}, + {"query_by", "customer_name"}, + {"filter_by", "product_price:<100"}, + // With nest, alias becomes the key + {"include_fields", "$Products(product_name:nest) as prod, product_price"} + }; + search_op = collectionManager.do_search(req_params, embedded_params, json_res, now_ts); + ASSERT_TRUE(search_op.ok()); + + res_obj = nlohmann::json::parse(json_res); + ASSERT_EQ(1, res_obj["found"].get()); + ASSERT_EQ(1, res_obj["hits"].size()); + ASSERT_EQ(2, res_obj["hits"][0]["document"].size()); + ASSERT_EQ(1, res_obj["hits"][0]["document"].count("prod")); + ASSERT_EQ(1, res_obj["hits"][0]["document"]["prod"].count("product_name")); + ASSERT_EQ("soap", res_obj["hits"][0]["document"]["prod"].at("product_name")); + ASSERT_EQ(1, res_obj["hits"][0]["document"].count("product_price")); + ASSERT_EQ(73.5, res_obj["hits"][0]["document"].at("product_price")); + + req_params = { + {"collection", "Products"}, + {"q", "soap"}, + {"query_by", "product_name"}, + {"filter_by", "$Customers(id:*)"}, + // With nest, alias becomes the key + {"include_fields", "$Customers(customer_name, product_price :nest) as CustomerPrices, product_name"} + }; + search_op = collectionManager.do_search(req_params, embedded_params, json_res, now_ts); + ASSERT_TRUE(search_op.ok()); + + res_obj = nlohmann::json::parse(json_res); + ASSERT_EQ(1, res_obj["found"].get()); + ASSERT_EQ(1, res_obj["hits"].size()); + ASSERT_EQ(2, res_obj["hits"][0]["document"].size()); + ASSERT_EQ(1, res_obj["hits"][0]["document"].count("product_name")); + ASSERT_EQ("soap", res_obj["hits"][0]["document"]["product_name"]); + ASSERT_EQ(1, res_obj["hits"][0]["document"].count("CustomerPrices")); + ASSERT_EQ(2, res_obj["hits"][0]["document"]["CustomerPrices"].size()); + + ASSERT_EQ("Joe", res_obj["hits"][0]["document"]["CustomerPrices"].at(0)["customer_name"]); + ASSERT_EQ(73.5, res_obj["hits"][0]["document"]["CustomerPrices"].at(0)["product_price"]); + + ASSERT_EQ("Dan", res_obj["hits"][0]["document"]["CustomerPrices"].at(1)["customer_name"]); + ASSERT_EQ(140, res_obj["hits"][0]["document"]["CustomerPrices"].at(1)["product_price"]); + schema_json = R"({ "name": "Users", @@ -2267,7 +2372,7 @@ TEST_F(CollectionJoinTest, IncludeExcludeFieldsByReference) { {"q", "R"}, {"query_by", "user_name"}, {"filter_by", "$Participants(org_id:=org_a) && $Links(repo_id:=repo_b)"}, - {"include_fields", "user_id, user_name, $Repos(repo_content), $Organizations(name) as org"}, + {"include_fields", "user_id, user_name, $Repos(repo_content:merge), $Organizations(name:merge) as org"}, {"exclude_fields", "$Participants(*), $Links(*), "} }; search_op = collectionManager.do_search(req_params, embedded_params, json_res, now_ts); @@ -2341,7 +2446,7 @@ TEST_F(CollectionJoinTest, FilterByReferenceArrayField) { std::map req_params = { {"collection", "songs"}, {"q", "*"}, - {"include_fields", "$genres(name) as genre"}, + {"include_fields", "$genres(name:merge) as genre"}, {"exclude_fields", "genres_sequence_id"}, }; nlohmann::json embedded_params; @@ -2372,7 +2477,7 @@ TEST_F(CollectionJoinTest, FilterByReferenceArrayField) { {"collection", "genres"}, {"q", "*"}, {"filter_by", "$songs(id: *)"}, - {"include_fields", "$songs(title) as song"}, + {"include_fields", "$songs(title:merge) as song"}, }; search_op_bool = collectionManager.do_search(req_params, embedded_params, json_res, now_ts); ASSERT_TRUE(search_op_bool.ok()); @@ -2720,7 +2825,7 @@ TEST_F(CollectionJoinTest, SortByReference) { {"query_by", "product_name"}, {"filter_by", "$Customers(customer_id:=customer_a)"}, {"sort_by", "$Customers(product_price:asc)"}, - {"include_fields", "product_id, $Customers(product_price)"}, + {"include_fields", "product_id, $Customers(product_price:merge)"}, }; search_op = collectionManager.do_search(req_params, embedded_params, json_res, now_ts); ASSERT_TRUE(search_op.ok()); @@ -2739,7 +2844,7 @@ TEST_F(CollectionJoinTest, SortByReference) { {"query_by", "product_name"}, {"filter_by", "$Customers(customer_id:=customer_a)"}, {"sort_by", "$Customers(product_price:desc)"}, - {"include_fields", "product_id, $Customers(product_price)"}, + {"include_fields", "product_id, $Customers(product_price:merge)"}, }; search_op = collectionManager.do_search(req_params, embedded_params, json_res, now_ts); ASSERT_TRUE(search_op.ok()); @@ -2759,7 +2864,7 @@ TEST_F(CollectionJoinTest, SortByReference) { {"query_by", "product_name"}, {"filter_by", "$Customers(customer_id:=customer_a)"}, {"sort_by", "$Customers(product_id:asc)"}, - {"include_fields", "product_id, $Customers(product_price)"}, + {"include_fields", "product_id, $Customers(product_price:merge)"}, }; search_op = collectionManager.do_search(req_params, embedded_params, json_res, now_ts); ASSERT_TRUE(search_op.ok()); @@ -2779,7 +2884,7 @@ TEST_F(CollectionJoinTest, SortByReference) { {"query_by", "product_name"}, {"filter_by", "$Customers(customer_id:=customer_a)"}, {"sort_by", "$Customers(_eval(product_available:true):asc)"}, - {"include_fields", "product_id, $Customers(product_price)"}, + {"include_fields", "product_id, $Customers(product_price:merge)"}, }; search_op = collectionManager.do_search(req_params, embedded_params, json_res, now_ts); ASSERT_TRUE(search_op.ok()); @@ -2798,7 +2903,7 @@ TEST_F(CollectionJoinTest, SortByReference) { {"query_by", "product_name"}, {"filter_by", "$Customers(customer_id:=customer_a)"}, {"sort_by", "$Customers(_eval(product_available:true):desc)"}, - {"include_fields", "product_id, $Customers(product_price)"}, + {"include_fields", "product_id, $Customers(product_price:merge)"}, }; search_op = collectionManager.do_search(req_params, embedded_params, json_res, now_ts); ASSERT_TRUE(search_op.ok()); @@ -2818,7 +2923,7 @@ TEST_F(CollectionJoinTest, SortByReference) { {"query_by", "product_name"}, {"filter_by", "$Customers(customer_id:=customer_a)"}, {"sort_by", "$Customers(product_price:desc)"}, - {"include_fields", "product_id, $Customers(product_price)"}, + {"include_fields", "product_id, $Customers(product_price:merge)"}, }; search_op = collectionManager.do_search(req_params, embedded_params, json_res, now_ts); ASSERT_TRUE(search_op.ok()); @@ -2837,7 +2942,7 @@ TEST_F(CollectionJoinTest, SortByReference) { {"q", R"("our")"}, {"query_by", "product_description"}, {"filter_by", "$Customers(customer_id:=customer_a)"}, - {"include_fields", "product_id, $Customers(product_price)"}, + {"include_fields", "product_id, $Customers(product_price:merge)"}, {"sort_by", "$Customers(product_price:desc)"}, }; search_op = collectionManager.do_search(req_params, embedded_params, json_res, now_ts); @@ -2857,7 +2962,7 @@ TEST_F(CollectionJoinTest, SortByReference) { {"q", "natural products"}, {"query_by", "embedding"}, {"filter_by", "$Customers(customer_id:=customer_a)"}, - {"include_fields", "product_id, $Customers(product_price)"}, + {"include_fields", "product_id, $Customers(product_price:merge)"}, {"sort_by", "$Customers(product_price:desc)"}, }; search_op = collectionManager.do_search(req_params, embedded_params, json_res, now_ts); @@ -2891,7 +2996,7 @@ TEST_F(CollectionJoinTest, SortByReference) { {"q", "*"}, {"vector_query", "embedding:(" + vec_string + ", flat_search_cutoff: 0)"}, {"filter_by", "$Customers(customer_id:=customer_a)"}, - {"include_fields", "product_id, $Customers(product_price)"}, + {"include_fields", "product_id, $Customers(product_price:merge)"}, {"sort_by", "$Customers(product_price:desc)"}, }; search_op = collectionManager.do_search(req_params, embedded_params, json_res, now_ts); @@ -2915,7 +3020,7 @@ TEST_F(CollectionJoinTest, SortByReference) { {"q", "soap"}, {"query_by", "product_name, embedding"}, {"filter_by", "$Customers(customer_id:=customer_a)"}, - {"include_fields", "product_id, $Customers(product_price)"}, + {"include_fields", "product_id, $Customers(product_price:merge)"}, {"sort_by", "$Customers(product_price:desc)"}, }; search_op = collectionManager.do_search(req_params, embedded_params, json_res, now_ts); @@ -2941,7 +3046,7 @@ TEST_F(CollectionJoinTest, SortByReference) { {"q", "natural products"}, {"query_by", "product_name, embedding"}, {"filter_by", "$Customers(customer_id:=customer_a)"}, - {"include_fields", "product_id, $Customers(product_price)"}, + {"include_fields", "product_id, $Customers(product_price:merge)"}, {"sort_by", "$Customers(product_price:desc)"}, }; search_op = collectionManager.do_search(req_params, embedded_params, json_res, now_ts); @@ -2966,7 +3071,7 @@ TEST_F(CollectionJoinTest, SortByReference) { {"query_by", "product_name"}, {"infix", "always"}, {"filter_by", "$Customers(customer_id:=customer_a)"}, - {"include_fields", "product_id, $Customers(product_price)"}, + {"include_fields", "product_id, $Customers(product_price:merge)"}, {"sort_by", "$Customers(product_price:desc)"}, }; search_op = collectionManager.do_search(req_params, embedded_params, json_res, now_ts); @@ -2985,7 +3090,7 @@ TEST_F(CollectionJoinTest, SortByReference) { {"collection", "Customers"}, {"q", "*"}, {"filter_by", "customer_name:= [Joe, Dan] && product_price:<100"}, - {"include_fields", "$Products(product_name), product_price"}, + {"include_fields", "$Products(product_name:merge), product_price"}, {"sort_by", "$Products(product_name:desc)"}, }; search_op = collectionManager.do_search(req_params, embedded_params, json_res, now_ts); @@ -3004,7 +3109,7 @@ TEST_F(CollectionJoinTest, SortByReference) { {"collection", "Customers"}, {"q", "*"}, {"filter_by", "customer_name:= [Joe, Dan] && product_price:<100"}, - {"include_fields", "$Products(product_name), product_price"}, + {"include_fields", "$Products(product_name:merge), product_price"}, {"sort_by", "$Products(product_name:asc)"}, }; search_op = collectionManager.do_search(req_params, embedded_params, json_res, now_ts); @@ -3166,7 +3271,7 @@ TEST_F(CollectionJoinTest, SortByReference) { {"collection", "Users"}, {"q", "*"}, {"filter_by", "$Links(repo_id:=[repo_a, repo_d])"}, - {"include_fields", "user_id, user_name, $Repos(repo_content, repo_stars), "}, + {"include_fields", "user_id, user_name, $Repos(repo_content, repo_stars:merge), "}, {"exclude_fields", "$Links(*), "}, {"sort_by", "$Repos(repo_stars: asc)"} }; @@ -3196,7 +3301,7 @@ TEST_F(CollectionJoinTest, SortByReference) { {"collection", "Users"}, {"q", "*"}, {"filter_by", "$Links(repo_id:=[repo_a, repo_d])"}, - {"include_fields", "user_id, user_name, $Repos(repo_content, repo_stars), "}, + {"include_fields", "user_id, user_name, $Repos(repo_content, repo_stars:merge), "}, {"exclude_fields", "$Links(*), "}, {"sort_by", "$Repos(repo_stars: desc)"} };