postgres/typesense
1
0
Fork 0
mirror of https://github.com/typesense/typesense.git synced 2025-05-19 05:08:43 +08:00
Code Issues Actions 1 Packages Projects Releases Wiki Activity

Merge pull request #1341 from happy-san/reference_includes

Browse Source 
Include reference document by merging or nesting.
This commit is contained in:
Kishore Nallan 2023-10-27 13:26:03 +05:30 committed by GitHub
commit 781e77d613
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
5 changed files with 251 additions and 85 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

7
include/collection.h
View File

@ -267,7 +267,8 @@ private:
std::vector<std::string>& processed_search_fields,
bool extract_only_string_fields,
bool enable_nested_fields,
const bool handle_wildcard = true);
const bool handle_wildcard = true,
const bool& include_id = false);
bool is_nested_array(const nlohmann::json& obj, std::vector<std::string> path_parts, size_t part_i) const;
@ -399,12 +400,14 @@ public:
static void remove_flat_fields(nlohmann::json& document);
static Option<bool> add_reference_fields(nlohmann::json& doc,
const std::string& ref_collection_name,
Collection *const ref_collection,
const std::string& alias,
const reference_filter_result_t& references,
const tsl::htrie_set<char>& ref_include_fields_full,
const tsl::htrie_set<char>& ref_exclude_fields_full,
const std::string& error_prefix, const bool& is_reference_array);
const std::string& error_prefix, const bool& is_reference_array,
const bool& nest_ref_doc);
static Option<bool> prune_doc(nlohmann::json& doc, const tsl::htrie_set<char>& include_names,
const tsl::htrie_set<char>& exclude_names, const std::string& parent_name = "",

9
include/field.h
View File

@ -496,9 +496,16 @@ namespace sort_field_const {
static const std::string vector_query = "_vector_query";
}
namespace ref_include {
static const std::string merge = "merge";
static const std::string nest = "nest";
}
struct ref_include_fields {
std::string expression;
std::string collection_name;
std::string fields;
std::string alias;
bool nest_ref_doc = false;
};
struct hnsw_index_t;

103
src/collection.cpp
View File

@ -1278,7 +1278,8 @@ Option<bool> Collection::extract_field_name(const std::string& field_name,
std::vector<std::string>& processed_search_fields,
const bool extract_only_string_fields,
const bool enable_nested_fields,
const bool handle_wildcard) {
const bool handle_wildcard,
const bool& include_id) {
// Reference to other collection
if (field_name[0] == '$') {
processed_search_fields.push_back(field_name);
@ -1294,6 +1295,12 @@ Option<bool> Collection::extract_field_name(const std::string& field_name,
if (is_wildcard && !handle_wildcard) {
return Option<bool>(400, "Pattern `" + field_name + "` is not allowed.");
}
if (is_wildcard && include_id && field_name.size() < 4 &&
(field_name == "*" || field_name == "i*" || field_name == "id*")) {
processed_search_fields.emplace_back("id");
}
// If wildcard, remove *
auto prefix_it = search_schema.equal_prefix_range(field_name.substr(0, field_name.size() - is_wildcard));
bool field_found = false;
@ -4440,12 +4447,14 @@ void Collection::remove_flat_fields(nlohmann::json& document) {
}
Option<bool> Collection::add_reference_fields(nlohmann::json& doc,
const std::string& ref_collection_name,
Collection *const ref_collection,
const std::string& alias,
const reference_filter_result_t& references,
const tsl::htrie_set<char>& ref_include_fields_full,
const tsl::htrie_set<char>& ref_exclude_fields_full,
const std::string& error_prefix, const bool& is_reference_array) {
const std::string& error_prefix, const bool& is_reference_array,
const bool& nest_ref_doc) {
// One-to-one relation.
if (!is_reference_array && references.count == 1) {
auto ref_doc_seq_id = references.docs[0];
@ -4463,15 +4472,23 @@ Option<bool> Collection::add_reference_fields(nlohmann::json& doc,
return Option<bool>(prune_op.code(), error_prefix + prune_op.error());
}
if (!alias.empty()) {
auto temp_doc = ref_doc;
ref_doc.clear();
for (const auto &item: temp_doc.items()) {
ref_doc[alias + item.key()] = item.value();
}
if (ref_doc.empty()) {
return Option<bool>(true);
}
doc.update(ref_doc);
if (nest_ref_doc) {
auto key = alias.empty() ? ref_collection_name : alias;
doc[key] = ref_doc;
} else {
if (!alias.empty()) {
auto temp_doc = ref_doc;
ref_doc.clear();
for (const auto &item: temp_doc.items()) {
ref_doc[alias + item.key()] = item.value();
}
}
doc.update(ref_doc);
}
return Option<bool>(true);
}
@ -4492,17 +4509,33 @@ Option<bool> Collection::add_reference_fields(nlohmann::json& doc,
return Option<bool>(prune_op.code(), error_prefix + prune_op.error());
}
if (!alias.empty()) {
auto temp_doc = ref_doc;
ref_doc.clear();
for (const auto &item: temp_doc.items()) {
ref_doc[alias + item.key()] = item.value();
}
if (ref_doc.empty()) {
continue;
}
for (auto ref_doc_it = ref_doc.begin(); ref_doc_it != ref_doc.end(); ref_doc_it++) {
// Add the values of ref_doc as JSON array into doc.
doc[ref_doc_it.key()] += ref_doc_it.value();
if (nest_ref_doc) {
auto key = alias.empty() ? ref_collection_name : alias;
if (doc.contains(key) && !doc[key].is_array()) {
return Option<bool>(400, "Could not include the reference document of `" + ref_collection_name +
"` collection. Expected `" + key + "` to be an array. Try " +
(alias.empty() ? "adding an" : "renaming the") + " alias.");
}
doc[key] += ref_doc;
} else {
for (auto ref_doc_it = ref_doc.begin(); ref_doc_it != ref_doc.end(); ref_doc_it++) {
auto const& ref_doc_key = ref_doc_it.key();
auto const& doc_key = alias + ref_doc_key;
if (doc.contains(doc_key) && !doc[doc_key].is_array()) {
return Option<bool>(400, "Could not include the value of `" + ref_doc_key +
"` key of the reference document of `" + ref_collection_name +
"` collection. Expected `" + doc_key + "` to be an array. Try " +
(alias.empty() ? "adding an" : "renaming the") + " alias.");
}
// Add the values of ref_doc as JSON array into doc.
doc[doc_key] += ref_doc_it.value();
}
}
}
@ -4592,11 +4625,7 @@ Option<bool> Collection::prune_doc(nlohmann::json& doc,
}
for (auto const& ref_include: ref_includes) {
auto const& ref = ref_include.expression;
size_t parenthesis_index = ref.find('(');
auto ref_collection_name = ref.substr(1, parenthesis_index - 1);
auto reference_fields = ref.substr(parenthesis_index + 1, ref.size() - parenthesis_index - 2);
auto const& ref_collection_name = ref_include.collection_name;
auto& cm = CollectionManager::get_instance();
auto ref_collection = cm.get_collection(ref_collection_name);
@ -4631,12 +4660,12 @@ Option<bool> Collection::prune_doc(nlohmann::json& doc,
}
std::vector<std::string> ref_include_fields_vec, ref_exclude_fields_vec;
StringUtils::split(reference_fields, ref_include_fields_vec, ",");
StringUtils::split(ref_include.fields, ref_include_fields_vec, ",");
auto exclude_reference_it = exclude_names.equal_prefix_range("$" + ref_collection_name);
if (exclude_reference_it.first != exclude_reference_it.second) {
auto ref_exclude = exclude_reference_it.first.key();
parenthesis_index = ref_exclude.find('(');
reference_fields = ref_exclude.substr(parenthesis_index + 1, ref_exclude.size() - parenthesis_index - 2);
auto parenthesis_index = ref_exclude.find('(');
auto reference_fields = ref_exclude.substr(parenthesis_index + 1, ref_exclude.size() - parenthesis_index - 2);
StringUtils::split(reference_fields, ref_exclude_fields_vec, ",");
}
@ -4664,10 +4693,12 @@ Option<bool> Collection::prune_doc(nlohmann::json& doc,
if (ref_collection->search_schema.count(field_name) == 0) {
continue;
}
add_reference_fields_op = add_reference_fields(doc, ref_collection.get(), ref_include.alias,
add_reference_fields_op = add_reference_fields(doc, ref_include.collection_name,
ref_collection.get(), ref_include.alias,
reference_filter_results.at(ref_collection_name),
ref_include_fields_full, ref_exclude_fields_full, error_prefix,
ref_collection->get_schema().at(field_name).is_array());
ref_collection->get_schema().at(field_name).is_array(),
ref_include.nest_ref_doc);
} else if (doc_has_reference) {
auto get_reference_field_op = ref_collection->get_referenced_in_field_with_lock(collection->name);
if (!get_reference_field_op.ok()) {
@ -4687,9 +4718,11 @@ Option<bool> Collection::prune_doc(nlohmann::json& doc,
result.count = ids.size();
result.docs = &ids[0];
add_reference_fields_op = add_reference_fields(doc, ref_collection.get(), ref_include.alias, result,
add_reference_fields_op = add_reference_fields(doc, ref_include.collection_name,
ref_collection.get(), ref_include.alias, result,
ref_include_fields_full, ref_exclude_fields_full, error_prefix,
collection->search_schema.at(field_name).is_array());
collection->search_schema.at(field_name).is_array(),
ref_include.nest_ref_doc);
result.docs = nullptr;
} else if (joined_coll_has_reference) {
auto joined_collection = cm.get_collection(joined_coll_having_reference);
@ -4720,9 +4753,11 @@ Option<bool> Collection::prune_doc(nlohmann::json& doc,
reference_filter_result_t result;
result.count = ids.size();
result.docs = &ids[0];
add_reference_fields_op = add_reference_fields(doc, ref_collection.get(), ref_include.alias, result,
add_reference_fields_op = add_reference_fields(doc, ref_include.collection_name,
ref_collection.get(), ref_include.alias, result,
ref_include_fields_full, ref_exclude_fields_full, error_prefix,
joined_collection->get_schema().at(reference_field_name).is_array());
joined_collection->get_schema().at(reference_field_name).is_array(),
ref_include.nest_ref_doc);
result.docs = nullptr;
}
@ -5580,7 +5615,7 @@ Option<bool> Collection::populate_include_exclude_fields(const spp::sparse_hash_
std::vector<std::string> exclude_fields_vec;
for(auto& f_name: include_fields) {
auto field_op = extract_field_name(f_name, search_schema, include_fields_vec, false, enable_nested_fields);
auto field_op = extract_field_name(f_name, search_schema, include_fields_vec, false, enable_nested_fields, true, true);
if(!field_op.ok()) {
if(field_op.code() == 404) {
// field need not be part of schema to be included (could be a stored value in the doc)
@ -5597,7 +5632,7 @@ Option<bool> Collection::populate_include_exclude_fields(const spp::sparse_hash_
continue;
}
auto field_op = extract_field_name(f_name, search_schema, exclude_fields_vec, false, enable_nested_fields);
auto field_op = extract_field_name(f_name, search_schema, exclude_fields_vec, false, enable_nested_fields, true, true);
if(!field_op.ok()) {
if(field_op.code() == 404) {
// field need not be part of schema to be excluded (could be a stored value in the doc)

28
src/collection_manager.cpp
View File

@ -967,14 +967,30 @@ void initialize_ref_include_fields_vec(const std::string& filter_query, std::vec
continue;
}
// Format: $ref_collection_name(field_1, field_2: include_strategy) as ref_alias
auto as_pos = include_field_exp.find(" as ");
auto ref_include = include_field_exp.substr(0, as_pos),
alias = (as_pos == std::string::npos) ? "" :
auto ref_include = include_field_exp.substr(0, as_pos);
auto alias = (as_pos == std::string::npos) ? "" :
include_field_exp.substr(as_pos + 4, include_field_exp.size() - (as_pos + 4));
// For an alias `foo`, we need append `foo.` to all the top level keys of reference doc.
ref_include_fields_vec.emplace_back(ref_include_fields{ref_include, alias.empty() ? alias :
StringUtils::trim(alias) + "."});
auto parenthesis_index = ref_include.find('(');
auto ref_collection_name = ref_include.substr(1, parenthesis_index - 1);
auto ref_fields = ref_include.substr(parenthesis_index + 1, ref_include.size() - parenthesis_index - 2);
auto nest_ref_doc = true;
auto colon_pos = ref_fields.find(':');
if (colon_pos != std::string::npos) {
auto include_strategy = ref_fields.substr(colon_pos + 1, ref_fields.size() - colon_pos - 1);
StringUtils::trim(include_strategy);
nest_ref_doc = include_strategy == ref_include::nest;
ref_fields = ref_fields.substr(0, colon_pos);
}
// For an alias `foo`,
// In case of "merge" reference doc, we need append `foo.` to all the top level keys of reference doc.
// In case of "nest" reference doc, `foo` becomes the key with reference doc as value.
auto ref_alias = !alias.empty() ? (StringUtils::trim(alias) + (nest_ref_doc ? "" : ".")) : "";
ref_include_fields_vec.emplace_back(ref_include_fields{ref_collection_name, ref_fields, ref_alias, nest_ref_doc});
auto open_paren_pos = include_field_exp.find('(');
if (open_paren_pos == std::string::npos) {
@ -993,7 +1009,7 @@ void initialize_ref_include_fields_vec(const std::string& filter_query, std::vec
// Get all the fields of the referenced collection in the filter but not mentioned in include_fields.
for (const auto &reference_collection_name: reference_collection_names) {
ref_include_fields_vec.emplace_back(ref_include_fields{"$" + reference_collection_name + "(*)", ""});
ref_include_fields_vec.emplace_back(ref_include_fields{reference_collection_name, "", "", true});
}
// Since no field of the collection is mentioned in include_fields, get all the fields.

189
test/collection_join_test.cpp
View File

@ -915,6 +915,7 @@ TEST_F(CollectionJoinTest, FilterByReference_SingleMatch) {
{"q", "Dan"},
{"query_by", "customer_name"},
{"filter_by", "$Products(rating:>3)"},
{"include_fields", "$Products(*:merge)"},
};
search_op_bool = collectionManager.do_search(req_params, embedded_params, json_res, now_ts);
@ -930,6 +931,7 @@ TEST_F(CollectionJoinTest, FilterByReference_SingleMatch) {
{"q", "Dan"},
{"query_by", "customer_name"},
{"filter_by", "$Products(id:*) && product_price:>100"},
{"include_fields", "$Products(*:merge)"},
};
search_op_bool = collectionManager.do_search(req_params, embedded_params, json_res, now_ts);
@ -1678,24 +1680,55 @@ TEST_F(CollectionJoinTest, IncludeExcludeFieldsByReference) {
ASSERT_EQ(1, res_obj["found"].get<size_t>());
ASSERT_EQ(1, res_obj["hits"].size());
// No fields are mentioned in `include_fields`, should include all fields of Products and Customers by default.
ASSERT_EQ(10, res_obj["hits"][0]["document"].size());
ASSERT_EQ(7, res_obj["hits"][0]["document"].size());
ASSERT_EQ(1, res_obj["hits"][0]["document"].count("id"));
ASSERT_EQ(1, res_obj["hits"][0]["document"].count("product_id"));
ASSERT_EQ(1, res_obj["hits"][0]["document"].count("product_name"));
ASSERT_EQ(1, res_obj["hits"][0]["document"].count("product_description"));
ASSERT_EQ(1, res_obj["hits"][0]["document"].count("embedding"));
ASSERT_EQ(1, res_obj["hits"][0]["document"].count("rating"));
ASSERT_EQ(1, res_obj["hits"][0]["document"].count("customer_id"));
ASSERT_EQ(1, res_obj["hits"][0]["document"].count("customer_name"));
ASSERT_EQ(1, res_obj["hits"][0]["document"].count("product_price"));
ASSERT_EQ(1, res_obj["hits"][0]["document"].count("product_id_sequence_id"));
// Default strategy of reference includes is nest. No alias was provided, collection name becomes the field name.
ASSERT_EQ(6, res_obj["hits"][0]["document"]["Customers"].size());
ASSERT_EQ(1, res_obj["hits"][0]["document"]["Customers"].count("customer_id"));
ASSERT_EQ(1, res_obj["hits"][0]["document"]["Customers"].count("customer_name"));
ASSERT_EQ(1, res_obj["hits"][0]["document"]["Customers"].count("id"));
ASSERT_EQ(1, res_obj["hits"][0]["document"]["Customers"].count("product_id"));
ASSERT_EQ(1, res_obj["hits"][0]["document"]["Customers"].count("product_id_sequence_id"));
ASSERT_EQ(1, res_obj["hits"][0]["document"]["Customers"].count("product_price"));
req_params = {
{"collection", "Products"},
{"q", "*"},
{"query_by", "product_name"},
{"filter_by", "$Customers(customer_id:=customer_a && product_price:<100)"},
{"include_fields", "*, $Customers(*:merge) as Customers"}
};
search_op = collectionManager.do_search(req_params, embedded_params, json_res, now_ts);
ASSERT_TRUE(search_op.ok());
res_obj = nlohmann::json::parse(json_res);
ASSERT_EQ(1, res_obj["found"].get<size_t>());
ASSERT_EQ(1, res_obj["hits"].size());
ASSERT_EQ(12, res_obj["hits"][0]["document"].size());
ASSERT_EQ(1, res_obj["hits"][0]["document"].count("id"));
ASSERT_EQ(1, res_obj["hits"][0]["document"].count("product_id"));
ASSERT_EQ(1, res_obj["hits"][0]["document"].count("product_name"));
ASSERT_EQ(1, res_obj["hits"][0]["document"].count("product_description"));
ASSERT_EQ(1, res_obj["hits"][0]["document"].count("embedding"));
ASSERT_EQ(1, res_obj["hits"][0]["document"].count("rating"));
ASSERT_EQ(1, res_obj["hits"][0]["document"].count("Customers.customer_id"));
ASSERT_EQ(1, res_obj["hits"][0]["document"].count("Customers.customer_name"));
ASSERT_EQ(1, res_obj["hits"][0]["document"].count("Customers.id"));
ASSERT_EQ(1, res_obj["hits"][0]["document"].count("Customers.product_id"));
ASSERT_EQ(1, res_obj["hits"][0]["document"].count("Customers.product_id_sequence_id"));
ASSERT_EQ(1, res_obj["hits"][0]["document"].count("Customers.product_price"));
req_params = {
{"collection", "Products"},
{"q", "*"},
{"query_by", "product_name"},
{"filter_by", "$Customers(customer_id:=customer_a && product_price:<100)"},
{"include_fields", "$Customers(bar)"}
{"include_fields", "$Customers(bar:merge)"}
};
search_op = collectionManager.do_search(req_params, embedded_params, json_res, now_ts);
ASSERT_TRUE(search_op.ok());
@ -1717,7 +1750,7 @@ TEST_F(CollectionJoinTest, IncludeExcludeFieldsByReference) {
{"q", "*"},
{"query_by", "product_name"},
{"filter_by", "$Customers(customer_id:=customer_a && product_price:<100)"},
{"include_fields", "$Customers(product_price)"}
{"include_fields", "$Customers(product_price:merge)"}
};
search_op = collectionManager.do_search(req_params, embedded_params, json_res, now_ts);
ASSERT_TRUE(search_op.ok());
@ -1734,7 +1767,7 @@ TEST_F(CollectionJoinTest, IncludeExcludeFieldsByReference) {
{"q", "*"},
{"query_by", "product_name"},
{"filter_by", "$Customers(customer_id:=customer_a && product_price:<100)"},
{"include_fields", "$Customers(product_price, customer_id)"}
{"include_fields", "$Customers(product_price, customer_id:merge)"}
};
search_op = collectionManager.do_search(req_params, embedded_params, json_res, now_ts);
ASSERT_TRUE(search_op.ok());
@ -1753,7 +1786,7 @@ TEST_F(CollectionJoinTest, IncludeExcludeFieldsByReference) {
{"q", "*"},
{"query_by", "product_name"},
{"filter_by", "$Customers(customer_id:=customer_a && product_price:<100)"},
{"include_fields", "*, $Customers(product_price, customer_id)"}
{"include_fields", "*, $Customers(product_price, customer_id:merge)"}
};
search_op = collectionManager.do_search(req_params, embedded_params, json_res, now_ts);
ASSERT_TRUE(search_op.ok());
@ -1769,7 +1802,7 @@ TEST_F(CollectionJoinTest, IncludeExcludeFieldsByReference) {
{"q", "*"},
{"query_by", "product_name"},
{"filter_by", "$Customers(customer_id:=customer_a && product_price:<100)"},
{"include_fields", "$Customers(product*)"}
{"include_fields", "$Customers(product*:merge)"}
};
search_op = collectionManager.do_search(req_params, embedded_params, json_res, now_ts);
ASSERT_TRUE(search_op.ok());
@ -1787,7 +1820,7 @@ TEST_F(CollectionJoinTest, IncludeExcludeFieldsByReference) {
{"q", "s"},
{"query_by", "product_name"},
{"filter_by", "$Customers(customer_id:=customer_a && product_price:<100)"},
{"include_fields", "$Customers(product*)"},
{"include_fields", "$Customers(product*:merge)"},
{"exclude_fields", "$Customers(product_id_sequence_id)"}
};
search_op = collectionManager.do_search(req_params, embedded_params, json_res, now_ts);
@ -1848,7 +1881,7 @@ TEST_F(CollectionJoinTest, IncludeExcludeFieldsByReference) {
{"q", "*"},
{"query_by", "product_name"},
{"filter_by", "product_name:soap && $Customers(product_price:>100)"},
{"include_fields", "product_name, $Customers(product_price)"},
{"include_fields", "product_name, $Customers(product_price:merge)"},
{"exclude_fields", ""}
};
search_op = collectionManager.do_search(req_params, embedded_params, json_res, now_ts);
@ -1869,7 +1902,7 @@ TEST_F(CollectionJoinTest, IncludeExcludeFieldsByReference) {
{"q", "soap"},
{"query_by", "product_name"},
{"filter_by", "$Customers(product_price: >0)"},
{"include_fields", "product_name, $Customers(customer_name, product_price)"},
{"include_fields", "product_name, $Customers(customer_name, product_price:merge)"},
{"exclude_fields", ""}
};
search_op = collectionManager.do_search(req_params, embedded_params, json_res, now_ts);
@ -1894,7 +1927,7 @@ TEST_F(CollectionJoinTest, IncludeExcludeFieldsByReference) {
{"q", "natural products"},
{"query_by", "embedding"},
{"filter_by", "$Customers(customer_id:=customer_a && product_price:<100)"},
{"include_fields", "product_name, $Customers(product_price)"},
{"include_fields", "product_name, $Customers(product_price:merge)"},
{"exclude_fields", ""}
};
search_op = collectionManager.do_search(req_params, embedded_params, json_res, now_ts);
@ -1924,7 +1957,7 @@ TEST_F(CollectionJoinTest, IncludeExcludeFieldsByReference) {
{"q", "*"},
{"vector_query", "embedding:(" + vec_string + ", flat_search_cutoff: 0)"},
{"filter_by", "$Customers(customer_id:=customer_a && product_price:<100)"},
{"include_fields", "product_name, $Customers(product_price)"},
{"include_fields", "product_name, $Customers(product_price : merge)"},
{"exclude_fields", ""}
};
search_op = collectionManager.do_search(req_params, embedded_params, json_res, now_ts);
@ -1944,7 +1977,7 @@ TEST_F(CollectionJoinTest, IncludeExcludeFieldsByReference) {
{"q", "soap"},
{"query_by", "product_name, embedding"},
{"filter_by", "$Customers(customer_id:=customer_a && product_price:<100)"},
{"include_fields", "product_name, $Customers(product_price)"},
{"include_fields", "product_name, $Customers(product_price: merge)"},
{"exclude_fields", ""}
};
search_op = collectionManager.do_search(req_params, embedded_params, json_res, now_ts);
@ -1966,7 +1999,7 @@ TEST_F(CollectionJoinTest, IncludeExcludeFieldsByReference) {
{"q", "natural products"},
{"query_by", "product_name, embedding"},
{"filter_by", "$Customers(customer_id:=customer_a && product_price:<100)"},
{"include_fields", "product_name, $Customers(product_price)"},
{"include_fields", "product_name, $Customers(product_price :merge)"},
{"exclude_fields", ""}
};
search_op = collectionManager.do_search(req_params, embedded_params, json_res, now_ts);
@ -1989,7 +2022,7 @@ TEST_F(CollectionJoinTest, IncludeExcludeFieldsByReference) {
{"query_by", "product_name"},
{"infix", "always"},
{"filter_by", "$Customers(customer_id:=customer_a && product_price:<100)"},
{"include_fields", "product_name, $Customers(product_price)"},
{"include_fields", "product_name, $Customers(product_price:merge)"},
{"exclude_fields", ""}
};
search_op = collectionManager.do_search(req_params, embedded_params, json_res, now_ts);
@ -2009,7 +2042,7 @@ TEST_F(CollectionJoinTest, IncludeExcludeFieldsByReference) {
{"q", "Dan"},
{"query_by", "customer_name"},
{"filter_by", "$Products(rating:>3)"},
{"include_fields", "$Products(product_name), product_price"}
{"include_fields", "$Products(product_name:merge), product_price"}
};
search_op = collectionManager.do_search(req_params, embedded_params, json_res, now_ts);
@ -2030,7 +2063,7 @@ TEST_F(CollectionJoinTest, IncludeExcludeFieldsByReference) {
{"q", "Joe"},
{"query_by", "customer_name"},
{"filter_by", "product_price:<100"},
{"include_fields", "$Products(product_name), product_price"}
{"include_fields", "$Products(product_name: merge), product_price"}
};
search_op = collectionManager.do_search(req_params, embedded_params, json_res, now_ts);
ASSERT_TRUE(search_op.ok());
@ -2045,12 +2078,37 @@ TEST_F(CollectionJoinTest, IncludeExcludeFieldsByReference) {
ASSERT_EQ(73.5, res_obj["hits"][0]["document"].at("product_price"));
// Add alias using `as`
req_params = {
{"collection", "Products"},
{"q", "soap"},
{"query_by", "product_name"},
{"filter_by", "$Customers(id:*)"},
{"include_fields", "id, $Customers(id :merge)"}
};
search_op = collectionManager.do_search(req_params, embedded_params, json_res, now_ts);
ASSERT_FALSE(search_op.ok());
ASSERT_EQ("Could not include the value of `id` key of the reference document of `Customers` collection."
" Expected `id` to be an array. Try adding an alias.", search_op.error());
req_params = {
{"collection", "Products"},
{"q", "soap"},
{"query_by", "product_name"},
{"filter_by", "$Customers(id:*)"},
{"include_fields", "id, $Customers(id :nest) as id"}
};
search_op = collectionManager.do_search(req_params, embedded_params, json_res, now_ts);
ASSERT_FALSE(search_op.ok());
ASSERT_EQ("Could not include the reference document of `Customers` collection."
" Expected `id` to be an array. Try renaming the alias.", search_op.error());
req_params = {
{"collection", "Customers"},
{"q", "Joe"},
{"query_by", "customer_name"},
{"filter_by", "product_price:<100"},
{"include_fields", "$Products(product_name) as p, product_price"}
// With merge, alias is prepended
{"include_fields", "$Products(product_name:merge) as prod, product_price"}
};
search_op = collectionManager.do_search(req_params, embedded_params, json_res, now_ts);
ASSERT_TRUE(search_op.ok());
@ -2059,11 +2117,58 @@ TEST_F(CollectionJoinTest, IncludeExcludeFieldsByReference) {
ASSERT_EQ(1, res_obj["found"].get<size_t>());
ASSERT_EQ(1, res_obj["hits"].size());
ASSERT_EQ(2, res_obj["hits"][0]["document"].size());
ASSERT_EQ(1, res_obj["hits"][0]["document"].count("p.product_name"));
ASSERT_EQ("soap", res_obj["hits"][0]["document"].at("p.product_name"));
ASSERT_EQ(1, res_obj["hits"][0]["document"].count("prod.product_name"));
ASSERT_EQ("soap", res_obj["hits"][0]["document"].at("prod.product_name"));
ASSERT_EQ(1, res_obj["hits"][0]["document"].count("product_price"));
ASSERT_EQ(73.5, res_obj["hits"][0]["document"].at("product_price"));
req_params = {
{"collection", "Customers"},
{"q", "Joe"},
{"query_by", "customer_name"},
{"filter_by", "product_price:<100"},
// With nest, alias becomes the key
{"include_fields", "$Products(product_name:nest) as prod, product_price"}
};
search_op = collectionManager.do_search(req_params, embedded_params, json_res, now_ts);
ASSERT_TRUE(search_op.ok());
res_obj = nlohmann::json::parse(json_res);
ASSERT_EQ(1, res_obj["found"].get<size_t>());
ASSERT_EQ(1, res_obj["hits"].size());
ASSERT_EQ(2, res_obj["hits"][0]["document"].size());
ASSERT_EQ(1, res_obj["hits"][0]["document"].count("prod"));
ASSERT_EQ(1, res_obj["hits"][0]["document"]["prod"].count("product_name"));
ASSERT_EQ("soap", res_obj["hits"][0]["document"]["prod"].at("product_name"));
ASSERT_EQ(1, res_obj["hits"][0]["document"].count("product_price"));
ASSERT_EQ(73.5, res_obj["hits"][0]["document"].at("product_price"));
req_params = {
{"collection", "Products"},
{"q", "soap"},
{"query_by", "product_name"},
{"filter_by", "$Customers(id:*)"},
// With nest, alias becomes the key
{"include_fields", "$Customers(customer_name, product_price :nest) as CustomerPrices, product_name"}
};
search_op = collectionManager.do_search(req_params, embedded_params, json_res, now_ts);
ASSERT_TRUE(search_op.ok());
res_obj = nlohmann::json::parse(json_res);
ASSERT_EQ(1, res_obj["found"].get<size_t>());
ASSERT_EQ(1, res_obj["hits"].size());
ASSERT_EQ(2, res_obj["hits"][0]["document"].size());
ASSERT_EQ(1, res_obj["hits"][0]["document"].count("product_name"));
ASSERT_EQ("soap", res_obj["hits"][0]["document"]["product_name"]);
ASSERT_EQ(1, res_obj["hits"][0]["document"].count("CustomerPrices"));
ASSERT_EQ(2, res_obj["hits"][0]["document"]["CustomerPrices"].size());
ASSERT_EQ("Joe", res_obj["hits"][0]["document"]["CustomerPrices"].at(0)["customer_name"]);
ASSERT_EQ(73.5, res_obj["hits"][0]["document"]["CustomerPrices"].at(0)["product_price"]);
ASSERT_EQ("Dan", res_obj["hits"][0]["document"]["CustomerPrices"].at(1)["customer_name"]);
ASSERT_EQ(140, res_obj["hits"][0]["document"]["CustomerPrices"].at(1)["product_price"]);
schema_json =
R"({
"name": "Users",
@ -2267,7 +2372,7 @@ TEST_F(CollectionJoinTest, IncludeExcludeFieldsByReference) {
{"q", "R"},
{"query_by", "user_name"},
{"filter_by", "$Participants(org_id:=org_a) && $Links(repo_id:=repo_b)"},
{"include_fields", "user_id, user_name, $Repos(repo_content), $Organizations(name) as org"},
{"include_fields", "user_id, user_name, $Repos(repo_content:merge), $Organizations(name:merge) as org"},
{"exclude_fields", "$Participants(*), $Links(*), "}
};
search_op = collectionManager.do_search(req_params, embedded_params, json_res, now_ts);
@ -2341,7 +2446,7 @@ TEST_F(CollectionJoinTest, FilterByReferenceArrayField) {
std::map<std::string, std::string> req_params = {
{"collection", "songs"},
{"q", "*"},
{"include_fields", "$genres(name) as genre"},
{"include_fields", "$genres(name:merge) as genre"},
{"exclude_fields", "genres_sequence_id"},
};
nlohmann::json embedded_params;
@ -2372,7 +2477,7 @@ TEST_F(CollectionJoinTest, FilterByReferenceArrayField) {
{"collection", "genres"},
{"q", "*"},
{"filter_by", "$songs(id: *)"},
{"include_fields", "$songs(title) as song"},
{"include_fields", "$songs(title:merge) as song"},
};
search_op_bool = collectionManager.do_search(req_params, embedded_params, json_res, now_ts);
ASSERT_TRUE(search_op_bool.ok());
@ -2720,7 +2825,7 @@ TEST_F(CollectionJoinTest, SortByReference) {
{"query_by", "product_name"},
{"filter_by", "$Customers(customer_id:=customer_a)"},
{"sort_by", "$Customers(product_price:asc)"},
{"include_fields", "product_id, $Customers(product_price)"},
{"include_fields", "product_id, $Customers(product_price:merge)"},
};
search_op = collectionManager.do_search(req_params, embedded_params, json_res, now_ts);
ASSERT_TRUE(search_op.ok());
@ -2739,7 +2844,7 @@ TEST_F(CollectionJoinTest, SortByReference) {
{"query_by", "product_name"},
{"filter_by", "$Customers(customer_id:=customer_a)"},
{"sort_by", "$Customers(product_price:desc)"},
{"include_fields", "product_id, $Customers(product_price)"},
{"include_fields", "product_id, $Customers(product_price:merge)"},
};
search_op = collectionManager.do_search(req_params, embedded_params, json_res, now_ts);
ASSERT_TRUE(search_op.ok());
@ -2759,7 +2864,7 @@ TEST_F(CollectionJoinTest, SortByReference) {
{"query_by", "product_name"},
{"filter_by", "$Customers(customer_id:=customer_a)"},
{"sort_by", "$Customers(product_id:asc)"},
{"include_fields", "product_id, $Customers(product_price)"},
{"include_fields", "product_id, $Customers(product_price:merge)"},
};
search_op = collectionManager.do_search(req_params, embedded_params, json_res, now_ts);
ASSERT_TRUE(search_op.ok());
@ -2779,7 +2884,7 @@ TEST_F(CollectionJoinTest, SortByReference) {
{"query_by", "product_name"},
{"filter_by", "$Customers(customer_id:=customer_a)"},
{"sort_by", "$Customers(_eval(product_available:true):asc)"},
{"include_fields", "product_id, $Customers(product_price)"},
{"include_fields", "product_id, $Customers(product_price:merge)"},
};
search_op = collectionManager.do_search(req_params, embedded_params, json_res, now_ts);
ASSERT_TRUE(search_op.ok());
@ -2798,7 +2903,7 @@ TEST_F(CollectionJoinTest, SortByReference) {
{"query_by", "product_name"},
{"filter_by", "$Customers(customer_id:=customer_a)"},
{"sort_by", "$Customers(_eval(product_available:true):desc)"},
{"include_fields", "product_id, $Customers(product_price)"},
{"include_fields", "product_id, $Customers(product_price:merge)"},
};
search_op = collectionManager.do_search(req_params, embedded_params, json_res, now_ts);
ASSERT_TRUE(search_op.ok());
@ -2818,7 +2923,7 @@ TEST_F(CollectionJoinTest, SortByReference) {
{"query_by", "product_name"},
{"filter_by", "$Customers(customer_id:=customer_a)"},
{"sort_by", "$Customers(product_price:desc)"},
{"include_fields", "product_id, $Customers(product_price)"},
{"include_fields", "product_id, $Customers(product_price:merge)"},
};
search_op = collectionManager.do_search(req_params, embedded_params, json_res, now_ts);
ASSERT_TRUE(search_op.ok());
@ -2837,7 +2942,7 @@ TEST_F(CollectionJoinTest, SortByReference) {
{"q", R"("our")"},
{"query_by", "product_description"},
{"filter_by", "$Customers(customer_id:=customer_a)"},
{"include_fields", "product_id, $Customers(product_price)"},
{"include_fields", "product_id, $Customers(product_price:merge)"},
{"sort_by", "$Customers(product_price:desc)"},
};
search_op = collectionManager.do_search(req_params, embedded_params, json_res, now_ts);
@ -2857,7 +2962,7 @@ TEST_F(CollectionJoinTest, SortByReference) {
{"q", "natural products"},
{"query_by", "embedding"},
{"filter_by", "$Customers(customer_id:=customer_a)"},
{"include_fields", "product_id, $Customers(product_price)"},
{"include_fields", "product_id, $Customers(product_price:merge)"},
{"sort_by", "$Customers(product_price:desc)"},
};
search_op = collectionManager.do_search(req_params, embedded_params, json_res, now_ts);
@ -2891,7 +2996,7 @@ TEST_F(CollectionJoinTest, SortByReference) {
{"q", "*"},
{"vector_query", "embedding:(" + vec_string + ", flat_search_cutoff: 0)"},
{"filter_by", "$Customers(customer_id:=customer_a)"},
{"include_fields", "product_id, $Customers(product_price)"},
{"include_fields", "product_id, $Customers(product_price:merge)"},
{"sort_by", "$Customers(product_price:desc)"},
};
search_op = collectionManager.do_search(req_params, embedded_params, json_res, now_ts);
@ -2915,7 +3020,7 @@ TEST_F(CollectionJoinTest, SortByReference) {
{"q", "soap"},
{"query_by", "product_name, embedding"},
{"filter_by", "$Customers(customer_id:=customer_a)"},
{"include_fields", "product_id, $Customers(product_price)"},
{"include_fields", "product_id, $Customers(product_price:merge)"},
{"sort_by", "$Customers(product_price:desc)"},
};
search_op = collectionManager.do_search(req_params, embedded_params, json_res, now_ts);
@ -2941,7 +3046,7 @@ TEST_F(CollectionJoinTest, SortByReference) {
{"q", "natural products"},
{"query_by", "product_name, embedding"},
{"filter_by", "$Customers(customer_id:=customer_a)"},
{"include_fields", "product_id, $Customers(product_price)"},
{"include_fields", "product_id, $Customers(product_price:merge)"},
{"sort_by", "$Customers(product_price:desc)"},
};
search_op = collectionManager.do_search(req_params, embedded_params, json_res, now_ts);
@ -2966,7 +3071,7 @@ TEST_F(CollectionJoinTest, SortByReference) {
{"query_by", "product_name"},
{"infix", "always"},
{"filter_by", "$Customers(customer_id:=customer_a)"},
{"include_fields", "product_id, $Customers(product_price)"},
{"include_fields", "product_id, $Customers(product_price:merge)"},
{"sort_by", "$Customers(product_price:desc)"},
};
search_op = collectionManager.do_search(req_params, embedded_params, json_res, now_ts);
@ -2985,7 +3090,7 @@ TEST_F(CollectionJoinTest, SortByReference) {
{"collection", "Customers"},
{"q", "*"},
{"filter_by", "customer_name:= [Joe, Dan] && product_price:<100"},
{"include_fields", "$Products(product_name), product_price"},
{"include_fields", "$Products(product_name:merge), product_price"},
{"sort_by", "$Products(product_name:desc)"},
};
search_op = collectionManager.do_search(req_params, embedded_params, json_res, now_ts);
@ -3004,7 +3109,7 @@ TEST_F(CollectionJoinTest, SortByReference) {
{"collection", "Customers"},
{"q", "*"},
{"filter_by", "customer_name:= [Joe, Dan] && product_price:<100"},
{"include_fields", "$Products(product_name), product_price"},
{"include_fields", "$Products(product_name:merge), product_price"},
{"sort_by", "$Products(product_name:asc)"},
};
search_op = collectionManager.do_search(req_params, embedded_params, json_res, now_ts);
@ -3166,7 +3271,7 @@ TEST_F(CollectionJoinTest, SortByReference) {
{"collection", "Users"},
{"q", "*"},
{"filter_by", "$Links(repo_id:=[repo_a, repo_d])"},
{"include_fields", "user_id, user_name, $Repos(repo_content, repo_stars), "},
{"include_fields", "user_id, user_name, $Repos(repo_content, repo_stars:merge), "},
{"exclude_fields", "$Links(*), "},
{"sort_by", "$Repos(repo_stars: asc)"}
};
@ -3196,7 +3301,7 @@ TEST_F(CollectionJoinTest, SortByReference) {
{"collection", "Users"},
{"q", "*"},
{"filter_by", "$Links(repo_id:=[repo_a, repo_d])"},
{"include_fields", "user_id, user_name, $Repos(repo_content, repo_stars), "},
{"include_fields", "user_id, user_name, $Repos(repo_content, repo_stars:merge), "},
{"exclude_fields", "$Links(*), "},
{"sort_by", "$Repos(repo_stars: desc)"}
};