mirror of
https://github.com/typesense/typesense.git
synced 2025-05-21 14:12:27 +08:00
Object reference field. (#1911)
* Including referenced documents of object's reference field shouldn't depend on whether the query excludes the reference field. * Refactor `Collection::include_references`. * Only copy `doc` when references are involved.
This commit is contained in:
Harpreet Sangar
GitHub
parent
6376e3951d
commit
bca83ef529
@ -467,7 +467,8 @@ public:
|
||||
|
||||
static Option<bool> include_references(nlohmann::json& doc, const uint32_t& seq_id, Collection *const collection,
|
||||
const std::map<std::string, reference_filter_result_t>& reference_filter_results,
|
||||
const std::vector<ref_include_exclude_fields>& ref_include_exclude_fields_vec);
|
||||
const std::vector<ref_include_exclude_fields>& ref_include_exclude_fields_vec,
|
||||
const nlohmann::json& original_doc);
|
||||
|
||||
Option<bool> prune_doc_with_lock(nlohmann::json& doc, const tsl::htrie_set<char>& include_names,
|
||||
const tsl::htrie_set<char>& exclude_names,
|
||||
|
||||
@ -5448,6 +5448,11 @@ Option<bool> Collection::prune_ref_doc(nlohmann::json& doc,
|
||||
const tsl::htrie_set<char>& ref_exclude_fields_full,
|
||||
const bool& is_reference_array,
|
||||
const ref_include_exclude_fields& ref_include_exclude) {
|
||||
nlohmann::json original_doc;
|
||||
if (!ref_include_exclude.nested_join_includes.empty()) {
|
||||
original_doc = doc;
|
||||
}
|
||||
|
||||
auto const& ref_collection_name = ref_include_exclude.collection_name;
|
||||
auto& cm = CollectionManager::get_instance();
|
||||
auto ref_collection = cm.get_collection(ref_collection_name);
|
||||
@ -5503,7 +5508,7 @@ Option<bool> Collection::prune_ref_doc(nlohmann::json& doc,
|
||||
ref_collection.get(),
|
||||
references.coll_to_references == nullptr ? refs :
|
||||
references.coll_to_references[0],
|
||||
ref_include_exclude.nested_join_includes);
|
||||
ref_include_exclude.nested_join_includes, original_doc);
|
||||
if (!nested_include_exclude_op.ok()) {
|
||||
return nested_include_exclude_op;
|
||||
}
|
||||
@ -5568,7 +5573,7 @@ Option<bool> Collection::prune_ref_doc(nlohmann::json& doc,
|
||||
ref_collection.get(),
|
||||
references.coll_to_references == nullptr ? refs :
|
||||
references.coll_to_references[i],
|
||||
ref_include_exclude.nested_join_includes);
|
||||
ref_include_exclude.nested_join_includes, original_doc);
|
||||
if (!nested_include_exclude_op.ok()) {
|
||||
return nested_include_exclude_op;
|
||||
}
|
||||
@ -5580,7 +5585,8 @@ Option<bool> Collection::prune_ref_doc(nlohmann::json& doc,
|
||||
|
||||
Option<bool> Collection::include_references(nlohmann::json& doc, const uint32_t& seq_id, Collection *const collection,
|
||||
const std::map<std::string, reference_filter_result_t>& reference_filter_results,
|
||||
const std::vector<ref_include_exclude_fields>& ref_include_exclude_fields_vec) {
|
||||
const std::vector<ref_include_exclude_fields>& ref_include_exclude_fields_vec,
|
||||
const nlohmann::json& original_doc) {
|
||||
for (auto const& ref_include_exclude: ref_include_exclude_fields_vec) {
|
||||
auto ref_collection_name = ref_include_exclude.collection_name;
|
||||
|
||||
@ -5656,13 +5662,26 @@ Option<bool> Collection::include_references(nlohmann::json& doc, const uint32_t&
|
||||
if (collection->object_reference_helper_fields.count(field_name) != 0) {
|
||||
std::vector<std::string> keys;
|
||||
StringUtils::split(field_name, keys, ".");
|
||||
if (!doc.contains(keys[0])) {
|
||||
return Option<bool>(400, "Could not find `" + keys[0] +
|
||||
"` in the document to include the referenced document.");
|
||||
auto const& key = keys[0];
|
||||
|
||||
if (!doc.contains(key)) {
|
||||
if (!original_doc.contains(key)) {
|
||||
return Option<bool>(400, "Could not find `" + key +
|
||||
"` key in the document to include the referenced document.");
|
||||
}
|
||||
|
||||
// The key is excluded from the doc by the query, inserting empty object(s) so referenced doc can be
|
||||
// included in it.
|
||||
if (original_doc[key].is_array()) {
|
||||
doc[key] = nlohmann::json::array();
|
||||
doc[key].insert(doc[key].begin(), original_doc[key].size(), nlohmann::json::object());
|
||||
} else {
|
||||
doc[key] = nlohmann::json::object();
|
||||
}
|
||||
}
|
||||
|
||||
if (doc[keys[0]].is_array()) {
|
||||
for (uint32_t i = 0; i < doc[keys[0]].size(); i++) {
|
||||
if (doc[key].is_array()) {
|
||||
for (uint32_t i = 0; i < doc[key].size(); i++) {
|
||||
uint32_t ref_doc_id;
|
||||
auto op = collection->get_object_array_related_id(field_name, seq_id, i, ref_doc_id);
|
||||
if (!op.ok()) {
|
||||
@ -5674,7 +5693,7 @@ Option<bool> Collection::include_references(nlohmann::json& doc, const uint32_t&
|
||||
}
|
||||
|
||||
reference_filter_result_t result(1, new uint32_t[1]{ref_doc_id});
|
||||
prune_doc_op = prune_ref_doc(doc[keys[0]][i], result,
|
||||
prune_doc_op = prune_ref_doc(doc[key][i], result,
|
||||
ref_include_fields_full, ref_exclude_fields_full,
|
||||
false, ref_include_exclude);
|
||||
if (!prune_doc_op.ok()) {
|
||||
@ -5688,7 +5707,7 @@ Option<bool> Collection::include_references(nlohmann::json& doc, const uint32_t&
|
||||
continue;
|
||||
}
|
||||
reference_filter_result_t result(ids.size(), &ids[0]);
|
||||
prune_doc_op = prune_ref_doc(doc[keys[0]], result, ref_include_fields_full, ref_exclude_fields_full,
|
||||
prune_doc_op = prune_ref_doc(doc[key], result, ref_include_fields_full, ref_exclude_fields_full,
|
||||
collection->search_schema.at(field_name).is_array(), ref_include_exclude);
|
||||
result.docs = nullptr;
|
||||
}
|
||||
@ -5764,6 +5783,11 @@ Option<bool> Collection::prune_doc(nlohmann::json& doc,
|
||||
const std::map<std::string, reference_filter_result_t>& reference_filter_results,
|
||||
Collection *const collection, const uint32_t& seq_id,
|
||||
const std::vector<ref_include_exclude_fields>& ref_include_exclude_fields_vec) {
|
||||
nlohmann::json original_doc;
|
||||
if (!ref_include_exclude_fields_vec.empty()) {
|
||||
original_doc = doc;
|
||||
}
|
||||
|
||||
// doc can only be an object
|
||||
auto it = doc.begin();
|
||||
while(it != doc.end()) {
|
||||
@ -5839,7 +5863,8 @@ Option<bool> Collection::prune_doc(nlohmann::json& doc,
|
||||
it++;
|
||||
}
|
||||
|
||||
return include_references(doc, seq_id, collection, reference_filter_results, ref_include_exclude_fields_vec);
|
||||
return include_references(doc, seq_id, collection, reference_filter_results, ref_include_exclude_fields_vec,
|
||||
original_doc);
|
||||
}
|
||||
|
||||
Option<bool> Collection::validate_alter_payload(nlohmann::json& schema_changes,
|
||||
|
||||
@ -3913,8 +3913,8 @@ TEST_F(CollectionJoinTest, FilterByObjectReferenceField) {
|
||||
auto now_ts = std::chrono::duration_cast<std::chrono::microseconds>(
|
||||
std::chrono::system_clock::now().time_since_epoch()).count();
|
||||
|
||||
auto search_op_bool = collectionManager.do_search(req_params, embedded_params, json_res, now_ts);
|
||||
ASSERT_TRUE(search_op_bool.ok());
|
||||
auto search_op = collectionManager.do_search(req_params, embedded_params, json_res, now_ts);
|
||||
ASSERT_TRUE(search_op.ok());
|
||||
|
||||
auto res_obj = nlohmann::json::parse(json_res);
|
||||
ASSERT_EQ(2, res_obj["found"].get<size_t>());
|
||||
@ -3931,14 +3931,39 @@ TEST_F(CollectionJoinTest, FilterByObjectReferenceField) {
|
||||
ASSERT_EQ("a", res_obj["hits"][1]["document"]["coll_id"]);
|
||||
ASSERT_EQ(0, res_obj["hits"][1]["document"]["object"].size());
|
||||
|
||||
req_params = {
|
||||
{"collection", "coll1"},
|
||||
{"q", "*"},
|
||||
{"include_fields", "$Products(product_id)"},
|
||||
{"exclude_fields", "object"}
|
||||
};
|
||||
search_op = collectionManager.do_search(req_params, embedded_params, json_res, now_ts);
|
||||
ASSERT_TRUE(search_op.ok());
|
||||
|
||||
res_obj = nlohmann::json::parse(json_res);
|
||||
ASSERT_EQ(2, res_obj["found"].get<size_t>());
|
||||
ASSERT_EQ(2, res_obj["hits"].size());
|
||||
ASSERT_EQ(3, res_obj["hits"][0]["document"].size());
|
||||
ASSERT_EQ("1", res_obj["hits"][0]["document"]["id"]);
|
||||
ASSERT_EQ(1, res_obj["hits"][0]["document"].count("object"));
|
||||
ASSERT_EQ(1, res_obj["hits"][0]["document"]["object"].size());
|
||||
ASSERT_EQ(1, res_obj["hits"][0]["document"]["object"].count("Products"));
|
||||
ASSERT_EQ(1, res_obj["hits"][0]["document"]["object"]["Products"].size());
|
||||
ASSERT_EQ(1, res_obj["hits"][0]["document"]["object"]["Products"].count("product_id"));
|
||||
ASSERT_EQ("product_c", res_obj["hits"][0]["document"]["object"]["Products"]["product_id"]);
|
||||
ASSERT_EQ(3, res_obj["hits"][1]["document"].size());
|
||||
ASSERT_EQ("0", res_obj["hits"][1]["document"]["id"]);
|
||||
ASSERT_EQ(1, res_obj["hits"][1]["document"].count("object"));
|
||||
ASSERT_EQ(0, res_obj["hits"][1]["document"]["object"].size());
|
||||
|
||||
req_params = {
|
||||
{"collection", "Products"},
|
||||
{"q", "*"},
|
||||
{"filter_by", "$coll1(id: *)"},
|
||||
{"include_fields", "$coll1(coll_id)"}
|
||||
};
|
||||
search_op_bool = collectionManager.do_search(req_params, embedded_params, json_res, now_ts);
|
||||
ASSERT_TRUE(search_op_bool.ok());
|
||||
search_op = collectionManager.do_search(req_params, embedded_params, json_res, now_ts);
|
||||
ASSERT_TRUE(search_op.ok());
|
||||
|
||||
res_obj = nlohmann::json::parse(json_res);
|
||||
ASSERT_EQ(1, res_obj["found"].get<size_t>());
|
||||
@ -3988,8 +4013,8 @@ TEST_F(CollectionJoinTest, FilterByObjectReferenceField) {
|
||||
{"q", "*"},
|
||||
{"include_fields", "$Products(product_id)"}
|
||||
};
|
||||
search_op_bool = collectionManager.do_search(req_params, embedded_params, json_res, now_ts);
|
||||
ASSERT_TRUE(search_op_bool.ok());
|
||||
search_op = collectionManager.do_search(req_params, embedded_params, json_res, now_ts);
|
||||
ASSERT_TRUE(search_op.ok());
|
||||
|
||||
res_obj = nlohmann::json::parse(json_res);
|
||||
ASSERT_EQ(2, res_obj["found"].get<size_t>());
|
||||
@ -4009,14 +4034,41 @@ TEST_F(CollectionJoinTest, FilterByObjectReferenceField) {
|
||||
ASSERT_EQ("a", res_obj["hits"][1]["document"]["coll_id"]);
|
||||
ASSERT_EQ(0, res_obj["hits"][1]["document"]["object"].size());
|
||||
|
||||
req_params = {
|
||||
{"collection", "coll2"},
|
||||
{"q", "*"},
|
||||
{"include_fields", "$Products(product_id)"},
|
||||
{"exclude_fields", "object"}
|
||||
};
|
||||
search_op = collectionManager.do_search(req_params, embedded_params, json_res, now_ts);
|
||||
ASSERT_TRUE(search_op.ok());
|
||||
|
||||
res_obj = nlohmann::json::parse(json_res);
|
||||
ASSERT_EQ(2, res_obj["found"].get<size_t>());
|
||||
ASSERT_EQ(2, res_obj["hits"].size());
|
||||
ASSERT_EQ(3, res_obj["hits"][0]["document"].size());
|
||||
ASSERT_EQ("1", res_obj["hits"][0]["document"]["id"]);
|
||||
ASSERT_EQ(1, res_obj["hits"][0]["document"].count("object"));
|
||||
ASSERT_EQ(1, res_obj["hits"][0]["document"]["object"].size());
|
||||
ASSERT_EQ(1, res_obj["hits"][0]["document"]["object"].count("Products"));
|
||||
ASSERT_EQ(2, res_obj["hits"][0]["document"]["object"]["Products"].size());
|
||||
ASSERT_EQ(1, res_obj["hits"][0]["document"]["object"]["Products"][0].count("product_id"));
|
||||
ASSERT_EQ("product_a", res_obj["hits"][0]["document"]["object"]["Products"][0]["product_id"]);
|
||||
ASSERT_EQ(1, res_obj["hits"][0]["document"]["object"]["Products"][1].count("product_id"));
|
||||
ASSERT_EQ("product_b", res_obj["hits"][0]["document"]["object"]["Products"][1]["product_id"]);
|
||||
ASSERT_EQ(3, res_obj["hits"][1]["document"].size());
|
||||
ASSERT_EQ("0", res_obj["hits"][1]["document"]["id"]);
|
||||
ASSERT_EQ(1, res_obj["hits"][1]["document"].count("object"));
|
||||
ASSERT_EQ(0, res_obj["hits"][1]["document"]["object"].size());
|
||||
|
||||
req_params = {
|
||||
{"collection", "Products"},
|
||||
{"q", "*"},
|
||||
{"filter_by", "$coll2(id: *)"},
|
||||
{"include_fields", "$coll2(coll_id)"}
|
||||
};
|
||||
search_op_bool = collectionManager.do_search(req_params, embedded_params, json_res, now_ts);
|
||||
ASSERT_TRUE(search_op_bool.ok());
|
||||
search_op = collectionManager.do_search(req_params, embedded_params, json_res, now_ts);
|
||||
ASSERT_TRUE(search_op.ok());
|
||||
|
||||
res_obj = nlohmann::json::parse(json_res);
|
||||
ASSERT_EQ(2, res_obj["found"].get<size_t>());
|
||||
@ -4071,8 +4123,8 @@ TEST_F(CollectionJoinTest, FilterByObjectReferenceField) {
|
||||
{"q", "*"},
|
||||
{"include_fields", "$Products(product_id)"}
|
||||
};
|
||||
search_op_bool = collectionManager.do_search(req_params, embedded_params, json_res, now_ts);
|
||||
ASSERT_TRUE(search_op_bool.ok());
|
||||
search_op = collectionManager.do_search(req_params, embedded_params, json_res, now_ts);
|
||||
ASSERT_TRUE(search_op.ok());
|
||||
|
||||
res_obj = nlohmann::json::parse(json_res);
|
||||
ASSERT_EQ(2, res_obj["found"].get<size_t>());
|
||||
@ -4098,8 +4150,8 @@ TEST_F(CollectionJoinTest, FilterByObjectReferenceField) {
|
||||
{"filter_by", "$coll3(id: *)"},
|
||||
{"include_fields", "$coll3(coll_id)"}
|
||||
};
|
||||
search_op_bool = collectionManager.do_search(req_params, embedded_params, json_res, now_ts);
|
||||
ASSERT_TRUE(search_op_bool.ok());
|
||||
search_op = collectionManager.do_search(req_params, embedded_params, json_res, now_ts);
|
||||
ASSERT_TRUE(search_op.ok());
|
||||
|
||||
res_obj = nlohmann::json::parse(json_res);
|
||||
ASSERT_EQ(2, res_obj["found"].get<size_t>());
|
||||
@ -4206,8 +4258,8 @@ TEST_F(CollectionJoinTest, FilterByObjectReferenceField) {
|
||||
{"q", "*"},
|
||||
{"include_fields", "$Portions(*, strategy:merge)"}
|
||||
};
|
||||
search_op_bool = collectionManager.do_search(req_params, embedded_params, json_res, now_ts);
|
||||
ASSERT_TRUE(search_op_bool.ok());
|
||||
search_op = collectionManager.do_search(req_params, embedded_params, json_res, now_ts);
|
||||
ASSERT_TRUE(search_op.ok());
|
||||
|
||||
res_obj = nlohmann::json::parse(json_res);
|
||||
ASSERT_EQ(2, res_obj["found"].get<size_t>());
|
||||
@ -4245,6 +4297,45 @@ TEST_F(CollectionJoinTest, FilterByObjectReferenceField) {
|
||||
ASSERT_EQ("g", res_obj["hits"][1]["document"]["portions"][0].at("unit"));
|
||||
ASSERT_EQ(10 , res_obj["hits"][1]["document"]["portions"][0].at("count"));
|
||||
|
||||
req_params = {
|
||||
{"collection", "Foods"},
|
||||
{"q", "*"},
|
||||
{"include_fields", "$Portions(*, strategy:merge)"},
|
||||
{"exclude_fields", "portions"}
|
||||
};
|
||||
search_op = collectionManager.do_search(req_params, embedded_params, json_res, now_ts);
|
||||
ASSERT_TRUE(search_op.ok());
|
||||
|
||||
res_obj = nlohmann::json::parse(json_res);
|
||||
ASSERT_EQ(2, res_obj["found"].get<size_t>());
|
||||
ASSERT_EQ(2, res_obj["hits"].size());
|
||||
ASSERT_EQ(3, res_obj["hits"][0]["document"].size());
|
||||
|
||||
ASSERT_EQ("1", res_obj["hits"][0]["document"]["id"]);
|
||||
ASSERT_EQ(1, res_obj["hits"][0]["document"].count("portions"));
|
||||
ASSERT_EQ(3, res_obj["hits"][0]["document"]["portions"].size());
|
||||
|
||||
ASSERT_EQ(4, res_obj["hits"][0]["document"]["portions"][0].size());
|
||||
ASSERT_EQ("portion_b", res_obj["hits"][0]["document"]["portions"][0].at("portion_id"));
|
||||
ASSERT_EQ(1 , res_obj["hits"][0]["document"]["portions"][0].at("quantity"));
|
||||
ASSERT_EQ("lt", res_obj["hits"][0]["document"]["portions"][0].at("unit"));
|
||||
|
||||
ASSERT_EQ(0, res_obj["hits"][0]["document"]["portions"][1].size());
|
||||
|
||||
ASSERT_EQ(4, res_obj["hits"][0]["document"]["portions"][2].size());
|
||||
ASSERT_EQ("portion_c", res_obj["hits"][0]["document"]["portions"][2].at("portion_id"));
|
||||
ASSERT_EQ(500 , res_obj["hits"][0]["document"]["portions"][2].at("quantity"));
|
||||
ASSERT_EQ("ml", res_obj["hits"][0]["document"]["portions"][2].at("unit"));
|
||||
|
||||
ASSERT_EQ("0", res_obj["hits"][1]["document"]["id"]);
|
||||
ASSERT_EQ(1, res_obj["hits"][1]["document"].count("portions"));
|
||||
ASSERT_EQ(1, res_obj["hits"][1]["document"]["portions"].size());
|
||||
|
||||
ASSERT_EQ(4, res_obj["hits"][1]["document"]["portions"][0].size());
|
||||
ASSERT_EQ("portion_a", res_obj["hits"][1]["document"]["portions"][0].at("portion_id"));
|
||||
ASSERT_EQ(500 , res_obj["hits"][1]["document"]["portions"][0].at("quantity"));
|
||||
ASSERT_EQ("g", res_obj["hits"][1]["document"]["portions"][0].at("unit"));
|
||||
|
||||
// recreate collection manager to ensure that it initializes `object_reference_helper_fields` correctly.
|
||||
collectionManager.dispose();
|
||||
delete store;
|
||||
@ -4263,8 +4354,8 @@ TEST_F(CollectionJoinTest, FilterByObjectReferenceField) {
|
||||
{"q", "*"},
|
||||
{"include_fields", "$Portions(*, strategy:merge)"}
|
||||
};
|
||||
search_op_bool = collectionManager.do_search(req_params, embedded_params, json_res, now_ts);
|
||||
ASSERT_TRUE(search_op_bool.ok());
|
||||
search_op = collectionManager.do_search(req_params, embedded_params, json_res, now_ts);
|
||||
ASSERT_TRUE(search_op.ok());
|
||||
|
||||
res_obj = nlohmann::json::parse(json_res);
|
||||
ASSERT_EQ(2, res_obj["found"].get<size_t>());
|
||||
@ -4321,8 +4412,8 @@ TEST_F(CollectionJoinTest, FilterByObjectReferenceField) {
|
||||
{"q", "*"},
|
||||
{"include_fields", "$Portions(*, strategy:merge)"}
|
||||
};
|
||||
search_op_bool = collectionManager.do_search(req_params, embedded_params, json_res, now_ts);
|
||||
ASSERT_TRUE(search_op_bool.ok());
|
||||
search_op = collectionManager.do_search(req_params, embedded_params, json_res, now_ts);
|
||||
ASSERT_TRUE(search_op.ok());
|
||||
|
||||
res_obj = nlohmann::json::parse(json_res);
|
||||
ASSERT_EQ(2, res_obj["found"].get<size_t>());
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user