diff --git a/include/field.h b/include/field.h index 8d9cbf83..03d82ec8 100644 --- a/include/field.h +++ b/include/field.h @@ -5,6 +5,7 @@ #include "art.h" #include "option.h" #include "string_utils.h" +#include "logger.h" #include "json.hpp" namespace field_types { @@ -192,6 +193,10 @@ struct field { bool found_default_sorting_field = false; for(const field & field: fields) { + if(field.name == "id") { + continue; + } + nlohmann::json field_val; field_val[fields::name] = field.name; field_val[fields::type] = field.type; @@ -263,6 +268,13 @@ struct field { size_t num_auto_detect_fields = 0; for(nlohmann::json & field_json: fields_json) { + if(field_json["name"] == "id") { + // No field should exist with the name "id" as it is reserved for internal use + // We cannot throw an error here anymore since that will break backward compatibility! + LOG(WARNING) << "Collection schema cannot contain a field with name `id`. Ignoring field."; + continue; + } + if(!field_json.is_object() || field_json.count(fields::name) == 0 || field_json.count(fields::type) == 0 || !field_json.at(fields::name).is_string() || !field_json.at(fields::type).is_string()) { @@ -463,6 +475,7 @@ namespace sort_field_const { static const std::string seq_id = "_seq_id"; static const std::string exclude_radius = "exclude_radius"; + static const std::string precision = "precision"; } struct sort_by { @@ -472,14 +485,16 @@ struct sort_by { // geo related fields int64_t geopoint; uint32_t exclude_radius; + uint32_t geo_precision; sort_by(const std::string & name, const std::string & order): - name(name), order(order), geopoint(0), exclude_radius(0) { + name(name), order(order), geopoint(0), exclude_radius(0), geo_precision(0) { } - sort_by(const std::string &name, const std::string &order, int64_t geopoint, uint32_t exclude_radius) : - name(name), order(order), geopoint(geopoint), exclude_radius(exclude_radius) { + sort_by(const std::string &name, const std::string &order, int64_t geopoint, + uint32_t exclude_radius, uint32_t geo_precision) : + name(name), order(order), geopoint(geopoint), exclude_radius(exclude_radius), geo_precision(geo_precision) { } @@ -488,6 +503,7 @@ struct sort_by { order = other.order; geopoint = other.geopoint; exclude_radius = other.exclude_radius; + geo_precision = other.geo_precision; return *this; } }; diff --git a/include/topster.h b/include/topster.h index 5550fbd0..19ba746c 100644 --- a/include/topster.h +++ b/include/topster.h @@ -16,6 +16,9 @@ struct KV { uint64_t distinct_key{}; int64_t scores[3]{}; // match score + 2 custom attributes + // to be used only in final aggregation + uint64_t* query_indices = nullptr; + KV(uint8_t field_id, uint16_t queryIndex, uint32_t token_bits, uint64_t key, uint64_t distinct_key, uint8_t match_score_index, const int64_t *scores): field_id(field_id), match_score_index(match_score_index), @@ -27,6 +30,69 @@ struct KV { } KV() = default; + + KV(KV& kv) = default; + + KV(KV&& kv) noexcept : field_id(kv.field_id), match_score_index(kv.match_score_index), + query_index(kv.query_index), array_index(kv.array_index), token_bits(kv.token_bits), + key(kv.key), distinct_key(kv.distinct_key) { + + scores[0] = kv.scores[0]; + scores[1] = kv.scores[1]; + scores[2] = kv.scores[2]; + + query_indices = kv.query_indices; + kv.query_indices = nullptr; + } + + KV& operator=(KV&& kv) noexcept { + if (this != &kv) { + field_id = kv.field_id; + match_score_index = kv.match_score_index; + query_index = kv.query_index; + array_index = kv.array_index; + token_bits = kv.token_bits; + key = kv.key; + distinct_key = kv.distinct_key; + + scores[0] = kv.scores[0]; + scores[1] = kv.scores[1]; + scores[2] = kv.scores[2]; + + delete[] query_indices; + query_indices = kv.query_indices; + kv.query_indices = nullptr; + } + + return *this; + } + + KV& operator=(KV& kv) noexcept { + if (this != &kv) { + field_id = kv.field_id; + match_score_index = kv.match_score_index; + query_index = kv.query_index; + array_index = kv.array_index; + token_bits = kv.token_bits; + key = kv.key; + distinct_key = kv.distinct_key; + + scores[0] = kv.scores[0]; + scores[1] = kv.scores[1]; + scores[2] = kv.scores[2]; + + delete[] query_indices; + query_indices = kv.query_indices; + kv.query_indices = nullptr; + } + + return *this; + } + + ~KV() { + delete [] query_indices; + query_indices = nullptr; + } }; /* diff --git a/src/collection.cpp b/src/collection.cpp index d0a4cc21..3f157723 100644 --- a/src/collection.cpp +++ b/src/collection.cpp @@ -338,7 +338,8 @@ void Collection::batch_index(std::vector> &index_batch res["code"] = index_record.indexed.code(); } - json_out[index_record.position] = res.dump(); + json_out[index_record.position] = res.dump(-1, ' ', false, + nlohmann::detail::error_handler_t::ignore); } } } @@ -753,36 +754,54 @@ Option Collection::search(const std::string & query, const std:: if(geo_parts.size() == 3) { // try to parse the exclude radius option - if(!StringUtils::begins_with(geo_parts[2], sort_field_const::exclude_radius)) { - return Option(400, error); - } + bool is_exclude_option = false; - std::vector exclude_parts; - StringUtils::split(geo_parts[2], exclude_parts, ":"); - - if(exclude_parts.size() != 2) { - return Option(400, error); - } - - std::vector exclude_value_parts; - StringUtils::split(exclude_parts[1], exclude_value_parts, " "); - - if(exclude_value_parts.size() != 2) { - return Option(400, error); - } - - if(!StringUtils::is_float(exclude_value_parts[0])) { - return Option(400, error); - } - - if(exclude_value_parts[1] == "km") { - sort_field_std.exclude_radius = std::stof(exclude_value_parts[0]) * 1000; - } else if(exclude_value_parts[1] == "mi") { - sort_field_std.exclude_radius = std::stof(exclude_value_parts[0]) * 1609.34; + if(StringUtils::begins_with(geo_parts[2], sort_field_const::exclude_radius)) { + is_exclude_option = true; + } else if(StringUtils::begins_with(geo_parts[2], sort_field_const::precision)) { + is_exclude_option = false; } else { - return Option(400, "Sort field's exclude radius " + return Option(400, error); + } + + std::vector param_parts; + StringUtils::split(geo_parts[2], param_parts, ":"); + + if(param_parts.size() != 2) { + return Option(400, error); + } + + std::vector param_value_parts; + StringUtils::split(param_parts[1], param_value_parts, " "); + + if(param_value_parts.size() != 2) { + return Option(400, error); + } + + if(!StringUtils::is_float(param_value_parts[0])) { + return Option(400, error); + } + + int32_t value_meters; + + if(param_value_parts[1] == "km") { + value_meters = std::stof(param_value_parts[0]) * 1000; + } else if(param_value_parts[1] == "mi") { + value_meters = std::stof(param_value_parts[0]) * 1609.34; + } else { + return Option(400, "Sort field's parameter " "unit must be either `km` or `mi`."); } + + if(value_meters <= 0) { + return Option(400, "Sort field's parameter must be a positive number."); + } + + if(is_exclude_option) { + sort_field_std.exclude_radius = value_meters; + } else { + sort_field_std.geo_precision = value_meters; + } } double lat = std::stod(geo_parts[0]); @@ -1387,12 +1406,22 @@ void Collection::aggregate_topster(size_t query_index, Topster& agg_topster, Top Topster* group_topster = group_topster_entry.second; for(const auto& map_kv: group_topster->kv_map) { map_kv.second->query_index += query_index; + if(map_kv.second->query_indices != nullptr) { + for(size_t i = 0; i < map_kv.second->query_indices[0]; i++) { + map_kv.second->query_indices[i+1] += query_index; + } + } agg_topster.add(map_kv.second); } } } else { for(const auto& map_kv: index_topster->kv_map) { map_kv.second->query_index += query_index; + if(map_kv.second->query_indices != nullptr) { + for(size_t i = 0; i < map_kv.second->query_indices[0]; i++) { + map_kv.second->query_indices[i+1] += query_index; + } + } agg_topster.add(map_kv.second); } } @@ -1497,20 +1526,29 @@ void Collection::highlight_result(const field &search_field, std::vector query_suggestion; std::set query_suggestion_tokens; - for (const art_leaf* token_leaf : searched_queries[field_order_kv->query_index]) { - // Must search for the token string fresh on that field for the given document since `token_leaf` - // is from the best matched field and need not be present in other fields of a document. - Index* index = indices[field_order_kv->key % num_memory_shards]; - art_leaf *actual_leaf = index->get_token_leaf(search_field.name, &token_leaf->key[0], token_leaf->key_len); + size_t qindex = 0; - //LOG(INFO) << "field: " << search_field.name << ", key: " << token_leaf->key; + do { + auto searched_query = + (field_order_kv->query_indices == nullptr) ? searched_queries[field_order_kv->query_index] : + searched_queries[field_order_kv->query_indices[qindex + 1]]; - if(actual_leaf != nullptr) { - query_suggestion.push_back(actual_leaf); - std::string token(reinterpret_cast(actual_leaf->key), actual_leaf->key_len-1); - query_suggestion_tokens.insert(token); + for (art_leaf* token_leaf : searched_query) { + // Must search for the token string fresh on that field for the given document since `token_leaf` + // is from the best matched field and need not be present in other fields of a document. + Index* index = indices[field_order_kv->key % num_memory_shards]; + art_leaf* actual_leaf = index->get_token_leaf(search_field.name, &token_leaf->key[0], token_leaf->key_len); + + if(actual_leaf != nullptr) { + query_suggestion.push_back(actual_leaf); + std::string token(reinterpret_cast(actual_leaf->key), actual_leaf->key_len - 1); + //LOG(INFO) << "field: " << search_field.name << ", key: " << token; + query_suggestion_tokens.insert(token); + } } - } + + qindex++; + } while(field_order_kv->query_indices != nullptr && qindex < field_order_kv->query_indices[0]); if(query_suggestion.size() != q_tokens.size()) { // can happen for compound query matched across 2 fields when some tokens are dropped @@ -1525,6 +1563,7 @@ void Collection::highlight_result(const field &search_field, q_token.size() + 1); if(actual_leaf != nullptr) { query_suggestion.push_back(actual_leaf); + query_suggestion_tokens.insert(q_token); } } } @@ -1614,9 +1653,10 @@ void Collection::highlight_result(const field &search_field, highlight.matched_tokens.emplace_back(); std::vector& matched_tokens = highlight.matched_tokens.back(); + bool found_first_match = false; while(tokenizer.next(raw_token, raw_token_index, tok_start, tok_end)) { - if(token_offsets.empty()) { + if(!found_first_match) { if(snippet_start_window.size() == highlight_affix_num_tokens + 1) { snippet_start_window.pop_front(); } @@ -1624,7 +1664,10 @@ void Collection::highlight_result(const field &search_field, snippet_start_window.push_back(tok_start); } - if (token_hits.count(raw_token) != 0 || + bool token_already_found = token_hits.count(raw_token) != 0; + + // ensures that the `snippet_start_offset` is always from a matched token, and not from query suggestion + if ((found_first_match && token_already_found) || (match_offset_index < match.offsets.size() && match.offsets[match_offset_index].offset == raw_token_index)) { @@ -1637,9 +1680,15 @@ void Collection::highlight_result(const field &search_field, } while(match_offset_index < match.offsets.size() && match.offsets[match_offset_index - 1].offset == match.offsets[match_offset_index].offset); - if(token_offsets.size() == 1) { + if(!found_first_match) { snippet_start_offset = snippet_start_window.front(); } + + found_first_match = true; + + } else if(query_suggestion_tokens.find(raw_token) != query_suggestion_tokens.end()) { + token_offsets.emplace(tok_start, tok_end); + token_hits.insert(raw_token); } if(raw_token_index == last_valid_offset + highlight_affix_num_tokens) { @@ -1669,6 +1718,11 @@ void Collection::highlight_result(const field &search_field, auto offset_it = token_offsets.begin(); std::stringstream highlighted_text; + // tokens from query might occur before actual snippet start offset: we skip that + while(offset_it != token_offsets.end() && offset_it->first < snippet_start_offset) { + offset_it++; + } + for(size_t i = snippet_start_offset; i <= snippet_end_offset; i++) { if(offset_it != token_offsets.end()) { if (i == offset_it->first) { diff --git a/src/index.cpp b/src/index.cpp index 3ddbb0f1..becff94b 100644 --- a/src/index.cpp +++ b/src/index.cpp @@ -1649,13 +1649,18 @@ void Index::search(const std::vector& field_query_tokens, auto& kvs = seq_id_kvs.second; // each `kv` can be from a different field std::sort(kvs.begin(), kvs.end(), Topster::is_greater); + kvs[0]->query_indices = new uint64_t[kvs.size() + 1]; + kvs[0]->query_indices[0] = kvs.size(); - // LOG(INFO) << "DOC ID: " << seq_id << ", score: " << kvs[0]->scores[kvs[0]->match_score_index]; + //LOG(INFO) << "DOC ID: " << seq_id << ", score: " << kvs[0]->scores[kvs[0]->match_score_index]; // to calculate existing aggregate scores across best matching fields spp::sparse_hash_map existing_field_kvs; - for(const auto kv: kvs) { - existing_field_kvs.emplace(kv->field_id, kv); + for(size_t kv_i = 0; kv_i < kvs.size(); kv_i++) { + existing_field_kvs.emplace(kvs[kv_i]->field_id, kvs[kv_i]); + kvs[0]->query_indices[kv_i+1] = kvs[kv_i]->query_index; + /*LOG(INFO) << "kv_i: " << kv_i << ", kvs[kv_i]->query_index: " << kvs[kv_i]->query_index << ", " + << "searched_query: " << searched_queries[kvs[kv_i]->query_index][0];*/ } uint32_t token_bits = (uint32_t(1) << 31); // top most bit set to guarantee atleast 1 bit set @@ -2098,6 +2103,11 @@ void Index::score_results(const std::vector & sort_fields, const uint16 dist = 0; } + if(sort_fields[i].geo_precision > 0) { + dist = dist + sort_fields[i].geo_precision - 1 - + (dist + sort_fields[i].geo_precision - 1) % sort_fields[i].geo_precision; + } + geopoint_distances[i].emplace(seq_id, dist); } diff --git a/test/collection_locale_test.cpp b/test/collection_locale_test.cpp index c6f5b63a..75fe60b1 100644 --- a/test/collection_locale_test.cpp +++ b/test/collection_locale_test.cpp @@ -263,7 +263,7 @@ TEST_F(CollectionLocaleTest, SearchAgainstThaiTextExactMatch) { ASSERT_EQ("ติดกับดักรายได้ปานกลาง", results["hits"][0]["highlights"][0]["snippet"].get()); - ASSERT_EQ("ข้อมูลรายคนหรือรายบริษัทในการเชื่อมโยงส่วนได้ส่วนเสีย", + ASSERT_EQ("ข้อมูลรายคนหรือรายบริษัทในการเชื่อมโยงส่วนได้ส่วนเสีย", results["hits"][1]["highlights"][0]["snippet"].get()); } diff --git a/test/collection_manager_test.cpp b/test/collection_manager_test.cpp index 8b3f507a..ac87bfc4 100644 --- a/test/collection_manager_test.cpp +++ b/test/collection_manager_test.cpp @@ -232,7 +232,6 @@ TEST_F(CollectionManagerTest, RestoreRecordsOnRestart) { std::vector facets; nlohmann::json results = collection1->search("thomas", search_fields, "", facets, sort_fields, {0}, 10, 1, FREQUENCY, {false}).get(); - LOG(INFO) << results; ASSERT_EQ(4, results["hits"].size()); std::unordered_map schema = collection1->get_schema(); diff --git a/test/collection_sorting_test.cpp b/test/collection_sorting_test.cpp index d85686a5..5c2283b0 100644 --- a/test/collection_sorting_test.cpp +++ b/test/collection_sorting_test.cpp @@ -756,7 +756,101 @@ TEST_F(CollectionSortingTest, GeoPointSortingWithExcludeRadius) { {}, geo_sort_fields, {0}, 10, 1, FREQUENCY); ASSERT_FALSE(res_op.ok()); - ASSERT_EQ("Sort field's exclude radius unit must be either `km` or `mi`.", res_op.error()); + ASSERT_EQ("Sort field's parameter unit must be either `km` or `mi`.", res_op.error()); + + geo_sort_fields = { sort_by("loc(32.24348, 77.1893, exclude_radius: -10 km)", "ASC") }; + res_op = coll1->search("*", {}, "loc: (32.24348, 77.1893, 20 km)", + {}, geo_sort_fields, {0}, 10, 1, FREQUENCY); + + ASSERT_FALSE(res_op.ok()); + ASSERT_EQ("Sort field's parameter must be a positive number.", res_op.error()); collectionManager.drop_collection("coll1"); -} \ No newline at end of file +} + +TEST_F(CollectionSortingTest, GeoPointSortingWithPrecision) { + Collection* coll1; + + std::vector fields = {field("title", field_types::STRING, false), + field("loc", field_types::GEOPOINT, false), + field("points", field_types::INT32, false),}; + + coll1 = collectionManager.get_collection("coll1").get(); + if (coll1 == nullptr) { + coll1 = collectionManager.create_collection("coll1", 1, fields, "points").get(); + } + + std::vector> records = { + {"Tibetan Colony", "32.24678, 77.19239"}, + {"Civil Hospital", "32.23959, 77.18763"}, + {"Johnson Lodge", "32.24751, 77.18814"}, + + {"Lion King Rock", "32.24493, 77.17038"}, + {"Jai Durga Handloom", "32.25749, 77.17583"}, + {"Panduropa", "32.26059, 77.21798"}, + + {"Police Station", "32.23743, 77.18639"}, + {"Panduropa Post", "32.26263, 77.2196"}, + }; + + for (size_t i = 0; i < records.size(); i++) { + nlohmann::json doc; + + std::vector lat_lng; + StringUtils::split(records[i][1], lat_lng, ", "); + + double lat = std::stod(lat_lng[0]); + double lng = std::stod(lat_lng[1]); + + doc["id"] = std::to_string(i); + doc["title"] = records[i][0]; + doc["loc"] = {lat, lng}; + doc["points"] = i; + + ASSERT_TRUE(coll1->add(doc.dump()).ok()); + } + + std::vector geo_sort_fields = { + sort_by("loc(32.24348, 77.1893, precision: 0.9 km)", "ASC"), + sort_by("points", "DESC"), + }; + + auto results = coll1->search("*", + {}, "loc: (32.24348, 77.1893, 20 km)", + {}, geo_sort_fields, {0}, 10, 1, FREQUENCY).get(); + + ASSERT_EQ(8, results["found"].get()); + + std::vector expected_ids = { + "6", "2", "1", "0", "3", "4", "7", "5" + }; + + for (size_t i = 0; i < expected_ids.size(); i++) { + ASSERT_STREQ(expected_ids[i].c_str(), results["hits"][i]["document"]["id"].get().c_str()); + } + + // badly formatted precision + + geo_sort_fields = { sort_by("loc(32.24348, 77.1893, precision 1 km)", "ASC") }; + auto res_op = coll1->search("*", {}, "loc: (32.24348, 77.1893, 20 km)", + {}, geo_sort_fields, {0}, 10, 1, FREQUENCY); + + ASSERT_FALSE(res_op.ok()); + ASSERT_EQ("Bad syntax for geopoint sorting field `loc`", res_op.error()); + + geo_sort_fields = { sort_by("loc(32.24348, 77.1893, precision: 1 meter)", "ASC") }; + res_op = coll1->search("*", {}, "loc: (32.24348, 77.1893, 20 km)", + {}, geo_sort_fields, {0}, 10, 1, FREQUENCY); + + ASSERT_FALSE(res_op.ok()); + ASSERT_EQ("Sort field's parameter unit must be either `km` or `mi`.", res_op.error()); + + geo_sort_fields = { sort_by("loc(32.24348, 77.1893, precision: -10 km)", "ASC") }; + res_op = coll1->search("*", {}, "loc: (32.24348, 77.1893, 20 km)", + {}, geo_sort_fields, {0}, 10, 1, FREQUENCY); + + ASSERT_FALSE(res_op.ok()); + ASSERT_EQ("Sort field's parameter must be a positive number.", res_op.error()); + + collectionManager.drop_collection("coll1"); +} diff --git a/test/collection_specific_test.cpp b/test/collection_specific_test.cpp index f829cf13..bf74f410 100644 --- a/test/collection_specific_test.cpp +++ b/test/collection_specific_test.cpp @@ -606,10 +606,117 @@ TEST_F(CollectionSpecificTest, TokensSpreadAcrossFields) { spp::sparse_hash_set(), 10, "", 30, 4, "", 40, {}, {}, {}, 0, "", "", {4, 1}).get(); - LOG(INFO) << results; - ASSERT_EQ("0", results["hits"][0]["document"]["id"].get()); ASSERT_EQ("1", results["hits"][1]["document"]["id"].get()); collectionManager.drop_collection("coll1"); } + +TEST_F(CollectionSpecificTest, GuardAgainstIdFieldInSchema) { + // The "id" field, if defined in the schema should be ignored + + std::vector fields = {field("title", field_types::STRING, false), + field("id", field_types::STRING, false), + field("points", field_types::INT32, false),}; + + nlohmann::json schema; + schema["name"] = "books"; + schema["fields"] = nlohmann::json::array(); + schema["fields"][0]["name"] = "title"; + schema["fields"][0]["type"] = "string"; + schema["fields"][1]["name"] = "id"; + schema["fields"][1]["type"] = "string"; + schema["fields"][2]["name"] = "points"; + schema["fields"][2]["type"] = "int32"; + + Collection* coll1 = collectionManager.create_collection(schema).get(); + + ASSERT_EQ(0, coll1->get_schema().count("id")); + + collectionManager.drop_collection("coll1"); +} + +TEST_F(CollectionSpecificTest, HandleBadCharactersInStringGracefully) { + std::vector fields = {field("title", field_types::STRING, false), + field("points", field_types::INT32, false),}; + + Collection* coll1 = collectionManager.create_collection("coll1", 1, fields, "points").get(); + std::string doc_str = "不推荐。\",\"price\":10.12,\"ratings\":5}"; + + auto add_op = coll1->add(doc_str); + ASSERT_FALSE(add_op.ok()); + + collectionManager.drop_collection("coll1"); +} + +TEST_F(CollectionSpecificTest, HighlightSecondaryFieldWithPrefixMatch) { + std::vector fields = {field("title", field_types::STRING, false), + field("description", field_types::STRING, false), + field("points", field_types::INT32, false),}; + + Collection* coll1 = collectionManager.create_collection("coll1", 1, fields, "points").get(); + + nlohmann::json doc1; + doc1["id"] = "0"; + doc1["title"] = "Functions and Equations"; + doc1["description"] = "Use a function to solve an equation."; + doc1["points"] = 100; + + nlohmann::json doc2; + doc2["id"] = "1"; + doc2["title"] = "Function of effort"; + doc2["description"] = "Learn all about it."; + doc2["points"] = 100; + + ASSERT_TRUE(coll1->add(doc1.dump()).ok()); + ASSERT_TRUE(coll1->add(doc2.dump()).ok()); + + auto results = coll1->search("function", {"title", "description"}, "", {}, {}, {0}, 10, + 1, FREQUENCY, {true, true}, + 10, spp::sparse_hash_set(), + spp::sparse_hash_set(), 10, "", 30, 4, "", 40, {}, {}, {}, 0, + "", "", {1, 1}).get(); + + ASSERT_EQ("0", results["hits"][0]["document"]["id"].get()); + + ASSERT_EQ(2, results["hits"][0]["highlights"].size()); + + ASSERT_EQ("Functions and Equations", + results["hits"][0]["highlights"][0]["snippet"].get()); + + ASSERT_EQ("Use a function to solve an equation.", + results["hits"][0]["highlights"][1]["snippet"].get()); + + collectionManager.drop_collection("coll1"); +} + +TEST_F(CollectionSpecificTest, HighlightWithDropTokens) { + std::vector fields = {field("description", field_types::STRING, false), + field("points", field_types::INT32, false),}; + + Collection* coll1 = collectionManager.create_collection("coll1", 1, fields, "points").get(); + + nlohmann::json doc1; + doc1["id"] = "0"; + doc1["description"] = "HPE Aruba AP-575 802.11ax Wireless Access Point - TAA Compliant - 2.40 GHz, " + "5 GHz - MIMO Technology - 1 x Network (RJ-45) - Gigabit Ethernet - Bluetooth 5"; + doc1["points"] = 100; + + ASSERT_TRUE(coll1->add(doc1.dump()).ok()); + + auto results = coll1->search("HPE Aruba AP-575 Technology Gigabit Bluetooth 5", {"description"}, "", {}, {}, {0}, 10, + 1, FREQUENCY, {true}, + 10, spp::sparse_hash_set(), + spp::sparse_hash_set(), 10, "", 30, 4, "description", 40, {}, {}, {}, 0, + "", "").get(); + + ASSERT_EQ(1, results["hits"][0]["highlights"].size()); + ASSERT_EQ("0", results["hits"][0]["document"]["id"].get()); + + ASSERT_EQ("HPE Aruba AP-575 802.11ax Wireless Access Point - " + "TAA Compliant - 2.40 GHz, 5 GHz - MIMO Technology - 1 x Network (RJ-45) - " + "Gigabit Ethernet - Bluetooth 5", + results["hits"][0]["highlights"][0]["snippet"].get()); + + collectionManager.drop_collection("coll1"); +} diff --git a/test/collection_test.cpp b/test/collection_test.cpp index efe6f215..9f924bf3 100644 --- a/test/collection_test.cpp +++ b/test/collection_test.cpp @@ -746,23 +746,23 @@ TEST_F(CollectionTest, ArrayStringFieldHighlight) { ASSERT_STREQ(id.c_str(), result_id.c_str()); } - ASSERT_EQ(4, results["hits"][0]["highlights"][0].size()); - ASSERT_STREQ(results["hits"][0]["highlights"][0]["field"].get().c_str(), "tags"); - ASSERT_EQ(2, results["hits"][0]["highlights"][0]["snippets"].size()); - ASSERT_STREQ("truth", results["hits"][0]["highlights"][0]["snippets"][0].get().c_str()); - ASSERT_STREQ("plain truth", results["hits"][0]["highlights"][0]["snippets"][1].get().c_str()); - ASSERT_EQ(2, results["hits"][0]["highlights"][0]["matched_tokens"].size()); - ASSERT_STREQ("truth", results["hits"][0]["highlights"][0]["matched_tokens"][0][0].get().c_str()); - ASSERT_STREQ("truth", results["hits"][0]["highlights"][0]["matched_tokens"][1][0].get().c_str()); - ASSERT_EQ(2, results["hits"][0]["highlights"][0]["indices"].size()); - ASSERT_EQ(1, results["hits"][0]["highlights"][0]["indices"][0]); - ASSERT_EQ(2, results["hits"][0]["highlights"][0]["indices"][1]); + ASSERT_EQ(3, results["hits"][0]["highlights"][0].size()); + ASSERT_STREQ("title", results["hits"][0]["highlights"][0]["field"].get().c_str()); + ASSERT_STREQ("Plain Truth", results["hits"][0]["highlights"][0]["snippet"].get().c_str()); + ASSERT_EQ(1, results["hits"][0]["highlights"][0]["matched_tokens"].size()); + ASSERT_STREQ("Truth", results["hits"][0]["highlights"][0]["matched_tokens"][0].get().c_str()); - ASSERT_EQ(3, results["hits"][0]["highlights"][1].size()); - ASSERT_STREQ("title", results["hits"][0]["highlights"][1]["field"].get().c_str()); - ASSERT_STREQ("Plain Truth", results["hits"][0]["highlights"][1]["snippet"].get().c_str()); - ASSERT_EQ(1, results["hits"][0]["highlights"][1]["matched_tokens"].size()); - ASSERT_STREQ("Truth", results["hits"][0]["highlights"][1]["matched_tokens"][0].get().c_str()); + ASSERT_EQ(4, results["hits"][0]["highlights"][1].size()); + ASSERT_STREQ(results["hits"][0]["highlights"][1]["field"].get().c_str(), "tags"); + ASSERT_EQ(2, results["hits"][0]["highlights"][1]["snippets"].size()); + ASSERT_STREQ("truth", results["hits"][0]["highlights"][1]["snippets"][0].get().c_str()); + ASSERT_STREQ("plain truth", results["hits"][0]["highlights"][1]["snippets"][1].get().c_str()); + ASSERT_EQ(2, results["hits"][0]["highlights"][1]["matched_tokens"].size()); + ASSERT_STREQ("truth", results["hits"][0]["highlights"][1]["matched_tokens"][0][0].get().c_str()); + ASSERT_STREQ("truth", results["hits"][0]["highlights"][1]["matched_tokens"][1][0].get().c_str()); + ASSERT_EQ(2, results["hits"][0]["highlights"][1]["indices"].size()); + ASSERT_EQ(1, results["hits"][0]["highlights"][1]["indices"][0]); + ASSERT_EQ(2, results["hits"][0]["highlights"][1]["indices"][1]); ASSERT_EQ(3, results["hits"][1]["highlights"][0].size()); ASSERT_STREQ("title", results["hits"][1]["highlights"][0]["field"].get().c_str()); @@ -2500,15 +2500,15 @@ TEST_F(CollectionTest, SearchHighlightFieldFully) { spp::sparse_hash_set(), 10, "", 5, 5, "title, tags").get(); ASSERT_EQ(2, res["hits"][0]["highlights"].size()); - ASSERT_EQ("LAZY", res["hits"][0]["highlights"][0]["values"][0].get()); ASSERT_EQ("The quick brown fox jumped over the lazy dog and ran straight to the forest to sleep.", - res["hits"][0]["highlights"][1]["value"].get()); + res["hits"][0]["highlights"][0]["value"].get()); + ASSERT_EQ(1, res["hits"][0]["highlights"][0]["matched_tokens"].size()); + ASSERT_STREQ("lazy", res["hits"][0]["highlights"][0]["matched_tokens"][0].get().c_str()); - ASSERT_EQ(1, res["hits"][0]["highlights"][1]["matched_tokens"].size()); - ASSERT_STREQ("lazy", res["hits"][0]["highlights"][1]["matched_tokens"][0].get().c_str()); - - ASSERT_EQ(1, res["hits"][0]["highlights"][0]["values"][0].size()); - ASSERT_STREQ("LAZY", res["hits"][0]["highlights"][0]["values"][0].get().c_str()); + ASSERT_EQ(1, res["hits"][0]["highlights"][1]["values"].size()); + ASSERT_EQ("LAZY", res["hits"][0]["highlights"][1]["values"][0].get()); + ASSERT_EQ(1, res["hits"][0]["highlights"][1]["snippets"].size()); + ASSERT_EQ("LAZY", res["hits"][0]["highlights"][1]["snippets"][0].get()); // excluded fields should not be returned in highlights section spp::sparse_hash_set excluded_fields = {"tags"}; @@ -3190,13 +3190,17 @@ TEST_F(CollectionTest, MultiFieldRelevance5) { ASSERT_STREQ("1", results["hits"][1]["document"]["id"].get().c_str()); ASSERT_STREQ("2", results["hits"][2]["document"]["id"].get().c_str()); - ASSERT_EQ(1, results["hits"][0]["highlights"].size()); - ASSERT_EQ("country", results["hits"][0]["highlights"][0]["field"].get()); - ASSERT_EQ("Canada", results["hits"][0]["highlights"][0]["snippet"].get()); + ASSERT_EQ(2, results["hits"][0]["highlights"].size()); + ASSERT_EQ("field_a", results["hits"][0]["highlights"][0]["field"].get()); + ASSERT_EQ("Canadia", results["hits"][0]["highlights"][0]["snippet"].get()); + ASSERT_EQ("country", results["hits"][0]["highlights"][1]["field"].get()); + ASSERT_EQ("Canada", results["hits"][0]["highlights"][1]["snippet"].get()); - ASSERT_EQ(1, results["hits"][1]["highlights"].size()); - ASSERT_EQ("company_name", results["hits"][1]["highlights"][0]["field"].get()); - ASSERT_EQ("Canaida Corp", results["hits"][1]["highlights"][0]["snippet"].get()); + ASSERT_EQ(2, results["hits"][1]["highlights"].size()); + ASSERT_EQ("field_a", results["hits"][1]["highlights"][0]["field"].get()); + ASSERT_EQ("Canadoo", results["hits"][1]["highlights"][0]["snippet"].get()); + ASSERT_EQ("company_name", results["hits"][1]["highlights"][1]["field"].get()); + ASSERT_EQ("Canaida Corp", results["hits"][1]["highlights"][1]["snippet"].get()); ASSERT_EQ(1, results["hits"][2]["highlights"].size()); ASSERT_EQ("field_a", results["hits"][2]["highlights"][0]["field"].get()); @@ -3361,7 +3365,7 @@ TEST_F(CollectionTest, MultiFieldHighlighting) { ASSERT_STREQ("0", results["hits"][0]["document"]["id"].get().c_str()); - ASSERT_EQ(2, results["hits"][0]["highlights"].size()); + ASSERT_EQ(3, results["hits"][0]["highlights"].size()); ASSERT_EQ("name", results["hits"][0]["highlights"][0]["field"].get()); ASSERT_EQ("Best Wireless Vehicle Charger", results["hits"][0]["highlights"][0]["snippet"].get()); @@ -3370,6 +3374,9 @@ TEST_F(CollectionTest, MultiFieldHighlighting) { ASSERT_EQ("Easily replenish your cell phone with this wireless charger.", results["hits"][0]["highlights"][1]["snippet"].get()); + ASSERT_EQ("categories", results["hits"][0]["highlights"][2]["field"].get()); + ASSERT_EQ("Car Chargers", results["hits"][0]["highlights"][2]["snippets"][0].get()); + results = coll1->search("John With Denver", {"description"}, "", {}, {}, {0}, 10, 1, FREQUENCY, {true}, 1, spp::sparse_hash_set(),