diff --git a/TODO.md b/TODO.md index 8c3bed2d..0c5afdb8 100644 --- a/TODO.md +++ b/TODO.md @@ -2,6 +2,13 @@ ## Pre-alpha +a) ~~Fix memory ratio (decreasing with indexing)~~ +b) ~~Speed up wildcard searches further~~ +c) Allow int64 in default sorting field +d) Use connection timeout for CURL rather than request timeout +e) Update role to set max memory ration at 0.80 +f) Async import + **Search index** - ~~Proper JSON as input~~ diff --git a/src/collection.cpp b/src/collection.cpp index e5495630..a3955d00 100644 --- a/src/collection.cpp +++ b/src/collection.cpp @@ -619,15 +619,6 @@ Option Collection::search(const std::string & query, const std:: filters.push_back(f); } - // for a wildcard query, if filter is not specified, use default_sorting_field as a catch-all - if(query == "*" && filters.empty()) { - field f = search_schema.at(default_sorting_field); - std::string max_value = f.is_float() ? std::to_string(std::numeric_limits::max()) : - std::to_string(std::numeric_limits::max()); - filter catch_all_filter = {f.name, {max_value}, LESS_THAN_EQUALS}; - filters.push_back(catch_all_filter); - } - // validate facet fields for(const std::string & field_name: facet_fields) { if(facet_schema.count(field_name) == 0) { diff --git a/src/index.cpp b/src/index.cpp index 6613e572..2fcc048d 100644 --- a/src/index.cpp +++ b/src/index.cpp @@ -1164,11 +1164,32 @@ void Index::search(Option & outcome, const uint8_t field_id = (uint8_t)(FIELD_LIMIT_NUM - 0); const std::string & field = search_fields[0]; + // if a filter is not specified, use the sorting index to generate the list of all document ids + if(filters.empty()) { + std::string all_records_field; + + // get the first non-optional field + for(const auto& kv: sort_schema) { + if(!kv.second.optional && kv.first != sort_field_const::text_match) { + all_records_field = kv.first; + break; + } + } + + const spp::sparse_hash_map *kvs = sort_index[all_records_field]; + filter_ids_length = kvs->size(); + filter_ids = new uint32_t[filter_ids_length]; + + size_t i = 0; + for(const auto& kv: *kvs) { + filter_ids[i++] = kv.first; + } + } + if(!curated_ids.empty()) { uint32_t *excluded_result_ids = nullptr; filter_ids_length = ArrayUtils::exclude_scalar(filter_ids, filter_ids_length, &curated_ids_sorted[0], curated_ids.size(), &excluded_result_ids); - delete [] filter_ids; filter_ids = excluded_result_ids; } diff --git a/test/collection_test.cpp b/test/collection_test.cpp index f22a5068..29ecd996 100644 --- a/test/collection_test.cpp +++ b/test/collection_test.cpp @@ -2259,7 +2259,7 @@ TEST_F(CollectionTest, OptionalFields) { infile.close(); - // first must be able to fetch all records (i.e. all must have been index) + // first must be able to fetch all records (i.e. all must have been indexed) auto res = coll1->search("*", {"title"}, "", {}, {}, 0, 10, 1, FREQUENCY, false).get(); ASSERT_EQ(6, res["found"].get());