From 09541e531180778bbca89ef4909f23bdd3fb9a5c Mon Sep 17 00:00:00 2001 From: kishorenc Date: Wed, 19 Aug 2020 07:28:21 +0530 Subject: [PATCH] Parameterize number of indices per collection. --- include/collection_manager.h | 11 +++--- include/config.h | 11 ------ include/core_api.h | 2 +- include/http_data.h | 10 ++++- include/http_server.h | 2 + src/collection.cpp | 1 + src/collection_manager.cpp | 20 +++++++--- src/core_api.cpp | 45 ++++++++++++++++------- src/http_server.cpp | 61 ++++++++++++++++++------------- src/main/main.cpp | 2 +- src/raft_server.cpp | 8 ++-- src/typesense_server_utils.cpp | 5 +-- test/collection_faceting_test.cpp | 10 ++--- test/collection_grouping_test.cpp | 4 +- test/collection_manager_test.cpp | 14 +++---- test/collection_override_test.cpp | 4 +- test/collection_sorting_test.cpp | 14 +++---- test/collection_test.cpp | 52 +++++++++++++------------- 18 files changed, 156 insertions(+), 120 deletions(-) diff --git a/include/collection_manager.h b/include/collection_manager.h index dc8c475a..01207f8a 100644 --- a/include/collection_manager.h +++ b/include/collection_manager.h @@ -29,12 +29,11 @@ private: static constexpr const char* COLLECTION_SEARCH_FIELDS_KEY = "fields"; static constexpr const char* COLLECTION_DEFAULT_SORTING_FIELD_KEY = "default_sorting_field"; static constexpr const char* COLLECTION_CREATED = "created_at"; + static constexpr const char* COLLECTION_NUM_INDICES = "num_indices"; std::string bootstrap_auth_key; std::string bootstrap_search_only_auth_key; - size_t default_num_indices; - float max_memory_ratio; CollectionManager(); @@ -60,8 +59,7 @@ public: CollectionManager(CollectionManager const&) = delete; void operator=(CollectionManager const&) = delete; - void init(Store *store, const size_t default_num_indices, const float max_memory_ratio, - const std::string & auth_key); + void init(Store *store, const float max_memory_ratio, const std::string & auth_key); Option load(const size_t init_batch_size=1000); @@ -75,7 +73,8 @@ public: bool auth_key_matches(const std::string& auth_key_sent, const std::string& action, const std::string& collection, std::map& params); - Option create_collection(const std::string name, const std::vector & fields, + Option create_collection(const std::string name, const size_t num_indices, + const std::vector & fields, const std::string & default_sorting_field, const uint64_t created_at = static_cast(std::time(nullptr))); @@ -106,6 +105,8 @@ public: Option delete_symlink(const std::string & symlink_name); + static const size_t DEFAULT_NUM_INDICES = 4; + static constexpr const char* NEXT_COLLECTION_ID_KEY = "$CI"; static constexpr const char* SYMLINK_PREFIX = "$SL"; }; \ No newline at end of file diff --git a/include/config.h b/include/config.h index 9df51978..b58e2748 100644 --- a/include/config.h +++ b/include/config.h @@ -29,8 +29,6 @@ private: bool enable_cors; - size_t indices_per_collection; - float max_memory_ratio; std::string config_file; @@ -43,7 +41,6 @@ public: this->api_port = 8108; this->peering_port = 8107; this->enable_cors = false; - this->indices_per_collection = 4; this->max_memory_ratio = 1.0f; } @@ -90,10 +87,6 @@ public: this->enable_cors = enable_cors; } - void set_indices_per_collection(size_t indices_per_collection) { - this->indices_per_collection = indices_per_collection; - } - // getters std::string get_data_dir() const { @@ -141,10 +134,6 @@ public: return this->enable_cors; } - size_t get_indices_per_collection() const { - return indices_per_collection; - } - std::string get_peering_address() const { return this->peering_address; } diff --git a/include/core_api.h b/include/core_api.h index e8e01c3a..6f29771b 100644 --- a/include/core_api.h +++ b/include/core_api.h @@ -69,6 +69,6 @@ bool get_metrics_json(http_req& req, http_res& res); // Misc helpers -bool async_write_request(void *data); +bool raft_write_send_response(void *data); static constexpr const char* SEND_RESPONSE_MSG = "send_response"; diff --git a/include/http_data.h b/include/http_data.h index 101d0e26..2aa7f1b6 100644 --- a/include/http_data.h +++ b/include/http_data.h @@ -15,7 +15,7 @@ extern "C" { } struct h2o_custom_timer_t { - h2o_timer_t timer{}; + h2o_timer_t timer; void *data; h2o_custom_timer_t(): data(nullptr) {} @@ -182,6 +182,14 @@ struct http_req { return content.dump(); } + + bool is_ending() { + return stream_state == "END" || stream_state == "NON_STREAMING"; + } + + bool is_starting() { + return stream_state == "START" || stream_state == "NON_STREAMING"; + } }; struct request_response { diff --git a/include/http_server.h b/include/http_server.h index 574297df..e3ea3273 100644 --- a/include/http_server.h +++ b/include/http_server.h @@ -127,6 +127,8 @@ public: void send_response(http_req* request, http_res* response); + static void destroy_request_response(http_req* request, http_res* response); + static void stream_response(http_req& request, http_res& response); uint64_t find_route(const std::vector & path_parts, const std::string & http_method, diff --git a/src/collection.cpp b/src/collection.cpp index a1219c93..3a12f0c2 100644 --- a/src/collection.cpp +++ b/src/collection.cpp @@ -132,6 +132,7 @@ nlohmann::json Collection::get_summary_json() { nlohmann::json json_response; json_response["name"] = name; + json_response["num_indices"] = num_indices; json_response["num_documents"] = num_documents; json_response["created_at"] = created_at; diff --git a/src/collection_manager.cpp b/src/collection_manager.cpp index 8f27f93e..6e628fd8 100644 --- a/src/collection_manager.cpp +++ b/src/collection_manager.cpp @@ -4,6 +4,9 @@ #include "collection_manager.h" #include "logger.h" +constexpr const char* CollectionManager::COLLECTION_NUM_INDICES; +constexpr const size_t CollectionManager::DEFAULT_NUM_INDICES; + CollectionManager::CollectionManager() { } @@ -31,6 +34,12 @@ Collection* CollectionManager::init_collection(const nlohmann::json & collection uint64_t created_at = collection_meta.find((const char*)COLLECTION_CREATED) != collection_meta.end() ? collection_meta[COLLECTION_CREATED].get() : 0; + size_t num_indices = collection_meta.count(COLLECTION_NUM_INDICES) != 0 ? + collection_meta[COLLECTION_NUM_INDICES].get() : + DEFAULT_NUM_INDICES; + + LOG(INFO) << "Loading collection with " << num_indices << " indices."; + Collection* collection = new Collection(this_collection_name, collection_meta[COLLECTION_ID_KEY].get(), created_at, @@ -38,7 +47,7 @@ Collection* CollectionManager::init_collection(const nlohmann::json & collection store, fields, default_sorting_field, - default_num_indices, + num_indices, max_memory_ratio); return collection; @@ -50,12 +59,10 @@ void CollectionManager::add_to_collections(Collection* collection) { } void CollectionManager::init(Store *store, - const size_t default_num_indices, const float max_memory_ratio, const std::string & auth_key) { this->store = store; this->bootstrap_auth_key = auth_key; - this->default_num_indices = default_num_indices; this->max_memory_ratio = max_memory_ratio; auth_manager.init(store); @@ -227,7 +234,9 @@ bool CollectionManager::auth_key_matches(const std::string& auth_key_sent, return auth_manager.authenticate(auth_key_sent, action, collection, params); } -Option CollectionManager::create_collection(const std::string name, const std::vector & fields, +Option CollectionManager::create_collection(const std::string name, + const size_t num_indices, + const std::vector & fields, const std::string & default_sorting_field, const uint64_t created_at) { if(store->contains(Collection::get_meta_key(name))) { @@ -273,9 +282,10 @@ Option CollectionManager::create_collection(const std::string name, collection_meta[COLLECTION_SEARCH_FIELDS_KEY] = fields_json; collection_meta[COLLECTION_DEFAULT_SORTING_FIELD_KEY] = default_sorting_field; collection_meta[COLLECTION_CREATED] = created_at; + collection_meta[COLLECTION_NUM_INDICES] = num_indices; Collection* new_collection = new Collection(name, next_collection_id, created_at, 0, store, fields, - default_sorting_field, this->default_num_indices, + default_sorting_field, num_indices, this->max_memory_ratio); next_collection_id++; diff --git a/src/core_api.cpp b/src/core_api.cpp index aead9d48..4ffa9059 100644 --- a/src/core_api.cpp +++ b/src/core_api.cpp @@ -42,6 +42,7 @@ bool get_collections(http_req & req, http_res & res) { } bool post_create_collection(http_req & req, http_res & res) { + const char* NUM_INDICES = "num_indices"; nlohmann::json req_json; try { @@ -61,6 +62,10 @@ bool post_create_collection(http_req & req, http_res & res) { return false; } + if(req_json.count(NUM_INDICES) == 0) { + req_json[NUM_INDICES] = CollectionManager::DEFAULT_NUM_INDICES; + } + if(req_json.count("fields") == 0) { res.set_400("Parameter `fields` is required."); return false; @@ -79,6 +84,16 @@ bool post_create_collection(http_req & req, http_res & res) { return false; } + if(!req_json[NUM_INDICES].is_number_unsigned()) { + res.set_400(std::string("`") + NUM_INDICES + "` should be a positive integer."); + return false; + } + + if(req_json[NUM_INDICES].get() == 0) { + res.set_400(std::string("`") + NUM_INDICES + "` should be a positive integer."); + return false; + } + if(collectionManager.get_collection(req_json["name"]) != nullptr) { res.set_409("Collection with name `" + req_json["name"].get() + "` already exists."); return false; @@ -125,7 +140,8 @@ bool post_create_collection(http_req & req, http_res & res) { const std::string & default_sorting_field = req_json[DEFAULT_SORTING_FIELD].get(); const Option & collection_op = - collectionManager.create_collection(req_json["name"], fields, default_sorting_field); + collectionManager.create_collection(req_json["name"], req_json[NUM_INDICES].get(), + fields, default_sorting_field); if(collection_op.ok()) { nlohmann::json json_response = collection_op.get()->get_summary_json(); @@ -571,14 +587,21 @@ bool post_import_documents(http_req& req, http_res& res) { if(!StringUtils::is_uint32_t(req.params[BATCH_SIZE])) { req.stream_state = "NON_STREAMING"; - res.set_400("Parameter `" + std::string(BATCH_SIZE) + "` must be an unsigned integer."); + res.set_400("Parameter `" + std::string(BATCH_SIZE) + "` must be a positive integer."); HttpServer::stream_response(req, res); return false; } const size_t IMPORT_BATCH_SIZE = std::stoi(req.params[BATCH_SIZE]); - LOG(INFO) << "Import batch size: " << IMPORT_BATCH_SIZE; + if(IMPORT_BATCH_SIZE == 0) { + res.set_400("Parameter `" + std::string(BATCH_SIZE) + "` must be a positive integer."); + return false; + } + + if(req.stream_state == "START" && req.body_index == 0) { + LOG(INFO) << "Import batch size: " << IMPORT_BATCH_SIZE; + } CollectionManager & collectionManager = CollectionManager::get_instance(); Collection* collection = collectionManager.get_collection(req.params["collection"]); @@ -1036,8 +1059,8 @@ bool del_key(http_req &req, http_res &res) { return true; } -bool async_write_request(void *data) { - //LOG(INFO) << "async_write_request called"; +bool raft_write_send_response(void *data) { + //LOG(INFO) << "raft_write_send_response called"; AsyncIndexArg* index_arg = static_cast(data); std::unique_ptr index_arg_guard(index_arg); @@ -1047,10 +1070,10 @@ bool async_write_request(void *data) { // route not found index_arg->res->set_400("Not found."); } else if(index_arg->req->route_hash != static_cast(ROUTE_CODES::ALREADY_HANDLED)) { - // call the underlying http handler route_path* found_rpath = nullptr; bool route_found = server->get_route(index_arg->req->route_hash, &found_rpath); if(route_found) { + // call request handler found_rpath->handler(*index_arg->req, *index_arg->res); async_res = found_rpath->async_res; } else { @@ -1058,17 +1081,11 @@ bool async_write_request(void *data) { } } - if(!async_res && index_arg->req->_req != nullptr) { - // we have to return a response to the client + if(!async_res) { + // only handle synchronous responses as async ones are handled by their handlers server->send_response(index_arg->req, index_arg->res); } - if(index_arg->req->_req == nullptr) { - // indicates raft serialized request and response -- lifecycle must be managed here - delete index_arg->req; - delete index_arg->res; - } - if(index_arg->promise != nullptr) { index_arg->promise->set_value(true); // returns control back to caller } diff --git a/src/http_server.cpp b/src/http_server.cpp index ef55e4f7..31bf29e3 100644 --- a/src/http_server.cpp +++ b/src/http_server.cpp @@ -71,7 +71,6 @@ void HttpServer::on_ssl_refresh_timeout(h2o_timer_t *entry) { int HttpServer::setup_ssl(const char *cert_file, const char *key_file) { // Set up a timer to refresh SSL config from disk. Also, initializing upfront so that destructor works ssl_refresh_timer = h2o_custom_timer_t(this); - ssl_refresh_timer.timer.expire_at = UINT64_MAX-1; h2o_timer_init(&ssl_refresh_timer.timer, on_ssl_refresh_timeout); h2o_timer_link(ctx.loop, SSL_REFRESH_INTERVAL_MS, &ssl_refresh_timer.timer); // every 8 hours @@ -278,14 +277,7 @@ void HttpServer::on_res_generator_dispose(void *self) { res_generator->rpath->handler(*res_generator->request, *res_generator->response); } - if(res_generator->request->defer_timer.data != nullptr) { - deferred_req_res_t* deferred_req_res = static_cast(res_generator->request->defer_timer.data); - delete deferred_req_res; - } - - // res_generator itself is reference counted, so we only delete the members - delete res_generator->request; - delete res_generator->response; + destroy_request_response(res_generator->request, res_generator->response); } int HttpServer::catch_all_handler(h2o_handler_t *_h2o_handler, h2o_req_t *req) { @@ -509,24 +501,15 @@ void HttpServer::defer_processing(http_req& req, http_res& res, size_t timeout_m if(req.defer_timer.data == nullptr) { auto deferred_req_res = new deferred_req_res_t{&req, &res, this}; req.defer_timer.data = deferred_req_res; - req.defer_timer.timer.expire_at = UINT64_MAX-1; h2o_timer_init(&req.defer_timer.timer, on_deferred_process_request); } + h2o_timer_unlink(&req.defer_timer.timer); h2o_timer_link(ctx.loop, timeout_ms, &req.defer_timer.timer); } -void HttpServer::send_response(http_req* request, http_res* response) { - h2o_req_t* req = request->_req; - h2o_generator_t& generator = *response->generator; - - h2o_iovec_t body = h2o_strdup(&req->pool, response->body.c_str(), SIZE_MAX); - req->res.status = response->status_code; - req->res.reason = http_res::get_status_reason(response->status_code); - h2o_add_header(&req->pool, &req->res.headers, H2O_TOKEN_CONTENT_TYPE, - nullptr, response->content_type_header.c_str(), response->content_type_header.size()); - h2o_start_response(req, &generator); - h2o_send(req, &body, 1, H2O_SEND_STATE_FINAL); +void HttpServer::send_message(const std::string & type, void* data) { + message_dispatcher->send_message(type, data); } int HttpServer::send_response(h2o_req_t *req, int status_code, const std::string & message) { @@ -540,8 +523,22 @@ int HttpServer::send_response(h2o_req_t *req, int status_code, const std::string return 0; } -void HttpServer::send_message(const std::string & type, void* data) { - message_dispatcher->send_message(type, data); +void HttpServer::send_response(http_req* request, http_res* response) { + if(request->_req == nullptr) { + // indicates serialized request and response -- lifecycle must be managed here + return destroy_request_response(request, response); + } + + h2o_req_t* req = request->_req; + h2o_generator_t& generator = *response->generator; + + h2o_iovec_t body = h2o_strdup(&req->pool, response->body.c_str(), SIZE_MAX); + req->res.status = response->status_code; + req->res.reason = http_res::get_status_reason(response->status_code); + h2o_add_header(&req->pool, &req->res.headers, H2O_TOKEN_CONTENT_TYPE, + nullptr, response->content_type_header.c_str(), response->content_type_header.size()); + h2o_start_response(req, &generator); + h2o_send(req, &body, 1, H2O_SEND_STATE_FINAL); } void HttpServer::response_proceed(h2o_generator_t *generator, h2o_req_t *req) { @@ -564,9 +561,10 @@ void HttpServer::response_proceed(h2o_generator_t *generator, h2o_req_t *req) { } void HttpServer::stream_response(http_req& request, http_res& response) { - if(!request._req) { - // underlying request is no longer available - return ; + if(request._req == nullptr) { + // serialized request or underlying request is no longer available + destroy_request_response(&request, &response); + return; } h2o_req_t* req = request._req; @@ -590,6 +588,16 @@ void HttpServer::stream_response(http_req& request, http_res& response) { h2o_send(req, &body, 1, state); } +void HttpServer::destroy_request_response(http_req* request, http_res* response) { + if(request->defer_timer.data != nullptr) { + deferred_req_res_t* deferred_req_res = static_cast(request->defer_timer.data); + delete deferred_req_res; + } + + delete request; + delete response; +} + void HttpServer::set_auth_handler(bool (*handler)(std::map& params, const route_path& rpath, const std::string& auth_key)) { auth_handler = handler; @@ -675,3 +683,4 @@ bool HttpServer::get_route(uint64_t hash, route_path** found_rpath) { uint64_t HttpServer::node_state() const { return replication_state->node_state(); } + diff --git a/src/main/main.cpp b/src/main/main.cpp index fc5ad311..9551038c 100644 --- a/src/main/main.cpp +++ b/src/main/main.cpp @@ -35,7 +35,7 @@ int main(int argc, char* argv[]) { Collection *collection = collectionManager.get_collection("github_top1k"); if(collection == nullptr) { - collection = collectionManager.create_collection("github_top1k", fields_to_index, "stars").get(); + collection = collectionManager.create_collection("github_top1k", 4, fields_to_index, "stars").get(); } int j = 0; diff --git a/src/raft_server.cpp b/src/raft_server.cpp index 03f54dc9..e3057afc 100644 --- a/src/raft_server.cpp +++ b/src/raft_server.cpp @@ -92,6 +92,7 @@ std::string ReplicationState::to_nodes_config(const butil::EndPoint& peering_end } void ReplicationState::write(http_req* request, http_res* response) { + // NOTE: this is executed on a different thread and runs concurrent to http thread if (!is_leader()) { if(node->leader_id().is_empty()) { // Handle no leader scenario @@ -187,11 +188,10 @@ void ReplicationState::on_apply(braft::Iterator& iter) { } else { // Parse request from the log response = new http_res; - http_req* remote_request = new http_req; - remote_request->deserialize(iter.data().to_string()); - remote_request->_req = nullptr; // indicates remote request - request = remote_request; + request = new http_req; + request->deserialize(iter.data().to_string()); + request->_req = nullptr; // indicates remote request } if(request->_req == nullptr && request->body == "INIT_SNAPSHOT") { diff --git a/src/typesense_server_utils.cpp b/src/typesense_server_utils.cpp index abff98cb..076ad979 100644 --- a/src/typesense_server_utils.cpp +++ b/src/typesense_server_utils.cpp @@ -329,8 +329,7 @@ int run_server(const Config & config, const std::string & version, void (*master Store store(db_dir); CollectionManager & collectionManager = CollectionManager::get_instance(); - collectionManager.init(&store, config.get_indices_per_collection(), - config.get_max_memory_ratio(), config.get_api_key()); + collectionManager.init(&store, config.get_max_memory_ratio(), config.get_api_key()); curl_global_init(CURL_GLOBAL_SSL); HttpClient & httpClient = HttpClient::get_instance(); @@ -348,7 +347,7 @@ int run_server(const Config & config, const std::string & version, void (*master server->set_auth_handler(handle_authentication); server->on(SEND_RESPONSE_MSG, on_send_response); - server->on(ReplicationState::REPLICATION_MSG, async_write_request); + server->on(ReplicationState::REPLICATION_MSG, raft_write_send_response); // first we start the peering service diff --git a/test/collection_faceting_test.cpp b/test/collection_faceting_test.cpp index 84ae0f67..7d430a7c 100644 --- a/test/collection_faceting_test.cpp +++ b/test/collection_faceting_test.cpp @@ -20,7 +20,7 @@ protected: system(("rm -rf "+state_dir_path+" && mkdir -p "+state_dir_path).c_str()); store = new Store(state_dir_path); - collectionManager.init(store, 4, 1.0, "auth_key"); + collectionManager.init(store, 1.0, "auth_key"); collectionManager.load(); } @@ -50,7 +50,7 @@ TEST_F(CollectionFacetingTest, FacetCounts) { coll_array_fields = collectionManager.get_collection("coll_array_fields"); if(coll_array_fields == nullptr) { - coll_array_fields = collectionManager.create_collection("coll_array_fields", fields, "age").get(); + coll_array_fields = collectionManager.create_collection("coll_array_fields", 4, fields, "age").get(); } std::string json_line; @@ -336,7 +336,7 @@ TEST_F(CollectionFacetingTest, FacetCountsBool) { coll1 = collectionManager.get_collection("coll1"); if (coll1 == nullptr) { - coll1 = collectionManager.create_collection("coll1", fields, "points").get(); + coll1 = collectionManager.create_collection("coll1", 4, fields, "points").get(); } nlohmann::json doc; @@ -390,7 +390,7 @@ TEST_F(CollectionFacetingTest, FacetCountsHighlighting) { coll1 = collectionManager.get_collection("coll1"); if(coll1 == nullptr) { - coll1 = collectionManager.create_collection("coll1", fields, "points").get(); + coll1 = collectionManager.create_collection("coll1", 4, fields, "points").get(); } nlohmann::json doc; @@ -507,7 +507,7 @@ TEST_F(CollectionFacetingTest, FacetStatOnFloatFields) { coll_float_fields = collectionManager.get_collection("coll_float_fields"); if(coll_float_fields == nullptr) { - coll_float_fields = collectionManager.create_collection("coll_float_fields", fields, "average").get(); + coll_float_fields = collectionManager.create_collection("coll_float_fields", 4, fields, "average").get(); } std::string json_line; diff --git a/test/collection_grouping_test.cpp b/test/collection_grouping_test.cpp index b680cc8b..07cfa639 100644 --- a/test/collection_grouping_test.cpp +++ b/test/collection_grouping_test.cpp @@ -18,7 +18,7 @@ protected: system(("rm -rf "+state_dir_path+" && mkdir -p "+state_dir_path).c_str()); store = new Store(state_dir_path); - collectionManager.init(store, 4, 1.0, "auth_key"); + collectionManager.init(store, 1.0, "auth_key"); collectionManager.load(); std::vector fields = { @@ -31,7 +31,7 @@ protected: coll_group = collectionManager.get_collection("coll_group"); if(coll_group == nullptr) { - coll_group = collectionManager.create_collection("coll_group", fields, "rating").get(); + coll_group = collectionManager.create_collection("coll_group", 4, fields, "rating").get(); } std::ifstream infile(std::string(ROOT_DIR)+"test/group_documents.jsonl"); diff --git a/test/collection_manager_test.cpp b/test/collection_manager_test.cpp index eed11192..902d61cd 100644 --- a/test/collection_manager_test.cpp +++ b/test/collection_manager_test.cpp @@ -20,7 +20,7 @@ protected: system(("rm -rf "+state_dir_path+" && mkdir -p "+state_dir_path).c_str()); store = new Store(state_dir_path); - collectionManager.init(store, 4, 1.0, "auth_key"); + collectionManager.init(store, 1.0, "auth_key"); collectionManager.load(); search_fields = { @@ -31,7 +31,7 @@ protected: }; sort_fields = { sort_by("points", "DESC") }; - collection1 = collectionManager.create_collection("collection1", search_fields, "points", 12345).get(); + collection1 = collectionManager.create_collection("collection1", 4, search_fields, "points", 12345).get(); } virtual void SetUp() { @@ -86,7 +86,7 @@ TEST_F(CollectionManagerTest, CollectionCreation) { "\"fields\":[{\"facet\":false,\"name\":\"title\",\"optional\":false,\"type\":\"string\"}," "{\"facet\":false,\"name\":\"starring\",\"optional\":false,\"type\":\"string\"}," "{\"facet\":true,\"name\":\"cast\",\"optional\":true,\"type\":\"string[]\"}," - "{\"facet\":false,\"name\":\"points\",\"optional\":false,\"type\":\"int32\"}],\"id\":0,\"name\":\"collection1\"}", + "{\"facet\":false,\"name\":\"points\",\"optional\":false,\"type\":\"int32\"}],\"id\":0,\"name\":\"collection1\",\"num_indices\":4}", collection_meta_json); ASSERT_EQ("1", next_collection_id); } @@ -124,7 +124,7 @@ TEST_F(CollectionManagerTest, GetAllCollections) { ASSERT_STREQ("collection1", collection_vec[0]->get_name().c_str()); // try creating one more collection - collectionManager.create_collection("collection2", search_fields, "points"); + collectionManager.create_collection("collection2", 4, search_fields, "points"); collection_vec = collectionManager.get_collections(); ASSERT_EQ(2, collection_vec.size()); @@ -212,7 +212,7 @@ TEST_F(CollectionManagerTest, RestoreRecordsOnRestart) { // create a new collection manager to ensure that it restores the records from the disk backed store CollectionManager & collectionManager2 = CollectionManager::get_instance(); - collectionManager2.init(store, 4, 1.0, "auth_key"); + collectionManager2.init(store, 1.0, "auth_key"); collectionManager2.load(); collection1 = collectionManager2.get_collection("collection1"); @@ -280,7 +280,7 @@ TEST_F(CollectionManagerTest, Symlinking) { std::string state_dir_path = "/tmp/typesense_test/cmanager_test_db"; system(("rm -rf "+state_dir_path+" && mkdir -p "+state_dir_path).c_str()); Store *store = new Store(state_dir_path); - cmanager.init(store, 4, 1.0, "auth_key"); + cmanager.init(store, 1.0, "auth_key"); cmanager.load(); // try resolving on a blank slate @@ -346,7 +346,7 @@ TEST_F(CollectionManagerTest, Symlinking) { // should be able to restore state on init CollectionManager & cmanager2 = CollectionManager::get_instance(); - cmanager2.init(store, 4, 1.0, "auth_key"); + cmanager2.init(store, 1.0, "auth_key"); cmanager2.load(); collection_option = cmanager2.resolve_symlink("company"); diff --git a/test/collection_override_test.cpp b/test/collection_override_test.cpp index ccc9997e..86a1452b 100644 --- a/test/collection_override_test.cpp +++ b/test/collection_override_test.cpp @@ -18,7 +18,7 @@ protected: system(("rm -rf "+state_dir_path+" && mkdir -p "+state_dir_path).c_str()); store = new Store(state_dir_path); - collectionManager.init(store, 4, 1.0, "auth_key"); + collectionManager.init(store, 1.0, "auth_key"); collectionManager.load(); std::ifstream infile(std::string(ROOT_DIR)+"test/multi_field_documents.jsonl"); @@ -31,7 +31,7 @@ protected: coll_mul_fields = collectionManager.get_collection("coll_mul_fields"); if(coll_mul_fields == nullptr) { - coll_mul_fields = collectionManager.create_collection("coll_mul_fields", fields, "points").get(); + coll_mul_fields = collectionManager.create_collection("coll_mul_fields", 4, fields, "points").get(); } std::string json_line; diff --git a/test/collection_sorting_test.cpp b/test/collection_sorting_test.cpp index 8cdb8104..8e101cd3 100644 --- a/test/collection_sorting_test.cpp +++ b/test/collection_sorting_test.cpp @@ -20,7 +20,7 @@ protected: system(("rm -rf "+state_dir_path+" && mkdir -p "+state_dir_path).c_str()); store = new Store(state_dir_path); - collectionManager.init(store, 4, 1.0, "auth_key"); + collectionManager.init(store, 1.0, "auth_key"); collectionManager.load(); } @@ -45,7 +45,7 @@ TEST_F(CollectionSortingTest, SortingOrder) { coll_mul_fields = collectionManager.get_collection("coll_mul_fields"); if(coll_mul_fields == nullptr) { - coll_mul_fields = collectionManager.create_collection("coll_mul_fields", fields, "points").get(); + coll_mul_fields = collectionManager.create_collection("coll_mul_fields", 4, fields, "points").get(); } std::string json_line; @@ -127,7 +127,7 @@ TEST_F(CollectionSortingTest, DefaultSortingFieldValidations) { std::vector sort_fields = { sort_by("age", "DESC"), sort_by("average", "DESC") }; - Option collection_op = collectionManager.create_collection("sample_collection", fields, "name"); + Option collection_op = collectionManager.create_collection("sample_collection", 4, fields, "name"); EXPECT_FALSE(collection_op.ok()); EXPECT_EQ("Default sorting field `name` must be a single valued numerical field.", collection_op.error()); collectionManager.drop_collection("sample_collection"); @@ -135,7 +135,7 @@ TEST_F(CollectionSortingTest, DefaultSortingFieldValidations) { // Default sorting field must exist as a field in schema sort_fields = { sort_by("age", "DESC"), sort_by("average", "DESC") }; - collection_op = collectionManager.create_collection("sample_collection", fields, "NOT-DEFINED"); + collection_op = collectionManager.create_collection("sample_collection", 4, fields, "NOT-DEFINED"); EXPECT_FALSE(collection_op.ok()); EXPECT_EQ("Default sorting field is defined as `NOT-DEFINED` but is not found in the schema.", collection_op.error()); collectionManager.drop_collection("sample_collection"); @@ -152,7 +152,7 @@ TEST_F(CollectionSortingTest, Int64AsDefaultSortingField) { coll_mul_fields = collectionManager.get_collection("coll_mul_fields"); if(coll_mul_fields == nullptr) { - coll_mul_fields = collectionManager.create_collection("coll_mul_fields", fields, "points").get(); + coll_mul_fields = collectionManager.create_collection("coll_mul_fields", 4, fields, "points").get(); } auto doc_str1 = "{\"title\": \"foo\", \"starring\": \"bar\", \"points\": 343234324234233234, \"cast\": [\"baz\"] }"; @@ -211,7 +211,7 @@ TEST_F(CollectionSortingTest, SortOnFloatFields) { coll_float_fields = collectionManager.get_collection("coll_float_fields"); if(coll_float_fields == nullptr) { - coll_float_fields = collectionManager.create_collection("coll_float_fields", fields, "score").get(); + coll_float_fields = collectionManager.create_collection("coll_float_fields", 4, fields, "score").get(); } std::string json_line; @@ -279,7 +279,7 @@ TEST_F(CollectionSortingTest, ThreeSortFieldsLimit) { coll1 = collectionManager.get_collection("coll1"); if(coll1 == nullptr) { - coll1 = collectionManager.create_collection("coll1", fields, "points").get(); + coll1 = collectionManager.create_collection("coll1", 4, fields, "points").get(); } nlohmann::json doc1; diff --git a/test/collection_test.cpp b/test/collection_test.cpp index 571561c3..461da75c 100644 --- a/test/collection_test.cpp +++ b/test/collection_test.cpp @@ -20,7 +20,7 @@ protected: system(("rm -rf "+state_dir_path+" && mkdir -p "+state_dir_path).c_str()); store = new Store(state_dir_path); - collectionManager.init(store, 4, 1.0, "auth_key"); + collectionManager.init(store, 1.0, "auth_key"); collectionManager.load(); std::ifstream infile(std::string(ROOT_DIR)+"test/documents.jsonl"); @@ -34,7 +34,7 @@ protected: collection = collectionManager.get_collection("collection"); if(collection == nullptr) { - collection = collectionManager.create_collection("collection", search_fields, "points").get(); + collection = collectionManager.create_collection("collection", 4, search_fields, "points").get(); } std::string json_line; @@ -581,7 +581,7 @@ TEST_F(CollectionTest, MultiOccurrenceString) { coll_multi_string = collectionManager.get_collection("coll_multi_string"); if (coll_multi_string == nullptr) { - coll_multi_string = collectionManager.create_collection("coll_multi_string", fields, "points").get(); + coll_multi_string = collectionManager.create_collection("coll_multi_string", 4, fields, "points").get(); } nlohmann::json document; @@ -609,7 +609,7 @@ TEST_F(CollectionTest, ArrayStringFieldHighlight) { coll_array_text = collectionManager.get_collection("coll_array_text"); if (coll_array_text == nullptr) { - coll_array_text = collectionManager.create_collection("coll_array_text", fields, "points").get(); + coll_array_text = collectionManager.create_collection("coll_array_text", 4, fields, "points").get(); } std::string json_line; @@ -774,7 +774,7 @@ TEST_F(CollectionTest, MultipleFields) { coll_mul_fields = collectionManager.get_collection("coll_mul_fields"); if(coll_mul_fields == nullptr) { - coll_mul_fields = collectionManager.create_collection("coll_mul_fields", fields, "points").get(); + coll_mul_fields = collectionManager.create_collection("coll_mul_fields", 4, fields, "points").get(); } std::string json_line; @@ -892,7 +892,7 @@ TEST_F(CollectionTest, FilterAndQueryFieldRestrictions) { coll_mul_fields = collectionManager.get_collection("coll_mul_fields"); if(coll_mul_fields == nullptr) { - coll_mul_fields = collectionManager.create_collection("coll_mul_fields", fields, "points").get(); + coll_mul_fields = collectionManager.create_collection("coll_mul_fields", 4, fields, "points").get(); } std::string json_line; @@ -984,12 +984,12 @@ TEST_F(CollectionTest, FilterOnNumericFields) { coll_array_fields = collectionManager.get_collection("coll_array_fields"); if(coll_array_fields == nullptr) { // ensure that default_sorting_field is a non-array numerical field - auto coll_op = collectionManager.create_collection("coll_array_fields", fields, "years"); + auto coll_op = collectionManager.create_collection("coll_array_fields", 4, fields, "years"); ASSERT_EQ(false, coll_op.ok()); ASSERT_STREQ("Default sorting field `years` must be a single valued numerical field.", coll_op.error().c_str()); // let's try again properly - coll_op = collectionManager.create_collection("coll_array_fields", fields, "age"); + coll_op = collectionManager.create_collection("coll_array_fields", 4, fields, "age"); coll_array_fields = coll_op.get(); } @@ -1129,7 +1129,7 @@ TEST_F(CollectionTest, FilterOnFloatFields) { coll_array_fields = collectionManager.get_collection("coll_array_fields"); if(coll_array_fields == nullptr) { - coll_array_fields = collectionManager.create_collection("coll_array_fields", fields, "age").get(); + coll_array_fields = collectionManager.create_collection("coll_array_fields", 4, fields, "age").get(); } std::string json_line; @@ -1284,7 +1284,7 @@ TEST_F(CollectionTest, ImportDocuments) { coll_mul_fields = collectionManager.get_collection("coll_mul_fields"); if(coll_mul_fields == nullptr) { - coll_mul_fields = collectionManager.create_collection("coll_mul_fields", fields, "points").get(); + coll_mul_fields = collectionManager.create_collection("coll_mul_fields", 4, fields, "points").get(); } // try importing records @@ -1393,7 +1393,7 @@ TEST_F(CollectionTest, QueryBoolFields) { coll_bool = collectionManager.get_collection("coll_bool"); if(coll_bool == nullptr) { - coll_bool = collectionManager.create_collection("coll_bool", fields, "rating").get(); + coll_bool = collectionManager.create_collection("coll_bool", 4, fields, "rating").get(); } std::string json_line; @@ -1498,7 +1498,7 @@ TEST_F(CollectionTest, FilterOnTextFields) { coll_array_fields = collectionManager.get_collection("coll_array_fields"); if(coll_array_fields == nullptr) { - coll_array_fields = collectionManager.create_collection("coll_array_fields", fields, "age").get(); + coll_array_fields = collectionManager.create_collection("coll_array_fields", 4, fields, "age").get(); } std::string json_line; @@ -1582,7 +1582,7 @@ TEST_F(CollectionTest, HandleBadlyFormedFilterQuery) { coll_array_fields = collectionManager.get_collection("coll_array_fields"); if(coll_array_fields == nullptr) { - coll_array_fields = collectionManager.create_collection("coll_array_fields", fields, "age").get(); + coll_array_fields = collectionManager.create_collection("coll_array_fields", 4, fields, "age").get(); } std::string json_line; @@ -1638,7 +1638,7 @@ TEST_F(CollectionTest, SearchingWithMissingFields) { coll_array_fields = collectionManager.get_collection("coll_array_fields"); if(coll_array_fields == nullptr) { - coll_array_fields = collectionManager.create_collection("coll_array_fields", fields, "age").get(); + coll_array_fields = collectionManager.create_collection("coll_array_fields", 4, fields, "age").get(); } std::string json_line; @@ -1693,7 +1693,7 @@ TEST_F(CollectionTest, IndexingWithBadData) { sample_collection = collectionManager.get_collection("sample_collection"); if(sample_collection == nullptr) { - sample_collection = collectionManager.create_collection("sample_collection", fields, "age").get(); + sample_collection = collectionManager.create_collection("sample_collection", 4, fields, "age").get(); } const Option & search_fields_missing_op1 = sample_collection->add("{\"namezz\": \"foo\", \"age\": 29, \"average\": 78}"); @@ -1773,7 +1773,7 @@ TEST_F(CollectionTest, EmptyIndexShouldNotCrash) { empty_coll = collectionManager.get_collection("empty_coll"); if(empty_coll == nullptr) { - empty_coll = collectionManager.create_collection("empty_coll", fields, "age").get(); + empty_coll = collectionManager.create_collection("empty_coll", 4, fields, "age").get(); } nlohmann::json results = empty_coll->search("a", {"name"}, "", {}, sort_fields, 0, 10, 1, FREQUENCY, false).get(); @@ -1793,7 +1793,7 @@ TEST_F(CollectionTest, IdFieldShouldBeAString) { coll1 = collectionManager.get_collection("coll1"); if(coll1 == nullptr) { - coll1 = collectionManager.create_collection("coll1", fields, "age").get(); + coll1 = collectionManager.create_collection("coll1", 4, fields, "age").get(); } nlohmann::json doc; @@ -1821,7 +1821,7 @@ TEST_F(CollectionTest, AnIntegerCanBePassedToAFloatField) { coll1 = collectionManager.get_collection("coll1"); if(coll1 == nullptr) { - coll1 = collectionManager.create_collection("coll1", fields, "average").get(); + coll1 = collectionManager.create_collection("coll1", 4, fields, "average").get(); } nlohmann::json doc; @@ -1849,7 +1849,7 @@ TEST_F(CollectionTest, DeletionOfADocument) { Collection *collection_for_del; collection_for_del = collectionManager.get_collection("collection_for_del"); if(collection_for_del == nullptr) { - collection_for_del = collectionManager.create_collection("collection_for_del", search_fields, "points").get(); + collection_for_del = collectionManager.create_collection("collection_for_del", 4, search_fields, "points").get(); } std::string json_line; @@ -1930,7 +1930,7 @@ TEST_F(CollectionTest, DeletionOfDocumentArrayFields) { coll1 = collectionManager.get_collection("coll1"); if(coll1 == nullptr) { - coll1 = collectionManager.create_collection("coll1", fields, "points").get(); + coll1 = collectionManager.create_collection("coll1", 4, fields, "points").get(); } nlohmann::json doc; @@ -2001,7 +2001,7 @@ TEST_F(CollectionTest, SearchLargeTextField) { coll_large_text = collectionManager.get_collection("coll_large_text"); if(coll_large_text == nullptr) { - coll_large_text = collectionManager.create_collection("coll_large_text", fields, "age").get(); + coll_large_text = collectionManager.create_collection("coll_large_text", 4, fields, "age").get(); } std::string json_line; @@ -2083,7 +2083,7 @@ TEST_F(CollectionTest, StringArrayFieldShouldNotAllowPlainString) { coll1 = collectionManager.get_collection("coll1"); if (coll1 == nullptr) { - coll1 = collectionManager.create_collection("coll1", fields, "points").get(); + coll1 = collectionManager.create_collection("coll1", 4, fields, "points").get(); } nlohmann::json doc; @@ -2108,7 +2108,7 @@ TEST_F(CollectionTest, SearchHighlightShouldFollowThreshold) { coll1 = collectionManager.get_collection("coll1"); if (coll1 == nullptr) { - coll1 = collectionManager.create_collection("coll1", fields, "points").get(); + coll1 = collectionManager.create_collection("coll1", 4, fields, "points").get(); } nlohmann::json doc; @@ -2151,7 +2151,7 @@ TEST_F(CollectionTest, SearchHighlightFieldFully) { coll1 = collectionManager.get_collection("coll1"); if (coll1 == nullptr) { - coll1 = collectionManager.create_collection("coll1", fields, "points").get(); + coll1 = collectionManager.create_collection("coll1", 4, fields, "points").get(); } nlohmann::json doc; @@ -2226,7 +2226,7 @@ TEST_F(CollectionTest, OptionalFields) { coll1 = collectionManager.get_collection("coll1"); if(coll1 == nullptr) { - coll1 = collectionManager.create_collection("coll1", fields, "max").get(); + coll1 = collectionManager.create_collection("coll1", 4, fields, "max").get(); } std::ifstream infile(std::string(ROOT_DIR)+"test/optional_fields.jsonl"); @@ -2290,7 +2290,7 @@ TEST_F(CollectionTest, OptionalFields) { field("score", field_types::INT32, false, true), }; - auto create_op = collectionManager.create_collection("coll2", fields, "score"); + auto create_op = collectionManager.create_collection("coll2", 4, fields, "score"); ASSERT_FALSE(create_op.ok()); ASSERT_STREQ("Default sorting field `score` cannot be an optional field.", create_op.error().c_str());