Parameterize number of indices per collection.

This commit is contained in:
kishorenc 2020-08-19 07:28:21 +05:30
parent 80a3e0fb87
commit 09541e5311
18 changed files with 156 additions and 120 deletions

View File

@ -29,12 +29,11 @@ private:
static constexpr const char* COLLECTION_SEARCH_FIELDS_KEY = "fields";
static constexpr const char* COLLECTION_DEFAULT_SORTING_FIELD_KEY = "default_sorting_field";
static constexpr const char* COLLECTION_CREATED = "created_at";
static constexpr const char* COLLECTION_NUM_INDICES = "num_indices";
std::string bootstrap_auth_key;
std::string bootstrap_search_only_auth_key;
size_t default_num_indices;
float max_memory_ratio;
CollectionManager();
@ -60,8 +59,7 @@ public:
CollectionManager(CollectionManager const&) = delete;
void operator=(CollectionManager const&) = delete;
void init(Store *store, const size_t default_num_indices, const float max_memory_ratio,
const std::string & auth_key);
void init(Store *store, const float max_memory_ratio, const std::string & auth_key);
Option<bool> load(const size_t init_batch_size=1000);
@ -75,7 +73,8 @@ public:
bool auth_key_matches(const std::string& auth_key_sent, const std::string& action,
const std::string& collection, std::map<std::string, std::string>& params);
Option<Collection*> create_collection(const std::string name, const std::vector<field> & fields,
Option<Collection*> create_collection(const std::string name, const size_t num_indices,
const std::vector<field> & fields,
const std::string & default_sorting_field,
const uint64_t created_at = static_cast<uint64_t>(std::time(nullptr)));
@ -106,6 +105,8 @@ public:
Option<bool> delete_symlink(const std::string & symlink_name);
static const size_t DEFAULT_NUM_INDICES = 4;
static constexpr const char* NEXT_COLLECTION_ID_KEY = "$CI";
static constexpr const char* SYMLINK_PREFIX = "$SL";
};

View File

@ -29,8 +29,6 @@ private:
bool enable_cors;
size_t indices_per_collection;
float max_memory_ratio;
std::string config_file;
@ -43,7 +41,6 @@ public:
this->api_port = 8108;
this->peering_port = 8107;
this->enable_cors = false;
this->indices_per_collection = 4;
this->max_memory_ratio = 1.0f;
}
@ -90,10 +87,6 @@ public:
this->enable_cors = enable_cors;
}
void set_indices_per_collection(size_t indices_per_collection) {
this->indices_per_collection = indices_per_collection;
}
// getters
std::string get_data_dir() const {
@ -141,10 +134,6 @@ public:
return this->enable_cors;
}
size_t get_indices_per_collection() const {
return indices_per_collection;
}
std::string get_peering_address() const {
return this->peering_address;
}

View File

@ -69,6 +69,6 @@ bool get_metrics_json(http_req& req, http_res& res);
// Misc helpers
bool async_write_request(void *data);
bool raft_write_send_response(void *data);
static constexpr const char* SEND_RESPONSE_MSG = "send_response";

View File

@ -15,7 +15,7 @@ extern "C" {
}
struct h2o_custom_timer_t {
h2o_timer_t timer{};
h2o_timer_t timer;
void *data;
h2o_custom_timer_t(): data(nullptr) {}
@ -182,6 +182,14 @@ struct http_req {
return content.dump();
}
bool is_ending() {
return stream_state == "END" || stream_state == "NON_STREAMING";
}
bool is_starting() {
return stream_state == "START" || stream_state == "NON_STREAMING";
}
};
struct request_response {

View File

@ -127,6 +127,8 @@ public:
void send_response(http_req* request, http_res* response);
static void destroy_request_response(http_req* request, http_res* response);
static void stream_response(http_req& request, http_res& response);
uint64_t find_route(const std::vector<std::string> & path_parts, const std::string & http_method,

View File

@ -132,6 +132,7 @@ nlohmann::json Collection::get_summary_json() {
nlohmann::json json_response;
json_response["name"] = name;
json_response["num_indices"] = num_indices;
json_response["num_documents"] = num_documents;
json_response["created_at"] = created_at;

View File

@ -4,6 +4,9 @@
#include "collection_manager.h"
#include "logger.h"
constexpr const char* CollectionManager::COLLECTION_NUM_INDICES;
constexpr const size_t CollectionManager::DEFAULT_NUM_INDICES;
CollectionManager::CollectionManager() {
}
@ -31,6 +34,12 @@ Collection* CollectionManager::init_collection(const nlohmann::json & collection
uint64_t created_at = collection_meta.find((const char*)COLLECTION_CREATED) != collection_meta.end() ?
collection_meta[COLLECTION_CREATED].get<uint64_t>() : 0;
size_t num_indices = collection_meta.count(COLLECTION_NUM_INDICES) != 0 ?
collection_meta[COLLECTION_NUM_INDICES].get<size_t>() :
DEFAULT_NUM_INDICES;
LOG(INFO) << "Loading collection with " << num_indices << " indices.";
Collection* collection = new Collection(this_collection_name,
collection_meta[COLLECTION_ID_KEY].get<uint32_t>(),
created_at,
@ -38,7 +47,7 @@ Collection* CollectionManager::init_collection(const nlohmann::json & collection
store,
fields,
default_sorting_field,
default_num_indices,
num_indices,
max_memory_ratio);
return collection;
@ -50,12 +59,10 @@ void CollectionManager::add_to_collections(Collection* collection) {
}
void CollectionManager::init(Store *store,
const size_t default_num_indices,
const float max_memory_ratio,
const std::string & auth_key) {
this->store = store;
this->bootstrap_auth_key = auth_key;
this->default_num_indices = default_num_indices;
this->max_memory_ratio = max_memory_ratio;
auth_manager.init(store);
@ -227,7 +234,9 @@ bool CollectionManager::auth_key_matches(const std::string& auth_key_sent,
return auth_manager.authenticate(auth_key_sent, action, collection, params);
}
Option<Collection*> CollectionManager::create_collection(const std::string name, const std::vector<field> & fields,
Option<Collection*> CollectionManager::create_collection(const std::string name,
const size_t num_indices,
const std::vector<field> & fields,
const std::string & default_sorting_field,
const uint64_t created_at) {
if(store->contains(Collection::get_meta_key(name))) {
@ -273,9 +282,10 @@ Option<Collection*> CollectionManager::create_collection(const std::string name,
collection_meta[COLLECTION_SEARCH_FIELDS_KEY] = fields_json;
collection_meta[COLLECTION_DEFAULT_SORTING_FIELD_KEY] = default_sorting_field;
collection_meta[COLLECTION_CREATED] = created_at;
collection_meta[COLLECTION_NUM_INDICES] = num_indices;
Collection* new_collection = new Collection(name, next_collection_id, created_at, 0, store, fields,
default_sorting_field, this->default_num_indices,
default_sorting_field, num_indices,
this->max_memory_ratio);
next_collection_id++;

View File

@ -42,6 +42,7 @@ bool get_collections(http_req & req, http_res & res) {
}
bool post_create_collection(http_req & req, http_res & res) {
const char* NUM_INDICES = "num_indices";
nlohmann::json req_json;
try {
@ -61,6 +62,10 @@ bool post_create_collection(http_req & req, http_res & res) {
return false;
}
if(req_json.count(NUM_INDICES) == 0) {
req_json[NUM_INDICES] = CollectionManager::DEFAULT_NUM_INDICES;
}
if(req_json.count("fields") == 0) {
res.set_400("Parameter `fields` is required.");
return false;
@ -79,6 +84,16 @@ bool post_create_collection(http_req & req, http_res & res) {
return false;
}
if(!req_json[NUM_INDICES].is_number_unsigned()) {
res.set_400(std::string("`") + NUM_INDICES + "` should be a positive integer.");
return false;
}
if(req_json[NUM_INDICES].get<size_t>() == 0) {
res.set_400(std::string("`") + NUM_INDICES + "` should be a positive integer.");
return false;
}
if(collectionManager.get_collection(req_json["name"]) != nullptr) {
res.set_409("Collection with name `" + req_json["name"].get<std::string>() + "` already exists.");
return false;
@ -125,7 +140,8 @@ bool post_create_collection(http_req & req, http_res & res) {
const std::string & default_sorting_field = req_json[DEFAULT_SORTING_FIELD].get<std::string>();
const Option<Collection*> & collection_op =
collectionManager.create_collection(req_json["name"], fields, default_sorting_field);
collectionManager.create_collection(req_json["name"], req_json[NUM_INDICES].get<size_t>(),
fields, default_sorting_field);
if(collection_op.ok()) {
nlohmann::json json_response = collection_op.get()->get_summary_json();
@ -571,14 +587,21 @@ bool post_import_documents(http_req& req, http_res& res) {
if(!StringUtils::is_uint32_t(req.params[BATCH_SIZE])) {
req.stream_state = "NON_STREAMING";
res.set_400("Parameter `" + std::string(BATCH_SIZE) + "` must be an unsigned integer.");
res.set_400("Parameter `" + std::string(BATCH_SIZE) + "` must be a positive integer.");
HttpServer::stream_response(req, res);
return false;
}
const size_t IMPORT_BATCH_SIZE = std::stoi(req.params[BATCH_SIZE]);
LOG(INFO) << "Import batch size: " << IMPORT_BATCH_SIZE;
if(IMPORT_BATCH_SIZE == 0) {
res.set_400("Parameter `" + std::string(BATCH_SIZE) + "` must be a positive integer.");
return false;
}
if(req.stream_state == "START" && req.body_index == 0) {
LOG(INFO) << "Import batch size: " << IMPORT_BATCH_SIZE;
}
CollectionManager & collectionManager = CollectionManager::get_instance();
Collection* collection = collectionManager.get_collection(req.params["collection"]);
@ -1036,8 +1059,8 @@ bool del_key(http_req &req, http_res &res) {
return true;
}
bool async_write_request(void *data) {
//LOG(INFO) << "async_write_request called";
bool raft_write_send_response(void *data) {
//LOG(INFO) << "raft_write_send_response called";
AsyncIndexArg* index_arg = static_cast<AsyncIndexArg*>(data);
std::unique_ptr<AsyncIndexArg> index_arg_guard(index_arg);
@ -1047,10 +1070,10 @@ bool async_write_request(void *data) {
// route not found
index_arg->res->set_400("Not found.");
} else if(index_arg->req->route_hash != static_cast<uint64_t>(ROUTE_CODES::ALREADY_HANDLED)) {
// call the underlying http handler
route_path* found_rpath = nullptr;
bool route_found = server->get_route(index_arg->req->route_hash, &found_rpath);
if(route_found) {
// call request handler
found_rpath->handler(*index_arg->req, *index_arg->res);
async_res = found_rpath->async_res;
} else {
@ -1058,17 +1081,11 @@ bool async_write_request(void *data) {
}
}
if(!async_res && index_arg->req->_req != nullptr) {
// we have to return a response to the client
if(!async_res) {
// only handle synchronous responses as async ones are handled by their handlers
server->send_response(index_arg->req, index_arg->res);
}
if(index_arg->req->_req == nullptr) {
// indicates raft serialized request and response -- lifecycle must be managed here
delete index_arg->req;
delete index_arg->res;
}
if(index_arg->promise != nullptr) {
index_arg->promise->set_value(true); // returns control back to caller
}

View File

@ -71,7 +71,6 @@ void HttpServer::on_ssl_refresh_timeout(h2o_timer_t *entry) {
int HttpServer::setup_ssl(const char *cert_file, const char *key_file) {
// Set up a timer to refresh SSL config from disk. Also, initializing upfront so that destructor works
ssl_refresh_timer = h2o_custom_timer_t(this);
ssl_refresh_timer.timer.expire_at = UINT64_MAX-1;
h2o_timer_init(&ssl_refresh_timer.timer, on_ssl_refresh_timeout);
h2o_timer_link(ctx.loop, SSL_REFRESH_INTERVAL_MS, &ssl_refresh_timer.timer); // every 8 hours
@ -278,14 +277,7 @@ void HttpServer::on_res_generator_dispose(void *self) {
res_generator->rpath->handler(*res_generator->request, *res_generator->response);
}
if(res_generator->request->defer_timer.data != nullptr) {
deferred_req_res_t* deferred_req_res = static_cast<deferred_req_res_t*>(res_generator->request->defer_timer.data);
delete deferred_req_res;
}
// res_generator itself is reference counted, so we only delete the members
delete res_generator->request;
delete res_generator->response;
destroy_request_response(res_generator->request, res_generator->response);
}
int HttpServer::catch_all_handler(h2o_handler_t *_h2o_handler, h2o_req_t *req) {
@ -509,24 +501,15 @@ void HttpServer::defer_processing(http_req& req, http_res& res, size_t timeout_m
if(req.defer_timer.data == nullptr) {
auto deferred_req_res = new deferred_req_res_t{&req, &res, this};
req.defer_timer.data = deferred_req_res;
req.defer_timer.timer.expire_at = UINT64_MAX-1;
h2o_timer_init(&req.defer_timer.timer, on_deferred_process_request);
}
h2o_timer_unlink(&req.defer_timer.timer);
h2o_timer_link(ctx.loop, timeout_ms, &req.defer_timer.timer);
}
void HttpServer::send_response(http_req* request, http_res* response) {
h2o_req_t* req = request->_req;
h2o_generator_t& generator = *response->generator;
h2o_iovec_t body = h2o_strdup(&req->pool, response->body.c_str(), SIZE_MAX);
req->res.status = response->status_code;
req->res.reason = http_res::get_status_reason(response->status_code);
h2o_add_header(&req->pool, &req->res.headers, H2O_TOKEN_CONTENT_TYPE,
nullptr, response->content_type_header.c_str(), response->content_type_header.size());
h2o_start_response(req, &generator);
h2o_send(req, &body, 1, H2O_SEND_STATE_FINAL);
void HttpServer::send_message(const std::string & type, void* data) {
message_dispatcher->send_message(type, data);
}
int HttpServer::send_response(h2o_req_t *req, int status_code, const std::string & message) {
@ -540,8 +523,22 @@ int HttpServer::send_response(h2o_req_t *req, int status_code, const std::string
return 0;
}
void HttpServer::send_message(const std::string & type, void* data) {
message_dispatcher->send_message(type, data);
void HttpServer::send_response(http_req* request, http_res* response) {
if(request->_req == nullptr) {
// indicates serialized request and response -- lifecycle must be managed here
return destroy_request_response(request, response);
}
h2o_req_t* req = request->_req;
h2o_generator_t& generator = *response->generator;
h2o_iovec_t body = h2o_strdup(&req->pool, response->body.c_str(), SIZE_MAX);
req->res.status = response->status_code;
req->res.reason = http_res::get_status_reason(response->status_code);
h2o_add_header(&req->pool, &req->res.headers, H2O_TOKEN_CONTENT_TYPE,
nullptr, response->content_type_header.c_str(), response->content_type_header.size());
h2o_start_response(req, &generator);
h2o_send(req, &body, 1, H2O_SEND_STATE_FINAL);
}
void HttpServer::response_proceed(h2o_generator_t *generator, h2o_req_t *req) {
@ -564,9 +561,10 @@ void HttpServer::response_proceed(h2o_generator_t *generator, h2o_req_t *req) {
}
void HttpServer::stream_response(http_req& request, http_res& response) {
if(!request._req) {
// underlying request is no longer available
return ;
if(request._req == nullptr) {
// serialized request or underlying request is no longer available
destroy_request_response(&request, &response);
return;
}
h2o_req_t* req = request._req;
@ -590,6 +588,16 @@ void HttpServer::stream_response(http_req& request, http_res& response) {
h2o_send(req, &body, 1, state);
}
void HttpServer::destroy_request_response(http_req* request, http_res* response) {
if(request->defer_timer.data != nullptr) {
deferred_req_res_t* deferred_req_res = static_cast<deferred_req_res_t*>(request->defer_timer.data);
delete deferred_req_res;
}
delete request;
delete response;
}
void HttpServer::set_auth_handler(bool (*handler)(std::map<std::string, std::string>& params, const route_path& rpath,
const std::string& auth_key)) {
auth_handler = handler;
@ -675,3 +683,4 @@ bool HttpServer::get_route(uint64_t hash, route_path** found_rpath) {
uint64_t HttpServer::node_state() const {
return replication_state->node_state();
}

View File

@ -35,7 +35,7 @@ int main(int argc, char* argv[]) {
Collection *collection = collectionManager.get_collection("github_top1k");
if(collection == nullptr) {
collection = collectionManager.create_collection("github_top1k", fields_to_index, "stars").get();
collection = collectionManager.create_collection("github_top1k", 4, fields_to_index, "stars").get();
}
int j = 0;

View File

@ -92,6 +92,7 @@ std::string ReplicationState::to_nodes_config(const butil::EndPoint& peering_end
}
void ReplicationState::write(http_req* request, http_res* response) {
// NOTE: this is executed on a different thread and runs concurrent to http thread
if (!is_leader()) {
if(node->leader_id().is_empty()) {
// Handle no leader scenario
@ -187,11 +188,10 @@ void ReplicationState::on_apply(braft::Iterator& iter) {
} else {
// Parse request from the log
response = new http_res;
http_req* remote_request = new http_req;
remote_request->deserialize(iter.data().to_string());
remote_request->_req = nullptr; // indicates remote request
request = remote_request;
request = new http_req;
request->deserialize(iter.data().to_string());
request->_req = nullptr; // indicates remote request
}
if(request->_req == nullptr && request->body == "INIT_SNAPSHOT") {

View File

@ -329,8 +329,7 @@ int run_server(const Config & config, const std::string & version, void (*master
Store store(db_dir);
CollectionManager & collectionManager = CollectionManager::get_instance();
collectionManager.init(&store, config.get_indices_per_collection(),
config.get_max_memory_ratio(), config.get_api_key());
collectionManager.init(&store, config.get_max_memory_ratio(), config.get_api_key());
curl_global_init(CURL_GLOBAL_SSL);
HttpClient & httpClient = HttpClient::get_instance();
@ -348,7 +347,7 @@ int run_server(const Config & config, const std::string & version, void (*master
server->set_auth_handler(handle_authentication);
server->on(SEND_RESPONSE_MSG, on_send_response);
server->on(ReplicationState::REPLICATION_MSG, async_write_request);
server->on(ReplicationState::REPLICATION_MSG, raft_write_send_response);
// first we start the peering service

View File

@ -20,7 +20,7 @@ protected:
system(("rm -rf "+state_dir_path+" && mkdir -p "+state_dir_path).c_str());
store = new Store(state_dir_path);
collectionManager.init(store, 4, 1.0, "auth_key");
collectionManager.init(store, 1.0, "auth_key");
collectionManager.load();
}
@ -50,7 +50,7 @@ TEST_F(CollectionFacetingTest, FacetCounts) {
coll_array_fields = collectionManager.get_collection("coll_array_fields");
if(coll_array_fields == nullptr) {
coll_array_fields = collectionManager.create_collection("coll_array_fields", fields, "age").get();
coll_array_fields = collectionManager.create_collection("coll_array_fields", 4, fields, "age").get();
}
std::string json_line;
@ -336,7 +336,7 @@ TEST_F(CollectionFacetingTest, FacetCountsBool) {
coll1 = collectionManager.get_collection("coll1");
if (coll1 == nullptr) {
coll1 = collectionManager.create_collection("coll1", fields, "points").get();
coll1 = collectionManager.create_collection("coll1", 4, fields, "points").get();
}
nlohmann::json doc;
@ -390,7 +390,7 @@ TEST_F(CollectionFacetingTest, FacetCountsHighlighting) {
coll1 = collectionManager.get_collection("coll1");
if(coll1 == nullptr) {
coll1 = collectionManager.create_collection("coll1", fields, "points").get();
coll1 = collectionManager.create_collection("coll1", 4, fields, "points").get();
}
nlohmann::json doc;
@ -507,7 +507,7 @@ TEST_F(CollectionFacetingTest, FacetStatOnFloatFields) {
coll_float_fields = collectionManager.get_collection("coll_float_fields");
if(coll_float_fields == nullptr) {
coll_float_fields = collectionManager.create_collection("coll_float_fields", fields, "average").get();
coll_float_fields = collectionManager.create_collection("coll_float_fields", 4, fields, "average").get();
}
std::string json_line;

View File

@ -18,7 +18,7 @@ protected:
system(("rm -rf "+state_dir_path+" && mkdir -p "+state_dir_path).c_str());
store = new Store(state_dir_path);
collectionManager.init(store, 4, 1.0, "auth_key");
collectionManager.init(store, 1.0, "auth_key");
collectionManager.load();
std::vector<field> fields = {
@ -31,7 +31,7 @@ protected:
coll_group = collectionManager.get_collection("coll_group");
if(coll_group == nullptr) {
coll_group = collectionManager.create_collection("coll_group", fields, "rating").get();
coll_group = collectionManager.create_collection("coll_group", 4, fields, "rating").get();
}
std::ifstream infile(std::string(ROOT_DIR)+"test/group_documents.jsonl");

View File

@ -20,7 +20,7 @@ protected:
system(("rm -rf "+state_dir_path+" && mkdir -p "+state_dir_path).c_str());
store = new Store(state_dir_path);
collectionManager.init(store, 4, 1.0, "auth_key");
collectionManager.init(store, 1.0, "auth_key");
collectionManager.load();
search_fields = {
@ -31,7 +31,7 @@ protected:
};
sort_fields = { sort_by("points", "DESC") };
collection1 = collectionManager.create_collection("collection1", search_fields, "points", 12345).get();
collection1 = collectionManager.create_collection("collection1", 4, search_fields, "points", 12345).get();
}
virtual void SetUp() {
@ -86,7 +86,7 @@ TEST_F(CollectionManagerTest, CollectionCreation) {
"\"fields\":[{\"facet\":false,\"name\":\"title\",\"optional\":false,\"type\":\"string\"},"
"{\"facet\":false,\"name\":\"starring\",\"optional\":false,\"type\":\"string\"},"
"{\"facet\":true,\"name\":\"cast\",\"optional\":true,\"type\":\"string[]\"},"
"{\"facet\":false,\"name\":\"points\",\"optional\":false,\"type\":\"int32\"}],\"id\":0,\"name\":\"collection1\"}",
"{\"facet\":false,\"name\":\"points\",\"optional\":false,\"type\":\"int32\"}],\"id\":0,\"name\":\"collection1\",\"num_indices\":4}",
collection_meta_json);
ASSERT_EQ("1", next_collection_id);
}
@ -124,7 +124,7 @@ TEST_F(CollectionManagerTest, GetAllCollections) {
ASSERT_STREQ("collection1", collection_vec[0]->get_name().c_str());
// try creating one more collection
collectionManager.create_collection("collection2", search_fields, "points");
collectionManager.create_collection("collection2", 4, search_fields, "points");
collection_vec = collectionManager.get_collections();
ASSERT_EQ(2, collection_vec.size());
@ -212,7 +212,7 @@ TEST_F(CollectionManagerTest, RestoreRecordsOnRestart) {
// create a new collection manager to ensure that it restores the records from the disk backed store
CollectionManager & collectionManager2 = CollectionManager::get_instance();
collectionManager2.init(store, 4, 1.0, "auth_key");
collectionManager2.init(store, 1.0, "auth_key");
collectionManager2.load();
collection1 = collectionManager2.get_collection("collection1");
@ -280,7 +280,7 @@ TEST_F(CollectionManagerTest, Symlinking) {
std::string state_dir_path = "/tmp/typesense_test/cmanager_test_db";
system(("rm -rf "+state_dir_path+" && mkdir -p "+state_dir_path).c_str());
Store *store = new Store(state_dir_path);
cmanager.init(store, 4, 1.0, "auth_key");
cmanager.init(store, 1.0, "auth_key");
cmanager.load();
// try resolving on a blank slate
@ -346,7 +346,7 @@ TEST_F(CollectionManagerTest, Symlinking) {
// should be able to restore state on init
CollectionManager & cmanager2 = CollectionManager::get_instance();
cmanager2.init(store, 4, 1.0, "auth_key");
cmanager2.init(store, 1.0, "auth_key");
cmanager2.load();
collection_option = cmanager2.resolve_symlink("company");

View File

@ -18,7 +18,7 @@ protected:
system(("rm -rf "+state_dir_path+" && mkdir -p "+state_dir_path).c_str());
store = new Store(state_dir_path);
collectionManager.init(store, 4, 1.0, "auth_key");
collectionManager.init(store, 1.0, "auth_key");
collectionManager.load();
std::ifstream infile(std::string(ROOT_DIR)+"test/multi_field_documents.jsonl");
@ -31,7 +31,7 @@ protected:
coll_mul_fields = collectionManager.get_collection("coll_mul_fields");
if(coll_mul_fields == nullptr) {
coll_mul_fields = collectionManager.create_collection("coll_mul_fields", fields, "points").get();
coll_mul_fields = collectionManager.create_collection("coll_mul_fields", 4, fields, "points").get();
}
std::string json_line;

View File

@ -20,7 +20,7 @@ protected:
system(("rm -rf "+state_dir_path+" && mkdir -p "+state_dir_path).c_str());
store = new Store(state_dir_path);
collectionManager.init(store, 4, 1.0, "auth_key");
collectionManager.init(store, 1.0, "auth_key");
collectionManager.load();
}
@ -45,7 +45,7 @@ TEST_F(CollectionSortingTest, SortingOrder) {
coll_mul_fields = collectionManager.get_collection("coll_mul_fields");
if(coll_mul_fields == nullptr) {
coll_mul_fields = collectionManager.create_collection("coll_mul_fields", fields, "points").get();
coll_mul_fields = collectionManager.create_collection("coll_mul_fields", 4, fields, "points").get();
}
std::string json_line;
@ -127,7 +127,7 @@ TEST_F(CollectionSortingTest, DefaultSortingFieldValidations) {
std::vector<sort_by> sort_fields = { sort_by("age", "DESC"), sort_by("average", "DESC") };
Option<Collection*> collection_op = collectionManager.create_collection("sample_collection", fields, "name");
Option<Collection*> collection_op = collectionManager.create_collection("sample_collection", 4, fields, "name");
EXPECT_FALSE(collection_op.ok());
EXPECT_EQ("Default sorting field `name` must be a single valued numerical field.", collection_op.error());
collectionManager.drop_collection("sample_collection");
@ -135,7 +135,7 @@ TEST_F(CollectionSortingTest, DefaultSortingFieldValidations) {
// Default sorting field must exist as a field in schema
sort_fields = { sort_by("age", "DESC"), sort_by("average", "DESC") };
collection_op = collectionManager.create_collection("sample_collection", fields, "NOT-DEFINED");
collection_op = collectionManager.create_collection("sample_collection", 4, fields, "NOT-DEFINED");
EXPECT_FALSE(collection_op.ok());
EXPECT_EQ("Default sorting field is defined as `NOT-DEFINED` but is not found in the schema.", collection_op.error());
collectionManager.drop_collection("sample_collection");
@ -152,7 +152,7 @@ TEST_F(CollectionSortingTest, Int64AsDefaultSortingField) {
coll_mul_fields = collectionManager.get_collection("coll_mul_fields");
if(coll_mul_fields == nullptr) {
coll_mul_fields = collectionManager.create_collection("coll_mul_fields", fields, "points").get();
coll_mul_fields = collectionManager.create_collection("coll_mul_fields", 4, fields, "points").get();
}
auto doc_str1 = "{\"title\": \"foo\", \"starring\": \"bar\", \"points\": 343234324234233234, \"cast\": [\"baz\"] }";
@ -211,7 +211,7 @@ TEST_F(CollectionSortingTest, SortOnFloatFields) {
coll_float_fields = collectionManager.get_collection("coll_float_fields");
if(coll_float_fields == nullptr) {
coll_float_fields = collectionManager.create_collection("coll_float_fields", fields, "score").get();
coll_float_fields = collectionManager.create_collection("coll_float_fields", 4, fields, "score").get();
}
std::string json_line;
@ -279,7 +279,7 @@ TEST_F(CollectionSortingTest, ThreeSortFieldsLimit) {
coll1 = collectionManager.get_collection("coll1");
if(coll1 == nullptr) {
coll1 = collectionManager.create_collection("coll1", fields, "points").get();
coll1 = collectionManager.create_collection("coll1", 4, fields, "points").get();
}
nlohmann::json doc1;

View File

@ -20,7 +20,7 @@ protected:
system(("rm -rf "+state_dir_path+" && mkdir -p "+state_dir_path).c_str());
store = new Store(state_dir_path);
collectionManager.init(store, 4, 1.0, "auth_key");
collectionManager.init(store, 1.0, "auth_key");
collectionManager.load();
std::ifstream infile(std::string(ROOT_DIR)+"test/documents.jsonl");
@ -34,7 +34,7 @@ protected:
collection = collectionManager.get_collection("collection");
if(collection == nullptr) {
collection = collectionManager.create_collection("collection", search_fields, "points").get();
collection = collectionManager.create_collection("collection", 4, search_fields, "points").get();
}
std::string json_line;
@ -581,7 +581,7 @@ TEST_F(CollectionTest, MultiOccurrenceString) {
coll_multi_string = collectionManager.get_collection("coll_multi_string");
if (coll_multi_string == nullptr) {
coll_multi_string = collectionManager.create_collection("coll_multi_string", fields, "points").get();
coll_multi_string = collectionManager.create_collection("coll_multi_string", 4, fields, "points").get();
}
nlohmann::json document;
@ -609,7 +609,7 @@ TEST_F(CollectionTest, ArrayStringFieldHighlight) {
coll_array_text = collectionManager.get_collection("coll_array_text");
if (coll_array_text == nullptr) {
coll_array_text = collectionManager.create_collection("coll_array_text", fields, "points").get();
coll_array_text = collectionManager.create_collection("coll_array_text", 4, fields, "points").get();
}
std::string json_line;
@ -774,7 +774,7 @@ TEST_F(CollectionTest, MultipleFields) {
coll_mul_fields = collectionManager.get_collection("coll_mul_fields");
if(coll_mul_fields == nullptr) {
coll_mul_fields = collectionManager.create_collection("coll_mul_fields", fields, "points").get();
coll_mul_fields = collectionManager.create_collection("coll_mul_fields", 4, fields, "points").get();
}
std::string json_line;
@ -892,7 +892,7 @@ TEST_F(CollectionTest, FilterAndQueryFieldRestrictions) {
coll_mul_fields = collectionManager.get_collection("coll_mul_fields");
if(coll_mul_fields == nullptr) {
coll_mul_fields = collectionManager.create_collection("coll_mul_fields", fields, "points").get();
coll_mul_fields = collectionManager.create_collection("coll_mul_fields", 4, fields, "points").get();
}
std::string json_line;
@ -984,12 +984,12 @@ TEST_F(CollectionTest, FilterOnNumericFields) {
coll_array_fields = collectionManager.get_collection("coll_array_fields");
if(coll_array_fields == nullptr) {
// ensure that default_sorting_field is a non-array numerical field
auto coll_op = collectionManager.create_collection("coll_array_fields", fields, "years");
auto coll_op = collectionManager.create_collection("coll_array_fields", 4, fields, "years");
ASSERT_EQ(false, coll_op.ok());
ASSERT_STREQ("Default sorting field `years` must be a single valued numerical field.", coll_op.error().c_str());
// let's try again properly
coll_op = collectionManager.create_collection("coll_array_fields", fields, "age");
coll_op = collectionManager.create_collection("coll_array_fields", 4, fields, "age");
coll_array_fields = coll_op.get();
}
@ -1129,7 +1129,7 @@ TEST_F(CollectionTest, FilterOnFloatFields) {
coll_array_fields = collectionManager.get_collection("coll_array_fields");
if(coll_array_fields == nullptr) {
coll_array_fields = collectionManager.create_collection("coll_array_fields", fields, "age").get();
coll_array_fields = collectionManager.create_collection("coll_array_fields", 4, fields, "age").get();
}
std::string json_line;
@ -1284,7 +1284,7 @@ TEST_F(CollectionTest, ImportDocuments) {
coll_mul_fields = collectionManager.get_collection("coll_mul_fields");
if(coll_mul_fields == nullptr) {
coll_mul_fields = collectionManager.create_collection("coll_mul_fields", fields, "points").get();
coll_mul_fields = collectionManager.create_collection("coll_mul_fields", 4, fields, "points").get();
}
// try importing records
@ -1393,7 +1393,7 @@ TEST_F(CollectionTest, QueryBoolFields) {
coll_bool = collectionManager.get_collection("coll_bool");
if(coll_bool == nullptr) {
coll_bool = collectionManager.create_collection("coll_bool", fields, "rating").get();
coll_bool = collectionManager.create_collection("coll_bool", 4, fields, "rating").get();
}
std::string json_line;
@ -1498,7 +1498,7 @@ TEST_F(CollectionTest, FilterOnTextFields) {
coll_array_fields = collectionManager.get_collection("coll_array_fields");
if(coll_array_fields == nullptr) {
coll_array_fields = collectionManager.create_collection("coll_array_fields", fields, "age").get();
coll_array_fields = collectionManager.create_collection("coll_array_fields", 4, fields, "age").get();
}
std::string json_line;
@ -1582,7 +1582,7 @@ TEST_F(CollectionTest, HandleBadlyFormedFilterQuery) {
coll_array_fields = collectionManager.get_collection("coll_array_fields");
if(coll_array_fields == nullptr) {
coll_array_fields = collectionManager.create_collection("coll_array_fields", fields, "age").get();
coll_array_fields = collectionManager.create_collection("coll_array_fields", 4, fields, "age").get();
}
std::string json_line;
@ -1638,7 +1638,7 @@ TEST_F(CollectionTest, SearchingWithMissingFields) {
coll_array_fields = collectionManager.get_collection("coll_array_fields");
if(coll_array_fields == nullptr) {
coll_array_fields = collectionManager.create_collection("coll_array_fields", fields, "age").get();
coll_array_fields = collectionManager.create_collection("coll_array_fields", 4, fields, "age").get();
}
std::string json_line;
@ -1693,7 +1693,7 @@ TEST_F(CollectionTest, IndexingWithBadData) {
sample_collection = collectionManager.get_collection("sample_collection");
if(sample_collection == nullptr) {
sample_collection = collectionManager.create_collection("sample_collection", fields, "age").get();
sample_collection = collectionManager.create_collection("sample_collection", 4, fields, "age").get();
}
const Option<nlohmann::json> & search_fields_missing_op1 = sample_collection->add("{\"namezz\": \"foo\", \"age\": 29, \"average\": 78}");
@ -1773,7 +1773,7 @@ TEST_F(CollectionTest, EmptyIndexShouldNotCrash) {
empty_coll = collectionManager.get_collection("empty_coll");
if(empty_coll == nullptr) {
empty_coll = collectionManager.create_collection("empty_coll", fields, "age").get();
empty_coll = collectionManager.create_collection("empty_coll", 4, fields, "age").get();
}
nlohmann::json results = empty_coll->search("a", {"name"}, "", {}, sort_fields, 0, 10, 1, FREQUENCY, false).get();
@ -1793,7 +1793,7 @@ TEST_F(CollectionTest, IdFieldShouldBeAString) {
coll1 = collectionManager.get_collection("coll1");
if(coll1 == nullptr) {
coll1 = collectionManager.create_collection("coll1", fields, "age").get();
coll1 = collectionManager.create_collection("coll1", 4, fields, "age").get();
}
nlohmann::json doc;
@ -1821,7 +1821,7 @@ TEST_F(CollectionTest, AnIntegerCanBePassedToAFloatField) {
coll1 = collectionManager.get_collection("coll1");
if(coll1 == nullptr) {
coll1 = collectionManager.create_collection("coll1", fields, "average").get();
coll1 = collectionManager.create_collection("coll1", 4, fields, "average").get();
}
nlohmann::json doc;
@ -1849,7 +1849,7 @@ TEST_F(CollectionTest, DeletionOfADocument) {
Collection *collection_for_del;
collection_for_del = collectionManager.get_collection("collection_for_del");
if(collection_for_del == nullptr) {
collection_for_del = collectionManager.create_collection("collection_for_del", search_fields, "points").get();
collection_for_del = collectionManager.create_collection("collection_for_del", 4, search_fields, "points").get();
}
std::string json_line;
@ -1930,7 +1930,7 @@ TEST_F(CollectionTest, DeletionOfDocumentArrayFields) {
coll1 = collectionManager.get_collection("coll1");
if(coll1 == nullptr) {
coll1 = collectionManager.create_collection("coll1", fields, "points").get();
coll1 = collectionManager.create_collection("coll1", 4, fields, "points").get();
}
nlohmann::json doc;
@ -2001,7 +2001,7 @@ TEST_F(CollectionTest, SearchLargeTextField) {
coll_large_text = collectionManager.get_collection("coll_large_text");
if(coll_large_text == nullptr) {
coll_large_text = collectionManager.create_collection("coll_large_text", fields, "age").get();
coll_large_text = collectionManager.create_collection("coll_large_text", 4, fields, "age").get();
}
std::string json_line;
@ -2083,7 +2083,7 @@ TEST_F(CollectionTest, StringArrayFieldShouldNotAllowPlainString) {
coll1 = collectionManager.get_collection("coll1");
if (coll1 == nullptr) {
coll1 = collectionManager.create_collection("coll1", fields, "points").get();
coll1 = collectionManager.create_collection("coll1", 4, fields, "points").get();
}
nlohmann::json doc;
@ -2108,7 +2108,7 @@ TEST_F(CollectionTest, SearchHighlightShouldFollowThreshold) {
coll1 = collectionManager.get_collection("coll1");
if (coll1 == nullptr) {
coll1 = collectionManager.create_collection("coll1", fields, "points").get();
coll1 = collectionManager.create_collection("coll1", 4, fields, "points").get();
}
nlohmann::json doc;
@ -2151,7 +2151,7 @@ TEST_F(CollectionTest, SearchHighlightFieldFully) {
coll1 = collectionManager.get_collection("coll1");
if (coll1 == nullptr) {
coll1 = collectionManager.create_collection("coll1", fields, "points").get();
coll1 = collectionManager.create_collection("coll1", 4, fields, "points").get();
}
nlohmann::json doc;
@ -2226,7 +2226,7 @@ TEST_F(CollectionTest, OptionalFields) {
coll1 = collectionManager.get_collection("coll1");
if(coll1 == nullptr) {
coll1 = collectionManager.create_collection("coll1", fields, "max").get();
coll1 = collectionManager.create_collection("coll1", 4, fields, "max").get();
}
std::ifstream infile(std::string(ROOT_DIR)+"test/optional_fields.jsonl");
@ -2290,7 +2290,7 @@ TEST_F(CollectionTest, OptionalFields) {
field("score", field_types::INT32, false, true),
};
auto create_op = collectionManager.create_collection("coll2", fields, "score");
auto create_op = collectionManager.create_collection("coll2", 4, fields, "score");
ASSERT_FALSE(create_op.ok());
ASSERT_STREQ("Default sorting field `score` cannot be an optional field.", create_op.error().c_str());