postgres/typesense
1
0
Fork 0
mirror of https://github.com/typesense/typesense.git synced 2025-05-19 13:12:22 +08:00
Code Issues Actions 1 Packages Projects Releases Wiki Activity

Tweak default value for thread pool size and parallel coll loading.

Browse Source
This commit is contained in:
kishorenc 2021-03-30 17:29:44 +05:30
parent 6c284dba17
commit 693fcb0e7e
3 changed files with 16 additions and 17 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
include/collection_manager.h
View File

@ -71,9 +71,6 @@ private:
std::atomic<float> max_memory_ratio;
/// During load, sleep `LOAD_THROTTLE_PERCENT`% of time taken for indexing to prevent CPU saturation
float LOAD_THROTTLE_PERCENT = 10;
CollectionManager();
~CollectionManager() = default;

22
include/config.h
View File

@ -40,10 +40,10 @@ private:
int log_slow_requests_time_ms;
size_t num_collections_parallel_load;
size_t num_documents_parallel_load;
uint32_t num_collections_parallel_load;
uint32_t num_documents_parallel_load;
size_t thread_pool_size;
uint32_t thread_pool_size;
protected:
@ -57,11 +57,9 @@ protected:
this->catch_up_min_sequence_diff = 3000;
this->catch_up_threshold_percentage = 95;
this->log_slow_requests_time_ms = -1;
this->num_collections_parallel_load = 4;
this->num_collections_parallel_load = 0; // will be set dynamically if not overridden
this->num_documents_parallel_load = 1000;
// will be set dynamically if not overridden
this->thread_pool_size = 0;
this->thread_pool_size = 0; // will be set dynamically if not overridden
}
Config(Config const&) {
@ -401,7 +399,7 @@ public:
}
if(reader.Exists("server", "num-collections-parallel-load")) {
this->num_collections_parallel_load = (int) reader.GetInteger("server", "num-collections-parallel-load", 4);
this->num_collections_parallel_load = (int) reader.GetInteger("server", "num-collections-parallel-load", 0);
}
if(reader.Exists("server", "num-documents-parallel-load")) {
@ -492,19 +490,19 @@ public:
}
if(options.exist("log-slow-requests-time-ms")) {
this->log_slow_requests_time_ms = options.exist("log-slow-requests-time-ms");
this->log_slow_requests_time_ms = options.get<int>("log-slow-requests-time-ms");
}
if(options.exist("num-collections-parallel-load")) {
this->num_collections_parallel_load = options.exist("num-collections-parallel-load");
this->num_collections_parallel_load = options.get<uint32_t>("num-collections-parallel-load");
}
if(options.exist("num-documents-parallel-load")) {
this->num_documents_parallel_load = options.exist("num-documents-parallel-load");
this->num_documents_parallel_load = options.get<uint32_t>("num-documents-parallel-load");
}
if(options.exist("thread-pool-size")) {
this->thread_pool_size = options.exist("thread-pool-size");
this->thread_pool_size = options.get<uint32_t>("thread-pool-size");
}
}

8
src/typesense_server_utils.cpp
View File

@ -347,7 +347,11 @@ int run_server(const Config & config, const std::string & version, void (*master
size_t thread_pool_size = config.get_thread_pool_size();
const size_t proc_count = std::max<size_t>(1, std::thread::hardware_concurrency());
const size_t num_threads = thread_pool_size == 0 ? std::max<size_t>(proc_count * 8, 16) : thread_pool_size;
const size_t num_threads = thread_pool_size == 0 ? (proc_count * 8) : thread_pool_size;
size_t num_collections_parallel_load = config.get_num_collections_parallel_load();
num_collections_parallel_load = (num_collections_parallel_load == 0) ?
(proc_count * 4) : num_collections_parallel_load;
LOG(INFO) << "Thread pool size: " << num_threads;
ThreadPool app_thread_pool(num_threads);
@ -385,7 +389,7 @@ int run_server(const Config & config, const std::string & version, void (*master
ReplicationState replication_state(&store, &app_thread_pool, server->get_message_dispatcher(),
ssl_enabled, config.get_catch_up_min_sequence_diff(),
config.get_catch_up_threshold_percentage(),
config.get_num_collections_parallel_load(),
num_collections_parallel_load,
config.get_num_documents_parallel_load());
std::thread raft_thread([&replication_state, &config, &state_dir, &app_thread_pool, &server_thread_pool]() {