postgres/typesense
1
0
Fork 0
mirror of https://github.com/typesense/typesense.git synced 2025-05-22 06:40:30 +08:00
Code Issues Actions 3 Packages Projects Releases Wiki Activity

Merge pull request #1305 from ozanarmagan/v0.25-join

Browse Source 
Avoid loading embedding model that is larger than free memory
This commit is contained in:
Kishore Nallan 2023-10-17 17:42:39 +05:30 committed by GitHub
commit acfd047b81
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
2 changed files with 16 additions and 2 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
include/system_metrics.h
View File

@ -173,4 +173,8 @@ public:
// compute
return compute_cpu_stats(cpu_data_prev, cpu_data_now);
}
static uint64_t get_memory_free_bytes() {
return get_memory_total_bytes() - get_memory_used_bytes();
}
};

14
src/text_embedder_manager.cpp
View File

@ -1,4 +1,5 @@
#include "text_embedder_manager.h"
#include "system_metrics.h"
TextEmbedderManager& TextEmbedderManager::get_instance() {
@ -107,8 +108,17 @@ Option<bool> TextEmbedderManager::validate_and_init_local_model(const nlohmann::
return Option<bool>(true);
}
const std::shared_ptr<TextEmbedder>& embedder = std::make_shared<TextEmbedder>(
get_model_name_without_namespace(model_name));
const auto& model_name_without_namespace = get_model_name_without_namespace(model_name);
const auto& free_memory = SystemMetrics::get_memory_free_bytes();
const auto& model_file_size = std::filesystem::file_size(abs_path);
// return error if (model file size * 1.15) is greater than free memory
if(model_file_size * 1.15 > free_memory) {
LOG(ERROR) << "Memory required to load the model exceeds free memory available.";
return Option<bool>(400, "Memory required to load the model exceeds free memory available.");
}
const std::shared_ptr<TextEmbedder>& embedder = std::make_shared<TextEmbedder>(model_name_without_namespace);
auto validate_op = embedder->validate();
if(!validate_op.ok()) {