From 5766b83f249f7c21b260966630fb7407d8af6d07 Mon Sep 17 00:00:00 2001 From: ozanarmagan Date: Tue, 17 Oct 2023 00:26:36 +0300 Subject: [PATCH 1/2] Avoid loading models that larger than free memory --- include/system_metrics.h | 4 ++++ src/text_embedder_manager.cpp | 15 +++++++++++++-- 2 files changed, 17 insertions(+), 2 deletions(-) diff --git a/include/system_metrics.h b/include/system_metrics.h index 678ae4f9..94363afa 100644 --- a/include/system_metrics.h +++ b/include/system_metrics.h @@ -173,4 +173,8 @@ public: // compute return compute_cpu_stats(cpu_data_prev, cpu_data_now); } + + static uint64_t get_memory_free_bytes() { + return get_memory_total_bytes() - get_memory_used_bytes(); + } }; \ No newline at end of file diff --git a/src/text_embedder_manager.cpp b/src/text_embedder_manager.cpp index 89400a79..912336dc 100644 --- a/src/text_embedder_manager.cpp +++ b/src/text_embedder_manager.cpp @@ -1,4 +1,5 @@ #include "text_embedder_manager.h" +#include "system_metrics.h" TextEmbedderManager& TextEmbedderManager::get_instance() { @@ -107,8 +108,18 @@ Option TextEmbedderManager::validate_and_init_local_model(const nlohmann:: return Option(true); } - const std::shared_ptr& embedder = std::make_shared( - get_model_name_without_namespace(model_name)); + const auto& model_name_without_namespace = get_model_name_without_namespace(model_name); + const auto& free_memory = SystemMetrics::get_memory_free_bytes(); + const auto& model_file_size = std::filesystem::file_size(abs_path); + + // return error if model file size is greater than free memory + if(model_file_size > free_memory) { + LOG(ERROR) << "Model file size is greater than free memory: " << model_file_size << " > " << free_memory; + return Option(400, "Model file size is greater than free memory"); + } + + + const std::shared_ptr& embedder = std::make_shared(model_name_without_namespace); auto validate_op = embedder->validate(); if(!validate_op.ok()) { From 8392e85891a9f502a41396e81f341ac7b83e7ac1 Mon Sep 17 00:00:00 2001 From: ozanarmagan Date: Tue, 17 Oct 2023 14:31:39 +0300 Subject: [PATCH 2/2] Use 'model size * 1.15` --- src/text_embedder_manager.cpp | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/src/text_embedder_manager.cpp b/src/text_embedder_manager.cpp index 912336dc..baa99ebf 100644 --- a/src/text_embedder_manager.cpp +++ b/src/text_embedder_manager.cpp @@ -112,13 +112,12 @@ Option TextEmbedderManager::validate_and_init_local_model(const nlohmann:: const auto& free_memory = SystemMetrics::get_memory_free_bytes(); const auto& model_file_size = std::filesystem::file_size(abs_path); - // return error if model file size is greater than free memory - if(model_file_size > free_memory) { - LOG(ERROR) << "Model file size is greater than free memory: " << model_file_size << " > " << free_memory; - return Option(400, "Model file size is greater than free memory"); + // return error if (model file size * 1.15) is greater than free memory + if(model_file_size * 1.15 > free_memory) { + LOG(ERROR) << "Memory required to load the model exceeds free memory available."; + return Option(400, "Memory required to load the model exceeds free memory available."); } - const std::shared_ptr& embedder = std::make_shared(model_name_without_namespace); auto validate_op = embedder->validate();