postgres/typesense
1
0
Fork 0
mirror of https://github.com/typesense/typesense.git synced 2025-05-19 21:22:25 +08:00
Code Issues Actions 1 Packages Projects Releases Wiki Activity

Review changes

Browse Source
This commit is contained in:
ozanarmagan 2023-10-09 02:04:43 +03:00
parent 238c5f00b9
commit cc54fb8176
7 changed files with 28 additions and 44 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
include/collection.h
View File

@ -213,7 +213,7 @@ private:
Option<bool> persist_collection_meta();
Option<bool> batch_alter_data(std::vector<field>& alter_fields,
Option<bool> batch_alter_data(const std::vector<field>& alter_fields,
const std::vector<field>& del_fields,
const std::string& this_fallback_field_type);
@ -566,6 +566,8 @@ public:
std::vector<std::string>& reordered_search_fields) const;
Option<bool> truncate_after_top_k(const std::string& field_name, size_t k);
static void process_embedding_field_delete(const std::string& model_name);
};
template<class T>

1
include/collection_manager.h
View File

@ -201,4 +201,5 @@ public:
Option<bool> upsert_preset(const std::string & preset_name, const nlohmann::json& preset_config);
Option<bool> delete_preset(const std::string & preset_name);
};

4
include/field.h
View File

@ -422,12 +422,12 @@ struct field {
static Option<bool> json_fields_to_fields(bool enable_nested_fields,
nlohmann::json& fields_json,
std::string& fallback_field_type,
std::vector<field>& the_fields, const std::string& collection_name = "");
std::vector<field>& the_fields);
static Option<bool> validate_and_init_embed_field(const tsl::htrie_map<char, field>& search_schema,
nlohmann::json& field_json,
const nlohmann::json& fields_json,
field& the_field, const std::string& collection_name);
field& the_field);
static bool flatten_obj(nlohmann::json& doc, nlohmann::json& value, bool has_array, bool has_obj_array,

21
src/collection.cpp
View File

@ -3765,7 +3765,7 @@ Option<bool> Collection::persist_collection_meta() {
return Option<bool>(true);
}
Option<bool> Collection::batch_alter_data(std::vector<field>& alter_fields,
Option<bool> Collection::batch_alter_data(const std::vector<field>& alter_fields,
const std::vector<field>& del_fields,
const std::string& this_fallback_field_type) {
// Update schema with additions (deletions can only be made later)
@ -3800,7 +3800,8 @@ Option<bool> Collection::batch_alter_data(std::vector<field>& alter_fields,
auto text_embedders = TextEmbedderManager::get_instance()._get_text_embedders();
auto model_name = f.embed[fields::model_config][fields::model_name].get<std::string>();
if(text_embedders.count(model_name) == 0) {
TextEmbedderManager::get_instance().validate_and_init_model(f.embed[fields::model_config], f.num_dim);
size_t dummy_num_dim = 0;
TextEmbedderManager::get_instance().validate_and_init_model(f.embed[fields::model_config], dummy_num_dim);
}
embedding_fields.emplace(f.name, f);
}
@ -4298,7 +4299,7 @@ Option<bool> Collection::validate_alter_payload(nlohmann::json& schema_changes,
}
if(!f.embed.empty()) {
auto validate_res = field::validate_and_init_embed_field(search_schema, schema_changes["fields"][json_array_index], schema_changes["fields"], f, name);
auto validate_res = field::validate_and_init_embed_field(search_schema, schema_changes["fields"][json_array_index], schema_changes["fields"], f);
if(!validate_res.ok()) {
return validate_res;
@ -5032,23 +5033,23 @@ Option<bool> Collection::truncate_after_top_k(const string &field_name, size_t k
}
void Collection::remove_embedding_field(const std::string& field_name) {
field del_field;
if(embedding_fields.find(field_name) != embedding_fields.end()) {
del_field = embedding_fields[field_name];
} else {
if(embedding_fields.find(field_name) == embedding_fields.end()) {
return;
}
auto del_field = embedding_fields[field_name];
embedding_fields.erase(field_name);
auto model_name = del_field.embed[fields::model_config]["model_name"].get<std::string>();
process_embedding_field_delete(model_name);
}
void Collection::process_embedding_field_delete(const std::string& model_name) {
auto collections = CollectionManager::get_instance().get_collections();
bool found = false;
for(auto& collection: collections) {
for(const auto& collection: collections) {
auto embedding_fields_other = collection->embedding_fields;
for(auto& embedding_field: embedding_fields_other) {

34
src/collection_manager.cpp
View File

@ -528,38 +528,20 @@ Option<nlohmann::json> CollectionManager::drop_collection(const std::string& col
s_lock.unlock();
auto embedding_fields = collection->get_embedding_fields();
std::unique_lock u_lock(mutex);
collections.erase(actual_coll_name);
collection_id_names.erase(collection->get_collection_id());
for(auto& embedding_field : embedding_fields) {
bool found = false;
auto model_name = embedding_field.embed[fields::model_config]["model_name"].get<std::string>();
for(auto& collection: collections) {
auto embedding_fields_other = collection.second->get_embedding_fields();
for(auto& embedding_field: embedding_fields_other) {
if(embedding_field.embed.count(fields::model_config) != 0) {
auto model_config = embedding_field.embed[fields::model_config];
if(model_config["model_name"].get<std::string>() == model_name) {
found = true;
break;
}
}
}
}
if(!found) {
LOG(INFO) << "Deleting text embedder: " << model_name;
TextEmbedderManager::get_instance().delete_text_embedder(embedding_field.embed[fields::model_config]["model_name"].get<std::string>());
}
}
u_lock.unlock();
const auto& embedding_fields = collection->get_embedding_fields();
for(const auto& embedding_field : embedding_fields) {
auto model_name = embedding_field.embed[fields::model_config]["model_name"].get<std::string>();
Collection::process_embedding_field_delete(model_name);
}
// don't hold any collection manager locks here, since this can take some time
delete collection;
@ -1287,7 +1269,7 @@ Option<Collection*> CollectionManager::create_collection(nlohmann::json& req_jso
std::string fallback_field_type;
std::vector<field> fields;
auto parse_op = field::json_fields_to_fields(req_json[ENABLE_NESTED_FIELDS].get<bool>(),
req_json["fields"], fallback_field_type, fields, req_json["name"]);
req_json["fields"], fallback_field_type, fields);
if(!parse_op.ok()) {
return Option<Collection*>(parse_op.code(), parse_op.error());

6
src/field.cpp
View File

@ -1081,7 +1081,7 @@ void field::compact_nested_fields(tsl::htrie_map<char, field>& nested_fields) {
}
Option<bool> field::json_fields_to_fields(bool enable_nested_fields, nlohmann::json &fields_json, string &fallback_field_type,
std::vector<field>& the_fields, const std::string& collection_name) {
std::vector<field>& the_fields) {
size_t num_auto_detect_fields = 0;
const tsl::htrie_map<char, field> dummy_search_schema;
@ -1094,7 +1094,7 @@ Option<bool> field::json_fields_to_fields(bool enable_nested_fields, nlohmann::j
}
if(!the_fields.empty() && !the_fields.back().embed.empty()) {
auto validate_res = validate_and_init_embed_field(dummy_search_schema, field_json, fields_json, the_fields.back(), collection_name);
auto validate_res = validate_and_init_embed_field(dummy_search_schema, field_json, fields_json, the_fields.back());
if(!validate_res.ok()) {
return validate_res;
}
@ -1110,7 +1110,7 @@ Option<bool> field::json_fields_to_fields(bool enable_nested_fields, nlohmann::j
Option<bool> field::validate_and_init_embed_field(const tsl::htrie_map<char, field>& search_schema, nlohmann::json& field_json,
const nlohmann::json& fields_json,
field& the_field, const std::string& collection_name) {
field& the_field) {
const std::string err_msg = "Property `" + fields::embed + "." + fields::from +
"` can only refer to string or string array fields.";

2
test/collection_vector_search_test.cpp
View File

@ -2386,8 +2386,6 @@ TEST_F(CollectionVectorTest, TestUnloadingModelsOnDrop) {
auto alter_op = coll->alter(alter_schema);
ASSERT_TRUE(alter_op.ok());
LOG(INFO) << "created second collection";
text_embedders = TextEmbedderManager::get_instance()._get_text_embedders();
ASSERT_EQ(1, text_embedders.size());