postgres/typesense
1
0
Fork 0
mirror of https://github.com/typesense/typesense.git synced 2025-05-21 22:33:27 +08:00
Code Issues Actions 3 Packages Projects Releases Wiki Activity

Fix for downloading models on every test run

Browse Source
This commit is contained in:
ozanarmagan 2023-03-16 17:07:31 +03:00
parent a07199b824
commit 783c8dcae2
2 changed files with 53 additions and 25 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

14
test/collection_all_fields_test.cpp
View File

@ -3,6 +3,7 @@
#include <vector>
#include <fstream>
#include <algorithm>
#include <filesystem>
#include <collection_manager.h>
#include "collection.h"
#include "text_embedder_manager.h"
@ -21,7 +22,7 @@ protected:
std::string state_dir_path = "/tmp/typesense_test/collection_all_fields";
LOG(INFO) << "Truncating and creating: " << state_dir_path;
system(("rm -rf "+state_dir_path+" && mkdir -p "+state_dir_path).c_str());
system(("rm -rf "+state_dir_path + "/models" + " && mkdir -p "+state_dir_path + "/models").c_str());
system("mkdir -p /tmp/typesense_test/models");
store = new Store(state_dir_path);
collectionManager.init(store, 1.0, "auth_key", quit);
@ -30,7 +31,6 @@ protected:
virtual void SetUp() {
setupCollection();
system("mkdir -p models");
}
virtual void TearDown() {
@ -1666,9 +1666,13 @@ TEST_F(CollectionAllFieldsTest, ModelPathWithoutCreateFrom) {
TEST_F(CollectionAllFieldsTest, CreateFromBasicValid) {
TextEmbedderManager::model_dir = "/tmp/typesense_test/collection_all_fields/models/";
HttpClient::get_instance().download_file(TextEmbedderManager::DEFAULT_MODEL_URL, TextEmbedderManager::get_absolute_model_path(TextEmbedderManager::DEFAULT_MODEL_NAME));
HttpClient::get_instance().download_file(TextEmbedderManager::DEFAULT_VOCAB_URL, TextEmbedderManager::get_absolute_vocab_path());
TextEmbedderManager::model_dir = "/tmp/typesense_test/models";
if(!std::filesystem::exists(std::filesystem::path(TextEmbedderManager::get_absolute_model_path(TextEmbedderManager::DEFAULT_MODEL_NAME)))){
HttpClient::get_instance().download_file(TextEmbedderManager::DEFAULT_MODEL_URL, TextEmbedderManager::get_absolute_model_path(TextEmbedderManager::DEFAULT_MODEL_NAME));
}
if(!std::filesystem::exists(std::filesystem::path(TextEmbedderManager::get_absolute_vocab_path()))){
HttpClient::get_instance().download_file(TextEmbedderManager::DEFAULT_VOCAB_URL, TextEmbedderManager::get_absolute_vocab_path());
}
field embedding = field("embedding", field_types::FLOAT_ARRAY, false);
embedding.create_from.push_back("name");

64
test/collection_test.cpp
View File

@ -3,6 +3,7 @@
#include <vector>
#include <fstream>
#include <algorithm>
#include <filesystem>
#include <collection_manager.h>
#include "collection.h"
#include "text_embedder_manager.h"
@ -24,7 +25,7 @@ protected:
std::string state_dir_path = "/tmp/typesense_test/collection";
LOG(INFO) << "Truncating and creating: " << state_dir_path;
system(("rm -rf "+state_dir_path+" && mkdir -p "+state_dir_path).c_str());
system(("rm -rf "+state_dir_path + "/models" + " && mkdir -p "+state_dir_path + "/models").c_str());
system("mkdir -p /tmp/typesense_test/models");
store = new Store(state_dir_path);
collectionManager.init(store, 1.0, "auth_key", quit);
@ -65,7 +66,6 @@ protected:
virtual void SetUp() {
setupCollection();
system("mkdir -p models");
}
virtual void TearDown() {
@ -4622,9 +4622,13 @@ TEST_F(CollectionTest, SemanticSearchTest) {
]
})"_json;
TextEmbedderManager::model_dir = "/tmp/typesense_test/collection/models/";
HttpClient::get_instance().download_file(TextEmbedderManager::DEFAULT_MODEL_URL, TextEmbedderManager::get_absolute_model_path(TextEmbedderManager::DEFAULT_MODEL_NAME));
HttpClient::get_instance().download_file(TextEmbedderManager::DEFAULT_VOCAB_URL, TextEmbedderManager::get_absolute_vocab_path());
TextEmbedderManager::model_dir = "/tmp/typesense_test/models";
if(!std::filesystem::exists(std::filesystem::path(TextEmbedderManager::get_absolute_model_path(TextEmbedderManager::DEFAULT_MODEL_NAME)))){
HttpClient::get_instance().download_file(TextEmbedderManager::DEFAULT_MODEL_URL, TextEmbedderManager::get_absolute_model_path(TextEmbedderManager::DEFAULT_MODEL_NAME));
}
if(!std::filesystem::exists(std::filesystem::path(TextEmbedderManager::get_absolute_vocab_path()))){
HttpClient::get_instance().download_file(TextEmbedderManager::DEFAULT_VOCAB_URL, TextEmbedderManager::get_absolute_vocab_path());
}
auto op = collectionManager.create_collection(schema);
ASSERT_TRUE(op.ok());
@ -4658,9 +4662,13 @@ TEST_F(CollectionTest, InvalidSemanticSearch) {
]
})"_json;
TextEmbedderManager::model_dir = "/tmp/typesense_test/collection/models/";
HttpClient::get_instance().download_file(TextEmbedderManager::DEFAULT_MODEL_URL, TextEmbedderManager::get_absolute_model_path(TextEmbedderManager::DEFAULT_MODEL_NAME));
HttpClient::get_instance().download_file(TextEmbedderManager::DEFAULT_VOCAB_URL, TextEmbedderManager::get_absolute_vocab_path());
TextEmbedderManager::model_dir = "/tmp/typesense_test/models";
if(!std::filesystem::exists(std::filesystem::path(TextEmbedderManager::get_absolute_model_path(TextEmbedderManager::DEFAULT_MODEL_NAME)))){
HttpClient::get_instance().download_file(TextEmbedderManager::DEFAULT_MODEL_URL, TextEmbedderManager::get_absolute_model_path(TextEmbedderManager::DEFAULT_MODEL_NAME));
}
if(!std::filesystem::exists(std::filesystem::path(TextEmbedderManager::get_absolute_vocab_path()))){
HttpClient::get_instance().download_file(TextEmbedderManager::DEFAULT_VOCAB_URL, TextEmbedderManager::get_absolute_vocab_path());
}
auto op = collectionManager.create_collection(schema);
LOG(INFO) << "op.error(): " << op.error();
@ -4690,9 +4698,13 @@ TEST_F(CollectionTest, HybridSearch) {
]
})"_json;
TextEmbedderManager::model_dir = "/tmp/typesense_test/collection/models/";
HttpClient::get_instance().download_file(TextEmbedderManager::DEFAULT_MODEL_URL, TextEmbedderManager::get_absolute_model_path(TextEmbedderManager::DEFAULT_MODEL_NAME));
HttpClient::get_instance().download_file(TextEmbedderManager::DEFAULT_VOCAB_URL, TextEmbedderManager::get_absolute_vocab_path());
TextEmbedderManager::model_dir = "/tmp/typesense_test/models";
if(!std::filesystem::exists(std::filesystem::path(TextEmbedderManager::get_absolute_model_path(TextEmbedderManager::DEFAULT_MODEL_NAME)))){
HttpClient::get_instance().download_file(TextEmbedderManager::DEFAULT_MODEL_URL, TextEmbedderManager::get_absolute_model_path(TextEmbedderManager::DEFAULT_MODEL_NAME));
}
if(!std::filesystem::exists(std::filesystem::path(TextEmbedderManager::get_absolute_vocab_path()))){
HttpClient::get_instance().download_file(TextEmbedderManager::DEFAULT_VOCAB_URL, TextEmbedderManager::get_absolute_vocab_path());
}
auto op = collectionManager.create_collection(schema);
ASSERT_TRUE(op.ok());
@ -4724,9 +4736,13 @@ TEST_F(CollectionTest, EmbedFielsTest) {
]
})"_json;
TextEmbedderManager::model_dir = "/tmp/typesense_test/collection/models/";
HttpClient::get_instance().download_file(TextEmbedderManager::DEFAULT_MODEL_URL, TextEmbedderManager::get_absolute_model_path(TextEmbedderManager::DEFAULT_MODEL_NAME));
HttpClient::get_instance().download_file(TextEmbedderManager::DEFAULT_VOCAB_URL, TextEmbedderManager::get_absolute_vocab_path());
TextEmbedderManager::model_dir = "/tmp/typesense_test/models";
if(!std::filesystem::exists(std::filesystem::path(TextEmbedderManager::get_absolute_model_path(TextEmbedderManager::DEFAULT_MODEL_NAME)))){
HttpClient::get_instance().download_file(TextEmbedderManager::DEFAULT_MODEL_URL, TextEmbedderManager::get_absolute_model_path(TextEmbedderManager::DEFAULT_MODEL_NAME));
}
if(!std::filesystem::exists(std::filesystem::path(TextEmbedderManager::get_absolute_vocab_path()))){
HttpClient::get_instance().download_file(TextEmbedderManager::DEFAULT_VOCAB_URL, TextEmbedderManager::get_absolute_vocab_path());
}
auto op = collectionManager.create_collection(schema);
ASSERT_TRUE(op.ok());
@ -4753,9 +4769,13 @@ TEST_F(CollectionTest, HybridSearchRankFusionTest) {
]
})"_json;
TextEmbedderManager::model_dir = "/tmp/typesense_test/collection/models/";
HttpClient::get_instance().download_file(TextEmbedderManager::DEFAULT_MODEL_URL, TextEmbedderManager::get_absolute_model_path(TextEmbedderManager::DEFAULT_MODEL_NAME));
HttpClient::get_instance().download_file(TextEmbedderManager::DEFAULT_VOCAB_URL, TextEmbedderManager::get_absolute_vocab_path());
TextEmbedderManager::model_dir = "/tmp/typesense_test/models";
if(!std::filesystem::exists(std::filesystem::path(TextEmbedderManager::get_absolute_model_path(TextEmbedderManager::DEFAULT_MODEL_NAME)))){
HttpClient::get_instance().download_file(TextEmbedderManager::DEFAULT_MODEL_URL, TextEmbedderManager::get_absolute_model_path(TextEmbedderManager::DEFAULT_MODEL_NAME));
}
if(!std::filesystem::exists(std::filesystem::path(TextEmbedderManager::get_absolute_vocab_path()))){
HttpClient::get_instance().download_file(TextEmbedderManager::DEFAULT_VOCAB_URL, TextEmbedderManager::get_absolute_vocab_path());
}
auto op = collectionManager.create_collection(schema);
ASSERT_TRUE(op.ok());
@ -4828,9 +4848,13 @@ TEST_F(CollectionTest, WildcardSearchWithEmbeddingField) {
]
})"_json;
TextEmbedderManager::model_dir = "/tmp/typesense_test/collection/models/";
HttpClient::get_instance().download_file(TextEmbedderManager::DEFAULT_MODEL_URL, TextEmbedderManager::get_absolute_model_path(TextEmbedderManager::DEFAULT_MODEL_NAME));
HttpClient::get_instance().download_file(TextEmbedderManager::DEFAULT_VOCAB_URL, TextEmbedderManager::get_absolute_vocab_path());
TextEmbedderManager::model_dir = "/tmp/typesense_test/models";
if(!std::filesystem::exists(std::filesystem::path(TextEmbedderManager::get_absolute_model_path(TextEmbedderManager::DEFAULT_MODEL_NAME)))){
HttpClient::get_instance().download_file(TextEmbedderManager::DEFAULT_MODEL_URL, TextEmbedderManager::get_absolute_model_path(TextEmbedderManager::DEFAULT_MODEL_NAME));
}
if(!std::filesystem::exists(std::filesystem::path(TextEmbedderManager::get_absolute_vocab_path()))){
HttpClient::get_instance().download_file(TextEmbedderManager::DEFAULT_VOCAB_URL, TextEmbedderManager::get_absolute_vocab_path());
}
auto op = collectionManager.create_collection(schema);
ASSERT_TRUE(op.ok());