postgres/typesense
1
0
Fork 0
mirror of https://github.com/typesense/typesense.git synced 2025-05-16 03:12:32 +08:00
Code Issues Actions 2 Packages Projects Releases Wiki Activity

Use a proper logger for logging!

Browse Source
This commit is contained in:
Kishore Nallan 2018-02-01 13:28:15 -05:00
parent 4fb5b2854e
commit 6b37d43fd7
25 changed files with 225 additions and 87 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

13
CMakeLists.txt
View File

@ -40,6 +40,7 @@ include(cmake/H2O.cmake)
include(cmake/RocksDB.cmake)
include(cmake/GoogleTest.cmake)
include(cmake/TestResources.cmake)
include(cmake/G3log.cmake)
FILE(GLOB SRC_FILES src/*.cpp)
@ -51,17 +52,19 @@ include_directories(${DEP_ROOT_DIR}/${MINIUTF_NAME})
include_directories(${DEP_ROOT_DIR}/${GTEST_NAME}/googletest/include)
include_directories(${DEP_ROOT_DIR}/${H2O_NAME}/include)
include_directories(${DEP_ROOT_DIR}/${ROCKSDB_NAME}/include)
include_directories(${DEP_ROOT_DIR}/${G3LOG_NAME}/src)
link_directories(${DEP_ROOT_DIR}/${GTEST_NAME}/googletest/build)
link_directories(${DEP_ROOT_DIR}/${FOR_NAME})
link_directories(${DEP_ROOT_DIR}/${MINIUTF_NAME}/build)
link_directories(${DEP_ROOT_DIR}/${H2O_NAME}/build)
link_directories(${DEP_ROOT_DIR}/${ROCKSDB_NAME})
link_directories(${DEP_ROOT_DIR}/${G3LOG_NAME}/build)
add_executable(typesense-server ${SRC_FILES} src/main/typesense_server.cpp)
add_executable(search ${SRC_FILES} src/main/main.cpp)
add_executable(benchmark ${SRC_FILES} src/main/benchmark.cpp)
add_executable(typesense_test ${SRC_FILES} test/array_test.cpp test/sorted_array_test.cpp test/art_test.cpp
add_executable(typesense_test ${SRC_FILES} test/main.cpp test/array_test.cpp test/sorted_array_test.cpp test/art_test.cpp
test/collection_test.cpp test/collection_manager_test.cpp
test/topster_test.cpp test/match_score_test.cpp test/store_test.cpp test/array_utils_test.cpp
test/string_utils_test.cpp)
@ -86,7 +89,7 @@ if(NOT APPLE)
list(APPEND ROCKSDB_LIBS rt)
endif()
target_link_libraries(typesense-server h2o-evloop for miniutf pthread ${CURL_LIBRARIES} ${ROCKSDB_LIBS} ${OPENSSL_LIBRARIES} dl ${STD_LIB})
target_link_libraries(search for miniutf pthread h2o-evloop ${CURL_LIBRARIES} ${ROCKSDB_LIBS} ${OPENSSL_LIBRARIES} dl ${STD_LIB})
target_link_libraries(benchmark for miniutf pthread ${CURL_LIBRARIES} h2o-evloop ${ROCKSDB_LIBS} ${OPENSSL_LIBRARIES} dl ${STD_LIB})
target_link_libraries(typesense_test h2o-evloop ${OPENSSL_LIBRARIES} pthread for miniutf ${ROCKSDB_LIBS} gtest gtest_main dl ${STD_LIB})
target_link_libraries(typesense-server h2o-evloop for miniutf g3logger pthread ${CURL_LIBRARIES} ${ROCKSDB_LIBS} ${OPENSSL_LIBRARIES} dl ${STD_LIB})
target_link_libraries(search for miniutf g3logger pthread h2o-evloop ${CURL_LIBRARIES} ${ROCKSDB_LIBS} ${OPENSSL_LIBRARIES} dl ${STD_LIB})
target_link_libraries(benchmark for miniutf g3logger pthread ${CURL_LIBRARIES} h2o-evloop ${ROCKSDB_LIBS} ${OPENSSL_LIBRARIES} dl ${STD_LIB})
target_link_libraries(typesense_test h2o-evloop ${OPENSSL_LIBRARIES} pthread for miniutf g3logger ${ROCKSDB_LIBS} gtest gtest_main dl ${STD_LIB})

2
README.md
View File

@ -78,7 +78,7 @@ Typesense requires the following dependencies:
* zlib
* OpenSSL (>=1.0.2)
* curl
* Compiler with good C++11 support (e.g. GCC >= 4.9)
* Compiler with good C++11 support (GCC >= 4.9.0, Apple Clang >= 8.0, Clang >= 3.9.0)
```
$ ./build.sh [--clean] [--depclean]

38
cmake/Easyloggingpp.cmake Normal file
View File

@ -0,0 +1,38 @@
# Download and build EASYLOGGINGPP
set(EASYLOGGINGPP_VERSION 9.95.3)
set(EASYLOGGINGPP_NAME easyloggingpp-${EASYLOGGINGPP_VERSION})
set(EASYLOGGINGPP_TAR_PATH ${DEP_ROOT_DIR}/${EASYLOGGINGPP_NAME}.tar.gz)
if(NOT EXISTS ${EASYLOGGINGPP_TAR_PATH})
message(STATUS "Downloading Easyloggingpp...")
file(DOWNLOAD https://github.com/muflihun/easyloggingpp/archive/v${EASYLOGGINGPP_VERSION}.tar.gz ${EASYLOGGINGPP_TAR_PATH})
endif()
if(NOT EXISTS ${DEP_ROOT_DIR}/${EASYLOGGINGPP_NAME})
message(STATUS "Extracting Easyloggingpp...")
execute_process(COMMAND ${CMAKE_COMMAND} -E tar xzf ${EASYLOGGINGPP_TAR_PATH} WORKING_DIRECTORY ${DEP_ROOT_DIR})
endif()
if(NOT EXISTS ${DEP_ROOT_DIR}/${EASYLOGGINGPP_NAME}/build)
message("Configuring Easyloggingpp...")
file(MAKE_DIRECTORY ${DEP_ROOT_DIR}/${EASYLOGGINGPP_NAME}/build)
execute_process(COMMAND ${CMAKE_COMMAND}
"-Dbuild_static_lib=ON"
"-H${DEP_ROOT_DIR}/${EASYLOGGINGPP_NAME}"
"-B${DEP_ROOT_DIR}/${EASYLOGGINGPP_NAME}/build"
RESULT_VARIABLE
EASYLOGGINGPP_CONFIGURE)
if(NOT EASYLOGGINGPP_CONFIGURE EQUAL 0)
message(FATAL_ERROR "Easyloggingpp Configure failed!")
endif()
message("Building Easyloggingpp locally...")
execute_process(COMMAND ${CMAKE_COMMAND} --build
"${DEP_ROOT_DIR}/${EASYLOGGINGPP_NAME}/build"
RESULT_VARIABLE
EASYLOGGINGPP_BUILD)
if(NOT EASYLOGGINGPP_BUILD EQUAL 0)
message(FATAL_ERROR "Easyloggingpp build failed!")
endif()
endif()

38
cmake/g3log.cmake Normal file
View File

@ -0,0 +1,38 @@
# Download and build g3log
set(G3LOG_VERSION 1.3)
set(G3LOG_NAME g3log-${G3LOG_VERSION})
set(G3LOG_TAR_PATH ${DEP_ROOT_DIR}/${G3LOG_NAME}.tar.gz)
if(NOT EXISTS ${G3LOG_TAR_PATH})
message(STATUS "Downloading G3log...")
file(DOWNLOAD https://github.com/KjellKod/g3log/archive/${G3LOG_VERSION}.tar.gz ${G3LOG_TAR_PATH})
endif()
if(NOT EXISTS ${DEP_ROOT_DIR}/${G3LOG_NAME})
message(STATUS "Extracting G3log...")
execute_process(COMMAND ${CMAKE_COMMAND} -E tar xzf ${G3LOG_TAR_PATH} WORKING_DIRECTORY ${DEP_ROOT_DIR})
endif()
if(NOT EXISTS ${DEP_ROOT_DIR}/${G3LOG_NAME}/build)
message("Configuring G3log...")
file(MAKE_DIRECTORY ${DEP_ROOT_DIR}/${G3LOG_NAME}/build)
execute_process(COMMAND ${CMAKE_COMMAND}
"-DCMAKE_BUILD_TYPE=Release"
"-H${DEP_ROOT_DIR}/${G3LOG_NAME}"
"-B${DEP_ROOT_DIR}/${G3LOG_NAME}/build"
RESULT_VARIABLE
G3LOG_CONFIGURE)
if(NOT G3LOG_CONFIGURE EQUAL 0)
message(FATAL_ERROR "G3log Configure failed!")
endif()
message("Building G3log locally...")
execute_process(COMMAND ${CMAKE_COMMAND} --build
"${DEP_ROOT_DIR}/${G3LOG_NAME}/build"
RESULT_VARIABLE
G3LOG_BUILD)
if(NOT G3LOG_BUILD EQUAL 0)
message(FATAL_ERROR "G3log build failed!")
endif()
endif()

7
include/cmdline.h
View File

@ -38,6 +38,7 @@
#include <algorithm>
#include <cxxabi.h>
#include <cstdlib>
#include "logger.h"
namespace cmdline{
@ -408,7 +409,7 @@ namespace cmdline{
args.push_back(buf);
for (size_t i=0; i<args.size(); i++)
std::cout<<"\""<<args[i]<<"\""<<std::endl;
LOG(INFO)<<"\""<<args[i]<<"\"";
return parse(args);
}
@ -588,12 +589,12 @@ namespace cmdline{
void check(int argc, bool ok){
if ((argc==1 && !ok) || exist("help")){
std::cerr<<usage();
LOG(FATAL)<<usage();
exit(0);
}
if (!ok){
std::cerr<<error()<<std::endl<<usage();
LOG(FATAL)<<error()<<std::endl<<usage();
exit(1);
}
}

22
include/console_logging_sink.h Normal file
View File

@ -0,0 +1,22 @@
#pragma once
#include <string>
#include <iostream>
#include <g3log/logmessage.hpp>
// https://github.com/KjellKod/g3sinks/blob/master/snippets/ColorCoutSink.hpp
struct ConsoleLoggingSink {
enum FG_Color {YELLOW = 33, RED = 31, GREEN=32, WHITE = 97};
void ReceiveLogMessage(g3::LogMessageMover logEntry) {
auto level = logEntry.get()._level;
if (g3::internal::wasFatal(level)) {
std::cerr << "\033[" << RED << "m"
<< logEntry.get().toString() << "\033[m";
} else {
std::cout << logEntry.get().toString();
}
}
};

3
include/exectime.h
View File

@ -1,6 +1,7 @@
#pragma once
#include <iostream>
#include <chrono>
#include "logger.h"
class ExecTime {
static auto begin = std::chrono::high_resolution_clock::now();
@ -12,6 +13,6 @@ class ExecTime {
static void log(std::string operation) {
long long int timeMicros = std::chrono::duration_cast<std::chrono::microseconds>(
std::chrono::high_resolution_clock::now() - begin).count();
std::cout << "Time taken for " << operation << ": " << timeMicros << "us" << std::endl;
LOG(INFO) << "Time taken for " << operation << ": " << timeMicros << "us";
}
};

5
include/logger.h Normal file
View File

@ -0,0 +1,5 @@
#pragma once
#include <g3log/g3log.hpp>
#include <g3log/logworker.hpp>
#include "console_logging_sink.h"

13
include/match_score.h
View File

@ -5,6 +5,7 @@
#include <queue>
#include <stdlib.h>
#include <limits>
#include "logger.h"
#ifdef DEBUG
#define D(x) x
@ -46,9 +47,9 @@ struct Match {
static void print_token_offsets(std::vector<std::vector<uint16_t>> &token_offsets) {
for(auto offsets: token_offsets) {
for(auto offset: offsets) {
std::cout << offset << ", ";
LOG(INFO) << offset << ", ";
}
std::cout << std::endl;
LOG(INFO) << "";
}
}
@ -114,7 +115,7 @@ struct Match {
addTopOfHeapToWindow(heap, window, token_offsets, token_offset);
}
D(std::cout << "Loop till window fills... doc_id: " << doc_id << std::endl;)
D(LOG(INFO) << "Loop till window fills... doc_id: " << doc_id;)
// Fill the queue with tokens within a given window frame size of the start offset
// At the same time, we also record the *last* occurrence of each token within the window
@ -124,7 +125,7 @@ struct Match {
addTopOfHeapToWindow(heap, window, token_offsets, token_offset);
}
D(std::cout << std::endl << "----" << std::endl);
D(LOG(INFO) << "----");
uint16_t prev_pos = MAX_DISPLACEMENT;
uint16_t num_match = 0;
@ -140,14 +141,14 @@ struct Match {
} else {
// Calculate the distance between the tokens within the window
// Ideally, this should be (NUM_TOKENS - 1) when all the tokens are adjacent to each other
D(std::cout << "prev_pos: " << prev_pos << " , curr_pos: " << token_offset[token_id] << std::endl);
D(LOG(INFO) << "prev_pos: " << prev_pos << " , curr_pos: " << token_offset[token_id]);
displacement += abs(token_offset[token_id]-prev_pos);
prev_pos = token_offset[token_id];
}
}
}
D(std::cout << std::endl << "!!!displacement: " << displacement << " | num_match: " << num_match << std::endl);
D(LOG(INFO) << std::endl << "!!!displacement: " << displacement << " | num_match: " << num_match);
// Track the best `displacement` and `num_match` seen so far across all the windows
// for a single token, displacement will be 0, while for 2 tokens minimum dispacement would be 1

13
include/replicator.h
View File

@ -8,7 +8,7 @@
#include <thread>
#include <string>
#include <iostream>
#include "logger.h"
struct ReplicationEvent {
std::string type;
@ -93,7 +93,7 @@ public:
while(true) {
IterateBatchHandler handler(server);
uint64_t latest_seq_num = store.get_latest_seq_number();
std::cout << "latest_seq_num: " << latest_seq_num << std::endl;
LOG(INFO) << "latest_seq_num: " << latest_seq_num;
HttpClient client(
master_host_port+"/replication/updates?seq_number="+std::to_string(latest_seq_num+1), api_key
@ -117,11 +117,10 @@ public:
store._get_db_unsafe()->Write(rocksdb::WriteOptions(), &write_batch);
}
} else {
std::cerr << "Replication error while fetching records from master, status_code="
<< status_code << std::endl;
LOG(FATAL) << "Replication error while fetching records from master, status_code=" << status_code;
if(status_code != 0) {
std::cerr << json_response << std::endl;
LOG(FATAL) << json_response;
}
}
@ -142,8 +141,8 @@ public:
try {
collection_meta = nlohmann::json::parse(replication_event->value);
} catch(...) {
std::cerr << "Failed to parse collection meta JSON." << std::endl;
std::cerr << "Replication event value: " << replication_event->value << std::endl;
LOG(FATAL) << "Failed to parse collection meta JSON.";
LOG(FATAL) << "Replication event value: " << replication_event->value;
delete replication_event;
exit(1);
}

7
include/store.h
View File

@ -11,6 +11,7 @@
#include <rocksdb/options.h>
#include <rocksdb/merge_operator.h>
#include <rocksdb/transaction_log.h>
#include "logger.h"
class UInt64AddOperator : public rocksdb::AssociativeMergeOperator {
public:
@ -68,7 +69,7 @@ public:
// open DB
rocksdb::Status s = rocksdb::DB::Open(options, state_dir_path, &db);
if(!s.ok()) {
std::cerr << s.ToString() << std::endl;
LOG(FATAL) << s.ToString();
}
assert(s.ok());
}
@ -195,10 +196,10 @@ public:
void print_memory_usage() {
std::string index_usage;
db->GetProperty("rocksdb.estimate-table-readers-mem", &index_usage);
std::cout << "rocksdb.estimate-table-readers-mem: " << index_usage << std::endl;
LOG(INFO) << "rocksdb.estimate-table-readers-mem: " << index_usage;
std::string memtable_usage;
db->GetProperty("rocksdb.cur-size-all-mem-tables", &memtable_usage);
std::cout << "rocksdb.cur-size-all-mem-tables: " << memtable_usage << std::endl;
LOG(INFO) << "rocksdb.cur-size-all-mem-tables: " << memtable_usage;
}
};

5
src/api.cpp
View File

@ -7,6 +7,7 @@
#include "string_utils.h"
#include "collection.h"
#include "collection_manager.h"
#include "logger.h"
nlohmann::json collection_summary_json(Collection *collection) {
nlohmann::json json_response;
@ -273,7 +274,7 @@ void get_search(http_req & req, http_res & res) {
//struct rusage r_usage;
//getrusage(RUSAGE_SELF,&r_usage);
//std::cout << "Memory usage: " << r_usage.ru_maxrss << std::endl;
//LOG(INFO) << "Memory usage: " << r_usage.ru_maxrss;
if(req.params.count(CALLBACK) == 0) {
res.send_200(results_json_str);
@ -281,7 +282,7 @@ void get_search(http_req & req, http_res & res) {
res.send_200(req.params[CALLBACK] + "(" + results_json_str + ");");
}
std::cout << "Time taken: " << timeMillis << "ms" << std::endl;
LOG(INFO) << "Time taken: " << timeMillis << "ms";
}
void get_collection_summary(http_req & req, http_res & res) {

15
src/art.cpp
View File

@ -13,6 +13,7 @@
#include <queue>
#include <stdint.h>
#include "art.h"
#include "logger.h"
/**
* Macros to manipulate pointer tags
@ -931,7 +932,7 @@ int art_topk_iter(const art_node *root, token_ordering token_order, size_t max_r
int idx;
switch (n->type) {
case NODE4:
//std::cout << "\nNODE4, SCORE: " << n->max_token_count << std::endl;
//LOG(INFO) << "\nNODE4, SCORE: " << n->max_token_count;
for (int i=0; i < n->num_children; i++) {
art_node* child = ((art_node4*)n)->children[i];
q.push(child);
@ -939,25 +940,25 @@ int art_topk_iter(const art_node *root, token_ordering token_order, size_t max_r
break;
case NODE16:
//std::cout << "\nNODE16, SCORE: " << n->max_token_count << std::endl;
//LOG(INFO) << "\nNODE16, SCORE: " << n->max_token_count;
for (int i=0; i < n->num_children; i++) {
q.push(((art_node16*)n)->children[i]);
}
break;
case NODE48:
//std::cout << "\nNODE48, SCORE: " << n->max_token_count << std::endl;
//LOG(INFO) << "\nNODE48, SCORE: " << n->max_token_count;
for (int i=0; i < 256; i++) {
idx = ((art_node48*)n)->keys[i];
if (!idx) continue;
art_node *child = ((art_node48*)n)->children[idx - 1];
//std::cout << "--PUSHING NODE48 CHILD WITH SCORE: " << get_score(child) << std::endl;
//LOG(INFO) << "--PUSHING NODE48 CHILD WITH SCORE: " << get_score(child);
q.push(child);
}
break;
case NODE256:
//std::cout << "\nNODE256, SCORE: " << n->max_token_count << std::endl;
//LOG(INFO) << "\nNODE256, SCORE: " << n->max_token_count;
for (int i=0; i < 256; i++) {
if (!((art_node256*)n)->children[i]) continue;
q.push(((art_node256*)n)->children[i]);
@ -1362,7 +1363,7 @@ int art_fuzzy_search(art_tree *t, const unsigned char *term, const int term_len,
}
long long int time_micro = microseconds(std::chrono::high_resolution_clock::now() - begin).count();
//!std::cout << "Time taken for fuzz: " << time_micro << "us, size of nodes: " << nodes.size() << std::endl;
//!LOG(INFO) << "Time taken for fuzz: " << time_micro << "us, size of nodes: " << nodes.size();
begin = std::chrono::high_resolution_clock::now();
@ -1377,7 +1378,7 @@ int art_fuzzy_search(art_tree *t, const unsigned char *term, const int term_len,
}
time_micro = microseconds(std::chrono::high_resolution_clock::now() - begin).count();
//!std::cout << "Time taken for art_topk_iter: " << time_micro << "us" << std::endl;
//!LOG(INFO) << "Time taken for art_topk_iter: " << time_micro << "us";
return 0;
}

3
src/collection.cpp
View File

@ -9,6 +9,7 @@
#include <thread>
#include <chrono>
#include <rocksdb/write_batch.h>
#include "logger.h"
Collection::Collection(const std::string name, const uint32_t collection_id, const uint32_t next_seq_id, Store *store,
const std::vector<field> &fields, const std::string & token_ranking_field):
@ -631,7 +632,7 @@ Option<nlohmann::json> Collection::search(std::string query, const std::vector<s
}
//long long int timeMillis = std::chrono::duration_cast<std::chrono::microseconds>(std::chrono::high_resolution_clock::now() - begin).count();
//!std::cout << "Time taken for result calc: " << timeMillis << "us" << std::endl;
//!LOG(INFO) << "Time taken for result calc: " << timeMillis << "us";
//!store->print_memory_usage();
return result;
}

12
src/http_server.cpp
View File

@ -6,6 +6,7 @@
#include <signal.h>
#include <h2o.h>
#include <iostream>
#include "logger.h"
struct h2o_custom_req_handler_t {
h2o_handler_t super;
@ -69,7 +70,7 @@ int HttpServer::setup_ssl(const char *cert_file, const char *key_file) {
int nid = NID_X9_62_prime256v1;
EC_KEY *key = EC_KEY_new_by_curve_name(nid);
if (key == NULL) {
std::cout << "Failed to create DH/ECDH." << std::endl;
LOG(INFO) << "Failed to create DH/ECDH.";
return -1;
}
@ -79,11 +80,11 @@ int HttpServer::setup_ssl(const char *cert_file, const char *key_file) {
SSL_CTX_set_options(accept_ctx->ssl_ctx, SSL_OP_NO_SSLv2);
if (SSL_CTX_use_certificate_file(accept_ctx->ssl_ctx, cert_file, SSL_FILETYPE_PEM) != 1) {
std::cout << "An error occurred while trying to load server certificate file:" << cert_file << std::endl;
LOG(INFO) << "An error occurred while trying to load server certificate file:" << cert_file;
return -1;
}
if (SSL_CTX_use_PrivateKey_file(accept_ctx->ssl_ctx, key_file, SSL_FILETYPE_PEM) != 1) {
std::cout << "An error occurred while trying to load private key file: " << key_file << std::endl;
LOG(INFO) << "An error occurred while trying to load private key file: " << key_file;
return -1;
}
@ -132,11 +133,10 @@ int HttpServer::run() {
h2o_multithread_register_receiver(message_queue, message_receiver, on_message);
if (create_listener() != 0) {
std::cerr << "Failed to listen on " << listen_address << ":" << listen_port << " - "
<< strerror(errno) << std::endl;
LOG(FATAL) << "Failed to listen on " << listen_address << ":" << listen_port << " - " << strerror(errno);
return 1;
} else {
std::cout << "Server has started. Ready to accept requests on port " << listen_port << std::endl;
LOG(INFO) << "Typesense has started. Ready to accept requests on port " << listen_port;
}
on(STOP_SERVER_MESSAGE, HttpServer::on_stop_server);

22
src/index.cpp
View File

@ -7,6 +7,7 @@
#include <match_score.h>
#include <string_utils.h>
#include <art.h>
#include "logger.h"
Index::Index(const std::string name, std::unordered_map<std::string, field> search_schema,
std::unordered_map<std::string, field> facet_schema, std::unordered_map<std::string, field> sort_schema):
@ -331,7 +332,7 @@ void Index::search_candidates(uint32_t* filter_ids, size_t filter_ids_length, st
std::vector<art_leaf *> query_suggestion = next_suggestion(token_candidates_vec, n);
/*for(auto i=0; i < query_suggestion.size(); i++) {
std::cout << "i: " << i << " - " << query_suggestion[i]->key << std::endl;
LOG(INFO) << "i: " << i << " - " << query_suggestion[i]->key;
}*/
// initialize results with the starting element (for further intersection)
@ -604,7 +605,7 @@ void Index::search(Option<uint32_t> & outcome, std::string query, const std::vec
delete [] all_result_ids;
//long long int timeMillis = std::chrono::duration_cast<std::chrono::microseconds>(std::chrono::high_resolution_clock::now() - begin).count();
//!std::cout << "Time taken for result calc: " << timeMillis << "us" << std::endl;
//!LOG(INFO) << "Time taken for result calc: " << timeMillis << "us";
outcome = Option<uint32_t>(field_order_kvs.size());
}
@ -684,7 +685,7 @@ void Index::search_field(std::string & query, const std::string & field, uint32_
const std::string token_cost_hash = token + std::to_string(costs[token_index]);
std::vector<art_leaf*> leaves;
//std::cout << "\nSearching for: " << token << " - cost: " << costs[token_index] << std::endl;
//LOG(INFO) << "\nSearching for: " << token << " - cost: " << costs[token_index];
if(token_cost_cache.count(token_cost_hash) != 0) {
leaves = token_cost_cache[token_cost_hash];
@ -774,12 +775,13 @@ void Index::search_field(std::string & query, const std::string & field, uint32_
}
void Index::log_leaves(const int cost, const std::string &token, const std::vector<art_leaf *> &leaves) const {
printf("Token: %s, cost: %d, candidates: \n", token.c_str(), cost);
LOG(INFO) << "Token: " << token << ", cost: " << cost;
for(size_t i=0; i < leaves.size(); i++) {
printf("%.*s, ", leaves[i]->key_len, leaves[i]->key);
printf("frequency: %d, max_score: %d\n", leaves[i]->values->ids.getLength(), leaves[i]->max_score);
LOG(INFO) << "frequency: " << leaves[i]->values->ids.getLength() << ", max_score: " << leaves[i]->max_score;
/*for(auto j=0; j<leaves[i]->values->ids.getLength(); j++) {
printf("id: %d\n", leaves[i]->values->ids.at(j));
LOG(INFO) << "id: " << leaves[i]->values->ids.at(j);
}*/
}
}
@ -887,11 +889,11 @@ void Index::score_results(const std::vector<sort_by> & sort_fields, const int &
<< ", words_present: " << match.words_present << ", match_score: " << match
<< ", primary_rank_score: " << primary_rank_score.intval << ", distance: " << (MAX_SEARCH_TOKENS - match.distance)
<< ", seq_id: " << seq_id << std::endl;
std::cout << os.str();*/
LOG(INFO) << os.str();*/
}
//long long int timeNanos = std::chrono::duration_cast<std::chrono::milliseconds>(std::chrono::high_resolution_clock::now() - begin).count();
//std::cout << "Time taken for results iteration: " << timeNanos << "ms" << std::endl;
//LOG(INFO) << "Time taken for results iteration: " << timeNanos << "ms";
for (auto it = leaf_to_indices.begin(); it != leaf_to_indices.end(); it++) {
delete [] it->second;
@ -1078,9 +1080,9 @@ Option<uint32_t> Index::remove(const uint32_t seq_id, nlohmann::json & document)
/*len = leaf->values->offset_index.getLength();
for(auto i=0; i<len; i++) {
std::cout << "i: " << i << ", val: " << leaf->values->offset_index.at(i) << std::endl;
LOG(INFO) << "i: " << i << ", val: " << leaf->values->offset_index.at(i);
}
std::cout << "----" << std::endl;*/
LOG(INFO) << "----";*/
if(leaf->values->ids.getLength() == 0) {
art_values* values = (art_values*) art_delete(search_index.at(name_field.first), key, key_len);

6
src/main/benchmark.cpp
View File

@ -38,7 +38,7 @@ int main(int argc, char* argv[]) {
}
infile.close();
cout << "FINISHED INDEXING!" << endl << flush;
std::cout << "FINISHED INDEXING!" << flush << std::endl;
/*std::vector<std::string> search_fields = {"title"};
@ -56,7 +56,7 @@ int main(int argc, char* argv[]) {
}*/
long long int timeMillis = std::chrono::duration_cast<std::chrono::milliseconds>(std::chrono::high_resolution_clock::now() - begin).count();
cout << "Time taken: " << timeMillis << "ms" << endl;
//cout << "Total: " << results_total << endl;
std::cout << "Time taken: " << timeMillis << "ms" << std::endl;
//std::cout << "Total: " << results_total << std::endl;
return 0;
}

2
src/main/main.cpp
View File

@ -44,7 +44,7 @@ int main(int argc, char* argv[]) {
std::ifstream infile(argv[1]);
std::string json_line;
cout << "BEGINNING Iteration: " << j << endl << flush;
std::cout << "BEGINNING Iteration: " << j << std::endl;
auto begin = std::chrono::high_resolution_clock::now();
int doc_id = 0;

37
src/main/typesense_server.cpp
View File

@ -11,11 +11,12 @@
#include "string_utils.h"
#include "replicator.h"
#include "typesense_version.h"
#include "logger.h"
HttpServer* server;
void catch_interrupt(int sig) {
std::cout << "Stopping Typesense server..." << std::endl;
LOG(INFO) << "Stopping Typesense server...";
signal(sig, SIG_IGN); // ignore for now as we want to shut down elegantly
server->stop();
}
@ -78,16 +79,35 @@ int main(int argc, char **argv) {
options.add<std::string>("ssl-certificate-key", 'e', "Path to the SSL certificate key file.", false, "");
options.add("enable-cors", '\0', "Enable CORS requests.");
options.add<std::string>("log-dir", '\0', "Path to the log file.", false, "");
options.parse_check(argc, argv);
auto log_worker = g3::LogWorker::createLogWorker();
std::string log_dir = options.get<std::string>("log-dir");
if(log_dir.empty()) {
log_worker->addSink(std2::make_unique<ConsoleLoggingSink>(),
&ConsoleLoggingSink::ReceiveLogMessage);
} else {
if(!directory_exists(log_dir)) {
std::cerr << "Typesense failed to start. " << "Log directory " << log_dir << " does not exist.";
return 1;
}
log_worker->addDefaultLogger("typesense", log_dir, "");
std::cout << "Typesense has started. Log directory is configured as: " << log_dir << std::endl;
}
g3::initializeLogging(log_worker.get());
signal(SIGINT, catch_interrupt);
std::cout << "Typesense version " << TYPESENSE_VERSION << std::endl;
LOG(INFO) << "Typesense version: " << TYPESENSE_VERSION;
if(!directory_exists(options.get<std::string>("data-dir"))) {
std::cerr << "Typesense failed to start. " << "Data directory " << options.get<std::string>("data-dir")
<< " does not exist." << std::endl;
LOG(FATAL) << "Typesense failed to start. " << "Data directory " << options.get<std::string>("data-dir")
<< " does not exist.";
return 1;
}
@ -97,10 +117,9 @@ int main(int argc, char **argv) {
options.get<std::string>("search-only-api-key"));
if(init_op.ok()) {
std::cout << "Finished loading collections from disk." << std::endl;
LOG(INFO) << "Finished loading collections from disk.";
} else {
std::cerr << "Typesense failed to start. " << "Could not load collections from disk: "
<< init_op.error() << std::endl;
LOG(FATAL)<< "Typesense failed to start. " << "Could not load collections from disk: " << init_op.error();
return 1;
}
@ -126,11 +145,11 @@ int main(int argc, char **argv) {
std::vector<std::string> parts;
StringUtils::split(master_host_port, parts, ":");
if(parts.size() != 3) {
std::cerr << "Invalid value for --master option. Usage: http(s)://<master_address>:<master_port>" << std::endl;
LOG(FATAL) << "Invalid value for --master option. Usage: http(s)://<master_address>:<master_port>";
return 1;
}
std::cout << "Typesense server started as a read-only replica... Spawning replication thread..." << std::endl;
LOG(INFO) << "Typesense is starting as a read-only replica... Spawning replication thread...";
std::thread replication_thread([&master_host_port, &store, &options]() {
Replicator::start(::server, master_host_port, options.get<std::string>("api-key"), store);
});

17
test/art_test.cpp
View File

@ -878,12 +878,6 @@ TEST(ArtTest, test_int32_million) {
}
encode_int32(5, chars);
/*std::cout << std::endl;
for(uint32_t i = 0; i < CHAR_LEN; i++) {
std::cout << (int)chars[i] << ", ";
}
std::cout << std::endl;*/
std::vector<const art_leaf*> results;
// ==
@ -972,16 +966,6 @@ TEST(ArtTest, test_int_range_byte_boundary) {
results.clear();
art_int32_search(&t, 255, GREATER_THAN, results);
ASSERT_EQ(44, results.size());
/*std::cout << std::endl;
for(auto i = 0; i < 1; i++) {
auto result = results[i];
for(auto j = 0; j < result->key_len; j++) {
std::cout << (int) result->key[j] << ", ";
}
std::cout << std::endl;
}*/
}
TEST(ArtTest, test_encode_int64) {
@ -1007,7 +991,6 @@ TEST(ArtTest, test_encode_int64) {
unsigned char chars_large_num[8] = {0, 0, 0, 0, 128, 0, 0, 199};
int64_t large_num = (int64_t)(std::numeric_limits<std::int32_t>::max()) + 200;
std::cout << std::endl << large_num << std::endl;
encode_int64(large_num, chars);
for(uint32_t i = 0; i < 8; i++) {
ASSERT_EQ(chars_large_num[i], chars[i]);

2
test/collection_manager_test.cpp
View File

@ -15,7 +15,7 @@ protected:
void setupCollection() {
std::string state_dir_path = "/tmp/typesense_test/coll_manager_test_db";
std::cout << "Truncating and creating: " << state_dir_path << std::endl;
LOG(INFO) << "Truncating and creating: " << state_dir_path;
system(("rm -rf "+state_dir_path+" && mkdir -p "+state_dir_path).c_str());
store = new Store(state_dir_path);

2
test/collection_test.cpp
View File

@ -17,7 +17,7 @@ protected:
void setupCollection() {
std::string state_dir_path = "/tmp/typesense_test/collection";
std::cout << "Truncating and creating: " << state_dir_path << std::endl;
LOG(INFO) << "Truncating and creating: " << state_dir_path;
system(("rm -rf "+state_dir_path+" && mkdir -p "+state_dir_path).c_str());
store = new Store(state_dir_path);

23
test/main.cpp Normal file
View File

@ -0,0 +1,23 @@
#include <gtest/gtest.h>
#include "logger.h"
class TypesenseTestEnvironment : public testing::Environment {
public:
virtual void SetUp() {
auto log_worker = g3::LogWorker::createLogWorker();
auto sink_handle = log_worker->addSink(std2::make_unique<ConsoleLoggingSink>(),
&ConsoleLoggingSink::ReceiveLogMessage);
g3::initializeLogging(log_worker.get());
}
virtual void TearDown() {
}
};
int main(int argc, char **argv) {
::testing::InitGoogleTest(&argc, argv);
::testing::AddGlobalTestEnvironment(new TypesenseTestEnvironment);
int exitCode = RUN_ALL_TESTS();
return exitCode;
}

4
test/store_test.cpp
View File

@ -5,7 +5,7 @@
TEST(StoreTest, GetUpdatesSince) {
std::string primary_store_path = "/tmp/typesense_test/primary_store_test";
std::cout << "Truncating and creating: " << primary_store_path << std::endl;
LOG(INFO) << "Truncating and creating: " << primary_store_path;
system(("rm -rf "+primary_store_path+" && mkdir -p "+primary_store_path).c_str());
// add some records, get the updates and restore them in a new store
@ -39,7 +39,7 @@ TEST(StoreTest, GetUpdatesSince) {
ASSERT_EQ(3, updates_op.get()->size());
std::string replica_store_path = "/tmp/typesense_test/replica_store_test";
std::cout << "Truncating and creating: " << replica_store_path << std::endl;
LOG(INFO) << "Truncating and creating: " << replica_store_path;
system(("rm -rf "+replica_store_path+" && mkdir -p "+replica_store_path).c_str());
Store replica_store(replica_store_path);

1
test/string_utils_test.cpp
View File

@ -17,6 +17,5 @@ TEST(StringUtilsTest, ShouldNormalizeString) {
// retain non-ascii unicode characters and should also lower case them
std::string alphanum_unicodechars = "abcÅà123";
StringUtils::normalize(alphanum_unicodechars);
std::cout << alphanum_unicodechars << std::endl;
ASSERT_STREQ("abcåà123", alphanum_unicodechars.c_str());
}