diff --git a/CMakeLists.txt b/CMakeLists.txt index 11ae50f4..95164726 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -40,6 +40,7 @@ include(cmake/H2O.cmake) include(cmake/RocksDB.cmake) include(cmake/GoogleTest.cmake) include(cmake/TestResources.cmake) +include(cmake/G3log.cmake) FILE(GLOB SRC_FILES src/*.cpp) @@ -51,17 +52,19 @@ include_directories(${DEP_ROOT_DIR}/${MINIUTF_NAME}) include_directories(${DEP_ROOT_DIR}/${GTEST_NAME}/googletest/include) include_directories(${DEP_ROOT_DIR}/${H2O_NAME}/include) include_directories(${DEP_ROOT_DIR}/${ROCKSDB_NAME}/include) +include_directories(${DEP_ROOT_DIR}/${G3LOG_NAME}/src) link_directories(${DEP_ROOT_DIR}/${GTEST_NAME}/googletest/build) link_directories(${DEP_ROOT_DIR}/${FOR_NAME}) link_directories(${DEP_ROOT_DIR}/${MINIUTF_NAME}/build) link_directories(${DEP_ROOT_DIR}/${H2O_NAME}/build) link_directories(${DEP_ROOT_DIR}/${ROCKSDB_NAME}) +link_directories(${DEP_ROOT_DIR}/${G3LOG_NAME}/build) add_executable(typesense-server ${SRC_FILES} src/main/typesense_server.cpp) add_executable(search ${SRC_FILES} src/main/main.cpp) add_executable(benchmark ${SRC_FILES} src/main/benchmark.cpp) -add_executable(typesense_test ${SRC_FILES} test/array_test.cpp test/sorted_array_test.cpp test/art_test.cpp +add_executable(typesense_test ${SRC_FILES} test/main.cpp test/array_test.cpp test/sorted_array_test.cpp test/art_test.cpp test/collection_test.cpp test/collection_manager_test.cpp test/topster_test.cpp test/match_score_test.cpp test/store_test.cpp test/array_utils_test.cpp test/string_utils_test.cpp) @@ -86,7 +89,7 @@ if(NOT APPLE) list(APPEND ROCKSDB_LIBS rt) endif() -target_link_libraries(typesense-server h2o-evloop for miniutf pthread ${CURL_LIBRARIES} ${ROCKSDB_LIBS} ${OPENSSL_LIBRARIES} dl ${STD_LIB}) -target_link_libraries(search for miniutf pthread h2o-evloop ${CURL_LIBRARIES} ${ROCKSDB_LIBS} ${OPENSSL_LIBRARIES} dl ${STD_LIB}) -target_link_libraries(benchmark for miniutf pthread ${CURL_LIBRARIES} h2o-evloop ${ROCKSDB_LIBS} ${OPENSSL_LIBRARIES} dl ${STD_LIB}) -target_link_libraries(typesense_test h2o-evloop ${OPENSSL_LIBRARIES} pthread for miniutf ${ROCKSDB_LIBS} gtest gtest_main dl ${STD_LIB}) +target_link_libraries(typesense-server h2o-evloop for miniutf g3logger pthread ${CURL_LIBRARIES} ${ROCKSDB_LIBS} ${OPENSSL_LIBRARIES} dl ${STD_LIB}) +target_link_libraries(search for miniutf g3logger pthread h2o-evloop ${CURL_LIBRARIES} ${ROCKSDB_LIBS} ${OPENSSL_LIBRARIES} dl ${STD_LIB}) +target_link_libraries(benchmark for miniutf g3logger pthread ${CURL_LIBRARIES} h2o-evloop ${ROCKSDB_LIBS} ${OPENSSL_LIBRARIES} dl ${STD_LIB}) +target_link_libraries(typesense_test h2o-evloop ${OPENSSL_LIBRARIES} pthread for miniutf g3logger ${ROCKSDB_LIBS} gtest gtest_main dl ${STD_LIB}) diff --git a/README.md b/README.md index 16b271d9..288ced33 100644 --- a/README.md +++ b/README.md @@ -78,7 +78,7 @@ Typesense requires the following dependencies: * zlib * OpenSSL (>=1.0.2) * curl -* Compiler with good C++11 support (e.g. GCC >= 4.9) +* Compiler with good C++11 support (GCC >= 4.9.0, Apple Clang >= 8.0, Clang >= 3.9.0) ``` $ ./build.sh [--clean] [--depclean] diff --git a/cmake/Easyloggingpp.cmake b/cmake/Easyloggingpp.cmake new file mode 100644 index 00000000..bd8efdbf --- /dev/null +++ b/cmake/Easyloggingpp.cmake @@ -0,0 +1,38 @@ +# Download and build EASYLOGGINGPP + +set(EASYLOGGINGPP_VERSION 9.95.3) +set(EASYLOGGINGPP_NAME easyloggingpp-${EASYLOGGINGPP_VERSION}) +set(EASYLOGGINGPP_TAR_PATH ${DEP_ROOT_DIR}/${EASYLOGGINGPP_NAME}.tar.gz) + +if(NOT EXISTS ${EASYLOGGINGPP_TAR_PATH}) + message(STATUS "Downloading Easyloggingpp...") + file(DOWNLOAD https://github.com/muflihun/easyloggingpp/archive/v${EASYLOGGINGPP_VERSION}.tar.gz ${EASYLOGGINGPP_TAR_PATH}) +endif() + +if(NOT EXISTS ${DEP_ROOT_DIR}/${EASYLOGGINGPP_NAME}) + message(STATUS "Extracting Easyloggingpp...") + execute_process(COMMAND ${CMAKE_COMMAND} -E tar xzf ${EASYLOGGINGPP_TAR_PATH} WORKING_DIRECTORY ${DEP_ROOT_DIR}) +endif() + +if(NOT EXISTS ${DEP_ROOT_DIR}/${EASYLOGGINGPP_NAME}/build) + message("Configuring Easyloggingpp...") + file(MAKE_DIRECTORY ${DEP_ROOT_DIR}/${EASYLOGGINGPP_NAME}/build) + execute_process(COMMAND ${CMAKE_COMMAND} + "-Dbuild_static_lib=ON" + "-H${DEP_ROOT_DIR}/${EASYLOGGINGPP_NAME}" + "-B${DEP_ROOT_DIR}/${EASYLOGGINGPP_NAME}/build" + RESULT_VARIABLE + EASYLOGGINGPP_CONFIGURE) + if(NOT EASYLOGGINGPP_CONFIGURE EQUAL 0) + message(FATAL_ERROR "Easyloggingpp Configure failed!") + endif() + + message("Building Easyloggingpp locally...") + execute_process(COMMAND ${CMAKE_COMMAND} --build + "${DEP_ROOT_DIR}/${EASYLOGGINGPP_NAME}/build" + RESULT_VARIABLE + EASYLOGGINGPP_BUILD) + if(NOT EASYLOGGINGPP_BUILD EQUAL 0) + message(FATAL_ERROR "Easyloggingpp build failed!") + endif() +endif() \ No newline at end of file diff --git a/cmake/g3log.cmake b/cmake/g3log.cmake new file mode 100644 index 00000000..9df76297 --- /dev/null +++ b/cmake/g3log.cmake @@ -0,0 +1,38 @@ +# Download and build g3log + +set(G3LOG_VERSION 1.3) +set(G3LOG_NAME g3log-${G3LOG_VERSION}) +set(G3LOG_TAR_PATH ${DEP_ROOT_DIR}/${G3LOG_NAME}.tar.gz) + +if(NOT EXISTS ${G3LOG_TAR_PATH}) + message(STATUS "Downloading G3log...") + file(DOWNLOAD https://github.com/KjellKod/g3log/archive/${G3LOG_VERSION}.tar.gz ${G3LOG_TAR_PATH}) +endif() + +if(NOT EXISTS ${DEP_ROOT_DIR}/${G3LOG_NAME}) + message(STATUS "Extracting G3log...") + execute_process(COMMAND ${CMAKE_COMMAND} -E tar xzf ${G3LOG_TAR_PATH} WORKING_DIRECTORY ${DEP_ROOT_DIR}) +endif() + +if(NOT EXISTS ${DEP_ROOT_DIR}/${G3LOG_NAME}/build) + message("Configuring G3log...") + file(MAKE_DIRECTORY ${DEP_ROOT_DIR}/${G3LOG_NAME}/build) + execute_process(COMMAND ${CMAKE_COMMAND} + "-DCMAKE_BUILD_TYPE=Release" + "-H${DEP_ROOT_DIR}/${G3LOG_NAME}" + "-B${DEP_ROOT_DIR}/${G3LOG_NAME}/build" + RESULT_VARIABLE + G3LOG_CONFIGURE) + if(NOT G3LOG_CONFIGURE EQUAL 0) + message(FATAL_ERROR "G3log Configure failed!") + endif() + + message("Building G3log locally...") + execute_process(COMMAND ${CMAKE_COMMAND} --build + "${DEP_ROOT_DIR}/${G3LOG_NAME}/build" + RESULT_VARIABLE + G3LOG_BUILD) + if(NOT G3LOG_BUILD EQUAL 0) + message(FATAL_ERROR "G3log build failed!") + endif() +endif() \ No newline at end of file diff --git a/include/cmdline.h b/include/cmdline.h index d7ee2927..67879084 100644 --- a/include/cmdline.h +++ b/include/cmdline.h @@ -38,6 +38,7 @@ #include #include #include +#include "logger.h" namespace cmdline{ @@ -408,7 +409,7 @@ namespace cmdline{ args.push_back(buf); for (size_t i=0; i +#include +#include + +// https://github.com/KjellKod/g3sinks/blob/master/snippets/ColorCoutSink.hpp + +struct ConsoleLoggingSink { + enum FG_Color {YELLOW = 33, RED = 31, GREEN=32, WHITE = 97}; + + void ReceiveLogMessage(g3::LogMessageMover logEntry) { + auto level = logEntry.get()._level; + + if (g3::internal::wasFatal(level)) { + std::cerr << "\033[" << RED << "m" + << logEntry.get().toString() << "\033[m"; + } else { + std::cout << logEntry.get().toString(); + } + } +}; \ No newline at end of file diff --git a/include/exectime.h b/include/exectime.h index 3e269eee..ef61c072 100644 --- a/include/exectime.h +++ b/include/exectime.h @@ -1,6 +1,7 @@ #pragma once #include #include +#include "logger.h" class ExecTime { static auto begin = std::chrono::high_resolution_clock::now(); @@ -12,6 +13,6 @@ class ExecTime { static void log(std::string operation) { long long int timeMicros = std::chrono::duration_cast( std::chrono::high_resolution_clock::now() - begin).count(); - std::cout << "Time taken for " << operation << ": " << timeMicros << "us" << std::endl; + LOG(INFO) << "Time taken for " << operation << ": " << timeMicros << "us"; } }; \ No newline at end of file diff --git a/include/logger.h b/include/logger.h new file mode 100644 index 00000000..e899d493 --- /dev/null +++ b/include/logger.h @@ -0,0 +1,5 @@ +#pragma once + +#include +#include +#include "console_logging_sink.h" \ No newline at end of file diff --git a/include/match_score.h b/include/match_score.h index cedb5eb0..10265e44 100644 --- a/include/match_score.h +++ b/include/match_score.h @@ -5,6 +5,7 @@ #include #include #include +#include "logger.h" #ifdef DEBUG #define D(x) x @@ -46,9 +47,9 @@ struct Match { static void print_token_offsets(std::vector> &token_offsets) { for(auto offsets: token_offsets) { for(auto offset: offsets) { - std::cout << offset << ", "; + LOG(INFO) << offset << ", "; } - std::cout << std::endl; + LOG(INFO) << ""; } } @@ -114,7 +115,7 @@ struct Match { addTopOfHeapToWindow(heap, window, token_offsets, token_offset); } - D(std::cout << "Loop till window fills... doc_id: " << doc_id << std::endl;) + D(LOG(INFO) << "Loop till window fills... doc_id: " << doc_id;) // Fill the queue with tokens within a given window frame size of the start offset // At the same time, we also record the *last* occurrence of each token within the window @@ -124,7 +125,7 @@ struct Match { addTopOfHeapToWindow(heap, window, token_offsets, token_offset); } - D(std::cout << std::endl << "----" << std::endl); + D(LOG(INFO) << "----"); uint16_t prev_pos = MAX_DISPLACEMENT; uint16_t num_match = 0; @@ -140,14 +141,14 @@ struct Match { } else { // Calculate the distance between the tokens within the window // Ideally, this should be (NUM_TOKENS - 1) when all the tokens are adjacent to each other - D(std::cout << "prev_pos: " << prev_pos << " , curr_pos: " << token_offset[token_id] << std::endl); + D(LOG(INFO) << "prev_pos: " << prev_pos << " , curr_pos: " << token_offset[token_id]); displacement += abs(token_offset[token_id]-prev_pos); prev_pos = token_offset[token_id]; } } } - D(std::cout << std::endl << "!!!displacement: " << displacement << " | num_match: " << num_match << std::endl); + D(LOG(INFO) << std::endl << "!!!displacement: " << displacement << " | num_match: " << num_match); // Track the best `displacement` and `num_match` seen so far across all the windows // for a single token, displacement will be 0, while for 2 tokens minimum dispacement would be 1 diff --git a/include/replicator.h b/include/replicator.h index b784e873..84220aac 100644 --- a/include/replicator.h +++ b/include/replicator.h @@ -8,7 +8,7 @@ #include #include #include - +#include "logger.h" struct ReplicationEvent { std::string type; @@ -93,7 +93,7 @@ public: while(true) { IterateBatchHandler handler(server); uint64_t latest_seq_num = store.get_latest_seq_number(); - std::cout << "latest_seq_num: " << latest_seq_num << std::endl; + LOG(INFO) << "latest_seq_num: " << latest_seq_num; HttpClient client( master_host_port+"/replication/updates?seq_number="+std::to_string(latest_seq_num+1), api_key @@ -117,11 +117,10 @@ public: store._get_db_unsafe()->Write(rocksdb::WriteOptions(), &write_batch); } } else { - std::cerr << "Replication error while fetching records from master, status_code=" - << status_code << std::endl; + LOG(FATAL) << "Replication error while fetching records from master, status_code=" << status_code; if(status_code != 0) { - std::cerr << json_response << std::endl; + LOG(FATAL) << json_response; } } @@ -142,8 +141,8 @@ public: try { collection_meta = nlohmann::json::parse(replication_event->value); } catch(...) { - std::cerr << "Failed to parse collection meta JSON." << std::endl; - std::cerr << "Replication event value: " << replication_event->value << std::endl; + LOG(FATAL) << "Failed to parse collection meta JSON."; + LOG(FATAL) << "Replication event value: " << replication_event->value; delete replication_event; exit(1); } diff --git a/include/store.h b/include/store.h index 8ca371d2..65df539f 100644 --- a/include/store.h +++ b/include/store.h @@ -11,6 +11,7 @@ #include #include #include +#include "logger.h" class UInt64AddOperator : public rocksdb::AssociativeMergeOperator { public: @@ -68,7 +69,7 @@ public: // open DB rocksdb::Status s = rocksdb::DB::Open(options, state_dir_path, &db); if(!s.ok()) { - std::cerr << s.ToString() << std::endl; + LOG(FATAL) << s.ToString(); } assert(s.ok()); } @@ -195,10 +196,10 @@ public: void print_memory_usage() { std::string index_usage; db->GetProperty("rocksdb.estimate-table-readers-mem", &index_usage); - std::cout << "rocksdb.estimate-table-readers-mem: " << index_usage << std::endl; + LOG(INFO) << "rocksdb.estimate-table-readers-mem: " << index_usage; std::string memtable_usage; db->GetProperty("rocksdb.cur-size-all-mem-tables", &memtable_usage); - std::cout << "rocksdb.cur-size-all-mem-tables: " << memtable_usage << std::endl; + LOG(INFO) << "rocksdb.cur-size-all-mem-tables: " << memtable_usage; } }; \ No newline at end of file diff --git a/src/api.cpp b/src/api.cpp index 8a43d80d..061dab33 100644 --- a/src/api.cpp +++ b/src/api.cpp @@ -7,6 +7,7 @@ #include "string_utils.h" #include "collection.h" #include "collection_manager.h" +#include "logger.h" nlohmann::json collection_summary_json(Collection *collection) { nlohmann::json json_response; @@ -273,7 +274,7 @@ void get_search(http_req & req, http_res & res) { //struct rusage r_usage; //getrusage(RUSAGE_SELF,&r_usage); - //std::cout << "Memory usage: " << r_usage.ru_maxrss << std::endl; + //LOG(INFO) << "Memory usage: " << r_usage.ru_maxrss; if(req.params.count(CALLBACK) == 0) { res.send_200(results_json_str); @@ -281,7 +282,7 @@ void get_search(http_req & req, http_res & res) { res.send_200(req.params[CALLBACK] + "(" + results_json_str + ");"); } - std::cout << "Time taken: " << timeMillis << "ms" << std::endl; + LOG(INFO) << "Time taken: " << timeMillis << "ms"; } void get_collection_summary(http_req & req, http_res & res) { diff --git a/src/art.cpp b/src/art.cpp index a2739e7f..d391ab8d 100644 --- a/src/art.cpp +++ b/src/art.cpp @@ -13,6 +13,7 @@ #include #include #include "art.h" +#include "logger.h" /** * Macros to manipulate pointer tags @@ -931,7 +932,7 @@ int art_topk_iter(const art_node *root, token_ordering token_order, size_t max_r int idx; switch (n->type) { case NODE4: - //std::cout << "\nNODE4, SCORE: " << n->max_token_count << std::endl; + //LOG(INFO) << "\nNODE4, SCORE: " << n->max_token_count; for (int i=0; i < n->num_children; i++) { art_node* child = ((art_node4*)n)->children[i]; q.push(child); @@ -939,25 +940,25 @@ int art_topk_iter(const art_node *root, token_ordering token_order, size_t max_r break; case NODE16: - //std::cout << "\nNODE16, SCORE: " << n->max_token_count << std::endl; + //LOG(INFO) << "\nNODE16, SCORE: " << n->max_token_count; for (int i=0; i < n->num_children; i++) { q.push(((art_node16*)n)->children[i]); } break; case NODE48: - //std::cout << "\nNODE48, SCORE: " << n->max_token_count << std::endl; + //LOG(INFO) << "\nNODE48, SCORE: " << n->max_token_count; for (int i=0; i < 256; i++) { idx = ((art_node48*)n)->keys[i]; if (!idx) continue; art_node *child = ((art_node48*)n)->children[idx - 1]; - //std::cout << "--PUSHING NODE48 CHILD WITH SCORE: " << get_score(child) << std::endl; + //LOG(INFO) << "--PUSHING NODE48 CHILD WITH SCORE: " << get_score(child); q.push(child); } break; case NODE256: - //std::cout << "\nNODE256, SCORE: " << n->max_token_count << std::endl; + //LOG(INFO) << "\nNODE256, SCORE: " << n->max_token_count; for (int i=0; i < 256; i++) { if (!((art_node256*)n)->children[i]) continue; q.push(((art_node256*)n)->children[i]); @@ -1362,7 +1363,7 @@ int art_fuzzy_search(art_tree *t, const unsigned char *term, const int term_len, } long long int time_micro = microseconds(std::chrono::high_resolution_clock::now() - begin).count(); - //!std::cout << "Time taken for fuzz: " << time_micro << "us, size of nodes: " << nodes.size() << std::endl; + //!LOG(INFO) << "Time taken for fuzz: " << time_micro << "us, size of nodes: " << nodes.size(); begin = std::chrono::high_resolution_clock::now(); @@ -1377,7 +1378,7 @@ int art_fuzzy_search(art_tree *t, const unsigned char *term, const int term_len, } time_micro = microseconds(std::chrono::high_resolution_clock::now() - begin).count(); - //!std::cout << "Time taken for art_topk_iter: " << time_micro << "us" << std::endl; + //!LOG(INFO) << "Time taken for art_topk_iter: " << time_micro << "us"; return 0; } diff --git a/src/collection.cpp b/src/collection.cpp index 62b8600b..4d6e3dfa 100644 --- a/src/collection.cpp +++ b/src/collection.cpp @@ -9,6 +9,7 @@ #include #include #include +#include "logger.h" Collection::Collection(const std::string name, const uint32_t collection_id, const uint32_t next_seq_id, Store *store, const std::vector &fields, const std::string & token_ranking_field): @@ -631,7 +632,7 @@ Option Collection::search(std::string query, const std::vector(std::chrono::high_resolution_clock::now() - begin).count(); - //!std::cout << "Time taken for result calc: " << timeMillis << "us" << std::endl; + //!LOG(INFO) << "Time taken for result calc: " << timeMillis << "us"; //!store->print_memory_usage(); return result; } diff --git a/src/http_server.cpp b/src/http_server.cpp index 27008a7e..8512b28e 100644 --- a/src/http_server.cpp +++ b/src/http_server.cpp @@ -6,6 +6,7 @@ #include #include #include +#include "logger.h" struct h2o_custom_req_handler_t { h2o_handler_t super; @@ -69,7 +70,7 @@ int HttpServer::setup_ssl(const char *cert_file, const char *key_file) { int nid = NID_X9_62_prime256v1; EC_KEY *key = EC_KEY_new_by_curve_name(nid); if (key == NULL) { - std::cout << "Failed to create DH/ECDH." << std::endl; + LOG(INFO) << "Failed to create DH/ECDH."; return -1; } @@ -79,11 +80,11 @@ int HttpServer::setup_ssl(const char *cert_file, const char *key_file) { SSL_CTX_set_options(accept_ctx->ssl_ctx, SSL_OP_NO_SSLv2); if (SSL_CTX_use_certificate_file(accept_ctx->ssl_ctx, cert_file, SSL_FILETYPE_PEM) != 1) { - std::cout << "An error occurred while trying to load server certificate file:" << cert_file << std::endl; + LOG(INFO) << "An error occurred while trying to load server certificate file:" << cert_file; return -1; } if (SSL_CTX_use_PrivateKey_file(accept_ctx->ssl_ctx, key_file, SSL_FILETYPE_PEM) != 1) { - std::cout << "An error occurred while trying to load private key file: " << key_file << std::endl; + LOG(INFO) << "An error occurred while trying to load private key file: " << key_file; return -1; } @@ -132,11 +133,10 @@ int HttpServer::run() { h2o_multithread_register_receiver(message_queue, message_receiver, on_message); if (create_listener() != 0) { - std::cerr << "Failed to listen on " << listen_address << ":" << listen_port << " - " - << strerror(errno) << std::endl; + LOG(FATAL) << "Failed to listen on " << listen_address << ":" << listen_port << " - " << strerror(errno); return 1; } else { - std::cout << "Server has started. Ready to accept requests on port " << listen_port << std::endl; + LOG(INFO) << "Typesense has started. Ready to accept requests on port " << listen_port; } on(STOP_SERVER_MESSAGE, HttpServer::on_stop_server); diff --git a/src/index.cpp b/src/index.cpp index 375b754d..c00d857e 100644 --- a/src/index.cpp +++ b/src/index.cpp @@ -7,6 +7,7 @@ #include #include #include +#include "logger.h" Index::Index(const std::string name, std::unordered_map search_schema, std::unordered_map facet_schema, std::unordered_map sort_schema): @@ -331,7 +332,7 @@ void Index::search_candidates(uint32_t* filter_ids, size_t filter_ids_length, st std::vector query_suggestion = next_suggestion(token_candidates_vec, n); /*for(auto i=0; i < query_suggestion.size(); i++) { - std::cout << "i: " << i << " - " << query_suggestion[i]->key << std::endl; + LOG(INFO) << "i: " << i << " - " << query_suggestion[i]->key; }*/ // initialize results with the starting element (for further intersection) @@ -604,7 +605,7 @@ void Index::search(Option & outcome, std::string query, const std::vec delete [] all_result_ids; //long long int timeMillis = std::chrono::duration_cast(std::chrono::high_resolution_clock::now() - begin).count(); - //!std::cout << "Time taken for result calc: " << timeMillis << "us" << std::endl; + //!LOG(INFO) << "Time taken for result calc: " << timeMillis << "us"; outcome = Option(field_order_kvs.size()); } @@ -684,7 +685,7 @@ void Index::search_field(std::string & query, const std::string & field, uint32_ const std::string token_cost_hash = token + std::to_string(costs[token_index]); std::vector leaves; - //std::cout << "\nSearching for: " << token << " - cost: " << costs[token_index] << std::endl; + //LOG(INFO) << "\nSearching for: " << token << " - cost: " << costs[token_index]; if(token_cost_cache.count(token_cost_hash) != 0) { leaves = token_cost_cache[token_cost_hash]; @@ -774,12 +775,13 @@ void Index::search_field(std::string & query, const std::string & field, uint32_ } void Index::log_leaves(const int cost, const std::string &token, const std::vector &leaves) const { - printf("Token: %s, cost: %d, candidates: \n", token.c_str(), cost); + LOG(INFO) << "Token: " << token << ", cost: " << cost; + for(size_t i=0; i < leaves.size(); i++) { printf("%.*s, ", leaves[i]->key_len, leaves[i]->key); - printf("frequency: %d, max_score: %d\n", leaves[i]->values->ids.getLength(), leaves[i]->max_score); + LOG(INFO) << "frequency: " << leaves[i]->values->ids.getLength() << ", max_score: " << leaves[i]->max_score; /*for(auto j=0; jvalues->ids.getLength(); j++) { - printf("id: %d\n", leaves[i]->values->ids.at(j)); + LOG(INFO) << "id: " << leaves[i]->values->ids.at(j); }*/ } } @@ -887,11 +889,11 @@ void Index::score_results(const std::vector & sort_fields, const int & << ", words_present: " << match.words_present << ", match_score: " << match << ", primary_rank_score: " << primary_rank_score.intval << ", distance: " << (MAX_SEARCH_TOKENS - match.distance) << ", seq_id: " << seq_id << std::endl; - std::cout << os.str();*/ + LOG(INFO) << os.str();*/ } //long long int timeNanos = std::chrono::duration_cast(std::chrono::high_resolution_clock::now() - begin).count(); - //std::cout << "Time taken for results iteration: " << timeNanos << "ms" << std::endl; + //LOG(INFO) << "Time taken for results iteration: " << timeNanos << "ms"; for (auto it = leaf_to_indices.begin(); it != leaf_to_indices.end(); it++) { delete [] it->second; @@ -1078,9 +1080,9 @@ Option Index::remove(const uint32_t seq_id, nlohmann::json & document) /*len = leaf->values->offset_index.getLength(); for(auto i=0; ivalues->offset_index.at(i) << std::endl; + LOG(INFO) << "i: " << i << ", val: " << leaf->values->offset_index.at(i); } - std::cout << "----" << std::endl;*/ + LOG(INFO) << "----";*/ if(leaf->values->ids.getLength() == 0) { art_values* values = (art_values*) art_delete(search_index.at(name_field.first), key, key_len); diff --git a/src/main/benchmark.cpp b/src/main/benchmark.cpp index b00372b8..ccf1dcf1 100644 --- a/src/main/benchmark.cpp +++ b/src/main/benchmark.cpp @@ -38,7 +38,7 @@ int main(int argc, char* argv[]) { } infile.close(); - cout << "FINISHED INDEXING!" << endl << flush; + std::cout << "FINISHED INDEXING!" << flush << std::endl; /*std::vector search_fields = {"title"}; @@ -56,7 +56,7 @@ int main(int argc, char* argv[]) { }*/ long long int timeMillis = std::chrono::duration_cast(std::chrono::high_resolution_clock::now() - begin).count(); - cout << "Time taken: " << timeMillis << "ms" << endl; - //cout << "Total: " << results_total << endl; + std::cout << "Time taken: " << timeMillis << "ms" << std::endl; + //std::cout << "Total: " << results_total << std::endl; return 0; } \ No newline at end of file diff --git a/src/main/main.cpp b/src/main/main.cpp index 6b8ce50d..edfb99cb 100644 --- a/src/main/main.cpp +++ b/src/main/main.cpp @@ -44,7 +44,7 @@ int main(int argc, char* argv[]) { std::ifstream infile(argv[1]); std::string json_line; - cout << "BEGINNING Iteration: " << j << endl << flush; + std::cout << "BEGINNING Iteration: " << j << std::endl; auto begin = std::chrono::high_resolution_clock::now(); int doc_id = 0; diff --git a/src/main/typesense_server.cpp b/src/main/typesense_server.cpp index 31ebbea6..b82c9dbc 100644 --- a/src/main/typesense_server.cpp +++ b/src/main/typesense_server.cpp @@ -11,11 +11,12 @@ #include "string_utils.h" #include "replicator.h" #include "typesense_version.h" +#include "logger.h" HttpServer* server; void catch_interrupt(int sig) { - std::cout << "Stopping Typesense server..." << std::endl; + LOG(INFO) << "Stopping Typesense server..."; signal(sig, SIG_IGN); // ignore for now as we want to shut down elegantly server->stop(); } @@ -78,16 +79,35 @@ int main(int argc, char **argv) { options.add("ssl-certificate-key", 'e', "Path to the SSL certificate key file.", false, ""); options.add("enable-cors", '\0', "Enable CORS requests."); + options.add("log-dir", '\0', "Path to the log file.", false, ""); options.parse_check(argc, argv); + auto log_worker = g3::LogWorker::createLogWorker(); + std::string log_dir = options.get("log-dir"); + + if(log_dir.empty()) { + log_worker->addSink(std2::make_unique(), + &ConsoleLoggingSink::ReceiveLogMessage); + } else { + if(!directory_exists(log_dir)) { + std::cerr << "Typesense failed to start. " << "Log directory " << log_dir << " does not exist."; + return 1; + } + + log_worker->addDefaultLogger("typesense", log_dir, ""); + std::cout << "Typesense has started. Log directory is configured as: " << log_dir << std::endl; + } + + g3::initializeLogging(log_worker.get()); + signal(SIGINT, catch_interrupt); - std::cout << "Typesense version " << TYPESENSE_VERSION << std::endl; + LOG(INFO) << "Typesense version: " << TYPESENSE_VERSION; if(!directory_exists(options.get("data-dir"))) { - std::cerr << "Typesense failed to start. " << "Data directory " << options.get("data-dir") - << " does not exist." << std::endl; + LOG(FATAL) << "Typesense failed to start. " << "Data directory " << options.get("data-dir") + << " does not exist."; return 1; } @@ -97,10 +117,9 @@ int main(int argc, char **argv) { options.get("search-only-api-key")); if(init_op.ok()) { - std::cout << "Finished loading collections from disk." << std::endl; + LOG(INFO) << "Finished loading collections from disk."; } else { - std::cerr << "Typesense failed to start. " << "Could not load collections from disk: " - << init_op.error() << std::endl; + LOG(FATAL)<< "Typesense failed to start. " << "Could not load collections from disk: " << init_op.error(); return 1; } @@ -126,11 +145,11 @@ int main(int argc, char **argv) { std::vector parts; StringUtils::split(master_host_port, parts, ":"); if(parts.size() != 3) { - std::cerr << "Invalid value for --master option. Usage: http(s)://:" << std::endl; + LOG(FATAL) << "Invalid value for --master option. Usage: http(s)://:"; return 1; } - std::cout << "Typesense server started as a read-only replica... Spawning replication thread..." << std::endl; + LOG(INFO) << "Typesense is starting as a read-only replica... Spawning replication thread..."; std::thread replication_thread([&master_host_port, &store, &options]() { Replicator::start(::server, master_host_port, options.get("api-key"), store); }); diff --git a/test/art_test.cpp b/test/art_test.cpp index 7de01547..8491f745 100644 --- a/test/art_test.cpp +++ b/test/art_test.cpp @@ -878,12 +878,6 @@ TEST(ArtTest, test_int32_million) { } encode_int32(5, chars); - /*std::cout << std::endl; - for(uint32_t i = 0; i < CHAR_LEN; i++) { - std::cout << (int)chars[i] << ", "; - } - std::cout << std::endl;*/ - std::vector results; // == @@ -972,16 +966,6 @@ TEST(ArtTest, test_int_range_byte_boundary) { results.clear(); art_int32_search(&t, 255, GREATER_THAN, results); ASSERT_EQ(44, results.size()); - - /*std::cout << std::endl; - for(auto i = 0; i < 1; i++) { - auto result = results[i]; - for(auto j = 0; j < result->key_len; j++) { - std::cout << (int) result->key[j] << ", "; - } - - std::cout << std::endl; - }*/ } TEST(ArtTest, test_encode_int64) { @@ -1007,7 +991,6 @@ TEST(ArtTest, test_encode_int64) { unsigned char chars_large_num[8] = {0, 0, 0, 0, 128, 0, 0, 199}; int64_t large_num = (int64_t)(std::numeric_limits::max()) + 200; - std::cout << std::endl << large_num << std::endl; encode_int64(large_num, chars); for(uint32_t i = 0; i < 8; i++) { ASSERT_EQ(chars_large_num[i], chars[i]); diff --git a/test/collection_manager_test.cpp b/test/collection_manager_test.cpp index f97de409..fbcf0f51 100644 --- a/test/collection_manager_test.cpp +++ b/test/collection_manager_test.cpp @@ -15,7 +15,7 @@ protected: void setupCollection() { std::string state_dir_path = "/tmp/typesense_test/coll_manager_test_db"; - std::cout << "Truncating and creating: " << state_dir_path << std::endl; + LOG(INFO) << "Truncating and creating: " << state_dir_path; system(("rm -rf "+state_dir_path+" && mkdir -p "+state_dir_path).c_str()); store = new Store(state_dir_path); diff --git a/test/collection_test.cpp b/test/collection_test.cpp index 60a5ede0..fca67ebe 100644 --- a/test/collection_test.cpp +++ b/test/collection_test.cpp @@ -17,7 +17,7 @@ protected: void setupCollection() { std::string state_dir_path = "/tmp/typesense_test/collection"; - std::cout << "Truncating and creating: " << state_dir_path << std::endl; + LOG(INFO) << "Truncating and creating: " << state_dir_path; system(("rm -rf "+state_dir_path+" && mkdir -p "+state_dir_path).c_str()); store = new Store(state_dir_path); diff --git a/test/main.cpp b/test/main.cpp new file mode 100644 index 00000000..0e0bce39 --- /dev/null +++ b/test/main.cpp @@ -0,0 +1,23 @@ +#include +#include "logger.h" + +class TypesenseTestEnvironment : public testing::Environment { +public: + virtual void SetUp() { + auto log_worker = g3::LogWorker::createLogWorker(); + auto sink_handle = log_worker->addSink(std2::make_unique(), + &ConsoleLoggingSink::ReceiveLogMessage); + g3::initializeLogging(log_worker.get()); + } + + virtual void TearDown() { + + } +}; + +int main(int argc, char **argv) { + ::testing::InitGoogleTest(&argc, argv); + ::testing::AddGlobalTestEnvironment(new TypesenseTestEnvironment); + int exitCode = RUN_ALL_TESTS(); + return exitCode; +} diff --git a/test/store_test.cpp b/test/store_test.cpp index 563a9050..0b0f9597 100644 --- a/test/store_test.cpp +++ b/test/store_test.cpp @@ -5,7 +5,7 @@ TEST(StoreTest, GetUpdatesSince) { std::string primary_store_path = "/tmp/typesense_test/primary_store_test"; - std::cout << "Truncating and creating: " << primary_store_path << std::endl; + LOG(INFO) << "Truncating and creating: " << primary_store_path; system(("rm -rf "+primary_store_path+" && mkdir -p "+primary_store_path).c_str()); // add some records, get the updates and restore them in a new store @@ -39,7 +39,7 @@ TEST(StoreTest, GetUpdatesSince) { ASSERT_EQ(3, updates_op.get()->size()); std::string replica_store_path = "/tmp/typesense_test/replica_store_test"; - std::cout << "Truncating and creating: " << replica_store_path << std::endl; + LOG(INFO) << "Truncating and creating: " << replica_store_path; system(("rm -rf "+replica_store_path+" && mkdir -p "+replica_store_path).c_str()); Store replica_store(replica_store_path); diff --git a/test/string_utils_test.cpp b/test/string_utils_test.cpp index 3664ce23..a74638b1 100644 --- a/test/string_utils_test.cpp +++ b/test/string_utils_test.cpp @@ -17,6 +17,5 @@ TEST(StringUtilsTest, ShouldNormalizeString) { // retain non-ascii unicode characters and should also lower case them std::string alphanum_unicodechars = "abcÅà123"; StringUtils::normalize(alphanum_unicodechars); - std::cout << alphanum_unicodechars << std::endl; ASSERT_STREQ("abcåà123", alphanum_unicodechars.c_str()); } \ No newline at end of file