From 8ec625b43451dae3693797c98e33823f82be6b0e Mon Sep 17 00:00:00 2001 From: Kishore Nallan Date: Wed, 17 May 2017 21:54:40 +0530 Subject: [PATCH] Fixed some build errors on benchmark/search targets. --- CMakeLists.txt | 4 +- src/api.cpp | 2 + src/array_utils.cpp | 3 +- src/http_server.cpp | 1 + src/main/benchmark.cpp | 4 +- src/main/main.cpp | 4 +- src/main/server.cpp | 265 ----------------------------------------- 7 files changed, 11 insertions(+), 272 deletions(-) delete mode 100644 src/main/server.cpp diff --git a/CMakeLists.txt b/CMakeLists.txt index 98f7f14b..1eca6791 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -85,5 +85,5 @@ endif() target_link_libraries(typesense-server h2o-evloop for pthread ${ROCKSDB_LIBS} ${OPENSSL_LIBRARIES} dl ${STD_LIB}) target_link_libraries(search for pthread h2o-evloop ${ROCKSDB_LIBS} ${OPENSSL_LIBRARIES} dl ${STD_LIB}) -target_link_libraries(benchmark for pthread ${ROCKSDB_LIBS} ${STD_LIB}) -target_link_libraries(typesense_test h2o-evloop ${OPENSSL_LIBRARIES} pthread for ${ROCKSDB_LIBS} gtest gtest_main ${STD_LIB}) +target_link_libraries(benchmark for pthread h2o-evloop ${ROCKSDB_LIBS} ${OPENSSL_LIBRARIES} dl ${STD_LIB}) +target_link_libraries(typesense_test h2o-evloop ${OPENSSL_LIBRARIES} pthread for ${ROCKSDB_LIBS} gtest gtest_main dl ${STD_LIB}) diff --git a/src/api.cpp b/src/api.cpp index ce2285db..e4416b6c 100644 --- a/src/api.cpp +++ b/src/api.cpp @@ -1,4 +1,6 @@ #include +#include +#include #include "api.h" #include "string_utils.h" #include "collection.h" diff --git a/src/array_utils.cpp b/src/array_utils.cpp index 69dce4ba..610aa96c 100644 --- a/src/array_utils.cpp +++ b/src/array_utils.cpp @@ -1,4 +1,5 @@ #include "array_utils.h" +#include size_t ArrayUtils::and_scalar(const uint32_t *A, const size_t lenA, const uint32_t *B, const size_t lenB, uint32_t *out) { @@ -79,7 +80,7 @@ size_t ArrayUtils::or_scalar(const uint32_t *A, const size_t lenA, // shrink fit *out = new uint32_t[res_index]; - std::memmove(*out, results, res_index * sizeof(uint32_t)); + memcpy(*out, results, res_index * sizeof(uint32_t)); delete[] results; return res_index; diff --git a/src/http_server.cpp b/src/http_server.cpp index aed2eeb1..94472a0f 100644 --- a/src/http_server.cpp +++ b/src/http_server.cpp @@ -1,6 +1,7 @@ #include "http_server.h" #include "string_utils.h" #include +#include h2o_globalconf_t HttpServer::config; h2o_context_t HttpServer::ctx; diff --git a/src/main/benchmark.cpp b/src/main/benchmark.cpp index d5585e7b..b9bfb38a 100644 --- a/src/main/benchmark.cpp +++ b/src/main/benchmark.cpp @@ -17,7 +17,7 @@ int main(int argc, char* argv[]) { system("rm -rf /tmp/typesense-data && mkdir -p /tmp/typesense-data"); std::vector fields_to_index = {field("title", field_types::STRING)}; - std::vector sort_fields = { sort_field("points", "DESC")}; + std::vector sort_fields = { field("points", "INT32")}; Store *store = new Store("/tmp/typesense-data"); CollectionManager & collectionManager = CollectionManager::get_instance(); collectionManager.init(store); @@ -48,7 +48,7 @@ int main(int argc, char* argv[]) { while(counter < 3000) { auto i = counter % 5; - auto results = collection->search(queries[i], search_fields, "", { }, {"points"}, 1, 100, MAX_SCORE, 0); + auto results = collection->search(queries[i], search_fields, "", { }, {sort_field("points", "DESC")}, 1, 100, MAX_SCORE, 0); results_total += results.size(); counter++; } diff --git a/src/main/main.cpp b/src/main/main.cpp index a7fc0117..ddde0e68 100644 --- a/src/main/main.cpp +++ b/src/main/main.cpp @@ -43,7 +43,7 @@ int main(int argc, char* argv[]) { const std::string state_dir_path = "/tmp/typesense-data"; std::vector fields_to_index = {field("title", field_types::STRING)}; - std::vector sort_fields = { sort_field("points", "DESC")}; + std::vector sort_fields = { field("points", "INT32")}; Store *store = new Store("/tmp/typesense-data"); CollectionManager & collectionManager = CollectionManager::get_instance(); @@ -69,7 +69,7 @@ int main(int argc, char* argv[]) { auto begin = std::chrono::high_resolution_clock::now(); std::vector search_fields = {"title"}; - collection->search("the", search_fields, "", {}, {"points"}, 1, 100, MAX_SCORE, 0); + collection->search("the", search_fields, "", {}, { sort_field("points", "DESC") }, 1, 100, MAX_SCORE, 0); long long int timeMillis = std::chrono::duration_cast(std::chrono::high_resolution_clock::now() - begin).count(); cout << "Time taken: " << timeMillis << "us" << endl; diff --git a/src/main/server.cpp b/src/main/server.cpp deleted file mode 100644 index 62f4d3b0..00000000 --- a/src/main/server.cpp +++ /dev/null @@ -1,265 +0,0 @@ -#define H2O_USE_LIBUV 0 - -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include "string_utils.h" -#include "collection.h" -#include "collection_manager.h" -#include "option.h" -#include - -#include "h2o.h" -#include "h2o/http1.h" -#include "h2o/http2.h" -#include "h2o/memcached.h" - -static h2o_globalconf_t config; -static h2o_context_t ctx; -static h2o_accept_ctx_t accept_ctx; -std::vector search_fields = {field("title", field_types::STRING), field("points", field_types::INT32)}; -std::vector sort_fields = { sort_field("points", "DESC")}; -Collection *collection; - -static h2o_pathconf_t *register_handler(h2o_hostconf_t *hostconf, const char *path, - int (*on_req)(h2o_handler_t *, h2o_req_t *)) { - h2o_pathconf_t *pathconf = h2o_config_register_path(hostconf, path, 0); - h2o_handler_t *handler = h2o_create_handler(pathconf, sizeof(*handler)); - handler->on_req = on_req; - return pathconf; -} - -std::map parse_query(const std::string& query) { - std::map query_map; - std::regex pattern("([\\w+%]+)=([^&]*)"); - - auto words_begin = std::sregex_iterator(query.begin(), query.end(), pattern); - auto words_end = std::sregex_iterator(); - - for (std::sregex_iterator i = words_begin; i != words_end; i++) { - std::string key = (*i)[1].str(); - std::string raw_value = (*i)[2].str(); - std::string value = StringUtils::url_decode(raw_value); - if(query_map.count(value) == 0) { - query_map[key] = value; - } else { - query_map[key] = query_map[key] + "&&" + value; - } - } - - return query_map; -} - -static int get_search(h2o_handler_t *self, h2o_req_t *req) { - static h2o_generator_t generator = {NULL, NULL}; - h2o_iovec_t query = req->query_at != SIZE_MAX ? - h2o_iovec_init(req->path.base + req->query_at, req->path.len - req->query_at) : - h2o_iovec_init(H2O_STRLIT("")); - - std::string query_str(query.base, query.len); - std::map query_map = parse_query(query_str); - const char *NUM_TYPOS = "num_typos"; - const char *PREFIX = "prefix"; - const char *TOKEN_ORDERING = "token_ordering"; - const char *FILTERS = "filters"; - - if(query_map.count(NUM_TYPOS) == 0) { - query_map[NUM_TYPOS] = "2"; - } - - if(query_map.count(PREFIX) == 0) { - query_map[PREFIX] = "false"; - } - - if(query_map.count(TOKEN_ORDERING) == 0) { - query_map[TOKEN_ORDERING] = "FREQUENCY"; - } - - std::string filter_str = query_map.count(FILTERS) != 0 ? query_map[FILTERS] : ""; - //std::cout << "filter_str: " << filter_str << std::endl; - - token_ordering token_order = (query_map[TOKEN_ORDERING] == "MAX_SCORE") ? MAX_SCORE : FREQUENCY; - - //printf("Query: %s\n", query_map["q"].c_str()); - auto begin = std::chrono::high_resolution_clock::now(); - - std::vector search_fields = {"title"}; - - nlohmann::json result = collection->search(query_map["q"], search_fields, filter_str, { }, - {"points"}, std::stoi(query_map[NUM_TYPOS]), 100, token_order, false); - std::string json_str = result.dump(); - //std::cout << "JSON:" << json_str << std::endl; - struct rusage r_usage; - getrusage(RUSAGE_SELF,&r_usage); - - //std::cout << "Memory usage: " << r_usage.ru_maxrss << std::endl; - - h2o_iovec_t body = h2o_strdup(&req->pool, json_str.c_str(), SIZE_MAX); - req->res.status = 200; - req->res.reason = "OK"; - h2o_add_header(&req->pool, &req->res.headers, H2O_TOKEN_CONTENT_TYPE, H2O_STRLIT("application/json; charset=utf-8")); - h2o_start_response(req, &generator); - h2o_send(req, &body, 1, 1); - - long long int timeMillis = std::chrono::duration_cast(std::chrono::high_resolution_clock::now() - begin).count(); - std::cout << "Time taken: " << timeMillis << "us" << std::endl; - return 0; -} - -static int post_add_document(h2o_handler_t *self, h2o_req_t *req) { - std::string document(req->entity.base, req->entity.len); - Option inserted_id_op = collection->add(document); - - nlohmann::json json_response; - static h2o_generator_t generator = {NULL, NULL}; - - if(!inserted_id_op.ok()) { - req->res.status = 400; - req->res.reason = "BAD REQUEST"; - json_response["message"] = inserted_id_op.error(); - } else { - req->res.status = 201; - req->res.reason = "CREATED"; - json_response["id"] = inserted_id_op.get(); - } - - h2o_add_header(&req->pool, &req->res.headers, H2O_TOKEN_CONTENT_TYPE, H2O_STRLIT("application/json; charset=utf-8")); - h2o_start_response(req, &generator); - - h2o_iovec_t body = h2o_strdup(&req->pool, json_response.dump().c_str(), SIZE_MAX); - h2o_send(req, &body, 1, 1); - return 0; -} - -static int delete_remove_document(h2o_handler_t *self, h2o_req_t *req) { - h2o_iovec_t query = req->query_at != SIZE_MAX ? - h2o_iovec_init(req->path.base + req->query_at, req->path.len - req->query_at) : - h2o_iovec_init(H2O_STRLIT("")); - - std::string query_str(query.base, query.len); - std::map query_map = parse_query(query_str); - - std::string doc_id = query_map["id"]; - - auto begin = std::chrono::high_resolution_clock::now(); - collection->remove(doc_id); - long long int time_micro = std::chrono::duration_cast( - std::chrono::high_resolution_clock::now() - begin).count(); - std::cout << "Time taken: " << time_micro << "us" << std::endl; - - nlohmann::json json_response; - json_response["id"] = doc_id; - json_response["status"] = "SUCCESS"; - - static h2o_generator_t generator = {NULL, NULL}; - req->res.status = 200; - req->res.reason = "OK"; - h2o_add_header(&req->pool, &req->res.headers, H2O_TOKEN_CONTENT_TYPE, H2O_STRLIT("application/json; charset=utf-8")); - h2o_start_response(req, &generator); - h2o_iovec_t body = h2o_strdup(&req->pool, json_response.dump().c_str(), SIZE_MAX); - h2o_send(req, &body, 1, 1); - return 0; -} - -static void on_accept(h2o_socket_t *listener, const char *err) { - h2o_socket_t *sock; - - if (err != NULL) { - return; - } - - if ((sock = h2o_evloop_socket_accept(listener)) == NULL) - return; - h2o_accept(&accept_ctx, sock); -} - -static int create_listener(void) { - struct sockaddr_in addr; - int fd, reuseaddr_flag = 1; - h2o_socket_t *sock; - - memset(&addr, 0, sizeof(addr)); - addr.sin_family = AF_INET; - addr.sin_addr.s_addr = htonl(INADDR_ANY); //htonl(INADDR_LOOPBACK); - addr.sin_port = htons(1088); - - if ((fd = socket(AF_INET, SOCK_STREAM, 0)) == -1 || - setsockopt(fd, SOL_SOCKET, SO_REUSEADDR, &reuseaddr_flag, sizeof(reuseaddr_flag)) != 0 || - bind(fd, (struct sockaddr *)&addr, sizeof(addr)) != 0 || - listen(fd, SOMAXCONN) != 0) { - return -1; - } - - sock = h2o_evloop_socket_create(ctx.loop, fd, H2O_SOCKET_FLAG_DONT_READ); - h2o_socket_read_start(sock, on_accept); - - return 0; -} - -void index_documents(std::string path_to_docs) { - std::ifstream infile(path_to_docs); -// std::ifstream infile(path_to_docs); - - std::string json_line; - - while (std::getline(infile, json_line)) { - collection->add(json_line); - } - - infile.close(); - std::cout << "FINISHED INDEXING!" << std::endl << std::flush; - struct rusage r_usage; - getrusage(RUSAGE_SELF,&r_usage); - - std::cout << "Memory usage: " << r_usage.ru_maxrss << std::endl; -} - -int main(int argc, char **argv) { - signal(SIGPIPE, SIG_IGN); - - Store *store = new Store("/tmp/typesense-data"); - - CollectionManager & collectionManager = CollectionManager::get_instance(); - collectionManager.init(store); - - collection = collectionManager.get_collection("collection"); - if(collection == nullptr) { - collection = collectionManager.create_collection("collection", search_fields, {}, sort_fields); - //index_documents(std::string(ROOT_DIR)+"test/documents.jsonl"); - if(argc > 1) { - index_documents(argv[1]); - } - } - - h2o_config_init(&config); - h2o_hostconf_t *hostconf = h2o_config_register_host(&config, h2o_iovec_init(H2O_STRLIT("default")), 65535); - register_handler(hostconf, "/add", post_add_document); - register_handler(hostconf, "/delete", delete_remove_document); - register_handler(hostconf, "/search", get_search); - - h2o_context_init(&ctx, h2o_evloop_create(), &config); - - accept_ctx.ctx = &ctx; - accept_ctx.hosts = config.hosts; - - if (create_listener() != 0) { - fprintf(stderr, "failed to listen to 127.0.0.1:1088:%s\n", strerror(errno)); - return 1; - } - - while (h2o_evloop_run(ctx.loop) == 0); - - return 0; -} \ No newline at end of file