postgres/typesense
1
0
Fork 0
mirror of https://github.com/typesense/typesense.git synced 2025-05-19 05:08:43 +08:00
Code Issues Actions 1 Packages Projects Releases Wiki Activity

Fixed some build errors on benchmark/search targets.

Browse Source
This commit is contained in:
Kishore Nallan 2017-05-17 21:54:40 +05:30
parent 3478aef573
commit 8ec625b434
7 changed files with 11 additions and 272 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
CMakeLists.txt
View File

@ -85,5 +85,5 @@ endif()
target_link_libraries(typesense-server h2o-evloop for pthread ${ROCKSDB_LIBS} ${OPENSSL_LIBRARIES} dl ${STD_LIB})
target_link_libraries(search for pthread h2o-evloop ${ROCKSDB_LIBS} ${OPENSSL_LIBRARIES} dl ${STD_LIB})
target_link_libraries(benchmark for pthread ${ROCKSDB_LIBS} ${STD_LIB})
target_link_libraries(typesense_test h2o-evloop ${OPENSSL_LIBRARIES} pthread for ${ROCKSDB_LIBS} gtest gtest_main ${STD_LIB})
target_link_libraries(benchmark for pthread h2o-evloop ${ROCKSDB_LIBS} ${OPENSSL_LIBRARIES} dl ${STD_LIB})
target_link_libraries(typesense_test h2o-evloop ${OPENSSL_LIBRARIES} pthread for ${ROCKSDB_LIBS} gtest gtest_main dl ${STD_LIB})

2
src/api.cpp
View File

@ -1,4 +1,6 @@
#include <regex>
#include <chrono>
#include <sys/resource.h>
#include "api.h"
#include "string_utils.h"
#include "collection.h"

3
src/array_utils.cpp
View File

@ -1,4 +1,5 @@
#include "array_utils.h"
#include <memory.h>
size_t ArrayUtils::and_scalar(const uint32_t *A, const size_t lenA,
const uint32_t *B, const size_t lenB, uint32_t *out) {
@ -79,7 +80,7 @@ size_t ArrayUtils::or_scalar(const uint32_t *A, const size_t lenA,
// shrink fit
*out = new uint32_t[res_index];
std::memmove(*out, results, res_index * sizeof(uint32_t));
memcpy(*out, results, res_index * sizeof(uint32_t));
delete[] results;
return res_index;

1
src/http_server.cpp
View File

@ -1,6 +1,7 @@
#include "http_server.h"
#include "string_utils.h"
#include <regex>
#include <signal.h>
h2o_globalconf_t HttpServer::config;
h2o_context_t HttpServer::ctx;

4
src/main/benchmark.cpp
View File

@ -17,7 +17,7 @@ int main(int argc, char* argv[]) {
system("rm -rf /tmp/typesense-data && mkdir -p /tmp/typesense-data");
std::vector<field> fields_to_index = {field("title", field_types::STRING)};
std::vector<sort_field> sort_fields = { sort_field("points", "DESC")};
std::vector<field> sort_fields = { field("points", "INT32")};
Store *store = new Store("/tmp/typesense-data");
CollectionManager & collectionManager = CollectionManager::get_instance();
collectionManager.init(store);
@ -48,7 +48,7 @@ int main(int argc, char* argv[]) {
while(counter < 3000) {
auto i = counter % 5;
auto results = collection->search(queries[i], search_fields, "", { }, {"points"}, 1, 100, MAX_SCORE, 0);
auto results = collection->search(queries[i], search_fields, "", { }, {sort_field("points", "DESC")}, 1, 100, MAX_SCORE, 0);
results_total += results.size();
counter++;
}

4
src/main/main.cpp
View File

@ -43,7 +43,7 @@ int main(int argc, char* argv[]) {
const std::string state_dir_path = "/tmp/typesense-data";
std::vector<field> fields_to_index = {field("title", field_types::STRING)};
std::vector<sort_field> sort_fields = { sort_field("points", "DESC")};
std::vector<field> sort_fields = { field("points", "INT32")};
Store *store = new Store("/tmp/typesense-data");
CollectionManager & collectionManager = CollectionManager::get_instance();
@ -69,7 +69,7 @@ int main(int argc, char* argv[]) {
auto begin = std::chrono::high_resolution_clock::now();
std::vector<std::string> search_fields = {"title"};
collection->search("the", search_fields, "", {}, {"points"}, 1, 100, MAX_SCORE, 0);
collection->search("the", search_fields, "", {}, { sort_field("points", "DESC") }, 1, 100, MAX_SCORE, 0);
long long int timeMillis =
std::chrono::duration_cast<std::chrono::microseconds>(std::chrono::high_resolution_clock::now() - begin).count();
cout << "Time taken: " << timeMillis << "us" << endl;

265
src/main/server.cpp
View File

@ -1,265 +0,0 @@
#define H2O_USE_LIBUV 0
#include <errno.h>
#include <limits.h>
#include <netinet/in.h>
#include <signal.h>
#include <stdio.h>
#include <stdlib.h>
#include <sys/socket.h>
#include <sys/stat.h>
#include <fstream>
#include <iostream>
#include <chrono>
#include <vector>
#include <string>
#include <map>
#include <regex>
#include "string_utils.h"
#include "collection.h"
#include "collection_manager.h"
#include "option.h"
#include <sys/resource.h>
#include "h2o.h"
#include "h2o/http1.h"
#include "h2o/http2.h"
#include "h2o/memcached.h"
static h2o_globalconf_t config;
static h2o_context_t ctx;
static h2o_accept_ctx_t accept_ctx;
std::vector<field> search_fields = {field("title", field_types::STRING), field("points", field_types::INT32)};
std::vector<sort_field> sort_fields = { sort_field("points", "DESC")};
Collection *collection;
static h2o_pathconf_t *register_handler(h2o_hostconf_t *hostconf, const char *path,
int (*on_req)(h2o_handler_t *, h2o_req_t *)) {
h2o_pathconf_t *pathconf = h2o_config_register_path(hostconf, path, 0);
h2o_handler_t *handler = h2o_create_handler(pathconf, sizeof(*handler));
handler->on_req = on_req;
return pathconf;
}
std::map<std::string, std::string> parse_query(const std::string& query) {
std::map<std::string, std::string> query_map;
std::regex pattern("([\\w+%]+)=([^&]*)");
auto words_begin = std::sregex_iterator(query.begin(), query.end(), pattern);
auto words_end = std::sregex_iterator();
for (std::sregex_iterator i = words_begin; i != words_end; i++) {
std::string key = (*i)[1].str();
std::string raw_value = (*i)[2].str();
std::string value = StringUtils::url_decode(raw_value);
if(query_map.count(value) == 0) {
query_map[key] = value;
} else {
query_map[key] = query_map[key] + "&&" + value;
}
}
return query_map;
}
static int get_search(h2o_handler_t *self, h2o_req_t *req) {
static h2o_generator_t generator = {NULL, NULL};
h2o_iovec_t query = req->query_at != SIZE_MAX ?
h2o_iovec_init(req->path.base + req->query_at, req->path.len - req->query_at) :
h2o_iovec_init(H2O_STRLIT(""));
std::string query_str(query.base, query.len);
std::map<std::string, std::string> query_map = parse_query(query_str);
const char *NUM_TYPOS = "num_typos";
const char *PREFIX = "prefix";
const char *TOKEN_ORDERING = "token_ordering";
const char *FILTERS = "filters";
if(query_map.count(NUM_TYPOS) == 0) {
query_map[NUM_TYPOS] = "2";
}
if(query_map.count(PREFIX) == 0) {
query_map[PREFIX] = "false";
}
if(query_map.count(TOKEN_ORDERING) == 0) {
query_map[TOKEN_ORDERING] = "FREQUENCY";
}
std::string filter_str = query_map.count(FILTERS) != 0 ? query_map[FILTERS] : "";
//std::cout << "filter_str: " << filter_str << std::endl;
token_ordering token_order = (query_map[TOKEN_ORDERING] == "MAX_SCORE") ? MAX_SCORE : FREQUENCY;
//printf("Query: %s\n", query_map["q"].c_str());
auto begin = std::chrono::high_resolution_clock::now();
std::vector<std::string> search_fields = {"title"};
nlohmann::json result = collection->search(query_map["q"], search_fields, filter_str, { },
{"points"}, std::stoi(query_map[NUM_TYPOS]), 100, token_order, false);
std::string json_str = result.dump();
//std::cout << "JSON:" << json_str << std::endl;
struct rusage r_usage;
getrusage(RUSAGE_SELF,&r_usage);
//std::cout << "Memory usage: " << r_usage.ru_maxrss << std::endl;
h2o_iovec_t body = h2o_strdup(&req->pool, json_str.c_str(), SIZE_MAX);
req->res.status = 200;
req->res.reason = "OK";
h2o_add_header(&req->pool, &req->res.headers, H2O_TOKEN_CONTENT_TYPE, H2O_STRLIT("application/json; charset=utf-8"));
h2o_start_response(req, &generator);
h2o_send(req, &body, 1, 1);
long long int timeMillis = std::chrono::duration_cast<std::chrono::microseconds>(std::chrono::high_resolution_clock::now() - begin).count();
std::cout << "Time taken: " << timeMillis << "us" << std::endl;
return 0;
}
static int post_add_document(h2o_handler_t *self, h2o_req_t *req) {
std::string document(req->entity.base, req->entity.len);
Option<std::string> inserted_id_op = collection->add(document);
nlohmann::json json_response;
static h2o_generator_t generator = {NULL, NULL};
if(!inserted_id_op.ok()) {
req->res.status = 400;
req->res.reason = "BAD REQUEST";
json_response["message"] = inserted_id_op.error();
} else {
req->res.status = 201;
req->res.reason = "CREATED";
json_response["id"] = inserted_id_op.get();
}
h2o_add_header(&req->pool, &req->res.headers, H2O_TOKEN_CONTENT_TYPE, H2O_STRLIT("application/json; charset=utf-8"));
h2o_start_response(req, &generator);
h2o_iovec_t body = h2o_strdup(&req->pool, json_response.dump().c_str(), SIZE_MAX);
h2o_send(req, &body, 1, 1);
return 0;
}
static int delete_remove_document(h2o_handler_t *self, h2o_req_t *req) {
h2o_iovec_t query = req->query_at != SIZE_MAX ?
h2o_iovec_init(req->path.base + req->query_at, req->path.len - req->query_at) :
h2o_iovec_init(H2O_STRLIT(""));
std::string query_str(query.base, query.len);
std::map<std::string, std::string> query_map = parse_query(query_str);
std::string doc_id = query_map["id"];
auto begin = std::chrono::high_resolution_clock::now();
collection->remove(doc_id);
long long int time_micro = std::chrono::duration_cast<std::chrono::microseconds>(
std::chrono::high_resolution_clock::now() - begin).count();
std::cout << "Time taken: " << time_micro << "us" << std::endl;
nlohmann::json json_response;
json_response["id"] = doc_id;
json_response["status"] = "SUCCESS";
static h2o_generator_t generator = {NULL, NULL};
req->res.status = 200;
req->res.reason = "OK";
h2o_add_header(&req->pool, &req->res.headers, H2O_TOKEN_CONTENT_TYPE, H2O_STRLIT("application/json; charset=utf-8"));
h2o_start_response(req, &generator);
h2o_iovec_t body = h2o_strdup(&req->pool, json_response.dump().c_str(), SIZE_MAX);
h2o_send(req, &body, 1, 1);
return 0;
}
static void on_accept(h2o_socket_t *listener, const char *err) {
h2o_socket_t *sock;
if (err != NULL) {
return;
}
if ((sock = h2o_evloop_socket_accept(listener)) == NULL)
return;
h2o_accept(&accept_ctx, sock);
}
static int create_listener(void) {
struct sockaddr_in addr;
int fd, reuseaddr_flag = 1;
h2o_socket_t *sock;
memset(&addr, 0, sizeof(addr));
addr.sin_family = AF_INET;
addr.sin_addr.s_addr = htonl(INADDR_ANY); //htonl(INADDR_LOOPBACK);
addr.sin_port = htons(1088);
if ((fd = socket(AF_INET, SOCK_STREAM, 0)) == -1 ||
setsockopt(fd, SOL_SOCKET, SO_REUSEADDR, &reuseaddr_flag, sizeof(reuseaddr_flag)) != 0 ||
bind(fd, (struct sockaddr *)&addr, sizeof(addr)) != 0 ||
listen(fd, SOMAXCONN) != 0) {
return -1;
}
sock = h2o_evloop_socket_create(ctx.loop, fd, H2O_SOCKET_FLAG_DONT_READ);
h2o_socket_read_start(sock, on_accept);
return 0;
}
void index_documents(std::string path_to_docs) {
std::ifstream infile(path_to_docs);
// std::ifstream infile(path_to_docs);
std::string json_line;
while (std::getline(infile, json_line)) {
collection->add(json_line);
}
infile.close();
std::cout << "FINISHED INDEXING!" << std::endl << std::flush;
struct rusage r_usage;
getrusage(RUSAGE_SELF,&r_usage);
std::cout << "Memory usage: " << r_usage.ru_maxrss << std::endl;
}
int main(int argc, char **argv) {
signal(SIGPIPE, SIG_IGN);
Store *store = new Store("/tmp/typesense-data");
CollectionManager & collectionManager = CollectionManager::get_instance();
collectionManager.init(store);
collection = collectionManager.get_collection("collection");
if(collection == nullptr) {
collection = collectionManager.create_collection("collection", search_fields, {}, sort_fields);
//index_documents(std::string(ROOT_DIR)+"test/documents.jsonl");
if(argc > 1) {
index_documents(argv[1]);
}
}
h2o_config_init(&config);
h2o_hostconf_t *hostconf = h2o_config_register_host(&config, h2o_iovec_init(H2O_STRLIT("default")), 65535);
register_handler(hostconf, "/add", post_add_document);
register_handler(hostconf, "/delete", delete_remove_document);
register_handler(hostconf, "/search", get_search);
h2o_context_init(&ctx, h2o_evloop_create(), &config);
accept_ctx.ctx = &ctx;
accept_ctx.hosts = config.hosts;
if (create_listener() != 0) {
fprintf(stderr, "failed to listen to 127.0.0.1:1088:%s\n", strerror(errno));
return 1;
}
while (h2o_evloop_run(ctx.loop) == 0);
return 0;
}