Force use of jemalloc via static linking.

This commit is contained in:
kishorenc 2020-07-21 06:55:25 +05:30
parent d02b678d62
commit 1ab9b65098
10 changed files with 243 additions and 67 deletions

View File

@ -22,6 +22,8 @@ if (APPLE)
# Prefer brew installated libraries
set(OPENSSL_ROOT_DIR /usr/local/opt/openssl@1.1)
set(ENV{OPENSSL_ROOT_DIR} /usr/local/opt/openssl@1.1)
set(SNAPPY_ROOT_DIR /usr/local/opt/snappy)
set(ZLIB_ROOT /usr/local/opt/zlib)
set(CMAKE_PREFIX_PATH /usr/local/opt/curl-openssl /usr/local/opt/icu4c)
@ -33,8 +35,17 @@ IF(WIN32 OR MSVC)
SET(CMAKE_FIND_LIBRARY_SUFFIXES .lib .a)
ELSE()
SET(CMAKE_FIND_LIBRARY_SUFFIXES .a)
set(ENV{CMAKE_FIND_LIBRARY_SUFFIXES} ".a")
ENDIF()
include(cmake/For.cmake)
include(cmake/H2O.cmake)
include(cmake/RocksDB.cmake)
include(cmake/GoogleTest.cmake)
include(cmake/TestResources.cmake)
include(cmake/Iconv.cmake)
include(cmake/Jemalloc.cmake)
FIND_PACKAGE(OpenSSL 1.1.1 REQUIRED)
FIND_PACKAGE(Snappy REQUIRED)
FIND_PACKAGE(ZLIB REQUIRED)
@ -47,13 +58,6 @@ FIND_PACKAGE(glog REQUIRED)
message("OpenSSL library: ${OPENSSL_LIBRARIES}")
include(cmake/For.cmake)
include(cmake/H2O.cmake)
include(cmake/RocksDB.cmake)
include(cmake/GoogleTest.cmake)
include(cmake/TestResources.cmake)
include(cmake/Iconv.cmake)
FILE(GLOB SRC_FILES src/*.cpp)
FILE(GLOB TEST_FILES test/*.cpp)
@ -68,52 +72,28 @@ include_directories(${DEP_ROOT_DIR}/${ROCKSDB_NAME}/include)
include_directories(${DEP_ROOT_DIR}/${ICONV_NAME}/include)
include_directories(${DEP_ROOT_DIR}/${BRPC_NAME}/include)
include_directories(${DEP_ROOT_DIR}/${BRAFT_NAME}/include)
# Write dependency include directories to a file
file(WRITE ${DEP_ROOT_DIR}/includes.txt "")
file(APPEND ${DEP_ROOT_DIR}/includes.txt "${DEP_ROOT_DIR}/${FOR_NAME}")
file(APPEND ${DEP_ROOT_DIR}/includes.txt " ${DEP_ROOT_DIR}/${H2O_NAME}/include")
file(APPEND ${DEP_ROOT_DIR}/includes.txt " ${DEP_ROOT_DIR}/${ROCKSDB_NAME}/include")
file(APPEND ${DEP_ROOT_DIR}/includes.txt " ${DEP_ROOT_DIR}/${ICONV_NAME}/include")
file(APPEND ${DEP_ROOT_DIR}/includes.txt " ${DEP_ROOT_DIR}/${BRPC_NAME}/include")
file(APPEND ${DEP_ROOT_DIR}/includes.txt " ${DEP_ROOT_DIR}/${BRAFT_NAME}/include")
include_directories(${DEP_ROOT_DIR}/${JEMALLOC_NAME}/include/jemalloc)
link_directories(${DEP_ROOT_DIR}/${GTEST_NAME}/googletest/build)
link_directories(${DEP_ROOT_DIR}/${FOR_NAME})
link_directories(${DEP_ROOT_DIR}/${H2O_NAME}/build)
link_directories(${DEP_ROOT_DIR}/${ROCKSDB_NAME})
link_directories(${DEP_ROOT_DIR}/${ICONV_NAME}/lib/.libs)
if (APPLE)
link_directories(${DEP_ROOT_DIR}/${BRPC_NAME}/lib)
link_directories(${DEP_ROOT_DIR}/${BRAFT_NAME}/lib)
endif()
link_directories(${DEP_ROOT_DIR}/${JEMALLOC_NAME}/lib)
# Write dependency libraries to a file
file(WRITE ${DEP_ROOT_DIR}/libs.txt "")
file(APPEND ${DEP_ROOT_DIR}/libs.txt " ${DEP_ROOT_DIR}/${FOR_NAME}")
file(APPEND ${DEP_ROOT_DIR}/libs.txt " ${DEP_ROOT_DIR}/${H2O_NAME}/build")
file(APPEND ${DEP_ROOT_DIR}/libs.txt " ${DEP_ROOT_DIR}/${ROCKSDB_NAME}")
file(APPEND ${DEP_ROOT_DIR}/libs.txt " ${DEP_ROOT_DIR}/${ICONV_NAME}/lib/.libs")
file(APPEND ${DEP_ROOT_DIR}/libs.txt " ${DEP_ROOT_DIR}/${BRPC_NAME}/lib")
file(APPEND ${DEP_ROOT_DIR}/libs.txt " ${DEP_ROOT_DIR}/${BRAFT_NAME}/lib")
set(JEMALLOC_ROOT_DIR "${DEP_ROOT_DIR}/${JEMALLOC_NAME}")
FIND_PACKAGE(Jemalloc REQUIRED)
add_executable(typesense-server ${SRC_FILES} src/main/typesense_server.cpp)
add_executable(search ${SRC_FILES} src/main/main.cpp)
add_executable(benchmark ${SRC_FILES} src/main/benchmark.cpp)
add_executable(typesense-test ${SRC_FILES} ${TEST_FILES})
add_library(typesense-core STATIC ${SRC_FILES})
target_compile_definitions(
typesense-server PRIVATE
TYPESENSE_VERSION="${TYPESENSE_VERSION}"
)
target_compile_definitions(
typesense-core PRIVATE
TYPESENSE_VERSION="${TYPESENSE_VERSION}"
)
target_compile_definitions(
benchmark PRIVATE
TYPESENSE_VERSION="${TYPESENSE_VERSION}"
@ -133,7 +113,7 @@ target_compile_definitions(
set(ROCKSDB_LIBS rocksdb ${SNAPPY_LIBRARIES} ${ZLIB_LIBRARIES})
if (APPLE)
set(STD_LIB -static-libstdc++) # https://stackoverflow.com/a/26543140/131050 (can't statically link libgcc on Mac)
set(STD_LIB "") # https://stackoverflow.com/a/26543140/131050 (can't statically link libgcc on Mac)
FIND_PACKAGE(ngHTTP2 REQUIRED)
set(CURL_LIBRARIES ${NGHTTP2_LIBRARIES} ${CURL_LIBRARIES}) # nghttp2 was not being statically built
@ -150,9 +130,9 @@ set(ICU_ALL_LIBRARIES ${ICU_I18N_LIBRARIES} ${ICU_LIBRARIES} ${ICU_DATA_LIBRARIE
set(CORE_LIBS h2o-evloop iconv ${CURL_LIBRARIES} for ${ICU_ALL_LIBRARIES} ${G3LOGGER_LIBRARIES}
${ROCKSDB_LIBS} ${OPENSSL_LIBRARIES} pthread dl ${STD_LIB})
set(CORE_LIBS braft brpc ${LevelDB_LIBRARIES} glog ${CORE_LIBS} ${GFLAGS_LIBRARIES} ${PROTOBUF_LIBRARIES} ${SYSTEM_LIBS})
set(CORE_LIBS ${JEMALLOC_LIBRARIES} braft brpc ${LevelDB_LIBRARIES} glog
${CORE_LIBS} ${GFLAGS_LIBRARIES} ${PROTOBUF_LIBRARIES} ${SYSTEM_LIBS})
target_link_libraries(typesense-core ${CORE_LIBS})
target_link_libraries(typesense-server ${CORE_LIBS})
target_link_libraries(search ${CORE_LIBS})
target_link_libraries(benchmark ${CORE_LIBS})

View File

@ -31,16 +31,3 @@ if [[ "$@" == *"--package-binary"* ]]; then
tar -cvzf $PROJECT_DIR/$BUILD_DIR/$RELEASE_NAME.tar.gz -C $PROJECT_DIR/$BUILD_DIR typesense-server typesense-server.md5.txt
echo "Built binary successfully: $PROJECT_DIR/$BUILD_DIR/$RELEASE_NAME.tar.gz"
fi
if [[ "$@" == *"--package-libs"* ]]; then
OS_FAMILY=$(echo `uname` | awk '{print tolower($0)}')
RELEASE_NAME=typesense-server-libs-$TYPESENSE_VERSION-$OS_FAMILY-amd64
LIBS=`cat $PROJECT_DIR/external-$SYSTEM_NAME/libs.txt`
TAR_PATHS=""
for lib in $LIBS; do
TAR_PATHS="$TAR_PATHS -C $lib `ls $lib/*.a | xargs basename`"
done
TAR_PATHS="$TAR_PATHS -C $PROJECT_DIR/$BUILD_DIR `ls $PROJECT_DIR/$BUILD_DIR/*.a | xargs basename`"
tar -cvzf $PROJECT_DIR/$BUILD_DIR/$RELEASE_NAME.tar.gz $TAR_PATHS
fi

34
cmake/Jemalloc.cmake Normal file
View File

@ -0,0 +1,34 @@
# Download and build Jemalloc
set(JEMALLOC_VERSION 5.2.1)
set(JEMALLOC_NAME jemalloc-${JEMALLOC_VERSION})
set(JEMALLOC_TAR_PATH ${DEP_ROOT_DIR}/${JEMALLOC_NAME}.tar.bz2)
if(NOT EXISTS ${JEMALLOC_TAR_PATH})
message(STATUS "Downloading ${JEMALLOC_NAME}...")
file(DOWNLOAD https://github.com/jemalloc/jemalloc/releases/download/${JEMALLOC_VERSION}/${JEMALLOC_NAME}.tar.bz2
${JEMALLOC_TAR_PATH})
endif()
if(NOT EXISTS ${DEP_ROOT_DIR}/${JEMALLOC_NAME})
message(STATUS "Extracting jemalloc...")
execute_process(COMMAND ${CMAKE_COMMAND} -E tar xzf ${JEMALLOC_TAR_PATH} WORKING_DIRECTORY ${DEP_ROOT_DIR})
endif()
if(NOT EXISTS ${DEP_ROOT_DIR}/${JEMALLOC_NAME}/Makefile)
message("Configuring jemalloc locally...")
# Builds with "--with-jemalloc-prefix=je_" on OSX
execute_process(COMMAND ./configure WORKING_DIRECTORY ${DEP_ROOT_DIR}/${JEMALLOC_NAME} RESULT_VARIABLE JEMALLOC_CONFIGURE)
if(NOT JEMALLOC_CONFIGURE EQUAL 0)
message(FATAL_ERROR "${JEMALLOC_NAME} configure failed!")
message("${RESULT_VARIABLE}")
endif()
endif()
if(NOT EXISTS ${DEP_ROOT_DIR}/${JEMALLOC_NAME}/lib/libjemalloc.a)
message("Building jemalloc locally...")
execute_process(COMMAND make "build_lib_static" WORKING_DIRECTORY ${DEP_ROOT_DIR}/${JEMALLOC_NAME})
if(NOT EXISTS ${DEP_ROOT_DIR}/${JEMALLOC_NAME}/lib/libjemalloc.a)
message(FATAL_ERROR "${JEMALLOC_NAME} build failed!")
endif()
endif()

View File

@ -0,0 +1,38 @@
# Tries to find Jemalloc headers and libraries.
#
# Usage of this module as follows:
#
# find_package(jemalloc)
#
# Variables used by this module, they can change the default behaviour and need
# to be set before calling find_package:
#
# JEMALLOC_ROOT_DIR Set this variable to the root installation of
# Jemalloc if the module has problems finding
# the proper installation path.
#
# Variables defined by this module:
#
# JEMALLOC_FOUND System has Jemalloc libs/headers
# JEMALLOC_LIBRARIES The Jemalloc libraries
# JEMALLOC_INCLUDE_DIR The location of Jemalloc headers
find_path(JEMALLOC_INCLUDE_DIR
NAMES jemalloc.h
HINTS ${JEMALLOC_ROOT_DIR}/include/jemalloc
NO_DEFAULT_PATH)
find_library(JEMALLOC_LIBRARIES
NAMES jemalloc
HINTS ${JEMALLOC_ROOT_DIR}/lib
NO_DEFAULT_PATH)
include(FindPackageHandleStandardArgs)
find_package_handle_standard_args(jemalloc DEFAULT_MSG
JEMALLOC_LIBRARIES
JEMALLOC_INCLUDE_DIR)
mark_as_advanced(
JEMALLOC_ROOT_DIR
JEMALLOC_LIBRARIES
JEMALLOC_INCLUDE_DIR)

View File

@ -46,18 +46,6 @@ if [[ "$@" == *"--package-binary"* ]]; then
echo "Built binary successfully: $PROJECT_DIR/$BUILD_DIR/$RELEASE_NAME.tar.gz"
fi
if [[ "$@" == *"--package-libs"* ]]; then
OS_FAMILY=linux
RELEASE_NAME=typesense-server-libs-$TYPESENSE_VERSION-$OS_FAMILY-amd64
LIBS=`cat $PROJECT_DIR/external-$SYSTEM_NAME/libs.txt`
TAR_PATHS=""
for lib in $LIBS; do
TAR_PATHS="$TAR_PATHS -C $PROJECT_DIR/../$lib `ls $PROJECT_DIR/../$lib/*.a | xargs basename`"
done
TAR_PATHS="$TAR_PATHS -C $PROJECT_DIR/$BUILD_DIR `ls $PROJECT_DIR/$BUILD_DIR/*.a | xargs basename`"
tar -cvzf $PROJECT_DIR/$BUILD_DIR/$RELEASE_NAME.tar.gz $TAR_PATHS
fi
#
#if [[ "$@" == *"--create-deb-upload"* ]]; then
# docker run -it -v $PROJECT_DIR:/typesense typesense/typesense-development:23-JUNE-2020-1 cmake -DTYPESENSE_VERSION=$TYPESENSE_VERSION \

109
include/cvt.h Normal file
View File

@ -0,0 +1,109 @@
/* Compact Variable Trie
================================================================================================================
ates, at, as, bet, to, tk
[ * ]
|
a b t
\
s t etØ k o
/ / /
Ø esØ Ø Ø Ø
BASIC DESIGN
============
* All siblings in the same block.
* Pointer to ONLY FIRST child node of each sibling.
* Each sibling node's children represented by their character/byte
* For root there are no siblings, so pointer only to `a` child.
ROOT -> [0|0|PTRA][3][a][b][t]
PTRA -> [0|PTRS][2|PTRE][4|PTRK[2][s][t][2][e][t][2][k][o]
PTRS -> [0|PTRØ][1|PTRT][1][Ø][2][e][Ø]
PTRT -> [0|L_PTRE][3|PTRØ][3][e][s][Ø][1][Ø] (path compression)
PTRØ -> [0|LEAF]
[TYPE+OFFSET][PTR_1]..[NUM_CHILDREN][A][B]..[X]
2 bytes for type+offset
6 bytes for address
1 byte for num_children
x bytes for bytes
Actual offset to the Nth node's children: (8*NUM_CHILDREN) + N + offset
if num_children >= 32:
Use bitset to represent children present
Read 32 bytes and do bitset operations to extract matched index
else:
Use array to represent children
Read `num_children` bytes and do sequential search
Removal of [be]
1. Realloc contents of PTR1 by removing "e" from the nodes list
2. Free PTR3
3. Realloc contents of ROOT by removing "b" from the nodes list
*/
#include <cstdint>
#include <cstddef>
class CVTrie {
private:
size_t size;
// [TYPE+OFFSET][PTR_1]... (8 bytes each)
// [NUM_PREFIX][A][B]..[X] (upto 33 bytes)
uint8_t* root;
const uintptr_t PTR_MASK = ~(1ULL << 48ULL);
public:
uint8_t* get_ptr(const uint8_t* tagged_ptr) {
// Unfortunately, right shifting of signed integer for sign extension is implementation-defined
// but works on all major compilers
return (uint8_t*)( (intptr_t)((uintptr_t)tagged_ptr << 16ULL) >> 16ULL );
}
uint8_t* tag_ptr(const uint8_t* ptr, const uint64_t data) {
return (uint8_t*)(((uintptr_t)ptr & PTR_MASK) | (data << 48ULL));
}
uint16_t get_data(const uint16_t* ptr) {
return (uintptr_t)(ptr) >> 56ULL;;
}
void add(const unsigned char* key, const size_t length, void* value) {
// If the key exists, augment the node, otherwise insert a new node
size_t num_siblings = 1;
uint8_t* buf = root;
size_t key_index = 0;
if(root == nullptr) {
// trie is empty
root = new uint8_t[8+1+1];
}
while(true) {
// for each sibling
for(auto sindex = 0; sindex < num_siblings; sindex++) {
unsigned char c = key[key_index];
}
}
}
};

View File

@ -186,7 +186,7 @@ public:
return Option<std::vector<std::string>*>(updates);
}
rocksdb::unique_ptr<rocksdb::TransactionLogIterator> iter;
std::unique_ptr<rocksdb::TransactionLogIterator> iter;
rocksdb::Status status = db->GetUpdatesSince(seq_number, &iter);
if(!status.ok()) {

View File

@ -2,6 +2,16 @@
#include "core_api.h"
#include "config.h"
extern "C" {
#include "jemalloc.h"
}
#ifdef __APPLE__
extern "C" {
extern void je_zone_register();
}
#endif
void master_server_routes() {
// collection management
server->post("/collections", post_create_collection);
@ -56,6 +66,15 @@ void replica_server_routes() {
}
int main(int argc, char **argv) {
#ifdef __APPLE__
// On OS X, je_zone_register registers jemalloc with the system allocator.
// We have to force the presence of these symbols on macOS by explicitly calling this method.
// See these issues:
// - https://github.com/jemalloc/jemalloc/issues/708
// - https://github.com/ClickHouse/ClickHouse/pull/11897
je_zone_register();
#endif
Config config;
cmdline::parser options;

View File

@ -1,7 +1,6 @@
#include <curl/curl.h>
#include <sys/stat.h>
#include <gflags/gflags.h>
#include <dlfcn.h>
#include <brpc/controller.h>
#include <brpc/server.h>
#include <braft/raft.h>
@ -18,6 +17,21 @@
HttpServer* server;
std::atomic<bool> quit_raft_service;
extern "C" {
typedef int (*mallctl_t)(const char *name, void *oldp, size_t *oldlenp, void *newp, size_t newlen);
}
bool using_jemalloc() {
// On OSX, jemalloc API is prefixed with "je_"
mallctl_t mallctl;
#ifdef __APPLE__
mallctl = (mallctl_t) ::dlsym(RTLD_DEFAULT, "je_mallctl");
#else
mallctl = (mallctl_t) ::dlsym(RTLD_DEFAULT, "mallctl");
#endif
return (mallctl != nullptr);
}
void catch_interrupt(int sig) {
LOG(INFO) << "Stopping Typesense server...";
signal(sig, SIG_IGN); // ignore for now as we want to shut down elegantly
@ -292,6 +306,13 @@ int start_raft_server(ReplicationState& replication_state, const std::string& st
int run_server(const Config & config, const std::string & version, void (*master_server_routes)()) {
LOG(INFO) << "Starting Typesense " << version << std::flush;
if(using_jemalloc()) {
LOG(INFO) << "Typesense is using jemalloc.";
} else {
LOG(WARNING) << "Typesense is NOT using jemalloc.";
}
quit_raft_service = false;
if(!directory_exists(config.get_data_dir())) {

View File

@ -194,7 +194,7 @@ TEST(ArtTest, test_art_insert_delete) {
// Delete, should get lineno back
art_values* values = (art_values*)art_delete(&t, (unsigned char*)buf, len);
EXPECT_EQ(line, values->ids.at(0));
free(values);
delete values;
// Check the size
ASSERT_TRUE(art_size(&t) == nlines - line);