apple/foundationdb
1
0
Fork 0
mirror of https://github.com/apple/foundationdb.git synced 2025-05-17 03:12:21 +08:00
Code Issues Packages Projects Releases Wiki Activity

Merge remote-tracking branch 'origin/main' into dd-refactor

Browse Source
This commit is contained in:
sfc-gh-tclinkenbeard 2022-02-09 14:29:16 -08:00
commit 3f0e2ae62e
12 changed files with 70 additions and 456 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
cmake/AddFdbTest.cmake
View File

@ -39,9 +39,6 @@ function(configure_testing)
endfunction() endfunction()
function(verify_testing) function(verify_testing)
if(NOT ENABLE_SIMULATION_TESTS)
return()
endif()
foreach(test_file IN LISTS fdb_test_files) foreach(test_file IN LISTS fdb_test_files)
message(SEND_ERROR "${test_file} found but it is not associated with a test") message(SEND_ERROR "${test_file} found but it is not associated with a test")
endforeach() endforeach()

16
cmake/CompileBoost.cmake
View File

@ -35,7 +35,7 @@ function(compile_boost)
endif() endif()
# Update the user-config.jam # Update the user-config.jam
set(BOOST_ADDITIONAL_COMPILE_OPTIOINS "") set(BOOST_ADDITIONAL_COMPILE_OPTIONS "")
foreach(flag IN LISTS BOOST_COMPILER_FLAGS COMPILE_BOOST_CXXFLAGS) foreach(flag IN LISTS BOOST_COMPILER_FLAGS COMPILE_BOOST_CXXFLAGS)
string(APPEND BOOST_ADDITIONAL_COMPILE_OPTIONS "<cxxflags>${flag} ") string(APPEND BOOST_ADDITIONAL_COMPILE_OPTIONS "<cxxflags>${flag} ")
endforeach() endforeach()
@ -49,8 +49,8 @@ function(compile_boost)
include(ExternalProject) include(ExternalProject)
set(BOOST_INSTALL_DIR "${CMAKE_BINARY_DIR}/boost_install") set(BOOST_INSTALL_DIR "${CMAKE_BINARY_DIR}/boost_install")
ExternalProject_add("${COMPILE_BOOST_TARGET}Project" ExternalProject_add("${COMPILE_BOOST_TARGET}Project"
URL "https://boostorg.jfrog.io/artifactory/main/release/1.77.0/source/boost_1_77_0.tar.bz2" URL "https://boostorg.jfrog.io/artifactory/main/release/1.78.0/source/boost_1_78_0.tar.bz2"
URL_HASH SHA256=fc9f85fc030e233142908241af7a846e60630aa7388de9a5fafb1f3a26840854 URL_HASH SHA256=8681f175d4bdb26c52222665793eef08490d7758529330f98d3b29dd0735bccc
CONFIGURE_COMMAND ${BOOTSTRAP_COMMAND} ${BOOTSTRAP_ARGS} --with-libraries=${BOOTSTRAP_LIBRARIES} --with-toolset=${BOOST_TOOLSET} CONFIGURE_COMMAND ${BOOTSTRAP_COMMAND} ${BOOTSTRAP_ARGS} --with-libraries=${BOOTSTRAP_LIBRARIES} --with-toolset=${BOOST_TOOLSET}
BUILD_COMMAND ${B2_COMMAND} link=static ${COMPILE_BOOST_BUILD_ARGS} --prefix=${BOOST_INSTALL_DIR} ${USER_CONFIG_FLAG} install BUILD_COMMAND ${B2_COMMAND} link=static ${COMPILE_BOOST_BUILD_ARGS} --prefix=${BOOST_INSTALL_DIR} ${USER_CONFIG_FLAG} install
BUILD_IN_SOURCE ON BUILD_IN_SOURCE ON
@ -89,12 +89,12 @@ set(Boost_USE_STATIC_LIBS ON)
# Clang and Gcc will have different name mangling to std::call_once, etc. # Clang and Gcc will have different name mangling to std::call_once, etc.
if (UNIX AND CMAKE_CXX_COMPILER_ID MATCHES "Clang$") if (UNIX AND CMAKE_CXX_COMPILER_ID MATCHES "Clang$")
list(APPEND CMAKE_PREFIX_PATH /opt/boost_1_72_0_clang) list(APPEND CMAKE_PREFIX_PATH /opt/boost_1_78_0_clang)
set(BOOST_HINT_PATHS /opt/boost_1_72_0_clang) set(BOOST_HINT_PATHS /opt/boost_1_78_0_clang)
message(STATUS "Using Clang version of boost::context") message(STATUS "Using Clang version of boost::context")
else () else ()
list(APPEND CMAKE_PREFIX_PATH /opt/boost_1_72_0) list(APPEND CMAKE_PREFIX_PATH /opt/boost_1_78_0)
set(BOOST_HINT_PATHS /opt/boost_1_72_0) set(BOOST_HINT_PATHS /opt/boost_1_78_0)
message(STATUS "Using g++ version of boost::context") message(STATUS "Using g++ version of boost::context")
endif () endif ()
@ -113,7 +113,7 @@ if(WIN32)
return() return()
endif() endif()
find_package(Boost 1.77.0 EXACT QUIET COMPONENTS context CONFIG PATHS ${BOOST_HINT_PATHS}) find_package(Boost 1.78.0 EXACT QUIET COMPONENTS context CONFIG PATHS ${BOOST_HINT_PATHS})
set(FORCE_BOOST_BUILD OFF CACHE BOOL "Forces cmake to build boost and ignores any installed boost") set(FORCE_BOOST_BUILD OFF CACHE BOOL "Forces cmake to build boost and ignores any installed boost")
if(Boost_FOUND AND NOT FORCE_BOOST_BUILD) if(Boost_FOUND AND NOT FORCE_BOOST_BUILD)

8
cmake/CompileRocksDB.cmake
View File

@ -1,6 +1,6 @@
# FindRocksDB # FindRocksDB
find_package(RocksDB 6.22.1) find_package(RocksDB 6.27.3)
include(ExternalProject) include(ExternalProject)
@ -22,6 +22,7 @@ if (RocksDB_FOUND)
-DWITH_SNAPPY=OFF -DWITH_SNAPPY=OFF
-DWITH_ZLIB=OFF -DWITH_ZLIB=OFF
-DWITH_ZSTD=OFF -DWITH_ZSTD=OFF
-DWITH_LIBURING=${WITH_LIBURING}
-DWITH_TSAN=${USE_TSAN} -DWITH_TSAN=${USE_TSAN}
-DWITH_ASAN=${USE_ASAN} -DWITH_ASAN=${USE_ASAN}
-DWITH_UBSAN=${USE_UBSAN} -DWITH_UBSAN=${USE_UBSAN}
@ -36,8 +37,8 @@ if (RocksDB_FOUND)
${BINARY_DIR}/librocksdb.a) ${BINARY_DIR}/librocksdb.a)
else() else()
ExternalProject_Add(rocksdb ExternalProject_Add(rocksdb
URL https://github.com/facebook/rocksdb/archive/v6.22.1.tar.gz URL https://github.com/facebook/rocksdb/archive/refs/tags/v6.27.3.tar.gz
URL_HASH SHA256=2df8f34a44eda182e22cf84dee7a14f17f55d305ff79c06fb3cd1e5f8831e00d URL_HASH SHA256=ee29901749b9132692b26f0a6c1d693f47d1a9ed8e3771e60556afe80282bf58
CMAKE_ARGS -DUSE_RTTI=1 -DPORTABLE=${PORTABLE_ROCKSDB} CMAKE_ARGS -DUSE_RTTI=1 -DPORTABLE=${PORTABLE_ROCKSDB}
-DCMAKE_CXX_STANDARD=${CMAKE_CXX_STANDARD} -DCMAKE_CXX_STANDARD=${CMAKE_CXX_STANDARD}
-DCMAKE_CXX_COMPILER=${CMAKE_CXX_COMPILER} -DCMAKE_CXX_COMPILER=${CMAKE_CXX_COMPILER}
@ -52,6 +53,7 @@ else()
-DWITH_SNAPPY=OFF -DWITH_SNAPPY=OFF
-DWITH_ZLIB=OFF -DWITH_ZLIB=OFF
-DWITH_ZSTD=OFF -DWITH_ZSTD=OFF
-DWITH_LIBURING=${WITH_LIBURING}
-DWITH_TSAN=${USE_TSAN} -DWITH_TSAN=${USE_TSAN}
-DWITH_ASAN=${USE_ASAN} -DWITH_ASAN=${USE_ASAN}
-DWITH_UBSAN=${USE_UBSAN} -DWITH_UBSAN=${USE_UBSAN}

2
cmake/FDBComponents.cmake
View File

@ -162,6 +162,8 @@ endif()
################################################################################ ################################################################################
set(SSD_ROCKSDB_EXPERIMENTAL ON CACHE BOOL "Build with experimental RocksDB support") set(SSD_ROCKSDB_EXPERIMENTAL ON CACHE BOOL "Build with experimental RocksDB support")
set(PORTABLE_ROCKSDB ON CACHE BOOL "Compile RocksDB in portable mode") # Set this to OFF to compile RocksDB with `-march=native`
set(WITH_LIBURING OFF CACHE BOOL "Build with liburing enabled") # Set this to ON to include liburing
# RocksDB is currently enabled by default for GCC but does not build with the latest # RocksDB is currently enabled by default for GCC but does not build with the latest
# Clang. # Clang.
if (SSD_ROCKSDB_EXPERIMENTAL AND GCC) if (SSD_ROCKSDB_EXPERIMENTAL AND GCC)

26
cmake/Finduring.cmake Normal file
View File

@ -0,0 +1,26 @@
# - Find liburing
#
# uring_INCLUDE_DIR - Where to find liburing.h
# uring_LIBRARIES - List of libraries when using uring.
# uring_FOUND - True if uring found.
find_path(uring_INCLUDE_DIR
NAMES liburing.h)
find_library(uring_LIBRARIES
NAMES liburing.a liburing)
include(FindPackageHandleStandardArgs)
find_package_handle_standard_args(uring
DEFAULT_MSG uring_LIBRARIES uring_INCLUDE_DIR)
mark_as_advanced(
uring_INCLUDE_DIR
uring_LIBRARIES)
if(uring_FOUND AND NOT TARGET uring::uring)
add_library(uring::uring UNKNOWN IMPORTED)
set_target_properties(uring::uring PROPERTIES
INTERFACE_INCLUDE_DIRECTORIES "${uring_INCLUDE_DIR}"
IMPORTED_LINK_INTERFACE_LANGUAGES "C"
IMPORTED_LOCATION "${uring_LIBRARIES}")
endif()

416
fdbcli/fdbcli.actor.cpp
View File

@ -644,422 +644,6 @@ ACTOR Future<Void> commitTransaction(Reference<ITransaction> tr) {
return Void(); return Void();
} }
// FIXME: Factor address parsing from coordinators, include, exclude
ACTOR Future<bool> coordinators(Database db, std::vector<StringRef> tokens, bool isClusterTLS) {
state StringRef setName;
StringRef nameTokenBegin = LiteralStringRef("description=");
for (auto tok = tokens.begin() + 1; tok != tokens.end(); ++tok)
if (tok->startsWith(nameTokenBegin)) {
setName = tok->substr(nameTokenBegin.size());
std::copy(tok + 1, tokens.end(), tok);
tokens.resize(tokens.size() - 1);
break;
}
bool automatic = tokens.size() == 2 && tokens[1] == LiteralStringRef("auto");
state Reference<IQuorumChange> change;
if (tokens.size() == 1 && setName.size()) {
change = noQuorumChange();
} else if (automatic) {
// Automatic quorum change
change = autoQuorumChange();
} else {
state std::set<NetworkAddress> addresses;
state std::vector<StringRef>::iterator t;
for (t = tokens.begin() + 1; t != tokens.end(); ++t) {
try {
// SOMEDAY: Check for keywords
auto const& addr = NetworkAddress::parse(t->toString());
if (addresses.count(addr)) {
fprintf(stderr, "ERROR: passed redundant coordinators: `%s'\n", addr.toString().c_str());
return true;
}
addresses.insert(addr);
} catch (Error& e) {
if (e.code() == error_code_connection_string_invalid) {
fprintf(stderr, "ERROR: '%s' is not a valid network endpoint address\n", t->toString().c_str());
return true;
}
throw;
}
}
std::vector<NetworkAddress> addressesVec(addresses.begin(), addresses.end());
change = specifiedQuorumChange(addressesVec);
}
if (setName.size())
change = nameQuorumChange(setName.toString(), change);
CoordinatorsResult r = wait(makeInterruptable(changeQuorum(db, change)));
// Real errors get thrown from makeInterruptable and printed by the catch block in cli(), but
// there are various results specific to changeConfig() that we need to report:
bool err = true;
switch (r) {
case CoordinatorsResult::INVALID_NETWORK_ADDRESSES:
fprintf(stderr, "ERROR: The specified network addresses are invalid\n");
break;
case CoordinatorsResult::SAME_NETWORK_ADDRESSES:
printf("No change (existing configuration satisfies request)\n");
err = false;
break;
case CoordinatorsResult::NOT_COORDINATORS:
fprintf(stderr, "ERROR: Coordination servers are not running on the specified network addresses\n");
break;
case CoordinatorsResult::DATABASE_UNREACHABLE:
fprintf(stderr, "ERROR: Database unreachable\n");
break;
case CoordinatorsResult::BAD_DATABASE_STATE:
fprintf(stderr,
"ERROR: The database is in an unexpected state from which changing coordinators might be unsafe\n");
break;
case CoordinatorsResult::COORDINATOR_UNREACHABLE:
fprintf(stderr, "ERROR: One of the specified coordinators is unreachable\n");
break;
case CoordinatorsResult::SUCCESS:
printf("Coordination state changed\n");
err = false;
break;
case CoordinatorsResult::NOT_ENOUGH_MACHINES:
fprintf(stderr, "ERROR: Too few fdbserver machines to provide coordination at the current redundancy level\n");
break;
default:
ASSERT(false);
};
return err;
}
// Includes the servers that could be IP addresses or localities back to the cluster.
ACTOR Future<bool> include(Database db, std::vector<StringRef> tokens) {
std::vector<AddressExclusion> addresses;
state std::vector<std::string> localities;
state bool failed = false;
state bool all = false;
for (auto t = tokens.begin() + 1; t != tokens.end(); ++t) {
if (*t == LiteralStringRef("all")) {
all = true;
} else if (*t == LiteralStringRef("failed")) {
failed = true;
} else if (t->startsWith(LocalityData::ExcludeLocalityPrefix) && t->toString().find(':') != std::string::npos) {
// if the token starts with 'locality_' prefix.
localities.push_back(t->toString());
} else {
auto a = AddressExclusion::parse(*t);
if (!a.isValid()) {
fprintf(stderr,
"ERROR: '%s' is neither a valid network endpoint address nor a locality\n",
t->toString().c_str());
if (t->toString().find(":tls") != std::string::npos)
printf(" Do not include the `:tls' suffix when naming a process\n");
return true;
}
addresses.push_back(a);
}
}
if (all) {
std::vector<AddressExclusion> includeAll;
includeAll.push_back(AddressExclusion());
wait(makeInterruptable(includeServers(db, includeAll, failed)));
wait(makeInterruptable(includeLocalities(db, localities, failed, all)));
} else {
if (!addresses.empty()) {
wait(makeInterruptable(includeServers(db, addresses, failed)));
}
if (!localities.empty()) {
// includes the servers that belong to given localities.
wait(makeInterruptable(includeLocalities(db, localities, failed, all)));
}
}
return false;
};
ACTOR Future<bool> exclude(Database db,
std::vector<StringRef> tokens,
Reference<ClusterConnectionFile> ccf,
Future<Void> warn) {
if (tokens.size() <= 1) {
state Future<std::vector<AddressExclusion>> fexclAddresses = makeInterruptable(getExcludedServers(db));
state Future<std::vector<std::string>> fexclLocalities = makeInterruptable(getExcludedLocalities(db));
wait(success(fexclAddresses) && success(fexclLocalities));
std::vector<AddressExclusion> exclAddresses = fexclAddresses.get();
std::vector<std::string> exclLocalities = fexclLocalities.get();
if (!exclAddresses.size() && !exclLocalities.size()) {
printf("There are currently no servers or localities excluded from the database.\n"
"To learn how to exclude a server, type `help exclude'.\n");
return false;
}
printf("There are currently %zu servers or localities being excluded from the database:\n",
exclAddresses.size() + exclLocalities.size());
for (const auto& e : exclAddresses)
printf(" %s\n", e.toString().c_str());
for (const auto& e : exclLocalities)
printf(" %s\n", e.c_str());
printf("To find out whether it is safe to remove one or more of these\n"
"servers from the cluster, type `exclude <addresses>'.\n"
"To return one of these servers to the cluster, type `include <addresses>'.\n");
return false;
} else {
state std::vector<AddressExclusion> exclusionVector;
state std::set<AddressExclusion> exclusionSet;
state std::vector<AddressExclusion> exclusionAddresses;
state std::unordered_set<std::string> exclusionLocalities;
state std::vector<std::string> noMatchLocalities;
state bool force = false;
state bool waitForAllExcluded = true;
state bool markFailed = false;
state std::vector<ProcessData> workers = wait(makeInterruptable(getWorkers(db)));
for (auto t = tokens.begin() + 1; t != tokens.end(); ++t) {
if (*t == LiteralStringRef("FORCE")) {
force = true;
} else if (*t == LiteralStringRef("no_wait")) {
waitForAllExcluded = false;
} else if (*t == LiteralStringRef("failed")) {
markFailed = true;
} else if (t->startsWith(LocalityData::ExcludeLocalityPrefix) &&
t->toString().find(':') != std::string::npos) {
std::set<AddressExclusion> localityAddresses = getAddressesByLocality(workers, t->toString());
if (localityAddresses.empty()) {
noMatchLocalities.push_back(t->toString());
} else {
// add all the server ipaddresses that belong to the given localities to the exclusionSet.
exclusionVector.insert(exclusionVector.end(), localityAddresses.begin(), localityAddresses.end());
exclusionSet.insert(localityAddresses.begin(), localityAddresses.end());
}
exclusionLocalities.insert(t->toString());
} else {
auto a = AddressExclusion::parse(*t);
if (!a.isValid()) {
fprintf(stderr,
"ERROR: '%s' is neither a valid network endpoint address nor a locality\n",
t->toString().c_str());
if (t->toString().find(":tls") != std::string::npos)
printf(" Do not include the `:tls' suffix when naming a process\n");
return true;
}
exclusionVector.push_back(a);
exclusionSet.insert(a);
exclusionAddresses.push_back(a);
}
}
if (exclusionAddresses.empty() && exclusionLocalities.empty()) {
fprintf(stderr, "ERROR: At least one valid network endpoint address or a locality is not provided\n");
return true;
}
if (!force) {
if (markFailed) {
state bool safe;
try {
bool _safe = wait(makeInterruptable(checkSafeExclusions(db, exclusionVector)));
safe = _safe;
} catch (Error& e) {
if (e.code() == error_code_actor_cancelled)
throw;
TraceEvent("CheckSafeExclusionsError").error(e);
safe = false;
}
if (!safe) {
std::string errorStr =
"ERROR: It is unsafe to exclude the specified servers at this time.\n"
"Please check that this exclusion does not bring down an entire storage team.\n"
"Please also ensure that the exclusion will keep a majority of coordinators alive.\n"
"You may add more storage processes or coordinators to make the operation safe.\n"
"Type `exclude FORCE failed <ADDRESS...>' to exclude without performing safety checks.\n";
printf("%s", errorStr.c_str());
return true;
}
}
StatusObject status = wait(makeInterruptable(StatusClient::statusFetcher(db)));
state std::string errorString =
"ERROR: Could not calculate the impact of this exclude on the total free space in the cluster.\n"
"Please try the exclude again in 30 seconds.\n"
"Type `exclude FORCE <ADDRESS...>' to exclude without checking free space.\n";
StatusObjectReader statusObj(status);
StatusObjectReader statusObjCluster;
if (!statusObj.get("cluster", statusObjCluster)) {
fprintf(stderr, "%s", errorString.c_str());
return true;
}
StatusObjectReader processesMap;
if (!statusObjCluster.get("processes", processesMap)) {
fprintf(stderr, "%s", errorString.c_str());
return true;
}
state int ssTotalCount = 0;
state int ssExcludedCount = 0;
state double worstFreeSpaceRatio = 1.0;
try {
for (auto proc : processesMap.obj()) {
bool storageServer = false;
StatusArray rolesArray = proc.second.get_obj()["roles"].get_array();
for (StatusObjectReader role : rolesArray) {
if (role["role"].get_str() == "storage") {
storageServer = true;
break;
}
}
// Skip non-storage servers in free space calculation
if (!storageServer)
continue;
StatusObjectReader process(proc.second);
std::string addrStr;
if (!process.get("address", addrStr)) {
fprintf(stderr, "%s", errorString.c_str());
return true;
}
NetworkAddress addr = NetworkAddress::parse(addrStr);
bool excluded =
(process.has("excluded") && process.last().get_bool()) || addressExcluded(exclusionSet, addr);
ssTotalCount++;
if (excluded)
ssExcludedCount++;
if (!excluded) {
StatusObjectReader disk;
if (!process.get("disk", disk)) {
fprintf(stderr, "%s", errorString.c_str());
return true;
}
int64_t total_bytes;
if (!disk.get("total_bytes", total_bytes)) {
fprintf(stderr, "%s", errorString.c_str());
return true;
}
int64_t free_bytes;
if (!disk.get("free_bytes", free_bytes)) {
fprintf(stderr, "%s", errorString.c_str());
return true;
}
worstFreeSpaceRatio = std::min(worstFreeSpaceRatio, double(free_bytes) / total_bytes);
}
}
} catch (...) // std::exception
{
fprintf(stderr, "%s", errorString.c_str());
return true;
}
if (ssExcludedCount == ssTotalCount ||
(1 - worstFreeSpaceRatio) * ssTotalCount / (ssTotalCount - ssExcludedCount) > 0.9) {
fprintf(stderr,
"ERROR: This exclude may cause the total free space in the cluster to drop below 10%%.\n"
"Type `exclude FORCE <ADDRESS...>' to exclude without checking free space.\n");
return true;
}
}
if (!exclusionAddresses.empty()) {
wait(makeInterruptable(excludeServers(db, exclusionAddresses, markFailed)));
}
if (!exclusionLocalities.empty()) {
wait(makeInterruptable(excludeLocalities(db, exclusionLocalities, markFailed)));
}
if (waitForAllExcluded) {
printf("Waiting for state to be removed from all excluded servers. This may take a while.\n");
printf("(Interrupting this wait with CTRL+C will not cancel the data movement.)\n");
}
if (warn.isValid())
warn.cancel();
state std::set<NetworkAddress> notExcludedServers =
wait(makeInterruptable(checkForExcludingServers(db, exclusionVector, waitForAllExcluded)));
std::map<IPAddress, std::set<uint16_t>> workerPorts;
for (auto addr : workers)
workerPorts[addr.address.ip].insert(addr.address.port);
// Print a list of all excluded addresses that don't have a corresponding worker
std::set<AddressExclusion> absentExclusions;
for (const auto& addr : exclusionVector) {
auto worker = workerPorts.find(addr.ip);
if (worker == workerPorts.end())
absentExclusions.insert(addr);
else if (addr.port > 0 && worker->second.count(addr.port) == 0)
absentExclusions.insert(addr);
}
for (const auto& exclusion : exclusionVector) {
if (absentExclusions.find(exclusion) != absentExclusions.end()) {
if (exclusion.port == 0) {
fprintf(stderr,
" %s(Whole machine) ---- WARNING: Missing from cluster!Be sure that you excluded the "
"correct machines before removing them from the cluster!\n",
exclusion.ip.toString().c_str());
} else {
fprintf(stderr,
" %s ---- WARNING: Missing from cluster! Be sure that you excluded the correct processes "
"before removing them from the cluster!\n",
exclusion.toString().c_str());
}
} else if (std::any_of(notExcludedServers.begin(), notExcludedServers.end(), [&](const NetworkAddress& a) {
return addressExcluded({ exclusion }, a);
})) {
if (exclusion.port == 0) {
fprintf(stderr,
" %s(Whole machine) ---- WARNING: Exclusion in progress! It is not safe to remove this "
"machine from the cluster\n",
exclusion.ip.toString().c_str());
} else {
fprintf(stderr,
" %s ---- WARNING: Exclusion in progress! It is not safe to remove this process from the "
"cluster\n",
exclusion.toString().c_str());
}
} else {
if (exclusion.port == 0) {
printf(" %s(Whole machine) ---- Successfully excluded. It is now safe to remove this machine "
"from the cluster.\n",
exclusion.ip.toString().c_str());
} else {
printf(
" %s ---- Successfully excluded. It is now safe to remove this process from the cluster.\n",
exclusion.toString().c_str());
}
}
}
for (const auto& locality : noMatchLocalities) {
fprintf(
stderr,
" %s ---- WARNING: Currently no servers found with this locality match! Be sure that you excluded "
"the correct locality.\n",
locality.c_str());
}
ClusterConnectionString ccs = wait(ccf->getStoredConnectionString());
bool foundCoordinator = false;
for (const auto& c : ccs.coordinators()) {
if (std::count(exclusionVector.begin(), exclusionVector.end(), AddressExclusion(c.ip, c.port)) ||
std::count(exclusionVector.begin(), exclusionVector.end(), AddressExclusion(c.ip))) {
fprintf(stderr, "WARNING: %s is a coordinator!\n", c.toString().c_str());
foundCoordinator = true;
}
}
if (foundCoordinator)
printf("Type `help coordinators' for information on how to change the\n"
"cluster's coordination servers before removing them.\n");
return false;
}
}
ACTOR Future<bool> createSnapshot(Database db, std::vector<StringRef> tokens) { ACTOR Future<bool> createSnapshot(Database db, std::vector<StringRef> tokens) {
state Standalone<StringRef> snapCmd; state Standalone<StringRef> snapCmd;
state UID snapUID = deterministicRandom()->randomUniqueID(); state UID snapUID = deterministicRandom()->randomUniqueID();

18
fdbserver/CMakeLists.txt
View File

@ -35,7 +35,7 @@ set(FDBSERVER_SRCS
DDTeamCollection.h DDTeamCollection.h
DiskQueue.actor.cpp DiskQueue.actor.cpp
EncryptKeyProxyInterface.h EncryptKeyProxyInterface.h
EncryptKeyProxy.actor.cpp EncryptKeyProxy.actor.cpp
fdbserver.actor.cpp fdbserver.actor.cpp
FDBExecHelper.actor.cpp FDBExecHelper.actor.cpp
FDBExecHelper.actor.h FDBExecHelper.actor.h
@ -299,14 +299,15 @@ add_library(fdb_sqlite STATIC
if (WITH_ROCKSDB_EXPERIMENTAL) if (WITH_ROCKSDB_EXPERIMENTAL)
add_definitions(-DSSD_ROCKSDB_EXPERIMENTAL) add_definitions(-DSSD_ROCKSDB_EXPERIMENTAL)
# Set this to 0 if you want to compile RocksDB with `-march=native`.
set(PORTABLE_ROCKSDB 1)
include(CompileRocksDB) include(CompileRocksDB)
# CompileRocksDB sets `lz4_LIBRARIES` to be the shared lib, we want to link # CompileRocksDB sets `lz4_LIBRARIES` to be the shared lib, we want to link
# statically, so find the static library here. # statically, so find the static library here.
find_library(lz4_STATIC_LIBRARIES find_library(lz4_STATIC_LIBRARIES
NAMES liblz4.a REQUIRED) NAMES liblz4.a REQUIRED)
if (WITH_LIBURING)
find_package(uring)
endif()
endif() endif()
# Suppress warnings in sqlite since it's third party # Suppress warnings in sqlite since it's third party
@ -326,8 +327,15 @@ target_include_directories(fdbserver PRIVATE
${CMAKE_CURRENT_SOURCE_DIR}/workloads) ${CMAKE_CURRENT_SOURCE_DIR}/workloads)
if (WITH_ROCKSDB_EXPERIMENTAL) if (WITH_ROCKSDB_EXPERIMENTAL)
add_dependencies(fdbserver rocksdb) add_dependencies(fdbserver rocksdb)
target_include_directories(fdbserver PRIVATE ${ROCKSDB_INCLUDE_DIR}) if(WITH_LIBURING)
target_link_libraries(fdbserver PRIVATE fdbclient fdb_sqlite ${ROCKSDB_LIBRARIES} ${lz4_STATIC_LIBRARIES}) target_include_directories(fdbserver PRIVATE ${ROCKSDB_INCLUDE_DIR} ${uring_INCLUDE_DIR})
target_link_libraries(fdbserver PRIVATE fdbclient fdb_sqlite ${ROCKSDB_LIBRARIES} ${uring_LIBRARIES} ${lz4_STATIC_LIBRARIES})
target_compile_definitions(fdbserver PRIVATE BOOST_ASIO_HAS_IO_URING=1 BOOST_ASIO_DISABLE_EPOLL=1)
else()
target_include_directories(fdbserver PRIVATE ${ROCKSDB_INCLUDE_DIR})
target_link_libraries(fdbserver PRIVATE fdbclient fdb_sqlite ${ROCKSDB_LIBRARIES} ${lz4_STATIC_LIBRARIES})
target_compile_definitions(fdbserver PRIVATE)
endif()
else() else()
target_link_libraries(fdbserver PRIVATE fdbclient fdb_sqlite) target_link_libraries(fdbserver PRIVATE fdbclient fdb_sqlite)
endif() endif()

17
fdbserver/KeyValueStoreRocksDB.actor.cpp
View File

@ -11,6 +11,11 @@
#include <rocksdb/version.h> #include <rocksdb/version.h>
#include <rocksdb/utilities/table_properties_collectors.h> #include <rocksdb/utilities/table_properties_collectors.h>
#include <rocksdb/rate_limiter.h> #include <rocksdb/rate_limiter.h>
#if defined __has_include
#if __has_include(<liburing.h>)
#include <liburing.h>
#endif
#endif
#include "fdbclient/SystemData.h" #include "fdbclient/SystemData.h"
#include "fdbserver/CoroFlow.h" #include "fdbserver/CoroFlow.h"
#include "flow/flow.h" #include "flow/flow.h"
@ -29,12 +34,12 @@
#ifdef SSD_ROCKSDB_EXPERIMENTAL #ifdef SSD_ROCKSDB_EXPERIMENTAL
// Enforcing rocksdb version to be 6.22.1 or greater. // Enforcing rocksdb version to be 6.27.3 or greater.
static_assert(ROCKSDB_MAJOR >= 6, "Unsupported rocksdb version. Update the rocksdb to 6.22.1 version"); static_assert(ROCKSDB_MAJOR >= 6, "Unsupported rocksdb version. Update the rocksdb to 6.27.3 version");
static_assert(ROCKSDB_MAJOR == 6 ? ROCKSDB_MINOR >= 22 : true, static_assert(ROCKSDB_MAJOR == 6 ? ROCKSDB_MINOR >= 27 : true,
"Unsupported rocksdb version. Update the rocksdb to 6.22.1 version"); "Unsupported rocksdb version. Update the rocksdb to 6.27.3 version");
static_assert((ROCKSDB_MAJOR == 6 && ROCKSDB_MINOR == 22) ? ROCKSDB_PATCH >= 1 : true, static_assert((ROCKSDB_MAJOR == 6 && ROCKSDB_MINOR == 27) ? ROCKSDB_PATCH >= 3 : true,
"Unsupported rocksdb version. Update the rocksdb to 6.22.1 version"); "Unsupported rocksdb version. Update the rocksdb to 6.27.3 version");
namespace { namespace {
using rocksdb::BackgroundErrorReason; using rocksdb::BackgroundErrorReason;

2
fdbservice/FDBService.cpp
View File

@ -29,7 +29,7 @@
#include <time.h> #include <time.h>
#include "flow/SimpleOpt.h" #include "flow/SimpleOpt.h"
#include "fdbmonitor/SimpleIni.h" #include "fdbclient/SimpleIni.h"
#include "fdbclient/versions.h" #include "fdbclient/versions.h"
// For PathFileExists // For PathFileExists

4
tests/CMakeLists.txt
View File

@ -118,6 +118,7 @@ if(WITH_PYTHON)
add_fdb_test(TEST_FILES randomSelector.txt IGNORE) add_fdb_test(TEST_FILES randomSelector.txt IGNORE)
add_fdb_test(TEST_FILES selectorCorrectness.txt IGNORE) add_fdb_test(TEST_FILES selectorCorrectness.txt IGNORE)
add_fdb_test(TEST_FILES IThreadPool.txt IGNORE) add_fdb_test(TEST_FILES IThreadPool.txt IGNORE)
add_fdb_test(TEST_FILES PerfUnitTests.toml IGNORE)
add_fdb_test(TEST_FILES fast/AtomicBackupCorrectness.toml) add_fdb_test(TEST_FILES fast/AtomicBackupCorrectness.toml)
add_fdb_test(TEST_FILES fast/AtomicBackupToDBCorrectness.toml) add_fdb_test(TEST_FILES fast/AtomicBackupToDBCorrectness.toml)
add_fdb_test(TEST_FILES fast/AtomicOps.toml) add_fdb_test(TEST_FILES fast/AtomicOps.toml)
@ -202,6 +203,9 @@ if(WITH_PYTHON)
add_fdb_test(TEST_FILES rare/TransactionTagApiCorrectness.toml) add_fdb_test(TEST_FILES rare/TransactionTagApiCorrectness.toml)
add_fdb_test(TEST_FILES rare/TransactionTagSwizzledApiCorrectness.toml) add_fdb_test(TEST_FILES rare/TransactionTagSwizzledApiCorrectness.toml)
add_fdb_test(TEST_FILES rare/WriteTagThrottling.toml) add_fdb_test(TEST_FILES rare/WriteTagThrottling.toml)
add_fdb_test(TEST_FILES rare/AllSimUnitTests.toml IGNORE)
add_fdb_test(TEST_FILES rare/StatusBuilderPerf.toml)
add_fdb_test(TEST_FILES rare/TLogVersionMessagesOverheadFactor.toml)
add_fdb_test( add_fdb_test(
TEST_FILES restarting/from_7.0.0/SnapIncrementalRestore-1.txt TEST_FILES restarting/from_7.0.0/SnapIncrementalRestore-1.txt
restarting/from_7.0.0/SnapIncrementalRestore-2.txt) restarting/from_7.0.0/SnapIncrementalRestore-2.txt)

7
tests/fast/ClientLibManagement.toml
View File

@ -1,7 +0,0 @@
[[test]]
testTitle = 'ClientLibManagement'
[[test.workload]]
testName = 'ClientLibManagement'
minTestFileSize = 0
maxTestFileSize = 1048576

7
tests/rare/Downgrade.toml
View File

@ -1,7 +0,0 @@
[[test]]
testTitle = 'Downgrade'
[[test.workload]]
testName = 'Downgrade'
oldKey = 'oldKey'
newKey = 'newKey'