apple/foundationdb
1
0
Fork 0
mirror of https://github.com/apple/foundationdb.git synced 2025-05-14 01:42:37 +08:00
Code Issues Packages Projects Releases Wiki Activity

Merge branch 'main' into feature-metacluster

Browse Source
This commit is contained in:
A.J. Beamon 2022-08-01 14:43:14 -07:00
commit 1b693a588a
13 changed files with 97 additions and 42 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
CMakeLists.txt
View File

@ -206,7 +206,7 @@ endif()
if (CMAKE_EXPORT_COMPILE_COMMANDS AND WITH_PYTHON)
add_custom_command(
OUTPUT ${CMAKE_CURRENT_SOURCE_DIR}/compile_commands.json
COMMAND $<TARGET_FILE:Python::Interpreter> ${CMAKE_CURRENT_SOURCE_DIR}/contrib/gen_compile_db.py
COMMAND $<TARGET_FILE:Python3::Interpreter> ${CMAKE_CURRENT_SOURCE_DIR}/contrib/gen_compile_db.py
ARGS -b ${CMAKE_CURRENT_BINARY_DIR} -s ${CMAKE_CURRENT_SOURCE_DIR} -o ${CMAKE_CURRENT_SOURCE_DIR}/compile_commands.json ${CMAKE_CURRENT_BINARY_DIR}/compile_commands.json
DEPENDS ${CMAKE_CURRENT_SOURCE_DIR}/contrib/gen_compile_db.py ${CMAKE_CURRENT_BINARY_DIR}/compile_commands.json
COMMENT "Build compile commands for IDE"

8
bindings/c/CMakeLists.txt
View File

@ -29,7 +29,7 @@ if(APPLE AND CMAKE_SYSTEM_PROCESSOR STREQUAL "arm64")
endif()
add_custom_command(OUTPUT ${asm_file} ${CMAKE_CURRENT_BINARY_DIR}/fdb_c_function_pointers.g.h
COMMAND $<TARGET_FILE:Python::Interpreter> ${CMAKE_CURRENT_SOURCE_DIR}/generate_asm.py ${os} ${cpu}
COMMAND $<TARGET_FILE:Python3::Interpreter> ${CMAKE_CURRENT_SOURCE_DIR}/generate_asm.py ${os} ${cpu}
${CMAKE_CURRENT_SOURCE_DIR}/fdb_c.cpp
${asm_file}
${CMAKE_CURRENT_BINARY_DIR}/fdb_c_function_pointers.g.h
@ -65,7 +65,7 @@ endif()
if(APPLE)
set(symbols ${CMAKE_CURRENT_BINARY_DIR}/fdb_c.symbols)
add_custom_command(OUTPUT ${symbols}
COMMAND $<TARGET_FILE:Python::Interpreter> ${CMAKE_CURRENT_SOURCE_DIR}/symbolify.py
COMMAND $<TARGET_FILE:Python3::Interpreter> ${CMAKE_CURRENT_SOURCE_DIR}/symbolify.py
${CMAKE_CURRENT_SOURCE_DIR}/foundationdb/fdb_c.h
${CMAKE_CURRENT_SOURCE_DIR}/foundationdb/fdb_c_internal.h
${symbols}
@ -457,7 +457,7 @@ elseif(NOT WIN32 AND NOT APPLE AND NOT USE_UBSAN) # Linux Only, non-ubsan only
)
add_custom_command(OUTPUT ${SHIM_LIB_GEN_SRC}
COMMAND $<TARGET_FILE:Python::Interpreter> ${IMPLIBSO_SRC_DIR}/implib-gen.py
COMMAND $<TARGET_FILE:Python3::Interpreter> ${IMPLIBSO_SRC_DIR}/implib-gen.py
--target ${CMAKE_SYSTEM_PROCESSOR}
--outdir ${SHIM_LIB_OUTPUT_DIR}
--dlopen-callback=fdb_shim_dlopen_callback
@ -484,7 +484,7 @@ elseif(NOT WIN32 AND NOT APPLE AND NOT USE_UBSAN) # Linux Only, non-ubsan only
target_include_directories(fdb_c_shim_lib_tester PUBLIC ${CMAKE_CURRENT_SOURCE_DIR} ${CMAKE_CURRENT_BINARY_DIR}/foundationdb/ ${CMAKE_SOURCE_DIR}/flow/include)
add_test(NAME fdb_c_shim_library_tests
COMMAND $<TARGET_FILE:Python::Interpreter> ${CMAKE_CURRENT_SOURCE_DIR}/test/fdb_c_shim_tests.py
COMMAND $<TARGET_FILE:Python3::Interpreter> ${CMAKE_CURRENT_SOURCE_DIR}/test/fdb_c_shim_tests.py
--build-dir ${CMAKE_BINARY_DIR}
--unit-tests-bin $<TARGET_FILE:fdb_c_shim_unit_tests>
--api-tester-bin $<TARGET_FILE:fdb_c_shim_api_tester>

2
bindings/python/CMakeLists.txt
View File

@ -68,7 +68,7 @@ endif()
set(setup_file_name foundationdb-${FDB_VERSION}.tar.gz)
set(package_file ${CMAKE_BINARY_DIR}/packages/foundationdb-${FDB_VERSION}${not_fdb_release_string}.tar.gz)
add_custom_command(OUTPUT ${package_file}
COMMAND $<TARGET_FILE:Python::Interpreter> setup.py sdist --formats=gztar &&
COMMAND $<TARGET_FILE:Python3::Interpreter> setup.py sdist --formats=gztar &&
${CMAKE_COMMAND} -E copy dist/${setup_file_name} ${package_file}
WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}
COMMENT "Create Python sdist package")

8
cmake/AddFdbTest.cmake
View File

@ -125,7 +125,7 @@ function(add_fdb_test)
list(TRANSFORM ADD_FDB_TEST_TEST_FILES PREPEND "${CMAKE_CURRENT_SOURCE_DIR}/")
if (ENABLE_SIMULATION_TESTS)
add_test(NAME ${test_name}
COMMAND $<TARGET_FILE:Python::Interpreter> ${TestRunner}
COMMAND $<TARGET_FILE:Python3::Interpreter> ${TestRunner}
-n ${test_name}
-b ${PROJECT_BINARY_DIR}
-t ${test_type}
@ -440,7 +440,7 @@ function(add_fdbclient_test)
message(STATUS "Adding Client test ${T_NAME}")
add_test(NAME "${T_NAME}"
WORKING_DIRECTORY ${T_WORKING_DIRECTORY}
COMMAND ${Python_EXECUTABLE} ${TMP_CLUSTER_CMD}
COMMAND ${Python3_EXECUTABLE} ${TMP_CLUSTER_CMD}
--
${T_COMMAND})
if (T_TEST_TIMEOUT)
@ -473,7 +473,7 @@ function(add_unavailable_fdbclient_test)
endif()
message(STATUS "Adding unavailable client test ${T_NAME}")
add_test(NAME "${T_NAME}"
COMMAND ${Python_EXECUTABLE} ${CMAKE_SOURCE_DIR}/tests/TestRunner/fake_cluster.py
COMMAND ${Python3_EXECUTABLE} ${CMAKE_SOURCE_DIR}/tests/TestRunner/fake_cluster.py
--output-dir ${CMAKE_BINARY_DIR}
--
${T_COMMAND})
@ -508,7 +508,7 @@ function(add_multi_fdbclient_test)
endif()
message(STATUS "Adding Client test ${T_NAME}")
add_test(NAME "${T_NAME}"
COMMAND ${Python_EXECUTABLE} ${CMAKE_SOURCE_DIR}/tests/TestRunner/tmp_multi_cluster.py
COMMAND ${Python3_EXECUTABLE} ${CMAKE_SOURCE_DIR}/tests/TestRunner/tmp_multi_cluster.py
--build-dir ${CMAKE_BINARY_DIR}
--clusters 3
--

4
cmake/FDBComponents.cmake
View File

@ -56,8 +56,8 @@ endif()
# Python Bindings
################################################################################
find_package(Python COMPONENTS Interpreter)
if(Python_Interpreter_FOUND)
find_package(Python3 COMPONENTS Interpreter)
if(Python3_Interpreter_FOUND)
set(WITH_PYTHON ON)
else()
message(WARNING "Could not found a suitable python interpreter")

37
fdbclient/NativeAPI.actor.cpp
View File

@ -2992,9 +2992,11 @@ Future<KeyRangeLocationInfo> getKeyLocation(Reference<TransactionState> trState,
isBackward,
version);
if (trState->tenant().present() && useTenant) {
if (trState->tenant().present() && useTenant && trState->tenantId == TenantInfo::INVALID_TENANT) {
return map(f, [trState](const KeyRangeLocationInfo& locationInfo) {
trState->tenantId = locationInfo.tenantEntry.id;
if (trState->tenantId == TenantInfo::INVALID_TENANT) {
trState->tenantId = locationInfo.tenantEntry.id;
}
return locationInfo;
});
} else {
@ -3132,10 +3134,12 @@ Future<std::vector<KeyRangeLocationInfo>> getKeyRangeLocations(Reference<Transac
trState->useProvisionalProxies,
version);
if (trState->tenant().present() && useTenant) {
if (trState->tenant().present() && useTenant && trState->tenantId == TenantInfo::INVALID_TENANT) {
return map(f, [trState](const std::vector<KeyRangeLocationInfo>& locationInfo) {
ASSERT(!locationInfo.empty());
trState->tenantId = locationInfo[0].tenantEntry.id;
if (trState->tenantId == TenantInfo::INVALID_TENANT) {
trState->tenantId = locationInfo[0].tenantEntry.id;
}
return locationInfo;
});
} else {
@ -5974,6 +5978,7 @@ ACTOR static Future<Void> commitDummyTransaction(Reference<TransactionState> trS
tr.trState->options = trState->options;
tr.trState->taskID = trState->taskID;
tr.trState->authToken = trState->authToken;
tr.trState->tenantId = trState->tenantId;
if (!trState->hasTenant()) {
tr.setOption(FDBTransactionOptions::RAW_ACCESS);
} else {
@ -5987,6 +5992,10 @@ ACTOR static Future<Void> commitDummyTransaction(Reference<TransactionState> trS
wait(tr.commit());
return Void();
} catch (Error& e) {
// If the tenant is gone, then our original transaction won't be able to commit
if (e.code() == error_code_unknown_tenant) {
return Void();
}
TraceEvent("CommitDummyTransactionError")
.errorUnsuppressed(e)
.detail("Key", range.begin)
@ -6159,19 +6168,17 @@ ACTOR static Future<Void> tryCommit(Reference<TransactionState> trState,
}
state Key tenantPrefix;
if (trState->tenant().present()) {
// skipApplyTenantPrefix is set only in the context of a commitDummyTransaction()
// (see member declaration)
if (trState->tenant().present() && !trState->skipApplyTenantPrefix) {
KeyRangeLocationInfo locationInfo = wait(getKeyLocation(trState,
""_sr,
&StorageServerInterface::getValue,
Reverse::False,
UseTenant::True,
req.transaction.read_snapshot));
// skipApplyTenantPrefix is set only in the context of a commitDummyTransaction()
// (see member declaration)
if (!trState->skipApplyTenantPrefix) {
applyTenantPrefix(req, locationInfo.tenantEntry.prefix);
tenantPrefixPrepended = TenantPrefixPrepended::True;
}
applyTenantPrefix(req, locationInfo.tenantEntry.prefix);
tenantPrefixPrepended = TenantPrefixPrepended::True;
tenantPrefix = locationInfo.tenantEntry.prefix;
}
CODE_PROBE(trState->skipApplyTenantPrefix, "Tenant prefix prepend skipped for dummy transaction");
@ -7623,10 +7630,14 @@ ACTOR Future<TenantMapEntry> blobGranuleGetTenantEntry(Transaction* self, Key ra
self->trState->useProvisionalProxies,
Reverse::False,
latestVersion));
self->trState->tenantId = l.tenantEntry.id;
if (self->trState->tenantId == TenantInfo::INVALID_TENANT) {
self->trState->tenantId = l.tenantEntry.id;
}
return l.tenantEntry;
} else {
self->trState->tenantId = cachedLocationInfo.get().tenantEntry.id;
if (self->trState->tenantId == TenantInfo::INVALID_TENANT) {
self->trState->tenantId = cachedLocationInfo.get().tenantEntry.id;
}
return cachedLocationInfo.get().tenantEntry;
}
}

34
fdbserver/CommitProxyServer.actor.cpp
View File

@ -283,16 +283,20 @@ bool verifyTenantPrefix(ProxyCommitData* const commitData, const CommitTransacti
if (!m.param1.startsWith(tenantPrefix)) {
TraceEvent(SevWarnAlways, "TenantPrefixMismatch")
.suppressFor(60)
.detail("Prefix", tenantPrefix.toHexString())
.detail("Key", m.param1.toHexString());
.detail("Tenant", req.tenantInfo.name)
.detail("TenantID", req.tenantInfo.tenantId)
.detail("Prefix", tenantPrefix)
.detail("Key", m.param1);
return false;
}
if (m.type == MutationRef::ClearRange && !m.param2.startsWith(tenantPrefix)) {
TraceEvent(SevWarnAlways, "TenantClearRangePrefixMismatch")
.suppressFor(60)
.detail("Prefix", tenantPrefix.toHexString())
.detail("Key", m.param2.toHexString());
.detail("Tenant", req.tenantInfo.name)
.detail("TenantID", req.tenantInfo.tenantId)
.detail("Prefix", tenantPrefix)
.detail("Key", m.param2);
return false;
} else if (m.type == MutationRef::SetVersionstampedKey) {
ASSERT(m.param1.size() >= 4);
@ -301,8 +305,10 @@ bool verifyTenantPrefix(ProxyCommitData* const commitData, const CommitTransacti
if (*offset < tenantPrefix.size()) {
TraceEvent(SevWarnAlways, "TenantVersionstampInvalidOffset")
.suppressFor(60)
.detail("Prefix", tenantPrefix.toHexString())
.detail("Key", m.param1.toHexString())
.detail("Tenant", req.tenantInfo.name)
.detail("TenantID", req.tenantInfo.tenantId)
.detail("Prefix", tenantPrefix)
.detail("Key", m.param1)
.detail("Offset", *offset);
return false;
}
@ -315,9 +321,11 @@ bool verifyTenantPrefix(ProxyCommitData* const commitData, const CommitTransacti
(!rc.begin.startsWith(tenantPrefix) || !rc.end.startsWith(tenantPrefix))) {
TraceEvent(SevWarnAlways, "TenantReadConflictPrefixMismatch")
.suppressFor(60)
.detail("Prefix", tenantPrefix.toHexString())
.detail("BeginKey", rc.begin.toHexString())
.detail("EndKey", rc.end.toHexString());
.detail("Tenant", req.tenantInfo.name)
.detail("TenantID", req.tenantInfo.tenantId)
.detail("Prefix", tenantPrefix)
.detail("BeginKey", rc.begin)
.detail("EndKey", rc.end);
return false;
}
}
@ -327,9 +335,11 @@ bool verifyTenantPrefix(ProxyCommitData* const commitData, const CommitTransacti
(!wc.begin.startsWith(tenantPrefix) || !wc.end.startsWith(tenantPrefix))) {
TraceEvent(SevWarnAlways, "TenantWriteConflictPrefixMismatch")
.suppressFor(60)
.detail("Prefix", tenantPrefix.toHexString())
.detail("BeginKey", wc.begin.toHexString())
.detail("EndKey", wc.end.toHexString());
.detail("Tenant", req.tenantInfo.name)
.detail("TenantID", req.tenantInfo.tenantId)
.detail("Prefix", tenantPrefix)
.detail("BeginKey", wc.begin)
.detail("EndKey", wc.end);
return false;
}
}

4
flow/Hostname.actor.cpp
View File

@ -43,7 +43,7 @@ ACTOR Future<Optional<NetworkAddress>> resolveImpl(Hostname* self) {
try {
std::vector<NetworkAddress> addresses =
wait(INetworkConnections::net()->resolveTCPEndpointWithDNSCache(self->host, self->service));
NetworkAddress address = addresses[deterministicRandom()->randomInt(0, addresses.size())];
NetworkAddress address = INetworkConnections::pickOneAddress(addresses);
address.flags = 0; // Reset the parsed address to public
address.fromHostname = NetworkAddressFromHostname::True;
if (self->isTLS) {
@ -84,7 +84,7 @@ Optional<NetworkAddress> Hostname::resolveBlocking() {
try {
std::vector<NetworkAddress> addresses =
INetworkConnections::net()->resolveTCPEndpointBlockingWithDNSCache(host, service);
NetworkAddress address = addresses[deterministicRandom()->randomInt(0, addresses.size())];
NetworkAddress address = INetworkConnections::pickOneAddress(addresses);
address.flags = 0; // Reset the parsed address to public
address.fromHostname = NetworkAddressFromHostname::True;
if (isTLS) {

16
flow/include/flow/network.h
View File

@ -736,6 +736,22 @@ public:
return static_cast<INetworkConnections*>((void*)g_network->global(INetwork::enNetworkConnections));
}
// If a DNS name can be resolved to both and IPv4 and IPv6 addresses, we want IPv6 addresses when running the
// clusters on IPv6.
// This function takes a vector of addresses and return a random one, preferring IPv6 over IPv4.
static NetworkAddress pickOneAddress(const std::vector<NetworkAddress>& addresses) {
std::vector<NetworkAddress> ipV6Addresses;
for (const NetworkAddress& addr : addresses) {
if (addr.isV6()) {
ipV6Addresses.push_back(addr);
}
}
if (ipV6Addresses.size() > 0) {
return ipV6Addresses[deterministicRandom()->randomInt(0, ipV6Addresses.size())];
}
return addresses[deterministicRandom()->randomInt(0, addresses.size())];
}
void removeCachedDNS(const std::string& host, const std::string& service) { dnsCache.remove(host, service); }
DNSCache dnsCache;

20
flow/network.cpp
View File

@ -279,7 +279,7 @@ Future<Reference<IConnection>> INetworkConnections::connect(const std::string& h
// Use map to create an actor that returns an endpoint or throws
Future<NetworkAddress> pickEndpoint =
map(resolveTCPEndpoint(host, service), [=](std::vector<NetworkAddress> const& addresses) -> NetworkAddress {
NetworkAddress addr = addresses[deterministicRandom()->randomInt(0, addresses.size())];
NetworkAddress addr = INetworkConnections::pickOneAddress(addresses);
addr.fromHostname = true;
if (isTLS) {
addr.flags = NetworkAddress::FLAG_TLS;
@ -347,4 +347,22 @@ TEST_CASE("/flow/network/ipaddress") {
return Void();
}
TEST_CASE("/flow/network/ipV6Preferred") {
std::vector<NetworkAddress> addresses;
for (int i = 0; i < 50; ++i) {
std::string s = fmt::format("{}.{}.{}.{}:{}", i, i, i, i, i);
addresses.push_back(NetworkAddress::parse(s));
}
std::string ipv6 = "[2001:db8:85a3::8a2e:370:7334]:4800";
addresses.push_back(NetworkAddress::parse(ipv6));
for (int i = 50; i < 100; ++i) {
std::string s = fmt::format("{}.{}.{}.{}:{}", i, i, i, i, i);
addresses.push_back(NetworkAddress::parse(s));
}
// Confirm IPv6 is always preferred.
ASSERT(INetworkConnections::pickOneAddress(addresses).toString() == ipv6);
return Void();
}
NetworkInfo::NetworkInfo() : handshakeLock(new FlowLock(FLOW_KNOBS->TLS_HANDSHAKE_LIMIT)) {}

2
tests/CMakeLists.txt
View File

@ -381,7 +381,7 @@ if(WITH_PYTHON)
if(NOT OPEN_FOR_IDE)
add_test(
NAME command_line_argument_test
COMMAND ${Python_EXECUTABLE} ${CMAKE_SOURCE_DIR}/tests/argument_parsing/test_argument_parsing.py ${CMAKE_BINARY_DIR}
COMMAND ${Python3_EXECUTABLE} ${CMAKE_SOURCE_DIR}/tests/argument_parsing/test_argument_parsing.py ${CMAKE_BINARY_DIR}
)
endif()

2
tests/CTestCustom.ctest.cmake
View File

@ -1 +1 @@
set(CTEST_CUSTOM_PRE_TEST ${CTEST_CUSTOM_PRE_TEST} "@Python_EXECUTABLE@ @PROJECT_SOURCE_DIR@/tests/TestRunner/TestDirectory.py @PROJECT_BINARY_DIR@")
set(CTEST_CUSTOM_PRE_TEST ${CTEST_CUSTOM_PRE_TEST} "@Python3_EXECUTABLE@ @PROJECT_SOURCE_DIR@/tests/TestRunner/TestDirectory.py @PROJECT_BINARY_DIR@")

0
tests/TestRunner/tmp_multi_cluster.py Normal file → Executable file
View File