apple/foundationdb
1
0
Fork 0
mirror of https://github.com/apple/foundationdb.git synced 2025-05-14 01:42:37 +08:00
Code Issues Packages Projects Releases Wiki Activity

Undo some changes that aren't needed

Browse Source
This commit is contained in:
A.J. Beamon 2022-07-07 14:48:36 -07:00
parent bda1444051
commit b9cc5389b1
20 changed files with 512 additions and 608 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
bindings/c/fdb_c.cpp
View File

@ -24,7 +24,6 @@
#define FDB_API_VERSION 720
#define FDB_INCLUDE_LEGACY_TYPES
#include "fdbclient/CoordinationInterface.h"
#include "fdbclient/MultiVersionTransaction.h"
#include "fdbclient/MultiVersionAssignmentVars.h"
#include "foundationdb/fdb_c.h"
@ -361,7 +360,8 @@ void fdb_cluster_destroy_v609(FDBCluster* c) {
// If it does and this is an external client loaded though the multi-version API, then it may inadvertently call
// the version of the function in the primary library if it was loaded into the global symbols.
fdb_error_t fdb_create_database_impl(const char* cluster_file_path, FDBDatabase** out_database) {
CATCH_AND_RETURN(*out_database = (FDBDatabase*)API->createDatabase(cluster_file_path).extractPtr(););
CATCH_AND_RETURN(*out_database =
(FDBDatabase*)API->createDatabase(cluster_file_path ? cluster_file_path : "").extractPtr(););
}
FDBFuture* fdb_cluster_create_database_v609(FDBCluster* c, uint8_t const* db_name, int db_name_length) {

77
documentation/sphinx/source/trace-logging.rst
View File

@ -1,77 +0,0 @@
#############
Trace Logging
#############
Overview
========
FoundationDB processes generate log files known as trace logs that include details about the sequence of events that a process performed and record a variety of metrics about that process. These log files are useful for observing the state of a cluster over time and for debugging issues that occur in the cluster, a client, or various tools.
Each file contains a sequence of events for a single process, ordered by time. Clients that use the multi-version or multi-threaded client features will generate :ref:`multiple log files simultaneously <mvc-logging>`. Each entry in the log file will be either an XML event or a JSON object that containts several mandatory fields and zero or more arbitrary fields. For example, the following is an event generated in the XML format::
<Event Severity="10" Time="1579736072.656689" Type="Net2Starting" ID="0000000000000000" Machine="1.1.1.1:4000" LogGroup="default"/>
Most FoundationDB processes generate trace logs in the format described in this document. This includes server and client processes, fdbcli, and backup and DR tooling.
Configuration
=============
File Format
-----------
FoundationDB trace logging
* Parameters
* Format
Trace Files
===========
* Filenames
* Rolling
Mandatory Fields
================
* Severity
* Time
* Type
* Machine
* LogGroup
* ID
Common Fields
=============
* Roles
Event Suppression
-----------------
* SuppressedEventCount
Errors
------
* Error
* ErrorDescription
* ErrorCode
Clients
-------
*
Rolled Events
=============
Counters
========
PeriodicEvents
==============
.. mvc-logging::
Multi-Version and Multi-Threaded Client Logging
===============================================

2
fdbcli/fdbcli.actor.cpp
View File

@ -1095,7 +1095,7 @@ ACTOR Future<int> cli(CLIOptions opt, LineNoise* plinenoise) {
if (!opt.exec.present()) {
printf("Using cluster file `%s'.\n", ccf->getLocation().c_str());
}
db = API->createDatabase(resolvedClusterFile.first.c_str());
db = API->createDatabase(opt.clusterFile.c_str());
} catch (Error& e) {
fprintf(stderr, "ERROR: %s (%d)\n", e.what(), e.code());
printf("Unable to connect to cluster from `%s'\n", ccf->getLocation().c_str());

7
fdbclient/CMakeLists.txt
View File

@ -23,13 +23,12 @@ vexillographer_compile(TARGET fdboptions_c LANG c OUT ${CMAKE_CURRENT_BINARY_DIR
add_custom_target(fdboptions DEPENDS ${CMAKE_CURRENT_BINARY_DIR}/include/fdbclient/FDBOptions.g.h)
add_dependencies(fdboptions fdboptions_c)
# ###############################################################################
################################################################################
# Build information
# ###############################################################################
################################################################################
configure_file(${CMAKE_CURRENT_SOURCE_DIR}/BuildFlags.h.in ${CMAKE_CURRENT_BINARY_DIR}/include/fdbclient/BuildFlags.h)
set(BUILD_AZURE_BACKUP OFF CACHE BOOL "Build Azure backup client")
if(BUILD_AZURE_BACKUP)
add_compile_definitions(BUILD_AZURE_BACKUP)
set(FDBCLIENT_SRCS
@ -64,6 +63,7 @@ if(BUILD_AZURE_BACKUP)
)
endif()
if(WITH_AWS_BACKUP)
add_compile_definitions(BUILD_AWS_BACKUP)
include(awssdk)
@ -83,7 +83,6 @@ target_include_directories(fdbclient_sampling PUBLIC "${CMAKE_CURRENT_SOURCE_DIR
add_dependencies(fdbclient_sampling fdboptions)
target_link_libraries(fdbclient_sampling PUBLIC fdbrpc_sampling msgpack PRIVATE rapidxml)
target_compile_definitions(fdbclient_sampling PRIVATE -DENABLE_SAMPLING)
if(WIN32)
add_dependencies(fdbclient_sampling_actors fdbclient_actors)
endif()

10
fdbclient/ClusterConnectionFile.actor.cpp
View File

@ -103,16 +103,6 @@ std::string ClusterConnectionFile::toString() const {
return "file://" + filename;
}
// Return the specified path of the cluster file
Optional<std::string> ClusterConnectionFile::getFilename() const {
return filename;
}
// Returns true because cluster files are supported through the C API
bool ClusterConnectionFile::supportedExternally() const {
return true;
}
// returns <resolved name, was default file>
std::pair<std::string, bool> ClusterConnectionFile::lookupClusterFileName(std::string const& filename) {
if (filename.length())

167
fdbclient/ClusterConnectionKey.actor.cpp Normal file
View File

@ -0,0 +1,167 @@
/*
* ClusterConnectionKey.actor.cpp
*
* This source file is part of the FoundationDB open source project
*
* Copyright 2013-2022 Apple Inc. and the FoundationDB project authors
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
#include "fdbclient/ClusterConnectionKey.actor.h"
#include "flow/actorcompiler.h" // has to be last include
// Creates a cluster connection record with a given connection string and saves it to the specified key. Needs to be
// persisted should be set to true unless this ClusterConnectionKey is being created with the value read from the
// key.
ClusterConnectionKey::ClusterConnectionKey(Database db,
Key connectionStringKey,
ClusterConnectionString const& contents,
ConnectionStringNeedsPersisted needsToBePersisted)
: IClusterConnectionRecord(needsToBePersisted), db(db), connectionStringKey(connectionStringKey) {
if (!needsToBePersisted) {
lastPersistedConnectionString = ValueRef(contents.toString());
}
cs = contents;
}
// Loads and parses the connection string at the specified key, throwing errors if the file cannot be read or the
// format is invalid.
ACTOR Future<Reference<ClusterConnectionKey>> ClusterConnectionKey::loadClusterConnectionKey(Database db,
Key connectionStringKey) {
state Transaction tr(db);
loop {
try {
Optional<Value> v = wait(tr.get(connectionStringKey));
if (!v.present()) {
throw connection_string_invalid();
}
return makeReference<ClusterConnectionKey>(db,
connectionStringKey,
ClusterConnectionString(v.get().toString()),
ConnectionStringNeedsPersisted::False);
} catch (Error& e) {
wait(tr.onError(e));
}
}
}
// Sets the connections string held by this object and persists it.
Future<Void> ClusterConnectionKey::setAndPersistConnectionString(ClusterConnectionString const& connectionString) {
cs = connectionString;
return success(persist());
}
// Get the connection string stored in the database.
ACTOR Future<ClusterConnectionString> ClusterConnectionKey::getStoredConnectionStringImpl(
Reference<ClusterConnectionKey> self) {
Reference<ClusterConnectionKey> cck =
wait(ClusterConnectionKey::loadClusterConnectionKey(self->db, self->connectionStringKey));
return cck->cs;
}
Future<ClusterConnectionString> ClusterConnectionKey::getStoredConnectionString() {
return getStoredConnectionStringImpl(Reference<ClusterConnectionKey>::addRef(this));
}
ACTOR Future<bool> ClusterConnectionKey::upToDateImpl(Reference<ClusterConnectionKey> self,
ClusterConnectionString* connectionString) {
try {
// the cluster file hasn't been created yet so there's nothing to check
if (self->needsToBePersisted())
return true;
Reference<ClusterConnectionKey> temp =
wait(ClusterConnectionKey::loadClusterConnectionKey(self->db, self->connectionStringKey));
*connectionString = temp->getConnectionString();
return connectionString->toString() == self->cs.toString();
} catch (Error& e) {
TraceEvent(SevWarnAlways, "ClusterKeyError").error(e).detail("Key", self->connectionStringKey);
return false; // Swallow the error and report that the file is out of date
}
}
// Checks whether the connection string in the database matches the connection string stored in memory. The cluster
// string stored in the database is returned via the reference parameter connectionString.
Future<bool> ClusterConnectionKey::upToDate(ClusterConnectionString& connectionString) {
return upToDateImpl(Reference<ClusterConnectionKey>::addRef(this), &connectionString);
}
// Returns the key where the connection string is stored.
std::string ClusterConnectionKey::getLocation() const {
return printable(connectionStringKey);
}
// Creates a copy of this object with a modified connection string but that isn't persisted.
Reference<IClusterConnectionRecord> ClusterConnectionKey::makeIntermediateRecord(
ClusterConnectionString const& connectionString) const {
return makeReference<ClusterConnectionKey>(db, connectionStringKey, connectionString);
}
// Returns a string representation of this cluster connection record. This will include the type of record and the
// key where the record is stored.
std::string ClusterConnectionKey::toString() const {
return "fdbkey://" + printable(connectionStringKey);
}
ACTOR Future<bool> ClusterConnectionKey::persistImpl(Reference<ClusterConnectionKey> self) {
self->setPersisted();
state Value newConnectionString = ValueRef(self->cs.toString());
try {
state Transaction tr(self->db);
loop {
try {
Optional<Value> existingConnectionString = wait(tr.get(self->connectionStringKey));
// Someone has already updated the connection string to what we want
if (existingConnectionString.present() && existingConnectionString.get() == newConnectionString) {
self->lastPersistedConnectionString = newConnectionString;
return true;
}
// Someone has updated the connection string to something we didn't expect, in which case we leave it
// alone. It's possible this could result in the stored string getting stuck if the connection string
// changes twice and only the first change is recorded. If the process that wrote the first change dies
// and no other process attempts to write the intermediate state, then only a newly opened connection
// key would be able to update the state.
else if (existingConnectionString.present() &&
existingConnectionString != self->lastPersistedConnectionString) {
TraceEvent(SevWarnAlways, "UnableToChangeConnectionKeyDueToMismatch")
.detail("ConnectionKey", self->connectionStringKey)
.detail("NewConnectionString", newConnectionString)
.detail("ExpectedStoredConnectionString", self->lastPersistedConnectionString)
.detail("ActualStoredConnectionString", existingConnectionString);
return false;
}
tr.set(self->connectionStringKey, newConnectionString);
wait(tr.commit());
self->lastPersistedConnectionString = newConnectionString;
return true;
} catch (Error& e) {
wait(tr.onError(e));
}
}
} catch (Error& e) {
TraceEvent(SevWarnAlways, "UnableToChangeConnectionKey")
.error(e)
.detail("ConnectionKey", self->connectionStringKey)
.detail("ConnectionString", self->cs.toString());
}
return false;
};
// Writes the connection string to the database
Future<bool> ClusterConnectionKey::persist() {
return persistImpl(Reference<ClusterConnectionKey>::addRef(this));
}

5
fdbclient/ClusterConnectionMemoryRecord.actor.cpp
View File

@ -57,11 +57,6 @@ std::string ClusterConnectionMemoryRecord::toString() const {
return "memory://" + id.toString();
}
// Returns true because memory connection records are supported through the C API
bool ClusterConnectionMemoryRecord::supportedExternally() const {
return true;
}
// This is a no-op for memory records. Returns true to indicate success.
Future<bool> ClusterConnectionMemoryRecord::persist() {
return true;

1
fdbclient/SpecialKeySpace.actor.cpp
View File

@ -36,7 +36,6 @@
#include "flow/Arena.h"
#include "flow/UnitTest.h"
#include "fdbclient/ManagementAPI.actor.h"
#include "fdbclient/MetaclusterManagement.actor.h"
#include "fdbclient/StatusClient.h"
#include "flow/actorcompiler.h" // This must be the last #include.

14
fdbclient/include/fdbclient/ClusterConnectionFile.h
View File

@ -25,9 +25,7 @@
#include "fdbclient/CoordinationInterface.h"
// An implementation of IClusterConnectionRecord backed by a file.
class ClusterConnectionFile : public IClusterConnectionRecord,
ThreadSafeReferenceCounted<ClusterConnectionFile>,
NonCopyable {
class ClusterConnectionFile : public IClusterConnectionRecord, ReferenceCounted<ClusterConnectionFile>, NonCopyable {
public:
// Loads and parses the file at 'filename', throwing errors if the file cannot be read or the format is invalid.
explicit ClusterConnectionFile(std::string const& filename);
@ -61,14 +59,8 @@ public:
// filename of the cluster file.
std::string toString() const override;
// Return the specified path of the cluster file
virtual Optional<std::string> getFilename() const override;
// Returns true because cluster files are supported through the C API
bool supportedExternally() const override;
void addref() override { ThreadSafeReferenceCounted<ClusterConnectionFile>::addref(); }
void delref() override { ThreadSafeReferenceCounted<ClusterConnectionFile>::delref(); }
void addref() override { ReferenceCounted<ClusterConnectionFile>::addref(); }
void delref() override { ReferenceCounted<ClusterConnectionFile>::delref(); }
// returns <resolved name, was default file>
static std::pair<std::string, bool> lookupClusterFileName(std::string const& filename);

159
fdbclient/include/fdbclient/ClusterConnectionKey.actor.h
View File

@ -17,9 +17,7 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
#pragma once
// When actually compiled (NO_INTELLISENSE), include the generated version of this file. In intellisense use the source
// version.
#if defined(NO_INTELLISENSE) && !defined(FDBCLIENT_CLUSTERCONNECTIONKEY_ACTOR_G_H)
@ -29,170 +27,63 @@
#define FDBCLIENT_CLUSTERCONNECTIONKEY_ACTOR_H
#include "fdbclient/CoordinationInterface.h"
#include "fdbclient/GenericTransactionHelper.h"
#include "fdbclient/NativeAPI.actor.h"
#include "flow/actorcompiler.h" // has to be last include
// An implementation of IClusterConnectionRecord backed by a key in a FoundationDB database.
template <class DB>
class ClusterConnectionKey : public IClusterConnectionRecord,
ThreadSafeReferenceCounted<ClusterConnectionKey<DB>>,
NonCopyable {
class ClusterConnectionKey : public IClusterConnectionRecord, ReferenceCounted<ClusterConnectionKey>, NonCopyable {
public:
// Creates a cluster connection record with a given connection string and saves it to the specified key. Needs to be
// persisted should be set to true unless this ClusterConnectionKey is being created with the value read from the
// key.
ClusterConnectionKey(DB db,
ClusterConnectionKey(Database db,
Key connectionStringKey,
ClusterConnectionString const& contents,
ConnectionStringNeedsPersisted needsToBePersisted = ConnectionStringNeedsPersisted::True)
: IClusterConnectionRecord(needsToBePersisted), db(db), connectionStringKey(connectionStringKey) {
if (!needsToBePersisted) {
lastPersistedConnectionString = ValueRef(contents.toString());
}
cs = contents;
}
ConnectionStringNeedsPersisted needsToBePersisted = ConnectionStringNeedsPersisted::True);
// Loads and parses the connection string at the specified key, throwing errors if the file cannot be read or
// the format is invalid.
ACTOR static Future<Reference<ClusterConnectionKey>> loadClusterConnectionKey(DB db, Key connectionStringKey) {
state Reference<typename DB::TransactionT> tr = db->createTransaction();
loop {
try {
typename transaction_future_type<Transaction, Optional<Value>>::type f = tr->get(connectionStringKey);
Optional<Value> v = wait(safeThreadFutureToFuture(f));
if (!v.present()) {
throw connection_string_invalid();
}
return makeReference<ClusterConnectionKey>(db,
connectionStringKey,
ClusterConnectionString(v.get().toString()),
ConnectionStringNeedsPersisted::False);
} catch (Error& e) {
wait(safeThreadFutureToFuture(tr->onError(e)));
}
}
}
// Loads and parses the connection string at the specified key, throwing errors if the file cannot be read or the
// format is invalid.
ACTOR static Future<Reference<ClusterConnectionKey>> loadClusterConnectionKey(Database db, Key connectionStringKey);
// Sets the connections string held by this object and persists it.
Future<Void> setAndPersistConnectionString(ClusterConnectionString const& connectionString) override {
cs = connectionString;
return success(persist());
}
Future<Void> setAndPersistConnectionString(ClusterConnectionString const&) override;
// Get the connection string stored in the database.
Future<ClusterConnectionString> getStoredConnectionString() override {
return getStoredConnectionStringImpl(Reference<ClusterConnectionKey>::addRef(this));
}
Future<ClusterConnectionString> getStoredConnectionString() override;
// Checks whether the connection string in the database matches the connection string stored in memory. The
// cluster string stored in the database is returned via the reference parameter connectionString.
Future<bool> upToDate(ClusterConnectionString& connectionString) override {
return upToDateImpl(Reference<ClusterConnectionKey>::addRef(this), &connectionString);
}
// Checks whether the connection string in the database matches the connection string stored in memory. The cluster
// string stored in the database is returned via the reference parameter connectionString.
Future<bool> upToDate(ClusterConnectionString& connectionString) override;
// Returns the key where the connection string is stored.
std::string getLocation() const override { return printable(connectionStringKey); }
std::string getLocation() const override;
// Creates a copy of this object with a modified connection string but that isn't persisted.
Reference<IClusterConnectionRecord> makeIntermediateRecord(
ClusterConnectionString const& connectionString) const override {
return makeReference<ClusterConnectionKey>(db, connectionStringKey, connectionString);
}
ClusterConnectionString const& connectionString) const override;
// Returns a string representation of this cluster connection record. This will include the type of record and
// the key where the record is stored.
std::string toString() const override { return "fdbkey://" + printable(connectionStringKey); }
// Returns a string representation of this cluster connection record. This will include the type of record and the
// key where the record is stored.
std::string toString() const override;
// Returns false because cluster connection keys are not supported through the C API
bool supportedExternally() const { return false; }
void addref() override { ThreadSafeReferenceCounted<ClusterConnectionKey>::addref(); }
void delref() override { ThreadSafeReferenceCounted<ClusterConnectionKey>::delref(); }
void addref() override { ReferenceCounted<ClusterConnectionKey>::addref(); }
void delref() override { ReferenceCounted<ClusterConnectionKey>::delref(); }
protected:
// Writes the connection string to the database
Future<bool> persist() override { return persistImpl(Reference<ClusterConnectionKey>::addRef(this)); }
Future<bool> persist() override;
private:
ACTOR static Future<ClusterConnectionString> getStoredConnectionStringImpl(Reference<ClusterConnectionKey> self) {
Reference<ClusterConnectionKey> cck =
wait(ClusterConnectionKey::loadClusterConnectionKey(self->db, self->connectionStringKey));
return cck->cs;
}
ACTOR static Future<ClusterConnectionString> getStoredConnectionStringImpl(Reference<ClusterConnectionKey> self);
ACTOR static Future<bool> upToDateImpl(Reference<ClusterConnectionKey> self,
ClusterConnectionString* connectionString) {
try {
// the cluster file hasn't been created yet so there's nothing to check
if (self->needsToBePersisted())
return true;
ClusterConnectionString* connectionString);
ACTOR static Future<bool> persistImpl(Reference<ClusterConnectionKey> self);
Reference<ClusterConnectionKey> temp =
wait(ClusterConnectionKey::loadClusterConnectionKey(self->db, self->connectionStringKey));
*connectionString = temp->getConnectionString();
return connectionString->toString() == self->cs.toString();
} catch (Error& e) {
TraceEvent(SevWarnAlways, "ClusterKeyError").error(e).detail("Key", self->connectionStringKey);
return false; // Swallow the error and report that the file is out of date
}
}
ACTOR static Future<bool> persistImpl(Reference<ClusterConnectionKey> self) {
self->setPersisted();
state Value newConnectionString = ValueRef(self->cs.toString());
try {
state Reference<typename DB::TransactionT> tr = self->db->createTransaction();
loop {
try {
typename transaction_future_type<Transaction, Optional<Value>>::type f =
tr->get(self->connectionStringKey);
Optional<Value> existingConnectionString = wait(safeThreadFutureToFuture(f));
// Someone has already updated the connection string to what we want
if (existingConnectionString.present() && existingConnectionString.get() == newConnectionString) {
self->lastPersistedConnectionString = newConnectionString;
return true;
}
// Someone has updated the connection string to something we didn't expect, in which case we leave
// it alone. It's possible this could result in the stored string getting stuck if the connection
// string changes twice and only the first change is recorded. If the process that wrote the first
// change dies and no other process attempts to write the intermediate state, then only a newly
// opened connection key would be able to update the state.
else if (existingConnectionString.present() &&
existingConnectionString != self->lastPersistedConnectionString) {
TraceEvent(SevWarnAlways, "UnableToChangeConnectionKeyDueToMismatch")
.detail("ConnectionKey", self->connectionStringKey)
.detail("NewConnectionString", newConnectionString)
.detail("ExpectedStoredConnectionString", self->lastPersistedConnectionString)
.detail("ActualStoredConnectionString", existingConnectionString);
return false;
}
tr->set(self->connectionStringKey, newConnectionString);
wait(safeThreadFutureToFuture(tr->commit()));
self->lastPersistedConnectionString = newConnectionString;
return true;
} catch (Error& e) {
wait(safeThreadFutureToFuture(tr->onError(e)));
}
}
} catch (Error& e) {
TraceEvent(SevWarnAlways, "UnableToChangeConnectionKey")
.error(e)
.detail("ConnectionKey", self->connectionStringKey)
.detail("ConnectionString", self->cs.toString());
}
return false;
}
// The database where the connection key is stored. Note that this does not need to be the same database as the
// one that the connection string would connect to.
DB db;
// The database where the connection key is stored. Note that this does not need to be the same database as the one
// that the connection string would connect to.
Database db;
Key connectionStringKey;
Optional<Value> lastPersistedConnectionString;
};
#include "flow/unactorcompiler.h"
#endif

9
fdbclient/include/fdbclient/ClusterConnectionMemoryRecord.h
View File

@ -26,7 +26,7 @@
// An implementation of IClusterConnectionRecord that is stored in memory only and not persisted.
class ClusterConnectionMemoryRecord : public IClusterConnectionRecord,
ThreadSafeReferenceCounted<ClusterConnectionMemoryRecord>,
ReferenceCounted<ClusterConnectionMemoryRecord>,
NonCopyable {
public:
// Creates a cluster file with a given connection string.
@ -56,11 +56,8 @@ public:
// record.
std::string toString() const override;
// Returns true because memory connection records are supported through the C API
bool supportedExternally() const override;
void addref() override { ThreadSafeReferenceCounted<ClusterConnectionMemoryRecord>::addref(); }
void delref() override { ThreadSafeReferenceCounted<ClusterConnectionMemoryRecord>::delref(); }
void addref() override { ReferenceCounted<ClusterConnectionMemoryRecord>::addref(); }
void delref() override { ReferenceCounted<ClusterConnectionMemoryRecord>::delref(); }
protected:
// This is a no-op for memory records. Returns true to indicate success.

7
fdbclient/include/fdbclient/CoordinationInterface.h
View File

@ -149,13 +149,6 @@ public:
// Signals to the connection record that it was successfully used to connect to a cluster.
void notifyConnected();
// If the connection string is backed by a file, returns the filename. Otherwise, returns an empty optional.
// The interface provides a default implementation that assumes no file-backing.
virtual Optional<std::string> getFilename() const { return Optional<std::string>(); }
// Returns true if this type of cluster connection record is exposed through the C API
virtual bool supportedExternally() const = 0;
virtual void addref() = 0;
virtual void delref() = 0;

639
fdbclient/include/fdbclient/GenericManagementAPI.actor.h
View File

@ -36,13 +36,11 @@ the contents of the system key space.
#include <map>
#include "fdbclient/ClientBooleanParams.h"
#include "fdbclient/DatabaseConfiguration.h"
#include "fdbclient/GenericTransactionHelper.h"
#include "fdbclient/Status.h"
#include "fdbclient/Subspace.h"
#include "fdbclient/DatabaseConfiguration.h"
#include "fdbclient/Status.h"
#include "fdbclient/SystemData.h"
#include "fdbclient/TenantManagement.actor.h"
#include "flow/actorcompiler.h" // has to be last include
// ConfigurationResult enumerates normal outcomes of changeConfig() and various error
@ -206,11 +204,10 @@ Future<Void> removeCachedRange(Reference<DB> db, KeyRangeRef range) {
return changeCachedRange(db, range, false);
}
ACTOR template <class Transaction>
Future<std::vector<ProcessData>> getWorkers(
Transaction tr,
typename transaction_future_type<Transaction, RangeResult>::type processClassesF,
typename transaction_future_type<Transaction, RangeResult>::type processDataF) {
ACTOR template <class Tr>
Future<std::vector<ProcessData>> getWorkers(Reference<Tr> tr,
typename Tr::template FutureT<RangeResult> processClassesF,
typename Tr::template FutureT<RangeResult> processDataF) {
// processClassesF and processDataF are used to hold standalone memory
processClassesF = tr->getRange(processClassKeys, CLIENT_KNOBS->TOO_MANY);
processDataF = tr->getRange(workerListKeys, CLIENT_KNOBS->TOO_MANY);
@ -248,31 +245,31 @@ Future<std::vector<ProcessData>> getWorkers(
// ConfigurationResult (or error).
// Accepts a full configuration in key/value format (from buildConfiguration)
ACTOR template <class Transaction>
Future<ConfigurationResult> changeConfigTransaction(Transaction tr,
std::map<std::string, std::string> m,
bool force,
Optional<UID> createId) {
state StringRef initIdKey = "\xff/init_id"_sr;
ACTOR template <class DB>
Future<ConfigurationResult> changeConfig(Reference<DB> db, std::map<std::string, std::string> m, bool force) {
state StringRef initIdKey = LiteralStringRef("\xff/init_id");
state Reference<typename DB::TransactionT> tr = db->createTransaction();
if (!m.size()) {
return ConfigurationResult::NO_OPTIONS_PROVIDED;
}
// make sure we have essential configuration options
std::string initKey = configKeysPrefix.toString() + "initialized";
state bool creating = m.count(initKey) != 0;
state Optional<UID> locked;
{
auto iter = m.find(databaseLockedKey.toString());
if (iter != m.end()) {
if (!createId.present()) {
if (!creating) {
return ConfigurationResult::LOCKED_NOT_NEW;
}
locked = UID::fromString(iter->second);
m.erase(iter);
}
}
if (createId.present()) {
m[initIdKey.toString()] = createId.get().toString();
if (creating) {
m[initIdKey.toString()] = deterministicRandom()->randomUniqueID().toString();
if (!isCompleteConfiguration(m)) {
return ConfigurationResult::INCOMPLETE_CONFIGURATION;
}
@ -286,267 +283,242 @@ Future<ConfigurationResult> changeConfigTransaction(Transaction tr,
state bool warnRocksDBIsExperimental = false;
state bool warnShardedRocksDBIsExperimental = false;
tr->setOption(FDBTransactionOptions::ACCESS_SYSTEM_KEYS);
tr->setOption(FDBTransactionOptions::PRIORITY_SYSTEM_IMMEDIATE);
tr->setOption(FDBTransactionOptions::LOCK_AWARE);
tr->setOption(FDBTransactionOptions::USE_PROVISIONAL_PROXIES);
loop {
try {
tr->setOption(FDBTransactionOptions::ACCESS_SYSTEM_KEYS);
tr->setOption(FDBTransactionOptions::PRIORITY_SYSTEM_IMMEDIATE);
tr->setOption(FDBTransactionOptions::LOCK_AWARE);
tr->setOption(FDBTransactionOptions::USE_PROVISIONAL_PROXIES);
if (!createId.present() && !force) {
state typename transaction_future_type<Transaction, RangeResult>::type fConfigF =
tr->getRange(configKeys, CLIENT_KNOBS->TOO_MANY);
state Future<RangeResult> fConfig = safeThreadFutureToFuture(fConfigF);
state typename transaction_future_type<Transaction, RangeResult>::type processClassesF;
state typename transaction_future_type<Transaction, RangeResult>::type processDataF;
state Future<std::vector<ProcessData>> fWorkers = getWorkers(tr, processClassesF, processDataF);
wait(success(fConfig) || tooLong);
if (!creating && !force) {
state typename DB::TransactionT::template FutureT<RangeResult> fConfigF =
tr->getRange(configKeys, CLIENT_KNOBS->TOO_MANY);
state Future<RangeResult> fConfig = safeThreadFutureToFuture(fConfigF);
state typename DB::TransactionT::template FutureT<RangeResult> processClassesF;
state typename DB::TransactionT::template FutureT<RangeResult> processDataF;
state Future<std::vector<ProcessData>> fWorkers = getWorkers(tr, processClassesF, processDataF);
wait(success(fConfig) || tooLong);
if (!fConfig.isReady()) {
return ConfigurationResult::DATABASE_UNAVAILABLE;
}
if (fConfig.isReady()) {
ASSERT(fConfig.get().size() < CLIENT_KNOBS->TOO_MANY);
state DatabaseConfiguration oldConfig;
oldConfig.fromKeyValues((VectorRef<KeyValueRef>)fConfig.get());
state DatabaseConfiguration newConfig = oldConfig;
for (auto kv : m) {
newConfig.set(kv.first, kv.second);
}
if (!newConfig.isValid()) {
return ConfigurationResult::INVALID_CONFIGURATION;
}
if (newConfig.tLogPolicy->attributeKeys().count("dcid") && newConfig.regions.size() > 0) {
return ConfigurationResult::REGION_REPLICATION_MISMATCH;
}
oldReplicationUsesDcId = oldReplicationUsesDcId || oldConfig.tLogPolicy->attributeKeys().count("dcid");
if (oldConfig.usableRegions != newConfig.usableRegions) {
// cannot change region configuration
std::map<Key, int32_t> dcId_priority;
for (auto& it : newConfig.regions) {
dcId_priority[it.dcId] = it.priority;
}
for (auto& it : oldConfig.regions) {
if (!dcId_priority.count(it.dcId) || dcId_priority[it.dcId] != it.priority) {
return ConfigurationResult::REGIONS_CHANGED;
}
}
// must only have one region with priority >= 0
int activeRegionCount = 0;
for (auto& it : newConfig.regions) {
if (it.priority >= 0) {
activeRegionCount++;
}
}
if (activeRegionCount > 1) {
return ConfigurationResult::MULTIPLE_ACTIVE_REGIONS;
}
}
state typename transaction_future_type<Transaction, RangeResult>::type fServerListF =
tr->getRange(serverListKeys, CLIENT_KNOBS->TOO_MANY);
state Future<RangeResult> fServerList =
(newConfig.regions.size()) ? safeThreadFutureToFuture(fServerListF) : Future<RangeResult>();
if (newConfig.usableRegions == 2) {
if (oldReplicationUsesDcId) {
state typename transaction_future_type<Transaction, RangeResult>::type fLocalityListF =
tr->getRange(tagLocalityListKeys, CLIENT_KNOBS->TOO_MANY);
state Future<RangeResult> fLocalityList = safeThreadFutureToFuture(fLocalityListF);
wait(success(fLocalityList) || tooLong);
if (!fLocalityList.isReady()) {
return ConfigurationResult::DATABASE_UNAVAILABLE;
}
RangeResult localityList = fLocalityList.get();
ASSERT(!localityList.more && localityList.size() < CLIENT_KNOBS->TOO_MANY);
std::set<Key> localityDcIds;
for (auto& s : localityList) {
auto dc = decodeTagLocalityListKey(s.key);
if (dc.present()) {
localityDcIds.insert(dc.get());
}
}
for (auto& it : newConfig.regions) {
if (localityDcIds.count(it.dcId) == 0) {
return ConfigurationResult::DCID_MISSING;
}
}
} else {
// all regions with priority >= 0 must be fully replicated
state std::vector<typename transaction_future_type<Transaction, Optional<Value>>::type>
replicasFuturesF;
state std::vector<Future<Optional<Value>>> replicasFutures;
for (auto& it : newConfig.regions) {
if (it.priority >= 0) {
replicasFuturesF.push_back(tr->get(datacenterReplicasKeyFor(it.dcId)));
replicasFutures.push_back(safeThreadFutureToFuture(replicasFuturesF.back()));
}
}
wait(waitForAll(replicasFutures) || tooLong);
for (auto& it : replicasFutures) {
if (!it.isReady()) {
return ConfigurationResult::DATABASE_UNAVAILABLE;
}
if (!it.get().present()) {
return ConfigurationResult::REGION_NOT_FULLY_REPLICATED;
}
}
}
}
if (newConfig.regions.size()) {
// all storage servers must be in one of the regions
wait(success(fServerList) || tooLong);
if (!fServerList.isReady()) {
if (!fConfig.isReady()) {
return ConfigurationResult::DATABASE_UNAVAILABLE;
}
RangeResult serverList = fServerList.get();
ASSERT(!serverList.more && serverList.size() < CLIENT_KNOBS->TOO_MANY);
std::set<Key> newDcIds;
for (auto& it : newConfig.regions) {
newDcIds.insert(it.dcId);
}
std::set<Optional<Key>> missingDcIds;
for (auto& s : serverList) {
auto ssi = decodeServerListValue(s.value);
if (!ssi.locality.dcId().present() || !newDcIds.count(ssi.locality.dcId().get())) {
missingDcIds.insert(ssi.locality.dcId());
if (fConfig.isReady()) {
ASSERT(fConfig.get().size() < CLIENT_KNOBS->TOO_MANY);
state DatabaseConfiguration oldConfig;
oldConfig.fromKeyValues((VectorRef<KeyValueRef>)fConfig.get());
state DatabaseConfiguration newConfig = oldConfig;
for (auto kv : m) {
newConfig.set(kv.first, kv.second);
}
if (!newConfig.isValid()) {
return ConfigurationResult::INVALID_CONFIGURATION;
}
}
if (missingDcIds.size() > (oldReplicationUsesDcId ? 1 : 0)) {
return ConfigurationResult::STORAGE_IN_UNKNOWN_DCID;
}
}
wait(success(fWorkers) || tooLong);
if (!fWorkers.isReady()) {
return ConfigurationResult::DATABASE_UNAVAILABLE;
}
if (newConfig.tLogPolicy->attributeKeys().count("dcid") && newConfig.regions.size() > 0) {
return ConfigurationResult::REGION_REPLICATION_MISMATCH;
}
if (newConfig.regions.size()) {
std::map<Optional<Key>, std::set<Optional<Key>>> dcId_zoneIds;
for (auto& it : fWorkers.get()) {
if (it.processClass.machineClassFitness(ProcessClass::Storage) <= ProcessClass::WorstFit) {
dcId_zoneIds[it.locality.dcId()].insert(it.locality.zoneId());
}
}
for (auto& region : newConfig.regions) {
if (dcId_zoneIds[region.dcId].size() <
std::max(newConfig.storageTeamSize, newConfig.tLogReplicationFactor)) {
return ConfigurationResult::NOT_ENOUGH_WORKERS;
}
if (region.satelliteTLogReplicationFactor > 0 && region.priority >= 0) {
int totalSatelliteProcesses = 0;
for (auto& sat : region.satellites) {
totalSatelliteProcesses += dcId_zoneIds[sat.dcId].size();
oldReplicationUsesDcId =
oldReplicationUsesDcId || oldConfig.tLogPolicy->attributeKeys().count("dcid");
if (oldConfig.usableRegions != newConfig.usableRegions) {
// cannot change region configuration
std::map<Key, int32_t> dcId_priority;
for (auto& it : newConfig.regions) {
dcId_priority[it.dcId] = it.priority;
}
if (totalSatelliteProcesses < region.satelliteTLogReplicationFactor) {
for (auto& it : oldConfig.regions) {
if (!dcId_priority.count(it.dcId) || dcId_priority[it.dcId] != it.priority) {
return ConfigurationResult::REGIONS_CHANGED;
}
}
// must only have one region with priority >= 0
int activeRegionCount = 0;
for (auto& it : newConfig.regions) {
if (it.priority >= 0) {
activeRegionCount++;
}
}
if (activeRegionCount > 1) {
return ConfigurationResult::MULTIPLE_ACTIVE_REGIONS;
}
}
state typename DB::TransactionT::template FutureT<RangeResult> fServerListF =
tr->getRange(serverListKeys, CLIENT_KNOBS->TOO_MANY);
state Future<RangeResult> fServerList =
(newConfig.regions.size()) ? safeThreadFutureToFuture(fServerListF) : Future<RangeResult>();
if (newConfig.usableRegions == 2) {
if (oldReplicationUsesDcId) {
state typename DB::TransactionT::template FutureT<RangeResult> fLocalityListF =
tr->getRange(tagLocalityListKeys, CLIENT_KNOBS->TOO_MANY);
state Future<RangeResult> fLocalityList = safeThreadFutureToFuture(fLocalityListF);
wait(success(fLocalityList) || tooLong);
if (!fLocalityList.isReady()) {
return ConfigurationResult::DATABASE_UNAVAILABLE;
}
RangeResult localityList = fLocalityList.get();
ASSERT(!localityList.more && localityList.size() < CLIENT_KNOBS->TOO_MANY);
std::set<Key> localityDcIds;
for (auto& s : localityList) {
auto dc = decodeTagLocalityListKey(s.key);
if (dc.present()) {
localityDcIds.insert(dc.get());
}
}
for (auto& it : newConfig.regions) {
if (localityDcIds.count(it.dcId) == 0) {
return ConfigurationResult::DCID_MISSING;
}
}
} else {
// all regions with priority >= 0 must be fully replicated
state std::vector<typename DB::TransactionT::template FutureT<Optional<Value>>>
replicasFuturesF;
state std::vector<Future<Optional<Value>>> replicasFutures;
for (auto& it : newConfig.regions) {
if (it.priority >= 0) {
replicasFuturesF.push_back(tr->get(datacenterReplicasKeyFor(it.dcId)));
replicasFutures.push_back(safeThreadFutureToFuture(replicasFuturesF.back()));
}
}
wait(waitForAll(replicasFutures) || tooLong);
for (auto& it : replicasFutures) {
if (!it.isReady()) {
return ConfigurationResult::DATABASE_UNAVAILABLE;
}
if (!it.get().present()) {
return ConfigurationResult::REGION_NOT_FULLY_REPLICATED;
}
}
}
}
if (newConfig.regions.size()) {
// all storage servers must be in one of the regions
wait(success(fServerList) || tooLong);
if (!fServerList.isReady()) {
return ConfigurationResult::DATABASE_UNAVAILABLE;
}
RangeResult serverList = fServerList.get();
ASSERT(!serverList.more && serverList.size() < CLIENT_KNOBS->TOO_MANY);
std::set<Key> newDcIds;
for (auto& it : newConfig.regions) {
newDcIds.insert(it.dcId);
}
std::set<Optional<Key>> missingDcIds;
for (auto& s : serverList) {
auto ssi = decodeServerListValue(s.value);
if (!ssi.locality.dcId().present() || !newDcIds.count(ssi.locality.dcId().get())) {
missingDcIds.insert(ssi.locality.dcId());
}
}
if (missingDcIds.size() > (oldReplicationUsesDcId ? 1 : 0)) {
return ConfigurationResult::STORAGE_IN_UNKNOWN_DCID;
}
}
wait(success(fWorkers) || tooLong);
if (!fWorkers.isReady()) {
return ConfigurationResult::DATABASE_UNAVAILABLE;
}
if (newConfig.regions.size()) {
std::map<Optional<Key>, std::set<Optional<Key>>> dcId_zoneIds;
for (auto& it : fWorkers.get()) {
if (it.processClass.machineClassFitness(ProcessClass::Storage) <= ProcessClass::WorstFit) {
dcId_zoneIds[it.locality.dcId()].insert(it.locality.zoneId());
}
}
for (auto& region : newConfig.regions) {
if (dcId_zoneIds[region.dcId].size() <
std::max(newConfig.storageTeamSize, newConfig.tLogReplicationFactor)) {
return ConfigurationResult::NOT_ENOUGH_WORKERS;
}
if (region.satelliteTLogReplicationFactor > 0 && region.priority >= 0) {
int totalSatelliteProcesses = 0;
for (auto& sat : region.satellites) {
totalSatelliteProcesses += dcId_zoneIds[sat.dcId].size();
}
if (totalSatelliteProcesses < region.satelliteTLogReplicationFactor) {
return ConfigurationResult::NOT_ENOUGH_WORKERS;
}
}
}
} else {
std::set<Optional<Key>> zoneIds;
for (auto& it : fWorkers.get()) {
if (it.processClass.machineClassFitness(ProcessClass::Storage) <= ProcessClass::WorstFit) {
zoneIds.insert(it.locality.zoneId());
}
}
if (zoneIds.size() < std::max(newConfig.storageTeamSize, newConfig.tLogReplicationFactor)) {
return ConfigurationResult::NOT_ENOUGH_WORKERS;
}
}
}
} else {
std::set<Optional<Key>> zoneIds;
for (auto& it : fWorkers.get()) {
if (it.processClass.machineClassFitness(ProcessClass::Storage) <= ProcessClass::WorstFit) {
zoneIds.insert(it.locality.zoneId());
if (newConfig.storageServerStoreType != oldConfig.storageServerStoreType &&
newConfig.storageMigrationType == StorageMigrationType::DISABLED) {
return ConfigurationResult::STORAGE_MIGRATION_DISABLED;
} else if (newConfig.storageMigrationType == StorageMigrationType::GRADUAL &&
newConfig.perpetualStorageWiggleSpeed == 0) {
warnPPWGradual = true;
} else if (newConfig.storageServerStoreType != oldConfig.storageServerStoreType &&
newConfig.storageServerStoreType == KeyValueStoreType::SSD_ROCKSDB_V1) {
warnRocksDBIsExperimental = true;
} else if (newConfig.storageServerStoreType != oldConfig.storageServerStoreType &&
newConfig.storageServerStoreType == KeyValueStoreType::SSD_SHARDED_ROCKSDB) {
warnShardedRocksDBIsExperimental = true;
}
if (newConfig.tenantMode != oldConfig.tenantMode) {
state typename DB::TransactionT::template FutureT<Optional<Value>>
metaclusterRegistrationFuture = tr->get(metaclusterRegistrationKey);
Optional<Value> metaclusterRegistration =
wait(safeThreadFutureToFuture(metaclusterRegistrationFuture));
if (metaclusterRegistration.present()) {
return ConfigurationResult::DATABASE_IS_REGISTERED;
}
}
}
if (zoneIds.size() < std::max(newConfig.storageTeamSize, newConfig.tLogReplicationFactor)) {
return ConfigurationResult::NOT_ENOUGH_WORKERS;
}
}
if (creating) {
tr->setOption(FDBTransactionOptions::INITIALIZE_NEW_DATABASE);
tr->addReadConflictRange(singleKeyRange(initIdKey));
} else if (m.size()) {
// might be used in an emergency transaction, so make sure it is retry-self-conflicting and
// CAUSAL_WRITE_RISKY
tr->setOption(FDBTransactionOptions::CAUSAL_WRITE_RISKY);
tr->addReadConflictRange(singleKeyRange(m.begin()->first));
}
if (newConfig.storageServerStoreType != oldConfig.storageServerStoreType &&
newConfig.storageMigrationType == StorageMigrationType::DISABLED) {
return ConfigurationResult::STORAGE_MIGRATION_DISABLED;
} else if (newConfig.storageMigrationType == StorageMigrationType::GRADUAL &&
newConfig.perpetualStorageWiggleSpeed == 0) {
warnPPWGradual = true;
} else if (newConfig.storageServerStoreType != oldConfig.storageServerStoreType &&
newConfig.storageServerStoreType == KeyValueStoreType::SSD_ROCKSDB_V1) {
warnRocksDBIsExperimental = true;
} else if (newConfig.storageServerStoreType != oldConfig.storageServerStoreType &&
newConfig.storageServerStoreType == KeyValueStoreType::SSD_SHARDED_ROCKSDB) {
warnShardedRocksDBIsExperimental = true;
if (locked.present()) {
ASSERT(creating);
tr->atomicOp(databaseLockedKey,
BinaryWriter::toValue(locked.get(), Unversioned())
.withPrefix(LiteralStringRef("0123456789"))
.withSuffix(LiteralStringRef("\x00\x00\x00\x00")),
MutationRef::SetVersionstampedValue);
}
if (newConfig.tenantMode != oldConfig.tenantMode) {
state
typename transaction_future_type<Transaction, Optional<Value>>::type metaclusterRegistrationFuture =
tr->get(metaclusterRegistrationKey);
Optional<Value> metaclusterRegistration = wait(safeThreadFutureToFuture(metaclusterRegistrationFuture));
if (metaclusterRegistration.present()) {
return ConfigurationResult::DATABASE_IS_REGISTERED;
}
for (auto i = m.begin(); i != m.end(); ++i) {
tr->set(StringRef(i->first), StringRef(i->second));
}
}
}
if (createId.present()) {
tr->setOption(FDBTransactionOptions::INITIALIZE_NEW_DATABASE);
tr->addReadConflictRange(singleKeyRange(initIdKey));
} else if (m.size()) {
// might be used in an emergency transaction, so make sure it is retry-self-conflicting and
// CAUSAL_WRITE_RISKY
tr->setOption(FDBTransactionOptions::CAUSAL_WRITE_RISKY);
tr->addReadConflictRange(singleKeyRange(m.begin()->first));
}
if (locked.present()) {
ASSERT(createId.present());
tr->atomicOp(databaseLockedKey,
BinaryWriter::toValue(locked.get(), Unversioned())
.withPrefix(LiteralStringRef("0123456789"))
.withSuffix(LiteralStringRef("\x00\x00\x00\x00")),
MutationRef::SetVersionstampedValue);
}
for (auto i = m.begin(); i != m.end(); ++i) {
tr->set(StringRef(i->first), StringRef(i->second));
}
tr->addReadConflictRange(singleKeyRange(moveKeysLockOwnerKey));
tr->set(moveKeysLockOwnerKey, versionKey);
if (warnPPWGradual) {
return ConfigurationResult::SUCCESS_WARN_PPW_GRADUAL;
} else if (warnRocksDBIsExperimental) {
return ConfigurationResult::SUCCESS_WARN_ROCKSDB_EXPERIMENTAL;
} else if (warnShardedRocksDBIsExperimental) {
return ConfigurationResult::SUCCESS_WARN_SHARDED_ROCKSDB_EXPERIMENTAL;
} else {
return ConfigurationResult::SUCCESS;
}
}
// Accepts a full configuration in key/value format (from buildConfiguration)
ACTOR template <class DB>
Future<ConfigurationResult> changeConfig(Reference<DB> db, std::map<std::string, std::string> m, bool force) {
state StringRef initIdKey = "\xff/init_id"_sr;
state Reference<typename DB::TransactionT> tr = db->createTransaction();
state Optional<UID> createId;
if (m.count(configKeysPrefix.toString() + "initialized") != 0) {
createId = deterministicRandom()->randomUniqueID();
}
loop {
try {
state ConfigurationResult result = wait(changeConfigTransaction(tr, m, force, createId));
wait(safeThreadFutureToFuture(tr->commit()));
return result;
tr->addReadConflictRange(singleKeyRange(moveKeysLockOwnerKey));
tr->set(moveKeysLockOwnerKey, versionKey);
break;
} catch (Error& e) {
state Error e1(e);
if ((e.code() == error_code_not_committed || e.code() == error_code_transaction_too_old) &&
createId.present()) {
if ((e.code() == error_code_not_committed || e.code() == error_code_transaction_too_old) && creating) {
// The database now exists. Determine whether we created it or it was already existing/created by
// someone else. The latter is an error.
tr->reset();
@ -559,17 +531,16 @@ Future<ConfigurationResult> changeConfig(Reference<DB> db, std::map<std::string,
state typename DB::TransactionT::template FutureT<Optional<Value>> vF = tr->get(initIdKey);
Optional<Value> v = wait(safeThreadFutureToFuture(vF));
if (v != createId.get().toString()) {
if (v != m[initIdKey.toString()])
return ConfigurationResult::DATABASE_ALREADY_CREATED;
} else if (m[configKeysPrefix.toString() + "storage_engine"] ==
std::to_string(KeyValueStoreType::SSD_ROCKSDB_V1)) {
else if (m[configKeysPrefix.toString() + "storage_engine"] ==
std::to_string(KeyValueStoreType::SSD_ROCKSDB_V1))
return ConfigurationResult::DATABASE_CREATED_WARN_ROCKSDB_EXPERIMENTAL;
} else if (m[configKeysPrefix.toString() + "storage_engine"] ==
std::to_string(KeyValueStoreType::SSD_SHARDED_ROCKSDB)) {
else if (m[configKeysPrefix.toString() + "storage_engine"] ==
std::to_string(KeyValueStoreType::SSD_SHARDED_ROCKSDB))
return ConfigurationResult::DATABASE_CREATED_WARN_SHARDED_ROCKSDB_EXPERIMENTAL;
} else {
else
return ConfigurationResult::DATABASE_CREATED;
}
} catch (Error& e2) {
wait(safeThreadFutureToFuture(tr->onError(e2)));
}
@ -578,77 +549,80 @@ Future<ConfigurationResult> changeConfig(Reference<DB> db, std::map<std::string,
wait(safeThreadFutureToFuture(tr->onError(e1)));
}
}
}
ACTOR template <class Transaction>
Future<ConfigurationResult> autoConfigTransaction(Transaction tr, ConfigureAutoResult conf) {
state Key versionKey = BinaryWriter::toValue(deterministicRandom()->randomUniqueID(), Unversioned());
if (!conf.address_class.size())
return ConfigurationResult::INCOMPLETE_CONFIGURATION; // FIXME: correct return type
tr->setOption(FDBTransactionOptions::ACCESS_SYSTEM_KEYS);
tr->setOption(FDBTransactionOptions::PRIORITY_SYSTEM_IMMEDIATE);
tr->setOption(FDBTransactionOptions::LOCK_AWARE);
tr->setOption(FDBTransactionOptions::USE_PROVISIONAL_PROXIES);
state typename transaction_future_type<Transaction, RangeResult>::type processClassesF;
state typename transaction_future_type<Transaction, RangeResult>::type processDataF;
std::vector<ProcessData> workers = wait(getWorkers(tr, processClassesF, processDataF));
std::map<NetworkAddress, Optional<Standalone<StringRef>>> address_processId;
for (auto& w : workers) {
address_processId[w.address] = w.locality.processId();
if (warnPPWGradual) {
return ConfigurationResult::SUCCESS_WARN_PPW_GRADUAL;
} else if (warnRocksDBIsExperimental) {
return ConfigurationResult::SUCCESS_WARN_ROCKSDB_EXPERIMENTAL;
} else if (warnShardedRocksDBIsExperimental) {
return ConfigurationResult::SUCCESS_WARN_SHARDED_ROCKSDB_EXPERIMENTAL;
} else {
return ConfigurationResult::SUCCESS;
}
for (auto& it : conf.address_class) {
if (it.second.classSource() == ProcessClass::CommandLineSource) {
tr->clear(processClassKeyFor(address_processId[it.first].get()));
} else {
tr->set(processClassKeyFor(address_processId[it.first].get()), processClassValue(it.second));
}
}
if (conf.address_class.size())
tr->set(processClassChangeKey, deterministicRandom()->randomUniqueID().toString());
if (conf.auto_logs != conf.old_logs)
tr->set(configKeysPrefix.toString() + "auto_logs", format("%d", conf.auto_logs));
if (conf.auto_commit_proxies != conf.old_commit_proxies)
tr->set(configKeysPrefix.toString() + "auto_commit_proxies", format("%d", conf.auto_commit_proxies));
if (conf.auto_grv_proxies != conf.old_grv_proxies)
tr->set(configKeysPrefix.toString() + "auto_grv_proxies", format("%d", conf.auto_grv_proxies));
if (conf.auto_resolvers != conf.old_resolvers)
tr->set(configKeysPrefix.toString() + "auto_resolvers", format("%d", conf.auto_resolvers));
if (conf.auto_replication != conf.old_replication) {
std::vector<StringRef> modes;
modes.push_back(conf.auto_replication);
std::map<std::string, std::string> m;
auto r = buildConfiguration(modes, m);
if (r != ConfigurationResult::SUCCESS)
return r;
for (auto& kv : m)
tr->set(kv.first, kv.second);
}
tr->addReadConflictRange(singleKeyRange(moveKeysLockOwnerKey));
tr->set(moveKeysLockOwnerKey, versionKey);
return ConfigurationResult::SUCCESS;
}
ACTOR template <class DB>
Future<ConfigurationResult> autoConfig(Reference<DB> db, ConfigureAutoResult conf) {
state Reference<typename DB::TransactionT> tr = db->createTransaction();
state Key versionKey = BinaryWriter::toValue(deterministicRandom()->randomUniqueID(), Unversioned());
if (!conf.address_class.size())
return ConfigurationResult::INCOMPLETE_CONFIGURATION; // FIXME: correct return type
loop {
try {
state ConfigurationResult result = wait(autoConfigTransaction(tr, conf));
wait(safeThreadFutureToFuture(tr->commit()));
return result;
tr->setOption(FDBTransactionOptions::ACCESS_SYSTEM_KEYS);
tr->setOption(FDBTransactionOptions::PRIORITY_SYSTEM_IMMEDIATE);
tr->setOption(FDBTransactionOptions::LOCK_AWARE);
tr->setOption(FDBTransactionOptions::USE_PROVISIONAL_PROXIES);
state typename DB::TransactionT::template FutureT<RangeResult> processClassesF;
state typename DB::TransactionT::template FutureT<RangeResult> processDataF;
std::vector<ProcessData> workers = wait(getWorkers(tr, processClassesF, processDataF));
std::map<NetworkAddress, Optional<Standalone<StringRef>>> address_processId;
for (auto& w : workers) {
address_processId[w.address] = w.locality.processId();
}
for (auto& it : conf.address_class) {
if (it.second.classSource() == ProcessClass::CommandLineSource) {
tr->clear(processClassKeyFor(address_processId[it.first].get()));
} else {
tr->set(processClassKeyFor(address_processId[it.first].get()), processClassValue(it.second));
}
}
if (conf.address_class.size())
tr->set(processClassChangeKey, deterministicRandom()->randomUniqueID().toString());
if (conf.auto_logs != conf.old_logs)
tr->set(configKeysPrefix.toString() + "auto_logs", format("%d", conf.auto_logs));
if (conf.auto_commit_proxies != conf.old_commit_proxies)
tr->set(configKeysPrefix.toString() + "auto_commit_proxies", format("%d", conf.auto_commit_proxies));
if (conf.auto_grv_proxies != conf.old_grv_proxies)
tr->set(configKeysPrefix.toString() + "auto_grv_proxies", format("%d", conf.auto_grv_proxies));
if (conf.auto_resolvers != conf.old_resolvers)
tr->set(configKeysPrefix.toString() + "auto_resolvers", format("%d", conf.auto_resolvers));
if (conf.auto_replication != conf.old_replication) {
std::vector<StringRef> modes;
modes.push_back(conf.auto_replication);
std::map<std::string, std::string> m;
auto r = buildConfiguration(modes, m);
if (r != ConfigurationResult::SUCCESS)
return r;
for (auto& kv : m)
tr->set(kv.first, kv.second);
}
tr->addReadConflictRange(singleKeyRange(moveKeysLockOwnerKey));
tr->set(moveKeysLockOwnerKey, versionKey);
return ConfigurationResult::SUCCESS;
} catch (Error& e) {
wait(safeThreadFutureToFuture(tr->onError(e)));
}
@ -666,23 +640,6 @@ Future<ConfigurationResult> changeConfig(Reference<DB> db, std::string const& mo
return changeConfig(db, m, force);
}
template <class Transaction>
Future<ConfigurationResult> changeConfigTransaction(Transaction tr,
std::vector<StringRef> const& modes,
Optional<ConfigureAutoResult> const& conf,
bool force,
Optional<UID> createId) {
if (modes.size() && modes[0] == LiteralStringRef("auto") && conf.present()) {
return autoConfigTransaction(tr, conf.get());
}
std::map<std::string, std::string> m;
auto r = buildConfiguration(modes, m);
if (r != ConfigurationResult::SUCCESS)
return r;
return changeConfigTransaction(tr, m, force, createId);
}
// Accepts a vector of configuration tokens
template <class DB>
Future<ConfigurationResult> changeConfig(Reference<DB> db,

1
fdbclient/include/fdbclient/IClientApi.h
View File

@ -22,7 +22,6 @@
#define FDBCLIENT_ICLIENTAPI_H
#pragma once
#include "fdbclient/CoordinationInterface.h"
#include "fdbclient/FDBOptions.g.h"
#include "fdbclient/FDBTypes.h"
#include "fdbclient/Tenant.h"

12
fdbclient/include/fdbclient/MultiVersionTransaction.h
View File

@ -522,8 +522,8 @@ class MultiVersionDatabase;
class MultiVersionTenant;
// An implementation of ITransaction that wraps a transaction created either locally or through a dynamically loaded
// external client. When needed (e.g on cluster version change), the MultiVersionTransaction can automatically
// replace its wrapped transaction with one from another client.
// external client. When needed (e.g on cluster version change), the MultiVersionTransaction can automatically replace
// its wrapped transaction with one from another client.
class MultiVersionTransaction : public ITransaction, ThreadSafeReferenceCounted<MultiVersionTransaction> {
public:
MultiVersionTransaction(Reference<MultiVersionDatabase> db,
@ -625,8 +625,8 @@ private:
// A lock that needs to be held if using timeoutTsav or currentTimeout
ThreadSpinLock timeoutLock;
// A single assignment var (i.e. promise) that gets set with an error when the timeout elapses or the
// transaction is reset or destroyed.
// A single assignment var (i.e. promise) that gets set with an error when the timeout elapses or the transaction
// is reset or destroyed.
Reference<ThreadSingleAssignmentVar<Void>> timeoutTsav;
// A reference to the current actor waiting for the timeout. This actor will set the timeoutTsav promise.
@ -793,8 +793,8 @@ public:
double getMainThreadBusyness() override;
// Returns the protocol version reported by the coordinator this client is connected to
// If an expected version is given, the future won't return until the protocol version is different than
// expected Note: this will never return if the server is running a protocol from FDB 5.0 or older
// If an expected version is given, the future won't return until the protocol version is different than expected
// Note: this will never return if the server is running a protocol from FDB 5.0 or older
ThreadFuture<ProtocolVersion> getServerProtocol(
Optional<ProtocolVersion> expectedVersion = Optional<ProtocolVersion>()) override;

2
fdbclient/include/fdbclient/SpecialKeySpace.actor.h
View File

@ -258,7 +258,7 @@ private:
static std::unordered_map<SpecialKeySpace::MODULE, KeyRange> moduleToBoundary;
// module command to special keys range
// management command to its special keys' range
static std::unordered_map<std::string, KeyRange> managementApiCommandToRange;
static std::unordered_map<std::string, KeyRange> actorLineageApiCommandToRange;

1
fdbclient/include/fdbclient/SystemData.h
View File

@ -26,7 +26,6 @@
#include "fdbclient/FDBTypes.h"
#include "fdbclient/BlobWorkerInterface.h" // TODO move the functions that depend on this out of here and into BlobWorkerInterface.h to remove this depdendency
#include "fdbclient/Metacluster.h"
#include "fdbclient/StorageServerInterface.h"
#include "Tenant.h"

1
fdbclient/include/fdbclient/TenantManagement.actor.h
View File

@ -28,6 +28,7 @@
#include <string>
#include <map>
#include "fdbclient/GenericTransactionHelper.h"
#include "fdbclient/Metacluster.h"
#include "fdbclient/SystemData.h"
#include "flow/actorcompiler.h" // has to be last include

1
fdbserver/ApplyMetadataMutation.cpp
View File

@ -20,6 +20,7 @@
#include "fdbclient/BackupAgent.actor.h"
#include "fdbclient/KeyBackedTypes.h" // for key backed map codecs for tss mapping
#include "fdbclient/Metacluster.h"
#include "fdbclient/MutationList.h"
#include "fdbclient/Notified.h"
#include "fdbclient/SystemData.h"

1
fdbserver/ClusterRecovery.actor.cpp
View File

@ -18,6 +18,7 @@
* limitations under the License.
*/
#include "fdbclient/Metacluster.h"
#include "fdbrpc/sim_validation.h"
#include "fdbserver/ApplyMetadataMutation.h"
#include "fdbserver/BackupProgress.actor.h"