/*
* fdbserver.actor.cpp
*
* This source file is part of the FoundationDB open source project
*
* Copyright 2013-2018 Apple Inc. and the FoundationDB project authors
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
// There's something in one of the files below that defines a macros
// a macro that makes boost interprocess break on Windows.
#define BOOST_DATE_TIME_NO_LIB
#include <algorithm>
#include <cctype>
#include <fstream>
#include <iterator>
#include <sstream>
#include <stdarg.h>
#include <stdio.h>
#include <time.h>
#include <boost/algorithm/string.hpp>
#include <boost/interprocess/managed_shared_memory.hpp>
#include "fdbclient/ActorLineageProfiler.h"
#include "fdbclient/ClusterConnectionFile.h"
#include "fdbclient/IKnobCollection.h"
#include "fdbclient/NativeAPI.actor.h"
#include "fdbclient/SystemData.h"
#include "fdbclient/versions.h"
#include "fdbclient/BuildFlags.h"
#include "fdbclient/WellKnownEndpoints.h"
#include "fdbmonitor/SimpleIni.h"
#include "fdbrpc/AsyncFileCached.actor.h"
#include "fdbrpc/Net2FileSystem.h"
#include "fdbrpc/PerfMetric.h"
#include "fdbrpc/simulator.h"
#include "fdbserver/ConflictSet.h"
#include "fdbserver/CoordinationInterface.h"
#include "fdbserver/CoroFlow.h"
#include "fdbserver/DataDistribution.actor.h"
#include "fdbserver/IKeyValueStore.h"
#include "fdbserver/MoveKeys.actor.h"
#include "fdbserver/NetworkTest.h"
#include "fdbserver/RestoreWorkerInterface.actor.h"
#include "fdbserver/ServerDBInfo.h"
#include "fdbserver/SimulatedCluster.h"
#include "fdbserver/Status.h"
#include "fdbserver/TesterInterface.actor.h"
#include "fdbserver/WorkerInterface.actor.h"
#include "fdbserver/pubsub.h"
#include "fdbserver/workloads/workloads.actor.h"
#include "flow/DeterministicRandom.h"
#include "flow/Platform.h"
#include "flow/ProtocolVersion.h"
#include "flow/SimpleOpt.h"
#include "flow/SystemMonitor.h"
#include "flow/TLSConfig.actor.h"
#include "flow/Tracing.h"
#include "flow/WriteOnlySet.h"
#include "flow/UnitTest.h"
#include "flow/FaultInjection.h"
#if defined(__linux__) || defined(__FreeBSD__)
#include <execinfo.h>
#include <signal.h>
#ifdef ALLOC_INSTRUMENTATION
#include <cxxabi.h>
#endif
#endif
#ifdef WIN32
#define NOMINMAX
#define WIN32_LEAN_AND_MEAN
#include <Windows.h>
#endif
#include "flow/actorcompiler.h" // This must be the last #include.
using namespace std::literals;
// clang-format off
enum {
OPT_CONNFILE, OPT_SEEDCONNFILE, OPT_SEEDCONNSTRING, OPT_ROLE, OPT_LISTEN, OPT_PUBLICADDR, OPT_DATAFOLDER, OPT_LOGFOLDER, OPT_PARENTPID, OPT_TRACER, OPT_NEWCONSOLE,
OPT_NOBOX, OPT_TESTFILE, OPT_RESTARTING, OPT_RESTORING, OPT_RANDOMSEED, OPT_KEY, OPT_MEMLIMIT, OPT_STORAGEMEMLIMIT, OPT_CACHEMEMLIMIT, OPT_MACHINEID,
OPT_DCID, OPT_MACHINE_CLASS, OPT_BUGGIFY, OPT_VERSION, OPT_BUILD_FLAGS, OPT_CRASHONERROR, OPT_HELP, OPT_NETWORKIMPL, OPT_NOBUFSTDOUT, OPT_BUFSTDOUTERR,
OPT_TRACECLOCK, OPT_NUMTESTERS, OPT_DEVHELP, OPT_ROLLSIZE, OPT_MAXLOGS, OPT_MAXLOGSSIZE, OPT_KNOB, OPT_UNITTESTPARAM, OPT_TESTSERVERS, OPT_TEST_ON_SERVERS, OPT_METRICSCONNFILE,
OPT_METRICSPREFIX, OPT_LOGGROUP, OPT_LOCALITY, OPT_IO_TRUST_SECONDS, OPT_IO_TRUST_WARN_ONLY, OPT_FILESYSTEM, OPT_PROFILER_RSS_SIZE, OPT_KVFILE,
OPT_TRACE_FORMAT, OPT_WHITELIST_BINPATH, OPT_BLOB_CREDENTIAL_FILE, OPT_CONFIG_PATH, OPT_USE_TEST_CONFIG_DB, OPT_FAULT_INJECTION, OPT_PROFILER, OPT_PRINT_SIMTIME,
};
CSimpleOpt::SOption g_rgOptions[] = {
{ OPT_CONNFILE, "-C", SO_REQ_SEP },
{ OPT_CONNFILE, "--cluster_file", SO_REQ_SEP },
{ OPT_SEEDCONNFILE, "--seed_cluster_file", SO_REQ_SEP },
{ OPT_SEEDCONNSTRING, "--seed_connection_string", SO_REQ_SEP },
{ OPT_ROLE, "-r", SO_REQ_SEP },
{ OPT_ROLE, "--role", SO_REQ_SEP },
{ OPT_PUBLICADDR, "-p", SO_REQ_SEP },
{ OPT_PUBLICADDR, "--public_address", SO_REQ_SEP },
{ OPT_LISTEN, "-l", SO_REQ_SEP },
{ OPT_LISTEN, "--listen_address", SO_REQ_SEP },
#ifdef __linux__
{ OPT_FILESYSTEM, "--data_filesystem", SO_REQ_SEP },
{ OPT_PROFILER_RSS_SIZE, "--rsssize", SO_REQ_SEP },
#endif
{ OPT_DATAFOLDER, "-d", SO_REQ_SEP },
{ OPT_DATAFOLDER, "--datadir", SO_REQ_SEP },
{ OPT_LOGFOLDER, "-L", SO_REQ_SEP },
{ OPT_LOGFOLDER, "--logdir", SO_REQ_SEP },
{ OPT_ROLLSIZE, "-Rs", SO_REQ_SEP },
{ OPT_ROLLSIZE, "--logsize", SO_REQ_SEP },
{ OPT_MAXLOGS, "--maxlogs", SO_REQ_SEP },
{ OPT_MAXLOGSSIZE, "--maxlogssize", SO_REQ_SEP },
{ OPT_LOGGROUP, "--loggroup", SO_REQ_SEP },
{ OPT_PARENTPID, "--parentpid", SO_REQ_SEP },
{ OPT_TRACER, "--tracer", SO_REQ_SEP },
#ifdef _WIN32
{ OPT_NEWCONSOLE, "-n", SO_NONE },
{ OPT_NEWCONSOLE, "--newconsole", SO_NONE },
{ OPT_NOBOX, "-q", SO_NONE },
{ OPT_NOBOX, "--no_dialog", SO_NONE },
#endif
{ OPT_KVFILE, "--kvfile", SO_REQ_SEP },
{ OPT_TESTFILE, "-f", SO_REQ_SEP },
{ OPT_TESTFILE, "--testfile", SO_REQ_SEP },
{ OPT_RESTARTING, "-R", SO_NONE },
{ OPT_RESTARTING, "--restarting", SO_NONE },
{ OPT_RANDOMSEED, "-s", SO_REQ_SEP },
{ OPT_RANDOMSEED, "--seed", SO_REQ_SEP },
{ OPT_KEY, "-k", SO_REQ_SEP },
{ OPT_KEY, "--key", SO_REQ_SEP },
{ OPT_MEMLIMIT, "-m", SO_REQ_SEP },
{ OPT_MEMLIMIT, "--memory", SO_REQ_SEP },
{ OPT_STORAGEMEMLIMIT, "-M", SO_REQ_SEP },
{ OPT_STORAGEMEMLIMIT, "--storage_memory", SO_REQ_SEP },
{ OPT_CACHEMEMLIMIT, "--cache_memory", SO_REQ_SEP },
{ OPT_MACHINEID, "-i", SO_REQ_SEP },
{ OPT_MACHINEID, "--machine_id", SO_REQ_SEP },
{ OPT_DCID, "-a", SO_REQ_SEP },
{ OPT_DCID, "--datacenter_id", SO_REQ_SEP },
{ OPT_MACHINE_CLASS, "-c", SO_REQ_SEP },
{ OPT_MACHINE_CLASS, "--class", SO_REQ_SEP },
{ OPT_BUGGIFY, "-b", SO_REQ_SEP },
{ OPT_BUGGIFY, "--buggify", SO_REQ_SEP },
{ OPT_VERSION, "-v", SO_NONE },
{ OPT_VERSION, "--version", SO_NONE },
{ OPT_BUILD_FLAGS, "--build_flags", SO_NONE },
{ OPT_CRASHONERROR, "--crash", SO_NONE },
{ OPT_NETWORKIMPL, "-N", SO_REQ_SEP },
{ OPT_NETWORKIMPL, "--network", SO_REQ_SEP },
{ OPT_NOBUFSTDOUT, "--unbufferedout", SO_NONE },
{ OPT_BUFSTDOUTERR, "--bufferedout", SO_NONE },
{ OPT_TRACECLOCK, "--traceclock", SO_REQ_SEP },
{ OPT_NUMTESTERS, "--num_testers", SO_REQ_SEP },
{ OPT_HELP, "-?", SO_NONE },
{ OPT_HELP, "-h", SO_NONE },
{ OPT_HELP, "--help", SO_NONE },
{ OPT_DEVHELP, "--dev-help", SO_NONE },
{ OPT_KNOB, "--knob_", SO_REQ_SEP },
{ OPT_UNITTESTPARAM, "--test_", SO_REQ_SEP },
{ OPT_LOCALITY, "--locality_", SO_REQ_SEP },
{ OPT_TESTSERVERS, "--testservers", SO_REQ_SEP },
{ OPT_TEST_ON_SERVERS, "--testonservers", SO_NONE },
{ OPT_METRICSCONNFILE, "--metrics_cluster", SO_REQ_SEP },
{ OPT_METRICSPREFIX, "--metrics_prefix", SO_REQ_SEP },
{ OPT_IO_TRUST_SECONDS, "--io_trust_seconds", SO_REQ_SEP },
{ OPT_IO_TRUST_WARN_ONLY, "--io_trust_warn_only", SO_NONE },
{ OPT_TRACE_FORMAT, "--trace_format", SO_REQ_SEP },
{ OPT_WHITELIST_BINPATH, "--whitelist_binpath", SO_REQ_SEP },
{ OPT_BLOB_CREDENTIAL_FILE, "--blob_credential_file", SO_REQ_SEP },
{ OPT_CONFIG_PATH, "--config_path", SO_REQ_SEP },
{ OPT_USE_TEST_CONFIG_DB, "--use_test_config_db", SO_NONE },
{ OPT_FAULT_INJECTION, "-fi", SO_REQ_SEP },
{ OPT_FAULT_INJECTION, "--fault_injection", SO_REQ_SEP },
{ OPT_PROFILER, "--profiler_", SO_REQ_SEP},
{ OPT_PRINT_SIMTIME, "--print_sim_time", SO_NONE },
#ifndef TLS_DISABLED
TLS_OPTION_FLAGS
#endif
SO_END_OF_OPTIONS
};
// clang-format on
extern void dsltest();
extern void pingtest();
extern void copyTest();
extern void versionedMapTest();
extern void createTemplateDatabase();
// FIXME: this really belongs in a header somewhere since it is actually used.
extern IPAddress determinePublicIPAutomatically(ClusterConnectionString const& ccs);
extern const char* getSourceVersion();
extern void flushTraceFileVoid();
extern const int MAX_CLUSTER_FILE_BYTES;
#ifdef ALLOC_INSTRUMENTATION
extern uint8_t* g_extra_memory;
#endif
bool enableFailures = true;
#define test_assert(x) \
if (!(x)) { \
std::cout << "Test failed: " #x << std::endl; \
return false; \
}
#ifdef _WIN32
#include <sddl.h>
// It is your
// responsibility to properly initialize the
// structure and to free the structure's
// lpSecurityDescriptor member when you have
// finished using it. To free the structure's
// lpSecurityDescriptor member, call the
// LocalFree function.
BOOL CreatePermissiveReadWriteDACL(SECURITY_ATTRIBUTES* pSA) {
UNSTOPPABLE_ASSERT(pSA != nullptr);
TCHAR* szSD = TEXT("D:") // Discretionary ACL
TEXT("(A;OICI;GR;;;AU)") // Allow read/write/execute to authenticated users
TEXT("(A;OICI;GA;;;BA)"); // Allow full control to administrators
return ConvertStringSecurityDescriptorToSecurityDescriptor(
szSD, SDDL_REVISION_1, &(pSA->lpSecurityDescriptor), nullptr);
}
#endif
class WorldReadablePermissions {
public:
WorldReadablePermissions() {
#ifdef _WIN32
sa.nLength = sizeof(SECURITY_ATTRIBUTES);
sa.bInheritHandle = FALSE;
if (!CreatePermissiveReadWriteDACL(&sa)) {
TraceEvent("Win32DACLCreationFail").GetLastError();
throw platform_error();
}
permission.set_permissions(&sa);
#elif (defined(__linux__) || defined(__APPLE__) || defined(__FreeBSD__))
// There is nothing to do here, since the default permissions are fine
#else
#error Port me!
#endif
}
virtual ~WorldReadablePermissions() {
#ifdef _WIN32
LocalFree(sa.lpSecurityDescriptor);
#elif (defined(__linux__) || defined(__APPLE__) || defined(__FreeBSD__))
// There is nothing to do here, since the default permissions are fine
#else
#error Port me!
#endif
}
boost::interprocess::permissions permission;
private:
WorldReadablePermissions(const WorldReadablePermissions& rhs) {}
#ifdef _WIN32
SECURITY_ATTRIBUTES sa;
#endif
};
UID getSharedMemoryMachineId() {
UID* machineId = nullptr;
int numTries = 0;
// Permissions object defaults to 0644 on *nix, but on windows defaults to allowing access to only the creator.
// On windows, this means that we have to create an elaborate workaround for DACLs
WorldReadablePermissions p;
std::string sharedMemoryIdentifier = "fdbserver_shared_memory_id";
loop {
try {
// "0" is the default parameter "addr"
boost::interprocess::managed_shared_memory segment(
boost::interprocess::open_or_create, sharedMemoryIdentifier.c_str(), 1000, 0, p.permission);
machineId = segment.find_or_construct<UID>("machineId")(deterministicRandom()->randomUniqueID());
if (!machineId)
criticalError(
FDB_EXIT_ERROR, "SharedMemoryError", "Could not locate or create shared memory - 'machineId'");
return *machineId;
} catch (boost::interprocess::interprocess_exception&) {
try {
// If the shared memory already exists, open it read-only in case it was created by another user
boost::interprocess::managed_shared_memory segment(boost::interprocess::open_read_only,
sharedMemoryIdentifier.c_str());
machineId = segment.find<UID>("machineId").first;
if (!machineId)
criticalError(FDB_EXIT_ERROR, "SharedMemoryError", "Could not locate shared memory - 'machineId'");
return *machineId;
} catch (boost::interprocess::interprocess_exception& ex) {
// Retry in case the shared memory was deleted in between the call to open_or_create and open_read_only
// Don't keep trying forever in case this is caused by some other problem
if (++numTries == 10)
criticalError(FDB_EXIT_ERROR,
"SharedMemoryError",
format("Could not open shared memory - %s", ex.what()).c_str());
}
}
}
}
ACTOR void failAfter(Future<Void> trigger, ISimulator::ProcessInfo* m = g_simulator.getCurrentProcess()) {
wait(trigger);
if (enableFailures) {
printf("Killing machine: %s at %f\n", m->address.toString().c_str(), now());
g_simulator.killProcess(m, ISimulator::KillInstantly);
}
}
void failAfter(Future<Void> trigger, Endpoint e) {
if (g_network == &g_simulator)
failAfter(trigger, g_simulator.getProcess(e));
}
ACTOR Future<Void> histogramReport() {
loop {
wait(delay(SERVER_KNOBS->HISTOGRAM_REPORT_INTERVAL));
GetHistogramRegistry().logReport();
}
}
void testSerializationSpeed() {
double tstart;
double build = 0, serialize = 0, deserialize = 0, copy = 0, deallocate = 0;
double bytes = 0;
double testBegin = timer();
for (int a = 0; a < 10000; a++) {
{
tstart = timer();
Arena batchArena;
VectorRef<CommitTransactionRef> batch;
batch.resize(batchArena, 1000);
for (int t = 0; t < batch.size(); t++) {
CommitTransactionRef& tr = batch[t];
tr.read_snapshot = 0;
for (int i = 0; i < 2; i++)
tr.mutations.push_back_deep(
batchArena,
MutationRef(MutationRef::SetValue, LiteralStringRef("KeyABCDE"), LiteralStringRef("SomeValu")));
tr.mutations.push_back_deep(
batchArena,
MutationRef(MutationRef::ClearRange, LiteralStringRef("BeginKey"), LiteralStringRef("EndKeyAB")));
}
build += timer() - tstart;
tstart = timer();
BinaryWriter wr(IncludeVersion());
wr << batch;
bytes += wr.getLength();
serialize += timer() - tstart;
for (int i = 0; i < 1; i++) {
tstart = timer();
Arena arena;
StringRef data(arena, StringRef((const uint8_t*)wr.getData(), wr.getLength()));
copy += timer() - tstart;
tstart = timer();
ArenaReader rd(arena, data, IncludeVersion());
VectorRef<CommitTransactionRef> batch2;
rd >> arena >> batch2;
deserialize += timer() - tstart;
}
tstart = timer();
}
deallocate += timer() - tstart;
}
double elapsed = (timer() - testBegin);
printf("Test speed: %0.1f MB/sec (%0.0f/sec)\n", bytes / 1e6 / elapsed, 1000000 / elapsed);
printf(" Build: %0.1f MB/sec\n", bytes / 1e6 / build);
printf(" Serialize: %0.1f MB/sec\n", bytes / 1e6 / serialize);
printf(" Copy: %0.1f MB/sec\n", bytes / 1e6 / copy);
printf(" Deserialize: %0.1f MB/sec\n", bytes / 1e6 / deserialize);
printf(" Deallocate: %0.1f MB/sec\n", bytes / 1e6 / deallocate);
printf(" Bytes: %0.1f MB\n", bytes / 1e6);
printf("\n");
}
std::string toHTML(const StringRef& binaryString) {
std::string s;
for (int i = 0; i < binaryString.size(); i++) {
uint8_t c = binaryString[i];
if (c == '<')
s += "<";
else if (c == '>')
s += ">";
else if (c == '&')
s += "&";
else if (c == '"')
s += """;
else if (c == ' ')
s += " ";
else if (c > 32 && c < 127)
s += c;
else
s += format("<span class=\"binary\">[%02x]</span>", c);
}
return s;
}
ACTOR Future<Void> dumpDatabase(Database cx, std::string outputFilename, KeyRange range = allKeys) {
try {
state Transaction tr(cx);
loop {
state FILE* output = fopen(outputFilename.c_str(), "wt");
try {
state KeySelectorRef iter = firstGreaterOrEqual(range.begin);
state Arena arena;
fprintf(output, "<html><head><style type=\"text/css\">.binary {color:red}</style></head><body>\n");
Version ver = wait(tr.getReadVersion());
fprintf(output, "<h3>Database version: %" PRId64 "</h3>", ver);
loop {
RangeResult results = wait(tr.getRange(iter, firstGreaterOrEqual(range.end), 1000));
for (int r = 0; r < results.size(); r++) {
std::string key = toHTML(results[r].key), value = toHTML(results[r].value);
fprintf(output, "<p>%s <b>:=</b> %s</p>\n", key.c_str(), value.c_str());
}
if (results.size() < 1000)
break;
iter = firstGreaterThan(KeyRef(arena, results[results.size() - 1].key));
}
fprintf(output, "</body></html>");
fclose(output);
TraceEvent("DatabaseDumped").detail("Filename", outputFilename);
return Void();
} catch (Error& e) {
fclose(output);
wait(tr.onError(e));
}
}
} catch (Error& e) {
TraceEvent(SevError, "DumpDatabaseError").error(e).detail("Filename", outputFilename);
throw;
}
}
void memoryTest();
void skipListTest();
Future<Void> startSystemMonitor(std::string dataFolder,
Optional<Standalone<StringRef>> dcId,
Optional<Standalone<StringRef>> zoneId,
Optional<Standalone<StringRef>> machineId) {
initializeSystemMonitorMachineState(
SystemMonitorMachineState(dataFolder, dcId, zoneId, machineId, g_network->getLocalAddress().ip));
systemMonitor();
return recurring(&systemMonitor, SERVER_KNOBS->SYSTEM_MONITOR_FREQUENCY, TaskPriority::FlushTrace);
}
void testIndexedSet();
#ifdef _WIN32
void parentWatcher(void* parentHandle) {
HANDLE parent = (HANDLE)parentHandle;
int signal = WaitForSingleObject(parent, INFINITE);
CloseHandle(parentHandle);
if (signal == WAIT_OBJECT_0)
criticalError(FDB_EXIT_SUCCESS, "ParentProcessExited", "Parent process exited");
TraceEvent(SevError, "ParentProcessWaitFailed").detail("RetCode", signal).GetLastError();
}
#else
void* parentWatcher(void* arg) {
int* parent_pid = (int*)arg;
while (1) {
sleep(1);
if (getppid() != *parent_pid)
criticalError(FDB_EXIT_SUCCESS, "ParentProcessExited", "Parent process exited");
}
}
#endif
static void printBuildInformation() {
printf("%s", jsonBuildInformation().c_str());
}
static void printVersion() {
printf("FoundationDB " FDB_VT_PACKAGE_NAME " (v" FDB_VT_VERSION ")\n");
printf("source version %s\n", getSourceVersion());
printf("protocol %" PRIx64 "\n", currentProtocolVersion.version());
}
static void printHelpTeaser(const char* name) {
fprintf(stderr, "Try `%s --help' for more information.\n", name);
}
static void printOptionUsage(std::string option, std::string description) {
static const std::string OPTION_INDENT(" ");
static const std::string DESCRIPTION_INDENT(" ");
static const int WIDTH = 80;
boost::algorithm::trim(option);
boost::algorithm::trim(description);
std::string result = OPTION_INDENT + option + "\n";
std::stringstream sstream(description);
if (sstream.eof()) {
printf("%s", result.c_str());
return;
}
std::string currWord;
sstream >> currWord;
std::string currLine(DESCRIPTION_INDENT + ' ' + currWord);
int currLength = currLine.size();
while (!sstream.eof()) {
sstream >> currWord;
if (currLength + static_cast<int>(currWord.size()) + 1 > WIDTH) {
result += currLine + '\n';
currLine = DESCRIPTION_INDENT + ' ' + currWord;
} else {
currLine += ' ' + currWord;
}
currLength = currLine.size();
}
result += currLine + '\n';
printf("%s", result.c_str());
}
static void printUsage(const char* name, bool devhelp) {
printf("FoundationDB " FDB_VT_PACKAGE_NAME " (v" FDB_VT_VERSION ")\n");
printf("Usage: %s -p ADDRESS [OPTIONS]\n\n", name);
printOptionUsage("-p ADDRESS, --public_address ADDRESS",
" Public address, specified as `IP_ADDRESS:PORT' or `auto:PORT'.");
printOptionUsage("-l ADDRESS, --listen_address ADDRESS",
" Listen address, specified as `IP_ADDRESS:PORT' (defaults to"
" public address).");
printOptionUsage("-C CONNFILE, --cluster_file CONNFILE",
" The path of a file containing the connection string for the"
" FoundationDB cluster. The default is first the value of the"
" FDB_CLUSTER_FILE environment variable, then `./fdb.cluster',"
" then `" +
platform::getDefaultClusterFilePath() + "'.");
printOptionUsage("--seed_cluster_file SEEDCONNFILE",
" The path of a seed cluster file which will be used to connect"
" if the -C cluster file does not exist. If the server connects"
" successfully using the seed file, then it copies the file to"
" the -C file location.");
printOptionUsage("--seed_connection_string SEEDCONNSTRING",
" The path of a seed connection string which will be used to connect"
" if the -C cluster file does not exist. If the server connects"
" successfully using the seed string, then it copies the string to"
" the -C file location.");
#ifdef __linux__
printOptionUsage("--data_filesystem PATH",
" Turns on validation that all data files are written to a drive"
" mounted at the specified PATH. This checks that the device at PATH"
" is currently mounted and that any data files get written to the"
" same device.");
#endif
printOptionUsage("-d PATH, --datadir PATH",
" Store data files in the given folder (must be unique for each"
" fdbserver instance on a given machine).");
printOptionUsage("-L PATH, --logdir PATH", " Store log files in the given folder (default is `.').");
printOptionUsage("--logsize SIZE",
"Roll over to a new log file after the current log file"
" exceeds SIZE bytes. The default value is 10MiB.");
printOptionUsage("--maxlogs SIZE, --maxlogssize SIZE",
" Delete the oldest log file when the total size of all log"
" files exceeds SIZE bytes. If set to 0, old log files will not"
" be deleted. The default value is 100MiB.");
printOptionUsage("--loggroup LOG_GROUP",
" Sets the LogGroup field with the specified value for all"
" events in the trace output (defaults to `default').");
printOptionUsage("--trace_format FORMAT",
" Select the format of the log files. xml (the default) and json"
" are supported.");
printOptionUsage("--tracer TRACER",
" Select a tracer for transaction tracing. Currently disabled"
" (the default) and log_file are supported.");
printOptionUsage("-i ID, --machine_id ID",
" Machine and zone identifier key (up to 16 hex characters)."
" Defaults to a random value shared by all fdbserver processes"
" on this machine.");
printOptionUsage("-a ID, --datacenter_id ID", " Data center identifier key (up to 16 hex characters).");
printOptionUsage("--locality_LOCALITYKEY LOCALITYVALUE",
" Define a locality key. LOCALITYKEY is case-insensitive though"
" LOCALITYVALUE is not.");
printOptionUsage("-m SIZE, --memory SIZE",
" Memory limit. The default value is 8GiB. When specified"
" without a unit, MiB is assumed.");
printOptionUsage("-M SIZE, --storage_memory SIZE",
" Maximum amount of memory used for storage. The default"
" value is 1GiB. When specified without a unit, MB is"
" assumed.");
printOptionUsage("--cache_memory SIZE",
" The amount of memory to use for caching disk pages."
" The default value is 2GiB. When specified without a unit,"
" MiB is assumed.");
printOptionUsage("-c CLASS, --class CLASS",
" Machine class (valid options are storage, transaction,"
" resolution, grv_proxy, commit_proxy, master, test, unset, stateless, log, router,"
" and cluster_controller).");
printOptionUsage("--profiler_",
"Set an actor profiler option. Supported options are:\n"
" collector -- None or FluentD (FluentD requires collector_endpoint to be set)\n"
" collector_endpoint -- IP:PORT of the fluentd server\n"
" collector_protocol -- UDP or TCP (default is UDP)");
#ifndef TLS_DISABLED
printf(TLS_HELP);
#endif
printOptionUsage("-v, --version", "Print version information and exit.");
printOptionUsage("-h, -?, --help", "Display this help and exit.");
if (devhelp) {
printf(" --build_flags Print build information and exit.\n");
printOptionUsage(
"-r ROLE, --role ROLE",
" Server role (valid options are fdbd, test, multitest,"
" simulation, networktestclient, networktestserver, restore"
" consistencycheck, kvfileintegritycheck, kvfilegeneratesums, unittests). The default is `fdbd'.");
#ifdef _WIN32
printOptionUsage("-n, --newconsole", " Create a new console.");
printOptionUsage("-q, --no_dialog", " Disable error dialog on crash.");
printOptionUsage("--parentpid PID", " Specify a process after whose termination to exit.");
#endif
printOptionUsage("-f TESTFILE, --testfile",
" Testfile to run, defaults to `tests/default.txt'. If role is `unittests', specifies which "
"unit tests to run as a search prefix.");
printOptionUsage("-R, --restarting", " Restart a previous simulation that was cleanly shut down.");
printOptionUsage("-s SEED, --seed SEED", " Random seed.");
printOptionUsage("-k KEY, --key KEY", "Target key for search role.");
printOptionUsage(
"--kvfile FILE",
"Input file (SQLite database file) for use by the 'kvfilegeneratesums' and 'kvfileintegritycheck' roles.");
printOptionUsage("-b [on,off], --buggify [on,off]", " Sets Buggify system state, defaults to `off'.");
printOptionUsage("-fi [on,off], --fault_injection [on,off]", " Sets fault injection, defaults to `on'.");
printOptionUsage("--crash", "Crash on serious errors instead of continuing.");
printOptionUsage("-N NETWORKIMPL, --network NETWORKIMPL",
" Select network implementation, `net2' (default),"
" `net2-threadpool'.");
printOptionUsage("--unbufferedout", " Do not buffer stdout and stderr.");
printOptionUsage("--bufferedout", " Buffer stdout and stderr.");
printOptionUsage("--traceclock CLOCKIMPL",
" Select clock source for trace files, `now' (default) or"
" `realtime'.");
printOptionUsage("--num_testers NUM",
" A multitester will wait for NUM testers before starting"
" (defaults to 1).");
printOptionUsage("--test_PARAMNAME PARAMVALUE",
" Set a UnitTest named parameter to the given value. Names are case sensitive.");
#ifdef __linux__
printOptionUsage("--rsssize SIZE",
" Turns on automatic heap profiling when RSS memory size exceeds"
" the given threshold. fdbserver needs to be compiled with"
" USE_GPERFTOOLS flag in order to use this feature.");
#endif
printOptionUsage("--testservers ADDRESSES",
" The addresses of networktestservers"
" specified as ADDRESS:PORT,ADDRESS:PORT...");
printOptionUsage("--testonservers", " Testers are recruited on servers.");
printOptionUsage("--metrics_cluster CONNFILE",
" The cluster file designating where this process will"
" store its metric data. By default metrics will be stored"
" in the same database the process is participating in.");
printOptionUsage("--metrics_prefix PREFIX",
" The prefix where this process will store its metric data."
" Must be specified if using a different database for metrics.");
printOptionUsage("--knob_KNOBNAME KNOBVALUE", " Changes a database knob. KNOBNAME should be lowercase.");
printOptionUsage("--io_trust_seconds SECONDS",
" Sets the time in seconds that a read or write operation is allowed to take"
" before timing out with an error. If an operation times out, all future"
" operations on that file will fail with an error as well. Only has an effect"
" when using AsyncFileKAIO in Linux.");
printOptionUsage("--io_trust_warn_only",
" Instead of failing when an I/O operation exceeds io_trust_seconds, just"
" log a warning to the trace log. Has no effect if io_trust_seconds is unspecified.");
} else {
printOptionUsage("--dev-help", "Display developer-specific help and exit.");
}
printf("\n"
"SIZE parameters may use one of the multiplicative suffixes B=1, KB=10^3,\n"
"KiB=2^10, MB=10^6, MiB=2^20, GB=10^9, GiB=2^30, TB=10^12, or TiB=2^40.\n");
}
extern bool g_crashOnError;
#if defined(ALLOC_INSTRUMENTATION) || defined(ALLOC_INSTRUMENTATION_STDOUT)
void* operator new(std::size_t size) {
void* p = malloc(size);
if (!p)
throw std::bad_alloc();
recordAllocation(p, size);
return p;
}
void operator delete(void* ptr) throw() {
recordDeallocation(ptr);
free(ptr);
}
// scalar, nothrow new and it matching delete
void* operator new(std::size_t size, const std::nothrow_t&) throw() {
void* p = malloc(size);
recordAllocation(p, size);
return p;
}
void operator delete(void* ptr, const std::nothrow_t&) throw() {
recordDeallocation(ptr);
free(ptr);
}
// array throwing new and matching delete[]
void* operator new[](std::size_t size) {
void* p = malloc(size);
if (!p)
throw std::bad_alloc();
recordAllocation(p, size);
return p;
}
void operator delete[](void* ptr) throw() {
recordDeallocation(ptr);
free(ptr);
}
// array, nothrow new and matching delete[]
void* operator new[](std::size_t size, const std::nothrow_t&) throw() {
void* p = malloc(size);
recordAllocation(p, size);
return p;
}
void operator delete[](void* ptr, const std::nothrow_t&) throw() {
recordDeallocation(ptr);
free(ptr);
}
#endif
Optional<bool> checkBuggifyOverride(const char* testFile) {
std::ifstream ifs;
ifs.open(testFile, std::ifstream::in);
if (!ifs.good())
return 0;
std::string cline;
while (ifs.good()) {
getline(ifs, cline);
std::string line = removeWhitespace(std::string(cline));
if (!line.size() || line.find(';') == 0)
continue;
size_t found = line.find('=');
if (found == std::string::npos)
// hmmm, not good
continue;
std::string attrib = removeWhitespace(line.substr(0, found));
std::string value = removeWhitespace(line.substr(found + 1));
if (attrib == "buggify") {
// Testspec uses `on` or `off` (without quotes).
// TOML uses literal `true` and `false`.
if (!strcmp(value.c_str(), "on") || !strcmp(value.c_str(), "true")) {
ifs.close();
return true;
} else if (!strcmp(value.c_str(), "off") || !strcmp(value.c_str(), "false")) {
ifs.close();
return false;
} else {
fprintf(stderr, "ERROR: Unknown buggify override state `%s'\n", value.c_str());
flushAndExit(FDB_EXIT_ERROR);
}
}
}
ifs.close();
return Optional<bool>();
}
// Takes a vector of public and listen address strings given via command line, and returns vector of NetworkAddress
// objects.
std::pair<NetworkAddressList, NetworkAddressList> buildNetworkAddresses(
const IClusterConnectionRecord& connectionRecord,
const std::vector<std::string>& publicAddressStrs,
std::vector<std::string>& listenAddressStrs) {
if (listenAddressStrs.size() > 0 && publicAddressStrs.size() != listenAddressStrs.size()) {
fprintf(stderr,
"ERROR: Listen addresses (if provided) should be equal to the number of public addresses in order.\n");
flushAndExit(FDB_EXIT_ERROR);
}
listenAddressStrs.resize(publicAddressStrs.size(), "public");
if (publicAddressStrs.size() > 2) {
fprintf(stderr, "ERROR: maximum 2 public/listen addresses are allowed\n");
flushAndExit(FDB_EXIT_ERROR);
}
NetworkAddressList publicNetworkAddresses;
NetworkAddressList listenNetworkAddresses;
auto& coordinators = connectionRecord.getConnectionString().coordinators();
ASSERT(coordinators.size() > 0);
for (int ii = 0; ii < publicAddressStrs.size(); ++ii) {
const std::string& publicAddressStr = publicAddressStrs[ii];
bool autoPublicAddress = StringRef(publicAddressStr).startsWith(LiteralStringRef("auto:"));
NetworkAddress currentPublicAddress;
if (autoPublicAddress) {
try {
const NetworkAddress& parsedAddress = NetworkAddress::parse("0.0.0.0:" + publicAddressStr.substr(5));
const IPAddress publicIP = determinePublicIPAutomatically(connectionRecord.getConnectionString());
currentPublicAddress = NetworkAddress(publicIP, parsedAddress.port, true, parsedAddress.isTLS());
} catch (Error& e) {
fprintf(stderr,
"ERROR: could not determine public address automatically from `%s': %s\n",
publicAddressStr.c_str(),
e.what());
throw;
}
} else {
try {
currentPublicAddress = NetworkAddress::parse(publicAddressStr);
} catch (Error&) {
fprintf(stderr,
"ERROR: Could not parse network address `%s' (specify as IP_ADDRESS:PORT)\n",
publicAddressStr.c_str());
throw;
}
}
if (ii == 0) {
publicNetworkAddresses.address = currentPublicAddress;
} else {
publicNetworkAddresses.secondaryAddress = currentPublicAddress;
}
if (!currentPublicAddress.isValid()) {
fprintf(stderr, "ERROR: %s is not a valid IP address\n", currentPublicAddress.toString().c_str());
flushAndExit(FDB_EXIT_ERROR);
}
const std::string& listenAddressStr = listenAddressStrs[ii];
NetworkAddress currentListenAddress;
if (listenAddressStr == "public") {
currentListenAddress = currentPublicAddress;
} else {
try {
currentListenAddress = NetworkAddress::parse(listenAddressStr);
} catch (Error&) {
fprintf(stderr,
"ERROR: Could not parse network address `%s' (specify as IP_ADDRESS:PORT)\n",
listenAddressStr.c_str());
throw;
}
if (currentListenAddress.isTLS() != currentPublicAddress.isTLS()) {
fprintf(stderr,
"ERROR: TLS state of listen address: %s is not equal to the TLS state of public address: %s.\n",
listenAddressStr.c_str(),
publicAddressStr.c_str());
flushAndExit(FDB_EXIT_ERROR);
}
}
if (ii == 0) {
listenNetworkAddresses.address = currentListenAddress;
} else {
listenNetworkAddresses.secondaryAddress = currentListenAddress;
}
bool hasSameCoord = std::all_of(coordinators.begin(), coordinators.end(), [&](const NetworkAddress& address) {
if (address.ip == currentPublicAddress.ip && address.port == currentPublicAddress.port) {
return address.isTLS() == currentPublicAddress.isTLS();
}
return true;
});
if (!hasSameCoord) {
fprintf(stderr,
"ERROR: TLS state of public address %s does not match in coordinator list.\n",
publicAddressStr.c_str());
flushAndExit(FDB_EXIT_ERROR);
}
}
if (publicNetworkAddresses.secondaryAddress.present() &&
publicNetworkAddresses.address.isTLS() == publicNetworkAddresses.secondaryAddress.get().isTLS()) {
fprintf(stderr, "ERROR: only one public address of each TLS state is allowed.\n");
flushAndExit(FDB_EXIT_ERROR);
}
return std::make_pair(publicNetworkAddresses, listenNetworkAddresses);
}
// moves files from 'dirSrc' to 'dirToMove' if their name contains 'role'
void restoreRoleFilesHelper(std::string dirSrc, std::string dirToMove, std::string role) {
std::vector<std::string> returnFiles = platform::listFiles(dirSrc, "");
for (const auto& fileEntry : returnFiles) {
if (fileEntry != "fdb.cluster" && fileEntry.find(role) != std::string::npos) {
// rename files
TraceEvent("RenamingSnapFile")
.detail("Oldname", dirSrc + "/" + fileEntry)
.detail("Newname", dirToMove + "/" + fileEntry);
renameFile(dirSrc + "/" + fileEntry, dirToMove + "/" + fileEntry);
}
}
}
namespace {
enum class ServerRole {
ConsistencyCheck,
CreateTemplateDatabase,
DSLTest,
FDBD,
KVFileGenerateIOLogChecksums,
KVFileIntegrityCheck,
MultiTester,
NetworkTestClient,
NetworkTestServer,
Restore,
SearchMutations,
Simulation,
SkipListTest,
Test,
VersionedMapTest,
UnitTests
};
struct CLIOptions {
std::string commandLine;
std::string fileSystemPath, dataFolder, connFile, seedConnFile, seedConnString, logFolder = ".", metricsConnFile,
metricsPrefix;
std::string logGroup = "default";
uint64_t rollsize = TRACE_DEFAULT_ROLL_SIZE;
uint64_t maxLogsSize = TRACE_DEFAULT_MAX_LOGS_SIZE;
bool maxLogsSizeSet = false;
int maxLogs = 0;
bool maxLogsSet = false;
ServerRole role = ServerRole::FDBD;
uint32_t randomSeed = platform::getRandomSeed();
const char* testFile = "tests/default.txt";
std::string kvFile;
std::string testServersStr;
std::string whitelistBinPaths;
std::vector<std::string> publicAddressStrs, listenAddressStrs;
NetworkAddressList publicAddresses, listenAddresses;
const char* targetKey = nullptr;
int64_t memLimit =
8LL << 30; // Nice to maintain the same default value for memLimit and SERVER_KNOBS->SERVER_MEM_LIMIT and
// SERVER_KNOBS->COMMIT_BATCHES_MEM_BYTES_HARD_LIMIT
uint64_t storageMemLimit = 1LL << 30;
bool buggifyEnabled = false, faultInjectionEnabled = true, restarting = false;
Optional<Standalone<StringRef>> zoneId;
Optional<Standalone<StringRef>> dcId;
ProcessClass processClass = ProcessClass(ProcessClass::UnsetClass, ProcessClass::CommandLineSource);
bool useNet2 = true;
bool useThreadPool = false;
std::vector<std::pair<std::string, std::string>> knobs;
std::map<std::string, std::string> manualKnobOverrides;
LocalityData localities;
int minTesterCount = 1;
bool testOnServers = false;
TLSConfig tlsConfig = TLSConfig(TLSEndpointType::SERVER);
double fileIoTimeout = 0.0;
bool fileIoWarnOnly = false;
uint64_t rsssize = -1;
std::vector<std::string> blobCredentials; // used for fast restore workers & backup workers
const char* blobCredsFromENV = nullptr;
std::string configPath;
ConfigDBType configDBType{ ConfigDBType::DISABLED };
Reference<IClusterConnectionRecord> connectionFile;
Standalone<StringRef> machineId;
UnitTestParameters testParams;
std::map<std::string, std::string> profilerConfig;
bool printSimTime = false;
static CLIOptions parseArgs(int argc, char* argv[]) {
CLIOptions opts;
opts.parseArgsInternal(argc, argv);
return opts;
}
private:
CLIOptions() = default;
void parseArgsInternal(int argc, char* argv[]) {
for (int a = 0; a < argc; a++) {
if (a)
commandLine += ' ';
commandLine += argv[a];
}
CSimpleOpt args(argc, argv, g_rgOptions, SO_O_EXACT);
if (argc == 1) {
printUsage(argv[0], false);
flushAndExit(FDB_EXIT_ERROR);
}
while (args.Next()) {
if (args.LastError() == SO_ARG_INVALID_DATA) {
fprintf(stderr, "ERROR: invalid argument to option `%s'\n", args.OptionText());
printHelpTeaser(argv[0]);
flushAndExit(FDB_EXIT_ERROR);
}
if (args.LastError() == SO_ARG_INVALID) {
fprintf(stderr, "ERROR: argument given for option `%s'\n", args.OptionText());
printHelpTeaser(argv[0]);
flushAndExit(FDB_EXIT_ERROR);
}
if (args.LastError() == SO_ARG_MISSING) {
fprintf(stderr, "ERROR: missing argument for option `%s'\n", args.OptionText());
printHelpTeaser(argv[0]);
flushAndExit(FDB_EXIT_ERROR);
}
if (args.LastError() == SO_OPT_INVALID) {
fprintf(stderr, "ERROR: unknown option: `%s'\n", args.OptionText());
printHelpTeaser(argv[0]);
flushAndExit(FDB_EXIT_ERROR);
}
if (args.LastError() != SO_SUCCESS) {
fprintf(stderr, "ERROR: error parsing options\n");
printHelpTeaser(argv[0]);
flushAndExit(FDB_EXIT_ERROR);
}
const char* sRole;
Optional<uint64_t> ti;
std::string argStr;
std::vector<std::string> tmpStrings;
switch (args.OptionId()) {
case OPT_HELP:
printUsage(argv[0], false);
flushAndExit(FDB_EXIT_SUCCESS);
break;
case OPT_DEVHELP:
printUsage(argv[0], true);
flushAndExit(FDB_EXIT_SUCCESS);
break;
case OPT_KNOB: {
std::string syn = args.OptionSyntax();
if (!StringRef(syn).startsWith(LiteralStringRef("--knob_"))) {
fprintf(stderr, "ERROR: unable to parse knob option '%s'\n", syn.c_str());
flushAndExit(FDB_EXIT_ERROR);
}
syn = syn.substr(7);
knobs.emplace_back(syn, args.OptionArg());
manualKnobOverrides[syn] = args.OptionArg();
break;
}
case OPT_PROFILER: {
std::string syn = args.OptionSyntax();
std::string_view key = syn;
auto prefix = "--profiler_"sv;
if (key.find(prefix) != 0) {
fprintf(stderr, "ERROR: unable to parse profiler option '%s'\n", syn.c_str());
flushAndExit(FDB_EXIT_ERROR);
}
key.remove_prefix(prefix.size());
profilerConfig.emplace(key, args.OptionArg());
break;
};
case OPT_UNITTESTPARAM: {
std::string syn = args.OptionSyntax();
if (!StringRef(syn).startsWith(LiteralStringRef("--test_"))) {
fprintf(stderr, "ERROR: unable to parse knob option '%s'\n", syn.c_str());
flushAndExit(FDB_EXIT_ERROR);
}
testParams.set(syn.substr(7), args.OptionArg());
break;
}
case OPT_LOCALITY: {
std::string syn = args.OptionSyntax();
if (!StringRef(syn).startsWith(LiteralStringRef("--locality_"))) {
fprintf(stderr, "ERROR: unable to parse locality key '%s'\n", syn.c_str());
flushAndExit(FDB_EXIT_ERROR);
}
syn = syn.substr(11);
std::transform(syn.begin(), syn.end(), syn.begin(), ::tolower);
localities.set(Standalone<StringRef>(syn), Standalone<StringRef>(std::string(args.OptionArg())));
break;
}
case OPT_VERSION:
printVersion();
flushAndExit(FDB_EXIT_SUCCESS);
break;
case OPT_BUILD_FLAGS:
printBuildInformation();
flushAndExit(FDB_EXIT_SUCCESS);
case OPT_NOBUFSTDOUT:
setvbuf(stdout, nullptr, _IONBF, 0);
setvbuf(stderr, nullptr, _IONBF, 0);
break;
case OPT_BUFSTDOUTERR:
setvbuf(stdout, nullptr, _IOFBF, BUFSIZ);
setvbuf(stderr, nullptr, _IOFBF, BUFSIZ);
break;
case OPT_ROLE:
sRole = args.OptionArg();
if (!strcmp(sRole, "fdbd"))
role = ServerRole::FDBD;
else if (!strcmp(sRole, "simulation"))
role = ServerRole::Simulation;
else if (!strcmp(sRole, "test"))
role = ServerRole::Test;
else if (!strcmp(sRole, "multitest"))
role = ServerRole::MultiTester;
else if (!strcmp(sRole, "skiplisttest"))
role = ServerRole::SkipListTest;
else if (!strcmp(sRole, "search"))
role = ServerRole::SearchMutations;
else if (!strcmp(sRole, "dsltest"))
role = ServerRole::DSLTest;
else if (!strcmp(sRole, "versionedmaptest"))
role = ServerRole::VersionedMapTest;
else if (!strcmp(sRole, "createtemplatedb"))
role = ServerRole::CreateTemplateDatabase;
else if (!strcmp(sRole, "networktestclient"))
role = ServerRole::NetworkTestClient;
else if (!strcmp(sRole, "networktestserver"))
role = ServerRole::NetworkTestServer;
else if (!strcmp(sRole, "restore"))
role = ServerRole::Restore;
else if (!strcmp(sRole, "kvfileintegritycheck"))
role = ServerRole::KVFileIntegrityCheck;
else if (!strcmp(sRole, "kvfilegeneratesums"))
role = ServerRole::KVFileGenerateIOLogChecksums;
else if (!strcmp(sRole, "consistencycheck"))
role = ServerRole::ConsistencyCheck;
else if (!strcmp(sRole, "unittests"))
role = ServerRole::UnitTests;
else {
fprintf(stderr, "ERROR: Unknown role `%s'\n", sRole);
printHelpTeaser(argv[0]);
flushAndExit(FDB_EXIT_ERROR);
}
break;
case OPT_PUBLICADDR:
argStr = args.OptionArg();
boost::split(tmpStrings, argStr, [](char c) { return c == ','; });
publicAddressStrs.insert(publicAddressStrs.end(), tmpStrings.begin(), tmpStrings.end());
break;
case OPT_LISTEN:
argStr = args.OptionArg();
boost::split(tmpStrings, argStr, [](char c) { return c == ','; });
listenAddressStrs.insert(listenAddressStrs.end(), tmpStrings.begin(), tmpStrings.end());
break;
case OPT_CONNFILE:
connFile = args.OptionArg();
break;
case OPT_LOGGROUP:
logGroup = args.OptionArg();
break;
case OPT_SEEDCONNFILE:
seedConnFile = args.OptionArg();
break;
case OPT_SEEDCONNSTRING:
seedConnString = args.OptionArg();
break;
#ifdef __linux__
case OPT_FILESYSTEM: {
fileSystemPath = args.OptionArg();
break;
}
case OPT_PROFILER_RSS_SIZE: {
const char* a = args.OptionArg();
char* end;
rsssize = strtoull(a, &end, 10);
if (*end) {
fprintf(stderr, "ERROR: Unrecognized memory size `%s'\n", a);
printHelpTeaser(argv[0]);
flushAndExit(FDB_EXIT_ERROR);
}
break;
}
#endif
case OPT_DATAFOLDER:
dataFolder = args.OptionArg();
break;
case OPT_LOGFOLDER:
logFolder = args.OptionArg();
break;
case OPT_NETWORKIMPL: {
const char* a = args.OptionArg();
if (!strcmp(a, "net2"))
useNet2 = true;
else if (!strcmp(a, "net2-threadpool")) {
useNet2 = true;
useThreadPool = true;
} else {
fprintf(stderr, "ERROR: Unknown network implementation `%s'\n", a);
printHelpTeaser(argv[0]);
flushAndExit(FDB_EXIT_ERROR);
}
break;
}
case OPT_TRACECLOCK: {
const char* a = args.OptionArg();
if (!strcmp(a, "realtime"))
g_trace_clock.store(TRACE_CLOCK_REALTIME);
else if (!strcmp(a, "now"))
g_trace_clock.store(TRACE_CLOCK_NOW);
else {
fprintf(stderr, "ERROR: Unknown clock source `%s'\n", a);
printHelpTeaser(argv[0]);
flushAndExit(FDB_EXIT_ERROR);
}
break;
}
case OPT_NUMTESTERS: {
const char* a = args.OptionArg();
if (!sscanf(a, "%d", &minTesterCount)) {
fprintf(stderr, "ERROR: Could not parse numtesters `%s'\n", a);
printHelpTeaser(argv[0]);
flushAndExit(FDB_EXIT_ERROR);
}
break;
}
case OPT_ROLLSIZE: {
const char* a = args.OptionArg();
ti = parse_with_suffix(a);
if (!ti.present()) {
fprintf(stderr, "ERROR: Could not parse logsize `%s'\n", a);
printHelpTeaser(argv[0]);
flushAndExit(FDB_EXIT_ERROR);
}
rollsize = ti.get();
break;
}
case OPT_MAXLOGSSIZE: {
const char* a = args.OptionArg();
ti = parse_with_suffix(a);
if (!ti.present()) {
fprintf(stderr, "ERROR: Could not parse maxlogssize `%s'\n", a);
printHelpTeaser(argv[0]);
flushAndExit(FDB_EXIT_ERROR);
}
maxLogsSize = ti.get();
maxLogsSizeSet = true;
break;
}
case OPT_MAXLOGS: {
const char* a = args.OptionArg();
char* end;
maxLogs = strtoull(a, &end, 10);
if (*end) {
fprintf(stderr, "ERROR: Unrecognized maximum number of logs `%s'\n", a);
printHelpTeaser(argv[0]);
flushAndExit(FDB_EXIT_ERROR);
}
maxLogsSet = true;
break;
}
#ifdef _WIN32
case OPT_PARENTPID: {
auto pid_str = args.OptionArg();
int parent_pid = atoi(pid_str);
auto pHandle = OpenProcess(SYNCHRONIZE, FALSE, parent_pid);
if (!pHandle) {
TraceEvent("ParentProcessOpenError").GetLastError();
fprintf(stderr, "Could not open parent process at pid %d (error %d)", parent_pid, GetLastError());
throw platform_error();
}
startThread(&parentWatcher, pHandle);
break;
}
case OPT_NEWCONSOLE:
FreeConsole();
AllocConsole();
freopen("CONIN$", "rb", stdin);
freopen("CONOUT$", "wb", stdout);
freopen("CONOUT$", "wb", stderr);
break;
case OPT_NOBOX:
SetErrorMode(SetErrorMode(0) | SEM_NOGPFAULTERRORBOX);
break;
#else
case OPT_PARENTPID: {
auto pid_str = args.OptionArg();
int* parent_pid = new (int);
*parent_pid = atoi(pid_str);
startThread(&parentWatcher, parent_pid);
break;
}
#endif
case OPT_TRACER: {
std::string arg = args.OptionArg();
std::string tracer;
std::transform(arg.begin(), arg.end(), std::back_inserter(tracer), [](char c) { return tolower(c); });
if (tracer == "none" || tracer == "disabled") {
openTracer(TracerType::DISABLED);
} else if (tracer == "logfile" || tracer == "file" || tracer == "log_file") {
openTracer(TracerType::LOG_FILE);
} else if (tracer == "network_lossy") {
openTracer(TracerType::NETWORK_LOSSY);
} else {
fprintf(stderr, "ERROR: Unknown or unsupported tracer: `%s'", args.OptionArg());
printHelpTeaser(argv[0]);
flushAndExit(FDB_EXIT_ERROR);
}
break;
}
case OPT_TESTFILE:
testFile = args.OptionArg();
break;
case OPT_KVFILE:
kvFile = args.OptionArg();
break;
case OPT_RESTARTING:
restarting = true;
break;
case OPT_RANDOMSEED: {
char* end;
randomSeed = (uint32_t)strtoul(args.OptionArg(), &end, 0);
if (*end) {
fprintf(stderr, "ERROR: Could not parse random seed `%s'\n", args.OptionArg());
printHelpTeaser(argv[0]);
flushAndExit(FDB_EXIT_ERROR);
}
break;
}
case OPT_MACHINEID: {
zoneId = std::string(args.OptionArg());
break;
}
case OPT_DCID: {
dcId = std::string(args.OptionArg());
break;
}
case OPT_MACHINE_CLASS:
sRole = args.OptionArg();
processClass = ProcessClass(sRole, ProcessClass::CommandLineSource);
if (processClass == ProcessClass::InvalidClass) {
fprintf(stderr, "ERROR: Unknown machine class `%s'\n", sRole);
printHelpTeaser(argv[0]);
flushAndExit(FDB_EXIT_ERROR);
}
break;
case OPT_KEY:
targetKey = args.OptionArg();
break;
case OPT_MEMLIMIT:
ti = parse_with_suffix(args.OptionArg(), "MiB");
if (!ti.present()) {
fprintf(stderr, "ERROR: Could not parse memory limit from `%s'\n", args.OptionArg());
printHelpTeaser(argv[0]);
flushAndExit(FDB_EXIT_ERROR);
}
memLimit = ti.get();
break;
case OPT_STORAGEMEMLIMIT:
ti = parse_with_suffix(args.OptionArg(), "MB");
if (!ti.present()) {
fprintf(stderr, "ERROR: Could not parse storage memory limit from `%s'\n", args.OptionArg());
printHelpTeaser(argv[0]);
flushAndExit(FDB_EXIT_ERROR);
}
storageMemLimit = ti.get();
break;
case OPT_CACHEMEMLIMIT:
ti = parse_with_suffix(args.OptionArg(), "MiB");
if (!ti.present()) {
fprintf(stderr, "ERROR: Could not parse cache memory limit from `%s'\n", args.OptionArg());
printHelpTeaser(argv[0]);
flushAndExit(FDB_EXIT_ERROR);
}
// SOMEDAY: ideally we'd have some better way to express that a knob should be elevated to formal
// parameter
knobs.emplace_back(
"page_cache_4k",
format("%ld", ti.get() / 4096 * 4096)); // The cache holds 4K pages, so we can truncate this to the
// next smaller multiple of 4K.
break;
case OPT_BUGGIFY:
if (!strcmp(args.OptionArg(), "on"))
buggifyEnabled = true;
else if (!strcmp(args.OptionArg(), "off"))
buggifyEnabled = false;
else {
fprintf(stderr, "ERROR: Unknown buggify state `%s'\n", args.OptionArg());
printHelpTeaser(argv[0]);
flushAndExit(FDB_EXIT_ERROR);
}
break;
case OPT_FAULT_INJECTION:
if (!strcmp(args.OptionArg(), "on"))
faultInjectionEnabled = true;
else if (!strcmp(args.OptionArg(), "off"))
faultInjectionEnabled = false;
else {
fprintf(stderr, "ERROR: Unknown fault injection state `%s'\n", args.OptionArg());
printHelpTeaser(argv[0]);
flushAndExit(FDB_EXIT_ERROR);
}
break;
case OPT_CRASHONERROR:
g_crashOnError = true;
break;
case OPT_TESTSERVERS:
testServersStr = args.OptionArg();
break;
case OPT_TEST_ON_SERVERS:
testOnServers = true;
break;
case OPT_METRICSCONNFILE:
metricsConnFile = args.OptionArg();
break;
case OPT_METRICSPREFIX:
metricsPrefix = args.OptionArg();
break;
case OPT_IO_TRUST_SECONDS: {
const char* a = args.OptionArg();
if (!sscanf(a, "%lf", &fileIoTimeout)) {
fprintf(stderr, "ERROR: Could not parse io_trust_seconds `%s'\n", a);
printHelpTeaser(argv[0]);
flushAndExit(FDB_EXIT_ERROR);
}
break;
}
case OPT_IO_TRUST_WARN_ONLY:
fileIoWarnOnly = true;
break;
case OPT_TRACE_FORMAT:
if (!selectTraceFormatter(args.OptionArg())) {
fprintf(stderr, "WARNING: Unrecognized trace format `%s'\n", args.OptionArg());
}
break;
case OPT_WHITELIST_BINPATH:
whitelistBinPaths = args.OptionArg();
break;
case OPT_BLOB_CREDENTIAL_FILE:
// Add blob credential following backup agent example
blobCredentials.push_back(args.OptionArg());
printf("blob credential file:%s\n", blobCredentials.back().c_str());
blobCredsFromENV = getenv("FDB_BLOB_CREDENTIALS");
if (blobCredsFromENV != nullptr) {
fprintf(stderr, "[WARNING] Set blob credetial via env variable is not tested yet\n");
TraceEvent(SevError, "FastRestoreGetBlobCredentialFile")
.detail("Reason", "Set blob credetial via env variable is not tested yet");
StringRef t((uint8_t*)blobCredsFromENV, strlen(blobCredsFromENV));
do {
StringRef file = t.eat(":");
if (file.size() != 0) {
blobCredentials.push_back(file.toString());
}
} while (t.size() != 0);
}
break;
case OPT_CONFIG_PATH:
configPath = args.OptionArg();
break;
case OPT_USE_TEST_CONFIG_DB:
configDBType = ConfigDBType::SIMPLE;
break;
case OPT_PRINT_SIMTIME:
printSimTime = true;
break;
#ifndef TLS_DISABLED
case TLSConfig::OPT_TLS_PLUGIN:
args.OptionArg();
break;
case TLSConfig::OPT_TLS_CERTIFICATES:
tlsConfig.setCertificatePath(args.OptionArg());
break;
case TLSConfig::OPT_TLS_PASSWORD:
tlsConfig.setPassword(args.OptionArg());
break;
case TLSConfig::OPT_TLS_CA_FILE:
tlsConfig.setCAPath(args.OptionArg());
break;
case TLSConfig::OPT_TLS_KEY:
tlsConfig.setKeyPath(args.OptionArg());
break;
case TLSConfig::OPT_TLS_VERIFY_PEERS:
tlsConfig.addVerifyPeers(args.OptionArg());
break;
#endif
}
}
try {
ProfilerConfig::instance().reset(profilerConfig);
} catch (ConfigError& e) {
printf("Error seting up profiler: %s", e.description.c_str());
flushAndExit(FDB_EXIT_ERROR);
}
if (seedConnString.length() && seedConnFile.length()) {
fprintf(
stderr, "%s\n", "--seed_cluster_file and --seed_connection_string may not both be specified at once.");
flushAndExit(FDB_EXIT_ERROR);
}
bool seedSpecified = seedConnFile.length() || seedConnString.length();
if (seedSpecified && !connFile.length()) {
fprintf(stderr,
"%s\n",
"If -seed_cluster_file or --seed_connection_string is specified, -C must be specified as well.");
flushAndExit(FDB_EXIT_ERROR);
}
if (metricsConnFile == connFile)
metricsConnFile = "";
if (metricsConnFile != "" && metricsPrefix == "") {
fprintf(stderr, "If a metrics cluster file is specified, a metrics prefix is required.\n");
flushAndExit(FDB_EXIT_ERROR);
}
bool autoPublicAddress =
std::any_of(publicAddressStrs.begin(), publicAddressStrs.end(), [](const std::string& addr) {
return StringRef(addr).startsWith(LiteralStringRef("auto:"));
});
if ((role != ServerRole::Simulation && role != ServerRole::CreateTemplateDatabase &&
role != ServerRole::KVFileIntegrityCheck && role != ServerRole::KVFileGenerateIOLogChecksums &&
role != ServerRole::UnitTests) ||
autoPublicAddress) {
if (seedSpecified && !fileExists(connFile)) {
std::string connectionString = seedConnString.length() ? seedConnString : "";
ClusterConnectionString ccs;
if (seedConnFile.length()) {
try {
connectionString = readFileBytes(seedConnFile, MAX_CLUSTER_FILE_BYTES);
} catch (Error& e) {
fprintf(stderr,
"%s\n",
ClusterConnectionFile::getErrorString(std::make_pair(seedConnFile, false), e).c_str());
throw;
}
}
try {
ccs = ClusterConnectionString(connectionString);
} catch (Error& e) {
fprintf(stderr, "%s\n", ClusterConnectionString::getErrorString(connectionString, e).c_str());
throw;
}
connectionFile = makeReference<ClusterConnectionFile>(connFile, ccs);
} else {
std::pair<std::string, bool> resolvedClusterFile;
try {
resolvedClusterFile = ClusterConnectionFile::lookupClusterFileName(connFile);
connectionFile = makeReference<ClusterConnectionFile>(resolvedClusterFile.first);
} catch (Error& e) {
fprintf(stderr, "%s\n", ClusterConnectionFile::getErrorString(resolvedClusterFile, e).c_str());
throw;
}
}
// failmon?
}
try {
if (!publicAddressStrs.empty()) {
std::tie(publicAddresses, listenAddresses) =
buildNetworkAddresses(*connectionFile, publicAddressStrs, listenAddressStrs);
}
} catch (Error&) {
printHelpTeaser(argv[0]);
flushAndExit(FDB_EXIT_ERROR);
}
if (role == ServerRole::ConsistencyCheck) {
if (!publicAddressStrs.empty()) {
fprintf(stderr, "ERROR: Public address cannot be specified for consistency check processes\n");
printHelpTeaser(argv[0]);
flushAndExit(FDB_EXIT_ERROR);
}
auto publicIP = determinePublicIPAutomatically(connectionFile->getConnectionString());
publicAddresses.address = NetworkAddress(publicIP, ::getpid());
}
if (role == ServerRole::Simulation) {
Optional<bool> buggifyOverride = checkBuggifyOverride(testFile);
if (buggifyOverride.present())
buggifyEnabled = buggifyOverride.get();
}
if (role == ServerRole::SearchMutations && !targetKey) {
fprintf(stderr, "ERROR: please specify a target key\n");
printHelpTeaser(argv[0]);
flushAndExit(FDB_EXIT_ERROR);
}
if (role == ServerRole::NetworkTestClient && !testServersStr.size()) {
fprintf(stderr, "ERROR: please specify --testservers\n");
printHelpTeaser(argv[0]);
flushAndExit(FDB_EXIT_ERROR);
}
// Interpret legacy "maxLogs" option in the most sensible and unsurprising way we can while eliminating its code
// path
if (maxLogsSet) {
if (maxLogsSizeSet) {
// This is the case where both options are set and we must deconflict.
auto maxLogsAsSize = maxLogs * rollsize;
// If either was unlimited, then the safe option here is to take the larger one.
// This means that is one of the two options specified a limited amount of logging
// then the option that specified "unlimited" will be ignored.
if (maxLogsSize == 0 || maxLogs == 0)
maxLogsSize = std::max(maxLogsSize, maxLogsAsSize);
else
maxLogsSize = std::min(maxLogsSize, maxLogs * rollsize);
} else {
maxLogsSize = maxLogs * rollsize;
}
}
if (!zoneId.present() &&
!(localities.isPresent(LocalityData::keyZoneId) && localities.isPresent(LocalityData::keyMachineId))) {
machineId = getSharedMemoryMachineId().toString();
}
if (!localities.isPresent(LocalityData::keyZoneId))
localities.set(LocalityData::keyZoneId, zoneId.present() ? zoneId : machineId);
if (!localities.isPresent(LocalityData::keyMachineId))
localities.set(LocalityData::keyMachineId, zoneId.present() ? zoneId : machineId);
if (!localities.isPresent(LocalityData::keyDcId) && dcId.present())
localities.set(LocalityData::keyDcId, dcId);
}
};
} // namespace
int main(int argc, char* argv[]) {
// TODO: Remove later, this is just to force the statics to be initialized
// otherwise the unit test won't run
#ifdef ENABLE_SAMPLING
ActorLineageSet _;
#endif
try {
platformInit();
#ifdef ALLOC_INSTRUMENTATION
g_extra_memory = new uint8_t[1000000];
#endif
registerCrashHandler();
// Set default of line buffering standard out and error
setvbuf(stdout, nullptr, _IOLBF, BUFSIZ);
setvbuf(stderr, nullptr, _IOLBF, BUFSIZ);
// Enables profiling on this thread (but does not start it)
registerThreadForProfiling();
#ifdef _WIN32
// Windows needs a gentle nudge to format floats correctly
//_set_output_format(_TWO_DIGIT_EXPONENT);
#endif
const auto opts = CLIOptions::parseArgs(argc, argv);
const auto role = opts.role;
#ifdef _WIN32
// For now, ignore all tests for Windows
if (role == ServerRole::Simulation || role == ServerRole::UnitTests || role == ServerRole::Test) {
printf("Windows tests are not supported yet\n");
flushAndExit(FDB_EXIT_SUCCESS);
}
#endif
if (role == ServerRole::Simulation)
printf("Random seed is %u...\n", opts.randomSeed);
if (opts.zoneId.present())
printf("ZoneId set to %s, dcId to %s\n", printable(opts.zoneId).c_str(), printable(opts.dcId).c_str());
setThreadLocalDeterministicRandomSeed(opts.randomSeed);
enableBuggify(opts.buggifyEnabled, BuggifyType::General);
enableFaultInjection(opts.faultInjectionEnabled);
IKnobCollection::setGlobalKnobCollection(IKnobCollection::Type::SERVER,
Randomize::True,
role == ServerRole::Simulation ? IsSimulated::True
: IsSimulated::False);
IKnobCollection::getMutableGlobalKnobCollection().setKnob("log_directory", KnobValue::create(opts.logFolder));
if (role != ServerRole::Simulation) {
IKnobCollection::getMutableGlobalKnobCollection().setKnob("commit_batches_mem_bytes_hard_limit",
KnobValue::create(int64_t{ opts.memLimit }));
}
for (const auto& [knobName, knobValueString] : opts.knobs) {
try {
auto& g_knobs = IKnobCollection::getMutableGlobalKnobCollection();
auto knobValue = g_knobs.parseKnobValue(knobName, knobValueString);
g_knobs.setKnob(knobName, knobValue);
} catch (Error& e) {
if (e.code() == error_code_invalid_option_value) {
fprintf(stderr,
"WARNING: Invalid value '%s' for knob option '%s'\n",
knobName.c_str(),
knobValueString.c_str());
TraceEvent(SevWarnAlways, "InvalidKnobValue")
.detail("Knob", printable(knobName))
.detail("Value", printable(knobValueString));
} else {
fprintf(stderr, "ERROR: Failed to set knob option '%s': %s\n", knobName.c_str(), e.what());
TraceEvent(SevError, "FailedToSetKnob")
.detail("Knob", printable(knobName))
.detail("Value", printable(knobValueString))
.error(e);
throw;
}
}
}
IKnobCollection::getMutableGlobalKnobCollection().setKnob("server_mem_limit",
KnobValue::create(int64_t{ opts.memLimit }));
// Reinitialize knobs in order to update knobs that are dependent on explicitly set knobs
IKnobCollection::getMutableGlobalKnobCollection().initialize(
Randomize::True, role == ServerRole::Simulation ? IsSimulated::True : IsSimulated::False);
// evictionPolicyStringToEnum will throw an exception if the string is not recognized as a valid
EvictablePageCache::evictionPolicyStringToEnum(FLOW_KNOBS->CACHE_EVICTION_POLICY);
if (opts.memLimit <= FLOW_KNOBS->PAGE_CACHE_4K) {
fprintf(stderr, "ERROR: --memory has to be larger than --cache_memory\n");
flushAndExit(FDB_EXIT_ERROR);
}
if (role == ServerRole::SkipListTest) {
skipListTest();
flushAndExit(FDB_EXIT_SUCCESS);
}
if (role == ServerRole::DSLTest) {
dsltest();
flushAndExit(FDB_EXIT_SUCCESS);
}
if (role == ServerRole::VersionedMapTest) {
versionedMapTest();
flushAndExit(FDB_EXIT_SUCCESS);
}
// Initialize the thread pool
CoroThreadPool::init();
// Ordinarily, this is done when the network is run. However, network thread should be set before TraceEvents
// are logged. This thread will eventually run the network, so call it now.
TraceEvent::setNetworkThread();
std::vector<Future<Void>> listenErrors;
if (role == ServerRole::Simulation || role == ServerRole::CreateTemplateDatabase) {
// startOldSimulator();
startNewSimulator(opts.printSimTime);
openTraceFile(NetworkAddress(), opts.rollsize, opts.maxLogsSize, opts.logFolder, "trace", opts.logGroup);
openTracer(TracerType(deterministicRandom()->randomInt(static_cast<int>(TracerType::DISABLED),
static_cast<int>(TracerType::SIM_END))));
} else {
g_network = newNet2(opts.tlsConfig, opts.useThreadPool, true);
g_network->addStopCallback(Net2FileSystem::stop);
FlowTransport::createInstance(false, 1, WLTOKEN_RESERVED_COUNT);
const bool expectsPublicAddress =
(role == ServerRole::FDBD || role == ServerRole::NetworkTestServer || role == ServerRole::Restore);
if (opts.publicAddressStrs.empty()) {
if (expectsPublicAddress) {
fprintf(stderr, "ERROR: The -p or --public_address option is required\n");
printHelpTeaser(argv[0]);
flushAndExit(FDB_EXIT_ERROR);
}
}
openTraceFile(
opts.publicAddresses.address, opts.rollsize, opts.maxLogsSize, opts.logFolder, "trace", opts.logGroup);
g_network->initTLS();
if (expectsPublicAddress) {
for (int ii = 0; ii < (opts.publicAddresses.secondaryAddress.present() ? 2 : 1); ++ii) {
const NetworkAddress& publicAddress =
ii == 0 ? opts.publicAddresses.address : opts.publicAddresses.secondaryAddress.get();
const NetworkAddress& listenAddress =
ii == 0 ? opts.listenAddresses.address : opts.listenAddresses.secondaryAddress.get();
try {
const Future<Void>& errorF = FlowTransport::transport().bind(publicAddress, listenAddress);
listenErrors.push_back(errorF);
if (errorF.isReady())
errorF.get();
} catch (Error& e) {
TraceEvent("BindError").error(e);
fprintf(stderr,
"Error initializing networking with public address %s and listen address %s (%s)\n",
publicAddress.toString().c_str(),
listenAddress.toString().c_str(),
e.what());
printHelpTeaser(argv[0]);
flushAndExit(FDB_EXIT_ERROR);
}
}
}
// Use a negative ioTimeout to indicate warn-only
Net2FileSystem::newFileSystem(opts.fileIoWarnOnly ? -opts.fileIoTimeout : opts.fileIoTimeout,
opts.fileSystemPath);
g_network->initMetrics();
FlowTransport::transport().initMetrics();
initTraceEventMetrics();
}
double start = timer(), startNow = now();
std::string cwd = "<unknown>";
try {
cwd = platform::getWorkingDirectory();
} catch (Error& e) {
// Allow for platform error by rethrowing all _other_ errors
if (e.code() != error_code_platform_error)
throw;
}
TraceEvent("ProgramStart")
.setMaxEventLength(12000)
.detail("RandomSeed", opts.randomSeed)
.detail("SourceVersion", getSourceVersion())
.detail("Version", FDB_VT_VERSION)
.detail("PackageName", FDB_VT_PACKAGE_NAME)
.detail("FileSystem", opts.fileSystemPath)
.detail("DataFolder", opts.dataFolder)
.detail("WorkingDirectory", cwd)
.detail("ClusterFile", opts.connectionFile ? opts.connectionFile->toString() : "")
.detail("ConnectionString",
opts.connectionFile ? opts.connectionFile->getConnectionString().toString() : "")
.detailf("ActualTime", "%lld", DEBUG_DETERMINISM ? 0 : time(nullptr))
.setMaxFieldLength(10000)
.detail("CommandLine", opts.commandLine)
.setMaxFieldLength(0)
.detail("BuggifyEnabled", opts.buggifyEnabled)
.detail("FaultInjectionEnabled", opts.faultInjectionEnabled)
.detail("MemoryLimit", opts.memLimit)
.trackLatest("ProgramStart");
Error::init();
std::set_new_handler(&platform::outOfMemory);
setMemoryQuota(opts.memLimit);
Future<Optional<Void>> f;
if (role == ServerRole::Simulation) {
TraceEvent("Simulation").detail("TestFile", opts.testFile);
auto histogramReportActor = histogramReport();
CLIENT_KNOBS->trace();
FLOW_KNOBS->trace();
SERVER_KNOBS->trace();
auto dataFolder = opts.dataFolder.size() ? opts.dataFolder : "simfdb";
std::vector<std::string> directories = platform::listDirectories(dataFolder);
const std::set<std::string> allowedDirectories = { ".", "..", "backups", "unittests" };
for (const auto& dir : directories) {
if (dir.size() != 32 && allowedDirectories.count(dir) == 0 && dir.find("snap") == std::string::npos) {
TraceEvent(SevError, "IncompatibleDirectoryFound")
.detail("DataFolder", dataFolder)
.detail("SuspiciousFile", dir);
fprintf(stderr,
"ERROR: Data folder `%s' had non fdb file `%s'; please use clean, fdb-only folder\n",
dataFolder.c_str(),
dir.c_str());
flushAndExit(FDB_EXIT_ERROR);
}
}
std::vector<std::string> files = platform::listFiles(dataFolder);
if ((files.size() > 1 || (files.size() == 1 && files[0] != "restartInfo.ini")) && !opts.restarting) {
TraceEvent(SevError, "IncompatibleFileFound").detail("DataFolder", dataFolder);
fprintf(stderr,
"ERROR: Data folder `%s' is non-empty; please use clean, fdb-only folder\n",
dataFolder.c_str());
flushAndExit(FDB_EXIT_ERROR);
} else if (files.empty() && opts.restarting) {
TraceEvent(SevWarnAlways, "FileNotFound").detail("DataFolder", dataFolder);
printf("ERROR: Data folder `%s' is empty, but restarting option selected. Run Phase 1 test first\n",
dataFolder.c_str());
flushAndExit(FDB_EXIT_ERROR);
}
int isRestoring = 0;
if (!opts.restarting) {
platform::eraseDirectoryRecursive(dataFolder);
platform::createDirectory(dataFolder);
} else {
CSimpleIni ini;
ini.SetUnicode();
std::string absDataFolder = abspath(dataFolder);
ini.LoadFile(joinPath(absDataFolder, "restartInfo.ini").c_str());
int backupFailed = true;
const char* isRestoringStr = ini.GetValue("RESTORE", "isRestoring", nullptr);
if (isRestoringStr) {
isRestoring = atoi(isRestoringStr);
const char* backupFailedStr = ini.GetValue("RESTORE", "BackupFailed", nullptr);
if (isRestoring && backupFailedStr) {
backupFailed = atoi(backupFailedStr);
}
}
if (isRestoring && !backupFailed) {
std::vector<std::string> returnList;
std::string ext = "";
returnList = platform::listDirectories(absDataFolder);
std::string snapStr = ini.GetValue("RESTORE", "RestoreSnapUID");
TraceEvent("RestoringDataFolder").detail("DataFolder", absDataFolder);
TraceEvent("RestoreSnapUID").detail("UID", snapStr);
// delete all files (except fdb.cluster) in non-snap directories
for (const auto& dirEntry : returnList) {
if (dirEntry == "." || dirEntry == "..") {
continue;
}
if (dirEntry.find(snapStr) != std::string::npos) {
continue;
}
std::string childf = absDataFolder + "/" + dirEntry;
std::vector<std::string> returnFiles = platform::listFiles(childf, ext);
for (const auto& fileEntry : returnFiles) {
if (fileEntry != "fdb.cluster" && fileEntry != "fitness") {
TraceEvent("DeletingNonSnapfiles").detail("FileBeingDeleted", childf + "/" + fileEntry);
deleteFile(childf + "/" + fileEntry);
}
}
}
// cleanup unwanted and partial directories
for (const auto& dirEntry : returnList) {
if (dirEntry == "." || dirEntry == "..") {
continue;
}
std::string dirSrc = absDataFolder + "/" + dirEntry;
// delete snap directories which are not part of restoreSnapUID
if (dirEntry.find(snapStr) == std::string::npos) {
if (dirEntry.find("snap") != std::string::npos) {
platform::eraseDirectoryRecursive(dirSrc);
}
continue;
}
// remove empty/partial snap directories
std::vector<std::string> childrenList = platform::listFiles(dirSrc);
if (childrenList.size() == 0) {
TraceEvent("RemovingEmptySnapDirectory").detail("DirBeingDeleted", dirSrc);
platform::eraseDirectoryRecursive(dirSrc);
continue;
}
}
// move snapshotted files to appropriate locations
for (const auto& dirEntry : returnList) {
if (dirEntry == "." || dirEntry == "..") {
continue;
}
std::string dirSrc = absDataFolder + "/" + dirEntry;
std::string origDir = dirEntry.substr(0, 32);
std::string dirToMove = absDataFolder + "/" + origDir;
if ((dirEntry.find("snap") != std::string::npos) &&
(dirEntry.find("tlog") != std::string::npos)) {
// restore tlog files
restoreRoleFilesHelper(dirSrc, dirToMove, "log");
} else if ((dirEntry.find("snap") != std::string::npos) &&
(dirEntry.find("storage") != std::string::npos)) {
// restore storage files
restoreRoleFilesHelper(dirSrc, dirToMove, "storage");
} else if ((dirEntry.find("snap") != std::string::npos) &&
(dirEntry.find("coord") != std::string::npos)) {
// restore coordinator files
restoreRoleFilesHelper(dirSrc, dirToMove, "coordination");
}
}
}
}
setupAndRun(dataFolder, opts.testFile, opts.restarting, (isRestoring >= 1), opts.whitelistBinPaths);
g_simulator.run();
} else if (role == ServerRole::FDBD) {
// Update the global blob credential files list so that both fast
// restore workers and backup workers can access blob storage.
std::vector<std::string>* pFiles =
(std::vector<std::string>*)g_network->global(INetwork::enBlobCredentialFiles);
if (pFiles != nullptr) {
for (auto& f : opts.blobCredentials) {
pFiles->push_back(f);
}
}
// Call fast restore for the class FastRestoreClass. This is a short-cut to run fast restore in circus
if (opts.processClass == ProcessClass::FastRestoreClass) {
printf("Run as fast restore worker\n");
ASSERT(opts.connectionFile);
auto dataFolder = opts.dataFolder;
if (!dataFolder.size())
dataFolder = format("fdb/%d/", opts.publicAddresses.address.port); // SOMEDAY: Better default
std::vector<Future<Void>> actors(listenErrors.begin(), listenErrors.end());
actors.push_back(restoreWorker(opts.connectionFile, opts.localities, dataFolder));
f = stopAfter(waitForAll(actors));
printf("Fast restore worker started\n");
g_network->run();
printf("g_network->run() done\n");
} else { // Call fdbd roles in conventional way
ASSERT(opts.connectionFile);
setupRunLoopProfiler();
auto dataFolder = opts.dataFolder;
if (!dataFolder.size())
dataFolder = format("fdb/%d/", opts.publicAddresses.address.port); // SOMEDAY: Better default
std::vector<Future<Void>> actors(listenErrors.begin(), listenErrors.end());
actors.push_back(fdbd(opts.connectionFile,
opts.localities,
opts.processClass,
dataFolder,
dataFolder,
opts.storageMemLimit,
opts.metricsConnFile,
opts.metricsPrefix,
opts.rsssize,
opts.whitelistBinPaths,
opts.configPath,
opts.manualKnobOverrides,
opts.configDBType));
actors.push_back(histogramReport());
// actors.push_back( recurring( []{}, .001 ) ); // for ASIO latency measurement
f = stopAfter(waitForAll(actors));
g_network->run();
}
} else if (role == ServerRole::MultiTester) {
setupRunLoopProfiler();
f = stopAfter(runTests(opts.connectionFile,
TEST_TYPE_FROM_FILE,
opts.testOnServers ? TEST_ON_SERVERS : TEST_ON_TESTERS,
opts.minTesterCount,
opts.testFile,
StringRef(),
opts.localities));
g_network->run();
} else if (role == ServerRole::Test) {
setupRunLoopProfiler();
auto m = startSystemMonitor(opts.dataFolder, opts.dcId, opts.zoneId, opts.zoneId);
f = stopAfter(runTests(
opts.connectionFile, TEST_TYPE_FROM_FILE, TEST_HERE, 1, opts.testFile, StringRef(), opts.localities));
g_network->run();
} else if (role == ServerRole::ConsistencyCheck) {
setupRunLoopProfiler();
auto m = startSystemMonitor(opts.dataFolder, opts.dcId, opts.zoneId, opts.zoneId);
f = stopAfter(runTests(opts.connectionFile,
TEST_TYPE_CONSISTENCY_CHECK,
TEST_HERE,
1,
opts.testFile,
StringRef(),
opts.localities));
g_network->run();
} else if (role == ServerRole::UnitTests) {
setupRunLoopProfiler();
auto m = startSystemMonitor(opts.dataFolder, opts.dcId, opts.zoneId, opts.zoneId);
f = stopAfter(runTests(opts.connectionFile,
TEST_TYPE_UNIT_TESTS,
TEST_HERE,
1,
opts.testFile,
StringRef(),
opts.localities,
opts.testParams));
g_network->run();
} else if (role == ServerRole::CreateTemplateDatabase) {
createTemplateDatabase();
} else if (role == ServerRole::NetworkTestClient) {
f = stopAfter(networkTestClient(opts.testServersStr));
g_network->run();
} else if (role == ServerRole::NetworkTestServer) {
f = stopAfter(networkTestServer());
g_network->run();
} else if (role == ServerRole::Restore) {
f = stopAfter(restoreWorker(opts.connectionFile, opts.localities, opts.dataFolder));
g_network->run();
} else if (role == ServerRole::KVFileIntegrityCheck) {
f = stopAfter(KVFileCheck(opts.kvFile, true));
g_network->run();
} else if (role == ServerRole::KVFileGenerateIOLogChecksums) {
Optional<Void> result;
try {
GenerateIOLogChecksumFile(opts.kvFile);
result = Void();
} catch (Error& e) {
fprintf(stderr, "Fatal Error: %s\n", e.what());
}
f = result;
}
int rc = FDB_EXIT_SUCCESS;
if (f.isValid() && f.isReady() && !f.isError() && !f.get().present()) {
rc = FDB_EXIT_ERROR;
}
int unseed = noUnseed ? 0 : deterministicRandom()->randomInt(0, 100001);
TraceEvent("ElapsedTime")
.detail("SimTime", now() - startNow)
.detail("RealTime", timer() - start)
.detail("RandomUnseed", unseed);
if (role == ServerRole::Simulation) {
printf("Unseed: %d\n", unseed);
printf("Elapsed: %f simsec, %f real seconds\n", now() - startNow, timer() - start);
}
// IFailureMonitor::failureMonitor().address_info.clear();
// we should have shut down ALL actors associated with this machine; let's list all of the ones still live
/*{
auto living = Actor::all;
printf("%d surviving actors:\n", living.size());
for(auto a = living.begin(); a != living.end(); ++a)
printf(" #%lld %s %p\n", (*a)->creationIndex, (*a)->getName(), (*a));
}
{
auto living = DatabaseContext::all;
printf("%d surviving DatabaseContexts:\n", living.size());
for(auto a = living.begin(); a != living.end(); ++a)
printf(" #%lld %p\n", (*a)->creationIndex, (*a));
}
{
auto living = TransactionData::all;
printf("%d surviving TransactionData(s):\n", living.size());
for(auto a = living.begin(); a != living.end(); ++a)
printf(" #%lld %p\n", (*a)->creationIndex, (*a));
}*/
/*cout << Actor::allActors.size() << " surviving actors:" << std::endl;
std::map<std::string,int> actorCount;
for(int i=0; i<Actor::allActors.size(); i++)
++actorCount[Actor::allActors[i]->getName()];
for(auto i = actorCount.rbegin(); !(i == actorCount.rend()); ++i)
std::cout << " " << i->second << " " << i->first << std::endl;*/
// std::cout << " " << Actor::allActors[i]->getName() << std::endl;
if (role == ServerRole::Simulation) {
unsigned long sevErrorEventsLogged = TraceEvent::CountEventsLoggedAt(SevError);
if (sevErrorEventsLogged > 0) {
printf("%lu SevError events logged\n", sevErrorEventsLogged);
rc = FDB_EXIT_ERROR;
}
}
// g_simulator.run();
#ifdef ALLOC_INSTRUMENTATION
{
std::cout << "Page Counts: " << FastAllocator<16>::pageCount << " " << FastAllocator<32>::pageCount << " "
<< FastAllocator<64>::pageCount << " " << FastAllocator<128>::pageCount << " "
<< FastAllocator<256>::pageCount << " " << FastAllocator<512>::pageCount << " "
<< FastAllocator<1024>::pageCount << " " << FastAllocator<2048>::pageCount << " "
<< FastAllocator<4096>::pageCount << " " << FastAllocator<8192>::pageCount << " "
<< FastAllocator<16384>::pageCount << std::endl;
std::vector<std::pair<std::string, const char*>> typeNames;
for (auto i = allocInstr.begin(); i != allocInstr.end(); ++i) {
std::string s;
#ifdef __linux__
char* demangled = abi::__cxa_demangle(i->first, nullptr, nullptr, nullptr);
if (demangled) {
s = demangled;
if (StringRef(s).startsWith(LiteralStringRef("(anonymous namespace)::")))
s = s.substr(LiteralStringRef("(anonymous namespace)::").size());
free(demangled);
} else
s = i->first;
#else
s = i->first;
if (StringRef(s).startsWith(LiteralStringRef("class `anonymous namespace'::")))
s = s.substr(LiteralStringRef("class `anonymous namespace'::").size());
else if (StringRef(s).startsWith(LiteralStringRef("class ")))
s = s.substr(LiteralStringRef("class ").size());
else if (StringRef(s).startsWith(LiteralStringRef("struct ")))
s = s.substr(LiteralStringRef("struct ").size());
#endif
typeNames.emplace_back(s, i->first);
}
std::sort(typeNames.begin(), typeNames.end());
for (int i = 0; i < typeNames.size(); i++) {
const char* n = typeNames[i].second;
auto& f = allocInstr[n];
printf("%+d\t%+d\t%d\t%d\t%s\n",
f.allocCount,
-f.deallocCount,
f.allocCount - f.deallocCount,
f.maxAllocated,
typeNames[i].first.c_str());
}
// We're about to exit and clean up data structures, this will wreak havoc on allocation recording
memSample_entered = true;
}
#endif
// printf("\n%d tests passed; %d tests failed\n", passCount, failCount);
flushAndExit(rc);
} catch (Error& e) {
fprintf(stderr, "Error: %s\n", e.what());
TraceEvent(SevError, "MainError").error(e);
// printf("\n%d tests passed; %d tests failed\n", passCount, failCount);
flushAndExit(FDB_EXIT_MAIN_ERROR);
} catch (boost::system::system_error& e) {
ASSERT_WE_THINK(false); // boost errors shouldn't leak
fprintf(stderr, "boost::system::system_error: %s (%d)", e.what(), e.code().value());
TraceEvent(SevError, "MainError").error(unknown_error()).detail("RootException", e.what());
// printf("\n%d tests passed; %d tests failed\n", passCount, failCount);
flushAndExit(FDB_EXIT_MAIN_EXCEPTION);
} catch (std::exception& e) {
fprintf(stderr, "std::exception: %s\n", e.what());
TraceEvent(SevError, "MainError").error(unknown_error()).detail("RootException", e.what());
// printf("\n%d tests passed; %d tests failed\n", passCount, failCount);
flushAndExit(FDB_EXIT_MAIN_EXCEPTION);
}
static_assert(LBLocalityData<StorageServerInterface>::Present, "Storage server interface should be load balanced");
static_assert(LBLocalityData<CommitProxyInterface>::Present, "Commit proxy interface should be load balanced");
static_assert(LBLocalityData<GrvProxyInterface>::Present, "GRV proxy interface should be load balanced");
static_assert(LBLocalityData<TLogInterface>::Present, "TLog interface should be load balanced");
static_assert(!LBLocalityData<MasterInterface>::Present, "Master interface should not be load balanced");
}