/*
 * fdbserver.actor.cpp
 *
 * This source file is part of the FoundationDB open source project
 *
 * Copyright 2013-2018 Apple Inc. and the FoundationDB project authors
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

// There's something in one of the files below that defines a macros
// a macro that makes boost interprocess break on Windows.
#define BOOST_DATE_TIME_NO_LIB

#include <algorithm>
#include <cctype>
#include <fstream>
#include <iterator>
#include <sstream>

#include <stdarg.h>
#include <stdio.h>
#include <time.h>

#include <boost/algorithm/string.hpp>
#include <boost/interprocess/managed_shared_memory.hpp>

#include "fdbclient/IKnobCollection.h"
#include "fdbclient/NativeAPI.actor.h"
#include "fdbclient/SystemData.h"
#include "fdbclient/versions.h"
#include "fdbclient/BuildFlags.h"
#include "fdbmonitor/SimpleIni.h"
#include "fdbrpc/AsyncFileCached.actor.h"
#include "fdbrpc/Net2FileSystem.h"
#include "fdbrpc/PerfMetric.h"
#include "fdbrpc/simulator.h"
#include "fdbserver/ConflictSet.h"
#include "fdbserver/CoordinationInterface.h"
#include "fdbserver/CoroFlow.h"
#include "fdbserver/DataDistribution.actor.h"
#include "fdbserver/IKeyValueStore.h"
#include "fdbserver/MoveKeys.actor.h"
#include "fdbserver/NetworkTest.h"
#include "fdbserver/RestoreWorkerInterface.actor.h"
#include "fdbserver/ServerDBInfo.h"
#include "fdbserver/SimulatedCluster.h"
#include "fdbserver/Status.h"
#include "fdbserver/TesterInterface.actor.h"
#include "fdbserver/WorkerInterface.actor.h"
#include "fdbserver/pubsub.h"
#include "fdbserver/workloads/workloads.actor.h"
#include "flow/DeterministicRandom.h"
#include "flow/Platform.h"
#include "flow/ProtocolVersion.h"
#include "flow/SimpleOpt.h"
#include "flow/SystemMonitor.h"
#include "flow/TLSConfig.actor.h"
#include "flow/Tracing.h"
#include "flow/UnitTest.h"

#if defined(__linux__) || defined(__FreeBSD__)
#include <execinfo.h>
#include <signal.h>
#ifdef ALLOC_INSTRUMENTATION
#include <cxxabi.h>
#endif
#endif

#ifdef WIN32
#define NOMINMAX
#define WIN32_LEAN_AND_MEAN
#include <Windows.h>
#endif

#include "flow/actorcompiler.h" // This must be the last #include.

// clang-format off
enum {
	OPT_CONNFILE, OPT_SEEDCONNFILE, OPT_SEEDCONNSTRING, OPT_ROLE, OPT_LISTEN, OPT_PUBLICADDR, OPT_DATAFOLDER, OPT_LOGFOLDER, OPT_PARENTPID, OPT_TRACER, OPT_NEWCONSOLE,
	OPT_NOBOX, OPT_TESTFILE, OPT_RESTARTING, OPT_RESTORING, OPT_RANDOMSEED, OPT_KEY, OPT_MEMLIMIT, OPT_STORAGEMEMLIMIT, OPT_CACHEMEMLIMIT, OPT_MACHINEID,
	OPT_DCID, OPT_MACHINE_CLASS, OPT_BUGGIFY, OPT_VERSION, OPT_BUILD_FLAGS, OPT_CRASHONERROR, OPT_HELP, OPT_NETWORKIMPL, OPT_NOBUFSTDOUT, OPT_BUFSTDOUTERR,
	OPT_TRACECLOCK, OPT_NUMTESTERS, OPT_DEVHELP, OPT_ROLLSIZE, OPT_MAXLOGS, OPT_MAXLOGSSIZE, OPT_KNOB, OPT_UNITTESTPARAM, OPT_TESTSERVERS, OPT_TEST_ON_SERVERS, OPT_METRICSCONNFILE,
	OPT_METRICSPREFIX, OPT_LOGGROUP, OPT_LOCALITY, OPT_IO_TRUST_SECONDS, OPT_IO_TRUST_WARN_ONLY, OPT_FILESYSTEM, OPT_PROFILER_RSS_SIZE, OPT_KVFILE,
	OPT_TRACE_FORMAT, OPT_WHITELIST_BINPATH, OPT_BLOB_CREDENTIAL_FILE, OPT_CONFIG_PATH, OPT_USE_TEST_CONFIG_DB,
};

CSimpleOpt::SOption g_rgOptions[] = {
	{ OPT_CONNFILE,              "-C",                          SO_REQ_SEP },
	{ OPT_CONNFILE,              "--cluster_file",              SO_REQ_SEP },
	{ OPT_SEEDCONNFILE,          "--seed_cluster_file",         SO_REQ_SEP },
	{ OPT_SEEDCONNSTRING,        "--seed_connection_string",    SO_REQ_SEP },
	{ OPT_ROLE,                  "-r",                          SO_REQ_SEP },
	{ OPT_ROLE,                  "--role",                      SO_REQ_SEP },
	{ OPT_PUBLICADDR,            "-p",                          SO_REQ_SEP },
	{ OPT_PUBLICADDR,            "--public_address",            SO_REQ_SEP },
	{ OPT_LISTEN,                "-l",                          SO_REQ_SEP },
	{ OPT_LISTEN,                "--listen_address",            SO_REQ_SEP },
#ifdef __linux__
	{ OPT_FILESYSTEM,           "--data_filesystem",           SO_REQ_SEP },
	{ OPT_PROFILER_RSS_SIZE,    "--rsssize",                   SO_REQ_SEP },
#endif
	{ OPT_DATAFOLDER,            "-d",                          SO_REQ_SEP },
	{ OPT_DATAFOLDER,            "--datadir",                   SO_REQ_SEP },
	{ OPT_LOGFOLDER,             "-L",                          SO_REQ_SEP },
	{ OPT_LOGFOLDER,             "--logdir",                    SO_REQ_SEP },
	{ OPT_ROLLSIZE,              "-Rs",                         SO_REQ_SEP },
	{ OPT_ROLLSIZE,              "--logsize",                   SO_REQ_SEP },
	{ OPT_MAXLOGS,               "--maxlogs",                   SO_REQ_SEP },
	{ OPT_MAXLOGSSIZE,           "--maxlogssize",               SO_REQ_SEP },
	{ OPT_LOGGROUP,              "--loggroup",                  SO_REQ_SEP },
	{ OPT_PARENTPID,             "--parentpid",                 SO_REQ_SEP },
	{ OPT_TRACER,                "--tracer",                    SO_REQ_SEP },
#ifdef _WIN32
	{ OPT_NEWCONSOLE,            "-n",                          SO_NONE },
	{ OPT_NEWCONSOLE,            "--newconsole",                SO_NONE },
	{ OPT_NOBOX,                 "-q",                          SO_NONE },
	{ OPT_NOBOX,                 "--no_dialog",                 SO_NONE },
#endif
	{ OPT_KVFILE,                "--kvfile",                    SO_REQ_SEP },
	{ OPT_TESTFILE,              "-f",                          SO_REQ_SEP },
	{ OPT_TESTFILE,              "--testfile",                  SO_REQ_SEP },
	{ OPT_RESTARTING,            "-R",                          SO_NONE },
	{ OPT_RESTARTING,            "--restarting",                SO_NONE },
	{ OPT_RANDOMSEED,            "-s",                          SO_REQ_SEP },
	{ OPT_RANDOMSEED,            "--seed",                      SO_REQ_SEP },
	{ OPT_KEY,                   "-k",                          SO_REQ_SEP },
	{ OPT_KEY,                   "--key",                       SO_REQ_SEP },
	{ OPT_MEMLIMIT,              "-m",                          SO_REQ_SEP },
	{ OPT_MEMLIMIT,              "--memory",                    SO_REQ_SEP },
	{ OPT_STORAGEMEMLIMIT,       "-M",                          SO_REQ_SEP },
	{ OPT_STORAGEMEMLIMIT,       "--storage_memory",            SO_REQ_SEP },
	{ OPT_CACHEMEMLIMIT,         "--cache_memory",              SO_REQ_SEP },
	{ OPT_MACHINEID,             "-i",                          SO_REQ_SEP },
	{ OPT_MACHINEID,             "--machine_id",                SO_REQ_SEP },
	{ OPT_DCID,                  "-a",                          SO_REQ_SEP },
	{ OPT_DCID,                  "--datacenter_id",             SO_REQ_SEP },
	{ OPT_MACHINE_CLASS,         "-c",                          SO_REQ_SEP },
	{ OPT_MACHINE_CLASS,         "--class",                     SO_REQ_SEP },
	{ OPT_BUGGIFY,               "-b",                          SO_REQ_SEP },
	{ OPT_BUGGIFY,               "--buggify",                   SO_REQ_SEP },
	{ OPT_VERSION,               "-v",                          SO_NONE },
	{ OPT_VERSION,               "--version",                   SO_NONE },
	{ OPT_BUILD_FLAGS,           "--build_flags",               SO_NONE },
	{ OPT_CRASHONERROR,          "--crash",                     SO_NONE },
	{ OPT_NETWORKIMPL,           "-N",                          SO_REQ_SEP },
	{ OPT_NETWORKIMPL,           "--network",                   SO_REQ_SEP },
	{ OPT_NOBUFSTDOUT,           "--unbufferedout",             SO_NONE },
	{ OPT_BUFSTDOUTERR,          "--bufferedout",               SO_NONE },
	{ OPT_TRACECLOCK,            "--traceclock",                SO_REQ_SEP },
	{ OPT_NUMTESTERS,            "--num_testers",               SO_REQ_SEP },
	{ OPT_HELP,                  "-?",                          SO_NONE },
	{ OPT_HELP,                  "-h",                          SO_NONE },
	{ OPT_HELP,                  "--help",                      SO_NONE },
	{ OPT_DEVHELP,               "--dev-help",                  SO_NONE },
	{ OPT_KNOB,                  "--knob_",                     SO_REQ_SEP },
	{ OPT_UNITTESTPARAM,         "--test_",                     SO_REQ_SEP },
	{ OPT_LOCALITY,              "--locality_",                 SO_REQ_SEP },
	{ OPT_TESTSERVERS,           "--testservers",               SO_REQ_SEP },
	{ OPT_TEST_ON_SERVERS,       "--testonservers",             SO_NONE },
	{ OPT_METRICSCONNFILE,       "--metrics_cluster",           SO_REQ_SEP },
	{ OPT_METRICSPREFIX,         "--metrics_prefix",            SO_REQ_SEP },
	{ OPT_IO_TRUST_SECONDS,      "--io_trust_seconds",          SO_REQ_SEP },
	{ OPT_IO_TRUST_WARN_ONLY,    "--io_trust_warn_only",        SO_NONE },
	{ OPT_TRACE_FORMAT      ,    "--trace_format",              SO_REQ_SEP },
	{ OPT_WHITELIST_BINPATH,     "--whitelist_binpath",         SO_REQ_SEP },
	{ OPT_BLOB_CREDENTIAL_FILE,  "--blob_credential_file",      SO_REQ_SEP },
	{ OPT_CONFIG_PATH,           "--config_path",               SO_REQ_SEP },
	{ OPT_USE_TEST_CONFIG_DB,    "--use_test_config_db",        SO_NONE },

#ifndef TLS_DISABLED
	TLS_OPTION_FLAGS
#endif

	SO_END_OF_OPTIONS
};

// clang-format on

extern void dsltest();
extern void pingtest();
extern void copyTest();
extern void versionedMapTest();
extern void createTemplateDatabase();
// FIXME: this really belongs in a header somewhere since it is actually used.
extern IPAddress determinePublicIPAutomatically(ClusterConnectionString const& ccs);

extern const char* getSourceVersion();

extern void flushTraceFileVoid();

extern const int MAX_CLUSTER_FILE_BYTES;

#ifdef ALLOC_INSTRUMENTATION
extern uint8_t* g_extra_memory;
#endif

bool enableFailures = true;

#define test_assert(x)                                                                                                 \
	if (!(x)) {                                                                                                        \
		cout << "Test failed: " #x << endl;                                                                            \
		return false;                                                                                                  \
	}

#ifdef _WIN32
#include <sddl.h>

// It is your
//    responsibility to properly initialize the
//    structure and to free the structure's
//    lpSecurityDescriptor member when you have
//    finished using it. To free the structure's
//    lpSecurityDescriptor member, call the
//    LocalFree function.
BOOL CreatePermissiveReadWriteDACL(SECURITY_ATTRIBUTES* pSA) {
	UNSTOPPABLE_ASSERT(pSA != nullptr);

	TCHAR* szSD = TEXT("D:") // Discretionary ACL
	    TEXT("(A;OICI;GR;;;AU)") // Allow read/write/execute to authenticated users
	    TEXT("(A;OICI;GA;;;BA)"); // Allow full control to administrators

	return ConvertStringSecurityDescriptorToSecurityDescriptor(
	    szSD, SDDL_REVISION_1, &(pSA->lpSecurityDescriptor), nullptr);
}
#endif

class WorldReadablePermissions {
public:
	WorldReadablePermissions() {
#ifdef _WIN32
		sa.nLength = sizeof(SECURITY_ATTRIBUTES);
		sa.bInheritHandle = FALSE;
		if (!CreatePermissiveReadWriteDACL(&sa)) {
			TraceEvent("Win32DACLCreationFail").GetLastError();
			throw platform_error();
		}
		permission.set_permissions(&sa);
#elif (defined(__linux__) || defined(__APPLE__) || defined(__FreeBSD__))
		// There is nothing to do here, since the default permissions are fine
#else
#error Port me!
#endif
	}

	virtual ~WorldReadablePermissions() {
#ifdef _WIN32
		LocalFree(sa.lpSecurityDescriptor);
#elif (defined(__linux__) || defined(__APPLE__) || defined(__FreeBSD__))
		// There is nothing to do here, since the default permissions are fine
#else
#error Port me!
#endif
	}

	boost::interprocess::permissions permission;

private:
	WorldReadablePermissions(const WorldReadablePermissions& rhs) {}
#ifdef _WIN32
	SECURITY_ATTRIBUTES sa;
#endif
};

UID getSharedMemoryMachineId() {
	UID* machineId = nullptr;
	int numTries = 0;

	// Permissions object defaults to 0644 on *nix, but on windows defaults to allowing access to only the creator.
	// On windows, this means that we have to create an elaborate workaround for DACLs
	WorldReadablePermissions p;
	std::string sharedMemoryIdentifier = "fdbserver_shared_memory_id";
	loop {
		try {
			// "0" is the default parameter "addr"
			boost::interprocess::managed_shared_memory segment(
			    boost::interprocess::open_or_create, sharedMemoryIdentifier.c_str(), 1000, 0, p.permission);
			machineId = segment.find_or_construct<UID>("machineId")(deterministicRandom()->randomUniqueID());
			if (!machineId)
				criticalError(
				    FDB_EXIT_ERROR, "SharedMemoryError", "Could not locate or create shared memory - 'machineId'");
			return *machineId;
		} catch (boost::interprocess::interprocess_exception&) {
			try {
				// If the shared memory already exists, open it read-only in case it was created by another user
				boost::interprocess::managed_shared_memory segment(boost::interprocess::open_read_only,
				                                                   sharedMemoryIdentifier.c_str());
				machineId = segment.find<UID>("machineId").first;
				if (!machineId)
					criticalError(FDB_EXIT_ERROR, "SharedMemoryError", "Could not locate shared memory - 'machineId'");
				return *machineId;
			} catch (boost::interprocess::interprocess_exception& ex) {
				// Retry in case the shared memory was deleted in between the call to open_or_create and open_read_only
				// Don't keep trying forever in case this is caused by some other problem
				if (++numTries == 10)
					criticalError(FDB_EXIT_ERROR,
					              "SharedMemoryError",
					              format("Could not open shared memory - %s", ex.what()).c_str());
			}
		}
	}
}

ACTOR void failAfter(Future<Void> trigger, ISimulator::ProcessInfo* m = g_simulator.getCurrentProcess()) {
	wait(trigger);
	if (enableFailures) {
		printf("Killing machine: %s at %f\n", m->address.toString().c_str(), now());
		g_simulator.killProcess(m, ISimulator::KillInstantly);
	}
}

void failAfter(Future<Void> trigger, Endpoint e) {
	if (g_network == &g_simulator)
		failAfter(trigger, g_simulator.getProcess(e));
}

ACTOR Future<Void> histogramReport() {
	loop {
		wait(delay(SERVER_KNOBS->HISTOGRAM_REPORT_INTERVAL));

		GetHistogramRegistry().logReport();
	}
}

void testSerializationSpeed() {
	double tstart;
	double build = 0, serialize = 0, deserialize = 0, copy = 0, deallocate = 0;
	double bytes = 0;
	double testBegin = timer();
	for (int a = 0; a < 10000; a++) {
		{
			tstart = timer();

			Arena batchArena;
			VectorRef<CommitTransactionRef> batch;
			batch.resize(batchArena, 1000);
			for (int t = 0; t < batch.size(); t++) {
				CommitTransactionRef& tr = batch[t];
				tr.read_snapshot = 0;
				for (int i = 0; i < 2; i++)
					tr.mutations.push_back_deep(
					    batchArena,
					    MutationRef(MutationRef::SetValue, LiteralStringRef("KeyABCDE"), LiteralStringRef("SomeValu")));
				tr.mutations.push_back_deep(
				    batchArena,
				    MutationRef(MutationRef::ClearRange, LiteralStringRef("BeginKey"), LiteralStringRef("EndKeyAB")));
			}

			build += timer() - tstart;

			tstart = timer();

			BinaryWriter wr(IncludeVersion());
			wr << batch;

			bytes += wr.getLength();

			serialize += timer() - tstart;

			for (int i = 0; i < 1; i++) {
				tstart = timer();
				Arena arena;
				StringRef data(arena, StringRef((const uint8_t*)wr.getData(), wr.getLength()));
				copy += timer() - tstart;

				tstart = timer();
				ArenaReader rd(arena, data, IncludeVersion());
				VectorRef<CommitTransactionRef> batch2;
				rd >> arena >> batch2;

				deserialize += timer() - tstart;
			}

			tstart = timer();
		}
		deallocate += timer() - tstart;
	}
	double elapsed = (timer() - testBegin);
	printf("Test speed: %0.1f MB/sec (%0.0f/sec)\n", bytes / 1e6 / elapsed, 1000000 / elapsed);
	printf("  Build: %0.1f MB/sec\n", bytes / 1e6 / build);
	printf("  Serialize: %0.1f MB/sec\n", bytes / 1e6 / serialize);
	printf("  Copy: %0.1f MB/sec\n", bytes / 1e6 / copy);
	printf("  Deserialize: %0.1f MB/sec\n", bytes / 1e6 / deserialize);
	printf("  Deallocate: %0.1f MB/sec\n", bytes / 1e6 / deallocate);
	printf("  Bytes: %0.1f MB\n", bytes / 1e6);
	printf("\n");
}

std::string toHTML(const StringRef& binaryString) {
	std::string s;

	for (int i = 0; i < binaryString.size(); i++) {
		uint8_t c = binaryString[i];
		if (c == '<')
			s += "&lt;";
		else if (c == '>')
			s += "&gt;";
		else if (c == '&')
			s += "&amp;";
		else if (c == '"')
			s += "&quot;";
		else if (c == ' ')
			s += "&nbsp;";
		else if (c > 32 && c < 127)
			s += c;
		else
			s += format("<span class=\"binary\">[%02x]</span>", c);
	}

	return s;
}

ACTOR Future<Void> dumpDatabase(Database cx, std::string outputFilename, KeyRange range = allKeys) {
	try {
		state Transaction tr(cx);
		loop {
			state FILE* output = fopen(outputFilename.c_str(), "wt");
			try {
				state KeySelectorRef iter = firstGreaterOrEqual(range.begin);
				state Arena arena;
				fprintf(output, "<html><head><style type=\"text/css\">.binary {color:red}</style></head><body>\n");
				Version ver = wait(tr.getReadVersion());
				fprintf(output, "<h3>Database version: %" PRId64 "</h3>", ver);

				loop {
					RangeResult results = wait(tr.getRange(iter, firstGreaterOrEqual(range.end), 1000));
					for (int r = 0; r < results.size(); r++) {
						std::string key = toHTML(results[r].key), value = toHTML(results[r].value);
						fprintf(output, "<p>%s <b>:=</b> %s</p>\n", key.c_str(), value.c_str());
					}
					if (results.size() < 1000)
						break;
					iter = firstGreaterThan(KeyRef(arena, results[results.size() - 1].key));
				}
				fprintf(output, "</body></html>");
				fclose(output);
				TraceEvent("DatabaseDumped").detail("Filename", outputFilename);
				return Void();
			} catch (Error& e) {
				fclose(output);
				wait(tr.onError(e));
			}
		}
	} catch (Error& e) {
		TraceEvent(SevError, "DumpDatabaseError").error(e).detail("Filename", outputFilename);
		throw;
	}
}

void memoryTest();
void skipListTest();

Future<Void> startSystemMonitor(std::string dataFolder,
                                Optional<Standalone<StringRef>> dcId,
                                Optional<Standalone<StringRef>> zoneId,
                                Optional<Standalone<StringRef>> machineId) {
	initializeSystemMonitorMachineState(
	    SystemMonitorMachineState(dataFolder, dcId, zoneId, machineId, g_network->getLocalAddress().ip));

	systemMonitor();
	return recurring(&systemMonitor, SERVER_KNOBS->SYSTEM_MONITOR_FREQUENCY, TaskPriority::FlushTrace);
}

void testIndexedSet();

#ifdef _WIN32
void parentWatcher(void* parentHandle) {
	HANDLE parent = (HANDLE)parentHandle;
	int signal = WaitForSingleObject(parent, INFINITE);
	CloseHandle(parentHandle);
	if (signal == WAIT_OBJECT_0)
		criticalError(FDB_EXIT_SUCCESS, "ParentProcessExited", "Parent process exited");
	TraceEvent(SevError, "ParentProcessWaitFailed").detail("RetCode", signal).GetLastError();
}
#else
void* parentWatcher(void* arg) {
	int* parent_pid = (int*)arg;
	while (1) {
		sleep(1);
		if (getppid() != *parent_pid)
			criticalError(FDB_EXIT_SUCCESS, "ParentProcessExited", "Parent process exited");
	}
}
#endif

static void printBuildInformation() {
	printf("%s", jsonBuildInformation().c_str());
}

static void printVersion() {
	printf("FoundationDB " FDB_VT_PACKAGE_NAME " (v" FDB_VT_VERSION ")\n");
	printf("source version %s\n", getSourceVersion());
	printf("protocol %" PRIx64 "\n", currentProtocolVersion.version());
}

static void printHelpTeaser(const char* name) {
	fprintf(stderr, "Try `%s --help' for more information.\n", name);
}

static void printOptionUsage(std::string option, std::string description) {
	static const std::string OPTION_INDENT("  ");
	static const std::string DESCRIPTION_INDENT("                ");
	static const int WIDTH = 80;

	boost::algorithm::trim(option);
	boost::algorithm::trim(description);

	std::string result = OPTION_INDENT + option + "\n";

	std::stringstream sstream(description);
	if (sstream.eof()) {
		printf(result.c_str());
		return;
	}

	std::string currWord;
	sstream >> currWord;

	std::string currLine(DESCRIPTION_INDENT + ' ' + currWord);
	int currLength = currLine.size();

	while (!sstream.eof()) {
		sstream >> currWord;

		if (currLength + static_cast<int>(currWord.size()) + 1 > WIDTH) {
			result += currLine + '\n';
			currLine = DESCRIPTION_INDENT + ' ' + currWord;
		} else {
			currLine += ' ' + currWord;
		}
		currLength = currLine.size();
	}
	result += currLine + '\n';

	printf(result.c_str());
}

static void printUsage(const char* name, bool devhelp) {
	printf("FoundationDB " FDB_VT_PACKAGE_NAME " (v" FDB_VT_VERSION ")\n");
	printf("Usage: %s -p ADDRESS [OPTIONS]\n\n", name);
	printOptionUsage("-p ADDRESS, --public_address ADDRESS",
	                 " Public address, specified as `IP_ADDRESS:PORT' or `auto:PORT'.");
	printOptionUsage("-l ADDRESS, --listen_address ADDRESS",
	                 " Listen address, specified as `IP_ADDRESS:PORT' (defaults to"
	                 " public address).");
	printOptionUsage("-C CONNFILE, --cluster_file CONNFILE",
	                 " The path of a file containing the connection string for the"
	                 " FoundationDB cluster. The default is first the value of the"
	                 " FDB_CLUSTER_FILE environment variable, then `./fdb.cluster',"
	                 " then `" +
	                     platform::getDefaultClusterFilePath() + "'.");
	printOptionUsage("--seed_cluster_file SEEDCONNFILE",
	                 " The path of a seed cluster file which will be used to connect"
	                 " if the -C cluster file does not exist. If the server connects"
	                 " successfully using the seed file, then it copies the file to"
	                 " the -C file location.");
	printOptionUsage("--seed_connection_string SEEDCONNSTRING",
	                 " The path of a seed connection string which will be used to connect"
	                 " if the -C cluster file does not exist. If the server connects"
	                 " successfully using the seed string, then it copies the string to"
	                 " the -C file location.");
#ifdef __linux__
	printOptionUsage("--data_filesystem PATH",
	                 " Turns on validation that all data files are written to a drive"
	                 " mounted at the specified PATH. This checks that the device at PATH"
	                 " is currently mounted and that any data files get written to the"
	                 " same device.");
#endif
	printOptionUsage("-d PATH, --datadir PATH",
	                 " Store data files in the given folder (must be unique for each"
	                 " fdbserver instance on a given machine).");
	printOptionUsage("-L PATH, --logdir PATH", " Store log files in the given folder (default is `.').");
	printOptionUsage("--logsize SIZE",
	                 "Roll over to a new log file after the current log file"
	                 " exceeds SIZE bytes. The default value is 10MiB.");
	printOptionUsage("--maxlogs SIZE, --maxlogssize SIZE",
	                 " Delete the oldest log file when the total size of all log"
	                 " files exceeds SIZE bytes. If set to 0, old log files will not"
	                 " be deleted. The default value is 100MiB.");
	printOptionUsage("--loggroup LOG_GROUP",
	                 " Sets the LogGroup field with the specified value for all"
	                 " events in the trace output (defaults to `default').");
	printOptionUsage("--trace_format FORMAT",
	                 " Select the format of the log files. xml (the default) and json"
	                 " are supported.");
	printOptionUsage("--tracer       TRACER",
	                 " Select a tracer for transaction tracing. Currently disabled"
	                 " (the default) and log_file are supported.");
	printOptionUsage("-i ID, --machine_id ID",
	                 " Machine and zone identifier key (up to 16 hex characters)."
	                 " Defaults to a random value shared by all fdbserver processes"
	                 " on this machine.");
	printOptionUsage("-a ID, --datacenter_id ID", " Data center identifier key (up to 16 hex characters).");
	printOptionUsage("--locality_LOCALITYKEY LOCALITYVALUE",
	                 " Define a locality key. LOCALITYKEY is case-insensitive though"
	                 " LOCALITYVALUE is not.");
	printOptionUsage("-m SIZE, --memory SIZE",
	                 " Memory limit. The default value is 8GiB. When specified"
	                 " without a unit, MiB is assumed.");
	printOptionUsage("-M SIZE, --storage_memory SIZE",
	                 " Maximum amount of memory used for storage. The default"
	                 " value is 1GiB. When specified without a unit, MB is"
	                 " assumed.");
	printOptionUsage("--cache_memory SIZE",
	                 " The amount of memory to use for caching disk pages."
	                 " The default value is 2GiB. When specified without a unit,"
	                 " MiB is assumed.");
	printOptionUsage("-c CLASS, --class CLASS",
	                 " Machine class (valid options are storage, transaction,"
	                 " resolution, grv_proxy, commit_proxy, master, test, unset, stateless, log, router,"
	                 " and cluster_controller).");
#ifndef TLS_DISABLED
	printf(TLS_HELP);
#endif
	printOptionUsage("-v, --version", "Print version information and exit.");
	printOptionUsage("-h, -?, --help", "Display this help and exit.");
	if (devhelp) {
		printf("  --build_flags  Print build information and exit.\n");
		printOptionUsage(
		    "-r ROLE, --role ROLE",
		    " Server role (valid options are fdbd, test, multitest,"
		    " simulation, networktestclient, networktestserver, restore"
		    " consistencycheck, kvfileintegritycheck, kvfilegeneratesums, unittests). The default is `fdbd'.");
#ifdef _WIN32
		printOptionUsage("-n, --newconsole", " Create a new console.");
		printOptionUsage("-q, --no_dialog", " Disable error dialog on crash.");
		printOptionUsage("--parentpid PID", " Specify a process after whose termination to exit.");
#endif
		printOptionUsage("-f TESTFILE, --testfile",
		                 " Testfile to run, defaults to `tests/default.txt'.  If role is `unittests', specifies which "
		                 "unit tests to run as a search prefix.");
		printOptionUsage("-R, --restarting", " Restart a previous simulation that was cleanly shut down.");
		printOptionUsage("-s SEED, --seed SEED", " Random seed.");
		printOptionUsage("-k KEY, --key KEY", "Target key for search role.");
		printOptionUsage(
		    "--kvfile FILE",
		    "Input file (SQLite database file) for use by the 'kvfilegeneratesums' and 'kvfileintegritycheck' roles.");
		printOptionUsage("-b [on,off], --buggify [on,off]", " Sets Buggify system state, defaults to `off'.");
		printOptionUsage("--crash", "Crash on serious errors instead of continuing.");
		printOptionUsage("-N NETWORKIMPL, --network NETWORKIMPL",
		                 " Select network implementation, `net2' (default),"
		                 " `net2-threadpool'.");
		printOptionUsage("--unbufferedout", " Do not buffer stdout and stderr.");
		printOptionUsage("--bufferedout", " Buffer stdout and stderr.");
		printOptionUsage("--traceclock CLOCKIMPL",
		                 " Select clock source for trace files, `now' (default) or"
		                 " `realtime'.");
		printOptionUsage("--num_testers NUM",
		                 " A multitester will wait for NUM testers before starting"
		                 " (defaults to 1).");
		printOptionUsage("--test_PARAMNAME PARAMVALUE",
		                 " Set a UnitTest named parameter to the given value.  Names are case sensitive.");
#ifdef __linux__
		printOptionUsage("--rsssize SIZE",
		                 " Turns on automatic heap profiling when RSS memory size exceeds"
		                 " the given threshold. fdbserver needs to be compiled with"
		                 " USE_GPERFTOOLS flag in order to use this feature.");
#endif
		printOptionUsage("--testservers ADDRESSES",
		                 " The addresses of networktestservers"
		                 " specified as ADDRESS:PORT,ADDRESS:PORT...");
		printOptionUsage("--testonservers", " Testers are recruited on servers.");
		printOptionUsage("--metrics_cluster CONNFILE",
		                 " The cluster file designating where this process will"
		                 " store its metric data. By default metrics will be stored"
		                 " in the same database the process is participating in.");
		printOptionUsage("--metrics_prefix PREFIX",
		                 " The prefix where this process will store its metric data."
		                 " Must be specified if using a different database for metrics.");
		printOptionUsage("--knob_KNOBNAME KNOBVALUE", " Changes a database knob. KNOBNAME should be lowercase.");
		printOptionUsage("--io_trust_seconds SECONDS",
		                 " Sets the time in seconds that a read or write operation is allowed to take"
		                 " before timing out with an error. If an operation times out, all future"
		                 " operations on that file will fail with an error as well. Only has an effect"
		                 " when using AsyncFileKAIO in Linux.");
		printOptionUsage("--io_trust_warn_only",
		                 " Instead of failing when an I/O operation exceeds io_trust_seconds, just"
		                 " log a warning to the trace log. Has no effect if io_trust_seconds is unspecified.");
	} else {
		printOptionUsage("--dev-help", "Display developer-specific help and exit.");
	}

	printf("\n"
	       "SIZE parameters may use one of the multiplicative suffixes B=1, KB=10^3,\n"
	       "KiB=2^10, MB=10^6, MiB=2^20, GB=10^9, GiB=2^30, TB=10^12, or TiB=2^40.\n");
}

extern bool g_crashOnError;

#if defined(ALLOC_INSTRUMENTATION) || defined(ALLOC_INSTRUMENTATION_STDOUT)
void* operator new(std::size_t size) {
	void* p = malloc(size);
	if (!p)
		throw std::bad_alloc();
	recordAllocation(p, size);
	return p;
}
void operator delete(void* ptr) throw() {
	recordDeallocation(ptr);
	free(ptr);
}

// scalar, nothrow new and it matching delete
void* operator new(std::size_t size, const std::nothrow_t&) throw() {
	void* p = malloc(size);
	recordAllocation(p, size);
	return p;
}
void operator delete(void* ptr, const std::nothrow_t&) throw() {
	recordDeallocation(ptr);
	free(ptr);
}

// array throwing new and matching delete[]
void* operator new[](std::size_t size) {
	void* p = malloc(size);
	if (!p)
		throw std::bad_alloc();
	recordAllocation(p, size);
	return p;
}
void operator delete[](void* ptr) throw() {
	recordDeallocation(ptr);
	free(ptr);
}

// array, nothrow new and matching delete[]
void* operator new[](std::size_t size, const std::nothrow_t&) throw() {
	void* p = malloc(size);
	recordAllocation(p, size);
	return p;
}
void operator delete[](void* ptr, const std::nothrow_t&) throw() {
	recordDeallocation(ptr);
	free(ptr);
}
#endif

Optional<bool> checkBuggifyOverride(const char* testFile) {
	std::ifstream ifs;
	ifs.open(testFile, std::ifstream::in);
	if (!ifs.good())
		return 0;

	std::string cline;

	while (ifs.good()) {
		getline(ifs, cline);
		std::string line = removeWhitespace(std::string(cline));
		if (!line.size() || line.find(';') == 0)
			continue;

		size_t found = line.find('=');
		if (found == std::string::npos)
			// hmmm, not good
			continue;
		std::string attrib = removeWhitespace(line.substr(0, found));
		std::string value = removeWhitespace(line.substr(found + 1));

		if (attrib == "buggify") {
			// Testspec uses `on` or `off` (without quotes).
			// TOML uses literal `true` and `false`.
			if (!strcmp(value.c_str(), "on") || !strcmp(value.c_str(), "true")) {
				ifs.close();
				return true;
			} else if (!strcmp(value.c_str(), "off") || !strcmp(value.c_str(), "false")) {
				ifs.close();
				return false;
			} else {
				fprintf(stderr, "ERROR: Unknown buggify override state `%s'\n", value.c_str());
				flushAndExit(FDB_EXIT_ERROR);
			}
		}
	}

	ifs.close();
	return Optional<bool>();
}

// Takes a vector of public and listen address strings given via command line, and returns vector of NetworkAddress
// objects.
std::pair<NetworkAddressList, NetworkAddressList> buildNetworkAddresses(const ClusterConnectionFile& connectionFile,
                                                                        const vector<std::string>& publicAddressStrs,
                                                                        vector<std::string>& listenAddressStrs) {
	if (listenAddressStrs.size() > 0 && publicAddressStrs.size() != listenAddressStrs.size()) {
		fprintf(stderr,
		        "ERROR: Listen addresses (if provided) should be equal to the number of public addresses in order.\n");
		flushAndExit(FDB_EXIT_ERROR);
	}
	listenAddressStrs.resize(publicAddressStrs.size(), "public");

	if (publicAddressStrs.size() > 2) {
		fprintf(stderr, "ERROR: maximum 2 public/listen addresses are allowed\n");
		flushAndExit(FDB_EXIT_ERROR);
	}

	NetworkAddressList publicNetworkAddresses;
	NetworkAddressList listenNetworkAddresses;

	auto& coordinators = connectionFile.getConnectionString().coordinators();
	ASSERT(coordinators.size() > 0);

	for (int ii = 0; ii < publicAddressStrs.size(); ++ii) {
		const std::string& publicAddressStr = publicAddressStrs[ii];
		bool autoPublicAddress = StringRef(publicAddressStr).startsWith(LiteralStringRef("auto:"));
		NetworkAddress currentPublicAddress;
		if (autoPublicAddress) {
			try {
				const NetworkAddress& parsedAddress = NetworkAddress::parse("0.0.0.0:" + publicAddressStr.substr(5));
				const IPAddress publicIP = determinePublicIPAutomatically(connectionFile.getConnectionString());
				currentPublicAddress = NetworkAddress(publicIP, parsedAddress.port, true, parsedAddress.isTLS());
			} catch (Error& e) {
				fprintf(stderr,
				        "ERROR: could not determine public address automatically from `%s': %s\n",
				        publicAddressStr.c_str(),
				        e.what());
				throw;
			}
		} else {
			try {
				currentPublicAddress = NetworkAddress::parse(publicAddressStr);
			} catch (Error&) {
				fprintf(stderr,
				        "ERROR: Could not parse network address `%s' (specify as IP_ADDRESS:PORT)\n",
				        publicAddressStr.c_str());
				throw;
			}
		}

		if (ii == 0) {
			publicNetworkAddresses.address = currentPublicAddress;
		} else {
			publicNetworkAddresses.secondaryAddress = currentPublicAddress;
		}

		if (!currentPublicAddress.isValid()) {
			fprintf(stderr, "ERROR: %s is not a valid IP address\n", currentPublicAddress.toString().c_str());
			flushAndExit(FDB_EXIT_ERROR);
		}

		const std::string& listenAddressStr = listenAddressStrs[ii];
		NetworkAddress currentListenAddress;
		if (listenAddressStr == "public") {
			currentListenAddress = currentPublicAddress;
		} else {
			try {
				currentListenAddress = NetworkAddress::parse(listenAddressStr);
			} catch (Error&) {
				fprintf(stderr,
				        "ERROR: Could not parse network address `%s' (specify as IP_ADDRESS:PORT)\n",
				        listenAddressStr.c_str());
				throw;
			}

			if (currentListenAddress.isTLS() != currentPublicAddress.isTLS()) {
				fprintf(stderr,
				        "ERROR: TLS state of listen address: %s is not equal to the TLS state of public address: %s.\n",
				        listenAddressStr.c_str(),
				        publicAddressStr.c_str());
				flushAndExit(FDB_EXIT_ERROR);
			}
		}

		if (ii == 0) {
			listenNetworkAddresses.address = currentListenAddress;
		} else {
			listenNetworkAddresses.secondaryAddress = currentListenAddress;
		}

		bool hasSameCoord = std::all_of(coordinators.begin(), coordinators.end(), [&](const NetworkAddress& address) {
			if (address.ip == currentPublicAddress.ip && address.port == currentPublicAddress.port) {
				return address.isTLS() == currentPublicAddress.isTLS();
			}
			return true;
		});
		if (!hasSameCoord) {
			fprintf(stderr,
			        "ERROR: TLS state of public address %s does not match in coordinator list.\n",
			        publicAddressStr.c_str());
			flushAndExit(FDB_EXIT_ERROR);
		}
	}

	if (publicNetworkAddresses.secondaryAddress.present() &&
	    publicNetworkAddresses.address.isTLS() == publicNetworkAddresses.secondaryAddress.get().isTLS()) {
		fprintf(stderr, "ERROR: only one public address of each TLS state is allowed.\n");
		flushAndExit(FDB_EXIT_ERROR);
	}

	return std::make_pair(publicNetworkAddresses, listenNetworkAddresses);
}

// moves files from 'dirSrc' to 'dirToMove' if their name contains 'role'
void restoreRoleFilesHelper(std::string dirSrc, std::string dirToMove, std::string role) {
	std::vector<std::string> returnFiles = platform::listFiles(dirSrc, "");
	for (const auto& fileEntry : returnFiles) {
		if (fileEntry != "fdb.cluster" && fileEntry.find(role) != std::string::npos) {
			// rename files
			TraceEvent("RenamingSnapFile")
			    .detail("Oldname", dirSrc + "/" + fileEntry)
			    .detail("Newname", dirToMove + "/" + fileEntry);
			renameFile(dirSrc + "/" + fileEntry, dirToMove + "/" + fileEntry);
		}
	}
}

namespace {
enum class ServerRole {
	ConsistencyCheck,
	CreateTemplateDatabase,
	DSLTest,
	FDBD,
	KVFileGenerateIOLogChecksums,
	KVFileIntegrityCheck,
	MultiTester,
	NetworkTestClient,
	NetworkTestServer,
	Restore,
	SearchMutations,
	Simulation,
	SkipListTest,
	Test,
	VersionedMapTest,
	UnitTests
};
struct CLIOptions {
	std::string commandLine;
	std::string fileSystemPath, dataFolder, connFile, seedConnFile, seedConnString, logFolder = ".", metricsConnFile,
	                                                                                metricsPrefix;
	std::string logGroup = "default";
	uint64_t rollsize = TRACE_DEFAULT_ROLL_SIZE;
	uint64_t maxLogsSize = TRACE_DEFAULT_MAX_LOGS_SIZE;
	bool maxLogsSizeSet = false;
	int maxLogs = 0;
	bool maxLogsSet = false;

	ServerRole role = ServerRole::FDBD;
	uint32_t randomSeed = platform::getRandomSeed();

	const char* testFile = "tests/default.txt";
	std::string kvFile;
	std::string testServersStr;
	std::string whitelistBinPaths;

	std::vector<std::string> publicAddressStrs, listenAddressStrs;
	NetworkAddressList publicAddresses, listenAddresses;

	const char* targetKey = nullptr;
	int64_t memLimit =
	    8LL << 30; // Nice to maintain the same default value for memLimit and SERVER_KNOBS->SERVER_MEM_LIMIT and
	               // SERVER_KNOBS->COMMIT_BATCHES_MEM_BYTES_HARD_LIMIT
	uint64_t storageMemLimit = 1LL << 30;
	bool buggifyEnabled = false, restarting = false;
	Optional<Standalone<StringRef>> zoneId;
	Optional<Standalone<StringRef>> dcId;
	ProcessClass processClass = ProcessClass(ProcessClass::UnsetClass, ProcessClass::CommandLineSource);
	bool useNet2 = true;
	bool useThreadPool = false;
	std::vector<std::pair<std::string, std::string>> knobs;
	std::map<std::string, std::string> manualKnobOverrides;
	LocalityData localities;
	int minTesterCount = 1;
	bool testOnServers = false;

	TLSConfig tlsConfig = TLSConfig(TLSEndpointType::SERVER);
	double fileIoTimeout = 0.0;
	bool fileIoWarnOnly = false;
	uint64_t rsssize = -1;
	std::vector<std::string> blobCredentials; // used for fast restore workers & backup workers
	const char* blobCredsFromENV = nullptr;

	std::string configPath;
	UseConfigDB useConfigDB{ UseConfigDB::DISABLED };

	Reference<ClusterConnectionFile> connectionFile;
	Standalone<StringRef> machineId;
	UnitTestParameters testParams;

	static CLIOptions parseArgs(int argc, char* argv[]) {
		CLIOptions opts;
		opts.parseArgsInternal(argc, argv);
		return opts;
	}

private:
	CLIOptions() = default;

	void parseArgsInternal(int argc, char* argv[]) {
		for (int a = 0; a < argc; a++) {
			if (a)
				commandLine += ' ';
			commandLine += argv[a];
		}

		CSimpleOpt args(argc, argv, g_rgOptions, SO_O_EXACT);

		if (argc == 1) {
			printUsage(argv[0], false);
			flushAndExit(FDB_EXIT_ERROR);
		}

		while (args.Next()) {
			if (args.LastError() == SO_ARG_INVALID_DATA) {
				fprintf(stderr, "ERROR: invalid argument to option `%s'\n", args.OptionText());
				printHelpTeaser(argv[0]);
				flushAndExit(FDB_EXIT_ERROR);
			}
			if (args.LastError() == SO_ARG_INVALID) {
				fprintf(stderr, "ERROR: argument given for option `%s'\n", args.OptionText());
				printHelpTeaser(argv[0]);
				flushAndExit(FDB_EXIT_ERROR);
			}
			if (args.LastError() == SO_ARG_MISSING) {
				fprintf(stderr, "ERROR: missing argument for option `%s'\n", args.OptionText());
				printHelpTeaser(argv[0]);
				flushAndExit(FDB_EXIT_ERROR);
			}
			if (args.LastError() == SO_OPT_INVALID) {
				fprintf(stderr, "ERROR: unknown option: `%s'\n", args.OptionText());
				printHelpTeaser(argv[0]);
				flushAndExit(FDB_EXIT_ERROR);
			}
			if (args.LastError() != SO_SUCCESS) {
				fprintf(stderr, "ERROR: error parsing options\n");
				printHelpTeaser(argv[0]);
				flushAndExit(FDB_EXIT_ERROR);
			}
			const char* sRole;
			Optional<uint64_t> ti;
			std::string argStr;
			std::vector<std::string> tmpStrings;

			switch (args.OptionId()) {
			case OPT_HELP:
				printUsage(argv[0], false);
				flushAndExit(FDB_EXIT_SUCCESS);
				break;
			case OPT_DEVHELP:
				printUsage(argv[0], true);
				flushAndExit(FDB_EXIT_SUCCESS);
				break;
			case OPT_KNOB: {
				std::string syn = args.OptionSyntax();
				if (!StringRef(syn).startsWith(LiteralStringRef("--knob_"))) {
					fprintf(stderr, "ERROR: unable to parse knob option '%s'\n", syn.c_str());
					flushAndExit(FDB_EXIT_ERROR);
				}
				syn = syn.substr(7);
				knobs.emplace_back(syn, args.OptionArg());
				manualKnobOverrides[syn] = args.OptionArg();
				break;
			}
			case OPT_UNITTESTPARAM: {
				std::string syn = args.OptionSyntax();
				if (!StringRef(syn).startsWith(LiteralStringRef("--test_"))) {
					fprintf(stderr, "ERROR: unable to parse knob option '%s'\n", syn.c_str());
					flushAndExit(FDB_EXIT_ERROR);
				}
				testParams.set(syn.substr(7), args.OptionArg());
				break;
			}
			case OPT_LOCALITY: {
				std::string syn = args.OptionSyntax();
				if (!StringRef(syn).startsWith(LiteralStringRef("--locality_"))) {
					fprintf(stderr, "ERROR: unable to parse locality key '%s'\n", syn.c_str());
					flushAndExit(FDB_EXIT_ERROR);
				}
				syn = syn.substr(11);
				std::transform(syn.begin(), syn.end(), syn.begin(), ::tolower);
				localities.set(Standalone<StringRef>(syn), Standalone<StringRef>(std::string(args.OptionArg())));
				break;
			}
			case OPT_VERSION:
				printVersion();
				flushAndExit(FDB_EXIT_SUCCESS);
				break;
			case OPT_BUILD_FLAGS:
				printBuildInformation();
				flushAndExit(FDB_EXIT_SUCCESS);
			case OPT_NOBUFSTDOUT:
				setvbuf(stdout, nullptr, _IONBF, 0);
				setvbuf(stderr, nullptr, _IONBF, 0);
				break;
			case OPT_BUFSTDOUTERR:
				setvbuf(stdout, nullptr, _IOFBF, BUFSIZ);
				setvbuf(stderr, nullptr, _IOFBF, BUFSIZ);
				break;
			case OPT_ROLE:
				sRole = args.OptionArg();
				if (!strcmp(sRole, "fdbd"))
					role = ServerRole::FDBD;
				else if (!strcmp(sRole, "simulation"))
					role = ServerRole::Simulation;
				else if (!strcmp(sRole, "test"))
					role = ServerRole::Test;
				else if (!strcmp(sRole, "multitest"))
					role = ServerRole::MultiTester;
				else if (!strcmp(sRole, "skiplisttest"))
					role = ServerRole::SkipListTest;
				else if (!strcmp(sRole, "search"))
					role = ServerRole::SearchMutations;
				else if (!strcmp(sRole, "dsltest"))
					role = ServerRole::DSLTest;
				else if (!strcmp(sRole, "versionedmaptest"))
					role = ServerRole::VersionedMapTest;
				else if (!strcmp(sRole, "createtemplatedb"))
					role = ServerRole::CreateTemplateDatabase;
				else if (!strcmp(sRole, "networktestclient"))
					role = ServerRole::NetworkTestClient;
				else if (!strcmp(sRole, "networktestserver"))
					role = ServerRole::NetworkTestServer;
				else if (!strcmp(sRole, "restore"))
					role = ServerRole::Restore;
				else if (!strcmp(sRole, "kvfileintegritycheck"))
					role = ServerRole::KVFileIntegrityCheck;
				else if (!strcmp(sRole, "kvfilegeneratesums"))
					role = ServerRole::KVFileGenerateIOLogChecksums;
				else if (!strcmp(sRole, "consistencycheck"))
					role = ServerRole::ConsistencyCheck;
				else if (!strcmp(sRole, "unittests"))
					role = ServerRole::UnitTests;
				else {
					fprintf(stderr, "ERROR: Unknown role `%s'\n", sRole);
					printHelpTeaser(argv[0]);
					flushAndExit(FDB_EXIT_ERROR);
				}
				break;
			case OPT_PUBLICADDR:
				argStr = args.OptionArg();
				boost::split(tmpStrings, argStr, [](char c) { return c == ','; });
				publicAddressStrs.insert(publicAddressStrs.end(), tmpStrings.begin(), tmpStrings.end());
				break;
			case OPT_LISTEN:
				argStr = args.OptionArg();
				boost::split(tmpStrings, argStr, [](char c) { return c == ','; });
				listenAddressStrs.insert(listenAddressStrs.end(), tmpStrings.begin(), tmpStrings.end());
				break;
			case OPT_CONNFILE:
				connFile = args.OptionArg();
				break;
			case OPT_LOGGROUP:
				logGroup = args.OptionArg();
				break;
			case OPT_SEEDCONNFILE:
				seedConnFile = args.OptionArg();
				break;
			case OPT_SEEDCONNSTRING:
				seedConnString = args.OptionArg();
				break;
#ifdef __linux__
			case OPT_FILESYSTEM: {
				fileSystemPath = args.OptionArg();
				break;
			}
			case OPT_PROFILER_RSS_SIZE: {
				const char* a = args.OptionArg();
				char* end;
				rsssize = strtoull(a, &end, 10);
				if (*end) {
					fprintf(stderr, "ERROR: Unrecognized memory size `%s'\n", a);
					printHelpTeaser(argv[0]);
					flushAndExit(FDB_EXIT_ERROR);
				}
				break;
			}
#endif
			case OPT_DATAFOLDER:
				dataFolder = args.OptionArg();
				break;
			case OPT_LOGFOLDER:
				logFolder = args.OptionArg();
				break;
			case OPT_NETWORKIMPL: {
				const char* a = args.OptionArg();
				if (!strcmp(a, "net2"))
					useNet2 = true;
				else if (!strcmp(a, "net2-threadpool")) {
					useNet2 = true;
					useThreadPool = true;
				} else {
					fprintf(stderr, "ERROR: Unknown network implementation `%s'\n", a);
					printHelpTeaser(argv[0]);
					flushAndExit(FDB_EXIT_ERROR);
				}
				break;
			}
			case OPT_TRACECLOCK: {
				const char* a = args.OptionArg();
				if (!strcmp(a, "realtime"))
					g_trace_clock.store(TRACE_CLOCK_REALTIME);
				else if (!strcmp(a, "now"))
					g_trace_clock.store(TRACE_CLOCK_NOW);
				else {
					fprintf(stderr, "ERROR: Unknown clock source `%s'\n", a);
					printHelpTeaser(argv[0]);
					flushAndExit(FDB_EXIT_ERROR);
				}
				break;
			}
			case OPT_NUMTESTERS: {
				const char* a = args.OptionArg();
				if (!sscanf(a, "%d", &minTesterCount)) {
					fprintf(stderr, "ERROR: Could not parse numtesters `%s'\n", a);
					printHelpTeaser(argv[0]);
					flushAndExit(FDB_EXIT_ERROR);
				}
				break;
			}
			case OPT_ROLLSIZE: {
				const char* a = args.OptionArg();
				ti = parse_with_suffix(a);
				if (!ti.present()) {
					fprintf(stderr, "ERROR: Could not parse logsize `%s'\n", a);
					printHelpTeaser(argv[0]);
					flushAndExit(FDB_EXIT_ERROR);
				}
				rollsize = ti.get();
				break;
			}
			case OPT_MAXLOGSSIZE: {
				const char* a = args.OptionArg();
				ti = parse_with_suffix(a);
				if (!ti.present()) {
					fprintf(stderr, "ERROR: Could not parse maxlogssize `%s'\n", a);
					printHelpTeaser(argv[0]);
					flushAndExit(FDB_EXIT_ERROR);
				}
				maxLogsSize = ti.get();
				maxLogsSizeSet = true;
				break;
			}
			case OPT_MAXLOGS: {
				const char* a = args.OptionArg();
				char* end;
				maxLogs = strtoull(a, &end, 10);
				if (*end) {
					fprintf(stderr, "ERROR: Unrecognized maximum number of logs `%s'\n", a);
					printHelpTeaser(argv[0]);
					flushAndExit(FDB_EXIT_ERROR);
				}
				maxLogsSet = true;
				break;
			}
#ifdef _WIN32
			case OPT_PARENTPID: {
				auto pid_str = args.OptionArg();
				int parent_pid = atoi(pid_str);
				auto pHandle = OpenProcess(SYNCHRONIZE, FALSE, parent_pid);
				if (!pHandle) {
					TraceEvent("ParentProcessOpenError").GetLastError();
					fprintf(stderr, "Could not open parent process at pid %d (error %d)", parent_pid, GetLastError());
					throw platform_error();
				}
				startThread(&parentWatcher, pHandle);
				break;
			}
			case OPT_NEWCONSOLE:
				FreeConsole();
				AllocConsole();
				freopen("CONIN$", "rb", stdin);
				freopen("CONOUT$", "wb", stdout);
				freopen("CONOUT$", "wb", stderr);
				break;
			case OPT_NOBOX:
				SetErrorMode(SetErrorMode(0) | SEM_NOGPFAULTERRORBOX);
				break;
#else
			case OPT_PARENTPID: {
				auto pid_str = args.OptionArg();
				int* parent_pid = new (int);
				*parent_pid = atoi(pid_str);
				startThread(&parentWatcher, parent_pid);
				break;
			}
#endif
			case OPT_TRACER: {
				std::string arg = args.OptionArg();
				std::string tracer;
				std::transform(arg.begin(), arg.end(), std::back_inserter(tracer), [](char c) { return tolower(c); });
				if (tracer == "none" || tracer == "disabled") {
					openTracer(TracerType::DISABLED);
				} else if (tracer == "logfile" || tracer == "file" || tracer == "log_file") {
					openTracer(TracerType::LOG_FILE);
				} else if (tracer == "network_lossy") {
					openTracer(TracerType::NETWORK_LOSSY);
				} else {
					fprintf(stderr, "ERROR: Unknown or unsupported tracer: `%s'", args.OptionArg());
					printHelpTeaser(argv[0]);
					flushAndExit(FDB_EXIT_ERROR);
				}
				break;
			}
			case OPT_TESTFILE:
				testFile = args.OptionArg();
				break;
			case OPT_KVFILE:
				kvFile = args.OptionArg();
				break;
			case OPT_RESTARTING:
				restarting = true;
				break;
			case OPT_RANDOMSEED: {
				char* end;
				randomSeed = (uint32_t)strtoul(args.OptionArg(), &end, 0);
				if (*end) {
					fprintf(stderr, "ERROR: Could not parse random seed `%s'\n", args.OptionArg());
					printHelpTeaser(argv[0]);
					flushAndExit(FDB_EXIT_ERROR);
				}
				break;
			}
			case OPT_MACHINEID: {
				zoneId = std::string(args.OptionArg());
				break;
			}
			case OPT_DCID: {
				dcId = std::string(args.OptionArg());
				break;
			}
			case OPT_MACHINE_CLASS:
				sRole = args.OptionArg();
				processClass = ProcessClass(sRole, ProcessClass::CommandLineSource);
				if (processClass == ProcessClass::InvalidClass) {
					fprintf(stderr, "ERROR: Unknown machine class `%s'\n", sRole);
					printHelpTeaser(argv[0]);
					flushAndExit(FDB_EXIT_ERROR);
				}
				break;
			case OPT_KEY:
				targetKey = args.OptionArg();
				break;
			case OPT_MEMLIMIT:
				ti = parse_with_suffix(args.OptionArg(), "MiB");
				if (!ti.present()) {
					fprintf(stderr, "ERROR: Could not parse memory limit from `%s'\n", args.OptionArg());
					printHelpTeaser(argv[0]);
					flushAndExit(FDB_EXIT_ERROR);
				}
				memLimit = ti.get();
				break;
			case OPT_STORAGEMEMLIMIT:
				ti = parse_with_suffix(args.OptionArg(), "MB");
				if (!ti.present()) {
					fprintf(stderr, "ERROR: Could not parse storage memory limit from `%s'\n", args.OptionArg());
					printHelpTeaser(argv[0]);
					flushAndExit(FDB_EXIT_ERROR);
				}
				storageMemLimit = ti.get();
				break;
			case OPT_CACHEMEMLIMIT:
				ti = parse_with_suffix(args.OptionArg(), "MiB");
				if (!ti.present()) {
					fprintf(stderr, "ERROR: Could not parse cache memory limit from `%s'\n", args.OptionArg());
					printHelpTeaser(argv[0]);
					flushAndExit(FDB_EXIT_ERROR);
				}
				// SOMEDAY: ideally we'd have some better way to express that a knob should be elevated to formal
				// parameter
				knobs.emplace_back(
				    "page_cache_4k",
				    format("%ld", ti.get() / 4096 * 4096)); // The cache holds 4K pages, so we can truncate this to the
				                                            // next smaller multiple of 4K.
				break;
			case OPT_BUGGIFY:
				if (!strcmp(args.OptionArg(), "on"))
					buggifyEnabled = true;
				else if (!strcmp(args.OptionArg(), "off"))
					buggifyEnabled = false;
				else {
					fprintf(stderr, "ERROR: Unknown buggify state `%s'\n", args.OptionArg());
					printHelpTeaser(argv[0]);
					flushAndExit(FDB_EXIT_ERROR);
				}
				break;
			case OPT_CRASHONERROR:
				g_crashOnError = true;
				break;
			case OPT_TESTSERVERS:
				testServersStr = args.OptionArg();
				break;
			case OPT_TEST_ON_SERVERS:
				testOnServers = true;
				break;
			case OPT_METRICSCONNFILE:
				metricsConnFile = args.OptionArg();
				break;
			case OPT_METRICSPREFIX:
				metricsPrefix = args.OptionArg();
				break;
			case OPT_IO_TRUST_SECONDS: {
				const char* a = args.OptionArg();
				if (!sscanf(a, "%lf", &fileIoTimeout)) {
					fprintf(stderr, "ERROR: Could not parse io_trust_seconds `%s'\n", a);
					printHelpTeaser(argv[0]);
					flushAndExit(FDB_EXIT_ERROR);
				}
				break;
			}
			case OPT_IO_TRUST_WARN_ONLY:
				fileIoWarnOnly = true;
				break;
			case OPT_TRACE_FORMAT:
				if (!selectTraceFormatter(args.OptionArg())) {
					fprintf(stderr, "WARNING: Unrecognized trace format `%s'\n", args.OptionArg());
				}
				break;
			case OPT_WHITELIST_BINPATH:
				whitelistBinPaths = args.OptionArg();
				break;
			case OPT_BLOB_CREDENTIAL_FILE:
				// Add blob credential following backup agent example
				blobCredentials.push_back(args.OptionArg());
				printf("blob credential file:%s\n", blobCredentials.back().c_str());

				blobCredsFromENV = getenv("FDB_BLOB_CREDENTIALS");
				if (blobCredsFromENV != nullptr) {
					fprintf(stderr, "[WARNING] Set blob credetial via env variable is not tested yet\n");
					TraceEvent(SevError, "FastRestoreGetBlobCredentialFile")
					    .detail("Reason", "Set blob credetial via env variable is not tested yet");
					StringRef t((uint8_t*)blobCredsFromENV, strlen(blobCredsFromENV));
					do {
						StringRef file = t.eat(":");
						if (file.size() != 0) {
							blobCredentials.push_back(file.toString());
						}
					} while (t.size() != 0);
				}
				break;
			case OPT_CONFIG_PATH:
				configPath = args.OptionArg();
				break;
			case OPT_USE_TEST_CONFIG_DB:
				useConfigDB = UseConfigDB::SIMPLE;
				break;

#ifndef TLS_DISABLED
			case TLSConfig::OPT_TLS_PLUGIN:
				args.OptionArg();
				break;
			case TLSConfig::OPT_TLS_CERTIFICATES:
				tlsConfig.setCertificatePath(args.OptionArg());
				break;
			case TLSConfig::OPT_TLS_PASSWORD:
				tlsConfig.setPassword(args.OptionArg());
				break;
			case TLSConfig::OPT_TLS_CA_FILE:
				tlsConfig.setCAPath(args.OptionArg());
				break;
			case TLSConfig::OPT_TLS_KEY:
				tlsConfig.setKeyPath(args.OptionArg());
				break;
			case TLSConfig::OPT_TLS_VERIFY_PEERS:
				tlsConfig.addVerifyPeers(args.OptionArg());
				break;
#endif
			}
		}

		if (seedConnString.length() && seedConnFile.length()) {
			fprintf(
			    stderr, "%s\n", "--seed_cluster_file and --seed_connection_string may not both be specified at once.");
			flushAndExit(FDB_EXIT_ERROR);
		}

		bool seedSpecified = seedConnFile.length() || seedConnString.length();

		if (seedSpecified && !connFile.length()) {
			fprintf(stderr,
			        "%s\n",
			        "If -seed_cluster_file or --seed_connection_string is specified, -C must be specified as well.");
			flushAndExit(FDB_EXIT_ERROR);
		}

		if (metricsConnFile == connFile)
			metricsConnFile = "";

		if (metricsConnFile != "" && metricsPrefix == "") {
			fprintf(stderr, "If a metrics cluster file is specified, a metrics prefix is required.\n");
			flushAndExit(FDB_EXIT_ERROR);
		}

		bool autoPublicAddress =
		    std::any_of(publicAddressStrs.begin(), publicAddressStrs.end(), [](const std::string& addr) {
			    return StringRef(addr).startsWith(LiteralStringRef("auto:"));
		    });
		if ((role != ServerRole::Simulation && role != ServerRole::CreateTemplateDatabase &&
		     role != ServerRole::KVFileIntegrityCheck && role != ServerRole::KVFileGenerateIOLogChecksums &&
		     role != ServerRole::UnitTests) ||
		    autoPublicAddress) {

			if (seedSpecified && !fileExists(connFile)) {
				std::string connectionString = seedConnString.length() ? seedConnString : "";
				ClusterConnectionString ccs;
				if (seedConnFile.length()) {
					try {
						connectionString = readFileBytes(seedConnFile, MAX_CLUSTER_FILE_BYTES);
					} catch (Error& e) {
						fprintf(stderr,
						        "%s\n",
						        ClusterConnectionFile::getErrorString(std::make_pair(seedConnFile, false), e).c_str());
						throw;
					}
				}

				try {
					ccs = ClusterConnectionString(connectionString);
				} catch (Error& e) {
					fprintf(stderr, "%s\n", ClusterConnectionString::getErrorString(connectionString, e).c_str());
					throw;
				}
				connectionFile = makeReference<ClusterConnectionFile>(connFile, ccs);
			} else {
				std::pair<std::string, bool> resolvedClusterFile;
				try {
					resolvedClusterFile = ClusterConnectionFile::lookupClusterFileName(connFile);
					connectionFile = makeReference<ClusterConnectionFile>(resolvedClusterFile.first);
				} catch (Error& e) {
					fprintf(stderr, "%s\n", ClusterConnectionFile::getErrorString(resolvedClusterFile, e).c_str());
					throw;
				}
			}

			// failmon?
		}

		try {
			if (!publicAddressStrs.empty()) {
				std::tie(publicAddresses, listenAddresses) =
				    buildNetworkAddresses(*connectionFile, publicAddressStrs, listenAddressStrs);
			}
		} catch (Error&) {
			printHelpTeaser(argv[0]);
			flushAndExit(FDB_EXIT_ERROR);
		}

		if (role == ServerRole::ConsistencyCheck) {
			if (!publicAddressStrs.empty()) {
				fprintf(stderr, "ERROR: Public address cannot be specified for consistency check processes\n");
				printHelpTeaser(argv[0]);
				flushAndExit(FDB_EXIT_ERROR);
			}
			auto publicIP = determinePublicIPAutomatically(connectionFile->getConnectionString());
			publicAddresses.address = NetworkAddress(publicIP, ::getpid());
		}

		if (role == ServerRole::Simulation) {
			Optional<bool> buggifyOverride = checkBuggifyOverride(testFile);
			if (buggifyOverride.present())
				buggifyEnabled = buggifyOverride.get();
		}

		if (role == ServerRole::SearchMutations && !targetKey) {
			fprintf(stderr, "ERROR: please specify a target key\n");
			printHelpTeaser(argv[0]);
			flushAndExit(FDB_EXIT_ERROR);
		}

		if (role == ServerRole::NetworkTestClient && !testServersStr.size()) {
			fprintf(stderr, "ERROR: please specify --testservers\n");
			printHelpTeaser(argv[0]);
			flushAndExit(FDB_EXIT_ERROR);
		}

		// Interpret legacy "maxLogs" option in the most sensible and unsurprising way we can while eliminating its code
		// path
		if (maxLogsSet) {
			if (maxLogsSizeSet) {
				// This is the case where both options are set and we must deconflict.
				auto maxLogsAsSize = maxLogs * rollsize;

				// If either was unlimited, then the safe option here is to take the larger one.
				//  This means that is one of the two options specified a limited amount of logging
				//  then the option that specified "unlimited" will be ignored.
				if (maxLogsSize == 0 || maxLogs == 0)
					maxLogsSize = std::max(maxLogsSize, maxLogsAsSize);
				else
					maxLogsSize = std::min(maxLogsSize, maxLogs * rollsize);
			} else {
				maxLogsSize = maxLogs * rollsize;
			}
		}
		if (!zoneId.present() &&
		    !(localities.isPresent(LocalityData::keyZoneId) && localities.isPresent(LocalityData::keyMachineId))) {
			machineId = getSharedMemoryMachineId().toString();
		}
		if (!localities.isPresent(LocalityData::keyZoneId))
			localities.set(LocalityData::keyZoneId, zoneId.present() ? zoneId : machineId);

		if (!localities.isPresent(LocalityData::keyMachineId))
			localities.set(LocalityData::keyMachineId, zoneId.present() ? zoneId : machineId);

		if (!localities.isPresent(LocalityData::keyDcId) && dcId.present())
			localities.set(LocalityData::keyDcId, dcId);
	}
};
} // namespace

int main(int argc, char* argv[]) {
	try {
		platformInit();

#ifdef ALLOC_INSTRUMENTATION
		g_extra_memory = new uint8_t[1000000];
#endif
		registerCrashHandler();

		// Set default of line buffering standard out and error
		setvbuf(stdout, nullptr, _IOLBF, BUFSIZ);
		setvbuf(stderr, nullptr, _IOLBF, BUFSIZ);

		// Enables profiling on this thread (but does not start it)
		registerThreadForProfiling();

#ifdef _WIN32
		// Windows needs a gentle nudge to format floats correctly
		//_set_output_format(_TWO_DIGIT_EXPONENT);
#endif

		const auto opts = CLIOptions::parseArgs(argc, argv);
		const auto role = opts.role;

		if (role == ServerRole::Simulation)
			printf("Random seed is %u...\n", opts.randomSeed);

		if (opts.zoneId.present())
			printf("ZoneId set to %s, dcId to %s\n", printable(opts.zoneId).c_str(), printable(opts.dcId).c_str());

		setThreadLocalDeterministicRandomSeed(opts.randomSeed);

		enableBuggify(opts.buggifyEnabled, BuggifyType::General);

		IKnobCollection::setGlobalKnobCollection(IKnobCollection::Type::SERVER,
		                                         Randomize::True,
		                                         role == ServerRole::Simulation ? IsSimulated::True
		                                                                        : IsSimulated::False);
		IKnobCollection::getMutableGlobalKnobCollection().setKnob("log_directory", KnobValue::create(opts.logFolder));
		if (role != ServerRole::Simulation) {
			IKnobCollection::getMutableGlobalKnobCollection().setKnob("commit_batches_mem_bytes_hard_limit",
			                                                          KnobValue::create(int64_t{ opts.memLimit }));
		}

		for (const auto& [knobName, knobValueString] : opts.knobs) {
			try {
				auto& g_knobs = IKnobCollection::getMutableGlobalKnobCollection();
				auto knobValue = g_knobs.parseKnobValue(knobName, knobValueString);
				g_knobs.setKnob(knobName, knobValue);
			} catch (Error& e) {
				if (e.code() == error_code_invalid_option_value) {
					fprintf(stderr,
					        "WARNING: Invalid value '%s' for knob option '%s'\n",
					        knobName.c_str(),
					        knobValueString.c_str());
					TraceEvent(SevWarnAlways, "InvalidKnobValue")
					    .detail("Knob", printable(knobName))
					    .detail("Value", printable(knobValueString));
				} else {
					fprintf(stderr, "ERROR: Failed to set knob option '%s': %s\n", knobName.c_str(), e.what());
					TraceEvent(SevError, "FailedToSetKnob")
					    .detail("Knob", printable(knobName))
					    .detail("Value", printable(knobValueString))
					    .error(e);
					throw;
				}
			}
		}
		IKnobCollection::getMutableGlobalKnobCollection().setKnob("server_mem_limit",
		                                                          KnobValue::create(int64_t{ opts.memLimit }));
		// Reinitialize knobs in order to update knobs that are dependent on explicitly set knobs
		IKnobCollection::getMutableGlobalKnobCollection().initialize(
		    Randomize::True, role == ServerRole::Simulation ? IsSimulated::True : IsSimulated::False);

		// evictionPolicyStringToEnum will throw an exception if the string is not recognized as a valid
		EvictablePageCache::evictionPolicyStringToEnum(FLOW_KNOBS->CACHE_EVICTION_POLICY);

		if (opts.memLimit <= FLOW_KNOBS->PAGE_CACHE_4K) {
			fprintf(stderr, "ERROR: --memory has to be larger than --cache_memory\n");
			flushAndExit(FDB_EXIT_ERROR);
		}

		if (role == ServerRole::SkipListTest) {
			skipListTest();
			flushAndExit(FDB_EXIT_SUCCESS);
		}

		if (role == ServerRole::DSLTest) {
			dsltest();
			flushAndExit(FDB_EXIT_SUCCESS);
		}

		if (role == ServerRole::VersionedMapTest) {
			versionedMapTest();
			flushAndExit(FDB_EXIT_SUCCESS);
		}

		// Initialize the thread pool
		CoroThreadPool::init();
		// Ordinarily, this is done when the network is run. However, network thread should be set before TraceEvents
		// are logged. This thread will eventually run the network, so call it now.
		TraceEvent::setNetworkThread();

		std::vector<Future<Void>> listenErrors;

		if (role == ServerRole::Simulation || role == ServerRole::CreateTemplateDatabase) {
			// startOldSimulator();
			startNewSimulator();
			openTraceFile(NetworkAddress(), opts.rollsize, opts.maxLogsSize, opts.logFolder, "trace", opts.logGroup);
			openTracer(TracerType(deterministicRandom()->randomInt(static_cast<int>(TracerType::DISABLED),
			                                                       static_cast<int>(TracerType::SIM_END))));
		} else {
			g_network = newNet2(opts.tlsConfig, opts.useThreadPool, true);
			g_network->addStopCallback(Net2FileSystem::stop);
			FlowTransport::createInstance(false, 1);

			const bool expectsPublicAddress =
			    (role == ServerRole::FDBD || role == ServerRole::NetworkTestServer || role == ServerRole::Restore);
			if (opts.publicAddressStrs.empty()) {
				if (expectsPublicAddress) {
					fprintf(stderr, "ERROR: The -p or --public_address option is required\n");
					printHelpTeaser(argv[0]);
					flushAndExit(FDB_EXIT_ERROR);
				}
			}

			openTraceFile(
			    opts.publicAddresses.address, opts.rollsize, opts.maxLogsSize, opts.logFolder, "trace", opts.logGroup);
			g_network->initTLS();

			if (expectsPublicAddress) {
				for (int ii = 0; ii < (opts.publicAddresses.secondaryAddress.present() ? 2 : 1); ++ii) {
					const NetworkAddress& publicAddress =
					    ii == 0 ? opts.publicAddresses.address : opts.publicAddresses.secondaryAddress.get();
					const NetworkAddress& listenAddress =
					    ii == 0 ? opts.listenAddresses.address : opts.listenAddresses.secondaryAddress.get();
					try {
						const Future<Void>& errorF = FlowTransport::transport().bind(publicAddress, listenAddress);
						listenErrors.push_back(errorF);
						if (errorF.isReady())
							errorF.get();
					} catch (Error& e) {
						TraceEvent("BindError").error(e);
						fprintf(stderr,
						        "Error initializing networking with public address %s and listen address %s (%s)\n",
						        publicAddress.toString().c_str(),
						        listenAddress.toString().c_str(),
						        e.what());
						printHelpTeaser(argv[0]);
						flushAndExit(FDB_EXIT_ERROR);
					}
				}
			}

			// Use a negative ioTimeout to indicate warn-only
			Net2FileSystem::newFileSystem(opts.fileIoWarnOnly ? -opts.fileIoTimeout : opts.fileIoTimeout,
			                              opts.fileSystemPath);
			g_network->initMetrics();
			FlowTransport::transport().initMetrics();
			initTraceEventMetrics();
		}

		double start = timer(), startNow = now();

		std::string cwd = "<unknown>";
		try {
			cwd = platform::getWorkingDirectory();
		} catch (Error& e) {
			// Allow for platform error by rethrowing all _other_ errors
			if (e.code() != error_code_platform_error)
				throw;
		}

		TraceEvent("ProgramStart")
		    .setMaxEventLength(12000)
		    .detail("RandomSeed", opts.randomSeed)
		    .detail("SourceVersion", getSourceVersion())
		    .detail("Version", FDB_VT_VERSION)
		    .detail("PackageName", FDB_VT_PACKAGE_NAME)
		    .detail("FileSystem", opts.fileSystemPath)
		    .detail("DataFolder", opts.dataFolder)
		    .detail("WorkingDirectory", cwd)
		    .detail("ClusterFile", opts.connectionFile ? opts.connectionFile->getFilename().c_str() : "")
		    .detail("ConnectionString",
		            opts.connectionFile ? opts.connectionFile->getConnectionString().toString() : "")
		    .detailf("ActualTime", "%lld", DEBUG_DETERMINISM ? 0 : time(nullptr))
		    .setMaxFieldLength(10000)
		    .detail("CommandLine", opts.commandLine)
		    .setMaxFieldLength(0)
		    .detail("BuggifyEnabled", opts.buggifyEnabled)
		    .detail("MemoryLimit", opts.memLimit)
		    .trackLatest("ProgramStart");

		Error::init();
		std::set_new_handler(&platform::outOfMemory);
		setMemoryQuota(opts.memLimit);

		Future<Optional<Void>> f;

		if (role == ServerRole::Simulation) {
			TraceEvent("Simulation").detail("TestFile", opts.testFile);

			auto histogramReportActor = histogramReport();

			CLIENT_KNOBS->trace();
			FLOW_KNOBS->trace();
			SERVER_KNOBS->trace();

			auto dataFolder = opts.dataFolder.size() ? opts.dataFolder : "simfdb";
			std::vector<std::string> directories = platform::listDirectories(dataFolder);
			const std::set<std::string> allowedDirectories = { ".", "..", "backups", "unittests" };

			for (const auto& dir : directories) {
				if (dir.size() != 32 && allowedDirectories.count(dir) == 0 && dir.find("snap") == std::string::npos) {

					TraceEvent(SevError, "IncompatibleDirectoryFound")
					    .detail("DataFolder", dataFolder)
					    .detail("SuspiciousFile", dir);

					fprintf(stderr,
					        "ERROR: Data folder `%s' had non fdb file `%s'; please use clean, fdb-only folder\n",
					        dataFolder.c_str(),
					        dir.c_str());

					flushAndExit(FDB_EXIT_ERROR);
				}
			}
			std::vector<std::string> files = platform::listFiles(dataFolder);
			if ((files.size() > 1 || (files.size() == 1 && files[0] != "restartInfo.ini")) && !opts.restarting) {
				TraceEvent(SevError, "IncompatibleFileFound").detail("DataFolder", dataFolder);
				fprintf(stderr,
				        "ERROR: Data folder `%s' is non-empty; please use clean, fdb-only folder\n",
				        dataFolder.c_str());
				flushAndExit(FDB_EXIT_ERROR);
			} else if (files.empty() && opts.restarting) {
				TraceEvent(SevWarnAlways, "FileNotFound").detail("DataFolder", dataFolder);
				printf("ERROR: Data folder `%s' is empty, but restarting option selected. Run Phase 1 test first\n",
				       dataFolder.c_str());
				flushAndExit(FDB_EXIT_ERROR);
			}

			int isRestoring = 0;
			if (!opts.restarting) {
				platform::eraseDirectoryRecursive(dataFolder);
				platform::createDirectory(dataFolder);
			} else {
				CSimpleIni ini;
				ini.SetUnicode();
				std::string absDataFolder = abspath(dataFolder);
				ini.LoadFile(joinPath(absDataFolder, "restartInfo.ini").c_str());
				int backupFailed = true;
				const char* isRestoringStr = ini.GetValue("RESTORE", "isRestoring", nullptr);
				if (isRestoringStr) {
					isRestoring = atoi(isRestoringStr);
					const char* backupFailedStr = ini.GetValue("RESTORE", "BackupFailed", nullptr);
					if (isRestoring && backupFailedStr) {
						backupFailed = atoi(backupFailedStr);
					}
				}
				if (isRestoring && !backupFailed) {
					std::vector<std::string> returnList;
					std::string ext = "";
					returnList = platform::listDirectories(absDataFolder);
					std::string snapStr = ini.GetValue("RESTORE", "RestoreSnapUID");

					TraceEvent("RestoringDataFolder").detail("DataFolder", absDataFolder);
					TraceEvent("RestoreSnapUID").detail("UID", snapStr);

					// delete all files (except fdb.cluster) in non-snap directories
					for (const auto& dirEntry : returnList) {
						if (dirEntry == "." || dirEntry == "..") {
							continue;
						}
						if (dirEntry.find(snapStr) != std::string::npos) {
							continue;
						}

						std::string childf = absDataFolder + "/" + dirEntry;
						std::vector<std::string> returnFiles = platform::listFiles(childf, ext);
						for (const auto& fileEntry : returnFiles) {
							if (fileEntry != "fdb.cluster" && fileEntry != "fitness") {
								TraceEvent("DeletingNonSnapfiles").detail("FileBeingDeleted", childf + "/" + fileEntry);
								deleteFile(childf + "/" + fileEntry);
							}
						}
					}
					// cleanup unwanted and partial directories
					for (const auto& dirEntry : returnList) {
						if (dirEntry == "." || dirEntry == "..") {
							continue;
						}
						std::string dirSrc = absDataFolder + "/" + dirEntry;
						// delete snap directories which are not part of restoreSnapUID
						if (dirEntry.find(snapStr) == std::string::npos) {
							if (dirEntry.find("snap") != std::string::npos) {
								platform::eraseDirectoryRecursive(dirSrc);
							}
							continue;
						}
						// remove empty/partial snap directories
						std::vector<std::string> childrenList = platform::listFiles(dirSrc);
						if (childrenList.size() == 0) {
							TraceEvent("RemovingEmptySnapDirectory").detail("DirBeingDeleted", dirSrc);
							platform::eraseDirectoryRecursive(dirSrc);
							continue;
						}
					}
					// move snapshotted files to appropriate locations
					for (const auto& dirEntry : returnList) {
						if (dirEntry == "." || dirEntry == "..") {
							continue;
						}
						std::string dirSrc = absDataFolder + "/" + dirEntry;
						std::string origDir = dirEntry.substr(0, 32);
						std::string dirToMove = absDataFolder + "/" + origDir;
						if ((dirEntry.find("snap") != std::string::npos) &&
						    (dirEntry.find("tlog") != std::string::npos)) {
							// restore tlog files
							restoreRoleFilesHelper(dirSrc, dirToMove, "log");
						} else if ((dirEntry.find("snap") != std::string::npos) &&
						           (dirEntry.find("storage") != std::string::npos)) {
							// restore storage files
							restoreRoleFilesHelper(dirSrc, dirToMove, "storage");
						} else if ((dirEntry.find("snap") != std::string::npos) &&
						           (dirEntry.find("coord") != std::string::npos)) {
							// restore coordinator files
							restoreRoleFilesHelper(dirSrc, dirToMove, "coordination");
						}
					}
				}
			}
			setupAndRun(dataFolder, opts.testFile, opts.restarting, (isRestoring >= 1), opts.whitelistBinPaths);
			g_simulator.run();
		} else if (role == ServerRole::FDBD) {
			// Update the global blob credential files list so that both fast
			// restore workers and backup workers can access blob storage.
			std::vector<std::string>* pFiles =
			    (std::vector<std::string>*)g_network->global(INetwork::enBlobCredentialFiles);
			if (pFiles != nullptr) {
				for (auto& f : opts.blobCredentials) {
					pFiles->push_back(f);
				}
			}

			// Call fast restore for the class FastRestoreClass. This is a short-cut to run fast restore in circus
			if (opts.processClass == ProcessClass::FastRestoreClass) {
				printf("Run as fast restore worker\n");
				ASSERT(opts.connectionFile);
				auto dataFolder = opts.dataFolder;
				if (!dataFolder.size())
					dataFolder = format("fdb/%d/", opts.publicAddresses.address.port); // SOMEDAY: Better default

				vector<Future<Void>> actors(listenErrors.begin(), listenErrors.end());
				actors.push_back(restoreWorker(opts.connectionFile, opts.localities, dataFolder));
				f = stopAfter(waitForAll(actors));
				printf("Fast restore worker started\n");
				g_network->run();
				printf("g_network->run() done\n");
			} else { // Call fdbd roles in conventional way
				ASSERT(opts.connectionFile);

				setupRunLoopProfiler();

				auto dataFolder = opts.dataFolder;
				if (!dataFolder.size())
					dataFolder = format("fdb/%d/", opts.publicAddresses.address.port); // SOMEDAY: Better default

				vector<Future<Void>> actors(listenErrors.begin(), listenErrors.end());
				actors.push_back(fdbd(opts.connectionFile,
				                      opts.localities,
				                      opts.processClass,
				                      dataFolder,
				                      dataFolder,
				                      opts.storageMemLimit,
				                      opts.metricsConnFile,
				                      opts.metricsPrefix,
				                      opts.rsssize,
				                      opts.whitelistBinPaths,
				                      opts.configPath,
				                      opts.manualKnobOverrides,
				                      opts.useConfigDB));
				actors.push_back(histogramReport());
				// actors.push_back( recurring( []{}, .001 ) );  // for ASIO latency measurement

				f = stopAfter(waitForAll(actors));
				g_network->run();
			}
		} else if (role == ServerRole::MultiTester) {
			setupRunLoopProfiler();
			f = stopAfter(runTests(opts.connectionFile,
			                       TEST_TYPE_FROM_FILE,
			                       opts.testOnServers ? TEST_ON_SERVERS : TEST_ON_TESTERS,
			                       opts.minTesterCount,
			                       opts.testFile,
			                       StringRef(),
			                       opts.localities));
			g_network->run();
		} else if (role == ServerRole::Test) {
			setupRunLoopProfiler();
			auto m = startSystemMonitor(opts.dataFolder, opts.dcId, opts.zoneId, opts.zoneId);
			f = stopAfter(runTests(
			    opts.connectionFile, TEST_TYPE_FROM_FILE, TEST_HERE, 1, opts.testFile, StringRef(), opts.localities));
			g_network->run();
		} else if (role == ServerRole::ConsistencyCheck) {
			setupRunLoopProfiler();

			auto m = startSystemMonitor(opts.dataFolder, opts.dcId, opts.zoneId, opts.zoneId);
			f = stopAfter(runTests(opts.connectionFile,
			                       TEST_TYPE_CONSISTENCY_CHECK,
			                       TEST_HERE,
			                       1,
			                       opts.testFile,
			                       StringRef(),
			                       opts.localities));
			g_network->run();
		} else if (role == ServerRole::UnitTests) {
			setupRunLoopProfiler();
			auto m = startSystemMonitor(opts.dataFolder, opts.dcId, opts.zoneId, opts.zoneId);
			f = stopAfter(runTests(opts.connectionFile,
			                       TEST_TYPE_UNIT_TESTS,
			                       TEST_HERE,
			                       1,
			                       opts.testFile,
			                       StringRef(),
			                       opts.localities,
			                       opts.testParams));
			g_network->run();
		} else if (role == ServerRole::CreateTemplateDatabase) {
			createTemplateDatabase();
		} else if (role == ServerRole::NetworkTestClient) {
			f = stopAfter(networkTestClient(opts.testServersStr));
			g_network->run();
		} else if (role == ServerRole::NetworkTestServer) {
			f = stopAfter(networkTestServer());
			g_network->run();
		} else if (role == ServerRole::Restore) {
			f = stopAfter(restoreWorker(opts.connectionFile, opts.localities, opts.dataFolder));
			g_network->run();
		} else if (role == ServerRole::KVFileIntegrityCheck) {
			f = stopAfter(KVFileCheck(opts.kvFile, true));
			g_network->run();
		} else if (role == ServerRole::KVFileGenerateIOLogChecksums) {
			Optional<Void> result;
			try {
				GenerateIOLogChecksumFile(opts.kvFile);
				result = Void();
			} catch (Error& e) {
				fprintf(stderr, "Fatal Error: %s\n", e.what());
			}

			f = result;
		}

		int rc = FDB_EXIT_SUCCESS;
		if (f.isValid() && f.isReady() && !f.isError() && !f.get().present()) {
			rc = FDB_EXIT_ERROR;
		}

		int unseed = noUnseed ? 0 : deterministicRandom()->randomInt(0, 100001);
		TraceEvent("ElapsedTime")
		    .detail("SimTime", now() - startNow)
		    .detail("RealTime", timer() - start)
		    .detail("RandomUnseed", unseed);

		if (role == ServerRole::Simulation) {
			printf("Unseed: %d\n", unseed);
			printf("Elapsed: %f simsec, %f real seconds\n", now() - startNow, timer() - start);
		}

		// IFailureMonitor::failureMonitor().address_info.clear();

		// we should have shut down ALL actors associated with this machine; let's list all of the ones still live
		/*{
		    auto living = Actor::all;
		    printf("%d surviving actors:\n", living.size());
		    for(auto a = living.begin(); a != living.end(); ++a)
		        printf("  #%lld %s %p\n", (*a)->creationIndex, (*a)->getName(), (*a));
		}

		{
		    auto living = DatabaseContext::all;
		    printf("%d surviving DatabaseContexts:\n", living.size());
		    for(auto a = living.begin(); a != living.end(); ++a)
		        printf("  #%lld %p\n", (*a)->creationIndex, (*a));
		}

		{
		    auto living = TransactionData::all;
		    printf("%d surviving TransactionData(s):\n", living.size());
		    for(auto a = living.begin(); a != living.end(); ++a)
		        printf("  #%lld %p\n", (*a)->creationIndex, (*a));
		}*/

		/*cout << Actor::allActors.size() << " surviving actors:" << endl;
		std::map<std::string,int> actorCount;
		for(int i=0; i<Actor::allActors.size(); i++)
		    ++actorCount[Actor::allActors[i]->getName()];
		for(auto i = actorCount.rbegin(); !(i == actorCount.rend()); ++i)
		    cout << "  " << i->second << " " << i->first << endl;*/
		//	cout << "  " << Actor::allActors[i]->getName() << endl;

		if (role == ServerRole::Simulation) {
			unsigned long sevErrorEventsLogged = TraceEvent::CountEventsLoggedAt(SevError);
			if (sevErrorEventsLogged > 0) {
				printf("%lu SevError events logged\n", sevErrorEventsLogged);
				rc = FDB_EXIT_ERROR;
			}
		}

		// g_simulator.run();

#ifdef ALLOC_INSTRUMENTATION
		{
			std::cout << "Page Counts: " << FastAllocator<16>::pageCount << " " << FastAllocator<32>::pageCount << " "
			          << FastAllocator<64>::pageCount << " " << FastAllocator<128>::pageCount << " "
			          << FastAllocator<256>::pageCount << " " << FastAllocator<512>::pageCount << " "
			          << FastAllocator<1024>::pageCount << " " << FastAllocator<2048>::pageCount << " "
			          << FastAllocator<4096>::pageCount << " " << FastAllocator<8192>::pageCount << " "
			          << FastAllocator<16384>::pageCount << std::endl;

			vector<std::pair<std::string, const char*>> typeNames;
			for (auto i = allocInstr.begin(); i != allocInstr.end(); ++i) {
				std::string s;

#ifdef __linux__
				char* demangled = abi::__cxa_demangle(i->first, nullptr, nullptr, nullptr);
				if (demangled) {
					s = demangled;
					if (StringRef(s).startsWith(LiteralStringRef("(anonymous namespace)::")))
						s = s.substr(LiteralStringRef("(anonymous namespace)::").size());
					free(demangled);
				} else
					s = i->first;
#else
				s = i->first;
				if (StringRef(s).startsWith(LiteralStringRef("class `anonymous namespace'::")))
					s = s.substr(LiteralStringRef("class `anonymous namespace'::").size());
				else if (StringRef(s).startsWith(LiteralStringRef("class ")))
					s = s.substr(LiteralStringRef("class ").size());
				else if (StringRef(s).startsWith(LiteralStringRef("struct ")))
					s = s.substr(LiteralStringRef("struct ").size());
#endif

				typeNames.emplace_back(s, i->first);
			}
			std::sort(typeNames.begin(), typeNames.end());
			for (int i = 0; i < typeNames.size(); i++) {
				const char* n = typeNames[i].second;
				auto& f = allocInstr[n];
				printf("%+d\t%+d\t%d\t%d\t%s\n",
				       f.allocCount,
				       -f.deallocCount,
				       f.allocCount - f.deallocCount,
				       f.maxAllocated,
				       typeNames[i].first.c_str());
			}

			// We're about to exit and clean up data structures, this will wreak havoc on allocation recording
			memSample_entered = true;
		}
#endif
		// printf("\n%d tests passed; %d tests failed\n", passCount, failCount);
		flushAndExit(rc);
	} catch (Error& e) {
		fprintf(stderr, "Error: %s\n", e.what());
		TraceEvent(SevError, "MainError").error(e);
		// printf("\n%d tests passed; %d tests failed\n", passCount, failCount);
		flushAndExit(FDB_EXIT_MAIN_ERROR);
	} catch (boost::system::system_error& e) {
		ASSERT_WE_THINK(false); // boost errors shouldn't leak
		fprintf(stderr, "boost::system::system_error: %s (%d)", e.what(), e.code().value());
		TraceEvent(SevError, "MainError").error(unknown_error()).detail("RootException", e.what());
		// printf("\n%d tests passed; %d tests failed\n", passCount, failCount);
		flushAndExit(FDB_EXIT_MAIN_EXCEPTION);
	} catch (std::exception& e) {
		fprintf(stderr, "std::exception: %s\n", e.what());
		TraceEvent(SevError, "MainError").error(unknown_error()).detail("RootException", e.what());
		// printf("\n%d tests passed; %d tests failed\n", passCount, failCount);
		flushAndExit(FDB_EXIT_MAIN_EXCEPTION);
	}

	static_assert(LBLocalityData<StorageServerInterface>::Present, "Storage server interface should be load balanced");
	static_assert(LBLocalityData<CommitProxyInterface>::Present, "Commit proxy interface should be load balanced");
	static_assert(LBLocalityData<GrvProxyInterface>::Present, "GRV proxy interface should be load balanced");
	static_assert(LBLocalityData<TLogInterface>::Present, "TLog interface should be load balanced");
	static_assert(!LBLocalityData<MasterInterface>::Present, "Master interface should not be load balanced");
}