apple/foundationdb
1
0
Fork 0
mirror of https://github.com/apple/foundationdb.git synced 2025-05-15 02:18:39 +08:00
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #6473 from sfc-gh-tclinkenbeard/change-data-hall

Browse Source 
Test renaming data hall on restart
This commit is contained in:
Trevor Clinkenbeard 2022-03-28 16:39:43 -07:00 committed by GitHub
commit ad98d64799
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
4 changed files with 59 additions and 41 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

11
fdbrpc/simulator.h
View File

@ -37,12 +37,6 @@ enum ClogMode { ClogDefault, ClogAll, ClogSend, ClogReceive };
class ISimulator : public INetwork {
public:
ISimulator()
: desiredCoordinators(1), physicalDatacenters(1), processesPerMachine(0), listenersPerProcess(1),
extraDB(nullptr), usableRegions(1), allowLogSetKills(true), tssMode(TSSMode::Disabled), isStopped(false),
lastConnectionFailure(0), connectionFailuresDisableDuration(0), speedUpSimulation(false),
backupAgents(BackupAgentType::WaitForType), drAgents(BackupAgentType::WaitForType), allSwapsDisabled(false) {}
// Order matters!
enum KillType {
KillInstantly,
@ -393,7 +387,7 @@ public:
int listenersPerProcess;
std::set<NetworkAddress> protectedAddresses;
std::map<NetworkAddress, ProcessInfo*> currentlyRebootingProcesses;
class ClusterConnectionString* extraDB;
std::unique_ptr<class ClusterConnectionString> extraDB;
Reference<IReplicationPolicy> storagePolicy;
Reference<IReplicationPolicy> tLogPolicy;
int32_t tLogWriteAntiQuorum;
@ -456,6 +450,9 @@ public:
return false;
}
ISimulator();
virtual ~ISimulator();
protected:
Mutex mutex;

57
fdbserver/SimulatedCluster.actor.cpp
View File

@ -53,6 +53,13 @@
extern "C" int g_expect_full_pointermap;
extern const char* getSourceVersion();
ISimulator::ISimulator()
: desiredCoordinators(1), physicalDatacenters(1), processesPerMachine(0), listenersPerProcess(1), usableRegions(1),
allowLogSetKills(true), tssMode(TSSMode::Disabled), isStopped(false), lastConnectionFailure(0),
connectionFailuresDisableDuration(0), speedUpSimulation(false), backupAgents(BackupAgentType::WaitForType),
drAgents(BackupAgentType::WaitForType), allSwapsDisabled(false) {}
ISimulator::~ISimulator() = default;
using namespace std::literals;
// TODO: Defining these here is just asking for ODR violations.
@ -266,6 +273,9 @@ class TestConfig {
configDBType = configDBTypeFromString(value);
}
}
if (attrib == "randomlyRenameZoneId") {
randomlyRenameZoneId = strcmp(value.c_str(), "true") == 0;
}
if (attrib == "blobGranulesEnabled") {
blobGranulesEnabled = strcmp(value.c_str(), "true") == 0;
}
@ -307,6 +317,7 @@ public:
stderrSeverity, machineCount, processesPerMachine, coordinators;
bool blobGranulesEnabled = false;
Optional<std::string> config;
bool randomlyRenameZoneId = false;
bool allowDefaultTenant = true;
bool allowDisablingTenants = true;
@ -364,7 +375,8 @@ public:
.add("extraMachineCountDC", &extraMachineCountDC)
.add("blobGranulesEnabled", &blobGranulesEnabled)
.add("allowDefaultTenant", &allowDefaultTenant)
.add("allowDisablingTenants", &allowDisablingTenants);
.add("allowDisablingTenants", &allowDisablingTenants)
.add("randomlyRenameZoneId", &randomlyRenameZoneId);
try {
auto file = toml::parse(testFile);
if (file.contains("configuration") && toml::find(file, "configuration").is_table()) {
@ -1039,6 +1051,11 @@ ACTOR Future<Void> restartSimulatedSystem(std::vector<Future<Void>>* systemActor
auto configDBType = testConfig.getConfigDBType();
// Randomly change data center id names to test that localities
// can be modified on cluster restart
bool renameZoneIds = testConfig.randomlyRenameZoneId ? deterministicRandom()->random01() < 0.1 : false;
TEST(renameZoneIds); // Zone ID names altered in restart test
// allows multiple ipAddr entries
ini.SetMultiKey();
@ -1059,7 +1076,7 @@ ACTOR Future<Void> restartSimulatedSystem(std::vector<Future<Void>>* systemActor
bool enableExtraDB = (testConfig.extraDB == 3);
ClusterConnectionString conn(ini.GetValue("META", "connectionString"));
if (enableExtraDB) {
g_simulator.extraDB = new ClusterConnectionString(ini.GetValue("META", "connectionString"));
g_simulator.extraDB = std::make_unique<ClusterConnectionString>(ini.GetValue("META", "connectionString"));
}
if (!testConfig.disableHostname) {
auto mockDNSStr = ini.GetValue("META", "mockDNS");
@ -1087,7 +1104,11 @@ ACTOR Future<Void> restartSimulatedSystem(std::vector<Future<Void>>* systemActor
if (zoneIDini == nullptr) {
zoneId = machineId;
} else {
zoneId = StringRef(zoneIDini);
auto zoneIdStr = std::string(zoneIDini);
if (renameZoneIds) {
zoneIdStr = "modified/" + zoneIdStr;
}
zoneId = Standalone<StringRef>(zoneIdStr);
}
ProcessClass::ClassType cType =
@ -1142,7 +1163,7 @@ ACTOR Future<Void> restartSimulatedSystem(std::vector<Future<Void>>* systemActor
}
LocalityData localities(Optional<Standalone<StringRef>>(), zoneId, machineId, dcUID);
localities.set(LiteralStringRef("data_hall"), dcUID);
localities.set("data_hall"_sr, dcUID);
// SOMEDAY: parse backup agent from test file
systemActors->push_back(reportErrors(
@ -2040,9 +2061,9 @@ void setupSimulatedSystem(std::vector<Future<Void>>* systemActors,
deterministicRandom()->randomShuffle(coordinatorAddresses);
ASSERT_EQ(coordinatorAddresses.size(), coordinatorCount);
ClusterConnectionString conn(coordinatorAddresses, LiteralStringRef("TestCluster:0"));
ClusterConnectionString conn(coordinatorAddresses, "TestCluster:0"_sr);
if (useHostname) {
conn = ClusterConnectionString(coordinatorHostnames, LiteralStringRef("TestCluster:0"));
conn = ClusterConnectionString(coordinatorHostnames, "TestCluster:0"_sr);
}
// If extraDB==0, leave g_simulator.extraDB as null because the test does not use DR.
@ -2050,21 +2071,21 @@ void setupSimulatedSystem(std::vector<Future<Void>>* systemActors,
// The DR database can be either a new database or itself
g_simulator.extraDB =
BUGGIFY
? (useHostname ? new ClusterConnectionString(coordinatorHostnames, LiteralStringRef("TestCluster:0"))
: new ClusterConnectionString(coordinatorAddresses, LiteralStringRef("TestCluster:0")))
? (useHostname ? std::make_unique<ClusterConnectionString>(coordinatorHostnames, "TestCluster:0"_sr)
: std::make_unique<ClusterConnectionString>(coordinatorAddresses, "TestCluster:0"_sr))
: (useHostname
? new ClusterConnectionString(extraCoordinatorHostnames, LiteralStringRef("ExtraCluster:0"))
: new ClusterConnectionString(extraCoordinatorAddresses, LiteralStringRef("ExtraCluster:0")));
? std::make_unique<ClusterConnectionString>(extraCoordinatorHostnames, "ExtraCluster:0"_sr)
: std::make_unique<ClusterConnectionString>(extraCoordinatorAddresses, "ExtraCluster:0"_sr));
} else if (testConfig.extraDB == 2) {
// The DR database is a new database
g_simulator.extraDB =
useHostname ? new ClusterConnectionString(extraCoordinatorHostnames, LiteralStringRef("ExtraCluster:0"))
: new ClusterConnectionString(extraCoordinatorAddresses, LiteralStringRef("ExtraCluster:0"));
useHostname ? std::make_unique<ClusterConnectionString>(extraCoordinatorHostnames, "ExtraCluster:0"_sr)
: std::make_unique<ClusterConnectionString>(extraCoordinatorAddresses, "ExtraCluster:0"_sr);
} else if (testConfig.extraDB == 3) {
// The DR database is the same database
g_simulator.extraDB =
useHostname ? new ClusterConnectionString(coordinatorHostnames, LiteralStringRef("TestCluster:0"))
: new ClusterConnectionString(coordinatorAddresses, LiteralStringRef("TestCluster:0"));
g_simulator.extraDB = useHostname
? std::make_unique<ClusterConnectionString>(coordinatorHostnames, "TestCluster:0"_sr)
: std::make_unique<ClusterConnectionString>(coordinatorAddresses, "TestCluster:0"_sr);
}
*pConnString = conn;
@ -2164,7 +2185,7 @@ void setupSimulatedSystem(std::vector<Future<Void>>* systemActors,
// check the sslEnablementMap using only one ip
LocalityData localities(Optional<Standalone<StringRef>>(), zoneId, machineId, dcUID);
localities.set(LiteralStringRef("data_hall"), dcUID);
localities.set("data_hall"_sr, dcUID);
systemActors->push_back(reportErrors(simulatedMachine(conn,
ips,
sslEnabled,
@ -2191,7 +2212,7 @@ void setupSimulatedSystem(std::vector<Future<Void>>* systemActors,
Standalone<StringRef> newMachineId(deterministicRandom()->randomUniqueID().toString());
LocalityData localities(Optional<Standalone<StringRef>>(), newZoneId, newMachineId, dcUID);
localities.set(LiteralStringRef("data_hall"), dcUID);
localities.set("data_hall"_sr, dcUID);
systemActors->push_back(reportErrors(simulatedMachine(*g_simulator.extraDB,
extraIps,
sslEnabled,
@ -2395,7 +2416,7 @@ ACTOR void setupAndRun(std::string dataFolder,
100.0));
// FIXME: snapshot restore does not support multi-region restore, hence restore it as single region always
if (restoring) {
startingConfiguration = LiteralStringRef("usable_regions=1");
startingConfiguration = "usable_regions=1"_sr;
}
} else {
g_expect_full_pointermap = 1;

29
fdbserver/workloads/SaveAndKill.actor.cpp
View File

@ -37,11 +37,9 @@ struct SaveAndKillWorkload : TestWorkload {
int isRestoring;
SaveAndKillWorkload(WorkloadContext const& wcx) : TestWorkload(wcx) {
restartInfo =
getOption(options, LiteralStringRef("restartInfoLocation"), LiteralStringRef("simfdb/restartInfo.ini"))
.toString();
testDuration = getOption(options, LiteralStringRef("testDuration"), 10.0);
isRestoring = getOption(options, LiteralStringRef("isRestoring"), 0);
restartInfo = getOption(options, "restartInfoLocation"_sr, "simfdb/restartInfo.ini"_sr).toString();
testDuration = getOption(options, "testDuration"_sr, 10.0);
isRestoring = getOption(options, "isRestoring"_sr, 0);
}
std::string description() const override { return "SaveAndKillWorkload"; }
@ -70,23 +68,22 @@ struct SaveAndKillWorkload : TestWorkload {
std::vector<ISimulator::ProcessInfo*> processes = g_simulator.getAllProcesses();
std::map<NetworkAddress, ISimulator::ProcessInfo*> rebootingProcesses = g_simulator.currentlyRebootingProcesses;
std::map<std::string, ISimulator::ProcessInfo*> allProcessesMap =
std::map<std::string, ISimulator::ProcessInfo*>();
for (auto it = rebootingProcesses.begin(); it != rebootingProcesses.end(); it++) {
if (allProcessesMap.find(it->second->dataFolder) == allProcessesMap.end())
allProcessesMap[it->second->dataFolder] = it->second;
std::map<std::string, ISimulator::ProcessInfo*> allProcessesMap;
for (const auto& [_, process] : rebootingProcesses) {
if (allProcessesMap.find(process->dataFolder) == allProcessesMap.end()) {
allProcessesMap[process->dataFolder] = process;
}
}
for (auto it = processes.begin(); it != processes.end(); it++) {
if (allProcessesMap.find((*it)->dataFolder) == allProcessesMap.end())
allProcessesMap[(*it)->dataFolder] = *it;
for (const auto& process : processes) {
if (allProcessesMap.find(process->dataFolder) == allProcessesMap.end()) {
allProcessesMap[process->dataFolder] = process;
}
}
ini.SetValue("META", "processCount", format("%d", allProcessesMap.size() - 1).c_str());
std::map<std::string, int> machines;
int j = 0;
for (auto processIterator = allProcessesMap.begin(); processIterator != allProcessesMap.end();
processIterator++) {
ISimulator::ProcessInfo* process = processIterator->second;
for (const auto& [_, process] : allProcessesMap) {
std::string machineId = printable(process->locality.machineId());
const char* machineIdString = machineId.c_str();
if (strcmp(process->name, "TestSystem") != 0) {

3
tests/restarting/from_7.1.0/ConfigureTestRestart-2.toml
View File

@ -1,3 +1,6 @@
[[configuration]]
randomlyRenameZoneId=true
[[test]]
testTitle='CloggedConfigureDatabaseTest'
runSetup=false