apple/foundationdb
1
0
Fork 0
mirror of https://github.com/apple/foundationdb.git synced 2025-06-02 03:12:12 +08:00
Code Issues Packages Projects Releases Wiki Activity

Don't inject TSS faults if speedUpSimulation is set

Browse Source
This commit is contained in:
Josh Slocum 2021-06-18 12:10:50 -05:00
parent e3c9d9f6a0
commit d1d2ca9285
3 changed files with 15 additions and 12 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
fdbserver/QuietDatabase.actor.cpp
View File

@ -358,11 +358,7 @@ ACTOR Future<int64_t> getMaxStorageServerQueueSize(Database cx, Reference<AsyncV
.detail("SS", servers[i].id());
throw attribute_not_found();
}
// Ignore TSS in add delay mode since it can purposefully freeze forever
if (!servers[i].isTss() || !g_network->isSimulated() ||
g_simulator.tssMode != ISimulator::TSSMode::EnabledAddDelay) {
messages.push_back(getStorageMetricsTimeout(servers[i].id(), itr->second));
}
messages.push_back(getStorageMetricsTimeout(servers[i].id(), itr->second));
}
wait(waitForAll(messages));

16
fdbserver/storageserver.actor.cpp
View File

@ -3446,6 +3446,15 @@ private:
}
};
ACTOR Future<Void> tssDelayForever() {
loop {
wait(delay(5.0));
if (g_simulator.speedUpSimulation) {
return Void();
}
}
}
ACTOR Future<Void> update(StorageServer* data, bool* pReceivedUpdate) {
state double start;
try {
@ -3466,12 +3475,13 @@ ACTOR Future<Void> update(StorageServer* data, bool* pReceivedUpdate) {
}
if (g_network->isSimulated() && data->isTss() && g_simulator.tssMode == ISimulator::TSSMode::EnabledAddDelay &&
data->tssFaultInjectTime.present() && data->tssFaultInjectTime.get() < now()) {
!g_simulator.speedUpSimulation && data->tssFaultInjectTime.present() &&
data->tssFaultInjectTime.get() < now()) {
if (deterministicRandom()->random01() < 0.01) {
TraceEvent(SevWarnAlways, "TSSInjectDelayForever", data->thisServerID);
// small random chance to just completely get stuck here, each tss should eventually hit this in this
// mode
wait(Never());
wait(tssDelayForever());
} else {
// otherwise pause for part of a second
double delayTime = deterministicRandom()->random01();
@ -3666,7 +3676,7 @@ ACTOR Future<Void> update(StorageServer* data, bool* pReceivedUpdate) {
// Drop non-private mutations if TSS fault injection is enabled in simulation, or if this is a TSS in
// quarantine.
if (g_network->isSimulated() && data->isTss() &&
if (g_network->isSimulated() && data->isTss() && !g_simulator.speedUpSimulation &&
g_simulator.tssMode == ISimulator::TSSMode::EnabledDropMutations &&
data->tssFaultInjectTime.present() && data->tssFaultInjectTime.get() < now() &&
(msg.type == MutationRef::SetValue || msg.type == MutationRef::ClearRange) &&

5
fdbserver/workloads/ConsistencyCheck.actor.cpp
View File

@ -1498,10 +1498,7 @@ struct ConsistencyCheckWorkload : TestWorkload {
.error(e);
// All shards should be available in quiscence
if (self->performQuiescentChecks &&
((g_network->isSimulated() &&
g_simulator.tssMode != ISimulator::TSSMode::EnabledAddDelay) ||
!storageServerInterfaces[j].isTss())) {
if (self->performQuiescentChecks) {
self->testFailure("Storage server unavailable");
return false;
}