diff --git a/fdbclient/ServerKnobs.cpp b/fdbclient/ServerKnobs.cpp index 7958ae1993..7bddd42b2e 100644 --- a/fdbclient/ServerKnobs.cpp +++ b/fdbclient/ServerKnobs.cpp @@ -582,6 +582,7 @@ void ServerKnobs::initialize(Randomize randomize, ClientKnobs* clientKnobs, IsSi init( MIN_AVAILABLE_SPACE, 1e8 ); init( MIN_AVAILABLE_SPACE_RATIO, 0.05 ); + init( MIN_AVAILABLE_SPACE_RATIO_SAFETY_BUFFER, 0.01 ); init( TARGET_AVAILABLE_SPACE_RATIO, 0.30 ); init( AVAILABLE_SPACE_UPDATE_DELAY, 5.0 ); diff --git a/fdbclient/ServerKnobs.h b/fdbclient/ServerKnobs.h index 9f06938334..bb7fb03a62 100644 --- a/fdbclient/ServerKnobs.h +++ b/fdbclient/ServerKnobs.h @@ -529,6 +529,7 @@ public: int64_t MIN_AVAILABLE_SPACE; double MIN_AVAILABLE_SPACE_RATIO; + double MIN_AVAILABLE_SPACE_RATIO_SAFETY_BUFFER; double TARGET_AVAILABLE_SPACE_RATIO; double AVAILABLE_SPACE_UPDATE_DELAY; diff --git a/fdbserver/DDTeamCollection.actor.cpp b/fdbserver/DDTeamCollection.actor.cpp index 68656c8604..34c661763e 100644 --- a/fdbserver/DDTeamCollection.actor.cpp +++ b/fdbserver/DDTeamCollection.actor.cpp @@ -5640,11 +5640,7 @@ public: std::pair>, bool> resTeam = req.reply.getFuture().get(); - std::set expectedServers{ UID(2, 0), UID(3, 0), UID(4, 0) }; - ASSERT(resTeam.first.present()); - auto servers = resTeam.first.get()->getServerIDs(); - const std::set selectedServers(servers.begin(), servers.end()); - ASSERT(expectedServers == selectedServers); + ASSERT(!resTeam.first.present()); return Void(); } diff --git a/fdbserver/TCInfo.actor.cpp b/fdbserver/TCInfo.actor.cpp index c1371136ee..5cf7f6b2dd 100644 --- a/fdbserver/TCInfo.actor.cpp +++ b/fdbserver/TCInfo.actor.cpp @@ -137,6 +137,23 @@ TCServerInfo::TCServerInfo(StorageServerInterface ssi, } } +bool TCServerInfo::hasHealthyAvailableSpace(double minAvailableSpaceRatio) const { + ASSERT(serverMetricsPresent()); + + auto& metrics = getServerMetrics(); + ASSERT(metrics.available.bytes >= 0); + ASSERT(metrics.capacity.bytes >= 0); + + double availableSpaceRatio; + if (metrics.capacity.bytes == 0) { + availableSpaceRatio = 0; + } else { + availableSpaceRatio = (((double)metrics.available.bytes) / metrics.capacity.bytes); + } + + return availableSpaceRatio >= minAvailableSpaceRatio; +} + Future TCServerInfo::updateServerMetrics() { return TCServerInfoImpl::updateServerMetrics(this); } @@ -381,8 +398,23 @@ double TCTeamInfo::getMinAvailableSpaceRatio(bool includeInFlight) const { return minRatio; } +bool TCTeamInfo::allServersHaveHealthyAvailableSpace() const { + bool result = true; + double minAvailableSpaceRatio = + SERVER_KNOBS->MIN_AVAILABLE_SPACE_RATIO + SERVER_KNOBS->MIN_AVAILABLE_SPACE_RATIO_SAFETY_BUFFER; + for (const auto& server : servers) { + if (!server->serverMetricsPresent() || !server->hasHealthyAvailableSpace(minAvailableSpaceRatio)) { + result = false; + break; + } + } + + return result; +} + bool TCTeamInfo::hasHealthyAvailableSpace(double minRatio) const { - return getMinAvailableSpaceRatio() >= minRatio && getMinAvailableSpace() > SERVER_KNOBS->MIN_AVAILABLE_SPACE; + return getMinAvailableSpaceRatio() >= minRatio && getMinAvailableSpace() > SERVER_KNOBS->MIN_AVAILABLE_SPACE && + allServersHaveHealthyAvailableSpace(); } bool TCTeamInfo::isOptimal() const { diff --git a/fdbserver/TCInfo.h b/fdbserver/TCInfo.h index f1886273f8..0170de1870 100644 --- a/fdbserver/TCInfo.h +++ b/fdbserver/TCInfo.h @@ -93,6 +93,8 @@ public: return (storeType == configStoreType || storeType == KeyValueStoreType::END); } + bool hasHealthyAvailableSpace(double minAvailableSpaceRatio) const; + Future updateServerMetrics(); static Future updateServerMetrics(Reference server); Future serverMetricsPolling(); @@ -218,4 +220,6 @@ private: // Calculate an "average" of the metrics replies that we received. Penalize teams from which we did not receive all // replies. int64_t getLoadAverage() const; + + bool allServersHaveHealthyAvailableSpace() const; };