apple/foundationdb
1
0
Fork 0
mirror of https://github.com/apple/foundationdb.git synced 2025-05-14 09:58:50 +08:00
Code Issues Packages Projects Releases Wiki Activity

Throttle status requests with MAX_STATUS_REQUESTS_PER_SECOND knob, whenever status batching is used.

Browse Source
This commit is contained in:
Trevor Clinkenbeard 2019-01-28 15:37:30 -08:00
parent dc2b740415
commit 5b89db811a
4 changed files with 18 additions and 2 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
fdbclient/StatusClient.actor.cpp
View File

@ -373,6 +373,9 @@ ACTOR Future<Optional<StatusObject>> clusterStatusFetcher(ClusterInterface cI, S
if (result.getError().code() == error_code_request_maybe_delivered)
messages->push_back(makeMessage("unreachable_cluster_controller",
("Unable to communicate with the cluster controller at " + cI.address().toString() + " to get status.").c_str()));
else if (result.getError().code() == error_code_server_overloaded)
messages->push_back(makeMessage("server_overloaded",
"The cluster controller is currently processing too many status requests, and is unable to respond"));
else
messages->push_back(makeMessage("status_incomplete_error", "Cluster encountered an error fetching status."));
}

15
fdbserver/ClusterController.actor.cpp
View File

@ -1829,8 +1829,19 @@ ACTOR Future<Void> statusServer(FutureStream< StatusRequest> requests,
// Get all requests that are ready right *now*, before GetStatus() begins.
// All of these requests will be responded to with the next GetStatus() result.
while (requests.isReady())
requests_batch.push_back(requests.pop());
// If requests are batched, do not respond to more than MAX_STATUS_REQUESTS_PER_SECOND
// requests per second
while (requests.isReady()) {
auto req = requests.pop();
if (SERVER_KNOBS->STATUS_MIN_TIME_BETWEEN_REQUESTS > 0.0 &&
requests_batch.size() + 1 >
SERVER_KNOBS->STATUS_MIN_TIME_BETWEEN_REQUESTS * SERVER_KNOBS->MAX_STATUS_REQUESTS_PER_SECOND) {
TraceEvent("TooManyStatusRequests").detail("BatchSize", requests_batch.size());
req.reply.sendError(server_overloaded());
} else {
requests_batch.push_back(req);
}
}
// Get status but trap errors to send back to client.
vector<std::pair<WorkerInterface, ProcessClass>> workers;

1
fdbserver/Knobs.cpp
View File

@ -402,6 +402,7 @@ ServerKnobs::ServerKnobs(bool randomize, ClientKnobs* clientKnobs) {
// Status
init( STATUS_MIN_TIME_BETWEEN_REQUESTS, 0.0 );
init( MAX_STATUS_REQUESTS_PER_SECOND, 256.0 );
init( CONFIGURATION_ROWS_TO_FETCH, 20000 );
// IPager

1
fdbserver/Knobs.h
View File

@ -339,6 +339,7 @@ public:
// Status
double STATUS_MIN_TIME_BETWEEN_REQUESTS;
double MAX_STATUS_REQUESTS_PER_SECOND;
int CONFIGURATION_ROWS_TO_FETCH;
// IPager