apple/foundationdb
1
0
Fork 0
mirror of https://github.com/apple/foundationdb.git synced 2025-05-15 18:32:18 +08:00
Code Issues Packages Projects Releases Wiki Activity

Throttle status requests with MAX_STATUS_REQUESTS_PER_SECOND knob, whenever status batching is used.

Browse Source
This commit is contained in:
Trevor Clinkenbeard 2019-01-28 15:37:30 -08:00
parent dc2b740415
commit 5b89db811a
4 changed files with 18 additions and 2 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
fdbclient/StatusClient.actor.cpp
View File

@ -373,6 +373,9 @@ ACTOR Future<Optional<StatusObject>> clusterStatusFetcher(ClusterInterface cI, S
if (result.getError().code() == error_code_request_maybe_delivered) if (result.getError().code() == error_code_request_maybe_delivered)
messages->push_back(makeMessage("unreachable_cluster_controller", messages->push_back(makeMessage("unreachable_cluster_controller",
("Unable to communicate with the cluster controller at " + cI.address().toString() + " to get status.").c_str())); ("Unable to communicate with the cluster controller at " + cI.address().toString() + " to get status.").c_str()));
else if (result.getError().code() == error_code_server_overloaded)
messages->push_back(makeMessage("server_overloaded",
"The cluster controller is currently processing too many status requests, and is unable to respond"));
else else
messages->push_back(makeMessage("status_incomplete_error", "Cluster encountered an error fetching status.")); messages->push_back(makeMessage("status_incomplete_error", "Cluster encountered an error fetching status."));
} }

15
fdbserver/ClusterController.actor.cpp
View File

@ -1829,8 +1829,19 @@ ACTOR Future<Void> statusServer(FutureStream< StatusRequest> requests,
// Get all requests that are ready right *now*, before GetStatus() begins. // Get all requests that are ready right *now*, before GetStatus() begins.
// All of these requests will be responded to with the next GetStatus() result. // All of these requests will be responded to with the next GetStatus() result.
while (requests.isReady()) // If requests are batched, do not respond to more than MAX_STATUS_REQUESTS_PER_SECOND
requests_batch.push_back(requests.pop()); // requests per second
while (requests.isReady()) {
auto req = requests.pop();
if (SERVER_KNOBS->STATUS_MIN_TIME_BETWEEN_REQUESTS > 0.0 &&
requests_batch.size() + 1 >
SERVER_KNOBS->STATUS_MIN_TIME_BETWEEN_REQUESTS * SERVER_KNOBS->MAX_STATUS_REQUESTS_PER_SECOND) {
TraceEvent("TooManyStatusRequests").detail("BatchSize", requests_batch.size());
req.reply.sendError(server_overloaded());
} else {
requests_batch.push_back(req);
}
}
// Get status but trap errors to send back to client. // Get status but trap errors to send back to client.
vector<std::pair<WorkerInterface, ProcessClass>> workers; vector<std::pair<WorkerInterface, ProcessClass>> workers;

1
fdbserver/Knobs.cpp
View File

@ -402,6 +402,7 @@ ServerKnobs::ServerKnobs(bool randomize, ClientKnobs* clientKnobs) {
// Status // Status
init( STATUS_MIN_TIME_BETWEEN_REQUESTS, 0.0 ); init( STATUS_MIN_TIME_BETWEEN_REQUESTS, 0.0 );
init( MAX_STATUS_REQUESTS_PER_SECOND, 256.0 );
init( CONFIGURATION_ROWS_TO_FETCH, 20000 ); init( CONFIGURATION_ROWS_TO_FETCH, 20000 );
// IPager // IPager

1
fdbserver/Knobs.h
View File

@ -339,6 +339,7 @@ public:
// Status // Status
double STATUS_MIN_TIME_BETWEEN_REQUESTS; double STATUS_MIN_TIME_BETWEEN_REQUESTS;
double MAX_STATUS_REQUESTS_PER_SECOND;
int CONFIGURATION_ROWS_TO_FETCH; int CONFIGURATION_ROWS_TO_FETCH;
// IPager // IPager