From 8a8914f046568da7ab34fb8eeb329f0533d51ee8 Mon Sep 17 00:00:00 2001
From: Evan Tschannen <ejt@apple.com>
Date: Fri, 22 Jun 2018 00:04:00 -0700
Subject: [PATCH] re-added the ability to configure the number of log routers.
Many log routers are needed to get a sufficient number of sockets involved in
copying data across the WAN
---
documentation/StatusSchema.json | 1 +
fdbcli/fdbcli.actor.cpp | 6 ++++++
fdbclient/DatabaseConfiguration.cpp | 6 +++++-
fdbclient/DatabaseConfiguration.h | 1 +
fdbclient/ManagementAPI.actor.cpp | 4 ++--
fdbserver/ClusterController.actor.cpp | 9 +++++----
fdbserver/SimulatedCluster.actor.cpp | 1 +
fdbserver/TagPartitionedLogSystem.actor.cpp | 2 +-
fdbserver/masterserver.actor.cpp | 2 +-
tests/fast/SidebandWithStatus.txt | 2 +-
tests/rare/LargeApiCorrectnessStatus.txt | 2 +-
tests/slow/DDBalanceAndRemoveStatus.txt | 2 +-
12 files changed, 26 insertions(+), 12 deletions(-)
diff --git a/documentation/StatusSchema.json b/documentation/StatusSchema.json
index 9ebbd6bd7c..075f319885 100644
--- a/documentation/StatusSchema.json
+++ b/documentation/StatusSchema.json
@@ -362,6 +362,7 @@
"remote_redundancy_mode":"remote_single",
"remote_log_replicas":3,
"remote_logs":5,
+ "log_routers":10,
"usable_regions":1,
"storage_replicas":1,
"resolvers":1,
diff --git a/fdbcli/fdbcli.actor.cpp b/fdbcli/fdbcli.actor.cpp
index a89783b842..7024155100 100644
--- a/fdbcli/fdbcli.actor.cpp
+++ b/fdbcli/fdbcli.actor.cpp
@@ -884,6 +884,12 @@ void printStatus(StatusObjectReader statusObj, StatusClient::StatusLevel level,
if (statusObjConfig.get("logs", intVal))
outputString += format("\n Desired Logs - %d", intVal);
+
+ if (statusObjConfig.get("remote_logs", intVal))
+ outputString += format("\n Desired Remote Logs - %d", intVal);
+
+ if (statusObjConfig.get("log_routers", intVal))
+ outputString += format("\n Desired Log Routers - %d", intVal);
}
catch (std::runtime_error& e) {
outputString = outputStringCache;
diff --git a/fdbclient/DatabaseConfiguration.cpp b/fdbclient/DatabaseConfiguration.cpp
index 13bc44f442..05e2ef0b20 100644
--- a/fdbclient/DatabaseConfiguration.cpp
+++ b/fdbclient/DatabaseConfiguration.cpp
@@ -29,7 +29,7 @@ DatabaseConfiguration::DatabaseConfiguration()
void DatabaseConfiguration::resetInternal() {
// does NOT reset rawConfiguration
initialized = false;
- masterProxyCount = resolverCount = desiredTLogCount = tLogWriteAntiQuorum = tLogReplicationFactor = storageTeamSize = -1;
+ masterProxyCount = resolverCount = desiredTLogCount = tLogWriteAntiQuorum = tLogReplicationFactor = storageTeamSize = desiredLogRouterCount = -1;
tLogDataStoreType = storageServerStoreType = KeyValueStoreType::END;
autoMasterProxyCount = CLIENT_KNOBS->DEFAULT_AUTO_PROXIES;
autoResolverCount = CLIENT_KNOBS->DEFAULT_AUTO_RESOLVERS;
@@ -297,6 +297,9 @@ StatusObject DatabaseConfiguration::toJSON(bool noPolicies) const {
if( resolverCount != -1 ) {
result["resolvers"] = resolverCount;
}
+ if( desiredLogRouterCount != -1 ) {
+ result["log_routers"] = desiredLogRouterCount;
+ }
if( remoteDesiredTLogCount != -1 ) {
result["remote_logs"] = remoteDesiredTLogCount;
}
@@ -336,6 +339,7 @@ bool DatabaseConfiguration::setInternal(KeyRef key, ValueRef value) {
else if (ck == LiteralStringRef("auto_logs")) parse(&autoDesiredTLogCount, value);
else if (ck == LiteralStringRef("storage_replication_policy")) parseReplicationPolicy(&storagePolicy, value);
else if (ck == LiteralStringRef("log_replication_policy")) parseReplicationPolicy(&tLogPolicy, value);
+ else if (ck == LiteralStringRef("log_routers")) parse(&desiredLogRouterCount, value);
else if (ck == LiteralStringRef("remote_logs")) parse(&remoteDesiredTLogCount, value);
else if (ck == LiteralStringRef("remote_log_replicas")) parse(&remoteTLogReplicationFactor, value);
else if (ck == LiteralStringRef("remote_log_policy")) parseReplicationPolicy(&remoteTLogPolicy, value);
diff --git a/fdbclient/DatabaseConfiguration.h b/fdbclient/DatabaseConfiguration.h
index 4c16b41ad5..e9f09a463f 100644
--- a/fdbclient/DatabaseConfiguration.h
+++ b/fdbclient/DatabaseConfiguration.h
@@ -159,6 +159,7 @@ struct DatabaseConfiguration {
KeyValueStoreType storageServerStoreType;
// Remote TLogs
+ int32_t desiredLogRouterCount;
int32_t remoteDesiredTLogCount;
int32_t remoteTLogReplicationFactor;
IRepPolicyRef remoteTLogPolicy;
diff --git a/fdbclient/ManagementAPI.actor.cpp b/fdbclient/ManagementAPI.actor.cpp
index 5bda5e37c0..d990be2a0a 100644
--- a/fdbclient/ManagementAPI.actor.cpp
+++ b/fdbclient/ManagementAPI.actor.cpp
@@ -65,14 +65,14 @@ std::map<std::string, std::string> configForToken( std::string const& mode ) {
std::string key = mode.substr(0, pos);
std::string value = mode.substr(pos+1);
- if( (key == "logs" || key == "proxies" || key == "resolvers" || key == "remote_logs" || key == "satellite_logs" || key == "usable_regions") && isInteger(value) ) {
+ if( (key == "logs" || key == "proxies" || key == "resolvers" || key == "remote_logs" || key == "log_routers" || key == "satellite_logs" || key == "usable_regions") && isInteger(value) ) {
out[p+key] = value;
}
if( key == "regions" ) {
json_spirit::mValue mv;
json_spirit::read_string( value, mv );
-
+
StatusObject regionObj;
regionObj["regions"] = mv;
out[p+key] = BinaryWriter::toValue(regionObj, IncludeVersion()).toString();
diff --git a/fdbserver/ClusterController.actor.cpp b/fdbserver/ClusterController.actor.cpp
index 868afb26d9..ce51234fbd 100644
--- a/fdbserver/ClusterController.actor.cpp
+++ b/fdbserver/ClusterController.actor.cpp
@@ -459,7 +459,6 @@ public:
std::set<Optional<Key>> remoteDC;
remoteDC.insert(req.dcId);
-
auto remoteLogs = getWorkersForTlogs( req.configuration, req.configuration.getRemoteTLogReplicationFactor(), req.configuration.getDesiredRemoteLogs(), req.configuration.getRemoteTLogPolicy(), id_used, false, remoteDC );
for(int i = 0; i < remoteLogs.size(); i++) {
@@ -895,13 +894,15 @@ public:
if(oldRemoteTLogFit < newRemoteTLogFit) return false;
+ int oldRouterCount = oldTLogFit.count * std::max<int>(1, db.config.desiredLogRouterCount / std::max(1,oldTLogFit.count));
+ int newRouterCount = newTLogFit.count * std::max<int>(1, db.config.desiredLogRouterCount / std::max(1,newTLogFit.count));
RoleFitness oldLogRoutersFit(log_routers, ProcessClass::LogRouter);
- RoleFitness newLogRoutersFit((db.config.usableRegions > 1 && dbi.recoveryState == RecoveryState::REMOTE_RECOVERED) ? getWorkersForRoleInDatacenter( *remoteDC.begin(), ProcessClass::LogRouter, newTLogFit.count, db.config, id_used, Optional<WorkerFitnessInfo>(), true ) : log_routers, ProcessClass::LogRouter);
+ RoleFitness newLogRoutersFit((db.config.usableRegions > 1 && dbi.recoveryState == RecoveryState::REMOTE_RECOVERED) ? getWorkersForRoleInDatacenter( *remoteDC.begin(), ProcessClass::LogRouter, newRouterCount, db.config, id_used, Optional<WorkerFitnessInfo>(), true ) : log_routers, ProcessClass::LogRouter);
- if(oldLogRoutersFit.count < oldTLogFit.count) {
+ if(oldLogRoutersFit.count < oldRouterCount) {
oldLogRoutersFit.worstFit = ProcessClass::NeverAssign;
}
- if(newLogRoutersFit.count < newTLogFit.count) {
+ if(newLogRoutersFit.count < newRouterCount) {
newLogRoutersFit.worstFit = ProcessClass::NeverAssign;
}
diff --git a/fdbserver/SimulatedCluster.actor.cpp b/fdbserver/SimulatedCluster.actor.cpp
index 5b5074b4f5..88c45287a0 100644
--- a/fdbserver/SimulatedCluster.actor.cpp
+++ b/fdbserver/SimulatedCluster.actor.cpp
@@ -863,6 +863,7 @@ void SimulationConfig::generateNormalConfig(int minimumReplication) {
ASSERT(false); // Programmer forgot to adjust cases.
}
+ if (g_random->random01() < 0.25) db.desiredLogRouterCount = g_random->randomInt(1,7);
if (g_random->random01() < 0.25) db.remoteDesiredTLogCount = g_random->randomInt(1,7);
}
diff --git a/fdbserver/TagPartitionedLogSystem.actor.cpp b/fdbserver/TagPartitionedLogSystem.actor.cpp
index 38ffd09407..b3eca87976 100644
--- a/fdbserver/TagPartitionedLogSystem.actor.cpp
+++ b/fdbserver/TagPartitionedLogSystem.actor.cpp
@@ -1600,7 +1600,7 @@ struct TagPartitionedLogSystem : ILogSystem, ReferenceCounted<TagPartitionedLogS
oldLogSystem->recruitmentID = logSystem->recruitmentID;
if(configuration.usableRegions > 1) {
- logSystem->logRouterTags = recr.tLogs.size();
+ logSystem->logRouterTags = recr.tLogs.size() * std::max<int>(1, configuration.desiredLogRouterCount / std::max<int>(1,recr.tLogs.size()));
logSystem->expectedLogSets++;
} else {
logSystem->logRouterTags = 0;
diff --git a/fdbserver/masterserver.actor.cpp b/fdbserver/masterserver.actor.cpp
index ce1a68ef29..9570f69955 100644
--- a/fdbserver/masterserver.actor.cpp
+++ b/fdbserver/masterserver.actor.cpp
@@ -311,7 +311,7 @@ ACTOR Future<Void> newTLogServers( Reference<MasterData> self, RecruitFromConfig
self->dcId_locality[remoteDcId] = loc;
}
- Future<RecruitRemoteFromConfigurationReply> fRemoteWorkers = brokenPromiseToNever( self->clusterController.recruitRemoteFromConfiguration.getReply( RecruitRemoteFromConfigurationRequest( self->configuration, remoteDcId, recr.tLogs.size() ) ) );
+ Future<RecruitRemoteFromConfigurationReply> fRemoteWorkers = brokenPromiseToNever( self->clusterController.recruitRemoteFromConfiguration.getReply( RecruitRemoteFromConfigurationRequest( self->configuration, remoteDcId, recr.tLogs.size() * std::max<int>(1, self->configuration.desiredLogRouterCount / std::max<int>(1, recr.tLogs.size())) ) ) );
Reference<ILogSystem> newLogSystem = wait( oldLogSystem->newEpoch( recr, fRemoteWorkers, self->configuration, self->cstate.myDBState.recoveryCount + 1, self->dcId_locality[recr.dcId], self->dcId_locality[remoteDcId], self->allTags, self->recruitmentStalled ) );
self->logSystem = newLogSystem;
diff --git a/tests/fast/SidebandWithStatus.txt b/tests/fast/SidebandWithStatus.txt
index 94f4616d9f..8bca27d0b4 100644
--- a/tests/fast/SidebandWithStatus.txt
+++ b/tests/fast/SidebandWithStatus.txt
@@ -5,7 +5,7 @@ testTitle=CloggedCausalConsistencyTest
testName=Status
testDuration=30.0
- schema={"cluster":{"layers":{"_valid":true,"_error":"some error description"},"processes":{"$map":{"version":"3.0.0","machine_id":"0ccb4e0feddb5583010f6b77d9d10ece","locality":{"$map":"value"},"class_source":{"$enum":["command_line","configure_auto","set_class"]},"class_type":{"$enum":["unset","storage","transaction","resolution","proxy","master","test"]},"roles":[{"query_queue_max":0,"input_bytes":{"hz":0,"counter":0,"roughness":0},"stored_bytes":12341234,"kvstore_used_bytes":12341234,"kvstore_available_bytes":12341234,"kvstore_free_bytes":12341234,"kvstore_total_bytes":12341234,"durable_bytes":{"hz":0,"counter":0,"roughness":0},"queue_disk_used_bytes":12341234,"queue_disk_available_bytes":12341234,"queue_disk_free_bytes":12341234,"queue_disk_total_bytes":12341234,"role":{"$enum":["master","proxy","log","storage","resolver","cluster_controller"]},"data_version":12341234,"data_version_lag":12341234,"id":"eb84471d68c12d1d26f692a50000003f","finished_queries":{"hz":0,"counter":0,"roughness":0}}],"command_line":"-r simulation","memory":{"available_bytes":0,"limit_bytes":0,"unused_allocated_memory":0,"used_bytes":0},"messages":[{"time":12345.12312,"type":"x","name":{"$enum":["file_open_error","incorrect_cluster_file_contents","process_error","io_error","io_timeout","platform_error","storage_server_lagging","(other FDB error messages)"]},"raw_log_message":"<stuff/>","description":"abc"}],"fault_domain":"0ccb4e0fdbdb5583010f6b77d9d10ece","excluded":false,"address":"1.2.3.4:1234","disk":{"free_bytes":3451233456234,"reads":{"hz":0,"counter":0,"sectors":0},"busy":0,"writes":{"hz":0,"counter":0,"sectors":0},"total_bytes":123412341234},"uptime_seconds":1234.2345,"cpu":{"usage_cores":0},"network":{"current_connections":0,"connections_established":{"hz":0},"connections_closed":{"hz":0},"connection_errors":{"hz":0},"megabits_sent":{"hz":0},"megabits_received":{"hz":0}}}},"old_logs":[{"logs":[{"id":"7f8d623d0cb9966e","healthy":true,"address":"1.2.3.4:1234"}],"log_replication_factor":3,"log_write_anti_quorum":0,"log_fault_tolerance":2,"remote_log_replication_factor":3,"remote_log_fault_tolerance":2,"satellite_log_replication_factor":3,"satellite_log_write_anti_quorum":0,"satellite_log_fault_tolerance":2}],"fault_tolerance":{"max_machine_failures_without_losing_availability":0,"max_machine_failures_without_losing_data":0},"qos":{"worst_queue_bytes_log_server":460,"performance_limited_by":{"reason_server_id":"7f8d623d0cb9966e","reason_id":0,"name":{"$enum":["workload","storage_server_write_queue_size","storage_server_write_bandwidth_mvcc","storage_server_readable_behind","log_server_mvcc_write_bandwidth","log_server_write_queue","storage_server_min_free_space","storage_server_min_free_space_ratio","log_server_min_free_space","log_server_min_free_space_ratio"]},"description":"The database is not being saturated by the workload."},"transactions_per_second_limit":0,"released_transactions_per_second":0,"limiting_queue_bytes_storage_server":0,"worst_queue_bytes_storage_server":0,"limiting_version_lag_storage_server":0,"worst_version_lag_storage_server":0},"incompatible_connections":[],"datacenter_version_difference":0,"database_available":true,"database_locked":false,"generation":2,"latency_probe":{"read_seconds":7,"immediate_priority_transaction_start_seconds":0,"batch_priority_transaction_start_seconds":0,"transaction_start_seconds":0,"commit_seconds":0.02},"clients":{"count":1,"supported_versions":[{"client_version":"3.0.0","connected_clients":[{"address":"127.0.0.1:9898","log_group":"default"}],"count":1,"protocol_version":"fdb00a400050001","source_version":"9430e1127b4991cbc5ab2b17f41cfffa5de07e9d"}]},"messages":[{"reasons":[{"description":"Blah."}],"unreachable_processes":[{"address":"1.2.3.4:1234"}],"name":{"$enum":["unreachable_master_worker","unreadable_configuration","full_replication_timeout","client_issues","unreachable_processes","immediate_priority_transaction_start_probe_timeout","batch_priority_transaction_start_probe_timeout","transaction_start_probe_timeout","read_probe_timeout","commit_probe_timeout","storage_servers_error","status_incomplete","layer_status_incomplete","database_availability_timeout"]},"issues":[{"name":{"$enum":["incorrect_cluster_file_contents"]},"description":"Cluster file contents do not match current cluster connection string. Verify cluster file is writable and has not been overwritten externally."}],"description":"abc"}],"recovery_state":{"required_resolvers":1,"required_proxies":1,"name":{"$enum":["reading_coordinated_state","locking_coordinated_state","locking_old_transaction_servers","reading_transaction_system_state","configuration_missing","configuration_never_created","configuration_invalid","recruiting_transaction_servers","initializing_transaction_servers","recovery_transaction","writing_coordinated_state","fully_recovered"]},"required_logs":3,"missing_logs":"7f8d623d0cb9966e","description":"Recovery complete."},"workload":{"operations":{"writes":{"hz":0,"counter":0,"roughness":0},"reads":{"hz":0,"counter":0,"roughness":0}},"bytes":{"written":{"hz":0,"counter":0,"roughness":0},"read":{"hz":0,"counter":0,"roughness":0}},"keys":{"read":{"hz":0,"counter":0,"roughness":0}},"transactions":{"started":{"hz":0,"counter":0,"roughness":0},"conflicted":{"hz":0,"counter":0,"roughness":0},"committed":{"hz":0,"counter":0,"roughness":0}}},"cluster_controller_timestamp":1415650089,"protocol_version":"fdb00a400050001","full_replication":true,"configuration":{"log_anti_quorum":0,"log_replicas":2,"log_replication_policy":"(zoneid^3x1)","redundancy_mode":"single","regions":[{"datacenters":[{"id":"mr","priority":1,"satellite":1}],"satellite_redundancy_mode":"one_satellite_single","satellite_log_replicas":1,"satellite_usable_dcs":1,"satellite_anti_quorum":0,"satellite_log_policy":"(zoneid^3x1)","satellite_logs":2}],"remote_redundancy_mode":"remote_single","remote_log_replicas":3,"remote_logs":5,"usable_regions":1,"storage_replicas":1,"resolvers":1,"storage_replication_policy":"(zoneid^3x1)","logs":2,"storage_engine":{"$enum":["ssd","ssd-1","ssd-2","memory","custom"]},"coordinators_count":1,"excluded_servers":[{"address":"10.0.4.1"}],"auto_proxies":3,"auto_resolvers":1,"auto_logs":3,"proxies":5},"data":{"least_operating_space_bytes_log_server":0,"average_partition_size_bytes":0,"state":{"healthy":true,"min_replicas_remaining":0,"name":{"$enum":["initializing","missing_data","healing","healthy_repartitioning","healthy_removing_server","healthy_rebalancing","healthy"]},"description":""},"least_operating_space_ratio_storage_server":0.1,"max_machine_failures_without_losing_availability":0,"total_disk_used_bytes":0,"total_kv_size_bytes":0,"partitions_count":2,"moving_data":{"total_written_bytes":0,"in_flight_bytes":0,"in_queue_bytes":0},"least_operating_space_bytes_storage_server":0,"max_machine_failures_without_losing_data":0},"machines":{"$map":{"network":{"megabits_sent":{"hz":0},"megabits_received":{"hz":0},"tcp_segments_retransmitted":{"hz":0}},"memory":{"free_bytes":0,"committed_bytes":0,"total_bytes":0},"contributing_workers":4,"datacenter_id":"6344abf1813eb05b","excluded":false,"address":"1.2.3.4","machine_id":"6344abf1813eb05b","locality":{"$map":"value"},"cpu":{"logical_core_utilization":0.4}}}},"client":{"coordinators":{"coordinators":[{"reachable":true,"address":"127.0.0.1:4701"}],"quorum_reachable":true},"database_status":{"available":true,"healthy":true},"messages":[{"name":{"$enum":["inconsistent_cluster_file","unreachable_cluster_controller","no_cluster_controller","status_incomplete_client","status_incomplete_coordinators","status_incomplete_error","status_incomplete_timeout","status_incomplete_cluster","quorum_not_reachable"]},"description":"The cluster file is not up to date."}],"timestamp":1415650089,"cluster_file":{"path":"/etc/foundationdb/fdb.cluster","up_to_date":true}}}
+ schema={"cluster":{"layers":{"_valid":true,"_error":"some error description"},"datacenter_version_difference":0,"processes":{"$map":{"fault_domain":"0ccb4e0fdbdb5583010f6b77d9d10ece","class_source":{"$enum":["command_line","configure_auto","set_class"]},"class_type":{"$enum":["unset","storage","transaction","resolution","proxy","master","test"]},"roles":[{"query_queue_max":0,"data_version_lag":12341234,"input_bytes":{"hz":0.0,"counter":0,"roughness":0.0},"kvstore_used_bytes":12341234,"stored_bytes":12341234,"kvstore_free_bytes":12341234,"durable_bytes":{"hz":0.0,"counter":0,"roughness":0.0},"id":"eb84471d68c12d1d26f692a50000003f","data_version":12341234,"role":{"$enum":["master","proxy","log","storage","resolver","cluster_controller"]},"queue_disk_available_bytes":12341234,"kvstore_available_bytes":12341234,"queue_disk_total_bytes":12341234,"queue_disk_used_bytes":12341234,"queue_disk_free_bytes":12341234,"kvstore_total_bytes":12341234,"finished_queries":{"hz":0.0,"counter":0,"roughness":0.0}}],"locality":{"$map":"value"},"messages":[{"description":"abc","type":"x","name":{"$enum":["file_open_error","incorrect_cluster_file_contents","process_error","io_error","io_timeout","platform_error","storage_server_lagging","(other FDB error messages)"]},"raw_log_message":"<stuff/>","time":12345.12312}],"address":"1.2.3.4:1234","command_line":"-r simulation","disk":{"free_bytes":3451233456234,"reads":{"hz":0.0,"counter":0,"sectors":0},"busy":0.0,"writes":{"hz":0.0,"counter":0,"sectors":0},"total_bytes":123412341234},"version":"3.0.0","excluded":false,"memory":{"available_bytes":0,"unused_allocated_memory":0,"limit_bytes":0,"used_bytes":0},"machine_id":"0ccb4e0feddb5583010f6b77d9d10ece","uptime_seconds":1234.2345,"cpu":{"usage_cores":0.0},"network":{"megabits_sent":{"hz":0.0},"megabits_received":{"hz":0.0},"connections_closed":{"hz":0.0},"connection_errors":{"hz":0.0},"current_connections":0,"connections_established":{"hz":0.0}}}},"clients":{"count":1,"supported_versions":[{"count":1,"protocol_version":"fdb00a400050001","client_version":"3.0.0","source_version":"9430e1127b4991cbc5ab2b17f41cfffa5de07e9d","connected_clients":[{"log_group":"default","address":"127.0.0.1:9898"}]}]},"qos":{"limiting_version_lag_storage_server":0,"released_transactions_per_second":0,"transactions_per_second_limit":0,"limiting_queue_bytes_storage_server":0,"performance_limited_by":{"reason_server_id":"7f8d623d0cb9966e","description":"The database is not being saturated by the workload.","reason_id":0,"name":{"$enum":["workload","storage_server_write_queue_size","storage_server_write_bandwidth_mvcc","storage_server_readable_behind","log_server_mvcc_write_bandwidth","log_server_write_queue","storage_server_min_free_space","storage_server_min_free_space_ratio","log_server_min_free_space","log_server_min_free_space_ratio"]}},"worst_version_lag_storage_server":0,"worst_queue_bytes_log_server":460,"worst_queue_bytes_storage_server":0},"incompatible_connections":[],"full_replication":true,"database_locked":false,"generation":2,"data":{"least_operating_space_bytes_log_server":0,"average_partition_size_bytes":0,"state":{"healthy":true,"description":"","name":{"$enum":["initializing","missing_data","healing","healthy_repartitioning","healthy_removing_server","healthy_rebalancing","healthy"]},"min_replicas_remaining":0},"least_operating_space_ratio_storage_server":0.1,"max_machine_failures_without_losing_availability":0,"total_disk_used_bytes":0,"total_kv_size_bytes":0,"max_machine_failures_without_losing_data":0,"moving_data":{"in_queue_bytes":0,"total_written_bytes":0,"in_flight_bytes":0},"least_operating_space_bytes_storage_server":0,"partitions_count":2},"fault_tolerance":{"max_machine_failures_without_losing_availability":0,"max_machine_failures_without_losing_data":0},"messages":[{"reasons":[{"description":"Blah."}],"unreachable_processes":[{"address":"1.2.3.4:1234"}],"name":{"$enum":["unreachable_master_worker","unreadable_configuration","full_replication_timeout","client_issues","unreachable_processes","immediate_priority_transaction_start_probe_timeout","batch_priority_transaction_start_probe_timeout","transaction_start_probe_timeout","read_probe_timeout","commit_probe_timeout","storage_servers_error","status_incomplete","layer_status_incomplete","database_availability_timeout"]},"issues":[{"name":{"$enum":["incorrect_cluster_file_contents"]},"description":"Cluster file contents do not match current cluster connection string. Verify cluster file is writable and has not been overwritten externally."}],"description":"abc"}],"database_available":true,"recovery_state":{"required_proxies":1,"name":{"$enum":["reading_coordinated_state","locking_coordinated_state","locking_old_transaction_servers","reading_transaction_system_state","configuration_missing","configuration_never_created","configuration_invalid","recruiting_transaction_servers","initializing_transaction_servers","recovery_transaction","writing_coordinated_state","fully_recovered"]},"missing_logs":"7f8d623d0cb9966e","required_resolvers":1,"required_logs":3,"description":"Recovery complete."},"workload":{"operations":{"writes":{"hz":0.0,"counter":0,"roughness":0.0},"reads":{"hz":0.0,"counter":0,"roughness":0.0}},"keys":{"read":{"hz":0.0,"counter":0,"roughness":0.0}},"bytes":{"read":{"hz":0.0,"counter":0,"roughness":0.0},"written":{"hz":0.0,"counter":0,"roughness":0.0}},"transactions":{"started":{"hz":0.0,"counter":0,"roughness":0.0},"conflicted":{"hz":0.0,"counter":0,"roughness":0.0},"committed":{"hz":0.0,"counter":0,"roughness":0.0}}},"cluster_controller_timestamp":1415650089,"protocol_version":"fdb00a400050001","configuration":{"resolvers":1,"regions":[{"satellite_redundancy_mode":"one_satellite_single","satellite_anti_quorum":0,"satellite_usable_dcs":1,"datacenters":[{"priority":1,"satellite":1,"id":"mr"}],"satellite_log_policy":"(zoneid^3x1)","satellite_log_replicas":1,"satellite_logs":2}],"remote_logs":5,"auto_logs":3,"logs":2,"log_anti_quorum":0,"storage_replicas":1,"log_replicas":2,"remote_redundancy_mode":"remote_single","storage_engine":{"$enum":["ssd","ssd-1","ssd-2","memory","custom"]},"coordinators_count":1,"log_replication_policy":"(zoneid^3x1)","log_routers":10,"storage_replication_policy":"(zoneid^3x1)","remote_log_replicas":3,"excluded_servers":[{"address":"10.0.4.1"}],"auto_proxies":3,"proxies":5,"usable_regions":1,"redundancy_mode":"single","auto_resolvers":1},"latency_probe":{"immediate_priority_transaction_start_seconds":0.0,"transaction_start_seconds":0.0,"batch_priority_transaction_start_seconds":0.0,"read_seconds":7,"commit_seconds":0.02},"machines":{"$map":{"network":{"megabits_sent":{"hz":0.0},"megabits_received":{"hz":0.0},"tcp_segments_retransmitted":{"hz":0.0}},"locality":{"$map":"value"},"memory":{"free_bytes":0,"committed_bytes":0,"total_bytes":0},"contributing_workers":4,"datacenter_id":"6344abf1813eb05b","excluded":false,"address":"1.2.3.4","machine_id":"6344abf1813eb05b","cpu":{"logical_core_utilization":0.4}}},"old_logs":[{"satellite_log_fault_tolerance":2,"logs":[{"healthy":true,"id":"7f8d623d0cb9966e","address":"1.2.3.4:1234"}],"satellite_log_write_anti_quorum":0,"remote_log_fault_tolerance":2,"log_fault_tolerance":2,"log_write_anti_quorum":0,"satellite_log_replication_factor":3,"remote_log_replication_factor":3,"log_replication_factor":3}]},"client":{"coordinators":{"coordinators":[{"reachable":true,"address":"127.0.0.1:4701"}],"quorum_reachable":true},"cluster_file":{"path":"/etc/foundationdb/fdb.cluster","up_to_date":true},"messages":[{"name":{"$enum":["inconsistent_cluster_file","unreachable_cluster_controller","no_cluster_controller","status_incomplete_client","status_incomplete_coordinators","status_incomplete_error","status_incomplete_timeout","status_incomplete_cluster","quorum_not_reachable"]},"description":"The cluster file is not up to date."}],"timestamp":1415650089,"database_status":{"available":true,"healthy":true}}}
testName=RandomClogging
testDuration=30.0
diff --git a/tests/rare/LargeApiCorrectnessStatus.txt b/tests/rare/LargeApiCorrectnessStatus.txt
index 7db5388026..4826de87f3 100644
--- a/tests/rare/LargeApiCorrectnessStatus.txt
+++ b/tests/rare/LargeApiCorrectnessStatus.txt
@@ -24,4 +24,4 @@ testTitle=ApiCorrectnessTest
testName=Status
testDuration=30.0
- schema={"cluster":{"layers":{"_valid":true,"_error":"some error description"},"processes":{"$map":{"version":"3.0.0","machine_id":"0ccb4e0feddb5583010f6b77d9d10ece","locality":{"$map":"value"},"class_source":{"$enum":["command_line","configure_auto","set_class"]},"class_type":{"$enum":["unset","storage","transaction","resolution","proxy","master","test"]},"roles":[{"query_queue_max":0,"input_bytes":{"hz":0,"counter":0,"roughness":0},"stored_bytes":12341234,"kvstore_used_bytes":12341234,"kvstore_available_bytes":12341234,"kvstore_free_bytes":12341234,"kvstore_total_bytes":12341234,"durable_bytes":{"hz":0,"counter":0,"roughness":0},"queue_disk_used_bytes":12341234,"queue_disk_available_bytes":12341234,"queue_disk_free_bytes":12341234,"queue_disk_total_bytes":12341234,"role":{"$enum":["master","proxy","log","storage","resolver","cluster_controller"]},"data_version":12341234,"data_version_lag":12341234,"id":"eb84471d68c12d1d26f692a50000003f","finished_queries":{"hz":0,"counter":0,"roughness":0}}],"command_line":"-r simulation","memory":{"available_bytes":0,"limit_bytes":0,"unused_allocated_memory":0,"used_bytes":0},"messages":[{"time":12345.12312,"type":"x","name":{"$enum":["file_open_error","incorrect_cluster_file_contents","process_error","io_error","io_timeout","platform_error","storage_server_lagging","(other FDB error messages)"]},"raw_log_message":"<stuff/>","description":"abc"}],"fault_domain":"0ccb4e0fdbdb5583010f6b77d9d10ece","excluded":false,"address":"1.2.3.4:1234","disk":{"free_bytes":3451233456234,"reads":{"hz":0,"counter":0,"sectors":0},"busy":0,"writes":{"hz":0,"counter":0,"sectors":0},"total_bytes":123412341234},"uptime_seconds":1234.2345,"cpu":{"usage_cores":0},"network":{"current_connections":0,"connections_established":{"hz":0},"connections_closed":{"hz":0},"connection_errors":{"hz":0},"megabits_sent":{"hz":0},"megabits_received":{"hz":0}}}},"old_logs":[{"logs":[{"id":"7f8d623d0cb9966e","healthy":true,"address":"1.2.3.4:1234"}],"log_replication_factor":3,"log_write_anti_quorum":0,"log_fault_tolerance":2,"remote_log_replication_factor":3,"remote_log_fault_tolerance":2,"satellite_log_replication_factor":3,"satellite_log_write_anti_quorum":0,"satellite_log_fault_tolerance":2}],"fault_tolerance":{"max_machine_failures_without_losing_availability":0,"max_machine_failures_without_losing_data":0},"qos":{"worst_queue_bytes_log_server":460,"performance_limited_by":{"reason_server_id":"7f8d623d0cb9966e","reason_id":0,"name":{"$enum":["workload","storage_server_write_queue_size","storage_server_write_bandwidth_mvcc","storage_server_readable_behind","log_server_mvcc_write_bandwidth","log_server_write_queue","storage_server_min_free_space","storage_server_min_free_space_ratio","log_server_min_free_space","log_server_min_free_space_ratio"]},"description":"The database is not being saturated by the workload."},"transactions_per_second_limit":0,"released_transactions_per_second":0,"limiting_queue_bytes_storage_server":0,"worst_queue_bytes_storage_server":0,"limiting_version_lag_storage_server":0,"worst_version_lag_storage_server":0},"incompatible_connections":[],"datacenter_version_difference":0,"database_available":true,"database_locked":false,"generation":2,"latency_probe":{"read_seconds":7,"immediate_priority_transaction_start_seconds":0,"batch_priority_transaction_start_seconds":0,"transaction_start_seconds":0,"commit_seconds":0.02},"clients":{"count":1,"supported_versions":[{"client_version":"3.0.0","connected_clients":[{"address":"127.0.0.1:9898","log_group":"default"}],"count":1,"protocol_version":"fdb00a400050001","source_version":"9430e1127b4991cbc5ab2b17f41cfffa5de07e9d"}]},"messages":[{"reasons":[{"description":"Blah."}],"unreachable_processes":[{"address":"1.2.3.4:1234"}],"name":{"$enum":["unreachable_master_worker","unreadable_configuration","full_replication_timeout","client_issues","unreachable_processes","immediate_priority_transaction_start_probe_timeout","batch_priority_transaction_start_probe_timeout","transaction_start_probe_timeout","read_probe_timeout","commit_probe_timeout","storage_servers_error","status_incomplete","layer_status_incomplete","database_availability_timeout"]},"issues":[{"name":{"$enum":["incorrect_cluster_file_contents"]},"description":"Cluster file contents do not match current cluster connection string. Verify cluster file is writable and has not been overwritten externally."}],"description":"abc"}],"recovery_state":{"required_resolvers":1,"required_proxies":1,"name":{"$enum":["reading_coordinated_state","locking_coordinated_state","locking_old_transaction_servers","reading_transaction_system_state","configuration_missing","configuration_never_created","configuration_invalid","recruiting_transaction_servers","initializing_transaction_servers","recovery_transaction","writing_coordinated_state","fully_recovered"]},"required_logs":3,"missing_logs":"7f8d623d0cb9966e","description":"Recovery complete."},"workload":{"operations":{"writes":{"hz":0,"counter":0,"roughness":0},"reads":{"hz":0,"counter":0,"roughness":0}},"bytes":{"written":{"hz":0,"counter":0,"roughness":0},"read":{"hz":0,"counter":0,"roughness":0}},"keys":{"read":{"hz":0,"counter":0,"roughness":0}},"transactions":{"started":{"hz":0,"counter":0,"roughness":0},"conflicted":{"hz":0,"counter":0,"roughness":0},"committed":{"hz":0,"counter":0,"roughness":0}}},"cluster_controller_timestamp":1415650089,"protocol_version":"fdb00a400050001","full_replication":true,"configuration":{"log_anti_quorum":0,"log_replicas":2,"log_replication_policy":"(zoneid^3x1)","redundancy_mode":"single","regions":[{"datacenters":[{"id":"mr","priority":1,"satellite":1}],"satellite_redundancy_mode":"one_satellite_single","satellite_log_replicas":1,"satellite_usable_dcs":1,"satellite_anti_quorum":0,"satellite_log_policy":"(zoneid^3x1)","satellite_logs":2}],"remote_redundancy_mode":"remote_single","remote_log_replicas":3,"remote_logs":5,"usable_regions":1,"storage_replicas":1,"resolvers":1,"storage_replication_policy":"(zoneid^3x1)","logs":2,"storage_engine":{"$enum":["ssd","ssd-1","ssd-2","memory","custom"]},"coordinators_count":1,"excluded_servers":[{"address":"10.0.4.1"}],"auto_proxies":3,"auto_resolvers":1,"auto_logs":3,"proxies":5},"data":{"least_operating_space_bytes_log_server":0,"average_partition_size_bytes":0,"state":{"healthy":true,"min_replicas_remaining":0,"name":{"$enum":["initializing","missing_data","healing","healthy_repartitioning","healthy_removing_server","healthy_rebalancing","healthy"]},"description":""},"least_operating_space_ratio_storage_server":0.1,"max_machine_failures_without_losing_availability":0,"total_disk_used_bytes":0,"total_kv_size_bytes":0,"partitions_count":2,"moving_data":{"total_written_bytes":0,"in_flight_bytes":0,"in_queue_bytes":0},"least_operating_space_bytes_storage_server":0,"max_machine_failures_without_losing_data":0},"machines":{"$map":{"network":{"megabits_sent":{"hz":0},"megabits_received":{"hz":0},"tcp_segments_retransmitted":{"hz":0}},"memory":{"free_bytes":0,"committed_bytes":0,"total_bytes":0},"contributing_workers":4,"datacenter_id":"6344abf1813eb05b","excluded":false,"address":"1.2.3.4","machine_id":"6344abf1813eb05b","locality":{"$map":"value"},"cpu":{"logical_core_utilization":0.4}}}},"client":{"coordinators":{"coordinators":[{"reachable":true,"address":"127.0.0.1:4701"}],"quorum_reachable":true},"database_status":{"available":true,"healthy":true},"messages":[{"name":{"$enum":["inconsistent_cluster_file","unreachable_cluster_controller","no_cluster_controller","status_incomplete_client","status_incomplete_coordinators","status_incomplete_error","status_incomplete_timeout","status_incomplete_cluster","quorum_not_reachable"]},"description":"The cluster file is not up to date."}],"timestamp":1415650089,"cluster_file":{"path":"/etc/foundationdb/fdb.cluster","up_to_date":true}}}
+ schema={"cluster":{"layers":{"_valid":true,"_error":"some error description"},"datacenter_version_difference":0,"processes":{"$map":{"fault_domain":"0ccb4e0fdbdb5583010f6b77d9d10ece","class_source":{"$enum":["command_line","configure_auto","set_class"]},"class_type":{"$enum":["unset","storage","transaction","resolution","proxy","master","test"]},"roles":[{"query_queue_max":0,"data_version_lag":12341234,"input_bytes":{"hz":0.0,"counter":0,"roughness":0.0},"kvstore_used_bytes":12341234,"stored_bytes":12341234,"kvstore_free_bytes":12341234,"durable_bytes":{"hz":0.0,"counter":0,"roughness":0.0},"id":"eb84471d68c12d1d26f692a50000003f","data_version":12341234,"role":{"$enum":["master","proxy","log","storage","resolver","cluster_controller"]},"queue_disk_available_bytes":12341234,"kvstore_available_bytes":12341234,"queue_disk_total_bytes":12341234,"queue_disk_used_bytes":12341234,"queue_disk_free_bytes":12341234,"kvstore_total_bytes":12341234,"finished_queries":{"hz":0.0,"counter":0,"roughness":0.0}}],"locality":{"$map":"value"},"messages":[{"description":"abc","type":"x","name":{"$enum":["file_open_error","incorrect_cluster_file_contents","process_error","io_error","io_timeout","platform_error","storage_server_lagging","(other FDB error messages)"]},"raw_log_message":"<stuff/>","time":12345.12312}],"address":"1.2.3.4:1234","command_line":"-r simulation","disk":{"free_bytes":3451233456234,"reads":{"hz":0.0,"counter":0,"sectors":0},"busy":0.0,"writes":{"hz":0.0,"counter":0,"sectors":0},"total_bytes":123412341234},"version":"3.0.0","excluded":false,"memory":{"available_bytes":0,"unused_allocated_memory":0,"limit_bytes":0,"used_bytes":0},"machine_id":"0ccb4e0feddb5583010f6b77d9d10ece","uptime_seconds":1234.2345,"cpu":{"usage_cores":0.0},"network":{"megabits_sent":{"hz":0.0},"megabits_received":{"hz":0.0},"connections_closed":{"hz":0.0},"connection_errors":{"hz":0.0},"current_connections":0,"connections_established":{"hz":0.0}}}},"clients":{"count":1,"supported_versions":[{"count":1,"protocol_version":"fdb00a400050001","client_version":"3.0.0","source_version":"9430e1127b4991cbc5ab2b17f41cfffa5de07e9d","connected_clients":[{"log_group":"default","address":"127.0.0.1:9898"}]}]},"qos":{"limiting_version_lag_storage_server":0,"released_transactions_per_second":0,"transactions_per_second_limit":0,"limiting_queue_bytes_storage_server":0,"performance_limited_by":{"reason_server_id":"7f8d623d0cb9966e","description":"The database is not being saturated by the workload.","reason_id":0,"name":{"$enum":["workload","storage_server_write_queue_size","storage_server_write_bandwidth_mvcc","storage_server_readable_behind","log_server_mvcc_write_bandwidth","log_server_write_queue","storage_server_min_free_space","storage_server_min_free_space_ratio","log_server_min_free_space","log_server_min_free_space_ratio"]}},"worst_version_lag_storage_server":0,"worst_queue_bytes_log_server":460,"worst_queue_bytes_storage_server":0},"incompatible_connections":[],"full_replication":true,"database_locked":false,"generation":2,"data":{"least_operating_space_bytes_log_server":0,"average_partition_size_bytes":0,"state":{"healthy":true,"description":"","name":{"$enum":["initializing","missing_data","healing","healthy_repartitioning","healthy_removing_server","healthy_rebalancing","healthy"]},"min_replicas_remaining":0},"least_operating_space_ratio_storage_server":0.1,"max_machine_failures_without_losing_availability":0,"total_disk_used_bytes":0,"total_kv_size_bytes":0,"max_machine_failures_without_losing_data":0,"moving_data":{"in_queue_bytes":0,"total_written_bytes":0,"in_flight_bytes":0},"least_operating_space_bytes_storage_server":0,"partitions_count":2},"fault_tolerance":{"max_machine_failures_without_losing_availability":0,"max_machine_failures_without_losing_data":0},"messages":[{"reasons":[{"description":"Blah."}],"unreachable_processes":[{"address":"1.2.3.4:1234"}],"name":{"$enum":["unreachable_master_worker","unreadable_configuration","full_replication_timeout","client_issues","unreachable_processes","immediate_priority_transaction_start_probe_timeout","batch_priority_transaction_start_probe_timeout","transaction_start_probe_timeout","read_probe_timeout","commit_probe_timeout","storage_servers_error","status_incomplete","layer_status_incomplete","database_availability_timeout"]},"issues":[{"name":{"$enum":["incorrect_cluster_file_contents"]},"description":"Cluster file contents do not match current cluster connection string. Verify cluster file is writable and has not been overwritten externally."}],"description":"abc"}],"database_available":true,"recovery_state":{"required_proxies":1,"name":{"$enum":["reading_coordinated_state","locking_coordinated_state","locking_old_transaction_servers","reading_transaction_system_state","configuration_missing","configuration_never_created","configuration_invalid","recruiting_transaction_servers","initializing_transaction_servers","recovery_transaction","writing_coordinated_state","fully_recovered"]},"missing_logs":"7f8d623d0cb9966e","required_resolvers":1,"required_logs":3,"description":"Recovery complete."},"workload":{"operations":{"writes":{"hz":0.0,"counter":0,"roughness":0.0},"reads":{"hz":0.0,"counter":0,"roughness":0.0}},"keys":{"read":{"hz":0.0,"counter":0,"roughness":0.0}},"bytes":{"read":{"hz":0.0,"counter":0,"roughness":0.0},"written":{"hz":0.0,"counter":0,"roughness":0.0}},"transactions":{"started":{"hz":0.0,"counter":0,"roughness":0.0},"conflicted":{"hz":0.0,"counter":0,"roughness":0.0},"committed":{"hz":0.0,"counter":0,"roughness":0.0}}},"cluster_controller_timestamp":1415650089,"protocol_version":"fdb00a400050001","configuration":{"resolvers":1,"regions":[{"satellite_redundancy_mode":"one_satellite_single","satellite_anti_quorum":0,"satellite_usable_dcs":1,"datacenters":[{"priority":1,"satellite":1,"id":"mr"}],"satellite_log_policy":"(zoneid^3x1)","satellite_log_replicas":1,"satellite_logs":2}],"remote_logs":5,"auto_logs":3,"logs":2,"log_anti_quorum":0,"storage_replicas":1,"log_replicas":2,"remote_redundancy_mode":"remote_single","storage_engine":{"$enum":["ssd","ssd-1","ssd-2","memory","custom"]},"coordinators_count":1,"log_replication_policy":"(zoneid^3x1)","log_routers":10,"storage_replication_policy":"(zoneid^3x1)","remote_log_replicas":3,"excluded_servers":[{"address":"10.0.4.1"}],"auto_proxies":3,"proxies":5,"usable_regions":1,"redundancy_mode":"single","auto_resolvers":1},"latency_probe":{"immediate_priority_transaction_start_seconds":0.0,"transaction_start_seconds":0.0,"batch_priority_transaction_start_seconds":0.0,"read_seconds":7,"commit_seconds":0.02},"machines":{"$map":{"network":{"megabits_sent":{"hz":0.0},"megabits_received":{"hz":0.0},"tcp_segments_retransmitted":{"hz":0.0}},"locality":{"$map":"value"},"memory":{"free_bytes":0,"committed_bytes":0,"total_bytes":0},"contributing_workers":4,"datacenter_id":"6344abf1813eb05b","excluded":false,"address":"1.2.3.4","machine_id":"6344abf1813eb05b","cpu":{"logical_core_utilization":0.4}}},"old_logs":[{"satellite_log_fault_tolerance":2,"logs":[{"healthy":true,"id":"7f8d623d0cb9966e","address":"1.2.3.4:1234"}],"satellite_log_write_anti_quorum":0,"remote_log_fault_tolerance":2,"log_fault_tolerance":2,"log_write_anti_quorum":0,"satellite_log_replication_factor":3,"remote_log_replication_factor":3,"log_replication_factor":3}]},"client":{"coordinators":{"coordinators":[{"reachable":true,"address":"127.0.0.1:4701"}],"quorum_reachable":true},"cluster_file":{"path":"/etc/foundationdb/fdb.cluster","up_to_date":true},"messages":[{"name":{"$enum":["inconsistent_cluster_file","unreachable_cluster_controller","no_cluster_controller","status_incomplete_client","status_incomplete_coordinators","status_incomplete_error","status_incomplete_timeout","status_incomplete_cluster","quorum_not_reachable"]},"description":"The cluster file is not up to date."}],"timestamp":1415650089,"database_status":{"available":true,"healthy":true}}}
diff --git a/tests/slow/DDBalanceAndRemoveStatus.txt b/tests/slow/DDBalanceAndRemoveStatus.txt
index 7e07285c3c..17cb5900b4 100644
--- a/tests/slow/DDBalanceAndRemoveStatus.txt
+++ b/tests/slow/DDBalanceAndRemoveStatus.txt
@@ -43,4 +43,4 @@ testTitle=DDBalance_test
testName=Status
testDuration=30.0
- schema={"cluster":{"layers":{"_valid":true,"_error":"some error description"},"processes":{"$map":{"version":"3.0.0","machine_id":"0ccb4e0feddb5583010f6b77d9d10ece","locality":{"$map":"value"},"class_source":{"$enum":["command_line","configure_auto","set_class"]},"class_type":{"$enum":["unset","storage","transaction","resolution","proxy","master","test"]},"roles":[{"query_queue_max":0,"input_bytes":{"hz":0,"counter":0,"roughness":0},"stored_bytes":12341234,"kvstore_used_bytes":12341234,"kvstore_available_bytes":12341234,"kvstore_free_bytes":12341234,"kvstore_total_bytes":12341234,"durable_bytes":{"hz":0,"counter":0,"roughness":0},"queue_disk_used_bytes":12341234,"queue_disk_available_bytes":12341234,"queue_disk_free_bytes":12341234,"queue_disk_total_bytes":12341234,"role":{"$enum":["master","proxy","log","storage","resolver","cluster_controller"]},"data_version":12341234,"data_version_lag":12341234,"id":"eb84471d68c12d1d26f692a50000003f","finished_queries":{"hz":0,"counter":0,"roughness":0}}],"command_line":"-r simulation","memory":{"available_bytes":0,"limit_bytes":0,"unused_allocated_memory":0,"used_bytes":0},"messages":[{"time":12345.12312,"type":"x","name":{"$enum":["file_open_error","incorrect_cluster_file_contents","process_error","io_error","io_timeout","platform_error","storage_server_lagging","(other FDB error messages)"]},"raw_log_message":"<stuff/>","description":"abc"}],"fault_domain":"0ccb4e0fdbdb5583010f6b77d9d10ece","excluded":false,"address":"1.2.3.4:1234","disk":{"free_bytes":3451233456234,"reads":{"hz":0,"counter":0,"sectors":0},"busy":0,"writes":{"hz":0,"counter":0,"sectors":0},"total_bytes":123412341234},"uptime_seconds":1234.2345,"cpu":{"usage_cores":0},"network":{"current_connections":0,"connections_established":{"hz":0},"connections_closed":{"hz":0},"connection_errors":{"hz":0},"megabits_sent":{"hz":0},"megabits_received":{"hz":0}}}},"old_logs":[{"logs":[{"id":"7f8d623d0cb9966e","healthy":true,"address":"1.2.3.4:1234"}],"log_replication_factor":3,"log_write_anti_quorum":0,"log_fault_tolerance":2,"remote_log_replication_factor":3,"remote_log_fault_tolerance":2,"satellite_log_replication_factor":3,"satellite_log_write_anti_quorum":0,"satellite_log_fault_tolerance":2}],"fault_tolerance":{"max_machine_failures_without_losing_availability":0,"max_machine_failures_without_losing_data":0},"qos":{"worst_queue_bytes_log_server":460,"performance_limited_by":{"reason_server_id":"7f8d623d0cb9966e","reason_id":0,"name":{"$enum":["workload","storage_server_write_queue_size","storage_server_write_bandwidth_mvcc","storage_server_readable_behind","log_server_mvcc_write_bandwidth","log_server_write_queue","storage_server_min_free_space","storage_server_min_free_space_ratio","log_server_min_free_space","log_server_min_free_space_ratio"]},"description":"The database is not being saturated by the workload."},"transactions_per_second_limit":0,"released_transactions_per_second":0,"limiting_queue_bytes_storage_server":0,"worst_queue_bytes_storage_server":0,"limiting_version_lag_storage_server":0,"worst_version_lag_storage_server":0},"incompatible_connections":[],"datacenter_version_difference":0,"database_available":true,"database_locked":false,"generation":2,"latency_probe":{"read_seconds":7,"immediate_priority_transaction_start_seconds":0,"batch_priority_transaction_start_seconds":0,"transaction_start_seconds":0,"commit_seconds":0.02},"clients":{"count":1,"supported_versions":[{"client_version":"3.0.0","connected_clients":[{"address":"127.0.0.1:9898","log_group":"default"}],"count":1,"protocol_version":"fdb00a400050001","source_version":"9430e1127b4991cbc5ab2b17f41cfffa5de07e9d"}]},"messages":[{"reasons":[{"description":"Blah."}],"unreachable_processes":[{"address":"1.2.3.4:1234"}],"name":{"$enum":["unreachable_master_worker","unreadable_configuration","full_replication_timeout","client_issues","unreachable_processes","immediate_priority_transaction_start_probe_timeout","batch_priority_transaction_start_probe_timeout","transaction_start_probe_timeout","read_probe_timeout","commit_probe_timeout","storage_servers_error","status_incomplete","layer_status_incomplete","database_availability_timeout"]},"issues":[{"name":{"$enum":["incorrect_cluster_file_contents"]},"description":"Cluster file contents do not match current cluster connection string. Verify cluster file is writable and has not been overwritten externally."}],"description":"abc"}],"recovery_state":{"required_resolvers":1,"required_proxies":1,"name":{"$enum":["reading_coordinated_state","locking_coordinated_state","locking_old_transaction_servers","reading_transaction_system_state","configuration_missing","configuration_never_created","configuration_invalid","recruiting_transaction_servers","initializing_transaction_servers","recovery_transaction","writing_coordinated_state","fully_recovered"]},"required_logs":3,"missing_logs":"7f8d623d0cb9966e","description":"Recovery complete."},"workload":{"operations":{"writes":{"hz":0,"counter":0,"roughness":0},"reads":{"hz":0,"counter":0,"roughness":0}},"bytes":{"written":{"hz":0,"counter":0,"roughness":0},"read":{"hz":0,"counter":0,"roughness":0}},"keys":{"read":{"hz":0,"counter":0,"roughness":0}},"transactions":{"started":{"hz":0,"counter":0,"roughness":0},"conflicted":{"hz":0,"counter":0,"roughness":0},"committed":{"hz":0,"counter":0,"roughness":0}}},"cluster_controller_timestamp":1415650089,"protocol_version":"fdb00a400050001","full_replication":true,"configuration":{"log_anti_quorum":0,"log_replicas":2,"log_replication_policy":"(zoneid^3x1)","redundancy_mode":"single","regions":[{"datacenters":[{"id":"mr","priority":1,"satellite":1}],"satellite_redundancy_mode":"one_satellite_single","satellite_log_replicas":1,"satellite_usable_dcs":1,"satellite_anti_quorum":0,"satellite_log_policy":"(zoneid^3x1)","satellite_logs":2}],"remote_redundancy_mode":"remote_single","remote_log_replicas":3,"remote_logs":5,"usable_regions":1,"storage_replicas":1,"resolvers":1,"storage_replication_policy":"(zoneid^3x1)","logs":2,"storage_engine":{"$enum":["ssd","ssd-1","ssd-2","memory","custom"]},"coordinators_count":1,"excluded_servers":[{"address":"10.0.4.1"}],"auto_proxies":3,"auto_resolvers":1,"auto_logs":3,"proxies":5},"data":{"least_operating_space_bytes_log_server":0,"average_partition_size_bytes":0,"state":{"healthy":true,"min_replicas_remaining":0,"name":{"$enum":["initializing","missing_data","healing","healthy_repartitioning","healthy_removing_server","healthy_rebalancing","healthy"]},"description":""},"least_operating_space_ratio_storage_server":0.1,"max_machine_failures_without_losing_availability":0,"total_disk_used_bytes":0,"total_kv_size_bytes":0,"partitions_count":2,"moving_data":{"total_written_bytes":0,"in_flight_bytes":0,"in_queue_bytes":0},"least_operating_space_bytes_storage_server":0,"max_machine_failures_without_losing_data":0},"machines":{"$map":{"network":{"megabits_sent":{"hz":0},"megabits_received":{"hz":0},"tcp_segments_retransmitted":{"hz":0}},"memory":{"free_bytes":0,"committed_bytes":0,"total_bytes":0},"contributing_workers":4,"datacenter_id":"6344abf1813eb05b","excluded":false,"address":"1.2.3.4","machine_id":"6344abf1813eb05b","locality":{"$map":"value"},"cpu":{"logical_core_utilization":0.4}}}},"client":{"coordinators":{"coordinators":[{"reachable":true,"address":"127.0.0.1:4701"}],"quorum_reachable":true},"database_status":{"available":true,"healthy":true},"messages":[{"name":{"$enum":["inconsistent_cluster_file","unreachable_cluster_controller","no_cluster_controller","status_incomplete_client","status_incomplete_coordinators","status_incomplete_error","status_incomplete_timeout","status_incomplete_cluster","quorum_not_reachable"]},"description":"The cluster file is not up to date."}],"timestamp":1415650089,"cluster_file":{"path":"/etc/foundationdb/fdb.cluster","up_to_date":true}}}
+ schema={"cluster":{"layers":{"_valid":true,"_error":"some error description"},"datacenter_version_difference":0,"processes":{"$map":{"fault_domain":"0ccb4e0fdbdb5583010f6b77d9d10ece","class_source":{"$enum":["command_line","configure_auto","set_class"]},"class_type":{"$enum":["unset","storage","transaction","resolution","proxy","master","test"]},"roles":[{"query_queue_max":0,"data_version_lag":12341234,"input_bytes":{"hz":0.0,"counter":0,"roughness":0.0},"kvstore_used_bytes":12341234,"stored_bytes":12341234,"kvstore_free_bytes":12341234,"durable_bytes":{"hz":0.0,"counter":0,"roughness":0.0},"id":"eb84471d68c12d1d26f692a50000003f","data_version":12341234,"role":{"$enum":["master","proxy","log","storage","resolver","cluster_controller"]},"queue_disk_available_bytes":12341234,"kvstore_available_bytes":12341234,"queue_disk_total_bytes":12341234,"queue_disk_used_bytes":12341234,"queue_disk_free_bytes":12341234,"kvstore_total_bytes":12341234,"finished_queries":{"hz":0.0,"counter":0,"roughness":0.0}}],"locality":{"$map":"value"},"messages":[{"description":"abc","type":"x","name":{"$enum":["file_open_error","incorrect_cluster_file_contents","process_error","io_error","io_timeout","platform_error","storage_server_lagging","(other FDB error messages)"]},"raw_log_message":"<stuff/>","time":12345.12312}],"address":"1.2.3.4:1234","command_line":"-r simulation","disk":{"free_bytes":3451233456234,"reads":{"hz":0.0,"counter":0,"sectors":0},"busy":0.0,"writes":{"hz":0.0,"counter":0,"sectors":0},"total_bytes":123412341234},"version":"3.0.0","excluded":false,"memory":{"available_bytes":0,"unused_allocated_memory":0,"limit_bytes":0,"used_bytes":0},"machine_id":"0ccb4e0feddb5583010f6b77d9d10ece","uptime_seconds":1234.2345,"cpu":{"usage_cores":0.0},"network":{"megabits_sent":{"hz":0.0},"megabits_received":{"hz":0.0},"connections_closed":{"hz":0.0},"connection_errors":{"hz":0.0},"current_connections":0,"connections_established":{"hz":0.0}}}},"clients":{"count":1,"supported_versions":[{"count":1,"protocol_version":"fdb00a400050001","client_version":"3.0.0","source_version":"9430e1127b4991cbc5ab2b17f41cfffa5de07e9d","connected_clients":[{"log_group":"default","address":"127.0.0.1:9898"}]}]},"qos":{"limiting_version_lag_storage_server":0,"released_transactions_per_second":0,"transactions_per_second_limit":0,"limiting_queue_bytes_storage_server":0,"performance_limited_by":{"reason_server_id":"7f8d623d0cb9966e","description":"The database is not being saturated by the workload.","reason_id":0,"name":{"$enum":["workload","storage_server_write_queue_size","storage_server_write_bandwidth_mvcc","storage_server_readable_behind","log_server_mvcc_write_bandwidth","log_server_write_queue","storage_server_min_free_space","storage_server_min_free_space_ratio","log_server_min_free_space","log_server_min_free_space_ratio"]}},"worst_version_lag_storage_server":0,"worst_queue_bytes_log_server":460,"worst_queue_bytes_storage_server":0},"incompatible_connections":[],"full_replication":true,"database_locked":false,"generation":2,"data":{"least_operating_space_bytes_log_server":0,"average_partition_size_bytes":0,"state":{"healthy":true,"description":"","name":{"$enum":["initializing","missing_data","healing","healthy_repartitioning","healthy_removing_server","healthy_rebalancing","healthy"]},"min_replicas_remaining":0},"least_operating_space_ratio_storage_server":0.1,"max_machine_failures_without_losing_availability":0,"total_disk_used_bytes":0,"total_kv_size_bytes":0,"max_machine_failures_without_losing_data":0,"moving_data":{"in_queue_bytes":0,"total_written_bytes":0,"in_flight_bytes":0},"least_operating_space_bytes_storage_server":0,"partitions_count":2},"fault_tolerance":{"max_machine_failures_without_losing_availability":0,"max_machine_failures_without_losing_data":0},"messages":[{"reasons":[{"description":"Blah."}],"unreachable_processes":[{"address":"1.2.3.4:1234"}],"name":{"$enum":["unreachable_master_worker","unreadable_configuration","full_replication_timeout","client_issues","unreachable_processes","immediate_priority_transaction_start_probe_timeout","batch_priority_transaction_start_probe_timeout","transaction_start_probe_timeout","read_probe_timeout","commit_probe_timeout","storage_servers_error","status_incomplete","layer_status_incomplete","database_availability_timeout"]},"issues":[{"name":{"$enum":["incorrect_cluster_file_contents"]},"description":"Cluster file contents do not match current cluster connection string. Verify cluster file is writable and has not been overwritten externally."}],"description":"abc"}],"database_available":true,"recovery_state":{"required_proxies":1,"name":{"$enum":["reading_coordinated_state","locking_coordinated_state","locking_old_transaction_servers","reading_transaction_system_state","configuration_missing","configuration_never_created","configuration_invalid","recruiting_transaction_servers","initializing_transaction_servers","recovery_transaction","writing_coordinated_state","fully_recovered"]},"missing_logs":"7f8d623d0cb9966e","required_resolvers":1,"required_logs":3,"description":"Recovery complete."},"workload":{"operations":{"writes":{"hz":0.0,"counter":0,"roughness":0.0},"reads":{"hz":0.0,"counter":0,"roughness":0.0}},"keys":{"read":{"hz":0.0,"counter":0,"roughness":0.0}},"bytes":{"read":{"hz":0.0,"counter":0,"roughness":0.0},"written":{"hz":0.0,"counter":0,"roughness":0.0}},"transactions":{"started":{"hz":0.0,"counter":0,"roughness":0.0},"conflicted":{"hz":0.0,"counter":0,"roughness":0.0},"committed":{"hz":0.0,"counter":0,"roughness":0.0}}},"cluster_controller_timestamp":1415650089,"protocol_version":"fdb00a400050001","configuration":{"resolvers":1,"regions":[{"satellite_redundancy_mode":"one_satellite_single","satellite_anti_quorum":0,"satellite_usable_dcs":1,"datacenters":[{"priority":1,"satellite":1,"id":"mr"}],"satellite_log_policy":"(zoneid^3x1)","satellite_log_replicas":1,"satellite_logs":2}],"remote_logs":5,"auto_logs":3,"logs":2,"log_anti_quorum":0,"storage_replicas":1,"log_replicas":2,"remote_redundancy_mode":"remote_single","storage_engine":{"$enum":["ssd","ssd-1","ssd-2","memory","custom"]},"coordinators_count":1,"log_replication_policy":"(zoneid^3x1)","log_routers":10,"storage_replication_policy":"(zoneid^3x1)","remote_log_replicas":3,"excluded_servers":[{"address":"10.0.4.1"}],"auto_proxies":3,"proxies":5,"usable_regions":1,"redundancy_mode":"single","auto_resolvers":1},"latency_probe":{"immediate_priority_transaction_start_seconds":0.0,"transaction_start_seconds":0.0,"batch_priority_transaction_start_seconds":0.0,"read_seconds":7,"commit_seconds":0.02},"machines":{"$map":{"network":{"megabits_sent":{"hz":0.0},"megabits_received":{"hz":0.0},"tcp_segments_retransmitted":{"hz":0.0}},"locality":{"$map":"value"},"memory":{"free_bytes":0,"committed_bytes":0,"total_bytes":0},"contributing_workers":4,"datacenter_id":"6344abf1813eb05b","excluded":false,"address":"1.2.3.4","machine_id":"6344abf1813eb05b","cpu":{"logical_core_utilization":0.4}}},"old_logs":[{"satellite_log_fault_tolerance":2,"logs":[{"healthy":true,"id":"7f8d623d0cb9966e","address":"1.2.3.4:1234"}],"satellite_log_write_anti_quorum":0,"remote_log_fault_tolerance":2,"log_fault_tolerance":2,"log_write_anti_quorum":0,"satellite_log_replication_factor":3,"remote_log_replication_factor":3,"log_replication_factor":3}]},"client":{"coordinators":{"coordinators":[{"reachable":true,"address":"127.0.0.1:4701"}],"quorum_reachable":true},"cluster_file":{"path":"/etc/foundationdb/fdb.cluster","up_to_date":true},"messages":[{"name":{"$enum":["inconsistent_cluster_file","unreachable_cluster_controller","no_cluster_controller","status_incomplete_client","status_incomplete_coordinators","status_incomplete_error","status_incomplete_timeout","status_incomplete_cluster","quorum_not_reachable"]},"description":"The cluster file is not up to date."}],"timestamp":1415650089,"database_status":{"available":true,"healthy":true}}}