From b19611010a652efb1e8357def764daf3b4d5124c Mon Sep 17 00:00:00 2001
From: "A.J. Beamon" <ajbeamon@apple.com>
Date: Wed, 19 Jul 2017 12:58:21 -0700
Subject: [PATCH 01/14] Add ability to disable options in specific bindings,
 use it to disable callbacks on external threads in java

---
 .../cie/foundationdb/test/AbstractTester.java |  3 +-
 fdbclient/vexillographer/fdb.options          |  3 +-
 fdbclient/vexillographer/java.cs              |  1 -
 fdbclient/vexillographer/vexillographer.cs    | 34 ++++++++++++-------
 4 files changed, 26 insertions(+), 15 deletions(-)

diff --git a/bindings/java/src-completable/test/com/apple/cie/foundationdb/test/AbstractTester.java b/bindings/java/src-completable/test/com/apple/cie/foundationdb/test/AbstractTester.java
index 2ef9abb340..e5cb4e596c 100644
--- a/bindings/java/src-completable/test/com/apple/cie/foundationdb/test/AbstractTester.java
+++ b/bindings/java/src-completable/test/com/apple/cie/foundationdb/test/AbstractTester.java
@@ -94,7 +94,8 @@ public abstract class AbstractTester {
             if (!args.useExternalClient()) {
                 throw new IllegalArgumentException("Cannot enable callbacks on external thread without using external client");
             }
-            fdb.options().setCallbacksOnExternalThreads();
+            throw new IllegalArgumentException("Cannot enable callbacks on external thread in Java");
+            //fdb.options().setCallbacksOnExternalThreads();
         }
         if (args.useExternalClient()) {
             fdb.options().setDisableLocalClient();
diff --git a/fdbclient/vexillographer/fdb.options b/fdbclient/vexillographer/fdb.options
index 6e70dd9eec..a9dc05ec7f 100644
--- a/fdbclient/vexillographer/fdb.options
+++ b/fdbclient/vexillographer/fdb.options
@@ -82,7 +82,8 @@ description is not currently required but encouraged.
     <Option name="disable_multi_version_client_api" code="60"
             description="Disables the multi-version client API and instead uses the local client directly. Must be set before setting up the network." />
     <Option name="callbacks_on_external_threads" code="61"
-            description="If set, callbacks from external client libraries can be called from threads created by the FoundationDB client library. Otherwise, callbacks will be called from either the thread used to add the callback or the network thread. Setting this option can improve performance when connected using an external client, but may not be safe to use in all environments. Must be set before setting up the network." />
+            description="If set, callbacks from external client libraries can be called from threads created by the FoundationDB client library. Otherwise, callbacks will be called from either the thread used to add the callback or the network thread. Setting this option can improve performance when connected using an external client, but may not be safe to use in all environments. Must be set before setting up the network." 
+            disableOn="java" />
     <Option name="external_client_library" code="62"
             paramType="String" paramDescription="path to client library"
             description="Adds an external client library for use by the multi-version client API. Must be set before setting up the network." />
diff --git a/fdbclient/vexillographer/java.cs b/fdbclient/vexillographer/java.cs
index bcce468bcf..8b94173413 100644
--- a/fdbclient/vexillographer/java.cs
+++ b/fdbclient/vexillographer/java.cs
@@ -168,7 +168,6 @@ namespace vexillographer
 
         private static void writePredicateClass(TextWriter outFile, Scope scope, IEnumerable<Option> options)
         {
-            string className = scope.ToString() + "s";
             outFile.WriteLine(
 @"package com.apple.cie.foundationdb;
 
diff --git a/fdbclient/vexillographer/vexillographer.cs b/fdbclient/vexillographer/vexillographer.cs
index 683cc96fcc..5582638a40 100644
--- a/fdbclient/vexillographer/vexillographer.cs
+++ b/fdbclient/vexillographer/vexillographer.cs
@@ -88,7 +88,7 @@ namespace vexillographer
             }
 
             IEnumerable<Option> options;
-            int result = parseOptions(args[0], out options);
+            int result = parseOptions(args[0], out options, args[1]);
             if (result != 0)
                 return result;
 
@@ -117,7 +117,7 @@ namespace vexillographer
                 Environment.GetCommandLineArgs()[0]);
         }
 
-        private static int parseOptions(string path, out IEnumerable<Option> options)
+        private static int parseOptions(string path, out IEnumerable<Option> options, string binding)
         {
 
             var list = new List<Option>();
@@ -133,17 +133,27 @@ namespace vexillographer
                         var paramTypeStr = oDoc.AttributeOrNull("paramType");
                         ParamType p = paramTypeStr == null ? ParamType.None : (ParamType)Enum.Parse(typeof(ParamType), paramTypeStr);
 						bool hidden = oDoc.AttributeOrNull("hidden") == "true";
+						string disableOn = oDoc.AttributeOrNull("disableOn");
+						bool disabled = false;
+						if(disableOn != null)
+						{
+							string[] disabledBindings = disableOn.Split(',');
+							disabled = disabledBindings.Contains(binding);
+						}
 
-                        list.Add(new Option
-                        {
-                            scope = s,
-                            name = oDoc.AttributeNonNull("name"),
-                            code = int.Parse(oDoc.AttributeNonNull("code")),
-                            paramType = p,
-                            paramDesc = oDoc.AttributeOrNull("paramDescription"),
-                            comment = oDoc.AttributeOrNull("description"),
-							hidden = hidden
-                        });
+						if (!disabled)
+						{
+							list.Add(new Option
+							{
+								scope = s,
+								name = oDoc.AttributeNonNull("name"),
+								code = int.Parse(oDoc.AttributeNonNull("code")),
+								paramType = p,
+								paramDesc = oDoc.AttributeOrNull("paramDescription"),
+								comment = oDoc.AttributeOrNull("description"),
+								hidden = hidden
+							});
+						}
                     }
                 }
                 options = list;

From 9d8350180008fb22f18096b7adf854628880b572 Mon Sep 17 00:00:00 2001
From: "A.J. Beamon" <ajbeamon@apple.com>
Date: Thu, 30 Nov 2017 13:57:05 -0800
Subject: [PATCH 02/14] fdbmonitor now deconfigures processes when the config
 file is deleted. Depending on the specified settings, running processes may
 or may not be killed in that case. If they are not killed, then they are
 rejoined with the config if it reappears.

In this implementation, the deletion or moving of a parent directory does not trigger an immediate reload of the configuration, but the change will be detected the next time the configuration is reloaded for other reasons.
---
 fdbmonitor/fdbmonitor.cpp | 259 +++++++++++++++++++++++++-------------
 1 file changed, 169 insertions(+), 90 deletions(-)

diff --git a/fdbmonitor/fdbmonitor.cpp b/fdbmonitor/fdbmonitor.cpp
index 374d7fe767..0f9df4036f 100644
--- a/fdbmonitor/fdbmonitor.cpp
+++ b/fdbmonitor/fdbmonitor.cpp
@@ -621,62 +621,64 @@ void load_conf(const char* confpath, uid_t &uid, gid_t &gid, sigset_t* mask, fdb
 	ini.SetUnicode();
 
 	SI_Error err = ini.LoadFile(confpath);
-	if (err<0) {
+	bool loadedConf = err >= 0;
+	if (!loadedConf) {
 		log_msg(LOG_ERR, "Unable to load configuration file %s (SI_Error: %d, errno: %d)\n", confpath, err, errno);
-		return;
 	}
 
-	uid_t _uid;
-	gid_t _gid;
+	if(loadedConf) {
+		uid_t _uid;
+		gid_t _gid;
 
-	const char* user = ini.GetValue("fdbmonitor", "user", NULL);
-	const char* group = ini.GetValue("fdbmonitor", "group", NULL);
+		const char* user = ini.GetValue("fdbmonitor", "user", NULL);
+		const char* group = ini.GetValue("fdbmonitor", "group", NULL);
 
-	if (user) {
-		errno = 0;
-		struct passwd* pw = getpwnam(user);
-		if (!pw) {
-			log_err( "getpwnam", errno, "Unable to lookup user %s", user );
-			return;
-		}
-		_uid = pw->pw_uid;
-	} else
-		_uid = geteuid();
+		if (user) {
+			errno = 0;
+			struct passwd* pw = getpwnam(user);
+			if (!pw) {
+				log_err( "getpwnam", errno, "Unable to lookup user %s", user );
+				return;
+			}
+			_uid = pw->pw_uid;
+		} else
+			_uid = geteuid();
 
-	if (group) {
-		errno = 0;
-		struct group* gr = getgrnam(group);
-		if (!gr) {
-			log_err( "getgrnam", errno, "Unable to lookup group %s", group );
-			return;
-		}
-		_gid = gr->gr_gid;
-	} else
-		_gid = getegid();
+		if (group) {
+			errno = 0;
+			struct group* gr = getgrnam(group);
+			if (!gr) {
+				log_err( "getgrnam", errno, "Unable to lookup group %s", group );
+				return;
+			}
+			_gid = gr->gr_gid;
+		} else
+			_gid = getegid();
 
-	/* Any change to uid or gid requires the process to be restarted to take effect */
-	if (uid != _uid || gid != _gid) {
-		std::vector<uint64_t> kill_ids;
-		for (auto i : id_pid) {
-			if(id_command[i.first]->kill_on_configuration_change) {
-				kill_ids.push_back(i.first);
+		/* Any change to uid or gid requires the process to be restarted to take effect */
+		if (uid != _uid || gid != _gid) {
+			std::vector<uint64_t> kill_ids;
+			for (auto i : id_pid) {
+				if(id_command[i.first]->kill_on_configuration_change) {
+					kill_ids.push_back(i.first);
+				}
+			}
+			for (auto i : kill_ids) {
+				kill_process(i);
+				delete id_command[i];
+				id_command.erase(i);
 			}
 		}
-		for (auto i : kill_ids) {
-			kill_process(i);
-			delete id_command[i];
-			id_command.erase(i);
-		}
-	}
 
-	uid = _uid;
-	gid = _gid;
+		uid = _uid;
+		gid = _gid;
+	}
 
 	std::list<uint64_t> kill_ids;
 	std::list<std::pair<uint64_t, Command*>> start_ids;
 
 	for (auto i : id_pid) {
-		if (ini.GetSectionSize(id_command[i.first]->ssection.c_str()) == -1) {
+		if (!loadedConf || ini.GetSectionSize(id_command[i.first]->ssection.c_str()) == -1) {
 			/* Server on this port no longer configured; deconfigure it and kill it if required */
 			log_msg(LOG_INFO, "Deconfigured %s\n", id_command[i.first]->ssection.c_str());
 
@@ -717,34 +719,36 @@ void load_conf(const char* confpath, uid_t &uid, gid_t &gid, sigset_t* mask, fdb
 
 	/* We've handled deconfigured sections, now look for newly
 	   configured sections */
-	CSimpleIniA::TNamesDepend sections;
-	ini.GetAllSections(sections);
-	for (auto i : sections) {
-		if (auto dot = strrchr(i.pItem, '.')) {
-			char* strtol_end;
+	if(loadedConf) {
+		CSimpleIniA::TNamesDepend sections;
+		ini.GetAllSections(sections);
+		for (auto i : sections) {
+			if (auto dot = strrchr(i.pItem, '.')) {
+				char* strtol_end;
 
-			uint64_t id = strtoull(dot + 1, &strtol_end, 10);
+				uint64_t id = strtoull(dot + 1, &strtol_end, 10);
 
-			if (*strtol_end != '\0' || !(id > 0)) {
-				log_msg(LOG_ERR, "Found bogus id in %s\n", i.pItem);
-			} else {
-				if (!id_pid.count(id)) {
-					/* Found something we haven't yet started */
-					Command *cmd;
+				if (*strtol_end != '\0' || !(id > 0)) {
+					log_msg(LOG_ERR, "Found bogus id in %s\n", i.pItem);
+				} else {
+					if (!id_pid.count(id)) {
+						/* Found something we haven't yet started */
+						Command *cmd;
 
-					auto itr = id_command.find(id);
-					if(itr != id_command.end()) {
-						cmd = itr->second;
-					}
-					else {
-						std::string section(i.pItem, dot - i.pItem);
-						cmd = new Command(ini, section, id, rfds, maxfd);
-						id_command[id] = cmd;
-					}
+						auto itr = id_command.find(id);
+						if(itr != id_command.end()) {
+							cmd = itr->second;
+						}
+						else {
+							std::string section(i.pItem, dot - i.pItem);
+							cmd = new Command(ini, section, id, rfds, maxfd);
+							id_command[id] = cmd;
+						}
 
-					if(cmd->fork_retry_time <= timer()) {
-						log_msg(LOG_INFO, "Starting %s\n", i.pItem);
-						start_process(cmd, id, uid, gid, 0, mask);
+						if(cmd->fork_retry_time <= timer()) {
+							log_msg(LOG_INFO, "Starting %s\n", i.pItem);
+							start_process(cmd, id, uid, gid, 0, mask);
+						}
 					}
 				}
 			}
@@ -785,6 +789,48 @@ void read_child_output( Command* cmd, int pipe_idx, fdb_fd_set fds ) {
 }
 
 #ifdef __APPLE__
+void watch_conf_dir( int kq, int* confd_fd, std::string confdir ) {
+	struct kevent ev;
+	std::string original = confdir;
+
+	while(true) {
+		/* If already watching, drop it and close */
+		if ( *confd_fd >= 0 ) {
+			EV_SET( &ev, *confd_fd, EVFILT_VNODE, EV_DELETE, NOTE_WRITE, 0, NULL );
+			kevent( kq, &ev, 1, NULL, 0, NULL );
+			close( *confd_fd );
+		}
+
+		confdir = original;
+		std::string child = confdir;
+
+		/* Find the nearest existing ancestor */
+		while( (*confd_fd = open( confdir.c_str(), O_EVTONLY )) < 0 && errno == ENOENT ) {
+			child = confdir;
+			confdir = parentDirectory(confdir);
+		}
+
+		if ( *confd_fd >= 0 ) {
+			EV_SET( &ev, *confd_fd, EVFILT_VNODE, EV_ADD | EV_CLEAR, NOTE_WRITE, 0, NULL );
+			kevent( kq, &ev, 1, NULL, 0, NULL );
+
+			/* If our child appeared since we last tested it, start over from the beginning */
+			if ( confdir != child && (access(child.c_str(), F_OK) == 0 || errno != ENOENT) ) {
+				continue;
+			}
+
+			if(confdir != original) {
+				log_msg(LOG_INFO, "Watching parent directory of missing directory %s\n", child.c_str());
+			}
+			else {
+				log_msg(LOG_INFO, "Watching conf dir %s\n", confdir.c_str());
+			}
+		}
+
+		return;
+	}
+}
+
 void watch_conf_file( int kq, int* conff_fd, const char* confpath ) {
 	struct kevent ev;
 
@@ -805,12 +851,8 @@ void watch_conf_file( int kq, int* conff_fd, const char* confpath ) {
 #endif
 
 #ifdef __linux__
-void fdbmon_stat(const char *path, struct stat *path_stat, bool is_link) {
-	int result = is_link ? lstat(path, path_stat) : stat(path, path_stat);
-	if(result) {
-		perror(is_link ? "lstat" : "stat");
-		exit(1);
-	}
+int fdbmon_stat(const char *path, struct stat *path_stat, bool is_link) {
+	return is_link ? lstat(path, path_stat) : stat(path, path_stat);
 }
 
 std::unordered_map<int, std::unordered_set<std::string>> set_watches(std::string path, int ifd) {
@@ -821,20 +863,32 @@ std::unordered_map<int, std::unordered_set<std::string>> set_watches(std::string
 		return additional_watch_wds;
 
 	int idx = 1;
-	while(idx != std::string::npos) {
+	bool exists = true;
+	while(idx != std::string::npos && exists) {
 		idx = path.find_first_of('/', idx+1);
 		std::string subpath = path.substr(0, idx);
 
 		int level = 0;
 		while(true) {
-			if(level++ == 100) {
-				log_msg(LOG_ERR, "Too many nested symlinks in path %s\n", path.c_str());
-				exit(1);
+			int result = fdbmon_stat(subpath.c_str(), &path_stat, true);
+			if(result != 0) {
+				if(errno == ENOENT) {
+					exists = false;
+				}
+				else {
+					perror("lstat");
+					exit(1);
+				}
 			}
 
-			fdbmon_stat(subpath.c_str(), &path_stat, true);
-			if(!S_ISLNK(path_stat.st_mode)) {
-				break;
+			if(exists) {
+				if(!S_ISLNK(path_stat.st_mode)) {
+					break;
+				}
+				else if(level++ == 100) {
+					log_msg(LOG_ERR, "Too many nested symlinks in path %s\n", path.c_str());
+					exit(1);
+				}
 			}
 
 			std::string parent = parentDirectory(subpath);
@@ -845,8 +899,21 @@ std::unordered_map<int, std::unordered_set<std::string>> set_watches(std::string
 				exit(1);
 			}
 
-			log_msg(LOG_INFO, "Watching parent directory of symlink %s (%d)\n", subpath.c_str(), wd);
-			additional_watch_wds[wd].insert(subpath.substr(parent.size()+1));
+			if(exists) {
+				log_msg(LOG_INFO, "Watching parent directory of symlink %s (%d)\n", subpath.c_str(), wd);
+				additional_watch_wds[wd].insert(subpath.substr(parent.size()+1));
+			}
+			else {
+				/* If the subpath appeared since we last checked, we should resume traversing the path */
+				int result = fdbmon_stat(subpath.c_str(), &path_stat, true);
+				if(result == 0 || errno != ENOENT) {
+					continue;
+				}
+
+				log_msg(LOG_INFO, "Watching parent directory of missing directory %s (%d)\n", subpath.c_str(), wd);
+				additional_watch_wds[wd].insert(subpath.substr(parent.size()+1));
+				break;
+			}
 
 			char buf[PATH_MAX+1];
 			ssize_t len = readlink(subpath.c_str(), buf, PATH_MAX);
@@ -1047,13 +1114,12 @@ int main(int argc, char** argv) {
 	EV_SET( &ev, SIGCHLD, EVFILT_SIGNAL, EV_ADD, 0, 0, NULL);
 	kevent( kq, &ev, 1, NULL, 0, NULL );
 
-	int confd_fd = open(confdir.c_str(), O_EVTONLY);
+	int confd_fd = -1;
+	int conff_fd = -1;
 
 	// Watch the directory holding the configuration file
-	EV_SET( &ev, confd_fd, EVFILT_VNODE, EV_ADD | EV_CLEAR, NOTE_WRITE, 0, NULL );
-	kevent( kq, &ev, 1, NULL, 0, NULL );
+	watch_conf_dir( kq, &confd_fd, confdir );
 
-	int conff_fd = -1;
 #endif
 
 #ifdef __linux__
@@ -1097,18 +1163,29 @@ int main(int argc, char** argv) {
 			}
 			conffile_wd = inotify_add_watch(ifd, confpath.c_str(), IN_CLOSE_WRITE);
 			if (conffile_wd < 0) {
-				perror("inotify_add_watch conf file");
-				exit(1); // Deleting the conf file causes fdbmonitor to terminate
+				if(errno != ENOENT) {
+					perror("inotify_add_watch conf file");
+					exit(1);
+				}
+				else {
+					log_msg(LOG_INFO, "Conf file has been deleted %s\n", confpath.c_str());
+				}
 			} else {
-				log_msg(LOG_INFO, "Watching config file %s\n", confpath.c_str());
+				log_msg(LOG_INFO, "Watching conf file %s\n", confpath.c_str());
 			}
 
 			confdir_wd = inotify_add_watch(ifd, confdir.c_str(), IN_CLOSE_WRITE | IN_MOVED_TO);
 			if (confdir_wd < 0) {
-				perror("inotify_add_watch conf dir");
-				exit(1);
+				if(errno != ENOENT) {
+					perror("inotify_add_watch conf dir");
+					exit(1);
+				}
+				else {
+					reload_additional_watches = true;
+					log_msg(LOG_INFO, "Conf dir has been deleted %s\n", confdir.c_str());
+				}
 			} else {
-				log_msg(LOG_INFO, "Watching config dir %s (%d)\n", confdir.c_str(), confdir_wd);
+				log_msg(LOG_INFO, "Watching conf dir %s (%d)\n", confdir.c_str(), confdir_wd);
 			}
 
 			if(reload_additional_watches) {
@@ -1120,6 +1197,7 @@ int main(int argc, char** argv) {
 #elif defined(__APPLE__)
 			load_conf( confpath.c_str(), uid, gid, &normal_mask, watched_fds, &maxfd );
 			watch_conf_file( kq, &conff_fd, confpath.c_str() );
+			watch_conf_dir( kq, &confd_fd, confdir );
 #endif
 		}
 
@@ -1215,6 +1293,7 @@ int main(int argc, char** argv) {
 
 		/* select() could have returned because received an exit signal */
 		if (exit_signal > 0) {
+			char *redone_confpath;
 			switch(exit_signal) {
 				case SIGHUP:
 					for(auto i : id_command) {

From b5ed01c0538426c3100ee0728ed7bf01643e43fb Mon Sep 17 00:00:00 2001
From: Evan Tschannen <ejt@apple.com>
Date: Tue, 2 Jan 2018 11:28:30 -0800
Subject: [PATCH 03/14] updated version to 5.2

---
 versions.target | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/versions.target b/versions.target
index dbaafd8d8b..68447f3feb 100644
--- a/versions.target
+++ b/versions.target
@@ -1,7 +1,7 @@
 <?xml version="1.0"?>
 <Project xmlns="http://schemas.microsoft.com/developer/msbuild/2003">
   <PropertyGroup>
-    <Version>5.1.0</Version>
-    <PackageName>5.1</PackageName>
+    <Version>5.2.0</Version>
+    <PackageName>5.2</PackageName>
   </PropertyGroup>
 </Project>

From 6f452ae9faf85aeef9aeb7d51de70b1edd9233c1 Mon Sep 17 00:00:00 2001
From: "A.J. Beamon" <ajbeamon@apple.com>
Date: Thu, 4 Jan 2018 10:30:02 -0800
Subject: [PATCH 04/14] Remove unused variable

---
 fdbmonitor/fdbmonitor.cpp | 1 -
 1 file changed, 1 deletion(-)

diff --git a/fdbmonitor/fdbmonitor.cpp b/fdbmonitor/fdbmonitor.cpp
index 0f9df4036f..7b36e6eaf1 100644
--- a/fdbmonitor/fdbmonitor.cpp
+++ b/fdbmonitor/fdbmonitor.cpp
@@ -1293,7 +1293,6 @@ int main(int argc, char** argv) {
 
 		/* select() could have returned because received an exit signal */
 		if (exit_signal > 0) {
-			char *redone_confpath;
 			switch(exit_signal) {
 				case SIGHUP:
 					for(auto i : id_command) {

From c0d37864cff1938b6c0a4446566ffaa7a653c88c Mon Sep 17 00:00:00 2001
From: "A.J. Beamon" <ajbeamon@apple.com>
Date: Thu, 4 Jan 2018 11:30:41 -0800
Subject: [PATCH 05/14] Remove the watch on parent of missing directory if we
 detect that the directory has just shown up.

---
 fdbmonitor/fdbmonitor.cpp | 1 +
 1 file changed, 1 insertion(+)

diff --git a/fdbmonitor/fdbmonitor.cpp b/fdbmonitor/fdbmonitor.cpp
index 334aab78eb..8a8e1faf05 100644
--- a/fdbmonitor/fdbmonitor.cpp
+++ b/fdbmonitor/fdbmonitor.cpp
@@ -941,6 +941,7 @@ std::unordered_map<int, std::unordered_set<std::string>> set_watches(std::string
 				/* If the subpath appeared since we last checked, we should resume traversing the path */
 				int result = fdbmon_stat(subpath.c_str(), &path_stat, true);
 				if(result == 0 || errno != ENOENT) {
+					inotify_rm_watch(ifd, wd);
 					continue;
 				}
 

From 8891804b1bab6c6921aae8286c60620b99782ccb Mon Sep 17 00:00:00 2001
From: "A.J. Beamon" <ajbeamon@apple.com>
Date: Thu, 4 Jan 2018 11:31:01 -0800
Subject: [PATCH 06/14] Add some comments to watch setting in fdbmonitor.

---
 fdbmonitor/fdbmonitor.cpp | 16 +++++++++++++++-
 1 file changed, 15 insertions(+), 1 deletion(-)

diff --git a/fdbmonitor/fdbmonitor.cpp b/fdbmonitor/fdbmonitor.cpp
index 8a8e1faf05..99b4477b92 100644
--- a/fdbmonitor/fdbmonitor.cpp
+++ b/fdbmonitor/fdbmonitor.cpp
@@ -889,6 +889,8 @@ int fdbmon_stat(const char *path, struct stat *path_stat, bool is_link) {
 	return is_link ? lstat(path, path_stat) : stat(path, path_stat);
 }
 
+/* Sets watches to track changes to all symlinks on a path.
+ * Also sets a watch on the last existing ancestor of a path if the full path doesn't exist. */
 std::unordered_map<int, std::unordered_set<std::string>> set_watches(std::string path, int ifd) {
 	std::unordered_map<int, std::unordered_set<std::string>> additional_watch_wds;
 	struct stat path_stat;
@@ -898,12 +900,17 @@ std::unordered_map<int, std::unordered_set<std::string>> set_watches(std::string
 
 	int idx = 1;
 	bool exists = true;
+
+	/* Check each level of the path, setting a watch on any symlinks.
+	 * Stop checking once we get to a part of the path that doesn't exist.
+	 * If we encounter a non-existing path, watch the closest existing ancestor. */
 	while(idx != std::string::npos && exists) {
 		idx = path.find_first_of('/', idx+1);
 		std::string subpath = path.substr(0, idx);
 
 		int level = 0;
 		while(true) {
+			/* Check path existence */
 			int result = fdbmon_stat(subpath.c_str(), &path_stat, true);
 			if(result != 0) {
 				if(errno == ENOENT) {
@@ -916,6 +923,7 @@ std::unordered_map<int, std::unordered_set<std::string>> set_watches(std::string
 			}
 
 			if(exists) {
+				/* Don't do anything for existing non-links */
 				if(!S_ISLNK(path_stat.st_mode)) {
 					break;
 				}
@@ -927,6 +935,7 @@ std::unordered_map<int, std::unordered_set<std::string>> set_watches(std::string
 
 			std::string parent = parentDirectory(subpath);
 
+			/* Watch the parent directory of the current path for changes */
 			int wd = inotify_add_watch(ifd, parent.c_str(), IN_CREATE | IN_MOVED_TO);
 			if (wd < 0) {
 				log_err("inotify_add_watch", errno, "Unable to add watch to parent directory %s", parent.c_str());
@@ -938,7 +947,7 @@ std::unordered_map<int, std::unordered_set<std::string>> set_watches(std::string
 				additional_watch_wds[wd].insert(subpath.substr(parent.size()+1));
 			}
 			else {
-				/* If the subpath appeared since we last checked, we should resume traversing the path */
+				/* If the subpath has appeared since we set the watch, we should cancel it and resume traversing the path */
 				int result = fdbmon_stat(subpath.c_str(), &path_stat, true);
 				if(result == 0 || errno != ENOENT) {
 					inotify_rm_watch(ifd, wd);
@@ -950,6 +959,7 @@ std::unordered_map<int, std::unordered_set<std::string>> set_watches(std::string
 				break;
 			}
 
+			/* Follow the symlink */
 			char buf[PATH_MAX+1];
 			ssize_t len = readlink(subpath.c_str(), buf, PATH_MAX);
 			if(len < 0) {
@@ -1198,12 +1208,15 @@ int main(int argc, char** argv) {
 		if (reload) {
 			reload = false;
 #ifdef __linux__
+			/* Remove existing watches on conf file and directory */
 			if(confdir_wd >= 0 && inotify_rm_watch(ifd, confdir_wd) < 0) {
 				log_msg(SevInfo, "Could not remove inotify conf dir watch, continuing...\n");
 			}
 			if(conffile_wd >= 0 && inotify_rm_watch(ifd, conffile_wd) < 0) {
 				log_msg(SevInfo, "Could not remove inotify conf file watch, continuing...\n");
 			}
+
+			/* Create new watches */
 			conffile_wd = inotify_add_watch(ifd, confpath.c_str(), IN_CLOSE_WRITE);
 			if (conffile_wd < 0) {
 				if(errno != ENOENT) {
@@ -1231,6 +1244,7 @@ int main(int argc, char** argv) {
 				log_msg(SevInfo, "Watching conf dir %s (%d)\n", confdir.c_str(), confdir_wd);
 			}
 
+			/* Reload watches on symlinks and/or the oldest existing ancestor */
 			if(reload_additional_watches) {
 				additional_watch_wds = set_watches(_confpath, ifd);
 			}

From b264a98aea7f487354a48cc6e9b51bff7a004832 Mon Sep 17 00:00:00 2001
From: Alex Miller <alexmiller@apple.com>
Date: Tue, 2 Jan 2018 17:32:14 -0800
Subject: [PATCH 07/14] Fix yet another VersionStamp DR bug.

In this episode, we discover that having a transaction retry loop in which the
transaction conditionally has write conflict ranges is potentially troublesome.

To simplify the problem, if we have two concurrent transaction loops:

    retry {
      if (rand() > .5) tr->set('x', rand());
      if (rand() > .5) tr->set('y', rand());
    }

and

    retry {
      x = tr->get('x')
      y = tr->get('y')
      if (x > y) {
        tr->set('y', x)
      }
      tr->commit();
    }

Is not guaranteed that x > y in the database after the second transaction
commits.  This is because it could read an older snapshot of x and y, in which
x was greater than y, and thus not invoke set.  This means that `tr` is now a
read-only transaction, which no-ops out of committing as an "optimization".  If
we add any write conflict range to `tr`, it then will conflict checked and
committed, which would guarantee that x>y when it commits.

Replace the first transaction with dumpData, and the second with version
upgrade transaction, and you have the bug that we're fixing, why, and how.
---
 fdbclient/DatabaseBackupAgent.actor.cpp | 12 ++++++++++--
 1 file changed, 10 insertions(+), 2 deletions(-)

diff --git a/fdbclient/DatabaseBackupAgent.actor.cpp b/fdbclient/DatabaseBackupAgent.actor.cpp
index 436b49a80a..09f74a76dc 100644
--- a/fdbclient/DatabaseBackupAgent.actor.cpp
+++ b/fdbclient/DatabaseBackupAgent.actor.cpp
@@ -1445,8 +1445,8 @@ public:
 				tr3.setOption(FDBTransactionOptions::ACCESS_SYSTEM_KEYS);
 				tr3.setOption(FDBTransactionOptions::LOCK_AWARE);
 				Version destVersion = wait(tr3.getReadVersion());
+				TraceEvent("DBA_switchover_version_upgrade").detail("src", commitVersion).detail("dest", destVersion);
 				if (destVersion <= commitVersion) {
-					TraceEvent("DBA_switchover_version_upgrade").detail("src", commitVersion).detail("dest", destVersion);
 					TEST(true);  // Forcing dest backup cluster to higher version
 					tr3.set(minRequiredCommitVersionKey, BinaryWriter::toValue(commitVersion+1, Unversioned()));
 					Void _ = wait(tr3.commit());
@@ -1549,12 +1549,20 @@ public:
 				if (lastApplied.present()) {
 					Version current = tr->getReadVersion().get();
 					Version applied = BinaryReader::fromStringRef<Version>(lastApplied.get(), Unversioned());
+					TraceEvent("DBA_abort_version_upgrade").detail("src", applied).detail("dest", current);
 					if (current <= applied) {
-						TraceEvent("DBA_abort_version_upgrade").detail("src", applied).detail("dest", current);
 						TEST(true);  // Upgrading version of local database.
 						// The +1 is because we want to make sure that a versionstamped operation can't reuse
 						// the same version as an already-applied transaction.
 						tr->set(minRequiredCommitVersionKey, BinaryWriter::toValue(applied+1, Unversioned()));
+					} else {
+						// We need to enforce that the read we did of the applyMutationsBeginKey is the most
+						// recent and up to date value, as the proxy might have accepted a commit previously
+						// queued by dumpData after our read. Transactions that don't have write conflict ranges
+						// have a no-op commit(), as they become snapshot transactions to which we don't promise
+						// strict serializability.  Therefore, we add an arbitrary write conflict range to
+						// request the strict serializability guarantee that is required.
+						tr->addWriteConflictRange(singleKeyRange(minRequiredCommitVersionKey));
 					}
 				}
 				Void _ = wait(tr->commit());

From 0f20068e82de98d6912b8c6e057679b95183e288 Mon Sep 17 00:00:00 2001
From: Stephen Atherton <s_atherton@apple.com>
Date: Thu, 4 Jan 2018 22:53:31 -0800
Subject: [PATCH 08/14] Renamed all TaskBucket backup tasks to more appropriate
 names.  Created the ability to make task aliases and used this to direct old
 task names to a task definition which will abort backups created before
 version 5.1.

---
 fdbclient/FileBackupAgent.actor.cpp | 112 +++++++++++++++++++++++-----
 fdbclient/TaskBucket.h              |   1 +
 flow/IDispatched.h                  |   2 +-
 3 files changed, 96 insertions(+), 19 deletions(-)

diff --git a/fdbclient/FileBackupAgent.actor.cpp b/fdbclient/FileBackupAgent.actor.cpp
index dd4bc3defa..ab3154cb2b 100644
--- a/fdbclient/FileBackupAgent.actor.cpp
+++ b/fdbclient/FileBackupAgent.actor.cpp
@@ -674,6 +674,86 @@ namespace fileBackup {
 		return Void();
 	}
 
+	ACTOR static Future<Void> abortOldBackup(FileBackupAgent* backupAgent, Reference<ReadYourWritesTransaction> tr, std::string tagName) {
+		tr->setOption(FDBTransactionOptions::ACCESS_SYSTEM_KEYS);
+		tr->setOption(FDBTransactionOptions::LOCK_AWARE);
+
+		state Subspace tagNames = backupAgent->subspace.get(BackupAgentBase::keyTagName);
+		Optional<Value> uidStr = wait(tr->get(tagNames.pack(Key(tagName))));
+		if (!uidStr.present()) {
+			TraceEvent(SevError, "BackupTagMissing").detail("tagName", tagName.c_str());
+			throw backup_error();
+		}
+		state UID uid = BinaryReader::fromStringRef<UID>(uidStr.get(), Unversioned());
+
+		state Subspace statusSpace = backupAgent->subspace.get(BackupAgentBase::keyStates).get(uid.toString());
+		state Subspace globalConfig = backupAgent->subspace.get(BackupAgentBase::keyConfig).get(uid.toString());
+		state Subspace newConfigSpace = uidPrefixKey(LiteralStringRef("uid->config/").withPrefix(fileBackupPrefixRange.begin), uid);
+
+		Optional<Value> statusStr = wait(tr->get(statusSpace.pack(FileBackupAgent::keyStateStatus)));
+		state EBackupState status = !statusStr.present() ? FileBackupAgent::STATE_NEVERRAN : BackupAgentBase::getState(statusStr.get().toString());
+
+		if (!backupAgent->isRunnable(status)) {
+			throw backup_unneeded();
+		}
+
+		TraceEvent(SevInfo, "FileBackupAbortOld")
+				.detail("tagName", tagName.c_str())
+				.detail("status", BackupAgentBase::getStateText(status));
+
+		// Clear the folder id will prevent future tasks from executing
+		tr->clear(singleKeyRange(StringRef(globalConfig.pack(FileBackupAgent::keyFolderId))));
+
+		Key configPath = uidPrefixKey(logRangesRange.begin, uid);
+		Key logsPath = uidPrefixKey(backupLogKeys.begin, uid);
+
+		tr->clear(KeyRangeRef(configPath, strinc(configPath)));
+		tr->clear(KeyRangeRef(logsPath, strinc(logsPath)));
+		tr->clear(newConfigSpace.range());
+
+		Key statusKey = StringRef(statusSpace.pack(FileBackupAgent::keyStateStatus));
+		tr->set(statusKey, StringRef(FileBackupAgent::getStateText(BackupAgentBase::STATE_ABORTED)));
+
+		return Void();
+	}
+
+	struct AbortOldBackupTask : TaskFuncBase {
+		static StringRef name;
+		ACTOR static Future<Void> _finish(Reference<ReadYourWritesTransaction> tr, Reference<TaskBucket> taskBucket, Reference<FutureBucket> futureBucket, Reference<Task> task) {
+			state FileBackupAgent backupAgent;
+			state std::string tagName = task->params[BackupAgentBase::keyConfigBackupTag].toString();
+
+			TEST(true);  // Canceling old backup task
+
+			TraceEvent(SevInfo, "FileBackupCancelOldTask")
+					.detail("task", printable(task->params[Task::reservedTaskParamKeyType]))
+					.detail("tagName", tagName);
+			Void _ = wait(abortOldBackup(&backupAgent, tr, tagName));
+
+			Void _ = wait(taskBucket->finish(tr, task));
+			return Void();
+		}
+
+		virtual StringRef getName() const {
+			TraceEvent(SevError, "FileBackupError").detail("cause", "AbortOldBackupTaskFunc::name() should never be called");
+			ASSERT(false);
+			return StringRef();
+		}
+
+		Future<Void> execute(Database cx, Reference<TaskBucket> tb, Reference<FutureBucket> fb, Reference<Task> task) { return Future<Void>(Void()); };
+		Future<Void> finish(Reference<ReadYourWritesTransaction> tr, Reference<TaskBucket> tb, Reference<FutureBucket> fb, Reference<Task> task) { return _finish(tr, tb, fb, task); };
+	};
+	StringRef AbortOldBackupTask::name = LiteralStringRef("abort_legacy_backup");
+	REGISTER_TASKFUNC(AbortOldBackupTask);
+	REGISTER_TASKFUNC_ALIAS(AbortOldBackupTask, file_backup_diff_logs);
+	REGISTER_TASKFUNC_ALIAS(AbortOldBackupTask, file_backup_log_range);
+	REGISTER_TASKFUNC_ALIAS(AbortOldBackupTask, file_backup_logs);
+	REGISTER_TASKFUNC_ALIAS(AbortOldBackupTask, file_backup_range);
+	REGISTER_TASKFUNC_ALIAS(AbortOldBackupTask, file_backup_restorable);
+	REGISTER_TASKFUNC_ALIAS(AbortOldBackupTask, file_finish_full_backup);
+	REGISTER_TASKFUNC_ALIAS(AbortOldBackupTask, file_finished_full_backup);
+	REGISTER_TASKFUNC_ALIAS(AbortOldBackupTask, file_start_full_backup);
+
 	std::function<void(Reference<Task>)> NOP_SETUP_TASK_FN = [](Reference<Task> task) { /* NOP */ };
 	ACTOR static Future<Key> addBackupTask(StringRef name,
 										   uint32_t version,
@@ -1011,7 +1091,7 @@ namespace fileBackup {
 		}
 
 	};
-	StringRef BackupRangeTaskFunc::name = LiteralStringRef("file_backup_range");
+	StringRef BackupRangeTaskFunc::name = LiteralStringRef("file_backup_write_range");
 	const uint32_t BackupRangeTaskFunc::version = 1;
 	REGISTER_TASKFUNC(BackupRangeTaskFunc);
 
@@ -1435,7 +1515,7 @@ namespace fileBackup {
 		}
 
 	};
-	StringRef BackupSnapshotDispatchTask::name = LiteralStringRef("file_backup_snapshot_dispatch");
+	StringRef BackupSnapshotDispatchTask::name = LiteralStringRef("file_backup_dispatch_ranges");
 	const uint32_t BackupSnapshotDispatchTask::version = 1;
 	REGISTER_TASKFUNC(BackupSnapshotDispatchTask);
 
@@ -1634,7 +1714,7 @@ namespace fileBackup {
 		}
 	};
 
-	StringRef BackupLogRangeTaskFunc::name = LiteralStringRef("file_backup_log_range");
+	StringRef BackupLogRangeTaskFunc::name = LiteralStringRef("file_backup_write_logs");
 	const uint32_t BackupLogRangeTaskFunc::version = 1;
 	REGISTER_TASKFUNC(BackupLogRangeTaskFunc);
 
@@ -1722,18 +1802,18 @@ namespace fileBackup {
 		Future<Void> execute(Database cx, Reference<TaskBucket> tb, Reference<FutureBucket> fb, Reference<Task> task) { return Void(); };
 		Future<Void> finish(Reference<ReadYourWritesTransaction> tr, Reference<TaskBucket> tb, Reference<FutureBucket> fb, Reference<Task> task) { return _finish(tr, tb, fb, task); };
 	};
-	StringRef BackupLogsDispatchTask::name = LiteralStringRef("file_backup_logs");
+	StringRef BackupLogsDispatchTask::name = LiteralStringRef("file_backup_dispatch_logs");
 	const uint32_t BackupLogsDispatchTask::version = 1;
 	REGISTER_TASKFUNC(BackupLogsDispatchTask);
 
-	struct FinishedFullBackupTaskFunc : BackupTaskFuncBase {
+	struct FileBackupFinishedTask : BackupTaskFuncBase {
 		static StringRef name;
 		static const uint32_t version;
 
 		StringRef getName() const { return name; };
 
 		ACTOR static Future<Void> _finish(Reference<ReadYourWritesTransaction> tr, Reference<TaskBucket> taskBucket, Reference<FutureBucket> futureBucket, Reference<Task> task) {
-			Void _ = wait(checkTaskVersion(tr->getDatabase(), task, FinishedFullBackupTaskFunc::name, FinishedFullBackupTaskFunc::version));
+			Void _ = wait(checkTaskVersion(tr->getDatabase(), task, FileBackupFinishedTask::name, FileBackupFinishedTask::version));
 
 			state BackupConfig backup(task);
 			state UID uid = backup.getUid();
@@ -1754,8 +1834,8 @@ namespace fileBackup {
 		}
 
 		ACTOR static Future<Key> addTask(Reference<ReadYourWritesTransaction> tr, Reference<TaskBucket> taskBucket, Reference<Task> parentTask, TaskCompletionKey completionKey, Reference<TaskFuture> waitFor = Reference<TaskFuture>()) {
-			Key key = wait(addBackupTask(FinishedFullBackupTaskFunc::name,
-										 FinishedFullBackupTaskFunc::version,
+			Key key = wait(addBackupTask(FileBackupFinishedTask::name,
+										 FileBackupFinishedTask::version,
 										 tr, taskBucket, completionKey,
 										 BackupConfig(parentTask), waitFor));
 			return key;
@@ -1764,13 +1844,9 @@ namespace fileBackup {
 		Future<Void> execute(Database cx, Reference<TaskBucket> tb, Reference<FutureBucket> fb, Reference<Task> task) { return Void(); };
 		Future<Void> finish(Reference<ReadYourWritesTransaction> tr, Reference<TaskBucket> tb, Reference<FutureBucket> fb, Reference<Task> task) { return _finish(tr, tb, fb, task); };
 	};
-	StringRef FinishedFullBackupTaskFunc::name = LiteralStringRef("file_finished_full_backup");
-	const uint32_t FinishedFullBackupTaskFunc::version = 1;
-	REGISTER_TASKFUNC(FinishedFullBackupTaskFunc);
-
-	// TODO:  Register a task that will finish/delete any tasks of these types:
-	//LiteralStringRef("file_backup_diff_logs");
-	//LiteralStringRef("file_finish_full_backup");
+	StringRef FileBackupFinishedTask::name = LiteralStringRef("file_backup_finished");
+	const uint32_t FileBackupFinishedTask::version = 1;
+	REGISTER_TASKFUNC(FileBackupFinishedTask);
 
 	struct BackupSnapshotManifest : BackupTaskFuncBase {
 		static StringRef name;
@@ -1918,7 +1994,7 @@ namespace fileBackup {
 		Future<Void> execute(Database cx, Reference<TaskBucket> tb, Reference<FutureBucket> fb, Reference<Task> task) { return _execute(cx, tb, fb, task); };
 		Future<Void> finish(Reference<ReadYourWritesTransaction> tr, Reference<TaskBucket> tb, Reference<FutureBucket> fb, Reference<Task> task) { return _finish(tr, tb, fb, task); };
 	};
-	StringRef BackupSnapshotManifest::name = LiteralStringRef("file_backup_snapshot_manifest");
+	StringRef BackupSnapshotManifest::name = LiteralStringRef("file_backup_write_snapshot_manifest");
 	const uint32_t BackupSnapshotManifest::version = 1;
 	REGISTER_TASKFUNC(BackupSnapshotManifest);
 
@@ -1978,7 +2054,7 @@ namespace fileBackup {
 
 			// If a clean stop is requested, the log and snapshot tasks will quit after the backup is restorable, then the following
 			// task will clean up and set the completed state.
-			Key _ = wait(FinishedFullBackupTaskFunc::addTask(tr, taskBucket, task, TaskCompletionKey::noSignal(), backupFinished));
+			Key _ = wait(FileBackupFinishedTask::addTask(tr, taskBucket, task, TaskCompletionKey::noSignal(), backupFinished));
 
 			Void _ = wait(taskBucket->finish(tr, task));
 			return Void();
@@ -1998,7 +2074,7 @@ namespace fileBackup {
 		Future<Void> execute(Database cx, Reference<TaskBucket> tb, Reference<FutureBucket> fb, Reference<Task> task) { return _execute(cx, tb, fb, task); };
 		Future<Void> finish(Reference<ReadYourWritesTransaction> tr, Reference<TaskBucket> tb, Reference<FutureBucket> fb, Reference<Task> task) { return _finish(tr, tb, fb, task); };
 	};
-	StringRef StartFullBackupTaskFunc::name = LiteralStringRef("file_start_full_backup");
+	StringRef StartFullBackupTaskFunc::name = LiteralStringRef("file_backup_start");
 	const uint32_t StartFullBackupTaskFunc::version = 1;
 	REGISTER_TASKFUNC(StartFullBackupTaskFunc);
 
diff --git a/fdbclient/TaskBucket.h b/fdbclient/TaskBucket.h
index f9f13bfea5..714808fadc 100644
--- a/fdbclient/TaskBucket.h
+++ b/fdbclient/TaskBucket.h
@@ -391,6 +391,7 @@ struct TaskFuncBase : IDispatched<TaskFuncBase, Standalone<StringRef>, std::func
 	};
 };
 #define REGISTER_TASKFUNC(TaskFunc) REGISTER_FACTORY(TaskFuncBase, TaskFunc, name)
+#define REGISTER_TASKFUNC_ALIAS(TaskFunc, Alias) REGISTER_DISPATCHED_ALIAS(TaskFunc, Alias, TaskFunc::name, LiteralStringRef(#Alias))
 
 struct TaskCompletionKey {
 	Future<Key> get(Reference<ReadYourWritesTransaction> tr, Reference<TaskBucket> taskBucket);
diff --git a/flow/IDispatched.h b/flow/IDispatched.h
index fe69de64e9..0d4e1e7f8e 100644
--- a/flow/IDispatched.h
+++ b/flow/IDispatched.h
@@ -41,7 +41,7 @@ struct IDispatched {
 };
 
 #define REGISTER_DISPATCHED(Type, Instance, Key, Func) struct Type##Instance { Type##Instance() { ASSERT(Type::dispatches().find(Key) == Type::dispatches().end()); Type::dispatches()[Key] = Func; } }; Type##Instance _Type##Instance
-
+#define REGISTER_DISPATCHED_ALIAS(Type, Instance, Target, Alias) struct Type##Instance { Type##Instance() { ASSERT(Type::dispatches().find(Alias) == Type::dispatches().end()); ASSERT(Type::dispatches().find(Target) != Type::dispatches().end()); Type::dispatches()[Alias] = Type::dispatches()[Target]; } }; Type##Instance _Type##Instance;
 #define REGISTER_COMMAND(Type, Instance, Key, Func) REGISTER_DISPATCHED(Type, Instance, Instance::Key, Instance::Func)
 
 /*

From 50151191152b7e0410ea06d2c3bc4861748bd01d Mon Sep 17 00:00:00 2001
From: "A.J. Beamon" <ajbeamon@apple.com>
Date: Fri, 5 Jan 2018 10:29:47 -0800
Subject: [PATCH 09/14] Generalize the message that gets displayed in status if
 a cluster file's contents are incorrect.

---
 fdbclient/NativeAPI.actor.cpp                        | 2 +-
 fdbclient/StatusClient.actor.cpp                     | 9 +++++----
 fdbserver/Status.actor.cpp                           | 6 +++---
 fdbserver/worker.actor.cpp                           | 2 +-
 tests/fast/SidebandWithStatus.txt                    | 2 +-
 tests/rare/LargeApiCorrectnessStatus.txt             | 2 +-
 tests/slow/DDBalanceAndRemoveStatus.txt              | 2 +-
 tests/status/separate_cannot_write_cluster_file.json | 4 ++--
 8 files changed, 15 insertions(+), 14 deletions(-)

diff --git a/fdbclient/NativeAPI.actor.cpp b/fdbclient/NativeAPI.actor.cpp
index 4fbc369ede..8a0bd7a682 100644
--- a/fdbclient/NativeAPI.actor.cpp
+++ b/fdbclient/NativeAPI.actor.cpp
@@ -490,7 +490,7 @@ ACTOR static Future<Void> monitorClientInfo( Reference<AsyncVar<Optional<Cluster
 
 			ClusterConnectionString fileConnectionString;
 			if (ccf && !ccf->fileContentsUpToDate(fileConnectionString)) {
-				req.issues = LiteralStringRef("unable_to_write_cluster_file");
+				req.issues = LiteralStringRef("incorrect_cluster_file_contents");
 				if(ccf->canGetFilename()) {
 					TraceEvent(SevWarnAlways, "IncorrectClusterFileContents").detail("Filename", ccf->getFilename())
 						.detail("ConnectionStringFromFile", fileConnectionString.toString())
diff --git a/fdbclient/StatusClient.actor.cpp b/fdbclient/StatusClient.actor.cpp
index c6ae17c09d..2814f58fd0 100644
--- a/fdbclient/StatusClient.actor.cpp
+++ b/fdbclient/StatusClient.actor.cpp
@@ -305,12 +305,13 @@ ACTOR Future<StatusObject> clientStatusFetcher(Reference<ClusterConnectionFile>
 	statusObj["cluster_file"] = statusObjClusterFile;
 
 	if (!contentsUpToDate){
-		std::string description = "Cluster file is not up to date.\nIt contains the connection string: ";
+		std::string description = "Cluster file contents do not match current cluster connection string.";
+		description += "\nThe file contains the connection string: ";
 		description += ClusterConnectionFile(f->getFilename()).getConnectionString().toString().c_str();
-		description += "\nThe original connection string is: ";
+		description += "\nThe current connection string is: ";
 		description += f->getConnectionString().toString().c_str();
-		description += "\nThis must mean that file permissions or other platform issues have prevented the file from being updated. To change coordinators without manual intervention, the cluster file and its containing folder must be writable by all servers and clients. If a majority of the coordinators referenced by the old connection string are lost, the database will stop working until the correct cluster file is distributed to all processes.";
-		messages->push_back(makeMessage("inconsistent_cluster_file", description.c_str()));
+		description += "\nVerify cluster file is writable and has not been overwritten externally. To change coordinators without manual intervention, the cluster file and its containing folder must be writable by all servers and clients. If a majority of the coordinators referenced by the old connection string are lost, the database will stop working until the correct cluster file is distributed to all processes.";
+		messages->push_back(makeMessage("incorrect_cluster_file_contents", description.c_str()));
 	}
 
 	return statusObj;
diff --git a/fdbserver/Status.actor.cpp b/fdbserver/Status.actor.cpp
index b83c5c905d..570d109654 100644
--- a/fdbserver/Status.actor.cpp
+++ b/fdbserver/Status.actor.cpp
@@ -1530,11 +1530,11 @@ static StatusObject faultToleranceStatusFetcher(DatabaseConfiguration configurat
 }
 
 static std::string getIssueDescription(std::string name) {
-	if(name == "unable_to_write_cluster_file") {
-		return "Unable to update cluster file.";
+	if(name == "incorrect_cluster_file_contents") {
+		return "Cluster file contents do not match current cluster connection string. Verify cluster file is writable and has not been overwritten externally.";
 	}
 
-	// FIXME: name and description will be the same unless the message is 'unable_to_write_cluster_file', which is currently the only possible message
+	// FIXME: name and description will be the same unless the message is 'incorrect_cluster_file_contents', which is currently the only possible message
 	return name;
 }
 
diff --git a/fdbserver/worker.actor.cpp b/fdbserver/worker.actor.cpp
index 20ad51a3e7..9ee05807be 100644
--- a/fdbserver/worker.actor.cpp
+++ b/fdbserver/worker.actor.cpp
@@ -446,7 +446,7 @@ ACTOR Future<Void> monitorServerDBInfo( Reference<AsyncVar<Optional<ClusterContr
 
 		ClusterConnectionString fileConnectionString;
 		if (connFile && !connFile->fileContentsUpToDate(fileConnectionString)) {
-			req.issues = LiteralStringRef("unable_to_write_cluster_file");
+			req.issues = LiteralStringRef("incorrect_cluster_file_contents");
 			if(connFile->canGetFilename()) {
 				TraceEvent(SevWarnAlways, "IncorrectClusterFileContents").detail("Filename", connFile->getFilename())
 					.detail("ConnectionStringFromFile", fileConnectionString.toString())
diff --git a/tests/fast/SidebandWithStatus.txt b/tests/fast/SidebandWithStatus.txt
index 2111530181..bc3d0115be 100644
--- a/tests/fast/SidebandWithStatus.txt
+++ b/tests/fast/SidebandWithStatus.txt
@@ -5,7 +5,7 @@ testTitle=CloggedCausalConsistencyTest
 
     testName=Status
     testDuration=30.0
-    schema={"cluster":{"layers":{"_valid":true,"_error":"some error description"},"processes":{"$map":{"fault_domain":"0ccb4e0fdbdb5583010f6b77d9d10ece","class_source":{"$enum":["command_line","configure_auto","set_class"]},"class_type":{"$enum":["unset","storage","transaction","resolution","proxy","master","test"]},"roles":[{"query_queue_max":0,"data_version_lag":12341234,"input_bytes":{"hz":0.0,"counter":0,"roughness":0.0},"kvstore_used_bytes":12341234,"stored_bytes":12341234,"data_version":12341234,"kvstore_free_bytes":12341234,"durable_bytes":{"hz":0.0,"counter":0,"roughness":0.0},"id":"eb84471d68c12d1d26f692a50000003f","persistent_disk_used_bytes":12341234,"role":{"$enum":["master","proxy","log","storage","resolver","cluster_controller"]},"persistent_disk_total_bytes":12341234,"queue_disk_total_bytes":12341234,"persistent_disk_free_bytes":12341234,"queue_disk_used_bytes":12341234,"queue_disk_free_bytes":12341234,"kvstore_total_bytes":12341234,"finished_queries":{"hz":0.0,"counter":0,"roughness":0.0}}],"locality":{"$map":"value"},"messages":[{"description":"abc","type":"x","name":{"$enum":["file_open_error","unable_to_write_cluster_file","process_error","io_error","io_timeout","platform_error","storage_server_lagging","(other FDB error messages)"]},"raw_log_message":"<stuff/>","time":12345.12312}],"address":"1.2.3.4:1234","command_line":"-r simulation","disk":{"free_bytes":3451233456234,"reads":{"hz":0.0,"counter":0,"sectors":0},"busy":0.0,"writes":{"hz":0.0,"counter":0,"sectors":0},"total_bytes":123412341234},"version":"3.0.0","excluded":false,"memory":{"available_bytes":0,"used_bytes":0,"limit_bytes":0},"machine_id":"0ccb4e0feddb5583010f6b77d9d10ece","uptime_seconds":1234.2345,"cpu":{"usage_cores":0.0},"network":{"megabits_sent":{"hz":0.0},"megabits_received":{"hz":0.0},"connections_closed":{"hz":0.0},"connection_errors":{"hz":0.0},"current_connections":0,"connections_established":{"hz":0.0}}}},"clients":{"count":1,"supported_versions":[{"count":1,"protocol_version":"fdb00a400050001","client_version":"3.0.0","source_version":"9430e1127b4991cbc5ab2b17f41cfffa5de07e9d","connected_clients":[{"log_group":"default","address":"127.0.0.1:9898"}]}]},"qos":{"limiting_version_lag_storage_server":0,"released_transactions_per_second":0,"transactions_per_second_limit":0,"limiting_queue_bytes_storage_server":0,"performance_limited_by":{"reason_server_id":"7f8d623d0cb9966e","description":"The database is not being saturated by the workload.","reason_id":0,"name":{"$enum":["workload","storage_server_write_queue_size","storage_server_write_bandwidth_mvcc","storage_server_readable_behind","log_server_mvcc_write_bandwidth","log_server_write_queue","storage_server_min_free_space","storage_server_min_free_space_ratio","log_server_min_free_space","log_server_min_free_space_ratio"]}},"worst_version_lag_storage_server":0,"worst_queue_bytes_log_server":460,"worst_queue_bytes_storage_server":0},"incompatible_connections":[],"database_locked":false,"generation":2,"data":{"least_operating_space_bytes_log_server":0,"average_partition_size_bytes":0,"state":{"healthy":true,"description":"","name":{"$enum":["initializing","missing_data","healing","healthy_repartitioning","healthy_removing_server","healthy_rebalancing","healthy"]},"min_replicas_remaining":0},"least_operating_space_ratio_storage_server":0.1,"max_machine_failures_without_losing_availability":0,"total_disk_used_bytes":0,"total_kv_size_bytes":0,"max_machine_failures_without_losing_data":0,"moving_data":{"in_queue_bytes":0,"total_written_bytes":0,"in_flight_bytes":0},"least_operating_space_bytes_storage_server":0,"partitions_count":2},"fault_tolerance":{"max_machine_failures_without_losing_availability":0,"max_machine_failures_without_losing_data":0},"messages":[{"reasons":[{"description":"Blah."}],"unreachable_processes":[{"address":"1.2.3.4:1234"}],"name":{"$enum":["unreachable_master_worker","unreadable_configuration","client_issues","unreachable_processes","immediate_priority_transaction_start_probe_timeout","batch_priority_transaction_start_probe_timeout","transaction_start_probe_timeout","read_probe_timeout","commit_probe_timeout","storage_servers_error","status_incomplete","layer_status_incomplete","database_availability_timeout"]},"issues":[{"name":{"$enum":["unable_to_write_cluster_file"]},"description":"Unable to update cluster file."}],"description":"abc"}],"database_available":true,"recovery_state":{"required_proxies":1,"name":{"$enum":["reading_coordinated_state","locking_coordinated_state","locking_old_transaction_servers","reading_transaction_system_state","configuration_missing","configuration_never_created","configuration_invalid","recruiting_transaction_servers","initializing_transaction_servers","recovery_transaction","writing_coordinated_state","fully_recovered"]},"missing_logs":"7f8d623d0cb9966e","required_resolvers":1,"required_logs":3,"description":"Recovery complete."},"workload":{"operations":{"writes":{"hz":0.0,"counter":0,"roughness":0.0},"reads":{"hz":0.0}},"bytes":{"written":{"hz":0.0,"counter":0,"roughness":0.0}},"transactions":{"started":{"hz":0.0,"counter":0,"roughness":0.0},"conflicted":{"hz":0.0,"counter":0,"roughness":0.0},"committed":{"hz":0.0,"counter":0,"roughness":0.0}}},"cluster_controller_timestamp":1415650089,"protocol_version":"fdb00a400050001","configuration":{"resolvers":1,"redundancy":{"factor":{"$enum":["single","double","triple","custom","two_datacenter","three_datacenter","three_data_hall","fast_recovery_double","fast_recovery_triple"]}},"logs":2,"storage_policy":"(zoneid^3x1)","storage_engine":{"$enum":["ssd","ssd-1","ssd-2","memory","custom"]},"coordinators_count":1,"excluded_servers":[{"address":"10.0.4.1"}],"proxies":5,"tlog_policy":"(zoneid^2x1)"},"latency_probe":{"immediate_priority_transaction_start_seconds":0.0,"transaction_start_seconds":0.0,"batch_priority_transaction_start_seconds":0.0,"read_seconds":7,"commit_seconds":0.02},"machines":{"$map":{"network":{"megabits_sent":{"hz":0.0},"megabits_received":{"hz":0.0},"tcp_segments_retransmitted":{"hz":0.0}},"locality":{"$map":"value"},"memory":{"free_bytes":0,"committed_bytes":0,"total_bytes":0},"contributing_workers":4,"datacenter_id":"6344abf1813eb05b","excluded":false,"address":"1.2.3.4","machine_id":"6344abf1813eb05b","cpu":{"logical_core_utilization":0.4}}},"old_logs":[{"log_write_anti_quorum":0,"log_fault_tolerance":2,"logs":[{"healthy":true,"id":"7f8d623d0cb9966e","address":"1.2.3.4:1234"}],"log_replication_factor":3}]},"client":{"coordinators":{"coordinators":[{"reachable":true,"address":"127.0.0.1:4701"}],"quorum_reachable":true},"cluster_file":{"path":"/etc/foundationdb/fdb.cluster","up_to_date":true},"messages":[{"name":{"$enum":["inconsistent_cluster_file","unreachable_cluster_controller","no_cluster_controller","status_incomplete_client","status_incomplete_coordinators","status_incomplete_error","status_incomplete_timeout","status_incomplete_cluster","quorum_not_reachable"]},"description":"The cluster file is not up to date."}],"timestamp":1415650089,"database_status":{"available":true,"healthy":true}}}
+    schema={"cluster":{"layers":{"_valid":true,"_error":"some error description"},"processes":{"$map":{"fault_domain":"0ccb4e0fdbdb5583010f6b77d9d10ece","class_source":{"$enum":["command_line","configure_auto","set_class"]},"class_type":{"$enum":["unset","storage","transaction","resolution","proxy","master","test"]},"roles":[{"query_queue_max":0,"data_version_lag":12341234,"input_bytes":{"hz":0.0,"counter":0,"roughness":0.0},"kvstore_used_bytes":12341234,"stored_bytes":12341234,"data_version":12341234,"kvstore_free_bytes":12341234,"durable_bytes":{"hz":0.0,"counter":0,"roughness":0.0},"id":"eb84471d68c12d1d26f692a50000003f","persistent_disk_used_bytes":12341234,"role":{"$enum":["master","proxy","log","storage","resolver","cluster_controller"]},"persistent_disk_total_bytes":12341234,"queue_disk_total_bytes":12341234,"persistent_disk_free_bytes":12341234,"queue_disk_used_bytes":12341234,"queue_disk_free_bytes":12341234,"kvstore_total_bytes":12341234,"finished_queries":{"hz":0.0,"counter":0,"roughness":0.0}}],"locality":{"$map":"value"},"messages":[{"description":"abc","type":"x","name":{"$enum":["file_open_error","incorrect_cluster_file_contents","process_error","io_error","io_timeout","platform_error","storage_server_lagging","(other FDB error messages)"]},"raw_log_message":"<stuff/>","time":12345.12312}],"address":"1.2.3.4:1234","command_line":"-r simulation","disk":{"free_bytes":3451233456234,"reads":{"hz":0.0,"counter":0,"sectors":0},"busy":0.0,"writes":{"hz":0.0,"counter":0,"sectors":0},"total_bytes":123412341234},"version":"3.0.0","excluded":false,"memory":{"available_bytes":0,"used_bytes":0,"limit_bytes":0},"machine_id":"0ccb4e0feddb5583010f6b77d9d10ece","uptime_seconds":1234.2345,"cpu":{"usage_cores":0.0},"network":{"megabits_sent":{"hz":0.0},"megabits_received":{"hz":0.0},"connections_closed":{"hz":0.0},"connection_errors":{"hz":0.0},"current_connections":0,"connections_established":{"hz":0.0}}}},"clients":{"count":1,"supported_versions":[{"count":1,"protocol_version":"fdb00a400050001","client_version":"3.0.0","source_version":"9430e1127b4991cbc5ab2b17f41cfffa5de07e9d","connected_clients":[{"log_group":"default","address":"127.0.0.1:9898"}]}]},"qos":{"limiting_version_lag_storage_server":0,"released_transactions_per_second":0,"transactions_per_second_limit":0,"limiting_queue_bytes_storage_server":0,"performance_limited_by":{"reason_server_id":"7f8d623d0cb9966e","description":"The database is not being saturated by the workload.","reason_id":0,"name":{"$enum":["workload","storage_server_write_queue_size","storage_server_write_bandwidth_mvcc","storage_server_readable_behind","log_server_mvcc_write_bandwidth","log_server_write_queue","storage_server_min_free_space","storage_server_min_free_space_ratio","log_server_min_free_space","log_server_min_free_space_ratio"]}},"worst_version_lag_storage_server":0,"worst_queue_bytes_log_server":460,"worst_queue_bytes_storage_server":0},"incompatible_connections":[],"database_locked":false,"generation":2,"data":{"least_operating_space_bytes_log_server":0,"average_partition_size_bytes":0,"state":{"healthy":true,"description":"","name":{"$enum":["initializing","missing_data","healing","healthy_repartitioning","healthy_removing_server","healthy_rebalancing","healthy"]},"min_replicas_remaining":0},"least_operating_space_ratio_storage_server":0.1,"max_machine_failures_without_losing_availability":0,"total_disk_used_bytes":0,"total_kv_size_bytes":0,"max_machine_failures_without_losing_data":0,"moving_data":{"in_queue_bytes":0,"total_written_bytes":0,"in_flight_bytes":0},"least_operating_space_bytes_storage_server":0,"partitions_count":2},"fault_tolerance":{"max_machine_failures_without_losing_availability":0,"max_machine_failures_without_losing_data":0},"messages":[{"reasons":[{"description":"Blah."}],"unreachable_processes":[{"address":"1.2.3.4:1234"}],"name":{"$enum":["unreachable_master_worker","unreadable_configuration","client_issues","unreachable_processes","immediate_priority_transaction_start_probe_timeout","batch_priority_transaction_start_probe_timeout","transaction_start_probe_timeout","read_probe_timeout","commit_probe_timeout","storage_servers_error","status_incomplete","layer_status_incomplete","database_availability_timeout"]},"issues":[{"name":{"$enum":["incorrect_cluster_file_contents"]},"description":"Cluster file contents do not match current cluster connection string. Verify cluster file is writable and has not been overwritten externally."}],"description":"abc"}],"database_available":true,"recovery_state":{"required_proxies":1,"name":{"$enum":["reading_coordinated_state","locking_coordinated_state","locking_old_transaction_servers","reading_transaction_system_state","configuration_missing","configuration_never_created","configuration_invalid","recruiting_transaction_servers","initializing_transaction_servers","recovery_transaction","writing_coordinated_state","fully_recovered"]},"missing_logs":"7f8d623d0cb9966e","required_resolvers":1,"required_logs":3,"description":"Recovery complete."},"workload":{"operations":{"writes":{"hz":0.0,"counter":0,"roughness":0.0},"reads":{"hz":0.0}},"bytes":{"written":{"hz":0.0,"counter":0,"roughness":0.0}},"transactions":{"started":{"hz":0.0,"counter":0,"roughness":0.0},"conflicted":{"hz":0.0,"counter":0,"roughness":0.0},"committed":{"hz":0.0,"counter":0,"roughness":0.0}}},"cluster_controller_timestamp":1415650089,"protocol_version":"fdb00a400050001","configuration":{"resolvers":1,"redundancy":{"factor":{"$enum":["single","double","triple","custom","two_datacenter","three_datacenter","three_data_hall","fast_recovery_double","fast_recovery_triple"]}},"logs":2,"storage_policy":"(zoneid^3x1)","storage_engine":{"$enum":["ssd","ssd-1","ssd-2","memory","custom"]},"coordinators_count":1,"excluded_servers":[{"address":"10.0.4.1"}],"proxies":5,"tlog_policy":"(zoneid^2x1)"},"latency_probe":{"immediate_priority_transaction_start_seconds":0.0,"transaction_start_seconds":0.0,"batch_priority_transaction_start_seconds":0.0,"read_seconds":7,"commit_seconds":0.02},"machines":{"$map":{"network":{"megabits_sent":{"hz":0.0},"megabits_received":{"hz":0.0},"tcp_segments_retransmitted":{"hz":0.0}},"locality":{"$map":"value"},"memory":{"free_bytes":0,"committed_bytes":0,"total_bytes":0},"contributing_workers":4,"datacenter_id":"6344abf1813eb05b","excluded":false,"address":"1.2.3.4","machine_id":"6344abf1813eb05b","cpu":{"logical_core_utilization":0.4}}},"old_logs":[{"log_write_anti_quorum":0,"log_fault_tolerance":2,"logs":[{"healthy":true,"id":"7f8d623d0cb9966e","address":"1.2.3.4:1234"}],"log_replication_factor":3}]},"client":{"coordinators":{"coordinators":[{"reachable":true,"address":"127.0.0.1:4701"}],"quorum_reachable":true},"cluster_file":{"path":"/etc/foundationdb/fdb.cluster","up_to_date":true},"messages":[{"name":{"$enum":["inconsistent_cluster_file","unreachable_cluster_controller","no_cluster_controller","status_incomplete_client","status_incomplete_coordinators","status_incomplete_error","status_incomplete_timeout","status_incomplete_cluster","quorum_not_reachable"]},"description":"The cluster file is not up to date."}],"timestamp":1415650089,"database_status":{"available":true,"healthy":true}}}
 
     testName=RandomClogging
     testDuration=30.0
diff --git a/tests/rare/LargeApiCorrectnessStatus.txt b/tests/rare/LargeApiCorrectnessStatus.txt
index ec44d4b2c8..fbb67df02d 100644
--- a/tests/rare/LargeApiCorrectnessStatus.txt
+++ b/tests/rare/LargeApiCorrectnessStatus.txt
@@ -24,4 +24,4 @@ testTitle=ApiCorrectnessTest
 
     testName=Status
     testDuration=30.0
-	schema={"cluster":{"layers":{"_valid":true,"_error":"some error description"},"processes":{"$map":{"fault_domain":"0ccb4e0fdbdb5583010f6b77d9d10ece","class_source":{"$enum":["command_line","configure_auto","set_class"]},"class_type":{"$enum":["unset","storage","transaction","resolution","proxy","master","test"]},"roles":[{"query_queue_max":0,"data_version_lag":12341234,"input_bytes":{"hz":0.0,"counter":0,"roughness":0.0},"kvstore_used_bytes":12341234,"stored_bytes":12341234,"data_version":12341234,"kvstore_free_bytes":12341234,"durable_bytes":{"hz":0.0,"counter":0,"roughness":0.0},"id":"eb84471d68c12d1d26f692a50000003f","persistent_disk_used_bytes":12341234,"role":{"$enum":["master","proxy","log","storage","resolver","cluster_controller"]},"persistent_disk_total_bytes":12341234,"queue_disk_total_bytes":12341234,"persistent_disk_free_bytes":12341234,"queue_disk_used_bytes":12341234,"queue_disk_free_bytes":12341234,"kvstore_total_bytes":12341234,"finished_queries":{"hz":0.0,"counter":0,"roughness":0.0}}],"locality":{"$map":"value"},"messages":[{"description":"abc","type":"x","name":{"$enum":["file_open_error","unable_to_write_cluster_file","process_error","io_error","io_timeout","platform_error","storage_server_lagging","(other FDB error messages)"]},"raw_log_message":"<stuff/>","time":12345.12312}],"address":"1.2.3.4:1234","command_line":"-r simulation","disk":{"free_bytes":3451233456234,"reads":{"hz":0.0,"counter":0,"sectors":0},"busy":0.0,"writes":{"hz":0.0,"counter":0,"sectors":0},"total_bytes":123412341234},"version":"3.0.0","excluded":false,"memory":{"available_bytes":0,"used_bytes":0,"limit_bytes":0},"machine_id":"0ccb4e0feddb5583010f6b77d9d10ece","uptime_seconds":1234.2345,"cpu":{"usage_cores":0.0},"network":{"megabits_sent":{"hz":0.0},"megabits_received":{"hz":0.0},"connections_closed":{"hz":0.0},"connection_errors":{"hz":0.0},"current_connections":0,"connections_established":{"hz":0.0}}}},"clients":{"count":1,"supported_versions":[{"count":1,"protocol_version":"fdb00a400050001","client_version":"3.0.0","source_version":"9430e1127b4991cbc5ab2b17f41cfffa5de07e9d","connected_clients":[{"log_group":"default","address":"127.0.0.1:9898"}]}]},"qos":{"limiting_version_lag_storage_server":0,"released_transactions_per_second":0,"transactions_per_second_limit":0,"limiting_queue_bytes_storage_server":0,"performance_limited_by":{"reason_server_id":"7f8d623d0cb9966e","description":"The database is not being saturated by the workload.","reason_id":0,"name":{"$enum":["workload","storage_server_write_queue_size","storage_server_write_bandwidth_mvcc","storage_server_readable_behind","log_server_mvcc_write_bandwidth","log_server_write_queue","storage_server_min_free_space","storage_server_min_free_space_ratio","log_server_min_free_space","log_server_min_free_space_ratio"]}},"worst_version_lag_storage_server":0,"worst_queue_bytes_log_server":460,"worst_queue_bytes_storage_server":0},"incompatible_connections":[],"database_locked":false,"generation":2,"data":{"least_operating_space_bytes_log_server":0,"average_partition_size_bytes":0,"state":{"healthy":true,"description":"","name":{"$enum":["initializing","missing_data","healing","healthy_repartitioning","healthy_removing_server","healthy_rebalancing","healthy"]},"min_replicas_remaining":0},"least_operating_space_ratio_storage_server":0.1,"max_machine_failures_without_losing_availability":0,"total_disk_used_bytes":0,"total_kv_size_bytes":0,"max_machine_failures_without_losing_data":0,"moving_data":{"in_queue_bytes":0,"total_written_bytes":0,"in_flight_bytes":0},"least_operating_space_bytes_storage_server":0,"partitions_count":2},"fault_tolerance":{"max_machine_failures_without_losing_availability":0,"max_machine_failures_without_losing_data":0},"messages":[{"reasons":[{"description":"Blah."}],"unreachable_processes":[{"address":"1.2.3.4:1234"}],"name":{"$enum":["unreachable_master_worker","unreadable_configuration","client_issues","unreachable_processes","immediate_priority_transaction_start_probe_timeout","batch_priority_transaction_start_probe_timeout","transaction_start_probe_timeout","read_probe_timeout","commit_probe_timeout","storage_servers_error","status_incomplete","layer_status_incomplete","database_availability_timeout"]},"issues":[{"name":{"$enum":["unable_to_write_cluster_file"]},"description":"Unable to update cluster file."}],"description":"abc"}],"database_available":true,"recovery_state":{"required_proxies":1,"name":{"$enum":["reading_coordinated_state","locking_coordinated_state","locking_old_transaction_servers","reading_transaction_system_state","configuration_missing","configuration_never_created","configuration_invalid","recruiting_transaction_servers","initializing_transaction_servers","recovery_transaction","writing_coordinated_state","fully_recovered"]},"missing_logs":"7f8d623d0cb9966e","required_resolvers":1,"required_logs":3,"description":"Recovery complete."},"workload":{"operations":{"writes":{"hz":0.0,"counter":0,"roughness":0.0},"reads":{"hz":0.0}},"bytes":{"written":{"hz":0.0,"counter":0,"roughness":0.0}},"transactions":{"started":{"hz":0.0,"counter":0,"roughness":0.0},"conflicted":{"hz":0.0,"counter":0,"roughness":0.0},"committed":{"hz":0.0,"counter":0,"roughness":0.0}}},"cluster_controller_timestamp":1415650089,"protocol_version":"fdb00a400050001","configuration":{"resolvers":1,"redundancy":{"factor":{"$enum":["single","double","triple","custom","two_datacenter","three_datacenter","three_data_hall","fast_recovery_double","fast_recovery_triple"]}},"logs":2,"storage_policy":"(zoneid^3x1)","storage_engine":{"$enum":["ssd","ssd-1","ssd-2","memory","custom"]},"coordinators_count":1,"excluded_servers":[{"address":"10.0.4.1"}],"proxies":5,"tlog_policy":"(zoneid^2x1)"},"latency_probe":{"immediate_priority_transaction_start_seconds":0.0,"transaction_start_seconds":0.0,"batch_priority_transaction_start_seconds":0.0,"read_seconds":7,"commit_seconds":0.02},"machines":{"$map":{"network":{"megabits_sent":{"hz":0.0},"megabits_received":{"hz":0.0},"tcp_segments_retransmitted":{"hz":0.0}},"locality":{"$map":"value"},"memory":{"free_bytes":0,"committed_bytes":0,"total_bytes":0},"contributing_workers":4,"datacenter_id":"6344abf1813eb05b","excluded":false,"address":"1.2.3.4","machine_id":"6344abf1813eb05b","cpu":{"logical_core_utilization":0.4}}},"old_logs":[{"log_write_anti_quorum":0,"log_fault_tolerance":2,"logs":[{"healthy":true,"id":"7f8d623d0cb9966e","address":"1.2.3.4:1234"}],"log_replication_factor":3}]},"client":{"coordinators":{"coordinators":[{"reachable":true,"address":"127.0.0.1:4701"}],"quorum_reachable":true},"cluster_file":{"path":"/etc/foundationdb/fdb.cluster","up_to_date":true},"messages":[{"name":{"$enum":["inconsistent_cluster_file","unreachable_cluster_controller","no_cluster_controller","status_incomplete_client","status_incomplete_coordinators","status_incomplete_error","status_incomplete_timeout","status_incomplete_cluster","quorum_not_reachable"]},"description":"The cluster file is not up to date."}],"timestamp":1415650089,"database_status":{"available":true,"healthy":true}}}
+	schema={"cluster":{"layers":{"_valid":true,"_error":"some error description"},"processes":{"$map":{"fault_domain":"0ccb4e0fdbdb5583010f6b77d9d10ece","class_source":{"$enum":["command_line","configure_auto","set_class"]},"class_type":{"$enum":["unset","storage","transaction","resolution","proxy","master","test"]},"roles":[{"query_queue_max":0,"data_version_lag":12341234,"input_bytes":{"hz":0.0,"counter":0,"roughness":0.0},"kvstore_used_bytes":12341234,"stored_bytes":12341234,"data_version":12341234,"kvstore_free_bytes":12341234,"durable_bytes":{"hz":0.0,"counter":0,"roughness":0.0},"id":"eb84471d68c12d1d26f692a50000003f","persistent_disk_used_bytes":12341234,"role":{"$enum":["master","proxy","log","storage","resolver","cluster_controller"]},"persistent_disk_total_bytes":12341234,"queue_disk_total_bytes":12341234,"persistent_disk_free_bytes":12341234,"queue_disk_used_bytes":12341234,"queue_disk_free_bytes":12341234,"kvstore_total_bytes":12341234,"finished_queries":{"hz":0.0,"counter":0,"roughness":0.0}}],"locality":{"$map":"value"},"messages":[{"description":"abc","type":"x","name":{"$enum":["file_open_error","incorrect_cluster_file_contents","process_error","io_error","io_timeout","platform_error","storage_server_lagging","(other FDB error messages)"]},"raw_log_message":"<stuff/>","time":12345.12312}],"address":"1.2.3.4:1234","command_line":"-r simulation","disk":{"free_bytes":3451233456234,"reads":{"hz":0.0,"counter":0,"sectors":0},"busy":0.0,"writes":{"hz":0.0,"counter":0,"sectors":0},"total_bytes":123412341234},"version":"3.0.0","excluded":false,"memory":{"available_bytes":0,"used_bytes":0,"limit_bytes":0},"machine_id":"0ccb4e0feddb5583010f6b77d9d10ece","uptime_seconds":1234.2345,"cpu":{"usage_cores":0.0},"network":{"megabits_sent":{"hz":0.0},"megabits_received":{"hz":0.0},"connections_closed":{"hz":0.0},"connection_errors":{"hz":0.0},"current_connections":0,"connections_established":{"hz":0.0}}}},"clients":{"count":1,"supported_versions":[{"count":1,"protocol_version":"fdb00a400050001","client_version":"3.0.0","source_version":"9430e1127b4991cbc5ab2b17f41cfffa5de07e9d","connected_clients":[{"log_group":"default","address":"127.0.0.1:9898"}]}]},"qos":{"limiting_version_lag_storage_server":0,"released_transactions_per_second":0,"transactions_per_second_limit":0,"limiting_queue_bytes_storage_server":0,"performance_limited_by":{"reason_server_id":"7f8d623d0cb9966e","description":"The database is not being saturated by the workload.","reason_id":0,"name":{"$enum":["workload","storage_server_write_queue_size","storage_server_write_bandwidth_mvcc","storage_server_readable_behind","log_server_mvcc_write_bandwidth","log_server_write_queue","storage_server_min_free_space","storage_server_min_free_space_ratio","log_server_min_free_space","log_server_min_free_space_ratio"]}},"worst_version_lag_storage_server":0,"worst_queue_bytes_log_server":460,"worst_queue_bytes_storage_server":0},"incompatible_connections":[],"database_locked":false,"generation":2,"data":{"least_operating_space_bytes_log_server":0,"average_partition_size_bytes":0,"state":{"healthy":true,"description":"","name":{"$enum":["initializing","missing_data","healing","healthy_repartitioning","healthy_removing_server","healthy_rebalancing","healthy"]},"min_replicas_remaining":0},"least_operating_space_ratio_storage_server":0.1,"max_machine_failures_without_losing_availability":0,"total_disk_used_bytes":0,"total_kv_size_bytes":0,"max_machine_failures_without_losing_data":0,"moving_data":{"in_queue_bytes":0,"total_written_bytes":0,"in_flight_bytes":0},"least_operating_space_bytes_storage_server":0,"partitions_count":2},"fault_tolerance":{"max_machine_failures_without_losing_availability":0,"max_machine_failures_without_losing_data":0},"messages":[{"reasons":[{"description":"Blah."}],"unreachable_processes":[{"address":"1.2.3.4:1234"}],"name":{"$enum":["unreachable_master_worker","unreadable_configuration","client_issues","unreachable_processes","immediate_priority_transaction_start_probe_timeout","batch_priority_transaction_start_probe_timeout","transaction_start_probe_timeout","read_probe_timeout","commit_probe_timeout","storage_servers_error","status_incomplete","layer_status_incomplete","database_availability_timeout"]},"issues":[{"name":{"$enum":["incorrect_cluster_file_contents"]},"description":"Cluster file contents do not match current cluster connection string. Verify cluster file is writable and has not been overwritten externally."}],"description":"abc"}],"database_available":true,"recovery_state":{"required_proxies":1,"name":{"$enum":["reading_coordinated_state","locking_coordinated_state","locking_old_transaction_servers","reading_transaction_system_state","configuration_missing","configuration_never_created","configuration_invalid","recruiting_transaction_servers","initializing_transaction_servers","recovery_transaction","writing_coordinated_state","fully_recovered"]},"missing_logs":"7f8d623d0cb9966e","required_resolvers":1,"required_logs":3,"description":"Recovery complete."},"workload":{"operations":{"writes":{"hz":0.0,"counter":0,"roughness":0.0},"reads":{"hz":0.0}},"bytes":{"written":{"hz":0.0,"counter":0,"roughness":0.0}},"transactions":{"started":{"hz":0.0,"counter":0,"roughness":0.0},"conflicted":{"hz":0.0,"counter":0,"roughness":0.0},"committed":{"hz":0.0,"counter":0,"roughness":0.0}}},"cluster_controller_timestamp":1415650089,"protocol_version":"fdb00a400050001","configuration":{"resolvers":1,"redundancy":{"factor":{"$enum":["single","double","triple","custom","two_datacenter","three_datacenter","three_data_hall","fast_recovery_double","fast_recovery_triple"]}},"logs":2,"storage_policy":"(zoneid^3x1)","storage_engine":{"$enum":["ssd","ssd-1","ssd-2","memory","custom"]},"coordinators_count":1,"excluded_servers":[{"address":"10.0.4.1"}],"proxies":5,"tlog_policy":"(zoneid^2x1)"},"latency_probe":{"immediate_priority_transaction_start_seconds":0.0,"transaction_start_seconds":0.0,"batch_priority_transaction_start_seconds":0.0,"read_seconds":7,"commit_seconds":0.02},"machines":{"$map":{"network":{"megabits_sent":{"hz":0.0},"megabits_received":{"hz":0.0},"tcp_segments_retransmitted":{"hz":0.0}},"locality":{"$map":"value"},"memory":{"free_bytes":0,"committed_bytes":0,"total_bytes":0},"contributing_workers":4,"datacenter_id":"6344abf1813eb05b","excluded":false,"address":"1.2.3.4","machine_id":"6344abf1813eb05b","cpu":{"logical_core_utilization":0.4}}},"old_logs":[{"log_write_anti_quorum":0,"log_fault_tolerance":2,"logs":[{"healthy":true,"id":"7f8d623d0cb9966e","address":"1.2.3.4:1234"}],"log_replication_factor":3}]},"client":{"coordinators":{"coordinators":[{"reachable":true,"address":"127.0.0.1:4701"}],"quorum_reachable":true},"cluster_file":{"path":"/etc/foundationdb/fdb.cluster","up_to_date":true},"messages":[{"name":{"$enum":["inconsistent_cluster_file","unreachable_cluster_controller","no_cluster_controller","status_incomplete_client","status_incomplete_coordinators","status_incomplete_error","status_incomplete_timeout","status_incomplete_cluster","quorum_not_reachable"]},"description":"The cluster file is not up to date."}],"timestamp":1415650089,"database_status":{"available":true,"healthy":true}}}
diff --git a/tests/slow/DDBalanceAndRemoveStatus.txt b/tests/slow/DDBalanceAndRemoveStatus.txt
index 3592d48cfa..c256d4f917 100644
--- a/tests/slow/DDBalanceAndRemoveStatus.txt
+++ b/tests/slow/DDBalanceAndRemoveStatus.txt
@@ -43,4 +43,4 @@ testTitle=DDBalance_test
 
     testName=Status
     testDuration=30.0
-    schema={"cluster":{"layers":{"_valid":true,"_error":"some error description"},"processes":{"$map":{"fault_domain":"0ccb4e0fdbdb5583010f6b77d9d10ece","class_source":{"$enum":["command_line","configure_auto","set_class"]},"class_type":{"$enum":["unset","storage","transaction","resolution","proxy","master","test"]},"roles":[{"query_queue_max":0,"data_version_lag":12341234,"input_bytes":{"hz":0.0,"counter":0,"roughness":0.0},"kvstore_used_bytes":12341234,"stored_bytes":12341234,"data_version":12341234,"kvstore_free_bytes":12341234,"durable_bytes":{"hz":0.0,"counter":0,"roughness":0.0},"id":"eb84471d68c12d1d26f692a50000003f","persistent_disk_used_bytes":12341234,"role":{"$enum":["master","proxy","log","storage","resolver","cluster_controller"]},"persistent_disk_total_bytes":12341234,"queue_disk_total_bytes":12341234,"persistent_disk_free_bytes":12341234,"queue_disk_used_bytes":12341234,"queue_disk_free_bytes":12341234,"kvstore_total_bytes":12341234,"finished_queries":{"hz":0.0,"counter":0,"roughness":0.0}}],"locality":{"$map":"value"},"messages":[{"description":"abc","type":"x","name":{"$enum":["file_open_error","unable_to_write_cluster_file","process_error","io_error","io_timeout","platform_error","storage_server_lagging","(other FDB error messages)"]},"raw_log_message":"<stuff/>","time":12345.12312}],"address":"1.2.3.4:1234","command_line":"-r simulation","disk":{"free_bytes":3451233456234,"reads":{"hz":0.0,"counter":0,"sectors":0},"busy":0.0,"writes":{"hz":0.0,"counter":0,"sectors":0},"total_bytes":123412341234},"version":"3.0.0","excluded":false,"memory":{"available_bytes":0,"used_bytes":0,"limit_bytes":0},"machine_id":"0ccb4e0feddb5583010f6b77d9d10ece","uptime_seconds":1234.2345,"cpu":{"usage_cores":0.0},"network":{"megabits_sent":{"hz":0.0},"megabits_received":{"hz":0.0},"connections_closed":{"hz":0.0},"connection_errors":{"hz":0.0},"current_connections":0,"connections_established":{"hz":0.0}}}},"clients":{"count":1,"supported_versions":[{"count":1,"protocol_version":"fdb00a400050001","client_version":"3.0.0","source_version":"9430e1127b4991cbc5ab2b17f41cfffa5de07e9d","connected_clients":[{"log_group":"default","address":"127.0.0.1:9898"}]}]},"qos":{"limiting_version_lag_storage_server":0,"released_transactions_per_second":0,"transactions_per_second_limit":0,"limiting_queue_bytes_storage_server":0,"performance_limited_by":{"reason_server_id":"7f8d623d0cb9966e","description":"The database is not being saturated by the workload.","reason_id":0,"name":{"$enum":["workload","storage_server_write_queue_size","storage_server_write_bandwidth_mvcc","storage_server_readable_behind","log_server_mvcc_write_bandwidth","log_server_write_queue","storage_server_min_free_space","storage_server_min_free_space_ratio","log_server_min_free_space","log_server_min_free_space_ratio"]}},"worst_version_lag_storage_server":0,"worst_queue_bytes_log_server":460,"worst_queue_bytes_storage_server":0},"incompatible_connections":[],"database_locked":false,"generation":2,"data":{"least_operating_space_bytes_log_server":0,"average_partition_size_bytes":0,"state":{"healthy":true,"description":"","name":{"$enum":["initializing","missing_data","healing","healthy_repartitioning","healthy_removing_server","healthy_rebalancing","healthy"]},"min_replicas_remaining":0},"least_operating_space_ratio_storage_server":0.1,"max_machine_failures_without_losing_availability":0,"total_disk_used_bytes":0,"total_kv_size_bytes":0,"max_machine_failures_without_losing_data":0,"moving_data":{"in_queue_bytes":0,"total_written_bytes":0,"in_flight_bytes":0},"least_operating_space_bytes_storage_server":0,"partitions_count":2},"fault_tolerance":{"max_machine_failures_without_losing_availability":0,"max_machine_failures_without_losing_data":0},"messages":[{"reasons":[{"description":"Blah."}],"unreachable_processes":[{"address":"1.2.3.4:1234"}],"name":{"$enum":["unreachable_master_worker","unreadable_configuration","client_issues","unreachable_processes","immediate_priority_transaction_start_probe_timeout","batch_priority_transaction_start_probe_timeout","transaction_start_probe_timeout","read_probe_timeout","commit_probe_timeout","storage_servers_error","status_incomplete","layer_status_incomplete","database_availability_timeout"]},"issues":[{"name":{"$enum":["unable_to_write_cluster_file"]},"description":"Unable to update cluster file."}],"description":"abc"}],"database_available":true,"recovery_state":{"required_proxies":1,"name":{"$enum":["reading_coordinated_state","locking_coordinated_state","locking_old_transaction_servers","reading_transaction_system_state","configuration_missing","configuration_never_created","configuration_invalid","recruiting_transaction_servers","initializing_transaction_servers","recovery_transaction","writing_coordinated_state","fully_recovered"]},"missing_logs":"7f8d623d0cb9966e","required_resolvers":1,"required_logs":3,"description":"Recovery complete."},"workload":{"operations":{"writes":{"hz":0.0,"counter":0,"roughness":0.0},"reads":{"hz":0.0}},"bytes":{"written":{"hz":0.0,"counter":0,"roughness":0.0}},"transactions":{"started":{"hz":0.0,"counter":0,"roughness":0.0},"conflicted":{"hz":0.0,"counter":0,"roughness":0.0},"committed":{"hz":0.0,"counter":0,"roughness":0.0}}},"cluster_controller_timestamp":1415650089,"protocol_version":"fdb00a400050001","configuration":{"resolvers":1,"redundancy":{"factor":{"$enum":["single","double","triple","custom","two_datacenter","three_datacenter","three_data_hall","fast_recovery_double","fast_recovery_triple"]}},"logs":2,"storage_policy":"(zoneid^3x1)","storage_engine":{"$enum":["ssd","ssd-1","ssd-2","memory","custom"]},"coordinators_count":1,"excluded_servers":[{"address":"10.0.4.1"}],"proxies":5,"tlog_policy":"(zoneid^2x1)"},"latency_probe":{"immediate_priority_transaction_start_seconds":0.0,"transaction_start_seconds":0.0,"batch_priority_transaction_start_seconds":0.0,"read_seconds":7,"commit_seconds":0.02},"machines":{"$map":{"network":{"megabits_sent":{"hz":0.0},"megabits_received":{"hz":0.0},"tcp_segments_retransmitted":{"hz":0.0}},"locality":{"$map":"value"},"memory":{"free_bytes":0,"committed_bytes":0,"total_bytes":0},"contributing_workers":4,"datacenter_id":"6344abf1813eb05b","excluded":false,"address":"1.2.3.4","machine_id":"6344abf1813eb05b","cpu":{"logical_core_utilization":0.4}}},"old_logs":[{"log_write_anti_quorum":0,"log_fault_tolerance":2,"logs":[{"healthy":true,"id":"7f8d623d0cb9966e","address":"1.2.3.4:1234"}],"log_replication_factor":3}]},"client":{"coordinators":{"coordinators":[{"reachable":true,"address":"127.0.0.1:4701"}],"quorum_reachable":true},"cluster_file":{"path":"/etc/foundationdb/fdb.cluster","up_to_date":true},"messages":[{"name":{"$enum":["inconsistent_cluster_file","unreachable_cluster_controller","no_cluster_controller","status_incomplete_client","status_incomplete_coordinators","status_incomplete_error","status_incomplete_timeout","status_incomplete_cluster","quorum_not_reachable"]},"description":"The cluster file is not up to date."}],"timestamp":1415650089,"database_status":{"available":true,"healthy":true}}}
+    schema={"cluster":{"layers":{"_valid":true,"_error":"some error description"},"processes":{"$map":{"fault_domain":"0ccb4e0fdbdb5583010f6b77d9d10ece","class_source":{"$enum":["command_line","configure_auto","set_class"]},"class_type":{"$enum":["unset","storage","transaction","resolution","proxy","master","test"]},"roles":[{"query_queue_max":0,"data_version_lag":12341234,"input_bytes":{"hz":0.0,"counter":0,"roughness":0.0},"kvstore_used_bytes":12341234,"stored_bytes":12341234,"data_version":12341234,"kvstore_free_bytes":12341234,"durable_bytes":{"hz":0.0,"counter":0,"roughness":0.0},"id":"eb84471d68c12d1d26f692a50000003f","persistent_disk_used_bytes":12341234,"role":{"$enum":["master","proxy","log","storage","resolver","cluster_controller"]},"persistent_disk_total_bytes":12341234,"queue_disk_total_bytes":12341234,"persistent_disk_free_bytes":12341234,"queue_disk_used_bytes":12341234,"queue_disk_free_bytes":12341234,"kvstore_total_bytes":12341234,"finished_queries":{"hz":0.0,"counter":0,"roughness":0.0}}],"locality":{"$map":"value"},"messages":[{"description":"abc","type":"x","name":{"$enum":["file_open_error","incorrect_cluster_file_contents","process_error","io_error","io_timeout","platform_error","storage_server_lagging","(other FDB error messages)"]},"raw_log_message":"<stuff/>","time":12345.12312}],"address":"1.2.3.4:1234","command_line":"-r simulation","disk":{"free_bytes":3451233456234,"reads":{"hz":0.0,"counter":0,"sectors":0},"busy":0.0,"writes":{"hz":0.0,"counter":0,"sectors":0},"total_bytes":123412341234},"version":"3.0.0","excluded":false,"memory":{"available_bytes":0,"used_bytes":0,"limit_bytes":0},"machine_id":"0ccb4e0feddb5583010f6b77d9d10ece","uptime_seconds":1234.2345,"cpu":{"usage_cores":0.0},"network":{"megabits_sent":{"hz":0.0},"megabits_received":{"hz":0.0},"connections_closed":{"hz":0.0},"connection_errors":{"hz":0.0},"current_connections":0,"connections_established":{"hz":0.0}}}},"clients":{"count":1,"supported_versions":[{"count":1,"protocol_version":"fdb00a400050001","client_version":"3.0.0","source_version":"9430e1127b4991cbc5ab2b17f41cfffa5de07e9d","connected_clients":[{"log_group":"default","address":"127.0.0.1:9898"}]}]},"qos":{"limiting_version_lag_storage_server":0,"released_transactions_per_second":0,"transactions_per_second_limit":0,"limiting_queue_bytes_storage_server":0,"performance_limited_by":{"reason_server_id":"7f8d623d0cb9966e","description":"The database is not being saturated by the workload.","reason_id":0,"name":{"$enum":["workload","storage_server_write_queue_size","storage_server_write_bandwidth_mvcc","storage_server_readable_behind","log_server_mvcc_write_bandwidth","log_server_write_queue","storage_server_min_free_space","storage_server_min_free_space_ratio","log_server_min_free_space","log_server_min_free_space_ratio"]}},"worst_version_lag_storage_server":0,"worst_queue_bytes_log_server":460,"worst_queue_bytes_storage_server":0},"incompatible_connections":[],"database_locked":false,"generation":2,"data":{"least_operating_space_bytes_log_server":0,"average_partition_size_bytes":0,"state":{"healthy":true,"description":"","name":{"$enum":["initializing","missing_data","healing","healthy_repartitioning","healthy_removing_server","healthy_rebalancing","healthy"]},"min_replicas_remaining":0},"least_operating_space_ratio_storage_server":0.1,"max_machine_failures_without_losing_availability":0,"total_disk_used_bytes":0,"total_kv_size_bytes":0,"max_machine_failures_without_losing_data":0,"moving_data":{"in_queue_bytes":0,"total_written_bytes":0,"in_flight_bytes":0},"least_operating_space_bytes_storage_server":0,"partitions_count":2},"fault_tolerance":{"max_machine_failures_without_losing_availability":0,"max_machine_failures_without_losing_data":0},"messages":[{"reasons":[{"description":"Blah."}],"unreachable_processes":[{"address":"1.2.3.4:1234"}],"name":{"$enum":["unreachable_master_worker","unreadable_configuration","client_issues","unreachable_processes","immediate_priority_transaction_start_probe_timeout","batch_priority_transaction_start_probe_timeout","transaction_start_probe_timeout","read_probe_timeout","commit_probe_timeout","storage_servers_error","status_incomplete","layer_status_incomplete","database_availability_timeout"]},"issues":[{"name":{"$enum":["incorrect_cluster_file_contents"]},"description":"Cluster file contents do not match current cluster connection string. Verify cluster file is writable and has not been overwritten externally."}],"description":"abc"}],"database_available":true,"recovery_state":{"required_proxies":1,"name":{"$enum":["reading_coordinated_state","locking_coordinated_state","locking_old_transaction_servers","reading_transaction_system_state","configuration_missing","configuration_never_created","configuration_invalid","recruiting_transaction_servers","initializing_transaction_servers","recovery_transaction","writing_coordinated_state","fully_recovered"]},"missing_logs":"7f8d623d0cb9966e","required_resolvers":1,"required_logs":3,"description":"Recovery complete."},"workload":{"operations":{"writes":{"hz":0.0,"counter":0,"roughness":0.0},"reads":{"hz":0.0}},"bytes":{"written":{"hz":0.0,"counter":0,"roughness":0.0}},"transactions":{"started":{"hz":0.0,"counter":0,"roughness":0.0},"conflicted":{"hz":0.0,"counter":0,"roughness":0.0},"committed":{"hz":0.0,"counter":0,"roughness":0.0}}},"cluster_controller_timestamp":1415650089,"protocol_version":"fdb00a400050001","configuration":{"resolvers":1,"redundancy":{"factor":{"$enum":["single","double","triple","custom","two_datacenter","three_datacenter","three_data_hall","fast_recovery_double","fast_recovery_triple"]}},"logs":2,"storage_policy":"(zoneid^3x1)","storage_engine":{"$enum":["ssd","ssd-1","ssd-2","memory","custom"]},"coordinators_count":1,"excluded_servers":[{"address":"10.0.4.1"}],"proxies":5,"tlog_policy":"(zoneid^2x1)"},"latency_probe":{"immediate_priority_transaction_start_seconds":0.0,"transaction_start_seconds":0.0,"batch_priority_transaction_start_seconds":0.0,"read_seconds":7,"commit_seconds":0.02},"machines":{"$map":{"network":{"megabits_sent":{"hz":0.0},"megabits_received":{"hz":0.0},"tcp_segments_retransmitted":{"hz":0.0}},"locality":{"$map":"value"},"memory":{"free_bytes":0,"committed_bytes":0,"total_bytes":0},"contributing_workers":4,"datacenter_id":"6344abf1813eb05b","excluded":false,"address":"1.2.3.4","machine_id":"6344abf1813eb05b","cpu":{"logical_core_utilization":0.4}}},"old_logs":[{"log_write_anti_quorum":0,"log_fault_tolerance":2,"logs":[{"healthy":true,"id":"7f8d623d0cb9966e","address":"1.2.3.4:1234"}],"log_replication_factor":3}]},"client":{"coordinators":{"coordinators":[{"reachable":true,"address":"127.0.0.1:4701"}],"quorum_reachable":true},"cluster_file":{"path":"/etc/foundationdb/fdb.cluster","up_to_date":true},"messages":[{"name":{"$enum":["inconsistent_cluster_file","unreachable_cluster_controller","no_cluster_controller","status_incomplete_client","status_incomplete_coordinators","status_incomplete_error","status_incomplete_timeout","status_incomplete_cluster","quorum_not_reachable"]},"description":"The cluster file is not up to date."}],"timestamp":1415650089,"database_status":{"available":true,"healthy":true}}}
diff --git a/tests/status/separate_cannot_write_cluster_file.json b/tests/status/separate_cannot_write_cluster_file.json
index e6f49d7c00..63eee515a7 100644
--- a/tests/status/separate_cannot_write_cluster_file.json
+++ b/tests/status/separate_cannot_write_cluster_file.json
@@ -100,8 +100,8 @@
                 "description" : "Some clients of this cluster have issues.",
                 "issues" : [
                     {
-                        "description" : "Unable to update cluster file.",
-                        "name" : "unable_to_write_cluster_file"
+                        "description" : "Cluster file contents do not match current cluster connection string. Verify cluster file is writable and has not been overwritten externally.",
+                        "name" : "incorrect_cluster_file_contents"
                     }
                 ],
                 "name" : "client_issues"

From 30067d2f53b30311b5a0c213e350f69f394c9565 Mon Sep 17 00:00:00 2001
From: "A.J. Beamon" <ajbeamon@apple.com>
Date: Fri, 5 Jan 2018 13:21:54 -0800
Subject: [PATCH 10/14] Whitespace fixes and removal of change to java's
 AbstractTester

---
 .../foundationdb/test/AbstractTester.java     |  3 +-
 fdbclient/vexillographer/vexillographer.cs    | 42 +++++++++----------
 2 files changed, 22 insertions(+), 23 deletions(-)

diff --git a/bindings/java/src/test/com/apple/foundationdb/test/AbstractTester.java b/bindings/java/src/test/com/apple/foundationdb/test/AbstractTester.java
index 8bbd4d2b04..d98521aa83 100644
--- a/bindings/java/src/test/com/apple/foundationdb/test/AbstractTester.java
+++ b/bindings/java/src/test/com/apple/foundationdb/test/AbstractTester.java
@@ -92,8 +92,7 @@ public abstract class AbstractTester {
             if (!args.useExternalClient()) {
                 throw new IllegalArgumentException("Cannot enable callbacks on external thread without using external client");
             }
-            throw new IllegalArgumentException("Cannot enable callbacks on external thread in Java");
-            //fdb.options().setCallbacksOnExternalThreads();
+            fdb.options().setCallbacksOnExternalThreads();
         }
         if (args.useExternalClient()) {
             fdb.options().setDisableLocalClient();
diff --git a/fdbclient/vexillographer/vexillographer.cs b/fdbclient/vexillographer/vexillographer.cs
index 5582638a40..f0bcc5af6b 100644
--- a/fdbclient/vexillographer/vexillographer.cs
+++ b/fdbclient/vexillographer/vexillographer.cs
@@ -132,28 +132,28 @@ namespace vexillographer
                     {
                         var paramTypeStr = oDoc.AttributeOrNull("paramType");
                         ParamType p = paramTypeStr == null ? ParamType.None : (ParamType)Enum.Parse(typeof(ParamType), paramTypeStr);
-						bool hidden = oDoc.AttributeOrNull("hidden") == "true";
-						string disableOn = oDoc.AttributeOrNull("disableOn");
-						bool disabled = false;
-						if(disableOn != null)
-						{
-							string[] disabledBindings = disableOn.Split(',');
-							disabled = disabledBindings.Contains(binding);
-						}
+                        bool hidden = oDoc.AttributeOrNull("hidden") == "true";
+                        string disableOn = oDoc.AttributeOrNull("disableOn");
+                        bool disabled = false;
+                        if(disableOn != null)
+                        {
+                            string[] disabledBindings = disableOn.Split(',');
+                            disabled = disabledBindings.Contains(binding);
+                        }
 
-						if (!disabled)
-						{
-							list.Add(new Option
-							{
-								scope = s,
-								name = oDoc.AttributeNonNull("name"),
-								code = int.Parse(oDoc.AttributeNonNull("code")),
-								paramType = p,
-								paramDesc = oDoc.AttributeOrNull("paramDescription"),
-								comment = oDoc.AttributeOrNull("description"),
-								hidden = hidden
-							});
-						}
+                        if (!disabled)
+                        {
+                            list.Add(new Option
+                            {
+                                scope = s,
+                                name = oDoc.AttributeNonNull("name"),
+                                code = int.Parse(oDoc.AttributeNonNull("code")),
+                                paramType = p,
+                                paramDesc = oDoc.AttributeOrNull("paramDescription"),
+                                comment = oDoc.AttributeOrNull("description"),
+                                hidden = hidden
+                            });
+                        }
                     }
                 }
                 options = list;

From 2763713cbc601745e0f4ce2e1686ed90aaf9d784 Mon Sep 17 00:00:00 2001
From: Stephen Atherton <s_atherton@apple.com>
Date: Fri, 5 Jan 2018 14:12:00 -0800
Subject: [PATCH 11/14] Bug fix, backup snapshot dispatch was calculating that
 all shards must be done immediately.

---
 fdbclient/FileBackupAgent.actor.cpp | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/fdbclient/FileBackupAgent.actor.cpp b/fdbclient/FileBackupAgent.actor.cpp
index dd4bc3defa..49a8faa622 100644
--- a/fdbclient/FileBackupAgent.actor.cpp
+++ b/fdbclient/FileBackupAgent.actor.cpp
@@ -1237,11 +1237,11 @@ namespace fileBackup {
 			// timeElapsed is between 0 and 1 and represents what portion of the shards we should have completed by now
 			double timeElapsed;
 			if(snapshotTargetEndVersion > snapshotBeginVersion)
-				timeElapsed = std::max(1.0, (double)(nextDispatchVersion - snapshotBeginVersion) / (snapshotTargetEndVersion - snapshotBeginVersion));
+				timeElapsed = std::min(1.0, (double)(nextDispatchVersion - snapshotBeginVersion) / (snapshotTargetEndVersion - snapshotBeginVersion));
 			else
 				timeElapsed = 1.0;
 
-			state int countExpectedShardsDone = std::min<int>(countAllShards, countAllShards * timeElapsed);
+			state int countExpectedShardsDone = countAllShards * timeElapsed;
 			state int countShardsToDispatch = std::max<int>(0, countExpectedShardsDone - countShardsDone);
 
 			TraceEvent("FileBackupSnapshotDispatchStats")

From f0219347926e88560e6369a8ee2d57db5cca3960 Mon Sep 17 00:00:00 2001
From: Alex Miller <alexmiller@apple.com>
Date: Tue, 2 Jan 2018 17:32:14 -0800
Subject: [PATCH 12/14] Fix yet another VersionStamp DR bug.

In this episode, we discover that having a transaction retry loop in which the
transaction conditionally has write conflict ranges is potentially troublesome.

To simplify the problem, if we have two concurrent transaction loops:

    retry {
      if (rand() > .5) tr->set('x', rand());
      if (rand() > .5) tr->set('y', rand());
    }

and

    retry {
      x = tr->get('x')
      y = tr->get('y')
      if (x > y) {
        tr->set('y', x)
      }
      tr->commit();
    }

Is not guaranteed that x > y in the database after the second transaction
commits.  This is because it could read an older snapshot of x and y, in which
x was greater than y, and thus not invoke set.  This means that `tr` is now a
read-only transaction, which no-ops out of committing as an "optimization".  If
we add any write conflict range to `tr`, it then will conflict checked and
committed, which would guarantee that x>y when it commits.

Replace the first transaction with dumpData, and the second with version
upgrade transaction, and you have the bug that we're fixing, why, and how.
---
 fdbclient/DatabaseBackupAgent.actor.cpp | 12 ++++++++++--
 1 file changed, 10 insertions(+), 2 deletions(-)

diff --git a/fdbclient/DatabaseBackupAgent.actor.cpp b/fdbclient/DatabaseBackupAgent.actor.cpp
index 436b49a80a..09f74a76dc 100644
--- a/fdbclient/DatabaseBackupAgent.actor.cpp
+++ b/fdbclient/DatabaseBackupAgent.actor.cpp
@@ -1445,8 +1445,8 @@ public:
 				tr3.setOption(FDBTransactionOptions::ACCESS_SYSTEM_KEYS);
 				tr3.setOption(FDBTransactionOptions::LOCK_AWARE);
 				Version destVersion = wait(tr3.getReadVersion());
+				TraceEvent("DBA_switchover_version_upgrade").detail("src", commitVersion).detail("dest", destVersion);
 				if (destVersion <= commitVersion) {
-					TraceEvent("DBA_switchover_version_upgrade").detail("src", commitVersion).detail("dest", destVersion);
 					TEST(true);  // Forcing dest backup cluster to higher version
 					tr3.set(minRequiredCommitVersionKey, BinaryWriter::toValue(commitVersion+1, Unversioned()));
 					Void _ = wait(tr3.commit());
@@ -1549,12 +1549,20 @@ public:
 				if (lastApplied.present()) {
 					Version current = tr->getReadVersion().get();
 					Version applied = BinaryReader::fromStringRef<Version>(lastApplied.get(), Unversioned());
+					TraceEvent("DBA_abort_version_upgrade").detail("src", applied).detail("dest", current);
 					if (current <= applied) {
-						TraceEvent("DBA_abort_version_upgrade").detail("src", applied).detail("dest", current);
 						TEST(true);  // Upgrading version of local database.
 						// The +1 is because we want to make sure that a versionstamped operation can't reuse
 						// the same version as an already-applied transaction.
 						tr->set(minRequiredCommitVersionKey, BinaryWriter::toValue(applied+1, Unversioned()));
+					} else {
+						// We need to enforce that the read we did of the applyMutationsBeginKey is the most
+						// recent and up to date value, as the proxy might have accepted a commit previously
+						// queued by dumpData after our read. Transactions that don't have write conflict ranges
+						// have a no-op commit(), as they become snapshot transactions to which we don't promise
+						// strict serializability.  Therefore, we add an arbitrary write conflict range to
+						// request the strict serializability guarantee that is required.
+						tr->addWriteConflictRange(singleKeyRange(minRequiredCommitVersionKey));
 					}
 				}
 				Void _ = wait(tr->commit());

From b86f68ceb81bded0a4fb06101e4fd3ce6e65c841 Mon Sep 17 00:00:00 2001
From: Stephen Atherton <s_atherton@apple.com>
Date: Fri, 5 Jan 2018 14:43:21 -0800
Subject: [PATCH 13/14] Added new test that combines atomic backup/restore. 
 Added randomization to delays in AtomicRestore workload.

---
 fdbserver/workloads/AtomicRestore.actor.cpp | 20 +++++++--------
 tests/slow/ApiCorrectnessAtomicRestore.txt  | 28 +++++++++++++++++++++
 tests/slow/WriteDuringReadAtomicRestore.txt |  2 +-
 3 files changed, 39 insertions(+), 11 deletions(-)
 create mode 100644 tests/slow/ApiCorrectnessAtomicRestore.txt

diff --git a/fdbserver/workloads/AtomicRestore.actor.cpp b/fdbserver/workloads/AtomicRestore.actor.cpp
index c09824a037..7b7b1f6eac 100644
--- a/fdbserver/workloads/AtomicRestore.actor.cpp
+++ b/fdbserver/workloads/AtomicRestore.actor.cpp
@@ -26,14 +26,14 @@
 
 //A workload which test the correctness of backup and restore process
 struct AtomicRestoreWorkload : TestWorkload {
-	double startAfter, switch1After;
+	double startAfter, restoreAfter;
 	Standalone<VectorRef<KeyRangeRef>> backupRanges;
 
 	AtomicRestoreWorkload(WorkloadContext const& wcx)
 		: TestWorkload(wcx) {
 
 		startAfter = getOption(options, LiteralStringRef("startAfter"), 10.0);
-		switch1After = getOption(options, LiteralStringRef("switch1After"), 20.0);
+		restoreAfter = getOption(options, LiteralStringRef("restoreAfter"), 20.0);
 		backupRanges.push_back_deep(backupRanges.arena(), normalKeys);
 	}
 
@@ -60,10 +60,10 @@ struct AtomicRestoreWorkload : TestWorkload {
 
 	ACTOR static Future<Void> _start(Database cx, AtomicRestoreWorkload* self) {
 		state FileBackupAgent backupAgent;
-		state Future<Void> switch1After = delay(self->switch1After);
 		state Future<Void> disabler = disableConnectionFailuresAfter(300, "atomicRestore");
-		Void _ = wait( delay(self->startAfter) );
-		TraceEvent("AR_Start");
+
+		Void _ = wait( delay(self->startAfter * g_random->random01()) );
+		TraceEvent("AtomicRestore_Start");
 
 		state std::string backupContainer = "file://simfdb/backups/";
 		try {
@@ -74,11 +74,11 @@ struct AtomicRestoreWorkload : TestWorkload {
 				throw;
 		}
 
-		TraceEvent("AS_Wait1");
+		TraceEvent("AtomicRestore_Wait");
 		int _ = wait( backupAgent.waitBackup(cx, BackupAgentBase::getDefaultTagName(), false) );
-		TraceEvent("AS_Ready1");
-		Void _ = wait( switch1After );
-		TraceEvent("AS_Switch1");
+		TraceEvent("AtomicRestore_BackupStart");
+		Void _ = wait( delay(self->restoreAfter * g_random->random01()) );
+		TraceEvent("AtomicRestore_RestoreStart");
 
 		loop {
 			std::vector<Future<Version>> restores;
@@ -102,7 +102,7 @@ struct AtomicRestoreWorkload : TestWorkload {
 			g_simulator.backupAgents = ISimulator::NoBackupAgents;
 		}
 
-		TraceEvent("AS_Done");
+		TraceEvent("AtomicRestore_Done");
 		return Void();
 	}
 };
diff --git a/tests/slow/ApiCorrectnessAtomicRestore.txt b/tests/slow/ApiCorrectnessAtomicRestore.txt
new file mode 100644
index 0000000000..4e1f6b612b
--- /dev/null
+++ b/tests/slow/ApiCorrectnessAtomicRestore.txt
@@ -0,0 +1,28 @@
+testTitle=ApiCorrectnessTest
+testName=ApiCorrectness
+runSetup=true
+clearAfterTest=true
+numKeys=5000
+onlyLowerCase=true
+shortKeysRatio=0.5
+minShortKeyLength=1
+maxShortKeyLength=3
+minLongKeyLength=1
+maxLongKeyLength=128
+minValueLength=1
+maxValueLength=1000
+numGets=1000
+numGetRanges=100
+numGetRangeSelectors=100
+numGetKeys=100
+numClears=100
+numClearRanges=10
+maxTransactionBytes=500000
+randomTestDuration=60
+timeout=2100
+
+testName=AtomicRestore
+startAfter=10.0
+restoreAfter=50.0
+clearAfterTest=false
+simBackupAgents=BackupToFile
diff --git a/tests/slow/WriteDuringReadAtomicRestore.txt b/tests/slow/WriteDuringReadAtomicRestore.txt
index 4e6c3bce36..2486e9de83 100644
--- a/tests/slow/WriteDuringReadAtomicRestore.txt
+++ b/tests/slow/WriteDuringReadAtomicRestore.txt
@@ -8,7 +8,7 @@ testTitle=WriteDuringReadTest
 
     testName=AtomicRestore
     startAfter=10.0
-    switch1After=50.0
+    restoreAfter=50.0
     clearAfterTest=false
     simBackupAgents=BackupToFile
     

From 96cb06cbc7289d34c9cd623968d54ef71eaefda8 Mon Sep 17 00:00:00 2001
From: Stephen Atherton <s_atherton@apple.com>
Date: Fri, 5 Jan 2018 23:06:39 -0800
Subject: [PATCH 14/14] Bug fixes.  Fdbbackup delete was broken.  Blobstore
 backup container deletion would do too much listing before deletions began
 due to list operations queueing up ahead of and starving the delete
 operations.  Created new knob and blob endpoint limit for concurrent list
 operations to fix this.  Increased blob request timeout default because some
 requests were taking longer.  Crash fixes in blobstore doRequest() which
 wasn't checking that response object is valid before using it in error
 conditions.   Filesystem-like backup container class (covering blobstore and
 local dirs) now ignores unrecognized filenames for describe() and expire()
 operations.

---
 fdbbackup/backup.actor.cpp          |  6 +--
 fdbclient/BackupContainer.actor.cpp | 73 ++++++++++++++++++-----------
 fdbclient/Knobs.cpp                 |  5 +-
 fdbclient/Knobs.h                   |  1 +
 fdbrpc/BlobStore.actor.cpp          | 53 +++++++++++++--------
 fdbrpc/BlobStore.h                  | 10 ++--
 6 files changed, 94 insertions(+), 54 deletions(-)

diff --git a/fdbbackup/backup.actor.cpp b/fdbbackup/backup.actor.cpp
index 205d7aa566..2e9019969a 100644
--- a/fdbbackup/backup.actor.cpp
+++ b/fdbbackup/backup.actor.cpp
@@ -1837,16 +1837,16 @@ ACTOR Future<Void> expireBackupData(const char *name, std::string destinationCon
 ACTOR Future<Void> deleteBackupContainer(const char *name, std::string destinationContainer) {
 	try {
 		state Reference<IBackupContainer> c = openBackupContainer(name, destinationContainer);
+		state int numDeleted = 0;
+		state Future<Void> done = c->deleteContainer(&numDeleted);
 
 		loop {
 			choose {
-				when ( Void _  = wait(c->deleteContainer()) ) {
+				when ( Void _ = wait(done) ) {
 					printf("The entire container has been deleted.\n");
 					break;
 				}
 				when ( Void _ = wait(delay(3)) ) {
-					int numDeleted = 0;
-					c->deleteContainer(&numDeleted);
 					printf("%d files have been deleted so far...\n", numDeleted);
 				}
 			}
diff --git a/fdbclient/BackupContainer.actor.cpp b/fdbclient/BackupContainer.actor.cpp
index af6e8bcd59..88f2365850 100644
--- a/fdbclient/BackupContainer.actor.cpp
+++ b/fdbclient/BackupContainer.actor.cpp
@@ -149,36 +149,42 @@ public:
 		return writeFile(format("ranges/%s/range,%lld,%s,%d", rangeVersionFolderString(version).c_str(), version, g_random->randomUniqueID().toString().c_str(), blockSize));
 	}
 
-	static RangeFile pathToRangeFile(std::string path, int64_t size) {
+	static bool pathToRangeFile(RangeFile &out, std::string path, int64_t size) {
 		std::string name = basename(path);
 		RangeFile f;
 		f.fileName = path;
 		f.fileSize = size;
 		int len;
-		if(sscanf(name.c_str(), "range,%lld,%*[^,],%u%n", &f.version, &f.blockSize, &len) == 2 && len == name.size())
-			return f;
-		throw restore_unknown_file_type();
+		if(sscanf(name.c_str(), "range,%lld,%*[^,],%u%n", &f.version, &f.blockSize, &len) == 2 && len == name.size()) {
+			out = f;
+			return true;
+		}
+		return false;
 	}
 
-	static LogFile pathToLogFile(std::string path, int64_t size) {
+	static bool pathToLogFile(LogFile &out, std::string path, int64_t size) {
 		std::string name = basename(path);
 		LogFile f;
 		f.fileName = path;
 		f.fileSize = size;
 		int len;
-		if(sscanf(name.c_str(), "log,%lld,%lld,%*[^,],%u%n", &f.beginVersion, &f.endVersion, &f.blockSize, &len) == 3 && len == name.size())
-			return f;
-		throw restore_unknown_file_type();
+		if(sscanf(name.c_str(), "log,%lld,%lld,%*[^,],%u%n", &f.beginVersion, &f.endVersion, &f.blockSize, &len) == 3 && len == name.size()) {
+			out = f;
+			return true;
+		}
+		return false;
 	}
 
-	static KeyspaceSnapshotFile pathToKeyspaceSnapshotFile(std::string path) {
+	static bool pathToKeyspaceSnapshotFile(KeyspaceSnapshotFile &out, std::string path) {
 		std::string name = basename(path);
 		KeyspaceSnapshotFile f;
 		f.fileName = path;
 		int len;
-		if(sscanf(name.c_str(), "snapshot,%lld,%lld,%lld%n", &f.beginVersion, &f.endVersion, &f.totalSize, &len) == 3 && len == name.size())
-			return f;
-		throw restore_unknown_file_type();
+		if(sscanf(name.c_str(), "snapshot,%lld,%lld,%lld%n", &f.beginVersion, &f.endVersion, &f.totalSize, &len) == 3 && len == name.size()) {
+			out = f;
+			return true;
+		}
+		return false;
 	}
 
 	// TODO:  Do this more efficiently, as the range file list for a snapshot could potentially be hundreds of megabytes.
@@ -236,14 +242,18 @@ public:
 
 		Version minVer = std::numeric_limits<Version>::max();
 		Version maxVer = std::numeric_limits<Version>::min();
+		RangeFile rf;
 
 		for(auto &f : fileNames) {
-			array.push_back(f);
-			RangeFile rf = pathToRangeFile(f, 0);
-			if(rf.version < minVer)
-				minVer = rf.version;
-			if(rf.version > maxVer)
-				maxVer = rf.version;
+			if(pathToRangeFile(rf, f, 0)) {
+				array.push_back(f);
+				if(rf.version < minVer)
+					minVer = rf.version;
+				if(rf.version > maxVer)
+					maxVer = rf.version;
+			}
+			else
+				throw restore_unknown_file_type();
 		}
 
 		doc.create("totalBytes") = totalBytes;
@@ -267,9 +277,9 @@ public:
 	Future<std::vector<LogFile>> listLogFiles(Version beginVersion = std::numeric_limits<Version>::min(), Version endVersion = std::numeric_limits<Version>::max()) {
 		return map(listFiles("logs/"), [=](const FilesAndSizesT &files) {
 			std::vector<LogFile> results;
+			LogFile lf;
 			for(auto &f : files) {
-				LogFile lf = pathToLogFile(f.first, f.second);
-				if(lf.endVersion > beginVersion && lf.beginVersion < endVersion)
+				if(pathToLogFile(lf, f.first, f.second) && lf.endVersion > beginVersion && lf.beginVersion < endVersion)
 					results.push_back(lf);
 			}
 			std::sort(results.begin(), results.end());
@@ -279,9 +289,9 @@ public:
 	Future<std::vector<RangeFile>> listRangeFiles(Version beginVersion = std::numeric_limits<Version>::min(), Version endVersion = std::numeric_limits<Version>::max()) {
 		return map(listFiles("ranges/"), [=](const FilesAndSizesT &files) {
 			std::vector<RangeFile> results;
+			RangeFile rf;
 			for(auto &f : files) {
-				RangeFile rf = pathToRangeFile(f.first, f.second);
-				if(rf.version >= beginVersion && rf.version <= endVersion)
+				if(pathToRangeFile(rf, f.first, f.second) && rf.version >= beginVersion && rf.version <= endVersion)
 					results.push_back(rf);
 			}
 			std::sort(results.begin(), results.end());
@@ -291,9 +301,9 @@ public:
 	Future<std::vector<KeyspaceSnapshotFile>> listKeyspaceSnapshots(Version beginVersion = std::numeric_limits<Version>::min(), Version endVersion = std::numeric_limits<Version>::max()) {
 		return map(listFiles("snapshots/"), [=](const FilesAndSizesT &files) {
 			std::vector<KeyspaceSnapshotFile> results;
+			KeyspaceSnapshotFile sf;
 			for(auto &f : files) {
-				KeyspaceSnapshotFile sf = pathToKeyspaceSnapshotFile(f.first);
-				if(sf.endVersion > beginVersion && sf.beginVersion < endVersion)
+				if(pathToKeyspaceSnapshotFile(sf, f.first) && sf.endVersion > beginVersion && sf.beginVersion < endVersion)
 					results.push_back(sf);
 			}
 			std::sort(results.begin(), results.end());
@@ -724,8 +734,8 @@ public:
 	// TODO:  If there is a need, this can be made faster by discovering common prefixes and listing levels of the folder structure in parallel.
 	ACTOR static Future<Void> deleteContainer_impl(Reference<BackupContainerBlobStore> bc, int *pNumDeleted) {
 		state PromiseStream<BlobStoreEndpoint::ListResult> resultStream;
-		state Future<Void> done = bc->m_bstore->listBucketStream(BACKUP_BUCKET, resultStream, bc->m_name + "/");
-		state std::vector<Future<Void>> deleteFutures;
+		state Future<Void> done = bc->m_bstore->listBucketStream(BACKUP_BUCKET, resultStream, bc->m_name + "/", '/', std::numeric_limits<int>::max());
+		state std::list<Future<Void>> deleteFutures;
 		loop {
 			choose {
 				when(Void _ = wait(done)) {
@@ -740,11 +750,20 @@ public:
 							return Void();
 						}));
 					}
+
+					while(deleteFutures.size() > CLIENT_KNOBS->BLOBSTORE_CONCURRENT_REQUESTS) {
+						Void _ = wait(deleteFutures.front());
+						deleteFutures.pop_front();
+					}
 				}
 			}
 		}
 
-		Void _ = wait(waitForAll(deleteFutures));
+		while(deleteFutures.size() > 0) {
+			Void _ = wait(deleteFutures.front());
+			deleteFutures.pop_front();
+		}
+
 		return Void();
 	}
 
diff --git a/fdbclient/Knobs.cpp b/fdbclient/Knobs.cpp
index 1afb7b7886..0828789ef2 100644
--- a/fdbclient/Knobs.cpp
+++ b/fdbclient/Knobs.cpp
@@ -142,9 +142,11 @@ ClientKnobs::ClientKnobs(bool randomize) {
 	init( BLOBSTORE_CONNECT_TIMEOUT,                10 );
 	init( BLOBSTORE_MAX_CONNECTION_LIFE,           120 );
 	init( BLOBSTORE_REQUEST_TRIES,                  10 );
-	init( BLOBSTORE_REQUEST_TIMEOUT,                30 );
+	init( BLOBSTORE_REQUEST_TIMEOUT,                60 );
 
 	init( BLOBSTORE_CONCURRENT_UPLOADS, BACKUP_TASKS_PER_AGENT*2 );
+	init( BLOBSTORE_CONCURRENT_LISTS,               20 );
+	init( BLOBSTORE_CONCURRENT_REQUESTS, BLOBSTORE_CONCURRENT_UPLOADS + BLOBSTORE_CONCURRENT_LISTS + 5);
 
 	init( BLOBSTORE_CONCURRENT_WRITES_PER_FILE,      5 );
 	init( BLOBSTORE_CONCURRENT_READS_PER_FILE,       3 );
@@ -156,7 +158,6 @@ ClientKnobs::ClientKnobs(bool randomize) {
 
 	// These are basically unlimited by default but can be used to reduce blob IO if needed
 	init( BLOBSTORE_REQUESTS_PER_SECOND,            200 );
-	init( BLOBSTORE_CONCURRENT_REQUESTS, (BACKUP_TASKS_PER_AGENT*2)+5 );
 	init( BLOBSTORE_MAX_SEND_BYTES_PER_SECOND,      1e9 );
 	init( BLOBSTORE_MAX_RECV_BYTES_PER_SECOND,      1e9 );
 
diff --git a/fdbclient/Knobs.h b/fdbclient/Knobs.h
index 7d2a70e734..6c82c45782 100644
--- a/fdbclient/Knobs.h
+++ b/fdbclient/Knobs.h
@@ -152,6 +152,7 @@ public:
 	int BLOBSTORE_MULTIPART_MAX_PART_SIZE;
 	int BLOBSTORE_MULTIPART_MIN_PART_SIZE;
 	int BLOBSTORE_CONCURRENT_UPLOADS;
+	int BLOBSTORE_CONCURRENT_LISTS;
 	int BLOBSTORE_CONCURRENT_WRITES_PER_FILE;
 	int BLOBSTORE_CONCURRENT_READS_PER_FILE;
 	int BLOBSTORE_READ_BLOCK_SIZE;
diff --git a/fdbrpc/BlobStore.actor.cpp b/fdbrpc/BlobStore.actor.cpp
index c9828ea34e..1747dd7731 100644
--- a/fdbrpc/BlobStore.actor.cpp
+++ b/fdbrpc/BlobStore.actor.cpp
@@ -59,6 +59,7 @@ BlobStoreEndpoint::BlobKnobs::BlobKnobs() {
 	multipart_max_part_size = CLIENT_KNOBS->BLOBSTORE_MULTIPART_MAX_PART_SIZE;
 	multipart_min_part_size = CLIENT_KNOBS->BLOBSTORE_MULTIPART_MIN_PART_SIZE;
 	concurrent_uploads = CLIENT_KNOBS->BLOBSTORE_CONCURRENT_UPLOADS;
+	concurrent_lists = CLIENT_KNOBS->BLOBSTORE_CONCURRENT_LISTS;
 	concurrent_reads_per_file = CLIENT_KNOBS->BLOBSTORE_CONCURRENT_READS_PER_FILE;
 	concurrent_writes_per_file = CLIENT_KNOBS->BLOBSTORE_CONCURRENT_WRITES_PER_FILE;
 	read_block_size = CLIENT_KNOBS->BLOBSTORE_READ_BLOCK_SIZE;
@@ -80,6 +81,7 @@ bool BlobStoreEndpoint::BlobKnobs::set(StringRef name, int value) {
 	TRY_PARAM(multipart_max_part_size, maxps);
 	TRY_PARAM(multipart_min_part_size, minps);
 	TRY_PARAM(concurrent_uploads, cu);
+	TRY_PARAM(concurrent_lists, cl);
 	TRY_PARAM(concurrent_reads_per_file, crpf);
 	TRY_PARAM(concurrent_writes_per_file, cwpf);
 	TRY_PARAM(read_block_size, rbs);
@@ -106,6 +108,7 @@ std::string BlobStoreEndpoint::BlobKnobs::getURLParameters() const {
 	_CHECK_PARAM(multipart_max_part_size, maxps);
 	_CHECK_PARAM(multipart_min_part_size, minps);
 	_CHECK_PARAM(concurrent_uploads, cu);
+	_CHECK_PARAM(concurrent_lists, cl);
 	_CHECK_PARAM(concurrent_reads_per_file, crpf);
 	_CHECK_PARAM(concurrent_writes_per_file, cwpf);
 	_CHECK_PARAM(read_block_size, rbs);
@@ -417,7 +420,6 @@ ACTOR Future<Reference<HTTP::Response>> doRequest_impl(Reference<BlobStoreEndpoi
 			// This must be done AFTER the connection is ready because if credentials are coming from disk they are refreshed
 			// when a new connection is established and setAuthHeaders() would need the updated secret.
 			bstore->setAuthHeaders(verb, resource, headers);
-
 			remoteAddress = rconn.conn->getPeerAddress();
 			Void _ = wait(bstore->requestRate->getAllowance(1));
 			state Reference<HTTP::Response> r = wait(timeoutError(HTTP::doRequest(rconn.conn, verb, resource, headers, &contentCopy, contentLen, bstore->sendRate, &bstore->s_stats.bytes_sent, bstore->recvRate), bstore->knobs.request_timeout));
@@ -478,16 +480,18 @@ ACTOR Future<Reference<HTTP::Response>> doRequest_impl(Reference<BlobStoreEndpoi
 		}
 
 		if(retryable) {
-			// If retrying, obey the Retry-After response header if present.
-			auto iRetryAfter = r->headers.find("Retry-After");
-			if(iRetryAfter != r->headers.end()) {
-				event.detail("RetryAfterHeader", iRetryAfter->second);
-				char *pEnd;
-				double retryAfter = strtod(iRetryAfter->second.c_str(), &pEnd);
-				if(*pEnd)  // If there were other characters then don't trust the parsed value, use a probably safe value of 5 minutes.
-					retryAfter = 300;
-				// Update delay
-				delay = std::max(delay, retryAfter);
+			// If r is valid then obey the Retry-After response header if present.
+			if(r) {
+				auto iRetryAfter = r->headers.find("Retry-After");
+				if(iRetryAfter != r->headers.end()) {
+					event.detail("RetryAfterHeader", iRetryAfter->second);
+					char *pEnd;
+					double retryAfter = strtod(iRetryAfter->second.c_str(), &pEnd);
+					if(*pEnd)  // If there were other characters then don't trust the parsed value, use a probably safe value of 5 minutes.
+						retryAfter = 300;
+					// Update delay
+					delay = std::max(delay, retryAfter);
+				}
 			}
 
 			// Log the delay then wait.
@@ -498,7 +502,7 @@ ACTOR Future<Reference<HTTP::Response>> doRequest_impl(Reference<BlobStoreEndpoi
 			// We can't retry, so throw something.
 
 			// This error code means the authentication header was not accepted, likely the account or key is wrong.
-			if(r->code == 406)
+			if(r && r->code == 406)
 				throw http_not_accepted();
 
 			throw http_request_failed();
@@ -526,8 +530,12 @@ ACTOR Future<Void> listBucketStream_impl(Reference<BlobStoreEndpoint> bstore, st
 	state std::vector<Future<Void>> subLists;
 
 	while(more) {
+		Void _ = wait(bstore->concurrentLists.take());
+		state FlowLock::Releaser listReleaser(bstore->concurrentLists, 1);
+
 		HTTP::Headers headers;
 		Reference<HTTP::Response> r = wait(bstore->doRequest("GET", resource + HTTP::urlEncode(lastFile), headers, NULL, 0, {200}));
+		listReleaser.release();
 
 		try {
 			BlobStoreEndpoint::ListResult result;
@@ -535,11 +543,13 @@ ACTOR Future<Void> listBucketStream_impl(Reference<BlobStoreEndpoint> bstore, st
 			json_spirit::mValue json;
 			json_spirit::read_string(r->content, json);
 			JSONDoc doc(json);
+
 			std::string isTruncated;
 			if (!doc.tryGet("truncated", more)) {
 				doc.get("ListBucketResult.IsTruncated", isTruncated);
 				more = isTruncated == "false" ? false : true;
 			}
+
 			if (doc.has("results")) {
 				for (auto &jsonObject : doc.at("results").get_array()) {
 					JSONDoc objectDoc(jsonObject);
@@ -549,6 +559,7 @@ ACTOR Future<Void> listBucketStream_impl(Reference<BlobStoreEndpoint> bstore, st
 					result.objects.push_back(std::move(object));
 				}
 			}
+
 			if(doc.has("ListBucketResult.Contents")) {
 				if (doc.at("ListBucketResult.Contents").type() == json_spirit::array_type) {
 					for (auto &jsonObject : doc.at("ListBucketResult.Contents").get_array()) {
@@ -572,22 +583,26 @@ ACTOR Future<Void> listBucketStream_impl(Reference<BlobStoreEndpoint> bstore, st
 					result.objects.push_back(std::move(object));
 				}
 			}
+
 			if(doc.has("CommonPrefixes")) {
 				for(auto &jsonObject : doc.at("CommonPrefixes").get_array()) {
 					JSONDoc objectDoc(jsonObject);
 					std::string p;
 					objectDoc.get("Prefix", p);
-					result.commonPrefixes.push_back(p);
-					if(maxDepth > 0) {
+					// If recursing, queue a sub-request, otherwise add the common prefix to the result.
+					if(maxDepth > 0)
 						subLists.push_back(bstore->listBucketStream(bucket, results, p, delimiter, maxDepth - 1));
-					}
+					else
+						result.commonPrefixes.push_back(p);
 				}
 			}
 
-			if(!result.objects.empty())
-				lastFile = result.objects.back().name;
-			if(!result.commonPrefixes.empty() && lastFile < result.commonPrefixes.back())
-				lastFile = result.commonPrefixes.back();
+			if(more) {
+				if(!result.objects.empty())
+					lastFile = result.objects.back().name;
+				if(!result.commonPrefixes.empty() && lastFile < result.commonPrefixes.back())
+					lastFile = result.commonPrefixes.back();
+			}
 
 			results.send(result);
 		} catch(Error &e) {
diff --git a/fdbrpc/BlobStore.h b/fdbrpc/BlobStore.h
index 3a01a2aa22..51f2b8abea 100644
--- a/fdbrpc/BlobStore.h
+++ b/fdbrpc/BlobStore.h
@@ -53,10 +53,11 @@ public:
 			request_tries,
 			request_timeout,
 			requests_per_second,
-			concurrent_requests,
 			multipart_max_part_size,
 			multipart_min_part_size,
+			concurrent_requests,
 			concurrent_uploads,
+			concurrent_lists,
 			concurrent_reads_per_file,
 			concurrent_writes_per_file,
 			read_block_size,
@@ -74,10 +75,11 @@ public:
 				"request_tries (or rt)                 Number of times to try each request until a parseable HTTP response other than 429 is received.",
 				"request_timeout (or rto)              Number of seconds to wait for a request to succeed after a connection is established.",
 				"requests_per_second (or rps)          Max number of requests to start per second.",
-				"concurrent_requests (or cr)           Max number of requests in progress at once.",
 				"multipart_max_part_size (or maxps)    Max part size for multipart uploads.",
 				"multipart_min_part_size (or minps)    Min part size for multipart uploads.",
+				"concurrent_requests (or cr)           Max number of total requests in progress at once, regardless of operation-specific concurrency limits.",
 				"concurrent_uploads (or cu)            Max concurrent uploads (part or whole) that can be in progress at once.",
+				"concurrent_lists (or cl)              Max concurrent list operations that can be in progress at once.",
 				"concurrent_reads_per_file (or crps)   Max concurrent reads in progress for any one file.",
 				"concurrent_writes_per_file (or cwps)  Max concurrent uploads in progress for any one file.",
 				"read_block_size (or rbs)              Block size in bytes to be used for reads.",
@@ -95,7 +97,8 @@ public:
 		sendRate(new SpeedLimit(knobs.max_send_bytes_per_second, 1)),
 		recvRate(new SpeedLimit(knobs.max_recv_bytes_per_second, 1)),
 		concurrentRequests(knobs.concurrent_requests),
-		concurrentUploads(knobs.concurrent_uploads) {
+		concurrentUploads(knobs.concurrent_uploads),
+		concurrentLists(knobs.concurrent_lists) {
 
 		if(host.empty())
 			throw connection_string_invalid();
@@ -133,6 +136,7 @@ public:
 	Reference<IRateControl> recvRate;
 	FlowLock concurrentRequests;
 	FlowLock concurrentUploads;
+	FlowLock concurrentLists;
 
 	Future<Void> updateSecret();