Added progress reporting to backup expiration. Simplified backup delete progress.

fdbbackup/backup.actor.cpp

@@ -1851,7 +1851,31 @@ ACTOR Future<Void> expireBackupData(const char *name, std::string destinationCon
 
 	try {
 		Reference<IBackupContainer> c = openBackupContainer(name, destinationContainer);
-		Void _ = wait(c->expireData(endVersion, force, restorableAfterVersion));
+
+		state IBackupContainer::ExpireProgress progress;
+		state std::string lastProgress;
+		state Future<Void> expire = c->expireData(endVersion, force, &progress, restorableAfterVersion);
+
+		loop {
+			choose {
+				when(Void _ = wait(delay(5))) {
+					std::string p = progress.toString();
+					if(p != lastProgress) {
+						int spaces = lastProgress.size() - p.size();
+						printf("\r%s%s", p.c_str(), (spaces > 0 ? std::string(spaces, ' ').c_str() : "") );
+						lastProgress = p;
+					}
+				}
+				when(Void _ = wait(expire)) {
+					break;
+				}
+			}
+		}
+
+		std::string p = progress.toString();
+		int spaces = lastProgress.size() - p.size();
+		printf("\r%s%s\n", p.c_str(), (spaces > 0 ? std::string(spaces, ' ').c_str() : "") );
+
 		if(endVersion < 0)
 			printf("All data before %lld versions (%lld days) prior to latest backup log has been deleted.\n", -endVersion, -endVersion / ((int64_t)24 * 3600 * CLIENT_KNOBS->CORE_VERSIONSPERSECOND));
 		else
@@ -1876,17 +1900,24 @@ ACTOR Future<Void> deleteBackupContainer(const char *name, std::string destinati
 		state int numDeleted = 0;
 		state Future<Void> done = c->deleteContainer(&numDeleted);
 
+		state int lastUpdate = -1;
+		printf("Deleting data...\n");
+
 		loop {
 			choose {
 				when ( Void _ = wait(done) ) {
 					printf("The entire container has been deleted.\n");
 					break;
 				}
-				when ( Void _ = wait(delay(3)) ) {
-					printf("%d files have been deleted so far...\n", numDeleted);
+				when ( Void _ = wait(delay(5)) ) {
+					if(numDeleted != lastUpdate) {
+						printf("\r%d...", numDeleted);
+						lastUpdate = numDeleted;
+					}
 				}
 			}
 		}
+		printf("\n");
 	}
 	catch (Error& e) {
 		if(e.code() == error_code_actor_cancelled)

fdbclient/BackupContainer.actor.cpp

@@ -47,6 +47,14 @@ Future<Void> IBackupFile::appendStringRefWithLen(Standalone<StringRef> s) {
 	return IBackupFile_impl::appendStringRefWithLen(Reference<IBackupFile>::addRef(this), s);
 }
 
+std::string IBackupContainer::ExpireProgress::toString() const {
+	std::string s = step + "...";
+	if(total > 0) {
+		s += format("%d/%d (%.2f%%)", done, total, double(done) / total * 100);
+	}
+	return s;
+}
+
 std::string formatTime(int64_t t) {
 	time_t curTime = (time_t)t;
 	char buffer[128];
@@ -731,7 +739,12 @@ public:
 		return describeBackup_impl(Reference<BackupContainerFileSystem>::addRef(this), deepScan, logStartVersionOverride);
 	}
 
-	ACTOR static Future<Void> expireData_impl(Reference<BackupContainerFileSystem> bc, Version expireEndVersion, bool force, Version restorableBeginVersion) {
+	ACTOR static Future<Void> expireData_impl(Reference<BackupContainerFileSystem> bc, Version expireEndVersion, bool force, ExpireProgress *progress, Version restorableBeginVersion) {
+		if(progress != nullptr) {
+			progress->step = "Describing backup";
+			progress->total = 0;
+		}
+
 		TraceEvent("BackupContainerFileSystemExpire1")
 			.detail("URL", bc->getURL())
 			.detail("ExpireEndVersion", expireEndVersion)
@@ -785,6 +798,9 @@ public:
 		state std::vector<LogFile> logs;
 		state std::vector<RangeFile> ranges;
 
+		if(progress != nullptr) {
+			progress->step = "Listing files";
+		}
 		// Get log files or range files that contain any data at or before expireEndVersion
 		Void _ = wait(store(bc->listLogFiles(scanBegin, expireEndVersion - 1), logs) && store(bc->listRangeFiles(scanBegin, expireEndVersion - 1), ranges));
 
@@ -844,11 +860,20 @@ public:
 		// We are about to start deleting files, at which point all data prior to expireEndVersion is considered
 		// 'unreliable' as some or all of it will be missing.  So before deleting anything, read unreliableEndVersion
 		// (don't use cached value in desc) and update its value if it is missing or < expireEndVersion
+		if(progress != nullptr) {
+			progress->step = "Initial metadata update";
+		}
 		Optional<Version> metaUnreliableEnd = wait(bc->unreliableEndVersion().get());
 		if(metaUnreliableEnd.orDefault(0) < expireEndVersion) {
 			Void _ = wait(bc->unreliableEndVersion().set(expireEndVersion));
 		}
 
+		if(progress != nullptr) {
+			progress->step = "Deleting files";
+			progress->total = toDelete.size();
+			progress->done = 0;
+		}
+
 		// Delete files, but limit parallelism because the file list could use a lot of memory and the corresponding
 		// delete actor states would use even more if they all existed at the same time.
 		state std::list<Future<Void>> deleteFutures;
@@ -868,10 +893,17 @@ public:
 
 			while(deleteFutures.size() > targetFuturesSize) {
 				Void _ = wait(deleteFutures.front());
+				if(progress != nullptr) {
+					++progress->done;
+				}
 				deleteFutures.pop_front();
 			}
 		}
 
+		if(progress != nullptr) {
+			progress->step = "Final metadata update";
+			progress->total = 0;
+		}
 		// Update the expiredEndVersion metadata to indicate that everything prior to that version has been
 		// successfully deleted if the current version is lower or missing
 		Optional<Version> metaExpiredEnd = wait(bc->expiredEndVersion().get());
@@ -883,8 +915,8 @@ public:
 	}
 
 	// Delete all data up to (but not including endVersion)
-	Future<Void> expireData(Version expireEndVersion, bool force, Version restorableBeginVersion) {
-		return expireData_impl(Reference<BackupContainerFileSystem>::addRef(this), expireEndVersion, force, restorableBeginVersion);
+	Future<Void> expireData(Version expireEndVersion, bool force, ExpireProgress *progress, Version restorableBeginVersion) {
+		return expireData_impl(Reference<BackupContainerFileSystem>::addRef(this), expireEndVersion, force, progress, restorableBeginVersion);
 	}
 
 	ACTOR static Future<Optional<RestorableFileSet>> getRestoreSet_impl(Reference<BackupContainerFileSystem> bc, Version targetVersion) {

fdbclient/BackupContainer.h

@@ -174,13 +174,19 @@ public:
 	// Open a file for read by name
 	virtual Future<Reference<IAsyncFile>> readFile(std::string name) = 0;
 
+	struct ExpireProgress {
+		std::string step;
+		int total;
+		int done;
+		std::string toString() const;
+	};
 	// Delete backup files which do not contain any data at or after (more recent than) expireEndVersion.
 	// If force is false, then nothing will be deleted unless there is a restorable snapshot which
 	//   - begins at or after expireEndVersion
 	//   - ends at or before restorableBeginVersion
 	// If force is true, data is deleted unconditionally which could leave the backup in an unusable state.  This is not recommended.
 	// Returns true if expiration was done.
-	virtual Future<Void> expireData(Version expireEndVersion, bool force = false, Version restorableBeginVersion = std::numeric_limits<Version>::max()) = 0;
+	virtual Future<Void> expireData(Version expireEndVersion, bool force = false, ExpireProgress *progress = nullptr, Version restorableBeginVersion = std::numeric_limits<Version>::max()) = 0;
 
 	// Delete entire container.  During the process, if pNumDeleted is not null it will be
 	// updated with the count of deleted files so that progress can be seen.