Support encryption for blob store backups (not yet tested)

fdbclient/AsyncFileS3BlobStore.actor.h

@@ -256,7 +256,7 @@ public:
 	    m_concurrentUploads(bstore->knobs.concurrent_writes_per_file) {
 
 		// Add first part
-		m_parts.push_back(Reference<Part>(new Part(1, m_bstore->knobs.multipart_min_part_size)));
+		m_parts.push_back(makeReference<Part>(1, m_bstore->knobs.multipart_min_part_size));
 	}
 };
 

fdbclient/BackupContainer.actor.cpp

@@ -58,6 +58,7 @@ ACTOR Future<Void> appendStringRefWithLen(Reference<IBackupFile> file, Standalon
 	wait(file->append(s.begin(), s.size()));
 	return Void();
 }
+
 } // namespace IBackupFile_impl
 
 Future<Void> IBackupFile::appendStringRefWithLen(Standalone<StringRef> s) {
@@ -278,7 +279,7 @@ Reference<IBackupContainer> IBackupContainer::openContainer(const std::string& u
 			for (auto c : resource)
 				if (!isalnum(c) && c != '_' && c != '-' && c != '.' && c != '/')
 					throw backup_invalid_url();
-			r = makeReference<BackupContainerS3BlobStore>(bstore, resource, backupParams);
+			r = makeReference<BackupContainerS3BlobStore>(bstore, resource, backupParams, encryptionKeyFileName);
 		}
 #ifdef BUILD_AZURE_BACKUP
 		else if (u.startsWith("azure://"_sr)) {
@@ -286,7 +287,8 @@ Reference<IBackupContainer> IBackupContainer::openContainer(const std::string& u
 			auto address = NetworkAddress::parse(u.eat("/"_sr).toString());
 			auto containerName = u.eat("/"_sr).toString();
 			auto accountName = u.eat("/"_sr).toString();
-			r = makeReference<BackupContainerAzureBlobStore>(address, containerName, accountName);
+			r = makeReference<BackupContainerAzureBlobStore>(
+			    address, containerName, accountName, encryptionKeyFileName);
 		}
 #endif
 		else {
@@ -334,7 +336,7 @@ ACTOR Future<std::vector<std::string>> listContainers_impl(std::string baseURL)
 			}
 
 			// Create a dummy container to parse the backup-specific parameters from the URL and get a final bucket name
-			BackupContainerS3BlobStore dummy(bstore, "dummy", backupParams);
+			BackupContainerS3BlobStore dummy(bstore, "dummy", backupParams, {});
 
 			std::vector<std::string> results = wait(BackupContainerS3BlobStore::listURLs(bstore, dummy.getBucket()));
 			return results;

fdbclient/BackupContainerAzureBlobStore.actor.cpp

@@ -19,6 +19,7 @@
  */
 
 #include "fdbclient/BackupContainerAzureBlobStore.h"
+#include "fdbrpc/AsyncFileEncrypted.h"
 
 #include "flow/actorcompiler.h" // This must be the last #include.
 
@@ -167,8 +168,13 @@ public:
 		if (!exists) {
 			throw file_not_found();
 		}
-		return Reference<IAsyncFile>(
+		Reference<IAsyncFile> f =
-		    new ReadFile(self->asyncTaskThread, self->containerName, fileName, self->client.get()));
+		    makeReference<ReadFile>(self->asyncTaskThread, self->containerName, fileName, self->client.get());
+		if (self->usesEncryption()) {
+			f = makeReference<AsyncFileEncrypted>(f, false);
+		}
+		f = makeReference<ReadFile>(self->asyncTaskThread, self->containerName, fileName, self->client.get());
+		return f;
 	}
 
 	ACTOR static Future<Reference<IBackupFile>> writeFile(BackupContainerAzureBlobStore* self, std::string fileName) {
@@ -177,10 +183,11 @@ public:
 			    auto outcome = client->create_append_blob(containerName, fileName).get();
 			    return Void();
 		    }));
-		return Reference<IBackupFile>(
+		auto f = makeReference<WriteFile>(self->asyncTaskThread, self->containerName, fileName, self->client.get());
-		    new BackupFile(fileName,
+		if (self->usesEncryption()) {
-		                   Reference<IAsyncFile>(new WriteFile(
+			f = makeReference<AsyncFileEncrypted>(f, true);
-		                       self->asyncTaskThread, self->containerName, fileName, self->client.get()))));
+		}
+		return makeReference<BackupFile>(fileName, f);
 	}
 
 	static void listFiles(AzureClient* client,
@@ -213,6 +220,16 @@ public:
 		}
 		return Void();
 	}
+
+	ACTOR static Future<Void> create(BackupContainerAzureBlobStore* self) {
+		state Future<Void> f1 =
+		    self->asyncTaskThread.execAsync([containerName = self->containerName, client = self->client.get()] {
+			    client->create_container(containerName).wait();
+			    return Void();
+		    });
+		state Future<Void> f2 = self->usesEncryption() ? self->encryptionSetupComplete() : Void();
+		return f1 && f2;
+	}
 };
 
 Future<bool> BackupContainerAzureBlobStore::blobExists(const std::string& fileName) {
@@ -225,10 +242,11 @@ Future<bool> BackupContainerAzureBlobStore::blobExists(const std::string& fileNa
 
 BackupContainerAzureBlobStore::BackupContainerAzureBlobStore(const NetworkAddress& address,
                                                              const std::string& accountName,
-                                                             const std::string& containerName)
+                                                             const std::string& containerName,
+                                                             const Optional<std::string>& encryptionKeyFileName)
   : containerName(containerName) {
+	setEncryptionKey(encryptionKeyFileName);
 	std::string accountKey = std::getenv("AZURE_KEY");
-
 	auto credential = std::make_shared<azure::storage_lite::shared_key_credential>(accountName, accountKey);
 	auto storageAccount = std::make_shared<azure::storage_lite::storage_account>(
 	    accountName, credential, false, format("http://%s/%s", address.toString().c_str(), accountName.c_str()));
@@ -244,10 +262,7 @@ void BackupContainerAzureBlobStore::delref() {
 }
 
 Future<Void> BackupContainerAzureBlobStore::create() {
-	return asyncTaskThread.execAsync([containerName = this->containerName, client = this->client.get()] {
+	return BackupContainerAzureBlobStoreImpl::create(this);
-		client->create_container(containerName).wait();
-		return Void();
-	});
 }
 Future<bool> BackupContainerAzureBlobStore::exists() {
 	return asyncTaskThread.execAsync([containerName = this->containerName, client = this->client.get()] {

fdbclient/BackupContainerAzureBlobStore.h

@@ -44,7 +44,8 @@ class BackupContainerAzureBlobStore final : public BackupContainerFileSystem,
 public:
 	BackupContainerAzureBlobStore(const NetworkAddress& address,
 	                              const std::string& accountName,
-	                              const std::string& containerName);
+	                              const std::string& containerName,
+	                              const Optional<std::string>& encryptionKeyFileName);
 
 	void addref() override;
 	void delref() override;

fdbclient/BackupContainerFileSystem.actor.cpp

@@ -23,6 +23,7 @@
 #include "fdbclient/BackupContainerFileSystem.h"
 #include "fdbclient/BackupContainerLocalDirectory.h"
 #include "fdbclient/JsonBuilder.h"
+#include "flow/StreamCipher.h"
 #include "flow/UnitTest.h"
 
 #include <algorithm>
@@ -1126,6 +1127,16 @@ public:
 		return false;
 	}
 
+	ACTOR static Future<Void> readEncryptionKey(std::string encryptionKeyFileName) {
+		state Reference<IAsyncFile> keyFile =
+		    wait(IAsyncFileSystem::filesystem()->open(encryptionKeyFileName, 0x0, 0400));
+		state std::array<uint8_t, 16> key;
+		int bytesRead = wait(keyFile->read(key.data(), key.size(), 0));
+		// TODO: Throw new error (fail gracefully)
+		ASSERT_EQ(bytesRead, key.size());
+		StreamCipher::Key::initializeKey(std::move(key));
+		return Void();
+	}
 }; // class BackupContainerFileSystemImpl
 
 Future<Reference<IBackupFile>> BackupContainerFileSystem::writeLogFile(Version beginVersion,
@@ -1432,6 +1443,17 @@ BackupContainerFileSystem::VersionProperty BackupContainerFileSystem::unreliable
 BackupContainerFileSystem::VersionProperty BackupContainerFileSystem::logType() {
 	return { Reference<BackupContainerFileSystem>::addRef(this), "mutation_log_type" };
 }
+bool BackupContainerFileSystem::usesEncryption() const {
+	return encryptionSetupFuture.isValid();
+}
+Future<Void> BackupContainerFileSystem::encryptionSetupComplete() const {
+	return encryptionSetupFuture;
+}
+void BackupContainerFileSystem::setEncryptionKey(Optional<std::string> const& encryptionKeyFileName) {
+	if (encryptionKeyFileName.present()) {
+		encryptionSetupFuture = BackupContainerFileSystemImpl::readEncryptionKey(encryptionKeyFileName.get());
+	}
+}
 
 namespace backup_test {
 

fdbclient/BackupContainerFileSystem.h

@@ -186,6 +186,14 @@ private:
 	Future<std::vector<RangeFile>> old_listRangeFiles(Version beginVersion, Version endVersion);
 
 	friend class BackupContainerFileSystemImpl;
+
+protected:
+	bool usesEncryption() const;
+	void setEncryptionKey(Optional<std::string> const& encryptionKeyFileName);
+	Future<Void> encryptionSetupComplete() const;
+
+private:
+	Future<Void> encryptionSetupFuture;
 };
 
 #endif

fdbclient/BackupContainerLocalDirectory.actor.cpp

@@ -23,7 +23,6 @@
 #include "fdbrpc/IAsyncFile.h"
 #include "flow/Platform.actor.h"
 #include "flow/Platform.h"
-#include "flow/StreamCipher.h"
 #include "fdbrpc/simulator.h"
 #include "flow/actorcompiler.h" // This must be the last #include.
 
@@ -132,25 +131,9 @@ std::string BackupContainerLocalDirectory::getURLFormat() {
 	return "file://</path/to/base/dir/>";
 }
 
-ACTOR static Future<Void> readEncryptionKey(std::string encryptionKeyFileName) {
-	state Reference<IAsyncFile> keyFile = wait(IAsyncFileSystem::filesystem()->open(encryptionKeyFileName, 0x0, 0400));
-	state std::array<uint8_t, 16> key;
-	int bytesRead = wait(keyFile->read(key.data(), key.size(), 0));
-	// TODO: Throw new error (fail gracefully)
-	ASSERT_EQ(bytesRead, key.size());
-	StreamCipher::Key::initializeKey(std::move(key));
-	return Void();
-}
-
-bool BackupContainerLocalDirectory::usesEncryption() const {
-	return encryptionSetupFuture.isValid();
-}
-
 BackupContainerLocalDirectory::BackupContainerLocalDirectory(const std::string& url,
                                                              const Optional<std::string>& encryptionKeyFileName) {
-	if (encryptionKeyFileName.present()) {
+	setEncryptionKey(encryptionKeyFileName);
-		encryptionSetupFuture = readEncryptionKey(encryptionKeyFileName.get());
-	}
 
 	std::string path;
 	if (url.find("file://") != 0) {
@@ -214,10 +197,9 @@ Future<std::vector<std::string>> BackupContainerLocalDirectory::listURLs(const s
 
 Future<Void> BackupContainerLocalDirectory::create() {
 	if (usesEncryption()) {
-		return encryptionSetupFuture;
+		return encryptionSetupComplete();
 	}
-	// TODO: Update this comment:
+	// No directory should be created here because create() can be called by any process working with the container URL,
-	// Nothing should be done here because create() can be called by any process working with the container URL,
 	// such as fdbbackup. Since "local directory" containers are by definition local to the machine they are
 	// accessed from, the container's creation (in this case the creation of a directory) must be ensured prior to
 	// every file creation, which is done in openFile(). Creating the directory here will result in unnecessary

fdbclient/BackupContainerLocalDirectory.h

@@ -54,8 +54,6 @@ public:
 
 private:
 	std::string m_path;
-	Future<Void> encryptionSetupFuture;
-	bool usesEncryption() const;
 };
 
 #endif

fdbclient/BackupContainerS3BlobStore.actor.cpp

@@ -20,6 +20,7 @@
 
 #include "fdbclient/AsyncFileS3BlobStore.actor.h"
 #include "fdbclient/BackupContainerS3BlobStore.h"
+#include "fdbrpc/AsyncFileEncrypted.h"
 #include "fdbrpc/AsyncFileReadAhead.actor.h"
 #include "flow/actorcompiler.h" // This must be the last #include.
 
@@ -103,6 +104,10 @@ public:
 			wait(bc->m_bstore->writeEntireFile(bc->m_bucket, bc->indexEntry(), ""));
 		}
 
+		if (bc->usesEncryption()) {
+			wait(bc->encryptionSetupComplete());
+		}
+
 		return Void();
 	}
 
@@ -137,9 +142,10 @@ std::string BackupContainerS3BlobStore::indexEntry() {
 
 BackupContainerS3BlobStore::BackupContainerS3BlobStore(Reference<S3BlobStoreEndpoint> bstore,
                                                        const std::string& name,
-                                                       const S3BlobStoreEndpoint::ParametersT& params)
+                                                       const S3BlobStoreEndpoint::ParametersT& params,
+                                                       const Optional<std::string>& encryptionKeyFileName)
   : m_bstore(bstore), m_name(name), m_bucket("FDB_BACKUPS_V2") {
-
+	setEncryptionKey(encryptionKeyFileName);
 	// Currently only one parameter is supported, "bucket"
 	for (const auto& [name, value] : params) {
 		if (name == "bucket") {
@@ -164,12 +170,16 @@ std::string BackupContainerS3BlobStore::getURLFormat() {
 }
 
 Future<Reference<IAsyncFile>> BackupContainerS3BlobStore::readFile(const std::string& path) {
-	return Reference<IAsyncFile>(new AsyncFileReadAheadCache(
+	Reference<IAsyncFile> f = makeReference<AsyncFileS3BlobStoreRead>(m_bstore, m_bucket, dataPath(path));
-	    Reference<IAsyncFile>(new AsyncFileS3BlobStoreRead(m_bstore, m_bucket, dataPath(path))),
+	if (usesEncryption()) {
+		f = makeReference<AsyncFileEncrypted>(f, false);
+	}
+	f = makeReference<AsyncFileReadAheadCache>(f,
 	                                           m_bstore->knobs.read_block_size,
 	                                           m_bstore->knobs.read_ahead_blocks,
 	                                           m_bstore->knobs.concurrent_reads_per_file,
-	    m_bstore->knobs.read_cache_blocks_per_file));
+	                                           m_bstore->knobs.read_cache_blocks_per_file);
+	return f;
 }
 
 Future<std::vector<std::string>> BackupContainerS3BlobStore::listURLs(Reference<S3BlobStoreEndpoint> bstore,
@@ -178,8 +188,11 @@ Future<std::vector<std::string>> BackupContainerS3BlobStore::listURLs(Reference<
 }
 
 Future<Reference<IBackupFile>> BackupContainerS3BlobStore::writeFile(const std::string& path) {
-	return Reference<IBackupFile>(new BackupContainerS3BlobStoreImpl::BackupFile(
+	Reference<IAsyncFile> f = makeReference<AsyncFileS3BlobStoreWrite>(m_bstore, m_bucket, dataPath(path));
-	    path, Reference<IAsyncFile>(new AsyncFileS3BlobStoreWrite(m_bstore, m_bucket, dataPath(path)))));
+	if (usesEncryption()) {
+		f = makeReference<AsyncFileEncrypted>(f, true);
+	}
+	return Future<Reference<IBackupFile>>(makeReference<BackupContainerS3BlobStoreImpl::BackupFile>(path, f));
 }
 
 Future<Void> BackupContainerS3BlobStore::deleteFile(const std::string& path) {

fdbclient/BackupContainerS3BlobStore.h

@@ -43,7 +43,8 @@ class BackupContainerS3BlobStore final : public BackupContainerFileSystem,
 public:
 	BackupContainerS3BlobStore(Reference<S3BlobStoreEndpoint> bstore,
 	                           const std::string& name,
-	                           const S3BlobStoreEndpoint::ParametersT& params);
+	                           const S3BlobStoreEndpoint::ParametersT& params,
+	                           const Optional<std::string>& encryptionKeyFileName);
 
 	void addref() override;
 	void delref() override;

fdbrpc/AsyncFileEncrypted.h

@@ -60,6 +60,7 @@ class AsyncFileEncrypted : public IAsyncFile, public ReferenceCounted<AsyncFileE
 	Future<Void> initialize();
 
 public:
+	// TODO: Remove boolean parameter here:
 	AsyncFileEncrypted(Reference<IAsyncFile>, bool canWrite);
 	void addref() override;
 	void delref() override;