Support passing encryption file to BackupContainer::openContainer

fdbclient/BackupAgent.actor.h

@@ -362,20 +362,22 @@ public:
 	                          Key outContainer,
 	                          int initialSnapshotIntervalSeconds,
 	                          int snapshotIntervalSeconds,
-	                          std::string tagName,
+	                          std::string const& tagName,
 	                          Standalone<VectorRef<KeyRangeRef>> backupRanges,
 	                          bool stopWhenDone = true,
 	                          bool partitionedLog = false,
-	                          bool incrementalBackupOnly = false);
+	                          bool incrementalBackupOnly = false,
+	                          Optional<std::string> const& encryptionKeyFileName = {});
 	Future<Void> submitBackup(Database cx,
 	                          Key outContainer,
 	                          int initialSnapshotIntervalSeconds,
 	                          int snapshotIntervalSeconds,
-	                          std::string tagName,
+	                          std::string const& tagName,
 	                          Standalone<VectorRef<KeyRangeRef>> backupRanges,
 	                          bool stopWhenDone = true,
 	                          bool partitionedLog = false,
-	                          bool incrementalBackupOnly = false) {
+	                          bool incrementalBackupOnly = false,
+	                          Optional<std::string> const& encryptionKeyFileName = {}) {
 		return runRYWTransactionFailIfLocked(cx, [=](Reference<ReadYourWritesTransaction> tr) {
 			return submitBackup(tr,
 			                    outContainer,
@@ -385,7 +387,8 @@ public:
 			                    backupRanges,
 			                    stopWhenDone,
 			                    partitionedLog,
-			                    incrementalBackupOnly);
+			                    incrementalBackupOnly,
+			                    encryptionKeyFileName);
 		});
 	}
 

fdbclient/BackupContainer.actor.cpp

@@ -253,7 +253,8 @@ std::vector<std::string> IBackupContainer::getURLFormats() {
 }
 
 // Get an IBackupContainer based on a container URL string
-Reference<IBackupContainer> IBackupContainer::openContainer(const std::string& url) {
+Reference<IBackupContainer> IBackupContainer::openContainer(const std::string& url,
+                                                            Optional<std::string> const& encryptionKeyFileName) {
 	static std::map<std::string, Reference<IBackupContainer>> m_cache;
 
 	Reference<IBackupContainer>& r = m_cache[url];
@@ -263,7 +264,7 @@ Reference<IBackupContainer> IBackupContainer::openContainer(const std::string& u
 	try {
 		StringRef u(url);
 		if (u.startsWith(LiteralStringRef("file://"))) {
-			r = Reference<IBackupContainer>(new BackupContainerLocalDirectory(url));
+			r = makeReference<BackupContainerLocalDirectory>(url, encryptionKeyFileName);
 		} else if (u.startsWith(LiteralStringRef("blobstore://"))) {
 			std::string resource;
 

fdbclient/BackupContainer.h

@@ -293,7 +293,8 @@ public:
 	                                                          Version beginVersion = -1) = 0;
 
 	// Get an IBackupContainer based on a container spec string
-	static Reference<IBackupContainer> openContainer(const std::string& url);
+	static Reference<IBackupContainer> openContainer(const std::string& url,
+	                                                 const Optional<std::string>& encryptionKeyFileName = {});
 	static std::vector<std::string> getURLFormats();
 	static Future<std::vector<std::string>> listContainers(const std::string& baseURL);
 

fdbclient/BackupContainerLocalDirectory.actor.cpp

@@ -23,6 +23,7 @@
 #include "fdbrpc/IAsyncFile.h"
 #include "flow/Platform.actor.h"
 #include "flow/Platform.h"
+#include "flow/StreamCipher.h"
 #include "fdbrpc/simulator.h"
 #include "flow/actorcompiler.h" // This must be the last #include.
 
@@ -131,7 +132,29 @@ std::string BackupContainerLocalDirectory::getURLFormat() {
 	return "file://</path/to/base/dir/>";
 }
 
-BackupContainerLocalDirectory::BackupContainerLocalDirectory(const std::string& url) {
+ACTOR static Future<Void> readEncryptionKey(std::string encryptionKeyFileName) {
+	state Reference<IAsyncFile> keyFile = wait(IAsyncFileSystem::filesystem()->open(encryptionKeyFileName, 0x0, 0400));
+	int64_t fileSize = wait(keyFile->size());
+	// TODO: Use new error code and avoid hard-coding expected size
+	if (fileSize != 16) {
+		throw internal_error();
+	}
+	state std::array<uint8_t, 16> key;
+	wait(success(keyFile->read(key.data(), key.size(), 0)));
+	StreamCipher::Key::initializeKey(std::move(key));
+	return Void();
+}
+
+bool BackupContainerLocalDirectory::usesEncryption() const {
+	return encryptionSetupFuture.isValid();
+}
+
+BackupContainerLocalDirectory::BackupContainerLocalDirectory(const std::string& url,
+                                                             const Optional<std::string>& encryptionKeyFileName) {
+	if (encryptionKeyFileName.present()) {
+		encryptionSetupFuture = readEncryptionKey(encryptionKeyFileName.get());
+	}
+
 	std::string path;
 	if (url.find("file://") != 0) {
 		TraceEvent(SevWarn, "BackupContainerLocalDirectory")
@@ -207,6 +230,9 @@ Future<bool> BackupContainerLocalDirectory::exists() {
 
 Future<Reference<IAsyncFile>> BackupContainerLocalDirectory::readFile(const std::string& path) {
 	int flags = IAsyncFile::OPEN_NO_AIO | IAsyncFile::OPEN_READONLY | IAsyncFile::OPEN_UNCACHED;
+	if (usesEncryption()) {
+		flags |= IAsyncFile::OPEN_ENCRYPTED;
+	}
 	// Simulation does not properly handle opening the same file from multiple machines using a shared filesystem,
 	// so create a symbolic link to make each file opening appear to be unique.  This could also work in production
 	// but only if the source directory is writeable which shouldn't be required for a restore.
@@ -260,6 +286,9 @@ Future<Reference<IAsyncFile>> BackupContainerLocalDirectory::readFile(const std:
 Future<Reference<IBackupFile>> BackupContainerLocalDirectory::writeFile(const std::string& path) {
 	int flags = IAsyncFile::OPEN_NO_AIO | IAsyncFile::OPEN_UNCACHED | IAsyncFile::OPEN_CREATE | IAsyncFile::OPEN_ATOMIC_WRITE_AND_CREATE |
 	            IAsyncFile::OPEN_READWRITE;
+	if (usesEncryption()) {
+		flags |= IAsyncFile::OPEN_ENCRYPTED;
+	}
 	std::string fullPath = joinPath(m_path, path);
 	platform::createDirectory(parentDirectory(fullPath));
 	std::string temp = fullPath + "." + deterministicRandom()->randomUniqueID().toString() + ".temp";

fdbclient/BackupContainerLocalDirectory.h

@@ -33,7 +33,7 @@ public:
 
 	static std::string getURLFormat();
 
-	BackupContainerLocalDirectory(const std::string& url);
+	BackupContainerLocalDirectory(const std::string& url, Optional<std::string> const& encryptionKeyFileName);
 
 	static Future<std::vector<std::string>> listURLs(const std::string& url);
 
@@ -54,6 +54,8 @@ public:
 
 private:
 	std::string m_path;
+	Future<Void> encryptionSetupFuture;
+	bool usesEncryption() const;
 };
 
 #endif

fdbclient/FileBackupAgent.actor.cpp

@@ -4506,6 +4506,7 @@ public:
 		}
 	}
 
+	// TODO: Get rid of all of these confusing boolean flags
 	ACTOR static Future<Void> submitBackup(FileBackupAgent* backupAgent,
 	                                       Reference<ReadYourWritesTransaction> tr,
 	                                       Key outContainer,
@@ -4515,7 +4516,8 @@ public:
 	                                       Standalone<VectorRef<KeyRangeRef>> backupRanges,
 	                                       bool stopWhenDone,
 	                                       bool partitionedLog,
-	                                       bool incrementalBackupOnly) {
+	                                       bool incrementalBackupOnly,
+	                                       Optional<std::string> encryptionKeyFileName) {
 		tr->setOption(FDBTransactionOptions::ACCESS_SYSTEM_KEYS);
 		tr->setOption(FDBTransactionOptions::LOCK_AWARE);
 		tr->setOption(FDBTransactionOptions::COMMIT_ON_FIRST_PROXY);
@@ -4553,7 +4555,7 @@ public:
 			backupContainer = joinPath(backupContainer, std::string("backup-") + nowStr.toString());
 		}
 
-		state Reference<IBackupContainer> bc = IBackupContainer::openContainer(backupContainer);
+		state Reference<IBackupContainer> bc = IBackupContainer::openContainer(backupContainer, encryptionKeyFileName);
 		try {
 			wait(timeoutError(bc->create(), 30));
 		} catch (Error& e) {
@@ -5631,11 +5633,12 @@ Future<Void> FileBackupAgent::submitBackup(Reference<ReadYourWritesTransaction>
                                            Key outContainer,
                                            int initialSnapshotIntervalSeconds,
                                            int snapshotIntervalSeconds,
-                                           std::string tagName,
+                                           std::string const& tagName,
                                            Standalone<VectorRef<KeyRangeRef>> backupRanges,
                                            bool stopWhenDone,
                                            bool partitionedLog,
-                                           bool incrementalBackupOnly) {
+                                           bool incrementalBackupOnly,
+                                           Optional<std::string> const& encryptionKeyFileName) {
 	return FileBackupAgentImpl::submitBackup(this,
 	                                         tr,
 	                                         outContainer,
@@ -5645,7 +5648,8 @@ Future<Void> FileBackupAgent::submitBackup(Reference<ReadYourWritesTransaction>
 	                                         backupRanges,
 	                                         stopWhenDone,
 	                                         partitionedLog,
-	                                         incrementalBackupOnly);
+	                                         incrementalBackupOnly,
+	                                         encryptionKeyFileName);
 }
 
 Future<Void> FileBackupAgent::discontinueBackup(Reference<ReadYourWritesTransaction> tr, Key tagName) {
@@ -5739,8 +5743,8 @@ ACTOR static Future<Void> writeKVs(Database cx, Standalone<VectorRef<KeyValueRef
 	state ReadYourWritesTransaction tr(cx);
 	loop {
 		try {
-			tr.setOption(FDBTransactionOptions::ACCESS_SYSTEM_KEYS);
+			tr.setOption(FDBTransactionOptions::READ_SYSTEM_KEYS);
-			tr.setOption(FDBTransactionOptions::LOCK_AWARE);
+			tr.setOption(FDBTransactionOptions::READ_LOCK_AWARE);
 			KeyRef k1 = kvs[begin].key;
 			KeyRef k2 = end < kvs.size() ? kvs[end].key : normalKeys.end;
 			TraceEvent(SevFRTestInfo, "TransformDatabaseContentsWriteKVReadBack")

fdbrpc/AsyncFileEncrypted.actor.cpp

@@ -241,11 +241,12 @@ TEST_CASE("fdbrpc/AsyncFileEncrypted") {
 	generateRandomData(&writeBuffer.front(), bytes);
 	state std::vector<unsigned char> readBuffer(bytes, 0);
 	ASSERT(g_network->isSimulated());
-	StreamCipher::Key::initializeRandomKey();
+	StreamCipher::Key::initializeRandomTestKey();
 	int flags = IAsyncFile::OPEN_READWRITE | IAsyncFile::OPEN_CREATE | IAsyncFile::OPEN_ATOMIC_WRITE_AND_CREATE |
 	            IAsyncFile::OPEN_UNBUFFERED | IAsyncFile::OPEN_ENCRYPTED | IAsyncFile::OPEN_UNCACHED |
 	            IAsyncFile::OPEN_NO_AIO;
-	state Reference<IAsyncFile> file = wait(IAsyncFileSystem::filesystem()->open(params.getDataDir(), flags, 0600));
+	state Reference<IAsyncFile> file =
+	    wait(IAsyncFileSystem::filesystem()->open(joinPath(params.getDataDir(), "test-encrypted-file"), flags, 0600));
 	state int bytesWritten = 0;
 	while (bytesWritten < bytes) {
 		chunkSize = std::min(deterministicRandom()->randomInt(0, 100), bytes - bytesWritten);

fdbrpc/AsyncFileEncrypted.h

@@ -18,8 +18,7 @@
  * limitations under the License.
  */
 
-#ifndef __FDBRPC_ASYNC_FILE_ENCRYPTED_H__
+#pragma once
-#define __FDBRPC_ASYNC_FILE_ENCRYPTED_H__
 
 #include "fdbrpc/IAsyncFile.h"
 #include "flow/FastRef.h"
@@ -75,5 +74,3 @@ public:
 	void releaseZeroCopy(void* data, int length, int64_t offset) override;
 	int64_t debugFD() const override;
 };
-
-#endif

flow/StreamCipher.cpp

@@ -44,11 +44,18 @@ void StreamCipher::cleanup() noexcept {
 	}
 }
 
-void StreamCipher::Key::initializeRandomKey() {
+void StreamCipher::Key::initializeKey(std::array<unsigned char, 16>&& arr) {
+	ASSERT(!globalKey);
+	globalKey = std::make_unique<Key>(ConstructorTag{});
+	globalKey->arr = std::move(arr);
+	memset(arr.data(), 0, arr.size());
+}
+
+void StreamCipher::Key::initializeRandomTestKey() {
 	ASSERT(g_network->isSimulated());
 	if (globalKey) return;
 	globalKey = std::make_unique<Key>(ConstructorTag{});
-	generateRandomData(globalKey.get()->arr.data(), globalKey.get()->arr.size());
+	generateRandomData(globalKey->arr.data(), globalKey->arr.size());
 }
 
 const StreamCipher::Key& StreamCipher::Key::getKey() {
@@ -56,6 +63,16 @@ const StreamCipher::Key& StreamCipher::Key::getKey() {
 	return *globalKey;
 }
 
+StreamCipher::Key::Key(Key&& rhs) : arr(std::move(rhs.arr)) {
+	memset(arr.data(), 0, arr.size());
+}
+
+StreamCipher::Key& StreamCipher::Key::operator=(Key&& rhs) {
+	arr = std::move(rhs.arr);
+	memset(arr.data(), 0, arr.size());
+	return *this;
+}
+
 StreamCipher::Key::~Key() {
 	memset(arr.data(), 0, arr.size());
 }
@@ -111,7 +128,7 @@ void forceLinkStreamCipherTests() {}
 // Tests both encryption and decryption of random data
 // using the StreamCipher class
 TEST_CASE("flow/StreamCipher") {
-	StreamCipher::Key::initializeRandomKey();
+	StreamCipher::Key::initializeRandomTestKey();
 	const auto& key = StreamCipher::Key::getKey();
 
 	StreamCipher::IV iv;

flow/StreamCipher.h

@@ -18,8 +18,7 @@
  * limitations under the License.
  */
 
-#ifndef __FLOW_STREAM_CIPHER_H__
+#pragma once
-#define __FLOW_STREAM_CIPHER_H__
 
 #include "flow/Arena.h"
 #include "flow/FastRef.h"
@@ -48,9 +47,12 @@ public:
 
 	public:
 		Key(ConstructorTag) {}
+		Key(Key&&);
+		Key& operator=(Key&&);
 		~Key();
 		unsigned char const* data() const { return arr.data(); }
-		static void initializeRandomKey();
+		static void initializeKey(decltype(arr)&&);
+		static void initializeRandomTestKey();
 		static const Key& getKey();
 		static void cleanup() noexcept;
 	};
@@ -75,5 +77,3 @@ public:
 	StringRef decrypt(unsigned char const* ciphertext, int len, Arena&);
 	StringRef finish(Arena&);
 };
-
-#endif