From 27e44c1bb9f9b7e6a3b1f40a026820791b51d066 Mon Sep 17 00:00:00 2001 From: sfc-gh-tclinkenbeard Date: Sat, 26 Jun 2021 11:15:12 -0700 Subject: [PATCH] Add support for encryption testing in BackupCorrectness workload --- fdbbackup/backup.actor.cpp | 7 +- fdbclient/AsyncTaskThread.actor.cpp | 2 +- fdbclient/BackupContainerFileSystem.actor.cpp | 79 +++++++++++-------- fdbclient/BackupContainerFileSystem.h | 13 +-- fdbrpc/AsyncFileEncrypted.actor.cpp | 6 +- fdbrpc/AsyncFileEncrypted.h | 1 - .../workloads/BackupCorrectness.actor.cpp | 77 ++++++++++++------ flow/StreamCipher.cpp | 4 +- flow/StreamCipher.h | 3 +- 9 files changed, 116 insertions(+), 76 deletions(-) diff --git a/fdbbackup/backup.actor.cpp b/fdbbackup/backup.actor.cpp index 0213bece1a..488ff04d76 100644 --- a/fdbbackup/backup.actor.cpp +++ b/fdbbackup/backup.actor.cpp @@ -2490,9 +2490,8 @@ ACTOR Future runFastRestoreTool(Database db, ACTOR Future dumpBackupData(const char* name, std::string destinationContainer, Version beginVersion, - Version endVersion, - Optional encryptionKeyFile) { - state Reference c = openBackupContainer(name, destinationContainer, encryptionKeyFile); + Version endVersion) { + state Reference c = openBackupContainer(name, destinationContainer); if (beginVersion < 0 || endVersion < 0) { BackupDescription desc = wait(c->describeBackup()); @@ -3994,7 +3993,7 @@ int main(int argc, char* argv[]) { case BackupType::DUMP: initTraceFile(); - f = stopAfter(dumpBackupData(argv[0], destinationContainer, dumpBegin, dumpEnd, encryptionKeyFile)); + f = stopAfter(dumpBackupData(argv[0], destinationContainer, dumpBegin, dumpEnd)); break; case BackupType::UNDEFINED: diff --git a/fdbclient/AsyncTaskThread.actor.cpp b/fdbclient/AsyncTaskThread.actor.cpp index 2e7c6e3596..050af68c29 100644 --- a/fdbclient/AsyncTaskThread.actor.cpp +++ b/fdbclient/AsyncTaskThread.actor.cpp @@ -83,6 +83,6 @@ TEST_CASE("/asynctaskthread/add") { clients.push_back(asyncTaskThreadClient(&asyncTaskThread, &sum, 100)); } wait(waitForAll(clients)); - ASSERT(sum == 1000); + ASSERT_EQ(sum, 1000); return Void(); } diff --git a/fdbclient/BackupContainerFileSystem.actor.cpp b/fdbclient/BackupContainerFileSystem.actor.cpp index 741ed55ce0..f0a3e80e0f 100644 --- a/fdbclient/BackupContainerFileSystem.actor.cpp +++ b/fdbclient/BackupContainerFileSystem.actor.cpp @@ -291,13 +291,13 @@ public: std::map> tagIndices; // tagId -> indices in files for (int i = 0; i < logs.size(); i++) { - ASSERT(logs[i].tagId >= 0); - ASSERT(logs[i].tagId < logs[i].totalTags); + ASSERT_GE(logs[i].tagId, 0); + ASSERT_LT(logs[i].tagId, logs[i].totalTags); auto& indices = tagIndices[logs[i].tagId]; // filter out if indices.back() is subset of files[i] or vice versa if (!indices.empty()) { if (logs[indices.back()].isSubset(logs[i])) { - ASSERT(logs[indices.back()].fileSize <= logs[i].fileSize); + ASSERT_LE(logs[indices.back()].fileSize, logs[i].fileSize); indices.back() = i; } else if (!logs[i].isSubset(logs[indices.back()])) { indices.push_back(i); @@ -865,7 +865,7 @@ public: int i = 0; for (int j = 1; j < logs.size(); j++) { if (logs[j].isSubset(logs[i])) { - ASSERT(logs[j].fileSize <= logs[i].fileSize); + ASSERT_LE(logs[j].fileSize, logs[i].fileSize); continue; } @@ -1033,10 +1033,10 @@ public: } static std::string versionFolderString(Version v, int smallestBucket) { - ASSERT(smallestBucket < 14); + ASSERT_LT(smallestBucket, 14); // Get a 0-padded fixed size representation of v std::string vFixedPrecision = format("%019lld", v); - ASSERT(vFixedPrecision.size() == 19); + ASSERT_EQ(vFixedPrecision.size(), 19); // Truncate smallestBucket from the fixed length representation vFixedPrecision.resize(vFixedPrecision.size() - smallestBucket); @@ -1127,12 +1127,37 @@ public: return false; } + ACTOR static Future createTestEncryptionKeyFile(std::string filename) { + state Reference keyFile = wait(IAsyncFileSystem::filesystem()->open( + filename, + IAsyncFile::OPEN_ATOMIC_WRITE_AND_CREATE | IAsyncFile::OPEN_READWRITE | IAsyncFile::OPEN_CREATE, + 0600)); + StreamCipher::Key::RawKeyType testKey; + generateRandomData(testKey.data(), testKey.size()); + keyFile->write(testKey.data(), testKey.size(), 0); + wait(keyFile->sync()); + return Void(); + } + ACTOR static Future readEncryptionKey(std::string encryptionKeyFileName) { - state Reference keyFile = - wait(IAsyncFileSystem::filesystem()->open(encryptionKeyFileName, 0x0, 0400)); - state std::array key; + state Reference keyFile; + try { + Reference _keyFile = + wait(IAsyncFileSystem::filesystem()->open(encryptionKeyFileName, 0x0, 0400)); + } catch (Error& e) { + TraceEvent(SevWarnAlways, "FailedToOpenEncryptionKeyFile") + .detail("FileName", encryptionKeyFileName) + .error(e); + throw e; + } + state StreamCipher::Key::RawKeyType key; int bytesRead = wait(keyFile->read(key.data(), key.size(), 0)); - // TODO: Throw new error (fail gracefully) + if (bytesRead != key.size()) { + TraceEvent(SevWarnAlways, "InvalidEncryptionKeyFileSize") + .detail("ExpectedSize", key.size()) + .detail("ActualSize", bytesRead); + throw invalid_encryption_key_file(); + } ASSERT_EQ(bytesRead, key.size()); StreamCipher::Key::initializeKey(std::move(key)); return Void(); @@ -1496,7 +1521,7 @@ ACTOR Future writeAndVerifyFile(Reference c, Reference> buf; buf.resize(buf.arena(), fileSize); int b = wait(inputFile->read(buf.begin(), buf.size(), 0)); - ASSERT(b == buf.size()); + ASSERT_EQ(b, buf.size()); ASSERT(buf == content); } return Void(); @@ -1510,7 +1535,7 @@ Version nextVersion(Version v) { // Write a snapshot file with only begin & end key ACTOR static Future testWriteSnapshotFile(Reference file, Key begin, Key end, uint32_t blockSize) { - ASSERT(blockSize > 3 * sizeof(uint32_t) + begin.size() + end.size()); + ASSERT_GT(blockSize, 3 * sizeof(uint32_t) + begin.size() + end.size()); uint32_t fileVersion = BACKUP_AGENT_SNAPSHOT_FILE_VERSION; // write Header @@ -1531,23 +1556,11 @@ ACTOR static Future testWriteSnapshotFile(Reference file, Key return Void(); } -ACTOR Future createTestEncryptionKeyFile(std::string filename) { - state Reference keyFile = wait(IAsyncFileSystem::filesystem()->open( - filename, - IAsyncFile::OPEN_ATOMIC_WRITE_AND_CREATE | IAsyncFile::OPEN_READWRITE | IAsyncFile::OPEN_CREATE, - 0600)); - std::array testKey; - generateRandomData(testKey.data(), testKey.size()); - keyFile->write(testKey.data(), testKey.size(), 0); - wait(keyFile->sync()); - return Void(); -} - ACTOR Future testBackupContainer(std::string url, Optional encryptionKeyFileName) { state FlowLock lock(100e6); if (encryptionKeyFileName.present()) { - wait(createTestEncryptionKeyFile(encryptionKeyFileName.get())); + wait(BackupContainerFileSystem::createTestEncryptionKeyFile(encryptionKeyFileName.get())); } printf("BackupContainerTest URL %s\n", url.c_str()); @@ -1638,9 +1651,9 @@ ACTOR Future testBackupContainer(std::string url, Optional en wait(waitForAll(writes)); state BackupFileList listing = wait(c->dumpFileList()); - ASSERT(listing.ranges.size() == nRangeFiles); - ASSERT(listing.logs.size() == logs.size()); - ASSERT(listing.snapshots.size() == snapshots.size()); + ASSERT_EQ(listing.ranges.size(), nRangeFiles); + ASSERT_EQ(listing.logs.size(), logs.size()); + ASSERT_EQ(listing.snapshots.size(), snapshots.size()); state BackupDescription desc = wait(c->describeBackup()); printf("\n%s\n", desc.toString().c_str()); @@ -1670,8 +1683,8 @@ ACTOR Future testBackupContainer(std::string url, Optional en // If there is an error, it must be backup_cannot_expire and we have to be on the last snapshot if (f.isError()) { - ASSERT(f.getError().code() == error_code_backup_cannot_expire); - ASSERT(i == listing.snapshots.size() - 1); + ASSERT_EQ(f.getError().code(), error_code_backup_cannot_expire); + ASSERT_EQ(i, listing.snapshots.size() - 1); wait(c->expireData(expireVersion, true)); } @@ -1687,9 +1700,9 @@ ACTOR Future testBackupContainer(std::string url, Optional en ASSERT(d.isError() && d.getError().code() == error_code_backup_does_not_exist); BackupFileList empty = wait(c->dumpFileList()); - ASSERT(empty.ranges.size() == 0); - ASSERT(empty.logs.size() == 0); - ASSERT(empty.snapshots.size() == 0); + ASSERT_EQ(empty.ranges.size(), 0); + ASSERT_EQ(empty.logs.size(), 0); + ASSERT_EQ(empty.snapshots.size(), 0); printf("BackupContainerTest URL=%s PASSED.\n", url.c_str()); diff --git a/fdbclient/BackupContainerFileSystem.h b/fdbclient/BackupContainerFileSystem.h index 8c0967e701..6acf2d87a7 100644 --- a/fdbclient/BackupContainerFileSystem.h +++ b/fdbclient/BackupContainerFileSystem.h @@ -153,6 +153,13 @@ public: bool logsOnly, Version beginVersion) final; + static Future createTestEncryptionKeyFile(std::string const& filename); + +protected: + bool usesEncryption() const; + void setEncryptionKey(Optional const& encryptionKeyFileName); + Future encryptionSetupComplete() const; + private: struct VersionProperty { VersionProperty(Reference bc, const std::string& name) @@ -187,12 +194,6 @@ private: friend class BackupContainerFileSystemImpl; -protected: - bool usesEncryption() const; - void setEncryptionKey(Optional const& encryptionKeyFileName); - Future encryptionSetupComplete() const; - -private: Future encryptionSetupFuture; }; diff --git a/fdbrpc/AsyncFileEncrypted.actor.cpp b/fdbrpc/AsyncFileEncrypted.actor.cpp index ffecfae3c1..6e33243df3 100644 --- a/fdbrpc/AsyncFileEncrypted.actor.cpp +++ b/fdbrpc/AsyncFileEncrypted.actor.cpp @@ -102,7 +102,7 @@ public: if (self->offsetInBlock == FLOW_KNOBS->ENCRYPTION_BLOCK_SIZE) { wait(self->writeLastBlockToFile()); self->offsetInBlock = 0; - ASSERT(self->currentBlock < std::numeric_limits::max()); + ASSERT_LT(self->currentBlock, std::numeric_limits::max()); ++self->currentBlock; self->encryptor = std::make_unique(StreamCipher::Key::getKey(), self->getIV(self->currentBlock)); @@ -205,7 +205,7 @@ Future AsyncFileEncrypted::writeLastBlockToFile() { } size_t AsyncFileEncrypted::RandomCache::evict() { - ASSERT(vec.size() == maxSize); + ASSERT_EQ(vec.size(), maxSize); auto index = deterministicRandom()->randomInt(0, maxSize); hashMap.erase(vec[index]); return index; @@ -263,7 +263,7 @@ TEST_CASE("fdbrpc/AsyncFileEncrypted") { while (bytesRead < bytes) { chunkSize = std::min(deterministicRandom()->randomInt(0, 100), bytes - bytesRead); int bytesReadInChunk = wait(file->read(&readBuffer[bytesRead], chunkSize, bytesRead)); - ASSERT(bytesReadInChunk == chunkSize); + ASSERT_EQ(bytesReadInChunk, chunkSize); bytesRead += bytesReadInChunk; } ASSERT(writeBuffer == readBuffer); diff --git a/fdbrpc/AsyncFileEncrypted.h b/fdbrpc/AsyncFileEncrypted.h index 6be26c8793..af123f2650 100644 --- a/fdbrpc/AsyncFileEncrypted.h +++ b/fdbrpc/AsyncFileEncrypted.h @@ -60,7 +60,6 @@ class AsyncFileEncrypted : public IAsyncFile, public ReferenceCounted initialize(); public: - // TODO: Remove boolean parameter here: AsyncFileEncrypted(Reference, bool canWrite); void addref() override; void delref() override; diff --git a/fdbserver/workloads/BackupCorrectness.actor.cpp b/fdbserver/workloads/BackupCorrectness.actor.cpp index abd3609325..070db26050 100644 --- a/fdbserver/workloads/BackupCorrectness.actor.cpp +++ b/fdbserver/workloads/BackupCorrectness.actor.cpp @@ -21,6 +21,7 @@ #include "fdbrpc/simulator.h" #include "fdbclient/BackupAgent.actor.h" #include "fdbclient/BackupContainer.h" +#include "fdbclient/BackupContainerFileSystem.h" #include "fdbserver/workloads/workloads.actor.h" #include "fdbserver/workloads/BulkSetup.actor.h" #include "flow/actorcompiler.h" // This must be the last #include. @@ -41,35 +42,39 @@ struct BackupAndRestoreCorrectnessWorkload : TestWorkload { bool allowPauses; bool shareLogRange; bool shouldSkipRestoreRanges; + Optional encryptionKeyFileName; BackupAndRestoreCorrectnessWorkload(WorkloadContext const& wcx) : TestWorkload(wcx) { locked = sharedRandomNumber % 2; - backupAfter = getOption(options, LiteralStringRef("backupAfter"), 10.0); - restoreAfter = getOption(options, LiteralStringRef("restoreAfter"), 35.0); - performRestore = getOption(options, LiteralStringRef("performRestore"), true); - backupTag = getOption(options, LiteralStringRef("backupTag"), BackupAgentBase::getDefaultTag()); - backupRangesCount = getOption(options, LiteralStringRef("backupRangesCount"), 5); - backupRangeLengthMax = getOption(options, LiteralStringRef("backupRangeLengthMax"), 1); + backupAfter = getOption(options, "backupAfter"_sr, 10.0); + restoreAfter = getOption(options, "restoreAfter"_sr, 35.0); + performRestore = getOption(options, "performRestore"_sr, true); + backupTag = getOption(options, "backupTag"_sr, BackupAgentBase::getDefaultTag()); + backupRangesCount = getOption(options, "backupRangesCount"_sr, 5); + backupRangeLengthMax = getOption(options, "backupRangeLengthMax"_sr, 1); abortAndRestartAfter = getOption(options, - LiteralStringRef("abortAndRestartAfter"), + "abortAndRestartAfter"_sr, deterministicRandom()->random01() < 0.5 ? deterministicRandom()->random01() * (restoreAfter - backupAfter) + backupAfter : 0.0); - differentialBackup = getOption( - options, LiteralStringRef("differentialBackup"), deterministicRandom()->random01() < 0.5 ? true : false); + differentialBackup = + getOption(options, "differentialBackup"_sr, deterministicRandom()->random01() < 0.5 ? true : false); stopDifferentialAfter = getOption(options, - LiteralStringRef("stopDifferentialAfter"), + "stopDifferentialAfter"_sr, differentialBackup ? deterministicRandom()->random01() * (restoreAfter - std::max(abortAndRestartAfter, backupAfter)) + std::max(abortAndRestartAfter, backupAfter) : 0.0); - agentRequest = getOption(options, LiteralStringRef("simBackupAgents"), true); - allowPauses = getOption(options, LiteralStringRef("allowPauses"), true); - shareLogRange = getOption(options, LiteralStringRef("shareLogRange"), false); - restorePrefixesToInclude = getOption(options, LiteralStringRef("restorePrefixesToInclude"), std::vector()); + agentRequest = getOption(options, "simBackupAgents"_sr, true); + allowPauses = getOption(options, "allowPauses"_sr, true); + shareLogRange = getOption(options, "shareLogRange"_sr, false); + restorePrefixesToInclude = getOption(options, "restorePrefixesToInclude"_sr, std::vector()); shouldSkipRestoreRanges = deterministicRandom()->random01() < 0.3 ? true : false; + if (getOption(options, "encrypted"_sr, false)) { + encryptionKeyFileName = "simfdb/test_encryption_key_file"; + } TraceEvent("BARW_ClientId").detail("Id", wcx.clientId); UID randomID = nondeterministicRandom()->randomUniqueID(); @@ -77,11 +82,10 @@ struct BackupAndRestoreCorrectnessWorkload : TestWorkload { if (shareLogRange) { bool beforePrefix = sharedRandomNumber & 1; if (beforePrefix) - backupRanges.push_back_deep(backupRanges.arena(), - KeyRangeRef(normalKeys.begin, LiteralStringRef("\xfe\xff\xfe"))); + backupRanges.push_back_deep(backupRanges.arena(), KeyRangeRef(normalKeys.begin, "\xfe\xff\xfe"_sr)); else backupRanges.push_back_deep(backupRanges.arena(), - KeyRangeRef(strinc(LiteralStringRef("\x00\x00\x01")), normalKeys.end)); + KeyRangeRef(strinc("\x00\x00\x01"_sr), normalKeys.end)); } else if (backupRangesCount <= 0) { backupRanges.push_back_deep(backupRanges.arena(), normalKeys); } else { @@ -265,7 +269,10 @@ struct BackupAndRestoreCorrectnessWorkload : TestWorkload { deterministicRandom()->randomInt(0, 100), tag.toString(), backupRanges, - stopDifferentialDelay ? false : true)); + stopDifferentialDelay ? false : true, + false, + false, + self->encryptionKeyFileName)); } catch (Error& e) { TraceEvent("BARW_DoBackupSubmitBackupException", randomID).error(e).detail("Tag", printable(tag)); if (e.code() != error_code_backup_unneeded && e.code() != error_code_backup_duplicate) @@ -456,6 +463,10 @@ struct BackupAndRestoreCorrectnessWorkload : TestWorkload { BackupAndRestoreCorrectnessWorkload::backupAgentRequests++; } + if (self->encryptionKeyFileName.present()) { + wait(BackupContainerFileSystem::createTestEncryptionKeyFile(self->encryptionKeyFileName.get())); + } + try { state Future startRestore = delay(self->restoreAfter); @@ -510,7 +521,7 @@ struct BackupAndRestoreCorrectnessWorkload : TestWorkload { TraceEvent("BARW_SubmitBackup2", randomID).detail("Tag", printable(self->backupTag)); try { extraBackup = backupAgent.submitBackup(cx, - LiteralStringRef("file://simfdb/backups/"), + "file://simfdb/backups/"_sr, deterministicRandom()->randomInt(0, 60), deterministicRandom()->randomInt(0, 100), self->backupTag.toString(), @@ -587,7 +598,11 @@ struct BackupAndRestoreCorrectnessWorkload : TestWorkload { range, Key(), Key(), - self->locked)); + self->locked, + false, + false, + ::invalidVersion, + self->encryptionKeyFileName)); } } else { multipleRangesInOneTag = true; @@ -606,7 +621,11 @@ struct BackupAndRestoreCorrectnessWorkload : TestWorkload { true, Key(), Key(), - self->locked)); + self->locked, + false, + false, + ::invalidVersion, + self->encryptionKeyFileName)); } // Sometimes kill and restart the restore @@ -632,7 +651,11 @@ struct BackupAndRestoreCorrectnessWorkload : TestWorkload { true, Key(), Key(), - self->locked); + self->locked, + false, + false, + ::invalidVersion, + self->encryptionKeyFileName); } } else { for (restoreIndex = 0; restoreIndex < restores.size(); restoreIndex++) { @@ -657,7 +680,11 @@ struct BackupAndRestoreCorrectnessWorkload : TestWorkload { self->restoreRanges[restoreIndex], Key(), Key(), - self->locked); + self->locked, + false, + false, + ::invalidVersion, + self->encryptionKeyFileName); } } } @@ -721,7 +748,7 @@ struct BackupAndRestoreCorrectnessWorkload : TestWorkload { .detail("TaskCount", taskCount) .detail("WaitCycles", waitCycles); printf("EndingNonZeroTasks: %ld\n", (long)taskCount); - wait(TaskBucket::debugPrintRange(cx, LiteralStringRef("\xff"), StringRef())); + wait(TaskBucket::debugPrintRange(cx, normalKeys.end, StringRef())); } loop { @@ -820,7 +847,7 @@ struct BackupAndRestoreCorrectnessWorkload : TestWorkload { } if (displaySystemKeys) { - wait(TaskBucket::debugPrintRange(cx, LiteralStringRef("\xff"), StringRef())); + wait(TaskBucket::debugPrintRange(cx, normalKeys.end, StringRef())); } TraceEvent("BARW_Complete", randomID).detail("BackupTag", printable(self->backupTag)); diff --git a/flow/StreamCipher.cpp b/flow/StreamCipher.cpp index e0066ae4a5..7a2c896564 100644 --- a/flow/StreamCipher.cpp +++ b/flow/StreamCipher.cpp @@ -44,7 +44,7 @@ void StreamCipher::cleanup() noexcept { } } -void StreamCipher::Key::initializeKey(std::array&& arr) { +void StreamCipher::Key::initializeKey(RawKeyType&& arr) { ASSERT(!globalKey); globalKey = std::make_unique(ConstructorTag{}); globalKey->arr = std::move(arr); @@ -180,7 +180,7 @@ TEST_CASE("flow/StreamCipher") { } const auto decrypted = decryptor.finish(arena); std::copy(decrypted.begin(), decrypted.end(), &decryptedtext[decryptedOffset]); - ASSERT(decryptedOffset + decrypted.size() == plaintext.size()); + ASSERT_EQ(decryptedOffset + decrypted.size(), plaintext.size()); decryptedtext.resize(decryptedOffset + decrypted.size()); } diff --git a/flow/StreamCipher.h b/flow/StreamCipher.h index f91dc272ce..57c2e0e436 100644 --- a/flow/StreamCipher.h +++ b/flow/StreamCipher.h @@ -46,12 +46,13 @@ public: struct ConstructorTag {}; public: + using RawKeyType = decltype(arr); Key(ConstructorTag) {} Key(Key&&); Key& operator=(Key&&); ~Key(); unsigned char const* data() const { return arr.data(); } - static void initializeKey(decltype(arr)&&); + static void initializeKey(RawKeyType&&); static void initializeRandomTestKey(); static const Key& getKey(); static void cleanup() noexcept;