apple/foundationdb
1
0
Fork 0
mirror of https://github.com/apple/foundationdb.git synced 2025-05-14 18:02:31 +08:00
Code Issues Packages Projects Releases Wiki Activity

Add support for encryption testing in BackupCorrectness workload

Browse Source
This commit is contained in:
sfc-gh-tclinkenbeard 2021-06-26 11:15:12 -07:00
parent c5b612510d
commit 27e44c1bb9
9 changed files with 116 additions and 76 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

7
fdbbackup/backup.actor.cpp
View File

@ -2490,9 +2490,8 @@ ACTOR Future<Void> runFastRestoreTool(Database db,
ACTOR Future<Void> dumpBackupData(const char* name,
std::string destinationContainer,
Version beginVersion,
Version endVersion,
Optional<std::string> encryptionKeyFile) {
state Reference<IBackupContainer> c = openBackupContainer(name, destinationContainer, encryptionKeyFile);
Version endVersion) {
state Reference<IBackupContainer> c = openBackupContainer(name, destinationContainer);
if (beginVersion < 0 || endVersion < 0) {
BackupDescription desc = wait(c->describeBackup());
@ -3994,7 +3993,7 @@ int main(int argc, char* argv[]) {
case BackupType::DUMP:
initTraceFile();
f = stopAfter(dumpBackupData(argv[0], destinationContainer, dumpBegin, dumpEnd, encryptionKeyFile));
f = stopAfter(dumpBackupData(argv[0], destinationContainer, dumpBegin, dumpEnd));
break;
case BackupType::UNDEFINED:

2
fdbclient/AsyncTaskThread.actor.cpp
View File

@ -83,6 +83,6 @@ TEST_CASE("/asynctaskthread/add") {
clients.push_back(asyncTaskThreadClient(&asyncTaskThread, &sum, 100));
}
wait(waitForAll(clients));
ASSERT(sum == 1000);
ASSERT_EQ(sum, 1000);
return Void();
}

79
fdbclient/BackupContainerFileSystem.actor.cpp
View File

@ -291,13 +291,13 @@ public:
std::map<int, std::vector<int>> tagIndices; // tagId -> indices in files
for (int i = 0; i < logs.size(); i++) {
ASSERT(logs[i].tagId >= 0);
ASSERT(logs[i].tagId < logs[i].totalTags);
ASSERT_GE(logs[i].tagId, 0);
ASSERT_LT(logs[i].tagId, logs[i].totalTags);
auto& indices = tagIndices[logs[i].tagId];
// filter out if indices.back() is subset of files[i] or vice versa
if (!indices.empty()) {
if (logs[indices.back()].isSubset(logs[i])) {
ASSERT(logs[indices.back()].fileSize <= logs[i].fileSize);
ASSERT_LE(logs[indices.back()].fileSize, logs[i].fileSize);
indices.back() = i;
} else if (!logs[i].isSubset(logs[indices.back()])) {
indices.push_back(i);
@ -865,7 +865,7 @@ public:
int i = 0;
for (int j = 1; j < logs.size(); j++) {
if (logs[j].isSubset(logs[i])) {
ASSERT(logs[j].fileSize <= logs[i].fileSize);
ASSERT_LE(logs[j].fileSize, logs[i].fileSize);
continue;
}
@ -1033,10 +1033,10 @@ public:
}
static std::string versionFolderString(Version v, int smallestBucket) {
ASSERT(smallestBucket < 14);
ASSERT_LT(smallestBucket, 14);
// Get a 0-padded fixed size representation of v
std::string vFixedPrecision = format("%019lld", v);
ASSERT(vFixedPrecision.size() == 19);
ASSERT_EQ(vFixedPrecision.size(), 19);
// Truncate smallestBucket from the fixed length representation
vFixedPrecision.resize(vFixedPrecision.size() - smallestBucket);
@ -1127,12 +1127,37 @@ public:
return false;
}
ACTOR static Future<Void> createTestEncryptionKeyFile(std::string filename) {
state Reference<IAsyncFile> keyFile = wait(IAsyncFileSystem::filesystem()->open(
filename,
IAsyncFile::OPEN_ATOMIC_WRITE_AND_CREATE | IAsyncFile::OPEN_READWRITE | IAsyncFile::OPEN_CREATE,
0600));
StreamCipher::Key::RawKeyType testKey;
generateRandomData(testKey.data(), testKey.size());
keyFile->write(testKey.data(), testKey.size(), 0);
wait(keyFile->sync());
return Void();
}
ACTOR static Future<Void> readEncryptionKey(std::string encryptionKeyFileName) {
state Reference<IAsyncFile> keyFile =
wait(IAsyncFileSystem::filesystem()->open(encryptionKeyFileName, 0x0, 0400));
state std::array<uint8_t, 16> key;
state Reference<IAsyncFile> keyFile;
try {
Reference<IAsyncFile> _keyFile =
wait(IAsyncFileSystem::filesystem()->open(encryptionKeyFileName, 0x0, 0400));
} catch (Error& e) {
TraceEvent(SevWarnAlways, "FailedToOpenEncryptionKeyFile")
.detail("FileName", encryptionKeyFileName)
.error(e);
throw e;
}
state StreamCipher::Key::RawKeyType key;
int bytesRead = wait(keyFile->read(key.data(), key.size(), 0));
// TODO: Throw new error (fail gracefully)
if (bytesRead != key.size()) {
TraceEvent(SevWarnAlways, "InvalidEncryptionKeyFileSize")
.detail("ExpectedSize", key.size())
.detail("ActualSize", bytesRead);
throw invalid_encryption_key_file();
}
ASSERT_EQ(bytesRead, key.size());
StreamCipher::Key::initializeKey(std::move(key));
return Void();
@ -1496,7 +1521,7 @@ ACTOR Future<Void> writeAndVerifyFile(Reference<IBackupContainer> c, Reference<I
state Standalone<VectorRef<uint8_t>> buf;
buf.resize(buf.arena(), fileSize);
int b = wait(inputFile->read(buf.begin(), buf.size(), 0));
ASSERT(b == buf.size());
ASSERT_EQ(b, buf.size());
ASSERT(buf == content);
}
return Void();
@ -1510,7 +1535,7 @@ Version nextVersion(Version v) {
// Write a snapshot file with only begin & end key
ACTOR static Future<Void> testWriteSnapshotFile(Reference<IBackupFile> file, Key begin, Key end, uint32_t blockSize) {
ASSERT(blockSize > 3 * sizeof(uint32_t) + begin.size() + end.size());
ASSERT_GT(blockSize, 3 * sizeof(uint32_t) + begin.size() + end.size());
uint32_t fileVersion = BACKUP_AGENT_SNAPSHOT_FILE_VERSION;
// write Header
@ -1531,23 +1556,11 @@ ACTOR static Future<Void> testWriteSnapshotFile(Reference<IBackupFile> file, Key
return Void();
}
ACTOR Future<Void> createTestEncryptionKeyFile(std::string filename) {
state Reference<IAsyncFile> keyFile = wait(IAsyncFileSystem::filesystem()->open(
filename,
IAsyncFile::OPEN_ATOMIC_WRITE_AND_CREATE | IAsyncFile::OPEN_READWRITE | IAsyncFile::OPEN_CREATE,
0600));
std::array<uint8_t, 16> testKey;
generateRandomData(testKey.data(), testKey.size());
keyFile->write(testKey.data(), testKey.size(), 0);
wait(keyFile->sync());
return Void();
}
ACTOR Future<Void> testBackupContainer(std::string url, Optional<std::string> encryptionKeyFileName) {
state FlowLock lock(100e6);
if (encryptionKeyFileName.present()) {
wait(createTestEncryptionKeyFile(encryptionKeyFileName.get()));
wait(BackupContainerFileSystem::createTestEncryptionKeyFile(encryptionKeyFileName.get()));
}
printf("BackupContainerTest URL %s\n", url.c_str());
@ -1638,9 +1651,9 @@ ACTOR Future<Void> testBackupContainer(std::string url, Optional<std::string> en
wait(waitForAll(writes));
state BackupFileList listing = wait(c->dumpFileList());
ASSERT(listing.ranges.size() == nRangeFiles);
ASSERT(listing.logs.size() == logs.size());
ASSERT(listing.snapshots.size() == snapshots.size());
ASSERT_EQ(listing.ranges.size(), nRangeFiles);
ASSERT_EQ(listing.logs.size(), logs.size());
ASSERT_EQ(listing.snapshots.size(), snapshots.size());
state BackupDescription desc = wait(c->describeBackup());
printf("\n%s\n", desc.toString().c_str());
@ -1670,8 +1683,8 @@ ACTOR Future<Void> testBackupContainer(std::string url, Optional<std::string> en
// If there is an error, it must be backup_cannot_expire and we have to be on the last snapshot
if (f.isError()) {
ASSERT(f.getError().code() == error_code_backup_cannot_expire);
ASSERT(i == listing.snapshots.size() - 1);
ASSERT_EQ(f.getError().code(), error_code_backup_cannot_expire);
ASSERT_EQ(i, listing.snapshots.size() - 1);
wait(c->expireData(expireVersion, true));
}
@ -1687,9 +1700,9 @@ ACTOR Future<Void> testBackupContainer(std::string url, Optional<std::string> en
ASSERT(d.isError() && d.getError().code() == error_code_backup_does_not_exist);
BackupFileList empty = wait(c->dumpFileList());
ASSERT(empty.ranges.size() == 0);
ASSERT(empty.logs.size() == 0);
ASSERT(empty.snapshots.size() == 0);
ASSERT_EQ(empty.ranges.size(), 0);
ASSERT_EQ(empty.logs.size(), 0);
ASSERT_EQ(empty.snapshots.size(), 0);
printf("BackupContainerTest URL=%s PASSED.\n", url.c_str());

13
fdbclient/BackupContainerFileSystem.h
View File

@ -153,6 +153,13 @@ public:
bool logsOnly,
Version beginVersion) final;
static Future<Void> createTestEncryptionKeyFile(std::string const& filename);
protected:
bool usesEncryption() const;
void setEncryptionKey(Optional<std::string> const& encryptionKeyFileName);
Future<Void> encryptionSetupComplete() const;
private:
struct VersionProperty {
VersionProperty(Reference<BackupContainerFileSystem> bc, const std::string& name)
@ -187,12 +194,6 @@ private:
friend class BackupContainerFileSystemImpl;
protected:
bool usesEncryption() const;
void setEncryptionKey(Optional<std::string> const& encryptionKeyFileName);
Future<Void> encryptionSetupComplete() const;
private:
Future<Void> encryptionSetupFuture;
};

6
fdbrpc/AsyncFileEncrypted.actor.cpp
View File

@ -102,7 +102,7 @@ public:
if (self->offsetInBlock == FLOW_KNOBS->ENCRYPTION_BLOCK_SIZE) {
wait(self->writeLastBlockToFile());
self->offsetInBlock = 0;
ASSERT(self->currentBlock < std::numeric_limits<uint16_t>::max());
ASSERT_LT(self->currentBlock, std::numeric_limits<uint16_t>::max());
++self->currentBlock;
self->encryptor = std::make_unique<EncryptionStreamCipher>(StreamCipher::Key::getKey(),
self->getIV(self->currentBlock));
@ -205,7 +205,7 @@ Future<Void> AsyncFileEncrypted::writeLastBlockToFile() {
}
size_t AsyncFileEncrypted::RandomCache::evict() {
ASSERT(vec.size() == maxSize);
ASSERT_EQ(vec.size(), maxSize);
auto index = deterministicRandom()->randomInt(0, maxSize);
hashMap.erase(vec[index]);
return index;
@ -263,7 +263,7 @@ TEST_CASE("fdbrpc/AsyncFileEncrypted") {
while (bytesRead < bytes) {
chunkSize = std::min(deterministicRandom()->randomInt(0, 100), bytes - bytesRead);
int bytesReadInChunk = wait(file->read(&readBuffer[bytesRead], chunkSize, bytesRead));
ASSERT(bytesReadInChunk == chunkSize);
ASSERT_EQ(bytesReadInChunk, chunkSize);
bytesRead += bytesReadInChunk;
}
ASSERT(writeBuffer == readBuffer);

1
fdbrpc/AsyncFileEncrypted.h
View File

@ -60,7 +60,6 @@ class AsyncFileEncrypted : public IAsyncFile, public ReferenceCounted<AsyncFileE
Future<Void> initialize();
public:
// TODO: Remove boolean parameter here:
AsyncFileEncrypted(Reference<IAsyncFile>, bool canWrite);
void addref() override;
void delref() override;

77
fdbserver/workloads/BackupCorrectness.actor.cpp
View File

@ -21,6 +21,7 @@
#include "fdbrpc/simulator.h"
#include "fdbclient/BackupAgent.actor.h"
#include "fdbclient/BackupContainer.h"
#include "fdbclient/BackupContainerFileSystem.h"
#include "fdbserver/workloads/workloads.actor.h"
#include "fdbserver/workloads/BulkSetup.actor.h"
#include "flow/actorcompiler.h" // This must be the last #include.
@ -41,35 +42,39 @@ struct BackupAndRestoreCorrectnessWorkload : TestWorkload {
bool allowPauses;
bool shareLogRange;
bool shouldSkipRestoreRanges;
Optional<std::string> encryptionKeyFileName;
BackupAndRestoreCorrectnessWorkload(WorkloadContext const& wcx) : TestWorkload(wcx) {
locked = sharedRandomNumber % 2;
backupAfter = getOption(options, LiteralStringRef("backupAfter"), 10.0);
restoreAfter = getOption(options, LiteralStringRef("restoreAfter"), 35.0);
performRestore = getOption(options, LiteralStringRef("performRestore"), true);
backupTag = getOption(options, LiteralStringRef("backupTag"), BackupAgentBase::getDefaultTag());
backupRangesCount = getOption(options, LiteralStringRef("backupRangesCount"), 5);
backupRangeLengthMax = getOption(options, LiteralStringRef("backupRangeLengthMax"), 1);
backupAfter = getOption(options, "backupAfter"_sr, 10.0);
restoreAfter = getOption(options, "restoreAfter"_sr, 35.0);
performRestore = getOption(options, "performRestore"_sr, true);
backupTag = getOption(options, "backupTag"_sr, BackupAgentBase::getDefaultTag());
backupRangesCount = getOption(options, "backupRangesCount"_sr, 5);
backupRangeLengthMax = getOption(options, "backupRangeLengthMax"_sr, 1);
abortAndRestartAfter =
getOption(options,
LiteralStringRef("abortAndRestartAfter"),
"abortAndRestartAfter"_sr,
deterministicRandom()->random01() < 0.5
? deterministicRandom()->random01() * (restoreAfter - backupAfter) + backupAfter
: 0.0);
differentialBackup = getOption(
options, LiteralStringRef("differentialBackup"), deterministicRandom()->random01() < 0.5 ? true : false);
differentialBackup =
getOption(options, "differentialBackup"_sr, deterministicRandom()->random01() < 0.5 ? true : false);
stopDifferentialAfter =
getOption(options,
LiteralStringRef("stopDifferentialAfter"),
"stopDifferentialAfter"_sr,
differentialBackup ? deterministicRandom()->random01() *
(restoreAfter - std::max(abortAndRestartAfter, backupAfter)) +
std::max(abortAndRestartAfter, backupAfter)
: 0.0);
agentRequest = getOption(options, LiteralStringRef("simBackupAgents"), true);
allowPauses = getOption(options, LiteralStringRef("allowPauses"), true);
shareLogRange = getOption(options, LiteralStringRef("shareLogRange"), false);
restorePrefixesToInclude = getOption(options, LiteralStringRef("restorePrefixesToInclude"), std::vector<std::string>());
agentRequest = getOption(options, "simBackupAgents"_sr, true);
allowPauses = getOption(options, "allowPauses"_sr, true);
shareLogRange = getOption(options, "shareLogRange"_sr, false);
restorePrefixesToInclude = getOption(options, "restorePrefixesToInclude"_sr, std::vector<std::string>());
shouldSkipRestoreRanges = deterministicRandom()->random01() < 0.3 ? true : false;
if (getOption(options, "encrypted"_sr, false)) {
encryptionKeyFileName = "simfdb/test_encryption_key_file";
}
TraceEvent("BARW_ClientId").detail("Id", wcx.clientId);
UID randomID = nondeterministicRandom()->randomUniqueID();
@ -77,11 +82,10 @@ struct BackupAndRestoreCorrectnessWorkload : TestWorkload {
if (shareLogRange) {
bool beforePrefix = sharedRandomNumber & 1;
if (beforePrefix)
backupRanges.push_back_deep(backupRanges.arena(),
KeyRangeRef(normalKeys.begin, LiteralStringRef("\xfe\xff\xfe")));
backupRanges.push_back_deep(backupRanges.arena(), KeyRangeRef(normalKeys.begin, "\xfe\xff\xfe"_sr));
else
backupRanges.push_back_deep(backupRanges.arena(),
KeyRangeRef(strinc(LiteralStringRef("\x00\x00\x01")), normalKeys.end));
KeyRangeRef(strinc("\x00\x00\x01"_sr), normalKeys.end));
} else if (backupRangesCount <= 0) {
backupRanges.push_back_deep(backupRanges.arena(), normalKeys);
} else {
@ -265,7 +269,10 @@ struct BackupAndRestoreCorrectnessWorkload : TestWorkload {
deterministicRandom()->randomInt(0, 100),
tag.toString(),
backupRanges,
stopDifferentialDelay ? false : true));
stopDifferentialDelay ? false : true,
false,
false,
self->encryptionKeyFileName));
} catch (Error& e) {
TraceEvent("BARW_DoBackupSubmitBackupException", randomID).error(e).detail("Tag", printable(tag));
if (e.code() != error_code_backup_unneeded && e.code() != error_code_backup_duplicate)
@ -456,6 +463,10 @@ struct BackupAndRestoreCorrectnessWorkload : TestWorkload {
BackupAndRestoreCorrectnessWorkload::backupAgentRequests++;
}
if (self->encryptionKeyFileName.present()) {
wait(BackupContainerFileSystem::createTestEncryptionKeyFile(self->encryptionKeyFileName.get()));
}
try {
state Future<Void> startRestore = delay(self->restoreAfter);
@ -510,7 +521,7 @@ struct BackupAndRestoreCorrectnessWorkload : TestWorkload {
TraceEvent("BARW_SubmitBackup2", randomID).detail("Tag", printable(self->backupTag));
try {
extraBackup = backupAgent.submitBackup(cx,
LiteralStringRef("file://simfdb/backups/"),
"file://simfdb/backups/"_sr,
deterministicRandom()->randomInt(0, 60),
deterministicRandom()->randomInt(0, 100),
self->backupTag.toString(),
@ -587,7 +598,11 @@ struct BackupAndRestoreCorrectnessWorkload : TestWorkload {
range,
Key(),
Key(),
self->locked));
self->locked,
false,
false,
::invalidVersion,
self->encryptionKeyFileName));
}
} else {
multipleRangesInOneTag = true;
@ -606,7 +621,11 @@ struct BackupAndRestoreCorrectnessWorkload : TestWorkload {
true,
Key(),
Key(),
self->locked));
self->locked,
false,
false,
::invalidVersion,
self->encryptionKeyFileName));
}
// Sometimes kill and restart the restore
@ -632,7 +651,11 @@ struct BackupAndRestoreCorrectnessWorkload : TestWorkload {
true,
Key(),
Key(),
self->locked);
self->locked,
false,
false,
::invalidVersion,
self->encryptionKeyFileName);
}
} else {
for (restoreIndex = 0; restoreIndex < restores.size(); restoreIndex++) {
@ -657,7 +680,11 @@ struct BackupAndRestoreCorrectnessWorkload : TestWorkload {
self->restoreRanges[restoreIndex],
Key(),
Key(),
self->locked);
self->locked,
false,
false,
::invalidVersion,
self->encryptionKeyFileName);
}
}
}
@ -721,7 +748,7 @@ struct BackupAndRestoreCorrectnessWorkload : TestWorkload {
.detail("TaskCount", taskCount)
.detail("WaitCycles", waitCycles);
printf("EndingNonZeroTasks: %ld\n", (long)taskCount);
wait(TaskBucket::debugPrintRange(cx, LiteralStringRef("\xff"), StringRef()));
wait(TaskBucket::debugPrintRange(cx, normalKeys.end, StringRef()));
}
loop {
@ -820,7 +847,7 @@ struct BackupAndRestoreCorrectnessWorkload : TestWorkload {
}
if (displaySystemKeys) {
wait(TaskBucket::debugPrintRange(cx, LiteralStringRef("\xff"), StringRef()));
wait(TaskBucket::debugPrintRange(cx, normalKeys.end, StringRef()));
}
TraceEvent("BARW_Complete", randomID).detail("BackupTag", printable(self->backupTag));

4
flow/StreamCipher.cpp
View File

@ -44,7 +44,7 @@ void StreamCipher::cleanup() noexcept {
}
}
void StreamCipher::Key::initializeKey(std::array<unsigned char, 16>&& arr) {
void StreamCipher::Key::initializeKey(RawKeyType&& arr) {
ASSERT(!globalKey);
globalKey = std::make_unique<Key>(ConstructorTag{});
globalKey->arr = std::move(arr);
@ -180,7 +180,7 @@ TEST_CASE("flow/StreamCipher") {
}
const auto decrypted = decryptor.finish(arena);
std::copy(decrypted.begin(), decrypted.end(), &decryptedtext[decryptedOffset]);
ASSERT(decryptedOffset + decrypted.size() == plaintext.size());
ASSERT_EQ(decryptedOffset + decrypted.size(), plaintext.size());
decryptedtext.resize(decryptedOffset + decrypted.size());
}

3
flow/StreamCipher.h
View File

@ -46,12 +46,13 @@ public:
struct ConstructorTag {};
public:
using RawKeyType = decltype(arr);
Key(ConstructorTag) {}
Key(Key&&);
Key& operator=(Key&&);
~Key();
unsigned char const* data() const { return arr.data(); }
static void initializeKey(decltype(arr)&&);
static void initializeKey(RawKeyType&&);
static void initializeRandomTestKey();
static const Key& getKey();
static void cleanup() noexcept;