apple/foundationdb
1
0
Fork 0
mirror of https://github.com/apple/foundationdb.git synced 2025-05-14 01:42:37 +08:00
Code Issues Packages Projects Releases Wiki Activity

Support encryption for blob store backups (not yet tested)

Browse Source
This commit is contained in:
sfc-gh-tclinkenbeard 2021-06-26 00:07:27 -07:00
parent d88f8f7653
commit 3d6515bd14
11 changed files with 94 additions and 51 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
fdbclient/AsyncFileS3BlobStore.actor.h
View File

@ -256,7 +256,7 @@ public:
m_concurrentUploads(bstore->knobs.concurrent_writes_per_file) {
// Add first part
m_parts.push_back(Reference<Part>(new Part(1, m_bstore->knobs.multipart_min_part_size)));
m_parts.push_back(makeReference<Part>(1, m_bstore->knobs.multipart_min_part_size));
}
};

8
fdbclient/BackupContainer.actor.cpp
View File

@ -58,6 +58,7 @@ ACTOR Future<Void> appendStringRefWithLen(Reference<IBackupFile> file, Standalon
wait(file->append(s.begin(), s.size()));
return Void();
}
} // namespace IBackupFile_impl
Future<Void> IBackupFile::appendStringRefWithLen(Standalone<StringRef> s) {
@ -278,7 +279,7 @@ Reference<IBackupContainer> IBackupContainer::openContainer(const std::string& u
for (auto c : resource)
if (!isalnum(c) && c != '_' && c != '-' && c != '.' && c != '/')
throw backup_invalid_url();
r = makeReference<BackupContainerS3BlobStore>(bstore, resource, backupParams);
r = makeReference<BackupContainerS3BlobStore>(bstore, resource, backupParams, encryptionKeyFileName);
}
#ifdef BUILD_AZURE_BACKUP
else if (u.startsWith("azure://"_sr)) {
@ -286,7 +287,8 @@ Reference<IBackupContainer> IBackupContainer::openContainer(const std::string& u
auto address = NetworkAddress::parse(u.eat("/"_sr).toString());
auto containerName = u.eat("/"_sr).toString();
auto accountName = u.eat("/"_sr).toString();
r = makeReference<BackupContainerAzureBlobStore>(address, containerName, accountName);
r = makeReference<BackupContainerAzureBlobStore>(
address, containerName, accountName, encryptionKeyFileName);
}
#endif
else {
@ -334,7 +336,7 @@ ACTOR Future<std::vector<std::string>> listContainers_impl(std::string baseURL)
}
// Create a dummy container to parse the backup-specific parameters from the URL and get a final bucket name
BackupContainerS3BlobStore dummy(bstore, "dummy", backupParams);
BackupContainerS3BlobStore dummy(bstore, "dummy", backupParams, {});
std::vector<std::string> results = wait(BackupContainerS3BlobStore::listURLs(bstore, dummy.getBucket()));
return results;

39
fdbclient/BackupContainerAzureBlobStore.actor.cpp
View File

@ -19,6 +19,7 @@
*/
#include "fdbclient/BackupContainerAzureBlobStore.h"
#include "fdbrpc/AsyncFileEncrypted.h"
#include "flow/actorcompiler.h" // This must be the last #include.
@ -167,8 +168,13 @@ public:
if (!exists) {
throw file_not_found();
}
return Reference<IAsyncFile>(
new ReadFile(self->asyncTaskThread, self->containerName, fileName, self->client.get()));
Reference<IAsyncFile> f =
makeReference<ReadFile>(self->asyncTaskThread, self->containerName, fileName, self->client.get());
if (self->usesEncryption()) {
f = makeReference<AsyncFileEncrypted>(f, false);
}
f = makeReference<ReadFile>(self->asyncTaskThread, self->containerName, fileName, self->client.get());
return f;
}
ACTOR static Future<Reference<IBackupFile>> writeFile(BackupContainerAzureBlobStore* self, std::string fileName) {
@ -177,10 +183,11 @@ public:
auto outcome = client->create_append_blob(containerName, fileName).get();
return Void();
}));
return Reference<IBackupFile>(
new BackupFile(fileName,
Reference<IAsyncFile>(new WriteFile(
self->asyncTaskThread, self->containerName, fileName, self->client.get()))));
auto f = makeReference<WriteFile>(self->asyncTaskThread, self->containerName, fileName, self->client.get());
if (self->usesEncryption()) {
f = makeReference<AsyncFileEncrypted>(f, true);
}
return makeReference<BackupFile>(fileName, f);
}
static void listFiles(AzureClient* client,
@ -213,6 +220,16 @@ public:
}
return Void();
}
ACTOR static Future<Void> create(BackupContainerAzureBlobStore* self) {
state Future<Void> f1 =
self->asyncTaskThread.execAsync([containerName = self->containerName, client = self->client.get()] {
client->create_container(containerName).wait();
return Void();
});
state Future<Void> f2 = self->usesEncryption() ? self->encryptionSetupComplete() : Void();
return f1 && f2;
}
};
Future<bool> BackupContainerAzureBlobStore::blobExists(const std::string& fileName) {
@ -225,10 +242,11 @@ Future<bool> BackupContainerAzureBlobStore::blobExists(const std::string& fileNa
BackupContainerAzureBlobStore::BackupContainerAzureBlobStore(const NetworkAddress& address,
const std::string& accountName,
const std::string& containerName)
const std::string& containerName,
const Optional<std::string>& encryptionKeyFileName)
: containerName(containerName) {
setEncryptionKey(encryptionKeyFileName);
std::string accountKey = std::getenv("AZURE_KEY");
auto credential = std::make_shared<azure::storage_lite::shared_key_credential>(accountName, accountKey);
auto storageAccount = std::make_shared<azure::storage_lite::storage_account>(
accountName, credential, false, format("http://%s/%s", address.toString().c_str(), accountName.c_str()));
@ -244,10 +262,7 @@ void BackupContainerAzureBlobStore::delref() {
}
Future<Void> BackupContainerAzureBlobStore::create() {
return asyncTaskThread.execAsync([containerName = this->containerName, client = this->client.get()] {
client->create_container(containerName).wait();
return Void();
});
return BackupContainerAzureBlobStoreImpl::create(this);
}
Future<bool> BackupContainerAzureBlobStore::exists() {
return asyncTaskThread.execAsync([containerName = this->containerName, client = this->client.get()] {

3
fdbclient/BackupContainerAzureBlobStore.h
View File

@ -44,7 +44,8 @@ class BackupContainerAzureBlobStore final : public BackupContainerFileSystem,
public:
BackupContainerAzureBlobStore(const NetworkAddress& address,
const std::string& accountName,
const std::string& containerName);
const std::string& containerName,
const Optional<std::string>& encryptionKeyFileName);
void addref() override;
void delref() override;

22
fdbclient/BackupContainerFileSystem.actor.cpp
View File

@ -23,6 +23,7 @@
#include "fdbclient/BackupContainerFileSystem.h"
#include "fdbclient/BackupContainerLocalDirectory.h"
#include "fdbclient/JsonBuilder.h"
#include "flow/StreamCipher.h"
#include "flow/UnitTest.h"
#include <algorithm>
@ -1126,6 +1127,16 @@ public:
return false;
}
ACTOR static Future<Void> readEncryptionKey(std::string encryptionKeyFileName) {
state Reference<IAsyncFile> keyFile =
wait(IAsyncFileSystem::filesystem()->open(encryptionKeyFileName, 0x0, 0400));
state std::array<uint8_t, 16> key;
int bytesRead = wait(keyFile->read(key.data(), key.size(), 0));
// TODO: Throw new error (fail gracefully)
ASSERT_EQ(bytesRead, key.size());
StreamCipher::Key::initializeKey(std::move(key));
return Void();
}
}; // class BackupContainerFileSystemImpl
Future<Reference<IBackupFile>> BackupContainerFileSystem::writeLogFile(Version beginVersion,
@ -1432,6 +1443,17 @@ BackupContainerFileSystem::VersionProperty BackupContainerFileSystem::unreliable
BackupContainerFileSystem::VersionProperty BackupContainerFileSystem::logType() {
return { Reference<BackupContainerFileSystem>::addRef(this), "mutation_log_type" };
}
bool BackupContainerFileSystem::usesEncryption() const {
return encryptionSetupFuture.isValid();
}
Future<Void> BackupContainerFileSystem::encryptionSetupComplete() const {
return encryptionSetupFuture;
}
void BackupContainerFileSystem::setEncryptionKey(Optional<std::string> const& encryptionKeyFileName) {
if (encryptionKeyFileName.present()) {
encryptionSetupFuture = BackupContainerFileSystemImpl::readEncryptionKey(encryptionKeyFileName.get());
}
}
namespace backup_test {

8
fdbclient/BackupContainerFileSystem.h
View File

@ -186,6 +186,14 @@ private:
Future<std::vector<RangeFile>> old_listRangeFiles(Version beginVersion, Version endVersion);
friend class BackupContainerFileSystemImpl;
protected:
bool usesEncryption() const;
void setEncryptionKey(Optional<std::string> const& encryptionKeyFileName);
Future<Void> encryptionSetupComplete() const;
private:
Future<Void> encryptionSetupFuture;
};
#endif

24
fdbclient/BackupContainerLocalDirectory.actor.cpp
View File

@ -23,7 +23,6 @@
#include "fdbrpc/IAsyncFile.h"
#include "flow/Platform.actor.h"
#include "flow/Platform.h"
#include "flow/StreamCipher.h"
#include "fdbrpc/simulator.h"
#include "flow/actorcompiler.h" // This must be the last #include.
@ -132,25 +131,9 @@ std::string BackupContainerLocalDirectory::getURLFormat() {
return "file://</path/to/base/dir/>";
}
ACTOR static Future<Void> readEncryptionKey(std::string encryptionKeyFileName) {
state Reference<IAsyncFile> keyFile = wait(IAsyncFileSystem::filesystem()->open(encryptionKeyFileName, 0x0, 0400));
state std::array<uint8_t, 16> key;
int bytesRead = wait(keyFile->read(key.data(), key.size(), 0));
// TODO: Throw new error (fail gracefully)
ASSERT_EQ(bytesRead, key.size());
StreamCipher::Key::initializeKey(std::move(key));
return Void();
}
bool BackupContainerLocalDirectory::usesEncryption() const {
return encryptionSetupFuture.isValid();
}
BackupContainerLocalDirectory::BackupContainerLocalDirectory(const std::string& url,
const Optional<std::string>& encryptionKeyFileName) {
if (encryptionKeyFileName.present()) {
encryptionSetupFuture = readEncryptionKey(encryptionKeyFileName.get());
}
setEncryptionKey(encryptionKeyFileName);
std::string path;
if (url.find("file://") != 0) {
@ -214,10 +197,9 @@ Future<std::vector<std::string>> BackupContainerLocalDirectory::listURLs(const s
Future<Void> BackupContainerLocalDirectory::create() {
if (usesEncryption()) {
return encryptionSetupFuture;
return encryptionSetupComplete();
}
// TODO: Update this comment:
// Nothing should be done here because create() can be called by any process working with the container URL,
// No directory should be created here because create() can be called by any process working with the container URL,
// such as fdbbackup. Since "local directory" containers are by definition local to the machine they are
// accessed from, the container's creation (in this case the creation of a directory) must be ensured prior to
// every file creation, which is done in openFile(). Creating the directory here will result in unnecessary

2
fdbclient/BackupContainerLocalDirectory.h
View File

@ -54,8 +54,6 @@ public:
private:
std::string m_path;
Future<Void> encryptionSetupFuture;
bool usesEncryption() const;
};
#endif

33
fdbclient/BackupContainerS3BlobStore.actor.cpp
View File

@ -20,6 +20,7 @@
#include "fdbclient/AsyncFileS3BlobStore.actor.h"
#include "fdbclient/BackupContainerS3BlobStore.h"
#include "fdbrpc/AsyncFileEncrypted.h"
#include "fdbrpc/AsyncFileReadAhead.actor.h"
#include "flow/actorcompiler.h" // This must be the last #include.
@ -103,6 +104,10 @@ public:
wait(bc->m_bstore->writeEntireFile(bc->m_bucket, bc->indexEntry(), ""));
}
if (bc->usesEncryption()) {
wait(bc->encryptionSetupComplete());
}
return Void();
}
@ -137,9 +142,10 @@ std::string BackupContainerS3BlobStore::indexEntry() {
BackupContainerS3BlobStore::BackupContainerS3BlobStore(Reference<S3BlobStoreEndpoint> bstore,
const std::string& name,
const S3BlobStoreEndpoint::ParametersT& params)
const S3BlobStoreEndpoint::ParametersT& params,
const Optional<std::string>& encryptionKeyFileName)
: m_bstore(bstore), m_name(name), m_bucket("FDB_BACKUPS_V2") {
setEncryptionKey(encryptionKeyFileName);
// Currently only one parameter is supported, "bucket"
for (const auto& [name, value] : params) {
if (name == "bucket") {
@ -164,12 +170,16 @@ std::string BackupContainerS3BlobStore::getURLFormat() {
}
Future<Reference<IAsyncFile>> BackupContainerS3BlobStore::readFile(const std::string& path) {
return Reference<IAsyncFile>(new AsyncFileReadAheadCache(
Reference<IAsyncFile>(new AsyncFileS3BlobStoreRead(m_bstore, m_bucket, dataPath(path))),
m_bstore->knobs.read_block_size,
m_bstore->knobs.read_ahead_blocks,
m_bstore->knobs.concurrent_reads_per_file,
m_bstore->knobs.read_cache_blocks_per_file));
Reference<IAsyncFile> f = makeReference<AsyncFileS3BlobStoreRead>(m_bstore, m_bucket, dataPath(path));
if (usesEncryption()) {
f = makeReference<AsyncFileEncrypted>(f, false);
}
f = makeReference<AsyncFileReadAheadCache>(f,
m_bstore->knobs.read_block_size,
m_bstore->knobs.read_ahead_blocks,
m_bstore->knobs.concurrent_reads_per_file,
m_bstore->knobs.read_cache_blocks_per_file);
return f;
}
Future<std::vector<std::string>> BackupContainerS3BlobStore::listURLs(Reference<S3BlobStoreEndpoint> bstore,
@ -178,8 +188,11 @@ Future<std::vector<std::string>> BackupContainerS3BlobStore::listURLs(Reference<
}
Future<Reference<IBackupFile>> BackupContainerS3BlobStore::writeFile(const std::string& path) {
return Reference<IBackupFile>(new BackupContainerS3BlobStoreImpl::BackupFile(
path, Reference<IAsyncFile>(new AsyncFileS3BlobStoreWrite(m_bstore, m_bucket, dataPath(path)))));
Reference<IAsyncFile> f = makeReference<AsyncFileS3BlobStoreWrite>(m_bstore, m_bucket, dataPath(path));
if (usesEncryption()) {
f = makeReference<AsyncFileEncrypted>(f, true);
}
return Future<Reference<IBackupFile>>(makeReference<BackupContainerS3BlobStoreImpl::BackupFile>(path, f));
}
Future<Void> BackupContainerS3BlobStore::deleteFile(const std::string& path) {

3
fdbclient/BackupContainerS3BlobStore.h
View File

@ -43,7 +43,8 @@ class BackupContainerS3BlobStore final : public BackupContainerFileSystem,
public:
BackupContainerS3BlobStore(Reference<S3BlobStoreEndpoint> bstore,
const std::string& name,
const S3BlobStoreEndpoint::ParametersT& params);
const S3BlobStoreEndpoint::ParametersT& params,
const Optional<std::string>& encryptionKeyFileName);
void addref() override;
void delref() override;

1
fdbrpc/AsyncFileEncrypted.h
View File

@ -60,6 +60,7 @@ class AsyncFileEncrypted : public IAsyncFile, public ReferenceCounted<AsyncFileE
Future<Void> initialize();
public:
// TODO: Remove boolean parameter here:
AsyncFileEncrypted(Reference<IAsyncFile>, bool canWrite);
void addref() override;
void delref() override;