From 670d40ef79e89f57475ff998c76d1d44e6c44381 Mon Sep 17 00:00:00 2001 From: Ata E Husain Bohra Date: Fri, 22 Apr 2022 08:53:39 -0700 Subject: [PATCH] FDB native KMS Connector Framework (#6846) * FDB native KMS Connector Framework Description Major changes includes: 1. Framework code to enable FDB native KMS connector implementation. 2. SERVER_KNOBS->KMS_CONNECTOR_TYPE controls the connector type selection. 3. KmsConnectorInterface endpoint definitions, every KMSConnector implementation needs to support defined endpoints. 4. Update EncryptKeyProxy to leverage KmsConnectorInterface endpoints to fetch encryption keys on-demand and/or periodic refreshes. Integrate SimKmsConnector implementation. 5. Implement SimKmsConnector by leveraging existing SimKeyProxy implementation. Testing Unit test: fdbserver/SimKmsConnector Simulation: EncryptKeyProxy --- fdbclient/ServerKnobs.cpp | 4 + fdbclient/ServerKnobs.h | 3 + fdbserver/CMakeLists.txt | 6 +- fdbserver/EncryptKeyProxy.actor.cpp | 228 +++++++++--------- fdbserver/KmsConnector.h | 43 ++++ fdbserver/KmsConnectorInterface.h | 144 +++++++++++ ...xy.actor.cpp => SimKmsConnector.actor.cpp} | 116 ++++----- fdbserver/SimKmsConnector.actor.h | 40 +++ fdbserver/workloads/UnitTests.actor.cpp | 4 +- flow/BlobCipher.h | 2 +- 10 files changed, 420 insertions(+), 170 deletions(-) create mode 100644 fdbserver/KmsConnector.h create mode 100644 fdbserver/KmsConnectorInterface.h rename fdbserver/{SimEncryptKmsProxy.actor.cpp => SimKmsConnector.actor.cpp} (60%) create mode 100644 fdbserver/SimKmsConnector.actor.h diff --git a/fdbclient/ServerKnobs.cpp b/fdbclient/ServerKnobs.cpp index 4b9f618d6f..5b90e0b930 100644 --- a/fdbclient/ServerKnobs.cpp +++ b/fdbclient/ServerKnobs.cpp @@ -855,6 +855,10 @@ void ServerKnobs::initialize(Randomize randomize, ClientKnobs* clientKnobs, IsSi init( ENCRYPTION_MODE, "AES-256-CTR"); init( SIM_KMS_MAX_KEYS, 4096); + // Support KmsConnector types are: + // KMS_CONNECTOR_TYPE_HTTP -> 1 + init( KMS_CONNECTOR_TYPE, "HttpKmsConnector"); + // Blob granlues init( BG_URL, isSimulated ? "file://fdbblob/" : "" ); // TODO: store in system key space or something, eventually init( BG_SNAPSHOT_FILE_TARGET_BYTES, 10000000 ); if( buggifySmallShards ) BG_SNAPSHOT_FILE_TARGET_BYTES = 100000; else if (simulationMediumShards || (randomize && BUGGIFY) ) BG_SNAPSHOT_FILE_TARGET_BYTES = 1000000; diff --git a/fdbclient/ServerKnobs.h b/fdbclient/ServerKnobs.h index 3dfccfc531..92674e55d0 100644 --- a/fdbclient/ServerKnobs.h +++ b/fdbclient/ServerKnobs.h @@ -822,6 +822,9 @@ public: std::string ENCRYPTION_MODE; int SIM_KMS_MAX_KEYS; + // Key Management Service (KMS) Connector + std::string KMS_CONNECTOR_TYPE; + // blob granule stuff // FIXME: configure url with database configuration instead of knob eventually std::string BG_URL; diff --git a/fdbserver/CMakeLists.txt b/fdbserver/CMakeLists.txt index 8ec4b437d3..f6d56ebe41 100644 --- a/fdbserver/CMakeLists.txt +++ b/fdbserver/CMakeLists.txt @@ -55,6 +55,8 @@ set(FDBSERVER_SRCS KeyValueStoreMemory.actor.cpp KeyValueStoreRocksDB.actor.cpp KeyValueStoreSQLite.actor.cpp + KmsConnector.h + KmsConnectorInterface.h KnobProtectiveGroups.cpp KnobProtectiveGroups.h Knobs.h @@ -134,8 +136,8 @@ set(FDBSERVER_SRCS ServerDBInfo.actor.h ServerDBInfo.h SigStack.cpp - SimEncryptKmsProxy.actor.cpp - SimEncryptKmsProxy.actor.h + SimKmsConnector.actor.h + SimKmsConnector.actor.cpp SimpleConfigConsumer.actor.cpp SimpleConfigConsumer.h SimulatedCluster.actor.cpp diff --git a/fdbserver/EncryptKeyProxy.actor.cpp b/fdbserver/EncryptKeyProxy.actor.cpp index 5995f027ec..20c0116fc7 100644 --- a/fdbserver/EncryptKeyProxy.actor.cpp +++ b/fdbserver/EncryptKeyProxy.actor.cpp @@ -21,11 +21,15 @@ #include "fdbrpc/Locality.h" #include "fdbrpc/Stats.h" #include "fdbserver/EncryptKeyProxyInterface.h" +#include "fdbserver/KmsConnector.h" +#include "fdbserver/KmsConnectorInterface.h" +#include "fdbserver/Knobs.h" #include "fdbserver/ServerDBInfo.actor.h" -#include "fdbserver/SimEncryptKmsProxy.actor.h" +#include "fdbserver/SimKmsConnector.actor.h" #include "fdbserver/WorkerInterface.actor.h" #include "fdbserver/ServerDBInfo.h" #include "flow/Arena.h" +#include "flow/EncryptUtils.h" #include "flow/Error.h" #include "flow/EventTypes.actor.h" #include "flow/FastRef.h" @@ -38,12 +42,10 @@ #include #include +#include #include "flow/actorcompiler.h" // This must be the last #include. -using EncryptDomainId = uint64_t; -using EncryptBaseCipherId = uint64_t; - namespace { bool canReplyWith(Error e) { switch (e.code()) { @@ -57,16 +59,16 @@ bool canReplyWith(Error e) { } // namespace struct EncryptBaseCipherKey { - EncryptDomainId domainId; - EncryptBaseCipherId baseCipherId; + EncryptCipherDomainId domainId; + EncryptCipherBaseKeyId baseCipherId; Standalone baseCipherKey; uint64_t creationTimeSec; bool noExpiry; EncryptBaseCipherKey() : domainId(0), baseCipherId(0), baseCipherKey(StringRef()), creationTimeSec(0), noExpiry(false) {} - explicit EncryptBaseCipherKey(EncryptDomainId dId, - EncryptBaseCipherId cipherId, + explicit EncryptBaseCipherKey(EncryptCipherDomainId dId, + EncryptCipherBaseKeyId cipherId, StringRef cipherKey, bool neverExpire) : domainId(dId), baseCipherId(cipherId), baseCipherKey(cipherKey), creationTimeSec(now()), noExpiry(neverExpire) { @@ -75,8 +77,8 @@ struct EncryptBaseCipherKey { bool isValid() { return noExpiry ? true : ((now() - creationTimeSec) < FLOW_KNOBS->ENCRYPT_CIPHER_KEY_CACHE_TTL); } }; -using EncryptBaseDomainIdCache = std::unordered_map; -using EncryptBaseCipherKeyIdCache = std::unordered_map; +using EncryptBaseDomainIdCache = std::unordered_map; +using EncryptBaseCipherKeyIdCache = std::unordered_map; struct EncryptKeyProxyData : NonCopyable, ReferenceCounted { public: @@ -87,6 +89,8 @@ public: EncryptBaseDomainIdCache baseCipherDomainIdCache; EncryptBaseCipherKeyIdCache baseCipherKeyIdCache; + std::unique_ptr kmsConnector; + CounterCollection ekpCacheMetrics; Counter baseCipherKeyIdCacheMisses; @@ -107,8 +111,8 @@ public: numResponseWithErrors("EKPNumResponseWithErrors", ekpCacheMetrics), numEncryptionKeyRefreshErrors("EKPNumEncryptionKeyRefreshErrors", ekpCacheMetrics) {} - void insertIntoBaseDomainIdCache(const EncryptDomainId domainId, - const EncryptBaseCipherId baseCipherId, + void insertIntoBaseDomainIdCache(const EncryptCipherDomainId domainId, + const EncryptCipherBaseKeyId baseCipherId, const StringRef baseCipherKey) { // Entries in domainId cache are eligible for periodic refreshes to support 'limiting lifetime of encryption // key' support if enabled on external KMS solutions. @@ -119,8 +123,8 @@ public: insertIntoBaseCipherIdCache(domainId, baseCipherId, baseCipherKey); } - void insertIntoBaseCipherIdCache(const EncryptDomainId domainId, - const EncryptBaseCipherId baseCipherId, + void insertIntoBaseCipherIdCache(const EncryptCipherDomainId domainId, + const EncryptCipherBaseKeyId baseCipherId, const StringRef baseCipherKey) { // Given an cipherKey is immutable, it is OK to NOT expire cached information. // TODO: Update cache to support LRU eviction policy to limit the total cache size. @@ -150,19 +154,37 @@ public: } }; +struct pair_hash { + template + std::size_t operator()(const std::pair& pair) const { + auto hash1 = std::hash{}(pair.first); + auto hash2 = std::hash{}(pair.second); + + // Equal hashes XOR would be ZERO. + return hash1 == hash2 ? hash1 : hash1 ^ hash2; + } +}; + ACTOR Future getCipherKeysByBaseCipherKeyIds(Reference ekpProxyData, - SimKmsProxyInterface simKmsInterface, + KmsConnectorInterface kmsConnectorInf, EKPGetBaseCipherKeysByIdsRequest req) { // Scan the cached cipher-keys and filter our baseCipherIds locally cached // for the rest, reachout to KMS to fetch the required details - std::vector> lookupCipherIds; + std::vector> lookupCipherIds; state std::vector cachedCipherDetails; state EKPGetBaseCipherKeysByIdsRequest keysByIds = req; state EKPGetBaseCipherKeysByIdsReply keyIdsReply; + // Dedup the requested pair + // TODO: endpoint serialization of std::unordered_set isn't working at the moment + std::unordered_set, pair_hash> dedupedCipherIds; for (const auto& item : req.baseCipherIds) { + dedupedCipherIds.emplace(item); + } + + for (const auto& item : dedupedCipherIds) { const auto itr = ekpProxyData->baseCipherKeyIdCache.find(item.first); if (itr != ekpProxyData->baseCipherKeyIdCache.end()) { ASSERT(itr->second.isValid()); @@ -176,38 +198,32 @@ ACTOR Future getCipherKeysByBaseCipherKeyIds(ReferencebaseCipherKeyIdCacheHits += cachedCipherDetails.size(); ekpProxyData->baseCipherKeyIdCacheMisses += lookupCipherIds.size(); - if (g_network->isSimulated()) { - if (!lookupCipherIds.empty()) { - try { - SimGetEncryptKeysByKeyIdsRequest simKeyIdsReq(lookupCipherIds); - SimGetEncryptKeysByKeyIdsReply simKeyIdsReply = - wait(simKmsInterface.encryptKeyLookupByKeyIds.getReply(simKeyIdsReq)); + if (!lookupCipherIds.empty()) { + try { + KmsConnLookupEKsByKeyIdsReq keysByIdsReq(lookupCipherIds); + KmsConnLookupEKsByKeyIdsRep keysByIdsRep = wait(kmsConnectorInf.ekLookupByIds.getReply(keysByIdsReq)); - for (const auto& item : simKeyIdsReply.encryptKeyDetails) { - keyIdsReply.baseCipherDetails.emplace_back( - item.encryptDomainId, item.encryptKeyId, item.encryptKey, keyIdsReply.arena); - } - - // Record the fetched cipher details to the local cache for the future references - // Note: cache warm-up is done after reponding to the caller - - for (auto& item : simKeyIdsReply.encryptKeyDetails) { - // DomainId isn't available here, the caller must know the encryption domainId - ekpProxyData->insertIntoBaseCipherIdCache(item.encryptDomainId, item.encryptKeyId, item.encryptKey); - } - } catch (Error& e) { - if (!canReplyWith(e)) { - TraceEvent("GetCipherKeysByIds", ekpProxyData->myId).error(e); - throw; - } - TraceEvent("GetCipherKeysByIds", ekpProxyData->myId).detail("ErrorCode", e.code()); - ekpProxyData->sendErrorResponse(keysByIds.reply, e); - return Void(); + for (const auto& item : keysByIdsRep.cipherKeyDetails) { + keyIdsReply.baseCipherDetails.emplace_back( + item.encryptDomainId, item.encryptKeyId, item.encryptKey, keyIdsReply.arena); } + + // Record the fetched cipher details to the local cache for the future references + // Note: cache warm-up is done after reponding to the caller + + for (auto& item : keysByIdsRep.cipherKeyDetails) { + // DomainId isn't available here, the caller must know the encryption domainId + ekpProxyData->insertIntoBaseCipherIdCache(item.encryptDomainId, item.encryptKeyId, item.encryptKey); + } + } catch (Error& e) { + if (!canReplyWith(e)) { + TraceEvent("GetCipherKeysByIds", ekpProxyData->myId).error(e); + throw; + } + TraceEvent("GetCipherKeysByIds", ekpProxyData->myId).detail("ErrorCode", e.code()); + ekpProxyData->sendErrorResponse(keysByIds.reply, e); + return Void(); } - } else { - // TODO: Call to non-FDB KMS connector process. - throw not_implemented(); } // Append cached cipherKeyDetails to the result-set @@ -221,7 +237,7 @@ ACTOR Future getCipherKeysByBaseCipherKeyIds(Reference getLatestCipherKeys(Reference ekpProxyData, - SimKmsProxyInterface simKmsInterface, + KmsConnectorInterface kmsConnectorInf, EKPGetLatestBaseCipherKeysRequest req) { // Scan the cached cipher-keys and filter our baseCipherIds locally cached // for the rest, reachout to KMS to fetch the required details @@ -231,49 +247,51 @@ ACTOR Future getLatestCipherKeys(Reference ekpProxyDa state EKPGetLatestBaseCipherKeysReply latestCipherReply; state Arena& arena = latestCipherReply.arena; + // Dedup the requested domainIds. + // TODO: endpoint serialization of std::unordered_set isn't working at the moment + std::unordered_set dedupedDomainIds; + for (EncryptCipherDomainId id : req.encryptDomainIds) { + dedupedDomainIds.emplace(id); + } + // First, check if the requested information is already cached by the server. // Ensure the cached information is within FLOW_KNOBS->ENCRYPT_CIPHER_KEY_CACHE_TTL time window. - std::vector lookupCipherDomains; - for (EncryptDomainId id : req.encryptDomainIds) { + std::vector lookupCipherDomains; + for (EncryptCipherDomainId id : dedupedDomainIds) { const auto itr = ekpProxyData->baseCipherDomainIdCache.find(id); if (itr != ekpProxyData->baseCipherDomainIdCache.end() && itr->second.isValid()) { cachedCipherDetails.emplace_back(id, itr->second.baseCipherId, itr->second.baseCipherKey, arena); } else { - lookupCipherDomains.push_back(id); + lookupCipherDomains.emplace_back(id); } } ekpProxyData->baseCipherDomainIdCacheHits += cachedCipherDetails.size(); ekpProxyData->baseCipherDomainIdCacheMisses += lookupCipherDomains.size(); - if (g_network->isSimulated()) { - if (!lookupCipherDomains.empty()) { - try { - SimGetEncryptKeysByDomainIdsRequest simKeysByDomainIdReq(lookupCipherDomains); - SimGetEncryptKeyByDomainIdReply simKeysByDomainIdRep = - wait(simKmsInterface.encryptKeyLookupByDomainId.getReply(simKeysByDomainIdReq)); + if (!lookupCipherDomains.empty()) { + try { + KmsConnLookupEKsByDomainIdsReq keysByDomainIdReq(lookupCipherDomains); + KmsConnLookupEKsByDomainIdsRep keysByDomainIdRep = + wait(kmsConnectorInf.ekLookupByDomainIds.getReply(keysByDomainIdReq)); - for (auto& item : simKeysByDomainIdRep.encryptKeyDetails) { - latestCipherReply.baseCipherDetails.emplace_back( - item.encryptDomainId, item.encryptKeyId, item.encryptKey, arena); + for (auto& item : keysByDomainIdRep.cipherKeyDetails) { + latestCipherReply.baseCipherDetails.emplace_back( + item.encryptDomainId, item.encryptKeyId, item.encryptKey, arena); - // Record the fetched cipher details to the local cache for the future references - ekpProxyData->insertIntoBaseDomainIdCache(item.encryptDomainId, item.encryptKeyId, item.encryptKey); - } - } catch (Error& e) { - if (!canReplyWith(e)) { - TraceEvent("GetLatestCipherKeys", ekpProxyData->myId).error(e); - throw; - } - TraceEvent("GetLatestCipherKeys", ekpProxyData->myId).detail("ErrorCode", e.code()); - ekpProxyData->sendErrorResponse(latestKeysReq.reply, e); - return Void(); + // Record the fetched cipher details to the local cache for the future references + ekpProxyData->insertIntoBaseDomainIdCache(item.encryptDomainId, item.encryptKeyId, item.encryptKey); } + } catch (Error& e) { + if (!canReplyWith(e)) { + TraceEvent("GetLatestCipherKeys", ekpProxyData->myId).error(e); + throw; + } + TraceEvent("GetLatestCipherKeys", ekpProxyData->myId).detail("ErrorCode", e.code()); + ekpProxyData->sendErrorResponse(latestKeysReq.reply, e); + return Void(); } - } else { - // TODO: Call to non-FDB KMS connector process. - throw not_implemented(); } for (auto& item : cachedCipherDetails) { @@ -287,27 +305,27 @@ ACTOR Future getLatestCipherKeys(Reference ekpProxyDa return Void(); } -ACTOR Future refreshEncryptionKeysUsingSimKms(Reference ekpProxyData, - SimKmsProxyInterface simKmsInterface) { +ACTOR Future refreshEncryptionKeysCore(Reference ekpProxyData, + KmsConnectorInterface kmsConnectorInf) { ASSERT(g_network->isSimulated()); - TraceEvent("RefreshEKs_Start", ekpProxyData->myId).detail("Inf", simKmsInterface.id()); + TraceEvent("RefreshEKs_Start", ekpProxyData->myId).detail("KmsConnInf", kmsConnectorInf.id()); try { - SimGetEncryptKeysByDomainIdsRequest req; + KmsConnLookupEKsByDomainIdsReq req; req.encryptDomainIds.reserve(ekpProxyData->baseCipherDomainIdCache.size()); for (auto& item : ekpProxyData->baseCipherDomainIdCache) { req.encryptDomainIds.emplace_back(item.first); } - SimGetEncryptKeyByDomainIdReply rep = wait(simKmsInterface.encryptKeyLookupByDomainId.getReply(req)); - for (auto& item : rep.encryptKeyDetails) { + KmsConnLookupEKsByDomainIdsRep rep = wait(kmsConnectorInf.ekLookupByDomainIds.getReply(req)); + for (auto& item : rep.cipherKeyDetails) { ekpProxyData->insertIntoBaseDomainIdCache(item.encryptDomainId, item.encryptKeyId, item.encryptKey); } - ekpProxyData->baseCipherKeysRefreshed += rep.encryptKeyDetails.size(); - TraceEvent("RefreshEKs_Done", ekpProxyData->myId).detail("KeyCount", rep.encryptKeyDetails.size()); + ekpProxyData->baseCipherKeysRefreshed += rep.cipherKeyDetails.size(); + TraceEvent("RefreshEKs_Done", ekpProxyData->myId).detail("KeyCount", rep.cipherKeyDetails.size()); } catch (Error& e) { if (!canReplyWith(e)) { TraceEvent("RefreshEncryptionKeys_Error").error(e); @@ -320,30 +338,34 @@ ACTOR Future refreshEncryptionKeysUsingSimKms(Reference refreshEncryptionKeysUsingKms(Reference ekpProxyData) { - wait(delay(0)); // compiler needs to be happy - throw not_implemented(); +void refreshEncryptionKeys(Reference ekpProxyData, KmsConnectorInterface kmsConnectorInf) { + Future ignored = refreshEncryptionKeysCore(ekpProxyData, kmsConnectorInf); } -void refreshEncryptionKeys(Reference ekpProxyData, SimKmsProxyInterface simKmsInterface) { - - Future ignored; +void activateKmsConnector(Reference ekpProxyData, KmsConnectorInterface kmsConnectorInf) { if (g_network->isSimulated()) { - ignored = refreshEncryptionKeysUsingSimKms(ekpProxyData, simKmsInterface); + ekpProxyData->kmsConnector = std::make_unique(); + } else if (SERVER_KNOBS->KMS_CONNECTOR_TYPE.compare("HttpKmsConnector")) { + throw not_implemented(); } else { - ignored = refreshEncryptionKeysUsingKms(ekpProxyData); + throw not_implemented(); } + + TraceEvent("EKP_ActiveKmsConnector", ekpProxyData->myId).detail("ConnectorType", SERVER_KNOBS->KMS_CONNECTOR_TYPE); + ekpProxyData->addActor.send(ekpProxyData->kmsConnector->connectorCore(kmsConnectorInf)); } ACTOR Future encryptKeyProxyServer(EncryptKeyProxyInterface ekpInterface, Reference> db) { state Reference self(new EncryptKeyProxyData(ekpInterface.id())); - state PromiseStream> addActor; state Future collection = actorCollection(self->addActor.getFuture()); self->addActor.send(traceRole(Role::ENCRYPT_KEY_PROXY, ekpInterface.id())); - state SimKmsProxyInterface simKmsProxyInf; + state KmsConnectorInterface kmsConnectorInf; + kmsConnectorInf.initEndpoints(); - TraceEvent("EKP_Start", self->myId).log(); + TraceEvent("EKP_Start", self->myId).detail("KmsConnectorInf", kmsConnectorInf.id()); + + activateKmsConnector(self, kmsConnectorInf); // Register a recurring task to refresh the cached Encryption keys. // Approach avoids external RPCs due to EncryptionKey refreshes for the inline write encryption codepath such as: @@ -352,31 +374,17 @@ ACTOR Future encryptKeyProxyServer(EncryptKeyProxyInterface ekpInterface, // FLOW_KNOB->ENCRRYPTION_KEY_REFRESH_INTERVAL_SEC, allowing the interactions with external Encryption Key Manager // mostly not co-inciding with FDB process encryption key refresh attempts. - if (g_network->isSimulated()) { - // In simulation construct an Encryption KMSProxy actor to satisfy encryption keys lookups otherwise satisfied - // by integrating external Encryption Key Management solutions. - - simKmsProxyInf.initEndpoints(); - self->addActor.send(simEncryptKmsProxyCore(simKmsProxyInf)); - - TraceEvent("EKP_InitSimKmsInf", self->myId).detail("Inf", simKmsProxyInf.id()); - - self->encryptionKeyRefresher = recurring([&]() { refreshEncryptionKeys(self, simKmsProxyInf); }, - FLOW_KNOBS->ENCRYPT_KEY_REFRESH_INTERVAL, - TaskPriority::Worker); - - } else { - // TODO: Add recurring actor to talk to external KMS proxy process - throw not_implemented(); - } + self->encryptionKeyRefresher = recurring([&]() { refreshEncryptionKeys(self, kmsConnectorInf); }, + FLOW_KNOBS->ENCRYPT_KEY_REFRESH_INTERVAL, + TaskPriority::Worker); try { loop choose { when(EKPGetBaseCipherKeysByIdsRequest req = waitNext(ekpInterface.getBaseCipherKeysByIds.getFuture())) { - wait(getCipherKeysByBaseCipherKeyIds(self, simKmsProxyInf, req)); + wait(getCipherKeysByBaseCipherKeyIds(self, kmsConnectorInf, req)); } when(EKPGetLatestBaseCipherKeysRequest req = waitNext(ekpInterface.getLatestBaseCipherKeys.getFuture())) { - wait(getLatestCipherKeys(self, simKmsProxyInf, req)); + wait(getLatestCipherKeys(self, kmsConnectorInf, req)); } when(HaltEncryptKeyProxyRequest req = waitNext(ekpInterface.haltEncryptKeyProxy.getFuture())) { TraceEvent("EKP_Halted", self->myId).detail("ReqID", req.requesterID); diff --git a/fdbserver/KmsConnector.h b/fdbserver/KmsConnector.h new file mode 100644 index 0000000000..e49b6b5c83 --- /dev/null +++ b/fdbserver/KmsConnector.h @@ -0,0 +1,43 @@ +/* + * KmsConnector.h + * + * This source file is part of the FoundationDB open source project + * + * Copyright 2013-2022 Apple Inc. and the FoundationDB project authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef KMS_CONNECTOR_H +#define KMS_CONNECTOR_H +#pragma once + +#include "fdbserver/KmsConnectorInterface.h" +#include "flow/Arena.h" +#include "flow/EncryptUtils.h" + +// FDB encryption needs to interact with external Key Management Services (KMS) solutions to lookup/refresh encryption +// keys. KmsConnector interface is an abstract interface enabling implementing specialized KMS connector +// implementations. +// FDB KMSConnector implementation should inherit from KmsConnector and implement pure virtual function, +// EncryptKeyProxyServer instantiates desired implementation object based on SERVER_KNOB->KMS_CONNECTOR_TYPE knob. + +class KmsConnector : public NonCopyable { +public: + KmsConnector() {} + virtual ~KmsConnector() {} + + virtual Future connectorCore(struct KmsConnectorInterface interf) = 0; +}; + +#endif diff --git a/fdbserver/KmsConnectorInterface.h b/fdbserver/KmsConnectorInterface.h new file mode 100644 index 0000000000..4c4c91aef5 --- /dev/null +++ b/fdbserver/KmsConnectorInterface.h @@ -0,0 +1,144 @@ +/* + * KmsConnectorInterface.h + * + * This source file is part of the FoundationDB open source project + * + * Copyright 2013-2022 Apple Inc. and the FoundationDB project authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef FDBSERVER_KMSCONNECTORINTERFACE_H +#define FDBSERVER_KMSCONNECTORINTERFACE_H +#pragma once + +#include "fdbrpc/fdbrpc.h" +#include "flow/EncryptUtils.h" +#include "flow/FileIdentifier.h" +#include "flow/Trace.h" +#include "flow/flow.h" +#include "flow/network.h" + +struct KmsConnectorInterface { + constexpr static FileIdentifier file_identifier = 2416711; + RequestStream> waitFailure; + RequestStream ekLookupByIds; + RequestStream ekLookupByDomainIds; + + KmsConnectorInterface() {} + + UID id() const { return ekLookupByIds.getEndpoint().token; } + template + void serialize(Archive& ar) { + if constexpr (!is_fb_function) { + ASSERT(ar.protocolVersion().isValid()); + } + serializer(ar, waitFailure); + if (Archive::isDeserializing) { + ekLookupByIds = + RequestStream(waitFailure.getEndpoint().getAdjustedEndpoint(1)); + ekLookupByDomainIds = + RequestStream(waitFailure.getEndpoint().getAdjustedEndpoint(2)); + } + } + + void initEndpoints() { + std::vector> streams; + streams.push_back(waitFailure.getReceiver()); + streams.push_back(ekLookupByIds.getReceiver(TaskPriority::Worker)); + streams.push_back(ekLookupByDomainIds.getReceiver(TaskPriority::Worker)); + FlowTransport::transport().addEndpoints(streams); + } +}; + +struct EncryptCipherKeyDetails { + constexpr static FileIdentifier file_identifier = 1227025; + EncryptCipherDomainId encryptDomainId; + EncryptCipherBaseKeyId encryptKeyId; + StringRef encryptKey; + + EncryptCipherKeyDetails() {} + explicit EncryptCipherKeyDetails(EncryptCipherDomainId dId, + EncryptCipherBaseKeyId keyId, + StringRef key, + Arena& arena) + : encryptDomainId(dId), encryptKeyId(keyId), encryptKey(StringRef(arena, key)) {} + + bool operator==(const EncryptCipherKeyDetails& toCompare) { + return encryptDomainId == toCompare.encryptDomainId && encryptKeyId == toCompare.encryptKeyId && + encryptKey.compare(toCompare.encryptKey) == 0; + } + + template + void serialize(Ar& ar) { + serializer(ar, encryptDomainId, encryptKeyId, encryptKey); + } +}; + +struct KmsConnLookupEKsByKeyIdsRep { + constexpr static FileIdentifier file_identifier = 2313778; + Arena arena; + std::vector cipherKeyDetails; + + KmsConnLookupEKsByKeyIdsRep() {} + + template + void serialize(Ar& ar) { + serializer(ar, arena, cipherKeyDetails); + } +}; + +struct KmsConnLookupEKsByKeyIdsReq { + constexpr static FileIdentifier file_identifier = 6913396; + std::vector> encryptKeyIds; + ReplyPromise reply; + + KmsConnLookupEKsByKeyIdsReq() {} + explicit KmsConnLookupEKsByKeyIdsReq( + const std::vector>& keyIds) + : encryptKeyIds(keyIds) {} + + template + void serialize(Ar& ar) { + serializer(ar, encryptKeyIds, reply); + } +}; + +struct KmsConnLookupEKsByDomainIdsRep { + constexpr static FileIdentifier file_identifier = 3009025; + Arena arena; + std::vector cipherKeyDetails; + + KmsConnLookupEKsByDomainIdsRep() {} + + template + void serialize(Ar& ar) { + serializer(ar, arena, cipherKeyDetails); + } +}; + +struct KmsConnLookupEKsByDomainIdsReq { + constexpr static FileIdentifier file_identifier = 9918682; + std::vector encryptDomainIds; + ReplyPromise reply; + + KmsConnLookupEKsByDomainIdsReq() {} + explicit KmsConnLookupEKsByDomainIdsReq(const std::vector& ids) : encryptDomainIds(ids) {} + + template + void serialize(Ar& ar) { + serializer(ar, encryptDomainIds, reply); + } +}; + +#endif diff --git a/fdbserver/SimEncryptKmsProxy.actor.cpp b/fdbserver/SimKmsConnector.actor.cpp similarity index 60% rename from fdbserver/SimEncryptKmsProxy.actor.cpp rename to fdbserver/SimKmsConnector.actor.cpp index c4b2e5b511..2ec45f5c8f 100644 --- a/fdbserver/SimEncryptKmsProxy.actor.cpp +++ b/fdbserver/SimKmsConnector.actor.cpp @@ -18,37 +18,38 @@ * limitations under the License. */ +#include "fdbserver/SimKmsConnector.actor.h" + +#include "fdbrpc/sim_validation.h" +#include "fdbserver/Knobs.h" +#include "flow/ActorCollection.h" +#include "flow/EncryptUtils.h" +#include "flow/Error.h" +#include "flow/FastRef.h" +#include "flow/IRandom.h" +#include "flow/ITrace.h" +#include "flow/network.h" +#include "flow/UnitTest.h" + #include #include #include -#include "fdbrpc/sim_validation.h" -#include "fdbserver/Knobs.h" -#include "fdbserver/SimEncryptKmsProxy.actor.h" -#include "flow/ActorCollection.h" -#include "flow/Error.h" -#include "flow/IRandom.h" -#include "flow/ITrace.h" -#include "flow/StreamCipher.h" -#include "flow/Trace.h" -#include "flow/UnitTest.h" #include "flow/actorcompiler.h" // This must be the last #include. -#include "flow/network.h" +using SimEncryptKey = std::string; struct SimEncryptKeyCtx { - SimEncryptKeyId id; + EncryptCipherBaseKeyId id; SimEncryptKey key; - SimEncryptKeyCtx() : id(0) {} - explicit SimEncryptKeyCtx(SimEncryptKeyId kId, const char* data) : id(kId), key(data) {} + explicit SimEncryptKeyCtx(EncryptCipherBaseKeyId kId, const char* data) : id(kId), key(data) {} }; -struct SimEncryptKmsProxyContext { +struct SimKmsConnectorContext { uint32_t maxEncryptionKeys; - std::unordered_map> simEncryptKeyStore; + std::unordered_map> simEncryptKeyStore; - SimEncryptKmsProxyContext() : maxEncryptionKeys(0) {} - explicit SimEncryptKmsProxyContext(uint32_t keyCount) : maxEncryptionKeys(keyCount) { + explicit SimKmsConnectorContext(uint32_t keyCount) : maxEncryptionKeys(keyCount) { uint8_t buffer[AES_256_KEY_LENGTH]; // Construct encryption keyStore. @@ -60,25 +61,26 @@ struct SimEncryptKmsProxyContext { } }; -ACTOR Future simEncryptKmsProxyCore(SimKmsProxyInterface interf) { - state SimEncryptKmsProxyContext kmsProxyCtx(SERVER_KNOBS->SIM_KMS_MAX_KEYS); - - ASSERT(kmsProxyCtx.simEncryptKeyStore.size() == SERVER_KNOBS->SIM_KMS_MAX_KEYS); - +ACTOR Future simKmsConnectorCore_impl(KmsConnectorInterface interf) { TraceEvent("SimEncryptKmsProxy_Init", interf.id()).detail("MaxEncryptKeys", SERVER_KNOBS->SIM_KMS_MAX_KEYS); state bool success = true; + state std::unique_ptr ctx = + std::make_unique(SERVER_KNOBS->SIM_KMS_MAX_KEYS); + + ASSERT_EQ(ctx->simEncryptKeyStore.size(), SERVER_KNOBS->SIM_KMS_MAX_KEYS); + loop { choose { - when(SimGetEncryptKeysByKeyIdsRequest req = waitNext(interf.encryptKeyLookupByKeyIds.getFuture())) { - state SimGetEncryptKeysByKeyIdsRequest keysByIdsReq = req; - state SimGetEncryptKeysByKeyIdsReply keysByIdsRep; + when(KmsConnLookupEKsByKeyIdsReq req = waitNext(interf.ekLookupByIds.getFuture())) { + state KmsConnLookupEKsByKeyIdsReq keysByIdsReq = req; + state KmsConnLookupEKsByKeyIdsRep keysByIdsRep; // Lookup corresponding EncryptKeyCtx for input keyId for (const auto& item : req.encryptKeyIds) { - const auto& itr = kmsProxyCtx.simEncryptKeyStore.find(item.first); - if (itr != kmsProxyCtx.simEncryptKeyStore.end()) { - keysByIdsRep.encryptKeyDetails.emplace_back( + const auto& itr = ctx->simEncryptKeyStore.find(item.first); + if (itr != ctx->simEncryptKeyStore.end()) { + keysByIdsRep.cipherKeyDetails.emplace_back( item.second, itr->first, StringRef(keysByIdsRep.arena, itr->second.get()->key), @@ -93,18 +95,18 @@ ACTOR Future simEncryptKmsProxyCore(SimKmsProxyInterface interf) { success ? keysByIdsReq.reply.send(keysByIdsRep) : keysByIdsReq.reply.sendError(encrypt_key_not_found()); } - when(SimGetEncryptKeysByDomainIdsRequest req = waitNext(interf.encryptKeyLookupByDomainId.getFuture())) { - state SimGetEncryptKeysByDomainIdsRequest keysByDomainIdReq = req; - state SimGetEncryptKeyByDomainIdReply keysByDomainIdRep; + when(KmsConnLookupEKsByDomainIdsReq req = waitNext(interf.ekLookupByDomainIds.getFuture())) { + state KmsConnLookupEKsByDomainIdsReq keysByDomainIdReq = req; + state KmsConnLookupEKsByDomainIdsRep keysByDomainIdRep; // Map encryptionDomainId to corresponding EncryptKeyCtx element using a modulo operation. This would // mean multiple domains gets mapped to the same encryption key which is fine, the EncryptKeyStore // guarantees that keyId -> plaintext encryptKey mapping is idempotent. - for (SimEncryptDomainId domainId : req.encryptDomainIds) { - SimEncryptKeyId keyId = domainId % SERVER_KNOBS->SIM_KMS_MAX_KEYS; - const auto& itr = kmsProxyCtx.simEncryptKeyStore.find(keyId); - if (itr != kmsProxyCtx.simEncryptKeyStore.end()) { - keysByDomainIdRep.encryptKeyDetails.emplace_back( + for (EncryptCipherDomainId domainId : req.encryptDomainIds) { + EncryptCipherBaseKeyId keyId = domainId % SERVER_KNOBS->SIM_KMS_MAX_KEYS; + const auto& itr = ctx->simEncryptKeyStore.find(keyId); + if (itr != ctx->simEncryptKeyStore.end()) { + keysByDomainIdRep.cipherKeyDetails.emplace_back( domainId, keyId, StringRef(itr->second.get()->key), keysByDomainIdRep.arena); } else { success = false; @@ -121,35 +123,38 @@ ACTOR Future simEncryptKmsProxyCore(SimKmsProxyInterface interf) { } } -void forceLinkSimEncryptKmsProxyTests() {} +Future SimKmsConnector::connectorCore(KmsConnectorInterface interf) { + return simKmsConnectorCore_impl(interf); +} +void forceLinkSimKmsConnectorTests() {} namespace { -ACTOR Future testRunWorkload(SimKmsProxyInterface inf, uint32_t nEncryptionKeys) { +ACTOR Future testRunWorkload(KmsConnectorInterface inf, uint32_t nEncryptionKeys) { state uint32_t maxEncryptionKeys = nEncryptionKeys; state int maxDomainIds = deterministicRandom()->randomInt(121, 295); state int maxIterations = deterministicRandom()->randomInt(786, 1786); - state std::unordered_map> domainIdKeyMap; + state std::unordered_map> domainIdKeyMap; state int i = 0; TraceEvent("RunWorkloadStart").detail("MaxDomainIds", maxDomainIds).detail("MaxIterations", maxIterations); { // construct domainId to EncryptKeyCtx map - SimGetEncryptKeysByDomainIdsRequest domainIdsReq; + KmsConnLookupEKsByDomainIdsReq domainIdsReq; for (i = 0; i < maxDomainIds; i++) { domainIdsReq.encryptDomainIds.push_back(i); } - SimGetEncryptKeyByDomainIdReply domainIdsReply = wait(inf.encryptKeyLookupByDomainId.getReply(domainIdsReq)); - for (auto& element : domainIdsReply.encryptKeyDetails) { + KmsConnLookupEKsByDomainIdsRep domainIdsRep = wait(inf.ekLookupByDomainIds.getReply(domainIdsReq)); + for (auto& element : domainIdsRep.cipherKeyDetails) { domainIdKeyMap.emplace( element.encryptDomainId, std::make_unique(element.encryptKeyId, element.encryptKey.toString().c_str())); } // randomly pick any domainId and validate if lookupByKeyId result matches - state std::unordered_map validationMap; - std::unordered_map idsToLookup; + state std::unordered_map validationMap; + std::unordered_map idsToLookup; for (i = 0; i < maxIterations; i++) { state int idx = deterministicRandom()->randomInt(0, maxDomainIds); state SimEncryptKeyCtx* ctx = domainIdKeyMap[idx].get(); @@ -157,27 +162,27 @@ ACTOR Future testRunWorkload(SimKmsProxyInterface inf, uint32_t nEncryptio idsToLookup.emplace(ctx->id, idx); } - state SimGetEncryptKeysByKeyIdsRequest keyIdsReq; + state KmsConnLookupEKsByKeyIdsReq keyIdsReq; for (const auto& item : idsToLookup) { keyIdsReq.encryptKeyIds.emplace_back(std::make_pair(item.first, item.second)); } - state SimGetEncryptKeysByKeyIdsReply keyIdsReply = wait(inf.encryptKeyLookupByKeyIds.getReply(keyIdsReq)); + state KmsConnLookupEKsByKeyIdsRep keyIdsReply = wait(inf.ekLookupByIds.getReply(keyIdsReq)); /* TraceEvent("Lookup") .detail("KeyIdReqSize", keyIdsReq.encryptKeyIds.size()) .detail("KeyIdsRepSz", keyIdsReply.encryptKeyDetails.size()) .detail("ValSz", validationMap.size()); */ - ASSERT(keyIdsReply.encryptKeyDetails.size() == validationMap.size()); - for (const auto& element : keyIdsReply.encryptKeyDetails) { - ASSERT(validationMap[element.encryptDomainId].compare(element.encryptKey) == 0); + ASSERT(keyIdsReply.cipherKeyDetails.size() == validationMap.size()); + for (const auto& element : keyIdsReply.cipherKeyDetails) { + ASSERT(validationMap[element.encryptKeyId].compare(element.encryptKey) == 0); } } { // Verify unknown key access returns the error - state SimGetEncryptKeysByKeyIdsRequest req; + state KmsConnLookupEKsByKeyIdsReq req; req.encryptKeyIds.emplace_back(std::make_pair(maxEncryptionKeys + 1, 1)); try { - SimGetEncryptKeysByKeyIdsReply reply = wait(inf.encryptKeyLookupByKeyIds.getReply(req)); + KmsConnLookupEKsByKeyIdsRep reply = wait(inf.ekLookupByIds.getReply(req)); } catch (Error& e) { ASSERT(e.code() == error_code_encrypt_key_not_found); } @@ -189,12 +194,13 @@ ACTOR Future testRunWorkload(SimKmsProxyInterface inf, uint32_t nEncryptio } // namespace -TEST_CASE("fdbserver/SimEncryptKmsProxy") { - state SimKmsProxyInterface inf; +TEST_CASE("fdbserver/SimKmsConnector") { + state KmsConnectorInterface inf; state uint32_t maxEncryptKeys = 64; + state SimKmsConnector connector; loop choose { - when(wait(simEncryptKmsProxyCore(inf))) { throw internal_error(); } + when(wait(connector.connectorCore(inf))) { throw internal_error(); } when(wait(testRunWorkload(inf, maxEncryptKeys))) { break; } } return Void(); diff --git a/fdbserver/SimKmsConnector.actor.h b/fdbserver/SimKmsConnector.actor.h new file mode 100644 index 0000000000..6a03dcda26 --- /dev/null +++ b/fdbserver/SimKmsConnector.actor.h @@ -0,0 +1,40 @@ +/* + * SimKmsConnector.actor.h + * + * This source file is part of the FoundationDB open source project + * + * Copyright 2013-2022 Apple Inc. and the FoundationDB project authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#pragma once + +#if defined(NO_INTELLISENSE) && !defined(FDBSERVER_SIMKMSCONNECTOR_ACTOR_G_H) +#define FDBSERVER_SIMKMSCONNECTOR_ACTOR_G_H +#include "fdbserver/SimKmsConnector.actor.g.h" +#elif !defined(FDBSERVER_SIMKMSCONNECTOR_ACTOR_H) +#define FDBSERVER_SIMKMSCONNECTOR_ACTOR_H + +#include "fdbserver/KmsConnector.h" +#include "flow/BlobCipher.h" + +#include "flow/actorcompiler.h" // This must be the last #include. + +class SimKmsConnector : public KmsConnector { +public: + SimKmsConnector() = default; + Future connectorCore(KmsConnectorInterface interf); +}; + +#endif \ No newline at end of file diff --git a/fdbserver/workloads/UnitTests.actor.cpp b/fdbserver/workloads/UnitTests.actor.cpp index 9a23f03a6f..25f2ed135a 100644 --- a/fdbserver/workloads/UnitTests.actor.cpp +++ b/fdbserver/workloads/UnitTests.actor.cpp @@ -35,7 +35,7 @@ void forceLinkBLockCiherTests(); void forceLinkParallelStreamTests(); void forceLinkSimExternalConnectionTests(); void forceLinkMutationLogReaderTests(); -void forceLinkSimEncryptKmsProxyTests(); +void forceLinkSimKmsConnectorTests(); void forceLinkIThreadPoolTests(); void forceLinkTokenSignTests(); void forceLinkVersionVectorTests(); @@ -84,7 +84,7 @@ struct UnitTestWorkload : TestWorkload { forceLinkParallelStreamTests(); forceLinkSimExternalConnectionTests(); forceLinkMutationLogReaderTests(); - forceLinkSimEncryptKmsProxyTests(); + forceLinkSimKmsConnectorTests(); forceLinkIThreadPoolTests(); forceLinkTokenSignTests(); forceLinkVersionVectorTests(); diff --git a/flow/BlobCipher.h b/flow/BlobCipher.h index 3c2e88a54e..75a4e8b947 100644 --- a/flow/BlobCipher.h +++ b/flow/BlobCipher.h @@ -25,7 +25,7 @@ #include #include -#if (!defined(TLS_DISABLED) && !defined(_WIN32)) +#if (!defined(TLS_DISABLED)) #define ENCRYPTION_ENABLED 1 #else #define ENCRYPTION_ENABLED 0