diff --git a/fdbclient/BlobCipher.cpp b/fdbclient/BlobCipher.cpp index 3047f7f1e4..bed09f201f 100644 --- a/fdbclient/BlobCipher.cpp +++ b/fdbclient/BlobCipher.cpp @@ -190,7 +190,7 @@ BlobCipherKeyIdCache::BlobCipherKeyIdCache(EncryptCipherDomainId dId, size_t* si BlobCipherKeyIdCacheKey BlobCipherKeyIdCache::getCacheKey(const EncryptCipherBaseKeyId& baseCipherKeyId, const EncryptCipherRandomSalt& salt) { - if (baseCipherKeyId == ENCRYPT_INVALID_CIPHER_KEY_ID || salt == ENCRYPT_INVALID_RANDOM_SALT) { + if (baseCipherKeyId == INVALID_ENCRYPT_CIPHER_KEY_ID || salt == INVALID_ENCRYPT_RANDOM_SALT) { throw encrypt_invalid_id(); } return std::make_pair(baseCipherKeyId, salt); @@ -200,9 +200,9 @@ Reference BlobCipherKeyIdCache::getLatestCipherKey() { if (!latestBaseCipherKeyId.present()) { return Reference(); } - ASSERT_NE(latestBaseCipherKeyId.get(), ENCRYPT_INVALID_CIPHER_KEY_ID); + ASSERT_NE(latestBaseCipherKeyId.get(), INVALID_ENCRYPT_CIPHER_KEY_ID); ASSERT(latestRandomSalt.present()); - ASSERT_NE(latestRandomSalt.get(), ENCRYPT_INVALID_RANDOM_SALT); + ASSERT_NE(latestRandomSalt.get(), INVALID_ENCRYPT_RANDOM_SALT); return getCipherByBaseCipherId(latestBaseCipherKeyId.get(), latestRandomSalt.get()); } @@ -221,7 +221,7 @@ Reference BlobCipherKeyIdCache::insertBaseCipherKey(const Encrypt int baseCipherLen, const int64_t refreshAt, const int64_t expireAt) { - ASSERT_GT(baseCipherId, ENCRYPT_INVALID_CIPHER_KEY_ID); + ASSERT_GT(baseCipherId, INVALID_ENCRYPT_CIPHER_KEY_ID); // BaseCipherKeys are immutable, given the routine invocation updates 'latestCipher', // ensure no key-tampering is done @@ -269,8 +269,8 @@ Reference BlobCipherKeyIdCache::insertBaseCipherKey(const Encrypt const EncryptCipherRandomSalt& salt, const int64_t refreshAt, const int64_t expireAt) { - ASSERT_NE(baseCipherId, ENCRYPT_INVALID_CIPHER_KEY_ID); - ASSERT_NE(salt, ENCRYPT_INVALID_RANDOM_SALT); + ASSERT_NE(baseCipherId, INVALID_ENCRYPT_CIPHER_KEY_ID); + ASSERT_NE(salt, INVALID_ENCRYPT_RANDOM_SALT); BlobCipherKeyIdCacheKey cacheKey = getCacheKey(baseCipherId, salt); @@ -332,7 +332,7 @@ Reference BlobCipherKeyCache::insertCipherKey(const EncryptCipher int baseCipherLen, const int64_t refreshAt, const int64_t expireAt) { - if (domainId == ENCRYPT_INVALID_DOMAIN_ID || baseCipherId == ENCRYPT_INVALID_CIPHER_KEY_ID) { + if (domainId == INVALID_ENCRYPT_DOMAIN_ID || baseCipherId == INVALID_ENCRYPT_CIPHER_KEY_ID) { throw encrypt_invalid_id(); } @@ -366,8 +366,8 @@ Reference BlobCipherKeyCache::insertCipherKey(const EncryptCipher const EncryptCipherRandomSalt& salt, const int64_t refreshAt, const int64_t expireAt) { - if (domainId == ENCRYPT_INVALID_DOMAIN_ID || baseCipherId == ENCRYPT_INVALID_CIPHER_KEY_ID || - salt == ENCRYPT_INVALID_RANDOM_SALT) { + if (domainId == INVALID_ENCRYPT_DOMAIN_ID || baseCipherId == INVALID_ENCRYPT_CIPHER_KEY_ID || + salt == INVALID_ENCRYPT_RANDOM_SALT) { throw encrypt_invalid_id(); } @@ -397,7 +397,7 @@ Reference BlobCipherKeyCache::insertCipherKey(const EncryptCipher } Reference BlobCipherKeyCache::getLatestCipherKey(const EncryptCipherDomainId& domainId) { - if (domainId == ENCRYPT_INVALID_DOMAIN_ID) { + if (domainId == INVALID_ENCRYPT_DOMAIN_ID) { TraceEvent(SevWarn, "BlobCipher.GetLatestCipherKeyInvalidID").detail("DomainId", domainId); throw encrypt_invalid_id(); } @@ -990,7 +990,7 @@ TEST_CASE("flow/BlobCipher") { cipherKeyCache->getLatestCipherKey(deterministicRandom()->randomInt(minDomainId, maxDomainId)); ASSERT(!latestKeyNonexists.isValid()); try { - cipherKeyCache->getLatestCipherKey(ENCRYPT_INVALID_DOMAIN_ID); + cipherKeyCache->getLatestCipherKey(INVALID_ENCRYPT_DOMAIN_ID); ASSERT(false); // shouldn't get here } catch (Error& e) { ASSERT_EQ(e.code(), error_code_encrypt_invalid_id); diff --git a/fdbclient/Tenant.cpp b/fdbclient/Tenant.cpp index 835dcb32b9..dd9c6c9796 100644 --- a/fdbclient/Tenant.cpp +++ b/fdbclient/Tenant.cpp @@ -21,20 +21,28 @@ #include "fdbclient/NativeAPI.actor.h" #include "fdbclient/SystemData.h" #include "fdbclient/Tenant.h" +#include "fdbrpc/TenantInfo.h" +#include "flow/BooleanParam.h" #include "libb64/encode.h" #include "flow/ApiVersion.h" #include "flow/UnitTest.h" +FDB_DEFINE_BOOLEAN_PARAM(EnforceValidTenantId); + Key TenantMapEntry::idToPrefix(int64_t id) { int64_t swapped = bigEndian64(id); return StringRef(reinterpret_cast(&swapped), TENANT_PREFIX_SIZE); } -int64_t TenantMapEntry::prefixToId(KeyRef prefix) { +int64_t TenantMapEntry::prefixToId(KeyRef prefix, EnforceValidTenantId enforceValidTenantId) { ASSERT(prefix.size() == TENANT_PREFIX_SIZE); int64_t id = *reinterpret_cast(prefix.begin()); id = bigEndian64(id); - ASSERT(id >= 0); + if (enforceValidTenantId) { + ASSERT(id >= 0); + } else if (id < 0) { + return TenantInfo::INVALID_TENANT; + } return id; } diff --git a/fdbclient/include/fdbclient/BlobCipher.h b/fdbclient/include/fdbclient/BlobCipher.h index 4c44ce9b69..9c61661955 100644 --- a/fdbclient/include/fdbclient/BlobCipher.h +++ b/fdbclient/include/fdbclient/BlobCipher.h @@ -140,9 +140,9 @@ private: #pragma pack(push, 1) // exact fit - no padding struct BlobCipherDetails { // Encryption domain boundary identifier. - EncryptCipherDomainId encryptDomainId = ENCRYPT_INVALID_DOMAIN_ID; + EncryptCipherDomainId encryptDomainId = INVALID_ENCRYPT_DOMAIN_ID; // BaseCipher encryption key identifier - EncryptCipherBaseKeyId baseCipherId = ENCRYPT_INVALID_CIPHER_KEY_ID; + EncryptCipherBaseKeyId baseCipherId = INVALID_ENCRYPT_CIPHER_KEY_ID; // Random salt EncryptCipherRandomSalt salt{}; diff --git a/fdbclient/include/fdbclient/CommitTransaction.h b/fdbclient/include/fdbclient/CommitTransaction.h index bbf194dc72..ab044549dc 100644 --- a/fdbclient/include/fdbclient/CommitTransaction.h +++ b/fdbclient/include/fdbclient/CommitTransaction.h @@ -143,7 +143,7 @@ struct MutationRef { const EncryptCipherDomainId& domainId, Arena& arena, BlobCipherMetrics::UsageType usageType) const { - ASSERT_NE(domainId, ENCRYPT_INVALID_DOMAIN_ID); + ASSERT_NE(domainId, INVALID_ENCRYPT_DOMAIN_ID); auto textCipherItr = cipherKeys.find(domainId); auto headerCipherItr = cipherKeys.find(ENCRYPT_HEADER_DOMAIN_ID); ASSERT(textCipherItr != cipherKeys.end() && textCipherItr->second.isValid()); diff --git a/fdbclient/include/fdbclient/EncryptKeyProxyInterface.h b/fdbclient/include/fdbclient/EncryptKeyProxyInterface.h index 3a88a855e0..5f4d56eb96 100644 --- a/fdbclient/include/fdbclient/EncryptKeyProxyInterface.h +++ b/fdbclient/include/fdbclient/EncryptKeyProxyInterface.h @@ -147,7 +147,7 @@ struct EKPGetBaseCipherKeysRequestInfo { EncryptCipherDomainNameRef domainName; EKPGetBaseCipherKeysRequestInfo() - : domainId(ENCRYPT_INVALID_DOMAIN_ID), baseCipherId(ENCRYPT_INVALID_CIPHER_KEY_ID) {} + : domainId(INVALID_ENCRYPT_DOMAIN_ID), baseCipherId(INVALID_ENCRYPT_CIPHER_KEY_ID) {} EKPGetBaseCipherKeysRequestInfo(const EncryptCipherDomainId dId, const EncryptCipherBaseKeyId bCId, StringRef name, @@ -205,7 +205,7 @@ struct EKPGetLatestCipherKeysRequestInfo { // {domainId, cipherBaseId} tuple EncryptCipherDomainNameRef domainName; - EKPGetLatestCipherKeysRequestInfo() : domainId(ENCRYPT_INVALID_DOMAIN_ID) {} + EKPGetLatestCipherKeysRequestInfo() : domainId(INVALID_ENCRYPT_DOMAIN_ID) {} EKPGetLatestCipherKeysRequestInfo(const EncryptCipherDomainId dId, StringRef name, Arena& arena) : domainId(dId), domainName(StringRef(arena, name)) {} diff --git a/fdbclient/include/fdbclient/GetEncryptCipherKeys.actor.h b/fdbclient/include/fdbclient/GetEncryptCipherKeys.actor.h index 1ccfed2997..1e91659f4a 100644 --- a/fdbclient/include/fdbclient/GetEncryptCipherKeys.actor.h +++ b/fdbclient/include/fdbclient/GetEncryptCipherKeys.actor.h @@ -269,7 +269,7 @@ Future getLatestEncryptCipherKeysForDomain(Reference domains; domains[domainId] = domainName; - domains[ENCRYPT_HEADER_DOMAIN_ID] = FDB_DEFAULT_ENCRYPT_DOMAIN_NAME; + domains[ENCRYPT_HEADER_DOMAIN_ID] = FDB_ENCRYPT_HEADER_DOMAIN_NAME; std::unordered_map> cipherKeys = wait(getLatestEncryptCipherKeys(db, domains, usageType)); ASSERT(cipherKeys.count(domainId) > 0); diff --git a/fdbclient/include/fdbclient/Tenant.h b/fdbclient/include/fdbclient/Tenant.h index d75629da10..b6341550aa 100644 --- a/fdbclient/include/fdbclient/Tenant.h +++ b/fdbclient/include/fdbclient/Tenant.h @@ -27,6 +27,7 @@ #include "fdbclient/VersionedMap.h" #include "fdbclient/KeyBackedTypes.h" #include "fdbrpc/TenantInfo.h" +#include "flow/BooleanParam.h" #include "flow/flat_buffers.h" typedef StringRef TenantNameRef; @@ -64,11 +65,13 @@ enum class TenantLockState { UNLOCKED, READ_ONLY, LOCKED }; constexpr int TENANT_PREFIX_SIZE = sizeof(int64_t); +FDB_DECLARE_BOOLEAN_PARAM(EnforceValidTenantId); + struct TenantMapEntry { constexpr static FileIdentifier file_identifier = 12247338; static Key idToPrefix(int64_t id); - static int64_t prefixToId(KeyRef prefix); + static int64_t prefixToId(KeyRef prefix, EnforceValidTenantId enforceTenantId = EnforceValidTenantId::True); static std::string tenantStateToString(TenantState tenantState); static TenantState stringToTenantState(std::string stateStr); diff --git a/fdbserver/ApplyMetadataMutation.cpp b/fdbserver/ApplyMetadataMutation.cpp index bcb9c8b6e5..2353e4326b 100644 --- a/fdbserver/ApplyMetadataMutation.cpp +++ b/fdbserver/ApplyMetadataMutation.cpp @@ -83,8 +83,8 @@ public: uid_applyMutationsData(proxyCommitData_.firstProxy ? &proxyCommitData_.uid_applyMutationsData : nullptr), commit(proxyCommitData_.commit), cx(proxyCommitData_.cx), committedVersion(&proxyCommitData_.committedVersion), storageCache(&proxyCommitData_.storageCache), tag_popped(&proxyCommitData_.tag_popped), - tssMapping(&proxyCommitData_.tssMapping), tenantMap(&proxyCommitData_.tenantMap), initialCommit(initialCommit_), - dbInfo(proxyCommitData_.db) {} + tssMapping(&proxyCommitData_.tssMapping), tenantMap(&proxyCommitData_.tenantMap), + tenantIdIndex(&proxyCommitData_.tenantIdIndex), initialCommit(initialCommit_), dbInfo(proxyCommitData_.db) {} ApplyMetadataMutationsImpl(const SpanContext& spanContext_, ResolverData& resolverData_, @@ -134,6 +134,7 @@ private: std::unordered_map* tssMapping = nullptr; std::map* tenantMap = nullptr; + std::unordered_map* tenantIdIndex = nullptr; // true if the mutations were already written to the txnStateStore as part of recovery bool initialCommit = false; @@ -659,13 +660,21 @@ private: void checkSetTenantMapPrefix(MutationRef m) { KeyRef prefix = TenantMetadata::tenantMap().subspace.begin; if (m.param1.startsWith(prefix)) { + TenantName tenantName = m.param1.removePrefix(prefix); + TenantMapEntry tenantEntry = TenantMapEntry::decode(m.param2); + if (tenantMap) { ASSERT(version != invalidVersion); - TenantName tenantName = m.param1.removePrefix(prefix); - TenantMapEntry tenantEntry = TenantMapEntry::decode(m.param2); - TraceEvent("CommitProxyInsertTenant", dbgid).detail("Tenant", tenantName).detail("Version", version); + TraceEvent("CommitProxyInsertTenant", dbgid) + .detail("Tenant", tenantName) + .detail("Id", tenantEntry.id) + .detail("Version", version); + (*tenantMap)[tenantName] = tenantEntry; + if (tenantIdIndex) { + (*tenantIdIndex)[tenantEntry.id] = tenantName; + } } if (!initialCommit) { @@ -1072,6 +1081,17 @@ private: auto startItr = tenantMap->lower_bound(startTenant); auto endItr = tenantMap->lower_bound(endTenant); + + if (tenantIdIndex) { + // Iterate over iterator-range and remove entries from TenantIdName map + // TODO: O(n) operation, optimize cpu + auto itr = startItr; + while (itr != endItr) { + tenantIdIndex->erase(itr->second.id); + itr++; + } + } + tenantMap->erase(startItr, endItr); } diff --git a/fdbserver/CommitProxyServer.actor.cpp b/fdbserver/CommitProxyServer.actor.cpp index 8e420ef531..51719c3177 100644 --- a/fdbserver/CommitProxyServer.actor.cpp +++ b/fdbserver/CommitProxyServer.actor.cpp @@ -30,7 +30,9 @@ #include "fdbclient/CommitProxyInterface.h" #include "fdbclient/NativeAPI.actor.h" #include "fdbclient/SystemData.h" +#include "fdbclient/Tenant.h" #include "fdbclient/TransactionLineage.h" +#include "fdbrpc/TenantInfo.h" #include "fdbrpc/sim_validation.h" #include "fdbserver/ApplyMetadataMutation.h" #include "fdbserver/ConflictSet.h" @@ -51,6 +53,7 @@ #include "fdbserver/WaitFailure.h" #include "fdbserver/WorkerInterface.actor.h" #include "flow/ActorCollection.h" +#include "flow/EncryptUtils.h" #include "flow/Error.h" #include "flow/IRandom.h" #include "flow/Knobs.h" @@ -874,6 +877,81 @@ ACTOR Future preresolutionProcessing(CommitBatchContext* self) { return Void(); } +namespace { +// Routine allows caller to find TenantName for a given TenantId. It returns empty optional when either TenantId is +// invalid or tenant is unknown +Optional getTenantName(ProxyCommitData* commitData, int64_t tenantId) { + if (tenantId != TenantInfo::INVALID_TENANT) { + auto itr = commitData->tenantIdIndex.find(tenantId); + if (itr != commitData->tenantIdIndex.end()) { + return Optional(itr->second); + } + } + + return Optional(); +} + +std::pair getEncryptDetailsFromMutationRef(ProxyCommitData* commitData, + MutationRef m) { + std::pair details(EncryptCipherDomainName(), + INVALID_ENCRYPT_DOMAIN_ID); + + // Possible scenarios: + // 1. Encryption domain (Tenant details) weren't explicitly provided, extract Tenant details using + // TenantPrefix (first 8 bytes of FDBKey) + // 2. Encryption domain isn't available, leverage 'default encryption domain' + + if (isSystemKey(m.param1)) { + // Encryption domain == FDB SystemKeyspace encryption domain + details.first = EncryptCipherDomainName(FDB_SYSTEM_KEYSPACE_ENCRYPT_DOMAIN_NAME); + details.second = SYSTEM_KEYSPACE_ENCRYPT_DOMAIN_ID; + } else if (commitData->tenantMap.empty()) { + // Cluster serves no-tenants; use 'default encryption domain' + } else if (isSingleKeyMutation((MutationRef::Type)m.type)) { + ASSERT_NE((MutationRef::Type)m.type, MutationRef::Type::ClearRange); + + if (m.param1.size() >= TENANT_PREFIX_SIZE) { + // Parse mutation key to determine mutation encryption domain + StringRef prefix = m.param1.substr(0, TENANT_PREFIX_SIZE); + int64_t tenantId = TenantMapEntry::prefixToId(prefix, EnforceValidTenantId::False); + if (tenantId != TenantInfo::INVALID_TENANT) { + Optional tenantName = getTenantName(commitData, tenantId); + if (tenantName.present()) { + details.first = tenantName.get(); + details.second = tenantId; + } + } else { + // Leverage 'default encryption domain' + } + } + } else { + // ClearRange is the 'only' MultiKey transaction allowed + ASSERT_EQ((MutationRef::Type)m.type, MutationRef::Type::ClearRange); + + // FIXME: Handle Clear-range transaction, actions needed: + // 1. Transaction range can spawn multiple encryption domains (tenants) + // 2. Transaction can be a multi-key transaction spawning multiple tenants + // For now fallback to 'default encryption domain' + + CODE_PROBE(true, "ClearRange mutation encryption"); + } + + // Unknown tenant, fallback to fdb default encryption domain + if (details.second == INVALID_ENCRYPT_DOMAIN_ID) { + ASSERT_EQ(details.first.size(), 0); + details.first = EncryptCipherDomainName(FDB_DEFAULT_ENCRYPT_DOMAIN_NAME); + details.second = FDB_DEFAULT_ENCRYPT_DOMAIN_ID; + + CODE_PROBE(true, "Default domain mutation encryption"); + } + + ASSERT_GT(details.first.size(), 0); + + return details; +} + +} // namespace + ACTOR Future getResolution(CommitBatchContext* self) { state double resolutionStart = now(); // Sending these requests is the fuzzy border between phase 1 and phase 2; it could conceivably overlap with @@ -918,18 +996,24 @@ ACTOR Future getResolution(CommitBatchContext* self) { state Future>> getCipherKeys; if (pProxyCommitData->isEncryptionEnabled) { static std::unordered_map defaultDomains = { - { SYSTEM_KEYSPACE_ENCRYPT_DOMAIN_ID, FDB_DEFAULT_ENCRYPT_DOMAIN_NAME }, - { ENCRYPT_HEADER_DOMAIN_ID, FDB_DEFAULT_ENCRYPT_DOMAIN_NAME } + { SYSTEM_KEYSPACE_ENCRYPT_DOMAIN_ID, FDB_SYSTEM_KEYSPACE_ENCRYPT_DOMAIN_NAME }, + { ENCRYPT_HEADER_DOMAIN_ID, FDB_ENCRYPT_HEADER_DOMAIN_NAME }, + { FDB_DEFAULT_ENCRYPT_DOMAIN_ID, FDB_DEFAULT_ENCRYPT_DOMAIN_NAME } }; std::unordered_map encryptDomains = defaultDomains; for (int t = 0; t < trs.size(); t++) { TenantInfo const& tenantInfo = trs[t].tenantInfo; int64_t tenantId = tenantInfo.tenantId; Optional const& tenantName = tenantInfo.name; - // TODO(yiwu): In raw access mode, use tenant prefix to figure out tenant id for user data if (tenantId != TenantInfo::INVALID_TENANT) { ASSERT(tenantName.present()); encryptDomains[tenantId] = tenantName.get(); + } else { + for (auto m : trs[t].transaction.mutations) { + std::pair details = + getEncryptDetailsFromMutationRef(pProxyCommitData, m); + encryptDomains[details.second] = details.first; + } } } getCipherKeys = getLatestEncryptCipherKeys(pProxyCommitData->db, encryptDomains, BlobCipherMetrics::TLOG); @@ -1157,18 +1241,24 @@ ACTOR Future applyMetadataToCommittedTransactions(CommitBatchContext* self } void writeMutation(CommitBatchContext* self, int64_t tenantId, const MutationRef& mutation) { - static_assert(TenantInfo::INVALID_TENANT == ENCRYPT_INVALID_DOMAIN_ID); - if (!self->pProxyCommitData->isEncryptionEnabled || tenantId == TenantInfo::INVALID_TENANT) { - // TODO(yiwu): In raw access mode, use tenant prefix to figure out tenant id for user data - bool isRawAccess = tenantId == TenantInfo::INVALID_TENANT && !isSystemKey(mutation.param1) && - !(mutation.type == MutationRef::ClearRange && isSystemKey(mutation.param2)) && - self->pProxyCommitData->db->get().client.tenantMode == TenantMode::REQUIRED; - CODE_PROBE(isRawAccess, "Raw access to tenant key space"); - self->toCommit.writeTypedMessage(mutation); - } else { + static_assert(TenantInfo::INVALID_TENANT == INVALID_ENCRYPT_DOMAIN_ID); + + if (self->pProxyCommitData->isEncryptionEnabled) { + EncryptCipherDomainId domainId = tenantId; + if (domainId == INVALID_ENCRYPT_DOMAIN_ID) { + std::pair p = + getEncryptDetailsFromMutationRef(self->pProxyCommitData, mutation); + domainId = p.second; + + CODE_PROBE(true, "Raw access mutation encryption"); + } + + ASSERT_NE(domainId, INVALID_ENCRYPT_DOMAIN_ID); + Arena arena; - self->toCommit.writeTypedMessage( - mutation.encrypt(self->cipherKeys, tenantId /*domainId*/, arena, BlobCipherMetrics::TLOG)); + self->toCommit.writeTypedMessage(mutation.encrypt(self->cipherKeys, domainId, arena, BlobCipherMetrics::TLOG)); + } else { + self->toCommit.writeTypedMessage(mutation); } } diff --git a/fdbserver/RESTKmsConnector.actor.cpp b/fdbserver/RESTKmsConnector.actor.cpp index 2aef038144..c1f0fbbc23 100644 --- a/fdbserver/RESTKmsConnector.actor.cpp +++ b/fdbserver/RESTKmsConnector.actor.cpp @@ -1102,9 +1102,8 @@ void testGetEncryptKeysByKeyIdsRequestBody(Reference ctx, A const int nKeys = deterministicRandom()->randomInt(7, 8); for (int i = 1; i < nKeys; i++) { EncryptCipherDomainId domainId = getRandomDomainId(); - EncryptCipherDomainNameRef domainName = domainId < 0 - ? StringRef(arena, std::string(FDB_DEFAULT_ENCRYPT_DOMAIN_NAME)) - : StringRef(arena, std::to_string(domainId)); + EncryptCipherDomainNameRef domainName = domainId < 0 ? StringRef(arena, FDB_DEFAULT_ENCRYPT_DOMAIN_NAME) + : StringRef(arena, std::to_string(domainId)); req.encryptKeyInfos.emplace_back_deep(req.arena, domainId, i, domainName); keyMap[i] = domainId; } @@ -1141,9 +1140,8 @@ void testGetEncryptKeysByDomainIdsRequestBody(Reference ctx const int nKeys = deterministicRandom()->randomInt(7, 25); for (int i = 1; i < nKeys; i++) { EncryptCipherDomainId domainId = getRandomDomainId(); - EncryptCipherDomainNameRef domainName = domainId < 0 - ? StringRef(arena, std::string(FDB_DEFAULT_ENCRYPT_DOMAIN_NAME)) - : StringRef(arena, std::to_string(domainId)); + EncryptCipherDomainNameRef domainName = domainId < 0 ? StringRef(arena, FDB_DEFAULT_ENCRYPT_DOMAIN_NAME) + : StringRef(arena, std::to_string(domainId)); KmsConnLookupDomainIdsReqInfoRef reqInfo(req.arena, domainId, domainName); if (domainInfoMap.insert({ domainId, reqInfo }).second) { req.encryptDomainInfos.push_back(req.arena, reqInfo); diff --git a/fdbserver/include/fdbserver/IEncryptionKeyProvider.actor.h b/fdbserver/include/fdbserver/IEncryptionKeyProvider.actor.h index f492e568c9..8c29fccfdb 100644 --- a/fdbserver/include/fdbserver/IEncryptionKeyProvider.actor.h +++ b/fdbserver/include/fdbserver/IEncryptionKeyProvider.actor.h @@ -27,10 +27,12 @@ #include "fdbclient/GetEncryptCipherKeys.actor.h" #include "fdbclient/Tenant.h" + #include "fdbserver/EncryptionOpsUtils.h" #include "fdbserver/ServerDBInfo.h" -#include "flow/Arena.h" +#include "flow/Arena.h" +#include "flow/EncryptUtils.h" #define XXH_INLINE_ALL #include "flow/xxhash.h" @@ -241,8 +243,8 @@ private: const KeyRef& end, EncryptCipherDomainNameRef* domainName) { int64_t domainId = SYSTEM_KEYSPACE_ENCRYPT_DOMAIN_ID; - int64_t beginTenantId = getTenant(begin, true /*inclusive*/); - int64_t endTenantId = getTenant(end, false /*inclusive*/); + int64_t beginTenantId = getTenantId(begin, true /*inclusive*/); + int64_t endTenantId = getTenantId(end, false /*inclusive*/); if (beginTenantId == endTenantId && beginTenantId != SYSTEM_KEYSPACE_ENCRYPT_DOMAIN_ID) { ASSERT(tenantPrefixIndex.isValid()); Key tenantPrefix = TenantMapEntry::idToPrefix(beginTenantId); @@ -256,22 +258,31 @@ private: } } if (domainId == SYSTEM_KEYSPACE_ENCRYPT_DOMAIN_ID) { - *domainName = FDB_DEFAULT_ENCRYPT_DOMAIN_NAME; + *domainName = FDB_SYSTEM_KEYSPACE_ENCRYPT_DOMAIN_NAME; } return domainId; } - int64_t getTenant(const KeyRef& key, bool inclusive) { + int64_t getTenantId(const KeyRef& key, bool inclusive) { // A valid tenant id is always a valid encrypt domain id. - static_assert(ENCRYPT_INVALID_DOMAIN_ID < 0); - if (key.size() < TENANT_PREFIX_SIZE || key >= systemKeys.begin) { + static_assert(INVALID_ENCRYPT_DOMAIN_ID == -1); + + if (key.size() && key >= systemKeys.begin) { return SYSTEM_KEYSPACE_ENCRYPT_DOMAIN_ID; } - // TODO(yiwu): Use TenantMapEntry::prefixToId() instead. - int64_t tenantId = bigEndian64(*reinterpret_cast(key.begin())); - if (tenantId < 0) { - return SYSTEM_KEYSPACE_ENCRYPT_DOMAIN_ID; + + if (key.size() < TENANT_PREFIX_SIZE) { + // Encryption domain information not available, leverage 'default encryption domain' + return FDB_DEFAULT_ENCRYPT_DOMAIN_ID; } + + StringRef prefix = key.substr(0, TENANT_PREFIX_SIZE); + int64_t tenantId = TenantMapEntry::prefixToId(prefix, EnforceValidTenantId::False); + if (tenantId == TenantInfo::INVALID_TENANT) { + // Encryption domain information not available, leverage 'default encryption domain' + return FDB_DEFAULT_ENCRYPT_DOMAIN_ID; + } + if (!inclusive && key.size() == TENANT_PREFIX_SIZE) { tenantId = tenantId - 1; } diff --git a/fdbserver/include/fdbserver/KmsConnectorInterface.h b/fdbserver/include/fdbserver/KmsConnectorInterface.h index 73afc1cbc0..ac31bc5d37 100644 --- a/fdbserver/include/fdbserver/KmsConnectorInterface.h +++ b/fdbserver/include/fdbserver/KmsConnectorInterface.h @@ -132,7 +132,7 @@ struct KmsConnLookupKeyIdsReqInfoRef { EncryptCipherDomainNameRef domainName; KmsConnLookupKeyIdsReqInfoRef() - : domainId(ENCRYPT_INVALID_DOMAIN_ID), baseCipherId(ENCRYPT_INVALID_CIPHER_KEY_ID) {} + : domainId(INVALID_ENCRYPT_DOMAIN_ID), baseCipherId(INVALID_ENCRYPT_CIPHER_KEY_ID) {} explicit KmsConnLookupKeyIdsReqInfoRef(Arena& arena, const EncryptCipherDomainId dId, const EncryptCipherBaseKeyId bCId, @@ -185,7 +185,7 @@ struct KmsConnLookupDomainIdsReqInfoRef { EncryptCipherDomainId domainId; EncryptCipherDomainNameRef domainName; - KmsConnLookupDomainIdsReqInfoRef() : domainId(ENCRYPT_INVALID_DOMAIN_ID) {} + KmsConnLookupDomainIdsReqInfoRef() : domainId(INVALID_ENCRYPT_DOMAIN_ID) {} explicit KmsConnLookupDomainIdsReqInfoRef(Arena& arena, const EncryptCipherDomainId dId, StringRef name) : domainId(dId), domainName(StringRef(arena, name)) {} explicit KmsConnLookupDomainIdsReqInfoRef(const EncryptCipherDomainId dId, StringRef name) diff --git a/fdbserver/include/fdbserver/ProxyCommitData.actor.h b/fdbserver/include/fdbserver/ProxyCommitData.actor.h index c6e173387a..be6d2aa9a1 100644 --- a/fdbserver/include/fdbserver/ProxyCommitData.actor.h +++ b/fdbserver/include/fdbserver/ProxyCommitData.actor.h @@ -20,6 +20,7 @@ #pragma once #include "fdbserver/EncryptionOpsUtils.h" +#include #if defined(NO_INTELLISENSE) && !defined(FDBSERVER_PROXYCOMMITDATA_ACTOR_G_H) #define FDBSERVER_PROXYCOMMITDATA_ACTOR_G_H #include "fdbserver/ProxyCommitData.actor.g.h" @@ -173,6 +174,7 @@ struct ProxyCommitData { UID dbgid; int64_t commitBatchesMemBytesCount; std::map tenantMap; + std::unordered_map tenantIdIndex; ProxyStats stats; MasterInterface master; std::vector resolvers; diff --git a/fdbserver/workloads/EncryptionOps.actor.cpp b/fdbserver/workloads/EncryptionOps.actor.cpp index c4b6a98a34..eb965f9598 100644 --- a/fdbserver/workloads/EncryptionOps.actor.cpp +++ b/fdbserver/workloads/EncryptionOps.actor.cpp @@ -221,6 +221,7 @@ struct EncryptionOpsWorkload : TestWorkload { cipherKeyCache->resetEncryptDomainId(id); } + cipherKeyCache->resetEncryptDomainId(FDB_DEFAULT_ENCRYPT_DOMAIN_ID); cipherKeyCache->resetEncryptDomainId(SYSTEM_KEYSPACE_ENCRYPT_DOMAIN_ID); cipherKeyCache->resetEncryptDomainId(ENCRYPT_HEADER_DOMAIN_ID); diff --git a/flow/include/flow/EncryptUtils.h b/flow/include/flow/EncryptUtils.h index e20af33d1e..8ac82e6de4 100644 --- a/flow/include/flow/EncryptUtils.h +++ b/flow/include/flow/EncryptUtils.h @@ -29,22 +29,27 @@ #include #include -#define ENCRYPT_INVALID_DOMAIN_ID -1 -#define ENCRYPT_INVALID_CIPHER_KEY_ID 0 -#define ENCRYPT_INVALID_RANDOM_SALT 0 - #define AUTH_TOKEN_SIZE 32 -#define SYSTEM_KEYSPACE_ENCRYPT_DOMAIN_ID -2 -#define ENCRYPT_HEADER_DOMAIN_ID -3 - -const std::string FDB_DEFAULT_ENCRYPT_DOMAIN_NAME = "FdbDefaultEncryptDomain"; - using EncryptCipherDomainId = int64_t; using EncryptCipherDomainNameRef = StringRef; +using EncryptCipherDomainName = Standalone; using EncryptCipherBaseKeyId = uint64_t; using EncryptCipherRandomSalt = uint64_t; +constexpr const EncryptCipherDomainId INVALID_ENCRYPT_DOMAIN_ID = -1; +constexpr const EncryptCipherDomainId SYSTEM_KEYSPACE_ENCRYPT_DOMAIN_ID = -2; +constexpr const EncryptCipherDomainId ENCRYPT_HEADER_DOMAIN_ID = -3; +constexpr const EncryptCipherDomainId FDB_DEFAULT_ENCRYPT_DOMAIN_ID = -4; + +constexpr const EncryptCipherBaseKeyId INVALID_ENCRYPT_CIPHER_KEY_ID = 0; + +constexpr const EncryptCipherRandomSalt INVALID_ENCRYPT_RANDOM_SALT = 0; + +const EncryptCipherDomainNameRef FDB_SYSTEM_KEYSPACE_ENCRYPT_DOMAIN_NAME = "FdbSystemKeyspaceEncryptDomain"_sr; +const EncryptCipherDomainNameRef FDB_DEFAULT_ENCRYPT_DOMAIN_NAME = "FdbDefaultEncryptDomain"_sr; +const EncryptCipherDomainNameRef FDB_ENCRYPT_HEADER_DOMAIN_NAME = "FdbEncryptHeaderDomain"_sr; + typedef enum { ENCRYPT_CIPHER_MODE_NONE = 0, ENCRYPT_CIPHER_MODE_AES_256_CTR = 1, diff --git a/flow/include/flow/error_definitions.h b/flow/include/flow/error_definitions.h index d4d33d9ef2..664078fe1e 100755 --- a/flow/include/flow/error_definitions.h +++ b/flow/include/flow/error_definitions.h @@ -316,28 +316,28 @@ ERROR( json_malformed, 2401, "JSON string was malformed") ERROR( json_eof_expected, 2402, "JSON string did not terminate where expected") // 2500 - disk snapshot based backup errors -ERROR( snap_disable_tlog_pop_failed, 2500, "Failed to disable tlog pops") -ERROR( snap_storage_failed, 2501, "Failed to snapshot storage nodes") -ERROR( snap_tlog_failed, 2502, "Failed to snapshot TLog nodes") -ERROR( snap_coord_failed, 2503, "Failed to snapshot coordinator nodes") -ERROR( snap_enable_tlog_pop_failed, 2504, "Failed to enable tlog pops") -ERROR( snap_path_not_whitelisted, 2505, "Snapshot create binary path not whitelisted") -ERROR( snap_not_fully_recovered_unsupported, 2506, "Unsupported when the cluster is not fully recovered") -ERROR( snap_log_anti_quorum_unsupported, 2507, "Unsupported when log anti quorum is configured") -ERROR( snap_with_recovery_unsupported, 2508, "Cluster recovery during snapshot operation not supported") -ERROR( snap_invalid_uid_string, 2509, "The given uid string is not a 32-length hex string") +ERROR( snap_disable_tlog_pop_failed, 2500, "Failed to disable tlog pops" ) +ERROR( snap_storage_failed, 2501, "Failed to snapshot storage nodes" ) +ERROR( snap_tlog_failed, 2502, "Failed to snapshot TLog nodes" ) +ERROR( snap_coord_failed, 2503, "Failed to snapshot coordinator nodes" ) +ERROR( snap_enable_tlog_pop_failed, 2504, "Failed to enable tlog pops" ) +ERROR( snap_path_not_whitelisted, 2505, "Snapshot create binary path not whitelisted" ) +ERROR( snap_not_fully_recovered_unsupported, 2506, "Unsupported when the cluster is not fully recovered" ) +ERROR( snap_log_anti_quorum_unsupported, 2507, "Unsupported when log anti quorum is configured" ) +ERROR( snap_with_recovery_unsupported, 2508, "Cluster recovery during snapshot operation not supported" ) +ERROR( snap_invalid_uid_string, 2509, "The given uid string is not a 32-length hex string" ) // 27XX - Encryption operations errors -ERROR( encrypt_ops_error, 2700, "Encryption operation error") -ERROR( encrypt_header_metadata_mismatch, 2701, "Encryption header metadata mismatch") -ERROR( encrypt_key_not_found, 2702, "Expected encryption key is missing") -ERROR( encrypt_key_ttl_expired, 2703, "Expected encryption key TTL has expired") -ERROR( encrypt_header_authtoken_mismatch, 2704, "Encryption header authentication token mismatch") -ERROR( encrypt_update_cipher, 2705, "Attempt to update encryption cipher key") -ERROR( encrypt_invalid_id, 2706, "Invalid encryption cipher details") -ERROR( encrypt_keys_fetch_failed, 2707, "Encryption keys fetch from external KMS failed") -ERROR( encrypt_invalid_kms_config, 2708, "Invalid encryption/kms configuration: discovery-url, validation-token, endpoint etc.") -ERROR( encrypt_unsupported, 2709, "Encryption not supported") +ERROR( encrypt_ops_error, 2700, "Encryption operation error" ) +ERROR( encrypt_header_metadata_mismatch, 2701, "Encryption header metadata mismatch" ) +ERROR( encrypt_key_not_found, 2702, "Expected encryption key is missing" ) +ERROR( encrypt_key_ttl_expired, 2703, "Expected encryption key TTL has expired" ) +ERROR( encrypt_header_authtoken_mismatch, 2704, "Encryption header authentication token mismatch" ) +ERROR( encrypt_update_cipher, 2705, "Attempt to update encryption cipher key" ) +ERROR( encrypt_invalid_id, 2706, "Invalid encryption cipher details" ) +ERROR( encrypt_keys_fetch_failed, 2707, "Encryption keys fetch from external KMS failed" ) +ERROR( encrypt_invalid_kms_config, 2708, "Invalid encryption/kms configuration: discovery-url, validation-token, endpoint etc." ) +ERROR( encrypt_unsupported, 2709, "Encryption not supported" ) // 4xxx Internal errors (those that should be generated only by bugs) are decimal 4xxx ERROR( unknown_error, 4000, "An unknown error occurred" ) // C++ exception not of type Error