Introduce "default encryption domain" (#8139)

* Introduce "default encryption domain"

Description

In current FDB native encryption data at-rest implementation,
an entity getting encrypted (mutation, KV and/or file) is categorized
into one of following encryption domains:
1. Tenant domain, where, Encryption domain == Tenant boundaries
2. FDB system keyspace - FDB metadata encryption domain
3. FDB Encryption Header domain - used to generate digest for
plaintext EncryptionHeader.

The scheme doesn't support encryption if an entity can't be categorized
into any of above mentioned encryption domains, for instance, non-tenant
mutations are NOT supported.

Patch extend the encryption support for mutations for which corresponding
Tenant information can't be obtained (Key length shorter than TenantPrefix)
and/or mutations do not belong to any valid Tenant
(FDB management cluster data) by mapping such mutations to a
"default encryption domain".

TODO

CommitProxy driven TLog encryption implementation requires every transaction
mutation to contain 1 KV, not crossing Tenant-boundaries. Only exception to
this rule is ClearRange mutations. For now ClearRange mutations are mapped
to 'default encryption domain', in subsequent patch appropriate handling
for ClearRange mutations shall be proposed.

Testing

devRunCorrectness - 100k

fdbclient/BlobCipher.cpp

@@ -190,7 +190,7 @@ BlobCipherKeyIdCache::BlobCipherKeyIdCache(EncryptCipherDomainId dId, size_t* si
 
 BlobCipherKeyIdCacheKey BlobCipherKeyIdCache::getCacheKey(const EncryptCipherBaseKeyId& baseCipherKeyId,
                                                           const EncryptCipherRandomSalt& salt) {
-	if (baseCipherKeyId == ENCRYPT_INVALID_CIPHER_KEY_ID || salt == ENCRYPT_INVALID_RANDOM_SALT) {
+	if (baseCipherKeyId == INVALID_ENCRYPT_CIPHER_KEY_ID || salt == INVALID_ENCRYPT_RANDOM_SALT) {
 		throw encrypt_invalid_id();
 	}
 	return std::make_pair(baseCipherKeyId, salt);
@@ -200,9 +200,9 @@ Reference<BlobCipherKey> BlobCipherKeyIdCache::getLatestCipherKey() {
 	if (!latestBaseCipherKeyId.present()) {
 		return Reference<BlobCipherKey>();
 	}
-	ASSERT_NE(latestBaseCipherKeyId.get(), ENCRYPT_INVALID_CIPHER_KEY_ID);
+	ASSERT_NE(latestBaseCipherKeyId.get(), INVALID_ENCRYPT_CIPHER_KEY_ID);
 	ASSERT(latestRandomSalt.present());
-	ASSERT_NE(latestRandomSalt.get(), ENCRYPT_INVALID_RANDOM_SALT);
+	ASSERT_NE(latestRandomSalt.get(), INVALID_ENCRYPT_RANDOM_SALT);
 
 	return getCipherByBaseCipherId(latestBaseCipherKeyId.get(), latestRandomSalt.get());
 }
@@ -221,7 +221,7 @@ Reference<BlobCipherKey> BlobCipherKeyIdCache::insertBaseCipherKey(const Encrypt
                                                                    int baseCipherLen,
                                                                    const int64_t refreshAt,
                                                                    const int64_t expireAt) {
-	ASSERT_GT(baseCipherId, ENCRYPT_INVALID_CIPHER_KEY_ID);
+	ASSERT_GT(baseCipherId, INVALID_ENCRYPT_CIPHER_KEY_ID);
 
 	// BaseCipherKeys are immutable, given the routine invocation updates 'latestCipher',
 	// ensure no key-tampering is done
@@ -269,8 +269,8 @@ Reference<BlobCipherKey> BlobCipherKeyIdCache::insertBaseCipherKey(const Encrypt
                                                                    const EncryptCipherRandomSalt& salt,
                                                                    const int64_t refreshAt,
                                                                    const int64_t expireAt) {
-	ASSERT_NE(baseCipherId, ENCRYPT_INVALID_CIPHER_KEY_ID);
+	ASSERT_NE(baseCipherId, INVALID_ENCRYPT_CIPHER_KEY_ID);
-	ASSERT_NE(salt, ENCRYPT_INVALID_RANDOM_SALT);
+	ASSERT_NE(salt, INVALID_ENCRYPT_RANDOM_SALT);
 
 	BlobCipherKeyIdCacheKey cacheKey = getCacheKey(baseCipherId, salt);
 
@@ -332,7 +332,7 @@ Reference<BlobCipherKey> BlobCipherKeyCache::insertCipherKey(const EncryptCipher
                                                              int baseCipherLen,
                                                              const int64_t refreshAt,
                                                              const int64_t expireAt) {
-	if (domainId == ENCRYPT_INVALID_DOMAIN_ID || baseCipherId == ENCRYPT_INVALID_CIPHER_KEY_ID) {
+	if (domainId == INVALID_ENCRYPT_DOMAIN_ID || baseCipherId == INVALID_ENCRYPT_CIPHER_KEY_ID) {
 		throw encrypt_invalid_id();
 	}
 
@@ -366,8 +366,8 @@ Reference<BlobCipherKey> BlobCipherKeyCache::insertCipherKey(const EncryptCipher
                                                              const EncryptCipherRandomSalt& salt,
                                                              const int64_t refreshAt,
                                                              const int64_t expireAt) {
-	if (domainId == ENCRYPT_INVALID_DOMAIN_ID || baseCipherId == ENCRYPT_INVALID_CIPHER_KEY_ID ||
+	if (domainId == INVALID_ENCRYPT_DOMAIN_ID || baseCipherId == INVALID_ENCRYPT_CIPHER_KEY_ID ||
-	    salt == ENCRYPT_INVALID_RANDOM_SALT) {
+	    salt == INVALID_ENCRYPT_RANDOM_SALT) {
 		throw encrypt_invalid_id();
 	}
 
@@ -397,7 +397,7 @@ Reference<BlobCipherKey> BlobCipherKeyCache::insertCipherKey(const EncryptCipher
 }
 
 Reference<BlobCipherKey> BlobCipherKeyCache::getLatestCipherKey(const EncryptCipherDomainId& domainId) {
-	if (domainId == ENCRYPT_INVALID_DOMAIN_ID) {
+	if (domainId == INVALID_ENCRYPT_DOMAIN_ID) {
 		TraceEvent(SevWarn, "BlobCipher.GetLatestCipherKeyInvalidID").detail("DomainId", domainId);
 		throw encrypt_invalid_id();
 	}
@@ -990,7 +990,7 @@ TEST_CASE("flow/BlobCipher") {
 	    cipherKeyCache->getLatestCipherKey(deterministicRandom()->randomInt(minDomainId, maxDomainId));
 	ASSERT(!latestKeyNonexists.isValid());
 	try {
-		cipherKeyCache->getLatestCipherKey(ENCRYPT_INVALID_DOMAIN_ID);
+		cipherKeyCache->getLatestCipherKey(INVALID_ENCRYPT_DOMAIN_ID);
 		ASSERT(false); // shouldn't get here
 	} catch (Error& e) {
 		ASSERT_EQ(e.code(), error_code_encrypt_invalid_id);

fdbclient/Tenant.cpp

@@ -21,20 +21,28 @@
 #include "fdbclient/NativeAPI.actor.h"
 #include "fdbclient/SystemData.h"
 #include "fdbclient/Tenant.h"
+#include "fdbrpc/TenantInfo.h"
+#include "flow/BooleanParam.h"
 #include "libb64/encode.h"
 #include "flow/ApiVersion.h"
 #include "flow/UnitTest.h"
 
+FDB_DEFINE_BOOLEAN_PARAM(EnforceValidTenantId);
+
 Key TenantMapEntry::idToPrefix(int64_t id) {
 	int64_t swapped = bigEndian64(id);
 	return StringRef(reinterpret_cast<const uint8_t*>(&swapped), TENANT_PREFIX_SIZE);
 }
 
-int64_t TenantMapEntry::prefixToId(KeyRef prefix) {
+int64_t TenantMapEntry::prefixToId(KeyRef prefix, EnforceValidTenantId enforceValidTenantId) {
 	ASSERT(prefix.size() == TENANT_PREFIX_SIZE);
 	int64_t id = *reinterpret_cast<const int64_t*>(prefix.begin());
 	id = bigEndian64(id);
-	ASSERT(id >= 0);
+	if (enforceValidTenantId) {
+		ASSERT(id >= 0);
+	} else if (id < 0) {
+		return TenantInfo::INVALID_TENANT;
+	}
 	return id;
 }
 

fdbclient/include/fdbclient/BlobCipher.h

@@ -140,9 +140,9 @@ private:
 #pragma pack(push, 1) // exact fit - no padding
 struct BlobCipherDetails {
 	// Encryption domain boundary identifier.
-	EncryptCipherDomainId encryptDomainId = ENCRYPT_INVALID_DOMAIN_ID;
+	EncryptCipherDomainId encryptDomainId = INVALID_ENCRYPT_DOMAIN_ID;
 	// BaseCipher encryption key identifier
-	EncryptCipherBaseKeyId baseCipherId = ENCRYPT_INVALID_CIPHER_KEY_ID;
+	EncryptCipherBaseKeyId baseCipherId = INVALID_ENCRYPT_CIPHER_KEY_ID;
 	// Random salt
 	EncryptCipherRandomSalt salt{};
 

fdbclient/include/fdbclient/CommitTransaction.h

@@ -143,7 +143,7 @@ struct MutationRef {
 	                    const EncryptCipherDomainId& domainId,
 	                    Arena& arena,
 	                    BlobCipherMetrics::UsageType usageType) const {
-		ASSERT_NE(domainId, ENCRYPT_INVALID_DOMAIN_ID);
+		ASSERT_NE(domainId, INVALID_ENCRYPT_DOMAIN_ID);
 		auto textCipherItr = cipherKeys.find(domainId);
 		auto headerCipherItr = cipherKeys.find(ENCRYPT_HEADER_DOMAIN_ID);
 		ASSERT(textCipherItr != cipherKeys.end() && textCipherItr->second.isValid());

fdbclient/include/fdbclient/EncryptKeyProxyInterface.h

@@ -147,7 +147,7 @@ struct EKPGetBaseCipherKeysRequestInfo {
 	EncryptCipherDomainNameRef domainName;
 
 	EKPGetBaseCipherKeysRequestInfo()
-	  : domainId(ENCRYPT_INVALID_DOMAIN_ID), baseCipherId(ENCRYPT_INVALID_CIPHER_KEY_ID) {}
+	  : domainId(INVALID_ENCRYPT_DOMAIN_ID), baseCipherId(INVALID_ENCRYPT_CIPHER_KEY_ID) {}
 	EKPGetBaseCipherKeysRequestInfo(const EncryptCipherDomainId dId,
 	                                const EncryptCipherBaseKeyId bCId,
 	                                StringRef name,
@@ -205,7 +205,7 @@ struct EKPGetLatestCipherKeysRequestInfo {
 	// {domainId, cipherBaseId} tuple
 	EncryptCipherDomainNameRef domainName;
 
-	EKPGetLatestCipherKeysRequestInfo() : domainId(ENCRYPT_INVALID_DOMAIN_ID) {}
+	EKPGetLatestCipherKeysRequestInfo() : domainId(INVALID_ENCRYPT_DOMAIN_ID) {}
 	EKPGetLatestCipherKeysRequestInfo(const EncryptCipherDomainId dId, StringRef name, Arena& arena)
 	  : domainId(dId), domainName(StringRef(arena, name)) {}
 

fdbclient/include/fdbclient/GetEncryptCipherKeys.actor.h

@@ -269,7 +269,7 @@ Future<TextAndHeaderCipherKeys> getLatestEncryptCipherKeysForDomain(Reference<As
                                                                     BlobCipherMetrics::UsageType usageType) {
 	std::unordered_map<EncryptCipherDomainId, EncryptCipherDomainNameRef> domains;
 	domains[domainId] = domainName;
-	domains[ENCRYPT_HEADER_DOMAIN_ID] = FDB_DEFAULT_ENCRYPT_DOMAIN_NAME;
+	domains[ENCRYPT_HEADER_DOMAIN_ID] = FDB_ENCRYPT_HEADER_DOMAIN_NAME;
 	std::unordered_map<EncryptCipherDomainId, Reference<BlobCipherKey>> cipherKeys =
 	    wait(getLatestEncryptCipherKeys(db, domains, usageType));
 	ASSERT(cipherKeys.count(domainId) > 0);

fdbclient/include/fdbclient/Tenant.h

@@ -27,6 +27,7 @@
 #include "fdbclient/VersionedMap.h"
 #include "fdbclient/KeyBackedTypes.h"
 #include "fdbrpc/TenantInfo.h"
+#include "flow/BooleanParam.h"
 #include "flow/flat_buffers.h"
 
 typedef StringRef TenantNameRef;
@@ -64,11 +65,13 @@ enum class TenantLockState { UNLOCKED, READ_ONLY, LOCKED };
 
 constexpr int TENANT_PREFIX_SIZE = sizeof(int64_t);
 
+FDB_DECLARE_BOOLEAN_PARAM(EnforceValidTenantId);
+
 struct TenantMapEntry {
 	constexpr static FileIdentifier file_identifier = 12247338;
 
 	static Key idToPrefix(int64_t id);
-	static int64_t prefixToId(KeyRef prefix);
+	static int64_t prefixToId(KeyRef prefix, EnforceValidTenantId enforceTenantId = EnforceValidTenantId::True);
 
 	static std::string tenantStateToString(TenantState tenantState);
 	static TenantState stringToTenantState(std::string stateStr);

fdbserver/ApplyMetadataMutation.cpp

@@ -83,8 +83,8 @@ public:
 	    uid_applyMutationsData(proxyCommitData_.firstProxy ? &proxyCommitData_.uid_applyMutationsData : nullptr),
 	    commit(proxyCommitData_.commit), cx(proxyCommitData_.cx), committedVersion(&proxyCommitData_.committedVersion),
 	    storageCache(&proxyCommitData_.storageCache), tag_popped(&proxyCommitData_.tag_popped),
-	    tssMapping(&proxyCommitData_.tssMapping), tenantMap(&proxyCommitData_.tenantMap), initialCommit(initialCommit_),
+	    tssMapping(&proxyCommitData_.tssMapping), tenantMap(&proxyCommitData_.tenantMap),
-	    dbInfo(proxyCommitData_.db) {}
+	    tenantIdIndex(&proxyCommitData_.tenantIdIndex), initialCommit(initialCommit_), dbInfo(proxyCommitData_.db) {}
 
 	ApplyMetadataMutationsImpl(const SpanContext& spanContext_,
 	                           ResolverData& resolverData_,
@@ -134,6 +134,7 @@ private:
 	std::unordered_map<UID, StorageServerInterface>* tssMapping = nullptr;
 
 	std::map<TenantName, TenantMapEntry>* tenantMap = nullptr;
+	std::unordered_map<int64_t, TenantName>* tenantIdIndex = nullptr;
 
 	// true if the mutations were already written to the txnStateStore as part of recovery
 	bool initialCommit = false;
@@ -659,13 +660,21 @@ private:
 	void checkSetTenantMapPrefix(MutationRef m) {
 		KeyRef prefix = TenantMetadata::tenantMap().subspace.begin;
 		if (m.param1.startsWith(prefix)) {
+			TenantName tenantName = m.param1.removePrefix(prefix);
+			TenantMapEntry tenantEntry = TenantMapEntry::decode(m.param2);
+
 			if (tenantMap) {
 				ASSERT(version != invalidVersion);
-				TenantName tenantName = m.param1.removePrefix(prefix);
-				TenantMapEntry tenantEntry = TenantMapEntry::decode(m.param2);
 
-				TraceEvent("CommitProxyInsertTenant", dbgid).detail("Tenant", tenantName).detail("Version", version);
+				TraceEvent("CommitProxyInsertTenant", dbgid)
+				    .detail("Tenant", tenantName)
+				    .detail("Id", tenantEntry.id)
+				    .detail("Version", version);
+
 				(*tenantMap)[tenantName] = tenantEntry;
+				if (tenantIdIndex) {
+					(*tenantIdIndex)[tenantEntry.id] = tenantName;
+				}
 			}
 
 			if (!initialCommit) {
@@ -1072,6 +1081,17 @@ private:
 
 				auto startItr = tenantMap->lower_bound(startTenant);
 				auto endItr = tenantMap->lower_bound(endTenant);
+
+				if (tenantIdIndex) {
+					// Iterate over iterator-range and remove entries from TenantIdName map
+					// TODO: O(n) operation, optimize cpu
+					auto itr = startItr;
+					while (itr != endItr) {
+						tenantIdIndex->erase(itr->second.id);
+						itr++;
+					}
+				}
+
 				tenantMap->erase(startItr, endItr);
 			}
 

fdbserver/CommitProxyServer.actor.cpp

@@ -30,7 +30,9 @@
 #include "fdbclient/CommitProxyInterface.h"
 #include "fdbclient/NativeAPI.actor.h"
 #include "fdbclient/SystemData.h"
+#include "fdbclient/Tenant.h"
 #include "fdbclient/TransactionLineage.h"
+#include "fdbrpc/TenantInfo.h"
 #include "fdbrpc/sim_validation.h"
 #include "fdbserver/ApplyMetadataMutation.h"
 #include "fdbserver/ConflictSet.h"
@@ -51,6 +53,7 @@
 #include "fdbserver/WaitFailure.h"
 #include "fdbserver/WorkerInterface.actor.h"
 #include "flow/ActorCollection.h"
+#include "flow/EncryptUtils.h"
 #include "flow/Error.h"
 #include "flow/IRandom.h"
 #include "flow/Knobs.h"
@@ -874,6 +877,81 @@ ACTOR Future<Void> preresolutionProcessing(CommitBatchContext* self) {
 	return Void();
 }
 
+namespace {
+// Routine allows caller to find TenantName for a given TenantId. It returns empty optional when either TenantId is
+// invalid or tenant is unknown
+Optional<TenantName> getTenantName(ProxyCommitData* commitData, int64_t tenantId) {
+	if (tenantId != TenantInfo::INVALID_TENANT) {
+		auto itr = commitData->tenantIdIndex.find(tenantId);
+		if (itr != commitData->tenantIdIndex.end()) {
+			return Optional<TenantName>(itr->second);
+		}
+	}
+
+	return Optional<TenantName>();
+}
+
+std::pair<EncryptCipherDomainName, EncryptCipherDomainId> getEncryptDetailsFromMutationRef(ProxyCommitData* commitData,
+                                                                                           MutationRef m) {
+	std::pair<EncryptCipherDomainName, EncryptCipherDomainId> details(EncryptCipherDomainName(),
+	                                                                  INVALID_ENCRYPT_DOMAIN_ID);
+
+	// Possible scenarios:
+	// 1. Encryption domain (Tenant details) weren't explicitly provided, extract Tenant details using
+	// TenantPrefix (first 8 bytes of FDBKey)
+	// 2. Encryption domain isn't available, leverage 'default encryption domain'
+
+	if (isSystemKey(m.param1)) {
+		// Encryption domain == FDB SystemKeyspace encryption domain
+		details.first = EncryptCipherDomainName(FDB_SYSTEM_KEYSPACE_ENCRYPT_DOMAIN_NAME);
+		details.second = SYSTEM_KEYSPACE_ENCRYPT_DOMAIN_ID;
+	} else if (commitData->tenantMap.empty()) {
+		// Cluster serves no-tenants; use 'default encryption domain'
+	} else if (isSingleKeyMutation((MutationRef::Type)m.type)) {
+		ASSERT_NE((MutationRef::Type)m.type, MutationRef::Type::ClearRange);
+
+		if (m.param1.size() >= TENANT_PREFIX_SIZE) {
+			// Parse mutation key to determine mutation encryption domain
+			StringRef prefix = m.param1.substr(0, TENANT_PREFIX_SIZE);
+			int64_t tenantId = TenantMapEntry::prefixToId(prefix, EnforceValidTenantId::False);
+			if (tenantId != TenantInfo::INVALID_TENANT) {
+				Optional<TenantName> tenantName = getTenantName(commitData, tenantId);
+				if (tenantName.present()) {
+					details.first = tenantName.get();
+					details.second = tenantId;
+				}
+			} else {
+				// Leverage 'default encryption domain'
+			}
+		}
+	} else {
+		// ClearRange is the 'only' MultiKey transaction allowed
+		ASSERT_EQ((MutationRef::Type)m.type, MutationRef::Type::ClearRange);
+
+		// FIXME: Handle Clear-range transaction, actions needed:
+		// 1. Transaction range can spawn multiple encryption domains (tenants)
+		// 2. Transaction can be a multi-key transaction spawning multiple tenants
+		// For now fallback to 'default encryption domain'
+
+		CODE_PROBE(true, "ClearRange mutation encryption");
+	}
+
+	// Unknown tenant, fallback to fdb default encryption domain
+	if (details.second == INVALID_ENCRYPT_DOMAIN_ID) {
+		ASSERT_EQ(details.first.size(), 0);
+		details.first = EncryptCipherDomainName(FDB_DEFAULT_ENCRYPT_DOMAIN_NAME);
+		details.second = FDB_DEFAULT_ENCRYPT_DOMAIN_ID;
+
+		CODE_PROBE(true, "Default domain mutation encryption");
+	}
+
+	ASSERT_GT(details.first.size(), 0);
+
+	return details;
+}
+
+} // namespace
+
 ACTOR Future<Void> getResolution(CommitBatchContext* self) {
 	state double resolutionStart = now();
 	// Sending these requests is the fuzzy border between phase 1 and phase 2; it could conceivably overlap with
@@ -918,18 +996,24 @@ ACTOR Future<Void> getResolution(CommitBatchContext* self) {
 	state Future<std::unordered_map<EncryptCipherDomainId, Reference<BlobCipherKey>>> getCipherKeys;
 	if (pProxyCommitData->isEncryptionEnabled) {
 		static std::unordered_map<EncryptCipherDomainId, EncryptCipherDomainNameRef> defaultDomains = {
-			{ SYSTEM_KEYSPACE_ENCRYPT_DOMAIN_ID, FDB_DEFAULT_ENCRYPT_DOMAIN_NAME },
+			{ SYSTEM_KEYSPACE_ENCRYPT_DOMAIN_ID, FDB_SYSTEM_KEYSPACE_ENCRYPT_DOMAIN_NAME },
-			{ ENCRYPT_HEADER_DOMAIN_ID, FDB_DEFAULT_ENCRYPT_DOMAIN_NAME }
+			{ ENCRYPT_HEADER_DOMAIN_ID, FDB_ENCRYPT_HEADER_DOMAIN_NAME },
+			{ FDB_DEFAULT_ENCRYPT_DOMAIN_ID, FDB_DEFAULT_ENCRYPT_DOMAIN_NAME }
 		};
 		std::unordered_map<EncryptCipherDomainId, EncryptCipherDomainNameRef> encryptDomains = defaultDomains;
 		for (int t = 0; t < trs.size(); t++) {
 			TenantInfo const& tenantInfo = trs[t].tenantInfo;
 			int64_t tenantId = tenantInfo.tenantId;
 			Optional<TenantNameRef> const& tenantName = tenantInfo.name;
-			// TODO(yiwu): In raw access mode, use tenant prefix to figure out tenant id for user data
 			if (tenantId != TenantInfo::INVALID_TENANT) {
 				ASSERT(tenantName.present());
 				encryptDomains[tenantId] = tenantName.get();
+			} else {
+				for (auto m : trs[t].transaction.mutations) {
+					std::pair<EncryptCipherDomainName, int64_t> details =
+					    getEncryptDetailsFromMutationRef(pProxyCommitData, m);
+					encryptDomains[details.second] = details.first;
+				}
 			}
 		}
 		getCipherKeys = getLatestEncryptCipherKeys(pProxyCommitData->db, encryptDomains, BlobCipherMetrics::TLOG);
@@ -1157,18 +1241,24 @@ ACTOR Future<Void> applyMetadataToCommittedTransactions(CommitBatchContext* self
 }
 
 void writeMutation(CommitBatchContext* self, int64_t tenantId, const MutationRef& mutation) {
-	static_assert(TenantInfo::INVALID_TENANT == ENCRYPT_INVALID_DOMAIN_ID);
+	static_assert(TenantInfo::INVALID_TENANT == INVALID_ENCRYPT_DOMAIN_ID);
-	if (!self->pProxyCommitData->isEncryptionEnabled || tenantId == TenantInfo::INVALID_TENANT) {
+
-		// TODO(yiwu): In raw access mode, use tenant prefix to figure out tenant id for user data
+	if (self->pProxyCommitData->isEncryptionEnabled) {
-		bool isRawAccess = tenantId == TenantInfo::INVALID_TENANT && !isSystemKey(mutation.param1) &&
+		EncryptCipherDomainId domainId = tenantId;
-		                   !(mutation.type == MutationRef::ClearRange && isSystemKey(mutation.param2)) &&
+		if (domainId == INVALID_ENCRYPT_DOMAIN_ID) {
-		                   self->pProxyCommitData->db->get().client.tenantMode == TenantMode::REQUIRED;
+			std::pair<EncryptCipherDomainName, EncryptCipherDomainId> p =
-		CODE_PROBE(isRawAccess, "Raw access to tenant key space");
+			    getEncryptDetailsFromMutationRef(self->pProxyCommitData, mutation);
-		self->toCommit.writeTypedMessage(mutation);
+			domainId = p.second;
-	} else {
+
+			CODE_PROBE(true, "Raw access mutation encryption");
+		}
+
+		ASSERT_NE(domainId, INVALID_ENCRYPT_DOMAIN_ID);
+
 		Arena arena;
-		self->toCommit.writeTypedMessage(
+		self->toCommit.writeTypedMessage(mutation.encrypt(self->cipherKeys, domainId, arena, BlobCipherMetrics::TLOG));
-		    mutation.encrypt(self->cipherKeys, tenantId /*domainId*/, arena, BlobCipherMetrics::TLOG));
+	} else {
+		self->toCommit.writeTypedMessage(mutation);
 	}
 }
 

fdbserver/RESTKmsConnector.actor.cpp

@@ -1102,9 +1102,8 @@ void testGetEncryptKeysByKeyIdsRequestBody(Reference<RESTKmsConnectorCtx> ctx, A
 	const int nKeys = deterministicRandom()->randomInt(7, 8);
 	for (int i = 1; i < nKeys; i++) {
 		EncryptCipherDomainId domainId = getRandomDomainId();
-		EncryptCipherDomainNameRef domainName = domainId < 0
+		EncryptCipherDomainNameRef domainName = domainId < 0 ? StringRef(arena, FDB_DEFAULT_ENCRYPT_DOMAIN_NAME)
-		                                            ? StringRef(arena, std::string(FDB_DEFAULT_ENCRYPT_DOMAIN_NAME))
+		                                                     : StringRef(arena, std::to_string(domainId));
-		                                            : StringRef(arena, std::to_string(domainId));
 		req.encryptKeyInfos.emplace_back_deep(req.arena, domainId, i, domainName);
 		keyMap[i] = domainId;
 	}
@@ -1141,9 +1140,8 @@ void testGetEncryptKeysByDomainIdsRequestBody(Reference<RESTKmsConnectorCtx> ctx
 	const int nKeys = deterministicRandom()->randomInt(7, 25);
 	for (int i = 1; i < nKeys; i++) {
 		EncryptCipherDomainId domainId = getRandomDomainId();
-		EncryptCipherDomainNameRef domainName = domainId < 0
+		EncryptCipherDomainNameRef domainName = domainId < 0 ? StringRef(arena, FDB_DEFAULT_ENCRYPT_DOMAIN_NAME)
-		                                            ? StringRef(arena, std::string(FDB_DEFAULT_ENCRYPT_DOMAIN_NAME))
+		                                                     : StringRef(arena, std::to_string(domainId));
-		                                            : StringRef(arena, std::to_string(domainId));
 		KmsConnLookupDomainIdsReqInfoRef reqInfo(req.arena, domainId, domainName);
 		if (domainInfoMap.insert({ domainId, reqInfo }).second) {
 			req.encryptDomainInfos.push_back(req.arena, reqInfo);

fdbserver/include/fdbserver/IEncryptionKeyProvider.actor.h

@@ -27,10 +27,12 @@
 
 #include "fdbclient/GetEncryptCipherKeys.actor.h"
 #include "fdbclient/Tenant.h"
+
 #include "fdbserver/EncryptionOpsUtils.h"
 #include "fdbserver/ServerDBInfo.h"
-#include "flow/Arena.h"
 
+#include "flow/Arena.h"
+#include "flow/EncryptUtils.h"
 #define XXH_INLINE_ALL
 #include "flow/xxhash.h"
 
@@ -241,8 +243,8 @@ private:
 	                                            const KeyRef& end,
 	                                            EncryptCipherDomainNameRef* domainName) {
 		int64_t domainId = SYSTEM_KEYSPACE_ENCRYPT_DOMAIN_ID;
-		int64_t beginTenantId = getTenant(begin, true /*inclusive*/);
+		int64_t beginTenantId = getTenantId(begin, true /*inclusive*/);
-		int64_t endTenantId = getTenant(end, false /*inclusive*/);
+		int64_t endTenantId = getTenantId(end, false /*inclusive*/);
 		if (beginTenantId == endTenantId && beginTenantId != SYSTEM_KEYSPACE_ENCRYPT_DOMAIN_ID) {
 			ASSERT(tenantPrefixIndex.isValid());
 			Key tenantPrefix = TenantMapEntry::idToPrefix(beginTenantId);
@@ -256,22 +258,31 @@ private:
 			}
 		}
 		if (domainId == SYSTEM_KEYSPACE_ENCRYPT_DOMAIN_ID) {
-			*domainName = FDB_DEFAULT_ENCRYPT_DOMAIN_NAME;
+			*domainName = FDB_SYSTEM_KEYSPACE_ENCRYPT_DOMAIN_NAME;
 		}
 		return domainId;
 	}
 
-	int64_t getTenant(const KeyRef& key, bool inclusive) {
+	int64_t getTenantId(const KeyRef& key, bool inclusive) {
 		// A valid tenant id is always a valid encrypt domain id.
-		static_assert(ENCRYPT_INVALID_DOMAIN_ID < 0);
+		static_assert(INVALID_ENCRYPT_DOMAIN_ID == -1);
-		if (key.size() < TENANT_PREFIX_SIZE || key >= systemKeys.begin) {
+
+		if (key.size() && key >= systemKeys.begin) {
 			return SYSTEM_KEYSPACE_ENCRYPT_DOMAIN_ID;
 		}
-		// TODO(yiwu): Use TenantMapEntry::prefixToId() instead.
+
-		int64_t tenantId = bigEndian64(*reinterpret_cast<const int64_t*>(key.begin()));
+		if (key.size() < TENANT_PREFIX_SIZE) {
-		if (tenantId < 0) {
+			// Encryption domain information not available, leverage 'default encryption domain'
-			return SYSTEM_KEYSPACE_ENCRYPT_DOMAIN_ID;
+			return FDB_DEFAULT_ENCRYPT_DOMAIN_ID;
 		}
+
+		StringRef prefix = key.substr(0, TENANT_PREFIX_SIZE);
+		int64_t tenantId = TenantMapEntry::prefixToId(prefix, EnforceValidTenantId::False);
+		if (tenantId == TenantInfo::INVALID_TENANT) {
+			// Encryption domain information not available, leverage 'default encryption domain'
+			return FDB_DEFAULT_ENCRYPT_DOMAIN_ID;
+		}
+
 		if (!inclusive && key.size() == TENANT_PREFIX_SIZE) {
 			tenantId = tenantId - 1;
 		}

fdbserver/include/fdbserver/KmsConnectorInterface.h

@@ -132,7 +132,7 @@ struct KmsConnLookupKeyIdsReqInfoRef {
 	EncryptCipherDomainNameRef domainName;
 
 	KmsConnLookupKeyIdsReqInfoRef()
-	  : domainId(ENCRYPT_INVALID_DOMAIN_ID), baseCipherId(ENCRYPT_INVALID_CIPHER_KEY_ID) {}
+	  : domainId(INVALID_ENCRYPT_DOMAIN_ID), baseCipherId(INVALID_ENCRYPT_CIPHER_KEY_ID) {}
 	explicit KmsConnLookupKeyIdsReqInfoRef(Arena& arena,
 	                                       const EncryptCipherDomainId dId,
 	                                       const EncryptCipherBaseKeyId bCId,
@@ -185,7 +185,7 @@ struct KmsConnLookupDomainIdsReqInfoRef {
 	EncryptCipherDomainId domainId;
 	EncryptCipherDomainNameRef domainName;
 
-	KmsConnLookupDomainIdsReqInfoRef() : domainId(ENCRYPT_INVALID_DOMAIN_ID) {}
+	KmsConnLookupDomainIdsReqInfoRef() : domainId(INVALID_ENCRYPT_DOMAIN_ID) {}
 	explicit KmsConnLookupDomainIdsReqInfoRef(Arena& arena, const EncryptCipherDomainId dId, StringRef name)
 	  : domainId(dId), domainName(StringRef(arena, name)) {}
 	explicit KmsConnLookupDomainIdsReqInfoRef(const EncryptCipherDomainId dId, StringRef name)

fdbserver/include/fdbserver/ProxyCommitData.actor.h

@@ -20,6 +20,7 @@
 
 #pragma once
 #include "fdbserver/EncryptionOpsUtils.h"
+#include <unordered_map>
 #if defined(NO_INTELLISENSE) && !defined(FDBSERVER_PROXYCOMMITDATA_ACTOR_G_H)
 #define FDBSERVER_PROXYCOMMITDATA_ACTOR_G_H
 #include "fdbserver/ProxyCommitData.actor.g.h"
@@ -173,6 +174,7 @@ struct ProxyCommitData {
 	UID dbgid;
 	int64_t commitBatchesMemBytesCount;
 	std::map<TenantName, TenantMapEntry> tenantMap;
+	std::unordered_map<int64_t, TenantName> tenantIdIndex;
 	ProxyStats stats;
 	MasterInterface master;
 	std::vector<ResolverInterface> resolvers;

fdbserver/workloads/EncryptionOps.actor.cpp

@@ -221,6 +221,7 @@ struct EncryptionOpsWorkload : TestWorkload {
 			cipherKeyCache->resetEncryptDomainId(id);
 		}
 
+		cipherKeyCache->resetEncryptDomainId(FDB_DEFAULT_ENCRYPT_DOMAIN_ID);
 		cipherKeyCache->resetEncryptDomainId(SYSTEM_KEYSPACE_ENCRYPT_DOMAIN_ID);
 		cipherKeyCache->resetEncryptDomainId(ENCRYPT_HEADER_DOMAIN_ID);
 

flow/include/flow/EncryptUtils.h

@@ -29,22 +29,27 @@
 #include <string>
 #include <string_view>
 
-#define ENCRYPT_INVALID_DOMAIN_ID -1
-#define ENCRYPT_INVALID_CIPHER_KEY_ID 0
-#define ENCRYPT_INVALID_RANDOM_SALT 0
-
 #define AUTH_TOKEN_SIZE 32
 
-#define SYSTEM_KEYSPACE_ENCRYPT_DOMAIN_ID -2
-#define ENCRYPT_HEADER_DOMAIN_ID -3
-
-const std::string FDB_DEFAULT_ENCRYPT_DOMAIN_NAME = "FdbDefaultEncryptDomain";
-
 using EncryptCipherDomainId = int64_t;
 using EncryptCipherDomainNameRef = StringRef;
+using EncryptCipherDomainName = Standalone<EncryptCipherDomainNameRef>;
 using EncryptCipherBaseKeyId = uint64_t;
 using EncryptCipherRandomSalt = uint64_t;
 
+constexpr const EncryptCipherDomainId INVALID_ENCRYPT_DOMAIN_ID = -1;
+constexpr const EncryptCipherDomainId SYSTEM_KEYSPACE_ENCRYPT_DOMAIN_ID = -2;
+constexpr const EncryptCipherDomainId ENCRYPT_HEADER_DOMAIN_ID = -3;
+constexpr const EncryptCipherDomainId FDB_DEFAULT_ENCRYPT_DOMAIN_ID = -4;
+
+constexpr const EncryptCipherBaseKeyId INVALID_ENCRYPT_CIPHER_KEY_ID = 0;
+
+constexpr const EncryptCipherRandomSalt INVALID_ENCRYPT_RANDOM_SALT = 0;
+
+const EncryptCipherDomainNameRef FDB_SYSTEM_KEYSPACE_ENCRYPT_DOMAIN_NAME = "FdbSystemKeyspaceEncryptDomain"_sr;
+const EncryptCipherDomainNameRef FDB_DEFAULT_ENCRYPT_DOMAIN_NAME = "FdbDefaultEncryptDomain"_sr;
+const EncryptCipherDomainNameRef FDB_ENCRYPT_HEADER_DOMAIN_NAME = "FdbEncryptHeaderDomain"_sr;
+
 typedef enum {
 	ENCRYPT_CIPHER_MODE_NONE = 0,
 	ENCRYPT_CIPHER_MODE_AES_256_CTR = 1,

flow/include/flow/error_definitions.h

@@ -316,28 +316,28 @@ ERROR( json_malformed, 2401, "JSON string was malformed")
 ERROR( json_eof_expected, 2402, "JSON string did not terminate where expected")
 
 // 2500 - disk snapshot based backup errors
-ERROR( snap_disable_tlog_pop_failed,  2500, "Failed to disable tlog pops")
+ERROR( snap_disable_tlog_pop_failed,  2500, "Failed to disable tlog pops" )
-ERROR( snap_storage_failed,  2501, "Failed to snapshot storage nodes")
+ERROR( snap_storage_failed,  2501, "Failed to snapshot storage nodes" )
-ERROR( snap_tlog_failed,  2502, "Failed to snapshot TLog nodes")
+ERROR( snap_tlog_failed,  2502, "Failed to snapshot TLog nodes" )
-ERROR( snap_coord_failed,  2503, "Failed to snapshot coordinator nodes")
+ERROR( snap_coord_failed,  2503, "Failed to snapshot coordinator nodes" )
-ERROR( snap_enable_tlog_pop_failed,  2504, "Failed to enable tlog pops")
+ERROR( snap_enable_tlog_pop_failed,  2504, "Failed to enable tlog pops" )
-ERROR( snap_path_not_whitelisted, 2505, "Snapshot create binary path not whitelisted")
+ERROR( snap_path_not_whitelisted, 2505, "Snapshot create binary path not whitelisted" )
-ERROR( snap_not_fully_recovered_unsupported, 2506, "Unsupported when the cluster is not fully recovered")
+ERROR( snap_not_fully_recovered_unsupported, 2506, "Unsupported when the cluster is not fully recovered" )
-ERROR( snap_log_anti_quorum_unsupported, 2507, "Unsupported when log anti quorum is configured")
+ERROR( snap_log_anti_quorum_unsupported, 2507, "Unsupported when log anti quorum is configured" )
-ERROR( snap_with_recovery_unsupported, 2508, "Cluster recovery during snapshot operation not supported")
+ERROR( snap_with_recovery_unsupported, 2508, "Cluster recovery during snapshot operation not supported" )
-ERROR( snap_invalid_uid_string, 2509, "The given uid string is not a 32-length hex string")
+ERROR( snap_invalid_uid_string, 2509, "The given uid string is not a 32-length hex string" )
 
 // 27XX - Encryption operations errors
-ERROR( encrypt_ops_error, 2700, "Encryption operation error")
+ERROR( encrypt_ops_error, 2700, "Encryption operation error" )
-ERROR( encrypt_header_metadata_mismatch, 2701, "Encryption header metadata mismatch")
+ERROR( encrypt_header_metadata_mismatch, 2701, "Encryption header metadata mismatch" )
-ERROR( encrypt_key_not_found, 2702, "Expected encryption key is missing")
+ERROR( encrypt_key_not_found, 2702, "Expected encryption key is missing" )
-ERROR( encrypt_key_ttl_expired, 2703, "Expected encryption key TTL has expired")
+ERROR( encrypt_key_ttl_expired, 2703, "Expected encryption key TTL has expired" )
-ERROR( encrypt_header_authtoken_mismatch, 2704, "Encryption header authentication token mismatch")
+ERROR( encrypt_header_authtoken_mismatch, 2704, "Encryption header authentication token mismatch" )
-ERROR( encrypt_update_cipher, 2705, "Attempt to update encryption cipher key")
+ERROR( encrypt_update_cipher, 2705, "Attempt to update encryption cipher key" )
-ERROR( encrypt_invalid_id, 2706, "Invalid encryption cipher details")
+ERROR( encrypt_invalid_id, 2706, "Invalid encryption cipher details" )
-ERROR( encrypt_keys_fetch_failed, 2707, "Encryption keys fetch from external KMS failed")
+ERROR( encrypt_keys_fetch_failed, 2707, "Encryption keys fetch from external KMS failed" )
-ERROR( encrypt_invalid_kms_config, 2708, "Invalid encryption/kms configuration: discovery-url, validation-token, endpoint etc.")
+ERROR( encrypt_invalid_kms_config, 2708, "Invalid encryption/kms configuration: discovery-url, validation-token, endpoint etc." )
-ERROR( encrypt_unsupported, 2709, "Encryption not supported")
+ERROR( encrypt_unsupported, 2709, "Encryption not supported" )
 
 // 4xxx Internal errors (those that should be generated only by bugs) are decimal 4xxx
 ERROR( unknown_error, 4000, "An unknown error occurred" )  // C++ exception not of type Error