Add metacluster awareness of the tenant count limit

fdbclient/Metacluster.cpp

@@ -29,4 +29,3 @@ json_spirit::mObject ClusterUsage::toJson() const {
 	obj["num_tenant_groups"] = numTenantGroups;
 	return obj;
 }
-

fdbclient/MetaclusterManagement.actor.cpp

@@ -51,6 +51,8 @@ KeyBackedMap<ClusterName,
     ManagementClusterMetadata::dataClusterConnectionRecords("metacluster/dataCluster/connectionString/"_sr);
 
 KeyBackedSet<Tuple> ManagementClusterMetadata::clusterCapacityIndex("metacluster/clusterCapacityIndex/"_sr);
+KeyBackedMap<ClusterName, int64_t, TupleCodec<ClusterName>, BinaryCodec<int64_t>>
+    ManagementClusterMetadata::clusterTenantCount("metacluster/clusterTenantCount/"_sr);
 KeyBackedSet<Tuple> ManagementClusterMetadata::clusterTenantIndex("metacluster/dataCluster/tenantMap/"_sr);
 KeyBackedSet<Tuple> ManagementClusterMetadata::clusterTenantGroupIndex("metacluster/dataCluster/tenantGroupMap/"_sr);
 

fdbclient/include/fdbclient/KeyBackedTypes.h

@@ -156,6 +156,12 @@ struct NullCodec {
 	static Standalone<StringRef> unpack(Standalone<StringRef> val) { return val; }
 };
 
+template <class T>
+struct BinaryCodec {
+	static Standalone<StringRef> pack(T val) { return BinaryWriter::toValue<T>(val, Unversioned()); }
+	static T unpack(Standalone<StringRef> val) { return BinaryReader::fromStringRef<T>(val, Unversioned()); }
+};
+
 template <typename ResultType>
 struct KeyBackedRangeResult {
 	std::vector<ResultType> results;
@@ -364,6 +370,16 @@ public:
 		    }));
 	}
 
+	// Get key's value or defaultValue if it doesn't exist
+	template <class Transaction>
+	Future<ValueType> getD(Transaction tr,
+	                       KeyType const& key,
+	                       Snapshot snapshot = Snapshot::False,
+	                       ValueType defaultValue = ValueType()) const {
+		return map(get(tr, key, snapshot),
+		           [=](Optional<ValueType> val) -> ValueType { return val.orDefault(defaultValue); });
+	}
+
 	// Returns a Property that can be get/set that represents key's entry in this this.
 	KeyBackedProperty<ValueType> getProperty(KeyType const& key) const {
 		return subspace.begin.withSuffix(KeyCodec::pack(key));
@@ -378,6 +394,13 @@ public:
 		return k.expectedSize() + v.expectedSize();
 	}
 
+	template <class Transaction>
+	void atomicOp(Transaction tr, KeyType const& key, ValueType const& val, MutationRef::Type type) {
+		Key k = subspace.begin.withSuffix(KeyCodec::pack(key));
+		Value v = ValueCodec::pack(val);
+		tr->atomicOp(k, v, type);
+	}
+
 	template <class Transaction>
 	void erase(Transaction tr, KeyType const& key) {
 		tr->clear(subspace.begin.withSuffix(KeyCodec::pack(key)));

fdbclient/include/fdbclient/MetaclusterManagement.actor.h

@@ -106,6 +106,9 @@ struct ManagementClusterMetadata {
 	// A set of non-full clusters where the key is the tuple (num tenant groups allocated, cluster name).
 	static KeyBackedSet<Tuple> clusterCapacityIndex;
 
+	// A map from cluster name to a count of tenants
+	static KeyBackedMap<ClusterName, int64_t, TupleCodec<ClusterName>, BinaryCodec<int64_t>> clusterTenantCount;
+
 	// A set of cluster/tenant pairings ordered by cluster
 	static KeyBackedSet<Tuple> clusterTenantIndex;
 
@@ -634,6 +637,11 @@ Future<Void> managementClusterPurgeDataCluster(Reference<DB> db, ClusterNameRef
 				ManagementClusterMetadata::tenantMetadata.tenantMap.erase(tr, entry.getString(1));
 			}
 
+			ManagementClusterMetadata::tenantMetadata.tenantCount.atomicOp(
+			    tr, -tenantEntries.results.size(), MutationRef::AddValue);
+			ManagementClusterMetadata::clusterTenantCount.atomicOp(
+			    tr, name, -tenantEntries.results.size(), MutationRef::AddValue);
+
 			// Erase all of the tenants processed in this transaction from the cluster tenant index
 			ManagementClusterMetadata::clusterTenantIndex.erase(
 			    tr,
@@ -1042,7 +1050,6 @@ struct CreateTenantImpl {
 		if (self->tenantName.startsWith("\xff"_sr)) {
 			throw invalid_tenant_name();
 		}
-
 		state Future<Optional<TenantMapEntry>> existingEntryFuture = tryGetTenantTransaction(tr, self->tenantName);
 		Optional<TenantMapEntry> existingEntry = wait(existingEntryFuture);
 
@@ -1058,6 +1065,19 @@ struct CreateTenantImpl {
 		self->tenantEntry.tenantState = TenantState::REGISTERING;
 		ManagementClusterMetadata::tenantMetadata.tenantMap.set(tr, self->tenantName, self->tenantEntry);
 
+		if (!existingEntry.present()) {
+			ManagementClusterMetadata::tenantMetadata.tenantCount.atomicOp(tr, 1, MutationRef::AddValue);
+			ManagementClusterMetadata::clusterTenantCount.atomicOp(
+			    tr, self->tenantEntry.assignedCluster.get(), 1, MutationRef::AddValue);
+
+			int64_t clusterTenantCount = wait(ManagementClusterMetadata::clusterTenantCount.getD(
+			    tr, self->tenantEntry.assignedCluster.get(), Snapshot::False, 0));
+
+			if (clusterTenantCount > CLIENT_KNOBS->MAX_TENANTS_PER_CLUSTER) {
+				throw cluster_no_capacity();
+			}
+		}
+
 		return std::make_pair(self->tenantEntry, true);
 	}
 
@@ -1213,6 +1233,11 @@ struct DeleteTenantImpl {
 		// Erase the tenant entry itself
 		ManagementClusterMetadata::tenantMetadata.tenantMap.erase(tr, tenantName);
 
+		// This is idempotent because this function is only called if the tenant is in the map
+		ManagementClusterMetadata::tenantMetadata.tenantCount.atomicOp(tr, -1, MutationRef::AddValue);
+		ManagementClusterMetadata::clusterTenantCount.atomicOp(
+		    tr, tenantEntry.assignedCluster.get(), -1, MutationRef::AddValue);
+
 		// Clean up cluster based tenant indices and remove the tenant group if it is empty
 		state DataClusterMetadata clusterMetadata = wait(getClusterTransaction(tr, tenantEntry.assignedCluster.get()));
 

fdbserver/workloads/TenantManagementConcurrencyWorkload.actor.cpp

@@ -195,7 +195,7 @@ struct TenantManagementConcurrencyWorkload : TestWorkload {
 
 			return Void();
 		} catch (Error& e) {
-			if (e.code() != error_code_tenant_already_exists) {
+			if (e.code() != error_code_tenant_already_exists && e.code() != error_code_cluster_no_capacity) {
 				TraceEvent(SevError, "CreateTenantFailure").error(e).detail("TenantName", tenant);
 			}
 

fdbserver/workloads/TenantManagementWorkload.actor.cpp

@@ -64,6 +64,7 @@ struct TenantManagementWorkload : TestWorkload {
 	const Key testParametersKey = "test_parameters"_sr;
 	const Value noTenantValue = "no_tenant"_sr;
 	const TenantName tenantNamePrefix = "tenant_management_workload_"_sr;
+	const ClusterName dataClusterName = "cluster1"_sr;
 	TenantName localTenantNamePrefix;
 	TenantName localTenantGroupNamePrefix;
 
@@ -165,7 +166,8 @@ struct TenantManagementWorkload : TestWorkload {
 
 			DataClusterEntry entry;
 			entry.capacity.numTenantGroups = 1e9;
-			wait(MetaclusterAPI::registerCluster(self->mvDb, "cluster1"_sr, g_simulator.extraDatabases[0], entry));
+			wait(MetaclusterAPI::registerCluster(
+			    self->mvDb, self->dataClusterName, g_simulator.extraDatabases[0], entry));
 		}
 
 		state Transaction tr(cx);
@@ -1397,20 +1399,40 @@ struct TenantManagementWorkload : TestWorkload {
 
 	// Verify that the tenant count matches the actual number of tenants in the cluster and that we haven't created too
 	// many
-	ACTOR static Future<Void> checkTenantCount(Database cx) {
+	ACTOR template <bool UseMetacluster, class DB>
-		state Reference<ReadYourWritesTransaction> tr = cx->createTransaction();
+	static Future<Void> checkTenantCount(TenantManagementWorkload* self, Reference<DB> db) {
+		state Reference<typename DB::TransactionT> tr = db->createTransaction();
 		loop {
 			try {
 				tr->setOption(FDBTransactionOptions::READ_SYSTEM_KEYS);
-				state int64_t tenantCount = wait(TenantMetadata::tenantCount.getD(tr, Snapshot::False, 0));
+				state int64_t tenantCount;
-				KeyBackedRangeResult<std::pair<TenantName, TenantMapEntry>> tenants =
+				state KeyBackedRangeResult<std::pair<TenantName, TenantMapEntry>> tenants;
-				    wait(TenantMetadata::tenantMap.getRange(tr, {}, {}, CLIENT_KNOBS->MAX_TENANTS_PER_CLUSTER + 1));
+
+				if (UseMetacluster) {
+					wait(store(tenantCount,
+					           MetaclusterAPI::ManagementClusterMetadata::tenantMetadata.tenantCount.getD(
+					               tr, Snapshot::False, 0)));
+					wait(store(tenants,
+					           MetaclusterAPI::ManagementClusterMetadata::tenantMetadata.tenantMap.getRange(
+					               tr, {}, {}, CLIENT_KNOBS->MAX_TENANTS_PER_CLUSTER + 1)));
+
+					int64_t clusterTenantCount =
+					    wait(MetaclusterAPI::ManagementClusterMetadata::clusterTenantCount.getD(
+					        tr, self->dataClusterName, Snapshot::False, 0));
+
+					ASSERT(clusterTenantCount == tenantCount);
+				} else {
+					wait(store(tenantCount, TenantMetadata::tenantCount.getD(tr, Snapshot::False, 0)));
+					wait(store(
+					    tenants,
+					    TenantMetadata::tenantMap.getRange(tr, {}, {}, CLIENT_KNOBS->MAX_TENANTS_PER_CLUSTER + 1)));
+					ASSERT(tenantCount <= CLIENT_KNOBS->MAX_TENANTS_PER_CLUSTER);
+				}
 
 				ASSERT(tenants.results.size() == tenantCount && !tenants.more);
-				ASSERT(tenantCount <= CLIENT_KNOBS->MAX_TENANTS_PER_CLUSTER);
 				return Void();
 			} catch (Error& e) {
-				wait(tr->onError(e));
+				wait(safeThreadFutureToFuture(tr->onError(e)));
 			}
 		}
 	}
@@ -1647,7 +1669,10 @@ struct TenantManagementWorkload : TestWorkload {
 		}
 
 		if (self->clientId == 0) {
-			wait(checkTenantCount(cx));
+			if (self->useMetacluster) {
+				wait(checkTenantCount<true>(self, self->mvDb));
+			}
+			wait(checkTenantCount<false>(self, self->dataDb.getReference()));
 		}
 
 		wait(compareTenants(self) && compareTenantGroups(self) && checkTenantTombstones(self));