solve some comments

fdbcli/DataDistributionCommand.actor.cpp

@@ -69,7 +69,7 @@ ACTOR Future<Void> setDDIgnoreRebalanceSwitch(Reference<IDatabase> db, uint8_t D
 		try {
 			state ThreadFuture<Optional<Value>> resultFuture = tr->get(rebalanceDDIgnoreKey);
 			Optional<Value> v = wait(safeThreadFutureToFuture(resultFuture));
-			uint8_t oldValue = 0; // nothing is disabled
+			uint8_t oldValue = DDIgnore::NONE; // nothing is disabled
 			if (v.present()) {
 				if (v.get().size() > 0) {
 					oldValue = BinaryReader::fromStringRef<uint8_t>(v.get(), Unversioned());

fdbcli/StatusCommand.actor.cpp

@@ -1128,8 +1128,8 @@ void printStatus(StatusObjectReader statusObj,
 					                "storage server failures.";
 				}
 				if (statusObjCluster.has("data_distribution_disabled_for_rebalance")) {
-					outputString += "\n\nWARNING: Data distribution is currently turned on but shard size balancing is "
-					                "currently disabled.";
+					outputString += "\n\nWARNING: Data distribution is currently turned on but one or both of shard "
+					                "size and read-load based balancing are disabled.";
 					// data_distribution_disabled_hex
 					if (statusObjCluster.has("data_distribution_disabled_hex")) {
 						outputString += " Ignore code: " + statusObjCluster["data_distribution_disabled_hex"].get_str();

fdbclient/ServerKnobs.h

@@ -610,7 +610,7 @@ public:
 	int64_t BYTES_READ_UNITS_PER_SAMPLE;
 	int64_t READ_HOT_SUB_RANGE_CHUNK_SIZE;
 	int64_t EMPTY_READ_PENALTY;
-	int DD_SHARD_COMPARE_LIMIT;
+	int DD_SHARD_COMPARE_LIMIT; // when read-aware DD is enabled, at most how many shards are compared together
 	bool READ_SAMPLING_ENABLED;
 
 	// Storage Server

fdbclient/SystemData.cpp

@@ -302,8 +302,8 @@ std::pair<std::vector<std::pair<UID, NetworkAddress>>, std::vector<std::pair<UID
 	return std::make_pair(logs, oldLogs);
 }
 
-const KeyRef serverKeysPrefix = "\xff/serverKeys/"_sr;
-const KeyRangeRef serverKeysRange = KeyRangeRef(serverKeysPrefix, "\xff/serverKeys0"_sr);
+const KeyRangeRef serverKeysRange = KeyRangeRef("\xff/serverKeys/"_sr, "\xff/serverKeys0"_sr);
+const KeyRef serverKeysPrefix = serverKeysRange.begin;
 const ValueRef serverKeysTrue = "1"_sr, // compatible with what was serverKeysTrue
     serverKeysTrueEmptyRange = "3"_sr, // the server treats the range as empty.
     serverKeysFalse;

fdbclient/SystemData.h

@@ -502,7 +502,9 @@ extern const KeyRangeRef monitorConfKeys;
 extern const KeyRef healthyZoneKey;
 extern const StringRef ignoreSSFailuresZoneString;
 extern const KeyRef rebalanceDDIgnoreKey;
-enum DDIgnore : uint8_t { REBALANCE_DISK = 1, REBALANCE_READ = 2, ALL = 3 };
+namespace DDIgnore {
+enum IgnoreType : uint8_t { NONE = 0, REBALANCE_DISK = 1, REBALANCE_READ = 2, ALL = 3 };
+}
 
 const Value healthyZoneValue(StringRef const& zoneId, Version version);
 std::pair<Key, Version> decodeHealthyZoneValue(ValueRef const&);

fdbserver/DataDistributionQueue.actor.cpp

@@ -1513,6 +1513,7 @@ inline double getWorstCpu(const HealthMetrics& metrics, const std::vector<UID>&
 		} else {
 			// assume the server is too busy to report its stats
 			cpu = std::max(cpu, 100.0);
+			break;
 		}
 	}
 	return cpu;

fdbserver/Status.actor.cpp

@@ -1570,7 +1570,7 @@ struct LoadConfigurationResult {
 	double healthyZoneSeconds;
 	bool rebalanceDDIgnored;
 	// FIXME: possible convert it to int if upgrade value can be resolved?
-	std::string rebalanceDDIgnoreHex; // any or combination of 0, 1, 2, see enum DDIgnore;
+	std::string rebalanceDDIgnoreHex; // any or combination of 0, 1, 2, see DDIgnore;
 	bool dataDistributionDisabled;
 
 	LoadConfigurationResult()

fdbserver/TCInfo.actor.cpp

@@ -402,7 +402,7 @@ double TCTeamInfo::getLoadReadBandwidth(bool includeInFlight, double inflightPen
 	}
 	return (size == 0 ? 0 : sum / size) +
 	       // we don't need to divide the inflight bandwidth because when added it the bandwidth is from single server
-	       (includeInFlight ? inflightPenalty * getReadInFlightToTeam() : 0);
+	       (includeInFlight ? inflightPenalty * getReadInFlightToTeam() / servers.size() : 0);
 }
 
 int64_t TCTeamInfo::getMinAvailableSpace(bool includeInFlight) const {

fdbserver/workloads/SpecialKeySpaceCorrectness.actor.cpp

@@ -1275,13 +1275,21 @@ struct SpecialKeySpaceCorrectnessWorkload : TestWorkload {
 				}
 			}
 			// set dd mode to 0 and disable DD for rebalance
+			state uint8_t ddIgnoreValue = DDIgnore::NONE;
+			if (deterministicRandom()->coinflip()) {
+				ddIgnoreValue |= DDIgnore::REBALANCE_READ;
+			}
+			if (deterministicRandom()->coinflip()) {
+				ddIgnoreValue |= DDIgnore::REBALANCE_DISK;
+			}
 			loop {
 				try {
 					tx->setOption(FDBTransactionOptions::RAW_ACCESS);
 					tx->setOption(FDBTransactionOptions::SPECIAL_KEY_SPACE_ENABLE_WRITES);
 					KeyRef ddPrefix = SpecialKeySpace::getManagementApiCommandPrefix("datadistribution");
 					tx->set(LiteralStringRef("mode").withPrefix(ddPrefix), LiteralStringRef("0"));
-					tx->set(LiteralStringRef("rebalance_ignored").withPrefix(ddPrefix), Value());
+					tx->set(LiteralStringRef("rebalance_ignored").withPrefix(ddPrefix),
+					        BinaryWriter::toValue(ddIgnoreValue, Unversioned()));
 					wait(tx->commit());
 					tx->reset();
 					break;
@@ -1306,8 +1314,8 @@ struct SpecialKeySpaceCorrectnessWorkload : TestWorkload {
 					ASSERT(BinaryReader::fromStringRef<int>(val2.get(), Unversioned()) == 0);
 					// check DD disabled for rebalance
 					Optional<Value> val3 = wait(tx->get(rebalanceDDIgnoreKey));
-					// default value "on"
-					ASSERT(val3.present());
+					ASSERT(val3.present() &&
+					       BinaryReader::fromStringRef<uint8_t>(val3.get(), Unversioned()) == ddIgnoreValue);
 					tx->reset();
 					break;
 				} catch (Error& e) {

tests/CMakeLists.txt

@@ -186,7 +186,6 @@ if(WITH_PYTHON)
   add_fdb_test(TEST_FILES fast/WriteDuringRead.toml)
   add_fdb_test(TEST_FILES fast/WriteDuringReadClean.toml)
   add_fdb_test(TEST_FILES noSim/RandomUnitTests.toml UNIT)
-  add_fdb_test(TEST_FILES noSim/ReadSkewReadWrite.toml IGNORE)
   if (WITH_ROCKSDB_EXPERIMENTAL)
     add_fdb_test(TEST_FILES noSim/KeyValueStoreRocksDBTest.toml)
     add_fdb_test(TEST_FILES fast/PhysicalShardMove.toml)
@@ -211,6 +210,7 @@ if(WITH_PYTHON)
   add_fdb_test(TEST_FILES rare/LargeApiCorrectnessStatus.toml)
   add_fdb_test(TEST_FILES rare/RYWDisable.toml)
   add_fdb_test(TEST_FILES rare/RandomReadWriteTest.toml)
+  add_fdb_test(TEST_FILES rare/ReadSkewReadWrite.toml)
   add_fdb_test(TEST_FILES rare/SpecificUnitTests.toml)
   add_fdb_test(TEST_FILES rare/SwizzledLargeApiCorrectness.toml)
   add_fdb_test(TEST_FILES rare/RedwoodCorrectnessBTree.toml)

tests/noSim/ReadSkewReadWrite.toml

@@ -1,24 +0,0 @@
-[[test]]
-testTitle = 'RandomReadWriteTest'
-connectionFailuresDisableDuration = 100000
-# waitForQuiescenceBegin= false
-# waitForQuiescenceEnd=false
-clearAfterTest = false #true
-runSetup = true # false
-timeout = 3600.0
-
-[[test.workload]]
-testName = 'ReadWrite'
-transactionsPerSecond = 100000
-testDuration = 900.0
-skewRound = 1
-nodeCount = 30000000
-valueBytes = 1000
-readsPerTransactionA = 8
-writesPerTransactionA = 0
-alpha = 0
-discardEdgeMeasurements = false
-hotServerFraction = 0.2
-hotServerReadFrac = 0.8
-# hotServerShardFraction = 0.3
-warmingDelay = 180.0

tests/rare/ReadSkewReadWrite.toml

@@ -1,15 +1,24 @@
 [[test]]
 testTitle = 'RandomReadWriteTest'
-simCheckRelocationDuration = true
 connectionFailuresDisableDuration = 100000
+waitForQuiescenceBegin= false
+waitForQuiescenceEnd=false
+clearAfterTest = true
+runSetup = true # false
+timeout = 3600.0
 
-    [[test.workload]]
-    testName = 'ReadWrite'
-    testDuration = 30.0
-    skewRound = 1
-    transactionsPerSecond = 2000
-    nodeCount = 150000
-    valueBytes = 128
-    discardEdgeMeasurements = false
-    warmingDelay = 10.0
-    hotServerFraction = 0.1
+[[test.workload]]
+testName = 'ReadWrite'
+transactionsPerSecond = 100000
+testDuration = 400.0
+skewRound = 1
+nodeCount = 30000 # 30000000
+valueBytes = 100
+readsPerTransactionA = 8
+writesPerTransactionA = 0
+alpha = 0
+discardEdgeMeasurements = false
+hotServerFraction = 0.2
+hotServerReadFrac = 0.8
+# hotServerShardFraction = 0.3
+warmingDelay = 180.0