From 3e79299898fa6b5dc07437743adb58fb7776574d Mon Sep 17 00:00:00 2001 From: Neethu Haneesha Bingi Date: Fri, 10 Sep 2021 15:33:17 -0700 Subject: [PATCH] Locality filter support to perpetual storage wiggler feature. --- .../sphinx/source/command-line-interface.rst | 7 ++- .../source/mr-status-json-schemas.rst.inc | 1 + .../source/perpetual-storage-wiggle.rst | 4 ++ fdbcli/ConfigureCommand.actor.cpp | 9 ++- fdbcli/fdbcli.actor.cpp | 1 + fdbclient/DatabaseConfiguration.cpp | 9 +++ fdbclient/DatabaseConfiguration.h | 1 + fdbclient/ManagementAPI.actor.cpp | 8 +++ fdbclient/Schemas.cpp | 1 + fdbclient/SystemData.cpp | 1 + fdbclient/SystemData.h | 1 + fdbserver/DataDistribution.actor.cpp | 58 ++++++++++++++++++- fdbserver/SimulatedCluster.actor.cpp | 2 +- 13 files changed, 95 insertions(+), 8 deletions(-) diff --git a/documentation/sphinx/source/command-line-interface.rst b/documentation/sphinx/source/command-line-interface.rst index 7617b5ad15..cacfccf56a 100644 --- a/documentation/sphinx/source/command-line-interface.rst +++ b/documentation/sphinx/source/command-line-interface.rst @@ -64,7 +64,7 @@ The ``commit`` command commits the current transaction. Any sets or clears execu configure --------- -The ``configure`` command changes the database configuration. Its syntax is ``configure [new|tss] [single|double|triple|three_data_hall|three_datacenter] [ssd|memory] [grv_proxies=] [commit_proxies=] [resolvers=] [logs=] [count=] [perpetual_storage_wiggle=] [storage_migration_type={disabled|aggressive|gradual}]``. +The ``configure`` command changes the database configuration. Its syntax is ``configure [new|tss] [single|double|triple|three_data_hall|three_datacenter] [ssd|memory] [grv_proxies=] [commit_proxies=] [resolvers=] [logs=] [count=] [perpetual_storage_wiggle=] [perpetual_storage_wiggle_locality=<:|0>] [storage_migration_type={disabled|aggressive|gradual}]``. The ``new`` option, if present, initializes a new database with the given configuration rather than changing the configuration of an existing one. When ``new`` is used, both a redundancy mode and a storage engine must be specified. @@ -112,7 +112,10 @@ For recommendations on appropriate values for process types in large clusters, s perpetual storage wiggle ^^^^^^^^^^^^^^^^^^^^^^^^ -Set the value speed (a.k.a., the number of processes that the Data Distributor should wiggle at a time). Currently, only 0 and 1 are supported. The value 0 means to disable the perpetual storage wiggle. For more details, see :ref:`perpetual-storage-wiggle`. +``perpetual_storage_wiggle`` sets the value speed (a.k.a., the number of processes that the Data Distributor should wiggle at a time). Currently, only 0 and 1 are supported. The value 0 means to disable the perpetual storage wiggle. +``perpetual_storage_wiggle_locality`` sets the process filter for wiggling. The processes that match the given locality key and locality value are only wiggled. The value 0 will disable the locality filter and matches all the processes for wiggling. + +For more details, see :ref:`perpetual-storage-wiggle`. storage migration type ^^^^^^^^^^^^^^^^^^^^^^ diff --git a/documentation/sphinx/source/mr-status-json-schemas.rst.inc b/documentation/sphinx/source/mr-status-json-schemas.rst.inc index 2b88704397..f07ae8054f 100644 --- a/documentation/sphinx/source/mr-status-json-schemas.rst.inc +++ b/documentation/sphinx/source/mr-status-json-schemas.rst.inc @@ -736,6 +736,7 @@ "proxies":6, // this field will be absent if a value has not been explicitly set "backup_worker_enabled":1, "perpetual_storage_wiggle": 0, + "perpetual_storage_wiggle_locality":"0" "storage_migration_type":{ "$enum":[ "disabled", diff --git a/documentation/sphinx/source/perpetual-storage-wiggle.rst b/documentation/sphinx/source/perpetual-storage-wiggle.rst index c469701657..43c4065362 100644 --- a/documentation/sphinx/source/perpetual-storage-wiggle.rst +++ b/documentation/sphinx/source/perpetual-storage-wiggle.rst @@ -40,6 +40,10 @@ Open perpetual storage wiggle: ``configure perpetual_storage_wiggle=1``. Disable perpetual storage wiggle on the cluster: ``configure perpetual_storage_wiggle=0``. +Open perpetual storage wiggle for only processes matching the given locality key and value: ``configure perpetual_storage_wiggle=1 perpetual_storage_wiggle_locality=:``. + +Disable perpetual storage wiggle locality matching filter, which wiggles all the processes: ``configure perpetual_storage_wiggle_locality=0``. + Monitor ======= diff --git a/fdbcli/ConfigureCommand.actor.cpp b/fdbcli/ConfigureCommand.actor.cpp index 586cdd126a..256a0cc30e 100644 --- a/fdbcli/ConfigureCommand.actor.cpp +++ b/fdbcli/ConfigureCommand.actor.cpp @@ -262,9 +262,9 @@ CommandFactory configureFactory( CommandHelp( "configure [new|tss]" "|" - "commit_proxies=|grv_proxies=|logs=|resolvers=>*|" - "count=|perpetual_storage_wiggle=|storage_migration_type={disabled|gradual|" - "aggressive}", + "commit_proxies=|grv_proxies=|logs=|resolvers=>*|count=|" + "perpetual_storage_wiggle=|perpetual_storage_wiggle_locality=<:|0>|" + "storage_migration_type={disabled|gradual|aggressive}", "change the database configuration", "The `new' option, if present, initializes a new database with the given configuration rather than changing " "the configuration of an existing one. When used, both a redundancy mode and a storage engine must be " @@ -292,6 +292,9 @@ CommandFactory configureFactory( "perpetual_storage_wiggle=: Set the value speed (a.k.a., the number of processes that the Data " "Distributor should wiggle at a time). Currently, only 0 and 1 are supported. The value 0 means to disable the " "perpetual storage wiggle.\n\n" + "perpetual_storage_wiggle_locality=<:|0>: Set the process filter for wiggling. " + "The processes that match the given locality key and locality value are only wiggled. The value 0 will disable " + "the locality filter and matches all the processes for wiggling.\n\n" "See the FoundationDB Administration Guide for more information.")); } // namespace fdb_cli diff --git a/fdbcli/fdbcli.actor.cpp b/fdbcli/fdbcli.actor.cpp index 2cf1946bf0..d1bf93be53 100644 --- a/fdbcli/fdbcli.actor.cpp +++ b/fdbcli/fdbcli.actor.cpp @@ -1171,6 +1171,7 @@ void configureGenerator(const char* text, const char* line, std::vectorinfo() != "dcid^2 x zoneid^2 x 1") && // We cannot specify regions with three_datacenter replication (perpetualStorageWiggleSpeed == 0 || perpetualStorageWiggleSpeed == 1) && + (perpetualStorageWiggleLocality.find(':') != std::string::npos || + !perpetualStorageWiggleLocality.compare("0")) && storageMigrationType != StorageMigrationType::UNSET)) { return false; } @@ -391,6 +394,7 @@ StatusObject DatabaseConfiguration::toJSON(bool noPolicies) const { result["backup_worker_enabled"] = (int32_t)backupWorkerEnabled; result["perpetual_storage_wiggle"] = perpetualStorageWiggleSpeed; + result["perpetual_storage_wiggle_locality"] = perpetualStorageWiggleLocality; result["storage_migration_type"] = storageMigrationType.toString(); return result; } @@ -545,6 +549,11 @@ bool DatabaseConfiguration::setInternal(KeyRef key, ValueRef value) { parse(®ions, value); } else if (ck == LiteralStringRef("perpetual_storage_wiggle")) { parse(&perpetualStorageWiggleSpeed, value); + } else if (ck == LiteralStringRef("perpetual_storage_wiggle_locality")) { + if (value.toString().find(':') == std::string::npos && value.toString().compare("0")) { + return false; + } + perpetualStorageWiggleLocality = value.toString(); } else if (ck == LiteralStringRef("storage_migration_type")) { parse((&type), value); storageMigrationType = (StorageMigrationType::MigrationType)type; diff --git a/fdbclient/DatabaseConfiguration.h b/fdbclient/DatabaseConfiguration.h index ccb11e985c..a390907529 100644 --- a/fdbclient/DatabaseConfiguration.h +++ b/fdbclient/DatabaseConfiguration.h @@ -245,6 +245,7 @@ struct DatabaseConfiguration { // Perpetual Storage Setting int32_t perpetualStorageWiggleSpeed; + std::string perpetualStorageWiggleLocality; // Storage Migration Type StorageMigrationType storageMigrationType; diff --git a/fdbclient/ManagementAPI.actor.cpp b/fdbclient/ManagementAPI.actor.cpp index a195e417db..0b3f1d9656 100644 --- a/fdbclient/ManagementAPI.actor.cpp +++ b/fdbclient/ManagementAPI.actor.cpp @@ -150,6 +150,14 @@ std::map configForToken(std::string const& mode) { } out[p + key] = value; } + if (key == "perpetual_storage_wiggle_locality") { + if (value.find(':') == std::string::npos && value.compare("0")) { + printf("Error: perpetual_storage_wiggle_locality should be in : " + "format or enter 0 to disable the locality match for wiggling.\n"); + return out; + } + out[p + key] = value; + } if (key == "storage_migration_type") { StorageMigrationType type; if (value == "disabled") { diff --git a/fdbclient/Schemas.cpp b/fdbclient/Schemas.cpp index bd20ec46f1..01044f2fc1 100644 --- a/fdbclient/Schemas.cpp +++ b/fdbclient/Schemas.cpp @@ -760,6 +760,7 @@ const KeyRef JSONSchemas::statusSchema = LiteralStringRef(R"statusSchema( "proxies":6, "backup_worker_enabled":1, "perpetual_storage_wiggle":0, + "perpetual_storage_wiggle_locality":"0", "storage_migration_type": { "$enum":[ "disabled", diff --git a/fdbclient/SystemData.cpp b/fdbclient/SystemData.cpp index c5e925e7c8..3545defccb 100644 --- a/fdbclient/SystemData.cpp +++ b/fdbclient/SystemData.cpp @@ -630,6 +630,7 @@ const KeyRangeRef configKeys(LiteralStringRef("\xff/conf/"), LiteralStringRef("\ const KeyRef configKeysPrefix = configKeys.begin; const KeyRef perpetualStorageWiggleKey(LiteralStringRef("\xff/conf/perpetual_storage_wiggle")); +const KeyRef perpetualStorageWiggleLocalityKey(LiteralStringRef("\xff/conf/perpetual_storage_wiggle_locality")); const KeyRef wigglingStorageServerKey(LiteralStringRef("\xff/storageWigglePID")); const KeyRef triggerDDTeamInfoPrintKey(LiteralStringRef("\xff/triggerDDTeamInfoPrint")); diff --git a/fdbclient/SystemData.h b/fdbclient/SystemData.h index f29f786404..64eb82c400 100644 --- a/fdbclient/SystemData.h +++ b/fdbclient/SystemData.h @@ -210,6 +210,7 @@ extern const KeyRangeRef configKeys; extern const KeyRef configKeysPrefix; extern const KeyRef perpetualStorageWiggleKey; +extern const KeyRef perpetualStorageWiggleLocalityKey; extern const KeyRef wigglingStorageServerKey; // Change the value of this key to anything and that will trigger detailed data distribution team info log. extern const KeyRef triggerDDTeamInfoPrintKey; diff --git a/fdbserver/DataDistribution.actor.cpp b/fdbserver/DataDistribution.actor.cpp index f5e135c152..f986f04385 100644 --- a/fdbserver/DataDistribution.actor.cpp +++ b/fdbserver/DataDistribution.actor.cpp @@ -4009,16 +4009,69 @@ ACTOR Future>> getSe // to a sorted PID set maintained by the data distributor. If now no storage server exists, the new Process ID is 0. ACTOR Future updateNextWigglingStoragePID(DDTeamCollection* teamCollection) { state ReadYourWritesTransaction tr(teamCollection->cx); - state Value writeValue; + state Value writeValue = LiteralStringRef(""); state const Key writeKey = wigglingStorageServerKey.withSuffix(teamCollection->primary ? "/primary"_sr : "/remote"_sr); loop { try { tr.setOption(FDBTransactionOptions::ACCESS_SYSTEM_KEYS); - Optional value = wait(tr.get(writeKey)); + Optional locality = wait(tr.get(perpetualStorageWiggleLocalityKey)); + if (teamCollection->pid2server_info.empty()) { writeValue = LiteralStringRef(""); + } else if (locality.present() && locality.get().toString().compare("0")) { + // if perpetual_storage_wiggle_locality has value and not 0(disabled). + state std::string localityKeyValue = locality.get().toString(); + int split = localityKeyValue.find(':'); + ASSERT(split != std::string::npos); + + // get key and value from perpetual_storage_wiggle_locality. + state std::string localityKey = localityKeyValue.substr(0, split); + state std::string localityValue = localityKeyValue.substr(split + 1); + state Value prevValue; + state int serverInfoSize = teamCollection->pid2server_info.size(); + + Optional value = wait(tr.get(writeKey)); + if (value.present()) { + prevValue = value.get(); + } else { + // if value not present, check for locality match of the first entry in pid2server_info. + auto& info_vec = teamCollection->pid2server_info.begin()->second; + if (info_vec.size() && info_vec[0]->lastKnownInterface.locality.get(localityKey) == localityValue) { + writeValue = teamCollection->pid2server_info.begin()->first; // first entry locality matched. + } else { + prevValue = teamCollection->pid2server_info.begin()->first; + serverInfoSize--; + } + } + + // If first entry of pid2server_info, did not match the locality. + if (!(writeValue.compare(LiteralStringRef("")))) { + while (true) { + auto nextIt = teamCollection->pid2server_info.upper_bound(prevValue); + if (nextIt == teamCollection->pid2server_info.end()) { + nextIt = teamCollection->pid2server_info.begin(); + } + + if (nextIt->second.size() && + nextIt->second[0]->lastKnownInterface.locality.get(localityKey) == localityValue) { + writeValue = nextIt->first; // locality matched + break; + } + serverInfoSize--; + if (!serverInfoSize) { + // None of the entries in pid2server_info matched the given locality. + writeValue = LiteralStringRef(""); + TraceEvent("PerpetualNextWigglingStoragePIDNotFound", teamCollection->distributorId) + .detail("WriteValue", "No process matched the given perpetualStorageWiggleLocality") + .detail("PerpetualStorageWiggleLocality", localityKeyValue); + break; + } + prevValue = nextIt->first; + } + } } else { + Optional value = wait(tr.get(writeKey)); Value pid = teamCollection->pid2server_info.begin()->first; if (value.present()) { auto nextIt = teamCollection->pid2server_info.upper_bound(value.get()); @@ -4031,6 +4084,7 @@ ACTOR Future updateNextWigglingStoragePID(DDTeamCollection* teamCollection writeValue = pid; } } + tr.set(writeKey, writeValue); wait(tr.commit()); break; diff --git a/fdbserver/SimulatedCluster.actor.cpp b/fdbserver/SimulatedCluster.actor.cpp index e0b1278131..a1a5eec86e 100644 --- a/fdbserver/SimulatedCluster.actor.cpp +++ b/fdbserver/SimulatedCluster.actor.cpp @@ -1781,7 +1781,7 @@ void setupSimulatedSystem(std::vector>* systemActors, } auto configDBType = testConfig.getConfigDBType(); for (auto kv : startingConfigJSON) { - if ("tss_storage_engine" == kv.first) { + if ("tss_storage_engine" == kv.first || "perpetual_storage_wiggle_locality" == kv.first) { continue; } startingConfigString += " ";