apple/foundationdb
1
0
Fork 0
mirror of https://github.com/apple/foundationdb.git synced 2025-05-21 22:33:17 +08:00
Code Issues Packages Projects Releases Wiki Activity

Add the function to return the special key for given management api command and option

Browse Source
This commit is contained in:
Chaoguang Lin 2020-07-17 12:32:42 -07:00
parent 96e5a4c827
commit 1e5096990e
3 changed files with 25 additions and 28 deletions

4
fdbclient/ManagementAPI.actor.cpp

@ -1282,9 +1282,7 @@ ACTOR Future<Void> excludeServers(Database cx, vector<AddressExclusion> servers,
loop { loop {
try{ try{
ryw.setOption( FDBTransactionOptions::SPECIAL_KEY_SPACE_CHANGE_CONFIGURATION); ryw.setOption( FDBTransactionOptions::SPECIAL_KEY_SPACE_CHANGE_CONFIGURATION);
Key optionKey = failed ? LiteralStringRef("\xff\xff/conf/options/failed/force") ryw.set(SpecialKeySpace::getManagementApiCommandOptionSpecialKey(failed ? "failed" : "exclude", "force"), ValueRef());
: LiteralStringRef("\xff\xff/conf/options/exclude/force");
ryw.set(optionKey, ValueRef());
for(auto& s : servers) { for(auto& s : servers) {
Key addr = failed ? SpecialKeySpace::getManagementApiCommandPrefix("failed").withSuffix(s.toString()) Key addr = failed ? SpecialKeySpace::getManagementApiCommandPrefix("failed").withSuffix(s.toString())
: SpecialKeySpace::getManagementApiCommandPrefix("exclude").withSuffix(s.toString()); : SpecialKeySpace::getManagementApiCommandPrefix("exclude").withSuffix(s.toString());

43
fdbclient/SpecialKeySpace.actor.cpp

@ -39,10 +39,14 @@ std::unordered_map<SpecialKeySpace::MODULE, KeyRange> SpecialKeySpace::moduleToB
}; };
std::unordered_map<std::string, KeyRange> SpecialKeySpace::managementApiCommandToRange = { std::unordered_map<std::string, KeyRange> SpecialKeySpace::managementApiCommandToRange = {
{ "exclude", KeyRangeRef(LiteralStringRef("excluded/"), LiteralStringRef("excluded0")).withPrefix(managementApiRange.begin) }, { "exclude",
{ "failed", KeyRangeRef(LiteralStringRef("failed/"), LiteralStringRef("failed0")).withPrefix(managementApiRange.begin) } KeyRangeRef(LiteralStringRef("excluded/"), LiteralStringRef("excluded0")).withPrefix(managementApiRange.begin) },
{ "failed",
KeyRangeRef(LiteralStringRef("failed/"), LiteralStringRef("failed0")).withPrefix(managementApiRange.begin) }
}; };
std::unordered_set<std::string> SpecialKeySpace::options = { "exclude/force", "failed/force" };
// This function will move the given KeySelector as far as possible to the standard form: // This function will move the given KeySelector as far as possible to the standard form:
// orEqual == false && offset == 1 (Standard form) // orEqual == false && offset == 1 (Standard form)
// If the corresponding key is not in the underlying key range, it will move over the range // If the corresponding key is not in the underlying key range, it will move over the range
@ -348,7 +352,6 @@ void SpecialKeySpace::set(ReadYourWritesTransaction* ryw, const KeyRef& key, con
if (!ryw->specialKeySpaceChangeConfiguration()) throw special_keys_write_disabled(); if (!ryw->specialKeySpaceChangeConfiguration()) throw special_keys_write_disabled();
// TODO : check value valid // TODO : check value valid
auto impl = writeImpls[key]; auto impl = writeImpls[key];
// TODO : do we need the separate error here to differentiate from read?
if (impl == nullptr) throw special_keys_no_write_module_found(); if (impl == nullptr) throw special_keys_no_write_module_found();
TraceEvent(SevDebug, "SpecialKeySpaceSet") TraceEvent(SevDebug, "SpecialKeySpaceSet")
.detail("Key", key.toString()) .detail("Key", key.toString())
@ -408,7 +411,6 @@ ACTOR Future<Void> commitActor(SpecialKeySpace* sks, ReadYourWritesTransaction*
state RangeMap<Key, std::pair<bool, Optional<Value>>, KeyRangeRef>::Ranges ranges = state RangeMap<Key, std::pair<bool, Optional<Value>>, KeyRangeRef>::Ranges ranges =
ryw->getSpecialKeySpaceWriteMap().containedRanges(specialKeys); ryw->getSpecialKeySpaceWriteMap().containedRanges(specialKeys);
state RangeMap<Key, std::pair<bool, Optional<Value>>, KeyRangeRef>::iterator iter = ranges.begin(); state RangeMap<Key, std::pair<bool, Optional<Value>>, KeyRangeRef>::iterator iter = ranges.begin();
// TODO : update this set container
state std::set<SpecialKeyRangeRWImpl*> writeModulePtrs; state std::set<SpecialKeyRangeRWImpl*> writeModulePtrs;
while (iter != ranges.end()) { while (iter != ranges.end()) {
std::pair<bool, Optional<Value>> entry = iter->value(); std::pair<bool, Optional<Value>> entry = iter->value();
@ -508,12 +510,12 @@ Future<Standalone<RangeResultRef>> DDStatsRangeImpl::getRange(ReadYourWritesTran
return ddMetricsGetRangeActor(ryw, kr); return ddMetricsGetRangeActor(ryw, kr);
} }
std::unordered_set<std::string> ManagementCommandsOptionsImpl::options = { "exclude/force", "failed/force" }; Key SpecialKeySpace::getManagementApiCommandOptionSpecialKey(const std::string& command, const std::string& option) {
Key prefix = LiteralStringRef("options/").withPrefix(managementApiRange.begin);
// Optional<Key> ManagementCommandsOptionsImpl::getOptionSpecialKey(const std::string& command, const std::string& auto pair = command + "/" + option;
// option) { auto pair = command + "/" + option; if (options.find(pair) != options.end()) { return ASSERT(options.find(pair) != options.end());
// Optional<Key>(getKeyRange().begin.withSuffix(pair)); } else return Optional<Key>(); return prefix.withSuffix(pair);
// } }
ManagementCommandsOptionsImpl::ManagementCommandsOptionsImpl(KeyRangeRef kr) : SpecialKeyRangeRWImpl(kr) {} ManagementCommandsOptionsImpl::ManagementCommandsOptionsImpl(KeyRangeRef kr) : SpecialKeyRangeRWImpl(kr) {}
@ -521,7 +523,7 @@ Future<Standalone<RangeResultRef>> ManagementCommandsOptionsImpl::getRange(ReadY
KeyRangeRef kr) const { KeyRangeRef kr) const {
Standalone<RangeResultRef> result; Standalone<RangeResultRef> result;
// Since we only have limit number of options, a brute force loop here is enough // Since we only have limit number of options, a brute force loop here is enough
for (const auto& option : options) { for (const auto& option : SpecialKeySpace::getManamentApiOptionsSet()) {
auto key = getKeyRange().begin.withSuffix(option); auto key = getKeyRange().begin.withSuffix(option);
// ignore all invalid keys // ignore all invalid keys
auto r = ryw->getSpecialKeySpaceWriteMap()[key]; auto r = ryw->getSpecialKeySpaceWriteMap()[key];
@ -534,15 +536,15 @@ Future<Standalone<RangeResultRef>> ManagementCommandsOptionsImpl::getRange(ReadY
void ManagementCommandsOptionsImpl::set(ReadYourWritesTransaction* ryw, const KeyRef& key, const ValueRef& value) { void ManagementCommandsOptionsImpl::set(ReadYourWritesTransaction* ryw, const KeyRef& key, const ValueRef& value) {
std::string option = key.removePrefix(getKeyRange().begin).toString(); std::string option = key.removePrefix(getKeyRange().begin).toString();
// ignore all invalid keys // ignore all invalid keys
if (options.find(option) != options.end()) { if (SpecialKeySpace::getManamentApiOptionsSet().find(option) != SpecialKeySpace::getManamentApiOptionsSet().end()) {
TraceEvent(SevDebug, "ManagementAPIOption").detail("Option", option).detail("Key", key); TraceEvent(SevDebug, "ManagementApiOption").detail("Option", option).detail("Key", key);
ryw->getSpecialKeySpaceWriteMap().insert(key, std::make_pair(true, Optional<Value>(value))); ryw->getSpecialKeySpaceWriteMap().insert(key, std::make_pair(true, Optional<Value>(value)));
} }
} }
void ManagementCommandsOptionsImpl::clear(ReadYourWritesTransaction* ryw, const KeyRangeRef& range) { void ManagementCommandsOptionsImpl::clear(ReadYourWritesTransaction* ryw, const KeyRangeRef& range) {
// Since we only have limit number of options, a brute force loop here is enough // Since we only have limit number of options, a brute force loop here is enough
for (const auto& option : options) { for (const auto& option : SpecialKeySpace::getManamentApiOptionsSet()) {
auto key = getKeyRange().begin.withSuffix(option); auto key = getKeyRange().begin.withSuffix(option);
// ignore all invalid keys // ignore all invalid keys
if (range.contains(key)) if (range.contains(key))
@ -554,7 +556,7 @@ void ManagementCommandsOptionsImpl::clear(ReadYourWritesTransaction* ryw, const
void ManagementCommandsOptionsImpl::clear(ReadYourWritesTransaction* ryw, const KeyRef& key) { void ManagementCommandsOptionsImpl::clear(ReadYourWritesTransaction* ryw, const KeyRef& key) {
std::string option = key.removePrefix(getKeyRange().begin).toString(); std::string option = key.removePrefix(getKeyRange().begin).toString();
// ignore all invalid keys // ignore all invalid keys
if (options.find(option) != options.end()) { if (SpecialKeySpace::getManamentApiOptionsSet().find(option) != SpecialKeySpace::getManamentApiOptionsSet().end()) {
// ryw->getSpecialKeySpaceWriteMap().insert(key, std::make_pair(true, Optional<Value>())); // ryw->getSpecialKeySpaceWriteMap().insert(key, std::make_pair(true, Optional<Value>()));
ryw->getSpecialKeySpaceWriteMap().rawErase(singleKeyRange(key)); ryw->getSpecialKeySpaceWriteMap().rawErase(singleKeyRange(key));
} }
@ -568,7 +570,7 @@ Future<Optional<std::string>> ManagementCommandsOptionsImpl::commit(ReadYourWrit
// read from rwModule // read from rwModule
ACTOR Future<Standalone<RangeResultRef>> rwModuleGetRangeActor(ReadYourWritesTransaction* ryw, KeyRangeRef range, ACTOR Future<Standalone<RangeResultRef>> rwModuleGetRangeActor(ReadYourWritesTransaction* ryw, KeyRangeRef range,
KeyRangeRef kr) { KeyRangeRef kr) {
KeyRangeRef krWithoutPrefix = kr.removePrefix(normalKeys.end); // TODO : need to update KeyRangeRef krWithoutPrefix = kr.removePrefix(normalKeys.end); // TODO : need to replace with a general encode/decode funciton
Standalone<RangeResultRef> resultWithoutPrefix = wait(ryw->getRange(krWithoutPrefix, CLIENT_KNOBS->TOO_MANY)); Standalone<RangeResultRef> resultWithoutPrefix = wait(ryw->getRange(krWithoutPrefix, CLIENT_KNOBS->TOO_MANY));
ASSERT(!resultWithoutPrefix.more && resultWithoutPrefix.size() < CLIENT_KNOBS->TOO_MANY); ASSERT(!resultWithoutPrefix.more && resultWithoutPrefix.size() < CLIENT_KNOBS->TOO_MANY);
Standalone<RangeResultRef> result; Standalone<RangeResultRef> result;
@ -695,7 +697,7 @@ ACTOR Future<bool> checkExclusion(Database db, std::vector<AddressExclusion>* ad
StatusObject status = wait(StatusClient::statusFetcher(db)); StatusObject status = wait(StatusClient::statusFetcher(db));
state std::string errorString = state std::string errorString =
"ERROR: Could not calculate the impact of this exclude on the total free space in the cluster.\n" "ERROR: Could not calculate the impact of this exclude on the total free space in the cluster.\n"
"Please try the exclude again in 30 seconds.\n"; // TODO : update msg here "Please try the exclude again in 30 seconds.\n";
StatusObjectReader statusObj(status); StatusObjectReader statusObj(status);
@ -773,8 +775,6 @@ ACTOR Future<bool> checkExclusion(Database db, std::vector<AddressExclusion>* ad
if (ssExcludedCount == ssTotalCount || if (ssExcludedCount == ssTotalCount ||
(1 - worstFreeSpaceRatio) * ssTotalCount / (ssTotalCount - ssExcludedCount) > 0.9) { (1 - worstFreeSpaceRatio) * ssTotalCount / (ssTotalCount - ssExcludedCount) > 0.9) {
std::string temp = "ERROR: This exclude may cause the total free space in the cluster to drop below 10%."; std::string temp = "ERROR: This exclude may cause the total free space in the cluster to drop below 10%.";
// "\nType `exclude FORCE <ADDRESS...>' to exclude without checking free space.\n"; // TODO : update message
// here
*msg = ManagementAPIError::toJsonString(false, markFailed ? "exclude failed" : "exclude", temp); *msg = ManagementAPIError::toJsonString(false, markFailed ? "exclude failed" : "exclude", temp);
return false; return false;
} }
@ -829,8 +829,8 @@ ACTOR Future<Optional<std::string>> excludeCommitActor(ReadYourWritesTransaction
addresses, exclusions, result)) addresses, exclusions, result))
return result; return result;
// If force option is not set, we need to do safety check // If force option is not set, we need to do safety check
auto force = ryw->getSpecialKeySpaceWriteMap()[failed ? LiteralStringRef("\xff\xff/conf/options/failed/force") auto force = ryw->getSpecialKeySpaceWriteMap()[SpecialKeySpace::getManagementApiCommandOptionSpecialKey(
: LiteralStringRef("\xff\xff/conf/options/exclude/force")]; failed ? "failed" : "exclude", "force")];
TraceEvent(SevDebug, "ExclusionCommit") TraceEvent(SevDebug, "ExclusionCommit")
.detail("Failed", failed) .detail("Failed", failed)
.detail("Force", force.first) .detail("Force", force.first)
@ -878,7 +878,6 @@ Future<Optional<std::string>> FailedServersRangeImpl::commit(ReadYourWritesTrans
ACTOR Future<Standalone<RangeResultRef>> ExclusionInProgressActor(ReadYourWritesTransaction* ryw, KeyRef prefix, ACTOR Future<Standalone<RangeResultRef>> ExclusionInProgressActor(ReadYourWritesTransaction* ryw, KeyRef prefix,
KeyRangeRef kr) { KeyRangeRef kr) {
state Standalone<RangeResultRef> result; state Standalone<RangeResultRef> result;
// TODO : get all inprogress excluded servers
state Transaction& tr = ryw->getTransaction(); state Transaction& tr = ryw->getTransaction();
tr.setOption(FDBTransactionOptions::READ_SYSTEM_KEYS); tr.setOption(FDBTransactionOptions::READ_SYSTEM_KEYS);
tr.setOption(FDBTransactionOptions::PRIORITY_SYSTEM_IMMEDIATE); // necessary? tr.setOption(FDBTransactionOptions::PRIORITY_SYSTEM_IMMEDIATE); // necessary?

6
fdbclient/SpecialKeySpace.actor.h

@ -156,6 +156,8 @@ public:
KeyRangeRef getKeyRange() const { return range; } KeyRangeRef getKeyRange() const { return range; }
static KeyRange getManamentApiCommandRange(std::string command) { return managementApiCommandToRange.at(command); } static KeyRange getManamentApiCommandRange(std::string command) { return managementApiCommandToRange.at(command); }
static Key getManagementApiCommandPrefix(std::string command) { return managementApiCommandToRange.at(command).begin; } static Key getManagementApiCommandPrefix(std::string command) { return managementApiCommandToRange.at(command).begin; }
static Key getManagementApiCommandOptionSpecialKey(const std::string& command, const std::string& option);
static const std::unordered_set<std::string>& getManamentApiOptionsSet() { return options; }
private: private:
ACTOR static Future<Optional<Value>> getActor(SpecialKeySpace* sks, ReadYourWritesTransaction* ryw, KeyRef key); ACTOR static Future<Optional<Value>> getActor(SpecialKeySpace* sks, ReadYourWritesTransaction* ryw, KeyRef key);
@ -175,6 +177,7 @@ private:
static std::unordered_map<SpecialKeySpace::MODULE, KeyRange> moduleToBoundary; static std::unordered_map<SpecialKeySpace::MODULE, KeyRange> moduleToBoundary;
static std::unordered_map<std::string, KeyRange> managementApiCommandToRange; // management command to its special keys' range static std::unordered_map<std::string, KeyRange> managementApiCommandToRange; // management command to its special keys' range
static std::unordered_set<std::string> options; // "<command>/<option>"
// Initialize module boundaries, used to handle cross_module_read // Initialize module boundaries, used to handle cross_module_read
void modulesBoundaryInit(); void modulesBoundaryInit();
@ -218,9 +221,6 @@ public:
void clear(ReadYourWritesTransaction* ryw, const KeyRangeRef& range) override; void clear(ReadYourWritesTransaction* ryw, const KeyRangeRef& range) override;
void clear(ReadYourWritesTransaction* ryw, const KeyRef& key) override; void clear(ReadYourWritesTransaction* ryw, const KeyRef& key) override;
Future<Optional<std::string>> commit(ReadYourWritesTransaction* ryw) override; Future<Optional<std::string>> commit(ReadYourWritesTransaction* ryw) override;
private:
static std::unordered_set<std::string> options; // "<command>/<option>"
}; };
class ExcludeServersRangeImpl : public SpecialKeyRangeRWImpl { class ExcludeServersRangeImpl : public SpecialKeyRangeRWImpl {