diff --git a/fdbserver/CMakeLists.txt b/fdbserver/CMakeLists.txt index 47a17ab10c..e710efb52b 100644 --- a/fdbserver/CMakeLists.txt +++ b/fdbserver/CMakeLists.txt @@ -50,10 +50,8 @@ set(FDBSERVER_SRCS KeyValueStoreMemory.actor.cpp KeyValueStoreRocksDB.actor.cpp KeyValueStoreSQLite.actor.cpp - ServerCheckpoint.actor.cpp - ServerCheckpoint.actor.h - RocksDBCheckpointUtils.actor.cpp - RocksDBCheckpointUtils.actor.h + KnobProtectiveGroups.cpp + KnobProtectiveGroups.h Knobs.h LatencyBandConfig.cpp LatencyBandConfig.h diff --git a/fdbserver/KnobProtectiveGroups.cpp b/fdbserver/KnobProtectiveGroups.cpp new file mode 100644 index 0000000000..6f7d37998a --- /dev/null +++ b/fdbserver/KnobProtectiveGroups.cpp @@ -0,0 +1,73 @@ +/* + * KnobProtectiveGroups.cpp + * + * This source file is part of the FoundationDB open source project + * + * Copyright 2013-2022 Apple Inc. and the FoundationDB project authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "fdbserver/KnobProtectiveGroups.h" + +#include + +#include "fdbclient/Knobs.h" +#include "fdbclient/ServerKnobCollection.h" +#include "fdbserver/Knobs.h" + +void KnobKeyValuePairs::set(const std::string& name, const ParsedKnobValue value) { + ASSERT(knobs.count(name) == 0); + + knobs[name] = value; +} + +const KnobKeyValuePairs::container_t& KnobKeyValuePairs::getKnobs() const { + return knobs; +} + +KnobProtectiveGroup::KnobProtectiveGroup(const KnobKeyValuePairs& overriddenKnobKeyValuePairs_) + : overriddenKnobs(overriddenKnobKeyValuePairs_) { + snapshotOriginalKnobs(); + assignKnobs(overriddenKnobs); +} + +KnobProtectiveGroup::~KnobProtectiveGroup() { + assignKnobs(originalKnobs); +} + +void KnobProtectiveGroup::snapshotOriginalKnobs() { + for (const auto& [name, _] : overriddenKnobs.getKnobs()) { + ParsedKnobValue value = CLIENT_KNOBS->getKnob(name); + if (std::get_if(&value)) { + value = SERVER_KNOBS->getKnob(name); + } + if (std::get_if(&value)) { + ASSERT(false); + } + originalKnobs.set(name, value); + TraceEvent("SnapshotKnobValue") + .detail("KnobName", name) + .detail("KnobValue", KnobValueRef::create(value).toString()); + } +} + +void KnobProtectiveGroup::assignKnobs(const KnobKeyValuePairs& overrideKnobs) { + auto& mutableServerKnobs = dynamic_cast(IKnobCollection::getMutableGlobalKnobCollection()); + + for (const auto& [name, value] : overrideKnobs.getKnobs()) { + Standalone valueRef = KnobValueRef::create(value); + ASSERT(mutableServerKnobs.trySetKnob(name, valueRef)); + TraceEvent("AssignKnobValue").detail("KnobName", name).detail("KnobValue", valueRef.toString()); + } +} \ No newline at end of file diff --git a/fdbserver/KnobProtectiveGroups.h b/fdbserver/KnobProtectiveGroups.h new file mode 100644 index 0000000000..3940d42fa0 --- /dev/null +++ b/fdbserver/KnobProtectiveGroups.h @@ -0,0 +1,59 @@ +/* + * KnobProtectiveGroups.h + * + * This source file is part of the FoundationDB open source project + * + * Copyright 2013-2022 Apple Inc. and the FoundationDB project authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef FDBSERVER_KNOBPROTECTIVEGROUPS_H +#define FDBSERVER_KNOBPROTECTIVEGROUPS_H + +#include +#include + +#include "fdbclient/IKnobCollection.h" + +// A list of knob key value pairs +class KnobKeyValuePairs { +public: + using container_t = std::unordered_map; +private: + // Here the knob value is directly stored, unlike KnobValue, for simplicity + container_t knobs; + +public: + // Sets a value for a given knob + void set(const std::string& name, const ParsedKnobValue value); + + // Gets a list of knobs for given type + const container_t& getKnobs() const; +}; + +// For knobs, temporarily change the values, the original values will be recovered +class KnobProtectiveGroup { + KnobKeyValuePairs overriddenKnobs; + KnobKeyValuePairs originalKnobs; + + // Snapshots the current knob values base on those knob keys in overriddenKnobs + void snapshotOriginalKnobs(); + + void assignKnobs(const KnobKeyValuePairs& overrideKnobs); +public: + KnobProtectiveGroup(const KnobKeyValuePairs& overridenKnobs_); + ~KnobProtectiveGroup(); +}; + +#endif // FDBSERVER_KNOBPROTECTIVEGROUPS_H \ No newline at end of file diff --git a/fdbserver/tester.actor.cpp b/fdbserver/tester.actor.cpp index 911eb14405..49c99c0f74 100644 --- a/fdbserver/tester.actor.cpp +++ b/fdbserver/tester.actor.cpp @@ -31,6 +31,7 @@ #include "fdbclient/ClusterInterface.h" #include "fdbclient/NativeAPI.actor.h" #include "fdbclient/SystemData.h" +#include "fdbserver/KnobProtectiveGroups.h" #include "fdbserver/TesterInterface.actor.h" #include "fdbserver/WorkerInterface.actor.h" #include "fdbserver/workloads/workloads.actor.h" @@ -1318,7 +1319,7 @@ std::vector readTOMLTests_(std::string fileName) { // First handle all test-level settings for (const auto& [k, v] : test.as_table()) { - if (k == "workload") { + if (k == "workload" || k == "knobs") { continue; } if (testSpecTestKeys.find(k) != testSpecTestKeys.end()) { @@ -1344,6 +1345,27 @@ std::vector readTOMLTests_(std::string fileName) { spec.options.push_back_deep(spec.options.arena(), workloadOptions); } + // And then copy the knob attributes to spec.overrideKnobs + try { + const toml::array& overrideKnobs = toml::find(test, "knobs").as_array(); + for (const toml::value& knob : overrideKnobs) { + for (const auto& [key_, value_] : knob.as_table()) { + const std::string key = key_; + const std::string& value = toml_to_string(value_); + ParsedKnobValue parsedValue = CLIENT_KNOBS->parseKnobValue(key, value); + if (std::get_if(&parsedValue)) { + parsedValue = SERVER_KNOBS->parseKnobValue(key, value); + } + if (std::get_if(&parsedValue)) { + ASSERT(false); + } + spec.overrideKnobs.set(key, parsedValue); + } + } + } catch (const std::out_of_range&) { + // no knob overridden + } + result.push_back(spec); } @@ -1529,9 +1551,12 @@ ACTOR Future runTests(Reference knobProtectiveGroup; for (; idx < tests.size(); idx++) { printf("Run test:%s start\n", tests[idx].title.toString().c_str()); + knobProtectiveGroup = std::make_unique(tests[idx].overrideKnobs); wait(success(runTest(cx, testers, tests[idx], dbInfo, defaultTenant))); + knobProtectiveGroup.reset(nullptr); printf("Run test:%s Done.\n", tests[idx].title.toString().c_str()); // do we handle a failure here? } diff --git a/fdbserver/workloads/workloads.actor.h b/fdbserver/workloads/workloads.actor.h index 34e9111f68..bf70332f04 100644 --- a/fdbserver/workloads/workloads.actor.h +++ b/fdbserver/workloads/workloads.actor.h @@ -27,6 +27,7 @@ #include "fdbclient/NativeAPI.actor.h" #include "fdbclient/DatabaseContext.h" // for clone() +#include "fdbserver/KnobProtectiveGroups.h" #include "fdbserver/TesterInterface.actor.h" #include "fdbrpc/simulator.h" #include "flow/actorcompiler.h" @@ -205,6 +206,8 @@ public: ISimulator::BackupAgentType simBackupAgents; // If set to true, then the simulation runs backup agents on the // workers. Can only be used in simulation. ISimulator::BackupAgentType simDrAgents; + + KnobKeyValuePairs overrideKnobs; }; ACTOR Future runWorkload(Database cx, diff --git a/flow/Knobs.cpp b/flow/Knobs.cpp index 6f26ec3662..9f5d43436d 100644 --- a/flow/Knobs.cpp +++ b/flow/Knobs.cpp @@ -346,6 +346,26 @@ bool Knobs::setKnob(std::string const& knob, std::string const& value) { return true; } +ParsedKnobValue Knobs::getKnob(const std::string& name) const { + if (double_knobs.count(name) > 0) { + return ParsedKnobValue{ *double_knobs.at(name).value }; + } + if (int64_knobs.count(name) > 0) { + return ParsedKnobValue{ *int64_knobs.at(name).value }; + } + if (int_knobs.count(name) > 0) { + return ParsedKnobValue{ *int_knobs.at(name).value }; + } + if (string_knobs.count(name) > 0) { + return ParsedKnobValue{ *string_knobs.at(name).value }; + } + if (bool_knobs.count(name) > 0) { + return ParsedKnobValue{ *bool_knobs.at(name).value }; + } + + return ParsedKnobValue{ NoKnobFound() }; +} + bool Knobs::isAtomic(std::string const& knob) const { if (double_knobs.count(knob)) { return double_knobs.find(knob)->second.atomic == Atomic::YES; diff --git a/flow/Knobs.h b/flow/Knobs.h index d9685953cb..7877c7ebca 100644 --- a/flow/Knobs.h +++ b/flow/Knobs.h @@ -76,11 +76,24 @@ protected: std::set explicitlySetKnobs; public: + // Sets an integer value to an integer knob, returns false if the knob does not exist or type mismatch bool setKnob(std::string const& name, int value); + + // Sets a boolean value to a bool knob, returns false if the knob does not exist or type mismatch bool setKnob(std::string const& name, bool value); + + // Sets an int64_t value to an int64_t knob, returns false if the knob does not exist or type mismatch bool setKnob(std::string const& name, int64_t value); + + // Sets a double value to a double knob, returns false if the knob does not exist or type mismatch bool setKnob(std::string const& name, double value); + + // Sets a string value to a string knob, returns false if the knob does not exist or type mismatch bool setKnob(std::string const& name, std::string const& value); + + // Gets the value of knob + ParsedKnobValue getKnob(const std::string& name) const; + ParsedKnobValue parseKnobValue(std::string const& name, std::string const& value) const; bool isAtomic(std::string const& knob) const; void trace() const;