Allow the TOML file assign knobs during test

In this patch, for a given test, it is possible to override the knob
values, e.g.

[[test]]

    [[test.knobs]]
    watch_timeout = 999

will set the client knob WATCH_TIMEOUT to 999 during the test. The
original value will be recovered after the test is over.

fdbserver/CMakeLists.txt

@@ -50,10 +50,8 @@ set(FDBSERVER_SRCS
   KeyValueStoreMemory.actor.cpp
   KeyValueStoreRocksDB.actor.cpp
   KeyValueStoreSQLite.actor.cpp
-  ServerCheckpoint.actor.cpp
-  ServerCheckpoint.actor.h
-  RocksDBCheckpointUtils.actor.cpp
-  RocksDBCheckpointUtils.actor.h
+  KnobProtectiveGroups.cpp
+  KnobProtectiveGroups.h
   Knobs.h
   LatencyBandConfig.cpp
   LatencyBandConfig.h

fdbserver/KnobProtectiveGroups.cpp

@@ -0,0 +1,73 @@
+/*
+ * KnobProtectiveGroups.cpp
+ *
+ * This source file is part of the FoundationDB open source project
+ *
+ * Copyright 2013-2022 Apple Inc. and the FoundationDB project authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include "fdbserver/KnobProtectiveGroups.h"
+
+#include <array>
+
+#include "fdbclient/Knobs.h"
+#include "fdbclient/ServerKnobCollection.h"
+#include "fdbserver/Knobs.h"
+
+void KnobKeyValuePairs::set(const std::string& name, const ParsedKnobValue value) {
+	ASSERT(knobs.count(name) == 0);
+
+	knobs[name] = value;
+}
+
+const KnobKeyValuePairs::container_t& KnobKeyValuePairs::getKnobs() const {
+	return knobs;
+}
+
+KnobProtectiveGroup::KnobProtectiveGroup(const KnobKeyValuePairs& overriddenKnobKeyValuePairs_)
+  : overriddenKnobs(overriddenKnobKeyValuePairs_) {
+	snapshotOriginalKnobs();
+	assignKnobs(overriddenKnobs);
+}
+
+KnobProtectiveGroup::~KnobProtectiveGroup() {
+	assignKnobs(originalKnobs);
+}
+
+void KnobProtectiveGroup::snapshotOriginalKnobs() {
+	for (const auto& [name, _] : overriddenKnobs.getKnobs()) {
+		ParsedKnobValue value = CLIENT_KNOBS->getKnob(name);
+		if (std::get_if<NoKnobFound>(&value)) {
+			value = SERVER_KNOBS->getKnob(name);
+		}
+		if (std::get_if<NoKnobFound>(&value)) {
+			ASSERT(false);
+		}
+		originalKnobs.set(name, value);
+		TraceEvent("SnapshotKnobValue")
+		    .detail("KnobName", name)
+		    .detail("KnobValue", KnobValueRef::create(value).toString());
+	}
+}
+
+void KnobProtectiveGroup::assignKnobs(const KnobKeyValuePairs& overrideKnobs) {
+	auto& mutableServerKnobs = dynamic_cast<ServerKnobCollection&>(IKnobCollection::getMutableGlobalKnobCollection());
+
+	for (const auto& [name, value] : overrideKnobs.getKnobs()) {
+		Standalone<KnobValueRef> valueRef = KnobValueRef::create(value);
+		ASSERT(mutableServerKnobs.trySetKnob(name, valueRef));
+		TraceEvent("AssignKnobValue").detail("KnobName", name).detail("KnobValue", valueRef.toString());
+	}
+}

fdbserver/KnobProtectiveGroups.h

@@ -0,0 +1,59 @@
+/*
+ * KnobProtectiveGroups.h
+ *
+ * This source file is part of the FoundationDB open source project
+ *
+ * Copyright 2013-2022 Apple Inc. and the FoundationDB project authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#ifndef FDBSERVER_KNOBPROTECTIVEGROUPS_H
+#define FDBSERVER_KNOBPROTECTIVEGROUPS_H
+
+#include <array>
+#include <unordered_map>
+
+#include "fdbclient/IKnobCollection.h"
+
+// A list of knob key value pairs
+class KnobKeyValuePairs {
+public:
+    using container_t = std::unordered_map<std::string, ParsedKnobValue>;
+private:
+    // Here the knob value is directly stored, unlike KnobValue, for simplicity
+    container_t knobs;
+
+public:
+    // Sets a value for a given knob
+    void set(const std::string& name, const ParsedKnobValue value);
+
+    // Gets a list of knobs for given type
+    const container_t& getKnobs() const;
+};
+
+// For knobs, temporarily change the values, the original values will be recovered 
+class KnobProtectiveGroup {
+    KnobKeyValuePairs overriddenKnobs;
+    KnobKeyValuePairs originalKnobs;
+
+    // Snapshots the current knob values base on those knob keys in overriddenKnobs
+    void snapshotOriginalKnobs();
+
+    void assignKnobs(const KnobKeyValuePairs& overrideKnobs);
+public:
+    KnobProtectiveGroup(const KnobKeyValuePairs& overridenKnobs_);
+    ~KnobProtectiveGroup();
+};
+
+#endif  // FDBSERVER_KNOBPROTECTIVEGROUPS_H

fdbserver/tester.actor.cpp

@@ -31,6 +31,7 @@
 #include "fdbclient/ClusterInterface.h"
 #include "fdbclient/NativeAPI.actor.h"
 #include "fdbclient/SystemData.h"
+#include "fdbserver/KnobProtectiveGroups.h"
 #include "fdbserver/TesterInterface.actor.h"
 #include "fdbserver/WorkerInterface.actor.h"
 #include "fdbserver/workloads/workloads.actor.h"
@@ -1318,7 +1319,7 @@ std::vector<TestSpec> readTOMLTests_(std::string fileName) {
 
 		// First handle all test-level settings
 		for (const auto& [k, v] : test.as_table()) {
-			if (k == "workload") {
+			if (k == "workload" || k == "knobs") {
 				continue;
 			}
 			if (testSpecTestKeys.find(k) != testSpecTestKeys.end()) {
@@ -1344,6 +1345,27 @@ std::vector<TestSpec> readTOMLTests_(std::string fileName) {
 			spec.options.push_back_deep(spec.options.arena(), workloadOptions);
 		}
 
+		// And then copy the knob attributes to spec.overrideKnobs
+		try {
+			const toml::array& overrideKnobs = toml::find(test, "knobs").as_array();
+			for (const toml::value& knob : overrideKnobs) {
+				for (const auto& [key_, value_] : knob.as_table()) {
+					const std::string key = key_;
+					const std::string& value = toml_to_string(value_);
+					ParsedKnobValue parsedValue = CLIENT_KNOBS->parseKnobValue(key, value);
+					if (std::get_if<NoKnobFound>(&parsedValue)) {
+						parsedValue = SERVER_KNOBS->parseKnobValue(key, value);
+					}
+					if (std::get_if<NoKnobFound>(&parsedValue)) {
+						ASSERT(false);
+					}
+					spec.overrideKnobs.set(key, parsedValue);
+				}
+			}
+		} catch (const std::out_of_range&) {
+			// no knob overridden
+		}
+
 		result.push_back(spec);
 	}
 
@@ -1529,9 +1551,12 @@ ACTOR Future<Void> runTests(Reference<AsyncVar<Optional<struct ClusterController
 
 	TraceEvent("TestsExpectedToPass").detail("Count", tests.size());
 	state int idx = 0;
+	state std::unique_ptr<KnobProtectiveGroup> knobProtectiveGroup;
 	for (; idx < tests.size(); idx++) {
 		printf("Run test:%s start\n", tests[idx].title.toString().c_str());
+		knobProtectiveGroup = std::make_unique<KnobProtectiveGroup>(tests[idx].overrideKnobs);
 		wait(success(runTest(cx, testers, tests[idx], dbInfo, defaultTenant)));
+		knobProtectiveGroup.reset(nullptr);
 		printf("Run test:%s Done.\n", tests[idx].title.toString().c_str());
 		// do we handle a failure here?
 	}

fdbserver/workloads/workloads.actor.h

@@ -27,6 +27,7 @@
 
 #include "fdbclient/NativeAPI.actor.h"
 #include "fdbclient/DatabaseContext.h" // for clone()
+#include "fdbserver/KnobProtectiveGroups.h"
 #include "fdbserver/TesterInterface.actor.h"
 #include "fdbrpc/simulator.h"
 #include "flow/actorcompiler.h"
@@ -205,6 +206,8 @@ public:
 	ISimulator::BackupAgentType simBackupAgents; // If set to true, then the simulation runs backup agents on the
 	                                             // workers. Can only be used in simulation.
 	ISimulator::BackupAgentType simDrAgents;
+
+	KnobKeyValuePairs overrideKnobs;
 };
 
 ACTOR Future<DistributedTestResults> runWorkload(Database cx,

flow/Knobs.cpp

@@ -346,6 +346,26 @@ bool Knobs::setKnob(std::string const& knob, std::string const& value) {
 	return true;
 }
 
+ParsedKnobValue Knobs::getKnob(const std::string& name) const {
+	if (double_knobs.count(name) > 0) {
+		return ParsedKnobValue{ *double_knobs.at(name).value };
+	}
+	if (int64_knobs.count(name) > 0) {
+		return ParsedKnobValue{ *int64_knobs.at(name).value };
+	}
+	if (int_knobs.count(name) > 0) {
+		return ParsedKnobValue{ *int_knobs.at(name).value };
+	}
+	if (string_knobs.count(name) > 0) {
+		return ParsedKnobValue{ *string_knobs.at(name).value };
+	}
+	if (bool_knobs.count(name) > 0) {
+		return ParsedKnobValue{ *bool_knobs.at(name).value };
+	}
+
+	return ParsedKnobValue{ NoKnobFound() };
+}
+
 bool Knobs::isAtomic(std::string const& knob) const {
 	if (double_knobs.count(knob)) {
 		return double_knobs.find(knob)->second.atomic == Atomic::YES;

flow/Knobs.h

@@ -76,11 +76,24 @@ protected:
 	std::set<std::string> explicitlySetKnobs;
 
 public:
+	// Sets an integer value to an integer knob, returns false if the knob does not exist or type mismatch
 	bool setKnob(std::string const& name, int value);
+
+	// Sets a boolean value to a bool knob, returns false if the knob does not exist or type mismatch
 	bool setKnob(std::string const& name, bool value);
+
+	// Sets an int64_t value to an int64_t knob, returns false if the knob does not exist or type mismatch
 	bool setKnob(std::string const& name, int64_t value);
+
+	// Sets a double value to a double knob, returns false if the knob does not exist or type mismatch
 	bool setKnob(std::string const& name, double value);
+
+	// Sets a string value to a string knob, returns false if the knob does not exist or type mismatch
 	bool setKnob(std::string const& name, std::string const& value);
+
+	// Gets the value of knob
+	ParsedKnobValue getKnob(const std::string& name) const;
+
 	ParsedKnobValue parseKnobValue(std::string const& name, std::string const& value) const;
 	bool isAtomic(std::string const& knob) const;
 	void trace() const;