simple benchmarks of IndexedSet, VersionedMap and std::map

fdbclient/CMakeLists.txt

@@ -71,6 +71,7 @@ set(FDBCLIENT_SRCS
   Tuple.h
   VersionedMap.actor.h
   VersionedMap.h
+  VersionedMap.cpp
   WriteMap.h
   json_spirit/json_spirit_error_position.h
   json_spirit/json_spirit_reader_template.h

fdbclient/VersionedMap.cpp

@@ -0,0 +1,56 @@
+#include "fdbclient/VersionedMap.h"
+#include "flow/TreeBenchmark.h"
+#include "flow/UnitTest.h"
+
+template <typename K>
+struct VersionedMapHarness {
+	using map = VersionedMap<K, int>;
+	using key_type = K;
+
+	struct result {
+		typename map::iterator it;
+
+		result(typename map::iterator it) : it(it) {}
+
+		result& operator++() {
+			++it;
+			return *this;
+		}
+
+		const K& operator*() const { return it.key(); }
+
+		const K& operator->() const { return it.key(); }
+
+		bool operator==(result const& k) const { return it == k.it; }
+	};
+
+	map s;
+
+	void insert(K const& k) { s.insert(k, 1); }
+	result find(K const& k) const { return result(s.atLatest().find(k)); }
+	result not_found() const { return result(s.atLatest().end()); }
+	result begin() const { return result(s.atLatest().begin()); }
+	result end() const { return result(s.atLatest().end()); }
+	result lower_bound(K const& k) const { return result(s.atLatest().lower_bound(k)); }
+	result upper_bound(K const& k) const { return result(s.atLatest().upper_bound(k)); }
+	void erase(K const& k) { s.erase(k); }
+};
+
+TEST_CASE("performance/map/int/VersionedMap") {
+    VersionedMapHarness<int> tree;
+
+	treeBenchmark(tree, *randomInt);
+
+	return Void();
+}
+
+TEST_CASE("performance/map/StringRef/VersionedMap") {
+	Arena arena;
+    VersionedMapHarness<StringRef> tree;
+    
+	treeBenchmark(tree, [&arena]() { return randomStr(arena); });
+
+	return Void();
+}
+
+void forceLinkVersionedMapTests() {}

fdbclient/VersionedMap.h

@@ -589,7 +589,7 @@ public:
 
 		UNSTOPPABLE_ASSERT(r->first == newOldestVersion);
 
-		vector<Tree> toFree;
+		std::vector<Tree> toFree;
 		toFree.reserve(10000);
 		auto newBegin = r;
 		Tree *lastRoot = nullptr;
@@ -679,7 +679,7 @@ public:
 		friend class VersionedMap<K,T>;
 		Tree root;
 		Version at;
-		vector< PTreeT const* > finger;
+		std::vector< PTreeT const* > finger;
 	};
 
 	class ViewAtVersion {

fdbserver/workloads/UnitTests.actor.cpp

@@ -25,6 +25,7 @@
 void forceLinkIndexedSetTests();
 void forceLinkDequeTests();
 void forceLinkFlowTests();
+void forceLinkVersionedMapTests();
 
 struct UnitTestWorkload : TestWorkload {
 	bool enabled;
@@ -43,6 +44,7 @@ struct UnitTestWorkload : TestWorkload {
 		forceLinkIndexedSetTests();
 		forceLinkDequeTests();
 		forceLinkFlowTests();
+		forceLinkVersionedMapTests();
 	}
 
 	virtual std::string description() { return "UnitTests"; }

flow/CMakeLists.txt

@@ -61,6 +61,7 @@ set(FLOW_SRCS
   ThreadSafeQueue.h
   Trace.cpp
   Trace.h
+  TreeBenchmark.h
   UnitTest.cpp
   UnitTest.h
   XmlTraceLogFormatter.cpp

flow/IndexedSet.cpp

@@ -31,8 +31,8 @@
 #include <cstring>
 #include <deque>
 #include <random>
+#include "flow/TreeBenchmark.h"
 #include "flow/UnitTest.h"
-
 template <class Node>
 int ISGetHeight(Node* n){
 	if (!n) return 0;
@@ -137,7 +137,121 @@ TEST_CASE("/flow/IndexedSet/erase 400k of 1M") {
 	return Void();
 }
 
-/*TEST_CASE("/flow/IndexedSet/performance") {
+TEST_CASE("/flow/IndexedSet/random ops") {
+	for (int t = 0; t < 100; t++) {
+		IndexedSet<int, int> is;
+		int rr = deterministicRandom()->randomInt(0, 600) * deterministicRandom()->randomInt(0, 600);
+		for (int n = 0; n < rr; n++) {
+			if (deterministicRandom()->random01() < (double)is.sumTo(is.end()) / rr * 2)
+				is.erase(is.lower_bound(deterministicRandom()->randomInt(0, 10000000)));
+			else
+				is.insert(deterministicRandom()->randomInt(0, 10000000), 3);
+		}
+
+		int b = deterministicRandom()->randomInt(0, 10000000);
+		// int e = b + deterministicRandom()->randomInt(0, 10);
+		int e = deterministicRandom()->randomInt(0, 10000000);
+		if (e < b) std::swap(b, e);
+		auto ib = is.lower_bound(b);
+		auto ie = is.lower_bound(e);
+
+		int original_count = is.sumTo(is.end()) / 3;
+		int original_incount = is.sumRange(ib, ie) / 3;
+
+		// printf("\n#%d Erasing %d of %d items\n", t, original_incount, original_count);
+
+		is.erase(ib, ie);
+		is.testonly_assertBalanced();
+
+		int count = 0, incount = 0;
+		for (auto i : is) {
+			++count;
+			if (i >= b && i < e) {
+				// printf("Remaining item: %d (%d - %d)\n", i, b, e);
+				incount++;
+			}
+		}
+
+		// printf("%d items remain, totalling %d\n", count, is.sumTo(is.end()));
+		// printf("%d items remain in erased range\n", incount);
+
+		ASSERT(incount == 0);
+		ASSERT(count == original_count - original_incount);
+		ASSERT(is.sumTo(is.end()) == count * 3);
+	}
+	return Void();
+}
+
+TEST_CASE("/flow/IndexedSet/strings") {
+	Map<std::string, int> myMap;
+	std::map<std::string, int> aMap;
+	myMap["Hello"] = 1;
+	myMap["Planet"] = 5;
+	for (auto i = myMap.begin(); i != myMap.end(); ++i) aMap[i->key] = i->value;
+
+	ASSERT(myMap.find(std::string("Hello"))->value == 1);
+	ASSERT(myMap.find(std::string("World")) == myMap.end());
+	ASSERT(myMap["Hello"] == 1);
+
+	auto a = myMap.upper_bound("A")->key;
+	auto x = myMap.lower_bound("M")->key;
+
+	ASSERT((a + x) == (std::string) "HelloPlanet");
+
+	return Void();
+}
+
+template <typename K>
+struct IndexedSetHarness {
+	using map = IndexedSet<K, int>;
+	using result = typename map::iterator;
+	using key_type = K;
+
+	map s;
+
+	void insert(K const& k) { s.insert(K(k), 1); }
+	result find(K const& k) const { return s.find(k); }
+	result not_found() const { return s.end(); }
+	result begin() const { return s.begin(); }
+	result end() const { return s.end(); }
+	result lower_bound(K const& k) const { return s.lower_bound(k); }
+	result upper_bound(K const& k) const { return s.upper_bound(k); }
+	void erase(K const& k) { s.erase(k); }
+};
+
+TEST_CASE("performance/map/StringRef/IndexedSet") {
+	Arena arena;
+
+	IndexedSetHarness<StringRef> is;
+	treeBenchmark(is, [&arena]() { return randomStr(arena); });
+
+	return Void();
+}
+
+TEST_CASE("performance/map/StringRef/StdMap") {
+	Arena arena;
+
+	MapHarness<StringRef> is;
+	treeBenchmark(is, [&arena]() { return randomStr(arena); });
+
+	return Void();
+}
+
+TEST_CASE("performance/map/int/IndexedSet") {
+	IndexedSetHarness<int> is;
+	treeBenchmark(is, &randomInt);
+
+	return Void();
+}
+
+TEST_CASE("performance/map/int/StdMap") {
+	MapHarness<int> is;
+	treeBenchmark(is, &randomInt);
+
+	return Void();
+}
+
+TEST_CASE("performance/flow/IndexedSet/integers") {
 	std::vector<int> x;
 	for (int i = 0; i<1000000; i++)
 		x.push_back(deterministicRandom()->randomInt(0, 10000000));
@@ -151,7 +265,6 @@ TEST_CASE("/flow/IndexedSet/erase 400k of 1M") {
 	double end = timer();
 	double kps = x.size() / 1000.0 / (end - start);
 	printf("%0.1f Kinsert/sec\n", kps);
-	ASSERT(kps >= 500);                                           //< Or something?
 
 	start = timer();
 	for (int i = 0; i<x.size(); i++)
@@ -159,7 +272,6 @@ TEST_CASE("/flow/IndexedSet/erase 400k of 1M") {
 	end = timer();
 	kps = x.size() / 1000.0 / (end - start);
 	printf("%0.1f Kfind/sec\n", kps);
-	ASSERT(kps >= 500);
 
 	{
 		//std::set<int> ss;
@@ -204,87 +316,41 @@ TEST_CASE("/flow/IndexedSet/erase 400k of 1M") {
 
 	printf("%0.1f Kerase/sec\n", x.size() / 1000.0 / (end - start));
 	is.testonly_assertBalanced();
-	for (int i = 0; i<x.size() / 2; i++)
+	for (int i = 0; i < x.size() / 2; i++) {
 		ASSERT(is.find(x[i]) == is.end());
-}*/
-
-TEST_CASE("/flow/IndexedSet/random ops") {
-	for (int t = 0; t<100; t++) {
-		IndexedSet<int, int> is;
-		int rr = deterministicRandom()->randomInt(0, 600) * deterministicRandom()->randomInt(0, 600);
-		for (int n = 0; n<rr; n++) {
-			if (deterministicRandom()->random01() < (double)is.sumTo(is.end()) / rr * 2)
-				is.erase(is.lower_bound(deterministicRandom()->randomInt(0, 10000000)));
-			else
-				is.insert(deterministicRandom()->randomInt(0, 10000000), 3);
-		}
-
-		int b = deterministicRandom()->randomInt(0, 10000000);
-		//int e = b + deterministicRandom()->randomInt(0, 10);
-		int e = deterministicRandom()->randomInt(0, 10000000);
-		if (e<b) std::swap(b, e);
-		auto ib = is.lower_bound(b);
-		auto ie = is.lower_bound(e);
-
-		int original_count = is.sumTo(is.end())/3;
-		int original_incount = is.sumRange(ib, ie)/3;
-
-		//printf("\n#%d Erasing %d of %d items\n", t, original_incount, original_count);
-
-		is.erase(ib, ie);
-		is.testonly_assertBalanced();
-
-		int count = 0, incount = 0;
-		for (auto i : is) {
-			++count;
-			if (i >= b && i < e) { 
-				//printf("Remaining item: %d (%d - %d)\n", i, b, e); 
-				incount++; 
-			}
-		}
-
-		//printf("%d items remain, totalling %d\n", count, is.sumTo(is.end()));
-		//printf("%d items remain in erased range\n", incount);
-
-		ASSERT(incount == 0);
-		ASSERT(count == original_count - original_incount);
-		ASSERT(is.sumTo(is.end()) == count*3);
 	}
+
 	return Void();
 }
 
-TEST_CASE("/flow/IndexedSet/strings") {
+TEST_CASE("performance/flow/IndexedSet/strings") {
+	constexpr size_t count = 1000000;
 	Map< std::string, int > myMap;
 	std::map< std::string, int > aMap;
-	myMap["Hello"] = 1;
+	double start, end;
-	myMap["Planet"] = 5;
+	int tt = 0;
-	for (auto i = myMap.begin(); i != myMap.end(); ++i)
-		aMap[i->key] = i->value;
 
-	ASSERT(myMap.find("Hello")->value == 1);
+	std::string const hello{ "Hello" };
-	ASSERT(myMap.find("World") == myMap.end());
+	myMap[hello] = 1;
-	ASSERT(myMap["Hello"] == 1);
+	aMap["Hello"] = 1;
 
-	auto a = myMap.upper_bound("A")->key;
+	start = timer();
-	auto x = myMap.lower_bound("M")->key;
 
-	ASSERT((a + x) == (std::string)"HelloPlanet");
+	for (size_t i = 0; i < count; i++) {
+		tt += myMap.find(hello)->value;
+	}
+	end = timer();
 
-	/* This was a performance test:
+	ASSERT(tt == count);
 
-		double start = timer();
+	printf("%0.1f Map.KfindStr/sec\n", count / 1000.0 / (end - start));
-		volatile int tt=0;
-		for(int i=0; i<1000000; i++)
-		tt += myMap.find( "Hello" )->value;
-		double end = timer();
-		printf("%0.1f Map.KfindStr/sec\n", 1000000/1000.0/(end-start));
 
-		start = timer();
+	start = timer();
-		for(int i=0; i<1000000; i++)
+	for (size_t i = 0; i < count; i++) {
-		aMap.find( "Hello" );
+		aMap.find(hello);
-		end = timer();
+	}
-		printf("%0.1f std::map.KfindStr/sec\n", 1000000/1000.0/(end-start));
+	end = timer();
-	*/
+	printf("%0.1f std::map.KfindStr/sec\n", count / 1000.0 / (end - start));
 
 	return Void();
 }

flow/TreeBenchmark.h

@@ -0,0 +1,123 @@
+/*
+ * IndexedSet.h
+ *
+ * This source file is part of the FoundationDB open source project
+ *
+ * Copyright 2020-2020 Apple Inc. and the FoundationDB project authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#ifndef FLOW_TREEBENCHMARK_H
+#define FLOW_TREEBENCHMARK_H
+#pragma once
+
+#include "flow/flow.h"
+
+struct opTimer {
+	double start = timer();
+	char const* name;
+	int opCount;
+
+	opTimer(char const* name, int opCount) : name(name), opCount(opCount) {}
+
+	~opTimer() { printf("%s: %0.1f Kop/s\n", name, (opCount / 1000.0) / (timer() - start)); }
+};
+
+template <typename F, typename T>
+void timedRun(char const* name, T& t, F f) {
+	opTimer timer(name, t.size());
+	for (auto& i : t) {
+		f(i);
+	}
+}
+
+template <typename K>
+struct MapHarness {
+	using map = std::map<K, int>;
+	using key_type = K;
+
+	struct result {
+		typename map::const_iterator it;
+
+		result(typename map::const_iterator it) : it(it) {}
+
+		result& operator++() {
+			it++;
+			return *this;
+		}
+
+		const K& operator*() const { return (*it).first; }
+
+		const K& operator->() const { return it->first; }
+
+		bool operator==(result const& k) const { return it == k.it; }
+	};
+
+	map s;
+
+	void insert(K const& k) { s.insert(std::pair<K, int>(k, 1)); }
+	result find(K const& k) const { return result(s.find(k)); }
+	result not_found() const { return result(s.end()); }
+	result begin() const { return result(s.begin()); }
+	result end() const { return result(s.end()); }
+	result lower_bound(K const& k) const { return result(s.lower_bound(k)); }
+	result upper_bound(K const& k) const { return result(s.upper_bound(k)); }
+	void erase(K const& k) { s.erase(k); }
+};
+
+template <typename T, typename F>
+void treeBenchmark(T& tree, F generateKey) {
+	using key = typename T::key_type;
+
+	int keyCount = 1000000;
+
+	std::vector<key> keys;
+	for (int i = 0; i < keyCount; i++) {
+		keys.push_back(generateKey());
+	}
+
+	timedRun("insert", keys, [&tree](key const& k) { tree.insert(k); });
+	timedRun("find", keys, [&tree](key const& k) { ASSERT(tree.find(k) != tree.not_found()); });
+	timedRun("lower_bound", keys, [&tree](key const & k) { ASSERT(tree.lower_bound(k) != tree.not_found()); });
+	timedRun("upper_bound", keys, [&tree](key const & k) { tree.upper_bound(k); });
+
+
+	std::sort(keys.begin(), keys.end());
+	keys.resize(std::unique(keys.begin(), keys.end()) - keys.begin());
+
+	auto iter = tree.lower_bound(*keys.begin());
+	timedRun("scan", keys, [&tree, &iter](key const& k) {
+		ASSERT(k == *iter);
+		++iter;
+	});
+	ASSERT(iter == tree.end());
+
+	timedRun("find (sorted)", keys, [&tree](key const& k) { ASSERT(tree.find(k) != tree.end()); });
+
+	std::random_shuffle(keys.begin(), keys.end());
+
+	timedRun("erase", keys, [&tree](key const& k) { tree.erase(k); });
+	ASSERT(tree.begin() == tree.end());
+}
+
+static inline StringRef randomStr(Arena& arena) {
+	size_t keySz = 100;
+	return StringRef(arena, deterministicRandom()->randomAlphaNumeric(keySz));
+}
+
+static inline int randomInt() {
+	return deterministicRandom()->randomInt(0, INT32_MAX);
+}
+
+#endif // FLOW_TREEBENCHMARK_H