Add a helper class to container StringSetGenerator and StringGenerator

fdbclient/RandomKeyValueUtils.cpp

@@ -30,15 +30,6 @@ void printNextN(T generator, int count = 10) {
 	fmt::print("\n");
 }
 
-TEST_CASE("/randomKeyValueUtils/g1") {
-//	printNextN(RandomStringGenerator("3..5/^a..c"), 100);
-//	printNextN(RandomKeyTupleGenerator("100::"), 100);
-//	printNextN(RandomKeySetGenerator("5::5/a..c"), 100);
-//	printNextN(RandomKeyTupleSetGenerator("1000000::10::3..5/^a..d,100::5..10/^a..f,10000::20..30/^a..z"), 100);
-	printNextN(RandomValueGenerator("10..1000/a..z"), 20);
-	return Void();
-}
-
 TEST_CASE("/randomKeyValueUtils/generate") {
 
 	printNextN(RandomIntGenerator(3, 10, false), 5);

fdbclient/include/fdbclient/RandomKeyValueUtils.h

@@ -28,13 +28,27 @@
 #include "flow/Error.h"
 #include "flow/IRandom.h"
 #include "fdbclient/FDBTypes.h"
+
+template <typename T>
+struct IGenerator {
+	virtual T next() = 0;
+	virtual T last() const = 0;
+	virtual std::string toString() const = 0;
+	virtual T next(int distance, bool wrap = false) {
+		throw unsupported_operation();
+	}
+	virtual ~IGenerator(){};
+};
+
+typedef IGenerator<Key> IKeyGenerator;
+
 // Random unsigned int generator which generates integers between and including first and last
 // Distribution can be uniform, skewed small, or skewed large
 // String Definition Format:  [^]first[..last]
 //   last is optional and defaults to first
 //   If ^ is present, the generated numbers skew toward first, otherwise are uniform random
 //   If either first or last begins with a letter character it will be interpreted as its ASCII byte value.
-struct RandomIntGenerator {
+struct RandomIntGenerator : IGenerator<unsigned int> {
 	enum Skew { LARGE, SMALL, NONE };
 
 	unsigned int min;
@@ -55,7 +69,7 @@ struct RandomIntGenerator {
 	}
 
 	RandomIntGenerator(unsigned int only = 0) : min(only), max(only) {}
-	RandomIntGenerator(unsigned int first, unsigned int last, bool skewTowardFirst) : min(first), max(last) {
+	RandomIntGenerator(unsigned int first, unsigned int last, bool skewTowardFirst = false) : min(first), max(last) {
 		if (first != last && skewTowardFirst) {
 			skew = (first < last) ? SMALL : LARGE;
 		}
@@ -126,7 +140,7 @@ struct RandomIntGenerator {
 // String Definition Format:  sizeRange[/byteRange]
 // sizeRange and byteRange are RandomIntGenerators
 // The default `byteRange` is 0:255
-struct RandomStringGenerator {
+struct RandomStringGenerator : IKeyGenerator {
 	RandomStringGenerator() {}
 	RandomStringGenerator(RandomIntGenerator size, RandomIntGenerator byteset) : size(size), bytes(byteset) {}
 	RandomStringGenerator(const char* cstr) : RandomStringGenerator(std::string(cstr)) {}
@@ -153,7 +167,7 @@ struct RandomStringGenerator {
 		return val;
 	}
 
-	Standalone<StringRef> last() { return val; };
+	Standalone<StringRef> last() const { return val; };
 
 	std::string toString() const { return fmt::format("{}/{}", size.toString(), bytes.toString()); }
 };
@@ -187,7 +201,7 @@ struct RandomValueGenerator {
 // Returns a random or nearby key at some distance from a vector of keys generated at init time.
 // Requires a RandomIntGenerator as the index generator for selecting which random next key to return.  The given index
 // generator should have a min of 0 and if it doesn't its min will be updated to 0.
-struct RandomStringSetGeneratorBase {
+struct RandomStringSetGeneratorBase : IKeyGenerator {
 	Arena arena;
 	std::vector<KeyRef> keys;
 	RandomIntGenerator indexGenerator;
@@ -198,6 +212,10 @@ struct RandomStringSetGeneratorBase {
 	void init(RandomIntGenerator originalIndexGenerator, KeyGen& keyGen) {
 		indexGenerator = originalIndexGenerator;
 		indexGenerator.min = 0;
+		if (indexGenerator.max == 0) {
+			fprintf(stdout, "JJJ1: error\n");
+		}
+		ASSERT(indexGenerator.max > 0);
 		std::set<Key> uniqueKeys;
 		int inserts = 0;
 		while (uniqueKeys.size() < indexGenerator.max) {
@@ -318,6 +336,42 @@ struct RandomKeyTupleGenerator {
 
 typedef RandomStringSetGenerator<RandomKeyTupleGenerator> RandomKeyTupleSetGenerator;
 
+// RandomKeyGenerator is a helper function to contain multiple KeyGenerators
+struct RandomKeyGenerator : IKeyGenerator {
+	std::vector<std::unique_ptr<IKeyGenerator>> keyGenerators;
+	Key val;
+
+
+	void addKeyGenerator(std::unique_ptr<IKeyGenerator> gen) {
+		keyGenerators.emplace_back(std::move(gen));
+	}
+
+	Key next() override {
+		int totalBytes = 0;
+		for (auto& g : keyGenerators) {
+			totalBytes += g->next().size();
+		}
+		val = makeString(totalBytes);
+		totalBytes = 0;
+		for (const auto& g : keyGenerators) {
+			memcpy(mutateString(val) + totalBytes, g->last().begin(), g->last().size());
+			totalBytes += g->last().size();
+		}
+		return val;
+	}
+
+	Key last() const override { return val; };
+
+	// TODO: the string is only for output, RandomKeyGenerator currently doesn't support constructing from a string.
+	std::string toString() const override {
+		std::string s;
+		for (auto const& g : keyGenerators) {
+			s += "[" + g->toString() + "]";
+		}
+		return s;
+	}
+};
+
 struct RandomMutationGenerator {
 	RandomKeyTupleSetGenerator keys;
 	RandomValueGenerator valueGen;

fdbserver/VersionedBTree.actor.cpp

@@ -10067,6 +10067,28 @@ TEST_CASE(":/redwood/pager/ArenaPage") {
 	return Void();
 }
 
+namespace {
+
+RandomKeyGenerator getDefaultKeyGenerator(int maxKeySize) {
+	RandomKeyGenerator keyGen;
+
+	int tupleSetNum = deterministicRandom()->randomInt(0, 10);
+	for (int i = 0; i < tupleSetNum && maxKeySize > 0; i++) {
+		int mKeySize = deterministicRandom()->randomInt(1, std::min(maxKeySize, 100));
+		maxKeySize -= mKeySize;
+		keyGen.addKeyGenerator(
+		    std::make_unique<RandomKeySetGenerator>(RandomIntGenerator(deterministicRandom()->randomInt(1, 5) * (i + 1)),
+		                                            RandomStringGenerator(RandomIntGenerator(1, mKeySize), RandomIntGenerator(1, 254))));
+	}
+	if (deterministicRandom()->coinflip() && maxKeySize > 0) {
+		keyGen.addKeyGenerator(std::make_unique<RandomStringGenerator>(RandomIntGenerator(1, maxKeySize), RandomIntGenerator(1, 254)));
+	}
+
+	return keyGen;
+}
+
+} // namespace
+
 TEST_CASE("Lredwood/correctness/btree") {
 	g_redwoodMetricsActor = Void(); // Prevent trace event metrics from starting
 	g_redwoodMetrics.clear();
@@ -10131,13 +10153,19 @@ TEST_CASE("Lredwood/correctness/btree") {
 	// Max number of records in the BTree or the versioned written map to visit
 	state int64_t maxRecordsRead = params.getInt("maxRecordsRead").orDefault(300e6);
 
-	state std::string keyGenerator = params.get("keyGenerator").orDefault("1000000::10::3..5/^a..d,100::5..10/^a..f,10000::20..30/^a..z");
+	state Optional<std::string> keyGenerator = params.get("keyGenerator");
 
 	state std::string valueGenerator = params.get("valueGenerator").orDefault("10..1000/a..z");
 
-	state std::unique_ptr<RandomKeyTupleSetGenerator> keyGen = std::make_unique<RandomKeyTupleSetGenerator>(keyGenerator);
+	state RandomKeyGenerator keyGen;
 
-	state std::unique_ptr<RandomValueGenerator> valGen = std::make_unique<RandomValueGenerator>(valueGenerator);
+	if (keyGenerator.present()) {
+		keyGen.addKeyGenerator(std::make_unique<RandomKeyTupleSetGenerator>(keyGenerator.get()));
+	} else {
+		keyGen = getDefaultKeyGenerator(2 * pageSize);
+	};
+
+	state RandomValueGenerator valGen = RandomValueGenerator(valueGenerator);
 
 	state EncodingType encodingType = static_cast<EncodingType>(encoding);
 	state EncryptionAtRestMode encryptionMode =
@@ -10173,7 +10201,7 @@ TEST_CASE("Lredwood/correctness/btree") {
 	printf("domainMode: %d\n", encryptionDomainMode);
 	printf("pageSize: %d\n", pageSize);
 	printf("extentSize: %d\n", extentSize);
-	printf("keyGenerator: %s\n", keyGenerator.c_str());
+	printf("keyGenerator: %s\n", keyGenerator.orDefault("default").c_str());
 	printf("valueGenerator: %s\n", valueGenerator.c_str());
 	printf("maxCommitSize: %d\n", maxCommitSize);
 	printf("setExistingKeyProbability: %f\n", setExistingKeyProbability);
@@ -10247,8 +10275,8 @@ TEST_CASE("Lredwood/correctness/btree") {
 
 		// Sometimes do a clear range
 		if (deterministicRandom()->random01() < clearProbability) {
-			Key start = keyGen->next();
-			Key end = (deterministicRandom()->random01() < .01) ? keyAfter(start) : keyGen->next();
+			Key start = keyGen.next();
+			Key end = (deterministicRandom()->random01() < .01) ? keyAfter(start) : keyGen.next();
 
 			// Sometimes replace start and/or end with a close actual (previously used) value
 			if (deterministicRandom()->random01() < clearExistingBoundaryProbability) {
@@ -10327,15 +10355,15 @@ TEST_CASE("Lredwood/correctness/btree") {
 			if (deterministicRandom()->random01() < clearPostSetProbability) {
 				KeyValue kv;
 				kv.key = range.begin;
-				kv.value = valGen->next();
+				kv.value = valGen.next();
 				btree->set(kv);
 				written[std::make_pair(kv.key.toString(), version)] = kv.value.toString();
 			}
 		} else {
 			// Set a key
 			KeyValue kv;
-			kv.key = keyGen->next();
-			kv.value = valGen->next();
+			kv.key = keyGen.next();
+			kv.value = valGen.next();
 			// Sometimes change key to a close previously used key
 			if (deterministicRandom()->random01() < setExistingKeyProbability) {
 				auto i = keys.upper_bound(kv.key);

flow/DeterministicRandom.cpp

@@ -43,6 +43,9 @@ double DeterministicRandom::random01() {
 }
 
 int DeterministicRandom::randomInt(int min, int maxPlusOne) {
+	if (min >= maxPlusOne) {
+		fprintf(stdout, "JJJ1:error\n");
+	}
 	ASSERT_LT(min, maxPlusOne);
 	unsigned int range;
 	if (maxPlusOne < 0)