Add a helper class to container StringSetGenerator and StringGenerator

This commit is contained in:
Jay Zhuang 2023-05-04 13:44:45 -07:00
parent fd680782b5
commit 561db510e0
4 changed files with 99 additions and 23 deletions

View File

@ -30,15 +30,6 @@ void printNextN(T generator, int count = 10) {
fmt::print("\n");
}
TEST_CASE("/randomKeyValueUtils/g1") {
// printNextN(RandomStringGenerator("3..5/^a..c"), 100);
// printNextN(RandomKeyTupleGenerator("100::"), 100);
// printNextN(RandomKeySetGenerator("5::5/a..c"), 100);
// printNextN(RandomKeyTupleSetGenerator("1000000::10::3..5/^a..d,100::5..10/^a..f,10000::20..30/^a..z"), 100);
printNextN(RandomValueGenerator("10..1000/a..z"), 20);
return Void();
}
TEST_CASE("/randomKeyValueUtils/generate") {
printNextN(RandomIntGenerator(3, 10, false), 5);

View File

@ -28,13 +28,27 @@
#include "flow/Error.h"
#include "flow/IRandom.h"
#include "fdbclient/FDBTypes.h"
template <typename T>
struct IGenerator {
virtual T next() = 0;
virtual T last() const = 0;
virtual std::string toString() const = 0;
virtual T next(int distance, bool wrap = false) {
throw unsupported_operation();
}
virtual ~IGenerator(){};
};
typedef IGenerator<Key> IKeyGenerator;
// Random unsigned int generator which generates integers between and including first and last
// Distribution can be uniform, skewed small, or skewed large
// String Definition Format: [^]first[..last]
// last is optional and defaults to first
// If ^ is present, the generated numbers skew toward first, otherwise are uniform random
// If either first or last begins with a letter character it will be interpreted as its ASCII byte value.
struct RandomIntGenerator {
struct RandomIntGenerator : IGenerator<unsigned int> {
enum Skew { LARGE, SMALL, NONE };
unsigned int min;
@ -55,7 +69,7 @@ struct RandomIntGenerator {
}
RandomIntGenerator(unsigned int only = 0) : min(only), max(only) {}
RandomIntGenerator(unsigned int first, unsigned int last, bool skewTowardFirst) : min(first), max(last) {
RandomIntGenerator(unsigned int first, unsigned int last, bool skewTowardFirst = false) : min(first), max(last) {
if (first != last && skewTowardFirst) {
skew = (first < last) ? SMALL : LARGE;
}
@ -126,7 +140,7 @@ struct RandomIntGenerator {
// String Definition Format: sizeRange[/byteRange]
// sizeRange and byteRange are RandomIntGenerators
// The default `byteRange` is 0:255
struct RandomStringGenerator {
struct RandomStringGenerator : IKeyGenerator {
RandomStringGenerator() {}
RandomStringGenerator(RandomIntGenerator size, RandomIntGenerator byteset) : size(size), bytes(byteset) {}
RandomStringGenerator(const char* cstr) : RandomStringGenerator(std::string(cstr)) {}
@ -153,7 +167,7 @@ struct RandomStringGenerator {
return val;
}
Standalone<StringRef> last() { return val; };
Standalone<StringRef> last() const { return val; };
std::string toString() const { return fmt::format("{}/{}", size.toString(), bytes.toString()); }
};
@ -187,7 +201,7 @@ struct RandomValueGenerator {
// Returns a random or nearby key at some distance from a vector of keys generated at init time.
// Requires a RandomIntGenerator as the index generator for selecting which random next key to return. The given index
// generator should have a min of 0 and if it doesn't its min will be updated to 0.
struct RandomStringSetGeneratorBase {
struct RandomStringSetGeneratorBase : IKeyGenerator {
Arena arena;
std::vector<KeyRef> keys;
RandomIntGenerator indexGenerator;
@ -198,6 +212,10 @@ struct RandomStringSetGeneratorBase {
void init(RandomIntGenerator originalIndexGenerator, KeyGen& keyGen) {
indexGenerator = originalIndexGenerator;
indexGenerator.min = 0;
if (indexGenerator.max == 0) {
fprintf(stdout, "JJJ1: error\n");
}
ASSERT(indexGenerator.max > 0);
std::set<Key> uniqueKeys;
int inserts = 0;
while (uniqueKeys.size() < indexGenerator.max) {
@ -318,6 +336,42 @@ struct RandomKeyTupleGenerator {
typedef RandomStringSetGenerator<RandomKeyTupleGenerator> RandomKeyTupleSetGenerator;
// RandomKeyGenerator is a helper function to contain multiple KeyGenerators
struct RandomKeyGenerator : IKeyGenerator {
std::vector<std::unique_ptr<IKeyGenerator>> keyGenerators;
Key val;
void addKeyGenerator(std::unique_ptr<IKeyGenerator> gen) {
keyGenerators.emplace_back(std::move(gen));
}
Key next() override {
int totalBytes = 0;
for (auto& g : keyGenerators) {
totalBytes += g->next().size();
}
val = makeString(totalBytes);
totalBytes = 0;
for (const auto& g : keyGenerators) {
memcpy(mutateString(val) + totalBytes, g->last().begin(), g->last().size());
totalBytes += g->last().size();
}
return val;
}
Key last() const override { return val; };
// TODO: the string is only for output, RandomKeyGenerator currently doesn't support constructing from a string.
std::string toString() const override {
std::string s;
for (auto const& g : keyGenerators) {
s += "[" + g->toString() + "]";
}
return s;
}
};
struct RandomMutationGenerator {
RandomKeyTupleSetGenerator keys;
RandomValueGenerator valueGen;

View File

@ -10067,6 +10067,28 @@ TEST_CASE(":/redwood/pager/ArenaPage") {
return Void();
}
namespace {
RandomKeyGenerator getDefaultKeyGenerator(int maxKeySize) {
RandomKeyGenerator keyGen;
int tupleSetNum = deterministicRandom()->randomInt(0, 10);
for (int i = 0; i < tupleSetNum && maxKeySize > 0; i++) {
int mKeySize = deterministicRandom()->randomInt(1, std::min(maxKeySize, 100));
maxKeySize -= mKeySize;
keyGen.addKeyGenerator(
std::make_unique<RandomKeySetGenerator>(RandomIntGenerator(deterministicRandom()->randomInt(1, 5) * (i + 1)),
RandomStringGenerator(RandomIntGenerator(1, mKeySize), RandomIntGenerator(1, 254))));
}
if (deterministicRandom()->coinflip() && maxKeySize > 0) {
keyGen.addKeyGenerator(std::make_unique<RandomStringGenerator>(RandomIntGenerator(1, maxKeySize), RandomIntGenerator(1, 254)));
}
return keyGen;
}
} // namespace
TEST_CASE("Lredwood/correctness/btree") {
g_redwoodMetricsActor = Void(); // Prevent trace event metrics from starting
g_redwoodMetrics.clear();
@ -10131,13 +10153,19 @@ TEST_CASE("Lredwood/correctness/btree") {
// Max number of records in the BTree or the versioned written map to visit
state int64_t maxRecordsRead = params.getInt("maxRecordsRead").orDefault(300e6);
state std::string keyGenerator = params.get("keyGenerator").orDefault("1000000::10::3..5/^a..d,100::5..10/^a..f,10000::20..30/^a..z");
state Optional<std::string> keyGenerator = params.get("keyGenerator");
state std::string valueGenerator = params.get("valueGenerator").orDefault("10..1000/a..z");
state std::unique_ptr<RandomKeyTupleSetGenerator> keyGen = std::make_unique<RandomKeyTupleSetGenerator>(keyGenerator);
state RandomKeyGenerator keyGen;
state std::unique_ptr<RandomValueGenerator> valGen = std::make_unique<RandomValueGenerator>(valueGenerator);
if (keyGenerator.present()) {
keyGen.addKeyGenerator(std::make_unique<RandomKeyTupleSetGenerator>(keyGenerator.get()));
} else {
keyGen = getDefaultKeyGenerator(2 * pageSize);
};
state RandomValueGenerator valGen = RandomValueGenerator(valueGenerator);
state EncodingType encodingType = static_cast<EncodingType>(encoding);
state EncryptionAtRestMode encryptionMode =
@ -10173,7 +10201,7 @@ TEST_CASE("Lredwood/correctness/btree") {
printf("domainMode: %d\n", encryptionDomainMode);
printf("pageSize: %d\n", pageSize);
printf("extentSize: %d\n", extentSize);
printf("keyGenerator: %s\n", keyGenerator.c_str());
printf("keyGenerator: %s\n", keyGenerator.orDefault("default").c_str());
printf("valueGenerator: %s\n", valueGenerator.c_str());
printf("maxCommitSize: %d\n", maxCommitSize);
printf("setExistingKeyProbability: %f\n", setExistingKeyProbability);
@ -10247,8 +10275,8 @@ TEST_CASE("Lredwood/correctness/btree") {
// Sometimes do a clear range
if (deterministicRandom()->random01() < clearProbability) {
Key start = keyGen->next();
Key end = (deterministicRandom()->random01() < .01) ? keyAfter(start) : keyGen->next();
Key start = keyGen.next();
Key end = (deterministicRandom()->random01() < .01) ? keyAfter(start) : keyGen.next();
// Sometimes replace start and/or end with a close actual (previously used) value
if (deterministicRandom()->random01() < clearExistingBoundaryProbability) {
@ -10327,15 +10355,15 @@ TEST_CASE("Lredwood/correctness/btree") {
if (deterministicRandom()->random01() < clearPostSetProbability) {
KeyValue kv;
kv.key = range.begin;
kv.value = valGen->next();
kv.value = valGen.next();
btree->set(kv);
written[std::make_pair(kv.key.toString(), version)] = kv.value.toString();
}
} else {
// Set a key
KeyValue kv;
kv.key = keyGen->next();
kv.value = valGen->next();
kv.key = keyGen.next();
kv.value = valGen.next();
// Sometimes change key to a close previously used key
if (deterministicRandom()->random01() < setExistingKeyProbability) {
auto i = keys.upper_bound(kv.key);

View File

@ -43,6 +43,9 @@ double DeterministicRandom::random01() {
}
int DeterministicRandom::randomInt(int min, int maxPlusOne) {
if (min >= maxPlusOne) {
fprintf(stdout, "JJJ1:error\n");
}
ASSERT_LT(min, maxPlusOne);
unsigned int range;
if (maxPlusOne < 0)