mirror of
https://github.com/apple/foundationdb.git
synced 2025-06-02 11:15:50 +08:00
Merge remote-tracking branch 'apple/main' into vgasiunas-fix-version-check-for-downgrades
This commit is contained in:
Vaidas Gasiunas
commit
6c94802f14
@ -2362,6 +2362,7 @@ ACTOR void setupAndRun(std::string dataFolder,
|
||||
allowList.addTrustedSubnet("abcd::/16"sv);
|
||||
state bool allowDefaultTenant = testConfig.allowDefaultTenant;
|
||||
state bool allowDisablingTenants = testConfig.allowDisablingTenants;
|
||||
state bool allowCreatingTenants = true;
|
||||
|
||||
// The RocksDB storage engine does not support the restarting tests because you cannot consistently get a clean
|
||||
// snapshot of the storage engine without a snapshotting file system.
|
||||
@ -2372,6 +2373,7 @@ ACTOR void setupAndRun(std::string dataFolder,
|
||||
// Disable the default tenant in restarting tests for now
|
||||
// TODO: persist the chosen default tenant in the restartInfo.ini file for the second test
|
||||
allowDefaultTenant = false;
|
||||
allowCreatingTenants = false;
|
||||
}
|
||||
|
||||
// TODO: Currently backup and restore related simulation tests are failing when run with rocksDB storage engine
|
||||
@ -2425,9 +2427,11 @@ ACTOR void setupAndRun(std::string dataFolder,
|
||||
TEST(true); // Simulation start
|
||||
|
||||
state Optional<TenantName> defaultTenant;
|
||||
state Standalone<VectorRef<TenantNameRef>> tenantsToCreate;
|
||||
state TenantMode tenantMode = TenantMode::DISABLED;
|
||||
if (allowDefaultTenant && deterministicRandom()->random01() < 0.5) {
|
||||
defaultTenant = "SimulatedDefaultTenant"_sr;
|
||||
tenantsToCreate.push_back_deep(tenantsToCreate.arena(), defaultTenant.get());
|
||||
if (deterministicRandom()->random01() < 0.9) {
|
||||
tenantMode = TenantMode::REQUIRED;
|
||||
} else {
|
||||
@ -2437,9 +2441,18 @@ ACTOR void setupAndRun(std::string dataFolder,
|
||||
tenantMode = TenantMode::OPTIONAL_TENANT;
|
||||
}
|
||||
|
||||
if (allowCreatingTenants && tenantMode != TenantMode::DISABLED && deterministicRandom()->random01() < 0.5) {
|
||||
int numTenants = deterministicRandom()->randomInt(1, 6);
|
||||
for (int i = 0; i < numTenants; ++i) {
|
||||
tenantsToCreate.push_back_deep(tenantsToCreate.arena(),
|
||||
TenantNameRef(format("SimulatedExtraTenant%04d", i)));
|
||||
}
|
||||
}
|
||||
|
||||
TraceEvent("SimulatedClusterTenantMode")
|
||||
.detail("UsingTenant", defaultTenant)
|
||||
.detail("TenantRequired", tenantMode.toString());
|
||||
.detail("TenantRequired", tenantMode.toString())
|
||||
.detail("TotalTenants", tenantsToCreate.size());
|
||||
|
||||
try {
|
||||
// systemActors.push_back( startSystemMonitor(dataFolder) );
|
||||
@ -2481,7 +2494,8 @@ ACTOR void setupAndRun(std::string dataFolder,
|
||||
startingConfiguration,
|
||||
LocalityData(),
|
||||
UnitTestParameters(),
|
||||
defaultTenant),
|
||||
defaultTenant,
|
||||
tenantsToCreate),
|
||||
isBuggifyEnabled(BuggifyType::General) ? 36000.0 : 5400.0));
|
||||
} catch (Error& e) {
|
||||
TraceEvent(SevError, "SetupAndRunError").error(e);
|
||||
|
||||
@ -122,15 +122,17 @@ ACTOR Future<Void> testerServerCore(TesterInterface interf,
|
||||
enum test_location_t { TEST_HERE, TEST_ON_SERVERS, TEST_ON_TESTERS };
|
||||
enum test_type_t { TEST_TYPE_FROM_FILE, TEST_TYPE_CONSISTENCY_CHECK, TEST_TYPE_UNIT_TESTS };
|
||||
|
||||
ACTOR Future<Void> runTests(Reference<IClusterConnectionRecord> connRecord,
|
||||
test_type_t whatToRun,
|
||||
test_location_t whereToRun,
|
||||
int minTestersExpected,
|
||||
std::string fileName = std::string(),
|
||||
StringRef startingConfiguration = StringRef(),
|
||||
LocalityData locality = LocalityData(),
|
||||
UnitTestParameters testOptions = UnitTestParameters(),
|
||||
Optional<TenantName> defaultTenant = Optional<TenantName>());
|
||||
ACTOR Future<Void> runTests(
|
||||
Reference<IClusterConnectionRecord> connRecord,
|
||||
test_type_t whatToRun,
|
||||
test_location_t whereToRun,
|
||||
int minTestersExpected,
|
||||
std::string fileName = std::string(),
|
||||
StringRef startingConfiguration = StringRef(),
|
||||
LocalityData locality = LocalityData(),
|
||||
UnitTestParameters testOptions = UnitTestParameters(),
|
||||
Optional<TenantName> defaultTenant = Optional<TenantName>(),
|
||||
Standalone<VectorRef<TenantNameRef>> tenantsToCreate = Standalone<VectorRef<TenantNameRef>>());
|
||||
|
||||
#include "flow/unactorcompiler.h"
|
||||
#endif
|
||||
|
||||
@ -3773,6 +3773,118 @@ ACTOR Future<GetMappedKeyValuesReply> mapKeyValues(StorageServer* data,
|
||||
return result;
|
||||
}
|
||||
|
||||
bool rangeIntersectsAnyTenant(TenantPrefixIndex& prefixIndex, KeyRangeRef range, Version ver) {
|
||||
auto view = prefixIndex.at(ver);
|
||||
auto beginItr = view.lastLessOrEqual(range.begin);
|
||||
auto endItr = view.lastLess(range.end);
|
||||
|
||||
// If the begin and end reference different spots in the tenant index, then the tenant pointed to
|
||||
// by endItr intersects the range
|
||||
if (beginItr != endItr) {
|
||||
return true;
|
||||
}
|
||||
|
||||
// If the iterators point to the same entry and that entry contains begin, then we are wholly in
|
||||
// one tenant
|
||||
if (beginItr != view.end() && range.begin.startsWith(beginItr.key())) {
|
||||
return true;
|
||||
}
|
||||
|
||||
return false;
|
||||
}
|
||||
|
||||
TEST_CASE("/fdbserver/storageserver/rangeIntersectsAnyTenant") {
|
||||
std::map<TenantName, TenantMapEntry> entries = { std::make_pair("tenant0"_sr, TenantMapEntry(0, ""_sr)),
|
||||
std::make_pair("tenant2"_sr, TenantMapEntry(2, ""_sr)),
|
||||
std::make_pair("tenant3"_sr, TenantMapEntry(3, ""_sr)),
|
||||
std::make_pair("tenant4"_sr, TenantMapEntry(4, ""_sr)),
|
||||
std::make_pair("tenant6"_sr, TenantMapEntry(6, ""_sr)) };
|
||||
TenantPrefixIndex index;
|
||||
index.createNewVersion(1);
|
||||
for (auto entry : entries) {
|
||||
index.insert(entry.second.prefix, entry.first);
|
||||
}
|
||||
|
||||
// Before all tenants
|
||||
ASSERT(!rangeIntersectsAnyTenant(index, KeyRangeRef(""_sr, "\x00"_sr), index.getLatestVersion()));
|
||||
|
||||
// After all tenants
|
||||
ASSERT(!rangeIntersectsAnyTenant(index, KeyRangeRef("\xfe"_sr, "\xff"_sr), index.getLatestVersion()));
|
||||
|
||||
// In between tenants
|
||||
ASSERT(!rangeIntersectsAnyTenant(
|
||||
index,
|
||||
KeyRangeRef(TenantMapEntry::idToPrefix(1), TenantMapEntry::idToPrefix(1).withSuffix("\xff"_sr)),
|
||||
index.getLatestVersion()));
|
||||
|
||||
// In between tenants with end intersecting tenant start
|
||||
ASSERT(!rangeIntersectsAnyTenant(
|
||||
index, KeyRangeRef(TenantMapEntry::idToPrefix(5), entries["tenant6"_sr].prefix), index.getLatestVersion()));
|
||||
|
||||
// Entire tenants
|
||||
ASSERT(rangeIntersectsAnyTenant(
|
||||
index, KeyRangeRef(entries["tenant0"_sr].prefix, TenantMapEntry::idToPrefix(1)), index.getLatestVersion()));
|
||||
ASSERT(rangeIntersectsAnyTenant(
|
||||
index, KeyRangeRef(entries["tenant2"_sr].prefix, entries["tenant3"_sr].prefix), index.getLatestVersion()));
|
||||
|
||||
// Partial tenants
|
||||
ASSERT(rangeIntersectsAnyTenant(
|
||||
index,
|
||||
KeyRangeRef(entries["tenant0"_sr].prefix, entries["tenant0"_sr].prefix.withSuffix("foo"_sr)),
|
||||
index.getLatestVersion()));
|
||||
ASSERT(rangeIntersectsAnyTenant(
|
||||
index,
|
||||
KeyRangeRef(entries["tenant3"_sr].prefix.withSuffix("foo"_sr), entries["tenant4"_sr].prefix),
|
||||
index.getLatestVersion()));
|
||||
ASSERT(rangeIntersectsAnyTenant(index,
|
||||
KeyRangeRef(entries["tenant4"_sr].prefix.withSuffix("bar"_sr),
|
||||
entries["tenant4"_sr].prefix.withSuffix("foo"_sr)),
|
||||
index.getLatestVersion()));
|
||||
|
||||
// Begin outside, end inside tenant
|
||||
ASSERT(rangeIntersectsAnyTenant(
|
||||
index,
|
||||
KeyRangeRef(TenantMapEntry::idToPrefix(1), entries["tenant2"_sr].prefix.withSuffix("foo"_sr)),
|
||||
index.getLatestVersion()));
|
||||
ASSERT(rangeIntersectsAnyTenant(
|
||||
index,
|
||||
KeyRangeRef(TenantMapEntry::idToPrefix(1), entries["tenant3"_sr].prefix.withSuffix("foo"_sr)),
|
||||
index.getLatestVersion()));
|
||||
|
||||
// Begin inside, end outside tenant
|
||||
ASSERT(rangeIntersectsAnyTenant(
|
||||
index,
|
||||
KeyRangeRef(entries["tenant3"_sr].prefix.withSuffix("foo"_sr), TenantMapEntry::idToPrefix(5)),
|
||||
index.getLatestVersion()));
|
||||
ASSERT(rangeIntersectsAnyTenant(
|
||||
index,
|
||||
KeyRangeRef(entries["tenant4"_sr].prefix.withSuffix("foo"_sr), TenantMapEntry::idToPrefix(5)),
|
||||
index.getLatestVersion()));
|
||||
|
||||
// Both inside different tenants
|
||||
ASSERT(rangeIntersectsAnyTenant(index,
|
||||
KeyRangeRef(entries["tenant0"_sr].prefix.withSuffix("foo"_sr),
|
||||
entries["tenant2"_sr].prefix.withSuffix("foo"_sr)),
|
||||
index.getLatestVersion()));
|
||||
ASSERT(rangeIntersectsAnyTenant(index,
|
||||
KeyRangeRef(entries["tenant0"_sr].prefix.withSuffix("foo"_sr),
|
||||
entries["tenant3"_sr].prefix.withSuffix("foo"_sr)),
|
||||
index.getLatestVersion()));
|
||||
ASSERT(rangeIntersectsAnyTenant(index,
|
||||
KeyRangeRef(entries["tenant2"_sr].prefix.withSuffix("foo"_sr),
|
||||
entries["tenant6"_sr].prefix.withSuffix("foo"_sr)),
|
||||
index.getLatestVersion()));
|
||||
|
||||
// Both outside tenants with tenant in the middle
|
||||
ASSERT(rangeIntersectsAnyTenant(
|
||||
index, KeyRangeRef(""_sr, TenantMapEntry::idToPrefix(1).withSuffix("foo"_sr)), index.getLatestVersion()));
|
||||
ASSERT(rangeIntersectsAnyTenant(index, KeyRangeRef(""_sr, "\xff"_sr), index.getLatestVersion()));
|
||||
ASSERT(rangeIntersectsAnyTenant(
|
||||
index, KeyRangeRef(TenantMapEntry::idToPrefix(5).withSuffix("foo"_sr), "\xff"_sr), index.getLatestVersion()));
|
||||
|
||||
return Void();
|
||||
}
|
||||
|
||||
// Most of the actor is copied from getKeyValuesQ. I tried to use templates but things become nearly impossible after
|
||||
// combining actor shenanigans with template shenanigans.
|
||||
ACTOR Future<Void> getMappedKeyValuesQ(StorageServer* data, GetMappedKeyValuesRequest req)
|
||||
@ -3859,13 +3971,7 @@ ACTOR Future<Void> getMappedKeyValuesQ(StorageServer* data, GetMappedKeyValuesRe
|
||||
throw tenant_name_required();
|
||||
}
|
||||
|
||||
auto view = data->tenantPrefixIndex.at(req.version);
|
||||
auto beginItr = view.lastLessOrEqual(begin);
|
||||
if (beginItr != view.end() && !begin.startsWith(beginItr.key())) {
|
||||
++beginItr;
|
||||
}
|
||||
auto endItr = view.lastLessOrEqual(end);
|
||||
if (beginItr != endItr) {
|
||||
if (rangeIntersectsAnyTenant(data->tenantPrefixIndex, KeyRangeRef(begin, end), req.version)) {
|
||||
throw tenant_name_required();
|
||||
}
|
||||
}
|
||||
|
||||
@ -1534,7 +1534,8 @@ ACTOR Future<Void> runTests(Reference<AsyncVar<Optional<struct ClusterController
|
||||
std::vector<TestSpec> tests,
|
||||
StringRef startingConfiguration,
|
||||
LocalityData locality,
|
||||
Optional<TenantName> defaultTenant) {
|
||||
Optional<TenantName> defaultTenant,
|
||||
Standalone<VectorRef<TenantNameRef>> tenantsToCreate) {
|
||||
state Database cx;
|
||||
state Reference<AsyncVar<ServerDBInfo>> dbInfo(new AsyncVar<ServerDBInfo>);
|
||||
state Future<Void> ccMonitor = monitorServerDBInfo(cc, LocalityData(), dbInfo); // FIXME: locality
|
||||
@ -1610,9 +1611,14 @@ ACTOR Future<Void> runTests(Reference<AsyncVar<Optional<struct ClusterController
|
||||
}
|
||||
}
|
||||
|
||||
if (useDB && defaultTenant.present()) {
|
||||
TraceEvent("CreatingDefaultTenant").detail("Tenant", defaultTenant.get());
|
||||
wait(ManagementAPI::createTenant(cx.getReference(), defaultTenant.get()));
|
||||
if (useDB) {
|
||||
std::vector<Future<Void>> tenantFutures;
|
||||
for (auto tenant : tenantsToCreate) {
|
||||
TraceEvent("CreatingTenant").detail("Tenant", tenant);
|
||||
tenantFutures.push_back(ManagementAPI::createTenant(cx.getReference(), tenant));
|
||||
}
|
||||
|
||||
wait(waitForAll(tenantFutures));
|
||||
}
|
||||
|
||||
if (useDB && waitForQuiescenceBegin) {
|
||||
@ -1694,7 +1700,8 @@ ACTOR Future<Void> runTests(Reference<AsyncVar<Optional<struct ClusterController
|
||||
int minTestersExpected,
|
||||
StringRef startingConfiguration,
|
||||
LocalityData locality,
|
||||
Optional<TenantName> defaultTenant) {
|
||||
Optional<TenantName> defaultTenant,
|
||||
Standalone<VectorRef<TenantNameRef>> tenantsToCreate) {
|
||||
state int flags = (at == TEST_ON_SERVERS ? 0 : GetWorkersRequest::TESTER_CLASS_ONLY) |
|
||||
GetWorkersRequest::NON_EXCLUDED_PROCESSES_ONLY;
|
||||
state Future<Void> testerTimeout = delay(600.0); // wait 600 sec for testers to show up
|
||||
@ -1725,7 +1732,7 @@ ACTOR Future<Void> runTests(Reference<AsyncVar<Optional<struct ClusterController
|
||||
for (int i = 0; i < workers.size(); i++)
|
||||
ts.push_back(workers[i].interf.testerInterface);
|
||||
|
||||
wait(runTests(cc, ci, ts, tests, startingConfiguration, locality, defaultTenant));
|
||||
wait(runTests(cc, ci, ts, tests, startingConfiguration, locality, defaultTenant, tenantsToCreate));
|
||||
return Void();
|
||||
}
|
||||
|
||||
@ -1763,7 +1770,8 @@ ACTOR Future<Void> runTests(Reference<IClusterConnectionRecord> connRecord,
|
||||
StringRef startingConfiguration,
|
||||
LocalityData locality,
|
||||
UnitTestParameters testOptions,
|
||||
Optional<TenantName> defaultTenant) {
|
||||
Optional<TenantName> defaultTenant,
|
||||
Standalone<VectorRef<TenantNameRef>> tenantsToCreate) {
|
||||
state TestSet testSet;
|
||||
state std::unique_ptr<KnobProtectiveGroup> knobProtectiveGroup(nullptr);
|
||||
auto cc = makeReference<AsyncVar<Optional<ClusterControllerFullInterface>>>();
|
||||
@ -1847,11 +1855,19 @@ ACTOR Future<Void> runTests(Reference<IClusterConnectionRecord> connRecord,
|
||||
actors.push_back(
|
||||
reportErrors(monitorServerDBInfo(cc, LocalityData(), db), "MonitorServerDBInfo")); // FIXME: Locality
|
||||
actors.push_back(reportErrors(testerServerCore(iTesters[0], connRecord, db, locality), "TesterServerCore"));
|
||||
tests = runTests(cc, ci, iTesters, testSet.testSpecs, startingConfiguration, locality, defaultTenant);
|
||||
tests = runTests(
|
||||
cc, ci, iTesters, testSet.testSpecs, startingConfiguration, locality, defaultTenant, tenantsToCreate);
|
||||
} else {
|
||||
tests = reportErrors(
|
||||
runTests(cc, ci, testSet.testSpecs, at, minTestersExpected, startingConfiguration, locality, defaultTenant),
|
||||
"RunTests");
|
||||
tests = reportErrors(runTests(cc,
|
||||
ci,
|
||||
testSet.testSpecs,
|
||||
at,
|
||||
minTestersExpected,
|
||||
startingConfiguration,
|
||||
locality,
|
||||
defaultTenant,
|
||||
tenantsToCreate),
|
||||
"RunTests");
|
||||
}
|
||||
|
||||
choose {
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user