From 2190ceb003fda847014d4ddc73ba3692c0ae2fee Mon Sep 17 00:00:00 2001 From: Josh Slocum Date: Wed, 27 Jul 2022 08:34:11 -0500 Subject: [PATCH 1/3] Adding chaos to ChangeFeedOperations --- tests/CMakeLists.txt | 1 + tests/fast/ChangeFeedOperations.toml | 25 +++++++++++++++ tests/fast/ChangeFeedOperationsMove.toml | 39 ++++++++++++++++++++++++ 3 files changed, 65 insertions(+) create mode 100644 tests/fast/ChangeFeedOperationsMove.toml diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt index 28896e4bff..b1cf78c7f7 100644 --- a/tests/CMakeLists.txt +++ b/tests/CMakeLists.txt @@ -140,6 +140,7 @@ if(WITH_PYTHON) add_fdb_test(TEST_FILES fast/CycleTest.toml) add_fdb_test(TEST_FILES fast/ChangeFeeds.toml) add_fdb_test(TEST_FILES fast/ChangeFeedOperations.toml) + add_fdb_test(TEST_FILES fast/ChangeFeedOperationsMove.toml) add_fdb_test(TEST_FILES fast/DataLossRecovery.toml) add_fdb_test(TEST_FILES fast/EncryptionOps.toml) # TODO: fix failures and renable the test diff --git a/tests/fast/ChangeFeedOperations.toml b/tests/fast/ChangeFeedOperations.toml index 3ba89beed0..ca3d47e08e 100644 --- a/tests/fast/ChangeFeedOperations.toml +++ b/tests/fast/ChangeFeedOperations.toml @@ -8,3 +8,28 @@ testTitle = 'ChangeFeedOperationsTest' [[test.workload]] testName = 'ChangeFeedOperations' + testDuration = 60.0 + + [[test.workload]] + testName = 'RandomClogging' + testDuration = 60.0 + + [[test.workload]] + testName = 'Rollback' + meanDelay = 30.0 + testDuration = 60.0 + + [[test.workload]] + testName = 'Attrition' + machinesToKill = 10 + machinesToLeave = 3 + reboot = true + testDuration = 60.0 + + [[test.workload]] + testName = 'Attrition' + machinesToKill = 10 + machinesToLeave = 3 + reboot = true + testDuration = 60.0 + diff --git a/tests/fast/ChangeFeedOperationsMove.toml b/tests/fast/ChangeFeedOperationsMove.toml new file mode 100644 index 0000000000..a9852f2760 --- /dev/null +++ b/tests/fast/ChangeFeedOperationsMove.toml @@ -0,0 +1,39 @@ +[configuration] +allowDefaultTenant = false + +# TODO add failure events, and then add a version that also supports randomMoveKeys + +[[test]] +testTitle = 'ChangeFeedOperationsTest' + + [[test.workload]] + testName = 'ChangeFeedOperations' + testDuration = 60.0 + + [[test.workload]] + testName = 'RandomMoveKeys' + testDuration = 60.0 + + [[test.workload]] + testName = 'RandomClogging' + testDuration = 60.0 + + [[test.workload]] + testName = 'Rollback' + meanDelay = 60.0 + testDuration = 60.0 + + [[test.workload]] + testName = 'Attrition' + machinesToKill = 10 + machinesToLeave = 3 + reboot = true + testDuration = 60.0 + + [[test.workload]] + testName = 'Attrition' + machinesToKill = 10 + machinesToLeave = 3 + reboot = true + testDuration = 60.0 + From ddad5b189ea8e2d6dca169daa2a8aa8554a9d7c0 Mon Sep 17 00:00:00 2001 From: Josh Slocum Date: Fri, 29 Jul 2022 07:31:22 -0500 Subject: [PATCH 2/3] fixing change feed ops test race with live reader and destroying --- fdbserver/workloads/ChangeFeedOperations.actor.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/fdbserver/workloads/ChangeFeedOperations.actor.cpp b/fdbserver/workloads/ChangeFeedOperations.actor.cpp index 05b1053ccb..07e3bf9d43 100644 --- a/fdbserver/workloads/ChangeFeedOperations.actor.cpp +++ b/fdbserver/workloads/ChangeFeedOperations.actor.cpp @@ -248,7 +248,7 @@ ACTOR Future liveReader(Database cx, Reference data, Version buffered.pop_front(); } if (buffered.empty()) { - if (data->poppingVersion < data->pendingCheck.front().first) { + if (data->poppingVersion < data->pendingCheck.front().first && !data->destroying) { fmt::print("DBG) {0} Buffered empty after ready for check, and data not popped! popped " "{1}, popping {2}, check {3}\n", data->key.printable(), @@ -256,7 +256,7 @@ ACTOR Future liveReader(Database cx, Reference data, Version data->poppingVersion, data->pendingCheck.front().first); } - ASSERT(data->poppingVersion >= data->pendingCheck.front().first); + ASSERT(data->poppingVersion >= data->pendingCheck.front().first || data->destroying); data->pendingCheck.pop_front(); } else { Version v = buffered.front().version; From c5c423f2b772d4824c6122aa9503ae15c3acaf79 Mon Sep 17 00:00:00 2001 From: Josh Slocum Date: Mon, 1 Aug 2022 17:42:32 -0500 Subject: [PATCH 3/3] Disabling update/clear post-stop for now in change feed ops --- fdbserver/workloads/ChangeFeedOperations.actor.cpp | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/fdbserver/workloads/ChangeFeedOperations.actor.cpp b/fdbserver/workloads/ChangeFeedOperations.actor.cpp index 07e3bf9d43..d6fd92bc5f 100644 --- a/fdbserver/workloads/ChangeFeedOperations.actor.cpp +++ b/fdbserver/workloads/ChangeFeedOperations.actor.cpp @@ -694,6 +694,14 @@ struct ChangeFeedOperationsWorkload : TestWorkload { state Transaction tr(cx); state Optional updateValue; + // FIXME: right now there is technically a bug in the change feed contract (mutations can appear in the stream + // at a higher version than the stop version) But because stopping a feed is sort of just an optimization, and + // no current user of change feeds currently relies on the stop version for correctness, it's fine to not test + // this for now + if (feedData->stopVersion.present()) { + return Void(); + } + // if value is already not set, don't do a clear, otherwise pick either if (feedData->lastCleared || deterministicRandom()->random01() > self->clearFrequency) { updateValue = feedData->nextValue();