Merge branch 'master' of https://github.com/apple/foundationdb into fix-reply-promise-stream-local-delivery

fdbclient/ServerKnobs.cpp

@@ -335,9 +335,7 @@ void ServerKnobs::initialize(Randomize randomize, ClientKnobs* clientKnobs, IsSi
 	// KeyValueStoreRocksDB
 	init( ROCKSDB_BACKGROUND_PARALLELISM,                          0 );
 	init( ROCKSDB_READ_PARALLELISM,                                4 );
-	// Use a smaller memtable in simulation to avoid OOMs.
-	int64_t memtableBytes = isSimulated ? 32 * 1024 : 512 * 1024 * 1024;
-	init( ROCKSDB_MEMTABLE_BYTES,                      memtableBytes );
+	init( ROCKSDB_MEMTABLE_BYTES,                  512 * 1024 * 1024 );
 	init( ROCKSDB_UNSAFE_AUTO_FSYNC,                           false );
 	init( ROCKSDB_PERIODIC_COMPACTION_SECONDS,                     0 );
 	init( ROCKSDB_PREFIX_LEN,                                      0 );

fdbserver/KeyValueStoreRocksDB.actor.cpp

@@ -7,7 +7,6 @@
 #include <rocksdb/slice_transform.h>
 #include <rocksdb/table.h>
 #include <rocksdb/utilities/table_properties_collectors.h>
-#include "fdbserver/CoroFlow.h"
 #include "flow/flow.h"
 #include "flow/IThreadPool.h"
 
@@ -284,9 +283,7 @@ struct RocksDBKeyValueStore : IKeyValueStore {
 				                              std::min(value.size(), size_t(a.maxLength)))));
 			} else {
 				if (!s.IsNotFound()) {
-					TraceEvent(SevError, "RocksDBError")
-					    .detail("Error", s.ToString())
-					    .detail("Method", "ReadValuePrefix");
+					TraceEvent(SevError, "RocksDBError").detail("Error", s.ToString()).detail("Method", "ReadValuePrefix");
 				}
 				a.result.send(Optional<Value>());
 			}
@@ -370,23 +367,8 @@ struct RocksDBKeyValueStore : IKeyValueStore {
 	std::unique_ptr<rocksdb::WriteBatch> writeBatch;
 
 	explicit RocksDBKeyValueStore(const std::string& path, UID id) : path(path), id(id) {
-		// In simluation, run the reader/writer threads as Coro threads (i.e. in the network thread. The storage engine
-		// is still multi-threaded as background compaction threads are still present. Reads/writes to disk will also
-		// block the network thread in a way that would be unacceptable in production but is a necessary evil here. When
-		// performing the reads in background threads in simulation, the event loop thinks there is no work to do and
-		// advances time faster than 1 sec/sec. By the time the blocking read actually finishes, simulation has advanced
-		// time by more than 5 seconds, so every read fails with a transaction_too_old error. Doing blocking IO on the
-		// main thread solves this issue. There are almost certainly better fixes, but my goal was to get a less
-		// invasive change merged first and work on a more realistic version if/when we think that would provide
-		// substantially more confidence in the correctness.
-		// TODO: Adapt the simulation framework to not advance time quickly when background reads/writes are occurring.
-		if (g_network->isSimulated()) {
-			writeThread = CoroThreadPool::createThreadPool();
-			readThreads = CoroThreadPool::createThreadPool();
-		} else {
-			writeThread = createGenericThreadPool();
-			readThreads = createGenericThreadPool();
-		}
+		writeThread = createGenericThreadPool();
+		readThreads = createGenericThreadPool();
 		writeThread->addThread(new Writer(db, id), "fdb-rocksdb-wr");
 		for (unsigned i = 0; i < SERVER_KNOBS->ROCKSDB_READ_PARALLELISM; ++i) {
 			readThreads->addThread(new Reader(db), "fdb-rocksdb-re");

fdbserver/SimulatedCluster.actor.cpp

@@ -232,7 +232,6 @@ public:
 	//	1 = "memory"
 	//	2 = "memory-radixtree-beta"
 	//	3 = "ssd-redwood-experimental"
-	//	4 = "ssd-rocksdb-experimental"
 	// Requires a comma-separated list of numbers WITHOUT whitespaces
 	std::vector<int> storageEngineExcludeTypes;
 	// Set the maximum TLog version that can be selected for a test
@@ -1253,7 +1252,7 @@ void SimulationConfig::setDatacenters(const TestConfig& testConfig) {
 
 // Sets storage engine based on testConfig details
 void SimulationConfig::setStorageEngine(const TestConfig& testConfig) {
-	int storage_engine_type = deterministicRandom()->randomInt(0, 5);
+	int storage_engine_type = deterministicRandom()->randomInt(0, 4);
 	if (testConfig.storageEngineType.present()) {
 		storage_engine_type = testConfig.storageEngineType.get();
 	} else {
@@ -1261,7 +1260,7 @@ void SimulationConfig::setStorageEngine(const TestConfig& testConfig) {
 		while (std::find(testConfig.storageEngineExcludeTypes.begin(),
 		                 testConfig.storageEngineExcludeTypes.end(),
 		                 storage_engine_type) != testConfig.storageEngineExcludeTypes.end()) {
-			storage_engine_type = deterministicRandom()->randomInt(0, 5);
+			storage_engine_type = deterministicRandom()->randomInt(0, 4);
 		}
 	}
 
@@ -1286,16 +1285,6 @@ void SimulationConfig::setStorageEngine(const TestConfig& testConfig) {
 		set_config("ssd-redwood-experimental");
 		break;
 	}
-	case 4: {
-		TEST(true); // Simulated cluster using RocksDB storage engine
-		set_config("ssd-rocksdb-experimental");
-		// Tests using the RocksDB engine are necessarily non-deterministic because of RocksDB
-		// background threads.
-		TraceEvent(SevWarn, "RocksDBNonDeterminism")
-		    .detail("Explanation", "The RocksDB storage engine is threaded and non-deterministic");
-		noUnseed = true;
-		break;
-	}
 	default:
 		ASSERT(false); // Programmer forgot to adjust cases.
 	}

tests/restarting/from_6.2.33/SnapCycleRestart-1.txt

@@ -1,5 +1,3 @@
-storageEngineExcludeTypes=4
-
 ;Take snap and do cycle test
 testTitle=SnapCyclePre
 clearAfterTest=false

tests/restarting/from_6.2.33/SnapCycleRestart-2.txt

@@ -1,5 +1,4 @@
 buggify=off
-storageEngineExcludeTypes=4
 
 testTitle=SnapCycleRestore
 runSetup=false

tests/restarting/from_6.2.33/SnapTestAttrition-1.txt

@@ -1,5 +1,3 @@
-storageEngineExcludeTypes=4
-
 ;write 1000 Keys ending with even numbers
 testTitle=SnapTestPre
 clearAfterTest=false

tests/restarting/from_6.2.33/SnapTestAttrition-2.txt

@@ -1,5 +1,4 @@
 buggify=off
-storageEngineExcludeTypes=4
 
 ; verify all keys are even numbered
 testTitle=SnapTestVerify

tests/restarting/from_6.2.33/SnapTestRestart-1.txt

@@ -1,5 +1,3 @@
-storageEngineExcludeTypes=4
-
 ;write 1000 Keys ending with even numbers
 testTitle=SnapTestPre
 clearAfterTest=false

tests/restarting/from_6.2.33/SnapTestRestart-2.txt

@@ -1,5 +1,4 @@
 buggify=off
-storageEngineExcludeTypes=4
 
 ; verify all keys are even numbered
 testTitle=SnapTestVerify

tests/restarting/from_6.2.33/SnapTestSimpleRestart-1.txt

@@ -1,5 +1,3 @@
-storageEngineExcludeTypes=4
-
 ;write 1000 Keys ending with even number
 testTitle=SnapSimplePre
 clearAfterTest=false

tests/restarting/from_6.2.33/SnapTestSimpleRestart-2.txt

@@ -1,5 +1,4 @@
 buggify=off
-storageEngineExcludeTypes=4
 
 ; verify all keys are even numbered
 testTitle=SnapSimpleVerify

tests/restarting/from_7.0.0/SnapIncrementalRestore-1.toml

@@ -1,8 +1,7 @@
 [configuration]
 logAntiQuorum = 0
-storageEngineExcludeTypes = [4]
 
-[[test]]
+[[test]] 
 testTitle = 'SubmitBackup'
 simBackupAgents = 'BackupToFile'
 clearAfterTest = false

tests/restarting/from_7.0.0/SnapIncrementalRestore-2.toml

@@ -1,6 +1,3 @@
-[configuration]
-storageEngineExcludeTypes = [4]
-
 [[test]]
 testTitle = 'RestoreBackup'
 simBackupAgents = 'BackupToFile'