Merge pull request #1386 from mpilman/features/client-simulator

Implemented JavaWorkload

bindings/java/CMakeLists.txt

@@ -51,6 +51,8 @@ set(JAVA_BINDING_SRCS
   src/main/com/apple/foundationdb/subspace/Subspace.java
   src/main/com/apple/foundationdb/Transaction.java
   src/main/com/apple/foundationdb/TransactionContext.java
+  src/main/com/apple/foundationdb/testing/AbstractWorkload.java
+  src/main/com/apple/foundationdb/testing/WorkloadContext.java
   src/main/com/apple/foundationdb/tuple/ByteArrayUtil.java
   src/main/com/apple/foundationdb/tuple/IterableComparator.java
   src/main/com/apple/foundationdb/tuple/package-info.java
@@ -127,6 +129,9 @@ target_include_directories(fdb_java PRIVATE ${JNI_INCLUDE_DIRS})
 target_link_libraries(fdb_java PRIVATE fdb_c)
 set_target_properties(fdb_java PROPERTIES
   LIBRARY_OUTPUT_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}/lib/${SYSTEM_NAME}/amd64/)
+if(APPLE)
+  set_target_properties(fdb_java PROPERTIES SUFFIX ".jnilib")
+endif()
 
 set(CMAKE_JAVA_COMPILE_FLAGS "-source" "1.8" "-target" "1.8")
 set(CMAKE_JNI_TARGET TRUE)
@@ -175,25 +180,34 @@ if(NOT OPEN_FOR_IDE)
   endif()
   if(WIN32)
     set(lib_destination "windows/amd64")
+    set(clib_destination "windows/amd64/fdb_c.dll")
   elseif(APPLE)
     set(lib_destination "osx/x86_64")
+    set(clib_destination "osx/x86_64/libfdb_c.jnilib")
   else()
     set(lib_destination "linux/amd64")
+    set(clib_destination "linux/amd64/libfdb_c.so")
   endif()
   set(lib_destination "${unpack_dir}/lib/${lib_destination}")
+  set(clib_destination "${unpack_dir}/lib/${clib_destination}")
   file(MAKE_DIRECTORY ${lib_destination})
   add_custom_command(OUTPUT ${CMAKE_CURRENT_BINARY_DIR}/lib_copied
     COMMAND ${CMAKE_COMMAND} -E copy $<TARGET_FILE:fdb_java> ${lib_destination} &&
     ${CMAKE_COMMAND} -E touch ${CMAKE_CURRENT_BINARY_DIR}/lib_copied
-    COMMENT "Copy library")
-  add_custom_target(copy_lib DEPENDS ${CMAKE_CURRENT_BINARY_DIR}/lib_copied)
+    COMMENT "Copy jni library for fat jar")
+  add_custom_command(OUTPUT ${clib_destination}
+    COMMAND ${CMAKE_COMMAND} -E copy $<TARGET_FILE:fdb_c> ${clib_destination}
+    COMMENT "Copy fdbc for fat jar")
+  add_custom_target(copy_lib DEPENDS ${CMAKE_CURRENT_BINARY_DIR}/lib_copied ${clib_destination})
   add_dependencies(copy_lib unpack_jar)
   set(target_jar ${jar_destination}/fdb-java-${CMAKE_PROJECT_VERSION}.jar)
   add_custom_command(OUTPUT ${target_jar}
     COMMAND ${Java_JAR_EXECUTABLE} cf ${target_jar} .
     WORKING_DIRECTORY ${unpack_dir}
+    DEPENDS ${CMAKE_CURRENT_BINARY_DIR}/lib_copied ${clib_destination}
     COMMENT "Build ${jar_destination}/fdb-java-${CMAKE_PROJECT_VERSION}.jar")
-  add_custom_target(fat-jar DEPENDS ${target_jar})
+  add_custom_target(fat-jar ALL DEPENDS ${target_jar})
+  add_dependencies(fat-jar fdb-java)
   add_dependencies(fat-jar copy_lib)
   add_dependencies(packages fat-jar)
 endif()

bindings/java/src/main/com/apple/foundationdb/FDB.java

@@ -104,10 +104,15 @@ public class FDB {
 	 * Called only once to create the FDB singleton.
 	 */
 	private FDB(int apiVersion) {
-		this.apiVersion = apiVersion;
+		this(apiVersion, true);
+	}
 
+	private FDB(int apiVersion, boolean controlRuntime) {
+		this.apiVersion = apiVersion;
 		options = new NetworkOptions(this::Network_setOption);
-		Runtime.getRuntime().addShutdownHook(new Thread(this::stopNetwork));
+		if (controlRuntime) {
+			Runtime.getRuntime().addShutdownHook(new Thread(this::stopNetwork));
+		}
 	}
 
 	/**
@@ -171,7 +176,14 @@ public class FDB {
 	 *
 	 * @return the FoundationDB API object
 	 */
-	public static synchronized FDB selectAPIVersion(final int version) throws FDBException {
+	public static FDB selectAPIVersion(final int version) throws FDBException {
+		return selectAPIVersion(version, true);
+	}
+
+	/**
+	   This function is called from C++ if the VM is controlled directly from FDB
+	 */
+	private static synchronized FDB selectAPIVersion(final int version, boolean controlRuntime) throws FDBException {
 		if(singleton != null) {
 			if(version != singleton.getAPIVersion()) {
 				throw new IllegalArgumentException(
@@ -185,7 +197,7 @@ public class FDB {
 			throw new IllegalArgumentException("API version not supported (maximum 610)");
 
 		Select_API_version(version);
-		FDB fdb = new FDB(version);
+		FDB fdb = new FDB(version, controlRuntime);
 
 		return singleton = fdb;
 	}

bindings/java/src/main/com/apple/foundationdb/testing/AbstractWorkload.java

@@ -0,0 +1,128 @@
+/*
+ * AbstractWorkload.java
+ *
+ * This source file is part of the FoundationDB open source project
+ *
+ * Copyright 2013-2019 Apple Inc. and the FoundationDB project authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.apple.foundationdb.testing;
+
+import com.apple.foundationdb.Database;
+
+import java.io.File;
+import java.io.IOException;
+import java.lang.reflect.Method;
+import java.net.URL;
+import java.net.URLClassLoader;
+import java.util.concurrent.Executor;
+import java.util.concurrent.ThreadPoolExecutor;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.SynchronousQueue;
+
+import java.util.Map;
+
+public abstract class AbstractWorkload {
+	private static final Class<?>[] parameters = new Class<?>[]{URL.class};
+	protected WorkloadContext context;
+	private ThreadPoolExecutor executorService;
+
+	public AbstractWorkload(WorkloadContext context) {
+		this.context = context;
+		executorService =
+        new ThreadPoolExecutor(1, 2,
+            10, TimeUnit.SECONDS,
+			new SynchronousQueue<>()) {
+			@Override
+			protected void beforeExecute(Thread t, Runnable r) {
+				setProcessID(context.getProcessID());
+				super.beforeExecute(t, r);
+			}
+		};
+	}
+
+	private Executor getExecutor() {
+		return executorService;
+	}
+
+	public abstract void setup(Database db);
+	public abstract void start(Database db);
+	public abstract boolean check(Database db);
+	public double getCheckTimeout() {
+		return 3000;
+	}
+
+	private void setup(Database db, long voidCallback) {
+		AbstractWorkload self = this;
+		getExecutor().execute(new Runnable(){
+			public void run() {
+				self.setup(db);
+				self.sendVoid(voidCallback);
+			}
+		});
+	}
+	private void start(Database db, long voidCallback) {
+		AbstractWorkload self = this;
+		getExecutor().execute(new Runnable(){
+			public void run() {
+				self.start(db);
+				self.sendVoid(voidCallback);
+			}
+		});
+	}
+	private void check(Database db, long boolCallback) {
+		AbstractWorkload self = this;
+		getExecutor().execute(new Runnable(){
+			public void run() {
+				boolean res = self.check(db);
+				self.sendBool(boolCallback, res);
+			}
+		});
+	}
+
+	private void shutdown() {
+		executorService.shutdown();
+	}
+
+	public native void log(int severity, String message, Map<String, String> details);
+	private native void setProcessID(long processID);
+	private native void sendVoid(long handle);
+	private native void sendBool(long handle, boolean value);
+
+	// Helper functions to add to the class path at Runtime - will be called
+	// from C++
+	private static void addFile(String s) throws IOException {
+		File f = new File(s);
+		addFile(f);
+	}
+
+	private static void addFile(File f) throws IOException {
+		addURL(f.toURI().toURL());
+	}
+
+	private static void addURL(URL u) throws IOException {
+		URLClassLoader sysLoader = (URLClassLoader) ClassLoader.getSystemClassLoader();
+		Class<URLClassLoader> sysClass = URLClassLoader.class;
+
+		try {
+			Method method = sysClass.getDeclaredMethod("addURL", parameters);
+			method.setAccessible(true);
+			method.invoke(sysLoader, new Object[]{u});
+		} catch (Throwable t) {
+			t.printStackTrace();
+			throw new IOException("Error, could not add URL to system classloader");
+		}
+	}
+}

bindings/java/src/main/com/apple/foundationdb/testing/WorkloadContext.java

@@ -0,0 +1,61 @@
+/*
+ * IWorkload.java
+ *
+ * This source file is part of the FoundationDB open source project
+ *
+ * Copyright 2013-2019 Apple Inc. and the FoundationDB project authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.apple.foundationdb.testing;
+
+import java.util.Map;
+
+public class WorkloadContext {
+	private Map<String, String> options;
+	private int clientId, clientCount;
+	long sharedRandomNumber, processID;
+
+	public WorkloadContext(Map<String, String> options, int clientId, int clientCount, long sharedRandomNumber, long processID)
+	{
+		this.options = options;
+		this.clientId = clientId;
+		this.clientCount = clientCount;
+		this.sharedRandomNumber = sharedRandomNumber;
+		this.processID = processID;
+	}
+
+	public String getOption(String name, String defaultValue) {
+		if (options.containsKey(name)) {
+			return options.get(name);
+		}
+		return defaultValue;
+	}
+
+	public int getClientId() {
+		return clientId;
+	}
+
+	public int getClientCount() {
+		return clientCount;
+	}
+
+	public long getSharedRandomNumber() {
+		return sharedRandomNumber;
+	}
+
+	public long getProcessID() {
+		return processID;
+	}
+}

documentation/sphinx/source/client-design.rst

@@ -14,6 +14,8 @@ FoundationDB supports language bindings for application development using the or
 
 * :doc:`data-modeling` explains recommended techniques for representing application data in the key-value store.
 
+* :doc:`client-testing` Explains how one can use workloads to test client code.
+
 * :doc:`api-general` contains information on FoundationDB clients applicable across all language bindings.
 
 * :doc:`known-limitations` describes both long-term design limitations of FoundationDB and short-term limitations applicable to the current version.
@@ -29,5 +31,6 @@ FoundationDB supports language bindings for application development using the or
     downloads
     developer-guide
     data-modeling
+    client-testing
     api-general
     known-limitations

documentation/sphinx/source/client-testing.rst

@@ -0,0 +1,239 @@
+###############
+Client Testing
+###############
+
+FoundationDB comes with its own testing framework. Tests are implemented as workloads. A workload is nothing more than a class
+that gets called by server processes running the ``tester`` role. Additionally, a ``fdbserver`` process can run a simulator that
+simulates a full fdb cluster with several machines and different configurations in one process. This simulator can run the same
+workloads you can run on a real cluster. It will also inject random failures like network partitions and disk failures.
+
+Currently, workloads can only be implemented in Java, support for other languages might come later.
+
+This tutorial explains how one can implement a workload, how one can orchestrate a workload on a cluster with multiple clients, and
+how one can run a workload within a simulator. Running in a simulator is also useful as it does not require any setup: you can simply
+run one command that will provide you with a fully functional FoundationDB cluster.
+
+Preparing the fdbserver Binary
+==============================
+
+In order to run a Java workload, ``fdbserver`` needs to be able to embed a JVM. Because of that it needs to be linked against JNI.
+The official FDB binaries do not link against JNI and therefore one can't use that to run a Java workload. Instead you need to
+download the sources and build them. Make sure that ``cmake`` can find Java and pass ``-DWITH_JAVA_WORKLOAD=ON`` to cmake.
+
+After FoundationDB was built, you can use ``bin/fdbserver`` to run the server. The jar file containing the client library can be
+found in ``packages/fdb-VERSION.jar``. Both of these are in the build directory.
+
+Implementing a Workload
+=======================
+
+In order to implement your own workload in Java you can simply create an implementation of the abstract class ``AbstractWorkload``.
+A minimal implementation will look like this:
+
+.. code-block:: java
+
+   package my.package;
+   import com.apple.foundationdb.testing.AbstractWorkload;
+   import com.apple.foundationdb.testing.WorkloadContext;
+
+   class MinimalWorkload extends AbstractWorkload {
+       public MinimalWorkload(WorkloadContext ctx) {
+           super(ctx);
+       }
+
+       @Override
+       public void setup(Database db) {
+           log(20, "WorkloadSetup", null);
+       }
+
+       @Override
+       public void start(Database db) {
+           log(20, "WorkloadStarted", null);
+       }
+
+       @Override
+       public boolean check(Database db) {
+           log(20, "WorkloadFailureCheck", null);
+           return true;
+       }
+   }
+
+The lifecycle of a test will look like this:
+
+1. All testers will create an instance of the ``AbstractWorkload`` implementation.
+2. All testers will (in parallel but not guaranteed exactly at the same time) call
+   ``setup`` and they will wait for all of them to finish. This phase can be used to
+   pre-populate data.
+3. All tester will then call start (again, in parallel) and wait for all of them to
+   finish.
+4. All testers will then call ``check`` on all testers and use the returned boolean
+   to determine whether the test succeeded.
+
+All these methods take a ``Database`` object as an argument. This object can be used
+to create and execute transactions against the cluster.
+
+When implementing workloads, an author has to follow these rules:
+
+- To write tracing to the trace-files one should use ``AbstractWorkload.log``. This
+  Method takes three arguments: an integer for severity (5 means debug, 10 means log,
+  20 means warning, 30 means warn always, and 40 is a severe error). If any tester
+  logs something of severity 40, the test run is considered to have failed.
+- In order to increase throughput on the cluster, an author might want to spawn several
+  threads. However, threads *MUST* only be spawn through the ``Executor`` instance one
+  can get from ``AbstractWorkload.getExecutor()``. Otherwise, a simulation test will
+  probably segfault. The reason for this is that we need to keep track of which simulated
+  machine a thread corresponds to internally.
+
+Within a workload you have access to the ``WorkloadContext`` which provides additional
+information about the current execution environment. The context can be accessed through
+``this.context`` and provides the following methods:
+
+- ``String getOption(String name, String defaultValue)``. A user can provide parameters to workloads
+  through a configuration file (explained further down). These parameters are provided to
+  all clients through the context and can be accessed with this method.
+- ``int getClientId()`` and ``int getClientCount()``. An author can determine how many
+  clients are running in the cluster and each of those will get a globally unique ID (a number
+  between 0 and clientCount - 1). This is useful for example if you want to generate transactions
+  that are guaranteed to not conflict with transactions from other clients.
+- ``int getSharedRandomNumber()``. At startup a random number will be generated. This will allow for
+  generating the same random numbers across several machines if this number is used as a seed.
+
+
+Running a Workload in the Simulator
+===================================
+
+We'll first walk how one can run a workload in a simulator. FoundationDB comes already with a large number
+of workloads. But some of them can't be run in simulation while other don't work on a real cluster. Most
+will work on both though. To look for examples how these can be ran, you can find configuration files in
+the ``tests`` directory in the FoundationDB source tree.
+
+We will now go through an example how you can write a relatively complex test and run it in the simulator.
+Writing and running tests in the simulator is a simple two-step process.
+
+1. Write the test.
+2. Run ``fdbserver`` in simulation mode and provide it with the test file.
+
+Write the Test
+--------------
+
+A workload is not a test. A test is a simple test file that tells the test orchestrator which workloads it
+should run and in which order. Additionally one can provide parameters to workloads through this file.
+
+A test file might look like this:
+
+.. code-block:: none
+
+   testTitle=MyTest
+     testName=JavaWorkload
+     workloadClass=my.package.MinimalWorkload
+     jvmOptions=-Djava.class.path=*PATH_TO_FDB_CLIENT_JAR*,*other options you want to pass to the JVM*
+     classPath=PATH_TO_JAR_OR_DIR_CONTAINING_WORKLOAD,OTHER_DEPENDENCIES
+
+     testName=Attrition
+     testDuration=5.0
+     reboot=true
+     machinesToKill=3
+
+   testTitle=AnotherTest
+     workloadClass=my.package.MinimalWorkload
+     workloadClass=my.package.MinimalWorkload
+     jvmOptions=-Djava.class.path=*PATH_TO_FDB_CLIENT_JAR*,*other options you want to pass to the JVM*
+     classPath=PATH_TO_JAR_OR_DIR_CONTAINING_WORKLOAD,OTHER_DEPENDENCIES
+     someOpion=foo
+
+     workloadClass=my.package.AnotherWorkload
+     workloadClass=my.package.AnotherWorkload
+     jvmOptions=-Djava.class.path=*PATH_TO_FDB_CLIENT_JAR*,*other options you want to pass to the JVM*
+     classPath=PATH_TO_JAR_OR_DIR_CONTAINING_WORKLOAD,OTHER_DEPENDENCIES
+     anotherOption=foo
+
+This test will do the following:
+
+1. First it will run ``MinimalWorkload`` without any parameter.
+2. After 5.0 seconds the simulator will reboot 3 random machines (this is what Attrition does
+   and this workload is provided by FoundationDB. This is one of the few workloads that only
+   work in the simulator).
+3. When all workloads are finished, it will run ``MinimalWorkload``
+   again. This time it will have the option ``someOption`` set to
+   ``foo``. Additionally it will run ``AnotherWorkload`` in parallel.
+
+How to set the Class Path correctly
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+As you can see from above example, we can set the classpath through two different mechanisms. However, one has
+to be careful as they can't be used interchangeably.
+
+- You can set a class path through the JVM argument ``-Djava.class.path=...``. This is how you have to pass the
+  path to the FoundationDB client library (as the client library is needed during the initialization phase). However,
+  only the first specified section will have any effect as the other Workloads will run in the same VM (and arguments,
+  by nature, can only be passed once).
+- The ``classPath`` option. This option will add all paths (directories or JAR-files) to the classPath of the JVM
+  while it is running. Not being able to add the path will result in a test failure. This is useful to add different
+  dependencies to different workloads. A path can appear more than once across sections. However, they must not
+  conflict with each other as we never remove something from the classpath.
+
+Run the simulator
+-----------------
+
+This step is very simple. You can simply run ``fdbserver`` with role simulator
+and pass the test with ``-f``:
+
+.. code-block:: sh
+
+   fdbserver -r simulator -f testfile.txt
+
+
+Running a Workload on an actual Cluster
+=======================================
+
+Running a workload on a cluster works basically the smae way. However, one must
+actually setup a cluster first. This cluster must run between one and many server
+processes with the class test. So above 2-step process becomes a bit more complex:
+
+1. Write the test (same as above).
+2. Set up a cluster with as many test clients as you want.
+3. Run the orchestor to actually execute the test.
+
+Step 1. is explained further up. For step 2., please refer to the general FoundationDB
+configuration. The main difference to a normal FoundationDB cluster is that some processes
+must have a test class assigned to them. This can be done in the ``foundationdb.conf``. For
+example this file would create a server with 8 processes of which 4 would act as test clients.
+
+.. code-block:: ini
+
+    [fdbmonitor]
+    user = foundationdb
+    group = foundationdb
+
+    [general]
+    restart_delay = 60
+    cluster_file = /etc/foundationdb/fdb.cluster
+
+    ## Default parameters for individual fdbserver processes
+    [fdbserver]
+    command = /usr/sbin/fdbserver
+    public_address = auto:$ID
+    listen_address = public
+    datadir = /var/lib/foundationdb/data/$ID
+    logdir = /var/log/foundationdb
+
+    [fdbserver.4500]
+    [fdbserver.4501]
+    [fdbserver.4502]
+    [fdbserver.4503]
+    [fdbserver.4510]
+    class = test
+    [fdbserver.4511]
+    class = test
+    [fdbserver.4512]
+    class = test
+    [fdbserver.4513]
+    class = test
+
+Running the actual test can be done with ``fdbserver`` as well. For this you can call the process
+with the ``multitest`` role:
+
+.. code-block:: sh
+
+   fdbserver -r multitest -f testfile.txt
+
+This command will block until all tests are completed.

fdbrpc/simulator.h

@@ -309,8 +309,8 @@ public:
 	virtual flowGlobalType global(int id) { return getCurrentProcess()->global(id); };
 	virtual void setGlobal(size_t id, flowGlobalType v) { getCurrentProcess()->setGlobal(id,v); };
 
-protected:
 	static thread_local ProcessInfo* currentProcess;
+protected:
 	Mutex mutex;
 
 private:

fdbserver/CMakeLists.txt

@@ -177,6 +177,18 @@ set(FDBSERVER_SRCS
   workloads/WriteBandwidth.actor.cpp
   workloads/WriteDuringRead.actor.cpp)
 
+set(java_workload_docstring "Build the Java workloads (makes fdbserver link against JNI)")
+if(FDB_RELEASE)
+  set(WITH_JAVA_WORKLOAD OFF CACHE BOOL "${java_workload_docstring}")
+elseif(WITH_JAVA)
+  set(WITH_JAVA_WORKLOAD ON CACHE BOOL "${java_workload_docstring}")
+else()
+  set(WITH_JAVA_WORKLOAD OFF CACHE BOOL "${java_workload_docstring}")
+endif()
+if(WITH_JAVA_WORKLOAD)
+  list(APPEND FDBSERVER_SRCS workloads/JavaWorkload.actor.cpp)
+endif()
+
 file(MAKE_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}/workloads)
 
 add_flow_target(EXECUTABLE NAME fdbserver SRCS ${FDBSERVER_SRCS})
@@ -184,6 +196,13 @@ target_include_directories(fdbserver PRIVATE
   ${CMAKE_CURRENT_BINARY_DIR}/workloads
   ${CMAKE_CURRENT_SOURCE_DIR}/workloads)
 target_link_libraries(fdbserver PRIVATE fdbclient)
+if(WITH_JAVA_WORKLOAD)
+  if(NOT JNI_FOUND)
+    message(SEND_ERROR "Trying to build Java workload but couldn't find JNI")
+  endif()
+  target_include_directories(fdbserver PRIVATE "${JNI_INCLUDE_DIRS}")
+  target_link_libraries(fdbserver PRIVATE "${JNI_LIBRARIES}")
+endif()
 if (GPERFTOOLS_FOUND)
   add_compile_definitions(USE_GPERFTOOLS)
   target_link_libraries(fdbserver PRIVATE gperftools)

fdbserver/workloads/JavaWorkload.actor.cpp

@@ -0,0 +1,603 @@
+#include "workloads.actor.h"
+#include <flow/ThreadHelper.actor.h>
+
+#include <jni.h>
+#include <fdbrpc/simulator.h>
+#include <fdbclient/IClientApi.h>
+#include <fdbclient/ThreadSafeTransaction.h>
+
+#include <memory>
+
+#include <flow/actorcompiler.h> // must be last include
+
+extern void flushTraceFileVoid();
+
+namespace {
+
+void printTrace(JNIEnv* env, jobject self, jint severity, jstring message, jobject details) {
+	jboolean isCopy;
+	const char* msg = env->GetStringUTFChars(message, &isCopy);
+	std::unordered_map<std::string, std::string> detailsMap;
+	if (details != nullptr) {
+		jclass mapClass = env->FindClass("java/util/Map");
+		jclass setClass = env->FindClass("java/util/Set");
+		jclass iteratorClass = env->FindClass("java/util/Iterator");
+		jmethodID keySetID = env->GetMethodID(mapClass, "keySet", "()Ljava/util/Set;");
+		jobject keySet = env->CallObjectMethod(details, keySetID);
+		jmethodID iteratorMethodID = env->GetMethodID(setClass, "iterator", "()Ljava/util/Iterator;");
+		jobject iterator = env->CallObjectMethod(keySet, iteratorMethodID);
+		jmethodID hasNextID = env->GetMethodID(iteratorClass, "hasNext", "()Z");
+		jmethodID nextID = env->GetMethodID(iteratorClass, "next", "()Ljava/lang/Object;");
+		jmethodID getID = env->GetMethodID(mapClass, "get", "(Ljava/lang/Object;)Ljava/lang/Object;");
+		while (env->CallBooleanMethod(iterator, hasNextID)) {
+			jobject next = env->CallObjectMethod(iterator, nextID);
+			jstring key = jstring(next);
+			jstring value = jstring(env->CallObjectMethod(details, getID, next));
+			auto keyStr = env->GetStringUTFChars(key, nullptr);
+			auto keyLen = env->GetStringUTFLength(key);
+			auto valueStr = env->GetStringUTFChars(value, nullptr);
+			auto valueLen = env->GetStringUTFLength(value);
+			detailsMap.emplace(std::string(keyStr, keyLen), std::string(valueStr, valueLen));
+			env->ReleaseStringUTFChars(key, keyStr);
+			env->ReleaseStringUTFChars(value, valueStr);
+			env->DeleteLocalRef(key);
+			env->DeleteLocalRef(value);
+		}
+	}
+	auto f = onMainThread([severity, &detailsMap, msg]() -> Future<Void> {
+		TraceEvent evt(Severity(severity), msg);
+		for (const auto& p : detailsMap) {
+			evt.detail(p.first, p.second);
+		}
+		return Void();
+	});
+	f.blockUntilReady();
+	if (isCopy) {
+		env->ReleaseStringUTFChars(message, msg);
+	}
+}
+
+void sendVoid(JNIEnv* env, jobject self, jlong promisePtr) {
+	auto f = onMainThread([promisePtr]() -> Future<Void> {
+		Promise<Void>* p = reinterpret_cast<Promise<Void>*>(promisePtr);
+		p->send(Void());
+		delete p;
+		return Void();
+	});
+}
+
+void sendBool(JNIEnv* env, jobject self, jlong promisePtr, jboolean value) {
+	auto f = onMainThread([promisePtr, value]() -> Future<Void> {
+		Promise<bool>* p = reinterpret_cast<Promise<bool>*>(promisePtr);
+		p->send(value);
+		delete p;
+		return Void();
+	});
+}
+
+void setProcessID(JNIEnv* env, jobject self, jlong processID) {
+	if (g_network->isSimulated()) {
+		g_simulator.currentProcess = reinterpret_cast<ISimulator::ProcessInfo*>(processID);
+	}
+}
+
+struct JVMContext {
+	JNIEnv* env = nullptr;
+	JavaVM* jvm = nullptr;
+	// the JVM requires char* args
+	std::vector<char*> jvmArgs;
+	jclass workloadClass;
+	jclass throwableClass;
+	jclass fdbClass;
+	jobject fdbObject;
+	bool success = true;
+	std::unique_ptr<JNINativeMethod[]>  workloadMethods;
+	int numWorkloadMethods;
+	//  this is a bit ugly - but JNINativeMethod requires
+	// char*  not const char *
+	std::vector<char*> charArrays;
+	std::set<std::string> classPath;
+
+	void setWorkloadMethods(const std::initializer_list<std::tuple<StringRef, StringRef, void*>>& methods) {
+		charArrays.reserve(charArrays.size() + 2*methods.size());
+		numWorkloadMethods = methods.size();
+		workloadMethods.reset(new JNINativeMethod[numWorkloadMethods]);
+		int i = 0;
+		for (const auto& t : methods) {
+			auto& w = workloadMethods[i];
+			StringRef nameStr = std::get<0>(t);
+			StringRef sigStr = std::get<1>(t);
+			charArrays.push_back(new char[nameStr.size() + 1]);
+			char* name = charArrays.back();
+			charArrays.push_back(new char[sigStr.size() + 1]);
+			char* sig = charArrays.back();
+			memcpy(name, nameStr.begin(), nameStr.size());
+			memcpy(sig, sigStr.begin(), sigStr.size());
+			name[nameStr.size()] = '\0';
+			sig[sigStr.size()] = '\0';
+			w.name = name;
+			w.signature = sig;
+			w.fnPtr = std::get<2>(t);
+			TraceEvent("PreparedNativeMethod")
+				.detail("Name", w.name)
+				.detail("Signature", w.signature)
+				.detail("Ptr", reinterpret_cast<uintptr_t>(w.fnPtr));
+			++i;
+		}
+	}
+
+	template<class Args>
+	JVMContext(Args&& jvmArgs)
+		: jvmArgs(std::forward<Args>(jvmArgs))
+		, fdbClass(nullptr)
+		, fdbObject(nullptr) {
+		setWorkloadMethods({
+				{
+					std::make_tuple<StringRef, StringRef, void*>(
+						LiteralStringRef("log"),
+						LiteralStringRef("(ILjava/lang/String;Ljava/util/Map;)V"),
+						reinterpret_cast<void*>(&printTrace))
+				},
+				{
+					std::make_tuple<StringRef, StringRef, void*>(
+						LiteralStringRef("sendVoid"),
+						LiteralStringRef("(J)V"),
+						reinterpret_cast<void*>(&sendVoid))
+				},
+				{ std::make_tuple<StringRef, StringRef, void*>(
+						LiteralStringRef("sendBool"),
+						LiteralStringRef("(JZ)V"),
+						reinterpret_cast<void*>(&sendBool))
+				},
+				{ std::make_tuple<StringRef, StringRef, void*>(
+						LiteralStringRef("setProcessID"),
+						LiteralStringRef("(J)V"),
+						reinterpret_cast<void*>(&setProcessID))
+				}});
+		init();
+	}
+
+	~JVMContext() {
+		TraceEvent(SevDebug, "JVMContextDestruct");
+		flushTraceFileVoid();
+		if (jvm) {
+			if (fdbObject) {
+				env->DeleteGlobalRef(fdbObject);
+			}
+			jvm->DestroyJavaVM();
+		}
+		for (auto& arr : jvmArgs) {
+			delete[] arr;
+		}
+		for (auto& arr : charArrays) {
+			delete[] arr;
+		}
+		TraceEvent(SevDebug, "JVMContextDestructDone");
+		flushTraceFileVoid();
+	}
+
+	bool addToClassPath(const std::string& path) {
+		TraceEvent("TryAddToClassPath")
+			.detail("Path", "path");
+		flushTraceFileVoid();
+		if (!success) {
+			return false;
+		}
+		if (classPath.count(path) > 0) {
+			// already added
+			return true;
+		}
+		auto addFileMethod = env->GetStaticMethodID(workloadClass, "addFile", "(Ljava/lang/String;)V");
+		if (!checkException()) {
+			return false;
+		}
+		auto p = env->NewStringUTF(path.c_str());
+		env->CallStaticVoidMethod(workloadClass, addFileMethod, p);
+		if (!checkException()) {
+			return false;
+		}
+		classPath.insert(path);
+		return true;
+	}
+
+	bool checkException() {
+		auto flag = env->ExceptionCheck();
+		if (flag) {
+			jthrowable exception = env->ExceptionOccurred();
+			TraceEvent(SevError, "JavaException");
+			env->ExceptionDescribe();
+			env->ExceptionClear();
+			return false;
+		}
+		return true;
+	}
+
+	void initializeFDB() {
+		if (!success) {
+			return;
+		}
+		fdbClass = env->FindClass("com/apple/foundationdb/FDB");
+		jmethodID selectMethod = env->GetStaticMethodID(fdbClass, "selectAPIVersion", "(IZ)Lcom/apple/foundationdb/FDB;");
+		if (!checkException()) {
+			success = false;
+			return;
+		}
+		fdbObject = env->CallStaticObjectMethod(fdbClass, selectMethod, jint(610), jboolean(false));
+		if (!checkException()) {
+			success = false;
+			return;
+		}
+	}
+
+	void init() {
+		TraceEvent(SevDebug, "InitializeJVM");
+		flushTraceFileVoid();
+		JavaVMInitArgs args;
+		args.version = JNI_VERSION_1_6;
+		args.ignoreUnrecognized = JNI_TRUE;
+		args.nOptions = jvmArgs.size();
+		std::unique_ptr<JavaVMOption[]> options(new JavaVMOption[args.nOptions]);
+		for (int i = 0; i < args.nOptions; ++i) {
+			options[i].optionString = jvmArgs[i];
+			TraceEvent(SevDebug, "AddJVMOption")
+				.detail("Option", reinterpret_cast<const char*>(options[i].optionString));
+			flushTraceFileVoid();
+		}
+		args.options = options.get();
+		{
+			TraceEvent evt(SevDebug, "StartVM");
+			for (int i = 0; i < args.nOptions; ++i) {
+				evt.detail(format("Option-%d", i), reinterpret_cast<const char*>(options[i].optionString));
+			}
+		}
+		flushTraceFileVoid();
+		auto res = JNI_CreateJavaVM(&jvm, reinterpret_cast<void**>(&env), &args);
+		if (res == JNI_ERR) {
+			success = false;
+			env->ExceptionDescribe();
+			return;
+		}
+		TraceEvent(SevDebug, "JVMStarted");
+		flushTraceFileVoid();
+		throwableClass = env->FindClass("java/lang/Throwable");
+		workloadClass = env->FindClass("com/apple/foundationdb/testing/AbstractWorkload");
+		if (workloadClass == nullptr) {
+			success = false;
+			TraceEvent(SevError, "ClassNotFound")
+				.detail("ClassName", "com/apple/foundationdb/testing/AbstractWorkload");
+			return;
+		}
+		TraceEvent(SevDebug, "RegisterNatives")
+			.detail("ThrowableClass", format("%x", reinterpret_cast<uintptr_t>(throwableClass)))
+			.detail("WorkloadClass", format("%x", reinterpret_cast<uintptr_t>(workloadClass)))
+			.detail("NumMethods", numWorkloadMethods);
+		flushTraceFileVoid();
+		env->RegisterNatives(workloadClass, workloadMethods.get(), numWorkloadMethods);
+		success = checkException() && success;
+		initializeFDB();
+	}
+
+};
+
+struct JavaWorkload : TestWorkload {
+	static const std::string name;
+	// From https://docs.oracle.com/javase/8/docs/technotes/guides/jni/spec/invocation.html#unloading_the_vm
+	// > Creation of multiple VMs in a single process is not supported.
+	// This means, that we have to share the VM across workloads.
+	static std::weak_ptr<JVMContext> globalVM;
+	std::shared_ptr<JVMContext> vm;
+	std::vector<std::string> classPath;
+
+	std::string className;
+
+	bool success = true;
+	jclass implClass;
+	jobject impl = nullptr;
+
+	explicit JavaWorkload(WorkloadContext const& wcx) : TestWorkload(wcx) {
+		className = getOption(options, LiteralStringRef("workloadClass"), LiteralStringRef("")).toString();
+		if (className == "") {
+			success = false;
+			return;
+		}
+		auto jvmOptions = getOption(options, LiteralStringRef("jvmOptions"), std::vector<std::string>{});
+		classPath = getOption(options, LiteralStringRef("classPath"), std::vector<std::string>{});
+		vm = globalVM.lock();
+		if (!vm) {
+			std::vector<char*> args;
+			args.reserve(jvmOptions.size());
+			for (const auto& opt : jvmOptions) {
+				char* option = new char[opt.size() + 1];
+				option[opt.size()] = '\0';
+				std::copy(opt.begin(), opt.end(), option);
+				args.emplace_back(option);
+			}
+			vm = std::make_shared<JVMContext>(args);
+			globalVM = vm;
+			success = vm->success;
+		} else {
+			success = vm->success;
+		}
+		if (success) {
+			TraceEvent("JVMRunning");
+			flushTraceFileVoid();
+			try {
+				createContext();
+			} catch (Error& e) {
+				success = false;
+				TraceEvent(SevError, "JavaContextCreationFailed")
+					.error(e);
+			}
+		}
+		TraceEvent(SevDebug, "JavaWorkloadConstructed")
+			.detail("Success", success);
+		flushTraceFileVoid();
+	}
+
+	~JavaWorkload() {
+		if (vm && impl) {
+			try {
+				auto shutdownID = getMethodID(vm->workloadClass, "shutdown", "()V");
+				vm->env->CallVoidMethod(impl, shutdownID);
+				if (!checkException()) {
+					TraceEvent(SevError, "JavaWorkloadShutdownFailed")
+						.detail("Reason", "AbstractWorkload::shutdown call");
+				}
+			} catch (Error& e) {
+				TraceEvent(SevError, "JavaWorkloadShutdownFailed")
+						.detail("Reason", "Exception");
+			}
+		}
+		TraceEvent(SevDebug, "DestroyJavaWorkload");
+		flushTraceFileVoid();
+		if (vm && vm->env && impl)
+			vm->env->DeleteGlobalRef(impl);
+		TraceEvent(SevDebug, "DestroyJavaWorkloadComplete");
+		flushTraceFileVoid();
+	}
+
+	bool checkException() {
+		return vm->checkException();
+	}
+
+	void createContext() {
+		TraceEvent("AddClassPaths")
+			.detail("Num", classPath.size());
+		flushTraceFileVoid();
+		for (const auto& p : classPath) {
+			if (!vm->addToClassPath(p)) {
+				TraceEvent("AddToClassPathFailed")
+					.detail("Path", p);
+				success = false;
+				return;
+			}
+			TraceEvent("AddToClassPath")
+				.detail("Path", p);
+		}
+		std::transform(className.begin(), className.end(), className.begin(), [](char c) {
+			if (c == '.') return '/';
+			return c;
+		});
+		implClass = vm->env->FindClass(className.c_str());
+		if (implClass == nullptr) {
+			success = false;
+			TraceEvent(SevError, "JavaWorkloadNotFound").detail("JavaClass", className);
+			return;
+		}
+		if (!vm->env->IsAssignableFrom(implClass, vm->workloadClass)) {
+			success = false;
+			TraceEvent(SevError, "JClassNotAWorkload").detail("Class", className);
+			return;
+		}
+		jint initalCapacity = options.size() * 2;
+		jclass hashMapCls = vm->env->FindClass("java/util/HashMap");
+		if (hashMapCls == nullptr) {
+			success = false;
+			TraceEvent(SevError, "ClassNotFound")
+				.detail("ClassName", "java/util/HashMap");
+			return;
+		}
+		jmethodID put = vm->env->GetMethodID(hashMapCls, "put", "(Ljava/lang/Object;Ljava/lang/Object;)Ljava/lang/Object;");
+		if (put == nullptr) {
+			checkException();
+			TraceEvent(SevError, "JavaMethodNotFound")
+				.detail("Class", "java/util/HashMap")
+				.detail("MethodName", "put")
+				.detail("Signature", "(Ljava/lang/Object;Ljava/lang/Object)Ljava/lang/Object;");
+			success = false;
+			return;
+		}
+		jmethodID constr = vm->env->GetMethodID(hashMapCls, "<init>", "(I)V");
+		jobject hashMap = vm->env->NewObject(hashMapCls, constr, initalCapacity);
+		if (hashMap == nullptr || !checkException()) {
+			TraceEvent(SevError, "JavaConstructionFailed")
+				.detail("Class", "java/util/HashMap");
+			success = false;
+			return;
+		}
+		for (auto& kv : options) {
+			auto key = vm->env->NewStringUTF(reinterpret_cast<const char*>(kv.key.begin()));
+			auto value = vm->env->NewStringUTF(reinterpret_cast<const char*>(kv.value.begin()));
+			vm->env->CallVoidMethod(hashMap, put, key, value);
+			vm->env->DeleteLocalRef(key);
+			vm->env->DeleteLocalRef(value);
+			kv.value = LiteralStringRef("");
+		}
+		auto workloadContextClass = findClass("com/apple/foundationdb/testing/WorkloadContext");
+		auto workloadContextConstructor = getMethodID(workloadContextClass, "<init>", "(Ljava/util/Map;IIJJ)V");
+		jlong processID = 0;
+		if (g_network->isSimulated()) {
+			processID = reinterpret_cast<jlong>(g_simulator.getCurrentProcess());
+		}
+		TraceEvent(SevDebug, "WorkloadContextConstructorFound")
+			.detail("FieldID", format("%x", reinterpret_cast<uintptr_t>(workloadContextConstructor)));
+		flushTraceFileVoid();
+		auto workloadContext = vm->env->NewObject(workloadContextClass, workloadContextConstructor, hashMap,
+												  jint(clientId), jint(clientCount), jlong(sharedRandomNumber),
+												  processID);
+		if (!checkException() || workloadContext == nullptr) {
+			success = false;
+			TraceEvent(SevError, "CouldNotCreateWorkloadContext");
+		}
+		TraceEvent(SevDebug, "WorkloadContextConstructed")
+			.detail("Object", format("%x", reinterpret_cast<uintptr_t>(workloadContext)));
+		flushTraceFileVoid();
+		auto implConstr = vm->env->GetMethodID(implClass, "<init>", "(Lcom/apple/foundationdb/testing/WorkloadContext;)V");
+		if (!checkException() || implConstr == nullptr) {
+			success = false;
+			TraceEvent(SevError, "JavaWorkloadNotDefaultConstructible").detail("Class", className);
+			return;
+		}
+		impl = vm->env->NewObject(implClass, implConstr, workloadContext);
+		if (!checkException() || impl == nullptr) {
+			success = false;
+			TraceEvent(SevError, "JavaWorkloadConstructionFailed").detail("Class", className);
+			return;
+		}
+		vm->env->NewGlobalRef(impl);
+	}
+
+	std::string description() override { return JavaWorkload::name; }
+
+	jclass findClass(const char* className) {
+		jclass res = vm->env->FindClass(className);
+		if (res == nullptr) {
+			checkException();
+			success = false;
+			TraceEvent(SevError, "ClassNotFound")
+				.detail("ClassName", className);
+			throw internal_error();
+		}
+		return res;
+	}
+
+	jmethodID getMethodID(jclass clazz, const char* name, const char* sig) {
+		auto res = vm->env->GetMethodID(clazz, name, sig);
+		if (!checkException() || res == nullptr) {
+			success = false;
+			TraceEvent(SevError, "JavaMethodNotFound")
+				.detail("Name", name)
+				.detail("Signature", sig);
+			throw internal_error();
+		}
+		return res;
+	}
+
+	jfieldID getStaticFieldID(jclass clazz, const char* name, const char* signature) {
+		auto res = vm->env->GetStaticFieldID(clazz, name, signature);
+		if (!checkException() || res == nullptr) {
+			success = false;
+			TraceEvent(SevError, "FieldNotFound")
+				.detail("FieldName", name)
+				.detail("Signature", signature);
+			throw internal_error();
+		}
+		return res;
+	}
+
+	jobject getStaticObjectField(jclass clazz, jfieldID field) {
+		auto res = vm->env->GetStaticObjectField(clazz, field);
+		if (!checkException() || res != nullptr) {
+			success = false;
+			TraceEvent(SevError, "CouldNotGetStaticObjectField");
+			throw operation_failed();
+		}
+		return res;
+	}
+
+	template<class Ret>
+	Future<Ret> callJava(Database const& db, const char* method, Ret failed) {
+		TraceEvent(SevDebug, "CallJava")
+			.detail("Method", method);
+		flushTraceFileVoid();
+		try {
+			auto cx = db.getPtr();
+			cx->addref();
+			// First we need an executor for the Database class
+			jmethodID executorMethod = getMethodID(vm->workloadClass, "getExecutor", "()Ljava/util/concurrent/Executor;");
+			jobject executor = vm->env->CallObjectMethod(impl, executorMethod);
+			if (!checkException()) {
+				success = false;
+				return failed;
+			}
+			if (executor == nullptr) {
+				TraceEvent(SevError, "JavaExecutorIsVoid");
+				success = false;
+				return failed;
+			}
+			Reference<IDatabase> database(new ThreadSafeDatabase(cx));
+			jlong databasePtr = reinterpret_cast<jlong>(database.extractPtr());
+			jclass databaseClass = findClass("com/apple/foundationdb/FDBDatabase");
+
+			// now we can create the Java Database object
+			auto sig =  "(JLjava/util/concurrent/Executor;)V";
+			jmethodID databaseConstructor = getMethodID(databaseClass, "<init>", sig);
+			jobject javaDatabase = vm->env->NewObject(databaseClass, databaseConstructor, databasePtr, executor);
+			if (!checkException() || javaDatabase == nullptr) {
+				TraceEvent(SevError, "ConstructingDatabaseFailed")
+					.detail("ConstructirSignature", sig);
+				success = false;
+				return failed;
+			}
+
+			auto p = new Promise<Ret>();
+			jmethodID methodID = getMethodID(vm->workloadClass, method, "(Lcom/apple/foundationdb/Database;J)V");
+			vm->env->CallVoidMethod(impl, methodID, javaDatabase, reinterpret_cast<jlong>(p));
+			checkException();
+			if (!checkException() || !success) {
+				delete p;
+				return failed;
+			}
+			return p->getFuture();
+			// and now we can call the method with the created Database
+		} catch (Error& e) {
+			TraceEvent("CallJavaFailed")
+				.error(e);
+			success = false;
+			return failed;
+		}
+	}
+
+	Future<Void> setup(Database const& cx) override {
+		if (!success) {
+			return Void();
+		}
+		return callJava<Void>(cx, "setup", Void());
+	}
+	Future<Void> start(Database const& cx) override {
+		if (!success) {
+			return Void();
+		}
+		return callJava<Void>(cx, "start", Void());
+	}
+	Future<bool> check(Database const& cx) override {
+		if (!success) {
+			return false;
+		}
+		return callJava<bool>(cx, "check", false);
+	}
+	void getMetrics(vector<PerfMetric>& m) override {
+		if (!success) {
+			return;
+		}
+	}
+
+	virtual double getCheckTimeout() {
+		if (!success) {
+			return 3000;
+		}
+		jmethodID methodID = vm->env->GetMethodID(implClass, "getCheckTimeout", "()D");
+		jdouble res = vm->env->CallDoubleMethod(impl, methodID);
+		checkException();
+		if (!success) {
+			return 3000;
+		}
+		return res;
+	}
+};
+
+const std::string JavaWorkload::name = "JavaWorkload";
+std::weak_ptr<JVMContext> JavaWorkload::globalVM;
+} // namespace
+
+WorkloadFactory<JavaWorkload> JavaWorkloadFactory(JavaWorkload::name.c_str());