From 6ee14bbb934ab06c82a9484d1e12ef262faef037 Mon Sep 17 00:00:00 2001 From: Alec Grieser Date: Fri, 2 Mar 2018 16:58:51 -0800 Subject: [PATCH] expose second versionstamp value type through vexillographer and add support in bindings and bindingtester --- bindings/bindingtester/tests/api.py | 76 +++++++++++++----- bindings/flow/tester/Tester.actor.cpp | 1 + .../apple/foundationdb/subspace/Subspace.java | 35 ++++++++- .../com/apple/foundationdb/tuple/Tuple.java | 77 +++++++++++++++++-- .../apple/foundationdb/tuple/TupleUtil.java | 8 +- .../foundationdb/test/AsyncStackTester.java | 9 ++- .../foundationdb/test/StackOperation.java | 3 +- .../apple/foundationdb/test/StackTester.java | 9 ++- .../test/VersionstampSmokeTest.java | 2 +- bindings/python/fdb/tuple.py | 23 ++++-- bindings/python/tests/tester.py | 20 ++++- fdbclient/vexillographer/fdb.options | 7 +- 12 files changed, 226 insertions(+), 44 deletions(-) diff --git a/bindings/bindingtester/tests/api.py b/bindings/bindingtester/tests/api.py index e31b8636fb..32cfee6324 100644 --- a/bindings/bindingtester/tests/api.py +++ b/bindings/bindingtester/tests/api.py @@ -40,6 +40,7 @@ class ApiTest(Test): self.stack_subspace = self.subspace['stack'] self.versionstamped_values = self.scratch['versionstamped_values'] + self.versionstamped_values_2 = self.scratch['versionstamped_values_2'] self.versionstamped_keys = self.scratch['versionstamped_keys'] def setup(self, args): @@ -150,7 +151,7 @@ class ApiTest(Test): snapshot_versions = ['GET_READ_VERSION_SNAPSHOT'] tuples = ['TUPLE_PACK', 'TUPLE_UNPACK', 'TUPLE_RANGE', 'TUPLE_SORT', 'SUB', 'ENCODE_FLOAT', 'ENCODE_DOUBLE', 'DECODE_DOUBLE', 'DECODE_FLOAT'] if 'versionstamp' in args.types: - tuples.append('TUPLE_PACK_WITH_VERSIONSTAMP') + tuples += ['TUPLE_PACK_VERSIONSTAMPED_KEY', 'TUPLE_PACK_VERSIONSTAMPED_VALUE'] resets = ['ON_ERROR', 'RESET', 'CANCEL'] read_conflicts = ['READ_CONFLICT_RANGE', 'READ_CONFLICT_KEY'] write_conflicts = ['WRITE_CONFLICT_RANGE', 'WRITE_CONFLICT_KEY', 'DISABLE_WRITE_CONFLICT'] @@ -349,17 +350,27 @@ class ApiTest(Test): elif op == 'VERSIONSTAMP': rand_str1 = self.random.random_string(100) key1 = self.versionstamped_values.pack((rand_str1,)) + key2 = self.versionstamped_values_2.pack((rand_str1,)) split = random.randint(0, 70) - rand_str2 = self.random.random_string(20 + split) + fdb.tuple.Versionstamp._UNSET_TR_VERSION + self.random.random_string(70 - split) - key2 = self.versionstamped_keys.pack() + rand_str2 - index = key2.find(fdb.tuple.Versionstamp._UNSET_TR_VERSION) - key2 += chr(index % 256) + chr(index / 256) + prefix = self.random.random_string(20 + split) + if prefix.endswith('\xff'): + # Necessary to make sure that the SET_VERSIONSTAMPED_VALUE check + # correctly finds where the version is supposed to fit in. + prefix += '\x00' + suffix = self.random.random_string(70 - split) + rand_str2 = prefix + fdb.tuple.Versionstamp._UNSET_TR_VERSION + suffix + key3 = self.versionstamped_keys.pack() + rand_str2 + index = len(self.versionstamped_keys.pack()) + len(prefix) + key3 += chr(index % 256) + chr(index / 256) instructions.push_args(u'SET_VERSIONSTAMPED_VALUE', key1, fdb.tuple.Versionstamp._UNSET_TR_VERSION + rand_str2) instructions.append('ATOMIC_OP') - instructions.push_args(u'SET_VERSIONSTAMPED_KEY', key2, rand_str1) + instructions.push_args(u'SET_VERSIONSTAMPED_VALUE_POS', key2, rand_str2 + struct.pack('= 0 and second_incomplete < 0: rand_str = self.random.random_string(100) - instructions.push_args(rand_str) - test_util.to_front(instructions, 1) - instructions.push_args(u'SET_VERSIONSTAMPED_KEY') - instructions.append('ATOMIC_OP') + if op == 'TUPLE_PACK_VERSIONSTAMPED_KEY': + instructions.push_args(rand_str) + test_util.to_front(instructions, 1) + instructions.push_args(u'SET_VERSIONSTAMPED_KEY') + instructions.append('ATOMIC_OP') + + version_value_key_2 = self.versionstamped_values_2.pack((rand_str,)) + versionstamped_value = fdb.tuple.pack(tup) + struct.pack(', Iterable { return TupleUtil.pack(elements, prefix); } + /** + * Deprecated. Use {@link #packVersionstampedKey() packVersionstampedKey()} instead. Because + * versionstamped keys and versionstamped values are serialized differently, this method was deprecated + * to encourage developers to choose appropriately between + * {@link #packVersionstampedKey() packVersionstampedKey()} and + * {@link #packVersionstampedValue() packVersionstampedValue}. + * + * @return a serialized representation of this {@code Tuple} for use with versionstamp ops. + * @throws IllegalArgumentException if there is not exactly one incomplete {@link Versionstamp} included in this {@code Tuple} + * @deprecated Since 5.2.0, replaced with {@link #packVersionstampedKey() packVersionstampedKey()} + */ + @Deprecated + public byte[] packWithVersionstamp() { + return packWithVersionstamp(null); + } + + /** + * Deprecated. Use {@link #packVersionstampedKey(byte[]) packVersionstampedKey()} instead. Because + * versionstamped keys and versionstamped values are serialized differently, this method was deprecated + * to encourage developers to choose appropriately between + * {@link #packVersionstampedKey(byte[]) packVersionstampedKey()} and + * {@link #packVersionstampedValue(byte[]) packVersionstampedValue}. + * + * @param prefix additional byte-array prefix to prepend to serialized bytes. + * @return a serialized representation of this {@code Tuple} for use with versionstamp ops. + * @throws IllegalArgumentException if there is not exactly one incomplete {@link Versionstamp} included in this {@code Tuple} + * @deprecated Since 5.2.0, replaced with {@link #packVersionstampedValue(byte[]) packVersionstampedKey()} + */ + @Deprecated + public byte[] packWithVersionstamp(byte[] prefix) { + return packVersionstampedKey(prefix); + } + /** * Get an encoded representation of this {@code Tuple} for use with * {@link com.apple.foundationdb.MutationType#SET_VERSIONSTAMPED_KEY MutationType.SET_VERSIONSTAMPED_KEY}. - * This works the same as the {@link #packWithVersionstamp(byte[]) one-paramter version of this method}, + * This works the same as the {@link #packVersionstampedKey(byte[]) one-paramter version of this method}, * but it does not add any prefix to the array. * * @return a serialized representation of this {@code Tuple} for use with versionstamp ops. * @throws IllegalArgumentException if there is not exactly one incomplete {@link Versionstamp} included in this {@code Tuple} */ - public byte[] packWithVersionstamp() { - return packWithVersionstamp(null); + public byte[] packVersionstampedKey() { + return packVersionstampedKey(null); } /** @@ -332,8 +365,42 @@ public class Tuple implements Comparable, Iterable { * @return a serialized representation of this {@code Tuple} for use with versionstamp ops. * @throws IllegalArgumentException if there is not exactly one incomplete {@link Versionstamp} included in this {@code Tuple} */ - public byte[] packWithVersionstamp(byte[] prefix) { - return TupleUtil.packWithVersionstamp(elements, prefix); + public byte[] packVersionstampedKey(byte[] prefix) { + return TupleUtil.packWithVersionstamp(elements, prefix, false); + } + + /** + * Get an encoded representation of this {@code Tuple} for use with + * {@link com.apple.foundationdb.MutationType#SET_VERSIONSTAMPED_VALUE_POS MutationType.SET_VERSIONSTAMPED_VALUE_POS}. + * This works the same as the {@link #packVersionstampedValue(byte[]) one-paramter version of this method}, + * but it does not add any prefix to the array. + * + * @return a serialized representation of this {@code Tuple} for use with versionstamp ops. + * @throws IllegalArgumentException if there is not exactly one incomplete {@link Versionstamp} included in this {@code Tuple} + */ + public byte[] packVersionstampedValue() { + return packVersionstampedValue(null); + } + + /** + * Get an encoded representation of this {@code Tuple} for use with + * {@link com.apple.foundationdb.MutationType#SET_VERSIONSTAMPED_VALUE_POS MutationType.SET_VERSIONSTAMPED_VALUE_POS}. + * There must be exactly one incomplete {@link Versionstamp} instance within this + * {@code Tuple} or this will throw an {@link IllegalArgumentException}. + * Each element is encoded to {@code byte}s and concatenated, the prefix + * is then prepended to the array, and then the index of the serialized incomplete + * {@link Versionstamp} is appended as a little-endian integer. This can then be passed + * as the value to + * {@link com.apple.foundationdb.Transaction#mutate(com.apple.foundationdb.MutationType, byte[], byte[]) Transaction.mutate()} + * with the {@code SET_VERSIONSTAMPED_VALUE_POS} {@link com.apple.foundationdb.MutationType}, and the transaction's + * version will then be filled in at commit time. + * + * @param prefix additional byte-array prefix to prepend to serialized bytes. + * @return a serialized representation of this {@code Tuple} for use with versionstamp ops. + * @throws IllegalArgumentException if there is not exactly one incomplete {@link Versionstamp} included in this {@code Tuple} + */ + public byte[] packVersionstampedValue(byte[] prefix) { + return TupleUtil.packWithVersionstamp(elements, prefix, true); } /** diff --git a/bindings/java/src/main/com/apple/foundationdb/tuple/TupleUtil.java b/bindings/java/src/main/com/apple/foundationdb/tuple/TupleUtil.java index 73291ed784..3316044dd1 100644 --- a/bindings/java/src/main/com/apple/foundationdb/tuple/TupleUtil.java +++ b/bindings/java/src/main/com/apple/foundationdb/tuple/TupleUtil.java @@ -578,7 +578,7 @@ class TupleUtil { } } - static byte[] packWithVersionstamp(List items, byte[] prefix) { + static byte[] packWithVersionstamp(List items, byte[] prefix, boolean forValue) { List encoded = new ArrayList(2 * items.size() + (prefix == null ? 1 : 2)); EncodeResult result = encodeAll(items, prefix, encoded); if(result.versionPos < 0) { @@ -587,7 +587,11 @@ class TupleUtil { if(result.versionPos > 0xffff) { throw new IllegalArgumentException("Tuple has incomplete version at position " + result.versionPos + " which is greater than the maximum " + 0xffff); } - encoded.add(ByteBuffer.allocate(2).order(ByteOrder.LITTLE_ENDIAN).putShort((short)result.versionPos).array()); + if (forValue) { + encoded.add(ByteBuffer.allocate(4).order(ByteOrder.LITTLE_ENDIAN).putInt(result.versionPos).array()); + } else { + encoded.add(ByteBuffer.allocate(2).order(ByteOrder.LITTLE_ENDIAN).putShort((short)result.versionPos).array()); + } return ByteArrayUtil.join(null, encoded); } } diff --git a/bindings/java/src/test/com/apple/foundationdb/test/AsyncStackTester.java b/bindings/java/src/test/com/apple/foundationdb/test/AsyncStackTester.java index a81df36a73..990791c668 100644 --- a/bindings/java/src/test/com/apple/foundationdb/test/AsyncStackTester.java +++ b/bindings/java/src/test/com/apple/foundationdb/test/AsyncStackTester.java @@ -360,7 +360,7 @@ public class AsyncStackTester { }, FDB.DEFAULT_EXECUTOR); }, FDB.DEFAULT_EXECUTOR); } - else if(op == StackOperation.TUPLE_PACK_WITH_VERSIONSTAMP) { + else if(op == StackOperation.TUPLE_PACK_VERSIONSTAMPED_KEY || op == StackOperation.TUPLE_PACK_VERSIONSTAMPED_VALUE) { return inst.popParams(2).thenComposeAsync(params -> { byte[] prefix = (byte[])params.get(0); int tupleSize = StackUtils.getInt(params.get(1)); @@ -372,7 +372,12 @@ public class AsyncStackTester { return; } try { - byte[] coded = tuple.packWithVersionstamp(prefix); + byte[] coded; + if(op == StackOperation.TUPLE_PACK_VERSIONSTAMPED_KEY) { + coded = tuple.packVersionstampedKey(prefix); + } else { + coded = tuple.packVersionstampedValue(prefix); + } inst.push("OK".getBytes()); inst.push(coded); } catch(IllegalArgumentException e) { diff --git a/bindings/java/src/test/com/apple/foundationdb/test/StackOperation.java b/bindings/java/src/test/com/apple/foundationdb/test/StackOperation.java index 7fdc7b9ff5..894fa3d4f2 100644 --- a/bindings/java/src/test/com/apple/foundationdb/test/StackOperation.java +++ b/bindings/java/src/test/com/apple/foundationdb/test/StackOperation.java @@ -60,7 +60,8 @@ enum StackOperation { SUB, CONCAT, TUPLE_PACK, - TUPLE_PACK_WITH_VERSIONSTAMP, + TUPLE_PACK_VERSIONSTAMPED_KEY, + TUPLE_PACK_VERSIONSTAMPED_VALUE, TUPLE_UNPACK, TUPLE_RANGE, TUPLE_SORT, diff --git a/bindings/java/src/test/com/apple/foundationdb/test/StackTester.java b/bindings/java/src/test/com/apple/foundationdb/test/StackTester.java index 2367c1fdc0..9374c0d4d0 100644 --- a/bindings/java/src/test/com/apple/foundationdb/test/StackTester.java +++ b/bindings/java/src/test/com/apple/foundationdb/test/StackTester.java @@ -321,7 +321,7 @@ public class StackTester { //System.out.println(inst.context.preStr + " - " + " -> result '" + ByteArrayUtil.printable(coded) + "'"); inst.push(coded); } - else if(op == StackOperation.TUPLE_PACK_WITH_VERSIONSTAMP) { + else if(op == StackOperation.TUPLE_PACK_VERSIONSTAMPED_KEY || op == StackOperation.TUPLE_PACK_VERSIONSTAMPED_VALUE) { List params = inst.popParams(2).join(); byte[] prefix = (byte[])params.get(0); int tupleSize = StackUtils.getInt(params.get(1)); @@ -331,7 +331,12 @@ public class StackTester { inst.push("ERROR: NONE".getBytes()); } else { try { - byte[] coded = tuple.packWithVersionstamp(prefix); + byte[] coded; + if(op == StackOperation.TUPLE_PACK_VERSIONSTAMPED_KEY) { + coded = tuple.packVersionstampedKey(prefix); + } else { + coded = tuple.packVersionstampedValue(prefix); + } inst.push("OK".getBytes()); inst.push(coded); } catch (IllegalArgumentException e) { diff --git a/bindings/java/src/test/com/apple/foundationdb/test/VersionstampSmokeTest.java b/bindings/java/src/test/com/apple/foundationdb/test/VersionstampSmokeTest.java index 556e92085e..9c6358d375 100644 --- a/bindings/java/src/test/com/apple/foundationdb/test/VersionstampSmokeTest.java +++ b/bindings/java/src/test/com/apple/foundationdb/test/VersionstampSmokeTest.java @@ -42,7 +42,7 @@ public class VersionstampSmokeTest { CompletableFuture trVersionFuture = db.run((Transaction tr) -> { // The incomplete Versionstamp will have tr's version information when committed. Tuple t = Tuple.from("prefix", Versionstamp.incomplete()); - tr.mutate(MutationType.SET_VERSIONSTAMPED_KEY, t.packWithVersionstamp(), new byte[0]); + tr.mutate(MutationType.SET_VERSIONSTAMPED_KEY, t.packVersionstampedKey(), new byte[0]); return tr.getVersionstamp(); }); diff --git a/bindings/python/fdb/tuple.py b/bindings/python/fdb/tuple.py index 766e659085..52237f6acc 100644 --- a/bindings/python/fdb/tuple.py +++ b/bindings/python/fdb/tuple.py @@ -374,7 +374,7 @@ def _encode(value, nested=False): # * if there is exactly one incomplete versionstamp member, it returns the tuple with the # two extra version bytes and the position of the version start # * if there is more than one incomplete versionstamp member, it throws an error -def _pack_maybe_with_versionstamp(t, prefix=None): +def _pack_maybe_with_versionstamp(t, prefix=None, for_value=False): if not isinstance(t, tuple): raise Exception("fdbtuple pack() expects a tuple, got a " + str(type(t))) @@ -384,7 +384,10 @@ def _pack_maybe_with_versionstamp(t, prefix=None): if version_pos >= 0: version_pos += len(prefix) if prefix is not None else 0 bytes_list.extend(child_bytes) - bytes_list.append(struct.pack('