apple/foundationdb
1
0
Fork 0
mirror of https://github.com/apple/foundationdb.git synced 2025-05-14 01:42:37 +08:00
Code Issues Packages Projects Releases Wiki Activity

Add versionstamp support in tuple (#7293)

Browse Source 
Tuple in C++ needs to support Versionstamp.
This commit is contained in:
Hao Fu 2022-06-02 17:44:10 -07:00 committed by GitHub
parent c886a6efe4
commit e7fa8e9f6f
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
11 changed files with 241 additions and 3 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

71
bindings/c/test/unit/unit_tests.cpp
View File

@ -992,6 +992,73 @@ GetMappedRangeResult getMappedIndexEntries(int beginId,
return getMappedIndexEntries(beginId, endId, tr, mapper, matchIndex);
}
TEST_CASE("versionstamp_unit_test") {
// a random 12 bytes long StringRef as a versionstamp
StringRef str = "\x01\x02\x03\x04\x05\x06\x07\x08\x09\x10\x11\x12"_sr;
Versionstamp vs(str), vs2(str);
ASSERT(vs == vs2);
ASSERT(vs.begin() != vs2.begin());
int64_t version = vs.getVersion();
int64_t version2 = vs2.getVersion();
int64_t versionExpected = ((int64_t)0x01 << 56) + ((int64_t)0x02 << 48) + ((int64_t)0x03 << 40) +
((int64_t)0x04 << 32) + (0x05 << 24) + (0x06 << 16) + (0x07 << 8) + 0x08;
ASSERT(version == versionExpected);
ASSERT(version2 == versionExpected);
int16_t batch = vs.getBatchNumber();
int16_t batch2 = vs2.getBatchNumber();
int16_t batchExpected = (0x09 << 8) + 0x10;
ASSERT(batch == batchExpected);
ASSERT(batch2 == batchExpected);
int16_t user = vs.getUserVersion();
int16_t user2 = vs2.getUserVersion();
int16_t userExpected = (0x11 << 8) + 0x12;
ASSERT(user == userExpected);
ASSERT(user2 == userExpected);
ASSERT(vs.size() == VERSIONSTAMP_TUPLE_SIZE);
ASSERT(vs2.size() == VERSIONSTAMP_TUPLE_SIZE);
}
TEST_CASE("tuple_support_versionstamp") {
// a random 12 bytes long StringRef as a versionstamp
StringRef str = "\x01\x02\x03\x04\x05\x06\x07\x08\x09\x10\x11\x12"_sr;
Versionstamp vs(str);
const Tuple t = Tuple().append(prefix).append(RECORD).appendVersionstamp(vs).append("{K[3]}"_sr).append("{...}"_sr);
ASSERT(t.getVersionstamp(2) == vs);
// verify the round-way pack-unpack path for a Tuple containing a versionstamp
StringRef result1 = t.pack();
Tuple t2 = Tuple::unpack(result1);
StringRef result2 = t2.pack();
ASSERT(t2.getVersionstamp(2) == vs);
ASSERT(result1.toString() == result2.toString());
}
TEST_CASE("tuple_fail_to_append_truncated_versionstamp") {
// a truncated 11 bytes long StringRef as a versionstamp
StringRef str = "\x01\x02\x03\x04\x05\x06\x07\x08\x09\x10\x11"_sr;
try {
Versionstamp truncatedVersionstamp(str);
} catch (Error& e) {
return;
}
UNREACHABLE();
}
TEST_CASE("tuple_fail_to_append_longer_versionstamp") {
// a longer than expected 13 bytes long StringRef as a versionstamp
StringRef str = "\x01\x02\x03\x04\x05\x06\x07\x08\x09\x10\x11"_sr;
try {
Versionstamp longerVersionstamp(str);
} catch (Error& e) {
return;
}
UNREACHABLE();
}
TEST_CASE("fdb_transaction_get_mapped_range") {
const int TOTAL_RECORDS = 20;
fillInRecords(TOTAL_RECORDS);
@ -1153,11 +1220,13 @@ void assertNotTuple(std::string str) {
TEST_CASE("fdb_transaction_get_mapped_range_fail_on_mapper_not_tuple") {
// A string that cannot be parsed as tuple.
// "\x15:\x152\x15E\x15\x09\x15\x02\x02MySimpleRecord$repeater-version\x00\x15\x013\x00\x00\x00\x00\x1aU\x90\xba\x00\x00\x00\x02\x15\x04"
// should fail at \x35
std::string mapper = {
'\x15', ':', '\x15', '2', '\x15', 'E', '\x15', '\t', '\x15', '\x02', '\x02', 'M',
'y', 'S', 'i', 'm', 'p', 'l', 'e', 'R', 'e', 'c', 'o', 'r',
'd', '$', 'r', 'e', 'p', 'e', 'a', 't', 'e', 'r', '-', 'v',
'e', 'r', 's', 'i', 'o', 'n', '\x00', '\x15', '\x01', '3', '\x00', '\x00',
'e', 'r', 's', 'i', 'o', 'n', '\x00', '\x15', '\x01', '\x35', '\x00', '\x00',
'\x00', '\x00', '\x1a', 'U', '\x90', '\xba', '\x00', '\x00', '\x00', '\x02', '\x15', '\x04'
};
assertNotTuple(mapper);

32
bindings/flow/Tuple.cpp
View File

@ -26,6 +26,7 @@ static_assert(std::numeric_limits<float>::is_iec559);
static_assert(std::numeric_limits<double>::is_iec559);
const size_t Uuid::SIZE = 16;
const size_t VERSIONSTAMP_TUPLE_SIZE = 12;
const uint8_t Tuple::NULL_CODE = 0x00;
const uint8_t Tuple::BYTES_CODE = 0x01;
@ -39,6 +40,7 @@ const uint8_t Tuple::DOUBLE_CODE = 0x21;
const uint8_t Tuple::FALSE_CODE = 0x26;
const uint8_t Tuple::TRUE_CODE = 0x27;
const uint8_t Tuple::UUID_CODE = 0x30;
const uint8_t Tuple::VERSIONSTAMP_96_CODE = 0x33;
static float bigEndianFloat(float orig) {
int32_t big = *(int32_t*)&orig;
@ -117,6 +119,8 @@ Tuple::Tuple(StringRef const& str) {
} else if (data[i] == NESTED_CODE) {
i += 1;
depth += 1;
} else if (data[i] == VERSIONSTAMP_96_CODE) {
i += 1 + VERSIONSTAMP_TUPLE_SIZE;
} else {
throw invalid_tuple_data_type();
}
@ -170,6 +174,15 @@ Tuple& Tuple::append(StringRef const& str, bool utf8) {
return *this;
}
Tuple& Tuple::appendVersionstamp(Versionstamp const& vs) {
offsets.push_back(data.size());
data.push_back(data.arena(), VERSIONSTAMP_96_CODE);
data.append(data.arena(), vs.begin(), vs.size());
return *this;
}
Tuple& Tuple::append(int32_t value) {
return append((int64_t)value);
}
@ -293,6 +306,8 @@ Tuple::ElementType Tuple::getType(size_t index) const {
return ElementType::BOOL;
} else if (code == UUID_CODE) {
return ElementType::UUID;
} else if (code == VERSIONSTAMP_96_CODE) {
return ElementType::VERSIONSTAMP;
} else {
throw invalid_tuple_data_type();
}
@ -425,6 +440,19 @@ double Tuple::getDouble(size_t index) const {
return bigEndianDouble(swap);
}
Versionstamp Tuple::getVersionstamp(size_t index) const {
if (index >= offsets.size()) {
throw invalid_tuple_index();
}
ASSERT_LT(offsets[index], data.size());
uint8_t code = data[offsets[index]];
if (code != VERSIONSTAMP_96_CODE) {
throw invalid_tuple_data_type();
}
size_t versionstampLength = VERSIONSTAMP_TUPLE_SIZE;
return Versionstamp(StringRef(data.begin() + offsets[index] + 1, versionstampLength));
}
Uuid Tuple::getUuid(size_t index) const {
if (index >= offsets.size()) {
throw invalid_tuple_index();
@ -507,6 +535,10 @@ Tuple Tuple::getNested(size_t index) const {
ASSERT_LE(i + 1 + sizeof(double), next_offset - 1);
dest.append(dest.arena(), data.begin() + i + 1, sizeof(double));
i += sizeof(double) + 1;
} else if (code == VERSIONSTAMP_96_CODE) {
ASSERT_LE(i + 1 + VERSIONSTAMP_TUPLE_SIZE, next_offset - 1);
dest.append(dest.arena(), data.begin() + i + 1, VERSIONSTAMP_TUPLE_SIZE);
i += VERSIONSTAMP_TUPLE_SIZE + 1;
} else if (code == NESTED_CODE) {
i += 1;
depth += 1;

6
bindings/flow/Tuple.h
View File

@ -24,6 +24,7 @@
#pragma once
#include "bindings/flow/fdb_flow.h"
#include "fdbclient/Versionstamp.h"
namespace FDB {
struct Uuid {
@ -60,6 +61,7 @@ struct Tuple {
Tuple& append(Uuid);
Tuple& appendNested(Tuple const&);
Tuple& appendNull();
Tuple& appendVersionstamp(Versionstamp const&);
StringRef pack() const { return StringRef(data.begin(), data.size()); }
@ -68,13 +70,14 @@ struct Tuple {
return append(t);
}
enum ElementType { NULL_TYPE, INT, BYTES, UTF8, BOOL, FLOAT, DOUBLE, UUID, NESTED };
enum ElementType { NULL_TYPE, INT, BYTES, UTF8, BOOL, FLOAT, DOUBLE, UUID, NESTED, VERSIONSTAMP };
// this is number of elements, not length of data
size_t size() const { return offsets.size(); }
ElementType getType(size_t index) const;
Standalone<StringRef> getString(size_t index) const;
Versionstamp getVersionstamp(size_t index) const;
int64_t getInt(size_t index) const;
bool getBool(size_t index) const;
float getFloat(size_t index) const;
@ -107,6 +110,7 @@ private:
static const uint8_t FALSE_CODE;
static const uint8_t TRUE_CODE;
static const uint8_t UUID_CODE;
static const uint8_t VERSIONSTAMP_96_CODE;
Tuple(const StringRef& data);
Tuple(Standalone<VectorRef<uint8_t>> data, std::vector<size_t> offsets);

10
bindings/flow/tester/Tester.actor.cpp
View File

@ -112,6 +112,12 @@ std::string tupleToString(Tuple const& tuple) {
str += format("%016llx%016llx", *(uint64_t*)u.getData().begin(), *(uint64_t*)(u.getData().begin() + 8));
} else if (type == Tuple::NESTED) {
str += tupleToString(tuple.getNested(i));
} else if (type == Tuple::VERSIONSTAMP) {
Versionstamp versionstamp = tuple.getVersionstamp(i);
str += format("Transaction Version: '%ld', BatchNumber: '%hd', UserVersion : '%hd'",
versionstamp.getVersion(),
versionstamp.getBatchNumber(),
versionstamp.getUserVersion());
} else {
ASSERT(false);
}
@ -1147,6 +1153,8 @@ struct TuplePackFunc : InstructionFunc {
tuple << itemTuple.getUuid(0);
} else if (type == Tuple::NESTED) {
tuple.appendNested(itemTuple.getNested(0));
} else if (type == Tuple::VERSIONSTAMP) {
tuple.appendVersionstamp(Versionstamp(itemTuple.getString(0)));
} else {
ASSERT(false);
}
@ -1226,6 +1234,8 @@ struct TupleRangeFunc : InstructionFunc {
tuple << itemTuple.getUuid(0);
} else if (type == Tuple::NESTED) {
tuple.appendNested(itemTuple.getNested(0));
} else if (type == Tuple::VERSIONSTAMP) {
tuple.appendVersionstamp(Versionstamp(itemTuple.getString(0)));
} else {
ASSERT(false);
}

2
fdbclient/CMakeLists.txt
View File

@ -149,6 +149,8 @@ set(FDBCLIENT_SRCS
VersionedMap.actor.h
VersionedMap.h
VersionedMap.cpp
Versionstamp.cpp
Versionstamp.h
VersionVector.h
VersionVector.cpp
WellKnownEndpoints.h

2
fdbclient/GlobalConfig.actor.cpp
View File

@ -119,6 +119,8 @@ void GlobalConfig::insert(KeyRef key, ValueRef value) {
any = t.getFloat(0);
} else if (t.getType(0) == Tuple::ElementType::DOUBLE) {
any = t.getDouble(0);
} else if (t.getType(0) == Tuple::ElementType::VERSIONSTAMP) {
any = t.getVersionstamp(0);
} else {
ASSERT(false);
}

27
fdbclient/Tuple.cpp
View File

@ -20,6 +20,8 @@
#include "fdbclient/Tuple.h"
const uint8_t VERSIONSTAMP_96_CODE = 0x33;
// TODO: Many functions copied from bindings/flow/Tuple.cpp. Merge at some point.
static float bigEndianFloat(float orig) {
int32_t big = *(int32_t*)&orig;
@ -75,6 +77,8 @@ Tuple::Tuple(StringRef const& str, bool exclude_incomplete) {
i += 1;
} else if (data[i] == '\x00') {
i += 1;
} else if (data[i] == VERSIONSTAMP_96_CODE) {
i += VERSIONSTAMP_TUPLE_SIZE + 1;
} else {
throw invalid_tuple_data_type();
}
@ -99,6 +103,15 @@ Tuple& Tuple::append(Tuple const& tuple) {
return *this;
}
Tuple& Tuple::appendVersionstamp(Versionstamp const& vs) {
offsets.push_back(data.size());
data.push_back(data.arena(), VERSIONSTAMP_96_CODE);
data.append(data.arena(), vs.begin(), vs.size());
return *this;
}
Tuple& Tuple::append(StringRef const& str, bool utf8) {
offsets.push_back(data.size());
@ -213,6 +226,8 @@ Tuple::ElementType Tuple::getType(size_t index) const {
return ElementType::DOUBLE;
} else if (code == 0x26 || code == 0x27) {
return ElementType::BOOL;
} else if (code == VERSIONSTAMP_96_CODE) {
return ElementType::VERSIONSTAMP;
} else {
throw invalid_tuple_data_type();
}
@ -361,6 +376,18 @@ double Tuple::getDouble(size_t index) const {
return bigEndianDouble(swap);
}
Versionstamp Tuple::getVersionstamp(size_t index) const {
if (index >= offsets.size()) {
throw invalid_tuple_index();
}
ASSERT_LT(offsets[index], data.size());
uint8_t code = data[offsets[index]];
if (code != VERSIONSTAMP_96_CODE) {
throw invalid_tuple_data_type();
}
return Versionstamp(StringRef(data.begin() + offsets[index] + 1, VERSIONSTAMP_TUPLE_SIZE));
}
KeyRange Tuple::range(Tuple const& tuple) const {
VectorRef<uint8_t> begin;
VectorRef<uint8_t> end;

5
fdbclient/Tuple.h
View File

@ -25,6 +25,7 @@
#include "flow/flow.h"
#include "fdbclient/FDBTypes.h"
#include "fdbclient/Versionstamp.h"
struct Tuple {
Tuple() {}
@ -47,6 +48,7 @@ struct Tuple {
Tuple& appendFloat(float);
Tuple& appendDouble(double);
Tuple& appendNull();
Tuple& appendVersionstamp(Versionstamp const&);
StringRef pack() const { return StringRef(data.begin(), data.size()); }
@ -55,7 +57,7 @@ struct Tuple {
return append(t);
}
enum ElementType { NULL_TYPE, INT, BYTES, UTF8, BOOL, FLOAT, DOUBLE };
enum ElementType { NULL_TYPE, INT, BYTES, UTF8, BOOL, FLOAT, DOUBLE, VERSIONSTAMP };
// this is number of elements, not length of data
size_t size() const { return offsets.size(); }
@ -68,6 +70,7 @@ struct Tuple {
StringRef subTupleRawString(size_t index) const;
ElementType getType(size_t index) const;
Standalone<StringRef> getString(size_t index) const;
Versionstamp getVersionstamp(size_t index) const;
int64_t getInt(size_t index, bool allow_incomplete = false) const;
bool getBool(size_t index) const;
float getFloat(size_t index) const;

44
fdbclient/Versionstamp.cpp Normal file
View File

@ -0,0 +1,44 @@
#include "fdbclient/Versionstamp.h"
Versionstamp::Versionstamp(StringRef str) {
if (str.size() != VERSIONSTAMP_TUPLE_SIZE) {
throw invalid_versionstamp_size();
}
data = str;
}
int16_t Versionstamp::getBatchNumber() const {
const uint8_t* begin = data.begin();
begin += 8;
int16_t batchNumber = *(int16_t*)(begin);
batchNumber = bigEndian16(batchNumber);
return batchNumber;
}
int16_t Versionstamp::getUserVersion() const {
const uint8_t* begin = data.begin();
begin += 10;
int16_t userVersion = *(int16_t*)(begin);
userVersion = bigEndian16(userVersion);
return userVersion;
}
const uint8_t* Versionstamp::begin() const {
return data.begin();
}
int64_t Versionstamp::getVersion() const {
const uint8_t* begin = data.begin();
int64_t version = *(int64_t*)begin;
version = bigEndian64(version);
return version;
}
size_t Versionstamp::size() const {
return VERSIONSTAMP_TUPLE_SIZE;
}
bool Versionstamp::operator==(const Versionstamp& other) const {
return getVersion() == other.getVersion() && getBatchNumber() == other.getBatchNumber() &&
getUserVersion() == other.getUserVersion();
}

44
fdbclient/Versionstamp.h Normal file
View File

@ -0,0 +1,44 @@
/*
* Versionstamp.h
*
* This source file is part of the FoundationDB open source project
*
* Copyright 2013-2022 Apple Inc. and the FoundationDB project authors
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
#ifndef FDBCLIENT_VERSIONSTAMP_H
#define FDBCLIENT_VERSIONSTAMP_H
#pragma once
#include "flow/Arena.h"
const size_t VERSIONSTAMP_TUPLE_SIZE = 12;
struct Versionstamp {
Versionstamp(StringRef);
int64_t getVersion() const;
int16_t getBatchNumber() const;
int16_t getUserVersion() const;
size_t size() const;
const uint8_t* begin() const;
bool operator==(const Versionstamp&) const;
private:
Standalone<StringRef> data;
};
#endif

1
flow/error_definitions.h
View File

@ -258,6 +258,7 @@ ERROR( directory_prefix_in_use, 2265, "Directory layer already has a conflicting
ERROR( invalid_destination_directory, 2266, "Target directory is invalid" )
ERROR( cannot_modify_root_directory, 2267, "Root directory cannot be modified" )
ERROR( invalid_uuid_size, 2268, "UUID is not sixteen bytes");
ERROR( invalid_versionstamp_size, 2269, "Versionstamp is not exactly twelve bytes");
// 2300 - backup and restore errors
ERROR( backup_error, 2300, "Backup error")