mirror of
https://github.com/apple/foundationdb.git
synced 2025-05-27 18:37:04 +08:00
Change to send back index of read_conflict_range not keys, which speeds up but fail to give narrowed keyrange
This commit is contained in:
chaoguang
parent
1cf3b5b807
commit
72d39a31f1
@ -104,7 +104,7 @@ struct CommitID {
|
||||
Version version; // returns invalidVersion if transaction conflicts
|
||||
uint16_t txnBatchId;
|
||||
Optional<Value> metadataVersion;
|
||||
Optional<Standalone<VectorRef<KeyRangeRef>>> conflictingKeyRanges;
|
||||
Optional<Standalone<VectorRef<int>>> conflictingKeyRanges;
|
||||
|
||||
template <class Ar>
|
||||
void serialize(Ar& ar) {
|
||||
@ -112,7 +112,7 @@ struct CommitID {
|
||||
}
|
||||
|
||||
CommitID() : version(invalidVersion), txnBatchId(0) {}
|
||||
CommitID( Version version, uint16_t txnBatchId, const Optional<Value>& metadataVersion, const Optional<Standalone<VectorRef<KeyRangeRef>>>& conflictingKeyRanges = Optional<Standalone<VectorRef<KeyRangeRef>>>() ) : version(version), txnBatchId(txnBatchId), metadataVersion(metadataVersion), conflictingKeyRanges(conflictingKeyRanges) {}
|
||||
CommitID( Version version, uint16_t txnBatchId, const Optional<Value>& metadataVersion, const Optional<Standalone<VectorRef<int>>>& conflictingKeyRanges = Optional<Standalone<VectorRef<int>>>() ) : version(version), txnBatchId(txnBatchId), metadataVersion(metadataVersion), conflictingKeyRanges(conflictingKeyRanges) {}
|
||||
};
|
||||
|
||||
struct CommitTransactionRequest : TimedRequest {
|
||||
|
||||
@ -2677,29 +2677,6 @@ ACTOR static Future<Void> tryCommit( Database cx, Reference<TransactionLogInfo>
|
||||
// clear the RYW transaction which contains previous conflicting keys
|
||||
tr->info.conflictingKeysRYW.reset();
|
||||
if (ci.conflictingKeyRanges.present()){
|
||||
// Since the performance of the proxy is important, we push the union of overlapped key range to client
|
||||
// Union all overlapped key ranges here
|
||||
std::vector<KeyRangeRef> unmergedConflictingKRs(ci.conflictingKeyRanges.get().begin(), ci.conflictingKeyRanges.get().end());
|
||||
Standalone<VectorRef<KeyRangeRef>> mergedKeyRanges;
|
||||
// At least one keyRange returned
|
||||
ASSERT(unmergedConflictingKRs.size());
|
||||
// Sort the keyranges by begin key, then union overlapping ranges from left to right
|
||||
std::sort(unmergedConflictingKRs.begin(), unmergedConflictingKRs.end(), [](KeyRangeRef a, KeyRangeRef b){
|
||||
return a.begin < b.begin;
|
||||
});
|
||||
auto next = unmergedConflictingKRs.begin();
|
||||
// At least one conflicting keyrange should be returned, which means curr is valid
|
||||
KeyRangeRef curr = *(next++);
|
||||
while (next != unmergedConflictingKRs.end()) {
|
||||
if (curr.end >= next->begin) {
|
||||
curr = KeyRangeRef(curr.begin, std::max(curr.end, next->end));
|
||||
} else {
|
||||
mergedKeyRanges.push_back(mergedKeyRanges.arena(), curr);
|
||||
curr = *next;
|
||||
}
|
||||
next++;
|
||||
}
|
||||
mergedKeyRanges.push_back(mergedKeyRanges.arena(), curr);
|
||||
// In general, if we want to use getRange to expose conflicting keys, we need to support all the parameters getRange provides.
|
||||
// It is difficult to take care of all corner cases of what getRange does.
|
||||
// Consequently, we use a hack way here to achieve it.
|
||||
@ -2714,7 +2691,10 @@ ACTOR static Future<Void> tryCommit( Database cx, Reference<TransactionLogInfo>
|
||||
// in case system keys are conflicting
|
||||
tr->info.conflictingKeysRYW->setOption(FDBTransactionOptions::ACCESS_SYSTEM_KEYS);
|
||||
tr->info.conflictingKeysRYW->clear(systemKeys);
|
||||
for (const KeyRangeRef & kr : mergedKeyRanges) {
|
||||
const auto cKRs = ci.conflictingKeyRanges.get();
|
||||
std::set<int> mergedIds(cKRs.begin(), cKRs.end());
|
||||
for (auto const & rCRIndex : mergedIds) {
|
||||
const KeyRangeRef & kr = req.transaction.read_conflict_ranges[rCRIndex];
|
||||
tr->info.conflictingKeysRYW->set(kr.begin, conflictingKeysTrue);
|
||||
tr->info.conflictingKeysRYW->set(kr.end, conflictingKeysFalse);
|
||||
}
|
||||
|
||||
@ -33,7 +33,7 @@ void clearConflictSet( ConflictSet*, Version );
|
||||
void destroyConflictSet(ConflictSet*);
|
||||
|
||||
struct ConflictBatch {
|
||||
explicit ConflictBatch( ConflictSet*, std::map< int, Standalone<VectorRef<KeyRangeRef>>>* conflictingKeyRangeMap = nullptr);
|
||||
explicit ConflictBatch( ConflictSet*, std::map< int, std::vector< int > >* conflictingKeyRangeMap = nullptr);
|
||||
~ConflictBatch();
|
||||
|
||||
enum TransactionCommitResult {
|
||||
@ -54,7 +54,7 @@ private:
|
||||
std::vector< std::pair<StringRef,StringRef> > combinedWriteConflictRanges;
|
||||
std::vector< struct ReadConflictRange > combinedReadConflictRanges;
|
||||
bool* transactionConflictStatus;
|
||||
std::map< int, Standalone< VectorRef< KeyRangeRef > > >* conflictingKeyRangeMap;
|
||||
std::map< int, std::vector< int > >* conflictingKeyRangeMap;
|
||||
|
||||
void checkIntraBatchConflicts();
|
||||
void combineWriteConflictRanges();
|
||||
|
||||
@ -1034,20 +1034,15 @@ ACTOR Future<Void> commitBatch(
|
||||
else {
|
||||
// If enable the option to report conflicting keys from resolvers, we union all conflicting key ranges here and send back through CommitID
|
||||
if (trs[t].transaction.report_conflicting_keys) {
|
||||
Standalone<VectorRef<KeyRangeRef>> conflictingEntries;
|
||||
int conflictingKeyRangeNum = 0;
|
||||
Standalone<VectorRef<int>> conflictingEntryIds;
|
||||
for (int resolverInd : transactionResolverMap[t]) {
|
||||
conflictingKeyRangeNum += resolution[resolverInd].conflictingKeyRangeMap[nextTr[resolverInd]].size();
|
||||
}
|
||||
conflictingEntries.reserve(conflictingEntries.arena(), conflictingKeyRangeNum);
|
||||
for (int resolverInd : transactionResolverMap[t]) {
|
||||
for (const KeyRangeRef & kr : resolution[resolverInd].conflictingKeyRangeMap[nextTr[resolverInd]]){
|
||||
conflictingEntries.push_back(conflictingEntries.arena(), kr);
|
||||
for (auto const & rCRIndex : resolution[resolverInd].conflictingKeyRangeMap[nextTr[resolverInd]]){
|
||||
conflictingEntryIds.push_back(conflictingEntryIds.arena(), rCRIndex);
|
||||
}
|
||||
}
|
||||
// At least one keyRange should be returned
|
||||
ASSERT(conflictingEntries.size());
|
||||
trs[t].reply.send(CommitID(invalidVersion, t, Optional<Value>(), Optional<Standalone<VectorRef<KeyRangeRef>>>(conflictingEntries.contents())));
|
||||
ASSERT(conflictingEntryIds.size());
|
||||
trs[t].reply.send(CommitID(invalidVersion, t, Optional<Value>(), Optional<Standalone<VectorRef<int>>>(conflictingEntryIds)));
|
||||
} else {
|
||||
trs[t].reply.sendError(not_committed());
|
||||
}
|
||||
|
||||
@ -132,7 +132,7 @@ ACTOR Future<Void> resolveBatch(
|
||||
|
||||
vector<int> commitList;
|
||||
vector<int> tooOldList;
|
||||
std::map< int, Standalone< VectorRef< KeyRangeRef > > > conflictingKeyRangeMap;
|
||||
std::map< int, std::vector< int > > conflictingKeyRangeMap;
|
||||
|
||||
// Detect conflicts
|
||||
double expire = now() + SERVER_KNOBS->SAMPLE_EXPIRATION_TIME;
|
||||
|
||||
@ -77,7 +77,7 @@ struct ResolveTransactionBatchReply {
|
||||
VectorRef<uint8_t> committed;
|
||||
Optional<UID> debugID;
|
||||
VectorRef<VectorRef<StateTransactionRef>> stateMutations; // [version][transaction#] -> (committed, [mutation#])
|
||||
std::map<int, Standalone<VectorRef<KeyRangeRef>>> conflictingKeyRangeMap; // transaction index -> conflicting keyRanges given by the resolver
|
||||
std::map<int, std::vector<int>> conflictingKeyRangeMap; // transaction index -> conflicting read_conflict_range ids given by the resolver
|
||||
|
||||
template <class Archive>
|
||||
void serialize(Archive& ar) {
|
||||
|
||||
@ -123,9 +123,10 @@ struct ReadConflictRange {
|
||||
StringRef begin, end;
|
||||
Version version;
|
||||
int transaction;
|
||||
Standalone< VectorRef< KeyRangeRef > >* conflictingKeyRange;
|
||||
ReadConflictRange( StringRef begin, StringRef end, Version version, int transaction, Standalone<VectorRef<KeyRangeRef>> * cKR = nullptr )
|
||||
: begin(begin), end(end), version(version), transaction(transaction), conflictingKeyRange(cKR)
|
||||
std::vector<int>* conflictingKeyRange;
|
||||
int indexInTx;
|
||||
ReadConflictRange( StringRef begin, StringRef end, Version version, int transaction, int indexInTx, std::vector<int> * cKR = nullptr )
|
||||
: begin(begin), end(end), version(version), transaction(transaction), indexInTx(indexInTx), conflictingKeyRange(cKR)
|
||||
{
|
||||
}
|
||||
bool operator<(const ReadConflictRange& rhs) const { return compare(begin, rhs.begin)<0; }
|
||||
@ -530,7 +531,7 @@ public:
|
||||
|
||||
int started = min(M,count);
|
||||
for(int i=0; i<started; i++){
|
||||
inProgress[i].init( ranges[i], header, transactionConflictStatus, ranges[i].conflictingKeyRange );
|
||||
inProgress[i].init( ranges[i], header, transactionConflictStatus, ranges[i].indexInTx, ranges[i].conflictingKeyRange );
|
||||
nextJob[i] = i+1;
|
||||
}
|
||||
nextJob[started-1] = 0;
|
||||
@ -547,7 +548,7 @@ public:
|
||||
}
|
||||
else {
|
||||
int temp = started++;
|
||||
inProgress[job].init( ranges[temp], header, transactionConflictStatus, ranges[temp].conflictingKeyRange );
|
||||
inProgress[job].init( ranges[temp], header, transactionConflictStatus, ranges[temp].indexInTx, ranges[temp].conflictingKeyRange );
|
||||
}
|
||||
}
|
||||
prevJob = job;
|
||||
@ -760,31 +761,24 @@ private:
|
||||
Version version;
|
||||
bool *result;
|
||||
int state;
|
||||
Standalone<VectorRef<KeyRangeRef>>* conflictingKeyRange; // null if report_conflicting_keys is not enabled.
|
||||
int indexInTx;
|
||||
std::vector<int>* conflictingKeyRange; // null if report_conflicting_keys is not enabled.
|
||||
|
||||
void init( const ReadConflictRange& r, Node* header, bool* tCS, Standalone<VectorRef<KeyRangeRef>>* cKR) {
|
||||
void init( const ReadConflictRange& r, Node* header, bool* tCS, int indexInTx, std::vector<int>* cKR) {
|
||||
this->start.init( r.begin, header );
|
||||
this->end.init( r.end, header );
|
||||
this->version = r.version;
|
||||
this->indexInTx = indexInTx;
|
||||
result = &tCS[ r.transaction ];
|
||||
conflictingKeyRange = cKR;
|
||||
this->state = 0;
|
||||
}
|
||||
|
||||
bool noConflict() { return true; }
|
||||
bool conflict(Node* start_node, Node* end_node) {
|
||||
bool conflict() {
|
||||
*result = true;
|
||||
if(conflictingKeyRange != nullptr){
|
||||
StringRef startKey(start.value);
|
||||
StringRef endKey(end.value);
|
||||
if (start_node != nullptr) {
|
||||
startKey = StringRef(start_node->value(), start_node->length());
|
||||
}
|
||||
if (end_node != nullptr) {
|
||||
endKey = StringRef(end_node->value(), end_node->length());
|
||||
}
|
||||
conflictingKeyRange->push_back_deep(conflictingKeyRange->arena(), KeyRangeRef(startKey, endKey));
|
||||
}
|
||||
if(conflictingKeyRange != nullptr)
|
||||
conflictingKeyRange->push_back(indexInTx);
|
||||
return true;
|
||||
}
|
||||
|
||||
@ -808,7 +802,7 @@ private:
|
||||
if (start.finger[l]->getMaxVersion(l) <= version)
|
||||
return noConflict();
|
||||
if (l==0)
|
||||
return conflict(nullptr, nullptr); // The whole read_conflict_range is conflicting with other transactions with higher version number
|
||||
return conflict(); // The whole read_conflict_range is conflicting with other transactions with higher version number
|
||||
}
|
||||
state = 1;
|
||||
case 1:
|
||||
@ -817,12 +811,12 @@ private:
|
||||
Node *e = end.finger[end.level];
|
||||
while (e->getMaxVersion(end.level) > version) {
|
||||
if (end.finished())
|
||||
return conflict(e, nullptr);
|
||||
return conflict();
|
||||
end.nextLevel();
|
||||
Node *f = end.finger[end.level];
|
||||
while (e != f){
|
||||
if (e->getMaxVersion(end.level) > version)
|
||||
return conflict(e, nullptr);
|
||||
return conflict();
|
||||
e = e->getNext(end.level);
|
||||
}
|
||||
}
|
||||
@ -834,7 +828,7 @@ private:
|
||||
Node *p = nextS;
|
||||
while (p != s){
|
||||
if (p->getMaxVersion(start.level) > version)
|
||||
return conflict(p, s);
|
||||
return conflict();
|
||||
p = p->getNext(start.level);
|
||||
}
|
||||
if (start.finger[start.level]->getMaxVersion(start.level) <= version)
|
||||
@ -844,7 +838,7 @@ private:
|
||||
if (nextS->length() == start.value.size() && !memcmp(nextS->value(), start.value.begin(), start.value.size()))
|
||||
return noConflict();
|
||||
else
|
||||
return conflict(nullptr, nextS);
|
||||
return conflict();
|
||||
}
|
||||
start.nextLevel();
|
||||
}
|
||||
@ -980,7 +974,7 @@ void destroyConflictSet(ConflictSet* cs) {
|
||||
delete cs;
|
||||
}
|
||||
|
||||
ConflictBatch::ConflictBatch( ConflictSet* cs, std::map< int, Standalone<VectorRef<KeyRangeRef>> >* conflictingKeyRangeMap )
|
||||
ConflictBatch::ConflictBatch( ConflictSet* cs, std::map< int, std::vector< int > >* conflictingKeyRangeMap )
|
||||
: cs(cs), transactionCount(0), conflictingKeyRangeMap(conflictingKeyRangeMap)
|
||||
{
|
||||
}
|
||||
@ -1016,7 +1010,7 @@ void ConflictBatch::addTransaction( const CommitTransactionRef& tr ) {
|
||||
points.emplace_back(range.begin, false, true, false, t, &info->readRanges[r].first);
|
||||
//points.back().keyEnd = StringRef(buf,range.second);
|
||||
points.emplace_back(range.end, false, false, false, t, &info->readRanges[r].second);
|
||||
combinedReadConflictRanges.emplace_back(range.begin, range.end, tr.read_snapshot, t, tr.report_conflicting_keys ? &(*conflictingKeyRangeMap)[t] : nullptr);
|
||||
combinedReadConflictRanges.emplace_back(range.begin, range.end, tr.read_snapshot, t, r, tr.report_conflicting_keys ? &(*conflictingKeyRangeMap)[t] : nullptr);
|
||||
}
|
||||
for(int r=0; r<tr.write_conflict_ranges.size(); r++) {
|
||||
const KeyRangeRef& range = tr.write_conflict_ranges[r];
|
||||
@ -1038,10 +1032,9 @@ public:
|
||||
for(int i=begin; i<end; i++)
|
||||
values[i] = true;
|
||||
}
|
||||
bool any( int begin, int end, int* conflictingIndex = nullptr ) {
|
||||
bool any( int begin, int end ) {
|
||||
for(int i=begin; i<end; i++)
|
||||
if (values[i]) {
|
||||
if (conflictingIndex != nullptr) *conflictingIndex = i;
|
||||
return true;
|
||||
}
|
||||
return false;
|
||||
@ -1134,9 +1127,9 @@ public:
|
||||
setBits(orValues, beginWord, lastWord+1, true);
|
||||
}
|
||||
|
||||
bool any(int begin, int end, int* conflictingIndex = nullptr) {
|
||||
bool any(int begin, int end) {
|
||||
bool a = orImpl(begin,end);
|
||||
bool b = debug.any(begin,end, conflictingIndex);
|
||||
bool b = debug.any(begin,end);
|
||||
ASSERT( a == b );
|
||||
return b;
|
||||
}
|
||||
@ -1163,17 +1156,9 @@ void ConflictBatch::checkIntraBatchConflicts() {
|
||||
if (transactionConflictStatus[t]) continue;
|
||||
bool conflict = tr.tooOld;
|
||||
for(int i=0; i<tr.readRanges.size(); i++){
|
||||
int startPointIndex = tr.readRanges[i].first;
|
||||
int endPointIndex = tr.readRanges[i].second;
|
||||
int conflictingIndex;
|
||||
if ( mcs.any( startPointIndex , endPointIndex, tr.reportConflictingKeys ? &conflictingIndex : nullptr ) ) {
|
||||
if ( mcs.any( tr.readRanges[i].first , tr.readRanges[i].second ) ) {
|
||||
if (tr.reportConflictingKeys){
|
||||
// The MiniConflictSet is difficult to change, use MiniConflictSet2 to hack here. (Future: use MiniConflictSet)
|
||||
if (points[conflictingIndex].begin)
|
||||
startPointIndex = conflictingIndex;
|
||||
else
|
||||
endPointIndex = conflictingIndex;
|
||||
(*conflictingKeyRangeMap)[t].push_back_deep((*conflictingKeyRangeMap)[t].arena(), KeyRangeRef(points[startPointIndex].key, points[endPointIndex].key));
|
||||
(*conflictingKeyRangeMap)[t].push_back(i);
|
||||
}
|
||||
conflict = true;
|
||||
break;
|
||||
|
||||
@ -192,13 +192,13 @@ struct ReportConflictingKeysWorkload : TestWorkload {
|
||||
ASSERT(startKey.value == conflictingKeysTrue);
|
||||
KeyValueRef endKey = conflictingKeyRanges[i+1];
|
||||
ASSERT(endKey.value == conflictingKeysFalse);
|
||||
KeyRange kr = KeyRangeRef(startKey.key, endKey.key);
|
||||
KeyRangeRef kr = KeyRangeRef(startKey.key, endKey.key);
|
||||
if (!std::any_of(readConflictRanges.begin(), readConflictRanges.end(), [&kr](KeyRange rCR) {
|
||||
// Read_conflict_range remains same in the resolver.
|
||||
// Thus, the returned keyrange is either the original read_conflict_range or merged
|
||||
// by several overlapped ones In either case, it contains at least one original
|
||||
// read_conflict_range
|
||||
return kr.intersects(rCR);
|
||||
return kr.contains(rCR);
|
||||
})) {
|
||||
++self->invalidReports;
|
||||
TraceEvent(SevError, "TestFailure").detail("Reason", "InvalidKeyRangeReturned");
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user