From 83d23512d24edfc676906bdb3aa44e828c2cc57c Mon Sep 17 00:00:00 2001
From: Dan Lambright <dlambrig@gmail.com>
Date: Thu, 28 Jul 2022 18:26:15 -0400
Subject: [PATCH] Track when version not found in version vector. (#7653)
* Track when version not found in version vector.
* Respond to comments
* include vv in event
* Respond to review comments.
* Fix bug where we logged too many lines
---
fdbclient/NativeAPI.actor.cpp | 45 +++++++++++++------
fdbclient/include/fdbclient/DatabaseContext.h | 1 +
2 files changed, 32 insertions(+), 14 deletions(-)
diff --git a/fdbclient/NativeAPI.actor.cpp b/fdbclient/NativeAPI.actor.cpp
index 8afbcfebff..dcf86db793 100644
--- a/fdbclient/NativeAPI.actor.cpp
+++ b/fdbclient/NativeAPI.actor.cpp
@@ -242,7 +242,7 @@ void DatabaseContext::getLatestCommitVersions(const Reference<LocationInfo>& loc
return;
}
- if (ssVersionVectorCache.getMaxVersion() != invalidVersion && readVersion > ssVersionVectorCache.getMaxVersion()) {
+ if (readVersion > ssVersionVectorCache.getMaxVersion()) {
if (!CLIENT_KNOBS->FORCE_GRV_CACHE_OFF && !info->options.skipGrvCache && info->options.useGrvCache) {
return;
} else {
@@ -255,16 +255,32 @@ void DatabaseContext::getLatestCommitVersions(const Reference<LocationInfo>& loc
std::map<Version, std::set<Tag>> versionMap; // order the versions to be returned
for (int i = 0; i < locationInfo->locations()->size(); i++) {
- UID uid = locationInfo->locations()->getId(i);
- if (ssidTagMapping.find(uid) != ssidTagMapping.end()) {
- Tag tag = ssidTagMapping[uid];
+ bool updatedVersionMap = false;
+ Version commitVersion = invalidVersion;
+ Tag tag = invalidTag;
+ auto iter = ssidTagMapping.find(locationInfo->locations()->getId(i));
+ if (iter != ssidTagMapping.end()) {
+ tag = iter->second;
if (ssVersionVectorCache.hasVersion(tag)) {
- Version commitVersion = ssVersionVectorCache.getVersion(tag); // latest commit version
+ commitVersion = ssVersionVectorCache.getVersion(tag); // latest commit version
if (commitVersion < readVersion) {
+ updatedVersionMap = true;
versionMap[commitVersion].insert(tag);
}
}
}
+ // commitVersion == readVersion is common, do not log.
+ if (!updatedVersionMap && commitVersion != readVersion) {
+ TraceEvent(SevDebug, "CommitVersionNotFoundForSS")
+ .detail("InSSIDMap", iter != ssidTagMapping.end() ? 1 : 0)
+ .detail("Tag", tag)
+ .detail("CommitVersion", commitVersion)
+ .detail("ReadVersion", readVersion)
+ .detail("VersionVector", ssVersionVectorCache.toString())
+ .setMaxEventLength(11000)
+ .setMaxFieldLength(10000);
+ ++transactionCommitVersionNotFoundForSS;
+ }
}
// insert the commit versions in the version vector.
@@ -1467,13 +1483,13 @@ DatabaseContext::DatabaseContext(Reference<AsyncVar<Reference<IClusterConnection
transactionsProcessBehind("ProcessBehind", cc), transactionsThrottled("Throttled", cc),
transactionsExpensiveClearCostEstCount("ExpensiveClearCostEstCount", cc),
transactionGrvFullBatches("NumGrvFullBatches", cc), transactionGrvTimedOutBatches("NumGrvTimedOutBatches", cc),
- latencies(1000), readLatencies(1000), commitLatencies(1000), GRVLatencies(1000), mutationsPerCommit(1000),
- bytesPerCommit(1000), bgLatencies(1000), bgGranulesPerRequest(1000), outstandingWatches(0), sharedStatePtr(nullptr),
- lastGrvTime(0.0), cachedReadVersion(0), lastRkBatchThrottleTime(0.0), lastRkDefaultThrottleTime(0.0),
- lastProxyRequestTime(0.0), transactionTracingSample(false), taskID(taskID), clientInfo(clientInfo),
- clientInfoMonitor(clientInfoMonitor), coordinator(coordinator), apiVersion(apiVersion), mvCacheInsertLocation(0),
- healthMetricsLastUpdated(0), detailedHealthMetricsLastUpdated(0),
- smoothMidShardSize(CLIENT_KNOBS->SHARD_STAT_SMOOTH_AMOUNT),
+ transactionCommitVersionNotFoundForSS("CommitVersionNotFoundForSS", cc), latencies(1000), readLatencies(1000),
+ commitLatencies(1000), GRVLatencies(1000), mutationsPerCommit(1000), bytesPerCommit(1000), bgLatencies(1000),
+ bgGranulesPerRequest(1000), outstandingWatches(0), sharedStatePtr(nullptr), lastGrvTime(0.0), cachedReadVersion(0),
+ lastRkBatchThrottleTime(0.0), lastRkDefaultThrottleTime(0.0), lastProxyRequestTime(0.0),
+ transactionTracingSample(false), taskID(taskID), clientInfo(clientInfo), clientInfoMonitor(clientInfoMonitor),
+ coordinator(coordinator), apiVersion(apiVersion), mvCacheInsertLocation(0), healthMetricsLastUpdated(0),
+ detailedHealthMetricsLastUpdated(0), smoothMidShardSize(CLIENT_KNOBS->SHARD_STAT_SMOOTH_AMOUNT),
specialKeySpace(std::make_unique<SpecialKeySpace>(specialKeys.begin, specialKeys.end, /* test */ false)),
connectToDatabaseEventCacheHolder(format("ConnectToDatabase/%s", dbId.toString().c_str())) {
@@ -1766,8 +1782,9 @@ DatabaseContext::DatabaseContext(const Error& err)
transactionsProcessBehind("ProcessBehind", cc), transactionsThrottled("Throttled", cc),
transactionsExpensiveClearCostEstCount("ExpensiveClearCostEstCount", cc),
transactionGrvFullBatches("NumGrvFullBatches", cc), transactionGrvTimedOutBatches("NumGrvTimedOutBatches", cc),
- latencies(1000), readLatencies(1000), commitLatencies(1000), GRVLatencies(1000), mutationsPerCommit(1000),
- bytesPerCommit(1000), bgLatencies(1000), bgGranulesPerRequest(1000), transactionTracingSample(false),
+ transactionCommitVersionNotFoundForSS("CommitVersionNotFoundForSS", cc), latencies(1000), readLatencies(1000),
+ commitLatencies(1000), GRVLatencies(1000), mutationsPerCommit(1000), bytesPerCommit(1000), bgLatencies(1000),
+ bgGranulesPerRequest(1000), transactionTracingSample(false),
smoothMidShardSize(CLIENT_KNOBS->SHARD_STAT_SMOOTH_AMOUNT),
connectToDatabaseEventCacheHolder(format("ConnectToDatabase/%s", dbId.toString().c_str())) {}
diff --git a/fdbclient/include/fdbclient/DatabaseContext.h b/fdbclient/include/fdbclient/DatabaseContext.h
index 0236aa3ccb..092a290a4c 100644
--- a/fdbclient/include/fdbclient/DatabaseContext.h
+++ b/fdbclient/include/fdbclient/DatabaseContext.h
@@ -527,6 +527,7 @@ public:
Counter transactionsExpensiveClearCostEstCount;
Counter transactionGrvFullBatches;
Counter transactionGrvTimedOutBatches;
+ Counter transactionCommitVersionNotFoundForSS;
ContinuousSample<double> latencies, readLatencies, commitLatencies, GRVLatencies, mutationsPerCommit,
bytesPerCommit, bgLatencies, bgGranulesPerRequest;