From 2f2ece073c24f85ec00c9986edd82a047071c0c4 Mon Sep 17 00:00:00 2001 From: "Bharadwaj V.R" Date: Wed, 13 Apr 2022 08:05:22 -0700 Subject: [PATCH 01/12] Add sw version tracking to DBCoreState --- fdbserver/ClusterRecovery.actor.cpp | 12 +++++ fdbserver/DBCoreState.h | 10 +++- fdbserver/Status.actor.cpp | 1 + flow/ProtocolVersion.h | 75 ++++++++++++++++++----------- 4 files changed, 68 insertions(+), 30 deletions(-) diff --git a/fdbserver/ClusterRecovery.actor.cpp b/fdbserver/ClusterRecovery.actor.cpp index af564b377d..3710f2f05b 100644 --- a/fdbserver/ClusterRecovery.actor.cpp +++ b/fdbserver/ClusterRecovery.actor.cpp @@ -1406,6 +1406,10 @@ ACTOR Future clusterRecoveryCore(Reference self) { wait(self->cstate.read()); + if (self->cstate.prevDBState.lowestCompatibleServerVersion > currentProtocolVersion) { + TraceEvent(SevWarnAlways, "IncompatbleServerVersion", self->dbgid).log(); + } + self->recoveryState = RecoveryState::LOCKING_CSTATE; TraceEvent(getRecoveryEventName(ClusterRecoveryEventType::CLUSTER_RECOVERY_STATE_EVENT_NAME).c_str(), self->dbgid) .detail("StatusCode", RecoveryStatus::locking_coordinated_state) @@ -1461,6 +1465,14 @@ ACTOR Future clusterRecoveryCore(Reference self) { DBCoreState newState = self->cstate.myDBState; newState.recoveryCount++; + newState.recoveryCount++; + if (self->cstate.myDBState.newestServerVersion.isInvalidMagic() || + self->cstate.myDBState.newestServerVersion < currentProtocolVersion) { + ASSERT(self->cstate.myDBState.lowestCompatibleServerVersion.isInvalidMagic() || + !self->cstate.myDBState.newestServerVersion.isInvalidMagic()); + newState.newestServerVersion = currentProtocolVersion; + newState.lowestCompatibleServerVersion = minCompatibleProtocolVersion; + } wait(self->cstate.write(newState) || recoverAndEndEpoch); self->recoveryState = RecoveryState::RECRUITING; diff --git a/fdbserver/DBCoreState.h b/fdbserver/DBCoreState.h index 7c06432498..9fdfd910f0 100644 --- a/fdbserver/DBCoreState.h +++ b/fdbserver/DBCoreState.h @@ -141,8 +141,13 @@ struct DBCoreState { DBRecoveryCount recoveryCount; // Increases with sequential successful recoveries. LogSystemType logSystemType; std::set pseudoLocalities; + ProtocolVersion newestServerVersion; + ProtocolVersion lowestCompatibleServerVersion; - DBCoreState() : logRouterTags(0), txsTags(0), recoveryCount(0), logSystemType(LogSystemType::empty) {} + DBCoreState() + : logRouterTags(0), txsTags(0), recoveryCount(0), logSystemType(LogSystemType::empty), + newestServerVersion(ProtocolVersion::invalidProtocolVersion), + lowestCompatibleServerVersion(ProtocolVersion::invalidProtocolVersion) {} std::vector getPriorCommittedLogServers() { std::vector priorCommittedLogServers; @@ -180,6 +185,9 @@ struct DBCoreState { if (ar.protocolVersion().hasShardedTxsTags()) { serializer(ar, txsTags); } + if (ar.protocolVersion().hasSWVersionTracking()) { + serializer(ar, newestServerVersion, lowestCompatibleServerVersion); + } } else if (ar.isDeserializing) { tLogs.push_back(CoreTLogSet()); serializer(ar, diff --git a/fdbserver/Status.actor.cpp b/fdbserver/Status.actor.cpp index f30f1638b2..0be742c3c2 100644 --- a/fdbserver/Status.actor.cpp +++ b/fdbserver/Status.actor.cpp @@ -2917,6 +2917,7 @@ ACTOR Future clusterGetStatus( statusObj["protocol_version"] = format("%" PRIx64, g_network->protocolVersion().version()); statusObj["connection_string"] = coordinators.ccr->getConnectionString().toString(); statusObj["bounce_impact"] = getBounceImpactInfo(statusCode); + // statusObj["latest_server_version"] = format("%" PRIx64, latestServerVersion.version()); state Optional configuration; state Optional loadResult; diff --git a/flow/ProtocolVersion.h b/flow/ProtocolVersion.h index a6561c91ab..059cf883dc 100644 --- a/flow/ProtocolVersion.h +++ b/flow/ProtocolVersion.h @@ -19,24 +19,31 @@ */ #pragma once -#include #include "flow/Trace.h" +#include -// This version impacts both communications and the deserialization of certain database and IKeyValueStore keys. +// This version impacts both communications and the deserialization of certain +// database and IKeyValueStore keys. // -// The convention is that 'x' and 'y' should match the major and minor version of the software, and 'z' should be 0. -// To make a change without a corresponding increase to the x.y version, increment the 'dev' digit. +// The convention is that 'x' and 'y' should match the major and minor version +// of the software, and 'z' should be 0. To make a change without a +// corresponding increase to the x.y version, increment the 'dev' digit. // -// The last 2 bytes (4 digits) of the protocol version do not affect compatibility. These two bytes are not currently -// used and should not be changed from 0. +// The last 2 bytes (4 digits) of the protocol version do not affect +// compatibility. These two bytes are not currently used and should not be +// changed from 0. // xyzdev // vvvv constexpr uint64_t currentProtocolVersionValue = 0x0FDB00B072000000LL; -// The first protocol version that cannot be downgraded from. Ordinarily, this will be two release versions larger -// than the current version, meaning that we only support downgrades between consecutive release versions. +// The first protocol version that cannot be downgraded from. Ordinarily, this +// will be two release versions larger than the current version, meaning that we +// only support downgrades between consecutive release versions. constexpr uint64_t minInvalidProtocolVersionValue = 0x0FDB00B074000000LL; +// The lowest protocol version that can be downgraded to. +constexpr uint64_t minCompatibleProtocolVersionValue = 0x0FDB00B071000000LL; + #define PROTOCOL_VERSION_FEATURE(v, x) \ static_assert((v & 0xF0FFFFLL) == 0 || v < 0x0FDB00B071000000LL, "Unexpected feature protocol version"); \ static_assert(v <= currentProtocolVersionValue, "Feature protocol version too large"); \ @@ -46,11 +53,11 @@ constexpr uint64_t minInvalidProtocolVersionValue = 0x0FDB00B074000000LL; constexpr bool has##x() const { return this->version() >= x ::protocolVersion; } \ static constexpr ProtocolVersion with##x() { return ProtocolVersion(x ::protocolVersion); } -// ProtocolVersion wraps a uint64_t to make it type safe. It will know about the current versions. -// The default constructor will initialize the version to 0 (which is an invalid -// version). ProtocolVersion objects should never be compared to version numbers -// directly. Instead one should always use the type-safe version types from which -// this class inherits all. +// ProtocolVersion wraps a uint64_t to make it type safe. It will know about the +// current versions. The default constructor will initialize the version to 0 +// (which is an invalid version). ProtocolVersion objects should never be +// compared to version numbers directly. Instead one should always use the +// type-safe version types from which this class inherits all. class ProtocolVersion { uint64_t _version; @@ -59,6 +66,7 @@ public: // constants static constexpr uint64_t objectSerializerFlag = 0x1000000000000000LL; static constexpr uint64_t compatibleProtocolVersionMask = 0xFFFFFFFFFFFF0000LL; static constexpr uint64_t minValidProtocolVersion = 0x0FDB00A200060001LL; + static constexpr uint64_t invalidProtocolVersion = 0x0FDB00A100000000LL; public: constexpr explicit ProtocolVersion(uint64_t version) : _version(version) {} @@ -68,12 +76,15 @@ public: return (other.version() & compatibleProtocolVersionMask) == (version() & compatibleProtocolVersionMask); } - // Returns a normalized protocol version that will be the same for all compatible versions + // Returns a normalized protocol version that will be the same for all + // compatible versions constexpr ProtocolVersion normalizedVersion() const { return ProtocolVersion(_version & compatibleProtocolVersionMask); } constexpr bool isValid() const { return version() >= minValidProtocolVersion; } + constexpr bool isInvalidMagic() const { return version() == invalidProtocolVersion; } + constexpr uint64_t version() const { return _version & versionFlagMask; } constexpr uint64_t versionWithFlags() const { return _version; } @@ -85,9 +96,10 @@ public: constexpr void removeAllFlags() { _version = version(); } // comparison operators - // Comparison operators ignore the flags - this is because the version flags are stored in the - // most significant byte which can make comparison confusing. Also, generally, when one wants to - // compare versions, we are usually not interested in the flags. + // Comparison operators ignore the flags - this is because the version flags + // are stored in the most significant byte which can make comparison + // confusing. Also, generally, when one wants to compare versions, we are + // usually not interested in the flags. constexpr bool operator==(const ProtocolVersion other) const { return version() == other.version(); } constexpr bool operator!=(const ProtocolVersion other) const { return version() != other.version(); } constexpr bool operator<=(const ProtocolVersion other) const { return version() <= other.version(); } @@ -96,11 +108,12 @@ public: constexpr bool operator>(const ProtocolVersion other) const { return version() > other.version(); } public: // introduced features - // The 5th digit from right is dev version, for example, 2 in 0x0FDB00B061020000LL; - // It was used to identify a protocol change (e.g., interface change) between major/minor versions (say 5.1 and 5.2) - // We stopped using the dev version consistently in the past. - // To ensure binaries work across patch releases (e.g., 6.2.0 to 6.2.22), we require that the protocol version be - // the same for each of them. + // The 5th digit from right is dev version, for example, 2 in + // 0x0FDB00B061020000LL; It was used to identify a protocol change (e.g., + // interface change) between major/minor versions (say 5.1 and 5.2) We stopped + // using the dev version consistently in the past. To ensure binaries work + // across patch releases (e.g., 6.2.0 to 6.2.22), we require that the protocol + // version be the same for each of them. PROTOCOL_VERSION_FEATURE(0x0FDB00A200090000LL, Watches); PROTOCOL_VERSION_FEATURE(0x0FDB00A2000D0000LL, MovableCoordinatedState); PROTOCOL_VERSION_FEATURE(0x0FDB00A340000000LL, ProcessID); @@ -164,6 +177,7 @@ public: // introduced features PROTOCOL_VERSION_FEATURE(0x0FDB00B071010000LL, Tenants); PROTOCOL_VERSION_FEATURE(0x0FDB00B071010000LL, StorageInterfaceReadiness); PROTOCOL_VERSION_FEATURE(0x0FDB00B071010000LL, ResolverPrivateMutations); + PROTOCOL_VERSION_FEATURE(0x0FDB00B072000000LL, SWVersionTracking); }; template <> @@ -175,15 +189,18 @@ struct Traceable : std::true_type { constexpr ProtocolVersion currentProtocolVersion(currentProtocolVersionValue); constexpr ProtocolVersion minInvalidProtocolVersion(minInvalidProtocolVersionValue); +constexpr ProtocolVersion minCompatibleProtocolVersion(minCompatibleProtocolVersionValue); -// This assert is intended to help prevent incrementing the leftmost digits accidentally. It will probably need to -// change when we reach version 10. +// This assert is intended to help prevent incrementing the leftmost digits +// accidentally. It will probably need to change when we reach version 10. static_assert(currentProtocolVersion.version() < 0x0FDB00B100000000LL, "Unexpected protocol version"); -// The last two bytes of the protocol version are currently masked out in compatibility checks. We do not use them, -// so prevent them from being inadvertently changed. +// The last two bytes of the protocol version are currently masked out in +// compatibility checks. We do not use them, so prevent them from being +// inadvertently changed. // -// We also do not modify the protocol version for patch releases, so prevent modifying the patch version digit. +// We also do not modify the protocol version for patch releases, so prevent +// modifying the patch version digit. static_assert((currentProtocolVersion.version() & 0xF0FFFFLL) == 0, "Unexpected protocol version"); // Downgrades must support at least one minor version. @@ -191,6 +208,6 @@ static_assert(minInvalidProtocolVersion.version() >= (currentProtocolVersion.version() & 0xFFFFFFFFFF000000LL) + 0x0000000002000000, "Downgrades must support one minor version"); -// The min invalid protocol version should be the smallest possible protocol version associated with a minor release -// version. +// The min invalid protocol version should be the smallest possible protocol +// version associated with a minor release version. static_assert((minInvalidProtocolVersion.version() & 0xFFFFFFLL) == 0, "Unexpected min invalid protocol version"); \ No newline at end of file From 831bee34f87e5efc90000e3ffb05ab56bbffc313 Mon Sep 17 00:00:00 2001 From: "Bharadwaj V.R" Date: Wed, 13 Apr 2022 08:23:02 -0700 Subject: [PATCH 02/12] Add server version to status json --- fdbclient/Schemas.cpp | 1 + fdbserver/Status.actor.cpp | 9 ++++++++- 2 files changed, 9 insertions(+), 1 deletion(-) diff --git a/fdbclient/Schemas.cpp b/fdbclient/Schemas.cpp index fbfd81558f..3f9dc7d518 100644 --- a/fdbclient/Schemas.cpp +++ b/fdbclient/Schemas.cpp @@ -695,6 +695,7 @@ const KeyRef JSONSchemas::statusSchema = LiteralStringRef(R"statusSchema( }, "cluster_controller_timestamp":1415650089, "protocol_version":"fdb00a400050001", + "latest_server_version":"fdb00a500040001", "connection_string":"a:a@127.0.0.1:4000", "full_replication":true, "maintenance_zone":"0ccb4e0fdbdb5583010f6b77d9d10ece", diff --git a/fdbserver/Status.actor.cpp b/fdbserver/Status.actor.cpp index 0be742c3c2..94651e092b 100644 --- a/fdbserver/Status.actor.cpp +++ b/fdbserver/Status.actor.cpp @@ -24,6 +24,7 @@ #include "fdbclient/KeyBackedTypes.h" #include "fdbserver/Status.h" #include "flow/ITrace.h" +#include "flow/ProtocolVersion.h" #include "flow/Trace.h" #include "fdbclient/NativeAPI.actor.h" #include "fdbclient/SystemData.h" @@ -1527,6 +1528,10 @@ ACTOR static Future logRangeWarningFetcher(Database cx, return Void(); } +ACTOR Future getLatestSoftwareVersion(Database cx) { + return currentProtocolVersion; +} + struct LoadConfigurationResult { bool fullReplication; Optional healthyZone; @@ -2917,7 +2922,9 @@ ACTOR Future clusterGetStatus( statusObj["protocol_version"] = format("%" PRIx64, g_network->protocolVersion().version()); statusObj["connection_string"] = coordinators.ccr->getConnectionString().toString(); statusObj["bounce_impact"] = getBounceImpactInfo(statusCode); - // statusObj["latest_server_version"] = format("%" PRIx64, latestServerVersion.version()); + + ProtocolVersion latestServerVersion = wait(getLatestSoftwareVersion(cx)); + statusObj["latest_server_version"] = format("%" PRIx64, latestServerVersion.version()); state Optional configuration; state Optional loadResult; From d9f7b9731591fef87b2f594da0e2ba663b9fc06d Mon Sep 17 00:00:00 2001 From: "Bharadwaj V.R" Date: Thu, 14 Apr 2022 07:19:41 -0700 Subject: [PATCH 03/12] Add software version to system key for status reports --- fdbclient/SystemData.cpp | 2 ++ fdbclient/SystemData.h | 8 ++++++++ fdbserver/ApplyMetadataMutation.cpp | 3 ++- fdbserver/ClusterRecovery.actor.cpp | 12 ++++++++++-- fdbserver/Status.actor.cpp | 19 +++++++++++++++---- 5 files changed, 37 insertions(+), 7 deletions(-) diff --git a/fdbclient/SystemData.cpp b/fdbclient/SystemData.cpp index 255fccc6c6..23b6e0aa66 100644 --- a/fdbclient/SystemData.cpp +++ b/fdbclient/SystemData.cpp @@ -43,6 +43,8 @@ const KeyRangeRef keyServersKeyServersKeys(LiteralStringRef("\xff/keyServers/\xf LiteralStringRef("\xff/keyServers/\xff/keyServers0")); const KeyRef keyServersKeyServersKey = keyServersKeyServersKeys.begin; +const KeyRef newestProtocolVersionKey = LiteralStringRef("\xff/newestProtocolVersion"); + const Key keyServersKey(const KeyRef& k) { return k.withPrefix(keyServersPrefix); } diff --git a/fdbclient/SystemData.h b/fdbclient/SystemData.h index 4069c68162..8e8c3dc19e 100644 --- a/fdbclient/SystemData.h +++ b/fdbclient/SystemData.h @@ -70,6 +70,14 @@ void decodeKeyServersValue(std::map const& tag_uid, extern const KeyRef clusterIdKey; +// Key whose value stores the newest software version. +// This is not used to enforce software version compatibility. This is just +// a copy of information stored in the DBCoreState. That information is +// used during recovery to make sure that only versions of software +// that are compatible with the database are run. +// "\xff/newestProtocolVersion" +extern const KeyRef newestProtocolVersionKey; + // "\xff/checkpoint/[[UID]] := [[CheckpointMetaData]]" extern const KeyRef checkpointPrefix; const Key checkpointKeyFor(UID checkpointID); diff --git a/fdbserver/ApplyMetadataMutation.cpp b/fdbserver/ApplyMetadataMutation.cpp index 90f987021f..93cd5f07b2 100644 --- a/fdbserver/ApplyMetadataMutation.cpp +++ b/fdbserver/ApplyMetadataMutation.cpp @@ -594,7 +594,8 @@ private: m.param1.startsWith(applyMutationsAddPrefixRange.begin) || m.param1.startsWith(applyMutationsRemovePrefixRange.begin) || m.param1.startsWith(tagLocalityListPrefix) || m.param1.startsWith(serverTagHistoryPrefix) || - m.param1.startsWith(testOnlyTxnStateStorePrefixRange.begin) || m.param1 == clusterIdKey) { + m.param1.startsWith(testOnlyTxnStateStorePrefixRange.begin) || m.param1 == clusterIdKey || + m.param1 == newestProtocolVersionKey) { txnStateStore->set(KeyValueRef(m.param1, m.param2)); } diff --git a/fdbserver/ClusterRecovery.actor.cpp b/fdbserver/ClusterRecovery.actor.cpp index 3710f2f05b..2e6ce111b9 100644 --- a/fdbserver/ClusterRecovery.actor.cpp +++ b/fdbserver/ClusterRecovery.actor.cpp @@ -25,6 +25,7 @@ #include "fdbserver/MasterInterface.h" #include "fdbserver/WaitFailure.h" +#include "flow/ProtocolVersion.h" #include "flow/actorcompiler.h" // This must be the last #include. static std::set const& normalClusterRecoveryErrors() { @@ -1408,6 +1409,7 @@ ACTOR Future clusterRecoveryCore(Reference self) { if (self->cstate.prevDBState.lowestCompatibleServerVersion > currentProtocolVersion) { TraceEvent(SevWarnAlways, "IncompatbleServerVersion", self->dbgid).log(); + throw internal_error(); } self->recoveryState = RecoveryState::LOCKING_CSTATE; @@ -1466,8 +1468,8 @@ ACTOR Future clusterRecoveryCore(Reference self) { DBCoreState newState = self->cstate.myDBState; newState.recoveryCount++; newState.recoveryCount++; - if (self->cstate.myDBState.newestServerVersion.isInvalidMagic() || - self->cstate.myDBState.newestServerVersion < currentProtocolVersion) { + if (self->cstate.prevDBState.newestServerVersion.isInvalidMagic() || + self->cstate.prevDBState.newestServerVersion < currentProtocolVersion) { ASSERT(self->cstate.myDBState.lowestCompatibleServerVersion.isInvalidMagic() || !self->cstate.myDBState.newestServerVersion.isInvalidMagic()); newState.newestServerVersion = currentProtocolVersion; @@ -1622,6 +1624,12 @@ ACTOR Future clusterRecoveryCore(Reference self) { tr.set(recoveryCommitRequest.arena, clusterIdKey, BinaryWriter::toValue(self->clusterId, Unversioned())); } + if (currentProtocolVersion > self->cstate.prevDBState.newestServerVersion) { + tr.set(recoveryCommitRequest.arena, + newestProtocolVersionKey, + BinaryWriter::toValue(currentProtocolVersion.version(), Unversioned())); + } + applyMetadataMutations(SpanID(), self->dbgid, recoveryCommitRequest.arena, diff --git a/fdbserver/Status.actor.cpp b/fdbserver/Status.actor.cpp index 94651e092b..ee26a6970d 100644 --- a/fdbserver/Status.actor.cpp +++ b/fdbserver/Status.actor.cpp @@ -1528,8 +1528,19 @@ ACTOR static Future logRangeWarningFetcher(Database cx, return Void(); } -ACTOR Future getLatestSoftwareVersion(Database cx) { - return currentProtocolVersion; +ACTOR Future getNewestProtocolVersion(Database cx) { + state ReadYourWritesTransaction tr(cx); + loop { + try { + tr.setOption(FDBTransactionOptions::ACCESS_SYSTEM_KEYS); + tr.setOption(FDBTransactionOptions::LOCK_AWARE); + Optional newestProtocolVersion = wait(tr.get(newestProtocolVersionKey)); + ASSERT(newestProtocolVersion.present()); + return BinaryReader::fromStringRef(newestProtocolVersion.get(), Unversioned()); + } catch (Error& e) { + wait(tr.onError(e)); + } + } } struct LoadConfigurationResult { @@ -2923,8 +2934,8 @@ ACTOR Future clusterGetStatus( statusObj["connection_string"] = coordinators.ccr->getConnectionString().toString(); statusObj["bounce_impact"] = getBounceImpactInfo(statusCode); - ProtocolVersion latestServerVersion = wait(getLatestSoftwareVersion(cx)); - statusObj["latest_server_version"] = format("%" PRIx64, latestServerVersion.version()); + ProtocolVersion newestProtocolVersion = wait(getNewestProtocolVersion(cx)); + statusObj["latest_server_version"] = format("%" PRIx64, newestProtocolVersion.version()); state Optional configuration; state Optional loadResult; From 78c4771f9d243224af6b1b0f459691b3e5a70080 Mon Sep 17 00:00:00 2001 From: "Bharadwaj V.R" Date: Thu, 14 Apr 2022 11:10:39 -0700 Subject: [PATCH 04/12] Fix ProtocolVersion.h formatting diffs --- ProtocolVersion.h | 196 +++++++++++++++++++++++++++++++++++++++++ flow/ProtocolVersion.h | 67 ++++++-------- 2 files changed, 225 insertions(+), 38 deletions(-) create mode 100644 ProtocolVersion.h diff --git a/ProtocolVersion.h b/ProtocolVersion.h new file mode 100644 index 0000000000..a6561c91ab --- /dev/null +++ b/ProtocolVersion.h @@ -0,0 +1,196 @@ +/* + * ProtocolVersion.h + * + * This source file is part of the FoundationDB open source project + * + * Copyright 2013-2022 Apple Inc. and the FoundationDB project authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#pragma once +#include +#include "flow/Trace.h" + +// This version impacts both communications and the deserialization of certain database and IKeyValueStore keys. +// +// The convention is that 'x' and 'y' should match the major and minor version of the software, and 'z' should be 0. +// To make a change without a corresponding increase to the x.y version, increment the 'dev' digit. +// +// The last 2 bytes (4 digits) of the protocol version do not affect compatibility. These two bytes are not currently +// used and should not be changed from 0. +// xyzdev +// vvvv +constexpr uint64_t currentProtocolVersionValue = 0x0FDB00B072000000LL; + +// The first protocol version that cannot be downgraded from. Ordinarily, this will be two release versions larger +// than the current version, meaning that we only support downgrades between consecutive release versions. +constexpr uint64_t minInvalidProtocolVersionValue = 0x0FDB00B074000000LL; + +#define PROTOCOL_VERSION_FEATURE(v, x) \ + static_assert((v & 0xF0FFFFLL) == 0 || v < 0x0FDB00B071000000LL, "Unexpected feature protocol version"); \ + static_assert(v <= currentProtocolVersionValue, "Feature protocol version too large"); \ + struct x { \ + static constexpr uint64_t protocolVersion = v; \ + }; \ + constexpr bool has##x() const { return this->version() >= x ::protocolVersion; } \ + static constexpr ProtocolVersion with##x() { return ProtocolVersion(x ::protocolVersion); } + +// ProtocolVersion wraps a uint64_t to make it type safe. It will know about the current versions. +// The default constructor will initialize the version to 0 (which is an invalid +// version). ProtocolVersion objects should never be compared to version numbers +// directly. Instead one should always use the type-safe version types from which +// this class inherits all. +class ProtocolVersion { + uint64_t _version; + +public: // constants + static constexpr uint64_t versionFlagMask = 0x0FFFFFFFFFFFFFFFLL; + static constexpr uint64_t objectSerializerFlag = 0x1000000000000000LL; + static constexpr uint64_t compatibleProtocolVersionMask = 0xFFFFFFFFFFFF0000LL; + static constexpr uint64_t minValidProtocolVersion = 0x0FDB00A200060001LL; + +public: + constexpr explicit ProtocolVersion(uint64_t version) : _version(version) {} + constexpr ProtocolVersion() : _version(0) {} + + constexpr bool isCompatible(ProtocolVersion other) const { + return (other.version() & compatibleProtocolVersionMask) == (version() & compatibleProtocolVersionMask); + } + + // Returns a normalized protocol version that will be the same for all compatible versions + constexpr ProtocolVersion normalizedVersion() const { + return ProtocolVersion(_version & compatibleProtocolVersionMask); + } + constexpr bool isValid() const { return version() >= minValidProtocolVersion; } + + constexpr uint64_t version() const { return _version & versionFlagMask; } + constexpr uint64_t versionWithFlags() const { return _version; } + + constexpr bool hasObjectSerializerFlag() const { return (_version & objectSerializerFlag) > 0; } + constexpr void addObjectSerializerFlag() { _version = _version | objectSerializerFlag; } + constexpr void removeObjectSerializerFlag() { + _version = hasObjectSerializerFlag() ? _version ^ objectSerializerFlag : _version; + } + constexpr void removeAllFlags() { _version = version(); } + + // comparison operators + // Comparison operators ignore the flags - this is because the version flags are stored in the + // most significant byte which can make comparison confusing. Also, generally, when one wants to + // compare versions, we are usually not interested in the flags. + constexpr bool operator==(const ProtocolVersion other) const { return version() == other.version(); } + constexpr bool operator!=(const ProtocolVersion other) const { return version() != other.version(); } + constexpr bool operator<=(const ProtocolVersion other) const { return version() <= other.version(); } + constexpr bool operator>=(const ProtocolVersion other) const { return version() >= other.version(); } + constexpr bool operator<(const ProtocolVersion other) const { return version() < other.version(); } + constexpr bool operator>(const ProtocolVersion other) const { return version() > other.version(); } + +public: // introduced features + // The 5th digit from right is dev version, for example, 2 in 0x0FDB00B061020000LL; + // It was used to identify a protocol change (e.g., interface change) between major/minor versions (say 5.1 and 5.2) + // We stopped using the dev version consistently in the past. + // To ensure binaries work across patch releases (e.g., 6.2.0 to 6.2.22), we require that the protocol version be + // the same for each of them. + PROTOCOL_VERSION_FEATURE(0x0FDB00A200090000LL, Watches); + PROTOCOL_VERSION_FEATURE(0x0FDB00A2000D0000LL, MovableCoordinatedState); + PROTOCOL_VERSION_FEATURE(0x0FDB00A340000000LL, ProcessID); + PROTOCOL_VERSION_FEATURE(0x0FDB00A400040000LL, OpenDatabase); + PROTOCOL_VERSION_FEATURE(0x0FDB00A446020000LL, Locality); + PROTOCOL_VERSION_FEATURE(0x0FDB00A460010000LL, MultiGenerationTLog); + PROTOCOL_VERSION_FEATURE(0x0FDB00A460010000LL, SharedMutations); + PROTOCOL_VERSION_FEATURE(0x0FDB00A551000000LL, InexpensiveMultiVersionClient); + PROTOCOL_VERSION_FEATURE(0x0FDB00A560010000LL, TagLocality); + PROTOCOL_VERSION_FEATURE(0x0FDB00B060000000LL, Fearless); + PROTOCOL_VERSION_FEATURE(0x0FDB00B061020000LL, EndpointAddrList); + PROTOCOL_VERSION_FEATURE(0x0FDB00B061030000LL, IPv6); + PROTOCOL_VERSION_FEATURE(0x0FDB00B061030000LL, TLogVersion); + PROTOCOL_VERSION_FEATURE(0x0FDB00B061070000LL, PseudoLocalities); + PROTOCOL_VERSION_FEATURE(0x0FDB00B061070000LL, ShardedTxsTags); + PROTOCOL_VERSION_FEATURE(0x0FDB00B062010001LL, TLogQueueEntryRef); + PROTOCOL_VERSION_FEATURE(0x0FDB00B062010001LL, GenerationRegVal); + PROTOCOL_VERSION_FEATURE(0x0FDB00B062010001LL, MovableCoordinatedStateV2); + PROTOCOL_VERSION_FEATURE(0x0FDB00B062010001LL, KeyServerValue); + PROTOCOL_VERSION_FEATURE(0x0FDB00B062010001LL, LogsValue); + PROTOCOL_VERSION_FEATURE(0x0FDB00B062010001LL, ServerTagValue); + PROTOCOL_VERSION_FEATURE(0x0FDB00B062010001LL, TagLocalityListValue); + PROTOCOL_VERSION_FEATURE(0x0FDB00B062010001LL, DatacenterReplicasValue); + PROTOCOL_VERSION_FEATURE(0x0FDB00B062010001LL, ProcessClassValue); + PROTOCOL_VERSION_FEATURE(0x0FDB00B062010001LL, WorkerListValue); + PROTOCOL_VERSION_FEATURE(0x0FDB00B062010001LL, BackupStartValue); + PROTOCOL_VERSION_FEATURE(0x0FDB00B062010001LL, LogRangeEncodeValue); + PROTOCOL_VERSION_FEATURE(0x0FDB00B062010001LL, HealthyZoneValue); + PROTOCOL_VERSION_FEATURE(0x0FDB00B062010001LL, DRBackupRanges); + PROTOCOL_VERSION_FEATURE(0x0FDB00B062010001LL, RegionConfiguration); + PROTOCOL_VERSION_FEATURE(0x0FDB00B062010001LL, ReplicationPolicy); + PROTOCOL_VERSION_FEATURE(0x0FDB00B062010001LL, BackupMutations); + PROTOCOL_VERSION_FEATURE(0x0FDB00B062010001LL, ClusterControllerPriorityInfo); + PROTOCOL_VERSION_FEATURE(0x0FDB00B062010001LL, ProcessIDFile); + PROTOCOL_VERSION_FEATURE(0x0FDB00B062010001LL, CloseUnusedConnection); + PROTOCOL_VERSION_FEATURE(0x0FDB00B063010000LL, DBCoreState); + PROTOCOL_VERSION_FEATURE(0x0FDB00B063010000LL, TagThrottleValue); + PROTOCOL_VERSION_FEATURE(0x0FDB00B063010000LL, StorageCacheValue); + PROTOCOL_VERSION_FEATURE(0x0FDB00B063010000LL, RestoreStatusValue); + PROTOCOL_VERSION_FEATURE(0x0FDB00B063010000LL, RestoreRequestValue); + PROTOCOL_VERSION_FEATURE(0x0FDB00B063010000LL, RestoreRequestDoneVersionValue); + PROTOCOL_VERSION_FEATURE(0x0FDB00B063010000LL, RestoreRequestTriggerValue); + PROTOCOL_VERSION_FEATURE(0x0FDB00B063010000LL, RestoreWorkerInterfaceValue); + PROTOCOL_VERSION_FEATURE(0x0FDB00B063010000LL, BackupProgressValue); + PROTOCOL_VERSION_FEATURE(0x0FDB00B063010000LL, KeyServerValueV2); + PROTOCOL_VERSION_FEATURE(0x0FDB00B063000000LL, UnifiedTLogSpilling); + PROTOCOL_VERSION_FEATURE(0x0FDB00B063010000LL, BackupWorker); + PROTOCOL_VERSION_FEATURE(0x0FDB00B063010000LL, ReportConflictingKeys); + PROTOCOL_VERSION_FEATURE(0x0FDB00B063010000LL, SmallEndpoints); + PROTOCOL_VERSION_FEATURE(0x0FDB00B063010000LL, CacheRole); + PROTOCOL_VERSION_FEATURE(0x0FDB00B070010000LL, StableInterfaces); + PROTOCOL_VERSION_FEATURE(0x0FDB00B070010001LL, ServerListValue); + PROTOCOL_VERSION_FEATURE(0x0FDB00B070010001LL, TagThrottleValueReason); + PROTOCOL_VERSION_FEATURE(0x0FDB00B070010001LL, SpanContext); + PROTOCOL_VERSION_FEATURE(0x0FDB00B070010001LL, TSS); + PROTOCOL_VERSION_FEATURE(0x0FDB00B071010000LL, ChangeFeed); + PROTOCOL_VERSION_FEATURE(0x0FDB00B071010000LL, BlobGranule); + PROTOCOL_VERSION_FEATURE(0x0FDB00B071010000LL, NetworkAddressHostnameFlag); + PROTOCOL_VERSION_FEATURE(0x0FDB00B071010000LL, StorageMetadata); + PROTOCOL_VERSION_FEATURE(0x0FDB00B071010000LL, PerpetualWiggleMetadata); + PROTOCOL_VERSION_FEATURE(0x0FDB00B071010000LL, Tenants); + PROTOCOL_VERSION_FEATURE(0x0FDB00B071010000LL, StorageInterfaceReadiness); + PROTOCOL_VERSION_FEATURE(0x0FDB00B071010000LL, ResolverPrivateMutations); +}; + +template <> +struct Traceable : std::true_type { + static std::string toString(const ProtocolVersion& protocolVersion) { + return format("0x%016lX", protocolVersion.version()); + } +}; + +constexpr ProtocolVersion currentProtocolVersion(currentProtocolVersionValue); +constexpr ProtocolVersion minInvalidProtocolVersion(minInvalidProtocolVersionValue); + +// This assert is intended to help prevent incrementing the leftmost digits accidentally. It will probably need to +// change when we reach version 10. +static_assert(currentProtocolVersion.version() < 0x0FDB00B100000000LL, "Unexpected protocol version"); + +// The last two bytes of the protocol version are currently masked out in compatibility checks. We do not use them, +// so prevent them from being inadvertently changed. +// +// We also do not modify the protocol version for patch releases, so prevent modifying the patch version digit. +static_assert((currentProtocolVersion.version() & 0xF0FFFFLL) == 0, "Unexpected protocol version"); + +// Downgrades must support at least one minor version. +static_assert(minInvalidProtocolVersion.version() >= + (currentProtocolVersion.version() & 0xFFFFFFFFFF000000LL) + 0x0000000002000000, + "Downgrades must support one minor version"); + +// The min invalid protocol version should be the smallest possible protocol version associated with a minor release +// version. +static_assert((minInvalidProtocolVersion.version() & 0xFFFFFFLL) == 0, "Unexpected min invalid protocol version"); \ No newline at end of file diff --git a/flow/ProtocolVersion.h b/flow/ProtocolVersion.h index 059cf883dc..2d968f03d8 100644 --- a/flow/ProtocolVersion.h +++ b/flow/ProtocolVersion.h @@ -22,23 +22,19 @@ #include "flow/Trace.h" #include -// This version impacts both communications and the deserialization of certain -// database and IKeyValueStore keys. +// This version impacts both communications and the deserialization of certain database and IKeyValueStore keys. // -// The convention is that 'x' and 'y' should match the major and minor version -// of the software, and 'z' should be 0. To make a change without a -// corresponding increase to the x.y version, increment the 'dev' digit. +// The convention is that 'x' and 'y' should match the major and minor version of the software, and 'z' should be 0. +// To make a change without a corresponding increase to the x.y version, increment the 'dev' digit. // -// The last 2 bytes (4 digits) of the protocol version do not affect -// compatibility. These two bytes are not currently used and should not be -// changed from 0. +// The last 2 bytes (4 digits) of the protocol version do not affect compatibility. These two bytes are not currently +// used and should not be changed from 0. // xyzdev // vvvv constexpr uint64_t currentProtocolVersionValue = 0x0FDB00B072000000LL; -// The first protocol version that cannot be downgraded from. Ordinarily, this -// will be two release versions larger than the current version, meaning that we -// only support downgrades between consecutive release versions. +// The first protocol version that cannot be downgraded from. Ordinarily, this will be two release versions larger +// than the current version, meaning that we only support downgrades between consecutive release versions. constexpr uint64_t minInvalidProtocolVersionValue = 0x0FDB00B074000000LL; // The lowest protocol version that can be downgraded to. @@ -53,11 +49,11 @@ constexpr uint64_t minCompatibleProtocolVersionValue = 0x0FDB00B071000000LL; constexpr bool has##x() const { return this->version() >= x ::protocolVersion; } \ static constexpr ProtocolVersion with##x() { return ProtocolVersion(x ::protocolVersion); } -// ProtocolVersion wraps a uint64_t to make it type safe. It will know about the -// current versions. The default constructor will initialize the version to 0 -// (which is an invalid version). ProtocolVersion objects should never be -// compared to version numbers directly. Instead one should always use the -// type-safe version types from which this class inherits all. +// ProtocolVersion wraps a uint64_t to make it type safe. It will know about the current versions. +// The default constructor will initialize the version to 0 (which is an invalid +// version). ProtocolVersion objects should never be compared to version numbers +// directly. Instead one should always use the type-safe version types from which +// this class inherits all. class ProtocolVersion { uint64_t _version; @@ -76,8 +72,7 @@ public: return (other.version() & compatibleProtocolVersionMask) == (version() & compatibleProtocolVersionMask); } - // Returns a normalized protocol version that will be the same for all - // compatible versions + // Returns a normalized protocol version that will be the same for all compatible versions constexpr ProtocolVersion normalizedVersion() const { return ProtocolVersion(_version & compatibleProtocolVersionMask); } @@ -96,10 +91,9 @@ public: constexpr void removeAllFlags() { _version = version(); } // comparison operators - // Comparison operators ignore the flags - this is because the version flags - // are stored in the most significant byte which can make comparison - // confusing. Also, generally, when one wants to compare versions, we are - // usually not interested in the flags. + // Comparison operators ignore the flags - this is because the version flags are stored in the + // most significant byte which can make comparison confusing. Also, generally, when one wants to + // compare versions, we are usually not interested in the flags. constexpr bool operator==(const ProtocolVersion other) const { return version() == other.version(); } constexpr bool operator!=(const ProtocolVersion other) const { return version() != other.version(); } constexpr bool operator<=(const ProtocolVersion other) const { return version() <= other.version(); } @@ -108,12 +102,11 @@ public: constexpr bool operator>(const ProtocolVersion other) const { return version() > other.version(); } public: // introduced features - // The 5th digit from right is dev version, for example, 2 in - // 0x0FDB00B061020000LL; It was used to identify a protocol change (e.g., - // interface change) between major/minor versions (say 5.1 and 5.2) We stopped - // using the dev version consistently in the past. To ensure binaries work - // across patch releases (e.g., 6.2.0 to 6.2.22), we require that the protocol - // version be the same for each of them. + // The 5th digit from right is dev version, for example, 2 in 0x0FDB00B061020000LL; + // It was used to identify a protocol change (e.g., interface change) between major/minor versions (say 5.1 and 5.2) + // We stopped using the dev version consistently in the past. + // To ensure binaries work across patch releases (e.g., 6.2.0 to 6.2.22), we require that the protocol version be + // the same for each of them. PROTOCOL_VERSION_FEATURE(0x0FDB00A200090000LL, Watches); PROTOCOL_VERSION_FEATURE(0x0FDB00A2000D0000LL, MovableCoordinatedState); PROTOCOL_VERSION_FEATURE(0x0FDB00A340000000LL, ProcessID); @@ -191,16 +184,14 @@ constexpr ProtocolVersion currentProtocolVersion(currentProtocolVersionValue); constexpr ProtocolVersion minInvalidProtocolVersion(minInvalidProtocolVersionValue); constexpr ProtocolVersion minCompatibleProtocolVersion(minCompatibleProtocolVersionValue); -// This assert is intended to help prevent incrementing the leftmost digits -// accidentally. It will probably need to change when we reach version 10. +// This assert is intended to help prevent incrementing the leftmost digits accidentally. It will probably need to +// change when we reach version 10. static_assert(currentProtocolVersion.version() < 0x0FDB00B100000000LL, "Unexpected protocol version"); -// The last two bytes of the protocol version are currently masked out in -// compatibility checks. We do not use them, so prevent them from being -// inadvertently changed. +// The last two bytes of the protocol version are currently masked out in compatibility checks. We do not use them, +// so prevent them from being inadvertently changed. // -// We also do not modify the protocol version for patch releases, so prevent -// modifying the patch version digit. +// We also do not modify the protocol version for patch releases, so prevent modifying the patch version digit. static_assert((currentProtocolVersion.version() & 0xF0FFFFLL) == 0, "Unexpected protocol version"); // Downgrades must support at least one minor version. @@ -208,6 +199,6 @@ static_assert(minInvalidProtocolVersion.version() >= (currentProtocolVersion.version() & 0xFFFFFFFFFF000000LL) + 0x0000000002000000, "Downgrades must support one minor version"); -// The min invalid protocol version should be the smallest possible protocol -// version associated with a minor release version. -static_assert((minInvalidProtocolVersion.version() & 0xFFFFFFLL) == 0, "Unexpected min invalid protocol version"); \ No newline at end of file +// The min invalid protocol version should be the smallest possible protocol version associated with a minor release +// version. +static_assert((minInvalidProtocolVersion.version() & 0xFFFFFFLL) == 0, "Unexpected min invalid protocol version"); From adfc4ca37988f4f99ce8772b46fdb6c3d2a5a7b6 Mon Sep 17 00:00:00 2001 From: "Bharadwaj V.R" Date: Thu, 14 Apr 2022 11:12:26 -0700 Subject: [PATCH 05/12] Revert accidental commit of a ProtocolVersion.h --- ProtocolVersion.h | 196 ---------------------------------------------- 1 file changed, 196 deletions(-) delete mode 100644 ProtocolVersion.h diff --git a/ProtocolVersion.h b/ProtocolVersion.h deleted file mode 100644 index a6561c91ab..0000000000 --- a/ProtocolVersion.h +++ /dev/null @@ -1,196 +0,0 @@ -/* - * ProtocolVersion.h - * - * This source file is part of the FoundationDB open source project - * - * Copyright 2013-2022 Apple Inc. and the FoundationDB project authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#pragma once -#include -#include "flow/Trace.h" - -// This version impacts both communications and the deserialization of certain database and IKeyValueStore keys. -// -// The convention is that 'x' and 'y' should match the major and minor version of the software, and 'z' should be 0. -// To make a change without a corresponding increase to the x.y version, increment the 'dev' digit. -// -// The last 2 bytes (4 digits) of the protocol version do not affect compatibility. These two bytes are not currently -// used and should not be changed from 0. -// xyzdev -// vvvv -constexpr uint64_t currentProtocolVersionValue = 0x0FDB00B072000000LL; - -// The first protocol version that cannot be downgraded from. Ordinarily, this will be two release versions larger -// than the current version, meaning that we only support downgrades between consecutive release versions. -constexpr uint64_t minInvalidProtocolVersionValue = 0x0FDB00B074000000LL; - -#define PROTOCOL_VERSION_FEATURE(v, x) \ - static_assert((v & 0xF0FFFFLL) == 0 || v < 0x0FDB00B071000000LL, "Unexpected feature protocol version"); \ - static_assert(v <= currentProtocolVersionValue, "Feature protocol version too large"); \ - struct x { \ - static constexpr uint64_t protocolVersion = v; \ - }; \ - constexpr bool has##x() const { return this->version() >= x ::protocolVersion; } \ - static constexpr ProtocolVersion with##x() { return ProtocolVersion(x ::protocolVersion); } - -// ProtocolVersion wraps a uint64_t to make it type safe. It will know about the current versions. -// The default constructor will initialize the version to 0 (which is an invalid -// version). ProtocolVersion objects should never be compared to version numbers -// directly. Instead one should always use the type-safe version types from which -// this class inherits all. -class ProtocolVersion { - uint64_t _version; - -public: // constants - static constexpr uint64_t versionFlagMask = 0x0FFFFFFFFFFFFFFFLL; - static constexpr uint64_t objectSerializerFlag = 0x1000000000000000LL; - static constexpr uint64_t compatibleProtocolVersionMask = 0xFFFFFFFFFFFF0000LL; - static constexpr uint64_t minValidProtocolVersion = 0x0FDB00A200060001LL; - -public: - constexpr explicit ProtocolVersion(uint64_t version) : _version(version) {} - constexpr ProtocolVersion() : _version(0) {} - - constexpr bool isCompatible(ProtocolVersion other) const { - return (other.version() & compatibleProtocolVersionMask) == (version() & compatibleProtocolVersionMask); - } - - // Returns a normalized protocol version that will be the same for all compatible versions - constexpr ProtocolVersion normalizedVersion() const { - return ProtocolVersion(_version & compatibleProtocolVersionMask); - } - constexpr bool isValid() const { return version() >= minValidProtocolVersion; } - - constexpr uint64_t version() const { return _version & versionFlagMask; } - constexpr uint64_t versionWithFlags() const { return _version; } - - constexpr bool hasObjectSerializerFlag() const { return (_version & objectSerializerFlag) > 0; } - constexpr void addObjectSerializerFlag() { _version = _version | objectSerializerFlag; } - constexpr void removeObjectSerializerFlag() { - _version = hasObjectSerializerFlag() ? _version ^ objectSerializerFlag : _version; - } - constexpr void removeAllFlags() { _version = version(); } - - // comparison operators - // Comparison operators ignore the flags - this is because the version flags are stored in the - // most significant byte which can make comparison confusing. Also, generally, when one wants to - // compare versions, we are usually not interested in the flags. - constexpr bool operator==(const ProtocolVersion other) const { return version() == other.version(); } - constexpr bool operator!=(const ProtocolVersion other) const { return version() != other.version(); } - constexpr bool operator<=(const ProtocolVersion other) const { return version() <= other.version(); } - constexpr bool operator>=(const ProtocolVersion other) const { return version() >= other.version(); } - constexpr bool operator<(const ProtocolVersion other) const { return version() < other.version(); } - constexpr bool operator>(const ProtocolVersion other) const { return version() > other.version(); } - -public: // introduced features - // The 5th digit from right is dev version, for example, 2 in 0x0FDB00B061020000LL; - // It was used to identify a protocol change (e.g., interface change) between major/minor versions (say 5.1 and 5.2) - // We stopped using the dev version consistently in the past. - // To ensure binaries work across patch releases (e.g., 6.2.0 to 6.2.22), we require that the protocol version be - // the same for each of them. - PROTOCOL_VERSION_FEATURE(0x0FDB00A200090000LL, Watches); - PROTOCOL_VERSION_FEATURE(0x0FDB00A2000D0000LL, MovableCoordinatedState); - PROTOCOL_VERSION_FEATURE(0x0FDB00A340000000LL, ProcessID); - PROTOCOL_VERSION_FEATURE(0x0FDB00A400040000LL, OpenDatabase); - PROTOCOL_VERSION_FEATURE(0x0FDB00A446020000LL, Locality); - PROTOCOL_VERSION_FEATURE(0x0FDB00A460010000LL, MultiGenerationTLog); - PROTOCOL_VERSION_FEATURE(0x0FDB00A460010000LL, SharedMutations); - PROTOCOL_VERSION_FEATURE(0x0FDB00A551000000LL, InexpensiveMultiVersionClient); - PROTOCOL_VERSION_FEATURE(0x0FDB00A560010000LL, TagLocality); - PROTOCOL_VERSION_FEATURE(0x0FDB00B060000000LL, Fearless); - PROTOCOL_VERSION_FEATURE(0x0FDB00B061020000LL, EndpointAddrList); - PROTOCOL_VERSION_FEATURE(0x0FDB00B061030000LL, IPv6); - PROTOCOL_VERSION_FEATURE(0x0FDB00B061030000LL, TLogVersion); - PROTOCOL_VERSION_FEATURE(0x0FDB00B061070000LL, PseudoLocalities); - PROTOCOL_VERSION_FEATURE(0x0FDB00B061070000LL, ShardedTxsTags); - PROTOCOL_VERSION_FEATURE(0x0FDB00B062010001LL, TLogQueueEntryRef); - PROTOCOL_VERSION_FEATURE(0x0FDB00B062010001LL, GenerationRegVal); - PROTOCOL_VERSION_FEATURE(0x0FDB00B062010001LL, MovableCoordinatedStateV2); - PROTOCOL_VERSION_FEATURE(0x0FDB00B062010001LL, KeyServerValue); - PROTOCOL_VERSION_FEATURE(0x0FDB00B062010001LL, LogsValue); - PROTOCOL_VERSION_FEATURE(0x0FDB00B062010001LL, ServerTagValue); - PROTOCOL_VERSION_FEATURE(0x0FDB00B062010001LL, TagLocalityListValue); - PROTOCOL_VERSION_FEATURE(0x0FDB00B062010001LL, DatacenterReplicasValue); - PROTOCOL_VERSION_FEATURE(0x0FDB00B062010001LL, ProcessClassValue); - PROTOCOL_VERSION_FEATURE(0x0FDB00B062010001LL, WorkerListValue); - PROTOCOL_VERSION_FEATURE(0x0FDB00B062010001LL, BackupStartValue); - PROTOCOL_VERSION_FEATURE(0x0FDB00B062010001LL, LogRangeEncodeValue); - PROTOCOL_VERSION_FEATURE(0x0FDB00B062010001LL, HealthyZoneValue); - PROTOCOL_VERSION_FEATURE(0x0FDB00B062010001LL, DRBackupRanges); - PROTOCOL_VERSION_FEATURE(0x0FDB00B062010001LL, RegionConfiguration); - PROTOCOL_VERSION_FEATURE(0x0FDB00B062010001LL, ReplicationPolicy); - PROTOCOL_VERSION_FEATURE(0x0FDB00B062010001LL, BackupMutations); - PROTOCOL_VERSION_FEATURE(0x0FDB00B062010001LL, ClusterControllerPriorityInfo); - PROTOCOL_VERSION_FEATURE(0x0FDB00B062010001LL, ProcessIDFile); - PROTOCOL_VERSION_FEATURE(0x0FDB00B062010001LL, CloseUnusedConnection); - PROTOCOL_VERSION_FEATURE(0x0FDB00B063010000LL, DBCoreState); - PROTOCOL_VERSION_FEATURE(0x0FDB00B063010000LL, TagThrottleValue); - PROTOCOL_VERSION_FEATURE(0x0FDB00B063010000LL, StorageCacheValue); - PROTOCOL_VERSION_FEATURE(0x0FDB00B063010000LL, RestoreStatusValue); - PROTOCOL_VERSION_FEATURE(0x0FDB00B063010000LL, RestoreRequestValue); - PROTOCOL_VERSION_FEATURE(0x0FDB00B063010000LL, RestoreRequestDoneVersionValue); - PROTOCOL_VERSION_FEATURE(0x0FDB00B063010000LL, RestoreRequestTriggerValue); - PROTOCOL_VERSION_FEATURE(0x0FDB00B063010000LL, RestoreWorkerInterfaceValue); - PROTOCOL_VERSION_FEATURE(0x0FDB00B063010000LL, BackupProgressValue); - PROTOCOL_VERSION_FEATURE(0x0FDB00B063010000LL, KeyServerValueV2); - PROTOCOL_VERSION_FEATURE(0x0FDB00B063000000LL, UnifiedTLogSpilling); - PROTOCOL_VERSION_FEATURE(0x0FDB00B063010000LL, BackupWorker); - PROTOCOL_VERSION_FEATURE(0x0FDB00B063010000LL, ReportConflictingKeys); - PROTOCOL_VERSION_FEATURE(0x0FDB00B063010000LL, SmallEndpoints); - PROTOCOL_VERSION_FEATURE(0x0FDB00B063010000LL, CacheRole); - PROTOCOL_VERSION_FEATURE(0x0FDB00B070010000LL, StableInterfaces); - PROTOCOL_VERSION_FEATURE(0x0FDB00B070010001LL, ServerListValue); - PROTOCOL_VERSION_FEATURE(0x0FDB00B070010001LL, TagThrottleValueReason); - PROTOCOL_VERSION_FEATURE(0x0FDB00B070010001LL, SpanContext); - PROTOCOL_VERSION_FEATURE(0x0FDB00B070010001LL, TSS); - PROTOCOL_VERSION_FEATURE(0x0FDB00B071010000LL, ChangeFeed); - PROTOCOL_VERSION_FEATURE(0x0FDB00B071010000LL, BlobGranule); - PROTOCOL_VERSION_FEATURE(0x0FDB00B071010000LL, NetworkAddressHostnameFlag); - PROTOCOL_VERSION_FEATURE(0x0FDB00B071010000LL, StorageMetadata); - PROTOCOL_VERSION_FEATURE(0x0FDB00B071010000LL, PerpetualWiggleMetadata); - PROTOCOL_VERSION_FEATURE(0x0FDB00B071010000LL, Tenants); - PROTOCOL_VERSION_FEATURE(0x0FDB00B071010000LL, StorageInterfaceReadiness); - PROTOCOL_VERSION_FEATURE(0x0FDB00B071010000LL, ResolverPrivateMutations); -}; - -template <> -struct Traceable : std::true_type { - static std::string toString(const ProtocolVersion& protocolVersion) { - return format("0x%016lX", protocolVersion.version()); - } -}; - -constexpr ProtocolVersion currentProtocolVersion(currentProtocolVersionValue); -constexpr ProtocolVersion minInvalidProtocolVersion(minInvalidProtocolVersionValue); - -// This assert is intended to help prevent incrementing the leftmost digits accidentally. It will probably need to -// change when we reach version 10. -static_assert(currentProtocolVersion.version() < 0x0FDB00B100000000LL, "Unexpected protocol version"); - -// The last two bytes of the protocol version are currently masked out in compatibility checks. We do not use them, -// so prevent them from being inadvertently changed. -// -// We also do not modify the protocol version for patch releases, so prevent modifying the patch version digit. -static_assert((currentProtocolVersion.version() & 0xF0FFFFLL) == 0, "Unexpected protocol version"); - -// Downgrades must support at least one minor version. -static_assert(minInvalidProtocolVersion.version() >= - (currentProtocolVersion.version() & 0xFFFFFFFFFF000000LL) + 0x0000000002000000, - "Downgrades must support one minor version"); - -// The min invalid protocol version should be the smallest possible protocol version associated with a minor release -// version. -static_assert((minInvalidProtocolVersion.version() & 0xFFFFFFLL) == 0, "Unexpected min invalid protocol version"); \ No newline at end of file From 3787ddae894d1962374063a440114237fe8d3abd Mon Sep 17 00:00:00 2001 From: "Bharadwaj V.R" Date: Fri, 15 Apr 2022 09:04:52 -0700 Subject: [PATCH 06/12] Add trace-based status reporting for server version --- fdbserver/ClusterRecovery.actor.cpp | 7 ++++++- fdbserver/ClusterRecovery.actor.h | 3 +++ fdbserver/Status.actor.cpp | 26 ++++++++++++++++++++++++-- 3 files changed, 33 insertions(+), 3 deletions(-) diff --git a/fdbserver/ClusterRecovery.actor.cpp b/fdbserver/ClusterRecovery.actor.cpp index 62696aa9cd..402b82896d 100644 --- a/fdbserver/ClusterRecovery.actor.cpp +++ b/fdbserver/ClusterRecovery.actor.cpp @@ -1409,7 +1409,7 @@ ACTOR Future clusterRecoveryCore(Reference self) { wait(self->cstate.read()); if (self->cstate.prevDBState.lowestCompatibleServerVersion > currentProtocolVersion) { - TraceEvent(SevWarnAlways, "IncompatbleServerVersion", self->dbgid).log(); + TraceEvent(SevWarnAlways, "IncompatibleServerVersion", self->dbgid).log(); throw internal_error(); } @@ -1478,6 +1478,11 @@ ACTOR Future clusterRecoveryCore(Reference self) { } wait(self->cstate.write(newState) || recoverAndEndEpoch); + TraceEvent("SWVersionCompatibilityChecked", self->dbgid) + .detail("NewestServerVersion", self->cstate.myDBState.newestServerVersion) + .detail("LowestCompatibleVersion", self->cstate.myDBState.lowestCompatibleServerVersion) + .trackLatest(self->swVersionCheckedEventHolder->trackingKey); + self->recoveryState = RecoveryState::RECRUITING; state std::vector seedServers; diff --git a/fdbserver/ClusterRecovery.actor.h b/fdbserver/ClusterRecovery.actor.h index d0deef911f..810bd35f7a 100644 --- a/fdbserver/ClusterRecovery.actor.h +++ b/fdbserver/ClusterRecovery.actor.h @@ -22,6 +22,7 @@ // When actually compiled (NO_INTELLISENSE), include the generated version of this file. In intellisense use the source // version. +#include "flow/Trace.h" #include #if defined(NO_INTELLISENSE) && !defined(FDBSERVER_CLUSTERRECOVERY_ACTOR_G_H) @@ -244,6 +245,7 @@ struct ClusterRecoveryData : NonCopyable, ReferenceCounted Future logger; + Reference swVersionCheckedEventHolder; Reference recoveredConfigEventHolder; Reference clusterRecoveryStateEventHolder; Reference clusterRecoveryGenerationsEventHolder; @@ -273,6 +275,7 @@ struct ClusterRecoveryData : NonCopyable, ReferenceCounted backupWorkerDoneRequests("BackupWorkerDoneRequests", cc), getLiveCommittedVersionRequests("GetLiveCommittedVersionRequests", cc), reportLiveCommittedVersionRequests("ReportLiveCommittedVersionRequests", cc), + swVersionCheckedEventHolder(makeReference("SWVersionCompatibilityChecked")), recoveredConfigEventHolder(makeReference("RecoveredConfig")) { clusterRecoveryStateEventHolder = makeReference( getRecoveryEventName(ClusterRecoveryEventType::CLUSTER_RECOVERY_STATE_EVENT_NAME)); diff --git a/fdbserver/Status.actor.cpp b/fdbserver/Status.actor.cpp index ee26a6970d..5fa1600ee7 100644 --- a/fdbserver/Status.actor.cpp +++ b/fdbserver/Status.actor.cpp @@ -1543,6 +1543,25 @@ ACTOR Future getNewestProtocolVersion(Database cx) { } } +ACTOR Future getNewestProtocolVersion(Database cx, WorkerDetails ccWorker) { + + try { + state Future swVersionF = timeoutError( + ccWorker.interf.eventLogRequest.getReply(EventLogRequest("SWVersionCompatibilityChecked"_sr)), 1.0); + + wait(success(swVersionF)); + const TraceEventFields& swVersionTrace = swVersionF.get(); + int64_t newestProtocolVersionValue = atoi(swVersionTrace.getValue("NewestServerVersion").c_str()); + return ProtocolVersion(newestProtocolVersionValue); + + } catch (Error& e) { + if (e.code() == error_code_actor_cancelled) + throw; + + return ProtocolVersion(); + } +} + struct LoadConfigurationResult { bool fullReplication; Optional healthyZone; @@ -2896,6 +2915,8 @@ ACTOR Future clusterGetStatus( messages.push_back(message); } + state ProtocolVersion newestProtocolVersion = wait(getNewestProtocolVersion(cx, ccWorker)); + // construct status information for cluster subsections state int statusCode = (int)RecoveryStatus::END; state JsonBuilderObject recoveryStateStatus = wait( @@ -2933,10 +2954,11 @@ ACTOR Future clusterGetStatus( statusObj["protocol_version"] = format("%" PRIx64, g_network->protocolVersion().version()); statusObj["connection_string"] = coordinators.ccr->getConnectionString().toString(); statusObj["bounce_impact"] = getBounceImpactInfo(statusCode); - - ProtocolVersion newestProtocolVersion = wait(getNewestProtocolVersion(cx)); statusObj["latest_server_version"] = format("%" PRIx64, newestProtocolVersion.version()); + // ProtocolVersion newestProtocolVersion = wait(getNewestProtocolVersion(cx)); + // statusObj["latest_server_version"] = format("%" PRIx64, newestProtocolVersion.version()); + state Optional configuration; state Optional loadResult; From 576957d3354e575e98903cf917bab3a28c734355 Mon Sep 17 00:00:00 2001 From: "Bharadwaj V.R" Date: Fri, 15 Apr 2022 09:52:42 -0700 Subject: [PATCH 07/12] Fix version status reporting --- fdbserver/Status.actor.cpp | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/fdbserver/Status.actor.cpp b/fdbserver/Status.actor.cpp index 5fa1600ee7..bff8b96b3e 100644 --- a/fdbserver/Status.actor.cpp +++ b/fdbserver/Status.actor.cpp @@ -1551,13 +1551,15 @@ ACTOR Future getNewestProtocolVersion(Database cx, WorkerDetail wait(success(swVersionF)); const TraceEventFields& swVersionTrace = swVersionF.get(); - int64_t newestProtocolVersionValue = atoi(swVersionTrace.getValue("NewestServerVersion").c_str()); + int64_t newestProtocolVersionValue = + std::stoull(swVersionTrace.getValue("NewestServerVersion").c_str(), nullptr, 16); return ProtocolVersion(newestProtocolVersionValue); } catch (Error& e) { if (e.code() == error_code_actor_cancelled) throw; + TraceEvent(SevWarnAlways, "SWVersionStatusFailed").error(e); return ProtocolVersion(); } } From ce5bcc0261e357842ec3b7ae0fa08053d1bc80f7 Mon Sep 17 00:00:00 2001 From: "Bharadwaj V.R" Date: Fri, 15 Apr 2022 10:57:11 -0700 Subject: [PATCH 08/12] Remove system key tracking newest server version --- fdbclient/SystemData.cpp | 2 -- fdbclient/SystemData.h | 8 -------- fdbserver/ApplyMetadataMutation.cpp | 3 +-- fdbserver/ClusterRecovery.actor.cpp | 6 ------ fdbserver/Status.actor.cpp | 18 ------------------ 5 files changed, 1 insertion(+), 36 deletions(-) diff --git a/fdbclient/SystemData.cpp b/fdbclient/SystemData.cpp index 23b6e0aa66..255fccc6c6 100644 --- a/fdbclient/SystemData.cpp +++ b/fdbclient/SystemData.cpp @@ -43,8 +43,6 @@ const KeyRangeRef keyServersKeyServersKeys(LiteralStringRef("\xff/keyServers/\xf LiteralStringRef("\xff/keyServers/\xff/keyServers0")); const KeyRef keyServersKeyServersKey = keyServersKeyServersKeys.begin; -const KeyRef newestProtocolVersionKey = LiteralStringRef("\xff/newestProtocolVersion"); - const Key keyServersKey(const KeyRef& k) { return k.withPrefix(keyServersPrefix); } diff --git a/fdbclient/SystemData.h b/fdbclient/SystemData.h index 8e8c3dc19e..4069c68162 100644 --- a/fdbclient/SystemData.h +++ b/fdbclient/SystemData.h @@ -70,14 +70,6 @@ void decodeKeyServersValue(std::map const& tag_uid, extern const KeyRef clusterIdKey; -// Key whose value stores the newest software version. -// This is not used to enforce software version compatibility. This is just -// a copy of information stored in the DBCoreState. That information is -// used during recovery to make sure that only versions of software -// that are compatible with the database are run. -// "\xff/newestProtocolVersion" -extern const KeyRef newestProtocolVersionKey; - // "\xff/checkpoint/[[UID]] := [[CheckpointMetaData]]" extern const KeyRef checkpointPrefix; const Key checkpointKeyFor(UID checkpointID); diff --git a/fdbserver/ApplyMetadataMutation.cpp b/fdbserver/ApplyMetadataMutation.cpp index 93cd5f07b2..90f987021f 100644 --- a/fdbserver/ApplyMetadataMutation.cpp +++ b/fdbserver/ApplyMetadataMutation.cpp @@ -594,8 +594,7 @@ private: m.param1.startsWith(applyMutationsAddPrefixRange.begin) || m.param1.startsWith(applyMutationsRemovePrefixRange.begin) || m.param1.startsWith(tagLocalityListPrefix) || m.param1.startsWith(serverTagHistoryPrefix) || - m.param1.startsWith(testOnlyTxnStateStorePrefixRange.begin) || m.param1 == clusterIdKey || - m.param1 == newestProtocolVersionKey) { + m.param1.startsWith(testOnlyTxnStateStorePrefixRange.begin) || m.param1 == clusterIdKey) { txnStateStore->set(KeyValueRef(m.param1, m.param2)); } diff --git a/fdbserver/ClusterRecovery.actor.cpp b/fdbserver/ClusterRecovery.actor.cpp index 402b82896d..e62ee0b060 100644 --- a/fdbserver/ClusterRecovery.actor.cpp +++ b/fdbserver/ClusterRecovery.actor.cpp @@ -1630,12 +1630,6 @@ ACTOR Future clusterRecoveryCore(Reference self) { tr.set(recoveryCommitRequest.arena, clusterIdKey, BinaryWriter::toValue(self->clusterId, Unversioned())); } - if (currentProtocolVersion > self->cstate.prevDBState.newestServerVersion) { - tr.set(recoveryCommitRequest.arena, - newestProtocolVersionKey, - BinaryWriter::toValue(currentProtocolVersion.version(), Unversioned())); - } - applyMetadataMutations(SpanID(), self->dbgid, recoveryCommitRequest.arena, diff --git a/fdbserver/Status.actor.cpp b/fdbserver/Status.actor.cpp index bff8b96b3e..d0d2334aac 100644 --- a/fdbserver/Status.actor.cpp +++ b/fdbserver/Status.actor.cpp @@ -1528,21 +1528,6 @@ ACTOR static Future logRangeWarningFetcher(Database cx, return Void(); } -ACTOR Future getNewestProtocolVersion(Database cx) { - state ReadYourWritesTransaction tr(cx); - loop { - try { - tr.setOption(FDBTransactionOptions::ACCESS_SYSTEM_KEYS); - tr.setOption(FDBTransactionOptions::LOCK_AWARE); - Optional newestProtocolVersion = wait(tr.get(newestProtocolVersionKey)); - ASSERT(newestProtocolVersion.present()); - return BinaryReader::fromStringRef(newestProtocolVersion.get(), Unversioned()); - } catch (Error& e) { - wait(tr.onError(e)); - } - } -} - ACTOR Future getNewestProtocolVersion(Database cx, WorkerDetails ccWorker) { try { @@ -2958,9 +2943,6 @@ ACTOR Future clusterGetStatus( statusObj["bounce_impact"] = getBounceImpactInfo(statusCode); statusObj["latest_server_version"] = format("%" PRIx64, newestProtocolVersion.version()); - // ProtocolVersion newestProtocolVersion = wait(getNewestProtocolVersion(cx)); - // statusObj["latest_server_version"] = format("%" PRIx64, newestProtocolVersion.version()); - state Optional configuration; state Optional loadResult; From 11f2827c8234eff1ca71847501add1b352c65bcd Mon Sep 17 00:00:00 2001 From: "Bharadwaj V.R" Date: Fri, 15 Apr 2022 13:57:58 -0700 Subject: [PATCH 09/12] Make names even across change and add lowest-compatible-version to status json --- fdbclient/Schemas.cpp | 3 ++- fdbserver/ClusterRecovery.actor.cpp | 22 ++++++++++----------- fdbserver/DBCoreState.h | 10 +++++----- fdbserver/Status.actor.cpp | 30 +++++++++++++++++++++++------ 4 files changed, 42 insertions(+), 23 deletions(-) diff --git a/fdbclient/Schemas.cpp b/fdbclient/Schemas.cpp index 3f9dc7d518..81cbb9c073 100644 --- a/fdbclient/Schemas.cpp +++ b/fdbclient/Schemas.cpp @@ -695,7 +695,8 @@ const KeyRef JSONSchemas::statusSchema = LiteralStringRef(R"statusSchema( }, "cluster_controller_timestamp":1415650089, "protocol_version":"fdb00a400050001", - "latest_server_version":"fdb00a500040001", + "newest_protocol_version":"fdb00a500040001", + "lowest_compatible_protocol_version":"fdb00a500040001", "connection_string":"a:a@127.0.0.1:4000", "full_replication":true, "maintenance_zone":"0ccb4e0fdbdb5583010f6b77d9d10ece", diff --git a/fdbserver/ClusterRecovery.actor.cpp b/fdbserver/ClusterRecovery.actor.cpp index e62ee0b060..d92ca2fe5a 100644 --- a/fdbserver/ClusterRecovery.actor.cpp +++ b/fdbserver/ClusterRecovery.actor.cpp @@ -1408,8 +1408,8 @@ ACTOR Future clusterRecoveryCore(Reference self) { wait(self->cstate.read()); - if (self->cstate.prevDBState.lowestCompatibleServerVersion > currentProtocolVersion) { - TraceEvent(SevWarnAlways, "IncompatibleServerVersion", self->dbgid).log(); + if (self->cstate.prevDBState.lowestCompatibleProtocolVersion > currentProtocolVersion) { + TraceEvent(SevWarnAlways, "IncompatibleProtocolVersion", self->dbgid).log(); throw internal_error(); } @@ -1469,18 +1469,18 @@ ACTOR Future clusterRecoveryCore(Reference self) { DBCoreState newState = self->cstate.myDBState; newState.recoveryCount++; newState.recoveryCount++; - if (self->cstate.prevDBState.newestServerVersion.isInvalidMagic() || - self->cstate.prevDBState.newestServerVersion < currentProtocolVersion) { - ASSERT(self->cstate.myDBState.lowestCompatibleServerVersion.isInvalidMagic() || - !self->cstate.myDBState.newestServerVersion.isInvalidMagic()); - newState.newestServerVersion = currentProtocolVersion; - newState.lowestCompatibleServerVersion = minCompatibleProtocolVersion; + if (self->cstate.prevDBState.newestProtocolVersion.isInvalidMagic() || + self->cstate.prevDBState.newestProtocolVersion < currentProtocolVersion) { + ASSERT(self->cstate.myDBState.lowestCompatibleProtocolVersion.isInvalidMagic() || + !self->cstate.myDBState.newestProtocolVersion.isInvalidMagic()); + newState.newestProtocolVersion = currentProtocolVersion; + newState.lowestCompatibleProtocolVersion = minCompatibleProtocolVersion; } wait(self->cstate.write(newState) || recoverAndEndEpoch); - TraceEvent("SWVersionCompatibilityChecked", self->dbgid) - .detail("NewestServerVersion", self->cstate.myDBState.newestServerVersion) - .detail("LowestCompatibleVersion", self->cstate.myDBState.lowestCompatibleServerVersion) + TraceEvent("ProtocolVersionCompatibilityChecked", self->dbgid) + .detail("NewestProtocolVersion", self->cstate.myDBState.newestProtocolVersion) + .detail("LowestCompatibleProtocolVersion", self->cstate.myDBState.lowestCompatibleProtocolVersion) .trackLatest(self->swVersionCheckedEventHolder->trackingKey); self->recoveryState = RecoveryState::RECRUITING; diff --git a/fdbserver/DBCoreState.h b/fdbserver/DBCoreState.h index 9fdfd910f0..0bb6e8936e 100644 --- a/fdbserver/DBCoreState.h +++ b/fdbserver/DBCoreState.h @@ -141,13 +141,13 @@ struct DBCoreState { DBRecoveryCount recoveryCount; // Increases with sequential successful recoveries. LogSystemType logSystemType; std::set pseudoLocalities; - ProtocolVersion newestServerVersion; - ProtocolVersion lowestCompatibleServerVersion; + ProtocolVersion newestProtocolVersion; + ProtocolVersion lowestCompatibleProtocolVersion; DBCoreState() : logRouterTags(0), txsTags(0), recoveryCount(0), logSystemType(LogSystemType::empty), - newestServerVersion(ProtocolVersion::invalidProtocolVersion), - lowestCompatibleServerVersion(ProtocolVersion::invalidProtocolVersion) {} + newestProtocolVersion(ProtocolVersion::invalidProtocolVersion), + lowestCompatibleProtocolVersion(ProtocolVersion::invalidProtocolVersion) {} std::vector getPriorCommittedLogServers() { std::vector priorCommittedLogServers; @@ -186,7 +186,7 @@ struct DBCoreState { serializer(ar, txsTags); } if (ar.protocolVersion().hasSWVersionTracking()) { - serializer(ar, newestServerVersion, lowestCompatibleServerVersion); + serializer(ar, newestProtocolVersion, lowestCompatibleProtocolVersion); } } else if (ar.isDeserializing) { tLogs.push_back(CoreTLogSet()); diff --git a/fdbserver/Status.actor.cpp b/fdbserver/Status.actor.cpp index d0d2334aac..29f7a55799 100644 --- a/fdbserver/Status.actor.cpp +++ b/fdbserver/Status.actor.cpp @@ -1528,7 +1528,20 @@ ACTOR static Future logRangeWarningFetcher(Database cx, return Void(); } -ACTOR Future getNewestProtocolVersion(Database cx, WorkerDetails ccWorker) { +struct ProtocolVersionData { + ProtocolVersion runningProtocolVersion; + ProtocolVersion newestProtocolVersion; + ProtocolVersion lowestCompatibleProtocolVersion; + ProtocolVersionData() + : runningProtocolVersion(currentProtocolVersion), newestProtocolVersion(ProtocolVersion()), + lowestCompatibleProtocolVersion(ProtocolVersion()) {} + + ProtocolVersionData(uint64_t newestProtocolVersionValue, uint64_t lowestCompatibleProtocolVersionValue) + : runningProtocolVersion(currentProtocolVersion), newestProtocolVersion(newestProtocolVersionValue), + lowestCompatibleProtocolVersion(lowestCompatibleProtocolVersionValue) {} +}; + +ACTOR Future getNewestProtocolVersion(Database cx, WorkerDetails ccWorker) { try { state Future swVersionF = timeoutError( @@ -1537,15 +1550,18 @@ ACTOR Future getNewestProtocolVersion(Database cx, WorkerDetail wait(success(swVersionF)); const TraceEventFields& swVersionTrace = swVersionF.get(); int64_t newestProtocolVersionValue = - std::stoull(swVersionTrace.getValue("NewestServerVersion").c_str(), nullptr, 16); - return ProtocolVersion(newestProtocolVersionValue); + std::stoull(swVersionTrace.getValue("NewestProtocolVersion").c_str(), nullptr, 16); + int64_t lowestCompatibleProtocolVersionValue = + std::stoull(swVersionTrace.getValue("LowestCompatibleProtocolVersion").c_str(), nullptr, 16); + return ProtocolVersionData(newestProtocolVersionValue, lowestCompatibleProtocolVersionValue); } catch (Error& e) { if (e.code() == error_code_actor_cancelled) throw; TraceEvent(SevWarnAlways, "SWVersionStatusFailed").error(e); - return ProtocolVersion(); + + return ProtocolVersionData(); } } @@ -2902,7 +2918,7 @@ ACTOR Future clusterGetStatus( messages.push_back(message); } - state ProtocolVersion newestProtocolVersion = wait(getNewestProtocolVersion(cx, ccWorker)); + state ProtocolVersionData protocolVersion = wait(getNewestProtocolVersion(cx, ccWorker)); // construct status information for cluster subsections state int statusCode = (int)RecoveryStatus::END; @@ -2941,7 +2957,9 @@ ACTOR Future clusterGetStatus( statusObj["protocol_version"] = format("%" PRIx64, g_network->protocolVersion().version()); statusObj["connection_string"] = coordinators.ccr->getConnectionString().toString(); statusObj["bounce_impact"] = getBounceImpactInfo(statusCode); - statusObj["latest_server_version"] = format("%" PRIx64, newestProtocolVersion.version()); + statusObj["newest_protocol_version"] = format("%" PRIx64, protocolVersion.newestProtocolVersion.version()); + statusObj["lowest_compatible_protocol_version"] = + format("%" PRIx64, protocolVersion.lowestCompatibleProtocolVersion.version()); state Optional configuration; state Optional loadResult; From a2449041eafdc521516b1c242c3978c71390d31a Mon Sep 17 00:00:00 2001 From: "Bharadwaj V.R" Date: Wed, 20 Apr 2022 08:45:53 -0700 Subject: [PATCH 10/12] Fix formatting of ProtocolVersion.h --- flow/ProtocolVersion.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/flow/ProtocolVersion.h b/flow/ProtocolVersion.h index e92e075288..c5479c1852 100644 --- a/flow/ProtocolVersion.h +++ b/flow/ProtocolVersion.h @@ -171,7 +171,7 @@ public: // introduced features PROTOCOL_VERSION_FEATURE(0x0FDB00B071010000LL, StorageInterfaceReadiness); PROTOCOL_VERSION_FEATURE(0x0FDB00B071010000LL, ResolverPrivateMutations); PROTOCOL_VERSION_FEATURE(0x0FDB00B072000000LL, OTELSpanContext); - PROTOCOL_VERSION_FEATURE(0x0FDB00B072000000LL, SWVersionTracking); + PROTOCOL_VERSION_FEATURE(0x0FDB00B072000000LL, SWVersionTracking); }; template <> From 96ab7c025434413f14c9fa73006ff9bfdb0f810d Mon Sep 17 00:00:00 2001 From: "Bharadwaj V.R" Date: Wed, 20 Apr 2022 08:52:14 -0700 Subject: [PATCH 11/12] Update fdbserver/Status.actor.cpp Co-authored-by: Trevor Clinkenbeard --- fdbserver/Status.actor.cpp | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/fdbserver/Status.actor.cpp b/fdbserver/Status.actor.cpp index 29f7a55799..3d802d385f 100644 --- a/fdbserver/Status.actor.cpp +++ b/fdbserver/Status.actor.cpp @@ -1533,8 +1533,7 @@ struct ProtocolVersionData { ProtocolVersion newestProtocolVersion; ProtocolVersion lowestCompatibleProtocolVersion; ProtocolVersionData() - : runningProtocolVersion(currentProtocolVersion), newestProtocolVersion(ProtocolVersion()), - lowestCompatibleProtocolVersion(ProtocolVersion()) {} + : runningProtocolVersion(currentProtocolVersion) {} ProtocolVersionData(uint64_t newestProtocolVersionValue, uint64_t lowestCompatibleProtocolVersionValue) : runningProtocolVersion(currentProtocolVersion), newestProtocolVersion(newestProtocolVersionValue), From 8a0ce5bfc75223f037ffd167e48e96abb048d0b3 Mon Sep 17 00:00:00 2001 From: "Bharadwaj V.R" Date: Wed, 20 Apr 2022 09:17:28 -0700 Subject: [PATCH 12/12] Rename isInvalidMagic and fix formatting --- fdbserver/ClusterRecovery.actor.cpp | 6 +++--- fdbserver/Status.actor.cpp | 3 +-- flow/ProtocolVersion.h | 2 +- 3 files changed, 5 insertions(+), 6 deletions(-) diff --git a/fdbserver/ClusterRecovery.actor.cpp b/fdbserver/ClusterRecovery.actor.cpp index e000e4a2c2..38fd147780 100644 --- a/fdbserver/ClusterRecovery.actor.cpp +++ b/fdbserver/ClusterRecovery.actor.cpp @@ -1469,10 +1469,10 @@ ACTOR Future clusterRecoveryCore(Reference self) { DBCoreState newState = self->cstate.myDBState; newState.recoveryCount++; newState.recoveryCount++; - if (self->cstate.prevDBState.newestProtocolVersion.isInvalidMagic() || + if (self->cstate.prevDBState.newestProtocolVersion.isInvalid() || self->cstate.prevDBState.newestProtocolVersion < currentProtocolVersion) { - ASSERT(self->cstate.myDBState.lowestCompatibleProtocolVersion.isInvalidMagic() || - !self->cstate.myDBState.newestProtocolVersion.isInvalidMagic()); + ASSERT(self->cstate.myDBState.lowestCompatibleProtocolVersion.isInvalid() || + !self->cstate.myDBState.newestProtocolVersion.isInvalid()); newState.newestProtocolVersion = currentProtocolVersion; newState.lowestCompatibleProtocolVersion = minCompatibleProtocolVersion; } diff --git a/fdbserver/Status.actor.cpp b/fdbserver/Status.actor.cpp index 3d802d385f..f086265d49 100644 --- a/fdbserver/Status.actor.cpp +++ b/fdbserver/Status.actor.cpp @@ -1532,8 +1532,7 @@ struct ProtocolVersionData { ProtocolVersion runningProtocolVersion; ProtocolVersion newestProtocolVersion; ProtocolVersion lowestCompatibleProtocolVersion; - ProtocolVersionData() - : runningProtocolVersion(currentProtocolVersion) {} + ProtocolVersionData() : runningProtocolVersion(currentProtocolVersion) {} ProtocolVersionData(uint64_t newestProtocolVersionValue, uint64_t lowestCompatibleProtocolVersionValue) : runningProtocolVersion(currentProtocolVersion), newestProtocolVersion(newestProtocolVersionValue), diff --git a/flow/ProtocolVersion.h b/flow/ProtocolVersion.h index c5479c1852..19700dff7f 100644 --- a/flow/ProtocolVersion.h +++ b/flow/ProtocolVersion.h @@ -78,7 +78,7 @@ public: } constexpr bool isValid() const { return version() >= minValidProtocolVersion; } - constexpr bool isInvalidMagic() const { return version() == invalidProtocolVersion; } + constexpr bool isInvalid() const { return version() == invalidProtocolVersion; } constexpr uint64_t version() const { return _version & versionFlagMask; } constexpr uint64_t versionWithFlags() const { return _version; }