From 6afce017441cf2747bc414a2238a1c24d992381b Mon Sep 17 00:00:00 2001 From: mpilman Date: Wed, 30 Jan 2019 13:53:23 -0800 Subject: [PATCH] Implementation complete (not yet working) --- fdbclient/ClientDBInfo.h | 1 + fdbclient/ClientWorkerInterface.h | 3 +++ fdbclient/ClusterInterface.h | 8 ++++++++ fdbclient/CoordinationInterface.h | 2 ++ fdbclient/FDBTypes.h | 1 + fdbclient/MasterProxyInterface.h | 11 +++++++++++ fdbclient/StorageServerInterface.h | 23 ++++++++++++++++++----- fdbrpc/FlowTransport.actor.cpp | 14 +++++++++++--- fdbrpc/FlowTransport.h | 2 ++ fdbrpc/Locality.h | 1 + fdbrpc/PerfMetric.h | 1 + fdbrpc/ReplicationPolicy.h | 2 +- fdbrpc/fdbrpc.h | 3 ++- fdbserver/ClusterRecruitmentInterface.h | 11 +++++++++++ fdbserver/CoordinationInterface.h | 8 ++++++++ fdbserver/LeaderElection.actor.cpp | 6 ++++-- fdbserver/MasterInterface.h | 6 ++++++ fdbserver/NetworkTest.h | 3 +++ fdbserver/ResolverInterface.h | 7 +++++++ fdbserver/RestoreInterface.h | 3 +++ fdbserver/ServerDBInfo.h | 1 + fdbserver/TLogInterface.h | 10 ++++++++++ fdbserver/TesterInterface.actor.h | 3 +++ fdbserver/WorkerInterface.actor.h | 15 +++++++++++++++ fdbserver/sqlite/btree.h | 3 +++ fdbserver/sqlite/sqlite3.amalgamation.c | 3 +++ fdbserver/workloads/Sideband.actor.cpp | 2 ++ flow/Arena.h | 3 +++ flow/Error.h | 2 ++ flow/FileIdentifier.h | 4 +--- flow/IRandom.h | 2 ++ flow/ObjectSerializer.h | 2 +- flow/Trace.h | 1 + flow/flow.h | 3 +++ flow/serialize.h | 11 ++++++++++- tests/TestRunner/TestRunner.py | 3 +++ 36 files changed, 167 insertions(+), 17 deletions(-) diff --git a/fdbclient/ClientDBInfo.h b/fdbclient/ClientDBInfo.h index 3468d0e2fa..cbefda7be0 100644 --- a/fdbclient/ClientDBInfo.h +++ b/fdbclient/ClientDBInfo.h @@ -27,6 +27,7 @@ // ClientDBInfo is all the information needed by a database client to access the database // It is returned (and kept up to date) by the OpenDatabaseRequest interface of ClusterInterface struct ClientDBInfo { + constexpr static FileIdentifier file_identifier = 5355080; UID id; // Changes each time anything else changes vector< MasterProxyInterface > proxies; double clientTxnInfoSampleRate; diff --git a/fdbclient/ClientWorkerInterface.h b/fdbclient/ClientWorkerInterface.h index 5ea6c499cd..4e3efde268 100644 --- a/fdbclient/ClientWorkerInterface.h +++ b/fdbclient/ClientWorkerInterface.h @@ -30,6 +30,7 @@ // Streams from WorkerInterface that are safe and useful to call from a client. // A ClientWorkerInterface is embedded as the first element of a WorkerInterface. struct ClientWorkerInterface { + constexpr static FileIdentifier file_identifier = 12418152; RequestStream< struct RebootRequest > reboot; RequestStream< struct ProfilerRequest > profiler; @@ -45,6 +46,7 @@ struct ClientWorkerInterface { }; struct RebootRequest { + constexpr static FileIdentifier file_identifier = 11913957; bool deleteData; bool checkData; @@ -57,6 +59,7 @@ struct RebootRequest { }; struct ProfilerRequest { + constexpr static FileIdentifier file_identifier = 15437862; ReplyPromise reply; enum class Type : std::int8_t { diff --git a/fdbclient/ClusterInterface.h b/fdbclient/ClusterInterface.h index c48529fa03..6130739b30 100644 --- a/fdbclient/ClusterInterface.h +++ b/fdbclient/ClusterInterface.h @@ -131,6 +131,7 @@ struct ClientVersionRef { }; struct OpenDatabaseRequest { + constexpr static FileIdentifier file_identifier = 2799502; // Sent by the native API to the cluster controller to open a database and track client // info changes. Returns immediately if the current client info id is different from // knownClientInfoID; otherwise returns when it next changes (or perhaps after a long interval) @@ -152,6 +153,7 @@ struct OpenDatabaseRequest { }; struct SystemFailureStatus { + constexpr static FileIdentifier file_identifier = 3194108; NetworkAddressList addresses; FailureStatus status; @@ -175,6 +177,7 @@ struct FailureMonitoringRequest { // The failureInformationVersion returned in reply should be passed back to the // next request to facilitate delta compression of the failure information. + constexpr static FileIdentifier file_identifier = 5867851; Optional senderStatus; Version failureInformationVersion; NetworkAddressList addresses; @@ -187,6 +190,7 @@ struct FailureMonitoringRequest { }; struct FailureMonitoringReply { + constexpr static FileIdentifier file_identifier = 6820325; VectorRef< SystemFailureStatus > changes; Version failureInformationVersion; bool allOthersFailed; // If true, changes are relative to all servers being failed, otherwise to the version given in the request @@ -201,6 +205,7 @@ struct FailureMonitoringReply { }; struct StatusRequest { + constexpr static FileIdentifier file_identifier = 14419140; ReplyPromise< struct StatusReply > reply; template @@ -210,6 +215,7 @@ struct StatusRequest { }; struct StatusReply { + constexpr static FileIdentifier file_identifier = 9980504; StatusObject statusObj; std::string statusStr; @@ -235,6 +241,7 @@ struct StatusReply { }; struct GetClientWorkersRequest { + constexpr static FileIdentifier file_identifier = 10771791; ReplyPromise> reply; GetClientWorkersRequest() {} @@ -246,6 +253,7 @@ struct GetClientWorkersRequest { }; struct ForceRecoveryRequest { + constexpr static FileIdentifier file_identifier = 14821350; Key dcId; ReplyPromise reply; diff --git a/fdbclient/CoordinationInterface.h b/fdbclient/CoordinationInterface.h index b8f91f01ee..7eee9d6aae 100644 --- a/fdbclient/CoordinationInterface.h +++ b/fdbclient/CoordinationInterface.h @@ -91,6 +91,7 @@ private: }; struct LeaderInfo { + constexpr static FileIdentifier file_identifier = 8338794; UID changeID; static const uint64_t mask = ~(127ll << 57); Value serializedInfo; @@ -126,6 +127,7 @@ struct LeaderInfo { }; struct GetLeaderRequest { + constexpr static FileIdentifier file_identifier = 214727; Key key; UID knownLeader; ReplyPromise< Optional > reply; diff --git a/fdbclient/FDBTypes.h b/fdbclient/FDBTypes.h index eb24e84814..f234ac4f25 100644 --- a/fdbclient/FDBTypes.h +++ b/fdbclient/FDBTypes.h @@ -540,6 +540,7 @@ struct Traceable : std::true_type { }; struct KeyValueStoreType { + constexpr static FileIdentifier file_identifier = 6560359; // These enumerated values are stored in the database configuration, so can NEVER be changed. Only add new ones just before END. enum StoreType { SSD_BTREE_V1, diff --git a/fdbclient/MasterProxyInterface.h b/fdbclient/MasterProxyInterface.h index 10721871a5..9d51ae1c18 100644 --- a/fdbclient/MasterProxyInterface.h +++ b/fdbclient/MasterProxyInterface.h @@ -30,6 +30,7 @@ #include "flow/Stats.h" struct MasterProxyInterface { + constexpr static FileIdentifier file_identifier = 8954922; enum { LocationAwareLoadBalance = 1 }; enum { AlwaysFresh = 1 }; @@ -71,6 +72,7 @@ struct MasterProxyInterface { }; struct CommitID { + constexpr static FileIdentifier file_identifier = 14254927; Version version; // returns invalidVersion if transaction conflicts uint16_t txnBatchId; Optional metadataVersion; @@ -85,6 +87,7 @@ struct CommitID { }; struct CommitTransactionRequest : TimedRequest { + constexpr static FileIdentifier file_identifier = 93948; enum { FLAG_IS_LOCK_AWARE = 0x1, FLAG_FIRST_IN_BATCH = 0x2 @@ -121,6 +124,7 @@ static inline int getBytes( CommitTransactionRequest const& r ) { } struct GetReadVersionReply { + constexpr static FileIdentifier file_identifier = 15709388; Version version; bool locked; Optional metadataVersion; @@ -132,6 +136,7 @@ struct GetReadVersionReply { }; struct GetReadVersionRequest : TimedRequest { + constexpr static FileIdentifier file_identifier = 838566; enum { PRIORITY_SYSTEM_IMMEDIATE = 15 << 24, // Highest possible priority, always executed even if writes are otherwise blocked PRIORITY_DEFAULT = 8 << 24, @@ -161,6 +166,7 @@ struct GetReadVersionRequest : TimedRequest { }; struct GetKeyServerLocationsReply { + constexpr static FileIdentifier file_identifier = 10636023; Arena arena; vector>> results; @@ -171,6 +177,7 @@ struct GetKeyServerLocationsReply { }; struct GetKeyServerLocationsRequest { + constexpr static FileIdentifier file_identifier = 9144680; Arena arena; KeyRef begin; Optional end; @@ -188,6 +195,7 @@ struct GetKeyServerLocationsRequest { }; struct GetRawCommittedVersionRequest { + constexpr static FileIdentifier file_identifier = 12954034; Optional debugID; ReplyPromise reply; @@ -200,6 +208,7 @@ struct GetRawCommittedVersionRequest { }; struct GetStorageServerRejoinInfoReply { + constexpr static FileIdentifier file_identifier = 9469225; Version version; Tag tag; Optional newTag; @@ -213,6 +222,7 @@ struct GetStorageServerRejoinInfoReply { }; struct GetStorageServerRejoinInfoRequest { + constexpr static FileIdentifier file_identifier = 994279; UID id; Optional dcId; ReplyPromise< GetStorageServerRejoinInfoReply > reply; @@ -227,6 +237,7 @@ struct GetStorageServerRejoinInfoRequest { }; struct TxnStateRequest { + constexpr static FileIdentifier file_identifier = 15250781; Arena arena; VectorRef data; Sequence sequence; diff --git a/fdbclient/StorageServerInterface.h b/fdbclient/StorageServerInterface.h index d535a81196..d016c2765c 100644 --- a/fdbclient/StorageServerInterface.h +++ b/fdbclient/StorageServerInterface.h @@ -30,11 +30,8 @@ #include "flow/Stats.h" struct StorageServerInterface { - enum { - BUSY_ALLOWED = 0, - BUSY_FORCE = 1, - BUSY_LOCAL = 2 - }; + constexpr static FileIdentifier file_identifier = 15302073; + enum { BUSY_ALLOWED = 0, BUSY_FORCE = 1, BUSY_LOCAL = 2 }; enum { LocationAwareLoadBalance = 1 }; enum { AlwaysFresh = 0 }; @@ -114,6 +111,7 @@ struct ServerCacheInfo { }; struct GetValueReply : public LoadBalancedReply { + constexpr static FileIdentifier file_identifier = 1378929; Optional value; GetValueReply() {} @@ -126,6 +124,7 @@ struct GetValueReply : public LoadBalancedReply { }; struct GetValueRequest : TimedRequest { + constexpr static FileIdentifier file_identifier = 8454530; Key key; Version version; Optional debugID; @@ -141,6 +140,7 @@ struct GetValueRequest : TimedRequest { }; struct WatchValueRequest { + constexpr static FileIdentifier file_identifier = 14747733; Key key; Optional value; Version version; @@ -157,6 +157,7 @@ struct WatchValueRequest { }; struct GetKeyValuesReply : public LoadBalancedReply { + constexpr static FileIdentifier file_identifier = 1783066; Arena arena; VectorRef data; Version version; // useful when latestVersion was requested @@ -169,6 +170,7 @@ struct GetKeyValuesReply : public LoadBalancedReply { }; struct GetKeyValuesRequest : TimedRequest { + constexpr static FileIdentifier file_identifier = 6795746; Arena arena; KeySelectorRef begin, end; Version version; // or latestVersion @@ -185,6 +187,7 @@ struct GetKeyValuesRequest : TimedRequest { }; struct GetKeyReply : public LoadBalancedReply { + constexpr static FileIdentifier file_identifier = 11226513; KeySelector sel; GetKeyReply() {} @@ -197,6 +200,7 @@ struct GetKeyReply : public LoadBalancedReply { }; struct GetKeyRequest : TimedRequest { + constexpr static FileIdentifier file_identifier = 10457870; Arena arena; KeySelectorRef sel; Version version; // or latestVersion @@ -212,6 +216,7 @@ struct GetKeyRequest : TimedRequest { }; struct GetShardStateRequest { + constexpr static FileIdentifier file_identifier = 15860168; enum waitMode { NO_WAIT = 0, FETCHING = 1, @@ -231,6 +236,7 @@ struct GetShardStateRequest { }; struct StorageMetrics { + constexpr static FileIdentifier file_identifier = 13622226; int64_t bytes; // total storage int64_t bytesPerKSecond; // network bandwidth (average over 10s) int64_t iosPerKSecond; @@ -283,6 +289,7 @@ struct StorageMetrics { struct WaitMetricsRequest { // Waits for any of the given minimum or maximum metrics to be exceeded, and then returns the current values // Send a reversed range for min, max to receive an immediate report + constexpr static FileIdentifier file_identifier = 1795961; Arena arena; KeyRangeRef keys; StorageMetrics min, max; @@ -301,6 +308,7 @@ struct WaitMetricsRequest { }; struct SplitMetricsReply { + constexpr static FileIdentifier file_identifier = 11530792; Standalone> splits; StorageMetrics used; @@ -311,6 +319,7 @@ struct SplitMetricsReply { }; struct SplitMetricsRequest { + constexpr static FileIdentifier file_identifier = 10463876; Arena arena; KeyRangeRef keys; StorageMetrics limits; @@ -329,6 +338,7 @@ struct SplitMetricsRequest { }; struct GetPhysicalMetricsReply { + constexpr static FileIdentifier file_identifier = 15491478; StorageMetrics load; StorageMetrics free; StorageMetrics capacity; @@ -340,6 +350,7 @@ struct GetPhysicalMetricsReply { }; struct GetPhysicalMetricsRequest { + constexpr static FileIdentifier file_identifier = 13290999; ReplyPromise reply; template @@ -350,6 +361,7 @@ struct GetPhysicalMetricsRequest { struct StorageQueuingMetricsRequest { // SOMEDAY: Send threshold value to avoid polling faster than the information changes? + constexpr static FileIdentifier file_identifier = 3978640; ReplyPromise reply; template @@ -359,6 +371,7 @@ struct StorageQueuingMetricsRequest { }; struct StorageQueuingMetricsReply { + constexpr static FileIdentifier file_identifier = 7633366; double localTime; int64_t instanceID; // changes if bytesDurable and bytesInput reset int64_t bytesDurable, bytesInput; diff --git a/fdbrpc/FlowTransport.actor.cpp b/fdbrpc/FlowTransport.actor.cpp index 60bff615dd..822a4a0b2b 100644 --- a/fdbrpc/FlowTransport.actor.cpp +++ b/fdbrpc/FlowTransport.actor.cpp @@ -329,7 +329,8 @@ struct Peer : NonCopyable { } pkt.connectPacketLength = sizeof(pkt) - sizeof(pkt.connectPacketLength); - pkt.protocolVersion = currentProtocolVersion; + pkt.protocolVersion = + g_network->useObjectSerializer() ? addObjectSerializerFlag(currentProtocolVersion) : currentProtocolVersion; pkt.connectionId = transport->transportId; PacketBuffer* pb_first = new PacketBuffer; @@ -1013,19 +1014,26 @@ static PacketID sendPacket( TransportData* self, ISerializeSource const& what, c TEST(true); // "Loopback" delivery // SOMEDAY: Would it be better to avoid (de)serialization by doing this check in flow? + Standalone copy; + if (g_network->useObjectSerializer()) { + ObjectWriter wr; + what.serializeBinaryWriter(wr); + copy = wr.toStringRef(); + } else { BinaryWriter wr( AssumeVersion(currentProtocolVersion) ); // we don't need to send using an object writer here. This is a loopback delivery // and therefore it is guaranteed that both versions will have exactly the // same structures - so the backwards compatability capabilities are never needed // here. what.serializeBinaryWriter(wr); - Standalone copy = wr.toValue(); + copy = wr.toStringRef(); + } #if VALGRIND VALGRIND_CHECK_MEM_IS_DEFINED(copy.begin(), copy.size()); #endif deliver(self, destination, ArenaReader(copy.arena(), copy, AssumeVersion(currentProtocolVersion)), false, - false); + g_network->useObjectSerializer()); return (PacketID)NULL; } else { diff --git a/fdbrpc/FlowTransport.h b/fdbrpc/FlowTransport.h index ef1f4a186c..63830296ef 100644 --- a/fdbrpc/FlowTransport.h +++ b/fdbrpc/FlowTransport.h @@ -24,12 +24,14 @@ #include #include "flow/network.h" +#include "flow/FileIdentifier.h" #pragma pack(push, 4) class Endpoint { public: // Endpoint represents a particular service (e.g. a serialized Promise or PromiseStream) // An endpoint is either "local" (used for receiving data) or "remote" (used for sending data) + constexpr static FileIdentifier file_identifier = 10618805; typedef UID Token; NetworkAddressList addresses; Token token; diff --git a/fdbrpc/Locality.h b/fdbrpc/Locality.h index 1b8b17ec2f..d7cd8da6e9 100644 --- a/fdbrpc/Locality.h +++ b/fdbrpc/Locality.h @@ -25,6 +25,7 @@ #include "flow/flow.h" struct ProcessClass { + constexpr static FileIdentifier file_identifier = 6697257; // This enum is stored in restartInfo.ini for upgrade tests, so be very careful about changing the existing items! enum ClassType { UnsetClass, StorageClass, TransactionClass, ResolutionClass, TesterClass, ProxyClass, MasterClass, StatelessClass, LogClass, ClusterControllerClass, LogRouterClass, DataDistributorClass, CoordinatorClass, RatekeeperClass, InvalidClass = -1 }; enum Fitness { BestFit, GoodFit, UnsetFit, OkayFit, WorstFit, ExcludeFit, NeverAssign }; //cannot be larger than 7 because of leader election mask diff --git a/fdbrpc/PerfMetric.h b/fdbrpc/PerfMetric.h index 8beea3e1fd..c5e07ff5ea 100644 --- a/fdbrpc/PerfMetric.h +++ b/fdbrpc/PerfMetric.h @@ -29,6 +29,7 @@ using std::vector; struct PerfMetric { + constexpr static FileIdentifier file_identifier = 5980618; PerfMetric() : m_name(""), m_value(0), m_averaged(false), m_format_code( "%.3g" ) {} PerfMetric( std::string name, double value, bool averaged ) : m_name(name), m_value(value), m_averaged(averaged), m_format_code( "%.3g" ) {} PerfMetric( std::string name, double value, bool averaged, std::string format_code ) : m_name(name), m_value(value), m_averaged(averaged), m_format_code(format_code) {} diff --git a/fdbrpc/ReplicationPolicy.h b/fdbrpc/ReplicationPolicy.h index 3d6d969144..7963b5b1c1 100644 --- a/fdbrpc/ReplicationPolicy.h +++ b/fdbrpc/ReplicationPolicy.h @@ -392,7 +392,7 @@ void serializeReplicationPolicy(Ar& ar, Reference& policy) { policy = Reference(pointer); } else if(name == LiteralStringRef("And")) { - PolicyAnd* pointer = new PolicyAnd({}); + PolicyAnd* pointer = new PolicyAnd{}; pointer->serialize(ar); policy = Reference(pointer); } diff --git a/fdbrpc/fdbrpc.h b/fdbrpc/fdbrpc.h index 4d466d77ab..26993e5514 100644 --- a/fdbrpc/fdbrpc.h +++ b/fdbrpc/fdbrpc.h @@ -112,8 +112,9 @@ template class ReplyPromise sealed { public: + constexpr static FileIdentifier file_identifier = (0x2 << 24) | FileIdentifierFor::value; template - void send(U && value) const { + void send(U&& value) const { sav->send(std::forward(value)); } template diff --git a/fdbserver/ClusterRecruitmentInterface.h b/fdbserver/ClusterRecruitmentInterface.h index 1af8ec9241..a79121cd5d 100644 --- a/fdbserver/ClusterRecruitmentInterface.h +++ b/fdbserver/ClusterRecruitmentInterface.h @@ -72,6 +72,7 @@ struct ClusterControllerFullInterface { }; struct RecruitFromConfigurationRequest { + constexpr static FileIdentifier file_identifier = 2023046; DatabaseConfiguration configuration; bool recruitSeedServers; int maxOldLogRouters; @@ -88,6 +89,7 @@ struct RecruitFromConfigurationRequest { }; struct RecruitFromConfigurationReply { + constexpr static FileIdentifier file_identifier = 2224085; vector tLogs; vector satelliteTLogs; vector proxies; @@ -106,6 +108,7 @@ struct RecruitFromConfigurationReply { }; struct RecruitRemoteFromConfigurationRequest { + constexpr static FileIdentifier file_identifier = 3235995; DatabaseConfiguration configuration; Optional dcId; int logRouterCount; @@ -122,6 +125,7 @@ struct RecruitRemoteFromConfigurationRequest { }; struct RecruitRemoteFromConfigurationReply { + constexpr static FileIdentifier file_identifier = 9091392; vector remoteTLogs; vector logRouters; @@ -132,6 +136,7 @@ struct RecruitRemoteFromConfigurationReply { }; struct RecruitStorageReply { + constexpr static FileIdentifier file_identifier = 15877089; WorkerInterface worker; ProcessClass processClass; @@ -142,6 +147,7 @@ struct RecruitStorageReply { }; struct RecruitStorageRequest { + constexpr static FileIdentifier file_identifier = 905920; std::vector>> excludeMachines; //< Don't recruit any of these machines std::vector excludeAddresses; //< Don't recruit any of these addresses std::vector>> includeDCs; @@ -155,6 +161,7 @@ struct RecruitStorageRequest { }; struct RegisterWorkerReply { + constexpr static FileIdentifier file_identifier = 16475696; ProcessClass processClass; ClusterControllerPriorityInfo priorityInfo; @@ -168,6 +175,7 @@ struct RegisterWorkerReply { }; struct RegisterWorkerRequest { + constexpr static FileIdentifier file_identifier = 14332605; WorkerInterface wi; ProcessClass initialClass; ProcessClass processClass; @@ -189,6 +197,7 @@ struct RegisterWorkerRequest { }; struct GetWorkersRequest { + constexpr static FileIdentifier file_identifier = 1254174; enum { TESTER_CLASS_ONLY = 0x1, NON_EXCLUDED_PROCESSES_ONLY = 0x2 }; int flags; @@ -204,6 +213,7 @@ struct GetWorkersRequest { }; struct RegisterMasterRequest { + constexpr static FileIdentifier file_identifier = 10773445; UID id; LocalityData mi; LogSystemConfig logSystemConfig; @@ -231,6 +241,7 @@ struct RegisterMasterRequest { }; struct GetServerDBInfoRequest { + constexpr static FileIdentifier file_identifier = 9467438; UID knownServerInfoID; Standalone> issues; std::vector incompatiblePeers; diff --git a/fdbserver/CoordinationInterface.h b/fdbserver/CoordinationInterface.h index 931fec5500..7d05639594 100644 --- a/fdbserver/CoordinationInterface.h +++ b/fdbserver/CoordinationInterface.h @@ -25,6 +25,7 @@ #include "fdbclient/CoordinationInterface.h" struct GenerationRegInterface { + constexpr static FileIdentifier file_identifier = 16726744; RequestStream< struct GenerationRegReadRequest > read; RequestStream< struct GenerationRegWriteRequest > write; @@ -52,6 +53,7 @@ struct GenerationRegInterface { }; struct UniqueGeneration { + constexpr static FileIdentifier file_identifier = 16684234; uint64_t generation; UID uid; UniqueGeneration() : generation(0) {} @@ -71,6 +73,7 @@ struct UniqueGeneration { }; struct GenerationRegReadRequest { + constexpr static FileIdentifier file_identifier = 8975311; Key key; UniqueGeneration gen; ReplyPromise reply; @@ -83,6 +86,7 @@ struct GenerationRegReadRequest { }; struct GenerationRegReadReply { + constexpr static FileIdentifier file_identifier = 12623609; Optional value; UniqueGeneration gen, rgen; GenerationRegReadReply() {} @@ -94,6 +98,7 @@ struct GenerationRegReadReply { }; struct GenerationRegWriteRequest { + constexpr static FileIdentifier file_identifier = 3521510; KeyValue kv; UniqueGeneration gen; ReplyPromise< UniqueGeneration > reply; @@ -116,6 +121,7 @@ struct LeaderElectionRegInterface : ClientLeaderRegInterface { }; struct CandidacyRequest { + constexpr static FileIdentifier file_identifier = 14473958; Key key; LeaderInfo myInfo; UID knownLeader, prevChangeID; @@ -131,6 +137,7 @@ struct CandidacyRequest { }; struct LeaderHeartbeatRequest { + constexpr static FileIdentifier file_identifier = 9495992; Key key; LeaderInfo myInfo; UID prevChangeID; @@ -146,6 +153,7 @@ struct LeaderHeartbeatRequest { }; struct ForwardRequest { + constexpr static FileIdentifier file_identifier = 13570359; Key key; Value conn; // a cluster connection string ReplyPromise reply; diff --git a/fdbserver/LeaderElection.actor.cpp b/fdbserver/LeaderElection.actor.cpp index 473498a445..de5e92cbfc 100644 --- a/fdbserver/LeaderElection.actor.cpp +++ b/fdbserver/LeaderElection.actor.cpp @@ -75,8 +75,10 @@ ACTOR Future changeLeaderCoordinators( ServerCoordinators coordinators, Va return Void(); } -ACTOR Future tryBecomeLeaderInternal( ServerCoordinators coordinators, Value proposedSerializedInterface, Reference> outSerializedLeader, bool hasConnected, Reference> asyncPriorityInfo ) { - state Reference>>> nominees( new AsyncVar>>() ); +ACTOR Future tryBecomeLeaderInternal(ServerCoordinators coordinators, Value proposedSerializedInterface, + Reference> outSerializedLeader, bool hasConnected, + Reference> asyncPriorityInfo) { + state Reference>>> nominees(new AsyncVar>>()); state LeaderInfo myInfo; state Future candidacies; state bool iAmLeader = false; diff --git a/fdbserver/MasterInterface.h b/fdbserver/MasterInterface.h index ac53d8ef2e..e6c35e12e3 100644 --- a/fdbserver/MasterInterface.h +++ b/fdbserver/MasterInterface.h @@ -31,6 +31,7 @@ typedef uint64_t DBRecoveryCount; struct MasterInterface { + constexpr static FileIdentifier file_identifier = 5979145; LocalityData locality; RequestStream< ReplyPromise > waitFailure; RequestStream< struct TLogRejoinRequest > tlogRejoin; // sent by tlog (whether or not rebooted) to communicate with a new master @@ -54,6 +55,7 @@ struct MasterInterface { }; struct TLogRejoinRequest { + constexpr static FileIdentifier file_identifier = 15692200; TLogInterface myInterface; ReplyPromise reply; // false means someone else registered, so we should re-register. true means this master is recovered, so don't send again to the same master. @@ -66,6 +68,7 @@ struct TLogRejoinRequest { }; struct ChangeCoordinatorsRequest { + constexpr static FileIdentifier file_identifier = 13605416; Standalone newConnectionString; ReplyPromise reply; // normally throws even on success! @@ -79,6 +82,7 @@ struct ChangeCoordinatorsRequest { }; struct ResolverMoveRef { + constexpr static FileIdentifier file_identifier = 11945475; KeyRangeRef range; int dest; @@ -104,6 +108,7 @@ struct ResolverMoveRef { }; struct GetCommitVersionReply { + constexpr static FileIdentifier file_identifier = 3568822; Standalone> resolverChanges; Version resolverChangesVersion; Version version; @@ -120,6 +125,7 @@ struct GetCommitVersionReply { }; struct GetCommitVersionRequest { + constexpr static FileIdentifier file_identifier = 16683181; uint64_t requestNum; uint64_t mostRecentProcessedRequestNum; UID requestingProxy; diff --git a/fdbserver/NetworkTest.h b/fdbserver/NetworkTest.h index 37057bf3a1..94ae95d0fe 100644 --- a/fdbserver/NetworkTest.h +++ b/fdbserver/NetworkTest.h @@ -24,6 +24,7 @@ #include "fdbclient/FDBTypes.h" #include "fdbrpc/fdbrpc.h" +#include "flow/FileIdentifier.h" struct NetworkTestInterface { RequestStream< struct NetworkTestRequest > test; @@ -33,6 +34,7 @@ struct NetworkTestInterface { }; struct NetworkTestRequest { + constexpr static FileIdentifier file_identifier = 4146513; Key key; uint32_t replySize; ReplyPromise reply; @@ -45,6 +47,7 @@ struct NetworkTestRequest { }; struct NetworkTestReply { + constexpr static FileIdentifier file_identifier = 14465374; Value value; NetworkTestReply() {} NetworkTestReply( Value value ) : value(value) {} diff --git a/fdbserver/ResolverInterface.h b/fdbserver/ResolverInterface.h index 6bc32ae62a..c011ce02c8 100644 --- a/fdbserver/ResolverInterface.h +++ b/fdbserver/ResolverInterface.h @@ -25,6 +25,7 @@ #include "fdbclient/FDBTypes.h" struct ResolverInterface { + constexpr static FileIdentifier file_identifier = 1755944; enum { LocationAwareLoadBalance = 1 }; enum { AlwaysFresh = 1 }; @@ -54,6 +55,7 @@ struct ResolverInterface { }; struct StateTransactionRef { + constexpr static FileIdentifier file_identifier = 6150271; StateTransactionRef() {} StateTransactionRef(const bool committed, VectorRef const& mutations) : committed(committed), mutations(mutations) {} StateTransactionRef(Arena &p, const StateTransactionRef &toCopy) : committed(toCopy.committed), mutations(p, toCopy.mutations) {} @@ -70,6 +72,7 @@ struct StateTransactionRef { }; struct ResolveTransactionBatchReply { + constexpr static FileIdentifier file_identifier = 15472264; Arena arena; VectorRef committed; Optional debugID; @@ -83,6 +86,7 @@ struct ResolveTransactionBatchReply { }; struct ResolveTransactionBatchRequest { + constexpr static FileIdentifier file_identifier = 16462858; Arena arena; Version prevVersion; @@ -100,6 +104,7 @@ struct ResolveTransactionBatchRequest { }; struct ResolutionMetricsRequest { + constexpr static FileIdentifier file_identifier = 11663527; ReplyPromise reply; template @@ -109,6 +114,7 @@ struct ResolutionMetricsRequest { }; struct ResolutionSplitReply { + constexpr static FileIdentifier file_identifier = 12137765; Key key; int64_t used; template @@ -119,6 +125,7 @@ struct ResolutionSplitReply { }; struct ResolutionSplitRequest { + constexpr static FileIdentifier file_identifier = 167535; KeyRange range; int64_t offset; bool front; diff --git a/fdbserver/RestoreInterface.h b/fdbserver/RestoreInterface.h index cf11686384..fc893c17f5 100644 --- a/fdbserver/RestoreInterface.h +++ b/fdbserver/RestoreInterface.h @@ -28,6 +28,7 @@ #include "fdbrpc/Locality.h" struct RestoreInterface { + constexpr static FileIdentifier file_identifier = 13398189; RequestStream< struct TestRequest > test; bool operator == (RestoreInterface const& r) const { return id() == r.id(); } @@ -46,6 +47,7 @@ struct RestoreInterface { }; struct TestRequest { + constexpr static FileIdentifier file_identifier = 14404487; int testData; ReplyPromise< struct TestReply > reply; @@ -59,6 +61,7 @@ struct TestRequest { }; struct TestReply { + constexpr static FileIdentifier file_identifier = 12075719; int replyData; TestReply() : replyData(0) {} diff --git a/fdbserver/ServerDBInfo.h b/fdbserver/ServerDBInfo.h index 2a5ca96135..fa50cd4d67 100644 --- a/fdbserver/ServerDBInfo.h +++ b/fdbserver/ServerDBInfo.h @@ -31,6 +31,7 @@ #include "fdbserver/LatencyBandConfig.h" struct ServerDBInfo { + constexpr static FileIdentifier file_identifier = 13838807; // This structure contains transient information which is broadcast to all workers for a database, // permitting them to communicate with each other. It is not available to the client. This mechanism // (see GetServerDBInfoRequest) is closely parallel to OpenDatabaseRequest for the client. diff --git a/fdbserver/TLogInterface.h b/fdbserver/TLogInterface.h index 87d2f92b31..39e9907322 100644 --- a/fdbserver/TLogInterface.h +++ b/fdbserver/TLogInterface.h @@ -29,6 +29,7 @@ #include struct TLogInterface { + constexpr static FileIdentifier file_identifier = 16308510; enum { LocationAwareLoadBalance = 1 }; enum { AlwaysFresh = 1 }; @@ -71,6 +72,7 @@ struct TLogInterface { }; struct TLogRecoveryFinishedRequest { + constexpr static FileIdentifier file_identifier = 8818668; ReplyPromise reply; TLogRecoveryFinishedRequest() {} @@ -82,6 +84,7 @@ struct TLogRecoveryFinishedRequest { }; struct TLogLockResult { + constexpr static FileIdentifier file_identifier = 11822027; Version end; Version knownCommittedVersion; @@ -92,6 +95,7 @@ struct TLogLockResult { }; struct TLogConfirmRunningRequest { + constexpr static FileIdentifier file_identifier = 10929130; Optional debugID; ReplyPromise reply; @@ -136,6 +140,7 @@ struct VerUpdateRef { }; struct TLogPeekReply { + constexpr static FileIdentifier file_identifier = 11365689; Arena arena; StringRef messages; Version end; @@ -151,6 +156,7 @@ struct TLogPeekReply { }; struct TLogPeekRequest { + constexpr static FileIdentifier file_identifier = 11001131; Arena arena; Version begin; Tag tag; @@ -168,6 +174,7 @@ struct TLogPeekRequest { }; struct TLogPopRequest { + constexpr static FileIdentifier file_identifier = 5556423; Arena arena; Version to; Version durableKnownCommittedVersion; @@ -201,6 +208,7 @@ struct TagMessagesRef { }; struct TLogCommitRequest { + constexpr static FileIdentifier file_identifier = 4022206; Arena arena; Version prevVersion, version, knownCommittedVersion, minKnownCommittedVersion; @@ -219,6 +227,7 @@ struct TLogCommitRequest { }; struct TLogQueuingMetricsRequest { + constexpr static FileIdentifier file_identifier = 7798476; ReplyPromise reply; template @@ -228,6 +237,7 @@ struct TLogQueuingMetricsRequest { }; struct TLogQueuingMetricsReply { + constexpr static FileIdentifier file_identifier = 12206626; double localTime; int64_t instanceID; // changes if bytesDurable and bytesInput reset int64_t bytesDurable, bytesInput; diff --git a/fdbserver/TesterInterface.actor.h b/fdbserver/TesterInterface.actor.h index e8ee49d342..afdc62be70 100644 --- a/fdbserver/TesterInterface.actor.h +++ b/fdbserver/TesterInterface.actor.h @@ -33,6 +33,7 @@ #include "flow/actorcompiler.h" // has to be last include struct WorkloadInterface { + constexpr static FileIdentifier file_identifier = 4454551; RequestStream> setup; RequestStream> start; RequestStream> check; @@ -48,6 +49,7 @@ struct WorkloadInterface { }; struct WorkloadRequest { + constexpr static FileIdentifier file_identifier = 8121024; Arena arena; StringRef title; int timeout; @@ -79,6 +81,7 @@ struct WorkloadRequest { }; struct TesterInterface { + constexpr static FileIdentifier file_identifier = 4465210; RequestStream recruitments; UID id() const { return recruitments.getEndpoint().token; } diff --git a/fdbserver/WorkerInterface.actor.h b/fdbserver/WorkerInterface.actor.h index d684109f0e..082dc072cf 100644 --- a/fdbserver/WorkerInterface.actor.h +++ b/fdbserver/WorkerInterface.actor.h @@ -41,6 +41,7 @@ #define DUMPTOKEN( name ) TraceEvent("DumpToken", recruited.id()).detail("Name", #name).detail("Token", name.getEndpoint().token) struct WorkerInterface { + constexpr static FileIdentifier file_identifier = 14712718; ClientWorkerInterface clientInterface; LocalityData locality; RequestStream< struct InitializeTLogRequest > tLog; @@ -89,6 +90,7 @@ struct WorkerDetails { }; struct InitializeTLogRequest { + constexpr static FileIdentifier file_identifier = 15604392; UID recruitmentID; LogSystemConfig recoverFrom; Version recoverAt; @@ -116,6 +118,7 @@ struct InitializeTLogRequest { }; struct InitializeLogRouterRequest { + constexpr static FileIdentifier file_identifier = 2976228; uint64_t recoveryCount; Tag routerTag; Version startVersion; @@ -132,6 +135,7 @@ struct InitializeLogRouterRequest { // FIXME: Rename to InitializeMasterRequest, etc struct RecruitMasterRequest { + constexpr static FileIdentifier file_identifier = 12684574; Arena arena; LifetimeToken lifetime; bool forceRecovery; @@ -147,6 +151,7 @@ struct RecruitMasterRequest { }; struct InitializeMasterProxyRequest { + constexpr static FileIdentifier file_identifier = 10344153; MasterInterface master; uint64_t recoveryCount; Version recoveryTransactionVersion; @@ -184,6 +189,7 @@ struct InitializeRatekeeperRequest { }; struct InitializeResolverRequest { + constexpr static FileIdentifier file_identifier = 7413317; uint64_t recoveryCount; int proxyCount; int resolverCount; @@ -196,6 +202,7 @@ struct InitializeResolverRequest { }; struct InitializeStorageReply { + constexpr static FileIdentifier file_identifier = 10390645; StorageServerInterface interf; Version addedVersion; @@ -206,6 +213,7 @@ struct InitializeStorageReply { }; struct InitializeStorageRequest { + constexpr static FileIdentifier file_identifier = 16665642; Tag seedTag; //< If this server will be passed to seedShardServers, this will be a tag, otherwise it is invalidTag UID reqId; UID interfaceId; @@ -219,6 +227,7 @@ struct InitializeStorageRequest { }; struct TraceBatchDumpRequest { + constexpr static FileIdentifier file_identifier = 8184121; ReplyPromise reply; template @@ -228,6 +237,7 @@ struct TraceBatchDumpRequest { }; struct LoadedReply { + constexpr static FileIdentifier file_identifier = 9956350; Standalone payload; UID id; @@ -238,6 +248,7 @@ struct LoadedReply { }; struct LoadedPingRequest { + constexpr static FileIdentifier file_identifier = 4590979; UID id; bool loadReply; Standalone payload; @@ -250,6 +261,7 @@ struct LoadedPingRequest { }; struct CoordinationPingMessage { + constexpr static FileIdentifier file_identifier = 9982747; UID clusterControllerId; int64_t timeStep; @@ -263,6 +275,7 @@ struct CoordinationPingMessage { }; struct SetMetricsLogRateRequest { + constexpr static FileIdentifier file_identifier = 4245995; uint32_t metricsLogsPerSecond; SetMetricsLogRateRequest() : metricsLogsPerSecond( 1 ) {} @@ -275,6 +288,7 @@ struct SetMetricsLogRateRequest { }; struct EventLogRequest { + constexpr static FileIdentifier file_identifier = 122319; bool getLastError; Standalone eventName; ReplyPromise< TraceEventFields > reply; @@ -309,6 +323,7 @@ struct DebugEntryRef { }; struct DiskStoreRequest { + constexpr static FileIdentifier file_identifier = 1986262; bool includePartialStores; ReplyPromise>> reply; diff --git a/fdbserver/sqlite/btree.h b/fdbserver/sqlite/btree.h index b0c748673d..bdf98c14bb 100644 --- a/fdbserver/sqlite/btree.h +++ b/fdbserver/sqlite/btree.h @@ -15,6 +15,9 @@ */ #ifndef _BTREE_H_ #define _BTREE_H_ +#ifndef NDEBUG +#define NDEBUG +#endif /* TODO: This definition is just included so other modules compile. It ** needs to be revisited. diff --git a/fdbserver/sqlite/sqlite3.amalgamation.c b/fdbserver/sqlite/sqlite3.amalgamation.c index 7f1376151e..c9df4ee492 100644 --- a/fdbserver/sqlite/sqlite3.amalgamation.c +++ b/fdbserver/sqlite/sqlite3.amalgamation.c @@ -1,3 +1,6 @@ +#ifndef NDEBUG +#define NDEBUG +#endif #ifndef NDEBUG #define SQLITE_DEBUG 1 #endif diff --git a/fdbserver/workloads/Sideband.actor.cpp b/fdbserver/workloads/Sideband.actor.cpp index 3877e64554..f474f7bec6 100644 --- a/fdbserver/workloads/Sideband.actor.cpp +++ b/fdbserver/workloads/Sideband.actor.cpp @@ -24,6 +24,7 @@ #include "flow/actorcompiler.h" // This must be the last #include. struct SidebandMessage { + constexpr static FileIdentifier file_identifier = 11862046; uint64_t key; Version commitVersion; @@ -37,6 +38,7 @@ struct SidebandMessage { }; struct SidebandInterface { + constexpr static FileIdentifier file_identifier = 15950544; RequestStream updates; UID id() const { return updates.getEndpoint().token; } diff --git a/flow/Arena.h b/flow/Arena.h index 4c1e60c6a0..53b096b207 100644 --- a/flow/Arena.h +++ b/flow/Arena.h @@ -27,6 +27,7 @@ #include "flow/Error.h" #include "flow/Trace.h" #include "flow/ObjectSerializerTraits.h" +#include "flow/FileIdentifier.h" #include #include #include @@ -512,6 +513,7 @@ extern std::string format(const char* form, ...); #pragma pack( push, 4 ) class StringRef { public: + constexpr static FileIdentifier file_identifier = 13300811; StringRef() : data(0), length(0) {} StringRef( Arena& p, const StringRef& toCopy ) : data( new (p) uint8_t[toCopy.size()] ), length( toCopy.size() ) { memcpy( (void*)data, toCopy.data, length ); @@ -778,6 +780,7 @@ template class VectorRef { public: using value_type = T; + static constexpr FileIdentifier file_identifier = (0x8 << 24) | FileIdentifierFor::value; // T must be trivially destructible (and copyable)! VectorRef() : data(0), m_size(0), m_capacity(0) {} diff --git a/flow/Error.h b/flow/Error.h index 18a3a1c9e6..f298752b08 100644 --- a/flow/Error.h +++ b/flow/Error.h @@ -29,6 +29,7 @@ #include #include "flow/Platform.h" #include "flow/Knobs.h" +#include "flow/FileIdentifier.h" #include "flow/ObjectSerializerTraits.h" enum { invalid_error_code = 0xffff }; @@ -41,6 +42,7 @@ public: class Error { public: + constexpr static FileIdentifier file_identifier = 14065384; int code() const { return error_code; } const char* name() const; const char* what() const; diff --git a/flow/FileIdentifier.h b/flow/FileIdentifier.h index 4bbb1d9d14..394356b55d 100644 --- a/flow/FileIdentifier.h +++ b/flow/FileIdentifier.h @@ -25,9 +25,7 @@ using FileIdentifier = uint32_t; template struct FileIdentifierFor { - //constexpr static FileIdentifier value = T::file_identifier; - // TODO: use file identifiers for different types - constexpr static FileIdentifier value = 0xffffff; + constexpr static FileIdentifier value = T::file_identifier; }; template <> diff --git a/flow/IRandom.h b/flow/IRandom.h index dbf14ac619..ca77984dca 100644 --- a/flow/IRandom.h +++ b/flow/IRandom.h @@ -23,6 +23,7 @@ #pragma once #include "flow/Platform.h" +#include "flow/FileIdentifier.h" #include "flow/ObjectSerializerTraits.h" #include #if (defined(__APPLE__)) @@ -35,6 +36,7 @@ class UID { uint64_t part[2]; public: + constexpr static FileIdentifier file_identifier = 15597147; UID() { part[0] = part[1] = 0; } UID( uint64_t a, uint64_t b ) { part[0]=a; part[1]=b; } std::string toString() const; diff --git a/flow/ObjectSerializer.h b/flow/ObjectSerializer.h index b10be8e835..73e1c3c9bc 100644 --- a/flow/ObjectSerializer.h +++ b/flow/ObjectSerializer.h @@ -101,7 +101,7 @@ class ObjectWriter { public: template void serialize(FileIdentifier file_identifier, Items const&... items) { - ASSERT(data = nullptr); // object serializer can only serialize one object + ASSERT(data == nullptr); // object serializer can only serialize one object int allocations = 0; auto allocator = [this, &allocations](size_t size_) { ++allocations; diff --git a/flow/Trace.h b/flow/Trace.h index c09e29374d..b937a9ea0a 100644 --- a/flow/Trace.h +++ b/flow/Trace.h @@ -57,6 +57,7 @@ enum Severity { class TraceEventFields { public: + constexpr static FileIdentifier file_identifier = 11262274; typedef std::pair Field; typedef std::vector FieldContainer; typedef FieldContainer::const_iterator FieldIterator; diff --git a/flow/flow.h b/flow/flow.h index 013a6e8ecd..0aac95eb0e 100644 --- a/flow/flow.h +++ b/flow/flow.h @@ -44,6 +44,7 @@ #include "flow/Deque.h" #include "flow/ThreadPrimitives.h" #include "flow/network.h" +#include "flow/FileIdentifier.h" #include @@ -114,6 +115,7 @@ Standalone concatenate( Iter b, Iter const& e ) { class Void { public: + constexpr static FileIdentifier file_identifier = 2010442; template void serialize(Ar&) {} }; @@ -123,6 +125,7 @@ class Never {}; template class ErrorOr { public: + constexpr static FileIdentifier file_identifier = (0x1 << 24) | FileIdentifierFor::value; ErrorOr() : error(default_error_or()) {} ErrorOr(Error const& error) : error(error) {} ErrorOr(const ErrorOr& o) : error(o.error) { diff --git a/flow/serialize.h b/flow/serialize.h index adaf56c560..3d82e761f2 100644 --- a/flow/serialize.h +++ b/flow/serialize.h @@ -715,6 +715,7 @@ private: struct ISerializeSource { virtual void serializePacketWriter(PacketWriter&, bool useObjectSerializer) const = 0; virtual void serializeBinaryWriter(BinaryWriter&) const = 0; + virtual void serializeBinaryWriter(ObjectWriter&) const = 0; }; template @@ -737,7 +738,12 @@ struct SerializeSource : MakeSerializeSource, T> { using value_type = T; T const& value; SerializeSource(T const& value) : value(value) {} - template void serialize(Ar& ar) const { ar << value; } + virtual void serializeBinaryWriter(ObjectWriter& w) const { + w.serialize(value); + } + template void serialize(Ar& ar) const { + ar << value; + } virtual T const& get() const { return value; } }; @@ -748,6 +754,9 @@ struct SerializeBoolAnd : MakeSerializeSource, T> { T const& value; SerializeBoolAnd( bool b, T const& value ) : b(b), value(value) {} template void serialize(Ar& ar) const { ar << b << value; } + virtual void serializeBinaryWriter(ObjectWriter& w) const { + ASSERT(false); + } virtual T const& get() const { // This is only used for the streaming serializer ASSERT(false); diff --git a/tests/TestRunner/TestRunner.py b/tests/TestRunner/TestRunner.py index 880a3cffa8..5fefc8502c 100755 --- a/tests/TestRunner/TestRunner.py +++ b/tests/TestRunner/TestRunner.py @@ -271,6 +271,9 @@ def run_simulation_test(basedir, options): if options.testtype == 'test': pargs.append('-C') pargs.append(os.path.join(args.builddir, 'fdb.cluster')) + else: + pargs.append('-S') + pargs.append('random') td = TestDirectory(basedir) if options.buggify: pargs.append('-b')