mirror of
https://github.com/apple/foundationdb.git
synced 2025-05-31 18:19:35 +08:00
Added beginVersion cmd line option and addressed code review comments
This commit is contained in:
Jon Fu
parent
4831f11e2a
commit
00c77ba2b4
fdbbackup
fdbclient
fdbserver/workloads
@ -135,6 +135,7 @@ enum {
|
||||
OPT_PREFIX_REMOVE,
|
||||
OPT_RESTORE_CLUSTERFILE_DEST,
|
||||
OPT_RESTORE_CLUSTERFILE_ORIG,
|
||||
OPT_RESTORE_BEGIN_VERSION,
|
||||
|
||||
// Shared constants
|
||||
OPT_CLUSTERFILE,
|
||||
@ -641,6 +642,7 @@ CSimpleOpt::SOption g_rgRestoreOptions[] = {
|
||||
{ OPT_BACKUPKEYS, "--keys", SO_REQ_SEP },
|
||||
{ OPT_WAITFORDONE, "-w", SO_NONE },
|
||||
{ OPT_WAITFORDONE, "--waitfordone", SO_NONE },
|
||||
{ OPT_RESTORE_BEGIN_VERSION, "--begin_version", SO_REQ_SEP },
|
||||
{ OPT_RESTORE_VERSION, "--version", SO_REQ_SEP },
|
||||
{ OPT_RESTORE_VERSION, "-v", SO_REQ_SEP },
|
||||
{ OPT_TRACE, "--log", SO_NONE },
|
||||
@ -2123,7 +2125,10 @@ Reference<IBackupContainer> openBackupContainer(const char *name, std::string de
|
||||
return c;
|
||||
}
|
||||
|
||||
ACTOR Future<Void> runRestore(Database db, std::string originalClusterFile, std::string tagName, std::string container, Standalone<VectorRef<KeyRangeRef>> ranges, Version targetVersion, std::string targetTimestamp, bool performRestore, bool verbose, bool waitForDone, std::string addPrefix, std::string removePrefix, bool incrementalBackupOnly) {
|
||||
ACTOR Future<Void> runRestore(Database db, std::string originalClusterFile, std::string tagName, std::string container,
|
||||
Standalone<VectorRef<KeyRangeRef>> ranges, Version beginVersion, Version targetVersion,
|
||||
std::string targetTimestamp, bool performRestore, bool verbose, bool waitForDone,
|
||||
std::string addPrefix, std::string removePrefix, bool incrementalBackupOnly) {
|
||||
if(ranges.empty()) {
|
||||
ranges.push_back_deep(ranges.arena(), normalKeys);
|
||||
}
|
||||
@ -2179,9 +2184,9 @@ ACTOR Future<Void> runRestore(Database db, std::string originalClusterFile, std:
|
||||
}
|
||||
|
||||
if (performRestore) {
|
||||
Version restoredVersion = wait(backupAgent.restore(db, origDb, KeyRef(tagName), KeyRef(container), ranges,
|
||||
waitForDone, targetVersion, verbose, KeyRef(addPrefix),
|
||||
KeyRef(removePrefix), true, incrementalBackupOnly));
|
||||
Version restoredVersion = wait(backupAgent.restore(
|
||||
db, origDb, KeyRef(tagName), KeyRef(container), ranges, waitForDone, targetVersion, verbose,
|
||||
KeyRef(addPrefix), KeyRef(removePrefix), true, incrementalBackupOnly, beginVersion));
|
||||
|
||||
if(waitForDone && verbose) {
|
||||
// If restore is now complete then report version restored
|
||||
@ -2960,6 +2965,7 @@ int main(int argc, char* argv[]) {
|
||||
std::string removePrefix;
|
||||
Standalone<VectorRef<KeyRangeRef>> backupKeys;
|
||||
int maxErrors = 20;
|
||||
Version beginVersion = invalidVersion;
|
||||
Version restoreVersion = invalidVersion;
|
||||
std::string restoreTimestamp;
|
||||
bool waitForDone = false;
|
||||
@ -3249,6 +3255,17 @@ int main(int argc, char* argv[]) {
|
||||
}
|
||||
break;
|
||||
}
|
||||
case OPT_RESTORE_BEGIN_VERSION: {
|
||||
const char* a = args->OptionArg();
|
||||
long long ver = 0;
|
||||
if (!sscanf(a, "%lld", &ver)) {
|
||||
fprintf(stderr, "ERROR: Could not parse database beginVersion `%s'\n", a);
|
||||
printHelpTeaser(argv[0]);
|
||||
return FDB_EXIT_ERROR;
|
||||
}
|
||||
beginVersion = ver;
|
||||
break;
|
||||
}
|
||||
case OPT_RESTORE_VERSION: {
|
||||
const char* a = args->OptionArg();
|
||||
long long ver = 0;
|
||||
@ -3753,7 +3770,9 @@ int main(int argc, char* argv[]) {
|
||||
|
||||
switch(restoreType) {
|
||||
case RESTORE_START:
|
||||
f = stopAfter( runRestore(db, restoreClusterFileOrig, tagName, restoreContainer, backupKeys, restoreVersion, restoreTimestamp, !dryRun, !quietDisplay, waitForDone, addPrefix, removePrefix, incrementalBackupOnly) );
|
||||
f = stopAfter(runRestore(db, restoreClusterFileOrig, tagName, restoreContainer, backupKeys,
|
||||
beginVersion, restoreVersion, restoreTimestamp, !dryRun, !quietDisplay,
|
||||
waitForDone, addPrefix, removePrefix, incrementalBackupOnly));
|
||||
break;
|
||||
case RESTORE_WAIT:
|
||||
f = stopAfter( success(ba.waitRestore(db, KeyRef(tagName), true)) );
|
||||
|
||||
@ -289,15 +289,16 @@ public:
|
||||
Future<Version> restore(Database cx, Optional<Database> cxOrig, Key tagName, Key url,
|
||||
Standalone<VectorRef<KeyRangeRef>> ranges, bool waitForComplete = true,
|
||||
Version targetVersion = -1, bool verbose = true, Key addPrefix = Key(),
|
||||
Key removePrefix = Key(), bool lockDB = true, bool incrementalBackupOnly = false);
|
||||
Key removePrefix = Key(), bool lockDB = true, bool incrementalBackupOnly = false,
|
||||
Version beginVersion = -1);
|
||||
Future<Version> restore(Database cx, Optional<Database> cxOrig, Key tagName, Key url, bool waitForComplete = true,
|
||||
Version targetVersion = -1, bool verbose = true, KeyRange range = normalKeys,
|
||||
Key addPrefix = Key(), Key removePrefix = Key(), bool lockDB = true,
|
||||
bool incrementalBackupOnly = false) {
|
||||
bool incrementalBackupOnly = false, Version beginVersion = -1) {
|
||||
Standalone<VectorRef<KeyRangeRef>> rangeRef;
|
||||
rangeRef.push_back_deep(rangeRef.arena(), range);
|
||||
return restore(cx, cxOrig, tagName, url, rangeRef, waitForComplete, targetVersion, verbose, addPrefix,
|
||||
removePrefix, lockDB, incrementalBackupOnly);
|
||||
removePrefix, lockDB, incrementalBackupOnly, beginVersion);
|
||||
}
|
||||
Future<Version> atomicRestore(Database cx, Key tagName, Standalone<VectorRef<KeyRangeRef>> ranges, Key addPrefix = Key(), Key removePrefix = Key());
|
||||
Future<Version> atomicRestore(Database cx, Key tagName, KeyRange range = normalKeys, Key addPrefix = Key(), Key removePrefix = Key()) {
|
||||
|
||||
@ -1364,24 +1364,31 @@ public:
|
||||
return getSnapshotFileKeyRange_impl(Reference<BackupContainerFileSystem>::addRef(this), file);
|
||||
}
|
||||
|
||||
static Optional<RestorableFileSet> getRestoreSetFromLogs(std::vector<LogFile> logs, Version targetVersion,
|
||||
RestorableFileSet restorable) {
|
||||
Version end = logs.begin()->endVersion;
|
||||
computeRestoreEndVersion(logs, &restorable.logs, &end, targetVersion);
|
||||
if (end >= targetVersion) {
|
||||
restorable.continuousBeginVersion = logs.begin()->beginVersion;
|
||||
restorable.continuousEndVersion = end;
|
||||
return Optional<RestorableFileSet>(restorable);
|
||||
}
|
||||
return Optional<RestorableFileSet>();
|
||||
}
|
||||
|
||||
ACTOR static Future<Optional<RestorableFileSet>> getRestoreSet_impl(Reference<BackupContainerFileSystem> bc,
|
||||
Version targetVersion, bool logsOnly) {
|
||||
Version targetVersion, bool logsOnly,
|
||||
Version beginVersion) {
|
||||
if (logsOnly) {
|
||||
state RestorableFileSet restorableSet;
|
||||
state std::vector<LogFile> logFiles;
|
||||
wait(store(logFiles, bc->listLogFiles(0, targetVersion, false)));
|
||||
Version begin = beginVersion == invalidVersion ? 0 : beginVersion;
|
||||
wait(store(logFiles, bc->listLogFiles(begin, targetVersion, false)));
|
||||
// List logs in version order so log continuity can be analyzed
|
||||
std::sort(logFiles.begin(), logFiles.end());
|
||||
if (!logFiles.empty()) {
|
||||
Version end = logFiles.begin()->endVersion;
|
||||
computeRestoreEndVersion(logFiles, &restorableSet.logs, &end, targetVersion);
|
||||
if (end >= targetVersion) {
|
||||
restorableSet.continuousBeginVersion = logFiles.begin()->beginVersion;
|
||||
restorableSet.continuousEndVersion = end;
|
||||
return Optional<RestorableFileSet>(restorableSet);
|
||||
}
|
||||
return getRestoreSetFromLogs(logFiles, targetVersion, restorableSet);
|
||||
}
|
||||
return Optional<RestorableFileSet>();
|
||||
}
|
||||
// Find the most recent keyrange snapshot to end at or before targetVersion
|
||||
state Optional<KeyspaceSnapshotFile> snapshot;
|
||||
@ -1453,21 +1460,17 @@ public:
|
||||
|
||||
// If there are logs and the first one starts at or before the snapshot begin version then proceed
|
||||
if(!logs.empty() && logs.front().beginVersion <= snapshot.get().beginVersion) {
|
||||
Version end = logs.begin()->endVersion;
|
||||
computeRestoreEndVersion(logs, &restorable.logs, &end, targetVersion);
|
||||
if (end >= targetVersion) {
|
||||
restorable.continuousBeginVersion = logs.begin()->beginVersion;
|
||||
restorable.continuousEndVersion = end;
|
||||
return Optional<RestorableFileSet>(restorable);
|
||||
}
|
||||
return getRestoreSetFromLogs(logs, targetVersion, restorable);
|
||||
}
|
||||
}
|
||||
|
||||
return Optional<RestorableFileSet>();
|
||||
}
|
||||
|
||||
Future<Optional<RestorableFileSet>> getRestoreSet(Version targetVersion, bool logsOnly) final {
|
||||
return getRestoreSet_impl(Reference<BackupContainerFileSystem>::addRef(this), targetVersion, logsOnly);
|
||||
Future<Optional<RestorableFileSet>> getRestoreSet(Version targetVersion, bool logsOnly,
|
||||
Version beginVersion) final {
|
||||
return getRestoreSet_impl(Reference<BackupContainerFileSystem>::addRef(this), targetVersion, logsOnly,
|
||||
beginVersion);
|
||||
}
|
||||
|
||||
private:
|
||||
|
||||
@ -282,7 +282,8 @@ public:
|
||||
|
||||
// Get exactly the files necessary to restore to targetVersion. Returns non-present if
|
||||
// restore to given version is not possible.
|
||||
virtual Future<Optional<RestorableFileSet>> getRestoreSet(Version targetVersion, bool logsOnly = false) = 0;
|
||||
virtual Future<Optional<RestorableFileSet>> getRestoreSet(Version targetVersion, bool logsOnly = false,
|
||||
Version beginVersion = -1) = 0;
|
||||
|
||||
// Get an IBackupContainer based on a container spec string
|
||||
static Reference<IBackupContainer> openContainer(std::string url);
|
||||
|
||||
@ -144,6 +144,9 @@ public:
|
||||
KeyBackedProperty<Key> batchFuture() {
|
||||
return configSpace.pack(LiteralStringRef(__FUNCTION__));
|
||||
}
|
||||
KeyBackedProperty<Version> beginVersion() {
|
||||
return configSpace.pack(LiteralStringRef(__FUNCTION__));
|
||||
}
|
||||
KeyBackedProperty<Version> restoreVersion() {
|
||||
return configSpace.pack(LiteralStringRef(__FUNCTION__));
|
||||
}
|
||||
@ -3407,6 +3410,7 @@ namespace fileBackup {
|
||||
state Reference<ReadYourWritesTransaction> tr(new ReadYourWritesTransaction(cx));
|
||||
state RestoreConfig restore(task);
|
||||
state Version restoreVersion;
|
||||
state Version beginVersion;
|
||||
state Reference<IBackupContainer> bc;
|
||||
|
||||
loop {
|
||||
@ -3417,6 +3421,8 @@ namespace fileBackup {
|
||||
wait(checkTaskVersion(tr->getDatabase(), task, name, version));
|
||||
Version _restoreVersion = wait(restore.restoreVersion().getOrThrow(tr));
|
||||
restoreVersion = _restoreVersion;
|
||||
Version _beginVersion = wait(restore.beginVersion().getOrThrow(tr));
|
||||
beginVersion = _beginVersion;
|
||||
wait(taskBucket->keepRunning(tr, task));
|
||||
|
||||
ERestoreState oldState = wait(restore.stateEnum().getD(tr));
|
||||
@ -3456,23 +3462,20 @@ namespace fileBackup {
|
||||
wait(tr->onError(e));
|
||||
}
|
||||
}
|
||||
state Future<Optional<bool>> logsOnly = restore.incrementalBackupOnly().get(tr);
|
||||
wait(success(logsOnly));
|
||||
state bool incremental = false;
|
||||
if (logsOnly.isReady() && logsOnly.get().present() && logsOnly.get().get()) {
|
||||
incremental = true;
|
||||
state bool incremental = wait(restore.incrementalBackupOnly().getOrThrow(tr));
|
||||
if (beginVersion == invalidVersion) {
|
||||
beginVersion = 0;
|
||||
}
|
||||
Optional<RestorableFileSet> restorable = wait(bc->getRestoreSet(restoreVersion, incremental));
|
||||
state Version beginVer = 0;
|
||||
Optional<RestorableFileSet> restorable = wait(bc->getRestoreSet(restoreVersion, incremental, beginVersion));
|
||||
if (!incremental) {
|
||||
beginVer = restorable.get().snapshot.beginVersion;
|
||||
beginVersion = restorable.get().snapshot.beginVersion;
|
||||
}
|
||||
|
||||
if(!restorable.present())
|
||||
throw restore_missing_data();
|
||||
|
||||
// First version for which log data should be applied
|
||||
Params.firstVersion().set(task, beginVer);
|
||||
Params.firstVersion().set(task, beginVersion);
|
||||
|
||||
// Convert the two lists in restorable (logs and ranges) to a single list of RestoreFiles.
|
||||
// Order does not matter, they will be put in order when written to the restoreFileMap below.
|
||||
@ -3901,7 +3904,7 @@ public:
|
||||
ACTOR static Future<Void> submitRestore(FileBackupAgent* backupAgent, Reference<ReadYourWritesTransaction> tr,
|
||||
Key tagName, Key backupURL, Standalone<VectorRef<KeyRangeRef>> ranges,
|
||||
Version restoreVersion, Key addPrefix, Key removePrefix, bool lockDB,
|
||||
bool incrementalBackupOnly, UID uid) {
|
||||
bool incrementalBackupOnly, Version beginVersion, UID uid) {
|
||||
KeyRangeMap<int> restoreRangeSet;
|
||||
for (auto& range : ranges) {
|
||||
restoreRangeSet.insert(range, 1);
|
||||
@ -3967,6 +3970,7 @@ public:
|
||||
restore.stateEnum().set(tr, ERestoreState::QUEUED);
|
||||
restore.restoreVersion().set(tr, restoreVersion);
|
||||
restore.incrementalBackupOnly().set(tr, incrementalBackupOnly);
|
||||
restore.beginVersion().set(tr, beginVersion);
|
||||
if (BUGGIFY && restoreRanges.size() == 1) {
|
||||
restore.restoreRange().set(tr, restoreRanges[0]);
|
||||
}
|
||||
@ -4484,7 +4488,8 @@ public:
|
||||
ACTOR static Future<Version> restore(FileBackupAgent* backupAgent, Database cx, Optional<Database> cxOrig,
|
||||
Key tagName, Key url, Standalone<VectorRef<KeyRangeRef>> ranges,
|
||||
bool waitForComplete, Version targetVersion, bool verbose, Key addPrefix,
|
||||
Key removePrefix, bool lockDB, bool incrementalBackupOnly, UID randomUid) {
|
||||
Key removePrefix, bool lockDB, bool incrementalBackupOnly,
|
||||
Version beginVersion, UID randomUid) {
|
||||
state Reference<IBackupContainer> bc = IBackupContainer::openContainer(url.toString());
|
||||
|
||||
state BackupDescription desc = wait(bc->describeBackup());
|
||||
@ -4500,7 +4505,8 @@ public:
|
||||
targetVersion = desc.contiguousLogEnd.get() - 1;
|
||||
}
|
||||
|
||||
Optional<RestorableFileSet> restoreSet = wait(bc->getRestoreSet(targetVersion, incrementalBackupOnly));
|
||||
Optional<RestorableFileSet> restoreSet =
|
||||
wait(bc->getRestoreSet(targetVersion, incrementalBackupOnly, beginVersion));
|
||||
|
||||
if(!restoreSet.present()) {
|
||||
TraceEvent(SevWarn, "FileBackupAgentRestoreNotPossible")
|
||||
@ -4520,7 +4526,7 @@ public:
|
||||
tr->setOption(FDBTransactionOptions::ACCESS_SYSTEM_KEYS);
|
||||
tr->setOption(FDBTransactionOptions::LOCK_AWARE);
|
||||
wait(submitRestore(backupAgent, tr, tagName, url, ranges, targetVersion, addPrefix, removePrefix,
|
||||
lockDB, incrementalBackupOnly, randomUid));
|
||||
lockDB, incrementalBackupOnly, beginVersion, randomUid));
|
||||
wait(tr->commit());
|
||||
break;
|
||||
} catch(Error &e) {
|
||||
@ -4657,7 +4663,7 @@ public:
|
||||
} else {
|
||||
TraceEvent("AS_StartRestore");
|
||||
Version ver = wait(restore(backupAgent, cx, cx, tagName, KeyRef(bc->getURL()), ranges, true, -1, true,
|
||||
addPrefix, removePrefix, true, false, randomUid));
|
||||
addPrefix, removePrefix, true, false, invalidVersion, randomUid));
|
||||
return ver;
|
||||
}
|
||||
}
|
||||
@ -4697,9 +4703,9 @@ Future<Void> FileBackupAgent::atomicParallelRestore(Database cx, Key tagName, St
|
||||
Future<Version> FileBackupAgent::restore(Database cx, Optional<Database> cxOrig, Key tagName, Key url,
|
||||
Standalone<VectorRef<KeyRangeRef>> ranges, bool waitForComplete,
|
||||
Version targetVersion, bool verbose, Key addPrefix, Key removePrefix,
|
||||
bool lockDB, bool incrementalBackupOnly) {
|
||||
bool lockDB, bool incrementalBackupOnly, Version beginVersion) {
|
||||
return FileBackupAgentImpl::restore(this, cx, cxOrig, tagName, url, ranges, waitForComplete, targetVersion, verbose,
|
||||
addPrefix, removePrefix, lockDB, incrementalBackupOnly,
|
||||
addPrefix, removePrefix, lockDB, incrementalBackupOnly, beginVersion,
|
||||
deterministicRandom()->randomUniqueID());
|
||||
}
|
||||
|
||||
|
||||
@ -75,6 +75,7 @@ struct IncrementalBackupWorkload : TestWorkload {
|
||||
state UID backupUID;
|
||||
TraceEvent("IBackupRestoreAttempt");
|
||||
wait(success(self->backupAgent.waitBackup(cx, self->tag.toString(), false, &backupContainer, &backupUID)));
|
||||
// TODO: add testing scenario for atomics and beginVersion
|
||||
wait(success(self->backupAgent.restore(cx, cx, Key(self->tag.toString()), Key(backupContainer->getURL()),
|
||||
true, -1, true, normalKeys, Key(), Key(), true, true)));
|
||||
TraceEvent("IBackupRestoreSuccess");
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user