From a892eec57d86727d5b50e4ce6580fe644baf4752 Mon Sep 17 00:00:00 2001 From: "A.J. Beamon" Date: Mon, 15 Mar 2021 16:41:22 -0700 Subject: [PATCH] Manually apply changes made to BackupContainer.actor.cpp on release-6.3 to the new files where that code is located. --- fdbclient/BackupContainerFileSystem.actor.cpp | 47 +++++++++++++------ .../BackupContainerLocalDirectory.actor.cpp | 21 ++++++--- 2 files changed, 47 insertions(+), 21 deletions(-) diff --git a/fdbclient/BackupContainerFileSystem.actor.cpp b/fdbclient/BackupContainerFileSystem.actor.cpp index 3eab8b701a..040796d35f 100644 --- a/fdbclient/BackupContainerFileSystem.actor.cpp +++ b/fdbclient/BackupContainerFileSystem.actor.cpp @@ -1431,30 +1431,41 @@ BackupContainerFileSystem::VersionProperty BackupContainerFileSystem::logType() namespace backup_test { int chooseFileSize(std::vector& sizes) { - int size = 1000; if (!sizes.empty()) { - size = sizes.back(); + int size = sizes.back(); sizes.pop_back(); + return size; } - return size; + return deterministicRandom()->randomInt(0, 2e6); } -ACTOR Future writeAndVerifyFile(Reference c, Reference f, int size) { - state Standalone content; - if (size > 0) { - content = makeString(size); - for (int i = 0; i < content.size(); ++i) - mutateString(content)[i] = (uint8_t)deterministicRandom()->randomInt(0, 256); +ACTOR Future writeAndVerifyFile(Reference c, Reference f, int size, FlowLock* lock) { + state Standalone> content; - wait(f->append(content.begin(), content.size())); + wait(lock->take(TaskPriority::DefaultYield, size)); + state FlowLock::Releaser releaser(*lock, size); + + printf("writeAndVerify size=%d file=%s\n", size, f->getFileName().c_str()); + content.resize(content.arena(), size); + for (int i = 0; i < content.size(); ++i) { + content[i] = (uint8_t)deterministicRandom()->randomInt(0, 256); + } + + state VectorRef sendBuf = content; + while (sendBuf.size() > 0) { + state int n = std::min(sendBuf.size(), deterministicRandom()->randomInt(1, 16384)); + wait(f->append(sendBuf.begin(), n)); + sendBuf.pop_front(n); } wait(f->finish()); + state Reference inputFile = wait(c->readFile(f->getFileName())); int64_t fileSize = wait(inputFile->size()); ASSERT(size == fileSize); if (size > 0) { - state Standalone buf = makeString(size); - int b = wait(inputFile->read(mutateString(buf), buf.size(), 0)); + state Standalone> buf; + buf.resize(buf.arena(), fileSize); + int b = wait(inputFile->read(buf.begin(), buf.size(), 0)); ASSERT(b == buf.size()); ASSERT(buf == content); } @@ -1491,6 +1502,8 @@ ACTOR static Future testWriteSnapshotFile(Reference file, Key } ACTOR static Future testBackupContainer(std::string url) { + state FlowLock lock(100e6); + printf("BackupContainerTest URL %s\n", url.c_str()); state Reference c = IBackupContainer::openContainer(url); @@ -1514,7 +1527,11 @@ ACTOR static Future testBackupContainer(std::string url) { state Version v = deterministicRandom()->randomInt64(0, std::numeric_limits::max() / 2); // List of sizes to use to test edge cases on underlying file implementations - state std::vector fileSizes = { 0, 10000000, 5000005 }; + state std::vector fileSizes = { 0 }; + if (StringRef(url).startsWith(LiteralStringRef("blob"))) { + fileSizes.push_back(CLIENT_KNOBS->BLOBSTORE_MULTIPART_MIN_PART_SIZE); + fileSizes.push_back(CLIENT_KNOBS->BLOBSTORE_MULTIPART_MIN_PART_SIZE + 10); + } loop { state Version logStart = v; @@ -1541,7 +1558,7 @@ ACTOR static Future testBackupContainer(std::string url) { int size = chooseFileSize(fileSizes); snapshotSizes.rbegin()->second += size; // Write in actual range file format, instead of random data. - // writes.push_back(writeAndVerifyFile(c, range, size)); + // writes.push_back(writeAndVerifyFile(c, range, size, &lock)); wait(testWriteSnapshotFile(range, begin, end, blockSize)); if (deterministicRandom()->random01() < .2) { @@ -1562,7 +1579,7 @@ ACTOR static Future testBackupContainer(std::string url) { state Reference log = wait(c->writeLogFile(logStart, v, 10)); logs[logStart] = log->getFileName(); int size = chooseFileSize(fileSizes); - writes.push_back(writeAndVerifyFile(c, log, size)); + writes.push_back(writeAndVerifyFile(c, log, size, &lock)); // Randomly stop after a snapshot has finished and all manually seeded file sizes have been used. if (fileSizes.empty() && !snapshots.empty() && snapshots.rbegin()->second.empty() && diff --git a/fdbclient/BackupContainerLocalDirectory.actor.cpp b/fdbclient/BackupContainerLocalDirectory.actor.cpp index 7aab62ef3f..e0c78a31bf 100644 --- a/fdbclient/BackupContainerLocalDirectory.actor.cpp +++ b/fdbclient/BackupContainerLocalDirectory.actor.cpp @@ -31,21 +31,29 @@ namespace { class BackupFile : public IBackupFile, ReferenceCounted { public: BackupFile(const std::string& fileName, Reference file, const std::string& finalFullPath) - : IBackupFile(fileName), m_file(file), m_finalFullPath(finalFullPath), m_writeOffset(0) { - m_buffer.reserve(m_buffer.arena(), CLIENT_KNOBS->BACKUP_LOCAL_FILE_WRITE_BLOCK); + : IBackupFile(fileName), m_file(file), m_finalFullPath(finalFullPath), m_writeOffset(0), m_blockSize(CLIENT_KNOBS->BACKUP_LOCAL_FILE_WRITE_BLOCK) { + if (BUGGIFY) { + m_blockSize = deterministicRandom()->randomInt(100, 20000); + } + m_buffer.reserve(m_buffer.arena(), m_blockSize); } Future append(const void* data, int len) override { m_buffer.append(m_buffer.arena(), (const uint8_t*)data, len); - if (m_buffer.size() >= CLIENT_KNOBS->BACKUP_LOCAL_FILE_WRITE_BLOCK) { - return flush(CLIENT_KNOBS->BACKUP_LOCAL_FILE_WRITE_BLOCK); + if (m_buffer.size() >= m_blockSize) { + return flush(m_blockSize); } return Void(); } Future flush(int size) { + // Avoid empty write + if (size == 0) { + return Void(); + } + ASSERT(size <= m_buffer.size()); // Keep a reference to the old buffer @@ -66,7 +74,7 @@ public: wait(f->m_file->sync()); std::string name = f->m_file->getFilename(); f->m_file.clear(); - renameFile(name, f->m_finalFullPath); + wait(IAsyncFileSystem::filesystem()->renameFile(name, f->m_finalFullPath)); return Void(); } @@ -82,6 +90,7 @@ private: Standalone> m_buffer; int64_t m_writeOffset; std::string m_finalFullPath; + int m_blockSize; }; ACTOR static Future listFiles_impl(std::string path, std::string m_path) { @@ -249,7 +258,7 @@ Future> BackupContainerLocalDirectory::readFile(const std: } Future> BackupContainerLocalDirectory::writeFile(const std::string& path) { - int flags = IAsyncFile::OPEN_NO_AIO | IAsyncFile::OPEN_CREATE | IAsyncFile::OPEN_ATOMIC_WRITE_AND_CREATE | + int flags = IAsyncFile::OPEN_NO_AIO | IAsyncFile::OPEN_UNCACHED | IAsyncFile::OPEN_CREATE | IAsyncFile::OPEN_ATOMIC_WRITE_AND_CREATE | IAsyncFile::OPEN_READWRITE; std::string fullPath = joinPath(m_path, path); platform::createDirectory(parentDirectory(fullPath));