mirror of
https://github.com/apple/foundationdb.git
synced 2025-06-03 03:41:53 +08:00
Upgrade to RocksDB 8.1.1 (#10039)
* Upgrade to RocksDB 8.1.1 * Revert an ASSERT on the previous rocksdb bug. The test was a guard to show the failure.
This commit is contained in:
He Liu
GitHub
parent
53fe07a709
commit
4c75cee2e3
@ -1,6 +1,6 @@
|
||||
# FindRocksDB
|
||||
|
||||
find_package(RocksDB 7.10.2)
|
||||
find_package(RocksDB 8.1.1)
|
||||
|
||||
include(ExternalProject)
|
||||
|
||||
@ -52,8 +52,8 @@ if(ROCKSDB_FOUND)
|
||||
${BINARY_DIR}/librocksdb.a)
|
||||
else()
|
||||
ExternalProject_Add(rocksdb
|
||||
URL https://github.com/facebook/rocksdb/archive/refs/tags/v7.10.2.tar.gz
|
||||
URL_HASH SHA256=4619ae7308cd3d11cdd36f0bfad3fb03a1ad399ca333f192b77b6b95b08e2f78
|
||||
URL https://github.com/facebook/rocksdb/archive/refs/tags/v8.1.1.tar.gz
|
||||
URL_HASH SHA256=9102704e169cfb53e7724a30750eeeb3e71307663852f01fa08d5a320e6155a8
|
||||
CMAKE_ARGS ${RocksDB_CMAKE_ARGS}
|
||||
BUILD_BYPRODUCTS <BINARY_DIR>/librocksdb.a
|
||||
INSTALL_COMMAND ""
|
||||
@ -75,4 +75,4 @@ message(STATUS "Found RocksDB includes: ${ROCKSDB_INCLUDE_DIR}")
|
||||
mark_as_advanced(
|
||||
ROCKSDB_LIBRARIES
|
||||
ROCKSDB_INCLUDE_DIR
|
||||
)
|
||||
)
|
||||
@ -49,6 +49,7 @@
|
||||
#endif
|
||||
#include "fdbclient/SystemData.h"
|
||||
#include "fdbserver/CoroFlow.h"
|
||||
#include "fdbserver/FDBRocksDBVersion.h"
|
||||
#include "fdbserver/RocksDBLogForwarder.h"
|
||||
#include "flow/ActorCollection.h"
|
||||
#include "flow/flow.h"
|
||||
@ -69,9 +70,10 @@
|
||||
|
||||
#ifdef SSD_ROCKSDB_EXPERIMENTAL
|
||||
|
||||
// Enforcing rocksdb version to be 7.10.2
|
||||
static_assert((ROCKSDB_MAJOR == 7 && ROCKSDB_MINOR == 10 && ROCKSDB_PATCH == 2),
|
||||
"Unsupported rocksdb version. Update the rocksdb to 7.10.2 version");
|
||||
// Enforcing rocksdb version.
|
||||
static_assert((ROCKSDB_MAJOR == FDB_ROCKSDB_MAJOR && ROCKSDB_MINOR == FDB_ROCKSDB_MINOR &&
|
||||
ROCKSDB_PATCH == FDB_ROCKSDB_PATCH),
|
||||
"Unsupported rocksdb version.");
|
||||
|
||||
namespace {
|
||||
using rocksdb::BackgroundErrorReason;
|
||||
@ -937,7 +939,8 @@ ACTOR Future<Void> rocksDBMetricLogger(UID id,
|
||||
{ "BloomFilterUseful", rocksdb::BLOOM_FILTER_USEFUL, 0 },
|
||||
{ "BloomFilterFullPositive", rocksdb::BLOOM_FILTER_FULL_POSITIVE, 0 },
|
||||
{ "BloomFilterTruePositive", rocksdb::BLOOM_FILTER_FULL_TRUE_POSITIVE, 0 },
|
||||
{ "BloomFilterMicros", rocksdb::BLOOM_FILTER_MICROS, 0 },
|
||||
// Deprecated in RocksDB 8.0
|
||||
// { "BloomFilterMicros", rocksdb::BLOOM_FILTER_MICROS, 0 },
|
||||
{ "MemtableHit", rocksdb::MEMTABLE_HIT, 0 },
|
||||
{ "MemtableMiss", rocksdb::MEMTABLE_MISS, 0 },
|
||||
{ "GetHitL0", rocksdb::GET_HIT_L0, 0 },
|
||||
@ -950,8 +953,9 @@ ACTOR Future<Void> rocksDBMetricLogger(UID id,
|
||||
{ "CountDBPrev", rocksdb::NUMBER_DB_PREV, 0 },
|
||||
{ "BloomFilterPrefixChecked", rocksdb::BLOOM_FILTER_PREFIX_CHECKED, 0 },
|
||||
{ "BloomFilterPrefixUseful", rocksdb::BLOOM_FILTER_PREFIX_USEFUL, 0 },
|
||||
{ "BlockCacheCompressedMiss", rocksdb::BLOCK_CACHE_COMPRESSED_MISS, 0 },
|
||||
{ "BlockCacheCompressedHit", rocksdb::BLOCK_CACHE_COMPRESSED_HIT, 0 },
|
||||
// Deprecated in RocksDB 8.0
|
||||
// { "BlockCacheCompressedMiss", rocksdb::BLOCK_CACHE_COMPRESSED_MISS, 0 },
|
||||
// { "BlockCacheCompressedHit", rocksdb::BLOCK_CACHE_COMPRESSED_HIT, 0 },
|
||||
{ "CountWalFileSyncs", rocksdb::WAL_FILE_SYNCED, 0 },
|
||||
{ "CountWalFileBytes", rocksdb::WAL_FILE_BYTES, 0 },
|
||||
{ "CompactReadBytes", rocksdb::COMPACT_READ_BYTES, 0 },
|
||||
@ -981,10 +985,11 @@ ACTOR Future<Void> rocksDBMetricLogger(UID id,
|
||||
rocksdb::COMPRESSION_TIMES_NANOS }, // enabled if rocksdb::StatsLevel > kExceptDetailedTimers(3)
|
||||
{ "DecompressionTimeNanos",
|
||||
rocksdb::DECOMPRESSION_TIMES_NANOS }, // enabled if rocksdb::StatsLevel > kExceptDetailedTimers(3)
|
||||
{ "HardRateLimitDelayCount",
|
||||
rocksdb::HARD_RATE_LIMIT_DELAY_COUNT }, // enabled if rocksdb::StatsLevel > kExceptHistogramOrTimers(1)
|
||||
{ "SoftRateLimitDelayCount",
|
||||
rocksdb::SOFT_RATE_LIMIT_DELAY_COUNT }, // enabled if rocksdb::StatsLevel > kExceptHistogramOrTimers(1)
|
||||
// Deprecated in RocksDB 8.0
|
||||
// { "HardRateLimitDelayCount",
|
||||
// rocksdb::HARD_RATE_LIMIT_DELAY_COUNT }, // enabled if rocksdb::StatsLevel > kExceptHistogramOrTimers(1)
|
||||
// { "SoftRateLimitDelayCount",
|
||||
// rocksdb::SOFT_RATE_LIMIT_DELAY_COUNT }, // enabled if rocksdb::StatsLevel > kExceptHistogramOrTimers(1)
|
||||
{ "WriteStall", rocksdb::WRITE_STALL }, // enabled if rocksdb::StatsLevel > kExceptHistogramOrTimers(1)
|
||||
};
|
||||
|
||||
|
||||
@ -27,6 +27,7 @@
|
||||
#endif
|
||||
#include "fdbclient/SystemData.h"
|
||||
#include "fdbserver/CoroFlow.h"
|
||||
#include "fdbserver/FDBRocksDBVersion.h"
|
||||
#include "flow/flow.h"
|
||||
#include "flow/IThreadPool.h"
|
||||
#include "flow/ThreadHelper.actor.h"
|
||||
@ -44,9 +45,10 @@
|
||||
|
||||
#ifdef SSD_ROCKSDB_EXPERIMENTAL
|
||||
|
||||
// Enforcing rocksdb version to be 7.10.2
|
||||
static_assert((ROCKSDB_MAJOR == 7 && ROCKSDB_MINOR == 10 && ROCKSDB_PATCH == 2),
|
||||
"Unsupported rocksdb version. Update the rocksdb to 7.10.2 version");
|
||||
// Enforcing rocksdb version.
|
||||
static_assert((ROCKSDB_MAJOR == FDB_ROCKSDB_MAJOR && ROCKSDB_MINOR == FDB_ROCKSDB_MINOR &&
|
||||
ROCKSDB_PATCH == FDB_ROCKSDB_PATCH),
|
||||
"Unsupported rocksdb version.");
|
||||
|
||||
const std::string rocksDataFolderSuffix = "-data";
|
||||
const std::string METADATA_SHARD_ID = "kvs-metadata";
|
||||
@ -187,6 +189,9 @@ rocksdb::ExportImportFilesMetaData getMetaData(const CheckpointMetaData& checkpo
|
||||
liveFileMetaData.oldest_blob_file_number = fileMetaData.oldest_blob_file_number;
|
||||
liveFileMetaData.oldest_ancester_time = fileMetaData.oldest_ancester_time;
|
||||
liveFileMetaData.file_creation_time = fileMetaData.file_creation_time;
|
||||
liveFileMetaData.smallest = fileMetaData.smallest;
|
||||
liveFileMetaData.largest = fileMetaData.largest;
|
||||
liveFileMetaData.file_type = rocksdb::kTableFile;
|
||||
liveFileMetaData.epoch_number = fileMetaData.epoch_number;
|
||||
liveFileMetaData.name = fileMetaData.name;
|
||||
liveFileMetaData.db_path = fileMetaData.db_path;
|
||||
@ -223,6 +228,8 @@ void populateMetaData(CheckpointMetaData* checkpoint, const rocksdb::ExportImpor
|
||||
liveFileMetaData.oldest_blob_file_number = fileMetaData.oldest_blob_file_number;
|
||||
liveFileMetaData.oldest_ancester_time = fileMetaData.oldest_ancester_time;
|
||||
liveFileMetaData.file_creation_time = fileMetaData.file_creation_time;
|
||||
liveFileMetaData.smallest = fileMetaData.smallest;
|
||||
liveFileMetaData.largest = fileMetaData.largest;
|
||||
liveFileMetaData.epoch_number = fileMetaData.epoch_number;
|
||||
liveFileMetaData.name = fileMetaData.name;
|
||||
liveFileMetaData.db_path = fileMetaData.db_path;
|
||||
@ -1562,7 +1569,8 @@ RocksDBMetrics::RocksDBMetrics(UID debugID, std::shared_ptr<rocksdb::Statistics>
|
||||
{ "BloomFilterUseful", rocksdb::BLOOM_FILTER_USEFUL, 0 },
|
||||
{ "BloomFilterFullPositive", rocksdb::BLOOM_FILTER_FULL_POSITIVE, 0 },
|
||||
{ "BloomFilterTruePositive", rocksdb::BLOOM_FILTER_FULL_TRUE_POSITIVE, 0 },
|
||||
{ "BloomFilterMicros", rocksdb::BLOOM_FILTER_MICROS, 0 },
|
||||
// Deprecated in RocksDB 8.0
|
||||
// { "BloomFilterMicros", rocksdb::BLOOM_FILTER_MICROS, 0 },
|
||||
{ "MemtableHit", rocksdb::MEMTABLE_HIT, 0 },
|
||||
{ "MemtableMiss", rocksdb::MEMTABLE_MISS, 0 },
|
||||
{ "GetHitL0", rocksdb::GET_HIT_L0, 0 },
|
||||
@ -1575,8 +1583,9 @@ RocksDBMetrics::RocksDBMetrics(UID debugID, std::shared_ptr<rocksdb::Statistics>
|
||||
{ "CountDBPrev", rocksdb::NUMBER_DB_PREV, 0 },
|
||||
{ "BloomFilterPrefixChecked", rocksdb::BLOOM_FILTER_PREFIX_CHECKED, 0 },
|
||||
{ "BloomFilterPrefixUseful", rocksdb::BLOOM_FILTER_PREFIX_USEFUL, 0 },
|
||||
{ "BlockCacheCompressedMiss", rocksdb::BLOCK_CACHE_COMPRESSED_MISS, 0 },
|
||||
{ "BlockCacheCompressedHit", rocksdb::BLOCK_CACHE_COMPRESSED_HIT, 0 },
|
||||
// Deprecated in RocksDB 8.0
|
||||
// { "BlockCacheCompressedMiss", rocksdb::BLOCK_CACHE_COMPRESSED_MISS, 0 },
|
||||
// { "BlockCacheCompressedHit", rocksdb::BLOCK_CACHE_COMPRESSED_HIT, 0 },
|
||||
{ "CountWalFileSyncs", rocksdb::WAL_FILE_SYNCED, 0 },
|
||||
{ "CountWalFileBytes", rocksdb::WAL_FILE_BYTES, 0 },
|
||||
{ "CompactReadBytes", rocksdb::COMPACT_READ_BYTES, 0 },
|
||||
@ -3996,11 +4005,11 @@ TEST_CASE("noSim/ShardedRocksDB/CheckpointRestore") {
|
||||
try {
|
||||
wait(testCheckpointRestore(kvStore, { rangeK }));
|
||||
} catch (Error& e) {
|
||||
TraceEvent("TestCheckpointRestoreError").errorUnsuppressed(e);
|
||||
TraceEvent(SevError, "TestCheckpointRestoreError").errorUnsuppressed(e);
|
||||
err = e;
|
||||
}
|
||||
// This will fail once RocksDB is upgraded to 8.1.
|
||||
ASSERT(err.code() == error_code_failed_to_restore_checkpoint);
|
||||
// ASSERT(err.code() == error_code_failed_to_restore_checkpoint);
|
||||
|
||||
try {
|
||||
wait(testCheckpointRestore(kvStore, { rangeKz }));
|
||||
|
||||
@ -36,6 +36,7 @@
|
||||
#include "fdbserver/MutationTracking.h"
|
||||
#include "fdbclient/StorageCheckpoint.h"
|
||||
#include "fdbserver/CoroFlow.h"
|
||||
#include "fdbserver/FDBRocksDBVersion.h"
|
||||
#include "fdbserver/Knobs.h"
|
||||
#include "flow/IThreadPool.h"
|
||||
#include "flow/ThreadHelper.actor.h"
|
||||
@ -45,9 +46,11 @@
|
||||
#include "flow/actorcompiler.h" // has to be last include
|
||||
|
||||
#ifdef SSD_ROCKSDB_EXPERIMENTAL
|
||||
// Enforcing rocksdb version to be 7.10.2
|
||||
static_assert((ROCKSDB_MAJOR == 7 && ROCKSDB_MINOR == 10 && ROCKSDB_PATCH == 2),
|
||||
"Unsupported rocksdb version. Update the rocksdb to 7.10.2 version");
|
||||
|
||||
// Enforcing rocksdb version.
|
||||
static_assert((ROCKSDB_MAJOR == FDB_ROCKSDB_MAJOR && ROCKSDB_MINOR == FDB_ROCKSDB_MINOR &&
|
||||
ROCKSDB_PATCH == FDB_ROCKSDB_PATCH),
|
||||
"Unsupported rocksdb version.");
|
||||
|
||||
namespace {
|
||||
|
||||
@ -90,6 +93,9 @@ rocksdb::ExportImportFilesMetaData getMetaData(const CheckpointMetaData& checkpo
|
||||
liveFileMetaData.oldest_blob_file_number = fileMetaData.oldest_blob_file_number;
|
||||
liveFileMetaData.oldest_ancester_time = fileMetaData.oldest_ancester_time;
|
||||
liveFileMetaData.file_creation_time = fileMetaData.file_creation_time;
|
||||
liveFileMetaData.smallest = fileMetaData.smallest;
|
||||
liveFileMetaData.largest = fileMetaData.largest;
|
||||
liveFileMetaData.file_type = rocksdb::kTableFile;
|
||||
liveFileMetaData.epoch_number = fileMetaData.epoch_number;
|
||||
liveFileMetaData.name = fileMetaData.name;
|
||||
liveFileMetaData.db_path = fileMetaData.db_path;
|
||||
|
||||
28
fdbserver/include/fdbserver/FDBRocksDBVersion.h
Normal file
28
fdbserver/include/fdbserver/FDBRocksDBVersion.h
Normal file
@ -0,0 +1,28 @@
|
||||
/*
|
||||
* FDBRocksDBVersion.h
|
||||
*
|
||||
* This source file is part of the FoundationDB open source project
|
||||
*
|
||||
* Copyright 2013-2023 Apple Inc. and the FoundationDB project authors
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
#ifndef FDBSERVER_FDBROCKSDBVERSION_H
|
||||
#define FDBSERVER_FDBROCKSDBVERSION_H
|
||||
|
||||
#define FDB_ROCKSDB_MAJOR 8
|
||||
#define FDB_ROCKSDB_MINOR 1
|
||||
#define FDB_ROCKSDB_PATCH 1
|
||||
|
||||
#endif
|
||||
@ -171,6 +171,12 @@ struct SstFileMetaData {
|
||||
// DEPRECATED: replaced by `directory` in base struct
|
||||
std::string db_path;
|
||||
|
||||
// These bounds define the effective key range for range tombstones
|
||||
// in this file.
|
||||
// Currently only used by CreateColumnFamilyWithImport().
|
||||
std::string smallest{}; // Smallest internal key served by table
|
||||
std::string largest{}; // Largest internal key served by table
|
||||
|
||||
template <class Ar>
|
||||
void serialize(Ar& ar) {
|
||||
serializer(ar,
|
||||
@ -195,7 +201,9 @@ struct SstFileMetaData {
|
||||
file_creation_time,
|
||||
epoch_number,
|
||||
name,
|
||||
db_path);
|
||||
db_path,
|
||||
smallest,
|
||||
largest);
|
||||
}
|
||||
};
|
||||
|
||||
@ -234,7 +242,9 @@ struct LiveFileMetaData : public SstFileMetaData {
|
||||
SstFileMetaData::db_path,
|
||||
column_family_name,
|
||||
level,
|
||||
fetched);
|
||||
fetched,
|
||||
SstFileMetaData::smallest,
|
||||
SstFileMetaData::largest);
|
||||
}
|
||||
};
|
||||
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user