book/rocksdb
1
0
Fork 0
mirror of https://github.com/facebook/rocksdb.git synced 2025-05-14 00:43:07 +08:00
Code Issues Packages Projects Releases Wiki Activity

Remove unused fields from FileMetaData (temporarily) (#10443)

Browse Source 
Summary:
FileMetaData::[min|max]_timestamp are not currently being used or
tracked by RocksDB, even when user-defined timestamp is enabled. Each of
them is a std::string which can occupy 32 bytes. Remove them for now.
They may be added back when we have a pressing need for them. When we do
add them back, consider store them in a more compact way, e.g. one
boolean flag and a byte array of size 16.

Per file min/max timestamp bounds are available as table properties.

Pull Request resolved: https://github.com/facebook/rocksdb/pull/10443

Test Plan: make check

Reviewed By: pdillinger

Differential Revision: D38292275

Pulled By: riversand963

fbshipit-source-id: 841dc4e855ad8f8481c80cb020603de9607c9c94
This commit is contained in:
Yanqin Jin 2022-08-01 17:56:13 -07:00 committed by Facebook GitHub Bot
parent cc2099803a
commit fbfcf5cbcd
17 changed files with 79 additions and 176 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
HISTORY.md
View File

@ -22,6 +22,7 @@
### Behavior Change
* Added checksum handshake during the copying of decompressed WAL fragment. This together with #9875, #10037, #10212, #10114 and #10319 provides end-to-end integrity protection for write batch during recovery.
* PosixLogger is removed and by default EnvLogger will be used for info logging. The behavior of the two loggers should be very similar when using the default Posix Env.
* Remove [min|max]_timestamp from VersionEdit for now since they are not tracked in MANIFEST anyway but consume two empty std::string (up to 64 bytes) for each file. Should they be added back in the future, we should store them more compactly.
## 7.5.0 (07/15/2022)
### New Features

3
db/compaction/compaction_job_test.cc
View File

@ -372,8 +372,7 @@ class CompactionJobTestBase : public testing::Test {
smallest_seqno, largest_seqno, false, Temperature::kUnknown,
oldest_blob_file_number, kUnknownOldestAncesterTime,
kUnknownFileCreationTime, kUnknownFileChecksum,
kUnknownFileChecksumFuncName, kDisableUserTimestamp,
kDisableUserTimestamp, kNullUniqueId64x2);
kUnknownFileChecksumFuncName, kNullUniqueId64x2);
mutex_.Lock();
EXPECT_OK(

3
db/compaction/compaction_picker_test.cc
View File

@ -116,8 +116,7 @@ class CompactionPickerTest : public testing::Test {
InternalKey(largest, largest_seq, kTypeValue), smallest_seq,
largest_seq, marked_for_compact, temperature, kInvalidBlobFileNumber,
kUnknownOldestAncesterTime, kUnknownFileCreationTime,
kUnknownFileChecksum, kUnknownFileChecksumFuncName,
kDisableUserTimestamp, kDisableUserTimestamp, kNullUniqueId64x2);
kUnknownFileChecksum, kUnknownFileChecksumFuncName, kNullUniqueId64x2);
f->compensated_file_size =
(compensated_file_size != 0) ? compensated_file_size : file_size;
f->oldest_ancester_time = oldest_ancestor_time;

5
db/db_impl/db_impl_compaction_flush.cc
View File

@ -1686,8 +1686,7 @@ Status DBImpl::ReFitLevel(ColumnFamilyData* cfd, int level, int target_level) {
f->smallest, f->largest, f->fd.smallest_seqno, f->fd.largest_seqno,
f->marked_for_compaction, f->temperature, f->oldest_blob_file_number,
f->oldest_ancester_time, f->file_creation_time, f->file_checksum,
f->file_checksum_func_name, f->min_timestamp, f->max_timestamp,
f->unique_id);
f->file_checksum_func_name, f->unique_id);
}
ROCKS_LOG_DEBUG(immutable_db_options_.info_log,
"[%s] Apply version edit:\n%s", cfd->GetName().c_str(),
@ -3313,7 +3312,7 @@ Status DBImpl::BackgroundCompaction(bool* made_progress,
f->fd.largest_seqno, f->marked_for_compaction, f->temperature,
f->oldest_blob_file_number, f->oldest_ancester_time,
f->file_creation_time, f->file_checksum, f->file_checksum_func_name,
f->min_timestamp, f->max_timestamp, f->unique_id);
f->unique_id);
ROCKS_LOG_BUFFER(
log_buffer,

14
db/db_impl/db_impl_experimental.cc
View File

@ -131,13 +131,13 @@ Status DBImpl::PromoteL0(ColumnFamilyHandle* column_family, int target_level) {
edit.SetColumnFamily(cfd->GetID());
for (const auto& f : l0_files) {
edit.DeleteFile(0, f->fd.GetNumber());
edit.AddFile(
target_level, f->fd.GetNumber(), f->fd.GetPathId(),
f->fd.GetFileSize(), f->smallest, f->largest, f->fd.smallest_seqno,
f->fd.largest_seqno, f->marked_for_compaction, f->temperature,
f->oldest_blob_file_number, f->oldest_ancester_time,
f->file_creation_time, f->file_checksum, f->file_checksum_func_name,
f->min_timestamp, f->max_timestamp, f->unique_id);
edit.AddFile(target_level, f->fd.GetNumber(), f->fd.GetPathId(),
f->fd.GetFileSize(), f->smallest, f->largest,
f->fd.smallest_seqno, f->fd.largest_seqno,
f->marked_for_compaction, f->temperature,
f->oldest_blob_file_number, f->oldest_ancester_time,
f->file_creation_time, f->file_checksum,
f->file_checksum_func_name, f->unique_id);
}
status = versions_->LogAndApply(cfd, *cfd->GetLatestMutableCFOptions(),

3
db/db_impl/db_impl_open.cc
View File

@ -1600,8 +1600,7 @@ Status DBImpl::WriteLevel0TableForRecovery(int job_id, ColumnFamilyData* cfd,
meta.marked_for_compaction, meta.temperature,
meta.oldest_blob_file_number, meta.oldest_ancester_time,
meta.file_creation_time, meta.file_checksum,
meta.file_checksum_func_name, meta.min_timestamp,
meta.max_timestamp, meta.unique_id);
meta.file_checksum_func_name, meta.unique_id);
for (const auto& blob : blob_file_additions) {
edit->AddBlobFile(blob);

15
db/experimental.cc
View File

@ -105,14 +105,13 @@ Status UpdateManifestForFilesState(
// Current state inconsistent with manifest
++files_updated;
edit.DeleteFile(level, number);
edit.AddFile(level, number, lf->fd.GetPathId(),
lf->fd.GetFileSize(), lf->smallest, lf->largest,
lf->fd.smallest_seqno, lf->fd.largest_seqno,
lf->marked_for_compaction, temp,
lf->oldest_blob_file_number,
lf->oldest_ancester_time, lf->file_creation_time,
lf->file_checksum, lf->file_checksum_func_name,
lf->min_timestamp, lf->max_timestamp, lf->unique_id);
edit.AddFile(
level, number, lf->fd.GetPathId(), lf->fd.GetFileSize(),
lf->smallest, lf->largest, lf->fd.smallest_seqno,
lf->fd.largest_seqno, lf->marked_for_compaction, temp,
lf->oldest_blob_file_number, lf->oldest_ancester_time,
lf->file_creation_time, lf->file_checksum,
lf->file_checksum_func_name, lf->unique_id);
}
}
} else {

3
db/external_sst_file_ingestion_job.cc
View File

@ -450,8 +450,7 @@ Status ExternalSstFileIngestionJob::Run() {
f.smallest_internal_key, f.largest_internal_key, f.assigned_seqno,
f.assigned_seqno, false, f.file_temperature, kInvalidBlobFileNumber,
oldest_ancester_time, current_time, f.file_checksum,
f.file_checksum_func_name, kDisableUserTimestamp, kDisableUserTimestamp,
f.unique_id);
f.file_checksum_func_name, f.unique_id);
f_metadata.temperature = f.file_temperature;
edit_.AddFile(f.picked_level, f_metadata);
}

3
db/flush_job.cc
View File

@ -1000,8 +1000,7 @@ Status FlushJob::WriteLevel0Table() {
meta_.marked_for_compaction, meta_.temperature,
meta_.oldest_blob_file_number, meta_.oldest_ancester_time,
meta_.file_creation_time, meta_.file_checksum,
meta_.file_checksum_func_name, meta_.min_timestamp,
meta_.max_timestamp, meta_.unique_id);
meta_.file_checksum_func_name, meta_.unique_id);
edit_->SetBlobFileAdditions(std::move(blob_file_additions));
}

2
db/import_column_family_job.cc
View File

@ -160,7 +160,7 @@ Status ImportColumnFamilyJob::Run() {
file_metadata.largest_seqno, false, file_metadata.temperature,
kInvalidBlobFileNumber, oldest_ancester_time, current_time,
kUnknownFileChecksum, kUnknownFileChecksumFuncName,
kDisableUserTimestamp, kDisableUserTimestamp, f.unique_id);
f.unique_id);
// If incoming sequence number is higher, update local sequence number.
if (file_metadata.largest_seqno > versions_->LastSequence()) {

1
db/repair.cc
View File

@ -651,7 +651,6 @@ class Repairer {
table->meta.temperature, table->meta.oldest_blob_file_number,
table->meta.oldest_ancester_time, table->meta.file_creation_time,
table->meta.file_checksum, table->meta.file_checksum_func_name,
table->meta.min_timestamp, table->meta.max_timestamp,
table->meta.unique_id);
}
assert(next_file_number_ > 0);

99
db/version_builder_test.cc
View File

@ -72,8 +72,7 @@ class VersionBuilderTest : public testing::Test {
/* marked_for_compact */ false, Temperature::kUnknown,
oldest_blob_file_number, kUnknownOldestAncesterTime,
kUnknownFileCreationTime, kUnknownFileChecksum,
kUnknownFileChecksumFuncName, kDisableUserTimestamp,
kDisableUserTimestamp, kNullUniqueId64x2);
kUnknownFileChecksumFuncName, kNullUniqueId64x2);
f->compensated_file_size = file_size;
f->num_entries = num_entries;
f->num_deletions = num_deletions;
@ -134,8 +133,7 @@ class VersionBuilderTest : public testing::Test {
GetInternalKey(largest), smallest_seqno, largest_seqno,
marked_for_compaction, Temperature::kUnknown, blob_file_number,
kUnknownOldestAncesterTime, kUnknownFileCreationTime,
kUnknownFileChecksum, kUnknownFileChecksumFuncName,
kDisableUserTimestamp, kDisableUserTimestamp, kNullUniqueId64x2);
kUnknownFileChecksum, kUnknownFileChecksumFuncName, kNullUniqueId64x2);
}
void UpdateVersionStorageInfo(VersionStorageInfo* vstorage) {
@ -180,8 +178,7 @@ TEST_F(VersionBuilderTest, ApplyAndSaveTo) {
2, 666, 0, 100U, GetInternalKey("301"), GetInternalKey("350"), 200, 200,
false, Temperature::kUnknown, kInvalidBlobFileNumber,
kUnknownOldestAncesterTime, kUnknownFileCreationTime,
kUnknownFileChecksum, kUnknownFileChecksumFuncName, kDisableUserTimestamp,
kDisableUserTimestamp, kNullUniqueId64x2);
kUnknownFileChecksum, kUnknownFileChecksumFuncName, kNullUniqueId64x2);
version_edit.DeleteFile(3, 27U);
EnvOptions env_options;
@ -224,8 +221,7 @@ TEST_F(VersionBuilderTest, ApplyAndSaveToDynamic) {
3, 666, 0, 100U, GetInternalKey("301"), GetInternalKey("350"), 200, 200,
false, Temperature::kUnknown, kInvalidBlobFileNumber,
kUnknownOldestAncesterTime, kUnknownFileCreationTime,
kUnknownFileChecksum, kUnknownFileChecksumFuncName, kDisableUserTimestamp,
kDisableUserTimestamp, kNullUniqueId64x2);
kUnknownFileChecksum, kUnknownFileChecksumFuncName, kNullUniqueId64x2);
version_edit.DeleteFile(0, 1U);
version_edit.DeleteFile(0, 88U);
@ -271,8 +267,7 @@ TEST_F(VersionBuilderTest, ApplyAndSaveToDynamic2) {
4, 666, 0, 100U, GetInternalKey("301"), GetInternalKey("350"), 200, 200,
false, Temperature::kUnknown, kInvalidBlobFileNumber,
kUnknownOldestAncesterTime, kUnknownFileCreationTime,
kUnknownFileChecksum, kUnknownFileChecksumFuncName, kDisableUserTimestamp,
kDisableUserTimestamp, kNullUniqueId64x2);
kUnknownFileChecksum, kUnknownFileChecksumFuncName, kNullUniqueId64x2);
version_edit.DeleteFile(0, 1U);
version_edit.DeleteFile(0, 88U);
version_edit.DeleteFile(4, 6U);
@ -308,32 +303,27 @@ TEST_F(VersionBuilderTest, ApplyMultipleAndSaveTo) {
2, 666, 0, 100U, GetInternalKey("301"), GetInternalKey("350"), 200, 200,
false, Temperature::kUnknown, kInvalidBlobFileNumber,
kUnknownOldestAncesterTime, kUnknownFileCreationTime,
kUnknownFileChecksum, kUnknownFileChecksumFuncName, kDisableUserTimestamp,
kDisableUserTimestamp, kNullUniqueId64x2);
kUnknownFileChecksum, kUnknownFileChecksumFuncName, kNullUniqueId64x2);
version_edit.AddFile(
2, 676, 0, 100U, GetInternalKey("401"), GetInternalKey("450"), 200, 200,
false, Temperature::kUnknown, kInvalidBlobFileNumber,
kUnknownOldestAncesterTime, kUnknownFileCreationTime,
kUnknownFileChecksum, kUnknownFileChecksumFuncName, kDisableUserTimestamp,
kDisableUserTimestamp, kNullUniqueId64x2);
kUnknownFileChecksum, kUnknownFileChecksumFuncName, kNullUniqueId64x2);
version_edit.AddFile(
2, 636, 0, 100U, GetInternalKey("601"), GetInternalKey("650"), 200, 200,
false, Temperature::kUnknown, kInvalidBlobFileNumber,
kUnknownOldestAncesterTime, kUnknownFileCreationTime,
kUnknownFileChecksum, kUnknownFileChecksumFuncName, kDisableUserTimestamp,
kDisableUserTimestamp, kNullUniqueId64x2);
kUnknownFileChecksum, kUnknownFileChecksumFuncName, kNullUniqueId64x2);
version_edit.AddFile(
2, 616, 0, 100U, GetInternalKey("501"), GetInternalKey("550"), 200, 200,
false, Temperature::kUnknown, kInvalidBlobFileNumber,
kUnknownOldestAncesterTime, kUnknownFileCreationTime,
kUnknownFileChecksum, kUnknownFileChecksumFuncName, kDisableUserTimestamp,
kDisableUserTimestamp, kNullUniqueId64x2);
kUnknownFileChecksum, kUnknownFileChecksumFuncName, kNullUniqueId64x2);
version_edit.AddFile(
2, 606, 0, 100U, GetInternalKey("701"), GetInternalKey("750"), 200, 200,
false, Temperature::kUnknown, kInvalidBlobFileNumber,
kUnknownOldestAncesterTime, kUnknownFileCreationTime,
kUnknownFileChecksum, kUnknownFileChecksumFuncName, kDisableUserTimestamp,
kDisableUserTimestamp, kNullUniqueId64x2);
kUnknownFileChecksum, kUnknownFileChecksumFuncName, kNullUniqueId64x2);
EnvOptions env_options;
constexpr TableCache* table_cache = nullptr;
@ -372,32 +362,27 @@ TEST_F(VersionBuilderTest, ApplyDeleteAndSaveTo) {
2, 666, 0, 100U, GetInternalKey("301"), GetInternalKey("350"), 200, 200,
false, Temperature::kUnknown, kInvalidBlobFileNumber,
kUnknownOldestAncesterTime, kUnknownFileCreationTime,
kUnknownFileChecksum, kUnknownFileChecksumFuncName, kDisableUserTimestamp,
kDisableUserTimestamp, kNullUniqueId64x2);
kUnknownFileChecksum, kUnknownFileChecksumFuncName, kNullUniqueId64x2);
version_edit.AddFile(
2, 676, 0, 100U, GetInternalKey("401"), GetInternalKey("450"), 200, 200,
false, Temperature::kUnknown, kInvalidBlobFileNumber,
kUnknownOldestAncesterTime, kUnknownFileCreationTime,
kUnknownFileChecksum, kUnknownFileChecksumFuncName, kDisableUserTimestamp,
kDisableUserTimestamp, kNullUniqueId64x2);
kUnknownFileChecksum, kUnknownFileChecksumFuncName, kNullUniqueId64x2);
version_edit.AddFile(
2, 636, 0, 100U, GetInternalKey("601"), GetInternalKey("650"), 200, 200,
false, Temperature::kUnknown, kInvalidBlobFileNumber,
kUnknownOldestAncesterTime, kUnknownFileCreationTime,
kUnknownFileChecksum, kUnknownFileChecksumFuncName, kDisableUserTimestamp,
kDisableUserTimestamp, kNullUniqueId64x2);
kUnknownFileChecksum, kUnknownFileChecksumFuncName, kNullUniqueId64x2);
version_edit.AddFile(
2, 616, 0, 100U, GetInternalKey("501"), GetInternalKey("550"), 200, 200,
false, Temperature::kUnknown, kInvalidBlobFileNumber,
kUnknownOldestAncesterTime, kUnknownFileCreationTime,
kUnknownFileChecksum, kUnknownFileChecksumFuncName, kDisableUserTimestamp,
kDisableUserTimestamp, kNullUniqueId64x2);
kUnknownFileChecksum, kUnknownFileChecksumFuncName, kNullUniqueId64x2);
version_edit.AddFile(
2, 606, 0, 100U, GetInternalKey("701"), GetInternalKey("750"), 200, 200,
false, Temperature::kUnknown, kInvalidBlobFileNumber,
kUnknownOldestAncesterTime, kUnknownFileCreationTime,
kUnknownFileChecksum, kUnknownFileChecksumFuncName, kDisableUserTimestamp,
kDisableUserTimestamp, kNullUniqueId64x2);
kUnknownFileChecksum, kUnknownFileChecksumFuncName, kNullUniqueId64x2);
ASSERT_OK(version_builder.Apply(&version_edit));
VersionEdit version_edit2;
@ -405,16 +390,14 @@ TEST_F(VersionBuilderTest, ApplyDeleteAndSaveTo) {
2, 808, 0, 100U, GetInternalKey("901"), GetInternalKey("950"), 200, 200,
false, Temperature::kUnknown, kInvalidBlobFileNumber,
kUnknownOldestAncesterTime, kUnknownFileCreationTime,
kUnknownFileChecksum, kUnknownFileChecksumFuncName, kDisableUserTimestamp,
kDisableUserTimestamp, kNullUniqueId64x2);
kUnknownFileChecksum, kUnknownFileChecksumFuncName, kNullUniqueId64x2);
version_edit2.DeleteFile(2, 616);
version_edit2.DeleteFile(2, 636);
version_edit.AddFile(
2, 806, 0, 100U, GetInternalKey("801"), GetInternalKey("850"), 200, 200,
false, Temperature::kUnknown, kInvalidBlobFileNumber,
kUnknownOldestAncesterTime, kUnknownFileCreationTime,
kUnknownFileChecksum, kUnknownFileChecksumFuncName, kDisableUserTimestamp,
kDisableUserTimestamp, kNullUniqueId64x2);
kUnknownFileChecksum, kUnknownFileChecksumFuncName, kNullUniqueId64x2);
ASSERT_OK(version_builder.Apply(&version_edit2));
ASSERT_OK(version_builder.SaveTo(&new_vstorage));
@ -525,8 +508,7 @@ TEST_F(VersionBuilderTest, ApplyFileDeletionAndAddition) {
largest_seqno, marked_for_compaction, Temperature::kUnknown,
kInvalidBlobFileNumber, kUnknownOldestAncesterTime,
kUnknownFileCreationTime, kUnknownFileChecksum,
kUnknownFileChecksumFuncName, kDisableUserTimestamp,
kDisableUserTimestamp, kNullUniqueId64x2);
kUnknownFileChecksumFuncName, kNullUniqueId64x2);
ASSERT_OK(builder.Apply(&addition));
@ -570,13 +552,12 @@ TEST_F(VersionBuilderTest, ApplyFileAdditionAlreadyInBase) {
constexpr SequenceNumber largest_seqno = 1000;
constexpr bool marked_for_compaction = false;
edit.AddFile(new_level, file_number, path_id, file_size,
GetInternalKey(smallest), GetInternalKey(largest),
smallest_seqno, largest_seqno, marked_for_compaction,
Temperature::kUnknown, kInvalidBlobFileNumber,
kUnknownOldestAncesterTime, kUnknownFileCreationTime,
kUnknownFileChecksum, kUnknownFileChecksumFuncName,
kDisableUserTimestamp, kDisableUserTimestamp, kNullUniqueId64x2);
edit.AddFile(
new_level, file_number, path_id, file_size, GetInternalKey(smallest),
GetInternalKey(largest), smallest_seqno, largest_seqno,
marked_for_compaction, Temperature::kUnknown, kInvalidBlobFileNumber,
kUnknownOldestAncesterTime, kUnknownFileCreationTime,
kUnknownFileChecksum, kUnknownFileChecksumFuncName, kNullUniqueId64x2);
const Status s = builder.Apply(&edit);
ASSERT_TRUE(s.IsCorruption());
@ -612,8 +593,7 @@ TEST_F(VersionBuilderTest, ApplyFileAdditionAlreadyApplied) {
marked_for_compaction, Temperature::kUnknown,
kInvalidBlobFileNumber, kUnknownOldestAncesterTime,
kUnknownFileCreationTime, kUnknownFileChecksum,
kUnknownFileChecksumFuncName, kDisableUserTimestamp,
kDisableUserTimestamp, kNullUniqueId64x2);
kUnknownFileChecksumFuncName, kNullUniqueId64x2);
ASSERT_OK(builder.Apply(&edit));
@ -626,8 +606,7 @@ TEST_F(VersionBuilderTest, ApplyFileAdditionAlreadyApplied) {
GetInternalKey(largest), smallest_seqno, largest_seqno,
marked_for_compaction, Temperature::kUnknown, kInvalidBlobFileNumber,
kUnknownOldestAncesterTime, kUnknownFileCreationTime,
kUnknownFileChecksum, kUnknownFileChecksumFuncName, kDisableUserTimestamp,
kDisableUserTimestamp, kNullUniqueId64x2);
kUnknownFileChecksum, kUnknownFileChecksumFuncName, kNullUniqueId64x2);
const Status s = builder.Apply(&other_edit);
ASSERT_TRUE(s.IsCorruption());
@ -663,8 +642,7 @@ TEST_F(VersionBuilderTest, ApplyFileAdditionAndDeletion) {
GetInternalKey(largest), smallest_seqno, largest_seqno,
marked_for_compaction, Temperature::kUnknown, kInvalidBlobFileNumber,
kUnknownOldestAncesterTime, kUnknownFileCreationTime,
kUnknownFileChecksum, kUnknownFileChecksumFuncName, kDisableUserTimestamp,
kDisableUserTimestamp, kNullUniqueId64x2);
kUnknownFileChecksum, kUnknownFileChecksumFuncName, kNullUniqueId64x2);
ASSERT_OK(builder.Apply(&addition));
@ -1233,8 +1211,7 @@ TEST_F(VersionBuilderTest, SaveBlobFilesToConcurrentJobs) {
smallest_seqno, largest_seqno, marked_for_compaction,
Temperature::kUnknown, blob_file_number,
kUnknownOldestAncesterTime, kUnknownFileCreationTime,
checksum_value, checksum_method, kDisableUserTimestamp,
kDisableUserTimestamp, kNullUniqueId64x2);
checksum_value, checksum_method, kNullUniqueId64x2);
edit.AddBlobFile(blob_file_number, total_blob_count, total_blob_bytes,
checksum_method, checksum_value);
@ -1321,8 +1298,7 @@ TEST_F(VersionBuilderTest, CheckConsistencyForBlobFiles) {
Temperature::kUnknown,
/* oldest_blob_file_number */ 16, kUnknownOldestAncesterTime,
kUnknownFileCreationTime, kUnknownFileChecksum,
kUnknownFileChecksumFuncName, kDisableUserTimestamp,
kDisableUserTimestamp, kNullUniqueId64x2);
kUnknownFileChecksumFuncName, kNullUniqueId64x2);
edit.AddFile(/* level */ 1, /* file_number */ 700, /* path_id */ 0,
/* file_size */ 100, /* smallest */ GetInternalKey("801"),
@ -1331,8 +1307,7 @@ TEST_F(VersionBuilderTest, CheckConsistencyForBlobFiles) {
Temperature::kUnknown,
/* oldest_blob_file_number */ 1000, kUnknownOldestAncesterTime,
kUnknownFileCreationTime, kUnknownFileChecksum,
kUnknownFileChecksumFuncName, kDisableUserTimestamp,
kDisableUserTimestamp, kNullUniqueId64x2);
kUnknownFileChecksumFuncName, kNullUniqueId64x2);
edit.AddBlobFile(/* blob_file_number */ 1000, /* total_blob_count */ 2000,
/* total_blob_bytes */ 200000,
/* checksum_method */ std::string(),
@ -1553,8 +1528,7 @@ TEST_F(VersionBuilderTest, MaintainLinkedSstsForBlobFiles) {
Temperature::kUnknown,
/* oldest_blob_file_number */ 1, kUnknownOldestAncesterTime,
kUnknownFileCreationTime, kUnknownFileChecksum,
kUnknownFileChecksumFuncName, kDisableUserTimestamp,
kDisableUserTimestamp, kNullUniqueId64x2);
kUnknownFileChecksumFuncName, kNullUniqueId64x2);
// Add an SST that does not reference any blob files.
edit.AddFile(
@ -1564,8 +1538,7 @@ TEST_F(VersionBuilderTest, MaintainLinkedSstsForBlobFiles) {
/* largest_seqno */ 2200, /* marked_for_compaction */ false,
Temperature::kUnknown, kInvalidBlobFileNumber, kUnknownOldestAncesterTime,
kUnknownFileCreationTime, kUnknownFileChecksum,
kUnknownFileChecksumFuncName, kDisableUserTimestamp,
kDisableUserTimestamp, kNullUniqueId64x2);
kUnknownFileChecksumFuncName, kNullUniqueId64x2);
// Delete a file that references a blob file.
edit.DeleteFile(/* level */ 1, /* file_number */ 6);
@ -1587,8 +1560,7 @@ TEST_F(VersionBuilderTest, MaintainLinkedSstsForBlobFiles) {
Temperature::kUnknown,
/* oldest_blob_file_number */ 3, kUnknownOldestAncesterTime,
kUnknownFileCreationTime, kUnknownFileChecksum,
kUnknownFileChecksumFuncName, kDisableUserTimestamp,
kDisableUserTimestamp, kNullUniqueId64x2);
kUnknownFileChecksumFuncName, kNullUniqueId64x2);
// Trivially move a file that does not reference any blob files.
edit.DeleteFile(/* level */ 1, /* file_number */ 13);
@ -1600,7 +1572,7 @@ TEST_F(VersionBuilderTest, MaintainLinkedSstsForBlobFiles) {
Temperature::kUnknown, kInvalidBlobFileNumber,
kUnknownOldestAncesterTime, kUnknownFileCreationTime,
kUnknownFileChecksum, kUnknownFileChecksumFuncName,
kDisableUserTimestamp, kDisableUserTimestamp, kNullUniqueId64x2);
kNullUniqueId64x2);
// Add one more SST file that references a blob file, then promptly
// delete it in a second version edit before the new version gets saved.
@ -1613,8 +1585,7 @@ TEST_F(VersionBuilderTest, MaintainLinkedSstsForBlobFiles) {
Temperature::kUnknown,
/* oldest_blob_file_number */ 5, kUnknownOldestAncesterTime,
kUnknownFileCreationTime, kUnknownFileChecksum,
kUnknownFileChecksumFuncName, kDisableUserTimestamp,
kDisableUserTimestamp, kNullUniqueId64x2);
kUnknownFileChecksumFuncName, kNullUniqueId64x2);
VersionEdit edit2;

32
db/version_edit.cc
View File

@ -192,16 +192,6 @@ bool VersionEdit::EncodeTo(std::string* dst) const {
PutVarint32(dst, NewFileCustomTag::kFileChecksumFuncName);
PutLengthPrefixedSlice(dst, Slice(f.file_checksum_func_name));
if (f.max_timestamp != kDisableUserTimestamp) {
if (f.min_timestamp.size() != f.max_timestamp.size()) {
assert(false);
return false;
}
PutVarint32(dst, NewFileCustomTag::kMinTimestamp);
PutLengthPrefixedSlice(dst, Slice(f.min_timestamp));
PutVarint32(dst, NewFileCustomTag::kMaxTimestamp);
PutLengthPrefixedSlice(dst, Slice(f.max_timestamp));
}
if (f.fd.GetPathId() != 0) {
PutVarint32(dst, NewFileCustomTag::kPathId);
char p = static_cast<char>(f.fd.GetPathId());
@ -339,10 +329,6 @@ const char* VersionEdit::DecodeNewFile4From(Slice* input) {
return "new-file4 custom field";
}
if (custom_tag == kTerminate) {
if (f.min_timestamp.size() != f.max_timestamp.size()) {
assert(false);
return "new-file4 custom field timestamp size mismatch error";
}
break;
}
if (!GetLengthPrefixedSlice(input, &field)) {
@ -403,12 +389,6 @@ const char* VersionEdit::DecodeNewFile4From(Slice* input) {
}
}
break;
case kMinTimestamp:
f.min_timestamp = field.ToString();
break;
case kMaxTimestamp:
f.max_timestamp = field.ToString();
break;
case kUniqueId:
if (!DecodeUniqueIdBytes(field.ToString(), &f.unique_id).ok()) {
f.unique_id = kNullUniqueId64x2;
@ -827,13 +807,6 @@ std::string VersionEdit::DebugString(bool hex_key) const {
r.append(" blob_file:");
AppendNumberTo(&r, f.oldest_blob_file_number);
}
if (f.min_timestamp != kDisableUserTimestamp) {
assert(f.max_timestamp != kDisableUserTimestamp);
r.append(" min_timestamp:");
r.append(Slice(f.min_timestamp).ToString(true));
r.append(" max_timestamp:");
r.append(Slice(f.max_timestamp).ToString(true));
}
r.append(" oldest_ancester_time:");
AppendNumberTo(&r, f.oldest_ancester_time);
r.append(" file_creation_time:");
@ -955,11 +928,6 @@ std::string VersionEdit::DebugJSON(int edit_num, bool hex_key) const {
jw << "FileSize" << f.fd.GetFileSize();
jw << "SmallestIKey" << f.smallest.DebugString(hex_key);
jw << "LargestIKey" << f.largest.DebugString(hex_key);
if (f.min_timestamp != kDisableUserTimestamp) {
assert(f.max_timestamp != kDisableUserTimestamp);
jw << "MinTimestamp" << Slice(f.min_timestamp).ToString(true);
jw << "MaxTimestamp" << Slice(f.max_timestamp).ToString(true);
}
jw << "OldestAncesterTime" << f.oldest_ancester_time;
jw << "FileCreationTime" << f.file_creation_time;
jw << "FileChecksum" << Slice(f.file_checksum).ToString(true);

14
db/version_edit.h
View File

@ -216,10 +216,6 @@ struct FileMetaData {
// File checksum function name
std::string file_checksum_func_name = kUnknownFileChecksumFuncName;
// Min (oldest) timestamp of keys in this file
std::string min_timestamp;
// Max (newest) timestamp of keys in this file
std::string max_timestamp;
// SST unique id
UniqueId64x2 unique_id{};
@ -234,7 +230,6 @@ struct FileMetaData {
uint64_t _oldest_ancester_time, uint64_t _file_creation_time,
const std::string& _file_checksum,
const std::string& _file_checksum_func_name,
std::string _min_timestamp, std::string _max_timestamp,
UniqueId64x2 _unique_id)
: fd(file, file_path_id, file_size, smallest_seq, largest_seq),
smallest(smallest_key),
@ -246,8 +241,6 @@ struct FileMetaData {
file_creation_time(_file_creation_time),
file_checksum(_file_checksum),
file_checksum_func_name(_file_checksum_func_name),
min_timestamp(std::move(_min_timestamp)),
max_timestamp(std::move(_max_timestamp)),
unique_id(std::move(_unique_id)) {
TEST_SYNC_POINT_CALLBACK("FileMetaData::FileMetaData", this);
}
@ -309,8 +302,7 @@ struct FileMetaData {
usage += sizeof(*this);
#endif // ROCKSDB_MALLOC_USABLE_SIZE
usage += smallest.size() + largest.size() + file_checksum.size() +
file_checksum_func_name.size() + min_timestamp.size() +
max_timestamp.size();
file_checksum_func_name.size();
return usage;
}
};
@ -435,8 +427,6 @@ class VersionEdit {
uint64_t oldest_ancester_time, uint64_t file_creation_time,
const std::string& file_checksum,
const std::string& file_checksum_func_name,
const std::string& min_timestamp,
const std::string& max_timestamp,
const UniqueId64x2& unique_id) {
assert(smallest_seqno <= largest_seqno);
new_files_.emplace_back(
@ -445,7 +435,7 @@ class VersionEdit {
smallest_seqno, largest_seqno, marked_for_compaction,
temperature, oldest_blob_file_number, oldest_ancester_time,
file_creation_time, file_checksum, file_checksum_func_name,
min_timestamp, max_timestamp, unique_id));
unique_id));
if (!HasLastSequence() || largest_seqno > GetLastSequence()) {
SetLastSequence(largest_seqno);
}

33
db/version_edit_test.cc
View File

@ -43,8 +43,8 @@ TEST_F(VersionEditTest, EncodeDecode) {
InternalKey("foo", kBig + 500 + i, kTypeValue),
InternalKey("zoo", kBig + 600 + i, kTypeDeletion),
kBig + 500 + i, kBig + 600 + i, false, Temperature::kUnknown,
kInvalidBlobFileNumber, 888, 678, "234", "crc32c", "123",
"345", kNullUniqueId64x2);
kInvalidBlobFileNumber, 888, 678, "234", "crc32c",
kNullUniqueId64x2);
edit.DeleteFile(4, kBig + 700 + i);
}
@ -63,24 +63,24 @@ TEST_F(VersionEditTest, EncodeDecodeNewFile4) {
InternalKey("zoo", kBig + 600, kTypeDeletion), kBig + 500,
kBig + 600, true, Temperature::kUnknown, kInvalidBlobFileNumber,
kUnknownOldestAncesterTime, kUnknownFileCreationTime,
kUnknownFileChecksum, kUnknownFileChecksumFuncName, "123", "234",
kUnknownFileChecksum, kUnknownFileChecksumFuncName,
kNullUniqueId64x2);
edit.AddFile(4, 301, 3, 100, InternalKey("foo", kBig + 501, kTypeValue),
InternalKey("zoo", kBig + 601, kTypeDeletion), kBig + 501,
kBig + 601, false, Temperature::kUnknown, kInvalidBlobFileNumber,
kUnknownOldestAncesterTime, kUnknownFileCreationTime,
kUnknownFileChecksum, kUnknownFileChecksumFuncName, "345", "543",
kUnknownFileChecksum, kUnknownFileChecksumFuncName,
kNullUniqueId64x2);
edit.AddFile(5, 302, 0, 100, InternalKey("foo", kBig + 502, kTypeValue),
InternalKey("zoo", kBig + 602, kTypeDeletion), kBig + 502,
kBig + 602, true, Temperature::kUnknown, kInvalidBlobFileNumber,
666, 888, kUnknownFileChecksum, kUnknownFileChecksumFuncName,
"456", "567", kNullUniqueId64x2);
kNullUniqueId64x2);
edit.AddFile(5, 303, 0, 100, InternalKey("foo", kBig + 503, kTypeBlobIndex),
InternalKey("zoo", kBig + 603, kTypeBlobIndex), kBig + 503,
kBig + 603, true, Temperature::kUnknown, 1001,
kUnknownOldestAncesterTime, kUnknownFileCreationTime,
kUnknownFileChecksum, kUnknownFileChecksumFuncName, "678", "789",
kUnknownFileChecksum, kUnknownFileChecksumFuncName,
kNullUniqueId64x2);
edit.DeleteFile(4, 700);
@ -112,14 +112,6 @@ TEST_F(VersionEditTest, EncodeDecodeNewFile4) {
ASSERT_EQ(kInvalidBlobFileNumber,
new_files[2].second.oldest_blob_file_number);
ASSERT_EQ(1001, new_files[3].second.oldest_blob_file_number);
ASSERT_EQ("123", new_files[0].second.min_timestamp);
ASSERT_EQ("234", new_files[0].second.max_timestamp);
ASSERT_EQ("345", new_files[1].second.min_timestamp);
ASSERT_EQ("543", new_files[1].second.max_timestamp);
ASSERT_EQ("456", new_files[2].second.min_timestamp);
ASSERT_EQ("567", new_files[2].second.max_timestamp);
ASSERT_EQ("678", new_files[3].second.min_timestamp);
ASSERT_EQ("789", new_files[3].second.max_timestamp);
}
TEST_F(VersionEditTest, ForwardCompatibleNewFile4) {
@ -129,13 +121,12 @@ TEST_F(VersionEditTest, ForwardCompatibleNewFile4) {
InternalKey("zoo", kBig + 600, kTypeDeletion), kBig + 500,
kBig + 600, true, Temperature::kUnknown, kInvalidBlobFileNumber,
kUnknownOldestAncesterTime, kUnknownFileCreationTime,
kUnknownFileChecksum, kUnknownFileChecksumFuncName, "123", "234",
kUnknownFileChecksum, kUnknownFileChecksumFuncName,
kNullUniqueId64x2);
edit.AddFile(4, 301, 3, 100, InternalKey("foo", kBig + 501, kTypeValue),
InternalKey("zoo", kBig + 601, kTypeDeletion), kBig + 501,
kBig + 601, false, Temperature::kUnknown, kInvalidBlobFileNumber,
686, 868, "234", "crc32c", kDisableUserTimestamp,
kDisableUserTimestamp, kNullUniqueId64x2);
686, 868, "234", "crc32c", kNullUniqueId64x2);
edit.DeleteFile(4, 700);
edit.SetComparatorName("foo");
@ -174,10 +165,6 @@ TEST_F(VersionEditTest, ForwardCompatibleNewFile4) {
ASSERT_EQ(3u, new_files[0].second.fd.GetPathId());
ASSERT_EQ(3u, new_files[1].second.fd.GetPathId());
ASSERT_EQ(1u, parsed.GetDeletedFiles().size());
ASSERT_EQ("123", new_files[0].second.min_timestamp);
ASSERT_EQ("234", new_files[0].second.max_timestamp);
ASSERT_EQ(kDisableUserTimestamp, new_files[1].second.min_timestamp);
ASSERT_EQ(kDisableUserTimestamp, new_files[1].second.max_timestamp);
}
TEST_F(VersionEditTest, NewFile4NotSupportedField) {
@ -188,7 +175,7 @@ TEST_F(VersionEditTest, NewFile4NotSupportedField) {
kBig + 600, true, Temperature::kUnknown, kInvalidBlobFileNumber,
kUnknownOldestAncesterTime, kUnknownFileCreationTime,
kUnknownFileChecksum, kUnknownFileChecksumFuncName,
kDisableUserTimestamp, kDisableUserTimestamp, kNullUniqueId64x2);
kNullUniqueId64x2);
edit.SetComparatorName("foo");
edit.SetLogNumber(kBig + 100);
@ -219,7 +206,7 @@ TEST_F(VersionEditTest, EncodeEmptyFile) {
Temperature::kUnknown, kInvalidBlobFileNumber,
kUnknownOldestAncesterTime, kUnknownFileCreationTime,
kUnknownFileChecksum, kUnknownFileChecksumFuncName,
kDisableUserTimestamp, kDisableUserTimestamp, kNullUniqueId64x2);
kNullUniqueId64x2);
std::string buffer;
ASSERT_TRUE(!edit.EncodeTo(&buffer));
}

3
db/version_set.cc
View File

@ -5550,8 +5550,7 @@ Status VersionSet::WriteCurrentStateToManifest(
f->marked_for_compaction, f->temperature,
f->oldest_blob_file_number, f->oldest_ancester_time,
f->file_creation_time, f->file_checksum,
f->file_checksum_func_name, f->min_timestamp,
f->max_timestamp, f->unique_id);
f->file_checksum_func_name, f->unique_id);
}
}

21
db/version_set_test.cc
View File

@ -50,8 +50,7 @@ class GenerateLevelFilesBriefTest : public testing::Test {
largest_seq, /* marked_for_compact */ false, Temperature::kUnknown,
kInvalidBlobFileNumber, kUnknownOldestAncesterTime,
kUnknownFileCreationTime, kUnknownFileChecksum,
kUnknownFileChecksumFuncName, kDisableUserTimestamp,
kDisableUserTimestamp, kNullUniqueId64x2);
kUnknownFileChecksumFuncName, kNullUniqueId64x2);
files_.push_back(f);
}
@ -159,8 +158,7 @@ class VersionStorageInfoTestBase : public testing::Test {
/* largest_seq */ 0, /* marked_for_compact */ false,
Temperature::kUnknown, oldest_blob_file_number,
kUnknownOldestAncesterTime, kUnknownFileCreationTime,
kUnknownFileChecksum, kUnknownFileChecksumFuncName,
kDisableUserTimestamp, kDisableUserTimestamp, kNullUniqueId64x2);
kUnknownFileChecksum, kUnknownFileChecksumFuncName, kNullUniqueId64x2);
f->compensated_file_size = file_size;
vstorage_.AddFile(level, f);
}
@ -3233,11 +3231,10 @@ class VersionSetTestMissingFiles : public VersionSetTestBase,
s = fs_->GetFileSize(fname, IOOptions(), &file_size, nullptr);
ASSERT_OK(s);
ASSERT_NE(0, file_size);
file_metas->emplace_back(
file_num, /*file_path_id=*/0, file_size, ikey, ikey, 0, 0, false,
Temperature::kUnknown, 0, 0, 0, kUnknownFileChecksum,
kUnknownFileChecksumFuncName, kDisableUserTimestamp,
kDisableUserTimestamp, kNullUniqueId64x2);
file_metas->emplace_back(file_num, /*file_path_id=*/0, file_size, ikey,
ikey, 0, 0, false, Temperature::kUnknown, 0, 0,
0, kUnknownFileChecksum,
kUnknownFileChecksumFuncName, kNullUniqueId64x2);
}
}
@ -3292,8 +3289,7 @@ TEST_F(VersionSetTestMissingFiles, ManifestFarBehindSst) {
FileMetaData meta = FileMetaData(
file_num, /*file_path_id=*/0, /*file_size=*/12, smallest_ikey,
largest_ikey, 0, 0, false, Temperature::kUnknown, 0, 0, 0,
kUnknownFileChecksum, kUnknownFileChecksumFuncName,
kDisableUserTimestamp, kDisableUserTimestamp, kNullUniqueId64x2);
kUnknownFileChecksum, kUnknownFileChecksumFuncName, kNullUniqueId64x2);
added_files.emplace_back(0, meta);
}
WriteFileAdditionAndDeletionToManifest(
@ -3348,8 +3344,7 @@ TEST_F(VersionSetTestMissingFiles, ManifestAheadofSst) {
FileMetaData meta = FileMetaData(
file_num, /*file_path_id=*/0, /*file_size=*/12, smallest_ikey,
largest_ikey, 0, 0, false, Temperature::kUnknown, 0, 0, 0,
kUnknownFileChecksum, kUnknownFileChecksumFuncName,
kDisableUserTimestamp, kDisableUserTimestamp, kNullUniqueId64x2);
kUnknownFileChecksum, kUnknownFileChecksumFuncName, kNullUniqueId64x2);
added_files.emplace_back(0, meta);
}
WriteFileAdditionAndDeletionToManifest(