apple/foundationdb
1
0
Fork 0
mirror of https://github.com/apple/foundationdb.git synced 2025-06-03 03:41:53 +08:00
Code Issues Packages Projects Releases Wiki Activity

Merge branch 'master' of https://github.com/apple/foundationdb into jfu-ibackup-correctness

Browse Source
This commit is contained in:
Jon Fu 2020-10-12 14:56:05 -04:00
commit ae4ba55445
12 changed files with 83 additions and 40 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
bindings/c/fdb_c.cpp
View File

@ -646,9 +646,9 @@ FDBFuture* fdb_transaction_get_estimated_range_size_bytes( FDBTransaction* tr, u
extern "C" DLLEXPORT
FDBFuture* fdb_transaction_get_range_split_points( FDBTransaction* tr, uint8_t const* begin_key_name,
int begin_key_name_length, uint8_t const* end_key_name, int end_key_name_length, int64_t chunkSize) {
int begin_key_name_length, uint8_t const* end_key_name, int end_key_name_length, int64_t chunk_size) {
KeyRangeRef range(KeyRef(begin_key_name, begin_key_name_length), KeyRef(end_key_name, end_key_name_length));
return (FDBFuture*)(TXN(tr)->getRangeSplitPoints(range, chunkSize).extractPtr());
return (FDBFuture*)(TXN(tr)->getRangeSplitPoints(range, chunk_size).extractPtr());
}
#include "fdb_c_function_pointers.g.h"

2
bindings/c/foundationdb/fdb_c.h
View File

@ -269,7 +269,7 @@ extern "C" {
DLLEXPORT WARN_UNUSED_RESULT FDBFuture*
fdb_transaction_get_range_split_points( FDBTransaction* tr, uint8_t const* begin_key_name,
int begin_key_name_length, uint8_t const* end_key_name, int end_key_name_length, int64_t chunkSize);
int begin_key_name_length, uint8_t const* end_key_name, int end_key_name_length, int64_t chunk_size);
#define FDB_KEYSEL_LAST_LESS_THAN(k, l) k, l, 0, 0
#define FDB_KEYSEL_LAST_LESS_OR_EQUAL(k, l) k, l, 1, 0

7
bindings/go/src/fdb/snapshot.go
View File

@ -87,7 +87,7 @@ func (s Snapshot) GetDatabase() Database {
return s.transaction.db
}
// GetEstimatedRangeSizeBytes will get an estimate for the number of bytes
// GetEstimatedRangeSizeBytes returns an estimate for the number of bytes
// stored in the given range.
func (s Snapshot) GetEstimatedRangeSizeBytes(r ExactRange) FutureInt64 {
beginKey, endKey := r.FDBRangeKeys()
@ -97,8 +97,9 @@ func (s Snapshot) GetEstimatedRangeSizeBytes(r ExactRange) FutureInt64 {
)
}
// GetRangeSplitPoints will return a list of keys that can divide the given range into
// chunks based on the chunk size provided.
// GetRangeSplitPoints returns a list of keys that can split the given range
// into (roughly) equally sized chunks based on chunkSize.
// Note: the returned split points contain the start key and end key of the given range.
func (s Snapshot) GetRangeSplitPoints(r ExactRange, chunkSize int64) FutureKeyArray {
beginKey, endKey := r.FDBRangeKeys()
return s.getRangeSplitPoints(

7
bindings/go/src/fdb/transaction.go
View File

@ -319,7 +319,7 @@ func (t *transaction) getEstimatedRangeSizeBytes(beginKey Key, endKey Key) Futur
}
}
// GetEstimatedRangeSizeBytes will get an estimate for the number of bytes
// GetEstimatedRangeSizeBytes returns an estimate for the number of bytes
// stored in the given range.
// Note: the estimated size is calculated based on the sampling done by FDB server. The sampling
// algorithm works roughly in this way: the larger the key-value pair is, the more likely it would
@ -348,8 +348,9 @@ func (t *transaction) getRangeSplitPoints(beginKey Key, endKey Key, chunkSize in
}
}
// GetRangeSplitPoints will return a list of keys that can divide the given range into
// chunks based on the chunk size provided.
// GetRangeSplitPoints returns a list of keys that can split the given range
// into (roughly) equally sized chunks based on chunkSize.
// Note: the returned split points contain the start key and end key of the given range.
func (t Transaction) GetRangeSplitPoints(r ExactRange, chunkSize int64) FutureKeyArray {
beginKey, endKey := r.FDBRangeKeys()
return t.getRangeSplitPoints(

6
bindings/java/src/main/com/apple/foundationdb/ReadTransaction.java
View File

@ -456,7 +456,8 @@ public interface ReadTransaction extends ReadTransactionContext {
CompletableFuture<Long> getEstimatedRangeSizeBytes(Range range);
/**
* Gets a list of keys that can split the given range into similar sized chunks based on chunkSize
* Gets a list of keys that can split the given range into (roughly) equally sized chunks based on <code>chunkSize</code>.
* Note: the returned split points contain the start key and end key of the given range.
*
* @param begin the beginning of the range (inclusive)
* @param end the end of the range (exclusive)
@ -466,7 +467,8 @@ public interface ReadTransaction extends ReadTransactionContext {
CompletableFuture<KeyArrayResult> getRangeSplitPoints(byte[] begin, byte[] end, long chunkSize);
/**
* Gets a list of keys that can split the given range into similar sized chunks based on chunkSize
* Gets a list of keys that can split the given range into (roughly) equally sized chunks based on <code>chunkSize</code>
* Note: the returned split points contain the start key and end key of the given range.
*
* @param range the range of the keys
*

12
documentation/sphinx/source/api-c.rst
View File

@ -301,6 +301,12 @@ See :ref:`developer-guide-programming-with-futures` for further (language-indepe
|future-get-return1| |future-get-return2|.
.. function:: fdb_error_t fdb_future_get_key_array( FDBFuture* f, FDBKey const** out_key_array, int* out_count)
Extracts an array of :type:`FDBKey` from an :type:`FDBFuture*` into a caller-provided variable of type ``FDBKey*``. The size of the array will also be extracted and passed back by a caller-provided variable of type ``int`` |future-warning|
|future-get-return1| |future-get-return2|.
.. function:: fdb_error_t fdb_future_get_key(FDBFuture* future, uint8_t const** out_key, int* out_key_length)
Extracts a key from an :type:`FDBFuture` into caller-provided variables of type ``uint8_t*`` (a pointer to the beginning of the key) and ``int`` (the length of the key). |future-warning|
@ -480,6 +486,12 @@ Applications must provide error handling and an appropriate retry loop around th
|future-return0| the estimated size of the key range given. |future-return1| call :func:`fdb_future_get_int64()` to extract the size, |future-return2|
.. function:: FDBFuture* fdb_transaction_get_range_split_points( FDBTransaction* tr, uint8_t const* begin_key_name, int begin_key_name_length, uint8_t const* end_key_name, int end_key_name_length, int64_t chunk_size)
Returns a list of keys that can split the given range into (roughly) equally sized chunks based on ``chunk_size``.
.. note:: The returned split points contain the start key and end key of the given range
|future-return0| the list of split points. |future-return1| call :func:`fdb_future_get_key_array()` to extract the array, |future-return2|
.. function:: FDBFuture* fdb_transaction_get_key(FDBTransaction* transaction, uint8_t const* key_name, int key_name_length, fdb_bool_t or_equal, int offset, fdb_bool_t snapshot)
Resolves a :ref:`key selector <key-selectors>` against the keys in the database snapshot represented by ``transaction``.

8
documentation/sphinx/source/api-python.rst
View File

@ -799,9 +799,15 @@ Transaction misc functions
.. method:: Transaction.get_estimated_range_size_bytes(begin_key, end_key)
Get the estimated byte size of the given key range. Returns a :class:`FutureInt64`.
Gets the estimated byte size of the given key range. Returns a :class:`FutureInt64`.
.. note:: The estimated size is calculated based on the sampling done by FDB server. The sampling algorithm works roughly in this way: the larger the key-value pair is, the more likely it would be sampled and the more accurate its sampled size would be. And due to that reason it is recommended to use this API to query against large ranges for accuracy considerations. For a rough reference, if the returned size is larger than 3MB, one can consider the size to be accurate.
.. method:: Transaction.get_range_split_points(self, begin_key, end_key, chunk_size)
Gets a list of keys that can split the given range into (roughly) equally sized chunks based on ``chunk_size``. Returns a :class:`FutureKeyArray`.
.. note:: The returned split points contain the start key and end key of the given range
.. _api-python-transaction-options:
Transaction misc functions

9
documentation/sphinx/source/api-ruby.rst
View File

@ -741,11 +741,16 @@ Most applications should use the read version that FoundationDB determines autom
Transaction misc functions
--------------------------
.. method:: Transaction.get_estimated_range_size_bytes(begin_key, end_key)
.. method:: Transaction.get_estimated_range_size_bytes(begin_key, end_key) -> Int64Future
Get the estimated byte size of the given key range. Returns a :class:`Int64Future`.
Gets the estimated byte size of the given key range. Returns a :class:`Int64Future`.
.. note:: The estimated size is calculated based on the sampling done by FDB server. The sampling algorithm works roughly in this way: the larger the key-value pair is, the more likely it would be sampled and the more accurate its sampled size would be. And due to that reason it is recommended to use this API to query against large ranges for accuracy considerations. For a rough reference, if the returned size is larger than 3MB, one can consider the size to be accurate.
.. method:: Transaction.get_range_split_points(begin_key, end_key, chunk_size) -> FutureKeyArray
Gets a list of keys that can split the given range into (roughly) equally sized chunks based on ``chunk_size``. Returns a :class:`FutureKeyArray`.
.. note:: The returned split points contain the start key and end key of the given range
.. method:: Transaction.get_approximate_size() -> Int64Future
|transaction-get-approximate-size-blurb|. Returns a :class:`Int64Future`.

13
documentation/sphinx/source/api-version-upgrade-guide.rst
View File

@ -9,6 +9,19 @@ This document provides an overview of changes that an application developer may
For more details about API versions, see :ref:`api-versions`.
.. _api-version-upgrade-guide-700:
API version 700
===============
General
-------
Python bindings
---------------
* The function ``get_estimated_range_size_bytes`` will now throw an error if the ``begin_key`` or ``end_key`` is ``None``.
.. _api-version-upgrade-guide-630:
API version 630

2
documentation/sphinx/source/release-notes/release-notes-700.rst
View File

@ -34,7 +34,7 @@ Status
Bindings
--------
* Python: The method ``get_estimated_range_size_bytes`` will now throw an error if the ``begin_key`` or ``end_key`` is ``None``. `(PR #3394) <https://github.com/apple/foundationdb/pull/3394>`_
* Python: The function ``get_estimated_range_size_bytes`` will now throw an error if the ``begin_key`` or ``end_key`` is ``None``. `(PR #3394) <https://github.com/apple/foundationdb/pull/3394>`_
Other Changes

51
fdbserver/workloads/IncrementalBackup.actor.cpp
View File

@ -61,36 +61,39 @@ struct IncrementalBackupWorkload : TestWorkload {
}
virtual Future<bool> check(Database const& cx) {
if (clientId || !waitForBackup) {
if (clientId) {
return true;
}
return _check(cx, this);
}
ACTOR static Future<bool> _check(Database cx, IncrementalBackupWorkload* self) {
state Reference<IBackupContainer> backupContainer;
state UID backupUID;
state Reference<ReadYourWritesTransaction> tr(new ReadYourWritesTransaction(cx));
state Version v = wait(tr->getReadVersion());
// Wait for backup container to be created and avoid race condition
TraceEvent("IBackupWaitContainer");
loop {
wait(success(self->backupAgent.waitBackup(cx, self->tag.toString(), false, &backupContainer, &backupUID)));
state bool e = wait(backupContainer->exists());
if (e) break;
wait(delay(5.0));
}
loop {
BackupDescription desc = wait(backupContainer->describeBackup(true));
TraceEvent("IBackupVersionGate")
.detail("MaxLogEndVersion", desc.maxLogEnd.present() ? desc.maxLogEnd.get() : invalidVersion)
.detail("ContiguousLogEndVersion",
desc.contiguousLogEnd.present() ? desc.contiguousLogEnd.get() : invalidVersion)
.detail("TargetVersion", v);
if (!desc.contiguousLogEnd.present()) continue;
if (desc.contiguousLogEnd.get() >= v) break;
// Avoid spamming requests with a delay
wait(delay(5.0));
if (self->waitForBackup) {
state Reference<IBackupContainer> backupContainer;
state UID backupUID;
state Reference<ReadYourWritesTransaction> tr(new ReadYourWritesTransaction(cx));
state Version v = wait(tr->getReadVersion());
// Wait for backup container to be created and avoid race condition
TraceEvent("IBackupWaitContainer");
loop {
wait(success(
self->backupAgent.waitBackup(cx, self->tag.toString(), false, &backupContainer, &backupUID)));
state bool e = wait(backupContainer->exists());
if (e) break;
wait(delay(5.0));
}
loop {
BackupDescription desc = wait(backupContainer->describeBackup(true));
TraceEvent("IBackupVersionGate")
.detail("MaxLogEndVersion", desc.maxLogEnd.present() ? desc.maxLogEnd.get() : invalidVersion)
.detail("ContiguousLogEndVersion",
desc.contiguousLogEnd.present() ? desc.contiguousLogEnd.get() : invalidVersion)
.detail("TargetVersion", v);
if (!desc.contiguousLogEnd.present()) continue;
if (desc.contiguousLogEnd.get() >= v) break;
// Avoid spamming requests with a delay
wait(delay(5.0));
}
}
if (self->stopBackup) {
wait(self->backupAgent.discontinueBackup(cx, self->tag));

2
tests/CMakeLists.txt
View File

@ -120,7 +120,7 @@ if(WITH_PYTHON)
add_fdb_test(TEST_FILES fast/CycleTest.toml)
add_fdb_test(TEST_FILES fast/FuzzApiCorrectness.toml)
add_fdb_test(TEST_FILES fast/FuzzApiCorrectnessClean.toml)
add_fdb_test(TEST_FILES fast/IncrementalBackup.toml)
add_fdb_test(TEST_FILES fast/IncrementalBackup.toml IGNORE)
add_fdb_test(TEST_FILES fast/IncrementTest.toml)
add_fdb_test(TEST_FILES fast/InventoryTestAlmostReadOnly.toml)
add_fdb_test(TEST_FILES fast/InventoryTestSomeWrites.toml)