apple/foundationdb
1
0
Fork 0
mirror of https://github.com/apple/foundationdb.git synced 2025-05-13 17:39:31 +08:00
Code Issues Packages Projects Releases Wiki Activity

Add tenant support to Python

Browse Source
This commit is contained in:
A.J. Beamon 2022-02-19 15:40:32 -08:00
parent ca653c77ee
commit ce03f5783d
4 changed files with 133 additions and 66 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
bindings/bindingtester/known_testers.py
View File

@ -58,8 +58,8 @@ _java_cmd = 'java -ea -cp %s:%s com.apple.foundationdb.test.' % (
# We could set min_api_version lower on some of these if the testers were updated to support them
testers = {
'python': Tester('python', 'python ' + _absolute_path('python/tests/tester.py'), 2040, 23, MAX_API_VERSION, types=ALL_TYPES),
'python3': Tester('python3', 'python3 ' + _absolute_path('python/tests/tester.py'), 2040, 23, MAX_API_VERSION, types=ALL_TYPES),
'python': Tester('python', 'python ' + _absolute_path('python/tests/tester.py'), 2040, 23, MAX_API_VERSION, types=ALL_TYPES, tenants_enabled=True),
'python3': Tester('python3', 'python3 ' + _absolute_path('python/tests/tester.py'), 2040, 23, MAX_API_VERSION, types=ALL_TYPES, tenants_enabled=True),
'ruby': Tester('ruby', _absolute_path('ruby/tests/tester.rb'), 2040, 23, MAX_API_VERSION),
'java': Tester('java', _java_cmd + 'StackTester', 2040, 510, MAX_API_VERSION, types=ALL_TYPES),
'java_async': Tester('java', _java_cmd + 'AsyncStackTester', 2040, 510, MAX_API_VERSION, types=ALL_TYPES),

1
bindings/python/fdb/__init__.py
View File

@ -88,6 +88,7 @@ def api_version(ver):
'predicates',
'Future',
'Database',
'Tenant',
'Transaction',
'KeyValue',
'KeySelector',

170
bindings/python/fdb/impl.py
View File

@ -198,9 +198,10 @@ def transactional(*tr_args, **tr_kwargs):
one of two actions, depending on the type of the parameter passed
to the function at call time.
If given a Database, a Transaction will be created and passed into
the wrapped code in place of the Database. After the function is
complete, the newly created transaction will be committed.
If given a Database or Tenant, a Transaction will be created and
passed into the wrapped code in place of the Database or Tenant.
After the function is complete, the newly created transaction
will be committed.
It is important to note that the wrapped method may be called
multiple times in the event of a commit failure, until the commit
@ -943,128 +944,114 @@ class FormerFuture(_FDBBase):
except:
pass
class Database(_FDBBase):
def __init__(self, dpointer):
self.dpointer = dpointer
self.options = _DatabaseOptions(self)
def __del__(self):
# print('Destroying database 0x%x' % self.dpointer)
self.capi.fdb_database_destroy(self.dpointer)
class _TransactionCreator(_FDBBase):
def get(self, key):
return Database.__database_getitem(self, key)
return _TransactionCreator.__creator_getitem(self, key)
def __getitem__(self, key):
if isinstance(key, slice):
return self.get_range(key.start, key.stop, reverse=(key.step == -1))
return Database.__database_getitem(self, key)
return _TransactionCreator.__creator_getitem(self, key)
def get_key(self, key_selector):
return Database.__database_get_key(self, key_selector)
return _TransactionCreator.__creator_get_key(self, key_selector)
def get_range(self, begin, end, limit=0, reverse=False, streaming_mode=StreamingMode.want_all):
return Database.__database_get_range(self, begin, end, limit, reverse, streaming_mode)
return _TransactionCreator.__creator_get_range(self, begin, end, limit, reverse, streaming_mode)
def get_range_startswith(self, prefix, *args, **kwargs):
return Database.__database_get_range_startswith(self, prefix, *args, **kwargs)
return _TransactionCreator.__creator_get_range_startswith(self, prefix, *args, **kwargs)
def set(self, key, value):
Database.__database_setitem(self, key, value)
_TransactionCreator.__creator_setitem(self, key, value)
def __setitem__(self, key, value):
Database.__database_setitem(self, key, value)
_TransactionCreator.__creator_setitem(self, key, value)
def clear(self, key):
Database.__database_delitem(self, key)
_TransactionCreator.__creator_delitem(self, key)
def clear_range(self, begin, end):
Database.__database_delitem(self, slice(begin, end))
_TransactionCreator.__creator_delitem(self, slice(begin, end))
def __delitem__(self, key_or_slice):
Database.__database_delitem(self, key_or_slice)
_TransactionCreator.__creator_delitem(self, key_or_slice)
def clear_range_startswith(self, prefix):
Database.__database_clear_range_startswith(self, prefix)
_TransactionCreator.__creator_clear_range_startswith(self, prefix)
def get_and_watch(self, key):
return Database.__database_get_and_watch(self, key)
return _TransactionCreator.__creator_get_and_watch(self, key)
def set_and_watch(self, key, value):
return Database.__database_set_and_watch(self, key, value)
return _TransactionCreator.__creator_set_and_watch(self, key, value)
def clear_and_watch(self, key):
return Database.__database_clear_and_watch(self, key)
return _TransactionCreator.__creator_clear_and_watch(self, key)
def create_transaction(self):
pointer = ctypes.c_void_p()
self.capi.fdb_database_create_transaction(self.dpointer, ctypes.byref(pointer))
return Transaction(pointer.value, self)
def _set_option(self, option, param, length):
self.capi.fdb_database_set_option(self.dpointer, option, param, length)
pass
def _atomic_operation(self, opcode, key, param):
Database.__database_atomic_operation(self, opcode, key, param)
_TransactionCreator.__creator_atomic_operation(self, opcode, key, param)
#### Transaction implementations ####
@staticmethod
@transactional
def __database_getitem(tr, key):
def __creator_getitem(tr, key):
return tr[key].value
@staticmethod
@transactional
def __database_get_key(tr, key_selector):
def __creator_get_key(tr, key_selector):
return tr.get_key(key_selector).value
@staticmethod
@transactional
def __database_get_range(tr, begin, end, limit, reverse, streaming_mode):
def __creator_get_range(tr, begin, end, limit, reverse, streaming_mode):
return tr.get_range(begin, end, limit, reverse, streaming_mode).to_list()
@staticmethod
@transactional
def __database_get_range_startswith(tr, prefix, *args, **kwargs):
def __creator_get_range_startswith(tr, prefix, *args, **kwargs):
return tr.get_range_startswith(prefix, *args, **kwargs).to_list()
@staticmethod
@transactional
def __database_setitem(tr, key, value):
def __creator_setitem(tr, key, value):
tr[key] = value
@staticmethod
@transactional
def __database_clear_range_startswith(tr, prefix):
def __creator_clear_range_startswith(tr, prefix):
tr.clear_range_startswith(prefix)
@staticmethod
@transactional
def __database_get_and_watch(tr, key):
def __creator_get_and_watch(tr, key):
v = tr.get(key)
return v, tr.watch(key)
@staticmethod
@transactional
def __database_set_and_watch(tr, key, value):
def __creator_set_and_watch(tr, key, value):
tr.set(key, value)
return tr.watch(key)
@staticmethod
@transactional
def __database_clear_and_watch(tr, key):
def __creator_clear_and_watch(tr, key):
del tr[key]
return tr.watch(key)
@staticmethod
@transactional
def __database_delitem(tr, key_or_slice):
def __creator_delitem(tr, key_or_slice):
del tr[key_or_slice]
@staticmethod
@transactional
def __database_atomic_operation(tr, opcode, key, param):
def __creator_atomic_operation(tr, opcode, key, param):
tr._atomic_operation(opcode, key, param)
# Asynchronous transactions
@ -1074,11 +1061,11 @@ class Database(_FDBBase):
From = asyncio.From
coroutine = asyncio.coroutine
class Database:
class TransactionCreator:
@staticmethod
@transactional
@coroutine
def __database_getitem(tr, key):
def __creator_getitem(tr, key):
# raise Return(( yield From( tr[key] ) ))
raise Return(tr[key])
yield None
@ -1086,26 +1073,26 @@ class Database(_FDBBase):
@staticmethod
@transactional
@coroutine
def __database_get_key(tr, key_selector):
def __creator_get_key(tr, key_selector):
raise Return(tr.get_key(key_selector))
yield None
@staticmethod
@transactional
@coroutine
def __database_get_range(tr, begin, end, limit, reverse, streaming_mode):
def __creator_get_range(tr, begin, end, limit, reverse, streaming_mode):
raise Return((yield From(tr.get_range(begin, end, limit, reverse, streaming_mode).to_list())))
@staticmethod
@transactional
@coroutine
def __database_get_range_startswith(tr, prefix, *args, **kwargs):
def __creator_get_range_startswith(tr, prefix, *args, **kwargs):
raise Return((yield From(tr.get_range_startswith(prefix, *args, **kwargs).to_list())))
@staticmethod
@transactional
@coroutine
def __database_setitem(tr, key, value):
def __creator_setitem(tr, key, value):
tr[key] = value
raise Return()
yield None
@ -1113,7 +1100,7 @@ class Database(_FDBBase):
@staticmethod
@transactional
@coroutine
def __database_clear_range_startswith(tr, prefix):
def __creator_clear_range_startswith(tr, prefix):
tr.clear_range_startswith(prefix)
raise Return()
yield None
@ -1121,7 +1108,7 @@ class Database(_FDBBase):
@staticmethod
@transactional
@coroutine
def __database_get_and_watch(tr, key):
def __creator_get_and_watch(tr, key):
v = tr.get(key)
raise Return(v, tr.watch(key))
yield None
@ -1129,7 +1116,7 @@ class Database(_FDBBase):
@staticmethod
@transactional
@coroutine
def __database_set_and_watch(tr, key, value):
def __creator_set_and_watch(tr, key, value):
tr.set(key, value)
raise Return(tr.watch(key))
yield None
@ -1137,7 +1124,7 @@ class Database(_FDBBase):
@staticmethod
@transactional
@coroutine
def __database_clear_and_watch(tr, key):
def __creator_clear_and_watch(tr, key):
del tr[key]
raise Return(tr.watch(key))
yield None
@ -1145,7 +1132,7 @@ class Database(_FDBBase):
@staticmethod
@transactional
@coroutine
def __database_delitem(tr, key_or_slice):
def __creator_delitem(tr, key_or_slice):
del tr[key_or_slice]
raise Return()
yield None
@ -1153,11 +1140,55 @@ class Database(_FDBBase):
@staticmethod
@transactional
@coroutine
def __database_atomic_operation(tr, opcode, key, param):
def __creator_atomic_operation(tr, opcode, key, param):
tr._atomic_operation(opcode, key, param)
raise Return()
yield None
return Database
return TransactionCreator
class Database(_TransactionCreator):
def __init__(self, dpointer):
self.dpointer = dpointer
self.options = _DatabaseOptions(self)
def __del__(self):
# print('Destroying database 0x%x' % self.dpointer)
self.capi.fdb_database_destroy(self.dpointer)
def _set_option(self, option, param, length):
self.capi.fdb_database_set_option(self.dpointer, option, param, length)
def open_tenant(self, name):
if not isinstance(name, bytes):
raise TypeError('Tenant name must be of type ' + bytes.__name__)
pointer = ctypes.c_void_p()
self.capi.fdb_database_open_tenant(self.dpointer, name, len(name), ctypes.byref(pointer))
return Tenant(pointer.value)
def create_transaction(self):
pointer = ctypes.c_void_p()
self.capi.fdb_database_create_transaction(self.dpointer, ctypes.byref(pointer))
return Transaction(pointer.value, self)
def allocate_tenant(self, name):
return FutureVoid(self.capi.fdb_database_allocate_tenant(self.dpointer, name, len(name)))
def delete_tenant(self, name):
return FutureVoid(self.capi.fdb_database_remove_tenant(self.dpointer, name, len(name)))
class Tenant(_TransactionCreator):
def __init__(self, tpointer):
self.tpointer = tpointer
def __del__(self):
self.capi.fdb_tenant_destroy(self.tpointer)
def create_transaction(self):
pointer = ctypes.c_void_p()
self.capi.fdb_tenant_create_transaction(self.tpointer, ctypes.byref(pointer))
return Transaction(pointer.value, self)
fill_operations()
@ -1458,6 +1489,10 @@ def init_c_api():
_capi.fdb_database_destroy.argtypes = [ctypes.c_void_p]
_capi.fdb_database_destroy.restype = None
_capi.fdb_database_open_tenant.argtypes = [ctypes.c_void_p, ctypes.c_void_p, ctypes.c_int, ctypes.POINTER(ctypes.c_void_p)]
_capi.fdb_database_open_tenant.restype = ctypes.c_int
_capi.fdb_database_open_tenant.errcheck = check_error_code
_capi.fdb_database_create_transaction.argtypes = [ctypes.c_void_p, ctypes.POINTER(ctypes.c_void_p)]
_capi.fdb_database_create_transaction.restype = ctypes.c_int
_capi.fdb_database_create_transaction.errcheck = check_error_code
@ -1466,6 +1501,19 @@ def init_c_api():
_capi.fdb_database_set_option.restype = ctypes.c_int
_capi.fdb_database_set_option.errcheck = check_error_code
_capi.fdb_database_allocate_tenant.argtypes = [ctypes.c_void_p, ctypes.c_void_p, ctypes.c_int]
_capi.fdb_database_allocate_tenant.restype = ctypes.c_void_p
_capi.fdb_database_remove_tenant.argtypes = [ctypes.c_void_p, ctypes.c_void_p, ctypes.c_int]
_capi.fdb_database_remove_tenant.restype = ctypes.c_void_p
_capi.fdb_tenant_destroy.argtypes = [ctypes.c_void_p]
_capi.fdb_tenant_destroy.restype = None
_capi.fdb_tenant_create_transaction.argtypes = [ctypes.c_void_p, ctypes.POINTER(ctypes.c_void_p)]
_capi.fdb_tenant_create_transaction.restype = ctypes.c_int
_capi.fdb_tenant_create_transaction.errcheck = check_error_code
_capi.fdb_transaction_destroy.argtypes = [ctypes.c_void_p]
_capi.fdb_transaction_destroy.restype = None
@ -1686,10 +1734,10 @@ def init(event_model=None):
raise asyncio.Return(self)
return it()
FDBRange.iterate = iterate
AT = Database.declare_asynchronous_transactions()
AT = _TransactionCreator.declare_asynchronous_transactions()
for name in dir(AT):
if name.startswith("_Database__database_"):
setattr(Database, name, getattr(AT, name))
if name.startswith("__TransactionCreator__creator_"):
setattr(_TransactionCreator, name, getattr(AT, name))
def to_list(self):
if self._mode == StreamingMode.iterator:

24
bindings/python/tests/tester.py
View File

@ -112,12 +112,13 @@ class Stack:
class Instruction:
def __init__(self, tr, stack, op, index, isDatabase=False, isSnapshot=False):
def __init__(self, tr, stack, op, index, isDatabase=False, isTenant=False, isSnapshot=False):
self.tr = tr
self.stack = stack
self.op = op
self.index = index
self.isDatabase = isDatabase
self.isTenant = isTenant
self.isSnapshot = isSnapshot
def pop(self, count=None, with_idx=False):
@ -277,6 +278,7 @@ class Tester:
def __init__(self, db, prefix):
self.db = db
self.tenant = None
self.instructions = self.db[fdb.tuple.range((prefix,))]
@ -317,7 +319,8 @@ class Tester:
def new_transaction(self):
with Tester.tr_map_lock:
Tester.tr_map[self.tr_name] = self.db.create_transaction()
tr_source = self.tenant if self.tenant is not None else self.db
Tester.tr_map[self.tr_name] = tr_source.create_transaction()
def switch_transaction(self, name):
self.tr_name = name
@ -335,18 +338,22 @@ class Tester:
# print("%d. Instruction is %s" % (idx, op))
isDatabase = op.endswith(six.u('_DATABASE'))
isTenant = op.endswith(six.u('_TENANT'))
isSnapshot = op.endswith(six.u('_SNAPSHOT'))
if isDatabase:
op = op[:-9]
obj = self.db
elif isTenant:
op = op[:-7]
obj = self.tenant if self.tenant else self.db
elif isSnapshot:
op = op[:-9]
obj = self.current_transaction().snapshot
else:
obj = self.current_transaction()
inst = Instruction(obj, self.stack, op, idx, isDatabase, isSnapshot)
inst = Instruction(obj, self.stack, op, idx, isDatabase, isTenant, isSnapshot)
try:
if inst.op == six.u("PUSH"):
@ -583,6 +590,17 @@ class Tester:
prefix = inst.pop()
Tester.wait_empty(self.db, prefix)
inst.push(b"WAITED_FOR_EMPTY")
elif inst.op == six.u("TENANT_CREATE"):
name = inst.pop()
inst.push(self.db.allocate_tenant(name))
elif inst.op == six.u("TENANT_DELETE"):
name = inst.pop()
inst.push(self.db.delete_tenant(name))
elif inst.op == six.u("TENANT_SET_ACTIVE"):
name = inst.pop()
self.tenant = self.db.open_tenant(name)
elif inst.op == six.u("TENANT_CLEAR_ACTIVE"):
self.tenant = None
elif inst.op == six.u("UNIT_TESTS"):
try:
test_db_options(db)