apple/foundationdb
1
0
Fork 0
mirror of https://github.com/apple/foundationdb.git synced 2025-05-14 18:02:31 +08:00
Code Issues Packages Projects Releases Wiki Activity

Re-enable exclude command ctest

Browse Source
This commit is contained in:
Chaoguang Lin 2021-08-09 21:13:59 +00:00
parent 857d5704da
commit 3b9cb1a85a
1 changed files with 19 additions and 9 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

26
bindings/python/tests/fdbcli_tests.py
View File

@ -332,9 +332,10 @@ def transaction(logger):
output7 = run_fdbcli_command('get', 'key') output7 = run_fdbcli_command('get', 'key')
assert output7 == "`key': not found" assert output7 == "`key': not found"
def get_fdb_process_addresses(): def get_fdb_process_addresses(logger):
# get all processes' network addresses # get all processes' network addresses
output = run_fdbcli_command('kill') output = run_fdbcli_command('kill')
logger.debug(output)
# except the first line, each line is one process # except the first line, each line is one process
addresses = output.split('\n')[1:] addresses = output.split('\n')[1:]
assert len(addresses) == process_number assert len(addresses) == process_number
@ -354,7 +355,7 @@ def coordinators(logger):
assert coordinator_list[0]['address'] == coordinators assert coordinator_list[0]['address'] == coordinators
# verify the cluster description # verify the cluster description
assert get_value_from_status_json(True, 'cluster', 'connection_string').startswith('{}:'.format(cluster_description)) assert get_value_from_status_json(True, 'cluster', 'connection_string').startswith('{}:'.format(cluster_description))
addresses = get_fdb_process_addresses() addresses = get_fdb_process_addresses(logger)
# set all 5 processes as coordinators and update the cluster description # set all 5 processes as coordinators and update the cluster description
new_cluster_description = 'a_simple_description' new_cluster_description = 'a_simple_description'
run_fdbcli_command('coordinators', *addresses, 'description={}'.format(new_cluster_description)) run_fdbcli_command('coordinators', *addresses, 'description={}'.format(new_cluster_description))
@ -369,7 +370,7 @@ def coordinators(logger):
@enable_logging() @enable_logging()
def exclude(logger): def exclude(logger):
# get all processes' network addresses # get all processes' network addresses
addresses = get_fdb_process_addresses() addresses = get_fdb_process_addresses(logger)
logger.debug("Cluster processes: {}".format(' '.join(addresses))) logger.debug("Cluster processes: {}".format(' '.join(addresses)))
# There should be no excluded process for now # There should be no excluded process for now
no_excluded_process_output = 'There are currently no servers or localities excluded from the database.' no_excluded_process_output = 'There are currently no servers or localities excluded from the database.'
@ -377,9 +378,15 @@ def exclude(logger):
assert no_excluded_process_output in output1 assert no_excluded_process_output in output1
# randomly pick one and exclude the process # randomly pick one and exclude the process
excluded_address = random.choice(addresses) excluded_address = random.choice(addresses)
# If we see "not enough space" error, use FORCE option to proceed
# this should be a safe operation as we do not need any storage space for the test
force = False
# sometimes we need to retry the exclude # sometimes we need to retry the exclude
while True: while True:
logger.debug("Excluding process: {}".format(excluded_address)) logger.debug("Excluding process: {}".format(excluded_address))
if force:
error_message = run_fdbcli_command_and_get_error('exclude', 'FORCE', excluded_address)
else:
error_message = run_fdbcli_command_and_get_error('exclude', excluded_address) error_message = run_fdbcli_command_and_get_error('exclude', excluded_address)
if error_message == 'WARNING: {} is a coordinator!'.format(excluded_address): if error_message == 'WARNING: {} is a coordinator!'.format(excluded_address):
# exclude coordinator will print the warning, verify the randomly selected process is the coordinator # exclude coordinator will print the warning, verify the randomly selected process is the coordinator
@ -387,6 +394,12 @@ def exclude(logger):
assert len(coordinator_list) == 1 assert len(coordinator_list) == 1
assert coordinator_list[0]['address'] == excluded_address assert coordinator_list[0]['address'] == excluded_address
break break
elif 'ERROR: This exclude may cause the total free space in the cluster to drop below 10%.' in error_message:
# exclude the process may cause the free space not enough
# use FORCE option to ignore it and proceed
assert not force
force = True
logger.debug("Use FORCE option to exclude the process")
elif not error_message: elif not error_message:
break break
else: else:
@ -450,10 +463,7 @@ if __name__ == '__main__':
throttle() throttle()
else: else:
assert process_number > 1, "Process number should be positive" assert process_number > 1, "Process number should be positive"
# the kill command which used to list processes seems to not work as expected sometime coordinators()
# which makes the test flaky. exclude()
# We need to figure out the reason and then re-enable these tests
#coordinators()
#exclude()