From 3b9cb1a85a60c4384155a9b8385f78ff383d9c54 Mon Sep 17 00:00:00 2001 From: Chaoguang Lin Date: Mon, 9 Aug 2021 21:13:59 +0000 Subject: [PATCH] Re-enable exclude command ctest --- bindings/python/tests/fdbcli_tests.py | 28 ++++++++++++++++++--------- 1 file changed, 19 insertions(+), 9 deletions(-) diff --git a/bindings/python/tests/fdbcli_tests.py b/bindings/python/tests/fdbcli_tests.py index 14224a6ca8..17a94e16b1 100755 --- a/bindings/python/tests/fdbcli_tests.py +++ b/bindings/python/tests/fdbcli_tests.py @@ -332,9 +332,10 @@ def transaction(logger): output7 = run_fdbcli_command('get', 'key') assert output7 == "`key': not found" -def get_fdb_process_addresses(): +def get_fdb_process_addresses(logger): # get all processes' network addresses output = run_fdbcli_command('kill') + logger.debug(output) # except the first line, each line is one process addresses = output.split('\n')[1:] assert len(addresses) == process_number @@ -354,7 +355,7 @@ def coordinators(logger): assert coordinator_list[0]['address'] == coordinators # verify the cluster description assert get_value_from_status_json(True, 'cluster', 'connection_string').startswith('{}:'.format(cluster_description)) - addresses = get_fdb_process_addresses() + addresses = get_fdb_process_addresses(logger) # set all 5 processes as coordinators and update the cluster description new_cluster_description = 'a_simple_description' run_fdbcli_command('coordinators', *addresses, 'description={}'.format(new_cluster_description)) @@ -369,7 +370,7 @@ def coordinators(logger): @enable_logging() def exclude(logger): # get all processes' network addresses - addresses = get_fdb_process_addresses() + addresses = get_fdb_process_addresses(logger) logger.debug("Cluster processes: {}".format(' '.join(addresses))) # There should be no excluded process for now no_excluded_process_output = 'There are currently no servers or localities excluded from the database.' @@ -377,16 +378,28 @@ def exclude(logger): assert no_excluded_process_output in output1 # randomly pick one and exclude the process excluded_address = random.choice(addresses) + # If we see "not enough space" error, use FORCE option to proceed + # this should be a safe operation as we do not need any storage space for the test + force = False # sometimes we need to retry the exclude while True: logger.debug("Excluding process: {}".format(excluded_address)) - error_message = run_fdbcli_command_and_get_error('exclude', excluded_address) + if force: + error_message = run_fdbcli_command_and_get_error('exclude', 'FORCE', excluded_address) + else: + error_message = run_fdbcli_command_and_get_error('exclude', excluded_address) if error_message == 'WARNING: {} is a coordinator!'.format(excluded_address): # exclude coordinator will print the warning, verify the randomly selected process is the coordinator coordinator_list = get_value_from_status_json(True, 'client', 'coordinators', 'coordinators') assert len(coordinator_list) == 1 assert coordinator_list[0]['address'] == excluded_address break + elif 'ERROR: This exclude may cause the total free space in the cluster to drop below 10%.' in error_message: + # exclude the process may cause the free space not enough + # use FORCE option to ignore it and proceed + assert not force + force = True + logger.debug("Use FORCE option to exclude the process") elif not error_message: break else: @@ -450,10 +463,7 @@ if __name__ == '__main__': throttle() else: assert process_number > 1, "Process number should be positive" - # the kill command which used to list processes seems to not work as expected sometime - # which makes the test flaky. - # We need to figure out the reason and then re-enable these tests - #coordinators() - #exclude() + coordinators() + exclude()