apple/foundationdb
1
0
Fork 0
mirror of https://github.com/apple/foundationdb.git synced 2025-05-14 09:58:50 +08:00
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #6919 from sfc-gh-vgasiunas/vgasiunas-client-tmp-dir

Browse Source 
Store client library copies in a build-local temporary directory
This commit is contained in:
Vaidas Gasiunas 2022-04-22 15:46:57 +02:00 committed by GitHub
commit 44c5f87ac9
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
10 changed files with 47 additions and 8 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
bindings/c/CMakeLists.txt
View File

@ -252,6 +252,8 @@ endif()
${CMAKE_CURRENT_BINARY_DIR}/libfdb_c_external.so
--test-dir
${CMAKE_SOURCE_DIR}/bindings/c/test/apitester/tests
--tmp-dir
@TMP_DIR@
)
add_fdbclient_test(

1
bindings/c/test/apitester/TesterOptions.h
View File

@ -38,6 +38,7 @@ public:
std::string logGroup;
std::string externalClientLibrary;
std::string externalClientDir;
std::string tmpDir;
bool disableLocalClient = false;
std::string testFile;
std::string inputPipeName;

10
bindings/c/test/apitester/fdb_c_api_tester.cpp
View File

@ -46,6 +46,7 @@ enum TesterOptionId {
OPT_KNOB,
OPT_EXTERNAL_CLIENT_LIBRARY,
OPT_EXTERNAL_CLIENT_DIRECTORY,
OPT_TMP_DIR,
OPT_DISABLE_LOCAL_CLIENT,
OPT_TEST_FILE,
OPT_INPUT_PIPE,
@ -67,6 +68,7 @@ CSimpleOpt::SOption TesterOptionDefs[] = //
{ OPT_KNOB, "--knob-", SO_REQ_SEP },
{ OPT_EXTERNAL_CLIENT_LIBRARY, "--external-client-library", SO_REQ_SEP },
{ OPT_EXTERNAL_CLIENT_DIRECTORY, "--external-client-dir", SO_REQ_SEP },
{ OPT_TMP_DIR, "--tmp-dir", SO_REQ_SEP },
{ OPT_DISABLE_LOCAL_CLIENT, "--disable-local-client", SO_NONE },
{ OPT_TEST_FILE, "-f", SO_REQ_SEP },
{ OPT_TEST_FILE, "--test-file", SO_REQ_SEP },
@ -100,6 +102,8 @@ void printProgramUsage(const char* execName) {
" Path to the external client library.\n"
" --external-client-dir DIR\n"
" Directory containing external client libraries.\n"
" --tmp-dir DIR\n"
" Directory for temporary files of the client.\n"
" --disable-local-client DIR\n"
" Disable the local client, i.e. use only external client libraries.\n"
" --input-pipe NAME\n"
@ -184,6 +188,9 @@ bool processArg(TesterOptions& options, const CSimpleOpt& args) {
case OPT_EXTERNAL_CLIENT_DIRECTORY:
options.externalClientDir = args.OptionArg();
break;
case OPT_TMP_DIR:
options.tmpDir = args.OptionArg();
break;
case OPT_DISABLE_LOCAL_CLIENT:
options.disableLocalClient = true;
break;
@ -243,6 +250,9 @@ void fdb_check(fdb_error_t e) {
}
void applyNetworkOptions(TesterOptions& options) {
if (!options.tmpDir.empty()) {
fdb_check(FdbApi::setOption(FDBNetworkOption::FDB_NET_OPTION_CLIENT_TMP_DIR, options.tmpDir));
}
if (!options.externalClientLibrary.empty()) {
fdb_check(FdbApi::setOption(FDBNetworkOption::FDB_NET_OPTION_DISABLE_LOCAL_CLIENT));
fdb_check(

17
bindings/c/test/apitester/run_c_api_tests.py
View File

@ -49,13 +49,16 @@ def initialize_logger_level(logging_level):
def run_tester(args, test_file):
cmd = [args.tester_binary, "--cluster-file",
args.cluster_file, "--test-file", test_file]
cmd = [args.tester_binary,
"--cluster-file", args.cluster_file,
"--test-file", test_file]
if args.external_client_library is not None:
cmd += ["--external-client-library", args.external_client_library]
if args.tmp_dir is not None:
cmd += ["--tmp-dir", args.tmp_dir]
if args.blob_granule_local_file_path is not None:
cmd += ["--blob-granule-local-file-path", args.blob_granule_local_file_path]
cmd += ["--blob-granule-local-file-path",
args.blob_granule_local_file_path]
get_logger().info('\nRunning tester \'%s\'...' % ' '.join(cmd))
proc = Popen(cmd, stdout=sys.stdout, stderr=sys.stderr)
@ -82,9 +85,11 @@ def run_tester(args, test_file):
get_logger().info('')
return ret_code
def run_tests(args):
num_failed = 0
test_files = [f for f in os.listdir(args.test_dir) if os.path.isfile(os.path.join(args.test_dir, f)) and f.endswith(".toml")]
test_files = [f for f in os.listdir(args.test_dir) if os.path.isfile(
os.path.join(args.test_dir, f)) and f.endswith(".toml")]
for test_file in test_files:
get_logger().info('=========================================================')
@ -112,6 +117,8 @@ def parse_args(argv):
help='The timeout in seconds for running each individual test. (default 300)')
parser.add_argument('--logging-level', type=str, default='INFO',
choices=['ERROR', 'WARNING', 'INFO', 'DEBUG'], help='Specifies the level of detail in the tester output (default=\'INFO\').')
parser.add_argument('--tmp-dir', type=str, default=None,
help='The directory for storing temporary files (default: None)')
parser.add_argument('--blob-granule-local-file-path', type=str, default=None,
help='Enable blob granule tests if set, value is path to local blob granule files')

7
bindings/go/src/fdb/generated.go
View File

@ -285,6 +285,13 @@ func (o NetworkOptions) SetDistributedClientTracer(param string) error {
return o.setOpt(90, []byte(param))
}
// Sets the directory for storing temporary files created by FDB client, such as temporary copies of client libraries. Defaults to /tmp
//
// Parameter: Client directory for temporary files.
func (o NetworkOptions) SetClientTmpDir(param string) error {
return o.setOpt(90, []byte(param))
}
// Set the size of the client location cache. Raising this value can boost performance in very large databases where clients access data in a near-random pattern. Defaults to 100000.
//
// Parameter: Max location cache entries

11
fdbclient/MultiVersionTransaction.actor.cpp
View File

@ -1998,8 +1998,9 @@ std::vector<std::pair<std::string, bool>> MultiVersionApi::copyExternalLibraryPe
for (int ii = 0; ii < threadCount; ++ii) {
std::string filename = basename(path);
char tempName[PATH_MAX + 12];
sprintf(tempName, "/tmp/%s-XXXXXX", filename.c_str());
constexpr int MAX_TMP_NAME_LENGTH = PATH_MAX + 12;
char tempName[MAX_TMP_NAME_LENGTH];
snprintf(tempName, MAX_TMP_NAME_LENGTH, "%s/%s-XXXXXX", tmpDir.c_str(), filename.c_str());
int tempFd = mkstemp(tempName);
int fd;
@ -2145,6 +2146,9 @@ void MultiVersionApi::setNetworkOptionInternal(FDBNetworkOptions::Option option,
// multiple client threads are not supported on windows.
threadCount = extractIntOption(value, 1, 1);
#endif
} else if (option == FDBNetworkOptions::CLIENT_TMP_DIR) {
validateOption(value, true, false, false);
tmpDir = abspath(value.get().toString());
} else {
forwardOption = true;
}
@ -2520,7 +2524,8 @@ void MultiVersionApi::loadEnvironmentVariableNetworkOptions() {
MultiVersionApi::MultiVersionApi()
: callbackOnMainThread(true), localClientDisabled(false), networkStartSetup(false), networkSetup(false),
bypassMultiClientApi(false), externalClient(false), apiVersion(0), threadCount(0), envOptionsLoaded(false) {}
bypassMultiClientApi(false), externalClient(false), apiVersion(0), threadCount(0), tmpDir("/tmp"),
envOptionsLoaded(false) {}
MultiVersionApi* MultiVersionApi::api = new MultiVersionApi();

1
fdbclient/MultiVersionTransaction.h
View File

@ -898,6 +898,7 @@ private:
int nextThread = 0;
int threadCount;
std::string tmpDir;
Mutex lock;
std::vector<std::pair<FDBNetworkOptions::Option, Optional<Standalone<StringRef>>>> options;

3
fdbclient/vexillographer/fdb.options
View File

@ -134,6 +134,9 @@ description is not currently required but encouraged.
<Option name="distributed_client_tracer" code="90"
paramType="String" paramDescription="Distributed tracer type. Choose from none, log_file, or network_lossy"
description="Set a tracer to run on the client. Should be set to the same value as the tracer set on the server." />
<Option name="client_tmp_dir" code="90"
paramType="String" paramDescription="Client directory for temporary files. "
description="Sets the directory for storing temporary files created by FDB client, such as temporary copies of client libraries. Defaults to /tmp" />
<Option name="supported_client_versions" code="1000"
paramType="String" paramDescription="[release version],[source version],[protocol version];..."
description="This option is set automatically to communicate the list of supported clients to the active client."

2
tests/TestRunner/tmp_cluster.py
View File

@ -111,6 +111,8 @@ if __name__ == "__main__":
cmd_args.append(str(cluster.log))
elif cmd == "@ETC_DIR@":
cmd_args.append(str(cluster.etc))
elif cmd == "@TMP_DIR@":
cmd_args.append(str(cluster.tmp_dir))
elif cmd.startswith("@DATA_DIR@"):
cmd_args.append(str(cluster.data) + cmd[len("@DATA_DIR@"):])
else:

1
tests/TestRunner/upgrade_test.py
View File

@ -246,6 +246,7 @@ class UpgradeTest:
'--api-version', str(self.api_version),
'--log',
'--log-dir', self.log,
'--tmp-dir', self.tmp_dir,
'--transaction-retry-limit', str(TRANSACTION_RETRY_LIMIT)]
if (RUN_WITH_GDB):
cmd_args = ['gdb', '-ex', 'run', '--args'] + cmd_args