apple/foundationdb
1
0
Fork 0
mirror of https://github.com/apple/foundationdb.git synced 2025-05-28 10:52:03 +08:00
Code Issues Packages Projects Releases Wiki Activity

Generate a shim library for the FDB C API ()

Browse Source 
* Adding sources of the Implib.so project

* Run C unit tests and API tests with the shim library

* Reuse compilation of C test binaries with and without shim library

* Resolve client library path from an environment variable

* Refactoring: Reusable module for downloading FDB binaries

* Testing client shim library with current version and last release version

* Tests for specifying client library over an environment variable

* Enable C shim library tests on ARM

* Restore the original path for including fdb_api.hpp
This commit is contained in:
Vaidas Gasiunas 2022-07-08 16:28:35 +02:00 committed by GitHub
parent 24d7e0d28b
commit 1e8feb9cb8
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
20 changed files with 1901 additions and 234 deletions

72
bindings/c/CMakeLists.txt

@ -141,6 +141,9 @@ if(NOT WIN32)
test/apitester/TesterWorkload.h
)
add_library(fdb_c_unit_tests_impl OBJECT ${UNIT_TEST_SRCS})
add_library(fdb_c_api_tester_impl OBJECT ${API_TESTER_SRCS})
if(OPEN_FOR_IDE)
add_library(fdb_c_performance_test OBJECT test/performance_test.c test/test.h)
add_library(fdb_c_ryw_benchmark OBJECT test/ryw_benchmark.c test/test.h)
@ -148,11 +151,9 @@ if(NOT WIN32)
add_library(fdb_c_client_memory_test OBJECT test/client_memory_test.cpp test/unit/fdb_api.cpp test/unit/fdb_api.hpp)
add_library(mako OBJECT ${MAKO_SRCS})
add_library(fdb_c_setup_tests OBJECT test/unit/setup_tests.cpp)
add_library(fdb_c_unit_tests OBJECT ${UNIT_TEST_SRCS})
add_library(fdb_c_unit_tests_version_510 OBJECT ${UNIT_TEST_VERSION_510_SRCS})
add_library(trace_partial_file_suffix_test OBJECT ${TRACE_PARTIAL_FILE_SUFFIX_TEST_SRCS})
add_library(disconnected_timeout_unit_tests OBJECT ${DISCONNECTED_TIMEOUT_UNIT_TEST_SRCS})
add_library(fdb_c_api_tester OBJECT ${API_TESTER_SRCS})
else()
add_executable(fdb_c_performance_test test/performance_test.c test/test.h)
add_executable(fdb_c_ryw_benchmark test/ryw_benchmark.c test/test.h)
@ -160,43 +161,46 @@ if(NOT WIN32)
add_executable(fdb_c_client_memory_test test/client_memory_test.cpp test/unit/fdb_api.cpp test/unit/fdb_api.hpp)
add_executable(mako ${MAKO_SRCS})
add_executable(fdb_c_setup_tests test/unit/setup_tests.cpp)
add_executable(fdb_c_unit_tests ${UNIT_TEST_SRCS})
add_executable(fdb_c_unit_tests)
target_link_libraries(fdb_c_unit_tests PRIVATE fdb_c fdb_c_unit_tests_impl)
add_executable(fdb_c_unit_tests_version_510 ${UNIT_TEST_VERSION_510_SRCS})
add_executable(trace_partial_file_suffix_test ${TRACE_PARTIAL_FILE_SUFFIX_TEST_SRCS})
add_executable(disconnected_timeout_unit_tests ${DISCONNECTED_TIMEOUT_UNIT_TEST_SRCS})
add_executable(fdb_c_api_tester ${API_TESTER_SRCS})
add_executable(fdb_c_api_tester)
target_link_libraries(fdb_c_api_tester PRIVATE fdb_c fdb_c_api_tester_impl)
strip_debug_symbols(fdb_c_performance_test)
strip_debug_symbols(fdb_c_ryw_benchmark)
strip_debug_symbols(fdb_c_txn_size_test)
strip_debug_symbols(fdb_c_client_memory_test)
endif()
target_link_libraries(fdb_c_performance_test PRIVATE fdb_c Threads::Threads)
target_link_libraries(fdb_c_ryw_benchmark PRIVATE fdb_c Threads::Threads)
target_link_libraries(fdb_c_txn_size_test PRIVATE fdb_c Threads::Threads)
target_link_libraries(fdb_c_client_memory_test PRIVATE fdb_c Threads::Threads)
target_include_directories(fdb_c_api_tester_impl PUBLIC ${CMAKE_CURRENT_SOURCE_DIR} ${CMAKE_CURRENT_BINARY_DIR}/foundationdb/ ${CMAKE_SOURCE_DIR}/flow/include ${CMAKE_BINARY_DIR}/flow/include)
if(USE_SANITIZER)
target_link_libraries(fdb_c_api_tester_impl PRIVATE fdb_cpp toml11_target Threads::Threads fmt::fmt boost_asan)
else()
target_link_libraries(fdb_c_api_tester_impl PRIVATE fdb_cpp toml11_target Threads::Threads fmt::fmt boost_target)
endif()
target_link_libraries(fdb_c_api_tester_impl PRIVATE SimpleOpt)
add_dependencies(fdb_c_setup_tests doctest)
add_dependencies(fdb_c_unit_tests doctest)
add_dependencies(fdb_c_unit_tests_impl doctest)
add_dependencies(fdb_c_unit_tests_version_510 doctest)
add_dependencies(disconnected_timeout_unit_tests doctest)
target_include_directories(fdb_c_setup_tests PUBLIC ${DOCTEST_INCLUDE_DIR})
target_include_directories(fdb_c_unit_tests PUBLIC ${DOCTEST_INCLUDE_DIR})
target_include_directories(fdb_c_unit_tests_impl PUBLIC ${DOCTEST_INCLUDE_DIR} ${CMAKE_CURRENT_SOURCE_DIR} ${CMAKE_CURRENT_BINARY_DIR}/foundationdb/)
target_include_directories(fdb_c_unit_tests_version_510 PUBLIC ${DOCTEST_INCLUDE_DIR} ${CMAKE_BINARY_DIR}/flow/include)
target_include_directories(disconnected_timeout_unit_tests PUBLIC ${DOCTEST_INCLUDE_DIR})
target_link_libraries(fdb_c_setup_tests PRIVATE fdb_c Threads::Threads)
target_link_libraries(fdb_c_unit_tests PRIVATE fdb_c Threads::Threads fdbclient rapidjson)
target_link_libraries(fdb_c_unit_tests_impl PRIVATE fdb_c Threads::Threads fdbclient rapidjson)
target_link_libraries(fdb_c_unit_tests_version_510 PRIVATE fdb_c Threads::Threads)
target_link_libraries(trace_partial_file_suffix_test PRIVATE fdb_c Threads::Threads flow)
target_link_libraries(disconnected_timeout_unit_tests PRIVATE fdb_c Threads::Threads)
if(USE_SANITIZER)
target_link_libraries(fdb_c_api_tester PRIVATE fdb_c fdb_cpp toml11_target Threads::Threads fmt::fmt boost_asan)
else()
target_link_libraries(fdb_c_api_tester PRIVATE fdb_c fdb_cpp toml11_target Threads::Threads fmt::fmt boost_target)
endif()
target_include_directories(fdb_c_api_tester PRIVATE "${CMAKE_SOURCE_DIR}/flow/include" "${CMAKE_BINARY_DIR}/flow/include")
target_link_libraries(fdb_c_api_tester PRIVATE SimpleOpt)
# do not set RPATH for mako
set_property(TARGET mako PROPERTY SKIP_BUILD_RPATH TRUE)
if (USE_SANITIZER)
@ -224,9 +228,9 @@ if(NOT WIN32)
DEPENDS fdb_c
COMMENT "Copy libfdb_c to use as external client for test")
add_custom_target(external_client DEPENDS ${CMAKE_CURRENT_BINARY_DIR}/libfdb_c_external.so)
add_dependencies(fdb_c_unit_tests external_client)
add_dependencies(fdb_c_unit_tests_impl external_client)
add_dependencies(disconnected_timeout_unit_tests external_client)
add_dependencies(fdb_c_api_tester external_client)
add_dependencies(fdb_c_api_tester_impl external_client)
add_fdbclient_test(
NAME fdb_c_setup_tests
@ -421,6 +425,40 @@ if (NOT WIN32 AND NOT APPLE AND NOT OPEN_FOR_IDE)
target_link_options(c_workloads PRIVATE "LINKER:--version-script=${CMAKE_CURRENT_SOURCE_DIR}/external_workload.map,-z,nodelete")
endif()
# Generate shim library in Linux builds
if (NOT WIN32 AND NOT APPLE AND NOT OPEN_FOR_IDE)
set(SHIM_LIB_OUTPUT_DIR ${CMAKE_CURRENT_BINARY_DIR})
set(SHIM_LIB_GEN_SRC
${SHIM_LIB_OUTPUT_DIR}/libfdb_c.so.init.c
${SHIM_LIB_OUTPUT_DIR}/libfdb_c.so.tramp.S)
add_custom_command(OUTPUT ${SHIM_LIB_GEN_SRC}
COMMAND $<TARGET_FILE:Python::Interpreter> ${CMAKE_SOURCE_DIR}/contrib/Implib.so/implib-gen.py
--target ${CMAKE_SYSTEM_PROCESSOR}
--outdir ${SHIM_LIB_OUTPUT_DIR}
--dlopen-callback=fdb_shim_dlopen_callback
$<TARGET_FILE:fdb_c>)
add_library(fdb_c_shim SHARED ${SHIM_LIB_GEN_SRC} fdb_c_shim.cpp)
target_link_options(fdb_c_shim PRIVATE "LINKER:--version-script=${CMAKE_CURRENT_SOURCE_DIR}/fdb_c.map,-z,nodelete,-z,noexecstack")
target_link_libraries(fdb_c_shim PUBLIC dl)
add_executable(fdb_c_shim_unit_tests)
target_link_libraries(fdb_c_shim_unit_tests PRIVATE fdb_c_shim fdb_c_unit_tests_impl)
add_executable(fdb_c_shim_api_tester)
target_link_libraries(fdb_c_shim_api_tester PRIVATE fdb_c_shim fdb_c_api_tester_impl)
add_test(NAME fdb_c_shim_library_tests
COMMAND $<TARGET_FILE:Python::Interpreter> ${CMAKE_CURRENT_SOURCE_DIR}/test/fdb_c_shim_tests.py
--build-dir ${CMAKE_BINARY_DIR}
--source-dir ${CMAKE_SOURCE_DIR}
)
endif() # End Linux only
# TODO: re-enable once the old vcxproj-based build system is removed.
#generate_export_header(fdb_c EXPORT_MACRO_NAME "DLLEXPORT"
# EXPORT_FILE_NAME ${CMAKE_CURRENT_BINARY_DIR}/foundationdb/fdb_c_export.h)

44
bindings/c/fdb_c_shim.cpp Normal file

@ -0,0 +1,44 @@
/*
* fdb_c_shim.cpp
*
* This source file is part of the FoundationDB open source project
*
* Copyright 2013-2022 Apple Inc. and the FoundationDB project authors
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
#if (defined(__linux__) || defined(__APPLE__) || defined(__FreeBSD__))
#include <dlfcn.h>
#include <stdio.h>
#include <stdlib.h>
#include <string>
static const char* FDB_C_CLIENT_LIBRARY_PATH = "FDB_C_CLIENT_LIBRARY_PATH";
// Callback that tries different library names
extern "C" void* fdb_shim_dlopen_callback(const char* libName) {
std::string libPath;
char* val = getenv(FDB_C_CLIENT_LIBRARY_PATH);
if (val) {
libPath = val;
} else {
libPath = libName;
}
return dlopen(libPath.c_str(), RTLD_LAZY | RTLD_GLOBAL);
}
#else
#error Port me!
#endif

208
bindings/c/test/fdb_c_shim_tests.py Normal file

@ -0,0 +1,208 @@
#!/usr/bin/env python3
from argparse import ArgumentParser, RawDescriptionHelpFormatter
from pathlib import Path
import platform
import shutil
import subprocess
import sys
import os
sys.path[:0] = [os.path.join(os.path.dirname(__file__), '..', '..', '..', 'tests', 'TestRunner')]
from binary_download import FdbBinaryDownloader, CURRENT_VERSION
from local_cluster import LocalCluster, random_secret_string
LAST_RELEASE_VERSION = "7.1.5"
TESTER_STATS_INTERVAL_SEC = 5
DEFAULT_TEST_FILE = "CApiCorrectnessMultiThr.toml"
def version_from_str(ver_str):
ver = [int(s) for s in ver_str.split(".")]
assert len(ver) == 3, "Invalid version string {}".format(ver_str)
return ver
def api_version_from_str(ver_str):
ver_tuple = version_from_str(ver_str)
return ver_tuple[0] * 100 + ver_tuple[1] * 10
def version_before(ver_str1, ver_str2):
return version_from_str(ver_str1) < version_from_str(ver_str2)
class TestEnv(LocalCluster):
def __init__(
self,
build_dir: str,
downloader: FdbBinaryDownloader,
version: str,
):
self.build_dir = Path(build_dir).resolve()
assert self.build_dir.exists(), "{} does not exist".format(build_dir)
assert self.build_dir.is_dir(), "{} is not a directory".format(build_dir)
self.tmp_dir = self.build_dir.joinpath("tmp", random_secret_string(16))
self.tmp_dir.mkdir(parents=True)
self.downloader = downloader
self.version = version
super().__init__(
self.tmp_dir,
self.downloader.binary_path(version, "fdbserver"),
self.downloader.binary_path(version, "fdbmonitor"),
self.downloader.binary_path(version, "fdbcli"),
1
)
self.set_env_var("LD_LIBRARY_PATH", self.downloader.lib_dir(version))
client_lib = self.downloader.lib_path(version)
assert client_lib.exists(), "{} does not exist".format(client_lib)
self.client_lib_external = self.tmp_dir.joinpath("libfdb_c_external.so")
shutil.copyfile(client_lib, self.client_lib_external)
def __enter__(self):
super().__enter__()
super().create_database()
return self
def __exit__(self, xc_type, exc_value, traceback):
super().__exit__(xc_type, exc_value, traceback)
shutil.rmtree(self.tmp_dir)
def exec_client_command(self, cmd_args, env_vars=None, expected_ret_code=0):
print("Executing test command: {}".format(
" ".join([str(c) for c in cmd_args])
))
tester_proc = subprocess.Popen(
cmd_args, stdout=sys.stdout, stderr=sys.stderr, env=env_vars
)
tester_retcode = tester_proc.wait()
assert tester_retcode == expected_ret_code, "Tester completed return code {}, but {} was expected".format(
tester_retcode, expected_ret_code)
class FdbCShimTests:
def __init__(
self,
args
):
self.build_dir = Path(args.build_dir).resolve()
assert self.build_dir.exists(), "{} does not exist".format(args.build_dir)
assert self.build_dir.is_dir(), "{} is not a directory".format(args.build_dir)
self.source_dir = Path(args.source_dir).resolve()
assert self.source_dir.exists(), "{} does not exist".format(args.source_dir)
assert self.source_dir.is_dir(), "{} is not a directory".format(args.source_dir)
self.api_tester_bin = self.build_dir.joinpath("bin", "fdb_c_shim_api_tester")
assert self.api_tester_bin.exists(), "{} does not exist".format(self.api_tester_bin)
self.unit_tests_bin = self.build_dir.joinpath("bin", "fdb_c_shim_unit_tests")
assert self.unit_tests_bin.exists(), "{} does not exist".format(self.unit_tests_bin)
self.api_test_dir = self.source_dir.joinpath("bindings", "c", "test", "apitester", "tests")
self.downloader = FdbBinaryDownloader(args.build_dir)
# binary downloads are currently available only for x86_64
self.platform = platform.machine()
if (self.platform == "x86_64"):
self.downloader.download_old_binaries(LAST_RELEASE_VERSION)
def build_c_api_tester_args(self, test_env, test_file):
test_file_path = self.api_test_dir.joinpath(test_file)
return [
self.api_tester_bin,
"--cluster-file",
test_env.cluster_file,
"--test-file",
test_file_path,
"--external-client-library",
test_env.client_lib_external,
"--disable-local-client",
"--api-version",
str(api_version_from_str(test_env.version)),
"--log",
"--log-dir",
test_env.log,
"--tmp-dir",
test_env.tmp_dir,
"--stats-interval",
str(TESTER_STATS_INTERVAL_SEC * 1000)
]
def run_c_api_test(self, version, test_file):
print('-' * 80)
print("C API Test - version: {}, workload: {}".format(version, test_file))
print('-' * 80)
with TestEnv(self.build_dir, self.downloader, version) as test_env:
cmd_args = self.build_c_api_tester_args(test_env, test_file)
env_vars = os.environ.copy()
env_vars["LD_LIBRARY_PATH"] = self.downloader.lib_dir(version)
test_env.exec_client_command(cmd_args, env_vars)
def run_c_unit_tests(self, version):
print('-' * 80)
print("C Unit Tests - version: {}".format(version))
print('-' * 80)
with TestEnv(self.build_dir, self.downloader, version) as test_env:
cmd_args = [
self.unit_tests_bin,
test_env.cluster_file,
"fdb",
test_env.client_lib_external
]
env_vars = os.environ.copy()
env_vars["LD_LIBRARY_PATH"] = self.downloader.lib_dir(version)
test_env.exec_client_command(cmd_args, env_vars)
def test_invalid_c_client_lib_env_var(self, version):
print('-' * 80)
print("Test invalid FDB_C_CLIENT_LIBRARY_PATH value")
print('-' * 80)
with TestEnv(self.build_dir, self.downloader, version) as test_env:
cmd_args = self.build_c_api_tester_args(test_env, DEFAULT_TEST_FILE)
env_vars = os.environ.copy()
env_vars["FDB_C_CLIENT_LIBRARY_PATH"] = "dummy"
test_env.exec_client_command(cmd_args, env_vars, 1)
def test_valid_c_client_lib_env_var(self, version):
print('-' * 80)
print("Test valid FDB_C_CLIENT_LIBRARY_PATH value")
print('-' * 80)
with TestEnv(self.build_dir, self.downloader, version) as test_env:
cmd_args = self.build_c_api_tester_args(test_env, DEFAULT_TEST_FILE)
env_vars = os.environ.copy()
env_vars["FDB_C_CLIENT_LIBRARY_PATH"] = self.downloader.lib_path(version)
test_env.exec_client_command(cmd_args, env_vars)
def run_tests(self):
# binary downloads are currently available only for x86_64
if (self.platform == "x86_64"):
self.run_c_api_test(LAST_RELEASE_VERSION, DEFAULT_TEST_FILE)
self.run_c_api_test(CURRENT_VERSION, DEFAULT_TEST_FILE)
self.run_c_unit_tests(CURRENT_VERSION)
self.test_invalid_c_client_lib_env_var(CURRENT_VERSION)
self.test_valid_c_client_lib_env_var(CURRENT_VERSION)
if __name__ == "__main__":
parser = ArgumentParser(
formatter_class=RawDescriptionHelpFormatter,
description="""
A script for testing FDB multi-version client in upgrade scenarios. Creates a local cluster,
generates a workload using fdb_c_api_tester with a specified test file, and performs
cluster upgrade according to the specified upgrade path. Checks if the workload successfully
progresses after each upgrade step.
""",
)
parser.add_argument(
"--build-dir",
"-b",
metavar="BUILD_DIRECTORY",
help="FDB build directory",
required=True,
)
parser.add_argument(
"--source-dir",
"-s",
metavar="SOURCE_DIRECTORY",
help="FDB source directory",
required=True,
)
args = parser.parse_args()
test = FdbCShimTests(args)
test.run_tests()

1
contrib/Implib.so/.gitattributes vendored Normal file

@ -0,0 +1 @@
arch/*/*.tpl linguist-vendored

144
contrib/Implib.so/.github/workflows/ci.yml vendored Normal file

@ -0,0 +1,144 @@
# TODO:
# * Android
# * FreeBSD
name: CI
on:
push:
paths-ignore:
- 'LICENSE.txt'
- 'README.md'
pull_request:
paths-ignore:
- 'LICENSE.txt'
- 'README.md'
jobs:
Baseline:
strategy:
fail-fast: false
matrix:
os: [ubuntu-18.04, ubuntu-20.04, ubuntu-latest]
cc: [[gcc, g++], [clang, clang++]]
py: [python3.6, python3.7, python3] # We need f-strings so 3.6+
runs-on: ${{ matrix.os }}
env:
CC: ${{ matrix.cc[0] }}
CXX: ${{ matrix.cc[1] }}
PYTHON: ${{ matrix.py }}
steps:
- uses: actions/checkout@v2
- name: Install deps
run: |
sudo add-apt-repository ppa:deadsnakes/ppa
sudo apt-get update
sudo apt-get install ${PYTHON}
sudo apt-get install ${PYTHON}-pip || true
- name: Run tests
run: scripts/travis.sh
PyPy: # Can't test this in matrix, pypy package has different names in old distros
runs-on: ubuntu-latest
env:
PYTHON: pypy3
steps:
- uses: actions/checkout@v2
- name: Install deps
run: sudo apt-get install ${PYTHON}
- name: Run tests
run: scripts/travis.sh
Pylint:
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v2
- name: Install deps
run: sudo apt-get install pylint
- name: Run tests
run: |
pylint implib-gen.py
pylint scripts/ld
Coverage:
runs-on: ubuntu-latest
environment: secrets
env:
COVERAGE: 1
CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }}
PYTHON: 'coverage run -a'
steps:
- uses: actions/checkout@v2
- name: Install deps
run: |
sudo apt-get install python3 python3-pip
sudo python3 -mpip install codecov
- name: Run tests
run: scripts/travis.sh
- name: Upload coverage
run: |
for t in tests/*; do
if test -d $t; then
(cd $t && coverage xml)
fi
done
codecov --required
x86:
runs-on: ubuntu-latest
env:
ARCH: i386-linux-gnueabi
steps:
- uses: actions/checkout@v2
- name: Install deps
run: sudo apt-get install gcc-multilib g++-multilib
- name: Run tests
run: scripts/travis.sh
arm-arm:
runs-on: ubuntu-latest
env:
ARCH: arm-linux-gnueabi
CFLAGS: -marm
steps:
- uses: actions/checkout@v2
- name: Install deps
run: sudo apt-get install qemu-user gcc-arm-linux-gnueabi g++-arm-linux-gnueabi binutils-arm-linux-gnueabi libc6-armel-cross libc6-dev-armel-cross
- name: Run tests
run: scripts/travis.sh
arm-thumb:
runs-on: ubuntu-latest
env:
ARCH: arm-linux-gnueabi
CFLAGS: -mthumb
steps:
- uses: actions/checkout@v2
- name: Install deps
run: sudo apt-get install qemu-user gcc-arm-linux-gnueabi g++-arm-linux-gnueabi binutils-arm-linux-gnueabi libc6-armel-cross libc6-dev-armel-cross
- name: Run tests
run: scripts/travis.sh
armhf-arm:
runs-on: ubuntu-latest
env:
ARCH: arm-linux-gnueabihf
CFLAGS: -marm
steps:
- uses: actions/checkout@v2
- name: Install deps
run: sudo apt-get install qemu-user gcc-arm-linux-gnueabihf g++-arm-linux-gnueabihf binutils-arm-linux-gnueabihf libc6-armhf-cross libc6-dev-armhf-cross
- name: Run tests
run: scripts/travis.sh
armhf-thumb:
runs-on: ubuntu-latest
env:
ARCH: arm-linux-gnueabihf
CFLAGS: -mthumb
steps:
- uses: actions/checkout@v2
- name: Install deps
run: sudo apt-get install qemu-user gcc-arm-linux-gnueabihf g++-arm-linux-gnueabihf binutils-arm-linux-gnueabihf libc6-armhf-cross libc6-dev-armhf-cross
- name: Run tests
run: scripts/travis.sh
aarch64:
runs-on: ubuntu-latest
env:
ARCH: aarch64-linux-gnueabi
steps:
- uses: actions/checkout@v2
- name: Install deps
run: sudo apt-get install qemu-user gcc-aarch64-linux-gnu g++-aarch64-linux-gnu binutils-aarch64-linux-gnu libc6-arm64-cross libc6-dev-arm64-cross
- name: Run tests
run: scripts/travis.sh

16
contrib/Implib.so/.gitignore vendored Normal file

@ -0,0 +1,16 @@
# Bytecode
__pycache__/
*.py[cod]
*.swp
.*.swp
# Temp files
*.tramp.S
*.init.c
*.so*
tests/*/a.out*
tests/*/*.log
# Coverage
*.coverage
*coverage.xml

23
contrib/Implib.so/.pylintrc Normal file

@ -0,0 +1,23 @@
[MESSAGES CONTROL]
disable = trailing-whitespace, # In copyrights
invalid-name, # Short variables like 'v'
unused-wildcard-import, # For 'from ... import ...'
fixme, # TODO/FIXME
unspecified-encoding, # Rely on default UTF-8
unused-argument, # Intentional
too-many-locals, too-many-branches, too-many-boolean-expressions, too-many-statements,
too-few-public-methods,
[FORMAT]
indent-string = ' '
indent-after-paren = 2
[IMPORT]
allow-wildcard-with-all = yes
[TYPECHECK]
disable = bad-option-value # C0209 is unsupported in older versions

21
contrib/Implib.so/LICENSE.txt Normal file

@ -0,0 +1,21 @@
The MIT License (MIT)
Copyright (c) 2017-2022 Yury Gribov
Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal
in the Software without restriction, including without limitation the rights
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
copies of the Software, and to permit persons to whom the Software is
furnished to do so, subject to the following conditions:
The above copyright notice and this permission notice shall be included in all
copies or substantial portions of the Software.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
SOFTWARE.

220
contrib/Implib.so/README.md Normal file

@ -0,0 +1,220 @@
[![License](http://img.shields.io/:license-MIT-blue.svg)](https://github.com/yugr/Implib.so/blob/master/LICENSE.txt)
[![Build Status](https://github.com/yugr/Implib.so/actions/workflows/ci.yml/badge.svg)](https://github.com/yugr/Implib.so/actions)
[![Total alerts](https://img.shields.io/lgtm/alerts/g/yugr/Implib.so.svg?logo=lgtm&logoWidth=18)](https://lgtm.com/projects/g/yugr/Implib.so/alerts/)
[![Codecov](https://codecov.io/gh/yugr/Implib.so/branch/master/graph/badge.svg)](https://codecov.io/gh/yugr/Implib.so)
# Motivation
In a nutshell, Implib.so is a simple equivalent of [Windows DLL import libraries](http://www.digitalmars.com/ctg/implib.html) for POSIX shared libraries.
On Linux/Android, if you link against shared library you normally use `-lxyz` compiler option which makes your application depend on `libxyz.so`. This would cause `libxyz.so` to be forcedly loaded at program startup (and its constructors to be executed) even if you never call any of its functions.
If you instead want to delay loading of `libxyz.so` (e.g. its unlikely to be used and you don't want to waste resources on it or [slow down startup time](https://lwn.net/Articles/341309/) or you want to select best platform-specific implementation at runtime), you can remove dependency from `LDFLAGS` and issue `dlopen` call manually. But this would cause `ld` to err because it won't be able to statically resolve symbols which are supposed to come from this shared library. At this point you have only two choices:
* emit normal calls to library functions and suppress link errors from `ld` via `-Wl,-z,nodefs`; this is undesired because you loose ability to detect link errors for other libraries statically
* load necessary function addresses at runtime via `dlsym` and call them via function pointers; this isn't very convenient because you have to keep track which symbols your program uses, properly cast function types and also somehow manage global function pointers
Implib.so provides an easy solution - link your program with a _wrapper_ which
* provides all necessary symbols to make linker happy
* loads wrapped library on first call to any of its functions
* redirects calls to library symbols
Generated wrapper code (often also called "shim" code or "shim" library) is analogous to Windows import libraries which achieve the same functionality for DLLs.
Implib.so can also be used to [reduce API provided by existing shared library](#reducing-external-interface-of-closed-source-library) or [rename it's exported symbols](#renaming-exported-interface-of-closed-source-library).
Implib.so was originally inspired by Stackoverflow question [Is there an elegant way to avoid dlsym when using dlopen in C?](https://stackoverflow.com/questions/45917816/is-there-an-elegant-way-to-avoid-dlsym-when-using-dlopen-in-c/47221180).
# Usage
A typical use-case would look like this:
```
$ implib-gen.py libxyz.so
```
This will generate code for host platform (presumably x86\_64). For other targets do
```
$ implib-gen.py --target $TARGET libxyz.so
```
where `TARGET` can be any of
* x86\_64-linux-gnu, x86\_64-none-linux-android
* i686-linux-gnu, i686-none-linux-android
* arm-linux-gnueabi, armel-linux-gnueabi, armv7-none-linux-androideabi
* arm-linux-gnueabihf (ARM hardfp ABI)
* aarch64-linux-gnu, aarch64-none-linux-android
* e2k-linux-gnu
Script generates two files: `libxyz.so.tramp.S` and `libxyz.so.init.c` which need to be linked to your application (instead of `-lxyz`):
```
$ gcc myfile1.c myfile2.c ... libxyz.so.tramp.S libxyz.so.init.c ... -ldl
```
Note that you need to link against libdl.so. On ARM in case your app is compiled to Thumb code (which e.g. Ubuntu's `arm-linux-gnueabihf-gcc` does by default) you'll also need to add `-mthumb-interwork`.
Application can then freely call functions from `libxyz.so` _without linking to it_. Library will be loaded (via `dlopen`) on first call to any of its functions. If you want to forcedly resolve all symbols (e.g. if you want to avoid delays further on) you can call `void libxyz_init_all()`.
Above command would perform a _lazy load_ i.e. load library on first call to one of it's symbols. If you want to load it at startup, run
```
$ implib-gen.py --no-lazy-load libxyz.so
```
If you don't want `dlopen` to be called automatically and prefer to load library yourself at program startup, run script as
```
$ implib-gen.py --no-dlopen libxys.so
```
If you do want to load library via `dlopen` but would prefer to call it yourself (e.g. with custom parameters or with modified library name), run script as
```
$ cat mycallback.c
#define _GNU_SOURCE
#include <dlfcn.h>
#include <stdio.h>
#include <stdlib.h>
#ifdef __cplusplus
extern "C"
#endif
// Callback that tries different library names
void *mycallback(const char *lib_name) {
lib_name = lib_name; // Please the compiler
void *h;
h = dlopen("libxyz.so", RTLD_LAZY);
if (h)
return h;
h = dlopen("libxyz-stub.so", RTLD_LAZY);
if (h)
return h;
fprintf(stderr, "dlopen failed: %s\n", dlerror());
exit(1);
}
$ implib-gen.py --dlopen-callback=mycallback libxyz.so
```
(callback must have signature `void *(*)(const char *lib_name)` and return handle of loaded library).
Finally to force library load and resolution of all symbols, call
void _LIBNAME_tramp_resolve_all(void);
# Wrapping vtables
By default the tool does not try to wrap vtables exported from the library. This can be enabled via `--vtables` flag:
```
$ implib-gen.py --vtables ...
```
# Reducing external interface of closed-source library
Sometimes you may want to reduce public interface of existing shared library (e.g. if it's a third-party lib which erroneously exports too many unrelated symbols).
To achieve this you can generate a wrapper with limited number of symbols and override the callback which loads the library to use `dlmopen` instead of `dlopen` (and thus does not pollute the global namespace):
```
$ cat mysymbols.txt
foo
bar
$ cat mycallback.c
#define _GNU_SOURCE
#include <dlfcn.h>
#include <stdio.h>
#include <stdlib.h>
#ifdef __cplusplus
extern "C"
#endif
// Dlopen callback that loads library to dedicated namespace
void *mycallback(const char *lib_name) {
void *h = dlmopen(LM_ID_NEWLM, lib_name, RTLD_LAZY | RTLD_DEEPBIND);
if (h)
return h;
fprintf(stderr, "dlmopen failed: %s\n", dlerror());
exit(1);
}
$ implib-gen.py --dlopen-callback=mycallback --symbol-list=mysymbols.txt libxyz.so
$ ... # Link your app with libxyz.tramp.S, libxyz.init.c and mycallback.c
```
Similar approach can be used if you want to provide a common interface for several libraries with partially intersecting interfaces (see [this example](tests/multilib/run.sh) for more details).
# Renaming exported interface of closed-source library
Sometimes you may need to rename API of existing shared library to avoid name clashes.
To achieve this you can generate a wrapper with _renamed_ symbols which call to old, non-renamed symbols in original library loaded via `dlmopen` instead of `dlopen` (to avoid polluting global namespace):
```
$ cat mycallback.c
... Same as before ...
$ implib-gen.py --dlopen-callback=mycallback --symbol_prefix=MYPREFIX_ libxyz.so
$ ... # Link your app with libxyz.tramp.S, libxyz.init.c and mycallback.c
```
# Linker wrapper
Generation of wrappers may be automated via linker wrapper `scripts/ld`.
Adding it to `PATH` (in front of normal `ld`) would by default result
in all dynamic libs (besides system ones) to be replaced with wrappers.
Explicit list of libraries can be specified by exporting
`IMPLIBSO_LD_OPTIONS` environment variable:
```
export IMPLIBSO_LD_OPTIONS='--wrap-libs attr,acl'
```
For more details run with
```
export IMPLIBSO_LD_OPTIONS=--help
```
Atm linker wrapper is only meant for testing.
# Overhead
Implib.so overhead on a fast path boils down to
* predictable direct jump to wrapper
* predictable untaken direct branch to initialization code
* load from trampoline table
* predictable indirect jump to real function
This is very similar to normal shlib call:
* predictable direct jump to PLT stub
* load from GOT
* predictable indirect jump to real function
so it should have equivalent performance.
# Limitations
The tool does not transparently support all features of POSIX shared libraries. In particular
* it can not provide wrappers for data symbols (except C++ virtual/RTTI tables)
* it makes first call to wrapped functions asynch signal unsafe (as it will call `dlopen` and library constructors)
* it may change semantics if there are multiple definitions of same symbol in different loaded shared objects (runtime symbol interposition is considered a bad practice though)
* it may change semantics because shared library constructors are delayed until when library is loaded
The tool also lacks the following very important features:
* proper support for multi-threading
* symbol versions are not handled at all
* support OSX
(none should be hard to add so let me know if you need it).
Finally, Implib.so is only lightly tested and there are some minor TODOs in code.
# Related work
As mentioned in introduction import libraries are first class citizens on Windows platform:
* [Wikipedia on Windows Import Libraries](https://en.wikipedia.org/wiki/Dynamic-link_library#Import_libraries)
* [MSDN on Linker Support for Delay-Loaded DLLs](https://msdn.microsoft.com/en-us/library/151kt790.aspx)
Delay-loaded libraries were once present on OSX (via `-lazy_lXXX` and `-lazy_library` options).
Lazy loading is supported by Solaris shared libraries but was never implemented in Linux. There have been [some discussions](https://www.sourceware.org/ml/libc-help/2013-02/msg00017.html) in libc-alpha but no patches were posted.
Implib.so-like functionality is used in [OpenGL loading libraries](https://www.khronos.org/opengl/wiki/OpenGL_Loading_Library) e.g. [GLEW](http://glew.sourceforge.net/) via custom project-specific scripts.

9
contrib/Implib.so/arch/README.md Normal file

@ -0,0 +1,9 @@
This folder contains target-specific config files and code snippets.
Basically to add a new target one needs to provide an .ini file with basic platform info
(like pointer sizes) and code templates for
* shim code which checks that real function address is available (and either jumps there or calls the slow path)
* the "slow path" code which
- saves function arguments (to avoid trashing them in next steps)
- calls code which loads the target library and locates function addresses
- restores saved arguments and returns

3
contrib/Implib.so/arch/aarch64/config.ini Normal file

@ -0,0 +1,3 @@
[Arch]
PointerSize = 8
SymbolReloc = R_AARCH64_ABS64

78
contrib/Implib.so/arch/aarch64/table.S.tpl vendored Normal file

@ -0,0 +1,78 @@
/*
* Copyright 2018-2020 Yury Gribov
*
* The MIT License (MIT)
*
* Use of this source code is governed by MIT license that can be
* found in the LICENSE.txt file.
*/
#define lr x30
#define ip0 x16
.data
.globl _${lib_suffix}_tramp_table
.hidden _${lib_suffix}_tramp_table
.align 8
_${lib_suffix}_tramp_table:
.zero $table_size
.text
.globl _${lib_suffix}_tramp_resolve
.hidden _${lib_suffix}_tramp_resolve
.globl _${lib_suffix}_save_regs_and_resolve
.hidden _${lib_suffix}_save_regs_and_resolve
.type _${lib_suffix}_save_regs_and_resolve, %function
_${lib_suffix}_save_regs_and_resolve:
.cfi_startproc
// Slow path which calls dlsym, taken only on first call.
// Registers are saved according to "Procedure Call Standard for the Arm® 64-bit Architecture".
// For DWARF directives, read https://www.imperialviolet.org/2017/01/18/cfi.html.
// Stack is aligned at 16 bytes
#define PUSH_PAIR(reg1, reg2) stp reg1, reg2, [sp, #-16]!; .cfi_adjust_cfa_offset 16; .cfi_rel_offset reg1, 0; .cfi_rel_offset reg2, 8
#define POP_PAIR(reg1, reg2) ldp reg1, reg2, [sp], #16; .cfi_adjust_cfa_offset -16; .cfi_restore reg2; .cfi_restore reg1
#define PUSH_WIDE_PAIR(reg1, reg2) stp reg1, reg2, [sp, #-32]!; .cfi_adjust_cfa_offset 32; .cfi_rel_offset reg1, 0; .cfi_rel_offset reg2, 16
#define POP_WIDE_PAIR(reg1, reg2) ldp reg1, reg2, [sp], #32; .cfi_adjust_cfa_offset -32; .cfi_restore reg2; .cfi_restore reg1
// Save only arguments (and lr)
PUSH_PAIR(x0, x1)
PUSH_PAIR(x2, x3)
PUSH_PAIR(x4, x5)
PUSH_PAIR(x6, x7)
PUSH_PAIR(x8, lr)
ldr x0, [sp, #80] // 16*5
PUSH_WIDE_PAIR(q0, q1)
PUSH_WIDE_PAIR(q2, q3)
PUSH_WIDE_PAIR(q4, q5)
PUSH_WIDE_PAIR(q6, q7)
// Stack is aligned at 16 bytes
bl _${lib_suffix}_tramp_resolve
// TODO: pop pc?
POP_WIDE_PAIR(q6, q7)
POP_WIDE_PAIR(q4, q5)
POP_WIDE_PAIR(q2, q3)
POP_WIDE_PAIR(q0, q1)
POP_PAIR(x8, lr)
POP_PAIR(x6, x7)
POP_PAIR(x4, x5)
POP_PAIR(x2, x3)
POP_PAIR(x0, x1)
br lr
.cfi_endproc

38
contrib/Implib.so/arch/aarch64/trampoline.S.tpl vendored Normal file

@ -0,0 +1,38 @@
/*
* Copyright 2018-2021 Yury Gribov
*
* The MIT License (MIT)
*
* Use of this source code is governed by MIT license that can be
* found in the LICENSE.txt file.
*/
.globl $sym
.p2align 4
.type $sym, %function
#ifdef IMPLIB_HIDDEN_SHIMS
.hidden $sym
#endif
$sym:
.cfi_startproc
1:
// Load address
// TODO: can we do this faster on newer ARMs?
adrp ip0, _${lib_suffix}_tramp_table+$offset
ldr ip0, [ip0, #:lo12:_${lib_suffix}_tramp_table+$offset]
cbz ip0, 2f
// Fast path
br ip0
2:
// Slow path
mov ip0, $number
stp ip0, lr, [sp, #-16]!; .cfi_adjust_cfa_offset 16; .cfi_rel_offset ip0, 0; .cfi_rel_offset lr, 8;
bl _${lib_suffix}_save_regs_and_resolve
ldp ip0, lr, [sp], #16; .cfi_adjust_cfa_offset -16; .cfi_restore lr; .cfi_restore ip0
b 1b
.cfi_endproc

115
contrib/Implib.so/arch/common/init.c.tpl vendored Normal file

@ -0,0 +1,115 @@
/*
* Copyright 2018-2020 Yury Gribov
*
* The MIT License (MIT)
*
* Use of this source code is governed by MIT license that can be
* found in the LICENSE.txt file.
*/
#include <dlfcn.h>
#include <stdlib.h>
#include <stdio.h>
#include <assert.h>
// Sanity check for ARM to avoid puzzling runtime crashes
#ifdef __arm__
# if defined __thumb__ && ! defined __THUMB_INTERWORK__
# error "ARM trampolines need -mthumb-interwork to work in Thumb mode"
# endif
#endif
#ifdef __cplusplus
extern "C" {
#endif
#define CHECK(cond, fmt, ...) do { \
if(!(cond)) { \
fprintf(stderr, "implib-gen: $load_name: " fmt "\n", ##__VA_ARGS__); \
assert(0 && "Assertion in generated code"); \
exit(1); \
} \
} while(0)
#define CALL_USER_CALLBACK $has_dlopen_callback
#define NO_DLOPEN $no_dlopen
#define LAZY_LOAD $lazy_load
static void *lib_handle;
static int is_lib_loading;
static void *load_library() {
if(lib_handle)
return lib_handle;
is_lib_loading = 1;
// TODO: dlopen and users callback must be protected w/ critical section (to avoid dlopening lib twice)
#if NO_DLOPEN
CHECK(0, "internal error"); // We shouldn't get here
#elif CALL_USER_CALLBACK
extern void *$dlopen_callback(const char *lib_name);
lib_handle = $dlopen_callback("$load_name");
CHECK(lib_handle, "callback '$dlopen_callback' failed to load library");
#else
lib_handle = dlopen("$load_name", RTLD_LAZY | RTLD_GLOBAL);
CHECK(lib_handle, "failed to load library: %s", dlerror());
#endif
is_lib_loading = 0;
return lib_handle;
}
#if ! NO_DLOPEN && ! LAZY_LOAD
static void __attribute__((constructor)) load_lib() {
load_library();
}
#endif
static void __attribute__((destructor)) unload_lib() {
if(lib_handle)
dlclose(lib_handle);
}
// TODO: convert to single 0-separated string
static const char *const sym_names[] = {
$sym_names
0
};
extern void *_${lib_suffix}_tramp_table[];
// Can be sped up by manually parsing library symtab...
void _${lib_suffix}_tramp_resolve(int i) {
assert((unsigned)i + 1 < sizeof(sym_names) / sizeof(sym_names[0]));
CHECK(!is_lib_loading, "library function '%s' called during library load", sym_names[i]);
void *h = 0;
#if NO_DLOPEN
// FIXME: instead of RTLD_NEXT we should search for loaded lib_handle
// as in https://github.com/jethrogb/ssltrace/blob/bf17c150a7/ssltrace.cpp#L74-L112
h = RTLD_NEXT;
#elif LAZY_LOAD
h = load_library();
#else
h = lib_handle;
CHECK(h, "failed to resolve symbol '%s', library failed to load", sym_names[i]);
#endif
// Dlsym is thread-safe so don't need to protect it.
_${lib_suffix}_tramp_table[i] = dlsym(h, sym_names[i]);
CHECK(_${lib_suffix}_tramp_table[i], "failed to resolve symbol '%s'", sym_names[i]);
}
// Helper for user to resolve all symbols
void _${lib_suffix}_tramp_resolve_all(void) {
size_t i;
for(i = 0; i + 1 < sizeof(sym_names) / sizeof(sym_names[0]); ++i)
_${lib_suffix}_tramp_resolve(i);
}
#ifdef __cplusplus
} // extern "C"
#endif

3
contrib/Implib.so/arch/x86_64/config.ini Normal file

@ -0,0 +1,3 @@
[Arch]
PointerSize = 8
SymbolReloc = R_X86_64_64

100
contrib/Implib.so/arch/x86_64/table.S.tpl vendored Normal file

@ -0,0 +1,100 @@
/*
* Copyright 2018-2020 Yury Gribov
*
* The MIT License (MIT)
*
* Use of this source code is governed by MIT license that can be
* found in the LICENSE.txt file.
*/
.data
.globl _${lib_suffix}_tramp_table
.hidden _${lib_suffix}_tramp_table
.align 8
_${lib_suffix}_tramp_table:
.zero $table_size
.text
.globl _${lib_suffix}_tramp_resolve
.hidden _${lib_suffix}_tramp_resolve
.globl _${lib_suffix}_save_regs_and_resolve
.hidden _${lib_suffix}_save_regs_and_resolve
.type _${lib_suffix}_save_regs_and_resolve, %function
_${lib_suffix}_save_regs_and_resolve:
.cfi_startproc
#define PUSH_REG(reg) pushq %reg ; .cfi_adjust_cfa_offset 8; .cfi_rel_offset reg, 0
#define POP_REG(reg) popq %reg ; .cfi_adjust_cfa_offset -8; .cfi_restore reg
#define DEC_STACK(d) subq $$d, %rsp; .cfi_adjust_cfa_offset d
#define INC_STACK(d) addq $$d, %rsp; .cfi_adjust_cfa_offset -d
#define PUSH_XMM_REG(reg) DEC_STACK(16); movdqa %reg, (%rsp); .cfi_rel_offset reg, 0
#define POP_XMM_REG(reg) movdqa (%rsp), %reg; .cfi_restore reg; INC_STACK(16)
// Slow path which calls dlsym, taken only on first call.
// All registers are stored to handle arbitrary calling conventions
// (except x87 FPU registers which do not have to be preserved).
// For Dwarf directives, read https://www.imperialviolet.org/2017/01/18/cfi.html.
// FIXME: AVX (YMM, ZMM) registers are NOT saved to simplify code.
PUSH_REG(rdi) // 16
mov 0x10(%rsp), %rdi
PUSH_REG(rax)
PUSH_REG(rbx) // 16
PUSH_REG(rcx)
PUSH_REG(rdx) // 16
PUSH_REG(rbp)
PUSH_REG(rsi) // 16
PUSH_REG(r8)
PUSH_REG(r9) // 16
PUSH_REG(r10)
PUSH_REG(r11) // 16
PUSH_REG(r12)
PUSH_REG(r13) // 16
PUSH_REG(r14)
PUSH_REG(r15) // 16
PUSH_XMM_REG(xmm0)
PUSH_XMM_REG(xmm1)
PUSH_XMM_REG(xmm2)
PUSH_XMM_REG(xmm3)
PUSH_XMM_REG(xmm4)
PUSH_XMM_REG(xmm5)
PUSH_XMM_REG(xmm6)
PUSH_XMM_REG(xmm7)
// Stack is just 8-byte aligned but callee will re-align to 16
call _${lib_suffix}_tramp_resolve
POP_XMM_REG(xmm7)
POP_XMM_REG(xmm6)
POP_XMM_REG(xmm5)
POP_XMM_REG(xmm4)
POP_XMM_REG(xmm3)
POP_XMM_REG(xmm2)
POP_XMM_REG(xmm1)
POP_XMM_REG(xmm0) // 16
POP_REG(r15)
POP_REG(r14) // 16
POP_REG(r13)
POP_REG(r12) // 16
POP_REG(r11)
POP_REG(r10) // 16
POP_REG(r9)
POP_REG(r8) // 16
POP_REG(rsi)
POP_REG(rbp) // 16
POP_REG(rdx)
POP_REG(rcx) // 16
POP_REG(rbx)
POP_REG(rax) // 16
POP_REG(rdi)
ret
.cfi_endproc

33
contrib/Implib.so/arch/x86_64/trampoline.S.tpl vendored Normal file

@ -0,0 +1,33 @@
/*
* Copyright 2018-2021 Yury Gribov
*
* The MIT License (MIT)
*
* Use of this source code is governed by MIT license that can be
* found in the LICENSE.txt file.
*/
.globl $sym
.p2align 4
.type $sym, %function
#ifdef IMPLIB_HIDDEN_SHIMS
.hidden $sym
#endif
$sym:
.cfi_startproc
// Intel opt. manual says to
// "make the fall-through code following a conditional branch be the likely target for a branch with a forward target"
// to hint static predictor.
cmpq $$0, _${lib_suffix}_tramp_table+$offset(%rip)
je 2f
1:
jmp *_${lib_suffix}_tramp_table+$offset(%rip)
2:
pushq $$$number
.cfi_adjust_cfa_offset 8
call _${lib_suffix}_save_regs_and_resolve
addq $$8, %rsp
.cfi_adjust_cfa_offset -8
jmp 1b
.cfi_endproc

553
contrib/Implib.so/implib-gen.py Executable file

@ -0,0 +1,553 @@
#!/usr/bin/python3
# Copyright 2017-2022 Yury Gribov
#
# The MIT License (MIT)
#
# Use of this source code is governed by MIT license that can be
# found in the LICENSE.txt file.
"""
Generates static import library for POSIX shared library
"""
import sys
import os.path
import re
import subprocess
import argparse
import string
import configparser
me = os.path.basename(__file__)
root = os.path.dirname(__file__)
def warn(msg):
"""Emits a nicely-decorated warning."""
sys.stderr.write(f'{me}: warning: {msg}\n')
def error(msg):
"""Emits a nicely-decorated error and exits."""
sys.stderr.write(f'{me}: error: {msg}\n')
sys.exit(1)
def run(args, stdin=''):
"""Runs external program and aborts on error."""
env = os.environ.copy()
# Force English language
env['LC_ALL'] = 'c'
try:
del env["LANG"]
except KeyError:
pass
with subprocess.Popen(args, stdin=subprocess.PIPE, stdout=subprocess.PIPE,
stderr=subprocess.PIPE, env=env) as p:
out, err = p.communicate(input=stdin.encode('utf-8'))
out = out.decode('utf-8')
err = err.decode('utf-8')
if p.returncode != 0 or err:
error(f"{args[0]} failed with retcode {p.returncode}:\n{err}")
return out, err
def make_toc(words, renames=None):
"Make an mapping of words to their indices in list"
renames = renames or {}
toc = {}
for i, n in enumerate(words):
name = renames.get(n, n)
toc[i] = name
return toc
def parse_row(words, toc, hex_keys):
"Make a mapping from column names to values"
vals = {k: (words[i] if i < len(words) else '') for i, k in toc.items()}
for k in hex_keys:
if vals[k]:
vals[k] = int(vals[k], 16)
return vals
def collect_syms(f):
"""Collect ELF dynamic symtab."""
# --dyn-syms does not always work for some reason so dump all symtabs
out, _ = run(['readelf', '-sW', f])
toc = None
syms = []
syms_set = set()
for line in out.splitlines():
line = line.strip()
if not line:
# Next symtab
toc = None
continue
words = re.split(r' +', line)
if line.startswith('Num'): # Header?
if toc is not None:
error("multiple headers in output of readelf")
# Colons are different across readelf versions so get rid of them.
toc = make_toc(map(lambda n: n.replace(':', ''), words))
elif toc is not None:
sym = parse_row(words, toc, ['Value'])
name = sym['Name']
if name in syms_set:
continue
syms_set.add(name)
sym['Size'] = int(sym['Size'], 0) # Readelf is inconistent on Size format
if '@' in name:
sym['Default'] = '@@' in name
name, ver = re.split(r'@+', name)
sym['Name'] = name
sym['Version'] = ver
else:
sym['Default'] = True
sym['Version'] = None
syms.append(sym)
if toc is None:
error(f"failed to analyze symbols in {f}")
# Also collected demangled names
if syms:
out, _ = run(['c++filt'], '\n'.join((sym['Name'] for sym in syms)))
for i, name in enumerate(out.split("\n")):
syms[i]['Demangled Name'] = name
return syms
def collect_relocs(f):
"""Collect ELF dynamic relocs."""
out, _ = run(['readelf', '-rW', f])
toc = None
rels = []
for line in out.splitlines():
line = line.strip()
if not line:
toc = None
continue
if line == 'There are no relocations in this file.':
return []
if re.match(r'^\s*Offset', line): # Header?
if toc is not None:
error("multiple headers in output of readelf")
words = re.split(r'\s\s+', line) # "Symbol's Name + Addend"
toc = make_toc(words)
elif toc is not None:
line = re.sub(r' \+ ', '+', line)
words = re.split(r'\s+', line)
rel = parse_row(words, toc, ['Offset', 'Info'])
rels.append(rel)
# Split symbolic representation
sym_name = 'Symbol\'s Name + Addend'
if sym_name not in rel and 'Symbol\'s Name' in rel:
# Adapt to different versions of readelf
rel[sym_name] = rel['Symbol\'s Name'] + '+0'
if rel[sym_name]:
p = rel[sym_name].split('+')
if len(p) == 1:
p = ['', p[0]]
rel[sym_name] = (p[0], int(p[1], 16))
if toc is None:
error(f"failed to analyze relocations in {f}")
return rels
def collect_sections(f):
"""Collect section info from ELF."""
out, _ = run(['readelf', '-SW', f])
toc = None
sections = []
for line in out.splitlines():
line = line.strip()
if not line:
continue
line = re.sub(r'\[\s+', '[', line)
words = re.split(r' +', line)
if line.startswith('[Nr]'): # Header?
if toc is not None:
error("multiple headers in output of readelf")
toc = make_toc(words, {'Addr' : 'Address'})
elif line.startswith('[') and toc is not None:
sec = parse_row(words, toc, ['Address', 'Off', 'Size'])
if 'A' in sec['Flg']: # Allocatable section?
sections.append(sec)
if toc is None:
error(f"failed to analyze sections in {f}")
return sections
def read_unrelocated_data(input_name, syms, secs):
"""Collect unrelocated data from ELF."""
data = {}
with open(input_name, 'rb') as f:
def is_symbol_in_section(sym, sec):
sec_end = sec['Address'] + sec['Size']
is_start_in_section = sec['Address'] <= sym['Value'] < sec_end
is_end_in_section = sym['Value'] + sym['Size'] <= sec_end
return is_start_in_section and is_end_in_section
for name, s in sorted(syms.items(), key=lambda s: s[1]['Value']):
# TODO: binary search (bisect)
sec = [sec for sec in secs if is_symbol_in_section(s, sec)]
if len(sec) != 1:
error(f"failed to locate section for interval [{s['Value']:x}, {s['Value'] + s['Size']:x})")
sec = sec[0]
f.seek(sec['Off'])
data[name] = f.read(s['Size'])
return data
def collect_relocated_data(syms, bites, rels, ptr_size, reloc_types):
"""Identify relocations for each symbol"""
data = {}
for name, s in sorted(syms.items()):
b = bites.get(name)
assert b is not None
if s['Demangled Name'].startswith('typeinfo name'):
data[name] = [('byte', int(x)) for x in b]
continue
data[name] = []
for i in range(0, len(b), ptr_size):
val = int.from_bytes(b[i*ptr_size:(i + 1)*ptr_size], byteorder='little')
data[name].append(('offset', val))
start = s['Value']
finish = start + s['Size']
# TODO: binary search (bisect)
for rel in rels:
if rel['Type'] in reloc_types and start <= rel['Offset'] < finish:
i = (rel['Offset'] - start) // ptr_size
assert i < len(data[name])
data[name][i] = 'reloc', rel
return data
def generate_vtables(cls_tables, cls_syms, cls_data):
"""Generate code for vtables"""
c_types = {
'reloc' : 'const void *',
'byte' : 'unsigned char',
'offset' : 'size_t'
}
ss = []
ss.append('''\
#ifdef __cplusplus
extern "C" {
#endif
''')
# Print externs
printed = set()
for name, data in sorted(cls_data.items()):
for typ, val in data:
if typ != 'reloc':
continue
sym_name, addend = val['Symbol\'s Name + Addend']
sym_name = re.sub(r'@.*', '', sym_name) # Can we pin version in C?
if sym_name not in cls_syms and sym_name not in printed:
ss.append(f'''\
extern const char {sym_name}[];
''')
# Collect variable infos
code_info = {}
for name, s in sorted(cls_syms.items()):
data = cls_data[name]
if s['Demangled Name'].startswith('typeinfo name'):
declarator = 'const unsigned char %s[]'
else:
field_types = (f'{c_types[typ]} field_{i};' for i, (typ, _) in enumerate(data))
declarator = 'const struct { %s } %%s' % ' '.join(field_types) # pylint: disable=C0209 # consider-using-f-string
vals = []
for typ, val in data:
if typ != 'reloc':
vals.append(str(val) + 'UL')
else:
sym_name, addend = val['Symbol\'s Name + Addend']
sym_name = re.sub(r'@.*', '', sym_name) # Can we pin version in C?
vals.append(f'(const char *)&{sym_name} + {addend}')
code_info[name] = (declarator, '{ %s }' % ', '.join(vals)) # pylint: disable= C0209 # consider-using-f-string
# Print declarations
for name, (decl, _) in sorted(code_info.items()):
type_name = name + '_type'
type_decl = decl % type_name
ss.append(f'''\
typedef {type_decl};
extern __attribute__((weak)) {type_name} {name};
''')
# Print definitions
for name, (_, init) in sorted(code_info.items()):
type_name = name + '_type'
ss.append(f'''\
const {type_name} {name} = {init};
''')
ss.append('''\
#ifdef __cplusplus
} // extern "C"
#endif
''')
return ''.join(ss)
def main():
"""Driver function"""
parser = argparse.ArgumentParser(description="Generate wrappers for shared library functions.",
formatter_class=argparse.RawDescriptionHelpFormatter,
epilog=f"""\
Examples:
$ python3 {me} /usr/lib/x86_64-linux-gnu/libaccountsservice.so.0
Generating libaccountsservice.so.0.tramp.S...
Generating libaccountsservice.so.0.init.c...
""")
parser.add_argument('library',
metavar='LIB',
help="Library to be wrapped.")
parser.add_argument('--verbose', '-v',
help="Print diagnostic info",
action='count',
default=0)
parser.add_argument('--dlopen-callback',
help="Call user-provided custom callback to load library instead of dlopen",
default='')
parser.add_argument('--dlopen',
help="Emit dlopen call (default)",
dest='dlopen', action='store_true', default=True)
parser.add_argument('--no-dlopen',
help="Do not emit dlopen call (user must load library himself)",
dest='dlopen', action='store_false')
parser.add_argument('--library-load-name',
help="Use custom name for dlopened library (default is LIB)")
parser.add_argument('--lazy-load',
help="Load library lazily on first call to one of it's functions (default)",
dest='lazy_load', action='store_true', default=True)
parser.add_argument('--no-lazy-load',
help="Load library eagerly at program start",
dest='lazy_load', action='store_false')
parser.add_argument('--vtables',
help="Intercept virtual tables (EXPERIMENTAL)",
dest='vtables', action='store_true', default=False)
parser.add_argument('--no-vtables',
help="Do not intercept virtual tables (default)",
dest='vtables', action='store_false')
parser.add_argument('--target',
help="Target platform triple e.g. x86_64-unknown-linux-gnu or arm-none-eabi "
"(atm x86_64, i[0-9]86, arm/armhf/armeabi, aarch64/armv8 "
"and e2k are supported)",
default=os.uname()[-1])
parser.add_argument('--symbol-list',
help="Path to file with symbols that should be present in wrapper "
"(all by default)")
parser.add_argument('--symbol-prefix',
metavar='PFX',
help="Prefix wrapper symbols with PFX",
default='')
parser.add_argument('-q', '--quiet',
help="Do not print progress info",
action='store_true')
parser.add_argument('--outdir', '-o',
help="Path to create wrapper at",
default='./')
args = parser.parse_args()
input_name = args.library
verbose = args.verbose
dlopen_callback = args.dlopen_callback
dlopen = args.dlopen
lazy_load = args.lazy_load
load_name = args.library_load_name or os.path.basename(input_name)
if args.target.startswith('arm'):
target = 'arm' # Handle armhf-..., armel-...
elif re.match(r'^i[0-9]86', args.target):
target = 'i386'
else:
target = args.target.split('-')[0]
quiet = args.quiet
outdir = args.outdir
if args.symbol_list is None:
funs = None
else:
with open(args.symbol_list, 'r') as f:
funs = []
for line in re.split(r'\r?\n', f.read()):
line = re.sub(r'#.*', '', line)
line = line.strip()
if line:
funs.append(line)
# Collect target info
target_dir = os.path.join(root, 'arch', target)
if not os.path.exists(target_dir):
error(f"unknown architecture '{target}'")
cfg = configparser.ConfigParser(inline_comment_prefixes=';')
cfg.read(target_dir + '/config.ini')
ptr_size = int(cfg['Arch']['PointerSize'])
symbol_reloc_types = set(re.split(r'\s*,\s*', cfg['Arch']['SymbolReloc']))
def is_exported(s):
return (s['Bind'] != 'LOCAL'
and s['Type'] != 'NOTYPE'
and s['Ndx'] != 'UND'
and s['Name'] not in ['', '_init', '_fini'])
syms = list(filter(is_exported, collect_syms(input_name)))
def is_data_symbol(s):
return (s['Type'] == 'OBJECT'
# Allow vtables if --vtables is on
and not (' for ' in s['Demangled Name'] and args.vtables))
exported_data = [s['Name'] for s in syms if is_data_symbol(s)]
if exported_data:
# TODO: we can generate wrappers for const data without relocations (or only code relocations)
warn(f"library '{input_name}' contains data symbols which won't be intercepted: "
+ ', '.join(exported_data))
# Collect functions
# TODO: warn if user-specified functions are missing
orig_funs = filter(lambda s: s['Type'] == 'FUNC', syms)
all_funs = set()
warn_versioned = False
for s in orig_funs:
if s['Version'] is not None:
# TODO: support versions
if not warn_versioned:
warn(f"library {input_name} contains versioned symbols which are NYI")
warn_versioned = True
if verbose:
print(f"Skipping versioned symbol {s['Name']}")
continue
all_funs.add(s['Name'])
if funs is None:
funs = sorted(list(all_funs))
if not funs and not quiet:
warn(f"no public functions were found in {input_name}")
else:
missing_funs = [name for name in funs if name not in all_funs]
if missing_funs:
warn("some user-specified functions are not present in library: " + ', '.join(missing_funs))
funs = [name for name in funs if name in all_funs]
if verbose:
print("Exported functions:")
for i, fun in enumerate(funs):
print(f" {i}: {fun}")
# Collect vtables
if args.vtables:
cls_tables = {}
cls_syms = {}
for s in syms:
m = re.match(r'^(vtable|typeinfo|typeinfo name) for (.*)', s['Demangled Name'])
if m is not None and is_exported(s):
typ, cls = m.groups()
name = s['Name']
cls_tables.setdefault(cls, {})[typ] = name
cls_syms[name] = s
if verbose:
print("Exported classes:")
for cls, _ in sorted(cls_tables.items()):
print(f" {cls}")
secs = collect_sections(input_name)
if verbose:
print("Sections:")
for sec in secs:
print(f" {sec['Name']}: [{sec['Address']:x}, {sec['Address'] + sec['Size']:x}), "
f"at {sec['Off']:x}")
bites = read_unrelocated_data(input_name, cls_syms, secs)
rels = collect_relocs(input_name)
if verbose:
print("Relocs:")
for rel in rels:
sym_add = rel['Symbol\'s Name + Addend']
print(f" {rel['Offset']}: {sym_add}")
cls_data = collect_relocated_data(cls_syms, bites, rels, ptr_size, symbol_reloc_types)
if verbose:
print("Class data:")
for name, data in sorted(cls_data.items()):
demangled_name = cls_syms[name]['Demangled Name']
print(f" {name} ({demangled_name}):")
for typ, val in data:
print(" " + str(val if typ != 'reloc' else val['Symbol\'s Name + Addend']))
# Generate assembly code
suffix = os.path.basename(load_name)
lib_suffix = re.sub(r'[^a-zA-Z_0-9]+', '_', suffix)
tramp_file = f'{suffix}.tramp.S'
with open(os.path.join(outdir, tramp_file), 'w') as f:
if not quiet:
print(f"Generating {tramp_file}...")
with open(target_dir + '/table.S.tpl', 'r') as t:
table_text = string.Template(t.read()).substitute(
lib_suffix=lib_suffix,
table_size=ptr_size*(len(funs) + 1))
f.write(table_text)
with open(target_dir + '/trampoline.S.tpl', 'r') as t:
tramp_tpl = string.Template(t.read())
for i, name in enumerate(funs):
tramp_text = tramp_tpl.substitute(
lib_suffix=lib_suffix,
sym=args.symbol_prefix + name,
offset=i*ptr_size,
number=i)
f.write(tramp_text)
# Generate C code
init_file = f'{suffix}.init.c'
with open(os.path.join(outdir, init_file), 'w') as f:
if not quiet:
print(f"Generating {init_file}...")
with open(os.path.join(root, 'arch/common/init.c.tpl'), 'r') as t:
if funs:
sym_names = ',\n '.join(f'"{name}"' for name in funs) + ','
else:
sym_names = ''
init_text = string.Template(t.read()).substitute(
lib_suffix=lib_suffix,
load_name=load_name,
dlopen_callback=dlopen_callback,
has_dlopen_callback=int(bool(dlopen_callback)),
no_dlopen=not int(dlopen),
lazy_load=int(lazy_load),
sym_names=sym_names)
f.write(init_text)
if args.vtables:
vtable_text = generate_vtables(cls_tables, cls_syms, cls_data)
f.write(vtable_text)
if __name__ == '__main__':
main()

223
tests/TestRunner/binary_download.py Normal file

@ -0,0 +1,223 @@
#!/usr/bin/env python3
import os
from pathlib import Path
import platform
import shutil
import stat
from urllib import request
import hashlib
from local_cluster import random_secret_string
SUPPORTED_PLATFORMS = ["x86_64", "aarch64"]
SUPPORTED_VERSIONS = [
"7.2.0",
"7.1.9",
"7.1.8",
"7.1.7",
"7.1.6",
"7.1.5",
"7.1.4",
"7.1.3",
"7.1.2",
"7.1.1",
"7.1.0",
"7.0.0",
"6.3.24",
"6.3.23",
"6.3.22",
"6.3.18",
"6.3.17",
"6.3.16",
"6.3.15",
"6.3.13",
"6.3.12",
"6.3.9",
"6.2.30",
"6.2.29",
"6.2.28",
"6.2.27",
"6.2.26",
"6.2.25",
"6.2.24",
"6.2.23",
"6.2.22",
"6.2.21",
"6.2.20",
"6.2.19",
"6.2.18",
"6.2.17",
"6.2.16",
"6.2.15",
"6.2.10",
"6.1.13",
"6.1.12",
"6.1.11",
"6.1.10",
"6.0.18",
"6.0.17",
"6.0.16",
"6.0.15",
"6.0.14",
"5.2.8",
"5.2.7",
"5.1.7",
"5.1.6",
]
FDB_DOWNLOAD_ROOT = "https://github.com/apple/foundationdb/releases/download/"
LOCAL_OLD_BINARY_REPO = "/opt/foundationdb/old/"
CURRENT_VERSION = "7.2.0"
MAX_DOWNLOAD_ATTEMPTS = 5
def make_executable_path(path):
st = os.stat(path)
os.chmod(path, st.st_mode | stat.S_IEXEC)
def compute_sha256(filename):
hash_function = hashlib.sha256()
with open(filename, "rb") as f:
while True:
data = f.read(128 * 1024)
if not data:
break
hash_function.update(data)
return hash_function.hexdigest()
def read_to_str(filename):
with open(filename, "r") as f:
return f.read()
class FdbBinaryDownloader:
def __init__(
self,
build_dir
):
self.build_dir = Path(build_dir).resolve()
assert self.build_dir.exists(), "{} does not exist".format(build_dir)
assert self.build_dir.is_dir(), "{} is not a directory".format(build_dir)
self.platform = platform.machine()
assert self.platform in SUPPORTED_PLATFORMS, "Unsupported platform {}".format(
self.platform
)
self.tmp_dir = self.build_dir.joinpath("tmp", random_secret_string(16))
self.tmp_dir.mkdir(parents=True)
self.download_dir = self.build_dir.joinpath("tmp", "old_binaries")
self.local_binary_repo = Path(LOCAL_OLD_BINARY_REPO)
if not self.local_binary_repo.exists():
self.local_binary_repo = None
# Check if the binaries for the given version are available in the local old binaries repository
def version_in_local_repo(self, version):
return (self.local_binary_repo is not None) and (self.local_binary_repo.joinpath(version).exists())
def binary_path(self, version, bin_name):
if version == CURRENT_VERSION:
return self.build_dir.joinpath("bin", bin_name)
elif self.version_in_local_repo(version):
return self.local_binary_repo.joinpath(version, "bin", "{}-{}".format(bin_name, version))
else:
return self.download_dir.joinpath(version, bin_name)
def lib_dir(self, version):
if version == CURRENT_VERSION:
return self.build_dir.joinpath("lib")
else:
return self.download_dir.joinpath(version)
def lib_path(self, version):
return self.lib_dir(version).joinpath("libfdb_c.so")
# Download an old binary of a given version from a remote repository
def download_old_binary(
self, version, target_bin_name, remote_bin_name, make_executable
):
local_file = self.download_dir.joinpath(version, target_bin_name)
if local_file.exists():
return
# Download to a temporary file and then replace the target file atomically
# to avoid consistency errors in case of multiple tests are downloading the
# same file in parallel
local_file_tmp = Path("{}.{}".format(str(local_file), random_secret_string(8)))
self.download_dir.joinpath(version).mkdir(parents=True, exist_ok=True)
remote_file = "{}{}/{}".format(FDB_DOWNLOAD_ROOT, version, remote_bin_name)
remote_sha256 = "{}.sha256".format(remote_file)
local_sha256 = Path("{}.sha256".format(local_file_tmp))
for attempt_cnt in range(MAX_DOWNLOAD_ATTEMPTS + 1):
if attempt_cnt == MAX_DOWNLOAD_ATTEMPTS:
assert False, "Failed to download {} after {} attempts".format(
local_file_tmp, MAX_DOWNLOAD_ATTEMPTS
)
try:
print("Downloading '{}' to '{}'...".format(remote_file, local_file_tmp))
request.urlretrieve(remote_file, local_file_tmp)
print("Downloading '{}' to '{}'...".format(remote_sha256, local_sha256))
request.urlretrieve(remote_sha256, local_sha256)
print("Download complete")
except Exception as e:
print("Retrying on error:", e)
continue
assert local_file_tmp.exists(), "{} does not exist".format(local_file_tmp)
assert local_sha256.exists(), "{} does not exist".format(local_sha256)
expected_checksum = read_to_str(local_sha256)
actual_checkum = compute_sha256(local_file_tmp)
if expected_checksum == actual_checkum:
print("Checksum OK")
break
print(
"Checksum mismatch. Expected: {} Actual: {}".format(
expected_checksum, actual_checkum
)
)
os.rename(local_file_tmp, local_file)
os.remove(local_sha256)
if make_executable:
make_executable_path(local_file)
# Copy a client library file from the local old binaries repository
# The file needs to be renamed to libfdb_c.so, because it is loaded with this name by fdbcli
def copy_clientlib_from_local_repo(self, version):
dest_lib_file = self.download_dir.joinpath(version, "libfdb_c.so")
if dest_lib_file.exists():
return
# Avoid race conditions in case of parallel test execution by first copying to a temporary file
# and then renaming it atomically
dest_file_tmp = Path("{}.{}".format(str(dest_lib_file), random_secret_string(8)))
src_lib_file = self.local_binary_repo.joinpath(version, "lib", "libfdb_c-{}.so".format(version))
assert src_lib_file.exists(), "Missing file {} in the local old binaries repository".format(src_lib_file)
self.download_dir.joinpath(version).mkdir(parents=True, exist_ok=True)
shutil.copyfile(src_lib_file, dest_file_tmp)
os.rename(dest_file_tmp, dest_lib_file)
assert dest_lib_file.exists(), "{} does not exist".format(dest_lib_file)
# Download all old binaries required for testing the specified upgrade path
def download_old_binaries(self, version):
if version == CURRENT_VERSION:
return
if self.version_in_local_repo(version):
self.copy_clientlib_from_local_repo(version)
return
self.download_old_binary(
version, "fdbserver", "fdbserver.{}".format(self.platform), True
)
self.download_old_binary(
version, "fdbmonitor", "fdbmonitor.{}".format(self.platform), True
)
self.download_old_binary(
version, "fdbcli", "fdbcli.{}".format(self.platform), True
)
self.download_old_binary(
version, "libfdb_c.so", "libfdb_c.{}.so".format(self.platform), False
)

231
tests/TestRunner/upgrade_test.py

@ -4,99 +4,24 @@ from argparse import ArgumentParser, RawDescriptionHelpFormatter
import glob
import os
from pathlib import Path
import platform
import random
import shutil
import stat
import subprocess
import sys
from threading import Thread, Event
import traceback
import time
from urllib import request
import hashlib
from binary_download import FdbBinaryDownloader, SUPPORTED_VERSIONS, CURRENT_VERSION
from local_cluster import LocalCluster, random_secret_string
SUPPORTED_PLATFORMS = ["x86_64"]
SUPPORTED_VERSIONS = [
"7.2.0",
"7.1.9",
"7.1.8",
"7.1.7",
"7.1.6",
"7.1.5",
"7.1.4",
"7.1.3",
"7.1.2",
"7.1.1",
"7.1.0",
"7.0.0",
"6.3.24",
"6.3.23",
"6.3.22",
"6.3.18",
"6.3.17",
"6.3.16",
"6.3.15",
"6.3.13",
"6.3.12",
"6.3.9",
"6.2.30",
"6.2.29",
"6.2.28",
"6.2.27",
"6.2.26",
"6.2.25",
"6.2.24",
"6.2.23",
"6.2.22",
"6.2.21",
"6.2.20",
"6.2.19",
"6.2.18",
"6.2.17",
"6.2.16",
"6.2.15",
"6.2.10",
"6.1.13",
"6.1.12",
"6.1.11",
"6.1.10",
"6.0.18",
"6.0.17",
"6.0.16",
"6.0.15",
"6.0.14",
"5.2.8",
"5.2.7",
"5.1.7",
"5.1.6",
]
CLUSTER_ACTIONS = ["wiggle"]
FDB_DOWNLOAD_ROOT = "https://github.com/apple/foundationdb/releases/download/"
LOCAL_OLD_BINARY_REPO = "/opt/foundationdb/old/"
CURRENT_VERSION = "7.2.0"
HEALTH_CHECK_TIMEOUT_SEC = 5
PROGRESS_CHECK_TIMEOUT_SEC = 30
TESTER_STATS_INTERVAL_SEC = 5
TRANSACTION_RETRY_LIMIT = 100
MAX_DOWNLOAD_ATTEMPTS = 5
RUN_WITH_GDB = False
def make_executable_path(path):
st = os.stat(path)
os.chmod(path, st.st_mode | stat.S_IEXEC)
def remove_file_no_fail(filename):
try:
os.remove(filename)
except OSError:
pass
def version_from_str(ver_str):
ver = [int(s) for s in ver_str.split(".")]
assert len(ver) == 3, "Invalid version string {}".format(ver_str)
@ -118,23 +43,6 @@ def random_sleep(min_sec, max_sec):
time.sleep(time_sec)
def compute_sha256(filename):
hash_function = hashlib.sha256()
with open(filename, "rb") as f:
while True:
data = f.read(128 * 1024)
if not data:
break
hash_function.update(data)
return hash_function.hexdigest()
def read_to_str(filename):
with open(filename, "r") as f:
return f.read()
class UpgradeTest:
def __init__(
self,
@ -143,28 +51,23 @@ class UpgradeTest:
self.build_dir = Path(args.build_dir).resolve()
assert self.build_dir.exists(), "{} does not exist".format(args.build_dir)
assert self.build_dir.is_dir(), "{} is not a directory".format(args.build_dir)
self.tester_bin = self.build_dir.joinpath("bin", "fdb_c_api_tester")
assert self.tester_bin.exists(), "{} does not exist".format(self.tester_bin)
self.upgrade_path = args.upgrade_path
self.used_versions = set(self.upgrade_path).difference(set(CLUSTER_ACTIONS))
for version in self.used_versions:
assert version in SUPPORTED_VERSIONS, "Unsupported version or cluster action {}".format(version)
self.platform = platform.machine()
assert self.platform in SUPPORTED_PLATFORMS, "Unsupported platform {}".format(
self.platform
)
self.tmp_dir = self.build_dir.joinpath("tmp", random_secret_string(16))
self.tmp_dir.mkdir(parents=True)
self.download_dir = self.build_dir.joinpath("tmp", "old_binaries")
self.local_binary_repo = Path(LOCAL_OLD_BINARY_REPO)
if not self.local_binary_repo.exists():
self.local_binary_repo = None
self.downloader = FdbBinaryDownloader(args.build_dir)
self.download_old_binaries()
self.create_external_lib_dir()
init_version = self.upgrade_path[0]
self.cluster = LocalCluster(
self.tmp_dir,
self.binary_path(init_version, "fdbserver"),
self.binary_path(init_version, "fdbmonitor"),
self.binary_path(init_version, "fdbcli"),
self.downloader.binary_path(init_version, "fdbserver"),
self.downloader.binary_path(init_version, "fdbmonitor"),
self.downloader.binary_path(init_version, "fdbcli"),
args.process_number,
create_config=False,
redundancy=args.redundancy
@ -187,116 +90,12 @@ class UpgradeTest:
self.tester_retcode = None
self.tester_proc = None
self.output_pipe = None
self.tester_bin = None
self.ctrl_pipe = None
# Check if the binaries for the given version are available in the local old binaries repository
def version_in_local_repo(self, version):
return (self.local_binary_repo is not None) and (self.local_binary_repo.joinpath(version).exists())
def binary_path(self, version, bin_name):
if version == CURRENT_VERSION:
return self.build_dir.joinpath("bin", bin_name)
elif self.version_in_local_repo(version):
return self.local_binary_repo.joinpath(version, "bin", "{}-{}".format(bin_name, version))
else:
return self.download_dir.joinpath(version, bin_name)
def lib_dir(self, version):
if version == CURRENT_VERSION:
return self.build_dir.joinpath("lib")
else:
return self.download_dir.joinpath(version)
# Download an old binary of a given version from a remote repository
def download_old_binary(
self, version, target_bin_name, remote_bin_name, make_executable
):
local_file = self.download_dir.joinpath(version, target_bin_name)
if local_file.exists():
return
# Download to a temporary file and then replace the target file atomically
# to avoid consistency errors in case of multiple tests are downloading the
# same file in parallel
local_file_tmp = Path("{}.{}".format(str(local_file), random_secret_string(8)))
self.download_dir.joinpath(version).mkdir(parents=True, exist_ok=True)
remote_file = "{}{}/{}".format(FDB_DOWNLOAD_ROOT, version, remote_bin_name)
remote_sha256 = "{}.sha256".format(remote_file)
local_sha256 = Path("{}.sha256".format(local_file_tmp))
for attempt_cnt in range(MAX_DOWNLOAD_ATTEMPTS + 1):
if attempt_cnt == MAX_DOWNLOAD_ATTEMPTS:
assert False, "Failed to download {} after {} attempts".format(
local_file_tmp, MAX_DOWNLOAD_ATTEMPTS
)
try:
print("Downloading '{}' to '{}'...".format(remote_file, local_file_tmp))
request.urlretrieve(remote_file, local_file_tmp)
print("Downloading '{}' to '{}'...".format(remote_sha256, local_sha256))
request.urlretrieve(remote_sha256, local_sha256)
print("Download complete")
except Exception as e:
print("Retrying on error:", e)
continue
assert local_file_tmp.exists(), "{} does not exist".format(local_file_tmp)
assert local_sha256.exists(), "{} does not exist".format(local_sha256)
expected_checksum = read_to_str(local_sha256)
actual_checkum = compute_sha256(local_file_tmp)
if expected_checksum == actual_checkum:
print("Checksum OK")
break
print(
"Checksum mismatch. Expected: {} Actual: {}".format(
expected_checksum, actual_checkum
)
)
os.rename(local_file_tmp, local_file)
os.remove(local_sha256)
if make_executable:
make_executable_path(local_file)
# Copy a client library file from the local old binaries repository
# The file needs to be renamed to libfdb_c.so, because it is loaded with this name by fdbcli
def copy_clientlib_from_local_repo(self, version):
dest_lib_file = self.download_dir.joinpath(version, "libfdb_c.so")
if dest_lib_file.exists():
return
# Avoid race conditions in case of parallel test execution by first copying to a temporary file
# and then renaming it atomically
dest_file_tmp = Path("{}.{}".format(str(dest_lib_file), random_secret_string(8)))
src_lib_file = self.local_binary_repo.joinpath(version, "lib", "libfdb_c-{}.so".format(version))
assert src_lib_file.exists(), "Missing file {} in the local old binaries repository".format(src_lib_file)
self.download_dir.joinpath(version).mkdir(parents=True, exist_ok=True)
shutil.copyfile(src_lib_file, dest_file_tmp)
os.rename(dest_file_tmp, dest_lib_file)
assert dest_lib_file.exists(), "{} does not exist".format(dest_lib_file)
# Download all old binaries required for testing the specified upgrade path
def download_old_binaries(self):
for version in self.used_versions:
if version == CURRENT_VERSION:
continue
if self.version_in_local_repo(version):
self.copy_clientlib_from_local_repo(version)
continue
self.download_old_binary(
version, "fdbserver", "fdbserver.{}".format(self.platform), True
)
self.download_old_binary(
version, "fdbmonitor", "fdbmonitor.{}".format(self.platform), True
)
self.download_old_binary(
version, "fdbcli", "fdbcli.{}".format(self.platform), True
)
self.download_old_binary(
version, "libfdb_c.so", "libfdb_c.{}.so".format(self.platform), False
)
self.downloader.download_old_binaries(version)
# Create a directory for external client libraries for MVC and fill it
# with the libraries necessary for the specified upgrade path
@ -304,7 +103,7 @@ class UpgradeTest:
self.external_lib_dir = self.tmp_dir.joinpath("client_libs")
self.external_lib_dir.mkdir(parents=True)
for version in self.used_versions:
src_file_path = self.lib_dir(version).joinpath("libfdb_c.so")
src_file_path = self.downloader.lib_path(version)
assert src_file_path.exists(), "{} does not exist".format(src_file_path)
target_file_path = self.external_lib_dir.joinpath(
"libfdb_c.{}.so".format(version)
@ -344,10 +143,10 @@ class UpgradeTest:
# Create and save a cluster configuration for the given version
def configure_version(self, version):
self.cluster.fdbmonitor_binary = self.binary_path(version, "fdbmonitor")
self.cluster.fdbserver_binary = self.binary_path(version, "fdbserver")
self.cluster.fdbcli_binary = self.binary_path(version, "fdbcli")
self.cluster.set_env_var = "LD_LIBRARY_PATH", self.lib_dir(version)
self.cluster.fdbmonitor_binary = self.downloader.binary_path(version, "fdbmonitor")
self.cluster.fdbserver_binary = self.downloader.binary_path(version, "fdbserver")
self.cluster.fdbcli_binary = self.downloader.binary_path(version, "fdbcli")
self.cluster.set_env_var("LD_LIBRARY_PATH", self.downloader.lib_dir(version))
if version_before(version, "7.1.0"):
self.cluster.use_legacy_conf_syntax = True
self.cluster.save_config()
@ -406,7 +205,7 @@ class UpgradeTest:
"--transaction-retry-limit",
str(TRANSACTION_RETRY_LIMIT),
"--stats-interval",
str(TESTER_STATS_INTERVAL_SEC*1000)
str(TESTER_STATS_INTERVAL_SEC * 1000)
]
if RUN_WITH_GDB:
cmd_args = ["gdb", "-ex", "run", "--args"] + cmd_args
@ -500,8 +299,6 @@ class UpgradeTest:
# - Start a thread for reading notifications from the tester
# - Trigger the upgrade steps and checks in the main thread
def exec_test(self, args):
self.tester_bin = self.build_dir.joinpath("bin", "fdb_c_api_tester")
assert self.tester_bin.exists(), "{} does not exist".format(self.tester_bin)
self.tester_proc = None
test_retcode = 1
try: