diff --git a/CMakeLists.txt b/CMakeLists.txt index 55d92d0591..96181a1111 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -201,9 +201,9 @@ configure_file(${CMAKE_CURRENT_SOURCE_DIR}/fdbclient/BuildFlags.h.in ${CMAKE_CUR if (CMAKE_EXPORT_COMPILE_COMMANDS AND WITH_PYTHON) add_custom_command( OUTPUT ${CMAKE_CURRENT_SOURCE_DIR}/compile_commands.json - COMMAND $<TARGET_FILE:Python::Interpreter> ${CMAKE_CURRENT_SOURCE_DIR}/build/gen_compile_db.py + COMMAND $<TARGET_FILE:Python::Interpreter> contrib/gen_compile_db.py ARGS -b ${CMAKE_CURRENT_BINARY_DIR} -s ${CMAKE_CURRENT_SOURCE_DIR} -o ${CMAKE_CURRENT_SOURCE_DIR}/compile_commands.json ${CMAKE_CURRENT_BINARY_DIR}/compile_commands.json - DEPENDS ${CMAKE_CURRENT_SOURCE_DIR}/build/gen_compile_db.py ${CMAKE_CURRENT_BINARY_DIR}/compile_commands.json + DEPENDS contrib/gen_compile_db.py ${CMAKE_CURRENT_BINARY_DIR}/compile_commands.json COMMENT "Build compile commands for IDE" ) add_custom_target(processed_compile_commands ALL DEPENDS ${CMAKE_CURRENT_SOURCE_DIR}/compile_commands.json ${CMAKE_CURRENT_BINARY_DIR}/compile_commands.json) diff --git a/build/Dockerfile.windows b/build/Dockerfile.windows deleted file mode 100644 index 0cca3c6f17..0000000000 --- a/build/Dockerfile.windows +++ /dev/null @@ -1,65 +0,0 @@ -# escape=` -ARG IMAGE_TAG=0.1.0 - -# Use the latest Windows Server Core image with .NET Framework 4.8. -FROM mcr.microsoft.com/dotnet/framework/sdk:4.8-windowsservercore-ltsc2019 - -# Restore the default Windows shell for correct batch processing. -SHELL ["cmd", "/S", "/C"] - -# Download the Build Tools bootstrapper. -ADD https://aka.ms/vs/16/release/vs_buildtools.exe C:\TEMP\vs_buildtools.exe - -# Install Build Tools with the Microsoft.VisualStudio.Workload.AzureBuildTools workload, excluding workloads and components with known issues. -RUN C:\TEMP\vs_buildtools.exe --quiet --wait --norestart --nocache ` - --installPath C:\BuildTools ` - --add Microsoft.VisualStudio.Workload.VCTools ` - --add Microsoft.VisualStudio.Component.TestTools.BuildTools ` - --add Microsoft.VisualStudio.Component.VC.ASAN ` - --add Microsoft.VisualStudio.Component.VC.CMake.Project ` - --add Microsoft.VisualStudio.Component.VC.Tools.x86.x64 ` - --add Microsoft.VisualStudio.Component.Windows10SDK.18362 ` - --add Microsoft.VisualStudio.Component.VC.Llvm.ClangToolset ` - --add Microsoft.VisualStudio.Component.VC.Llvm.Clang ` - || IF "%ERRORLEVEL%"=="3010" EXIT 0 - -# Install Choco package manager -RUN powershell -Command "iwr https://chocolatey.org/install.ps1 -UseBasicParsing | iex" -RUN powershell -Command choco install 7zip -y - -# Download boost -# We want to make this as early as possible as downloading and unpacking boost takes a very long time -RUN mkdir C:\Downloads && ` - powershell -Command iwr -Uri https://dl.bintray.com/boostorg/release/1.72.0/source/boost_1_72_0.7z -OutFile C:\Downloads\boost_1_72_0.7z && ` - powershell -Command (Get-FileHash C:\Downloads\boost_1_72_0.7z).Hash -eq \"247a91dd7e4d9dd3c4b954b532fbc167ba62dc15ab834e5ad893d7c3f9eb5f0f\" && ` - cd \Downloads && "C:\Program Files\7-Zip\7z" x boost_1_72_0.7z && del boost_1_72_0.7z - -# install other dependencies -RUN powershell -Command choco install cmake openjdk12 python -y - -# add cmake to the path -RUN setx path "%path%;C:\Program Files\CMake\bin;C:\Program Files\7-Zip;C:\Program Files (x86)\Microsoft Visual Studio\installer" - -# Compile boost context -RUN C:\BuildTools\Common7\Tools\VsDevCmd.bat && ` - cd \Downloads\boost_1_72_0 && ` - .\bootstrap.bat && ` - powershell -Command ` - .\b2 variant=release address-model=64 architecture=x86 link=static --with-context --build-type=minimal --layout=system ` - -j((Get-WmiObject -Class Win32_Processor).NumberOfLogicalProcessors) - -# CMake's find_package wouldn't be able to find this otherwise -RUN setx CMAKE_PREFIX_PATH "C:\Downloads\boost_1_72_0\stage\" - -LABEL version=${IMAGE_TAG} -ENV DOCKER_IMAGEVER=${IMAGE_TAG} - -# Enable Windows Update Service (which is required to get .Net Core which is a dependency for wix) and install .Net framework -RUN powershell "Set-Service -Name wuauserv -StartupType Manual; Install-WindowsFeature -Name NET-Framework-Features -Verbose" - -# Install WIX -RUN powershell -Command choco install wixtoolset --version 3.11.2 -y - -# Define the entry point for the docker container. -# This entry point starts the developer command prompt and launches the PowerShell shell. -ENTRYPOINT ["C:\\BuildTools\\Common7\\Tools\\VsDevCmd.bat", "&&", "powershell.exe", "-NoLogo", "-ExecutionPolicy", "Bypass"] \ No newline at end of file diff --git a/build/Dockerfile.windows.devel b/build/Dockerfile.windows.devel deleted file mode 100644 index d45f4b173d..0000000000 --- a/build/Dockerfile.windows.devel +++ /dev/null @@ -1,9 +0,0 @@ -# Use the latest FoundationDB Windows Build image -FROM foundationdb/foundationdb-build-windows:0.1.0 - -# Install git -RUN powershell -Command choco install git -y - -# Define the entry point for the docker container. -# This entry point starts the developer command prompt and launches the PowerShell shell. -ENTRYPOINT ["C:\\BuildTools\\Common7\\Tools\\VsDevCmd.bat", "&&", "powershell.exe", "-NoLogo", "-ExecutionPolicy", "Bypass"] \ No newline at end of file diff --git a/build/artifacts/.gitkeep b/build/artifacts/.gitkeep deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/build/build.ps1 b/build/build.ps1 deleted file mode 100644 index 371a1f7214..0000000000 --- a/build/build.ps1 +++ /dev/null @@ -1,63 +0,0 @@ -param ( - [string]$SourceDir = (Resolve-Path "$PSScriptRoot\.."), - [string]$ImageName = "fdb-windows", - # By default we want to leave one CPU core for the OS so the user has some minimal control over the system - [string]$Cpus = (Get-CimInstance -ClassName Win32_Processor -Filter "DeviceID='CPU0'").NumberOfLogicalProcessors - 2, - # We want to leave at least 1GB of memory for the OS - [string]$Memory = (Get-CimInstance -ClassName Win32_ComputerSystem).TotalPhysicalMemory - 2*[Math]::Pow(2, 30), - [Parameter(Mandatory=$true)][string]$BuildDir, - [switch]$DryRun = $false, - [switch]$ForceConfigure = $false, - [switch]$SkipDockerBuild = $false, - [Parameter(Position=0)][string]$Target = "installer" -) - -$BuildDir = Resolve-Path $BuildDir -# we don't want a trailing \ in the build dir -if ($BuildDir.EndsWith("\")) { - $BuildDir = $BuildDir.Substring(0, $BuildDir.Length - 1) -} -$exponent = 0 -$Memory = $Memory.ToUpper() -if ($Memory.EndsWith("K")) { - $exponent = 10 -} elseif ($Memory.EndsWith("M")) { - $exponent = 20 -} elseif ($Memory.EndsWith("G")) { - $exponent = 30 -} elseif ($Memory.EndsWith("T")) { - $exponent = 40 -} -if ($exponent -gt 0) { - $Memory = $Memory.Substring(0, $Memory.Length - 1) * [Math]::Pow(2, $exponent) -} - -$buildCommand = [string]::Format("Get-Content {0}\build\Dockerfile.windows.devel | docker build -t {1} -m {2} -", - $SourceDir, $ImageName, [Math]::Min(16 * [Math]::Pow(2, 30), $Memory)) -if ($DryRun -and !$SkipDockerBuild) { - Write-Output $buildCommand -} elseif (!$SkipDockerBuild) { - Invoke-Expression -Command $buildCommand -} - -# Write build instructions into file -$cmdFile = "docker_command.ps1" -$batFile = "$BuildDir\$cmdFile" -$batFileDocker = "C:\fdbbuild\$cmdFile" -# "C:\BuildTools\Common7\Tools\VsDevCmd.bat" | Out-File $batFile -"cd \fdbbuild" | Out-File -Append $batFile -if ($ForceConfigure -or ![System.IO.File]::Exists("$BuildDir\CMakeCache.txt") -or ($Target -eq "configure")) { - "cmake -G ""Visual Studio 16 2019"" -A x64 -T""ClangCL"" -S C:\foundationdb -B C:\fdbbuild --debug-trycompile" | Out-File -Append $batFile -} -if ($Target -ne "configure") { - "msbuild /p:CL_MPCount=$Cpus /m /p:UseMultiToolTask=true /p:Configuration=Release foundationdb.sln" | Out-File -Append $batFile -} - -$dockerCommand = "powershell.exe -NoLogo -ExecutionPolicy Bypass -File $batFileDocker" -$runCommand = [string]::Format("docker run -v {0}:C:\foundationdb -v {1}:C:\fdbbuild --name fdb-build -m {2} --cpus={3} --rm {4} ""{5}""", - $SourceDir, $BuildDir, $Memory, $Cpus, $ImageName, $dockerCommand); -if ($DryRun) { - Write-Output $runCommand -} else { - Invoke-Expression $runCommand -} \ No newline at end of file diff --git a/build/cmake/Dockerfile b/build/cmake/Dockerfile deleted file mode 100644 index 0452606a1f..0000000000 --- a/build/cmake/Dockerfile +++ /dev/null @@ -1,44 +0,0 @@ -FROM centos:6 -LABEL version=0.0.4 - -RUN yum install -y yum-utils -RUN yum-config-manager --enable rhel-server-rhscl-7-rpms -RUN yum -y install centos-release-scl -RUN yum install -y devtoolset-7 - -# install cmake -RUN curl -L https://github.com/Kitware/CMake/releases/download/v3.13.4/cmake-3.13.4-Linux-x86_64.tar.gz > /tmp/cmake.tar.gz &&\ - echo "563a39e0a7c7368f81bfa1c3aff8b590a0617cdfe51177ddc808f66cc0866c76 /tmp/cmake.tar.gz" > /tmp/cmake-sha.txt &&\ - sha256sum -c /tmp/cmake-sha.txt &&\ - cd /tmp && tar xf cmake.tar.gz && cp -r cmake-3.13.4-Linux-x86_64/* /usr/local/ - -# install boost -RUN curl -L https://boostorg.jfrog.io/artifactory/main/release/1.67.0/source/boost_1_67_0.tar.bz2 > /tmp/boost.tar.bz2 &&\ - cd /tmp && echo "2684c972994ee57fc5632e03bf044746f6eb45d4920c343937a465fd67a5adba boost.tar.bz2" > boost-sha.txt &&\ - sha256sum -c boost-sha.txt && tar xf boost.tar.bz2 && cp -r boost_1_72_0/boost /usr/local/include/ &&\ - rm -rf boost.tar.bz2 boost_1_72_0 - -# install mono (for actorcompiler) -RUN yum install -y epel-release -RUN yum install -y mono-core - -# install Java -RUN yum install -y java-1.8.0-openjdk-devel - -# install LibreSSL -RUN curl https://ftp.openbsd.org/pub/OpenBSD/LibreSSL/libressl-2.8.2.tar.gz > /tmp/libressl.tar.gz &&\ - cd /tmp && echo "b8cb31e59f1294557bfc80f2a662969bc064e83006ceef0574e2553a1c254fd5 libressl.tar.gz" > libressl-sha.txt &&\ - sha256sum -c libressl-sha.txt && tar xf libressl.tar.gz &&\ - cd libressl-2.8.2 && cd /tmp/libressl-2.8.2 && scl enable devtoolset-7 -- ./configure --prefix=/usr/local/stow/libressl CFLAGS="-fPIC -O3" --prefix=/usr/local &&\ - cd /tmp/libressl-2.8.2 && scl enable devtoolset-7 -- make -j`nproc` install &&\ - rm -rf /tmp/libressl-2.8.2 /tmp/libressl.tar.gz - - -# install dependencies for bindings and documentation -# python 2.7 is required for the documentation -RUN yum install -y rh-python36-python-devel rh-ruby24 golang python27 - -# install packaging tools -RUN yum install -y rpm-build debbuild - -CMD scl enable devtoolset-7 python27 rh-python36 rh-ruby24 -- bash diff --git a/build/cmake/build.sh b/build/cmake/build.sh deleted file mode 100644 index ff02e78080..0000000000 --- a/build/cmake/build.sh +++ /dev/null @@ -1,279 +0,0 @@ -#!/usr/bin/env bash - -arguments_usage() { - cat <<EOF -usage: build.sh [-h] [commands] - -h: print this help message and - abort execution - - Will execute the passed commands - in the order they were passed -EOF -} - -arguments_parse() { - local __res=0 - while getopts ":h" opt - do - case ${opt} in - h ) - arguments_usage - __res=2 - break - ;; - \? ) - echo "Unknown option ${opt}" - arguments_usage - __res=1 - break - ;; - esac - done - shift $((OPTIND -1)) - commands=("$@") - return ${__res} -} - -configure() { - local __res=0 - for _ in 1 - do - cmake ../foundationdb ${CMAKE_EXTRA_ARGS} - __res=$? - if [ ${__res} -ne 0 ] - then - break - fi - done - return ${__res} -} - -build_fast() { - local __res=0 - for _ in 1 - do - make -j`nproc` - __res=$? - if [ ${__res} -ne 0 ] - then - break - fi - done - return ${__res} -} - -build() { - local __res=0 - for _ in 1 - do - configure - __res=$? - if [ ${__res} -ne 0 ] - then - break - fi - build_fast - __res=$? - if [ ${__res} -ne 0 ] - then - break - fi - done - return ${__res} -} - -package_fast() { - local __res=0 - for _ in 1 - do - make -j`nproc` packages - cpack - cpack -G RPM -D GENERATE_EL6=ON - __res=$? - if [ ${__res} -ne 0 ] - then - break - fi - done - return ${__res} -} - -package() { - local __res=0 - for _ in 1 - do - build - __res=$? - if [ ${__res} -ne 0 ] - then - break - fi - package_fast - __res=$? - if [ ${__res} -ne 0 ] - then - break - fi - done - return ${__res} -} - -rpm() { - local __res=0 - for _ in 1 - do - configure - __res=$? - if [ ${__res} -ne 0 ] - then - break - fi - build_fast - __res=$? - if [ ${__res} -ne 0 ] - then - break - fi - fakeroot cpack -G RPM -D GENERATE_EL6=ON - fakeroot cpack -G RPM - __res=$? - if [ ${__res} -ne 0 ] - then - break - fi - done - return ${__res} -} - -deb() { - local __res=0 - for _ in 1 - do - configure - __res=$? - if [ ${__res} -ne 0 ] - then - break - fi - build_fast - __res=$? - if [ ${__res} -ne 0 ] - then - break - fi - fakeroot cpack -G DEB - __res=$? - if [ ${__res} -ne 0 ] - then - break - fi - done - return ${__res} -} - -test-fast() { - local __res=0 - for _ in 1 - do - ctest -j`nproc` ${CTEST_EXTRA_ARGS} - __res=$? - if [ ${__res} -ne 0 ] - then - break - fi - done - return ${__res} -} - -test() { - local __res=0 - for _ in 1 - do - build - __res=$? - if [ ${__res} -ne 0 ] - then - break - fi - test-fast - __res=$? - if [ ${__res} -ne 0 ] - then - break - fi - done - return ${__res} -} - -main() { - local __res=0 - for _ in 1 - do - arguments_parse "$@" - __res=$? - if [ ${__res} -ne 0 ] - then - if [ ${__res} -eq 2 ] - then - # in this case there was no error - # We still want to exit the script - __res=0 - fi - break - fi - echo "Num commands ${#commands[@]}" - for command in "${commands[@]}" - do - echo "Command: ${command}" - case ${command} in - configure ) - configure - __res=$? - ;; - build ) - build - __res=$? - ;; - build/fast ) - build_fast - __res=$? - ;; - package ) - package - __res=$? - ;; - package/fast ) - package_fast - __res=$? - ;; - rpm ) - rpm - __res=$? - ;; - deb ) - deb - __res=$? - ;; - test-fast) - test-fast - __res=$? - ;; - test) - test - __res=$? - ;; - * ) - echo "ERROR: Command not found ($command)" - __res=1 - ;; - esac - if [ ${__res} -ne 0 ] - then - break - fi - done - done - return ${__res} -} - -main "$@" diff --git a/build/cmake/centos6-test/Dockerfile b/build/cmake/centos6-test/Dockerfile deleted file mode 100644 index a12d3cd2a1..0000000000 --- a/build/cmake/centos6-test/Dockerfile +++ /dev/null @@ -1,3 +0,0 @@ -FROM centos:6 - -RUN yum install -y yum-utils upstart initscripts diff --git a/build/cmake/centos7-test/Dockerfile b/build/cmake/centos7-test/Dockerfile deleted file mode 100644 index 2be4b61f06..0000000000 --- a/build/cmake/centos7-test/Dockerfile +++ /dev/null @@ -1,3 +0,0 @@ -FROM centos:7 - -RUN yum install -y yum-utils systemd sysvinit-tools diff --git a/build/cmake/debian-test/Dockerfile b/build/cmake/debian-test/Dockerfile deleted file mode 100644 index 5d42b627c0..0000000000 --- a/build/cmake/debian-test/Dockerfile +++ /dev/null @@ -1,3 +0,0 @@ -FROM ubuntu:16.04 - -RUN apt-get update && apt-get install -y systemd python diff --git a/build/cmake/docker-compose.yml b/build/cmake/docker-compose.yml deleted file mode 100644 index 90edaeeac3..0000000000 --- a/build/cmake/docker-compose.yml +++ /dev/null @@ -1,65 +0,0 @@ -version: "3" - -services: - - common: &common - image: foundationdb-build:0.0.4 - build: - context: . - dockerfile: Dockerfile - - build-setup: &build-setup - <<: *common - depends_on: [common] - #debuginfo builds need the build path to be longer than - #the path where debuginfo sources are places. Crazy, yes, - #see the manual for CPACK_RPM_BUILD_SOURCE_DIRS_PREFIX. - volumes: - - ../..:/foundationdb/deep/directory/as/debuginfo/doesnt/work/otherwise/foundationdb - - ${BUILDDIR}:/foundationdb/deep/directory/as/debuginfo/doesnt/work/otherwise/build - working_dir: /foundationdb/deep/directory/as/debuginfo/doesnt/work/otherwise/build - - configure: &configure - <<: *build-setup - command: scl enable devtoolset-7 rh-ruby24 rh-python36 python27 -- bash ../foundationdb/build/cmake/build.sh configure - - build: &build - <<: *build-setup - command: scl enable devtoolset-7 rh-ruby24 rh-python36 python27 -- bash ../foundationdb/build/cmake/build.sh build - - build-fast: &build-fast - <<: *build-setup - command: scl enable devtoolset-7 rh-ruby24 rh-python36 python27 -- bash ../foundationdb/build/cmake/build.sh build/fast - - rpm: &rpm - <<: *build-setup - command: scl enable devtoolset-7 rh-ruby24 rh-python36 python27 -- bash ../foundationdb/build/cmake/build.sh rpm - - deb: &deb - <<: *build-setup - command: scl enable devtoolset-7 rh-ruby24 rh-python36 python27 -- bash ../foundationdb/build/cmake/build.sh deb - - package: &package - <<: *build-setup - command: scl enable devtoolset-7 rh-ruby24 rh-python36 python27 -- bash ../foundationdb/build/cmake/build.sh package - - package-fast: &package-fast - <<: *build-setup - command: scl enable devtoolset-7 rh-ruby24 rh-python36 python27 -- bash ../foundationdb/build/cmake/build.sh package/fast - - test-fast: &test-fast - <<: *build-setup - command: scl enable devtoolset-7 rh-ruby24 rh-python36 python27 -- bash ../foundationdb/build/cmake/build.sh test-fast - - test: &test - <<: *build-setup - command: scl enable devtoolset-7 rh-ruby24 rh-python36 python27 -- bash ../foundationdb/build/cmake/build.sh test - - snowflake-ci: &snowflake-ci - <<: *build-setup - command: scl enable devtoolset-7 rh-ruby24 rh-python36 python27 -- bash ../foundationdb/build/cmake/build.sh package test-fast - - shell: - <<: *build-setup - command: scl enable devtoolset-7 rh-ruby24 rh-python36 python27 -- bash - diff --git a/build/cmake/docker.ini b/build/cmake/docker.ini deleted file mode 100644 index 367644b503..0000000000 --- a/build/cmake/docker.ini +++ /dev/null @@ -1,17 +0,0 @@ -[centos6] -name = fdb-centos6 -location = centos6-test -packages = ^.*el6((?!debuginfo).)*\.rpm$ -format = rpm - -[centos7] -name = fdb-centos7 -location = centos7-test -packages = ^.*el7((?!debuginfo).)*\.rpm$ -format = rpm - -[ubuntu_16_04] -name = fdb-debian -location = debian-test -packages = ^.*\.deb$ -format = deb diff --git a/build/cmake/package_tester/deb_tests.sh b/build/cmake/package_tester/deb_tests.sh deleted file mode 100644 index 0dc85d005b..0000000000 --- a/build/cmake/package_tester/deb_tests.sh +++ /dev/null @@ -1,32 +0,0 @@ -#!/usr/bin/env bash - -source_dir="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null && pwd )" - -source ${source_dir}/modules/globals.sh -source ${source_dir}/modules/util.sh -source ${source_dir}/modules/deb.sh -source ${source_dir}/modules/tests.sh -source ${source_dir}/modules/test_args.sh - -main() { - local __res=0 - enterfun - for _ in 1 - do - test_args_parse "$@" - __res=$? - if [ ${__res} -eq 2 ] - then - __res=0 - break - elif [ ${__res} -ne 0 ] - then - break - fi - tests_main - done - exitfun - return ${__res} -} - -main "$@" diff --git a/build/cmake/package_tester/fdb_c_app/CMakeLists.txt b/build/cmake/package_tester/fdb_c_app/CMakeLists.txt deleted file mode 100644 index 60ed25f0ca..0000000000 --- a/build/cmake/package_tester/fdb_c_app/CMakeLists.txt +++ /dev/null @@ -1,5 +0,0 @@ -cmake_minimum_required(VERSION 2.8.0) -project(fdb_c_app C) -find_package(FoundationDB-Client REQUIRED) -add_executable(app app.c) -target_link_libraries(app PRIVATE fdb_c) diff --git a/build/cmake/package_tester/fdb_c_app/app.c b/build/cmake/package_tester/fdb_c_app/app.c deleted file mode 100644 index 6fe24068f9..0000000000 --- a/build/cmake/package_tester/fdb_c_app/app.c +++ /dev/null @@ -1,7 +0,0 @@ -#define FDB_API_VERSION 710 -#include <foundationdb/fdb_c.h> - -int main(int argc, char* argv[]) { - fdb_select_api_version(710); - return 0; -} diff --git a/build/cmake/package_tester/modules/arguments.sh b/build/cmake/package_tester/modules/arguments.sh deleted file mode 100644 index afe46ee035..0000000000 --- a/build/cmake/package_tester/modules/arguments.sh +++ /dev/null @@ -1,114 +0,0 @@ -#!/usr/bin/env bash - -if [ -z ${arguments_sh_included+x} ] -then - arguments_sh_included=1 - - source ${source_dir}/modules/util.sh - - arguments_usage() { - cat <<EOF -usage: test_packages.sh [-h] [commands] - -h: print this help message and - abort execution - -b DIR: point set the fdb build directory - (this is a required argument). - -s DIR: Path to fdb source directory. - -p STR: Colon-separated list of package - file names (without path) to - test. - -c PATH: Path to a ini-file with the docker - configuration - -t TEST: One of DEB, RPM, ALL - -n TESTS: Colon separated list of test names - to run (will run all if this option - is not set) - -j NUM Number of threads the tester should - run in parallel. - -P STR Pruning strategy for docker container - (Can be ALL|FAILED|SUCCEEDED|NONE) - Defaults to "SUCCEEDED" - - Will execute the passed commands - in the order they were passed -EOF - } - - arguments_parse() { - local __res=0 - run_deb_tests=1 - run_rpm_tests=1 - docker_parallelism=1 - pruning_strategy=SUCCEEDED - while getopts ":hb:s:p:c:t:n:j:P:" opt - do - case ${opt} in - h ) - arguments_usage - __res=2 - break - ;; - b ) - fdb_build="${OPTARG}" - ;; - s ) - fdb_source="${OPTARG}" - ;; - p ) - fdb_packages="${OPTARG}" - ;; - c ) - docker_ini="${OPTARG}" - ;; - t ) - if [ "${OPTARG}" = "DEB" ] - then - run_rpm_tests=0 - elif [ "${OPTARG}" = "RPM" ] - then - run_deb_tests=0 - elif [ "${OPTARG}" != "ALL" ] - then - echo -e "${RED}No such test: ${OPTARG}${NC}" - echo "Note: Currently known tests are: RPM, DEB, and ALL" - exit 1 - fi - ;; - n ) - tests_to_run="${OPTARG}" - ;; - j ) - docker_parallelism="${OPTARG}" - if [[ $docker_parallelism =~ "^[0-9]+$" ]] - then - echo -e "${RED}Error: -j expects a number, ${OPTARG}, is not a number" >&2 - __res=1 - break - elif [ $docker_parallelism -lt 1 ] - then - echo -e "${RED}Error: -j ${OPTARG} makes no sense" >&2 - __res=1 - break - fi - ;; - P ) - pruning_strategy="${OPTARG}" - if ! [[ "${pruning_strategy}" =~ ^(ALL|FAILED|SUCCEEDED|NONE)$ ]] - then - fail "Unknown pruning strategy ${pruning_strategy}" - fi - ;; - \? ) - curr_index="$((OPTIND-1))" - echo "Unknown option ${@:${curr_index}:1}" - arguments_usage - __res=1 - break - ;; - esac - done - shift $((OPTIND -1)) - commands=("$@") - return ${__res} - } -fi diff --git a/build/cmake/package_tester/modules/config.sh b/build/cmake/package_tester/modules/config.sh deleted file mode 100644 index 70fc35692c..0000000000 --- a/build/cmake/package_tester/modules/config.sh +++ /dev/null @@ -1,123 +0,0 @@ -#!/usr/bin/env bash - -if [ -z ${config_sh_included+x} ] -then - config_sh_included=1 - - source ${source_dir}/modules/util.sh - - config_load_vms() { - local __res=0 - enterfun - for _ in 1 - do - if [ -z "${docker_ini+x}"] - then - docker_file="${source_dir}/../docker.ini" - fi - # parse the ini file and read it into an - # associative array - eval "$(awk -F ' *= *' '{ if ($1 ~ /^\[/) section=$1; else if ($1 !~ /^$/) printf "ini_%s%s=\47%s\47\n", $1, section, $2 }' ${docker_file})" - vms=( "${!ini_name[@]}" ) - if [ $? -ne 0 ] - then - echo "ERROR: Could not parse config-file ${docker_file}" - __res=1 - break - fi - done - exitfun - return ${__res} - } - - config_find_packages() { - local __res=0 - enterfun - for _ in 1 - do - cd ${fdb_build} - while read f - do - if [[ "${f}" =~ .*"clients".* || "${f}" =~ .*"server".* ]] - then - if [ -z ${fdb_packages+x} ] - then - fdb_packages="${f}" - else - fdb_packages="${fdb_packages}:${f}" - fi - fi - done <<< "$(ls *.deb *.rpm)" - if [ $? -ne 0 ] - then - __res=1 - break - fi - done - exitfun - return ${__res} - } - - get_fdb_source() { - local __res=0 - enterfun - cd ${source_dir} - while true - do - if [ -d .git ] - then - # we found the root - pwd - break - fi - if [ `pwd` = "/" ] - then - __res=1 - break - fi - cd .. - done - exitfun - return ${__res} - } - - fdb_build=0 - - config_verify() { - local __res=0 - enterfun - for _ in 1 - do - if [ -z ${fdb_source+x} ] - then - fdb_source=`get_fdb_source` - fi - if [ ! -d "${fdb_build}" ] - then - __res=1 - echo "ERROR: Could not find fdb build dir: ${fdb_build}" - echo " Either set the environment variable fdb_build or" - echo " pass it with -b <PATH_TO_BUILD>" - fi - if [ ! -f "${fdb_source}/flow/Net2.actor.cpp" ] - then - __res=1 - echo "ERROR: ${fdb_source} does not appear to be a fdb source" - echo " directory. Either pass it with -s or set" - echo " the environment variable fdb_source." - fi - if [ ${__res} -ne 0 ] - then - break - fi - config_load_vms - __res=$? - if [ ${__res} -ne 0 ] - then - break - fi - done - exitfun - return ${__res} - } -fi diff --git a/build/cmake/package_tester/modules/deb.sh b/build/cmake/package_tester/modules/deb.sh deleted file mode 100644 index fcef96805a..0000000000 --- a/build/cmake/package_tester/modules/deb.sh +++ /dev/null @@ -1,40 +0,0 @@ -#!/usr/bin/env bash - -if [ -z "${deb_sh_included}" ] -then - deb_sh_included=1 - - source ${source_dir}/modules/util.sh - - install_build_tools() { - apt-get -y install cmake gcc - } - - install() { - local __res=0 - enterfun - echo "Install FoundationDB" - cd /build/packages - package_names=() - for f in "${package_files[@]}" - do - package_name="$(dpkg -I ${f} | grep Package | sed 's/.*://')" - package_names+=( "${package_name}" ) - done - dpkg -i ${package_files[@]} - apt-get -yf -o Dpkg::Options::="--force-confold" install - __res=$? - sleep 5 - exitfun - return ${__res} - } - - uninstall() { - local __res=0 - enterfun - apt-get -y remove ${package_names[@]} - __res=$? - exitfun - return ${__res} - } -fi diff --git a/build/cmake/package_tester/modules/docker.sh b/build/cmake/package_tester/modules/docker.sh deleted file mode 100644 index d38b939957..0000000000 --- a/build/cmake/package_tester/modules/docker.sh +++ /dev/null @@ -1,186 +0,0 @@ -#!/usr/bin/env bash - -if [ -z "${docker_sh_included+x}" ] -then - docker_sh_included=1 - source ${source_dir}/modules/util.sh - source ${source_dir}/modules/config.sh - source ${source_dir}/modules/tests.sh - - failed_tests=() - - docker_ids=() - docker_threads=() - docker_logs=() - docker_error_logs=() - - docker_wait_any() { - local __res=0 - enterfun - while [ "${#docker_threads[@]}" -gt 0 ] - do - IFS=";" read -ra res <${pipe_file} - docker_id=${res[0]} - result=${res[1]} - i=0 - for (( idx=0; idx<${#docker_ids[@]}; idx++ )) - do - if [ "${docker_id}" = "${docker_ids[idx]}" ] - then - i=idx - break - fi - done - if [ "${result}" -eq 0 ] - then - echo -e "${GREEN}Test succeeded: ${docker_threads[$i]}" - echo -e "\tDocker-ID: ${docker_ids[$i]} " - echo -e "\tLog-File: ${docker_logs[$i]}" - echo -e "\tErr-File: ${docker_error_logs[$i]} ${NC}" - else - echo -e "${RED}Test FAILED: ${docker_threads[$i]}" - echo -e "\tDocker-ID: ${docker_ids[$i]} " - echo -e "\tLog-File: ${docker_logs[$i]}" - echo -e "\tErr-File: ${docker_error_logs[$i]} ${NC}" - failed_tests+=( "${docker_threads[$i]}" ) - fi - n=$((i+1)) - docker_ids=( "${docker_ids[@]:0:$i}" "${docker_ids[@]:$n}" ) - docker_threads=( "${docker_threads[@]:0:$i}" "${docker_threads[@]:$n}" ) - docker_logs=( "${docker_logs[@]:0:$i}" "${docker_logs[@]:$n}" ) - docker_error_logs=( "${docker_error_logs[@]:0:$i}" "${docker_error_logs[@]:$n}" ) - break - done - exitfun - return "${__res}" - } - - docker_wait_all() { - local __res=0 - while [ "${#docker_threads[@]}" -gt 0 ] - do - docker_wait_any - if [ "$?" -ne 0 ] - then - __res=1 - fi - done - return ${__res} - } - - docker_run() { - local __res=0 - enterfun - for _ in 1 - do - echo "Testing the following:" - echo "======================" - for K in "${vms[@]}" - do - curr_packages=( $(cd ${fdb_build}/packages; ls | grep -P ${ini_packages[${K}]} ) ) - echo "Will test the following ${#curr_packages[@]} packages in docker-image ${K}:" - for p in "${curr_packages[@]}" - do - echo " ${p}" - done - echo - done - log_dir="${fdb_build}/pkg_tester" - pipe_file="${fdb_build}/pkg_tester.pipe" - lock_file="${fdb_build}/pkg_tester.lock" - if [ -p "${pipe_file}" ] - then - rm "${pipe_file}" - successOr "Could not delete old pipe file" - fi - if [ -f "${lock_file}" ] - then - rm "${lock_file}" - successOr "Could not delete old pipe file" - fi - touch "${lock_file}" - successOr "Could not create lock file" - mkfifo "${pipe_file}" - successOr "Could not create pipe file" - mkdir -p "${log_dir}" - # setup the containers - # TODO: shall we make this parallel as well? - for vm in "${vms[@]}" - do - curr_name="${ini_name[$vm]}" - curr_location="${ini_location[$vm]}" - if [[ "$curr_location" = /* ]] - then - cd "${curr_location}" - else - cd ${source_dir}/../${curr_location} - fi - docker_buid_logs="${log_dir}/docker_build_${curr_name}" - docker build . -t ${curr_name} 1> "${docker_buid_logs}.log" 2> "${docker_buid_logs}.err" - successOr "Building Docker image ${curr_name} failed - see ${docker_buid_logs}.log and ${docker_buid_logs}.err" - done - if [ ! -z "${tests_to_run+x}"] - then - tests=() - IFS=';' read -ra tests <<< "${tests_to_run}" - fi - for vm in "${vms[@]}" - do - curr_name="${ini_name[$vm]}" - curr_format="${ini_format[$vm]}" - curr_packages=( $(cd ${fdb_build}/packages; ls | grep -P ${ini_packages[${vm}]} ) ) - for curr_test in "${tests[@]}" - do - if [ "${#docker_ids[@]}" -ge "${docker_parallelism}" ] - then - docker_wait_any - fi - log_file="${log_dir}/${curr_name}_${curr_test}.log" - err_file="${log_dir}/${curr_name}_${curr_test}.err" - docker_id=$( docker run -d -v "${fdb_source}:/foundationdb"\ - -v "${fdb_build}:/build"\ - ${curr_name} /sbin/init ) - echo "Starting Test ${curr_name}/${curr_test} Docker-ID: ${docker_id}" - { - docker exec "${docker_id}" bash \ - /foundationdb/build/cmake/package_tester/${curr_format}_tests.sh -n ${curr_test} ${curr_packages[@]}\ - 2> ${err_file} 1> ${log_file} - res=$? - if [ "${pruning_strategy}" = "ALL" ] - then - docker kill "${docker_id}" > /dev/null - elif [ "${res}" -eq 0 ] && [ "${pruning_strategy}" = "SUCCEEDED" ] - then - docker kill "${docker_id}" > /dev/null - elif [ "${res}" -ne 0 ] && [ "${pruning_strategy}" = "FAILED" ] - then - docker kill "${docker_id}" > /dev/null - fi - flock "${lock_file}" echo "${docker_id};${res}" >> "${pipe_file}" - } & - docker_ids+=( "${docker_id}" ) - docker_threads+=( "${curr_name}/${curr_test}" ) - docker_logs+=( "${log_file}" ) - docker_error_logs+=( "${err_file}" ) - done - done - docker_wait_all - rm ${pipe_file} - if [ "${#failed_tests[@]}" -eq 0 ] - then - echo -e "${GREEN}SUCCESS${NC}" - else - echo -e "${RED}FAILURE" - echo "The following tests failed:" - for t in "${failed_tests[@]}" - do - echo " - ${t}" - done - echo -e "${NC}" - __res=1 - fi - done - exitfun - return "${__res}" - } -fi diff --git a/build/cmake/package_tester/modules/globals.sh b/build/cmake/package_tester/modules/globals.sh deleted file mode 100644 index 795a4adc66..0000000000 --- a/build/cmake/package_tester/modules/globals.sh +++ /dev/null @@ -1,23 +0,0 @@ -#!/usr/bin/env bash - -# This module has to be included first and only once. -# This is because of a limitation of older bash versions -# that doesn't allow us to declare associative arrays -# globally. - -if [ -z "${global_sh_included+x}"] -then - global_sh_included=1 -else - echo "global.sh can only be included once" - exit 1 -fi - -declare -A ini_name -declare -A ini_location -declare -A ini_packages -declare -A ini_format -declare -A test_start_state -declare -A test_exit_state -declare -a tests -declare -a vms diff --git a/build/cmake/package_tester/modules/rpm.sh b/build/cmake/package_tester/modules/rpm.sh deleted file mode 100644 index 866bde558e..0000000000 --- a/build/cmake/package_tester/modules/rpm.sh +++ /dev/null @@ -1,45 +0,0 @@ -#!/usr/bin/env bash - -if [ -z "${rpm_sh_included}" ] -then - rpm_sh_included=1 - - source ${source_dir}/modules/util.sh - - conf_save_extension=".rpmsave" - - install_build_tools() { - yum -y install cmake gcc - } - - install() { - local __res=0 - enterfun - cd /build/packages - package_names=() - for f in "${package_files[@]}" - do - package_names+=( "$(rpm -qp ${f})" ) - done - yum install -y ${package_files[@]} - __res=$? - # give the server some time to come up - sleep 5 - exitfun - return ${__res} - } - - uninstall() { - local __res=0 - enterfun - if [ "$1" == "purge" ] - then - yum remove --purge -y ${package_names[@]} - else - yum remove -y ${package_names[@]} - fi - __res=$? - exitfun - return ${__res} - } -fi diff --git a/build/cmake/package_tester/modules/test_args.sh b/build/cmake/package_tester/modules/test_args.sh deleted file mode 100644 index bb88da945f..0000000000 --- a/build/cmake/package_tester/modules/test_args.sh +++ /dev/null @@ -1,49 +0,0 @@ -#!/usr/bin/env bash - -if [ -z ${test_args_sh_included+x} ] -then - test_args_sh_included=1 - - source ${source_dir}/modules/util.sh - - test_args_usage() { - me=`basename "$0"` - echo "usage: ${me} [-h] files..." - cat <<EOF - -n TEST: The name of the test to run - - Will execute the passed commands - in the order they were passed -EOF - } - - test_args_parse() { - local __res=0 - run_deb_tests=1 - run_rpm_tests=1 - while getopts ":hn:" opt - do - case ${opt} in - h ) - test_args_usage - __res=2 - break - ;; - n ) - echo "test_name=${OPTARG}" - test_name="${OPTARG}" - ;; - \? ) - curr_index="$((OPTIND-1))" - echo "Unknown option ${@:${curr_index}:1}" - arguments_usage - __res=1 - break - ;; - esac - done - shift $((OPTIND -1)) - package_files=("$@") - return ${__res} - } -fi diff --git a/build/cmake/package_tester/modules/testing.sh b/build/cmake/package_tester/modules/testing.sh deleted file mode 100644 index e7b5582a87..0000000000 --- a/build/cmake/package_tester/modules/testing.sh +++ /dev/null @@ -1,64 +0,0 @@ -#!/usr/bin/env bash - -if [ -z "${testing_sh_included+x}" ] -then - testing_sh_included=1 - - source ${source_dir}/modules/util.sh - - desired_state() { - case $1 in - CLEAN ) - : - ;; - INSTALLED ) - install - ;; - esac - } - - tests_healthy() { - enterfun - local __res=0 - for _ in 1 - do - cd / - fdbcli --exec status - if [ $? -ne 0 ] - then - return 1 - fi - healthy="$(fdbcli --exec status | grep HEALTHY | wc -l)" - if [ -z "${healthy}" ] - then - __res=1 - break - fi - done - exitfun - return ${__res} - } - - tests_clean() { - uninstall purge - successOr "FoundationDB was not uninstalled correctly" - # systemd/initd are not running, so we have to kill manually here - pidof fdbmonitor | xargs kill - tests_clean_nouninstall - rm -rf /etc/foundationdb - rm -rf /var/lib/foundationdb - rm -rf /var/log/foundationdb - } - - tests_main() { - new_state="${test_start_state[${test_name}]}" - echo "Setting desired state ${new_state} for ${test_name}" - desired_state "${new_state}" - ${test_name} - successOr "${test_name} Failed" - echo "=======================================================================" - echo "Test $t successfully finished" - echo "=======================================================================" - current_state="${test_exit_state[${test_name}]}" - } -fi diff --git a/build/cmake/package_tester/modules/tests.sh b/build/cmake/package_tester/modules/tests.sh deleted file mode 100644 index 2495e21a94..0000000000 --- a/build/cmake/package_tester/modules/tests.sh +++ /dev/null @@ -1,143 +0,0 @@ -#!/usr/bin/env bash - -# In this file the tests are formulated which -# should run in the docker container to test -# whether the RPM and DEB packages work properly. -# -# In order to add a test, a user first has to -# add the name of the test to the `tests` array -# which is defined in this file. -# -# Then, she must define the state this test -# expects the container to be in. To do that, -# a value for the test has to be added to the -# associative array `test_start_state`. Valid -# values are: -# -# - INSTALLED: In this case, the test will be -# started with a freshly installed FDB, but -# no other changes were made to the container. -# - CLEAN: This simply means that the container -# will run a minimal version of the OS (as defined -# in the corresponsing Dockerfile) -# -# A test is then simply a bash function with the -# same name as the test. It can use the predefined -# bash functions `install` and `uninstall` to either -# install or uninstall FDB on the container. The FDB -# build directory can be found in `/build`, the -# source code will be located in `/foundationdb` - -if [ -z "${tests_sh_included+x}" ] -then - tests_sh_included=1 - - source ${source_dir}/modules/util.sh - source ${source_dir}/modules/testing.sh - - tests=( "fresh_install" "keep_config" ) - test_start_state=([fresh_install]=INSTALLED [keep_config]=CLEAN ) - - fresh_install() { - tests_healthy - successOr "Fresh installation is not clean" - # test that we can read from and write to fdb - cd / - timeout 2 fdbcli --exec 'writemode on ; set foo bar' - successOr "Cannot write to database" - getresult="$(timeout 2 fdbcli --exec 'get foo')" - successOr "Get on database failed" - if [ "${getresult}" != "\`foo' is \`bar'" ] - then - fail "value was not set correctly" - fi - timeout 2 fdbcli --exec 'writemode on ; clear foo' - successOr "Deleting value failed" - getresult="$(timeout 2 fdbcli --exec 'get foo')" - successOr "Get on database failed" - if [ "${getresult}" != "\`foo': not found" ] - then - fail "value was not cleared correctly" - fi - PYTHON_TARGZ_NAME="$(ls /build/packages | grep 'foundationdb-[0-9.]*\.tar\.gz' | sed 's/\.tar\.gz$//')" - tar -C /tmp -xvf /build/packages/${PYTHON_TARGZ_NAME}.tar.gz - pushd /tmp/${PYTHON_TARGZ_NAME} - python setup.py install - successOr "Installing python bindings failed" - popd - python -c 'import fdb; fdb.api_version(710)' - successOr "Loading python bindings failed" - - # Test cmake and pkg-config integration: https://github.com/apple/foundationdb/issues/1483 - install_build_tools - rm -rf build-fdb_c_app - mkdir build-fdb_c_app - pushd build-fdb_c_app - cmake /foundationdb/build/cmake/package_tester/fdb_c_app && make - successOr "FoundationDB-Client cmake integration failed" - cc /foundationdb/build/cmake/package_tester/fdb_c_app/app.c `pkg-config --libs --cflags foundationdb-client` - successOr "FoundationDB-Client pkg-config integration failed" - popd - } - - keep_config() { - mkdir /etc/foundationdb - description=$(LC_CTYPE=C tr -dc A-Za-z0-9 < /dev/urandom | head -c 8) - random_str=$(LC_CTYPE=C tr -dc A-Za-z0-9 < /dev/urandom | head -c 8) - successOr "Could not create secret" - echo $description:$random_str@127.0.0.1:4500 > /tmp/fdb.cluster - successOr "Could not create fdb.cluster file" - sed '/\[fdbserver.4500\]/a \[fdbserver.4501\]' /foundationdb/packaging/foundationdb.conf > /tmp/foundationdb.conf - successOr "Could not change foundationdb.conf file" - # we need to keep these files around for testing that the install didn't change them - cp /tmp/fdb.cluster /etc/foundationdb/fdb.cluster - cp /tmp/foundationdb.conf /etc/foundationdb/foundationdb.conf - - install - successOr "FoundationDB install failed" - # make sure we are not in build directory as there is a fdbc.cluster file there - echo "Configure new database - Install isn't supposed to do this for us" - echo "as there was an existing configuration" - cd / - timeout 2 fdbcli --exec 'configure new single ssd' - successOr "Couldn't configure new database" - tests_healthy - num_processes="$(timeout 2 fdbcli --exec 'status' | grep "FoundationDB processes" | sed -e 's/.*- //')" - if [ "${num_processes}" -ne 2 ] - then - fail Number of processes incorrect after config change - fi - - differences="$(diff /tmp/fdb.cluster /etc/foundationdb/fdb.cluster)" - if [ -n "${differences}" ] - then - fail Install changed configuration files - fi - differences="$(diff /tmp/foundationdb.conf /etc/foundationdb/foundationdb.conf)" - if [ -n "${differences}" ] - then - fail Install changed configuration files - fi - - uninstall - # make sure config didn't get deleted - # RPM, however, renames the file on remove, so we need to check for this - conffile="/etc/foundationdb/foundationdb.conf${conf_save_extension}" - if [ ! -f /etc/foundationdb/fdb.cluster ] || [ ! -f "${conffile}" ] - then - fail "Uninstall removed configuration" - fi - differences="$(diff /tmp/foundationdb.conf ${conffile})" - if [ -n "${differences}" ] - then - fail "${conffile} changed during remove" - fi - differences="$(diff /tmp/fdb.cluster /etc/foundationdb/fdb.cluster)" - if [ -n "${differences}" ] - then - fail "/etc/foundationdb/fdb.cluster changed during remove" - fi - - return 0 - } -fi diff --git a/build/cmake/package_tester/modules/util.sh b/build/cmake/package_tester/modules/util.sh deleted file mode 100644 index c3d643bdfc..0000000000 --- a/build/cmake/package_tester/modules/util.sh +++ /dev/null @@ -1,40 +0,0 @@ -#!/usr/bin/env bash - -if [ -z ${util_sh_included+x} ] -then - util_sh_included=1 - - # for colored output - RED='\033[0;31m' - GREEN='\033[0;32m' - YELLOW='\033[1;33m' - NC='\033[0m' # No Color - - - enterfun() { - pushd . > /dev/null - } - - exitfun() { - popd > /dev/null - } - - fail() { - false - successOr ${@:1} - } - - successOr() { - local __res=$? - if [ ${__res} -ne 0 ] - then - if [ "$#" -gt 1 ] - then - >&2 echo -e "${RED}${@:1} ${NC}" - fi - exit ${__res} - fi - return 0 - } - -fi diff --git a/build/cmake/package_tester/rpm_tests.sh b/build/cmake/package_tester/rpm_tests.sh deleted file mode 100755 index a88bfb4f15..0000000000 --- a/build/cmake/package_tester/rpm_tests.sh +++ /dev/null @@ -1,32 +0,0 @@ -#!/usr/bin/env bash - -source_dir="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null && pwd )" - -source ${source_dir}/modules/globals.sh -source ${source_dir}/modules/util.sh -source ${source_dir}/modules/rpm.sh -source ${source_dir}/modules/tests.sh -source ${source_dir}/modules/test_args.sh - -main() { - local __res=0 - enterfun - for _ in 1 - do - test_args_parse "$@" - __res=$? - if [ ${__res} -eq 2 ] - then - __res=0 - break - elif [ ${__res} -ne 0 ] - then - break - fi - tests_main - done - exitfun - return ${__res} -} - -main "$@" diff --git a/build/cmake/package_tester/test_packages.sh b/build/cmake/package_tester/test_packages.sh deleted file mode 100755 index 05642073d8..0000000000 --- a/build/cmake/package_tester/test_packages.sh +++ /dev/null @@ -1,35 +0,0 @@ -#!/usr/bin/env bash - -source_dir="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null && pwd )" - -source ${source_dir}/modules/globals.sh -source ${source_dir}/modules/config.sh -source ${source_dir}/modules/util.sh -source ${source_dir}/modules/arguments.sh -source ${source_dir}/modules/docker.sh - -main() { - local __res=0 - enterfun - for _ in 1 - do - arguments_parse "$@" - if [ $? -ne 0 ] - then - __res=1 - break - fi - config_verify - if [ $? -ne 0 ] - then - __res=1 - break - fi - docker_run - __res=$? - done - exitfun - return ${__res} -} - -main "$@" diff --git a/build/docker-compose.yaml b/build/docker-compose.yaml deleted file mode 100644 index 4dcc30c683..0000000000 --- a/build/docker-compose.yaml +++ /dev/null @@ -1,105 +0,0 @@ -version: "3" - -services: - common: &common - image: foundationdb/foundationdb-build:0.1.24 - - build-setup: &build-setup - <<: *common - depends_on: [common] - volumes: - - ..:/__this_is_some_very_long_name_dir_needed_to_fix_a_bug_with_debug_rpms__/foundationdb - working_dir: /__this_is_some_very_long_name_dir_needed_to_fix_a_bug_with_debug_rpms__/foundationdb - environment: - - MAKEJOBS=1 - - USE_CCACHE=1 - - BUILD_DIR=./work - - release-setup: &release-setup - <<: *build-setup - environment: - - MAKEJOBS=1 - - USE_CCACHE=1 - - RELEASE=true - - BUILD_DIR=./work - - snapshot-setup: &snapshot-setup - <<: *build-setup - - build-docs: - <<: *build-setup - volumes: - - ..:/foundationdb - working_dir: /foundationdb - command: scl enable devtoolset-8 python27 rh-python36 rh-ruby24 -- bash -c 'make -j "$${MAKEJOBS}" docpackage' - - - release-packages: &release-packages - <<: *release-setup - command: scl enable devtoolset-8 rh-python36 rh-ruby24 -- bash -c 'make -j "$${MAKEJOBS}" packages' - - snapshot-packages: &snapshot-packages - <<: *build-setup - command: scl enable devtoolset-8 rh-python36 rh-ruby24 -- bash -c 'make -j "$${MAKEJOBS}" packages' - - prb-packages: - <<: *snapshot-packages - - - release-bindings: &release-bindings - <<: *release-setup - command: scl enable devtoolset-8 rh-python36 rh-ruby24 -- bash -c 'make -j "$${MAKEJOBS}" bindings' - - snapshot-bindings: &snapshot-bindings - <<: *build-setup - command: scl enable devtoolset-8 rh-python36 rh-ruby24 -- bash -c 'make -j "$${MAKEJOBS}" bindings' - - prb-bindings: - <<: *snapshot-bindings - - - snapshot-cmake: &snapshot-cmake - <<: *build-setup - command: scl enable devtoolset-8 rh-python36 rh-ruby24 -- bash -c 'mkdir -p "$${BUILD_DIR}" && cd "$${BUILD_DIR}" && cmake -G "Ninja" -DFDB_RELEASE=0 -DUSE_WERROR=ON /__this_is_some_very_long_name_dir_needed_to_fix_a_bug_with_debug_rpms__/foundationdb && ninja -v -j "$${MAKEJOBS}" "packages" "strip_targets" && cpack' - - prb-cmake: - <<: *snapshot-cmake - - - snapshot-bindings-cmake: &snapshot-bindings-cmake - <<: *build-setup - command: scl enable devtoolset-8 rh-python36 rh-ruby24 -- bash -c 'mkdir -p "$${BUILD_DIR}" && cd "$${BUILD_DIR}" && cmake -G "Ninja" -DFDB_RELEASE=0 -DUSE_WERROR=ON /__this_is_some_very_long_name_dir_needed_to_fix_a_bug_with_debug_rpms__/foundationdb && ninja -v -j "$${MAKEJOBS}" "bindings/all"' - - prb-bindings-cmake: - <<: *snapshot-bindings-cmake - - - snapshot-cmake: &snapshot-testpackages - <<: *build-setup - command: scl enable devtoolset-8 rh-python36 rh-ruby24 -- bash -c 'mkdir -p "$${BUILD_DIR}" && cd "$${BUILD_DIR}" && cmake -G "Ninja" -DFDB_RELEASE=0 /__this_is_some_very_long_name_dir_needed_to_fix_a_bug_with_debug_rpms__/foundationdb && ninja -v -j "$${MAKEJOBS}"' - - prb-testpackages: - <<: *snapshot-testpackages - - - snapshot-ctest: &snapshot-ctest - <<: *build-setup - command: scl enable devtoolset-8 rh-python36 rh-ruby24 -- bash -c 'mkdir -p "$${BUILD_DIR}" && cd "$${BUILD_DIR}" && cmake -G "Ninja" -DFDB_RELEASE=1 -DUSE_WERROR=ON /__this_is_some_very_long_name_dir_needed_to_fix_a_bug_with_debug_rpms__/foundationdb && ninja -v -j "$${MAKEJOBS}" && ctest -j "$${MAKEJOBS}" --output-on-failure' - - prb-ctest: - <<: *snapshot-ctest - - - snapshot-correctness: &snapshot-correctness - <<: *build-setup - command: scl enable devtoolset-8 rh-python36 rh-ruby24 -- bash -c 'mkdir -p "$${BUILD_DIR}" && cd "$${BUILD_DIR}" && cmake -G "Ninja" -DFDB_RELEASE=1 -DUSE_WERROR=ON /__this_is_some_very_long_name_dir_needed_to_fix_a_bug_with_debug_rpms__/foundationdb && ninja -v -j "$${MAKEJOBS}" && ctest -j "$${MAKEJOBS}" --output-on-failure' - - prb-correctness: - <<: *snapshot-correctness - - - shell: - <<: *build-setup - volumes: - - ..:/foundationdb - entrypoint: /bin/bash diff --git a/build/docker/centos6/build/Dockerfile b/build/docker/centos6/build/Dockerfile deleted file mode 100644 index 0a1fbbd70a..0000000000 --- a/build/docker/centos6/build/Dockerfile +++ /dev/null @@ -1,290 +0,0 @@ -FROM centos:6 - -WORKDIR /tmp - -RUN sed -i -e '/enabled/d' /etc/yum.repos.d/CentOS-Base.repo && \ - sed -i -e '/gpgcheck=1/a enabled=0' /etc/yum.repos.d/CentOS-Base.repo && \ - sed -i -n '/6.1/q;p' /etc/yum.repos.d/CentOS-Vault.repo && \ - sed -i -e "s/6\.0/$(cut -d\ -f3 /etc/redhat-release)/g" /etc/yum.repos.d/CentOS-Vault.repo && \ - sed -i -e 's/enabled=0/enabled=1/g' /etc/yum.repos.d/CentOS-Vault.repo && \ - yum install -y \ - centos-release-scl-rh \ - epel-release \ - scl-utils \ - yum-utils && \ - yum-config-manager --enable rhel-server-rhscl-7-rpms && \ - sed -i -e 's/#baseurl=/baseurl=/g' \ - -e 's/mirror.centos.org/vault.centos.org/g' \ - -e 's/mirrorlist=/#mirrorlist=/g' \ - /etc/yum.repos.d/CentOS-SCLo-scl-rh.repo && \ - yum install -y \ - binutils-devel \ - curl \ - debbuild \ - devtoolset-8 \ - devtoolset-8-libasan-devel \ - devtoolset-8-libtsan-devel \ - devtoolset-8-libubsan-devel \ - devtoolset-8-valgrind-devel \ - dos2unix \ - dpkg \ - gettext-devel \ - git \ - golang \ - java-1.8.0-openjdk-devel \ - libcurl-devel \ - libuuid-devel \ - libxslt \ - lz4 \ - lz4-devel \ - lz4-static \ - mono-devel \ - redhat-lsb-core \ - rpm-build \ - tcl-devel \ - unzip \ - wget \ - rh-python36 \ - rh-python36-python-devel \ - rh-ruby24 && \ - yum clean all && \ - rm -rf /var/cache/yum - -# build/install autoconf -- same version installed by yum in centos7 -RUN curl -Ls http://ftp.gnu.org/gnu/autoconf/autoconf-2.69.tar.gz -o autoconf.tar.gz && \ - echo "954bd69b391edc12d6a4a51a2dd1476543da5c6bbf05a95b59dc0dd6fd4c2969 autoconf.tar.gz" > autoconf-sha.txt && \ - sha256sum -c autoconf-sha.txt && \ - mkdir autoconf && \ - tar --strip-components 1 --no-same-owner --directory autoconf -xf autoconf.tar.gz && \ - cd autoconf && \ - ./configure && \ - make && \ - make install && \ - cd ../ && \ - rm -rf /tmp/* - -# build/install automake -- same version installed by yum in centos7 -RUN curl -Ls http://ftp.gnu.org/gnu/automake/automake-1.13.4.tar.gz -o automake.tar.gz && \ - echo "4c93abc0bff54b296f41f92dd3aa1e73e554265a6f719df465574983ef6f878c automake.tar.gz" > automake-sha.txt && \ - sha256sum -c automake-sha.txt && \ - mkdir automake && \ - tar --strip-components 1 --no-same-owner --directory automake -xf automake.tar.gz && \ - cd automake && \ - ./configure && \ - make && \ - make install && \ - cd ../ && \ - rm -rf /tmp/* - -# build/install git -RUN source /opt/rh/devtoolset-8/enable && \ - curl -Ls https://github.com/git/git/archive/v2.30.0.tar.gz -o git.tar.gz && \ - echo "8db4edd1a0a74ebf4b78aed3f9e25c8f2a7db3c00b1aaee94d1e9834fae24e61 git.tar.gz" > git-sha.txt && \ - sha256sum -c git-sha.txt && \ - mkdir git && \ - tar --strip-components 1 --no-same-owner --directory git -xf git.tar.gz && \ - cd git && \ - make configure && \ - ./configure && \ - make && \ - make install && \ - cd ../ && \ - rm -rf /tmp/* - -# build/install ninja -RUN source /opt/rh/devtoolset-8/enable && \ - curl -Ls https://github.com/ninja-build/ninja/archive/v1.9.0.zip -o ninja.zip && \ - echo "8e2e654a418373f10c22e4cc9bdbe9baeca8527ace8d572e0b421e9d9b85b7ef ninja.zip" > ninja-sha.txt && \ - sha256sum -c ninja-sha.txt && \ - unzip ninja.zip && \ - cd ninja-1.9.0 && \ - ./configure.py --bootstrap && \ - cp ninja /usr/bin && \ - cd .. && \ - rm -rf /tmp/* - -# install cmake -RUN curl -Ls https://github.com/Kitware/CMake/releases/download/v3.13.4/cmake-3.13.4-Linux-x86_64.tar.gz -o cmake.tar.gz && \ - echo "563a39e0a7c7368f81bfa1c3aff8b590a0617cdfe51177ddc808f66cc0866c76 cmake.tar.gz" > cmake-sha.txt && \ - sha256sum -c cmake-sha.txt && \ - mkdir cmake && \ - tar --strip-components 1 --no-same-owner --directory cmake -xf cmake.tar.gz && \ - cp -r cmake/* /usr/local/ && \ - rm -rf /tmp/* - -# build/install LLVM -RUN source /opt/rh/devtoolset-8/enable && \ - source /opt/rh/rh-python36/enable && \ - curl -Ls https://github.com/llvm/llvm-project/releases/download/llvmorg-10.0.0/llvm-project-10.0.0.tar.xz -o llvm.tar.xz && \ - echo "6287a85f4a6aeb07dbffe27847117fe311ada48005f2b00241b523fe7b60716e llvm.tar.xz" > llvm-sha.txt && \ - sha256sum -c llvm-sha.txt && \ - mkdir llvm-project && \ - tar --strip-components 1 --no-same-owner --directory llvm-project -xf llvm.tar.xz && \ - mkdir -p llvm-project/build && \ - cd llvm-project/build && \ - cmake \ - -DCMAKE_BUILD_TYPE=Release \ - -G Ninja \ - -DLLVM_INCLUDE_EXAMPLES=OFF \ - -DLLVM_INCLUDE_TESTS=OFF \ - -DLLVM_ENABLE_PROJECTS="clang;clang-tools-extra;compiler-rt;libcxx;libcxxabi;libunwind;lld;lldb" \ - -DLLVM_STATIC_LINK_CXX_STDLIB=ON \ - ../llvm && \ - cmake --build . && \ - cmake --build . --target install && \ - cd ../.. && \ - rm -rf /tmp/* - -# build/install openssl -RUN source /opt/rh/devtoolset-8/enable && \ - curl -Ls https://www.openssl.org/source/openssl-1.1.1h.tar.gz -o openssl.tar.gz && \ - echo "5c9ca8774bd7b03e5784f26ae9e9e6d749c9da2438545077e6b3d755a06595d9 openssl.tar.gz" > openssl-sha.txt && \ - sha256sum -c openssl-sha.txt && \ - mkdir openssl && \ - tar --strip-components 1 --no-same-owner --directory openssl -xf openssl.tar.gz && \ - cd openssl && \ - ./config CFLAGS="-fPIC -O3" --prefix=/usr/local && \ - make -j`nproc` && \ - make -j1 install && \ - ln -sv /usr/local/lib64/lib*.so.1.1 /usr/lib64/ && \ - cd .. && \ - rm -rf /tmp/* - -# install rocksdb to /opt -RUN curl -Ls https://github.com/facebook/rocksdb/archive/v6.10.1.tar.gz -o rocksdb.tar.gz && \ - echo "d573d2f15cdda883714f7e0bc87b814a8d4a53a82edde558f08f940e905541ee rocksdb.tar.gz" > rocksdb-sha.txt && \ - sha256sum -c rocksdb-sha.txt && \ - tar --directory /opt -xf rocksdb.tar.gz && \ - rm -rf /tmp/* - -# install boost 1.67 to /opt -RUN curl -Ls https://boostorg.jfrog.io/artifactory/main/release/1.67.0/source/boost_1_67_0.tar.bz2 -o boost_1_67_0.tar.bz2 && \ - echo "2684c972994ee57fc5632e03bf044746f6eb45d4920c343937a465fd67a5adba boost_1_67_0.tar.bz2" > boost-sha-67.txt && \ - sha256sum -c boost-sha-67.txt && \ - tar --no-same-owner --directory /opt -xjf boost_1_67_0.tar.bz2 && \ - rm -rf /opt/boost_1_67_0/libs && \ - rm -rf /tmp/* - -# install boost 1.72 to /opt -RUN source /opt/rh/devtoolset-8/enable && \ - curl -Ls https://boostorg.jfrog.io/artifactory/main/release/1.72.0/source/boost_1_72_0.tar.bz2 -o boost_1_72_0.tar.bz2 && \ - echo "59c9b274bc451cf91a9ba1dd2c7fdcaf5d60b1b3aa83f2c9fa143417cc660722 boost_1_72_0.tar.bz2" > boost-sha-72.txt && \ - sha256sum -c boost-sha-72.txt && \ - tar --no-same-owner --directory /opt -xjf boost_1_72_0.tar.bz2 && \ - cd /opt/boost_1_72_0 &&\ - ./bootstrap.sh --with-libraries=context &&\ - ./b2 link=static cxxflags=-std=c++14 --prefix=/opt/boost_1_72_0 install &&\ - rm -rf /opt/boost_1_72_0/libs && \ - rm -rf /tmp/* - -# jemalloc (needed for FDB after 6.3) -RUN source /opt/rh/devtoolset-8/enable && \ - curl -Ls https://github.com/jemalloc/jemalloc/releases/download/5.2.1/jemalloc-5.2.1.tar.bz2 -o jemalloc-5.2.1.tar.bz2 && \ - echo "34330e5ce276099e2e8950d9335db5a875689a4c6a56751ef3b1d8c537f887f6 jemalloc-5.2.1.tar.bz2" > jemalloc-sha.txt && \ - sha256sum -c jemalloc-sha.txt && \ - mkdir jemalloc && \ - tar --strip-components 1 --no-same-owner --no-same-permissions --directory jemalloc -xjf jemalloc-5.2.1.tar.bz2 && \ - cd jemalloc && \ - ./configure --enable-static --disable-cxx && \ - make && \ - make install && \ - cd .. && \ - rm -rf /tmp/* - -# Install CCACHE -RUN source /opt/rh/devtoolset-8/enable && \ - curl -Ls https://github.com/ccache/ccache/releases/download/v4.0/ccache-4.0.tar.gz -o ccache.tar.gz && \ - echo "ac97af86679028ebc8555c99318352588ff50f515fc3a7f8ed21a8ad367e3d45 ccache.tar.gz" > ccache-sha256.txt && \ - sha256sum -c ccache-sha256.txt && \ - mkdir ccache &&\ - tar --strip-components 1 --no-same-owner --directory ccache -xf ccache.tar.gz && \ - mkdir build && \ - cd build && \ - cmake -G Ninja -DCMAKE_BUILD_TYPE=Release -DZSTD_FROM_INTERNET=ON ../ccache && \ - cmake --build . --target install && \ - cd .. && \ - rm -rf /tmp/* - -# build/install toml -RUN source /opt/rh/devtoolset-8/enable && \ - curl -Ls https://github.com/ToruNiina/toml11/archive/v3.4.0.tar.gz -o toml.tar.gz && \ - echo "bc6d733efd9216af8c119d8ac64a805578c79cc82b813e4d1d880ca128bd154d toml.tar.gz" > toml-sha256.txt && \ - sha256sum -c toml-sha256.txt && \ - mkdir toml && \ - tar --strip-components 1 --no-same-owner --directory toml -xf toml.tar.gz && \ - mkdir build && \ - cd build && \ - cmake -G Ninja -DCMAKE_BUILD_TYPE=Release -Dtoml11_BUILD_TEST=OFF ../toml && \ - cmake --build . --target install && \ - cd .. && \ - rm -rf /tmp/* - -# download old fdbserver binaries -ARG FDB_VERSION="6.2.29" -RUN mkdir -p /opt/foundationdb/old && \ - curl -Ls https://www.foundationdb.org/downloads/misc/fdbservers-${FDB_VERSION}.tar.gz | \ - tar --no-same-owner --directory /opt/foundationdb/old -xz && \ - chmod +x /opt/foundationdb/old/* && \ - ln -sf /opt/foundationdb/old/fdbserver-${FDB_VERSION} /opt/foundationdb/old/fdbserver - -# build/install distcc -RUN source /opt/rh/devtoolset-8/enable && \ - source /opt/rh/rh-python36/enable && \ - curl -Ls https://github.com/distcc/distcc/archive/v3.3.5.tar.gz -o distcc.tar.gz && \ - echo "13a4b3ce49dfc853a3de550f6ccac583413946b3a2fa778ddf503a9edc8059b0 distcc.tar.gz" > distcc-sha256.txt && \ - sha256sum -c distcc-sha256.txt && \ - mkdir distcc && \ - tar --strip-components 1 --no-same-owner --directory distcc -xf distcc.tar.gz && \ - cd distcc && \ - ./autogen.sh && \ - ./configure && \ - make && \ - make install && \ - cd .. && \ - rm -rf /tmp/* - -RUN curl -Ls https://github.com/manticoresoftware/manticoresearch/raw/master/misc/junit/ctest2junit.xsl -o /opt/ctest2junit.xsl - -# # Setting this environment variable switches from OpenSSL to BoringSSL -# ENV OPENSSL_ROOT_DIR=/opt/boringssl -# -# # install BoringSSL: TODO: They don't seem to have releases(?) I picked today's master SHA. -# RUN cd /opt &&\ -# git clone https://boringssl.googlesource.com/boringssl &&\ -# cd boringssl &&\ -# git checkout e796cc65025982ed1fb9ef41b3f74e8115092816 &&\ -# mkdir build -# -# # ninja doesn't respect CXXFLAGS, and the boringssl CMakeLists doesn't expose an option to define __STDC_FORMAT_MACROS -# # also, enable -fPIC. -# # this is moderately uglier than creating a patchfile, but easier to maintain. -# RUN cd /opt/boringssl &&\ -# for f in crypto/fipsmodule/rand/fork_detect_test.cc \ -# include/openssl/bn.h \ -# ssl/test/bssl_shim.cc ; do \ -# perl -p -i -e 's/#include <inttypes.h>/#define __STDC_FORMAT_MACROS 1\n#include <inttypes.h>/g;' $f ; \ -# done &&\ -# perl -p -i -e 's/-Werror/-Werror -fPIC/' CMakeLists.txt &&\ -# git diff -# -# RUN cd /opt/boringssl/build &&\ -# scl enable devtoolset-8 rh-python36 rh-ruby24 -- cmake -GNinja -DCMAKE_BUILD_TYPE=Release .. &&\ -# scl enable devtoolset-8 rh-python36 rh-ruby24 -- ninja &&\ -# ./ssl/ssl_test &&\ -# mkdir -p ../lib && cp crypto/libcrypto.a ssl/libssl.a ../lib -# -# # Localize time zone -# ARG TIMEZONEINFO=America/Los_Angeles -# RUN rm -f /etc/localtime && ln -s /usr/share/zoneinfo/${TIMEZONEINFO} /etc/localtime -# -# LABEL version=${IMAGE_TAG} -# ENV DOCKER_IMAGEVER=${IMAGE_TAG} -# ENV JAVA_HOME=/usr/lib/jvm/java-1.8.0 -# ENV CC=/opt/rh/devtoolset-8/root/usr/bin/gcc -# ENV CXX=/opt/rh/devtoolset-8/root/usr/bin/g++ -# -# ENV CCACHE_NOHASHDIR=true -# ENV CCACHE_UMASK=0000 -# ENV CCACHE_SLOPPINESS="file_macro,time_macros,include_file_mtime,include_file_ctime,file_stat_matches" -# -# CMD scl enable devtoolset-8 rh-python36 rh-ruby24 -- bash diff --git a/build/docker/centos6/devel/Dockerfile b/build/docker/centos6/devel/Dockerfile deleted file mode 100644 index c5c9db2914..0000000000 --- a/build/docker/centos6/devel/Dockerfile +++ /dev/null @@ -1,84 +0,0 @@ -ARG REPOSITORY=foundationdb/build -ARG VERSION=centos6-latest -FROM ${REPOSITORY}:${VERSION} - -# add vscode server -RUN yum repolist && \ - yum -y install \ - bash-completion \ - byobu \ - cgdb \ - emacs-nox \ - jq \ - the_silver_searcher \ - tmux \ - tree \ - vim \ - zsh && \ - yum clean all && \ - rm -rf /var/cache/yum - -WORKDIR /tmp -RUN source /opt/rh/devtoolset-8/enable && \ - source /opt/rh/rh-python36/enable && \ - pip3 install \ - lxml \ - psutil \ - python-dateutil \ - subprocess32 && \ - mkdir fdb-joshua && \ - cd fdb-joshua && \ - git clone https://github.com/FoundationDB/fdb-joshua . && \ - pip3 install /tmp/fdb-joshua && \ - cd /tmp && \ - curl -Ls https://amazon-eks.s3.us-west-2.amazonaws.com/1.18.9/2020-11-02/bin/linux/amd64/kubectl -o kubectl && \ - echo "3dbe69e6deb35fbd6fec95b13d20ac1527544867ae56e3dae17e8c4d638b25b9 kubectl" > kubectl.txt && \ - sha256sum -c kubectl.txt && \ - mv kubectl /usr/local/bin/kubectl && \ - chmod 755 /usr/local/bin/kubectl && \ - curl https://awscli.amazonaws.com/awscli-exe-linux-x86_64-2.0.30.zip -o "awscliv2.zip" && \ - echo "7ee475f22c1b35cc9e53affbf96a9ffce91706e154a9441d0d39cbf8366b718e awscliv2.zip" > awscliv2.txt && \ - sha256sum -c awscliv2.txt && \ - unzip -qq awscliv2.zip && \ - ./aws/install && \ - rm -rf /tmp/* - -ARG FDB_VERSION="6.2.29" -RUN mkdir -p /usr/lib/foundationdb/plugins && \ - curl -Ls https://www.foundationdb.org/downloads/misc/joshua_tls_library.tar.gz | \ - tar --strip-components=1 --no-same-owner --directory /usr/lib/foundationdb/plugins -xz && \ - ln -sf /usr/lib/foundationdb/plugins/FDBGnuTLS.so /usr/lib/foundationdb/plugins/fdb-libressl-plugin.so && \ - curl -Ls https://www.foundationdb.org/downloads/${FDB_VERSION}/linux/libfdb_c_${FDB_VERSION}.so -o /usr/lib64/libfdb_c_${FDB_VERSION}.so && \ - ln -sf /usr/lib64/libfdb_c_${FDB_VERSION}.so /usr/lib64/libfdb_c.so - -WORKDIR /root -RUN rm -f /root/anaconda-ks.cfg && \ - printf '%s\n' \ - 'source /opt/rh/devtoolset-8/enable' \ - 'source /opt/rh/rh-python36/enable' \ - 'source /opt/rh/rh-ruby26/enable' \ - '' \ - 'function cmk_ci() {' \ - ' cmake -S ${HOME}/src/foundationdb -B ${HOME}/build_output -D USE_CCACHE=ON -D USE_WERROR=ON -D RocksDB_ROOT=/opt/rocksdb-6.10.1 -D RUN_JUNIT_TESTS=ON -D RUN_JAVA_INTEGRATION_TESTS=ON -G Ninja && \' \ - ' ninja -v -C ${HOME}/build_output -j 84 all packages strip_targets' \ - '}' \ - 'function cmk() {' \ - ' cmake -S ${HOME}/src/foundationdb -B ${HOME}/build_output -D USE_CCACHE=ON -D USE_WERROR=ON -D RocksDB_ROOT=/opt/rocksdb-6.10.1 -D RUN_JUNIT_TESTS=ON -D RUN_JAVA_INTEGRATION_TESTS=ON -G Ninja && \' \ - ' ninja -C ${HOME}/build_output -j 84' \ - '}' \ - 'function ct() {' \ - ' cd ${HOME}/build_output && ctest -j 32 --no-compress-output -T test --output-on-failure' \ - '}' \ - 'function j() {' \ - ' python3 -m joshua.joshua "${@}"' \ - '}' \ - 'function jsd() {' \ - ' j start --tarball $(find ${HOME}/build_output/packages -name correctness\*.tar.gz) "${@}"' \ - '}' \ - '' \ - 'USER_BASHRC="$HOME/src/.bashrc.local"' \ - 'if test -f "$USER_BASHRC"; then' \ - ' source $USER_BASHRC' \ - 'fi' \ - '' \ - >> .bashrc diff --git a/build/docker/centos6/distcc/Dockerfile b/build/docker/centos6/distcc/Dockerfile deleted file mode 100644 index a96e67dff2..0000000000 --- a/build/docker/centos6/distcc/Dockerfile +++ /dev/null @@ -1,24 +0,0 @@ -ARG REPOSITORY=foundationdb/build -ARG VERSION=centos6-latest -FROM ${REPOSITORY}:${VERSION} - -RUN useradd distcc && \ - source /opt/rh/devtoolset-8/enable && \ - source /opt/rh/rh-python36/enable && \ - update-distcc-symlinks - -EXPOSE 3632 -EXPOSE 3633 -USER distcc -ENV ALLOW 0.0.0.0/0 - -ENTRYPOINT distccd \ - --daemon \ - --enable-tcp-insecure \ - --no-detach \ - --port 3632 \ - --log-stderr \ - --log-level info \ - --listen 0.0.0.0 \ - --allow ${ALLOW} \ - --jobs `nproc` \ No newline at end of file diff --git a/build/docker/centos7/build/Dockerfile b/build/docker/centos7/build/Dockerfile deleted file mode 100644 index de376d2557..0000000000 --- a/build/docker/centos7/build/Dockerfile +++ /dev/null @@ -1,247 +0,0 @@ -FROM centos:7 - -WORKDIR /tmp -COPY mono-project.com.rpmkey.pgp ./ -RUN rpmkeys --import mono-project.com.rpmkey.pgp && \ - curl -Ls https://download.mono-project.com/repo/centos7-stable.repo -o /etc/yum.repos.d/mono-centos7-stable.repo && \ - yum repolist && \ - yum install -y \ - centos-release-scl-rh \ - epel-release \ - scl-utils \ - yum-utils && \ - yum-config-manager --add-repo https://download.docker.com/linux/centos/docker-ce.repo && \ - yum install -y \ - autoconf \ - automake \ - binutils-devel \ - curl \ - debbuild \ - devtoolset-8 \ - devtoolset-8-libasan-devel \ - devtoolset-8-libtsan-devel \ - devtoolset-8-libubsan-devel \ - devtoolset-8-systemtap-sdt-devel \ - docker-ce \ - dos2unix \ - dpkg \ - gettext-devel \ - git \ - golang \ - java-11-openjdk-devel \ - libcurl-devel \ - libuuid-devel \ - libxslt \ - lz4 \ - lz4-devel \ - lz4-static \ - mono-devel \ - redhat-lsb-core \ - rpm-build \ - tcl-devel \ - unzip \ - wget && \ - if [ "$(uname -p)" == "aarch64" ]; then \ - yum install -y \ - rh-python38 \ - rh-python38-python-devel \ - rh-ruby27; \ - else \ - yum install -y \ - rh-python36 \ - rh-python36-python-devel \ - rh-ruby26; \ - fi && \ - yum clean all && \ - rm -rf /var/cache/yum - -# build/install git -RUN source /opt/rh/devtoolset-8/enable && \ - curl -Ls https://github.com/git/git/archive/v2.30.0.tar.gz -o git.tar.gz && \ - echo "8db4edd1a0a74ebf4b78aed3f9e25c8f2a7db3c00b1aaee94d1e9834fae24e61 git.tar.gz" > git-sha.txt && \ - sha256sum -c git-sha.txt && \ - mkdir git && \ - tar --strip-components 1 --no-same-owner --directory git -xf git.tar.gz && \ - cd git && \ - make configure && \ - ./configure && \ - make && \ - make install && \ - cd ../ && \ - rm -rf /tmp/* - -# build/install ninja -RUN source /opt/rh/devtoolset-8/enable && \ - curl -Ls https://github.com/ninja-build/ninja/archive/v1.9.0.zip -o ninja.zip && \ - echo "8e2e654a418373f10c22e4cc9bdbe9baeca8527ace8d572e0b421e9d9b85b7ef ninja.zip" > ninja-sha.txt && \ - sha256sum -c ninja-sha.txt && \ - unzip ninja.zip && \ - cd ninja-1.9.0 && \ - ./configure.py --bootstrap && \ - cp ninja /usr/bin && \ - cd .. && \ - rm -rf /tmp/* - -# install cmake -RUN if [ "$(uname -p)" == "aarch64" ]; then \ - curl -Ls https://github.com/Kitware/CMake/releases/download/v3.19.6/cmake-3.19.6-Linux-aarch64.tar.gz -o cmake.tar.gz; \ - echo "69ec045c6993907a4f4a77349d0a0668f1bd3ce8bc5f6fbab6dc7a7e2ffc4f80 cmake.tar.gz" > cmake-sha.txt; \ - else \ - curl -Ls https://github.com/Kitware/CMake/releases/download/v3.13.4/cmake-3.13.4-Linux-x86_64.tar.gz -o cmake.tar.gz; \ - echo "563a39e0a7c7368f81bfa1c3aff8b590a0617cdfe51177ddc808f66cc0866c76 cmake.tar.gz" > cmake-sha.txt; \ - fi && \ - sha256sum -c cmake-sha.txt && \ - mkdir cmake && \ - tar --strip-components 1 --no-same-owner --directory cmake -xf cmake.tar.gz && \ - cp -r cmake/* /usr/local/ && \ - rm -rf /tmp/* - -# build/install LLVM -RUN source /opt/rh/devtoolset-8/enable && \ - curl -Ls https://github.com/llvm/llvm-project/releases/download/llvmorg-11.0.0/llvm-project-11.0.0.tar.xz -o llvm.tar.xz && \ - echo "b7b639fc675fa1c86dd6d0bc32267be9eb34451748d2efd03f674b773000e92b llvm.tar.xz" > llvm-sha.txt && \ - sha256sum -c llvm-sha.txt && \ - mkdir llvm-project && \ - tar --strip-components 1 --no-same-owner --directory llvm-project -xf llvm.tar.xz && \ - mkdir -p llvm-project/build && \ - cd llvm-project/build && \ - cmake \ - -DCMAKE_BUILD_TYPE=Release \ - -G Ninja \ - -DLLVM_INCLUDE_EXAMPLES=OFF \ - -DLLVM_INCLUDE_TESTS=OFF \ - -DLLVM_ENABLE_PROJECTS="clang;clang-tools-extra;compiler-rt;libcxx;libcxxabi;libunwind;lld;lldb" \ - -DLLVM_STATIC_LINK_CXX_STDLIB=ON \ - ../llvm && \ - cmake --build . && \ - cmake --build . --target install && \ - cd ../.. && \ - rm -rf /tmp/* - -# build/install openssl -RUN source /opt/rh/devtoolset-8/enable && \ - curl -Ls https://www.openssl.org/source/openssl-1.1.1h.tar.gz -o openssl.tar.gz && \ - echo "5c9ca8774bd7b03e5784f26ae9e9e6d749c9da2438545077e6b3d755a06595d9 openssl.tar.gz" > openssl-sha.txt && \ - sha256sum -c openssl-sha.txt && \ - mkdir openssl && \ - tar --strip-components 1 --no-same-owner --directory openssl -xf openssl.tar.gz && \ - cd openssl && \ - ./config CFLAGS="-fPIC -O3" --prefix=/usr/local && \ - make -j`nproc` && \ - make -j1 install && \ - ln -sv /usr/local/lib64/lib*.so.1.1 /usr/lib64/ && \ - cd .. && \ - rm -rf /tmp/* - -# install rocksdb to /opt -RUN curl -Ls https://github.com/facebook/rocksdb/archive/v6.10.1.tar.gz -o rocksdb.tar.gz && \ - echo "d573d2f15cdda883714f7e0bc87b814a8d4a53a82edde558f08f940e905541ee rocksdb.tar.gz" > rocksdb-sha.txt && \ - sha256sum -c rocksdb-sha.txt && \ - tar --directory /opt -xf rocksdb.tar.gz && \ - rm -rf /tmp/* - -# install boost 1.67 to /opt -RUN curl -Ls https://boostorg.jfrog.io/artifactory/main/release/1.67.0/source/boost_1_67_0.tar.bz2 -o boost_1_67_0.tar.bz2 && \ - echo "2684c972994ee57fc5632e03bf044746f6eb45d4920c343937a465fd67a5adba boost_1_67_0.tar.bz2" > boost-sha-67.txt && \ - sha256sum -c boost-sha-67.txt && \ - tar --no-same-owner --directory /opt -xjf boost_1_67_0.tar.bz2 && \ - rm -rf /opt/boost_1_67_0/libs && \ - rm -rf /tmp/* - -# install boost 1.72 to /opt -RUN source /opt/rh/devtoolset-8/enable && \ - curl -Ls https://boostorg.jfrog.io/artifactory/main/release/1.72.0/source/boost_1_72_0.tar.bz2 -o boost_1_72_0.tar.bz2 && \ - echo "59c9b274bc451cf91a9ba1dd2c7fdcaf5d60b1b3aa83f2c9fa143417cc660722 boost_1_72_0.tar.bz2" > boost-sha-72.txt && \ - sha256sum -c boost-sha-72.txt && \ - tar --no-same-owner --directory /opt -xjf boost_1_72_0.tar.bz2 && \ - cd /opt/boost_1_72_0 &&\ - ./bootstrap.sh --with-libraries=context &&\ - ./b2 link=static cxxflags=-std=c++14 --prefix=/opt/boost_1_72_0 install &&\ - rm -rf /opt/boost_1_72_0/libs && \ - rm -rf /tmp/* - -# jemalloc (needed for FDB after 6.3) -RUN source /opt/rh/devtoolset-8/enable && \ - curl -Ls https://github.com/jemalloc/jemalloc/releases/download/5.2.1/jemalloc-5.2.1.tar.bz2 -o jemalloc-5.2.1.tar.bz2 && \ - echo "34330e5ce276099e2e8950d9335db5a875689a4c6a56751ef3b1d8c537f887f6 jemalloc-5.2.1.tar.bz2" > jemalloc-sha.txt && \ - sha256sum -c jemalloc-sha.txt && \ - mkdir jemalloc && \ - tar --strip-components 1 --no-same-owner --no-same-permissions --directory jemalloc -xjf jemalloc-5.2.1.tar.bz2 && \ - cd jemalloc && \ - ./configure --enable-static --disable-cxx && \ - make && \ - make install && \ - cd .. && \ - rm -rf /tmp/* - -# Install CCACHE -RUN source /opt/rh/devtoolset-8/enable && \ - curl -Ls https://github.com/ccache/ccache/releases/download/v4.0/ccache-4.0.tar.gz -o ccache.tar.gz && \ - echo "ac97af86679028ebc8555c99318352588ff50f515fc3a7f8ed21a8ad367e3d45 ccache.tar.gz" > ccache-sha256.txt && \ - sha256sum -c ccache-sha256.txt && \ - mkdir ccache &&\ - tar --strip-components 1 --no-same-owner --directory ccache -xf ccache.tar.gz && \ - mkdir build && \ - cd build && \ - cmake -G Ninja -DCMAKE_BUILD_TYPE=Release -DZSTD_FROM_INTERNET=ON ../ccache && \ - cmake --build . --target install && \ - cd .. && \ - rm -rf /tmp/* - -# build/install toml -RUN source /opt/rh/devtoolset-8/enable && \ - curl -Ls https://github.com/ToruNiina/toml11/archive/v3.4.0.tar.gz -o toml.tar.gz && \ - echo "bc6d733efd9216af8c119d8ac64a805578c79cc82b813e4d1d880ca128bd154d toml.tar.gz" > toml-sha256.txt && \ - sha256sum -c toml-sha256.txt && \ - mkdir toml && \ - tar --strip-components 1 --no-same-owner --directory toml -xf toml.tar.gz && \ - mkdir build && \ - cd build && \ - cmake -G Ninja -DCMAKE_BUILD_TYPE=Release -Dtoml11_BUILD_TEST=OFF ../toml && \ - cmake --build . --target install && \ - cd .. && \ - rm -rf /tmp/* - -# download old fdbserver binaries -ARG FDB_VERSION="6.2.29" -RUN mkdir -p /opt/foundationdb/old && \ - curl -Ls https://www.foundationdb.org/downloads/misc/fdbservers-${FDB_VERSION}.tar.gz | \ - tar --no-same-owner --directory /opt/foundationdb/old -xz && \ - chmod +x /opt/foundationdb/old/* && \ - ln -sf /opt/foundationdb/old/fdbserver-${FDB_VERSION} /opt/foundationdb/old/fdbserver - -# build/install distcc -RUN source /opt/rh/devtoolset-8/enable && \ - if [ "$(uname -p)" == "aarch64" ]; then \ - source /opt/rh/rh-python38/enable; \ - else \ - source /opt/rh/rh-python36/enable; \ - fi && \ - curl -Ls https://github.com/distcc/distcc/archive/v3.3.5.tar.gz -o distcc.tar.gz && \ - echo "13a4b3ce49dfc853a3de550f6ccac583413946b3a2fa778ddf503a9edc8059b0 distcc.tar.gz" > distcc-sha256.txt && \ - sha256sum -c distcc-sha256.txt && \ - mkdir distcc && \ - tar --strip-components 1 --no-same-owner --directory distcc -xf distcc.tar.gz && \ - cd distcc && \ - ./autogen.sh && \ - ./configure && \ - make && \ - make install && \ - cd .. && \ - rm -rf /tmp/* - -# valgrind -RUN source /opt/rh/devtoolset-8/enable && \ - curl -Ls https://sourceware.org/pub/valgrind/valgrind-3.17.0.tar.bz2 -o valgrind-3.17.0.tar.bz2 && \ - echo "ad3aec668e813e40f238995f60796d9590eee64a16dff88421430630e69285a2 valgrind-3.17.0.tar.bz2" > valgrind-sha.txt && \ - sha256sum -c valgrind-sha.txt && \ - mkdir valgrind && \ - tar --strip-components 1 --no-same-owner --no-same-permissions --directory valgrind -xjf valgrind-3.17.0.tar.bz2 && \ - cd valgrind && \ - ./configure && \ - make && \ - make install && \ - cd .. && \ - rm -rf /tmp/* - -RUN curl -Ls https://github.com/manticoresoftware/manticoresearch/raw/master/misc/junit/ctest2junit.xsl -o /opt/ctest2junit.xsl diff --git a/build/docker/centos7/build/mono-project.com.rpmkey.pgp b/build/docker/centos7/build/mono-project.com.rpmkey.pgp deleted file mode 100644 index 4d7f8726d4..0000000000 --- a/build/docker/centos7/build/mono-project.com.rpmkey.pgp +++ /dev/null @@ -1,40 +0,0 @@ ------BEGIN PGP PUBLIC KEY BLOCK----- -Version: SKS 1.1.6 -Comment: Hostname: sks.pod01.fleetstreetops.com - -mQENBFPfqCcBCADctOzyTxfWvf40Nlb+AMkcJyb505WSbzhWU8yPmBNAJOnbwueMsTkNMHEO -u8fGRNxRWj5o/Db1N7EoSQtK3OgFnBef8xquUyrzA1nJ2aPfUWX+bhTG1TwyrtLaOssFRz6z -/h/ChUIFvt2VZCw+Yx4BiKi+tvgwrHTYB/Yf2J9+R/1O6949n6veFFRBfgPOL0djhvRqXzhv -FjJkh4xhTaGVeOnRR3+YQkblmti2n6KYl0n2kNB40ujSqpTloSfnR5tmJpz00WoOA9MJBdvH -txTTn8l6rVzXbm4mW9ZmB1kht/BgWaNLaIisW5AZSkQKer35wOWf0G7Gw+cWHq+I7W9pABEB -AAG0OlhhbWFyaW4gUHVibGljIEplbmtpbnMgKGF1dG8tc2lnbmluZykgPHJlbGVuZ0B4YW1h -cmluLmNvbT6JARwEEAECAAYFAlQIhKQACgkQyQ+cuQ4frQyc1wf+MCusJK4ANLWikbgiSSx1 -qMBveBlLKLEdCxYY+B9rc/pRDw448iBdd+nuSVdbRoqLgoN8gHbClboP+i22yw+mga0KASD7 -b1mpdYB0npR3H73zbYArn3qTV8s/yUXkIAEFUtj0yoEuv8KjO8P7nZJh8OuqqAupUVN0s3Kj -ONqXqi6Ro3fvVEZWOUFZl/FmY5KmXlpcw+YwE5CaNhJ2WunrjFTDqynRU/LeoPEKuwyYvfo9 -37zJFCrpAUMTr/9QpEKmV61H7fEHA9oHq97FBwWfjOU0l2mrXt1zJ97xVd2DXxrZodlkiY6B -76rhaT4ZhltY1E7WB2Z9WPfTe1Y6jz4fZ4kBHAQQAQgABgUCWEyoiAAKCRABFQplW72BAn/P -CAC0GkRBR3JTmG8WGeQMLb/o6Gon9cxpLnKv1GgFbHSM7XYMe7ySh5zxORwFuECuJ5+qcA6c -Ve/kJAV8rewLULL9yvHK3oK7R8zoVGbFVm+lyoxiaXpkkWg21Mb8IubiO+tA/dJc7hKQSpoI -0+dmJNaNrTVwqj0tQ8e0OL9KvBOYwFbSe06bocSNPVmKCt0EOvpGcQfzFw5UEjJVkqFn/moU -rSxj0YsJpwRXB1pOsBaQC6r9oCgUvxPf4H77U07+ImXzxRWInVPYFSXSiBA7p+hzvsikmZEl -iIAia8mTteUF1GeK4kafUk6iZZUfBlCIb9sV4O9Vvv8W0VjK4Vg6O2UAiQE4BBMBAgAiBQJT -36gnAhsDBgsJCAcDAgYVCAIJCgsEFgIDAQIeAQIXgAAKCRCmoZs409gx75DoB/9h5p8u1cUS -y6Mp2PjjW398LJZaqWwaa2W/lcLEKN7oWTC5Yf5BEuVsO9270pVln9Cv7hiqcbC8kywk+sZv -RsYO3uoTRwsmImc/7uaK382hey1A2hvkH5fYHmY/5Z/Z0bm/A0k0chhG2ycjWjZXYLZ96I0V -U3ZBQBHoh3qRtgWq4yWTsCJBX+FKPBdmkIpgcPXQw+hak0mj2sILqjScRZT1Oe+WJsMNMaLa -8dSdw+pPm8NM/VGLmO9iTTDApuAsRixpCYLdJY+ThGNrKe6xDswQo8gr3gbBkJi0wLRDP2Rz -q7rD0TC2PxOaWOZ7hmyz+EhjLcjZhHNJTaa+NV0k8YAwuQENBFPfqCcBCACtc7HssC9S3PxJ -m1youvGfYLhm+KzMO+gIoy7R32VXIZNxrkMYzaeerqSsMwxdhEjyOscT+rJbRGZ+9iPOGeh4 -AqZlzzOuxQ/Lg5h+2mGVXe0Avb+A2zC56mLSQCL3W8NjABUZdknnc1YIf9Dz05fy4jPEttNS -y+Rzte0ITLH1Hy/PKBrlF5n+G1/86f3L5n1ZZXmV3vi+rXT/OyEh9xRS4usmR6kVh4o2XGlI -zUrUjhZvb4lxrHfWgzKlWFoUSydaZDk7eikTKF692RiSSpLbDLW2sNOdzT2eqv2B8CJRF5sL -bD6BB3dAbH7KfqKiCT3xcCZhNEZw+M+GcRO/HNbnABEBAAGJAR8EGAECAAkFAlPfqCcCGwwA -CgkQpqGbONPYMe+sNQgAwjm9PJ45t7NBNTXn1zadoQQbPqz9qAlWiII0k+zzJCTTVqgyIXJY -I6zdNiB/Oh1Xajs/T9z9tL54+LLqgtZKa0lzDOmcxn6Iujf3a1MFdYxKgaQtT2ADxAimuBoz -3Y1ohxXgAs2+VISWYoPBI+UWhYqg11zq3uwpFIYQBRgkVydCxefCxY19okNp9FPC7KJPpJkO -NgDAK693Y9mOZXSq+XeGhjy3Sxesl0PYLIfV33z+vCpc2o1dDA5wuycgfqupNQITkQm6gPOH -1jLu8Vttm4fdEtVMcqkn8dJFomo3JW3qxI7IWwjbVRg10G8LGAuBbD6CA0dGSf8PkHFYv2Xs -dQ== -=MWcF ------END PGP PUBLIC KEY BLOCK----- \ No newline at end of file diff --git a/build/docker/centos7/devel/Dockerfile b/build/docker/centos7/devel/Dockerfile deleted file mode 100644 index 98f1923c17..0000000000 --- a/build/docker/centos7/devel/Dockerfile +++ /dev/null @@ -1,113 +0,0 @@ -ARG REPOSITORY=foundationdb/build -ARG VERSION=centos7-latest -FROM ${REPOSITORY}:${VERSION} - -# add vscode server -RUN yum-config-manager --add-repo=https://copr.fedorainfracloud.org/coprs/carlwgeorge/ripgrep/repo/epel-7/carlwgeorge-ripgrep-epel-7.repo && \ - yum repolist && \ - yum -y install \ - bash-completion \ - byobu \ - cgdb \ - emacs-nox \ - fish \ - jq \ - ripgrep \ - the_silver_searcher \ - tmux \ - tree \ - vim \ - zsh && \ - yum clean all && \ - rm -rf /var/cache/yum - -WORKDIR /tmp -RUN source /opt/rh/devtoolset-8/enable && \ - source /opt/rh/rh-python36/enable && \ - pip3 install \ - lxml \ - psutil \ - python-dateutil \ - subprocess32 && \ - mkdir fdb-joshua && \ - cd fdb-joshua && \ - git clone https://github.com/FoundationDB/fdb-joshua . && \ - pip3 install /tmp/fdb-joshua && \ - cd /tmp && \ - curl -Ls https://amazon-eks.s3.us-west-2.amazonaws.com/1.18.9/2020-11-02/bin/linux/amd64/kubectl -o kubectl && \ - echo "3dbe69e6deb35fbd6fec95b13d20ac1527544867ae56e3dae17e8c4d638b25b9 kubectl" > kubectl.txt && \ - sha256sum -c kubectl.txt && \ - mv kubectl /usr/local/bin/kubectl && \ - chmod 755 /usr/local/bin/kubectl && \ - curl https://awscli.amazonaws.com/awscli-exe-linux-x86_64-2.0.30.zip -o "awscliv2.zip" && \ - echo "7ee475f22c1b35cc9e53affbf96a9ffce91706e154a9441d0d39cbf8366b718e awscliv2.zip" > awscliv2.txt && \ - sha256sum -c awscliv2.txt && \ - unzip -qq awscliv2.zip && \ - ./aws/install && \ - rm -rf /tmp/* - -ARG FDB_VERSION="6.2.29" -RUN mkdir -p /usr/lib/foundationdb/plugins && \ - curl -Ls https://www.foundationdb.org/downloads/misc/joshua_tls_library.tar.gz | \ - tar --strip-components=1 --no-same-owner --directory /usr/lib/foundationdb/plugins -xz && \ - ln -sf /usr/lib/foundationdb/plugins/FDBGnuTLS.so /usr/lib/foundationdb/plugins/fdb-libressl-plugin.so && \ - curl -Ls https://www.foundationdb.org/downloads/${FDB_VERSION}/linux/libfdb_c_${FDB_VERSION}.so -o /usr/lib64/libfdb_c_${FDB_VERSION}.so && \ - ln -sf /usr/lib64/libfdb_c_${FDB_VERSION}.so /usr/lib64/libfdb_c.so - -WORKDIR /root -RUN curl -Ls https://update.code.visualstudio.com/latest/server-linux-x64/stable -o /tmp/vscode-server-linux-x64.tar.gz && \ - mkdir -p .vscode-server/bin/latest && \ - tar --strip-components 1 --no-same-owner --directory .vscode-server/bin/latest -xf /tmp/vscode-server-linux-x64.tar.gz && \ - touch .vscode-server/bin/latest/0 && \ - rm -rf /tmp/* -RUN rm -f /root/anaconda-ks.cfg && \ - printf '%s\n' \ - '#!/usr/bin/env bash' \ - 'set -Eeuo pipefail' \ - '' \ - 'mkdir -p ~/.docker' \ - 'cat > ~/.docker/config.json << EOF' \ - '{' \ - ' "proxies":' \ - ' {' \ - ' "default":' \ - ' {' \ - ' "httpProxy": "${HTTP_PROXY}",' \ - ' "httpsProxy": "${HTTPS_PROXY}",' \ - ' "noProxy": "${NO_PROXY}"' \ - ' }' \ - ' }' \ - '}' \ - 'EOF' \ - > docker_proxy.sh && \ - chmod 755 docker_proxy.sh && \ - printf '%s\n' \ - 'source /opt/rh/devtoolset-8/enable' \ - 'source /opt/rh/rh-python36/enable' \ - 'source /opt/rh/rh-ruby26/enable' \ - '' \ - 'function cmk_ci() {' \ - ' cmake -S ${HOME}/src/foundationdb -B ${HOME}/build_output -D USE_CCACHE=ON -D USE_WERROR=ON -D RocksDB_ROOT=/opt/rocksdb-6.10.1 -D RUN_JUNIT_TESTS=ON -D RUN_JAVA_INTEGRATION_TESTS=ON -G Ninja && \' \ - ' ninja -v -C ${HOME}/build_output -j 84 all packages strip_targets' \ - '}' \ - 'function cmk() {' \ - ' cmake -S ${HOME}/src/foundationdb -B ${HOME}/build_output -D USE_CCACHE=ON -D USE_WERROR=ON -D RocksDB_ROOT=/opt/rocksdb-6.10.1 -D RUN_JUNIT_TESTS=ON -D RUN_JAVA_INTEGRATION_TESTS=ON -G Ninja && \' \ - ' ninja -C ${HOME}/build_output -j 84' \ - '}' \ - 'function ct() {' \ - ' cd ${HOME}/build_output && ctest -j 32 --no-compress-output -T test --output-on-failure' \ - '}' \ - 'function j() {' \ - ' python3 -m joshua.joshua "${@}"' \ - '}' \ - 'function jsd() {' \ - ' j start --tarball $(find ${HOME}/build_output/packages -name correctness\*.tar.gz) "${@}"' \ - '}' \ - '' \ - 'USER_BASHRC="$HOME/src/.bashrc.local"' \ - 'if test -f "$USER_BASHRC"; then' \ - ' source $USER_BASHRC' \ - 'fi' \ - '' \ - 'bash ${HOME}/docker_proxy.sh' \ - >> .bashrc diff --git a/build/docker/centos7/distcc/Dockerfile b/build/docker/centos7/distcc/Dockerfile deleted file mode 100644 index 785e6bee93..0000000000 --- a/build/docker/centos7/distcc/Dockerfile +++ /dev/null @@ -1,24 +0,0 @@ -ARG REPOSITORY=foundationdb/build -ARG VERSION=centos7-latest -FROM ${REPOSITORY}:${VERSION} - -RUN useradd distcc && \ - source /opt/rh/devtoolset-8/enable && \ - source /opt/rh/rh-python36/enable && \ - update-distcc-symlinks - -EXPOSE 3632 -EXPOSE 3633 -USER distcc -ENV ALLOW 0.0.0.0/0 - -ENTRYPOINT distccd \ - --daemon \ - --enable-tcp-insecure \ - --no-detach \ - --port 3632 \ - --log-stderr \ - --log-level info \ - --listen 0.0.0.0 \ - --allow ${ALLOW} \ - --jobs `nproc` \ No newline at end of file diff --git a/build/docker/centos7/ycsb/Dockerfile b/build/docker/centos7/ycsb/Dockerfile deleted file mode 100644 index a8b60230b3..0000000000 --- a/build/docker/centos7/ycsb/Dockerfile +++ /dev/null @@ -1,20 +0,0 @@ -ARG REPOSITORY=foundationdb/build -ARG VERSION=centos7-latest -FROM ${REPOSITORY}:${VERSION} - -ENV YCSB_VERSION=ycsb-foundationdb-binding-0.17.0 \ - PATH=${PATH}:/usr/bin - -RUN cd /opt \ - && eval curl "-Ls https://github.com/brianfrankcooper/YCSB/releases/download/0.17.0/ycsb-foundationdb-binding-0.17.0.tar.gz" \ - | tar -xzvf - - -RUN rm -Rf /opt/${YCSB_VERSION}/lib/fdb-java-5.2.5.jar - -# COPY The Appropriate fdb-java-.jar Aaron from packages -# COPY binary RPM for foundationd-db -# Install Binary - -WORKDIR "/opt/${YCSB_VERSION}" - -ENTRYPOINT ["bin/ycsb.sh"] \ No newline at end of file diff --git a/build/gen_dev_docker.sh b/build/gen_dev_docker.sh deleted file mode 100755 index 89129d5a86..0000000000 --- a/build/gen_dev_docker.sh +++ /dev/null @@ -1,99 +0,0 @@ -#!/usr/bin/env bash - -set -e - -# we first check whether the user is in the group docker -user=$(id -un) -DIR_UUID=$(cat /dev/urandom | tr -dc 'a-zA-Z0-9' | fold -w 32 | head -n 1) -group=$(id -gn) -uid=$(id -u) -gid=$(id -g) -gids=( $(id -G) ) -groups=( $(id -Gn) ) -tmpdir="/tmp/fdb-docker-${DIR_UUID}" -image=fdb-dev - -pushd . -mkdir ${tmpdir} -cd ${tmpdir} - -echo - -cat <<EOF >> Dockerfile -FROM foundationdb/foundationdb-dev:0.11.1 -RUN yum install -y sudo -RUN echo '%sudo ALL=(ALL) NOPASSWD:ALL' >> /etc/sudoers -RUN groupadd -g 1100 sudo -EOF - -num_groups=${#gids[@]} -additional_groups="-G sudo" -for ((i=0;i<num_groups;i++)) -do - echo "RUN groupadd -g ${gids[$i]} ${groups[$i]} || true" >> Dockerfile - if [ ${gids[i]} -ne ${gid} ] - then - additional_groups="${additional_groups},${gids[$i]}" - fi -done - -cat <<EOF >> Dockerfile -RUN useradd -u ${uid} -g ${gid} ${additional_groups} -m ${user} - -USER ${user} -CMD scl enable devtoolset-8 rh-python36 rh-ruby24 -- bash - -EOF - -echo "Created ${tmpdir}" -echo "Buidling Docker container ${image}" -sudo docker build -t ${image} . - -popd - -echo "Writing startup script" -mkdir -p $HOME/bin -cat <<EOF > $HOME/bin/fdb-dev -#!/usr/bin/bash - -if [ -d "\${CCACHE_DIR}" ] -then - args="-v \${CCACHE_DIR}:\${CCACHE_DIR}" - args="\${args} -e CCACHE_DIR=\${CCACHE_DIR}" - args="\${args} -e CCACHE_UMASK=\${CCACHE_UMASK}" - ccache_args=\$args -fi - -if [ -t 1 ] ; then - TERMINAL_ARGS=-it `# Run in interactive mode and simulate a TTY` -else - TERMINAL_ARGS=-i `# Run in interactive mode` -fi - -sudo docker run --rm `# delete (temporary) image after return` \\ - \${TERMINAL_ARGS} \\ - --privileged=true `# Run in privileged mode ` \\ - --cap-add=SYS_PTRACE \\ - --security-opt seccomp=unconfined \\ - -v "${HOME}:${HOME}" `# Mount home directory` \\ - -w="\$(pwd)" \\ - \${ccache_args} \\ - ${image} "\$@" -EOF - -cat <<EOF > $HOME/bin/clangd -#!/usr/bin/bash - -fdb-dev scl enable devtoolset-8 rh-python36 rh-ruby24 -- clangd -EOF - -if [[ ":$PATH:" != *":$HOME/bin:"* ]] -then - echo "WARNING: $HOME/bin is not in your PATH!" - echo -e "\tThis can cause problems with some scripts (like fdb-clangd)" -fi -chmod +x $HOME/bin/fdb-dev -chmod +x $HOME/bin/clangd -echo "To start the dev docker image run $HOME/bin/fdb-dev" -echo "$HOME/bin/clangd can be used for IDE integration" -echo "You can edit these files but be aware that this script will overwrite your changes if you rerun it" diff --git a/build/get_package_name.sh b/build/get_package_name.sh deleted file mode 100755 index c2c94d126b..0000000000 --- a/build/get_package_name.sh +++ /dev/null @@ -1,3 +0,0 @@ -#!/usr/bin/env bash - -cat $1 | grep '<PackageName>' | sed -e 's,^[^>]*>,,' -e 's,<.*,,' diff --git a/build/get_version.sh b/build/get_version.sh deleted file mode 100755 index a7a2a179f2..0000000000 --- a/build/get_version.sh +++ /dev/null @@ -1,4 +0,0 @@ -#!/usr/bin/env bash - -cat $1 | grep '<Version>' | sed -e 's,^[^>]*>,,' -e 's,<.*,,' - diff --git a/build/txt-to-toml.py b/build/txt-to-toml.py deleted file mode 100755 index 68d1dcdbb5..0000000000 --- a/build/txt-to-toml.py +++ /dev/null @@ -1,63 +0,0 @@ -#!/usr/bin/env python -import sys - - -def main(): - if len(sys.argv) != 2: - print("Usage: txt-to-toml.py [src.txt]") - return 1 - - filename = sys.argv[1] - - indent = " " - in_workload = False - first_test = False - keys_before_test = False - - for line in open(filename): - k = "" - v = "" - - if line.strip().startswith(";"): - print((indent if in_workload else "") + line.strip().replace(";", "#")) - continue - - if "=" in line: - (k, v) = line.strip().split("=") - (k, v) = (k.strip(), v.strip()) - - if k == "testTitle": - first_test = True - if in_workload: - print("") - in_workload = False - if keys_before_test: - print("") - keys_before_test = False - print("[[test]]") - - if k == "testName": - in_workload = True - print("") - print(indent + "[[test.workload]]") - - if not first_test: - keys_before_test = True - - if v.startswith("."): - v = "0" + v - - if any(c.isalpha() or c in ["/", "!"] for c in v): - if v != "true" and v != "false": - v = "'" + v + "'" - - if k == "buggify": - print("buggify = " + ("true" if v == "'on'" else "false")) - elif k: - print((indent if in_workload else "") + k + " = " + v) - - return 0 - - -if __name__ == "__main__": - sys.exit(main()) diff --git a/build/gen_compile_db.py b/contrib/gen_compile_db.py similarity index 100% rename from build/gen_compile_db.py rename to contrib/gen_compile_db.py