diff --git a/cmake/InstallLayout.cmake b/cmake/InstallLayout.cmake index 6caa871759..a0e5285cba 100644 --- a/cmake/InstallLayout.cmake +++ b/cmake/InstallLayout.cmake @@ -185,12 +185,6 @@ install(DIRECTORY "${script_dir}/clients/usr/lib/cmake" DESTINATION usr/lib COMPONENT clients-versioned) -################################################################################ -# Move Docker Setup -################################################################################ - -file(COPY "${PROJECT_SOURCE_DIR}/packaging/docker" DESTINATION "${PROJECT_BINARY_DIR}/packages/") - ################################################################################ # General CPack configuration ################################################################################ @@ -228,7 +222,7 @@ set(CPACK_COMPONENT_CLIENTS-TGZ_DISPLAY_NAME "foundationdb-clients") set(CPACK_COMPONENT_CLIENTS-VERSIONED_DISPLAY_NAME "foundationdb${PROJECT_VERSION}-clients") -# MacOS needs a file exiension for the LICENSE file +# MacOS needs a file extension for the LICENSE file configure_file(${CMAKE_SOURCE_DIR}/LICENSE ${CMAKE_BINARY_DIR}/License.txt COPYONLY) ################################################################################ diff --git a/fdbclient/ServerKnobs.cpp b/fdbclient/ServerKnobs.cpp index f5b10a9bd7..d1ce8e1aca 100644 --- a/fdbclient/ServerKnobs.cpp +++ b/fdbclient/ServerKnobs.cpp @@ -38,7 +38,7 @@ void ServerKnobs::initialize(Randomize randomize, ClientKnobs* clientKnobs, IsSi init( MAX_COMMIT_BATCH_INTERVAL, 2.0 ); if( randomize && BUGGIFY ) MAX_COMMIT_BATCH_INTERVAL = 0.5; // Each commit proxy generates a CommitTransactionBatchRequest at least this often, so that versions always advance smoothly MAX_COMMIT_BATCH_INTERVAL = std::min(MAX_COMMIT_BATCH_INTERVAL, MAX_READ_TRANSACTION_LIFE_VERSIONS/double(2*VERSIONS_PER_SECOND)); // Ensure that the proxy commits 2 times every MAX_READ_TRANSACTION_LIFE_VERSIONS, otherwise the master will not give out versions fast enough init( ENABLE_VERSION_VECTOR, true ); - init( ENABLE_VERSION_VECTOR_TLOG_UNICAST, true ); + init( ENABLE_VERSION_VECTOR_TLOG_UNICAST, false ); // TLogs init( TLOG_TIMEOUT, 0.4 ); //cannot buggify because of availability diff --git a/fdbclient/rapidjson/rapidjson.h b/fdbclient/rapidjson/rapidjson.h index 5567cb868a..1a6d6f3beb 100644 --- a/fdbclient/rapidjson/rapidjson.h +++ b/fdbclient/rapidjson/rapidjson.h @@ -449,7 +449,7 @@ RAPIDJSON_NAMESPACE_END */ #define RAPIDJSON_STATIC_ASSERT(x) \ typedef ::RAPIDJSON_NAMESPACE::StaticAssertTest)> \ - RAPIDJSON_JOIN(StaticAssertTypedef, __LINE__) RAPIDJSON_STATIC_ASSERT_UNUSED_ATTRIBUTE + RAPIDJSON_JOIN(StaticAssertTypedef, __LINE__) RAPIDJSON_STATIC_ASSERT_UNUSED_ATTRIBUTE #endif /////////////////////////////////////////////////////////////////////////////// diff --git a/flow/TDMetric.actor.h b/flow/TDMetric.actor.h index 03287984d3..00aa399f52 100644 --- a/flow/TDMetric.actor.h +++ b/flow/TDMetric.actor.h @@ -343,11 +343,11 @@ auto tuple_map_impl(F f, index_sequence, const Tuples&... ts) // tuple_map( f(a,b), (a1,a2,a3), (b1,b2,b3) ) = (f(a1,b1), f(a2,b2), f(a3,b3)) template -auto tuple_map(F f, const Tuple& t, const Tuples&... ts) -> decltype( - tuple_map_impl(f, - typename make_index_sequence_impl<0, index_sequence<>, std::tuple_size::value>::type(), - t, - ts...)) { +auto tuple_map(F f, const Tuple& t, const Tuples&... ts) -> decltype(tuple_map_impl( + f, + typename make_index_sequence_impl<0, index_sequence<>, std::tuple_size::value>::type(), + t, + ts...)) { return tuple_map_impl( f, typename make_index_sequence_impl<0, index_sequence<>, std::tuple_size::value>::type(), t, ts...); } diff --git a/packaging/docker/Dockerfile b/packaging/docker/Dockerfile new file mode 100644 index 0000000000..c76fc2e9f1 --- /dev/null +++ b/packaging/docker/Dockerfile @@ -0,0 +1,194 @@ +# Dockerfile +# +# This source file is part of the FoundationDB open source project +# +# Copyright 2013-2021 Apple Inc. and the FoundationDB project authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +FROM centos:7.9.2009 as base + +RUN yum install -y \ + epel-release-7-11 \ + centos-release-scl-2-3.el7.centos && \ + yum install -y \ + bind-utils-9.11.4-26.P2.el7_9.7 \ + binutils-2.27-44.base.el7_9.1 \ + curl-7.29.0-59.el7_9.1 \ + gdb-7.6.1-120.el7 \ + hostname-3.13-3.el7_7.1 \ + jq-1.6-2.el7 \ + less-458-9.el7 \ + libubsan-7.3.1-5.16.el7 \ + lsof-4.87-6.el7 \ + net-tools-2.0-0.25.20131004git.el7 \ + nmap-ncat-6.40-19.el7 \ + perf-3.10.0-1160.45.1.el7 \ + perl-5.16.3-299.el7_9 \ + procps-ng-3.3.10-28.el7 \ + strace-4.24-6.el7 \ + sysstat-10.1.5-19.el7 \ + tar-1.26-35.el7 \ + tcpdump-4.9.2-4.el7_7.1 \ + telnet-0.17-66.el7 \ + traceroute-2.0.22-2.el7 \ + unzip-6.0-22.el7_9 \ + vim-enhanced-7.4.629-8.el7_9 && \ + yum clean all && \ + rm -rf /var/cache/yum + +WORKDIR /tmp + +RUN curl -Ls https://github.com/krallin/tini/releases/download/v0.19.0/tini-amd64 -o tini && \ + echo "93dcc18adc78c65a028a84799ecf8ad40c936fdfc5f2a57b1acda5a8117fa82c tini" > tini-amd64.sha256sum && \ + sha256sum -c tini-amd64.sha256sum && \ + chmod +x tini && \ + mv tini /usr/bin/ && \ + rm -rf /tmp/* + +WORKDIR / + +FROM golang:1.16.7-bullseye AS go-build + +COPY fdbkubernetesmonitor/ /fdbkubernetesmonitor +WORKDIR /fdbkubernetesmonitor +RUN go build -o /fdb-kubernetes-monitor *.go + +FROM base as foundationdb-base + +WORKDIR /tmp +ARG FDB_VERSION=6.3.22 +ARG FDB_LIBRARY_VERSIONS="${FDB_VERSION}" +ARG FDB_WEBSITE=https://www.foundationdb.org + +RUN mkdir -p /var/fdb/{logs,tmp,lib} && \ + mkdir -p /usr/lib/fdb/multiversion && \ + echo ${FDB_VERSION} > /var/fdb/version + +# Set up a non-root user +RUN groupadd --gid 4059 fdb && \ + useradd --gid 4059 --uid 4059 --no-create-home --shell /bin/bash fdb && \ + chown -R fdb:fdb /var/fdb + +COPY website /tmp/website/ + +# Install FoundationDB Binaries +RUN curl $FDB_WEBSITE/downloads/$FDB_VERSION/linux/fdb_$FDB_VERSION.tar.gz | tar zxf - --strip-components=1 && \ + for file in fdbbackup fdbcli fdbdr fdbmonitor fdbrestore fdbserver backup_agent dr_agent fastrestore_tool; do \ + chmod u+x $file; \ + mv $file /usr/bin; \ + done + +# Install additional FoundationDB Client Libraries +RUN for version in $FDB_LIBRARY_VERSIONS; do \ + curl $FDB_WEBSITE/downloads/$version/linux/libfdb_c_$version.so -o /usr/lib/fdb/multiversion/libfdb_c_${version%.*}.so; \ + done + +# Install additional FoundationDB Client Libraries (for sidecar) +RUN mkdir -p /var/fdb/lib && \ + for version in $FDB_LIBRARY_VERSIONS; do \ + curl $FDB_WEBSITE/downloads/$version/linux/libfdb_c_$version.so -o /var/fdb/lib/libfdb_c_${version%.*}.so; \ + done + +RUN curl $FDB_WEBSITE/downloads/$FDB_VERSION/linux/libfdb_c_$FDB_VERSION.so -o /usr/lib/libfdb_c.so + +RUN rm -rf /tmp/* +WORKDIR / + +FROM foundationdb-base as fdb-kubernetes-monitor + +# Install the kubernetes monitor binary +COPY --from=go-build /fdb-kubernetes-monitor /usr/bin/ + +# Runtime Configuration Options +USER fdb +WORKDIR /var/fdb +VOLUME /var/fdb/data +ENTRYPOINT ["/usr/bin/fdb-kubernetes-monitor"] + +FROM foundationdb-base as foundationdb-kubernetes-sidecar + +RUN yum -y install \ + rh-python38-2.0-4.el7 \ + yum clean all && \ + rm -rf /var/cache/yum && \ + source /opt/rh/rh-python38/enable && \ + pip3 install watchdog==0.9.0 + +WORKDIR / +ADD entrypoint.bash sidecar.py / +RUN chmod a+x /entrypoint.bash /sidecar.py +USER fdb +VOLUME /var/input-files +VOLUME /var/output-files +ENV LISTEN_PORT 8080 +ENTRYPOINT ["/usr/bin/tini", "-g", "--", "/entrypoint.bash"] + +FROM foundationdb-base as foundationdb + +WORKDIR /tmp +RUN curl -LsO https://raw.githubusercontent.com/brendangregg/FlameGraph/90533539b75400297092f973163b8a7b067c66d3/stackcollapse-perf.pl && \ + curl -LsO https://raw.githubusercontent.com/brendangregg/FlameGraph/90533539b75400297092f973163b8a7b067c66d3/flamegraph.pl && \ + echo "a682ac46497d6fdbf9904d1e405d3aea3ad255fcb156f6b2b1a541324628dfc0 flamegraph.pl" > flamegraph.sha256sum && \ + echo "5bcfb73ff2c2ab7bf2ad2b851125064780b58c51cc602335ec0001bec92679a5 stackcollapse-perf.pl" >> flamegraph.sha256sum && \ + sha256sum -c flamegraph.sha256sum && \ + chmod +x stackcollapse-perf.pl flamegraph.pl && \ + mv stackcollapse-perf.pl flamegraph.pl /usr/bin && \ + rm -rf /tmp/* +WORKDIR / +# Set Up Runtime Scripts and Directories +ADD fdb.bash /var/fdb/scripts/ +RUN chmod a+x /var/fdb/scripts/fdb.bash +VOLUME /var/fdb/data +ENV FDB_PORT 4500 +ENV FDB_CLUSTER_FILE /var/fdb/fdb.cluster +ENV FDB_NETWORKING_MODE container +ENV FDB_COORDINATOR "" +ENV FDB_COORDINATOR_PORT 4500 +ENV FDB_CLUSTER_FILE_CONTENTS "" +ENV FDB_PROCESS_CLASS unset +ENTRYPOINT ["/usr/bin/tini", "-g", "--", "/var/fdb/scripts/fdb.bash"] + +FROM base as ycsb + +RUN yum -y install \ + java-11-openjdk-11.0.13.0.8-1.el7_9 && \ + yum clean all && \ + rm -rf /var/cache/yum + +WORKDIR /tmp +RUN curl -Ls https://amazon-eks.s3.amazonaws.com/1.19.6/2021-01-05/bin/linux/amd64/kubectl -o kubectl && \ + echo "08ff68159bbcb844455167abb1d0de75bbfe5ae1b051f81ab060a1988027868a kubectl" > kubectl.txt && \ + sha256sum -c kubectl.txt && \ + mv kubectl /usr/local/bin/kubectl && \ + chmod 755 /usr/local/bin/kubectl && \ + curl https://awscli.amazonaws.com/awscli-exe-linux-x86_64-2.2.43.zip -o "awscliv2.zip" && \ + echo "9a8b3c4e7f72bbcc55e341dce3af42479f2730c225d6d265ee6f9162cfdebdfd awscliv2.zip" > awscliv2.txt && \ + sha256sum -c awscliv2.txt && \ + unzip -qq awscliv2.zip && \ + ./aws/install && \ + rm -rf /tmp/* + +# TODO: Log4J complains that it's eating the HTracer logs. Even without it, we get per-operation +# time series graphs of throughput, median, 90, 99, 99.9 and 99.99 (in usec). +ADD run_ycsb.sh /usr/local/bin/run_ycsb.sh +RUN mkdir -p /var/log/fdb-trace-logs && \ + chmod +x /usr/local/bin/run_ycsb.sh + +ADD YCSB /YCSB +WORKDIR /YCSB +ENV FDB_NETWORK_OPTION_EXTERNAL_CLIENT_DIRECTORY=/var/dynamic-conf/lib/multiversion/ +ENV FDB_NETWORK_OPTION_TRACE_ENABLE=/var/log/fdb-trace-logs +ENV LD_LIBRARY_PATH=/var/dynamic-conf/lib/ +ENV BUCKET="" +CMD ["run_ycsb.sh"] diff --git a/packaging/docker/Dockerfile.eks b/packaging/docker/Dockerfile.eks index bc05b4d5a6..d08e25bd5e 100644 --- a/packaging/docker/Dockerfile.eks +++ b/packaging/docker/Dockerfile.eks @@ -1,76 +1,152 @@ +# Dockerfile +# +# This source file is part of the FoundationDB open source project +# +# Copyright 2013-2021 Apple Inc. and the FoundationDB project authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# FROM amazonlinux:2.0.20210326.0 as base RUN yum install -y \ - binutils \ - bind-utils \ - curl \ - gdb \ - jq \ - less \ - lsof \ - nc \ - net-tools \ - perf \ - perl \ - procps \ - python38 \ - python3-pip \ - strace \ - tar \ - traceroute \ - telnet \ - tcpdump \ - unzip \ - vim + bind-utils-9.11.4-26.P2.amzn2.5.2 \ + binutils-2.29.1-30.amzn2 \ + curl-7.76.1-7.amzn2.0.2 \ + gdb-8.0.1-36.amzn2.0.1 \ + hostname-3.13-3.amzn2.0.2 \ + jq-1.5-1.amzn2.0.2 \ + less-458-9.amzn2.0.2 \ + libsanitizer-7.3.1-13.amzn2 \ + lsof-4.87-6.amzn2 \ + net-tools-2.0-0.22.20131004git.amzn2.0.2 \ + nmap-ncat-6.40-13.amzn2 \ + perf-4.14.252-195.483.amzn2 \ + perl-5.16.3-299.amzn2.0.1 \ + procps-ng-3.3.10-26.amzn2 \ + strace-4.26-1.amzn2.0.1 \ + sysstat-10.1.5-12.amzn2 \ + tar-1.26-35.amzn2 \ + tcpdump-4.9.2-4.amzn2.1 \ + telnet-0.17-65.amzn2 \ + traceroute-2.0.22-2.amzn2.0.1 \ + unzip-6.0-43.amzn2 \ + vim-enhanced-8.1.1602-1.amzn2 && \ + yum clean all && \ + rm -rf /var/cache/yum -#todo: nload, iperf, numademo +WORKDIR /tmp -RUN curl "https://awscli.amazonaws.com/awscli-exe-linux-x86_64-2.0.30.zip" -o "awscliv2.zip" \ - && unzip awscliv2.zip && ./aws/install && rm -rf aws +RUN curl -Ls https://github.com/krallin/tini/releases/download/v0.19.0/tini-amd64 -o tini && \ + echo "93dcc18adc78c65a028a84799ecf8ad40c936fdfc5f2a57b1acda5a8117fa82c tini" > tini-amd64.sha256sum && \ + sha256sum -c tini-amd64.sha256sum && \ + chmod +x tini && \ + mv tini /usr/bin/ && \ + rm -rf /tmp/* -COPY misc/tini-amd64.sha256sum /tmp/ -COPY misc/flamegraph.sha256sum /tmp/ -# Adding tini as PID 1 https://github.com/krallin/tini -ARG TINI_VERSION=v0.19.0 -RUN curl -sLO https://github.com/krallin/tini/releases/download/${TINI_VERSION}/tini-amd64 && \ - sha256sum -c /tmp/tini-amd64.sha256sum && \ - chmod +x tini-amd64 && \ - mv tini-amd64 /usr/bin/tini +WORKDIR / -COPY sidecar/requirements.txt /tmp -RUN pip3 install -r /tmp/requirements.txt +FROM golang:1.16.7-bullseye AS go-build -# Install flamegraph -RUN curl -sLO https://raw.githubusercontent.com/brendangregg/FlameGraph/90533539b75400297092f973163b8a7b067c66d3/stackcollapse-perf.pl && \ - curl -sLO https://raw.githubusercontent.com/brendangregg/FlameGraph/90533539b75400297092f973163b8a7b067c66d3/flamegraph.pl && \ - sha256sum -c /tmp/flamegraph.sha256sum && \ - chmod +x stackcollapse-perf.pl flamegraph.pl && \ - mv stackcollapse-perf.pl flamegraph.pl /usr/bin +COPY fdbkubernetesmonitor/ /fdbkubernetesmonitor +WORKDIR /fdbkubernetesmonitor +RUN go build -o /fdb-kubernetes-monitor *.go -# TODO: Only used by sidecar +FROM base as foundationdb-base + +WORKDIR /tmp +ARG FDB_VERSION=6.3.22 +ARG FDB_LIBRARY_VERSIONS="${FDB_VERSION}" +ARG FDB_WEBSITE=https://www.foundationdb.org + +RUN mkdir -p /var/fdb/{logs,tmp,lib} && \ + mkdir -p /usr/lib/fdb/multiversion && \ + echo ${FDB_VERSION} > /var/fdb/version + +# Set up a non-root user RUN groupadd --gid 4059 fdb && \ - useradd --gid 4059 --uid 4059 --no-create-home --shell /bin/bash fdb + useradd --gid 4059 --uid 4059 --no-create-home --shell /bin/bash fdb && \ + chown -R fdb:fdb /var/fdb -ARG FDB_VERSION +COPY website /tmp/website/ -# These are the output of the current build (not stripped) -COPY --chown=root bin /usr/bin/ -COPY --chown=root lib/libfdb_c.so /var/fdb/lib/ -RUN mv /var/fdb/lib/libfdb_c.so /var/fdb/lib/libfdb_c_${FDB_VERSION%.*}.so -RUN ln -s /var/fdb/lib/libfdb_c_${FDB_VERSION%.*}.so /var/fdb/lib/libfdb_c.so -# ------------------------------------------------- +# Install FoundationDB Binaries +RUN curl $FDB_WEBSITE/downloads/$FDB_VERSION/linux/fdb_$FDB_VERSION.tar.gz | tar zxf - --strip-components=1 && \ + for file in fdbbackup fdbcli fdbdr fdbmonitor fdbrestore fdbserver backup_agent dr_agent fastrestore_tool; do \ + chmod u+x $file; \ + mv $file /usr/bin; \ + done -FROM base as foundationdb +# Install additional FoundationDB Client Libraries +RUN for version in $FDB_LIBRARY_VERSIONS; do \ + curl $FDB_WEBSITE/downloads/$version/linux/libfdb_c_$version.so -o /usr/lib/fdb/multiversion/libfdb_c_${version%.*}.so; \ + done -COPY release/*.bash /var/fdb/scripts/ -RUN mkdir -p /var/fdb/logs +# Install additional FoundationDB Client Libraries (for sidecar) +RUN mkdir -p /var/fdb/lib && \ + for version in $FDB_LIBRARY_VERSIONS; do \ + curl $FDB_WEBSITE/downloads/$version/linux/libfdb_c_$version.so -o /var/fdb/lib/libfdb_c_${version%.*}.so; \ + done -# TODO: FDB_ADDITIONAL_VERSIONS -RUN mkdir -p /usr/lib/fdb/multiversion +RUN curl $FDB_WEBSITE/downloads/$FDB_VERSION/linux/libfdb_c_$FDB_VERSION.so -o /usr/lib/libfdb_c.so -VOLUME /var/fdb/data +RUN rm -rf /tmp/* +WORKDIR / + +FROM foundationdb-base as fdb-kubernetes-monitor + +# Install the kubernetes monitor binary +COPY --from=go-build /fdb-kubernetes-monitor /usr/bin/ # Runtime Configuration Options +USER fdb +WORKDIR /var/fdb +VOLUME /var/fdb/data +ENTRYPOINT ["/usr/bin/fdb-kubernetes-monitor"] + +FROM foundationdb-base as foundationdb-kubernetes-sidecar + +RUN yum -y install \ + python3-3.7.10-1.amzn2.0.1 \ + python3-pip-20.2.2-1.amzn2.0.3 && \ + yum clean all && \ + rm -rf /var/cache/yum && \ + pip3 install watchdog==0.9.0 + +WORKDIR / +ADD entrypoint.bash sidecar.py / +RUN chmod a+x /entrypoint.bash /sidecar.py +USER fdb +VOLUME /var/input-files +VOLUME /var/output-files +ENV LISTEN_PORT 8080 +ENTRYPOINT ["/usr/bin/tini", "-g", "--", "/entrypoint.bash"] + +FROM foundationdb-base as foundationdb + +WORKDIR /tmp +RUN curl -LsO https://raw.githubusercontent.com/brendangregg/FlameGraph/90533539b75400297092f973163b8a7b067c66d3/stackcollapse-perf.pl && \ + curl -LsO https://raw.githubusercontent.com/brendangregg/FlameGraph/90533539b75400297092f973163b8a7b067c66d3/flamegraph.pl && \ + echo "a682ac46497d6fdbf9904d1e405d3aea3ad255fcb156f6b2b1a541324628dfc0 flamegraph.pl" > flamegraph.sha256sum && \ + echo "5bcfb73ff2c2ab7bf2ad2b851125064780b58c51cc602335ec0001bec92679a5 stackcollapse-perf.pl" >> flamegraph.sha256sum && \ + sha256sum -c flamegraph.sha256sum && \ + chmod +x stackcollapse-perf.pl flamegraph.pl && \ + mv stackcollapse-perf.pl flamegraph.pl /usr/bin && \ + rm -rf /tmp/* +WORKDIR / +# Set Up Runtime Scripts and Directories +ADD fdb.bash /var/fdb/scripts/ +RUN chmod a+x /var/fdb/scripts/fdb.bash +VOLUME /var/fdb/data ENV FDB_PORT 4500 ENV FDB_CLUSTER_FILE /var/fdb/fdb.cluster ENV FDB_NETWORKING_MODE container @@ -78,28 +154,38 @@ ENV FDB_COORDINATOR "" ENV FDB_COORDINATOR_PORT 4500 ENV FDB_CLUSTER_FILE_CONTENTS "" ENV FDB_PROCESS_CLASS unset +ENTRYPOINT ["/usr/bin/tini", "-g", "--", "/var/fdb/scripts/fdb.bash"] -ENTRYPOINT ["/usr/bin/tini", "-g", "--"] -CMD /var/fdb/scripts/fdb.bash +FROM base as ycsb -# ------------------------------------------------- +RUN yum -y install \ + java-11-amazon-corretto-11.0.13+8-1.amzn2 && \ + yum clean all && \ + rm -rf /var/cache/yum -FROM base AS sidecar +WORKDIR /tmp +RUN curl -Ls https://amazon-eks.s3.amazonaws.com/1.19.6/2021-01-05/bin/linux/amd64/kubectl -o kubectl && \ + echo "08ff68159bbcb844455167abb1d0de75bbfe5ae1b051f81ab060a1988027868a kubectl" > kubectl.txt && \ + sha256sum -c kubectl.txt && \ + mv kubectl /usr/local/bin/kubectl && \ + chmod 755 /usr/local/bin/kubectl && \ + curl https://awscli.amazonaws.com/awscli-exe-linux-x86_64-2.2.43.zip -o "awscliv2.zip" && \ + echo "9a8b3c4e7f72bbcc55e341dce3af42479f2730c225d6d265ee6f9162cfdebdfd awscliv2.zip" > awscliv2.txt && \ + sha256sum -c awscliv2.txt && \ + unzip -qq awscliv2.zip && \ + ./aws/install && \ + rm -rf /tmp/* -COPY sidecar/entrypoint.bash / -COPY sidecar/sidecar.py / -RUN chmod a+x /sidecar.py /entrypoint.bash +# TODO: Log4J complains that it's eating the HTracer logs. Even without it, we get per-operation +# time series graphs of throughput, median, 90, 99, 99.9 and 99.99 (in usec). +ADD run_ycsb.sh /usr/local/bin/run_ycsb.sh +RUN mkdir -p /var/log/fdb-trace-logs && \ + chmod +x /usr/local/bin/run_ycsb.sh -VOLUME /var/input-files -VOLUME /var/output-files - -ARG FDB_VERSION - -RUN echo ${FDB_VERSION} ; echo ${FDB_VERSION}> /var/fdb/version -RUN mkdir -p /var/fdb/lib - -ENV LISTEN_PORT 8080 - -USER fdb - -ENTRYPOINT ["/usr/bin/tini", "-g", "--", "/entrypoint.bash"] \ No newline at end of file +ADD YCSB /YCSB +WORKDIR /YCSB +ENV FDB_NETWORK_OPTION_EXTERNAL_CLIENT_DIRECTORY=/var/dynamic-conf/lib/multiversion/ +ENV FDB_NETWORK_OPTION_TRACE_ENABLE=/var/log/fdb-trace-logs +ENV LD_LIBRARY_PATH=/var/dynamic-conf/lib/ +ENV BUCKET="" +CMD ["run_ycsb.sh"] diff --git a/packaging/docker/build-eks-docker.sh b/packaging/docker/build-eks-docker.sh deleted file mode 100755 index 3161e559fb..0000000000 --- a/packaging/docker/build-eks-docker.sh +++ /dev/null @@ -1,62 +0,0 @@ -#!/usr/bin/env bash -set -Eeuo pipefail -set -x - -DOCKER_ROOT="$(realpath "$(dirname "${BASH_SOURCE[0]}")")" -BUILD_OUTPUT=$(realpath "${DOCKER_ROOT}"/../..) - -echo Docker root: "${DOCKER_ROOT}" -echo Build output: "${BUILD_OUTPUT}" - -cd "${DOCKER_ROOT}" - -## eg: CMAKE_PROJECT_VERSION:STATIC=7.0.0 -FDB_VERSION=$(grep CMAKE_PROJECT_VERSION: "${BUILD_OUTPUT}"/CMakeCache.txt | cut -d '=' -f 2) - -# Options (passed via environment variables) - -# Feel free to customize the image tag. -# TODO: add a mechanism to set TAG=FDB_VERSION when we're building public releases. -TAG=${TAG:-${FDB_VERSION}-${OKTETO_NAME}} -ECR=${ECR:-112664522426.dkr.ecr.us-west-2.amazonaws.com} - -echo Building with tag "${TAG}" - -# Login to ECR -# TODO: Move this to a common place instead of repeatedly copy-pasting it. -aws ecr get-login-password | docker login --username AWS --password-stdin "${ECR}" - -docker pull "${ECR}"/amazonlinux:2.0.20210326.0 -docker tag "${ECR}"/amazonlinux:2.0.20210326.0 amazonlinux:2.0.20210326.0 - - - -# derived variables -IMAGE=foundationdb/foundationdb:${TAG} -SIDECAR=foundationdb/foundationdb-kubernetes-sidecar:${TAG}-1 -STRIPPED=${STRIPPED:-false} - - - - - -if $STRIPPED; then - rsync -av --delete --exclude=*.xml "${BUILD_OUTPUT}"/packages/bin . - rsync -av --delete --exclude=*.a --exclude=*.xml "${BUILD_OUTPUT}"/packages/lib . -else - rsync -av --delete --exclude=*.xml "${BUILD_OUTPUT}"/bin . - rsync -av --delete --exclude=*.a --exclude=*.xml "${BUILD_OUTPUT}"/lib . -fi - -BUILD_ARGS="--build-arg FDB_VERSION=$FDB_VERSION" - - - -docker build ${BUILD_ARGS} -t "${IMAGE}" --target foundationdb -f Dockerfile.eks . -docker build ${BUILD_ARGS} -t "${SIDECAR}" --target sidecar -f Dockerfile.eks . - -docker tag "${IMAGE}" "${ECR}"/"${IMAGE}" -docker tag "${SIDECAR}" "${ECR}"/"${SIDECAR}" - -docker push "${ECR}"/"${IMAGE}" -docker push "${ECR}"/"${SIDECAR}" diff --git a/packaging/docker/build-images.sh b/packaging/docker/build-images.sh new file mode 100755 index 0000000000..6b0c3ecda7 --- /dev/null +++ b/packaging/docker/build-images.sh @@ -0,0 +1,275 @@ +#!/usr/bin/env bash +set -Eeuo pipefail +script_dir=$(cd "$(dirname "${BASH_SOURCE[0]}")" &>/dev/null && pwd -P) +reset=$(tput sgr0) +blue=$(tput setaf 4) + +function logg() { + printf "${blue}##### $(date +"%H:%M:%S") # %-56.55s #####${reset}\n" "${1}" +} + +function pushd () { + command pushd "$@" > /dev/null +} + +function popd () { + command popd > /dev/null +} + +function create_fake_website_directory() { + fdb_binaries=( 'fdbbackup' 'fdbcli' 'fdbserver' 'fdbmonitor' ) + fake_fdb_binaries=( 'backup_agent' 'dr_agent' 'fastrestore_tool' 'fdbdr' 'fdbrestore' ) + logg "PREPARING WEBSITE" + website_directory="${script_dir}/website" + rm -rf "${website_directory}" + mkdir -p "${website_directory}/downloads/${fdb_version}/linux/bin" + pushd "${website_directory}/downloads/${fdb_version}/linux/bin" || exit 127 + ############################################################################ + # there are four intended paths here: + # 1) fetch the unstripped binaries and client library from artifactory_base_url + # 2) fetch the stripped binaries and multiple client library versions + # from artifactory_base_url + # 3) copy the unstripped binaries and client library from the current local + # build_output of foundationdb + # 4) copy the stripped binaries and client library from the current local + # build_output of foundationdb + ############################################################################ + logg "FETCHING BINARIES" + case "${stripped_binaries_and_from_where}" in + "unstripped_artifactory") + logg "DOWNLOADING BINARIES TAR FILE" + curl -Ls "${artifactory_base_url}/${fdb_version}/release/api/foundationdb-binaries-${fdb_version}-linux.tar.gz" | tar -xzf - + ;; + "stripped_artifactory") + for file in "${fdb_binaries[@]}"; do + logg "DOWNLOADING ${file}" + curl -Ls "${artifactory_base_url}/${fdb_version}/release/files/linux/bin/${file}" -o "${file}" + chmod 755 "${file}" + done + ;; + "unstripped_local") + for file in "${fdb_binaries[@]}"; do + logg "COPYING ${file}" + cp -pr "${build_output_directory}/bin/${file}" "${file}" + chmod 755 "${file}" + done + ;; + "stripped_local") + for file in "${fdb_binaries[@]}"; do + logg "COPYING ${file}" + cp -pr "${build_output_directory}/packaging/bin/${file}" "${file}" + chmod 755 "${file}" + done + ;; + esac + # dont download files that are binary duplicates of fdbbackup, recreate the + # symlinks (to fdbbackup in the same directory) + logg "CREATING fdbbackup SYMLINKS" + for fake in "${fake_fdb_binaries[@]}"; do + logg "CREATING ${fake}" + ln -sf fdbbackup "${fake}" + done + popd || exit 128 + # re-create the same file that is present in the downloads path of + # foundationdb.org such that it can be copied into the Docker image and + # referenced with a file:// url by the container image build + logg "CREATING BINARIES TAR FILE" + tar -czf "${website_directory}/downloads/${fdb_version}/linux/fdb_${fdb_version}.tar.gz" --directory "${website_directory}/downloads/${fdb_version}/linux/bin" . + rm -rf "${website_directory}/downloads/${fdb_version}/linux/bin" + + ############################################################################ + # this follows the same logic as the case statement above, they are separate + # because it allows for the simplification of the steps that create the + # symlinks and the binaries tarball + ############################################################################ + logg "FETCHING CLIENT LIBRARY" + case "${stripped_binaries_and_from_where}" in + "unstripped_artifactory") + for version in "${fdb_library_versions[@]}"; do + logg "FETCHING ${version} CLIENT LIBRARY" + destination_directory="${website_directory}/downloads/${version}/linux" + destination_filename="libfdb_c_${version}.so" + mkdir -p "${destination_directory}" + pushd "${destination_directory}" || exit 127 + curl -Ls "${artifactory_base_url}/${version}/release/api/fdb-server-${version}-linux.tar.gz" | tar -xzf - ./lib/libfdb_c.so --strip-components 2 + mv "libfdb_c.so" "${destination_filename}" + chmod 755 "${destination_filename}" + popd || exit 128 + done + ;; + "stripped_artifactory") + for version in "${fdb_library_versions[@]}"; do + logg "FETCHING ${version} CLIENT LIBRARY" + destination_directory="${website_directory}/downloads/${version}/linux" + destination_filename="libfdb_c_${version}.so" + mkdir -p "${destination_directory}" + pushd "${destination_directory}" || exit 127 + curl -Ls "${artifactory_base_url}/${version}/release/files/linux/lib/libfdb_c.so" -o "${destination_filename}" + chmod 755 "${destination_filename}" + popd || exit 128 + done + ;; + "unstripped_local") + logg "COPYING UNSTRIPPED CLIENT LIBRARY" + cp -pr "${build_output_directory}/lib/libfdb_c.so" "${website_directory}/downloads/${fdb_version}/linux/libfdb_c_${fdb_version}.so" + ;; + "stripped_local") + logg "COPYING STRIPPED CLIENT LIBRARY" + cp -pr "${build_output_directory}/packaging/lib/libfdb_c.so" "${website_directory}/downloads/${fdb_version}/linux/libfdb_c_${fdb_version}.so" + ;; + esac + # override fdb_website variable that is passed to Docker build + fdb_website="file:///tmp/website" +} + +function compile_ycsb() { + logg "COMPILING YCSB" + if [ "${use_development_java_bindings}" == "true" ]; then + logg "INSTALL JAVA BINDINGS" + foundationdb_java_version="${fdb_version}-PRERELEASE" + mvn install:install-file \ + --batch-mode \ + -Dorg.slf4j.simpleLogger.log.org.apache.maven.cli.transfer.Slf4jMavenTransferListener=warn \ + -Dfile="${build_output_directory}/packages/fdb-java-${foundationdb_java_version}.jar" \ + -DgroupId=org.foundationdb \ + -DartifactId=fdb-java \ + -Dversion="${foundationdb_java_version}" \ + -Dpackaging=jar \ + -DgeneratePom=true + else + foundationdb_java_version="${fdb_version}" + fi + rm -rf "${script_dir}/YCSB" + mkdir -p "${script_dir}/YCSB" + pushd "${script_dir}/YCSB" || exit 127 + if [ -d "${HOME}/src/YCSB" ]; then + rsync -av "${HOME}"/src/YCSB/. . + else + git clone https://github.com/FoundationDB/YCSB.git . + fi + sed -i "s/[0-9]\+.[0-9]\+.[0-9]\+<\/foundationdb.version>/${foundationdb_java_version}<\/foundationdb.version>/g" pom.xml + mvn --batch-mode -Dorg.slf4j.simpleLogger.log.org.apache.maven.cli.transfer.Slf4jMavenTransferListener=warn -pl site.ycsb:foundationdb-binding -am clean package + mkdir -p core/target/dependency + # shellcheck disable=SC2046 + cp $(find "${HOME}/.m2" -name jax\*.jar) core/target/dependency/ + # shellcheck disable=SC2046 + cp $(find "${HOME}/.m2" -name htrace\*.jar) core/target/dependency/ + # shellcheck disable=SC2046 + cp $(find "${HOME}/.m2" -name HdrHistogram\*.jar) core/target/dependency/ + rm -rf .git + popd || exit 128 +} + +function build_and_push_images(){ + declare -a tags_to_push=() + for image in "${image_list[@]}"; do + logg "BUILDING ${image}" + image_tag="${tag_base}${image}:${fdb_version}" + if [ "${image}" == "foundationdb-kubernetes-sidecar" ]; then + image_tag="${image_tag}-1" + fi + if [ "${dockerfile_name}" == "Dockerfile.eks" ]; then + image_tag="${image_tag}-debug" + fi + if [ "${image}" == "ycsb" ]; then + compile_ycsb + fi + logg "TAG ${image_tag#${registry}/foundationdb/}" + docker build \ + --label "org.foundationdb.version=${fdb_version}" \ + --label "org.foundationdb.build_date=${build_date}" \ + --label "org.foundationdb.commit=${commit_sha}" \ + --progress plain \ + --build-arg FDB_VERSION="${fdb_version}" \ + --build-arg FDB_LIBRARY_VERSIONS="${fdb_library_versions[*]}" \ + --build-arg FDB_WEBSITE="${fdb_website}" \ + --tag "${image_tag}" \ + --file "${dockerfile_name}" \ + --target "${image}" . + if [ "${image}" == 'foundationdb' ] || [ "${image}" == 'foundationdb-kubernetes-sidecar' ] || [ "${image}" == 'ycsb' ] ; then + tags_to_push+=("${image_tag}") + fi + done + + if [ "${push_docker_images}" == "true" ]; then + for tag in "${tags_to_push[@]}"; do + logg "PUSH ${tag}" + docker push "${tag}" + done + fi +} + +echo "${blue}################################################################################${reset}" +logg "STARTING ${0}" +echo "${blue}################################################################################${reset}" + +################################################################################ +# The intent of this script is to build the set of docker images needed to run +# FoundationDB in kubernetes from binaries that are not available the website: +# https://foundationdb.org/downloads +# +# The docker file itself will pull released binaries from the foundationdb +# website. If the intent is to build images for an already released version of +# FoundationDB, a simple docker build command will work. +# +# This script has enough stupid built into it that trying to come up with a set +# of sensible default options that will work everywhere has gotten silly. Below +# are a set of variable definitions that need to be set for this script to +# execute to completion the defaults are based on the FoundationDB development +# environment used by the team at Apple, they will not work for everyone cloning +# this project. +# +# Use this script with care. +################################################################################ +artifactory_base_url="${ARTIFACTORY_URL:-https://artifactory.foundationdb.org}" +aws_region="us-west-2" +aws_account_id=$(aws --output text sts get-caller-identity --query 'Account') +build_date=$(date +"%Y-%m-%dT%H:%M:%S%z") +build_output_directory="${script_dir}/../../build_output" +commit_sha=$(git rev-parse --verify HEAD --short=10) +fdb_version=$(awk '/^[[:space:]]+VERSION[[:space:]]+[[:digit:]]+\.[[:digit:]]+\.[[:digit:]]+(-rc[[:digit:]])?/{print $2}' "${script_dir}/../../CMakeLists.txt") +fdb_library_versions=( '5.1.7' '6.1.13' '6.2.30' "${fdb_version}" ) +fdb_website="https://www.foundationdb.org" +image_list=( + 'base' + # 'go-build' + 'foundationdb-base' + 'foundationdb' + # 'foundationdb-kubernetes-monitor' + 'foundationdb-kubernetes-sidecar' + 'ycsb' +) +registry="" +tag_base="foundationdb/" +# THESE CONTROL THE PATH OF FUNCTIONS THAT ARE CALLED BELOW +stripped_binaries_and_from_where="stripped_local" # MUST BE ONE OF ( "unstripped_artifactory" "stripped_artifactory" "unstripped_local" "stripped_local" ) +dockerfile_name="Dockerfile" +use_development_java_bindings="false" +push_docker_images="false" + +if [ -n "${OKTETO_NAMESPACE+x}" ]; then + logg "RUNNING IN OKTETO/AWS" + # these are defaults for the Apple development environment + aws_region=$(curl -s "http://169.254.169.254/latest/meta-data/placement/region") + aws_account_id=$(aws --output text sts get-caller-identity --query 'Account') + build_output_directory="${HOME}/build_output" + fdb_library_versions=( "${fdb_version}" ) + registry="${aws_account_id}.dkr.ecr.${aws_region}.amazonaws.com" + tag_base="${registry}/foundationdb/" + stripped_binaries_and_from_where="unstripped_local" # MUST BE ONE OF ( "unstripped_artifactory" "stripped_artifactory" "unstripped_local" "stripped_local" ) + dockerfile_name="Dockerfile.eks" + use_development_java_bindings="true" + push_docker_images="true" +else + echo "Dear ${USER}, you probably need to edit this file before running it. " + echo "${0} has a very narrow set of situations where it will be successful," + echo "or even useful, when executed unedited" + exit 1 +fi + +create_fake_website_directory +build_and_push_images + +echo "${blue}################################################################################${reset}" +logg "COMPLETED ${0}" +echo "${blue}################################################################################${reset}" diff --git a/packaging/docker/build-release-docker.sh b/packaging/docker/build-release-docker.sh deleted file mode 100755 index bd13c22382..0000000000 --- a/packaging/docker/build-release-docker.sh +++ /dev/null @@ -1,62 +0,0 @@ -#!/usr/bin/env bash -set -Eeuo pipefail -set -x - -DOCKER_ROOT="$(realpath "$(dirname "${BASH_SOURCE[0]}")")" -BUILD_OUTPUT=$(realpath "${DOCKER_ROOT}"/../..) - -echo Docker root: "${DOCKER_ROOT}" -echo Build output: "${BUILD_OUTPUT}" - -cd "${DOCKER_ROOT}" - -## eg: CMAKE_PROJECT_VERSION:STATIC=7.0.0 -FDB_VERSION=$(grep CMAKE_PROJECT_VERSION: "${BUILD_OUTPUT}"/CMakeCache.txt | cut -d '=' -f 2) - -# Options (passed via environment variables) - -# Feel free to customize the image tag. -# TODO: add a mechanism to set TAG=FDB_VERSION when we're building public releases. -TAG=${TAG:-${FDB_VERSION}-${OKTETO_NAME}} -ECR=${ECR:-112664522426.dkr.ecr.us-west-2.amazonaws.com} - -echo Building with tag "${TAG}" - -# Login to ECR -# TODO: Move this to a common place instead of repeatedly copy-pasting it. -aws ecr get-login-password | docker login --username AWS --password-stdin "${ECR}" - -docker pull "${ECR}"/ubuntu:18.04 -docker tag "${ECR}"/ubuntu:18.04 ubuntu:18.04 -docker pull "${ECR}"/python:3.9-slim -docker tag "${ECR}"/python:3.9-slim python:3.9-slim - -# derived variables -IMAGE=foundationdb/foundationdb:"${TAG}" -SIDECAR=foundationdb/foundationdb-kubernetes-sidecar:"${TAG}"-1 -STRIPPED=${STRIPPED:-false} - -WEBSITE_BIN_DIR=website/downloads/"${FDB_VERSION}"/linux -TARBALL=${WEBSITE_BIN_DIR}/fdb_"${FDB_VERSION}".tar.gz -mkdir -p "${WEBSITE_BIN_DIR}" - -if $STRIPPED; then - tar -C ~/build_output/packages/ -zcvf "${TARBALL}" bin lib - cp ~/build_output/packages/lib/libfdb_c.so "${WEBSITE_BIN_DIR}"/libfdb_c_"${FDB_VERSION}".so -else - tar -C ~/build_output/ -zcvf "${TARBALL}" bin lib - cp ~/build_output/lib/libfdb_c.so "${WEBSITE_BIN_DIR}"/libfdb_c_"${FDB_VERSION}".so -fi - -BUILD_ARGS="--build-arg FDB_VERSION=${FDB_VERSION}" -BUILD_ARGS+=" --build-arg FDB_WEBSITE=file:///mnt/website" -BUILD_ARGS+=" --build-arg FDB_ADDITIONAL_VERSIONS=${FDB_VERSION}" - -docker build ${BUILD_ARGS} -t "${IMAGE}" -f release/Dockerfile . -docker build ${BUILD_ARGS} -t "${SIDECAR}" -f sidecar/Dockerfile . - -docker tag "${IMAGE}" "${ECR}"/"${IMAGE}" -docker tag "${SIDECAR}" "${ECR}"/"${SIDECAR}" - -docker push "${ECR}"/"${IMAGE}" -docker push "${ECR}"/"${SIDECAR}" diff --git a/packaging/docker/build-ycsb-docker.sh b/packaging/docker/build-ycsb-docker.sh deleted file mode 100755 index 38b1765123..0000000000 --- a/packaging/docker/build-ycsb-docker.sh +++ /dev/null @@ -1,64 +0,0 @@ -#!/usr/bin/env bash -set -Eeuo pipefail -set -x - -DOCKER_ROOT="$(realpath "$(dirname "${BASH_SOURCE[0]}")")" -BUILD_OUTPUT=$(realpath "${DOCKER_ROOT}"/../..) - -echo Docker root: "${DOCKER_ROOT}" -echo Build output: "${BUILD_OUTPUT}" - -cd "${DOCKER_ROOT}" - -## eg: CMAKE_PROJECT_VERSION:STATIC=7.0.0 -FDB_VERSION=$(grep CMAKE_PROJECT_VERSION: "${BUILD_OUTPUT}"/CMakeCache.txt | cut -d '=' -f 2) - -# Options (passed via environment variables) - -# Feel free to customize the image tag. -# TODO: add a mechanism to set TAG=FDB_VERSION when we're building public releases. -TAG=${TAG:-${FDB_VERSION}-${OKTETO_NAME}} -ECR=${ECR:-112664522426.dkr.ecr.us-west-2.amazonaws.com} - -echo Building with tag "${TAG}" - -# Login to ECR -# TODO: Move this to a common place instead of repeatedly copy-pasting it. -aws ecr get-login-password | docker login --username AWS --password-stdin "${ECR}" - -docker pull "${ECR}"/openjdk:17-slim -docker tag "${ECR}"/openjdk:17-slim openjdk:17-slim - - - -# derived variables -IMAGE=foundationdb/ycsb:"${TAG}" - -# mvn install fdb-java, compile YCSB -mvn install:install-file \ - -Dfile="${BUILD_OUTPUT}"/packages/fdb-java-"${FDB_VERSION}"-PRERELEASE.jar \ - -DgroupId=org.foundationdb \ - -DartifactId=fdb-java \ - -Dversion="${FDB_VERSION}"-PRERELEASE \ - -Dpackaging=jar \ - -DgeneratePom=true -mkdir "${DOCKER_ROOT}"/YCSB && cd "${DOCKER_ROOT}"/YCSB -git clone https://github.com/FoundationDB/YCSB.git . -sed -i "s/[0-9]\+.[0-9]\+.[0-9]\+<\/foundationdb.version>/${FDB_VERSION}-PRERELEASE<\/foundationdb.version>/g" pom.xml -mvn -pl site.ycsb:foundationdb-binding -am clean package -mkdir -p core/target/dependency -# shellcheck disable=SC2046 -cp $(find ~/.m2/ -name jax\*.jar) core/target/dependency/ -# shellcheck disable=SC2046 -cp $(find ~/.m2/ -name htrace\*.jar) core/target/dependency/ -# shellcheck disable=SC2046 -cp $(find ~/.m2/ -name HdrHistogram\*.jar) core/target/dependency/ -rm -rf .git && cd .. - -docker build -t "${IMAGE}" -f ycsb/Dockerfile . - - -docker tag "${IMAGE}" "${ECR}"/"${IMAGE}" - - -docker push "${ECR}"/"${IMAGE}" diff --git a/packaging/docker/sidecar/entrypoint.bash b/packaging/docker/entrypoint.bash similarity index 90% rename from packaging/docker/sidecar/entrypoint.bash rename to packaging/docker/entrypoint.bash index b6678fc831..ba28eb31db 100755 --- a/packaging/docker/sidecar/entrypoint.bash +++ b/packaging/docker/entrypoint.bash @@ -24,4 +24,8 @@ if [[ -n "$ADDITIONAL_ENV_FILE" ]]; then source $ADDITIONAL_ENV_FILE fi +if [[ -f "/opt/rh/rh-python38/enable" ]]; then + source /opt/rh/rh-python38/enable +fi + exec /sidecar.py $* diff --git a/packaging/docker/release/fdb.bash b/packaging/docker/fdb.bash similarity index 100% rename from packaging/docker/release/fdb.bash rename to packaging/docker/fdb.bash diff --git a/packaging/docker/release/Dockerfile b/packaging/docker/release/Dockerfile deleted file mode 100644 index 7df65e63c6..0000000000 --- a/packaging/docker/release/Dockerfile +++ /dev/null @@ -1,92 +0,0 @@ -# Dockerfile -# -# This source file is part of the FoundationDB open source project -# -# Copyright 2013-2021 Apple Inc. and the FoundationDB project authors -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -FROM ubuntu:18.04 - -RUN apt-get update && \ - apt-get install -y curl>=7.58.0-2ubuntu3.6 \ - dnsutils>=1:9.11.3+dfsg-1ubuntu1.7 \ - lsof>=4.89+dfsg-0.1 \ - tcptraceroute>=1.5beta7+debian-4build1 \ - telnet>=0.17-41 \ - netcat>=1.10-41.1 \ - strace>=4.21-1ubuntu1 \ - tcpdump>=4.9.3-0ubuntu0.18.04.1 \ - less>=487-0.1 \ - vim>=2:8.0.1453-1ubuntu1.4 \ - net-tools>=1.60+git20161116.90da8a0-1ubuntu1 \ - jq>=1.5+dfsg-2 \ - openssl>=1.1.1-1ubuntu2.1~18.04.9 && \ - rm -rf /var/lib/apt/lists/* - -COPY misc/tini-amd64.sha256sum /tmp/ -# Adding tini as PID 1 https://github.com/krallin/tini -ARG TINI_VERSION=v0.19.0 -RUN curl -sLO https://github.com/krallin/tini/releases/download/${TINI_VERSION}/tini-amd64 && \ - sha256sum -c /tmp/tini-amd64.sha256sum && \ - chmod +x tini-amd64 && \ - mv tini-amd64 /usr/bin/tini - -ARG FDB_VERSION -ARG FDB_ADDITIONAL_VERSIONS="5.1.7" -ARG FDB_WEBSITE=https://www.foundationdb.org - -WORKDIR /var/fdb/tmp -COPY website /mnt/website/ - -# Install FoundationDB Binaries -RUN curl $FDB_WEBSITE/downloads/$FDB_VERSION/linux/fdb_$FDB_VERSION.tar.gz | tar zxf - --strip-components=1 && \ - chmod u+x fdbbackup fdbcli fdbdr fdbmonitor fdbrestore fdbserver backup_agent dr_agent && \ - mv fdbbackup fdbcli fdbdr fdbmonitor fdbrestore fdbserver backup_agent dr_agent /usr/bin && \ - rm -r /var/fdb/tmp - -WORKDIR / - -## TODO: Can unify everything above this line -## TODO: we can almost unify the additional client library download, -## but sidecar.py expects them in a different location, -## with a different naming convention. - -RUN curl $FDB_WEBSITE/downloads/$FDB_VERSION/linux/libfdb_c_$FDB_VERSION.so -o /usr/lib/libfdb_c.so - -# Set Up Runtime Scripts and Directories -ADD release/*.bash /var/fdb/scripts/ -RUN chmod a+x /var/fdb/scripts/*.bash - -# Install additional FoundationDB Client Libraries -RUN /var/fdb/scripts/download_multiversion_libraries.bash $FDB_WEBSITE $FDB_ADDITIONAL_VERSIONS - -RUN rm -rf /mnt/website - -RUN mkdir -p /var/fdb/logs - -VOLUME /var/fdb/data - -# Runtime Configuration Options - -ENV FDB_PORT 4500 -ENV FDB_CLUSTER_FILE /var/fdb/fdb.cluster -ENV FDB_NETWORKING_MODE container -ENV FDB_COORDINATOR "" -ENV FDB_COORDINATOR_PORT 4500 -ENV FDB_CLUSTER_FILE_CONTENTS "" -ENV FDB_PROCESS_CLASS unset - -ENTRYPOINT ["/usr/bin/tini", "-g", "--"] -CMD /var/fdb/scripts/fdb.bash diff --git a/packaging/docker/release/create_cluster_file.bash b/packaging/docker/release/create_cluster_file.bash deleted file mode 100755 index c1bb959b8e..0000000000 --- a/packaging/docker/release/create_cluster_file.bash +++ /dev/null @@ -1,52 +0,0 @@ -#! /bin/bash - -# -# create_cluster_file.bash -# -# This source file is part of the FoundationDB open source project -# -# Copyright 2013-2018 Apple Inc. and the FoundationDB project authors -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -# This script creates a cluster file for a server or client. -# This takes the cluster file path from the FDB_CLUSTER_FILE -# environment variable, with a default of /etc/foundationdb/fdb.cluster -# -# The name of the coordinator must be defined in the FDB_COORDINATOR environment -# variable, and it must be a name that can be resolved through DNS. - -function create_cluster_file() { - FDB_CLUSTER_FILE=${FDB_CLUSTER_FILE:-/etc/foundationdb/fdb.cluster} - mkdir -p $(dirname $FDB_CLUSTER_FILE) - - if [[ -n "$FDB_CLUSTER_FILE_CONTENTS" ]]; then - echo "$FDB_CLUSTER_FILE_CONTENTS" > $FDB_CLUSTER_FILE - elif [[ -n $FDB_COORDINATOR ]]; then - coordinator_ip=$(dig +short $FDB_COORDINATOR) - if [[ -z "$coordinator_ip" ]]; then - echo "Failed to look up coordinator address for $FDB_COORDINATOR" 1>&2 - exit 1 - fi - coordinator_port=${FDB_COORDINATOR_PORT:-4500} - echo "docker:docker@$coordinator_ip:$coordinator_port" > $FDB_CLUSTER_FILE - else - echo "FDB_COORDINATOR environment variable not defined" 1>&2 - exit 1 - fi -} - -if [[ "${BASH_SOURCE[0]}" == "${0}" ]]; then - create_cluster_file "$@" -fi diff --git a/packaging/docker/release/create_server_environment.bash b/packaging/docker/release/create_server_environment.bash deleted file mode 100755 index 51a782f991..0000000000 --- a/packaging/docker/release/create_server_environment.bash +++ /dev/null @@ -1,43 +0,0 @@ -#! /bin/bash - -# -# create_server_environment.bash -# -# This source file is part of the FoundationDB open source project -# -# Copyright 2013-2018 Apple Inc. and the FoundationDB project authors -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -source /var/fdb/scripts/create_cluster_file.bash - -function create_server_environment() { - env_file=/var/fdb/.fdbenv - - if [[ "$FDB_NETWORKING_MODE" == "host" ]]; then - public_ip=127.0.0.1 - elif [[ "$FDB_NETWORKING_MODE" == "container" ]]; then - public_ip=$(hostname -i | awk '{print $1}') - else - echo "Unknown FDB Networking mode \"$FDB_NETWORKING_MODE\"" 1>&2 - exit 1 - fi - - echo "export PUBLIC_IP=$public_ip" > $env_file - if [[ -z $FDB_COORDINATOR && -z "$FDB_CLUSTER_FILE_CONTENTS" ]]; then - FDB_CLUSTER_FILE_CONTENTS="docker:docker@$public_ip:$FDB_PORT" - fi - - create_cluster_file -} diff --git a/packaging/docker/release/download_multiversion_libraries.bash b/packaging/docker/release/download_multiversion_libraries.bash deleted file mode 100755 index 1cd5770ff3..0000000000 --- a/packaging/docker/release/download_multiversion_libraries.bash +++ /dev/null @@ -1,31 +0,0 @@ -#! /bin/bash - -# -# download_multiversion_libraries.bash -# -# This source file is part of the FoundationDB open source project -# -# Copyright 2013-2018 Apple Inc. and the FoundationDB project authors -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -mkdir -p /usr/lib/fdb/multiversion -website=$1 -shift -for version in $*; do - origin=$website/downloads/$version/linux/libfdb_c_$version.so - destination=/usr/lib/fdb/multiversion/libfdb_c_$version.so - echo "Downloading $origin to $destination" - curl $origin -o $destination -done diff --git a/packaging/docker/ycsb/run_ycsb.sh b/packaging/docker/run_ycsb.sh old mode 100644 new mode 100755 similarity index 100% rename from packaging/docker/ycsb/run_ycsb.sh rename to packaging/docker/run_ycsb.sh diff --git a/packaging/docker/sidecar/sidecar.py b/packaging/docker/sidecar.py similarity index 100% rename from packaging/docker/sidecar/sidecar.py rename to packaging/docker/sidecar.py diff --git a/packaging/docker/sidecar/Dockerfile b/packaging/docker/sidecar/Dockerfile deleted file mode 100644 index b2d76693ec..0000000000 --- a/packaging/docker/sidecar/Dockerfile +++ /dev/null @@ -1,75 +0,0 @@ -# Dockerfile -# -# This source file is part of the FoundationDB open source project -# -# Copyright 2018-2019 Apple Inc. and the FoundationDB project authors -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -FROM python:3.9-slim - -RUN apt-get update && \ - apt-get install -y --no-install-recommends curl && \ - rm -rf /var/lub/apt/lists/* - -COPY misc/tini-amd64.sha256sum /tmp/ -# Adding tini as PID 1 https://github.com/krallin/tini -ARG TINI_VERSION=v0.19.0 -RUN curl -sLO https://github.com/krallin/tini/releases/download/${TINI_VERSION}/tini-amd64 && \ - sha256sum -c /tmp/tini-amd64.sha256sum && \ - chmod +x tini-amd64 && \ - mv tini-amd64 /usr/bin/tini - -COPY sidecar/requirements.txt /tmp -RUN pip install -r tmp/requirements.txt - -ARG FDB_VERSION= -ARG FDB_ADDITIONAL_VERSIONS="6.2.30 6.1.13" -ARG FDB_WEBSITE=https://www.foundationdb.org - -WORKDIR /var/fdb/tmp -COPY website /mnt/website/ - -# Install FoundationDB Binaries -RUN curl $FDB_WEBSITE/downloads/$FDB_VERSION/linux/fdb_$FDB_VERSION.tar.gz | tar zxf - --strip-components=1 && \ - chmod u+x fdbbackup fdbcli fdbdr fdbmonitor fdbrestore fdbserver backup_agent dr_agent && \ - mv fdbbackup fdbcli fdbdr fdbmonitor fdbrestore fdbserver backup_agent dr_agent /usr/bin && \ - rm -r /var/fdb/tmp - -WORKDIR / - -# Set Up Runtime Scripts and Directories -ADD sidecar/entrypoint.bash sidecar/sidecar.py / -RUN chmod a+x /entrypoint.bash /sidecar.py - -# Install additional FoundationDB Client Libraries -RUN mkdir -p /var/fdb/lib && \ - for version in $FDB_ADDITIONAL_VERSIONS; do curl $FDB_WEBSITE/downloads/$version/linux/libfdb_c_$version.so -o /var/fdb/lib/libfdb_c_${version%.*}.so; done - -RUN rm -rf /mnt/website - -RUN echo ${FDB_VERSION} > /var/fdb/version && \ - mkdir -p /var/fdb/lib && \ - groupadd --gid 4059 fdb && \ - useradd --gid 4059 --uid 4059 --no-create-home --shell /bin/bash fdb - - -VOLUME /var/input-files -VOLUME /var/output-files - -USER fdb - -ENV LISTEN_PORT 8080 - -ENTRYPOINT ["/usr/bin/tini", "-g", "--", "/entrypoint.bash"] diff --git a/packaging/docker/sidecar/requirements.txt b/packaging/docker/sidecar/requirements.txt deleted file mode 100644 index c7fcc8bac8..0000000000 --- a/packaging/docker/sidecar/requirements.txt +++ /dev/null @@ -1 +0,0 @@ -watchdog==0.9.0 \ No newline at end of file diff --git a/packaging/docker/ycsb/Dockerfile b/packaging/docker/ycsb/Dockerfile deleted file mode 100644 index 1cf7b1e537..0000000000 --- a/packaging/docker/ycsb/Dockerfile +++ /dev/null @@ -1,32 +0,0 @@ -FROM openjdk:17-slim AS RUN - -WORKDIR /tmp -RUN apt-get update && \ - apt-get install -y curl zip && \ - curl -Ls https://amazon-eks.s3.amazonaws.com/1.19.6/2021-01-05/bin/linux/amd64/kubectl -o kubectl && \ - echo "08ff68159bbcb844455167abb1d0de75bbfe5ae1b051f81ab060a1988027868a kubectl" > kubectl.txt && \ - sha256sum -c kubectl.txt && \ - mv kubectl /usr/local/bin/kubectl && \ - chmod 755 /usr/local/bin/kubectl && \ - curl https://awscli.amazonaws.com/awscli-exe-linux-x86_64-2.2.43.zip -o "awscliv2.zip" && \ - echo "9a8b3c4e7f72bbcc55e341dce3af42479f2730c225d6d265ee6f9162cfdebdfd awscliv2.zip" > awscliv2.txt && \ - sha256sum -c awscliv2.txt && \ - unzip -qq awscliv2.zip && \ - ./aws/install && \ - rm -rf /tmp/* - -ADD YCSB /YCSB -WORKDIR /YCSB - -ENV FDB_NETWORK_OPTION_EXTERNAL_CLIENT_DIRECTORY=/var/dynamic-conf/lib/multiversion/ -ENV FDB_NETWORK_OPTION_TRACE_ENABLE=/var/log/fdb-trace-logs -ENV LD_LIBRARY_PATH=/var/dynamic-conf/lib/ -ENV BUCKET=backup-112664522426-us-west-2 - -# TODO: Log4J complains that it's eating the HTracer logs. Even without it, we get per-operation -# time series graphs of throughput, median, 90, 99, 99.9 and 99.99 (in usec). -COPY ycsb/run_ycsb.sh /usr/local/bin/run_ycsb.sh -RUN mkdir -p /var/log/fdb-trace-logs && \ - chmod +x /usr/local/bin/run_ycsb.sh - -CMD ["run_ycsb.sh"]