From 67bd4ddea0203bb271bbad44ed2bb36a3ad1146b Mon Sep 17 00:00:00 2001 From: Jon Fu Date: Tue, 2 Nov 2021 16:24:40 -0400 Subject: [PATCH 1/5] Add a wait(delay(0)) to storage server termination to avoid a rare segfault --- fdbserver/storageserver.actor.cpp | 23 +++++++++++++++++++---- 1 file changed, 19 insertions(+), 4 deletions(-) diff --git a/fdbserver/storageserver.actor.cpp b/fdbserver/storageserver.actor.cpp index c7bb89afc6..467793bf2f 100644 --- a/fdbserver/storageserver.actor.cpp +++ b/fdbserver/storageserver.actor.cpp @@ -6110,9 +6110,16 @@ ACTOR Future storageServer(IKeyValueStore* persistentData, // (ClusterController, and from there to the DataDistributionTeamCollection) if (!recruitReply.isSet()) recruitReply.sendError(recruitment_failed()); - if (storageServerTerminated(self, persistentData, e)) + if (e.code() == error_code_actor_cancelled) + throw e; + + // If the storage server dies while something that uses self is still on the stack, + // we want that actor to complete before we terminate and that memory goes out of scope + state Error err = e; + wait(delay(0)); + if (storageServerTerminated(self, persistentData, err)) return Void(); - throw e; + throw err; } } @@ -6320,9 +6327,17 @@ ACTOR Future storageServer(IKeyValueStore* persistentData, } catch (Error& e) { if (recovered.canBeSet()) recovered.send(Void()); - if (storageServerTerminated(self, persistentData, e)) + + if (e.code() == error_code_actor_cancelled) + throw e; + + // If the storage server dies while something that uses self is still on the stack, + // we want that actor to complete before we terminate and that memory goes out of scope + state Error err = e; + wait(delay(0)); + if (storageServerTerminated(self, persistentData, err)) return Void(); - throw e; + throw err; } } From 396cd58b2107310ab67570cad5cf8f3e58d83184 Mon Sep 17 00:00:00 2001 From: Jon Fu Date: Thu, 4 Nov 2021 16:05:23 -0400 Subject: [PATCH 2/5] cancel ss core and ss actor collection after termination and before context switch --- fdbserver/storageserver.actor.cpp | 33 ++++++++++++++++++++----------- 1 file changed, 22 insertions(+), 11 deletions(-) diff --git a/fdbserver/storageserver.actor.cpp b/fdbserver/storageserver.actor.cpp index 467793bf2f..0b535fc528 100644 --- a/fdbserver/storageserver.actor.cpp +++ b/fdbserver/storageserver.actor.cpp @@ -6063,6 +6063,7 @@ ACTOR Future storageServer(IKeyValueStore* persistentData, Reference const> db, std::string folder) { state StorageServer self(persistentData, db, ssi); + state Future ssCore; if (ssi.isTss()) { self.setTssPair(ssi.tssPairID.get()); ASSERT(self.isTss()); @@ -6102,7 +6103,8 @@ ACTOR Future storageServer(IKeyValueStore* persistentData, recruitReply.send(rep); self.byteSampleRecovery = Void(); - wait(storageServerCore(&self, ssi)); + ssCore = storageServerCore(&self, ssi); + wait(ssCore); throw internal_error(); } catch (Error& e) { @@ -6110,15 +6112,19 @@ ACTOR Future storageServer(IKeyValueStore* persistentData, // (ClusterController, and from there to the DataDistributionTeamCollection) if (!recruitReply.isSet()) recruitReply.sendError(recruitment_failed()); - if (e.code() == error_code_actor_cancelled) - throw e; // If the storage server dies while something that uses self is still on the stack, // we want that actor to complete before we terminate and that memory goes out of scope state Error err = e; - wait(delay(0)); - if (storageServerTerminated(self, persistentData, err)) + if (storageServerTerminated(self, persistentData, err)) { + ssCore.cancel(); + self.actors.clear(true); + wait(delay(0)); return Void(); + } + ssCore.cancel(); + self.actors.clear(true); + wait(delay(0)); throw err; } } @@ -6267,6 +6273,7 @@ ACTOR Future storageServer(IKeyValueStore* persistentData, Promise recovered, Reference connRecord) { state StorageServer self(persistentData, db, ssi); + state Future ssCore; self.folder = folder; try { @@ -6321,22 +6328,26 @@ ACTOR Future storageServer(IKeyValueStore* persistentData, TraceEvent("StorageServerStartingCore", self.thisServerID).detail("TimeTaken", now() - start); // wait( delay(0) ); // To make sure self->zkMasterInfo.onChanged is available to wait on - wait(storageServerCore(&self, ssi)); + ssCore = storageServerCore(&self, ssi); + wait(ssCore); throw internal_error(); } catch (Error& e) { if (recovered.canBeSet()) recovered.send(Void()); - if (e.code() == error_code_actor_cancelled) - throw e; - // If the storage server dies while something that uses self is still on the stack, // we want that actor to complete before we terminate and that memory goes out of scope state Error err = e; - wait(delay(0)); - if (storageServerTerminated(self, persistentData, err)) + if (storageServerTerminated(self, persistentData, err)) { + ssCore.cancel(); + self.actors.clear(true); + wait(delay(0)); return Void(); + } + ssCore.cancel(); + self.actors.clear(true); + wait(delay(0)); throw err; } } From 9a07a117d28cb21997ddba633dbd9ad55179829c Mon Sep 17 00:00:00 2001 From: Tao Lin Date: Wed, 17 Nov 2021 16:23:20 -0800 Subject: [PATCH 3/5] Not warn on constant-logical-operand when compile with clang (#5990) --- cmake/ConfigureCompiler.cmake | 1 + 1 file changed, 1 insertion(+) diff --git a/cmake/ConfigureCompiler.cmake b/cmake/ConfigureCompiler.cmake index 160c46f5ae..5641d32ead 100644 --- a/cmake/ConfigureCompiler.cmake +++ b/cmake/ConfigureCompiler.cmake @@ -288,6 +288,7 @@ else() -Wno-undefined-var-template -Wno-unknown-warning-option -Wno-unused-parameter + -Wno-constant-logical-operand ) if (USE_CCACHE) add_compile_options( From 124e7eb4f251903b90d5b8addc8742bcafc9e5ae Mon Sep 17 00:00:00 2001 From: Aaron Molitor Date: Mon, 15 Nov 2021 12:51:55 -0800 Subject: [PATCH 4/5] refactor docker directory - make release images ALL based from centos:7 - keep eks images that are based from amazonlinux:2, but create strong alignment with release (centos7) images - use multi-stage Dockerfile(s) - have a single build-images script - remove per image directories (which would only contain scripts that are copied into images) - remove empty requirements.txt file for sidecar.py (the pip command, package and version are in the Dockerfile) - don't copy docker into PROJECT_BINARY_DIR - incorporate latest changes already in master --- cmake/InstallLayout.cmake | 8 +- fdbkubernetesmonitor/README.md | 4 +- packaging/docker/Dockerfile | 194 ++++++++++++ packaging/docker/Dockerfile.eks | 228 +++++++++------ packaging/docker/build-eks-docker.sh | 62 ---- packaging/docker/build-images.sh | 275 ++++++++++++++++++ packaging/docker/build-release-docker.sh | 62 ---- packaging/docker/build-ycsb-docker.sh | 64 ---- .../docker/{sidecar => }/entrypoint.bash | 4 + packaging/docker/{release => }/fdb.bash | 0 packaging/docker/kubernetes/Dockerfile | 86 ------ packaging/docker/release/Dockerfile | 100 ------- packaging/docker/{ycsb => }/run_ycsb.sh | 0 packaging/docker/{sidecar => }/sidecar.py | 0 packaging/docker/sidecar/Dockerfile | 77 ----- packaging/docker/sidecar/requirements.txt | 0 packaging/docker/ycsb/Dockerfile | 32 -- 17 files changed, 613 insertions(+), 583 deletions(-) create mode 100644 packaging/docker/Dockerfile delete mode 100755 packaging/docker/build-eks-docker.sh create mode 100755 packaging/docker/build-images.sh delete mode 100755 packaging/docker/build-release-docker.sh delete mode 100755 packaging/docker/build-ycsb-docker.sh rename packaging/docker/{sidecar => }/entrypoint.bash (90%) rename packaging/docker/{release => }/fdb.bash (100%) delete mode 100644 packaging/docker/kubernetes/Dockerfile delete mode 100644 packaging/docker/release/Dockerfile rename packaging/docker/{ycsb => }/run_ycsb.sh (100%) mode change 100644 => 100755 rename packaging/docker/{sidecar => }/sidecar.py (100%) delete mode 100644 packaging/docker/sidecar/Dockerfile delete mode 100644 packaging/docker/sidecar/requirements.txt delete mode 100644 packaging/docker/ycsb/Dockerfile diff --git a/cmake/InstallLayout.cmake b/cmake/InstallLayout.cmake index 6caa871759..a0e5285cba 100644 --- a/cmake/InstallLayout.cmake +++ b/cmake/InstallLayout.cmake @@ -185,12 +185,6 @@ install(DIRECTORY "${script_dir}/clients/usr/lib/cmake" DESTINATION usr/lib COMPONENT clients-versioned) -################################################################################ -# Move Docker Setup -################################################################################ - -file(COPY "${PROJECT_SOURCE_DIR}/packaging/docker" DESTINATION "${PROJECT_BINARY_DIR}/packages/") - ################################################################################ # General CPack configuration ################################################################################ @@ -228,7 +222,7 @@ set(CPACK_COMPONENT_CLIENTS-TGZ_DISPLAY_NAME "foundationdb-clients") set(CPACK_COMPONENT_CLIENTS-VERSIONED_DISPLAY_NAME "foundationdb${PROJECT_VERSION}-clients") -# MacOS needs a file exiension for the LICENSE file +# MacOS needs a file extension for the LICENSE file configure_file(${CMAKE_SOURCE_DIR}/LICENSE ${CMAKE_BINARY_DIR}/License.txt COPYONLY) ################################################################################ diff --git a/fdbkubernetesmonitor/README.md b/fdbkubernetesmonitor/README.md index 901b1c9c72..b9539c01cd 100644 --- a/fdbkubernetesmonitor/README.md +++ b/fdbkubernetesmonitor/README.md @@ -5,8 +5,8 @@ To test this, run the following commands from the root of the FoundationDB repository: ```bash -docker build -t foundationdb/foundationdb-kubernetes:6.3.13-local --build-arg FDB_VERSION=6.3.13 --build-arg FDB_LIBRARY_VERSIONS="6.3.13 6.2.30 6.1.13" -f packaging/docker/kubernetes/Dockerfile . -docker build -t foundationdb/foundationdb-kubernetes:6.3.15-local --build-arg FDB_VERSION=6.3.15 --build-arg FDB_LIBRARY_VERSIONS="6.3.15 6.2.30 6.1.13" -f packaging/docker/kubernetes/Dockerfile . +docker build -t foundationdb/foundationdb-kubernetes:6.3.13-local --target fdb-kubernetes-monitor --build-arg FDB_VERSION=6.3.13 --build-arg FDB_LIBRARY_VERSIONS="6.3.13 6.2.30 6.1.13" -f packaging/docker/Dockerfile . +docker build -t foundationdb/foundationdb-kubernetes:6.3.15-local --target fdb-kubernetes-monitor --build-arg FDB_VERSION=6.3.15 --build-arg FDB_LIBRARY_VERSIONS="6.3.15 6.2.30 6.1.13" -f packaging/docker/Dockerfile . kubectl apply -f packaging/docker/kubernetes/test_config.yaml # Wait for the pods to become ready ips=$(kubectl get pod -l app=fdb-kubernetes-example -o json | jq -j '[[.items|.[]|select(.status.podIP!="")]|limit(3;.[])|.status.podIP+":4501"]|join(",")') diff --git a/packaging/docker/Dockerfile b/packaging/docker/Dockerfile new file mode 100644 index 0000000000..c76fc2e9f1 --- /dev/null +++ b/packaging/docker/Dockerfile @@ -0,0 +1,194 @@ +# Dockerfile +# +# This source file is part of the FoundationDB open source project +# +# Copyright 2013-2021 Apple Inc. and the FoundationDB project authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +FROM centos:7.9.2009 as base + +RUN yum install -y \ + epel-release-7-11 \ + centos-release-scl-2-3.el7.centos && \ + yum install -y \ + bind-utils-9.11.4-26.P2.el7_9.7 \ + binutils-2.27-44.base.el7_9.1 \ + curl-7.29.0-59.el7_9.1 \ + gdb-7.6.1-120.el7 \ + hostname-3.13-3.el7_7.1 \ + jq-1.6-2.el7 \ + less-458-9.el7 \ + libubsan-7.3.1-5.16.el7 \ + lsof-4.87-6.el7 \ + net-tools-2.0-0.25.20131004git.el7 \ + nmap-ncat-6.40-19.el7 \ + perf-3.10.0-1160.45.1.el7 \ + perl-5.16.3-299.el7_9 \ + procps-ng-3.3.10-28.el7 \ + strace-4.24-6.el7 \ + sysstat-10.1.5-19.el7 \ + tar-1.26-35.el7 \ + tcpdump-4.9.2-4.el7_7.1 \ + telnet-0.17-66.el7 \ + traceroute-2.0.22-2.el7 \ + unzip-6.0-22.el7_9 \ + vim-enhanced-7.4.629-8.el7_9 && \ + yum clean all && \ + rm -rf /var/cache/yum + +WORKDIR /tmp + +RUN curl -Ls https://github.com/krallin/tini/releases/download/v0.19.0/tini-amd64 -o tini && \ + echo "93dcc18adc78c65a028a84799ecf8ad40c936fdfc5f2a57b1acda5a8117fa82c tini" > tini-amd64.sha256sum && \ + sha256sum -c tini-amd64.sha256sum && \ + chmod +x tini && \ + mv tini /usr/bin/ && \ + rm -rf /tmp/* + +WORKDIR / + +FROM golang:1.16.7-bullseye AS go-build + +COPY fdbkubernetesmonitor/ /fdbkubernetesmonitor +WORKDIR /fdbkubernetesmonitor +RUN go build -o /fdb-kubernetes-monitor *.go + +FROM base as foundationdb-base + +WORKDIR /tmp +ARG FDB_VERSION=6.3.22 +ARG FDB_LIBRARY_VERSIONS="${FDB_VERSION}" +ARG FDB_WEBSITE=https://www.foundationdb.org + +RUN mkdir -p /var/fdb/{logs,tmp,lib} && \ + mkdir -p /usr/lib/fdb/multiversion && \ + echo ${FDB_VERSION} > /var/fdb/version + +# Set up a non-root user +RUN groupadd --gid 4059 fdb && \ + useradd --gid 4059 --uid 4059 --no-create-home --shell /bin/bash fdb && \ + chown -R fdb:fdb /var/fdb + +COPY website /tmp/website/ + +# Install FoundationDB Binaries +RUN curl $FDB_WEBSITE/downloads/$FDB_VERSION/linux/fdb_$FDB_VERSION.tar.gz | tar zxf - --strip-components=1 && \ + for file in fdbbackup fdbcli fdbdr fdbmonitor fdbrestore fdbserver backup_agent dr_agent fastrestore_tool; do \ + chmod u+x $file; \ + mv $file /usr/bin; \ + done + +# Install additional FoundationDB Client Libraries +RUN for version in $FDB_LIBRARY_VERSIONS; do \ + curl $FDB_WEBSITE/downloads/$version/linux/libfdb_c_$version.so -o /usr/lib/fdb/multiversion/libfdb_c_${version%.*}.so; \ + done + +# Install additional FoundationDB Client Libraries (for sidecar) +RUN mkdir -p /var/fdb/lib && \ + for version in $FDB_LIBRARY_VERSIONS; do \ + curl $FDB_WEBSITE/downloads/$version/linux/libfdb_c_$version.so -o /var/fdb/lib/libfdb_c_${version%.*}.so; \ + done + +RUN curl $FDB_WEBSITE/downloads/$FDB_VERSION/linux/libfdb_c_$FDB_VERSION.so -o /usr/lib/libfdb_c.so + +RUN rm -rf /tmp/* +WORKDIR / + +FROM foundationdb-base as fdb-kubernetes-monitor + +# Install the kubernetes monitor binary +COPY --from=go-build /fdb-kubernetes-monitor /usr/bin/ + +# Runtime Configuration Options +USER fdb +WORKDIR /var/fdb +VOLUME /var/fdb/data +ENTRYPOINT ["/usr/bin/fdb-kubernetes-monitor"] + +FROM foundationdb-base as foundationdb-kubernetes-sidecar + +RUN yum -y install \ + rh-python38-2.0-4.el7 \ + yum clean all && \ + rm -rf /var/cache/yum && \ + source /opt/rh/rh-python38/enable && \ + pip3 install watchdog==0.9.0 + +WORKDIR / +ADD entrypoint.bash sidecar.py / +RUN chmod a+x /entrypoint.bash /sidecar.py +USER fdb +VOLUME /var/input-files +VOLUME /var/output-files +ENV LISTEN_PORT 8080 +ENTRYPOINT ["/usr/bin/tini", "-g", "--", "/entrypoint.bash"] + +FROM foundationdb-base as foundationdb + +WORKDIR /tmp +RUN curl -LsO https://raw.githubusercontent.com/brendangregg/FlameGraph/90533539b75400297092f973163b8a7b067c66d3/stackcollapse-perf.pl && \ + curl -LsO https://raw.githubusercontent.com/brendangregg/FlameGraph/90533539b75400297092f973163b8a7b067c66d3/flamegraph.pl && \ + echo "a682ac46497d6fdbf9904d1e405d3aea3ad255fcb156f6b2b1a541324628dfc0 flamegraph.pl" > flamegraph.sha256sum && \ + echo "5bcfb73ff2c2ab7bf2ad2b851125064780b58c51cc602335ec0001bec92679a5 stackcollapse-perf.pl" >> flamegraph.sha256sum && \ + sha256sum -c flamegraph.sha256sum && \ + chmod +x stackcollapse-perf.pl flamegraph.pl && \ + mv stackcollapse-perf.pl flamegraph.pl /usr/bin && \ + rm -rf /tmp/* +WORKDIR / +# Set Up Runtime Scripts and Directories +ADD fdb.bash /var/fdb/scripts/ +RUN chmod a+x /var/fdb/scripts/fdb.bash +VOLUME /var/fdb/data +ENV FDB_PORT 4500 +ENV FDB_CLUSTER_FILE /var/fdb/fdb.cluster +ENV FDB_NETWORKING_MODE container +ENV FDB_COORDINATOR "" +ENV FDB_COORDINATOR_PORT 4500 +ENV FDB_CLUSTER_FILE_CONTENTS "" +ENV FDB_PROCESS_CLASS unset +ENTRYPOINT ["/usr/bin/tini", "-g", "--", "/var/fdb/scripts/fdb.bash"] + +FROM base as ycsb + +RUN yum -y install \ + java-11-openjdk-11.0.13.0.8-1.el7_9 && \ + yum clean all && \ + rm -rf /var/cache/yum + +WORKDIR /tmp +RUN curl -Ls https://amazon-eks.s3.amazonaws.com/1.19.6/2021-01-05/bin/linux/amd64/kubectl -o kubectl && \ + echo "08ff68159bbcb844455167abb1d0de75bbfe5ae1b051f81ab060a1988027868a kubectl" > kubectl.txt && \ + sha256sum -c kubectl.txt && \ + mv kubectl /usr/local/bin/kubectl && \ + chmod 755 /usr/local/bin/kubectl && \ + curl https://awscli.amazonaws.com/awscli-exe-linux-x86_64-2.2.43.zip -o "awscliv2.zip" && \ + echo "9a8b3c4e7f72bbcc55e341dce3af42479f2730c225d6d265ee6f9162cfdebdfd awscliv2.zip" > awscliv2.txt && \ + sha256sum -c awscliv2.txt && \ + unzip -qq awscliv2.zip && \ + ./aws/install && \ + rm -rf /tmp/* + +# TODO: Log4J complains that it's eating the HTracer logs. Even without it, we get per-operation +# time series graphs of throughput, median, 90, 99, 99.9 and 99.99 (in usec). +ADD run_ycsb.sh /usr/local/bin/run_ycsb.sh +RUN mkdir -p /var/log/fdb-trace-logs && \ + chmod +x /usr/local/bin/run_ycsb.sh + +ADD YCSB /YCSB +WORKDIR /YCSB +ENV FDB_NETWORK_OPTION_EXTERNAL_CLIENT_DIRECTORY=/var/dynamic-conf/lib/multiversion/ +ENV FDB_NETWORK_OPTION_TRACE_ENABLE=/var/log/fdb-trace-logs +ENV LD_LIBRARY_PATH=/var/dynamic-conf/lib/ +ENV BUCKET="" +CMD ["run_ycsb.sh"] diff --git a/packaging/docker/Dockerfile.eks b/packaging/docker/Dockerfile.eks index 38fc61f56e..d08e25bd5e 100644 --- a/packaging/docker/Dockerfile.eks +++ b/packaging/docker/Dockerfile.eks @@ -15,45 +15,37 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. - +# FROM amazonlinux:2.0.20210326.0 as base RUN yum install -y \ - binutils \ - bind-utils \ - curl \ - gdb \ - jq \ - less \ - libsanitizer \ - lsof \ - nc \ - net-tools \ - perf \ - perl \ - procps \ - python38 \ - python3-pip \ - strace \ - tar \ - traceroute \ - telnet \ - tcpdump \ - unzip \ - vim && \ + bind-utils-9.11.4-26.P2.amzn2.5.2 \ + binutils-2.29.1-30.amzn2 \ + curl-7.76.1-7.amzn2.0.2 \ + gdb-8.0.1-36.amzn2.0.1 \ + hostname-3.13-3.amzn2.0.2 \ + jq-1.5-1.amzn2.0.2 \ + less-458-9.amzn2.0.2 \ + libsanitizer-7.3.1-13.amzn2 \ + lsof-4.87-6.amzn2 \ + net-tools-2.0-0.22.20131004git.amzn2.0.2 \ + nmap-ncat-6.40-13.amzn2 \ + perf-4.14.252-195.483.amzn2 \ + perl-5.16.3-299.amzn2.0.1 \ + procps-ng-3.3.10-26.amzn2 \ + strace-4.26-1.amzn2.0.1 \ + sysstat-10.1.5-12.amzn2 \ + tar-1.26-35.amzn2 \ + tcpdump-4.9.2-4.amzn2.1 \ + telnet-0.17-65.amzn2 \ + traceroute-2.0.22-2.amzn2.0.1 \ + unzip-6.0-43.amzn2 \ + vim-enhanced-8.1.1602-1.amzn2 && \ yum clean all && \ rm -rf /var/cache/yum -# TODO: nload, iperf, numademo +WORKDIR /tmp -RUN curl https://awscli.amazonaws.com/awscli-exe-linux-x86_64-2.2.43.zip -o "awscliv2.zip" && \ - echo "9a8b3c4e7f72bbcc55e341dce3af42479f2730c225d6d265ee6f9162cfdebdfd awscliv2.zip" > awscliv2.txt && \ - sha256sum -c awscliv2.txt && \ - unzip -qq awscliv2.zip && \ - ./aws/install && \ - rm -rf /tmp/* - -# Adding tini as PID 1 https://github.com/krallin/tini RUN curl -Ls https://github.com/krallin/tini/releases/download/v0.19.0/tini-amd64 -o tini && \ echo "93dcc18adc78c65a028a84799ecf8ad40c936fdfc5f2a57b1acda5a8117fa82c tini" > tini-amd64.sha256sum && \ sha256sum -c tini-amd64.sha256sum && \ @@ -61,49 +53,100 @@ RUN curl -Ls https://github.com/krallin/tini/releases/download/v0.19.0/tini-amd6 mv tini /usr/bin/ && \ rm -rf /tmp/* -# Install flamegraph +WORKDIR / + +FROM golang:1.16.7-bullseye AS go-build + +COPY fdbkubernetesmonitor/ /fdbkubernetesmonitor +WORKDIR /fdbkubernetesmonitor +RUN go build -o /fdb-kubernetes-monitor *.go + +FROM base as foundationdb-base + +WORKDIR /tmp +ARG FDB_VERSION=6.3.22 +ARG FDB_LIBRARY_VERSIONS="${FDB_VERSION}" +ARG FDB_WEBSITE=https://www.foundationdb.org + +RUN mkdir -p /var/fdb/{logs,tmp,lib} && \ + mkdir -p /usr/lib/fdb/multiversion && \ + echo ${FDB_VERSION} > /var/fdb/version + +# Set up a non-root user +RUN groupadd --gid 4059 fdb && \ + useradd --gid 4059 --uid 4059 --no-create-home --shell /bin/bash fdb && \ + chown -R fdb:fdb /var/fdb + +COPY website /tmp/website/ + +# Install FoundationDB Binaries +RUN curl $FDB_WEBSITE/downloads/$FDB_VERSION/linux/fdb_$FDB_VERSION.tar.gz | tar zxf - --strip-components=1 && \ + for file in fdbbackup fdbcli fdbdr fdbmonitor fdbrestore fdbserver backup_agent dr_agent fastrestore_tool; do \ + chmod u+x $file; \ + mv $file /usr/bin; \ + done + +# Install additional FoundationDB Client Libraries +RUN for version in $FDB_LIBRARY_VERSIONS; do \ + curl $FDB_WEBSITE/downloads/$version/linux/libfdb_c_$version.so -o /usr/lib/fdb/multiversion/libfdb_c_${version%.*}.so; \ + done + +# Install additional FoundationDB Client Libraries (for sidecar) +RUN mkdir -p /var/fdb/lib && \ + for version in $FDB_LIBRARY_VERSIONS; do \ + curl $FDB_WEBSITE/downloads/$version/linux/libfdb_c_$version.so -o /var/fdb/lib/libfdb_c_${version%.*}.so; \ + done + +RUN curl $FDB_WEBSITE/downloads/$FDB_VERSION/linux/libfdb_c_$FDB_VERSION.so -o /usr/lib/libfdb_c.so + +RUN rm -rf /tmp/* +WORKDIR / + +FROM foundationdb-base as fdb-kubernetes-monitor + +# Install the kubernetes monitor binary +COPY --from=go-build /fdb-kubernetes-monitor /usr/bin/ + +# Runtime Configuration Options +USER fdb +WORKDIR /var/fdb +VOLUME /var/fdb/data +ENTRYPOINT ["/usr/bin/fdb-kubernetes-monitor"] + +FROM foundationdb-base as foundationdb-kubernetes-sidecar + +RUN yum -y install \ + python3-3.7.10-1.amzn2.0.1 \ + python3-pip-20.2.2-1.amzn2.0.3 && \ + yum clean all && \ + rm -rf /var/cache/yum && \ + pip3 install watchdog==0.9.0 + +WORKDIR / +ADD entrypoint.bash sidecar.py / +RUN chmod a+x /entrypoint.bash /sidecar.py +USER fdb +VOLUME /var/input-files +VOLUME /var/output-files +ENV LISTEN_PORT 8080 +ENTRYPOINT ["/usr/bin/tini", "-g", "--", "/entrypoint.bash"] + +FROM foundationdb-base as foundationdb + +WORKDIR /tmp RUN curl -LsO https://raw.githubusercontent.com/brendangregg/FlameGraph/90533539b75400297092f973163b8a7b067c66d3/stackcollapse-perf.pl && \ curl -LsO https://raw.githubusercontent.com/brendangregg/FlameGraph/90533539b75400297092f973163b8a7b067c66d3/flamegraph.pl && \ echo "a682ac46497d6fdbf9904d1e405d3aea3ad255fcb156f6b2b1a541324628dfc0 flamegraph.pl" > flamegraph.sha256sum && \ echo "5bcfb73ff2c2ab7bf2ad2b851125064780b58c51cc602335ec0001bec92679a5 stackcollapse-perf.pl" >> flamegraph.sha256sum && \ sha256sum -c flamegraph.sha256sum && \ chmod +x stackcollapse-perf.pl flamegraph.pl && \ - mv stackcollapse-perf.pl flamegraph.pl /usr/bin - -ARG FDB_VERSION -ARG FDB_ADDITIONAL_VERSIONS="6.3.12 6.2.30 6.1.13 5.1.7" -ARG FDB_WEBSITE=https://www.foundationdb.org - -# Install additional FoundationDB Client Libraries -RUN mkdir -p /usr/lib/fdb/multiversion && \ - for version in $FDB_ADDITIONAL_VERSIONS; do \ - curl $FDB_WEBSITE/downloads/$version/linux/libfdb_c_$version.so -o /usr/lib/fdb/multiversion/libfdb_c_$version.so; \ - done && \ - rm -rf /mnt/website - -COPY --chown=root bin /usr/bin/ -COPY --chown=root lib/libfdb_c.so /var/fdb/lib/ -RUN mv /var/fdb/lib/libfdb_c.so /var/fdb/lib/libfdb_c_${FDB_VERSION%.*}.so -RUN ln -s /var/fdb/lib/libfdb_c_${FDB_VERSION%.*}.so /var/fdb/lib/libfdb_c.so - -# =========================== END OF LAYER: base =============================== - -FROM base as foundationdb - -ARG FDB_VERSION - + mv stackcollapse-perf.pl flamegraph.pl /usr/bin && \ + rm -rf /tmp/* WORKDIR / - # Set Up Runtime Scripts and Directories -ADD release/fdb.bash /var/fdb/scripts/ +ADD fdb.bash /var/fdb/scripts/ RUN chmod a+x /var/fdb/scripts/fdb.bash - -RUN mkdir -p /var/fdb/logs - VOLUME /var/fdb/data - -# Runtime Configuration Options - ENV FDB_PORT 4500 ENV FDB_CLUSTER_FILE /var/fdb/fdb.cluster ENV FDB_NETWORKING_MODE container @@ -111,35 +154,38 @@ ENV FDB_COORDINATOR "" ENV FDB_COORDINATOR_PORT 4500 ENV FDB_CLUSTER_FILE_CONTENTS "" ENV FDB_PROCESS_CLASS unset - ENTRYPOINT ["/usr/bin/tini", "-g", "--", "/var/fdb/scripts/fdb.bash"] -# =========================== END OF LAYER: foundationdb =============================== +FROM base as ycsb -FROM base AS sidecar -WORKDIR / +RUN yum -y install \ + java-11-amazon-corretto-11.0.13+8-1.amzn2 && \ + yum clean all && \ + rm -rf /var/cache/yum -ARG FDB_VERSION +WORKDIR /tmp +RUN curl -Ls https://amazon-eks.s3.amazonaws.com/1.19.6/2021-01-05/bin/linux/amd64/kubectl -o kubectl && \ + echo "08ff68159bbcb844455167abb1d0de75bbfe5ae1b051f81ab060a1988027868a kubectl" > kubectl.txt && \ + sha256sum -c kubectl.txt && \ + mv kubectl /usr/local/bin/kubectl && \ + chmod 755 /usr/local/bin/kubectl && \ + curl https://awscli.amazonaws.com/awscli-exe-linux-x86_64-2.2.43.zip -o "awscliv2.zip" && \ + echo "9a8b3c4e7f72bbcc55e341dce3af42479f2730c225d6d265ee6f9162cfdebdfd awscliv2.zip" > awscliv2.txt && \ + sha256sum -c awscliv2.txt && \ + unzip -qq awscliv2.zip && \ + ./aws/install && \ + rm -rf /tmp/* -# Set Up Runtime Scripts and Directories +# TODO: Log4J complains that it's eating the HTracer logs. Even without it, we get per-operation +# time series graphs of throughput, median, 90, 99, 99.9 and 99.99 (in usec). +ADD run_ycsb.sh /usr/local/bin/run_ycsb.sh +RUN mkdir -p /var/log/fdb-trace-logs && \ + chmod +x /usr/local/bin/run_ycsb.sh -ADD sidecar/entrypoint.bash sidecar/sidecar.py / -RUN chmod a+x /entrypoint.bash /sidecar.py -RUN pip3 install watchdog==0.9.0 - -RUN echo ${FDB_VERSION} > /var/fdb/version && \ - mkdir -p /var/fdb/lib && \ - groupadd --gid 4059 fdb && \ - useradd --gid 4059 --uid 4059 --no-create-home --shell /bin/bash fdb - -VOLUME /var/input-files -VOLUME /var/output-files - -USER fdb - -# Runtime Configuration Options - -ENV LISTEN_PORT 8080 - -ENTRYPOINT ["/usr/bin/tini", "-g", "--", "/entrypoint.bash"] -# =========================== END OF LAYER: sidecar =============================== +ADD YCSB /YCSB +WORKDIR /YCSB +ENV FDB_NETWORK_OPTION_EXTERNAL_CLIENT_DIRECTORY=/var/dynamic-conf/lib/multiversion/ +ENV FDB_NETWORK_OPTION_TRACE_ENABLE=/var/log/fdb-trace-logs +ENV LD_LIBRARY_PATH=/var/dynamic-conf/lib/ +ENV BUCKET="" +CMD ["run_ycsb.sh"] diff --git a/packaging/docker/build-eks-docker.sh b/packaging/docker/build-eks-docker.sh deleted file mode 100755 index 3161e559fb..0000000000 --- a/packaging/docker/build-eks-docker.sh +++ /dev/null @@ -1,62 +0,0 @@ -#!/usr/bin/env bash -set -Eeuo pipefail -set -x - -DOCKER_ROOT="$(realpath "$(dirname "${BASH_SOURCE[0]}")")" -BUILD_OUTPUT=$(realpath "${DOCKER_ROOT}"/../..) - -echo Docker root: "${DOCKER_ROOT}" -echo Build output: "${BUILD_OUTPUT}" - -cd "${DOCKER_ROOT}" - -## eg: CMAKE_PROJECT_VERSION:STATIC=7.0.0 -FDB_VERSION=$(grep CMAKE_PROJECT_VERSION: "${BUILD_OUTPUT}"/CMakeCache.txt | cut -d '=' -f 2) - -# Options (passed via environment variables) - -# Feel free to customize the image tag. -# TODO: add a mechanism to set TAG=FDB_VERSION when we're building public releases. -TAG=${TAG:-${FDB_VERSION}-${OKTETO_NAME}} -ECR=${ECR:-112664522426.dkr.ecr.us-west-2.amazonaws.com} - -echo Building with tag "${TAG}" - -# Login to ECR -# TODO: Move this to a common place instead of repeatedly copy-pasting it. -aws ecr get-login-password | docker login --username AWS --password-stdin "${ECR}" - -docker pull "${ECR}"/amazonlinux:2.0.20210326.0 -docker tag "${ECR}"/amazonlinux:2.0.20210326.0 amazonlinux:2.0.20210326.0 - - - -# derived variables -IMAGE=foundationdb/foundationdb:${TAG} -SIDECAR=foundationdb/foundationdb-kubernetes-sidecar:${TAG}-1 -STRIPPED=${STRIPPED:-false} - - - - - -if $STRIPPED; then - rsync -av --delete --exclude=*.xml "${BUILD_OUTPUT}"/packages/bin . - rsync -av --delete --exclude=*.a --exclude=*.xml "${BUILD_OUTPUT}"/packages/lib . -else - rsync -av --delete --exclude=*.xml "${BUILD_OUTPUT}"/bin . - rsync -av --delete --exclude=*.a --exclude=*.xml "${BUILD_OUTPUT}"/lib . -fi - -BUILD_ARGS="--build-arg FDB_VERSION=$FDB_VERSION" - - - -docker build ${BUILD_ARGS} -t "${IMAGE}" --target foundationdb -f Dockerfile.eks . -docker build ${BUILD_ARGS} -t "${SIDECAR}" --target sidecar -f Dockerfile.eks . - -docker tag "${IMAGE}" "${ECR}"/"${IMAGE}" -docker tag "${SIDECAR}" "${ECR}"/"${SIDECAR}" - -docker push "${ECR}"/"${IMAGE}" -docker push "${ECR}"/"${SIDECAR}" diff --git a/packaging/docker/build-images.sh b/packaging/docker/build-images.sh new file mode 100755 index 0000000000..6b0c3ecda7 --- /dev/null +++ b/packaging/docker/build-images.sh @@ -0,0 +1,275 @@ +#!/usr/bin/env bash +set -Eeuo pipefail +script_dir=$(cd "$(dirname "${BASH_SOURCE[0]}")" &>/dev/null && pwd -P) +reset=$(tput sgr0) +blue=$(tput setaf 4) + +function logg() { + printf "${blue}##### $(date +"%H:%M:%S") # %-56.55s #####${reset}\n" "${1}" +} + +function pushd () { + command pushd "$@" > /dev/null +} + +function popd () { + command popd > /dev/null +} + +function create_fake_website_directory() { + fdb_binaries=( 'fdbbackup' 'fdbcli' 'fdbserver' 'fdbmonitor' ) + fake_fdb_binaries=( 'backup_agent' 'dr_agent' 'fastrestore_tool' 'fdbdr' 'fdbrestore' ) + logg "PREPARING WEBSITE" + website_directory="${script_dir}/website" + rm -rf "${website_directory}" + mkdir -p "${website_directory}/downloads/${fdb_version}/linux/bin" + pushd "${website_directory}/downloads/${fdb_version}/linux/bin" || exit 127 + ############################################################################ + # there are four intended paths here: + # 1) fetch the unstripped binaries and client library from artifactory_base_url + # 2) fetch the stripped binaries and multiple client library versions + # from artifactory_base_url + # 3) copy the unstripped binaries and client library from the current local + # build_output of foundationdb + # 4) copy the stripped binaries and client library from the current local + # build_output of foundationdb + ############################################################################ + logg "FETCHING BINARIES" + case "${stripped_binaries_and_from_where}" in + "unstripped_artifactory") + logg "DOWNLOADING BINARIES TAR FILE" + curl -Ls "${artifactory_base_url}/${fdb_version}/release/api/foundationdb-binaries-${fdb_version}-linux.tar.gz" | tar -xzf - + ;; + "stripped_artifactory") + for file in "${fdb_binaries[@]}"; do + logg "DOWNLOADING ${file}" + curl -Ls "${artifactory_base_url}/${fdb_version}/release/files/linux/bin/${file}" -o "${file}" + chmod 755 "${file}" + done + ;; + "unstripped_local") + for file in "${fdb_binaries[@]}"; do + logg "COPYING ${file}" + cp -pr "${build_output_directory}/bin/${file}" "${file}" + chmod 755 "${file}" + done + ;; + "stripped_local") + for file in "${fdb_binaries[@]}"; do + logg "COPYING ${file}" + cp -pr "${build_output_directory}/packaging/bin/${file}" "${file}" + chmod 755 "${file}" + done + ;; + esac + # dont download files that are binary duplicates of fdbbackup, recreate the + # symlinks (to fdbbackup in the same directory) + logg "CREATING fdbbackup SYMLINKS" + for fake in "${fake_fdb_binaries[@]}"; do + logg "CREATING ${fake}" + ln -sf fdbbackup "${fake}" + done + popd || exit 128 + # re-create the same file that is present in the downloads path of + # foundationdb.org such that it can be copied into the Docker image and + # referenced with a file:// url by the container image build + logg "CREATING BINARIES TAR FILE" + tar -czf "${website_directory}/downloads/${fdb_version}/linux/fdb_${fdb_version}.tar.gz" --directory "${website_directory}/downloads/${fdb_version}/linux/bin" . + rm -rf "${website_directory}/downloads/${fdb_version}/linux/bin" + + ############################################################################ + # this follows the same logic as the case statement above, they are separate + # because it allows for the simplification of the steps that create the + # symlinks and the binaries tarball + ############################################################################ + logg "FETCHING CLIENT LIBRARY" + case "${stripped_binaries_and_from_where}" in + "unstripped_artifactory") + for version in "${fdb_library_versions[@]}"; do + logg "FETCHING ${version} CLIENT LIBRARY" + destination_directory="${website_directory}/downloads/${version}/linux" + destination_filename="libfdb_c_${version}.so" + mkdir -p "${destination_directory}" + pushd "${destination_directory}" || exit 127 + curl -Ls "${artifactory_base_url}/${version}/release/api/fdb-server-${version}-linux.tar.gz" | tar -xzf - ./lib/libfdb_c.so --strip-components 2 + mv "libfdb_c.so" "${destination_filename}" + chmod 755 "${destination_filename}" + popd || exit 128 + done + ;; + "stripped_artifactory") + for version in "${fdb_library_versions[@]}"; do + logg "FETCHING ${version} CLIENT LIBRARY" + destination_directory="${website_directory}/downloads/${version}/linux" + destination_filename="libfdb_c_${version}.so" + mkdir -p "${destination_directory}" + pushd "${destination_directory}" || exit 127 + curl -Ls "${artifactory_base_url}/${version}/release/files/linux/lib/libfdb_c.so" -o "${destination_filename}" + chmod 755 "${destination_filename}" + popd || exit 128 + done + ;; + "unstripped_local") + logg "COPYING UNSTRIPPED CLIENT LIBRARY" + cp -pr "${build_output_directory}/lib/libfdb_c.so" "${website_directory}/downloads/${fdb_version}/linux/libfdb_c_${fdb_version}.so" + ;; + "stripped_local") + logg "COPYING STRIPPED CLIENT LIBRARY" + cp -pr "${build_output_directory}/packaging/lib/libfdb_c.so" "${website_directory}/downloads/${fdb_version}/linux/libfdb_c_${fdb_version}.so" + ;; + esac + # override fdb_website variable that is passed to Docker build + fdb_website="file:///tmp/website" +} + +function compile_ycsb() { + logg "COMPILING YCSB" + if [ "${use_development_java_bindings}" == "true" ]; then + logg "INSTALL JAVA BINDINGS" + foundationdb_java_version="${fdb_version}-PRERELEASE" + mvn install:install-file \ + --batch-mode \ + -Dorg.slf4j.simpleLogger.log.org.apache.maven.cli.transfer.Slf4jMavenTransferListener=warn \ + -Dfile="${build_output_directory}/packages/fdb-java-${foundationdb_java_version}.jar" \ + -DgroupId=org.foundationdb \ + -DartifactId=fdb-java \ + -Dversion="${foundationdb_java_version}" \ + -Dpackaging=jar \ + -DgeneratePom=true + else + foundationdb_java_version="${fdb_version}" + fi + rm -rf "${script_dir}/YCSB" + mkdir -p "${script_dir}/YCSB" + pushd "${script_dir}/YCSB" || exit 127 + if [ -d "${HOME}/src/YCSB" ]; then + rsync -av "${HOME}"/src/YCSB/. . + else + git clone https://github.com/FoundationDB/YCSB.git . + fi + sed -i "s/[0-9]\+.[0-9]\+.[0-9]\+<\/foundationdb.version>/${foundationdb_java_version}<\/foundationdb.version>/g" pom.xml + mvn --batch-mode -Dorg.slf4j.simpleLogger.log.org.apache.maven.cli.transfer.Slf4jMavenTransferListener=warn -pl site.ycsb:foundationdb-binding -am clean package + mkdir -p core/target/dependency + # shellcheck disable=SC2046 + cp $(find "${HOME}/.m2" -name jax\*.jar) core/target/dependency/ + # shellcheck disable=SC2046 + cp $(find "${HOME}/.m2" -name htrace\*.jar) core/target/dependency/ + # shellcheck disable=SC2046 + cp $(find "${HOME}/.m2" -name HdrHistogram\*.jar) core/target/dependency/ + rm -rf .git + popd || exit 128 +} + +function build_and_push_images(){ + declare -a tags_to_push=() + for image in "${image_list[@]}"; do + logg "BUILDING ${image}" + image_tag="${tag_base}${image}:${fdb_version}" + if [ "${image}" == "foundationdb-kubernetes-sidecar" ]; then + image_tag="${image_tag}-1" + fi + if [ "${dockerfile_name}" == "Dockerfile.eks" ]; then + image_tag="${image_tag}-debug" + fi + if [ "${image}" == "ycsb" ]; then + compile_ycsb + fi + logg "TAG ${image_tag#${registry}/foundationdb/}" + docker build \ + --label "org.foundationdb.version=${fdb_version}" \ + --label "org.foundationdb.build_date=${build_date}" \ + --label "org.foundationdb.commit=${commit_sha}" \ + --progress plain \ + --build-arg FDB_VERSION="${fdb_version}" \ + --build-arg FDB_LIBRARY_VERSIONS="${fdb_library_versions[*]}" \ + --build-arg FDB_WEBSITE="${fdb_website}" \ + --tag "${image_tag}" \ + --file "${dockerfile_name}" \ + --target "${image}" . + if [ "${image}" == 'foundationdb' ] || [ "${image}" == 'foundationdb-kubernetes-sidecar' ] || [ "${image}" == 'ycsb' ] ; then + tags_to_push+=("${image_tag}") + fi + done + + if [ "${push_docker_images}" == "true" ]; then + for tag in "${tags_to_push[@]}"; do + logg "PUSH ${tag}" + docker push "${tag}" + done + fi +} + +echo "${blue}################################################################################${reset}" +logg "STARTING ${0}" +echo "${blue}################################################################################${reset}" + +################################################################################ +# The intent of this script is to build the set of docker images needed to run +# FoundationDB in kubernetes from binaries that are not available the website: +# https://foundationdb.org/downloads +# +# The docker file itself will pull released binaries from the foundationdb +# website. If the intent is to build images for an already released version of +# FoundationDB, a simple docker build command will work. +# +# This script has enough stupid built into it that trying to come up with a set +# of sensible default options that will work everywhere has gotten silly. Below +# are a set of variable definitions that need to be set for this script to +# execute to completion the defaults are based on the FoundationDB development +# environment used by the team at Apple, they will not work for everyone cloning +# this project. +# +# Use this script with care. +################################################################################ +artifactory_base_url="${ARTIFACTORY_URL:-https://artifactory.foundationdb.org}" +aws_region="us-west-2" +aws_account_id=$(aws --output text sts get-caller-identity --query 'Account') +build_date=$(date +"%Y-%m-%dT%H:%M:%S%z") +build_output_directory="${script_dir}/../../build_output" +commit_sha=$(git rev-parse --verify HEAD --short=10) +fdb_version=$(awk '/^[[:space:]]+VERSION[[:space:]]+[[:digit:]]+\.[[:digit:]]+\.[[:digit:]]+(-rc[[:digit:]])?/{print $2}' "${script_dir}/../../CMakeLists.txt") +fdb_library_versions=( '5.1.7' '6.1.13' '6.2.30' "${fdb_version}" ) +fdb_website="https://www.foundationdb.org" +image_list=( + 'base' + # 'go-build' + 'foundationdb-base' + 'foundationdb' + # 'foundationdb-kubernetes-monitor' + 'foundationdb-kubernetes-sidecar' + 'ycsb' +) +registry="" +tag_base="foundationdb/" +# THESE CONTROL THE PATH OF FUNCTIONS THAT ARE CALLED BELOW +stripped_binaries_and_from_where="stripped_local" # MUST BE ONE OF ( "unstripped_artifactory" "stripped_artifactory" "unstripped_local" "stripped_local" ) +dockerfile_name="Dockerfile" +use_development_java_bindings="false" +push_docker_images="false" + +if [ -n "${OKTETO_NAMESPACE+x}" ]; then + logg "RUNNING IN OKTETO/AWS" + # these are defaults for the Apple development environment + aws_region=$(curl -s "http://169.254.169.254/latest/meta-data/placement/region") + aws_account_id=$(aws --output text sts get-caller-identity --query 'Account') + build_output_directory="${HOME}/build_output" + fdb_library_versions=( "${fdb_version}" ) + registry="${aws_account_id}.dkr.ecr.${aws_region}.amazonaws.com" + tag_base="${registry}/foundationdb/" + stripped_binaries_and_from_where="unstripped_local" # MUST BE ONE OF ( "unstripped_artifactory" "stripped_artifactory" "unstripped_local" "stripped_local" ) + dockerfile_name="Dockerfile.eks" + use_development_java_bindings="true" + push_docker_images="true" +else + echo "Dear ${USER}, you probably need to edit this file before running it. " + echo "${0} has a very narrow set of situations where it will be successful," + echo "or even useful, when executed unedited" + exit 1 +fi + +create_fake_website_directory +build_and_push_images + +echo "${blue}################################################################################${reset}" +logg "COMPLETED ${0}" +echo "${blue}################################################################################${reset}" diff --git a/packaging/docker/build-release-docker.sh b/packaging/docker/build-release-docker.sh deleted file mode 100755 index bd13c22382..0000000000 --- a/packaging/docker/build-release-docker.sh +++ /dev/null @@ -1,62 +0,0 @@ -#!/usr/bin/env bash -set -Eeuo pipefail -set -x - -DOCKER_ROOT="$(realpath "$(dirname "${BASH_SOURCE[0]}")")" -BUILD_OUTPUT=$(realpath "${DOCKER_ROOT}"/../..) - -echo Docker root: "${DOCKER_ROOT}" -echo Build output: "${BUILD_OUTPUT}" - -cd "${DOCKER_ROOT}" - -## eg: CMAKE_PROJECT_VERSION:STATIC=7.0.0 -FDB_VERSION=$(grep CMAKE_PROJECT_VERSION: "${BUILD_OUTPUT}"/CMakeCache.txt | cut -d '=' -f 2) - -# Options (passed via environment variables) - -# Feel free to customize the image tag. -# TODO: add a mechanism to set TAG=FDB_VERSION when we're building public releases. -TAG=${TAG:-${FDB_VERSION}-${OKTETO_NAME}} -ECR=${ECR:-112664522426.dkr.ecr.us-west-2.amazonaws.com} - -echo Building with tag "${TAG}" - -# Login to ECR -# TODO: Move this to a common place instead of repeatedly copy-pasting it. -aws ecr get-login-password | docker login --username AWS --password-stdin "${ECR}" - -docker pull "${ECR}"/ubuntu:18.04 -docker tag "${ECR}"/ubuntu:18.04 ubuntu:18.04 -docker pull "${ECR}"/python:3.9-slim -docker tag "${ECR}"/python:3.9-slim python:3.9-slim - -# derived variables -IMAGE=foundationdb/foundationdb:"${TAG}" -SIDECAR=foundationdb/foundationdb-kubernetes-sidecar:"${TAG}"-1 -STRIPPED=${STRIPPED:-false} - -WEBSITE_BIN_DIR=website/downloads/"${FDB_VERSION}"/linux -TARBALL=${WEBSITE_BIN_DIR}/fdb_"${FDB_VERSION}".tar.gz -mkdir -p "${WEBSITE_BIN_DIR}" - -if $STRIPPED; then - tar -C ~/build_output/packages/ -zcvf "${TARBALL}" bin lib - cp ~/build_output/packages/lib/libfdb_c.so "${WEBSITE_BIN_DIR}"/libfdb_c_"${FDB_VERSION}".so -else - tar -C ~/build_output/ -zcvf "${TARBALL}" bin lib - cp ~/build_output/lib/libfdb_c.so "${WEBSITE_BIN_DIR}"/libfdb_c_"${FDB_VERSION}".so -fi - -BUILD_ARGS="--build-arg FDB_VERSION=${FDB_VERSION}" -BUILD_ARGS+=" --build-arg FDB_WEBSITE=file:///mnt/website" -BUILD_ARGS+=" --build-arg FDB_ADDITIONAL_VERSIONS=${FDB_VERSION}" - -docker build ${BUILD_ARGS} -t "${IMAGE}" -f release/Dockerfile . -docker build ${BUILD_ARGS} -t "${SIDECAR}" -f sidecar/Dockerfile . - -docker tag "${IMAGE}" "${ECR}"/"${IMAGE}" -docker tag "${SIDECAR}" "${ECR}"/"${SIDECAR}" - -docker push "${ECR}"/"${IMAGE}" -docker push "${ECR}"/"${SIDECAR}" diff --git a/packaging/docker/build-ycsb-docker.sh b/packaging/docker/build-ycsb-docker.sh deleted file mode 100755 index 38b1765123..0000000000 --- a/packaging/docker/build-ycsb-docker.sh +++ /dev/null @@ -1,64 +0,0 @@ -#!/usr/bin/env bash -set -Eeuo pipefail -set -x - -DOCKER_ROOT="$(realpath "$(dirname "${BASH_SOURCE[0]}")")" -BUILD_OUTPUT=$(realpath "${DOCKER_ROOT}"/../..) - -echo Docker root: "${DOCKER_ROOT}" -echo Build output: "${BUILD_OUTPUT}" - -cd "${DOCKER_ROOT}" - -## eg: CMAKE_PROJECT_VERSION:STATIC=7.0.0 -FDB_VERSION=$(grep CMAKE_PROJECT_VERSION: "${BUILD_OUTPUT}"/CMakeCache.txt | cut -d '=' -f 2) - -# Options (passed via environment variables) - -# Feel free to customize the image tag. -# TODO: add a mechanism to set TAG=FDB_VERSION when we're building public releases. -TAG=${TAG:-${FDB_VERSION}-${OKTETO_NAME}} -ECR=${ECR:-112664522426.dkr.ecr.us-west-2.amazonaws.com} - -echo Building with tag "${TAG}" - -# Login to ECR -# TODO: Move this to a common place instead of repeatedly copy-pasting it. -aws ecr get-login-password | docker login --username AWS --password-stdin "${ECR}" - -docker pull "${ECR}"/openjdk:17-slim -docker tag "${ECR}"/openjdk:17-slim openjdk:17-slim - - - -# derived variables -IMAGE=foundationdb/ycsb:"${TAG}" - -# mvn install fdb-java, compile YCSB -mvn install:install-file \ - -Dfile="${BUILD_OUTPUT}"/packages/fdb-java-"${FDB_VERSION}"-PRERELEASE.jar \ - -DgroupId=org.foundationdb \ - -DartifactId=fdb-java \ - -Dversion="${FDB_VERSION}"-PRERELEASE \ - -Dpackaging=jar \ - -DgeneratePom=true -mkdir "${DOCKER_ROOT}"/YCSB && cd "${DOCKER_ROOT}"/YCSB -git clone https://github.com/FoundationDB/YCSB.git . -sed -i "s/[0-9]\+.[0-9]\+.[0-9]\+<\/foundationdb.version>/${FDB_VERSION}-PRERELEASE<\/foundationdb.version>/g" pom.xml -mvn -pl site.ycsb:foundationdb-binding -am clean package -mkdir -p core/target/dependency -# shellcheck disable=SC2046 -cp $(find ~/.m2/ -name jax\*.jar) core/target/dependency/ -# shellcheck disable=SC2046 -cp $(find ~/.m2/ -name htrace\*.jar) core/target/dependency/ -# shellcheck disable=SC2046 -cp $(find ~/.m2/ -name HdrHistogram\*.jar) core/target/dependency/ -rm -rf .git && cd .. - -docker build -t "${IMAGE}" -f ycsb/Dockerfile . - - -docker tag "${IMAGE}" "${ECR}"/"${IMAGE}" - - -docker push "${ECR}"/"${IMAGE}" diff --git a/packaging/docker/sidecar/entrypoint.bash b/packaging/docker/entrypoint.bash similarity index 90% rename from packaging/docker/sidecar/entrypoint.bash rename to packaging/docker/entrypoint.bash index 165f11bce4..b9a62cfabd 100755 --- a/packaging/docker/sidecar/entrypoint.bash +++ b/packaging/docker/entrypoint.bash @@ -24,4 +24,8 @@ if [[ -n "$ADDITIONAL_ENV_FILE" ]]; then source $ADDITIONAL_ENV_FILE fi +if [[ -f "/opt/rh/rh-python38/enable" ]]; then + source /opt/rh/rh-python38/enable +fi + exec /sidecar.py $* diff --git a/packaging/docker/release/fdb.bash b/packaging/docker/fdb.bash similarity index 100% rename from packaging/docker/release/fdb.bash rename to packaging/docker/fdb.bash diff --git a/packaging/docker/kubernetes/Dockerfile b/packaging/docker/kubernetes/Dockerfile deleted file mode 100644 index de529138b7..0000000000 --- a/packaging/docker/kubernetes/Dockerfile +++ /dev/null @@ -1,86 +0,0 @@ -# Dockerfile -# -# This source file is part of the FoundationDB open source project -# -# Copyright 2021 Apple Inc. and the FoundationDB project authors -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -# This docker image assumes that the context for the docker build is pointed -# at the root of the foundationdb repository. - -# Build the Kubernetes monitor - -FROM golang:1.16.7-bullseye AS go-build - -COPY fdbkubernetesmonitor/ /fdbkubernetesmonitor -WORKDIR /fdbkubernetesmonitor -RUN go build -o /fdb-kubernetes-monitor *.go - -# Build the main image - -FROM centos:7.9.2009 - -RUN yum install -y \ - binutils-2.27-44.base.el7 \ - bind-utils-9.11.4-26.P2.el7_9.7 \ - curl-7.29.0-59.el7_9.1 \ - less-458-9.el7 \ - lsof-4.87-6.el7 \ - nano-2.3.1-10.el7 \ - nmap-ncat-6.40-19.el7 \ - net-tools-2.0-0.25.20131004git.el7 \ - strace-4.24-6.el7 \ - tar-1.26-35.el7 \ - telnet-0.17-66.el7 \ - traceroute-2.0.22-2.el7 \ - tcpdump-4.9.2-4.el7_7.1 \ - vim-enhanced-7.4.629-8.el7_9 \ - && yum clean all - -ARG FDB_VERSION -ARG FDB_LIBRARY_VERSIONS="${FDB_VERSION}" -ARG FDB_WEBSITE=https://www.foundationdb.org - -COPY packaging/docker/website /mnt/website/ - -# Install FoundationDB Binaries -RUN mkdir -p /var/fdb/logs && mkdir -p /var/fdb/tmp && \ - curl $FDB_WEBSITE/downloads/$FDB_VERSION/linux/fdb_$FDB_VERSION.tar.gz | tar zxf - --strip-components=1 && \ - chmod u+x fdbbackup fdbcli fdbdr fdbmonitor fdbrestore fdbserver backup_agent dr_agent && \ - mv fdbbackup fdbcli fdbdr fdbmonitor fdbrestore fdbserver backup_agent dr_agent /usr/bin && \ - echo ${FDB_VERSION} > /var/fdb/version - -# Install additional FoundationDB Client Libraries -RUN mkdir -p /usr/lib/fdb/multiversion && \ - for version in $FDB_LIBRARY_VERSIONS; do curl $FDB_WEBSITE/downloads/$version/linux/libfdb_c_$version.so -o /usr/lib/fdb/multiversion/libfdb_c_${version%.*}.so; done - -# Clean up temporary directories -RUN rm -rf /mnt/website && rm -r /var/fdb/tmp - -# Install the kubernetes monitor binary -COPY --from=go-build /fdb-kubernetes-monitor /usr/bin/ - -# Set up a non-root user - -RUN groupadd --gid 4059 fdb && \ - useradd --gid 4059 --uid 4059 --no-create-home --shell /bin/bash fdb && \ - chown -R fdb:fdb /var/fdb - -# Runtime Configuration Options - -USER fdb -WORKDIR /var/fdb -ENTRYPOINT ["/usr/bin/fdb-kubernetes-monitor"] -VOLUME /var/fdb/data diff --git a/packaging/docker/release/Dockerfile b/packaging/docker/release/Dockerfile deleted file mode 100644 index fc58f64d49..0000000000 --- a/packaging/docker/release/Dockerfile +++ /dev/null @@ -1,100 +0,0 @@ -# Dockerfile -# -# This source file is part of the FoundationDB open source project -# -# Copyright 2013-2021 Apple Inc. and the FoundationDB project authors -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -FROM ubuntu:18.04 - -RUN apt-get update && \ - apt-get install -y \ - curl \ - dnsutils \ - jq \ - less \ - linux-tools-generic \ - lsof \ - net-tools \ - netcat \ - openssl \ - perl \ - strace \ - tcpdump \ - tcptraceroute \ - telnet \ - vim && \ - rm -rf /var/lib/apt/lists/* - -WORKDIR /tmp -# Adding tini as PID 1 https://github.com/krallin/tini -RUN curl -Ls https://github.com/krallin/tini/releases/download/v0.19.0/tini-amd64 -o tini && \ - echo "93dcc18adc78c65a028a84799ecf8ad40c936fdfc5f2a57b1acda5a8117fa82c tini" > tini-amd64.sha256sum && \ - sha256sum -c tini-amd64.sha256sum && \ - chmod +x tini && \ - mv tini /usr/bin/ && \ - rm -rf /tmp/* - -# Install flamegraph -RUN curl -LsO https://raw.githubusercontent.com/brendangregg/FlameGraph/90533539b75400297092f973163b8a7b067c66d3/stackcollapse-perf.pl && \ - curl -LsO https://raw.githubusercontent.com/brendangregg/FlameGraph/90533539b75400297092f973163b8a7b067c66d3/flamegraph.pl && \ - echo "a682ac46497d6fdbf9904d1e405d3aea3ad255fcb156f6b2b1a541324628dfc0 flamegraph.pl" > flamegraph.sha256sum && \ - echo "5bcfb73ff2c2ab7bf2ad2b851125064780b58c51cc602335ec0001bec92679a5 stackcollapse-perf.pl" >> flamegraph.sha256sum && \ - sha256sum -c flamegraph.sha256sum && \ - chmod +x stackcollapse-perf.pl flamegraph.pl && \ - mv stackcollapse-perf.pl flamegraph.pl /usr/bin - -ARG FDB_VERSION -ARG FDB_ADDITIONAL_VERSIONS="5.1.7" -ARG FDB_WEBSITE=https://www.foundationdb.org - -COPY website /mnt/website/ - -RUN curl $FDB_WEBSITE/downloads/$FDB_VERSION/linux/libfdb_c_$FDB_VERSION.so -o /usr/lib/libfdb_c.so - -# Install FoundationDB Binaries -RUN curl $FDB_WEBSITE/downloads/$FDB_VERSION/linux/fdb_$FDB_VERSION.tar.gz | tar zxf - --strip-components=1 && \ - chmod u+x fdbbackup fdbcli fdbdr fdbmonitor fdbrestore fdbserver backup_agent dr_agent && \ - mv fdbbackup fdbcli fdbdr fdbmonitor fdbrestore fdbserver backup_agent dr_agent /usr/bin && \ - rm -rf /tmp/* - -# Install additional FoundationDB Client Libraries -RUN mkdir -p /usr/lib/fdb/multiversion && \ - for version in $FDB_ADDITIONAL_VERSIONS; do \ - curl $FDB_WEBSITE/downloads/$version/linux/libfdb_c_$version.so -o /usr/lib/fdb/multiversion/libfdb_c_$version.so; \ - done && \ - rm -rf /mnt/website - -WORKDIR / - -# Set Up Runtime Scripts and Directories -ADD release/fdb.bash /var/fdb/scripts/ -RUN chmod a+x /var/fdb/scripts/fdb.bash - -RUN mkdir -p /var/fdb/logs - -VOLUME /var/fdb/data - -# Runtime Configuration Options - -ENV FDB_PORT 4500 -ENV FDB_CLUSTER_FILE /var/fdb/fdb.cluster -ENV FDB_NETWORKING_MODE container -ENV FDB_COORDINATOR "" -ENV FDB_COORDINATOR_PORT 4500 -ENV FDB_CLUSTER_FILE_CONTENTS "" -ENV FDB_PROCESS_CLASS unset - -ENTRYPOINT ["/usr/bin/tini", "-g", "--", "/var/fdb/scripts/fdb.bash"] diff --git a/packaging/docker/ycsb/run_ycsb.sh b/packaging/docker/run_ycsb.sh old mode 100644 new mode 100755 similarity index 100% rename from packaging/docker/ycsb/run_ycsb.sh rename to packaging/docker/run_ycsb.sh diff --git a/packaging/docker/sidecar/sidecar.py b/packaging/docker/sidecar.py similarity index 100% rename from packaging/docker/sidecar/sidecar.py rename to packaging/docker/sidecar.py diff --git a/packaging/docker/sidecar/Dockerfile b/packaging/docker/sidecar/Dockerfile deleted file mode 100644 index 3c281f8987..0000000000 --- a/packaging/docker/sidecar/Dockerfile +++ /dev/null @@ -1,77 +0,0 @@ -# Dockerfile -# -# This source file is part of the FoundationDB open source project -# -# Copyright 2013-2021 Apple Inc. and the FoundationDB project authors -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -FROM python:3.9-slim - -RUN apt-get update && \ - apt-get install -y --no-install-recommends \ - curl && \ - pip install watchdog==0.9.0 && \ - rm -rf /var/lib/apt/lists/* - -WORKDIR /tmp -# Adding tini as PID 1 https://github.com/krallin/tini -RUN curl -Ls https://github.com/krallin/tini/releases/download/v0.19.0/tini-amd64 -o tini && \ - echo "93dcc18adc78c65a028a84799ecf8ad40c936fdfc5f2a57b1acda5a8117fa82c tini" > tini-amd64.sha256sum && \ - sha256sum -c tini-amd64.sha256sum && \ - chmod +x tini && \ - mv tini /usr/bin/ && \ - rm -rf /tmp/* - -ARG FDB_VERSION -ARG FDB_ADDITIONAL_VERSIONS="6.2.30 6.1.13" -ARG FDB_WEBSITE=https://www.foundationdb.org - -COPY website /mnt/website/ - -# Install FoundationDB Binaries -RUN curl $FDB_WEBSITE/downloads/$FDB_VERSION/linux/fdb_$FDB_VERSION.tar.gz | tar zxf - --strip-components=1 && \ - chmod u+x fdbbackup fdbcli fdbdr fdbmonitor fdbrestore fdbserver backup_agent dr_agent && \ - mv fdbbackup fdbcli fdbdr fdbmonitor fdbrestore fdbserver backup_agent dr_agent /usr/bin && \ - rm -rf /tmp/* - -# Install additional FoundationDB Client Libraries -RUN mkdir -p /var/fdb/lib && \ - for version in $FDB_ADDITIONAL_VERSIONS; do \ - curl $FDB_WEBSITE/downloads/$version/linux/libfdb_c_$version.so -o /var/fdb/lib/libfdb_c_${version%.*}.so; \ - done && \ - rm -rf /mnt/website - -WORKDIR / - -# Set Up Runtime Scripts and Directories -ADD sidecar/entrypoint.bash sidecar/sidecar.py / -RUN chmod a+x /entrypoint.bash /sidecar.py - -RUN echo ${FDB_VERSION} > /var/fdb/version && \ - mkdir -p /var/fdb/lib && \ - groupadd --gid 4059 fdb && \ - useradd --gid 4059 --uid 4059 --no-create-home --shell /bin/bash fdb - - -VOLUME /var/input-files -VOLUME /var/output-files - -USER fdb - -# Runtime Configuration Options - -ENV LISTEN_PORT 8080 - -ENTRYPOINT ["/usr/bin/tini", "-g", "--", "/entrypoint.bash"] diff --git a/packaging/docker/sidecar/requirements.txt b/packaging/docker/sidecar/requirements.txt deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/packaging/docker/ycsb/Dockerfile b/packaging/docker/ycsb/Dockerfile deleted file mode 100644 index 1cf7b1e537..0000000000 --- a/packaging/docker/ycsb/Dockerfile +++ /dev/null @@ -1,32 +0,0 @@ -FROM openjdk:17-slim AS RUN - -WORKDIR /tmp -RUN apt-get update && \ - apt-get install -y curl zip && \ - curl -Ls https://amazon-eks.s3.amazonaws.com/1.19.6/2021-01-05/bin/linux/amd64/kubectl -o kubectl && \ - echo "08ff68159bbcb844455167abb1d0de75bbfe5ae1b051f81ab060a1988027868a kubectl" > kubectl.txt && \ - sha256sum -c kubectl.txt && \ - mv kubectl /usr/local/bin/kubectl && \ - chmod 755 /usr/local/bin/kubectl && \ - curl https://awscli.amazonaws.com/awscli-exe-linux-x86_64-2.2.43.zip -o "awscliv2.zip" && \ - echo "9a8b3c4e7f72bbcc55e341dce3af42479f2730c225d6d265ee6f9162cfdebdfd awscliv2.zip" > awscliv2.txt && \ - sha256sum -c awscliv2.txt && \ - unzip -qq awscliv2.zip && \ - ./aws/install && \ - rm -rf /tmp/* - -ADD YCSB /YCSB -WORKDIR /YCSB - -ENV FDB_NETWORK_OPTION_EXTERNAL_CLIENT_DIRECTORY=/var/dynamic-conf/lib/multiversion/ -ENV FDB_NETWORK_OPTION_TRACE_ENABLE=/var/log/fdb-trace-logs -ENV LD_LIBRARY_PATH=/var/dynamic-conf/lib/ -ENV BUCKET=backup-112664522426-us-west-2 - -# TODO: Log4J complains that it's eating the HTracer logs. Even without it, we get per-operation -# time series graphs of throughput, median, 90, 99, 99.9 and 99.99 (in usec). -COPY ycsb/run_ycsb.sh /usr/local/bin/run_ycsb.sh -RUN mkdir -p /var/log/fdb-trace-logs && \ - chmod +x /usr/local/bin/run_ycsb.sh - -CMD ["run_ycsb.sh"] From 18243351e73f0f6b7180e23e9d422c9ec949cb2c Mon Sep 17 00:00:00 2001 From: Lukas Joswiak Date: Wed, 17 Nov 2021 15:25:36 -0800 Subject: [PATCH 5/5] Fix possible data race Transactions (created on a separate thread) can read the `globals` field at the same time as `setGlobal` is called on the main thread, causing a potential race. TSAN surfaced this issue. --- flow/Net2.actor.cpp | 4 ++-- flow/network.h | 3 ++- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/flow/Net2.actor.cpp b/flow/Net2.actor.cpp index f728f570ee..b97e3f4a2f 100644 --- a/flow/Net2.actor.cpp +++ b/flow/Net2.actor.cpp @@ -199,7 +199,7 @@ public: flowGlobalType global(int id) const override { return (globals.size() > id) ? globals[id] : nullptr; } void setGlobal(size_t id, flowGlobalType v) override { - globals.resize(std::max(globals.size(), id + 1)); + ASSERT(id < globals.size()); globals[id] = v; } @@ -1187,7 +1187,7 @@ struct PromiseTask final : public Task, public FastAllocated { // 5MB for loading files into memory Net2::Net2(const TLSConfig& tlsConfig, bool useThreadPool, bool useMetrics) - : useThreadPool(useThreadPool), reactor(this), + : globals(enumGlobal::COUNT), useThreadPool(useThreadPool), reactor(this), #ifndef TLS_DISABLED sslContextVar({ ReferencedObject::from( boost::asio::ssl::context(boost::asio::ssl::context::tls)) }), diff --git a/flow/network.h b/flow/network.h index 3cc41639b3..cd7ea691b8 100644 --- a/flow/network.h +++ b/flow/network.h @@ -540,7 +540,8 @@ public: enGlobalConfig = 14, enChaosMetrics = 15, enDiskFailureInjector = 16, - enBitFlipper = 17 + enBitFlipper = 17, + COUNT // Add new fields before this enumerator }; virtual void longTaskCheck(const char* name) {}