apple/foundationdb
1
0
Fork 0
mirror of https://github.com/apple/foundationdb.git synced 2025-05-17 03:12:21 +08:00
Code Issues Packages Projects Releases Wiki Activity

rebase

Browse Source
This commit is contained in:
Dan Lambright 2021-12-10 10:54:17 -05:00
commit 8aef860a8b
21 changed files with 642 additions and 603 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

8
cmake/InstallLayout.cmake
View File

@ -185,12 +185,6 @@ install(DIRECTORY "${script_dir}/clients/usr/lib/cmake"
DESTINATION usr/lib DESTINATION usr/lib
COMPONENT clients-versioned) COMPONENT clients-versioned)
################################################################################
# Move Docker Setup
################################################################################
file(COPY "${PROJECT_SOURCE_DIR}/packaging/docker" DESTINATION "${PROJECT_BINARY_DIR}/packages/")
################################################################################ ################################################################################
# General CPack configuration # General CPack configuration
################################################################################ ################################################################################
@ -228,7 +222,7 @@ set(CPACK_COMPONENT_CLIENTS-TGZ_DISPLAY_NAME "foundationdb-clients")
set(CPACK_COMPONENT_CLIENTS-VERSIONED_DISPLAY_NAME "foundationdb${PROJECT_VERSION}-clients") set(CPACK_COMPONENT_CLIENTS-VERSIONED_DISPLAY_NAME "foundationdb${PROJECT_VERSION}-clients")
# MacOS needs a file exiension for the LICENSE file # MacOS needs a file extension for the LICENSE file
configure_file(${CMAKE_SOURCE_DIR}/LICENSE ${CMAKE_BINARY_DIR}/License.txt COPYONLY) configure_file(${CMAKE_SOURCE_DIR}/LICENSE ${CMAKE_BINARY_DIR}/License.txt COPYONLY)
################################################################################ ################################################################################

2
fdbclient/ServerKnobs.cpp
View File

@ -38,7 +38,7 @@ void ServerKnobs::initialize(Randomize randomize, ClientKnobs* clientKnobs, IsSi
init( MAX_COMMIT_BATCH_INTERVAL, 2.0 ); if( randomize && BUGGIFY ) MAX_COMMIT_BATCH_INTERVAL = 0.5; // Each commit proxy generates a CommitTransactionBatchRequest at least this often, so that versions always advance smoothly init( MAX_COMMIT_BATCH_INTERVAL, 2.0 ); if( randomize && BUGGIFY ) MAX_COMMIT_BATCH_INTERVAL = 0.5; // Each commit proxy generates a CommitTransactionBatchRequest at least this often, so that versions always advance smoothly
MAX_COMMIT_BATCH_INTERVAL = std::min(MAX_COMMIT_BATCH_INTERVAL, MAX_READ_TRANSACTION_LIFE_VERSIONS/double(2*VERSIONS_PER_SECOND)); // Ensure that the proxy commits 2 times every MAX_READ_TRANSACTION_LIFE_VERSIONS, otherwise the master will not give out versions fast enough MAX_COMMIT_BATCH_INTERVAL = std::min(MAX_COMMIT_BATCH_INTERVAL, MAX_READ_TRANSACTION_LIFE_VERSIONS/double(2*VERSIONS_PER_SECOND)); // Ensure that the proxy commits 2 times every MAX_READ_TRANSACTION_LIFE_VERSIONS, otherwise the master will not give out versions fast enough
init( ENABLE_VERSION_VECTOR, true ); init( ENABLE_VERSION_VECTOR, true );
init( ENABLE_VERSION_VECTOR_TLOG_UNICAST, true ); init( ENABLE_VERSION_VECTOR_TLOG_UNICAST, false );
// TLogs // TLogs
init( TLOG_TIMEOUT, 0.4 ); //cannot buggify because of availability init( TLOG_TIMEOUT, 0.4 ); //cannot buggify because of availability

2
fdbclient/rapidjson/rapidjson.h
View File

@ -449,7 +449,7 @@ RAPIDJSON_NAMESPACE_END
*/ */
#define RAPIDJSON_STATIC_ASSERT(x) \ #define RAPIDJSON_STATIC_ASSERT(x) \
typedef ::RAPIDJSON_NAMESPACE::StaticAssertTest<sizeof(::RAPIDJSON_NAMESPACE::STATIC_ASSERTION_FAILURE<bool(x)>)> \ typedef ::RAPIDJSON_NAMESPACE::StaticAssertTest<sizeof(::RAPIDJSON_NAMESPACE::STATIC_ASSERTION_FAILURE<bool(x)>)> \
RAPIDJSON_JOIN(StaticAssertTypedef, __LINE__) RAPIDJSON_STATIC_ASSERT_UNUSED_ATTRIBUTE RAPIDJSON_JOIN(StaticAssertTypedef, __LINE__) RAPIDJSON_STATIC_ASSERT_UNUSED_ATTRIBUTE
#endif #endif
/////////////////////////////////////////////////////////////////////////////// ///////////////////////////////////////////////////////////////////////////////

10
flow/TDMetric.actor.h
View File

@ -343,11 +343,11 @@ auto tuple_map_impl(F f, index_sequence<Is...>, const Tuples&... ts)
// tuple_map( f(a,b), (a1,a2,a3), (b1,b2,b3) ) = (f(a1,b1), f(a2,b2), f(a3,b3)) // tuple_map( f(a,b), (a1,a2,a3), (b1,b2,b3) ) = (f(a1,b1), f(a2,b2), f(a3,b3))
template <typename F, typename Tuple, typename... Tuples> template <typename F, typename Tuple, typename... Tuples>
auto tuple_map(F f, const Tuple& t, const Tuples&... ts) -> decltype( auto tuple_map(F f, const Tuple& t, const Tuples&... ts) -> decltype(tuple_map_impl(
tuple_map_impl(f, f,
typename make_index_sequence_impl<0, index_sequence<>, std::tuple_size<Tuple>::value>::type(), typename make_index_sequence_impl<0, index_sequence<>, std::tuple_size<Tuple>::value>::type(),
t, t,
ts...)) { ts...)) {
return tuple_map_impl( return tuple_map_impl(
f, typename make_index_sequence_impl<0, index_sequence<>, std::tuple_size<Tuple>::value>::type(), t, ts...); f, typename make_index_sequence_impl<0, index_sequence<>, std::tuple_size<Tuple>::value>::type(), t, ts...);
} }

194
packaging/docker/Dockerfile Normal file
View File

@ -0,0 +1,194 @@
# Dockerfile
#
# This source file is part of the FoundationDB open source project
#
# Copyright 2013-2021 Apple Inc. and the FoundationDB project authors
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
FROM centos:7.9.2009 as base
RUN yum install -y \
epel-release-7-11 \
centos-release-scl-2-3.el7.centos && \
yum install -y \
bind-utils-9.11.4-26.P2.el7_9.7 \
binutils-2.27-44.base.el7_9.1 \
curl-7.29.0-59.el7_9.1 \
gdb-7.6.1-120.el7 \
hostname-3.13-3.el7_7.1 \
jq-1.6-2.el7 \
less-458-9.el7 \
libubsan-7.3.1-5.16.el7 \
lsof-4.87-6.el7 \
net-tools-2.0-0.25.20131004git.el7 \
nmap-ncat-6.40-19.el7 \
perf-3.10.0-1160.45.1.el7 \
perl-5.16.3-299.el7_9 \
procps-ng-3.3.10-28.el7 \
strace-4.24-6.el7 \
sysstat-10.1.5-19.el7 \
tar-1.26-35.el7 \
tcpdump-4.9.2-4.el7_7.1 \
telnet-0.17-66.el7 \
traceroute-2.0.22-2.el7 \
unzip-6.0-22.el7_9 \
vim-enhanced-7.4.629-8.el7_9 && \
yum clean all && \
rm -rf /var/cache/yum
WORKDIR /tmp
RUN curl -Ls https://github.com/krallin/tini/releases/download/v0.19.0/tini-amd64 -o tini && \
echo "93dcc18adc78c65a028a84799ecf8ad40c936fdfc5f2a57b1acda5a8117fa82c tini" > tini-amd64.sha256sum && \
sha256sum -c tini-amd64.sha256sum && \
chmod +x tini && \
mv tini /usr/bin/ && \
rm -rf /tmp/*
WORKDIR /
FROM golang:1.16.7-bullseye AS go-build
COPY fdbkubernetesmonitor/ /fdbkubernetesmonitor
WORKDIR /fdbkubernetesmonitor
RUN go build -o /fdb-kubernetes-monitor *.go
FROM base as foundationdb-base
WORKDIR /tmp
ARG FDB_VERSION=6.3.22
ARG FDB_LIBRARY_VERSIONS="${FDB_VERSION}"
ARG FDB_WEBSITE=https://www.foundationdb.org
RUN mkdir -p /var/fdb/{logs,tmp,lib} && \
mkdir -p /usr/lib/fdb/multiversion && \
echo ${FDB_VERSION} > /var/fdb/version
# Set up a non-root user
RUN groupadd --gid 4059 fdb && \
useradd --gid 4059 --uid 4059 --no-create-home --shell /bin/bash fdb && \
chown -R fdb:fdb /var/fdb
COPY website /tmp/website/
# Install FoundationDB Binaries
RUN curl $FDB_WEBSITE/downloads/$FDB_VERSION/linux/fdb_$FDB_VERSION.tar.gz | tar zxf - --strip-components=1 && \
for file in fdbbackup fdbcli fdbdr fdbmonitor fdbrestore fdbserver backup_agent dr_agent fastrestore_tool; do \
chmod u+x $file; \
mv $file /usr/bin; \
done
# Install additional FoundationDB Client Libraries
RUN for version in $FDB_LIBRARY_VERSIONS; do \
curl $FDB_WEBSITE/downloads/$version/linux/libfdb_c_$version.so -o /usr/lib/fdb/multiversion/libfdb_c_${version%.*}.so; \
done
# Install additional FoundationDB Client Libraries (for sidecar)
RUN mkdir -p /var/fdb/lib && \
for version in $FDB_LIBRARY_VERSIONS; do \
curl $FDB_WEBSITE/downloads/$version/linux/libfdb_c_$version.so -o /var/fdb/lib/libfdb_c_${version%.*}.so; \
done
RUN curl $FDB_WEBSITE/downloads/$FDB_VERSION/linux/libfdb_c_$FDB_VERSION.so -o /usr/lib/libfdb_c.so
RUN rm -rf /tmp/*
WORKDIR /
FROM foundationdb-base as fdb-kubernetes-monitor
# Install the kubernetes monitor binary
COPY --from=go-build /fdb-kubernetes-monitor /usr/bin/
# Runtime Configuration Options
USER fdb
WORKDIR /var/fdb
VOLUME /var/fdb/data
ENTRYPOINT ["/usr/bin/fdb-kubernetes-monitor"]
FROM foundationdb-base as foundationdb-kubernetes-sidecar
RUN yum -y install \
rh-python38-2.0-4.el7 \
yum clean all && \
rm -rf /var/cache/yum && \
source /opt/rh/rh-python38/enable && \
pip3 install watchdog==0.9.0
WORKDIR /
ADD entrypoint.bash sidecar.py /
RUN chmod a+x /entrypoint.bash /sidecar.py
USER fdb
VOLUME /var/input-files
VOLUME /var/output-files
ENV LISTEN_PORT 8080
ENTRYPOINT ["/usr/bin/tini", "-g", "--", "/entrypoint.bash"]
FROM foundationdb-base as foundationdb
WORKDIR /tmp
RUN curl -LsO https://raw.githubusercontent.com/brendangregg/FlameGraph/90533539b75400297092f973163b8a7b067c66d3/stackcollapse-perf.pl && \
curl -LsO https://raw.githubusercontent.com/brendangregg/FlameGraph/90533539b75400297092f973163b8a7b067c66d3/flamegraph.pl && \
echo "a682ac46497d6fdbf9904d1e405d3aea3ad255fcb156f6b2b1a541324628dfc0 flamegraph.pl" > flamegraph.sha256sum && \
echo "5bcfb73ff2c2ab7bf2ad2b851125064780b58c51cc602335ec0001bec92679a5 stackcollapse-perf.pl" >> flamegraph.sha256sum && \
sha256sum -c flamegraph.sha256sum && \
chmod +x stackcollapse-perf.pl flamegraph.pl && \
mv stackcollapse-perf.pl flamegraph.pl /usr/bin && \
rm -rf /tmp/*
WORKDIR /
# Set Up Runtime Scripts and Directories
ADD fdb.bash /var/fdb/scripts/
RUN chmod a+x /var/fdb/scripts/fdb.bash
VOLUME /var/fdb/data
ENV FDB_PORT 4500
ENV FDB_CLUSTER_FILE /var/fdb/fdb.cluster
ENV FDB_NETWORKING_MODE container
ENV FDB_COORDINATOR ""
ENV FDB_COORDINATOR_PORT 4500
ENV FDB_CLUSTER_FILE_CONTENTS ""
ENV FDB_PROCESS_CLASS unset
ENTRYPOINT ["/usr/bin/tini", "-g", "--", "/var/fdb/scripts/fdb.bash"]
FROM base as ycsb
RUN yum -y install \
java-11-openjdk-11.0.13.0.8-1.el7_9 && \
yum clean all && \
rm -rf /var/cache/yum
WORKDIR /tmp
RUN curl -Ls https://amazon-eks.s3.amazonaws.com/1.19.6/2021-01-05/bin/linux/amd64/kubectl -o kubectl && \
echo "08ff68159bbcb844455167abb1d0de75bbfe5ae1b051f81ab060a1988027868a kubectl" > kubectl.txt && \
sha256sum -c kubectl.txt && \
mv kubectl /usr/local/bin/kubectl && \
chmod 755 /usr/local/bin/kubectl && \
curl https://awscli.amazonaws.com/awscli-exe-linux-x86_64-2.2.43.zip -o "awscliv2.zip" && \
echo "9a8b3c4e7f72bbcc55e341dce3af42479f2730c225d6d265ee6f9162cfdebdfd awscliv2.zip" > awscliv2.txt && \
sha256sum -c awscliv2.txt && \
unzip -qq awscliv2.zip && \
./aws/install && \
rm -rf /tmp/*
# TODO: Log4J complains that it's eating the HTracer logs. Even without it, we get per-operation
# time series graphs of throughput, median, 90, 99, 99.9 and 99.99 (in usec).
ADD run_ycsb.sh /usr/local/bin/run_ycsb.sh
RUN mkdir -p /var/log/fdb-trace-logs && \
chmod +x /usr/local/bin/run_ycsb.sh
ADD YCSB /YCSB
WORKDIR /YCSB
ENV FDB_NETWORK_OPTION_EXTERNAL_CLIENT_DIRECTORY=/var/dynamic-conf/lib/multiversion/
ENV FDB_NETWORK_OPTION_TRACE_ENABLE=/var/log/fdb-trace-logs
ENV LD_LIBRARY_PATH=/var/dynamic-conf/lib/
ENV BUCKET=""
CMD ["run_ycsb.sh"]

236
packaging/docker/Dockerfile.eks
View File

@ -1,76 +1,152 @@
# Dockerfile
#
# This source file is part of the FoundationDB open source project
#
# Copyright 2013-2021 Apple Inc. and the FoundationDB project authors
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
FROM amazonlinux:2.0.20210326.0 as base FROM amazonlinux:2.0.20210326.0 as base
RUN yum install -y \ RUN yum install -y \
binutils \ bind-utils-9.11.4-26.P2.amzn2.5.2 \
bind-utils \ binutils-2.29.1-30.amzn2 \
curl \ curl-7.76.1-7.amzn2.0.2 \
gdb \ gdb-8.0.1-36.amzn2.0.1 \
jq \ hostname-3.13-3.amzn2.0.2 \
less \ jq-1.5-1.amzn2.0.2 \
lsof \ less-458-9.amzn2.0.2 \
nc \ libsanitizer-7.3.1-13.amzn2 \
net-tools \ lsof-4.87-6.amzn2 \
perf \ net-tools-2.0-0.22.20131004git.amzn2.0.2 \
perl \ nmap-ncat-6.40-13.amzn2 \
procps \ perf-4.14.252-195.483.amzn2 \
python38 \ perl-5.16.3-299.amzn2.0.1 \
python3-pip \ procps-ng-3.3.10-26.amzn2 \
strace \ strace-4.26-1.amzn2.0.1 \
tar \ sysstat-10.1.5-12.amzn2 \
traceroute \ tar-1.26-35.amzn2 \
telnet \ tcpdump-4.9.2-4.amzn2.1 \
tcpdump \ telnet-0.17-65.amzn2 \
unzip \ traceroute-2.0.22-2.amzn2.0.1 \
vim unzip-6.0-43.amzn2 \
vim-enhanced-8.1.1602-1.amzn2 && \
yum clean all && \
rm -rf /var/cache/yum
#todo: nload, iperf, numademo WORKDIR /tmp
RUN curl "https://awscli.amazonaws.com/awscli-exe-linux-x86_64-2.0.30.zip" -o "awscliv2.zip" \ RUN curl -Ls https://github.com/krallin/tini/releases/download/v0.19.0/tini-amd64 -o tini && \
&& unzip awscliv2.zip && ./aws/install && rm -rf aws echo "93dcc18adc78c65a028a84799ecf8ad40c936fdfc5f2a57b1acda5a8117fa82c tini" > tini-amd64.sha256sum && \
sha256sum -c tini-amd64.sha256sum && \
chmod +x tini && \
mv tini /usr/bin/ && \
rm -rf /tmp/*
COPY misc/tini-amd64.sha256sum /tmp/ WORKDIR /
COPY misc/flamegraph.sha256sum /tmp/
# Adding tini as PID 1 https://github.com/krallin/tini
ARG TINI_VERSION=v0.19.0
RUN curl -sLO https://github.com/krallin/tini/releases/download/${TINI_VERSION}/tini-amd64 && \
sha256sum -c /tmp/tini-amd64.sha256sum && \
chmod +x tini-amd64 && \
mv tini-amd64 /usr/bin/tini
COPY sidecar/requirements.txt /tmp FROM golang:1.16.7-bullseye AS go-build
RUN pip3 install -r /tmp/requirements.txt
# Install flamegraph COPY fdbkubernetesmonitor/ /fdbkubernetesmonitor
RUN curl -sLO https://raw.githubusercontent.com/brendangregg/FlameGraph/90533539b75400297092f973163b8a7b067c66d3/stackcollapse-perf.pl && \ WORKDIR /fdbkubernetesmonitor
curl -sLO https://raw.githubusercontent.com/brendangregg/FlameGraph/90533539b75400297092f973163b8a7b067c66d3/flamegraph.pl && \ RUN go build -o /fdb-kubernetes-monitor *.go
sha256sum -c /tmp/flamegraph.sha256sum && \
chmod +x stackcollapse-perf.pl flamegraph.pl && \
mv stackcollapse-perf.pl flamegraph.pl /usr/bin
# TODO: Only used by sidecar FROM base as foundationdb-base
WORKDIR /tmp
ARG FDB_VERSION=6.3.22
ARG FDB_LIBRARY_VERSIONS="${FDB_VERSION}"
ARG FDB_WEBSITE=https://www.foundationdb.org
RUN mkdir -p /var/fdb/{logs,tmp,lib} && \
mkdir -p /usr/lib/fdb/multiversion && \
echo ${FDB_VERSION} > /var/fdb/version
# Set up a non-root user
RUN groupadd --gid 4059 fdb && \ RUN groupadd --gid 4059 fdb && \
useradd --gid 4059 --uid 4059 --no-create-home --shell /bin/bash fdb useradd --gid 4059 --uid 4059 --no-create-home --shell /bin/bash fdb && \
chown -R fdb:fdb /var/fdb
ARG FDB_VERSION COPY website /tmp/website/
# These are the output of the current build (not stripped) # Install FoundationDB Binaries
COPY --chown=root bin /usr/bin/ RUN curl $FDB_WEBSITE/downloads/$FDB_VERSION/linux/fdb_$FDB_VERSION.tar.gz | tar zxf - --strip-components=1 && \
COPY --chown=root lib/libfdb_c.so /var/fdb/lib/ for file in fdbbackup fdbcli fdbdr fdbmonitor fdbrestore fdbserver backup_agent dr_agent fastrestore_tool; do \
RUN mv /var/fdb/lib/libfdb_c.so /var/fdb/lib/libfdb_c_${FDB_VERSION%.*}.so chmod u+x $file; \
RUN ln -s /var/fdb/lib/libfdb_c_${FDB_VERSION%.*}.so /var/fdb/lib/libfdb_c.so mv $file /usr/bin; \
# ------------------------------------------------- done
FROM base as foundationdb # Install additional FoundationDB Client Libraries
RUN for version in $FDB_LIBRARY_VERSIONS; do \
curl $FDB_WEBSITE/downloads/$version/linux/libfdb_c_$version.so -o /usr/lib/fdb/multiversion/libfdb_c_${version%.*}.so; \
done
COPY release/*.bash /var/fdb/scripts/ # Install additional FoundationDB Client Libraries (for sidecar)
RUN mkdir -p /var/fdb/logs RUN mkdir -p /var/fdb/lib && \
for version in $FDB_LIBRARY_VERSIONS; do \
curl $FDB_WEBSITE/downloads/$version/linux/libfdb_c_$version.so -o /var/fdb/lib/libfdb_c_${version%.*}.so; \
done
# TODO: FDB_ADDITIONAL_VERSIONS RUN curl $FDB_WEBSITE/downloads/$FDB_VERSION/linux/libfdb_c_$FDB_VERSION.so -o /usr/lib/libfdb_c.so
RUN mkdir -p /usr/lib/fdb/multiversion
VOLUME /var/fdb/data RUN rm -rf /tmp/*
WORKDIR /
FROM foundationdb-base as fdb-kubernetes-monitor
# Install the kubernetes monitor binary
COPY --from=go-build /fdb-kubernetes-monitor /usr/bin/
# Runtime Configuration Options # Runtime Configuration Options
USER fdb
WORKDIR /var/fdb
VOLUME /var/fdb/data
ENTRYPOINT ["/usr/bin/fdb-kubernetes-monitor"]
FROM foundationdb-base as foundationdb-kubernetes-sidecar
RUN yum -y install \
python3-3.7.10-1.amzn2.0.1 \
python3-pip-20.2.2-1.amzn2.0.3 && \
yum clean all && \
rm -rf /var/cache/yum && \
pip3 install watchdog==0.9.0
WORKDIR /
ADD entrypoint.bash sidecar.py /
RUN chmod a+x /entrypoint.bash /sidecar.py
USER fdb
VOLUME /var/input-files
VOLUME /var/output-files
ENV LISTEN_PORT 8080
ENTRYPOINT ["/usr/bin/tini", "-g", "--", "/entrypoint.bash"]
FROM foundationdb-base as foundationdb
WORKDIR /tmp
RUN curl -LsO https://raw.githubusercontent.com/brendangregg/FlameGraph/90533539b75400297092f973163b8a7b067c66d3/stackcollapse-perf.pl && \
curl -LsO https://raw.githubusercontent.com/brendangregg/FlameGraph/90533539b75400297092f973163b8a7b067c66d3/flamegraph.pl && \
echo "a682ac46497d6fdbf9904d1e405d3aea3ad255fcb156f6b2b1a541324628dfc0 flamegraph.pl" > flamegraph.sha256sum && \
echo "5bcfb73ff2c2ab7bf2ad2b851125064780b58c51cc602335ec0001bec92679a5 stackcollapse-perf.pl" >> flamegraph.sha256sum && \
sha256sum -c flamegraph.sha256sum && \
chmod +x stackcollapse-perf.pl flamegraph.pl && \
mv stackcollapse-perf.pl flamegraph.pl /usr/bin && \
rm -rf /tmp/*
WORKDIR /
# Set Up Runtime Scripts and Directories
ADD fdb.bash /var/fdb/scripts/
RUN chmod a+x /var/fdb/scripts/fdb.bash
VOLUME /var/fdb/data
ENV FDB_PORT 4500 ENV FDB_PORT 4500
ENV FDB_CLUSTER_FILE /var/fdb/fdb.cluster ENV FDB_CLUSTER_FILE /var/fdb/fdb.cluster
ENV FDB_NETWORKING_MODE container ENV FDB_NETWORKING_MODE container
@ -78,28 +154,38 @@ ENV FDB_COORDINATOR ""
ENV FDB_COORDINATOR_PORT 4500 ENV FDB_COORDINATOR_PORT 4500
ENV FDB_CLUSTER_FILE_CONTENTS "" ENV FDB_CLUSTER_FILE_CONTENTS ""
ENV FDB_PROCESS_CLASS unset ENV FDB_PROCESS_CLASS unset
ENTRYPOINT ["/usr/bin/tini", "-g", "--", "/var/fdb/scripts/fdb.bash"]
ENTRYPOINT ["/usr/bin/tini", "-g", "--"] FROM base as ycsb
CMD /var/fdb/scripts/fdb.bash
# ------------------------------------------------- RUN yum -y install \
java-11-amazon-corretto-11.0.13+8-1.amzn2 && \
yum clean all && \
rm -rf /var/cache/yum
FROM base AS sidecar WORKDIR /tmp
RUN curl -Ls https://amazon-eks.s3.amazonaws.com/1.19.6/2021-01-05/bin/linux/amd64/kubectl -o kubectl && \
echo "08ff68159bbcb844455167abb1d0de75bbfe5ae1b051f81ab060a1988027868a kubectl" > kubectl.txt && \
sha256sum -c kubectl.txt && \
mv kubectl /usr/local/bin/kubectl && \
chmod 755 /usr/local/bin/kubectl && \
curl https://awscli.amazonaws.com/awscli-exe-linux-x86_64-2.2.43.zip -o "awscliv2.zip" && \
echo "9a8b3c4e7f72bbcc55e341dce3af42479f2730c225d6d265ee6f9162cfdebdfd awscliv2.zip" > awscliv2.txt && \
sha256sum -c awscliv2.txt && \
unzip -qq awscliv2.zip && \
./aws/install && \
rm -rf /tmp/*
COPY sidecar/entrypoint.bash / # TODO: Log4J complains that it's eating the HTracer logs. Even without it, we get per-operation
COPY sidecar/sidecar.py / # time series graphs of throughput, median, 90, 99, 99.9 and 99.99 (in usec).
RUN chmod a+x /sidecar.py /entrypoint.bash ADD run_ycsb.sh /usr/local/bin/run_ycsb.sh
RUN mkdir -p /var/log/fdb-trace-logs && \
chmod +x /usr/local/bin/run_ycsb.sh
VOLUME /var/input-files ADD YCSB /YCSB
VOLUME /var/output-files WORKDIR /YCSB
ENV FDB_NETWORK_OPTION_EXTERNAL_CLIENT_DIRECTORY=/var/dynamic-conf/lib/multiversion/
ARG FDB_VERSION ENV FDB_NETWORK_OPTION_TRACE_ENABLE=/var/log/fdb-trace-logs
ENV LD_LIBRARY_PATH=/var/dynamic-conf/lib/
RUN echo ${FDB_VERSION} ; echo ${FDB_VERSION}> /var/fdb/version ENV BUCKET=""
RUN mkdir -p /var/fdb/lib CMD ["run_ycsb.sh"]
ENV LISTEN_PORT 8080
USER fdb
ENTRYPOINT ["/usr/bin/tini", "-g", "--", "/entrypoint.bash"]

62
packaging/docker/build-eks-docker.sh
View File

@ -1,62 +0,0 @@
#!/usr/bin/env bash
set -Eeuo pipefail
set -x
DOCKER_ROOT="$(realpath "$(dirname "${BASH_SOURCE[0]}")")"
BUILD_OUTPUT=$(realpath "${DOCKER_ROOT}"/../..)
echo Docker root: "${DOCKER_ROOT}"
echo Build output: "${BUILD_OUTPUT}"
cd "${DOCKER_ROOT}"
## eg: CMAKE_PROJECT_VERSION:STATIC=7.0.0
FDB_VERSION=$(grep CMAKE_PROJECT_VERSION: "${BUILD_OUTPUT}"/CMakeCache.txt | cut -d '=' -f 2)
# Options (passed via environment variables)
# Feel free to customize the image tag.
# TODO: add a mechanism to set TAG=FDB_VERSION when we're building public releases.
TAG=${TAG:-${FDB_VERSION}-${OKTETO_NAME}}
ECR=${ECR:-112664522426.dkr.ecr.us-west-2.amazonaws.com}
echo Building with tag "${TAG}"
# Login to ECR
# TODO: Move this to a common place instead of repeatedly copy-pasting it.
aws ecr get-login-password | docker login --username AWS --password-stdin "${ECR}"
docker pull "${ECR}"/amazonlinux:2.0.20210326.0
docker tag "${ECR}"/amazonlinux:2.0.20210326.0 amazonlinux:2.0.20210326.0
# derived variables
IMAGE=foundationdb/foundationdb:${TAG}
SIDECAR=foundationdb/foundationdb-kubernetes-sidecar:${TAG}-1
STRIPPED=${STRIPPED:-false}
if $STRIPPED; then
rsync -av --delete --exclude=*.xml "${BUILD_OUTPUT}"/packages/bin .
rsync -av --delete --exclude=*.a --exclude=*.xml "${BUILD_OUTPUT}"/packages/lib .
else
rsync -av --delete --exclude=*.xml "${BUILD_OUTPUT}"/bin .
rsync -av --delete --exclude=*.a --exclude=*.xml "${BUILD_OUTPUT}"/lib .
fi
BUILD_ARGS="--build-arg FDB_VERSION=$FDB_VERSION"
docker build ${BUILD_ARGS} -t "${IMAGE}" --target foundationdb -f Dockerfile.eks .
docker build ${BUILD_ARGS} -t "${SIDECAR}" --target sidecar -f Dockerfile.eks .
docker tag "${IMAGE}" "${ECR}"/"${IMAGE}"
docker tag "${SIDECAR}" "${ECR}"/"${SIDECAR}"
docker push "${ECR}"/"${IMAGE}"
docker push "${ECR}"/"${SIDECAR}"

275
packaging/docker/build-images.sh Executable file
View File

@ -0,0 +1,275 @@
#!/usr/bin/env bash
set -Eeuo pipefail
script_dir=$(cd "$(dirname "${BASH_SOURCE[0]}")" &>/dev/null && pwd -P)
reset=$(tput sgr0)
blue=$(tput setaf 4)
function logg() {
printf "${blue}##### $(date +"%H:%M:%S") # %-56.55s #####${reset}\n" "${1}"
}
function pushd () {
command pushd "$@" > /dev/null
}
function popd () {
command popd > /dev/null
}
function create_fake_website_directory() {
fdb_binaries=( 'fdbbackup' 'fdbcli' 'fdbserver' 'fdbmonitor' )
fake_fdb_binaries=( 'backup_agent' 'dr_agent' 'fastrestore_tool' 'fdbdr' 'fdbrestore' )
logg "PREPARING WEBSITE"
website_directory="${script_dir}/website"
rm -rf "${website_directory}"
mkdir -p "${website_directory}/downloads/${fdb_version}/linux/bin"
pushd "${website_directory}/downloads/${fdb_version}/linux/bin" || exit 127
############################################################################
# there are four intended paths here:
# 1) fetch the unstripped binaries and client library from artifactory_base_url
# 2) fetch the stripped binaries and multiple client library versions
# from artifactory_base_url
# 3) copy the unstripped binaries and client library from the current local
# build_output of foundationdb
# 4) copy the stripped binaries and client library from the current local
# build_output of foundationdb
############################################################################
logg "FETCHING BINARIES"
case "${stripped_binaries_and_from_where}" in
"unstripped_artifactory")
logg "DOWNLOADING BINARIES TAR FILE"
curl -Ls "${artifactory_base_url}/${fdb_version}/release/api/foundationdb-binaries-${fdb_version}-linux.tar.gz" | tar -xzf -
;;
"stripped_artifactory")
for file in "${fdb_binaries[@]}"; do
logg "DOWNLOADING ${file}"
curl -Ls "${artifactory_base_url}/${fdb_version}/release/files/linux/bin/${file}" -o "${file}"
chmod 755 "${file}"
done
;;
"unstripped_local")
for file in "${fdb_binaries[@]}"; do
logg "COPYING ${file}"
cp -pr "${build_output_directory}/bin/${file}" "${file}"
chmod 755 "${file}"
done
;;
"stripped_local")
for file in "${fdb_binaries[@]}"; do
logg "COPYING ${file}"
cp -pr "${build_output_directory}/packaging/bin/${file}" "${file}"
chmod 755 "${file}"
done
;;
esac
# dont download files that are binary duplicates of fdbbackup, recreate the
# symlinks (to fdbbackup in the same directory)
logg "CREATING fdbbackup SYMLINKS"
for fake in "${fake_fdb_binaries[@]}"; do
logg "CREATING ${fake}"
ln -sf fdbbackup "${fake}"
done
popd || exit 128
# re-create the same file that is present in the downloads path of
# foundationdb.org such that it can be copied into the Docker image and
# referenced with a file:// url by the container image build
logg "CREATING BINARIES TAR FILE"
tar -czf "${website_directory}/downloads/${fdb_version}/linux/fdb_${fdb_version}.tar.gz" --directory "${website_directory}/downloads/${fdb_version}/linux/bin" .
rm -rf "${website_directory}/downloads/${fdb_version}/linux/bin"
############################################################################
# this follows the same logic as the case statement above, they are separate
# because it allows for the simplification of the steps that create the
# symlinks and the binaries tarball
############################################################################
logg "FETCHING CLIENT LIBRARY"
case "${stripped_binaries_and_from_where}" in
"unstripped_artifactory")
for version in "${fdb_library_versions[@]}"; do
logg "FETCHING ${version} CLIENT LIBRARY"
destination_directory="${website_directory}/downloads/${version}/linux"
destination_filename="libfdb_c_${version}.so"
mkdir -p "${destination_directory}"
pushd "${destination_directory}" || exit 127
curl -Ls "${artifactory_base_url}/${version}/release/api/fdb-server-${version}-linux.tar.gz" | tar -xzf - ./lib/libfdb_c.so --strip-components 2
mv "libfdb_c.so" "${destination_filename}"
chmod 755 "${destination_filename}"
popd || exit 128
done
;;
"stripped_artifactory")
for version in "${fdb_library_versions[@]}"; do
logg "FETCHING ${version} CLIENT LIBRARY"
destination_directory="${website_directory}/downloads/${version}/linux"
destination_filename="libfdb_c_${version}.so"
mkdir -p "${destination_directory}"
pushd "${destination_directory}" || exit 127
curl -Ls "${artifactory_base_url}/${version}/release/files/linux/lib/libfdb_c.so" -o "${destination_filename}"
chmod 755 "${destination_filename}"
popd || exit 128
done
;;
"unstripped_local")
logg "COPYING UNSTRIPPED CLIENT LIBRARY"
cp -pr "${build_output_directory}/lib/libfdb_c.so" "${website_directory}/downloads/${fdb_version}/linux/libfdb_c_${fdb_version}.so"
;;
"stripped_local")
logg "COPYING STRIPPED CLIENT LIBRARY"
cp -pr "${build_output_directory}/packaging/lib/libfdb_c.so" "${website_directory}/downloads/${fdb_version}/linux/libfdb_c_${fdb_version}.so"
;;
esac
# override fdb_website variable that is passed to Docker build
fdb_website="file:///tmp/website"
}
function compile_ycsb() {
logg "COMPILING YCSB"
if [ "${use_development_java_bindings}" == "true" ]; then
logg "INSTALL JAVA BINDINGS"
foundationdb_java_version="${fdb_version}-PRERELEASE"
mvn install:install-file \
--batch-mode \
-Dorg.slf4j.simpleLogger.log.org.apache.maven.cli.transfer.Slf4jMavenTransferListener=warn \
-Dfile="${build_output_directory}/packages/fdb-java-${foundationdb_java_version}.jar" \
-DgroupId=org.foundationdb \
-DartifactId=fdb-java \
-Dversion="${foundationdb_java_version}" \
-Dpackaging=jar \
-DgeneratePom=true
else
foundationdb_java_version="${fdb_version}"
fi
rm -rf "${script_dir}/YCSB"
mkdir -p "${script_dir}/YCSB"
pushd "${script_dir}/YCSB" || exit 127
if [ -d "${HOME}/src/YCSB" ]; then
rsync -av "${HOME}"/src/YCSB/. .
else
git clone https://github.com/FoundationDB/YCSB.git .
fi
sed -i "s/<foundationdb.version>[0-9]\+.[0-9]\+.[0-9]\+<\/foundationdb.version>/<foundationdb.version>${foundationdb_java_version}<\/foundationdb.version>/g" pom.xml
mvn --batch-mode -Dorg.slf4j.simpleLogger.log.org.apache.maven.cli.transfer.Slf4jMavenTransferListener=warn -pl site.ycsb:foundationdb-binding -am clean package
mkdir -p core/target/dependency
# shellcheck disable=SC2046
cp $(find "${HOME}/.m2" -name jax\*.jar) core/target/dependency/
# shellcheck disable=SC2046
cp $(find "${HOME}/.m2" -name htrace\*.jar) core/target/dependency/
# shellcheck disable=SC2046
cp $(find "${HOME}/.m2" -name HdrHistogram\*.jar) core/target/dependency/
rm -rf .git
popd || exit 128
}
function build_and_push_images(){
declare -a tags_to_push=()
for image in "${image_list[@]}"; do
logg "BUILDING ${image}"
image_tag="${tag_base}${image}:${fdb_version}"
if [ "${image}" == "foundationdb-kubernetes-sidecar" ]; then
image_tag="${image_tag}-1"
fi
if [ "${dockerfile_name}" == "Dockerfile.eks" ]; then
image_tag="${image_tag}-debug"
fi
if [ "${image}" == "ycsb" ]; then
compile_ycsb
fi
logg "TAG ${image_tag#${registry}/foundationdb/}"
docker build \
--label "org.foundationdb.version=${fdb_version}" \
--label "org.foundationdb.build_date=${build_date}" \
--label "org.foundationdb.commit=${commit_sha}" \
--progress plain \
--build-arg FDB_VERSION="${fdb_version}" \
--build-arg FDB_LIBRARY_VERSIONS="${fdb_library_versions[*]}" \
--build-arg FDB_WEBSITE="${fdb_website}" \
--tag "${image_tag}" \
--file "${dockerfile_name}" \
--target "${image}" .
if [ "${image}" == 'foundationdb' ] || [ "${image}" == 'foundationdb-kubernetes-sidecar' ] || [ "${image}" == 'ycsb' ] ; then
tags_to_push+=("${image_tag}")
fi
done
if [ "${push_docker_images}" == "true" ]; then
for tag in "${tags_to_push[@]}"; do
logg "PUSH ${tag}"
docker push "${tag}"
done
fi
}
echo "${blue}################################################################################${reset}"
logg "STARTING ${0}"
echo "${blue}################################################################################${reset}"
################################################################################
# The intent of this script is to build the set of docker images needed to run
# FoundationDB in kubernetes from binaries that are not available the website:
# https://foundationdb.org/downloads
#
# The docker file itself will pull released binaries from the foundationdb
# website. If the intent is to build images for an already released version of
# FoundationDB, a simple docker build command will work.
#
# This script has enough stupid built into it that trying to come up with a set
# of sensible default options that will work everywhere has gotten silly. Below
# are a set of variable definitions that need to be set for this script to
# execute to completion the defaults are based on the FoundationDB development
# environment used by the team at Apple, they will not work for everyone cloning
# this project.
#
# Use this script with care.
################################################################################
artifactory_base_url="${ARTIFACTORY_URL:-https://artifactory.foundationdb.org}"
aws_region="us-west-2"
aws_account_id=$(aws --output text sts get-caller-identity --query 'Account')
build_date=$(date +"%Y-%m-%dT%H:%M:%S%z")
build_output_directory="${script_dir}/../../build_output"
commit_sha=$(git rev-parse --verify HEAD --short=10)
fdb_version=$(awk '/^[[:space:]]+VERSION[[:space:]]+[[:digit:]]+\.[[:digit:]]+\.[[:digit:]]+(-rc[[:digit:]])?/{print $2}' "${script_dir}/../../CMakeLists.txt")
fdb_library_versions=( '5.1.7' '6.1.13' '6.2.30' "${fdb_version}" )
fdb_website="https://www.foundationdb.org"
image_list=(
'base'
# 'go-build'
'foundationdb-base'
'foundationdb'
# 'foundationdb-kubernetes-monitor'
'foundationdb-kubernetes-sidecar'
'ycsb'
)
registry=""
tag_base="foundationdb/"
# THESE CONTROL THE PATH OF FUNCTIONS THAT ARE CALLED BELOW
stripped_binaries_and_from_where="stripped_local" # MUST BE ONE OF ( "unstripped_artifactory" "stripped_artifactory" "unstripped_local" "stripped_local" )
dockerfile_name="Dockerfile"
use_development_java_bindings="false"
push_docker_images="false"
if [ -n "${OKTETO_NAMESPACE+x}" ]; then
logg "RUNNING IN OKTETO/AWS"
# these are defaults for the Apple development environment
aws_region=$(curl -s "http://169.254.169.254/latest/meta-data/placement/region")
aws_account_id=$(aws --output text sts get-caller-identity --query 'Account')
build_output_directory="${HOME}/build_output"
fdb_library_versions=( "${fdb_version}" )
registry="${aws_account_id}.dkr.ecr.${aws_region}.amazonaws.com"
tag_base="${registry}/foundationdb/"
stripped_binaries_and_from_where="unstripped_local" # MUST BE ONE OF ( "unstripped_artifactory" "stripped_artifactory" "unstripped_local" "stripped_local" )
dockerfile_name="Dockerfile.eks"
use_development_java_bindings="true"
push_docker_images="true"
else
echo "Dear ${USER}, you probably need to edit this file before running it. "
echo "${0} has a very narrow set of situations where it will be successful,"
echo "or even useful, when executed unedited"
exit 1
fi
create_fake_website_directory
build_and_push_images
echo "${blue}################################################################################${reset}"
logg "COMPLETED ${0}"
echo "${blue}################################################################################${reset}"

62
packaging/docker/build-release-docker.sh
View File

@ -1,62 +0,0 @@
#!/usr/bin/env bash
set -Eeuo pipefail
set -x
DOCKER_ROOT="$(realpath "$(dirname "${BASH_SOURCE[0]}")")"
BUILD_OUTPUT=$(realpath "${DOCKER_ROOT}"/../..)
echo Docker root: "${DOCKER_ROOT}"
echo Build output: "${BUILD_OUTPUT}"
cd "${DOCKER_ROOT}"
## eg: CMAKE_PROJECT_VERSION:STATIC=7.0.0
FDB_VERSION=$(grep CMAKE_PROJECT_VERSION: "${BUILD_OUTPUT}"/CMakeCache.txt | cut -d '=' -f 2)
# Options (passed via environment variables)
# Feel free to customize the image tag.
# TODO: add a mechanism to set TAG=FDB_VERSION when we're building public releases.
TAG=${TAG:-${FDB_VERSION}-${OKTETO_NAME}}
ECR=${ECR:-112664522426.dkr.ecr.us-west-2.amazonaws.com}
echo Building with tag "${TAG}"
# Login to ECR
# TODO: Move this to a common place instead of repeatedly copy-pasting it.
aws ecr get-login-password | docker login --username AWS --password-stdin "${ECR}"
docker pull "${ECR}"/ubuntu:18.04
docker tag "${ECR}"/ubuntu:18.04 ubuntu:18.04
docker pull "${ECR}"/python:3.9-slim
docker tag "${ECR}"/python:3.9-slim python:3.9-slim
# derived variables
IMAGE=foundationdb/foundationdb:"${TAG}"
SIDECAR=foundationdb/foundationdb-kubernetes-sidecar:"${TAG}"-1
STRIPPED=${STRIPPED:-false}
WEBSITE_BIN_DIR=website/downloads/"${FDB_VERSION}"/linux
TARBALL=${WEBSITE_BIN_DIR}/fdb_"${FDB_VERSION}".tar.gz
mkdir -p "${WEBSITE_BIN_DIR}"
if $STRIPPED; then
tar -C ~/build_output/packages/ -zcvf "${TARBALL}" bin lib
cp ~/build_output/packages/lib/libfdb_c.so "${WEBSITE_BIN_DIR}"/libfdb_c_"${FDB_VERSION}".so
else
tar -C ~/build_output/ -zcvf "${TARBALL}" bin lib
cp ~/build_output/lib/libfdb_c.so "${WEBSITE_BIN_DIR}"/libfdb_c_"${FDB_VERSION}".so
fi
BUILD_ARGS="--build-arg FDB_VERSION=${FDB_VERSION}"
BUILD_ARGS+=" --build-arg FDB_WEBSITE=file:///mnt/website"
BUILD_ARGS+=" --build-arg FDB_ADDITIONAL_VERSIONS=${FDB_VERSION}"
docker build ${BUILD_ARGS} -t "${IMAGE}" -f release/Dockerfile .
docker build ${BUILD_ARGS} -t "${SIDECAR}" -f sidecar/Dockerfile .
docker tag "${IMAGE}" "${ECR}"/"${IMAGE}"
docker tag "${SIDECAR}" "${ECR}"/"${SIDECAR}"
docker push "${ECR}"/"${IMAGE}"
docker push "${ECR}"/"${SIDECAR}"

64
packaging/docker/build-ycsb-docker.sh
View File

@ -1,64 +0,0 @@
#!/usr/bin/env bash
set -Eeuo pipefail
set -x
DOCKER_ROOT="$(realpath "$(dirname "${BASH_SOURCE[0]}")")"
BUILD_OUTPUT=$(realpath "${DOCKER_ROOT}"/../..)
echo Docker root: "${DOCKER_ROOT}"
echo Build output: "${BUILD_OUTPUT}"
cd "${DOCKER_ROOT}"
## eg: CMAKE_PROJECT_VERSION:STATIC=7.0.0
FDB_VERSION=$(grep CMAKE_PROJECT_VERSION: "${BUILD_OUTPUT}"/CMakeCache.txt | cut -d '=' -f 2)
# Options (passed via environment variables)
# Feel free to customize the image tag.
# TODO: add a mechanism to set TAG=FDB_VERSION when we're building public releases.
TAG=${TAG:-${FDB_VERSION}-${OKTETO_NAME}}
ECR=${ECR:-112664522426.dkr.ecr.us-west-2.amazonaws.com}
echo Building with tag "${TAG}"
# Login to ECR
# TODO: Move this to a common place instead of repeatedly copy-pasting it.
aws ecr get-login-password | docker login --username AWS --password-stdin "${ECR}"
docker pull "${ECR}"/openjdk:17-slim
docker tag "${ECR}"/openjdk:17-slim openjdk:17-slim
# derived variables
IMAGE=foundationdb/ycsb:"${TAG}"
# mvn install fdb-java, compile YCSB
mvn install:install-file \
-Dfile="${BUILD_OUTPUT}"/packages/fdb-java-"${FDB_VERSION}"-PRERELEASE.jar \
-DgroupId=org.foundationdb \
-DartifactId=fdb-java \
-Dversion="${FDB_VERSION}"-PRERELEASE \
-Dpackaging=jar \
-DgeneratePom=true
mkdir "${DOCKER_ROOT}"/YCSB && cd "${DOCKER_ROOT}"/YCSB
git clone https://github.com/FoundationDB/YCSB.git .
sed -i "s/<foundationdb.version>[0-9]\+.[0-9]\+.[0-9]\+<\/foundationdb.version>/<foundationdb.version>${FDB_VERSION}-PRERELEASE<\/foundationdb.version>/g" pom.xml
mvn -pl site.ycsb:foundationdb-binding -am clean package
mkdir -p core/target/dependency
# shellcheck disable=SC2046
cp $(find ~/.m2/ -name jax\*.jar) core/target/dependency/
# shellcheck disable=SC2046
cp $(find ~/.m2/ -name htrace\*.jar) core/target/dependency/
# shellcheck disable=SC2046
cp $(find ~/.m2/ -name HdrHistogram\*.jar) core/target/dependency/
rm -rf .git && cd ..
docker build -t "${IMAGE}" -f ycsb/Dockerfile .
docker tag "${IMAGE}" "${ECR}"/"${IMAGE}"
docker push "${ECR}"/"${IMAGE}"

4
packaging/docker/sidecar/entrypoint.bash → packaging/docker/entrypoint.bash
View File

@ -24,4 +24,8 @@ if [[ -n "$ADDITIONAL_ENV_FILE" ]]; then
source $ADDITIONAL_ENV_FILE source $ADDITIONAL_ENV_FILE
fi fi
if [[ -f "/opt/rh/rh-python38/enable" ]]; then
source /opt/rh/rh-python38/enable
fi
exec /sidecar.py $* exec /sidecar.py $*

0
packaging/docker/release/fdb.bash → packaging/docker/fdb.bash
View File

92
packaging/docker/release/Dockerfile
View File

@ -1,92 +0,0 @@
# Dockerfile
#
# This source file is part of the FoundationDB open source project
#
# Copyright 2013-2021 Apple Inc. and the FoundationDB project authors
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
FROM ubuntu:18.04
RUN apt-get update && \
apt-get install -y curl>=7.58.0-2ubuntu3.6 \
dnsutils>=1:9.11.3+dfsg-1ubuntu1.7 \
lsof>=4.89+dfsg-0.1 \
tcptraceroute>=1.5beta7+debian-4build1 \
telnet>=0.17-41 \
netcat>=1.10-41.1 \
strace>=4.21-1ubuntu1 \
tcpdump>=4.9.3-0ubuntu0.18.04.1 \
less>=487-0.1 \
vim>=2:8.0.1453-1ubuntu1.4 \
net-tools>=1.60+git20161116.90da8a0-1ubuntu1 \
jq>=1.5+dfsg-2 \
openssl>=1.1.1-1ubuntu2.1~18.04.9 && \
rm -rf /var/lib/apt/lists/*
COPY misc/tini-amd64.sha256sum /tmp/
# Adding tini as PID 1 https://github.com/krallin/tini
ARG TINI_VERSION=v0.19.0
RUN curl -sLO https://github.com/krallin/tini/releases/download/${TINI_VERSION}/tini-amd64 && \
sha256sum -c /tmp/tini-amd64.sha256sum && \
chmod +x tini-amd64 && \
mv tini-amd64 /usr/bin/tini
ARG FDB_VERSION
ARG FDB_ADDITIONAL_VERSIONS="5.1.7"
ARG FDB_WEBSITE=https://www.foundationdb.org
WORKDIR /var/fdb/tmp
COPY website /mnt/website/
# Install FoundationDB Binaries
RUN curl $FDB_WEBSITE/downloads/$FDB_VERSION/linux/fdb_$FDB_VERSION.tar.gz | tar zxf - --strip-components=1 && \
chmod u+x fdbbackup fdbcli fdbdr fdbmonitor fdbrestore fdbserver backup_agent dr_agent && \
mv fdbbackup fdbcli fdbdr fdbmonitor fdbrestore fdbserver backup_agent dr_agent /usr/bin && \
rm -r /var/fdb/tmp
WORKDIR /
## TODO: Can unify everything above this line
## TODO: we can almost unify the additional client library download,
## but sidecar.py expects them in a different location,
## with a different naming convention.
RUN curl $FDB_WEBSITE/downloads/$FDB_VERSION/linux/libfdb_c_$FDB_VERSION.so -o /usr/lib/libfdb_c.so
# Set Up Runtime Scripts and Directories
ADD release/*.bash /var/fdb/scripts/
RUN chmod a+x /var/fdb/scripts/*.bash
# Install additional FoundationDB Client Libraries
RUN /var/fdb/scripts/download_multiversion_libraries.bash $FDB_WEBSITE $FDB_ADDITIONAL_VERSIONS
RUN rm -rf /mnt/website
RUN mkdir -p /var/fdb/logs
VOLUME /var/fdb/data
# Runtime Configuration Options
ENV FDB_PORT 4500
ENV FDB_CLUSTER_FILE /var/fdb/fdb.cluster
ENV FDB_NETWORKING_MODE container
ENV FDB_COORDINATOR ""
ENV FDB_COORDINATOR_PORT 4500
ENV FDB_CLUSTER_FILE_CONTENTS ""
ENV FDB_PROCESS_CLASS unset
ENTRYPOINT ["/usr/bin/tini", "-g", "--"]
CMD /var/fdb/scripts/fdb.bash

52
packaging/docker/release/create_cluster_file.bash
View File

@ -1,52 +0,0 @@
#! /bin/bash
#
# create_cluster_file.bash
#
# This source file is part of the FoundationDB open source project
#
# Copyright 2013-2018 Apple Inc. and the FoundationDB project authors
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
# This script creates a cluster file for a server or client.
# This takes the cluster file path from the FDB_CLUSTER_FILE
# environment variable, with a default of /etc/foundationdb/fdb.cluster
#
# The name of the coordinator must be defined in the FDB_COORDINATOR environment
# variable, and it must be a name that can be resolved through DNS.
function create_cluster_file() {
FDB_CLUSTER_FILE=${FDB_CLUSTER_FILE:-/etc/foundationdb/fdb.cluster}
mkdir -p $(dirname $FDB_CLUSTER_FILE)
if [[ -n "$FDB_CLUSTER_FILE_CONTENTS" ]]; then
echo "$FDB_CLUSTER_FILE_CONTENTS" > $FDB_CLUSTER_FILE
elif [[ -n $FDB_COORDINATOR ]]; then
coordinator_ip=$(dig +short $FDB_COORDINATOR)
if [[ -z "$coordinator_ip" ]]; then
echo "Failed to look up coordinator address for $FDB_COORDINATOR" 1>&2
exit 1
fi
coordinator_port=${FDB_COORDINATOR_PORT:-4500}
echo "docker:docker@$coordinator_ip:$coordinator_port" > $FDB_CLUSTER_FILE
else
echo "FDB_COORDINATOR environment variable not defined" 1>&2
exit 1
fi
}
if [[ "${BASH_SOURCE[0]}" == "${0}" ]]; then
create_cluster_file "$@"
fi

43
packaging/docker/release/create_server_environment.bash
View File

@ -1,43 +0,0 @@
#! /bin/bash
#
# create_server_environment.bash
#
# This source file is part of the FoundationDB open source project
#
# Copyright 2013-2018 Apple Inc. and the FoundationDB project authors
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
source /var/fdb/scripts/create_cluster_file.bash
function create_server_environment() {
env_file=/var/fdb/.fdbenv
if [[ "$FDB_NETWORKING_MODE" == "host" ]]; then
public_ip=127.0.0.1
elif [[ "$FDB_NETWORKING_MODE" == "container" ]]; then
public_ip=$(hostname -i | awk '{print $1}')
else
echo "Unknown FDB Networking mode \"$FDB_NETWORKING_MODE\"" 1>&2
exit 1
fi
echo "export PUBLIC_IP=$public_ip" > $env_file
if [[ -z $FDB_COORDINATOR && -z "$FDB_CLUSTER_FILE_CONTENTS" ]]; then
FDB_CLUSTER_FILE_CONTENTS="docker:docker@$public_ip:$FDB_PORT"
fi
create_cluster_file
}

31
packaging/docker/release/download_multiversion_libraries.bash
View File

@ -1,31 +0,0 @@
#! /bin/bash
#
# download_multiversion_libraries.bash
#
# This source file is part of the FoundationDB open source project
#
# Copyright 2013-2018 Apple Inc. and the FoundationDB project authors
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
mkdir -p /usr/lib/fdb/multiversion
website=$1
shift
for version in $*; do
origin=$website/downloads/$version/linux/libfdb_c_$version.so
destination=/usr/lib/fdb/multiversion/libfdb_c_$version.so
echo "Downloading $origin to $destination"
curl $origin -o $destination
done

0
packaging/docker/ycsb/run_ycsb.sh → packaging/docker/run_ycsb.sh Normal file → Executable file
View File

0
packaging/docker/sidecar/sidecar.py → packaging/docker/sidecar.py
View File

75
packaging/docker/sidecar/Dockerfile
View File

@ -1,75 +0,0 @@
# Dockerfile
#
# This source file is part of the FoundationDB open source project
#
# Copyright 2018-2019 Apple Inc. and the FoundationDB project authors
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
FROM python:3.9-slim
RUN apt-get update && \
apt-get install -y --no-install-recommends curl && \
rm -rf /var/lub/apt/lists/*
COPY misc/tini-amd64.sha256sum /tmp/
# Adding tini as PID 1 https://github.com/krallin/tini
ARG TINI_VERSION=v0.19.0
RUN curl -sLO https://github.com/krallin/tini/releases/download/${TINI_VERSION}/tini-amd64 && \
sha256sum -c /tmp/tini-amd64.sha256sum && \
chmod +x tini-amd64 && \
mv tini-amd64 /usr/bin/tini
COPY sidecar/requirements.txt /tmp
RUN pip install -r tmp/requirements.txt
ARG FDB_VERSION=
ARG FDB_ADDITIONAL_VERSIONS="6.2.30 6.1.13"
ARG FDB_WEBSITE=https://www.foundationdb.org
WORKDIR /var/fdb/tmp
COPY website /mnt/website/
# Install FoundationDB Binaries
RUN curl $FDB_WEBSITE/downloads/$FDB_VERSION/linux/fdb_$FDB_VERSION.tar.gz | tar zxf - --strip-components=1 && \
chmod u+x fdbbackup fdbcli fdbdr fdbmonitor fdbrestore fdbserver backup_agent dr_agent && \
mv fdbbackup fdbcli fdbdr fdbmonitor fdbrestore fdbserver backup_agent dr_agent /usr/bin && \
rm -r /var/fdb/tmp
WORKDIR /
# Set Up Runtime Scripts and Directories
ADD sidecar/entrypoint.bash sidecar/sidecar.py /
RUN chmod a+x /entrypoint.bash /sidecar.py
# Install additional FoundationDB Client Libraries
RUN mkdir -p /var/fdb/lib && \
for version in $FDB_ADDITIONAL_VERSIONS; do curl $FDB_WEBSITE/downloads/$version/linux/libfdb_c_$version.so -o /var/fdb/lib/libfdb_c_${version%.*}.so; done
RUN rm -rf /mnt/website
RUN echo ${FDB_VERSION} > /var/fdb/version && \
mkdir -p /var/fdb/lib && \
groupadd --gid 4059 fdb && \
useradd --gid 4059 --uid 4059 --no-create-home --shell /bin/bash fdb
VOLUME /var/input-files
VOLUME /var/output-files
USER fdb
ENV LISTEN_PORT 8080
ENTRYPOINT ["/usr/bin/tini", "-g", "--", "/entrypoint.bash"]

1
packaging/docker/sidecar/requirements.txt
View File

@ -1 +0,0 @@
watchdog==0.9.0

32
packaging/docker/ycsb/Dockerfile
View File

@ -1,32 +0,0 @@
FROM openjdk:17-slim AS RUN
WORKDIR /tmp
RUN apt-get update && \
apt-get install -y curl zip && \
curl -Ls https://amazon-eks.s3.amazonaws.com/1.19.6/2021-01-05/bin/linux/amd64/kubectl -o kubectl && \
echo "08ff68159bbcb844455167abb1d0de75bbfe5ae1b051f81ab060a1988027868a kubectl" > kubectl.txt && \
sha256sum -c kubectl.txt && \
mv kubectl /usr/local/bin/kubectl && \
chmod 755 /usr/local/bin/kubectl && \
curl https://awscli.amazonaws.com/awscli-exe-linux-x86_64-2.2.43.zip -o "awscliv2.zip" && \
echo "9a8b3c4e7f72bbcc55e341dce3af42479f2730c225d6d265ee6f9162cfdebdfd awscliv2.zip" > awscliv2.txt && \
sha256sum -c awscliv2.txt && \
unzip -qq awscliv2.zip && \
./aws/install && \
rm -rf /tmp/*
ADD YCSB /YCSB
WORKDIR /YCSB
ENV FDB_NETWORK_OPTION_EXTERNAL_CLIENT_DIRECTORY=/var/dynamic-conf/lib/multiversion/
ENV FDB_NETWORK_OPTION_TRACE_ENABLE=/var/log/fdb-trace-logs
ENV LD_LIBRARY_PATH=/var/dynamic-conf/lib/
ENV BUCKET=backup-112664522426-us-west-2
# TODO: Log4J complains that it's eating the HTracer logs. Even without it, we get per-operation
# time series graphs of throughput, median, 90, 99, 99.9 and 99.99 (in usec).
COPY ycsb/run_ycsb.sh /usr/local/bin/run_ycsb.sh
RUN mkdir -p /var/log/fdb-trace-logs && \
chmod +x /usr/local/bin/run_ycsb.sh
CMD ["run_ycsb.sh"]