apple/foundationdb
1
0
Fork 0
mirror of https://github.com/apple/foundationdb.git synced 2025-05-24 16:20:15 +08:00
Code Issues Packages Projects Releases Wiki Activity

Move the new Kubernetes image to centos 7.

Browse Source
This commit is contained in:
John Brownlee 2021-09-21 12:12:43 -07:00
parent 1a5069a047
commit a6b903e7f8
4 changed files with 61 additions and 42 deletions
fdbkubernetesmonitor
README.mdmain.gomonitor.go
packaging/docker/kubernetes
Dockerfile

38
fdbkubernetesmonitor/README.md

@ -4,28 +4,34 @@ This package provides a launcher program for running FoundationDB in Kubernetes.
To test this, run the following commands from the root of the FoundationDB To test this, run the following commands from the root of the FoundationDB
repository: repository:
docker build -t foundationdb/foundationdb-kubernetes:6.3.13-local --build-arg FDB_VERSION=6.3.13 --build-arg FDB_LIBRARY_VERSIONS="6.3.13 6.2.30 6.1.13" -f packaging/docker/kubernetes/Dockerfile . ```bash
docker build -t foundationdb/foundationdb-kubernetes:6.3.15-local --build-arg FDB_VERSION=6.3.15 --build-arg FDB_LIBRARY_VERSIONS="6.3.15 6.2.30 6.1.13" -f packaging/docker/kubernetes/Dockerfile . docker build -t foundationdb/foundationdb-kubernetes:6.3.13-local --build-arg FDB_VERSION=6.3.13 --build-arg FDB_LIBRARY_VERSIONS="6.3.13 6.2.30 6.1.13" -f packaging/docker/kubernetes/Dockerfile .
kubectl apply -f packaging/docker/kubernetes/test_config.yaml docker build -t foundationdb/foundationdb-kubernetes:6.3.15-local --build-arg FDB_VERSION=6.3.15 --build-arg FDB_LIBRARY_VERSIONS="6.3.15 6.2.30 6.1.13" -f packaging/docker/kubernetes/Dockerfile .
# Wait for the pods to become ready kubectl apply -f packaging/docker/kubernetes/test_config.yaml
ips=$(kubectl get pod -l app=fdb-kubernetes-example -o json | jq -j '[[.items|.[]|select(.status.podIP!="")]|limit(3;.[])|.status.podIP+":4501"]|join(",")') # Wait for the pods to become ready
cat packaging/docker/kubernetes/test_config.yaml | sed -e "s/fdb.cluster: \"\"/fdb.cluster: \"test:test@$ips\"/" -e "s/\"serverCount\": 0/\"serverCount\": 1/" | kubectl apply -f - ips=$(kubectl get pod -l app=fdb-kubernetes-example -o json | jq -j '[[.items|.[]|select(.status.podIP!="")]|limit(3;.[])|.status.podIP+":4501"]|join(",")')
kubectl get pod -l app=fdb-kubernetes-example -o name | xargs -I {} kubectl annotate {} foundationdb.org/outdated-config-map-seen=$(date +%s) --overwrite sed -e "s/fdb.cluster: \"\"/fdb.cluster: \"test:test@$ips\"/" -e "s/\"serverCount\": 0/\"serverCount\": 1/" packaging/docker/kubernetes/test_config.yaml | kubectl apply -f -
# Watch the logs for the fdb-kubernetes-example pods to confirm that they have launched the fdbserver processes. kubectl get pod -l app=fdb-kubernetes-example -o name | xargs -I {} kubectl annotate {} foundationdb.org/outdated-config-map-seen=$(date +%s) --overwrite
kubectl exec -it sts/fdb-kubernetes-example -- fdbcli --exec "configure new double ssd" # Watch the logs for the fdb-kubernetes-example pods to confirm that they have launched the fdbserver processes.
kubectl exec -it sts/fdb-kubernetes-example -- fdbcli --exec "configure new double ssd"
```
This will set up a cluster in your Kubernetes environment using a statefulset, to provide a simple subset of what the Kubernetes operator does to set up the cluster. This will set up a cluster in your Kubernetes environment using a statefulset, to provide a simple subset of what the Kubernetes operator does to set up the cluster. Note: This assumes that you are running Docker Desktop on your local machine, with Kubernetes configured through Docker Desktop.
You can then make changes to the data in the config map and update the fdbserver processes: You can then make changes to the data in the config map and update the fdbserver processes:
cat packaging/docker/kubernetes/test_config.yaml | sed -e "s/fdb.cluster: \"\"/fdb.cluster: \"test:test@$ips\"/" -e "s/\"serverCount\": 0/\"serverCount\": 1/" | kubectl apply -f - ```bash
sed -e "s/fdb.cluster: \"\"/fdb.cluster: \"test:test@$ips\"/" -e "s/\"serverCount\": 0/\"serverCount\": 1/" packaging/docker/kubernetes/test_config.yaml | kubectl apply -f -
# You can apply an annotation to speed up the propagation of config # You can apply an annotation to speed up the propagation of config
kubectl get pod -l app=fdb-kubernetes-example -o name | xargs -I {} kubectl annotate {} foundationdb.org/outdated-config-map-seen=$(date +%s) --overwrite kubectl get pod -l app=fdb-kubernetes-example -o name | xargs -I {} kubectl annotate {} foundationdb.org/outdated-config-map-seen=$(date +%s) --overwrite
# Watch the logs for the fdb-kubernetes-example pods to confirm that they have reloaded their configuration, and then do a bounce. # Watch the logs for the fdb-kubernetes-example pods to confirm that they have reloaded their configuration, and then do a bounce.
kubectl exec -it sts/fdb-kubernetes-example -- fdbcli --exec "kill; kill all; status" kubectl exec -it sts/fdb-kubernetes-example -- fdbcli --exec "kill; kill all; status"
```
Once you are done, you can tear down the example with the following command: Once you are done, you can tear down the example with the following command:
kubectl delete -f packaging/docker/kubernetes/test_config.yaml; kubectl delete pvc -l app=fdb-kubernetes-example ```bash
kubectl delete -f packaging/docker/kubernetes/test_config.yaml; kubectl delete pvc -l app=fdb-kubernetes-example
```

9
fdbkubernetesmonitor/main.go

@ -103,20 +103,21 @@ func main() {
} }
mode := executionMode(executionModeString) mode := executionMode(executionModeString)
if mode == executionModeLauncher { switch mode {
case executionModeLauncher:
customEnvironment, err := loadAdditionalEnvironment(logger) customEnvironment, err := loadAdditionalEnvironment(logger)
if err != nil { if err != nil {
logger.Error(err, "Error loading additional environment") logger.Error(err, "Error loading additional environment")
os.Exit(1) os.Exit(1)
} }
StartMonitor(logger, fmt.Sprintf("%s/%s", inputDir, monitorConfFile), customEnvironment) StartMonitor(logger, fmt.Sprintf("%s/%s", inputDir, monitorConfFile), customEnvironment)
} else if mode == executionModeInit { case executionModeInit:
err = CopyFiles(logger, outputDir, copyDetails, requiredCopies) err = CopyFiles(logger, outputDir, copyDetails, requiredCopies)
if err != nil { if err != nil {
logger.Error(err, "Error copying files") logger.Error(err, "Error copying files")
os.Exit(1) os.Exit(1)
} }
} else if mode == executionModeSidecar { case executionModeSidecar:
if mainContainerVersion != currentContainerVersion { if mainContainerVersion != currentContainerVersion {
err = CopyFiles(logger, outputDir, copyDetails, requiredCopies) err = CopyFiles(logger, outputDir, copyDetails, requiredCopies)
if err != nil { if err != nil {
@ -126,7 +127,7 @@ func main() {
done := make(chan bool) done := make(chan bool)
<-done <-done
} }
} else { default:
logger.Error(nil, "Unknown execution mode", "mode", mode) logger.Error(nil, "Unknown execution mode", "mode", mode)
os.Exit(1) os.Exit(1)
} }

24
fdbkubernetesmonitor/monitor.go

@ -22,6 +22,7 @@ package main
import ( import (
"bufio" "bufio"
"encoding/json" "encoding/json"
"fmt"
"io" "io"
"os" "os"
"os/exec" "os/exec"
@ -123,13 +124,9 @@ func (monitor *Monitor) LoadConfiguration() {
configuration.BinaryPath = path.Join(sharedBinaryDir, configuration.Version, "fdbserver") configuration.BinaryPath = path.Join(sharedBinaryDir, configuration.Version, "fdbserver")
} }
binaryStat, err := os.Stat(configuration.BinaryPath) err = checkOwnerExecutable(configuration.BinaryPath)
if err != nil { if err != nil {
monitor.Logger.Error(err, "Error checking binary path for latest configuration", "configuration", configuration, "binaryPath", configuration.BinaryPath) monitor.Logger.Error(err, "Error with binary path for latest configuration", "configuration", configuration, "binaryPath", configuration.BinaryPath)
return
}
if binaryStat.Mode()&0o100 == 0 {
monitor.Logger.Error(nil, "New binary path is not executable", "configuration", configuration, "binaryPath", configuration.BinaryPath)
return return
} }
@ -142,6 +139,19 @@ func (monitor *Monitor) LoadConfiguration() {
monitor.acceptConfiguration(configuration, configurationBytes) monitor.acceptConfiguration(configuration, configurationBytes)
} }
// checkOwnerExecutable validates that a path is a file that exists and is
// executable by its owner.
func checkOwnerExecutable(path string) error {
binaryStat, err := os.Stat(path)
if err != nil {
return err
}
if binaryStat.Mode()&0o100 == 0 {
return fmt.Errorf("Binary is not executable")
}
return nil
}
// acceptConfiguration is called when the monitor process parses and accepts // acceptConfiguration is called when the monitor process parses and accepts
// a configuration from the local config file. // a configuration from the local config file.
func (monitor *Monitor) acceptConfiguration(configuration *ProcessConfiguration, configurationBytes []byte) { func (monitor *Monitor) acceptConfiguration(configuration *ProcessConfiguration, configurationBytes []byte) {
@ -217,7 +227,7 @@ func (monitor *Monitor) RunProcess(processNumber int) {
if cmd.Process != nil { if cmd.Process != nil {
pid = cmd.Process.Pid pid = cmd.Process.Pid
} else { } else {
logger.Error(nil, "No Process information availale for subprocess") logger.Error(nil, "No Process information available for subprocess")
} }
startTime := time.Now() startTime := time.Now()

32
packaging/docker/kubernetes/Dockerfile

@ -30,22 +30,24 @@ RUN go build -o /fdb-kubernetes-monitor ./...
# Build the main image # Build the main image
FROM ubuntu:18.04 FROM centos:7.9.2009
RUN apt-get update && \ RUN yum install -y \
apt-get install -y curl>=7.58.0-2ubuntu3.6 \ binutils-2.27-44.base.el7 \
dnsutils>=1:9.11.3+dfsg-1ubuntu1.7 \ bind-utils-9.11.4-26.P2.el7_9.7 \
lsof>=4.89+dfsg-0.1 \ curl-7.29.0-59.el7_9.1 \
tcptraceroute>=1.5beta7+debian-4build1 \ less-458-9.el7 \
telnet>=0.17-41 \ lsof-4.87-6.el7 \
netcat>=1.10-41.1 \ nano-2.3.1-10.el7 \
strace>=4.21-1ubuntu1 \ nmap-ncat-6.40-19.el7 \
tcpdump>=4.9.3-0ubuntu0.18.04.1 \ net-tools-2.0-0.25.20131004git.el7 \
less>=487-0.1 \ strace-4.24-6.el7 \
vim>=2:8.0.1453-1ubuntu1.4 \ tar-1.26-35.el7 \
net-tools>=1.60+git20161116.90da8a0-1ubuntu1 \ telnet-0.17-66.el7 \
jq>=1.5+dfsg-2 && \ traceroute-2.0.22-2.el7 \
rm -rf /var/lib/apt/lists/* tcpdump-4.9.2-4.el7_7.1 \
vim-enhanced-7.4.629-8.el7_9 \
&& yum clean all
ARG FDB_VERSION ARG FDB_VERSION
ARG FDB_LIBRARY_VERSIONS="${FDB_VERSION}" ARG FDB_LIBRARY_VERSIONS="${FDB_VERSION}"