mirror of
https://github.com/timescale/timescaledb.git
synced 2025-05-16 10:33:27 +08:00
Run update tests in parallel
This change makes the update tests from each prior version of TimescaleDB to run in parallel instead of in serial order. This speeds up the testing significantly.
This commit is contained in:
Erik Nordström
Erik Nordström
parent
7fb6fbacdd
commit
b7a4851cd0
12
.travis.yml
12
.travis.yml
@ -76,16 +76,24 @@ jobs:
|
||||
stage: test
|
||||
env: PG_VERSION=9.6.6 PG_GIT_TAG=REL9_6_6
|
||||
name: "Update tests (versions w/o constraints support) 9.6"
|
||||
before_install:
|
||||
install:
|
||||
after_failure:
|
||||
after_script:
|
||||
script:
|
||||
- PGTEST_TMPDIR=/tmp/ bash -x ./scripts/test_updates_no_constraints.sh
|
||||
- bash -x ./scripts/test_updates_no_constraints.sh
|
||||
|
||||
# This tests the ability to upgrade to the latest version from versions with constraint support
|
||||
- if: (type = pull_request) OR (type = cron)
|
||||
stage: test
|
||||
env: PG_VERSION=9.6.6 PG_GIT_TAG=REL9_6_6
|
||||
name: "Update tests (versions w/ constraints support) 9.6"
|
||||
before_install:
|
||||
install:
|
||||
after_failure:
|
||||
after_script:
|
||||
script:
|
||||
- PGTEST_TMPDIR=/tmp/ bash -x ./scripts/test_updates_with_constraints.sh
|
||||
- bash -x ./scripts/test_updates_with_constraints.sh
|
||||
|
||||
# This tests the formatting of a PR.
|
||||
- if: (type = pull_request) OR (type = cron) OR NOT (branch = master)
|
||||
|
||||
@ -5,7 +5,7 @@
|
||||
#
|
||||
SCRIPT_DIR=$(dirname $0)
|
||||
BASE_DIR=${PWD}/${SCRIPT_DIR}/..
|
||||
PG_VERSION=${PG_VERSION:-10.4}
|
||||
PG_VERSION=${PG_VERSION:-9.6.5}
|
||||
PG_IMAGE_TAG=${PG_IMAGE_TAG:-${PG_VERSION}-alpine}
|
||||
BUILD_CONTAINER_NAME=${BUILD_CONTAINER_NAME:-pgbuild}
|
||||
BUILD_IMAGE_NAME=${BUILD_IMAGE_NAME:-$USER/pgbuild}
|
||||
@ -15,7 +15,7 @@ GIT_TAG=$(git -C ${BASE_DIR} rev-parse --short --verify HEAD)
|
||||
GIT_ID=$(git -C ${BASE_DIR} describe --dirty | sed -e "s|/|_|g")
|
||||
TAG_NAME=${TAG_NAME:-$GIT_ID}
|
||||
BUILD_TYPE=${BUILD_TYPE:-Debug}
|
||||
USE_OPENSSL=${USE_OPENSSL:-True}
|
||||
USE_OPENSSL=${USE_OPENSSL:-true}
|
||||
PUSH_PG_IMAGE=${PUSH_PG_IMAGE:-false}
|
||||
|
||||
# Full image identifiers
|
||||
@ -96,10 +96,10 @@ fi
|
||||
|
||||
if ! postgres_build_image_exists; then
|
||||
if ! fetch_postgres_build_image; then
|
||||
create_postgres_build_image || exit -1
|
||||
create_postgres_build_image || exit 1
|
||||
fi
|
||||
fi
|
||||
|
||||
build_timescaledb || exit -1
|
||||
build_timescaledb || exit 1
|
||||
|
||||
message_and_exit
|
||||
|
||||
@ -6,18 +6,28 @@ set -o pipefail
|
||||
SCRIPT_DIR=$(dirname $0)
|
||||
BASE_DIR=${PWD}/${SCRIPT_DIR}/..
|
||||
TEST_VERSION=${TEST_VERSION:-v2}
|
||||
PGTEST_TMPDIR=${PGTEST_TMPDIR:-$(mktemp -d 2>/dev/null || mktemp -d -t 'timescaledb_update_test')}
|
||||
TEST_TMPDIR=${TEST_TMPDIR:-$(mktemp -d 2>/dev/null || mktemp -d -t 'timescaledb_update_test' || mkdir -p /tmp/${RANDOM})}
|
||||
UPDATE_PG_PORT=${UPDATE_PG_PORT:-6432}
|
||||
CLEAN_PG_PORT=${CLEAN_PG_PORT:-6433}
|
||||
PG_VERSION=${PG_VERSION:-9.6.5} # Need 9.6.x version since we are
|
||||
# upgrading the extension from
|
||||
# versions that didn't support PG10.
|
||||
GIT_ID=$(git -C ${BASE_DIR} describe --dirty | sed -e "s|/|_|g")
|
||||
UPDATE_FROM_IMAGE=${UPDATE_FROM_IMAGE:-timescale/timescaledb}
|
||||
UPDATE_FROM_TAG=${UPDATE_FROM_TAG:-0.1.0}
|
||||
UPDATE_TO_IMAGE=${UPDATE_TO_IMAGE:-update_test}
|
||||
UPDATE_TO_TAG=${UPDATE_TO_TAG:-latest}
|
||||
UPDATE_TO_TAG=${UPDATE_TO_TAG:-${GIT_ID}}
|
||||
DO_CLEANUP=true
|
||||
|
||||
# PID of the current shell
|
||||
PID=$$
|
||||
|
||||
# Container names. Append shell PID so that we can run this script in parallel
|
||||
CONTAINER_ORIG=timescaledb-orig-${PID}
|
||||
CONTAINER_CLEAN_RESTORE=timescaledb-clean-restore-${PID}
|
||||
CONTAINER_UPDATED=timescaledb-updated-${PID}
|
||||
CONTAINER_CLEAN_RERUN=timescaledb-clean-rerun-${PID}
|
||||
|
||||
export PG_VERSION
|
||||
|
||||
while getopts "d" opt;
|
||||
@ -33,28 +43,36 @@ done
|
||||
|
||||
shift $((OPTIND-1))
|
||||
|
||||
if "$DO_CLEANUP" = "true"; then
|
||||
trap cleanup EXIT
|
||||
fi
|
||||
trap cleanup EXIT
|
||||
|
||||
remove_containers() {
|
||||
docker rm -vf ${CONTAINER_ORIG} 2>/dev/null
|
||||
docker rm -vf ${CONTAINER_CLEAN_RESTORE} 2>/dev/null
|
||||
docker rm -vf ${CONTAINER_UPDATED} 2>/dev/null
|
||||
docker rm -vf ${CONTAINER_CLEAN_RERUN} 2>/dev/null
|
||||
docker volume rm -f ${CLEAN_VOLUME} 2>/dev/null
|
||||
docker volume rm -f ${UPDATE_VOLUME} 2>/dev/null
|
||||
}
|
||||
|
||||
cleanup() {
|
||||
# Save status here so that we can return the status of the last
|
||||
# command in the script and not the last command of the cleanup
|
||||
# function
|
||||
status="$?"
|
||||
local status="$?"
|
||||
set +e # do not exit immediately on failure in cleanup handler
|
||||
if [ $status -eq 0 ]; then
|
||||
rm -rf ${PGTEST_TMPDIR}
|
||||
docker rm -vf timescaledb-orig timescaledb-clean-restore timescaledb-updated 2>/dev/null
|
||||
if [ "$DO_CLEANUP" = "true" ]; then
|
||||
rm -rf ${TEST_TMPDIR}
|
||||
sleep 1
|
||||
remove_containers
|
||||
fi
|
||||
echo "Exit status is $status"
|
||||
exit $status
|
||||
echo "Test with pid ${PID} exited with code ${status}"
|
||||
exit ${status}
|
||||
}
|
||||
|
||||
docker_exec() {
|
||||
# Echo to stderr
|
||||
>&2 echo -e "\033[1m$1\033[0m: $2"
|
||||
docker exec -it $1 /bin/bash -c "$2"
|
||||
docker exec $1 /bin/bash -c "$2"
|
||||
}
|
||||
|
||||
docker_pgcmd() {
|
||||
@ -67,15 +85,15 @@ docker_pgscript() {
|
||||
|
||||
docker_pgtest() {
|
||||
>&2 echo -e "\033[1m$1\033[0m: $2"
|
||||
docker exec $1 psql -X -v ECHO=ALL -v ON_ERROR_STOP=1 -h localhost -U postgres -d single -f $2 > ${PGTEST_TMPDIR}/$1.out || exit $?
|
||||
docker exec $1 psql -X -v ECHO=ALL -v ON_ERROR_STOP=1 -h localhost -U postgres -d single -f $2 > ${TEST_TMPDIR}/$1.out || exit $?
|
||||
}
|
||||
|
||||
docker_pgdiff() {
|
||||
>&2 echo -e "\033[1m$1 vs $2\033[0m: $2"
|
||||
docker_pgtest $1 $3
|
||||
docker_pgtest $2 $3
|
||||
echo "RUNNING: diff ${PGTEST_TMPDIR}/$1.out ${PGTEST_TMPDIR}/$2.out "
|
||||
diff ${PGTEST_TMPDIR}/$1.out ${PGTEST_TMPDIR}/$2.out | tee ${PGTEST_TMPDIR}/update_test.output
|
||||
echo "RUNNING: diff ${TEST_TMPDIR}/$1.out ${TEST_TMPDIR}/$2.out "
|
||||
diff ${TEST_TMPDIR}/$1.out ${TEST_TMPDIR}/$2.out | tee ${TEST_TMPDIR}/update_test.output
|
||||
}
|
||||
|
||||
docker_run() {
|
||||
@ -90,8 +108,8 @@ docker_run_vol() {
|
||||
|
||||
wait_for_pg() {
|
||||
set +e
|
||||
for i in {1..10}; do
|
||||
sleep 2
|
||||
for i in {1..20}; do
|
||||
sleep 0.5
|
||||
|
||||
docker_exec $1 "pg_isready -U postgres"
|
||||
|
||||
@ -100,7 +118,7 @@ wait_for_pg() {
|
||||
# ideal. Apperently, pg_isready is not always a good
|
||||
# indication of whether the DB is actually ready to accept
|
||||
# queries
|
||||
sleep 2
|
||||
sleep 0.2
|
||||
set -e
|
||||
return 0
|
||||
fi
|
||||
@ -110,42 +128,46 @@ wait_for_pg() {
|
||||
|
||||
VERSION=`echo ${UPDATE_FROM_TAG} | sed 's/\([0-9]\{0,\}\.[0-9]\{0,\}\.[0-9]\{0,\}\).*/\1/g'`
|
||||
echo "Testing from version ${VERSION} (test version ${TEST_VERSION})"
|
||||
echo "Using temporary directory $PGTEST_TMPDIR"
|
||||
echo "Using temporary directory ${TEST_TMPDIR}"
|
||||
|
||||
docker rm -f timescaledb-orig timescaledb-updated timescaledb-clean-restore timescaledb-clean-rerun 2>/dev/null || true
|
||||
IMAGE_NAME=update_test TAG_NAME=latest bash ${SCRIPT_DIR}/docker-build.sh
|
||||
remove_containers || true
|
||||
|
||||
docker_run timescaledb-orig ${UPDATE_FROM_IMAGE}:${UPDATE_FROM_TAG}
|
||||
docker_run timescaledb-clean-restore ${UPDATE_TO_IMAGE}:${UPDATE_TO_TAG}
|
||||
docker_run timescaledb-clean-rerun ${UPDATE_TO_IMAGE}:${UPDATE_TO_TAG}
|
||||
IMAGE_NAME=${UPDATE_TO_IMAGE} TAG_NAME=${UPDATE_TO_TAG} PG_VERSION=${PG_VERSION} bash ${SCRIPT_DIR}/docker-build.sh
|
||||
|
||||
CLEAN_VOLUME=$(docker inspect timescaledb-clean-restore --format='{{range .Mounts }}{{.Name}}{{end}}')
|
||||
UPDATE_VOLUME=$(docker inspect timescaledb-orig --format='{{range .Mounts }}{{.Name}}{{end}}')
|
||||
docker_run ${CONTAINER_ORIG} ${UPDATE_FROM_IMAGE}:${UPDATE_FROM_TAG}
|
||||
docker_run ${CONTAINER_CLEAN_RESTORE} ${UPDATE_TO_IMAGE}:${UPDATE_TO_TAG}
|
||||
docker_run ${CONTAINER_CLEAN_RERUN} ${UPDATE_TO_IMAGE}:${UPDATE_TO_TAG}
|
||||
|
||||
CLEAN_VOLUME=$(docker inspect ${CONTAINER_CLEAN_RESTORE} --format='{{range .Mounts }}{{.Name}}{{end}}')
|
||||
UPDATE_VOLUME=$(docker inspect ${CONTAINER_ORIG} --format='{{range .Mounts }}{{.Name}}{{end}}')
|
||||
|
||||
echo "Executing setup script on ${VERSION}"
|
||||
docker_pgscript timescaledb-orig /src/test/sql/updates/setup.${TEST_VERSION}.sql
|
||||
docker rm -f timescaledb-orig
|
||||
docker_pgscript ${CONTAINER_ORIG} /src/test/sql/updates/setup.${TEST_VERSION}.sql
|
||||
|
||||
docker_run_vol timescaledb-updated ${UPDATE_VOLUME}:/var/lib/postgresql/data ${UPDATE_TO_IMAGE}:${UPDATE_TO_TAG}
|
||||
# Remove container but keep volume
|
||||
docker rm -f ${CONTAINER_ORIG}
|
||||
|
||||
echo "Running update container"
|
||||
docker_run_vol ${CONTAINER_UPDATED} ${UPDATE_VOLUME}:/var/lib/postgresql/data ${UPDATE_TO_IMAGE}:${UPDATE_TO_TAG}
|
||||
|
||||
echo "Executing ALTER EXTENSION timescaledb UPDATE"
|
||||
docker_pgcmd timescaledb-updated "ALTER EXTENSION timescaledb UPDATE"
|
||||
docker_pgcmd ${CONTAINER_UPDATED} "ALTER EXTENSION timescaledb UPDATE"
|
||||
|
||||
docker_exec timescaledb-updated "pg_dump -h localhost -U postgres -Fc single > /tmp/single.sql"
|
||||
docker cp timescaledb-updated:/tmp/single.sql ${PGTEST_TMPDIR}/single.sql
|
||||
docker_exec ${CONTAINER_UPDATED} "pg_dump -h localhost -U postgres -Fc single > /tmp/single.sql"
|
||||
docker cp ${CONTAINER_UPDATED}:/tmp/single.sql ${TEST_TMPDIR}/single.sql
|
||||
|
||||
echo "Executing setup script on clean"
|
||||
docker_pgscript timescaledb-clean-rerun /src/test/sql/updates/setup.${TEST_VERSION}.sql
|
||||
docker_pgscript ${CONTAINER_CLEAN_RERUN} /src/test/sql/updates/setup.${TEST_VERSION}.sql
|
||||
|
||||
echo "Testing updated vs clean"
|
||||
docker_pgdiff timescaledb-updated timescaledb-clean-rerun /src/test/sql/updates/test-rerun.sql
|
||||
docker_pgdiff ${CONTAINER_UPDATED} ${CONTAINER_CLEAN_RERUN} /src/test/sql/updates/test-rerun.sql
|
||||
|
||||
echo "Restoring database on clean version"
|
||||
docker cp ${PGTEST_TMPDIR}/single.sql timescaledb-clean-restore:/tmp/single.sql
|
||||
docker_exec timescaledb-clean-restore "createdb -h localhost -U postgres single"
|
||||
docker_pgcmd timescaledb-clean-restore "ALTER DATABASE single SET timescaledb.restoring='on'"
|
||||
docker_exec timescaledb-clean-restore "pg_restore -h localhost -U postgres -d single /tmp/single.sql"
|
||||
docker_pgcmd timescaledb-clean-restore "ALTER DATABASE single SET timescaledb.restoring='off'"
|
||||
docker cp ${TEST_TMPDIR}/single.sql ${CONTAINER_CLEAN_RESTORE}:/tmp/single.sql
|
||||
docker_exec ${CONTAINER_CLEAN_RESTORE} "createdb -h localhost -U postgres single"
|
||||
docker_pgcmd ${CONTAINER_CLEAN_RESTORE} "ALTER DATABASE single SET timescaledb.restoring='on'"
|
||||
docker_exec ${CONTAINER_CLEAN_RESTORE} "pg_restore -h localhost -U postgres -d single /tmp/single.sql"
|
||||
docker_pgcmd ${CONTAINER_CLEAN_RESTORE} "ALTER DATABASE single SET timescaledb.restoring='off'"
|
||||
|
||||
echo "Testing restored"
|
||||
docker_pgdiff timescaledb-updated timescaledb-clean-restore /src/test/sql/updates/post.${TEST_VERSION}.sql
|
||||
docker_pgdiff ${CONTAINER_UPDATED} ${CONTAINER_CLEAN_RESTORE} /src/test/sql/updates/post.${TEST_VERSION}.sql
|
||||
|
||||
97
scripts/test_updates.sh
Normal file
97
scripts/test_updates.sh
Normal file
@ -0,0 +1,97 @@
|
||||
#!/bin/bash
|
||||
|
||||
set -o pipefail
|
||||
|
||||
SCRIPT_DIR=$(dirname $0)
|
||||
TEST_TMPDIR=${TEST_TMPDIR:-$(mktemp -d 2>/dev/null || mktemp -d -t 'timescaledb_update_test' || mkdir -p /tmp/$RANDOM )}
|
||||
BASE_DIR=${PWD}/${SCRIPT_DIR}/..
|
||||
TAGS=${TAGS:-}
|
||||
TEST_VERSION=${TEST_VERSION:-}
|
||||
GIT_ID=$(git -C ${BASE_DIR} describe --dirty | sed -e "s|/|_|g")
|
||||
UPDATE_TO_IMAGE=${UPDATE_TO_IMAGE:-update_test}
|
||||
UPDATE_TO_TAG=${UPDATE_TO_TAG:-${GIT_ID}}
|
||||
PG_VERSION=${PG_VERSION:-9.6.5} # Need 9.6.x version since we are
|
||||
# upgrading the extension from
|
||||
# versions that didn't support PG10.
|
||||
|
||||
FAILED_TEST=
|
||||
|
||||
# Declare a hash table to keep test names keyed by pid
|
||||
declare -A tests
|
||||
|
||||
while getopts "c" opt;
|
||||
do
|
||||
case $opt in
|
||||
c)
|
||||
echo "Forcing cleanup of build image"
|
||||
docker rmi -f ${UPDATE_TO_IMAGE}:${UPDATE_TO_TAG}
|
||||
;;
|
||||
esac
|
||||
done
|
||||
|
||||
cleanup() {
|
||||
local exit_code="$?"
|
||||
set +e # do not exit immediately on failure
|
||||
echo "Waiting for remaining tests to finish..."
|
||||
wait
|
||||
if [ -f ${TEST_TMPDIR}/${FAILED_TEST}.log ]; then
|
||||
echo "###### Failed test log below #####"
|
||||
cat ${TEST_TMPDIR}/${FAILED_TEST}.log
|
||||
fi
|
||||
rm -rf ${TEST_TMPDIR}
|
||||
echo "exit code is $exit_code"
|
||||
return $exit_code
|
||||
}
|
||||
|
||||
kill_all_tests() {
|
||||
local exit_code="$?"
|
||||
set +e # do not exit immediately on failure
|
||||
echo "Killing all tests"
|
||||
kill ${!tests[@]} 2>/dev/null
|
||||
return $exit_code
|
||||
}
|
||||
|
||||
trap kill_all_tests INT HUP
|
||||
trap cleanup EXIT
|
||||
|
||||
if [ -z "${TEST_VERSION}" ]; then
|
||||
echo "No TEST_VERSION specified"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
if [ -z "${TAGS}" ]; then
|
||||
echo "No TAGS specified"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# Build the docker image with current source here so that the parallel
|
||||
# tests don't all compete in trying to build it first
|
||||
IMAGE_NAME=${UPDATE_TO_IMAGE} TAG_NAME=${UPDATE_TO_TAG} PG_VERSION=${PG_VERSION} bash ${SCRIPT_DIR}/docker-build.sh
|
||||
|
||||
# Run update tests in parallel
|
||||
for tag in ${TAGS};
|
||||
do
|
||||
UPDATE_FROM_TAG=${tag} TEST_VERSION=${TEST_VERSION} $(dirname $0)/test_update_from_tag.sh > ${TEST_TMPDIR}/${tag}.log 2>&1 &
|
||||
|
||||
tests[$!]=${tag}
|
||||
echo "Launched test ${tag} with pid $!"
|
||||
done
|
||||
|
||||
# Need to wait on each pid in a loop to return the exit status of each
|
||||
echo "Waiting for tests to finish..."
|
||||
|
||||
# Since we are iterating a hash table, the tests are not going to be
|
||||
# in order started. But it doesn't matter.
|
||||
for pid in ${!tests[@]};
|
||||
do
|
||||
wait $pid;
|
||||
exit_code=$?
|
||||
echo "Test ${tests[$pid]} (pid $pid) exited with code $exit_code"
|
||||
|
||||
if [ $exit_code -ne 0 ]; then
|
||||
FAILED_TEST=${tests[$pid]}
|
||||
kill_all_tests
|
||||
exit $exit_code
|
||||
fi
|
||||
$(exit $exit_code)
|
||||
done
|
||||
@ -7,9 +7,8 @@
|
||||
set -e
|
||||
set -o pipefail
|
||||
|
||||
SCRIPT_DIR=$(dirname $0)
|
||||
TAGS="0.1.0 0.2.0 0.3.0 0.4.0 0.4.1 0.4.2"
|
||||
TEST_VERSION="v1"
|
||||
|
||||
for tag in ${TAGS};
|
||||
do
|
||||
UPDATE_FROM_TAG=${tag} TEST_VERSION="v1" $(dirname $0)/test_update_from_tag.sh
|
||||
done
|
||||
. ${SCRIPT_DIR}/test_updates.sh
|
||||
|
||||
@ -10,9 +10,9 @@
|
||||
set -e
|
||||
set -o pipefail
|
||||
|
||||
TAGS="0.5.0 0.6.0 0.6.1 0.7.0-pg9.6 0.7.1-pg9.6 0.8.0-pg9.6 0.9.0-pg9.6 0.9.1-pg9.6 0.9.2-pg9.6 0.10.0-pg9.6 0.10.1-pg9.6 0.11.0-pg9.6 0.12.0-pg9.6 1.0.0-rc1-pg9.6 1.0.0-rc2-pg9.6 1.0.0-rc3-pg9.6"
|
||||
SCRIPT_DIR=$(dirname $0)
|
||||
|
||||
for tag in ${TAGS};
|
||||
do
|
||||
UPDATE_FROM_TAG=${tag} TEST_VERSION="v2" $(dirname $0)/test_update_from_tag.sh
|
||||
done
|
||||
TAGS="0.5.0 0.6.0 0.6.1 0.7.0-pg9.6 0.7.1-pg9.6 0.8.0-pg9.6 0.9.0-pg9.6 0.9.1-pg9.6 0.9.2-pg9.6 0.10.0-pg9.6 0.10.1-pg9.6 0.11.0-pg9.6 0.12.0-pg9.6 1.0.0-rc1-pg9.6 1.0.0-rc2-pg9.6 1.0.0-rc3-pg9.6"
|
||||
TEST_VERSION="v2"
|
||||
|
||||
. ${SCRIPT_DIR}/test_updates.sh
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user