Run update tests in parallel

This change makes the update tests from each prior version of
TimescaleDB to run in parallel instead of in serial order.

This speeds up the testing significantly.

.travis.yml

@@ -76,16 +76,24 @@ jobs:
       stage: test
       env: PG_VERSION=9.6.6 PG_GIT_TAG=REL9_6_6
       name: "Update tests (versions w/o constraints support) 9.6"
+      before_install:
+      install:
+      after_failure:
+      after_script:
       script:
-        - PGTEST_TMPDIR=/tmp/ bash -x ./scripts/test_updates_no_constraints.sh
+        - bash -x ./scripts/test_updates_no_constraints.sh
 
     # This tests the ability to upgrade to the latest version from versions with constraint support
     - if: (type = pull_request) OR (type = cron)
       stage: test
       env: PG_VERSION=9.6.6 PG_GIT_TAG=REL9_6_6
       name: "Update tests (versions w/ constraints support) 9.6"
+      before_install:
+      install:
+      after_failure:
+      after_script:
       script:
-        - PGTEST_TMPDIR=/tmp/ bash -x ./scripts/test_updates_with_constraints.sh
+        - bash -x ./scripts/test_updates_with_constraints.sh
 
     # This tests the formatting of a PR.
     - if: (type = pull_request) OR (type = cron) OR NOT (branch = master)

scripts/docker-build.sh

@@ -5,7 +5,7 @@
 #
 SCRIPT_DIR=$(dirname $0)
 BASE_DIR=${PWD}/${SCRIPT_DIR}/..
-PG_VERSION=${PG_VERSION:-10.4}
+PG_VERSION=${PG_VERSION:-9.6.5}
 PG_IMAGE_TAG=${PG_IMAGE_TAG:-${PG_VERSION}-alpine}
 BUILD_CONTAINER_NAME=${BUILD_CONTAINER_NAME:-pgbuild}
 BUILD_IMAGE_NAME=${BUILD_IMAGE_NAME:-$USER/pgbuild}
@@ -15,7 +15,7 @@ GIT_TAG=$(git -C ${BASE_DIR} rev-parse --short --verify HEAD)
 GIT_ID=$(git -C ${BASE_DIR} describe --dirty | sed -e "s|/|_|g")
 TAG_NAME=${TAG_NAME:-$GIT_ID}
 BUILD_TYPE=${BUILD_TYPE:-Debug}
-USE_OPENSSL=${USE_OPENSSL:-True}
+USE_OPENSSL=${USE_OPENSSL:-true}
 PUSH_PG_IMAGE=${PUSH_PG_IMAGE:-false}
 
 # Full image identifiers
@@ -96,10 +96,10 @@ fi
 
 if ! postgres_build_image_exists; then
     if ! fetch_postgres_build_image; then
-        create_postgres_build_image || exit -1
+        create_postgres_build_image || exit 1
     fi
 fi
 
-build_timescaledb || exit -1
+build_timescaledb || exit 1
 
 message_and_exit

scripts/test_update_from_tag.sh

@@ -6,18 +6,28 @@ set -o pipefail
 SCRIPT_DIR=$(dirname $0)
 BASE_DIR=${PWD}/${SCRIPT_DIR}/..
 TEST_VERSION=${TEST_VERSION:-v2}
-PGTEST_TMPDIR=${PGTEST_TMPDIR:-$(mktemp -d 2>/dev/null || mktemp -d -t 'timescaledb_update_test')}
+TEST_TMPDIR=${TEST_TMPDIR:-$(mktemp -d 2>/dev/null || mktemp -d -t 'timescaledb_update_test' || mkdir -p /tmp/${RANDOM})}
 UPDATE_PG_PORT=${UPDATE_PG_PORT:-6432}
 CLEAN_PG_PORT=${CLEAN_PG_PORT:-6433}
 PG_VERSION=${PG_VERSION:-9.6.5} # Need 9.6.x version since we are
                                 # upgrading the extension from
                                 # versions that didn't support PG10.
+GIT_ID=$(git -C ${BASE_DIR} describe --dirty | sed -e "s|/|_|g")
 UPDATE_FROM_IMAGE=${UPDATE_FROM_IMAGE:-timescale/timescaledb}
 UPDATE_FROM_TAG=${UPDATE_FROM_TAG:-0.1.0}
 UPDATE_TO_IMAGE=${UPDATE_TO_IMAGE:-update_test}
-UPDATE_TO_TAG=${UPDATE_TO_TAG:-latest}
+UPDATE_TO_TAG=${UPDATE_TO_TAG:-${GIT_ID}}
 DO_CLEANUP=true
 
+# PID of the current shell
+PID=$$
+
+# Container names. Append shell PID so that we can run this script in parallel
+CONTAINER_ORIG=timescaledb-orig-${PID}
+CONTAINER_CLEAN_RESTORE=timescaledb-clean-restore-${PID}
+CONTAINER_UPDATED=timescaledb-updated-${PID}
+CONTAINER_CLEAN_RERUN=timescaledb-clean-rerun-${PID}
+
 export PG_VERSION
 
 while getopts "d" opt;
@@ -33,28 +43,36 @@ done
 
 shift $((OPTIND-1))
 
-if "$DO_CLEANUP" = "true"; then
 trap cleanup EXIT
-fi
+
+remove_containers() {
+    docker rm -vf ${CONTAINER_ORIG} 2>/dev/null
+    docker rm -vf ${CONTAINER_CLEAN_RESTORE} 2>/dev/null
+    docker rm -vf ${CONTAINER_UPDATED}  2>/dev/null
+    docker rm -vf ${CONTAINER_CLEAN_RERUN} 2>/dev/null
+    docker volume rm -f ${CLEAN_VOLUME} 2>/dev/null
+    docker volume rm -f ${UPDATE_VOLUME} 2>/dev/null
+}
 
 cleanup() {
     # Save status here so that we can return the status of the last
     # command in the script and not the last command of the cleanup
     # function
-    status="$?"
+    local status="$?"
     set +e # do not exit immediately on failure in cleanup handler
-    if [ $status -eq 0 ]; then
+    if [ "$DO_CLEANUP" = "true" ]; then
-        rm -rf ${PGTEST_TMPDIR}
+        rm -rf ${TEST_TMPDIR}
-        docker rm -vf timescaledb-orig timescaledb-clean-restore timescaledb-updated 2>/dev/null
+        sleep 1
+        remove_containers
     fi
-    echo "Exit status is $status"
+    echo "Test with pid ${PID} exited with code ${status}"
-    exit $status
+    exit ${status}
 }
 
 docker_exec() {
     # Echo to stderr
     >&2 echo -e "\033[1m$1\033[0m: $2"
-    docker exec -it $1 /bin/bash -c "$2"
+    docker exec $1 /bin/bash -c "$2"
 }
 
 docker_pgcmd() {
@@ -67,15 +85,15 @@ docker_pgscript() {
 
 docker_pgtest() {
     >&2 echo -e "\033[1m$1\033[0m: $2"
-    docker exec $1 psql -X -v ECHO=ALL -v ON_ERROR_STOP=1 -h localhost -U postgres -d single -f $2 > ${PGTEST_TMPDIR}/$1.out || exit $?
+    docker exec $1 psql -X -v ECHO=ALL -v ON_ERROR_STOP=1 -h localhost -U postgres -d single -f $2 > ${TEST_TMPDIR}/$1.out || exit $?
 }
 
 docker_pgdiff() {
     >&2 echo -e "\033[1m$1 vs $2\033[0m: $2"
     docker_pgtest $1 $3
     docker_pgtest $2 $3
-    echo "RUNNING:  diff ${PGTEST_TMPDIR}/$1.out ${PGTEST_TMPDIR}/$2.out "
+    echo "RUNNING:  diff ${TEST_TMPDIR}/$1.out ${TEST_TMPDIR}/$2.out "
-    diff ${PGTEST_TMPDIR}/$1.out ${PGTEST_TMPDIR}/$2.out | tee ${PGTEST_TMPDIR}/update_test.output
+    diff ${TEST_TMPDIR}/$1.out ${TEST_TMPDIR}/$2.out | tee ${TEST_TMPDIR}/update_test.output
 }
 
 docker_run() {
@@ -90,8 +108,8 @@ docker_run_vol() {
 
 wait_for_pg() {
     set +e
-    for i in {1..10}; do
+    for i in {1..20}; do
-        sleep 2
+        sleep 0.5
 
         docker_exec $1 "pg_isready -U postgres"
 
@@ -100,7 +118,7 @@ wait_for_pg() {
             # ideal. Apperently, pg_isready is not always a good
             # indication of whether the DB is actually ready to accept
             # queries
-            sleep 2
+            sleep 0.2
             set -e
             return 0
         fi
@@ -110,42 +128,46 @@ wait_for_pg() {
 
 VERSION=`echo ${UPDATE_FROM_TAG} | sed 's/\([0-9]\{0,\}\.[0-9]\{0,\}\.[0-9]\{0,\}\).*/\1/g'`
 echo "Testing from version ${VERSION} (test version ${TEST_VERSION})"
-echo "Using temporary directory $PGTEST_TMPDIR"
+echo "Using temporary directory ${TEST_TMPDIR}"
 
-docker rm -f timescaledb-orig timescaledb-updated timescaledb-clean-restore timescaledb-clean-rerun 2>/dev/null || true
+remove_containers || true
-IMAGE_NAME=update_test TAG_NAME=latest bash ${SCRIPT_DIR}/docker-build.sh
 
-docker_run timescaledb-orig ${UPDATE_FROM_IMAGE}:${UPDATE_FROM_TAG}
+IMAGE_NAME=${UPDATE_TO_IMAGE} TAG_NAME=${UPDATE_TO_TAG} PG_VERSION=${PG_VERSION} bash ${SCRIPT_DIR}/docker-build.sh
-docker_run timescaledb-clean-restore ${UPDATE_TO_IMAGE}:${UPDATE_TO_TAG}
-docker_run timescaledb-clean-rerun ${UPDATE_TO_IMAGE}:${UPDATE_TO_TAG}
 
-CLEAN_VOLUME=$(docker inspect timescaledb-clean-restore --format='{{range .Mounts }}{{.Name}}{{end}}')
+docker_run ${CONTAINER_ORIG} ${UPDATE_FROM_IMAGE}:${UPDATE_FROM_TAG}
-UPDATE_VOLUME=$(docker inspect timescaledb-orig --format='{{range .Mounts }}{{.Name}}{{end}}')
+docker_run ${CONTAINER_CLEAN_RESTORE} ${UPDATE_TO_IMAGE}:${UPDATE_TO_TAG}
+docker_run ${CONTAINER_CLEAN_RERUN} ${UPDATE_TO_IMAGE}:${UPDATE_TO_TAG}
+
+CLEAN_VOLUME=$(docker inspect ${CONTAINER_CLEAN_RESTORE} --format='{{range .Mounts }}{{.Name}}{{end}}')
+UPDATE_VOLUME=$(docker inspect ${CONTAINER_ORIG} --format='{{range .Mounts }}{{.Name}}{{end}}')
 
 echo "Executing setup script on ${VERSION}"
-docker_pgscript timescaledb-orig /src/test/sql/updates/setup.${TEST_VERSION}.sql
+docker_pgscript ${CONTAINER_ORIG} /src/test/sql/updates/setup.${TEST_VERSION}.sql
-docker rm -f timescaledb-orig
 
-docker_run_vol timescaledb-updated ${UPDATE_VOLUME}:/var/lib/postgresql/data ${UPDATE_TO_IMAGE}:${UPDATE_TO_TAG}
+# Remove container but keep volume
+docker rm -f ${CONTAINER_ORIG}
+
+echo "Running update container"
+docker_run_vol ${CONTAINER_UPDATED} ${UPDATE_VOLUME}:/var/lib/postgresql/data ${UPDATE_TO_IMAGE}:${UPDATE_TO_TAG}
 
 echo "Executing ALTER EXTENSION timescaledb UPDATE"
-docker_pgcmd timescaledb-updated "ALTER EXTENSION timescaledb UPDATE"
+docker_pgcmd ${CONTAINER_UPDATED} "ALTER EXTENSION timescaledb UPDATE"
 
-docker_exec timescaledb-updated "pg_dump -h localhost -U postgres -Fc single > /tmp/single.sql"
+docker_exec ${CONTAINER_UPDATED} "pg_dump -h localhost -U postgres -Fc single > /tmp/single.sql"
-docker cp timescaledb-updated:/tmp/single.sql ${PGTEST_TMPDIR}/single.sql
+docker cp ${CONTAINER_UPDATED}:/tmp/single.sql ${TEST_TMPDIR}/single.sql
 
 echo "Executing setup script on clean"
-docker_pgscript timescaledb-clean-rerun /src/test/sql/updates/setup.${TEST_VERSION}.sql
+docker_pgscript ${CONTAINER_CLEAN_RERUN} /src/test/sql/updates/setup.${TEST_VERSION}.sql
 
 echo "Testing updated vs clean"
-docker_pgdiff timescaledb-updated timescaledb-clean-rerun /src/test/sql/updates/test-rerun.sql
+docker_pgdiff ${CONTAINER_UPDATED} ${CONTAINER_CLEAN_RERUN} /src/test/sql/updates/test-rerun.sql
 
 echo "Restoring database on clean version"
-docker cp ${PGTEST_TMPDIR}/single.sql timescaledb-clean-restore:/tmp/single.sql
+docker cp ${TEST_TMPDIR}/single.sql ${CONTAINER_CLEAN_RESTORE}:/tmp/single.sql
-docker_exec timescaledb-clean-restore "createdb -h localhost -U postgres single"
+docker_exec ${CONTAINER_CLEAN_RESTORE} "createdb -h localhost -U postgres single"
-docker_pgcmd timescaledb-clean-restore "ALTER DATABASE single SET timescaledb.restoring='on'"
+docker_pgcmd ${CONTAINER_CLEAN_RESTORE} "ALTER DATABASE single SET timescaledb.restoring='on'"
-docker_exec timescaledb-clean-restore "pg_restore -h localhost -U postgres -d single /tmp/single.sql"
+docker_exec ${CONTAINER_CLEAN_RESTORE} "pg_restore -h localhost -U postgres -d single /tmp/single.sql"
-docker_pgcmd timescaledb-clean-restore "ALTER DATABASE single SET timescaledb.restoring='off'"
+docker_pgcmd ${CONTAINER_CLEAN_RESTORE} "ALTER DATABASE single SET timescaledb.restoring='off'"
 
 echo "Testing restored"
-docker_pgdiff timescaledb-updated timescaledb-clean-restore /src/test/sql/updates/post.${TEST_VERSION}.sql
+docker_pgdiff ${CONTAINER_UPDATED} ${CONTAINER_CLEAN_RESTORE} /src/test/sql/updates/post.${TEST_VERSION}.sql

scripts/test_updates.sh

@@ -0,0 +1,97 @@
+#!/bin/bash
+
+set -o pipefail
+
+SCRIPT_DIR=$(dirname $0)
+TEST_TMPDIR=${TEST_TMPDIR:-$(mktemp -d 2>/dev/null || mktemp -d -t 'timescaledb_update_test' || mkdir -p /tmp/$RANDOM )}
+BASE_DIR=${PWD}/${SCRIPT_DIR}/..
+TAGS=${TAGS:-}
+TEST_VERSION=${TEST_VERSION:-}
+GIT_ID=$(git -C ${BASE_DIR} describe --dirty | sed -e "s|/|_|g")
+UPDATE_TO_IMAGE=${UPDATE_TO_IMAGE:-update_test}
+UPDATE_TO_TAG=${UPDATE_TO_TAG:-${GIT_ID}}
+PG_VERSION=${PG_VERSION:-9.6.5} # Need 9.6.x version since we are
+                                # upgrading the extension from
+                                # versions that didn't support PG10.
+
+FAILED_TEST=
+
+# Declare a hash table to keep test names keyed by pid
+declare -A tests
+
+while getopts "c" opt;
+do
+    case $opt in
+        c)
+            echo "Forcing cleanup of build image"
+            docker rmi -f ${UPDATE_TO_IMAGE}:${UPDATE_TO_TAG}
+            ;;
+    esac
+done
+
+cleanup() {
+    local exit_code="$?"
+    set +e # do not exit immediately on failure
+    echo "Waiting for remaining tests to finish..."
+    wait
+    if [ -f ${TEST_TMPDIR}/${FAILED_TEST}.log ]; then
+        echo "###### Failed test log below #####"
+        cat ${TEST_TMPDIR}/${FAILED_TEST}.log
+    fi
+    rm -rf ${TEST_TMPDIR}
+    echo "exit code is $exit_code"
+    return $exit_code
+}
+
+kill_all_tests() {
+    local exit_code="$?"
+    set +e # do not exit immediately on failure
+    echo "Killing all tests"
+    kill ${!tests[@]} 2>/dev/null
+    return $exit_code
+}
+
+trap kill_all_tests INT HUP
+trap cleanup EXIT
+
+if [ -z "${TEST_VERSION}" ]; then
+    echo "No TEST_VERSION specified"
+    exit 1
+fi
+
+if [ -z "${TAGS}" ]; then
+    echo "No TAGS specified"
+    exit 1
+fi
+
+# Build the docker image with current source here so that the parallel
+# tests don't all compete in trying to build it first
+IMAGE_NAME=${UPDATE_TO_IMAGE} TAG_NAME=${UPDATE_TO_TAG} PG_VERSION=${PG_VERSION} bash ${SCRIPT_DIR}/docker-build.sh
+
+# Run update tests in parallel
+for tag in ${TAGS};
+do
+    UPDATE_FROM_TAG=${tag} TEST_VERSION=${TEST_VERSION} $(dirname $0)/test_update_from_tag.sh > ${TEST_TMPDIR}/${tag}.log 2>&1 &
+
+    tests[$!]=${tag}
+    echo "Launched test ${tag} with pid $!"
+done
+
+# Need to wait on each pid in a loop to return the exit status of each
+echo "Waiting for tests to finish..."
+
+# Since we are iterating a hash table, the tests are not going to be
+# in order started. But it doesn't matter.
+for pid in ${!tests[@]};
+do
+    wait $pid;
+    exit_code=$?
+    echo "Test ${tests[$pid]} (pid $pid) exited with code $exit_code"
+
+    if [ $exit_code -ne 0 ]; then
+        FAILED_TEST=${tests[$pid]}
+        kill_all_tests
+        exit $exit_code
+    fi
+    $(exit $exit_code)
+done

scripts/test_updates_no_constraints.sh

@@ -7,9 +7,8 @@
 set -e
 set -o pipefail
 
+SCRIPT_DIR=$(dirname $0)
 TAGS="0.1.0 0.2.0 0.3.0 0.4.0 0.4.1 0.4.2"
+TEST_VERSION="v1"
 
-for tag in ${TAGS};
+. ${SCRIPT_DIR}/test_updates.sh
-do
-    UPDATE_FROM_TAG=${tag} TEST_VERSION="v1" $(dirname $0)/test_update_from_tag.sh
-done

scripts/test_updates_with_constraints.sh

@@ -10,9 +10,9 @@
 set -e
 set -o pipefail
 
-TAGS="0.5.0 0.6.0 0.6.1 0.7.0-pg9.6 0.7.1-pg9.6 0.8.0-pg9.6 0.9.0-pg9.6 0.9.1-pg9.6 0.9.2-pg9.6 0.10.0-pg9.6 0.10.1-pg9.6 0.11.0-pg9.6 0.12.0-pg9.6 1.0.0-rc1-pg9.6 1.0.0-rc2-pg9.6 1.0.0-rc3-pg9.6"
+SCRIPT_DIR=$(dirname $0)
 
-for tag in ${TAGS};
+TAGS="0.5.0 0.6.0 0.6.1 0.7.0-pg9.6 0.7.1-pg9.6 0.8.0-pg9.6 0.9.0-pg9.6 0.9.1-pg9.6 0.9.2-pg9.6 0.10.0-pg9.6 0.10.1-pg9.6 0.11.0-pg9.6 0.12.0-pg9.6 1.0.0-rc1-pg9.6 1.0.0-rc2-pg9.6 1.0.0-rc3-pg9.6"
-do
+TEST_VERSION="v2"
-    UPDATE_FROM_TAG=${tag} TEST_VERSION="v2" $(dirname $0)/test_update_from_tag.sh
+
-done
+. ${SCRIPT_DIR}/test_updates.sh