Add memory spike test for out of order inserts

Out of order, random inserts into TimescaleDB in the past
sometimes caused continuously increasing memory usage.
These checks are being added to ensure that there are no
memory spike regressions via a cron Travis job.

.travis.yml

@@ -52,6 +52,17 @@ jobs:
         # Version >= 9.6.3 is required by TimescaleDB
         - PG_MAJOR=9.6 PG_MINOR_COMPILE=3 bash -x ./scripts/docker-run-abi-test.sh
 
+    - if: type = cron
+      stage: test
+      env:
+        - PG_VERSION=10.0 PG_GIT_TAG=REL_10_0
+        - secure: "TurskgAodG44RKCUPfoUb4JlT+UZwaPjUN73MS4+2ED2ayf46iUpcMODovDf7CPQRriMAvii/5xQ9UWTGYZejJy/Si3aE0fQeSKHIOPmuHqmmBkC3jgK9A5vOMpaAwljSNvIAMfwhIjVsq87nyvxHYWReDWl5SrdxTvh91RJvOiMRNdqwBbnj2j1cC30783ICGZUazrUyhDDnXispyBJYFxlHJmQJpfs6428dTRNJY/X8nVDXFwQkRjlIXarquTsrifCfzdq40Ykb8/1CS0295R8/kdKamQqXM0aS7iIkw3JFmZIz08OHKc7wcJuv6KSnCfNSftJqy2+nb1gkyIqkr3mrpYUa+2tKVwpA6OfET2S2kC5B3bOTcyBgihhnivAOS9zG8PTouLSUzqZzo+H893xdaMlX4kX3oOrOqzbj8YvTw6Dgw8PWERfnOngpKmKJFb8yboXPmGY7luSVxOplnPnYO0jaAXPU4ESsFYL3EBG5Zo4rVIuA5g7aJ0Ex+e5GQWiHDM1ubUo2vcuVVlurs/Aj8eSpNhqo0S5MjI05Jcz82X+9+asFcP+QV81UrueiXTlQ/7buAyMyVtKOgOhZwR4ImA1zzWkJjNNbZMJqeO9tPEgU7IxrqLhMgIpBB/pn0gzJZ8LN7C2oyznMYGzS5lrYaavojFUViv8vFyAQ24="
+        - secure: "S8WfPsrVGg21vZPxjrmy1X19T7zpHNgaSAoqYGzJdDZcNYuZbI87+tP3VQZo4MxjP3Ag46zTI5K+6aMjCqYqFGKufZdEaSpPSLmppFbXALgSp/rOkzp1M9HhGj0QVP7QcoPbva5Bjnj3zVi9CMAo6MwtnGAvzqBLOsdtnXTxEmKYyeNgwUVbQi2nzO38GeLgabaFL4XD3f3bmHE5jG6QAwZ/L7+jajsD1biH4sOiRLCRshtvermtYxh3SK6rML2kiwATABxTd1xDdMVRO/llmDBN373tHK6mufJyIhrwG9oTprSSDDIJlEbSPCkH0uvJBGcDvOAWf9tr4TGs/beawx3ELcXqR/4EisFqvMiVL4Vpt1gMBg9gue4Y0wpHZYoBOOef2gFtuyL7PjYe9koYJAJxg9DJ29DMxIjuCLTnbHN6yLa9425pdvxcuNEJ2K2zD0ZWoLSVueL0MxLhOZJsqlMkmmrFyI7y5cj1XvJF3vcsa5yK3S7sXroKUzeLu6QMh8hO3jD/2IvdQJ7Huy5uiJk2a+KUceLzruxLbwPF4b8SEoTquJ9NFGtagCG77lbIbLltLs2fUf+JnM8ipdpoMibVRFBOahP8NZwpL/NFzG3WLH3tDncj/c3fneMiNv3FUHLf6ufOVOayQiAEidszcMct2R20qIxggXKymrjZA3k="
+      before_script:
+        - git clone --progress --verbose https://$USR:$PASSWORD@bitbucket.org/440-labs/tsdb-dev-tools.git /tmp/tsdb-dev-tools
+      script:
+        - bash ./scripts/docker-run-memory-test.sh
+
     - stage: test
       env:
         - secure: "jy4DQH2syPR2v13igCNPTr044h3H/ilbJk6FifDMxGZVrOZR0dnkBx3O7qJMQOkEQvNxKsoq41k6HCP16qcgt4+HjxhcZonz5hKIiF8IpcB9r+TIlZunNTx7HjSNFZ3WCnham4AvMEthBHgAttRUhscy39ELCNUEobKS/youi7OHLOEXXShc84yTh3aSuGR3SnDVK1diLN5ufX6tN20pc3QvLMGZmA/jmJFcIQHGilhWGwwiJ45LSLwM9slvgGKbTM/K6btVBMOUnjM0h5WqPjRjDUL2tF+iZLEIpY8lFN/MQCnj0vP/BryDdoVPZS3TDQYwYuvASevQ4sOmULnM770jFqzClq4zkeM2GhMq67aYMmXjblu/qcLeCjZL+vfjMKpBMUydK/bCb097HvdRWDEPA0zItKWX9Kd6lVf2XbJCCh0ljp5REJEyk+plJ2V12nLpOPwY6zTtzcoTxEN6wcvUJfHAdNovpp63hWTnbAbEZamIdxwyCqpzThDobeD354TeXFUaKvrUw00iAiIhGL2QvwapaCbhlwM6NQAmdU3tMy3nZpka6bRI1kjyTh7CXfdwXV98ZJSiPdUFxyIgFNI2dKiL3BI1pvFDfq3mnmi3WqzZHCaQqDKNEtUrzxC40swIJGLcLUiqc5xX37P47jNDWrNIRDs8IdbM0tS9pFM="

scripts/docker-run-memory-test.sh

@@ -0,0 +1,82 @@
+#!/bin/bash
+
+set -e
+set -o pipefail
+
+SCRIPT_DIR=$(dirname $0)
+DO_CLEANUP=true
+
+while getopts "d" opt;
+do
+    case $opt in
+        d)
+            DO_CLEANUP=false
+            echo "!!Debug mode: Containers and temporary directory will be left on disk"
+            echo
+            ;;
+    esac
+done
+
+shift $((OPTIND-1))
+
+if "$DO_CLEANUP" = "true"; then
+    trap cleanup EXIT
+fi
+
+cleanup() {
+    # Save status here so that we can return the status of the last
+    # command in the script and not the last command of the cleanup
+    # function
+    status="$?"
+    set +e # do not exit immediately on failure in cleanup handler
+    # docker rm -vf timescaledb-valgrind 2>/dev/null
+    docker rm -vf timescaledb-memory 2>/dev/null
+    echo "Exit status is $status"
+    exit $status
+}
+
+docker_exec() {
+    # Echo to stderr
+    >&2 echo -e "\033[1m$1\033[0m: $2"
+    docker exec -it $1 /bin/bash -c "$2"
+}
+
+docker_run() {
+    docker run -d --name $1 $2
+    wait_for_pg $1
+}
+
+wait_for_pg() {
+    set +e
+    for i in {1..10}; do
+        sleep 2
+
+        docker_exec $1 "pg_isready -U postgres"
+
+        if [[ $? == 0 ]] ; then
+            # this makes the test less flaky, although not
+            # ideal. Apperently, pg_isready is not always a good
+            # indication of whether the DB is actually ready to accept
+            # queries
+            sleep 5
+            set -e
+            return 0
+        fi
+    done
+    exit 1
+}
+
+
+docker rm -f timescaledb-memory 2>/dev/null || true
+IMAGE_NAME=memory_test TAG_NAME=latest bash ${SCRIPT_DIR}/docker-build.sh
+
+docker_run timescaledb-memory memory_test:latest
+
+echo "Installing python3 and psutil"
+docker_exec timescaledb-memory "apk add --no-cache python3 && python3 -m ensurepip && pip3 install --upgrade pip && apk add --update build-base python3-dev py-psutil"
+
+echo "Copying necessary scripts from host to docker container"
+docker cp -a /tmp/tsdb-dev-tools timescaledb-memory:/tmp/tsdb-dev-tools
+
+echo "Testing"
+docker_exec timescaledb-memory "python3 /tmp/tsdb-dev-tools/test_memory_spikes.py & sleep 5 && psql -U postgres -d postgres -h localhost -v ECHO=all -X -f /tmp/tsdb-dev-tools/sql/out_of_order_random_direct.sql"