diff --git a/scripts/test_update_smoke.sh b/scripts/test_update_smoke.sh index 76736de3b..41c16c821 100644 --- a/scripts/test_update_smoke.sh +++ b/scripts/test_update_smoke.sh @@ -2,6 +2,9 @@ # Run smoke tests to test that updating between versions work. # +# Usage: +# bash test_update_smoke.sh postgres://... +# # This is based on our update tests but doing some tweaks to ensure we # can run it on Forge (or any other PostgreSQL server that only permit # a single user and single database). @@ -13,7 +16,7 @@ # cannot (necessarily) be done with multinode setups. Feel free to try # though. # -# The following environment variables can be set: +# The following environment variables can be set: # - UPDATE_FROM_TAG is the version to update from (optional). # # - UPDATE_TO_TAG is the version to update to (optional). @@ -33,33 +36,69 @@ SCRIPT_DIR=$(dirname $0) BASE_DIR=${PWD}/${SCRIPT_DIR}/.. SCRATCHDIR=`mktemp -d -t 'smoketest-XXXX'` +LOGFILE="$SCRATCHDIR/update-test.log" DUMPFILE="$SCRATCHDIR/smoke.dump" UPGRADE_OUT="$SCRATCHDIR/upgrade.out" CLEAN_OUT="$SCRATCHDIR/clean.out" RESTORE_OUT="$SCRATCHDIR/restore.out" TEST_VERSION=${TEST_VERSION:-v6} +UPDATE_FROM_TAG=${UPDATE_FROM_TAG:-1.7.4} +UPDATE_TO_TAG=${UPDATE_TO_TAG:-2.0.1} +# We do not have superuser privileges when running smoke tests. +WITH_SUPERUSER=false -while getopts "d" opt; +while getopts "ds:t:" opt; do case $opt in d) cleanup=1 ;; + s) + UPDATE_FROM_TAG=$OPTARG + ;; + t) + UPDATE_TO_TAG=$OPTARG + ;; esac done shift $((OPTIND-1)) -UPDATE_FROM_TAG=${1:-${UPDATE_FROM_TAG:-1.7.4}} -UPDATE_TO_TAG=${2:-${UPDATE_TO_TAG:-2.0.1}} - -# We do not have superuser privileges when running smoke tests. -WITH_SUPERUSER=false +echo "**** pg_dump at " `which pg_dump` +echo "**** pg_restore at" `which pg_restore` # Extra options to pass to psql -PGOPTS="-v TEST_VERSION=${TEST_VERSION} -v WITH_SUPERUSER=${WITH_SUPERUSER}" +PGOPTS="-v TEST_VERSION=${TEST_VERSION} -v WITH_SUPERUSER=${WITH_SUPERUSER} -v WITH_CHUNK=false" PSQL="psql -qX $PGOPTS" +# If we are providing a URI for the connection, we parse it here and +# set the PG??? variables since that is the only reliable way to +# provide connection information to psql, pg_dump, and pg_restore. +# +# To work with Forge, we need to only set PGPASSWORD when the password +# is available and leave it unset otherwise. If the user has either +# set PGPASSWORD or has the password in a .pgpass file, it will be +# picked up and used for the connection. +if [[ $# -gt 0 ]]; then + parts=($(echo $1 | perl -mURI::Split=uri_split -ne '@F = uri_split($_); print join(" ", split(qr/[:@]/, $F[1]), substr($F[2], 1))')) + export PGUSER=${parts[0]} + if [[ ${#parts[@]} -eq 5 ]]; then + # Cloud has 5 fields + export PGPASSWORD=${parts[1]} + export PGHOST=${parts[2]} + export PGPORT=${parts[3]} + export PGDATABASE=${parts[4]} + elif [[ ${#parts[@]} -eq 4 ]]; then + # Forge has 4 fields + export PGHOST=${parts[1]} + export PGPORT=${parts[2]} + export PGDATABASE=${parts[3]} + else + echo "Malformed URL '$1'" 1>&2 + exit 2 + fi +fi + cleanup() { rm -rf $SCRATCHDIR } @@ -70,11 +109,12 @@ if [[ "x$cleanup" != "x" ]]; then fi missing_versions() { - $PSQL -t <>$LOGFILE 2>&1 +$PSQL -f pre.cleanup.sql >>$LOGFILE 2>&1 +$PSQL -c "CREATE EXTENSION timescaledb VERSION '${UPDATE_FROM_TAG}'" >>$LOGFILE 2>&1 $PSQL -c "\dx" # Run setup on Upgrade -$PSQL -f pre.smoke.sql -$PSQL -f setup.${TEST_VERSION}.sql +$PSQL -f pre.smoke.sql >>$LOGFILE 2>&1 +$PSQL -f setup.${TEST_VERSION}.sql >>$LOGFILE 2>&1 # Run update on Upgrade. You now have a 2.0.2 version in Upgrade. -$PSQL -c "ALTER EXTENSION timescaledb UPDATE TO '${UPDATE_TO_TAG}'" +$PSQL -c "ALTER EXTENSION timescaledb UPDATE TO '${UPDATE_TO_TAG}'" >>$LOGFILE 2>&1 -# Dump the contents of Upgrade -pg_dump -Fc -f $DUMPFILE +echo -n "Dumping the contents of Upgrade..." +pg_dump -Fc -f $DUMPFILE >>$LOGFILE 2>&1 +echo "done" # Run the post scripts on Upgrade to get UpgradeOut to compare # with. We can now discard Upgrade database. +echo -n "Collecting post-update status..." $PSQL -f post.${TEST_VERSION}.sql >$UPGRADE_OUT +echo "done" -: ---- Create a 2.0.2 version Clean ---- -$PSQL -c "DROP EXTENSION IF EXISTS timescaledb CASCADE" -$PSQL -f pre.cleanup.sql -$PSQL -c "CREATE EXTENSION timescaledb VERSION '${UPDATE_TO_TAG}'" +echo "---- Create a ${UPDATE_TO_TAG} version Clean ----" +$PSQL -c "DROP EXTENSION IF EXISTS timescaledb CASCADE" >>$LOGFILE 2>&1 +$PSQL -f pre.cleanup.sql >>$LOGFILE 2>&1 +$PSQL -c "CREATE EXTENSION timescaledb VERSION '${UPDATE_TO_TAG}'" >>$LOGFILE 2>&1 +$PSQL -c "\dx" -: ---- Run the setup scripts on Clean, with post-update actions ---- -$PSQL -f pre.smoke.sql -$PSQL -f setup.${TEST_VERSION}.sql -#$PSQL -f post.repair.sql +echo "---- Run the setup scripts on Clean, with post-update actions ----" +$PSQL -f pre.smoke.sql >>$LOGFILE 2>&1 +$PSQL -f setup.${TEST_VERSION}.sql >>$LOGFILE 2>&1 -: ---- Run the post scripts on Clean to get output CleanOut ---- +echo "---- Run the post scripts on Clean to get output CleanOut ----" $PSQL -f post.${TEST_VERSION}.sql >$CLEAN_OUT -: ---- Create a 2.0.2 version Restore ---- -$PSQL -c "DROP EXTENSION IF EXISTS timescaledb CASCADE" -$PSQL -f pre.cleanup.sql -$PSQL -c "CREATE EXTENSION timescaledb VERSION '${UPDATE_TO_TAG}'" +$PSQL -f cleanup.${TEST_VERSION}.sql 2>&1 >>$LOGFILE -: ---- Restore the UpgradeDump into Restore ---- -$PSQL -c "SELECT timescaledb_pre_restore()" -pg_restore -d $PGDATABASE $DUMPFILE || true -$PSQL -c "SELECT timescaledb_post_restore()" +echo "---- Create a ${UPDATE_TO_TAG} version Restore ----" +$PSQL -c "DROP EXTENSION IF EXISTS timescaledb CASCADE" >>$LOGFILE 2>&1 +$PSQL -f pre.cleanup.sql >>$LOGFILE 2>&1 +$PSQL -c "CREATE EXTENSION timescaledb VERSION '${UPDATE_TO_TAG}'" >>$LOGFILE 2>&1 +$PSQL -c "\dx" -: ---- Run the post scripts on Restore to get a RestoreOut ---- +echo "---- Restore the UpgradeDump into Restore ----" +echo -n "Restoring dump..." +$PSQL -c "SELECT timescaledb_pre_restore()" >>$LOGFILE 2>&1 +pg_restore -d $PGDATABASE $DUMPFILE >>$LOGFILE 2>&1 || true +$PSQL -c "SELECT timescaledb_post_restore()" >>$LOGFILE 2>&1 +echo "done" + +echo "---- Run the post scripts on Restore to get a RestoreOut ----" $PSQL -f post.${TEST_VERSION}.sql >$RESTORE_OUT -: ---- Compare UpgradeOut with CleanOut and make sure they are identical ---- +echo "---- Compare UpgradeOut with CleanOut and make sure they are identical ----" diff -u $UPGRADE_OUT $CLEAN_OUT && echo "No difference between $UPGRADE_OUT and $CLEAN_OUT" | tee $SCRATCHDIR/upgrade-clean.diff -: ---- Compare RestoreOut with CleanOut and make sure they are identical ---- +echo "---- Compare RestoreOut with CleanOut and make sure they are identical ----" diff -u $RESTORE_OUT $CLEAN_OUT && echo "No difference between $RESTORE_OUT and $CLEAN_OUT" | tee $SCRATCHDIR/restore-clean.diff +$PSQL -f cleanup.${TEST_VERSION}.sql >>$LOGFILE 2>&1 diff --git a/test/sql/updates/cleanup.bigint.sql b/test/sql/updates/cleanup.bigint.sql new file mode 100644 index 000000000..6fb610d76 --- /dev/null +++ b/test/sql/updates/cleanup.bigint.sql @@ -0,0 +1,5 @@ +-- This file and its contents are licensed under the Apache License 2.0. +-- Please see the included NOTICE for copyright information and +-- LICENSE-APACHE for a copy of the license. + +DROP TABLE PUBLIC."two_Partitions" CASCADE; diff --git a/test/sql/updates/cleanup.compression.sql b/test/sql/updates/cleanup.compression.sql new file mode 100644 index 000000000..4d8809702 --- /dev/null +++ b/test/sql/updates/cleanup.compression.sql @@ -0,0 +1,6 @@ +-- This file and its contents are licensed under the Apache License 2.0. +-- Please see the included NOTICE for copyright information and +-- LICENSE-APACHE for a copy of the license. + +DROP TABLE compress; +DROP TYPE custom_type_for_compression; diff --git a/test/sql/updates/cleanup.continuous_aggs.v2.sql b/test/sql/updates/cleanup.continuous_aggs.v2.sql new file mode 100644 index 000000000..1345e129e --- /dev/null +++ b/test/sql/updates/cleanup.continuous_aggs.v2.sql @@ -0,0 +1,44 @@ +-- This file and its contents are licensed under the Apache License 2.0. +-- Please see the included NOTICE for copyright information and +-- LICENSE-APACHE for a copy of the license. + +-- Collect information about different features so that we can pick +-- the right usage. Some of these are changed in the same version, but +-- we keep them separate anyway so that we can do additional checking +-- if necessary. +SELECT extversion < '2.0.0' AS has_refresh_mat_view, + extversion < '2.0.0' AS has_drop_chunks_old_interface, + extversion < '2.0.0' AS has_ignore_invalidations_older_than, + extversion < '2.0.0' AS has_max_interval_per_job, + extversion >= '2.0.0' AS has_create_mat_view, + extversion >= '2.0.0' AS has_continuous_aggs_policy + FROM pg_extension + WHERE extname = 'timescaledb' \gset + +\if :has_create_mat_view +\set :DROP_CAGG DROP MATERIALIZED VIEW +\else +\set :DROP_CAGG DROP VIEW +\endif + +\if :has_continuous_aggs_policy +SELECT drop_continuous_aggregate_policy('mat_drop'); +\endif + +:DROP_CAGG mat_drop +DROP TABLE drop_test; +:DROP_CAGG mat_conflict; +DROP TABLE conflict_test; +:DROP_CAGG mat_inttime; +DROP FUNCTION integer_now_test; +DROP TABLE int_time_test; +:DROP_CAGG mat_inval; +DROP TABLE inval_test; +:DROP_CAGG mat_ignoreinval; +:DROP_CAGG cagg.realtime_mat; +DROP SCHEMA cagg; +:DROP_CAGG mat_before; +DROP TABLE conditions_before; +DROP TYPE custom_type; + + diff --git a/test/sql/updates/cleanup.policies.sql b/test/sql/updates/cleanup.policies.sql new file mode 100644 index 000000000..42114b7e7 --- /dev/null +++ b/test/sql/updates/cleanup.policies.sql @@ -0,0 +1,26 @@ +-- This file and its contents are licensed under the Apache License 2.0. +-- Please see the included NOTICE for copyright information and +-- LICENSE-APACHE for a copy of the license. + +DO LANGUAGE PLPGSQL $$ +DECLARE + ts_version TEXT; +BEGIN + + SELECT extversion INTO ts_version FROM pg_extension WHERE extname = 'timescaledb'; + + PERFORM remove_reorder_policy('policy_test_timestamptz'); + + -- some policy API functions got renamed for 2.0 so we need to make + -- sure to use the right name for the version + IF ts_version < '2.0.0' THEN + PERFORM remove_drop_chunks_policy('policy_test_timestamptz'); + PERFORM remove_compress_chunks_policy('policy_test_timestamptz'); + ELSE + PERFORM remove_retention_policy('policy_test_timestamptz'); + PERFORM remove_compression_policy('policy_test_timestamptz'); + END IF; +END +$$; + +DROP TABLE policy_test_timestamptz; diff --git a/test/sql/updates/cleanup.v2.sql b/test/sql/updates/cleanup.v2.sql new file mode 100644 index 000000000..367c0c100 --- /dev/null +++ b/test/sql/updates/cleanup.v2.sql @@ -0,0 +1,10 @@ +-- This file and its contents are licensed under the Apache License 2.0. +-- Please see the included NOTICE for copyright information and +-- LICENSE-APACHE for a copy of the license. + +\ir cleanup.bigint.sql +\ir cleanup.constraints.sql +\ir cleanup.insert_bigint.v2.sql +\ir cleanup.timestamp.sql +\ir cleanup.insert_timestamp.sql +\ir cleanup.drop_meta.sql diff --git a/test/sql/updates/cleanup.v6.sql b/test/sql/updates/cleanup.v6.sql new file mode 100644 index 000000000..48ba2216a --- /dev/null +++ b/test/sql/updates/cleanup.v6.sql @@ -0,0 +1,8 @@ +-- This file and its contents are licensed under the Apache License 2.0. +-- Please see the included NOTICE for copyright information and +-- LICENSE-APACHE for a copy of the license. + +\ir cleanup.v2.sql +\ir cleanup.continuous_aggs.v2.sql +\ir cleanup.compression.sql +\ir cleanup.policies.sql diff --git a/test/sql/updates/post.continuous_aggs.sql b/test/sql/updates/post.continuous_aggs.sql index f055ec9f3..6451e09f9 100644 --- a/test/sql/updates/post.continuous_aggs.sql +++ b/test/sql/updates/post.continuous_aggs.sql @@ -2,7 +2,9 @@ -- Please see the included NOTICE for copyright information and -- LICENSE-APACHE for a copy of the license. +\x on SELECT * FROM mat_before ORDER BY bucket, location; +\x off INSERT INTO conditions_before SELECT generate_series('2018-12-01 00:00'::timestamp, '2018-12-31 00:00'::timestamp, '1 day'), 'POR', 165, 75, 40, 70, NULL, (1,2)::custom_type, 2, true; @@ -11,12 +13,16 @@ SELECT generate_series('2018-12-01 00:00'::timestamp, '2018-12-31 00:00'::timest INSERT INTO conditions_before SELECT generate_series('2017-12-01 00:00'::timestamp, '2017-12-31 00:00'::timestamp, '1 day'), 'POR', 1065, 75, 40, 70, NULL, (1,2)::custom_type, 2, true; +\x on SELECT * FROM mat_before ORDER BY bucket, location; +\x off CALL refresh_continuous_aggregate('mat_before',NULL,NULL); --the max of the temp for the POR should now be 165 +\x on SELECT * FROM mat_before ORDER BY bucket, location; +\x off -- Output the ACLs for each internal cagg object SELECT cl.oid::regclass::text AS reloid, diff --git a/test/sql/updates/post.continuous_aggs.v2.sql b/test/sql/updates/post.continuous_aggs.v2.sql index 87edceb46..a87f7fdc4 100644 --- a/test/sql/updates/post.continuous_aggs.v2.sql +++ b/test/sql/updates/post.continuous_aggs.v2.sql @@ -6,11 +6,15 @@ \d cagg.* +\x on SELECT * FROM cagg.realtime_mat ORDER BY bucket, location; +\x off CALL refresh_continuous_aggregate('cagg.realtime_mat',NULL,NULL); +\x on SELECT * FROM cagg.realtime_mat ORDER BY bucket, location; +\x off SELECT view_name, materialized_only, materialization_hypertable_name FROM timescaledb_information.continuous_aggregates diff --git a/test/sql/updates/setup.compression.sql b/test/sql/updates/setup.compression.sql index 5446c9fbc..7962a02f0 100644 --- a/test/sql/updates/setup.compression.sql +++ b/test/sql/updates/setup.compression.sql @@ -34,4 +34,4 @@ SELECT compress_chunk(chunk.schema_name|| '.' || chunk.table_name) as count_comp FROM _timescaledb_catalog.chunk chunk INNER JOIN _timescaledb_catalog.hypertable hypertable ON (chunk.hypertable_id = hypertable.id) WHERE hypertable.table_name = 'compress' and chunk.compressed_chunk_id IS NULL -ORDER BY chunk.id; \ No newline at end of file +ORDER BY chunk.id; diff --git a/test/sql/updates/setup.continuous_aggs.v2.sql b/test/sql/updates/setup.continuous_aggs.v2.sql index 00eb4399e..df279e6aa 100644 --- a/test/sql/updates/setup.continuous_aggs.v2.sql +++ b/test/sql/updates/setup.continuous_aggs.v2.sql @@ -58,7 +58,7 @@ BEGIN min(allnull) as min_allnull, max(temperature) as max_temp, sum(temperature)+sum(humidity) as agg_sum_expr, - avg(humidity), + avg(humidity) AS avg_humidity, stddev(humidity), bit_and(bit_int), bit_or(bit_int), @@ -103,7 +103,7 @@ BEGIN min(allnull) as min_allnull, max(temperature) as max_temp, sum(temperature)+sum(humidity) as agg_sum_expr, - avg(humidity), + avg(humidity) AS avg_humidity, stddev(humidity), bit_and(bit_int), bit_or(bit_int), @@ -171,7 +171,7 @@ BEGIN min(allnull) as min_allnull, max(temperature) as max_temp, sum(temperature)+sum(humidity) as agg_sum_expr, - avg(humidity), + avg(humidity) AS avg_humidity, stddev(humidity), bit_and(bit_int), bit_or(bit_int), @@ -215,7 +215,7 @@ BEGIN min(allnull) as min_allnull, max(temperature) as max_temp, sum(temperature)+sum(humidity) as agg_sum_expr, - avg(humidity), + avg(humidity) AS avg_humidity, stddev(humidity), bit_and(bit_int), bit_or(bit_int), diff --git a/test/sql/updates/setup.drop_meta.sql b/test/sql/updates/setup.drop_meta.sql index 31ffe9cd6..13d51df21 100644 --- a/test/sql/updates/setup.drop_meta.sql +++ b/test/sql/updates/setup.drop_meta.sql @@ -3,5 +3,7 @@ -- LICENSE-APACHE for a copy of the license. -- DROP some chunks to test metadata cleanup +\if :WITH_CHUNK DROP TABLE _timescaledb_internal._hyper_1_2_chunk; DROP TABLE _timescaledb_internal._hyper_1_3_chunk; +\endif