From f57f34f1b285e20dd90b8882fbb8907e84fc5265 Mon Sep 17 00:00:00 2001 From: Olof Rensfelt Date: Thu, 17 Nov 2016 21:48:56 +0100 Subject: [PATCH] add script to generate csv --- .../unit/expected_outputs/empty_result.csv | 1 + .../expected_outputs/q1_response_fields.csv | 12 ++-- .../{input_data => import_data}/README.md | 0 .../batch1_dev1.tsv | 0 .../batch1_dev2.tsv | 0 .../batch2_dev1.tsv | 0 sql/tests/unit/ioql_test.sql | 71 ------------------- sql/tests/unit/run.sh | 24 ++++--- sql/tests/unit/{ => setup}/add_test_inputs.sh | 6 +- .../unit/{ => setup}/add_test_outputs.sh | 2 +- sql/tests/unit/setup/add_test_utils.sh | 17 +++++ sql/tests/unit/{ => setup}/start_tests.sql | 0 sql/tests/unit/setup/test_utils.sql | 47 ++++++++++++ sql/tests/unit/tests/ioql_empty_test.sql | 42 +++++++++++ .../kafka_offset_functions_test.sql | 0 sql/tests/unit/{ => tests}/names_test.sql | 2 - sql/tests/unit/utils/generate_expected_csv.sh | 41 +++++++++++ sql/tests/unit/utils/import_batch.sh | 28 ++++++++ 18 files changed, 199 insertions(+), 94 deletions(-) create mode 100644 sql/tests/unit/expected_outputs/empty_result.csv rename sql/tests/unit/{input_data => import_data}/README.md (100%) rename sql/tests/unit/{input_data => import_data}/batch1_dev1.tsv (100%) rename sql/tests/unit/{input_data => import_data}/batch1_dev2.tsv (100%) rename sql/tests/unit/{input_data => import_data}/batch2_dev1.tsv (100%) delete mode 100644 sql/tests/unit/ioql_test.sql rename sql/tests/unit/{ => setup}/add_test_inputs.sh (94%) rename sql/tests/unit/{ => setup}/add_test_outputs.sh (96%) create mode 100755 sql/tests/unit/setup/add_test_utils.sh rename sql/tests/unit/{ => setup}/start_tests.sql (100%) create mode 100644 sql/tests/unit/setup/test_utils.sql create mode 100644 sql/tests/unit/tests/ioql_empty_test.sql rename sql/tests/unit/{ => tests}/kafka_offset_functions_test.sql (100%) rename sql/tests/unit/{ => tests}/names_test.sql (97%) create mode 100755 sql/tests/unit/utils/generate_expected_csv.sh create mode 100755 sql/tests/unit/utils/import_batch.sh diff --git a/sql/tests/unit/expected_outputs/empty_result.csv b/sql/tests/unit/expected_outputs/empty_result.csv new file mode 100644 index 000000000..cbe85dcb4 --- /dev/null +++ b/sql/tests/unit/expected_outputs/empty_result.csv @@ -0,0 +1 @@ +time bigint; bool_1 boolean; device_id text; field_only_dev2 double precision; field_only_ref2 text; nUm_1 double precision; num_2 double precision; string_1 text; string_2 text diff --git a/sql/tests/unit/expected_outputs/q1_response_fields.csv b/sql/tests/unit/expected_outputs/q1_response_fields.csv index d1fe5aed8..a09efc324 100644 --- a/sql/tests/unit/expected_outputs/q1_response_fields.csv +++ b/sql/tests/unit/expected_outputs/q1_response_fields.csv @@ -1,6 +1,6 @@ -time bigint; bool_1 boolean; device_id text; field_only_dev2 double precision; field_only_ref2 text; num_1 double precision; num_2 double precision; string_1 text; string_2 text -1257897600000000000;false;dev1;null;null;null;5;const;five -1257894001000000000;true;dev1;null;null;null;4;const;four -1257894000000001000;true;dev1;null;null;null;3;const;three -1257894000000000000;true;dev1;null;null;null;1;const;one -1257894000000000000;false;dev1;null;null;null;2;const;two +time bigint; bool_1 boolean; device_id text; field_only_dev2 double precision; field_only_ref2 text; nUm_1 double precision; num_2 double precision; string_1 text; string_2 text +1257897600000000000;f;dev1;null;null;1.5;5;const;five +1257894001000000000;t;dev1;null;null;1.5;4;const;four +1257894000000001000;t;dev1;null;null;1.5;3;const;three +1257894000000000000;t;dev1;null;null;1.5;1;const;one +1257894000000000000;f;dev1;null;null;1.5;2;const;two diff --git a/sql/tests/unit/input_data/README.md b/sql/tests/unit/import_data/README.md similarity index 100% rename from sql/tests/unit/input_data/README.md rename to sql/tests/unit/import_data/README.md diff --git a/sql/tests/unit/input_data/batch1_dev1.tsv b/sql/tests/unit/import_data/batch1_dev1.tsv similarity index 100% rename from sql/tests/unit/input_data/batch1_dev1.tsv rename to sql/tests/unit/import_data/batch1_dev1.tsv diff --git a/sql/tests/unit/input_data/batch1_dev2.tsv b/sql/tests/unit/import_data/batch1_dev2.tsv similarity index 100% rename from sql/tests/unit/input_data/batch1_dev2.tsv rename to sql/tests/unit/import_data/batch1_dev2.tsv diff --git a/sql/tests/unit/input_data/batch2_dev1.tsv b/sql/tests/unit/import_data/batch2_dev1.tsv similarity index 100% rename from sql/tests/unit/input_data/batch2_dev1.tsv rename to sql/tests/unit/import_data/batch2_dev1.tsv diff --git a/sql/tests/unit/ioql_test.sql b/sql/tests/unit/ioql_test.sql deleted file mode 100644 index abb43b102..000000000 --- a/sql/tests/unit/ioql_test.sql +++ /dev/null @@ -1,71 +0,0 @@ - -DROP FUNCTION IF EXISTS unit_tests.test_ioql_queries_empty(); -CREATE FUNCTION unit_tests.test_ioql_queries_empty() -RETURNS test_result -AS -$$ -DECLARE -message test_result; -diffcount integer; -result json[]; -expected json[]; -expected_row json; -cursor REFCURSOR; -rowvar record; -expected_record record; -result_jsonb jsonb[]; -expected_cursor REFCURSOR; -BEGIN - - PERFORM insert_data_one_partition('test_input_data.batch1_dev1', get_partition_for_key('dev1'::text, 10 :: SMALLINT), 10 :: SMALLINT); - - -- Test json output from query (q1_response_json.csv) - SELECT Array (select * FROM ioql_exec_query(new_ioql_query(namespace_name => '33_testNs'))) into result; - SELECT Array (select * FROM test_outputs.q1_response_json) into expected; - - IF to_jsonb(result) != to_jsonb(expected) THEN - SELECT assert.fail('Bad json return from rows from query.') INTO message; - RETURN message; - END IF; - - -- Test cursor based queries, compare to json table (q1_response_json.csv) - SELECT ioql_exec_query_record_cursor(new_ioql_query(namespace_name => '33_testNs'), 'cursor') into cursor; - - FOREACH expected_row IN ARRAY expected - LOOP - FETCH cursor into rowvar; - IF FOUND = FALSE THEN - raise exception 'Expected row: %v got nothing', to_jsonb(expected_row); - EXIT; - END IF; - - IF to_jsonb(expected_row) != to_jsonb(rowvar) THEN - raise exception 'Expected row: %v got: %v', to_jsonb(expected_row), to_jsonb(rowvar); - END IF; - END LOOP; - - -- Test cursor based queries, compare to records table (q1_response_fields.csv) - - CLOSE cursor; - - SELECT ioql_exec_query_record_cursor(new_ioql_query(namespace_name => '33_testNs'), 'cursor') into cursor; - - FOR expected_record in SELECT * FROM test_outputs.q1_response_fields - LOOP - FETCH cursor into rowvar; - IF FOUND = FALSE THEN - raise exception 'Expected row: %v got nothing', to_jsonb(expected_record); - EXIT; - END IF; - - -- Record comparison fails on different types of columns - IF expected_record != rowvar THEN - raise exception 'Expected row: %v got: %v', to_jsonb(expected_record), to_jsonb(rowvar); - END IF; - END LOOP; - - SELECT assert.ok('End of test.') INTO message; - RETURN message; -END -$$ -LANGUAGE plpgsql; \ No newline at end of file diff --git a/sql/tests/unit/run.sh b/sql/tests/unit/run.sh index b6f8d8fbc..9eb261186 100755 --- a/sql/tests/unit/run.sh +++ b/sql/tests/unit/run.sh @@ -13,30 +13,32 @@ INSTALL_DB=${INSTALL_DB:-Test1} echo "Connecting to $POSTGRES_HOST as user $POSTGRES_USER" NODES="Test1 test2" -SETUPDIR="../../setup" +DB_SETUPDIR="../../setup" if [ $RESET_POSTGRES_DB == "true" ]; then echo "Cleaning up DB" - $SETUPDIR/setup_meta.sh - $SETUPDIR/add_cluster_user.sh postgres + $DB_SETUPDIR/setup_meta.sh + $DB_SETUPDIR/add_cluster_user.sh postgres for node in $NODES; do - $SETUPDIR/setup_node.sh $node - $SETUPDIR/setup_kafka.sh $node - $SETUPDIR/add_node.sh $node $POSTGRES_HOST + $DB_SETUPDIR/setup_node.sh $node + $DB_SETUPDIR/setup_kafka.sh $node + $DB_SETUPDIR/add_node.sh $node $POSTGRES_HOST done - ./add_test_inputs.sh - ./add_test_outputs.sh - + ./setup/add_test_inputs.sh + ./setup/add_test_outputs.sh psql -U $POSTGRES_USER -h $POSTGRES_HOST -v ON_ERROR_STOP=1 -d $INSTALL_DB -f ../../plpgunit/install/1.install-unit-test.sql + + ./setup/add_test_utils.sh + fi if [ "$#" -ne 0 ]; then tests="$@" else - tests=`ls -1 *_test.sql` + tests=`ls -1 tests/*_test.sql` fi for test in $tests; do @@ -44,4 +46,4 @@ for test in $tests; do psql -U $POSTGRES_USER -h $POSTGRES_HOST -d $INSTALL_DB -v ON_ERROR_STOP=1 -f $test done -psql -U $POSTGRES_USER -h $POSTGRES_HOST -d $INSTALL_DB -v ON_ERROR_STOP=1 -f ./start_tests.sql +psql -U $POSTGRES_USER -h $POSTGRES_HOST -d $INSTALL_DB -v ON_ERROR_STOP=1 -f ./setup/start_tests.sql diff --git a/sql/tests/unit/add_test_inputs.sh b/sql/tests/unit/setup/add_test_inputs.sh similarity index 94% rename from sql/tests/unit/add_test_inputs.sh rename to sql/tests/unit/setup/add_test_inputs.sh index efac99aec..62f9a9074 100755 --- a/sql/tests/unit/add_test_inputs.sh +++ b/sql/tests/unit/setup/add_test_inputs.sh @@ -21,7 +21,7 @@ NAMESPACE="33_testNs" psql -U $POSTGRES_USER -h $POSTGRES_HOST -d $INSTALL_DB_META -v ON_ERROR_STOP=1 < '33_testNs', + select_items => ARRAY[new_select_item('nUm_1'::text, + NULL::aggregate_function_type)]), 'test_outputs', 'empty_result'); + + PERFORM test_utils.test_query(new_ioql_query(namespace_name => '33_testNs', + select_items => ARRAY[new_select_item('nUm_1'::text, 'SUM'::aggregate_function_type)], + aggregate => new_aggregate(2592000000000000, NULL)), 'test_outputs', 'empty_result'); + + PERFORM test_utils.test_query(new_ioql_query(namespace_name => '33_testNs', + select_items => ARRAY[new_select_item('nUm_1'::text, 'AVG'::aggregate_function_type)], + aggregate => new_aggregate(2592000000000000, NULL)), 'test_outputs', 'empty_result'); + + PERFORM test_utils.test_query(new_ioql_query(namespace_name => '33_testNs', + select_items => ARRAY[new_select_item('nUm_1'::text, 'MAX'::aggregate_function_type)], + aggregate => new_aggregate(2592000000000000, NULL)), 'test_outputs', 'empty_result'); + + PERFORM test_utils.test_query(new_ioql_query(namespace_name => '33_testNs', + select_items => ARRAY[new_select_item('nUm_1'::text, 'MIN'::aggregate_function_type)], + aggregate => new_aggregate(2592000000000000, NULL)), 'test_outputs', 'empty_result'); + + PERFORM test_utils.test_query(new_ioql_query(namespace_name => '33_testNs', + select_items => ARRAY[new_select_item('nUm_1'::text, 'COUNT'::aggregate_function_type)], + aggregate => new_aggregate(2592000000000000, NULL)), 'test_outputs', 'empty_result'); + + PERFORM test_utils.test_query(new_ioql_query(namespace_name => '33_testNs', + select_items => ARRAY[new_select_item('nUm_1'::text, 'SUM'::aggregate_function_type)], + aggregate => new_aggregate(2592000000000000, 'device_id')), 'test_outputs', 'empty_result'); + + SELECT assert.ok('End of test.') INTO message; + RETURN message; +END +$$ +LANGUAGE plpgsql; diff --git a/sql/tests/unit/kafka_offset_functions_test.sql b/sql/tests/unit/tests/kafka_offset_functions_test.sql similarity index 100% rename from sql/tests/unit/kafka_offset_functions_test.sql rename to sql/tests/unit/tests/kafka_offset_functions_test.sql diff --git a/sql/tests/unit/names_test.sql b/sql/tests/unit/tests/names_test.sql similarity index 97% rename from sql/tests/unit/names_test.sql rename to sql/tests/unit/tests/names_test.sql index 4fed268ad..aa68450ee 100644 --- a/sql/tests/unit/names_test.sql +++ b/sql/tests/unit/tests/names_test.sql @@ -1,5 +1,3 @@ -\ir ../../main/names.sql - DROP FUNCTION IF EXISTS unit_tests.names_tests_start_set(); CREATE FUNCTION unit_tests.names_tests_start_set() RETURNS test_result diff --git a/sql/tests/unit/utils/generate_expected_csv.sh b/sql/tests/unit/utils/generate_expected_csv.sh new file mode 100755 index 000000000..4863f30fa --- /dev/null +++ b/sql/tests/unit/utils/generate_expected_csv.sh @@ -0,0 +1,41 @@ +#!/bin/bash + +# +# NOTE ! This script will not populate the db. You have to make sure the right data is imported before the query +# +set -u +set -e + +PWD=`pwd` +DIR=`dirname $0` + +POSTGRES_HOST=${POSTGRES_HOST:-localhost} +POSTGRES_USER=${POSTGRES_USER:-postgres} +QUERY_DB=${QUERY_DB:-Test1} + +if [ "$#" -ne 2 ] ; then + echo "usage: $0 query outputfile" + echo "ex: $0 \"new_ioql_query(namespace_name => '33_testNs')\" result.csv " + echo "NOTE ! This script will not populate the db. You have to make sure the right data is imported before the query" + exit 1 +fi + +QUERY=$1 +OUTPUTFILE=$2 +TMPFILE="output.tmp" + +echo "Connecting to $POSTGRES_HOST as user $POSTGRES_USER and with db $QUERY_DB" +psql -U $POSTGRES_USER -h $POSTGRES_HOST -d $QUERY_DB -v ON_ERROR_STOP=1 < $OUTPUTFILE +cat $TMPFILE >> $OUTPUTFILE +#rm $TMPFILE + +cd $PWD \ No newline at end of file diff --git a/sql/tests/unit/utils/import_batch.sh b/sql/tests/unit/utils/import_batch.sh new file mode 100755 index 000000000..df979dcbc --- /dev/null +++ b/sql/tests/unit/utils/import_batch.sh @@ -0,0 +1,28 @@ + #!/bin/bash + +# To avoid pw writing, add localhost:5432:*:postgres:test to ~/.pgpass +set -u +set -e + +PWD=`pwd` +DIR=`dirname $0` + +if [ "$#" -ne 2 ] ; then + echo "usage: $0 batch_name partition_value" + echo "ex: $0 test_input_data.batch1_dev1 dev1" + exit 1 +fi + +POSTGRES_HOST=${POSTGRES_HOST:-localhost} +POSTGRES_USER=${POSTGRES_USER:-postgres} +INSTALL_DB_MAIN=${INSTALL_DB_MAIN:-Test1} + +echo "Connecting to $POSTGRES_HOST as user $POSTGRES_USER and with db $INSTALL_DB_MAIN" + +cd $DIR +psql -U $POSTGRES_USER -h $POSTGRES_HOST -d $INSTALL_DB_MAIN -v ON_ERROR_STOP=1 <