add script to generate csv

sql/tests/unit/expected_outputs/empty_result.csv

@@ -0,0 +1 @@
+time bigint; bool_1 boolean; device_id text; field_only_dev2 double precision; field_only_ref2 text; nUm_1 double precision; num_2 double precision; string_1 text; string_2 text

sql/tests/unit/expected_outputs/q1_response_fields.csv

@@ -1,6 +1,6 @@
-time bigint; bool_1 boolean; device_id text; field_only_dev2 double precision; field_only_ref2 text; num_1 double precision; num_2 double precision; string_1 text; string_2 text
+time bigint; bool_1 boolean; device_id text; field_only_dev2 double precision; field_only_ref2 text; nUm_1 double precision; num_2 double precision; string_1 text; string_2 text
-1257897600000000000;false;dev1;null;null;null;5;const;five
+1257897600000000000;f;dev1;null;null;1.5;5;const;five
-1257894001000000000;true;dev1;null;null;null;4;const;four
+1257894001000000000;t;dev1;null;null;1.5;4;const;four
-1257894000000001000;true;dev1;null;null;null;3;const;three
+1257894000000001000;t;dev1;null;null;1.5;3;const;three
-1257894000000000000;true;dev1;null;null;null;1;const;one
+1257894000000000000;t;dev1;null;null;1.5;1;const;one
-1257894000000000000;false;dev1;null;null;null;2;const;two
+1257894000000000000;f;dev1;null;null;1.5;2;const;two

sql/tests/unit/ioql_test.sql

@@ -1,71 +0,0 @@
-
-DROP FUNCTION IF EXISTS unit_tests.test_ioql_queries_empty();
-CREATE FUNCTION unit_tests.test_ioql_queries_empty()
-RETURNS test_result
-AS
-$$
-DECLARE
-message test_result;
-diffcount integer;
-result json[];
-expected json[];
-expected_row json;
-cursor REFCURSOR;
-rowvar record;
-expected_record record;
-result_jsonb jsonb[];
-expected_cursor REFCURSOR;
-BEGIN
-
-    PERFORM insert_data_one_partition('test_input_data.batch1_dev1', get_partition_for_key('dev1'::text, 10 :: SMALLINT), 10 :: SMALLINT);
-
-    -- Test json output from query (q1_response_json.csv)
-    SELECT Array (select * FROM ioql_exec_query(new_ioql_query(namespace_name => '33_testNs'))) into result;
-    SELECT Array (select * FROM test_outputs.q1_response_json) into expected;
-
-    IF to_jsonb(result) != to_jsonb(expected) THEN
-		SELECT assert.fail('Bad json return from rows from query.') INTO message;
-		RETURN message;
-	END IF;
-    
-    -- Test cursor based queries, compare to json table (q1_response_json.csv)
-    SELECT ioql_exec_query_record_cursor(new_ioql_query(namespace_name => '33_testNs'), 'cursor') into cursor;
-
-    FOREACH expected_row IN ARRAY expected
-    LOOP
-        FETCH cursor into rowvar;
-        IF FOUND = FALSE THEN 
-            raise exception 'Expected row: %v got nothing', to_jsonb(expected_row);
-            EXIT;
-        END IF;
-        
-        IF to_jsonb(expected_row) != to_jsonb(rowvar) THEN
-            raise exception 'Expected row: %v got: %v', to_jsonb(expected_row), to_jsonb(rowvar);
-        END IF;
-    END LOOP;
-
-    -- Test cursor based queries, compare to records table (q1_response_fields.csv)
-
-    CLOSE cursor;
-
-    SELECT ioql_exec_query_record_cursor(new_ioql_query(namespace_name => '33_testNs'), 'cursor') into cursor;
-
-    FOR expected_record in SELECT * FROM test_outputs.q1_response_fields 
-    LOOP
-        FETCH cursor into rowvar;
-        IF FOUND = FALSE THEN 
-            raise exception 'Expected row: %v got nothing', to_jsonb(expected_record);
-            EXIT;
-        END IF;
-
-        -- Record comparison fails on different types of columns
-        IF expected_record != rowvar THEN
-            raise exception 'Expected row: %v got: %v', to_jsonb(expected_record), to_jsonb(rowvar);
-        END IF;
-    END LOOP;
-
-    SELECT assert.ok('End of test.') INTO message;
-	RETURN message;
-END
-$$
-LANGUAGE plpgsql;

sql/tests/unit/run.sh

@@ -13,30 +13,32 @@ INSTALL_DB=${INSTALL_DB:-Test1}
 echo "Connecting to $POSTGRES_HOST as user $POSTGRES_USER"
 
 NODES="Test1 test2"
-SETUPDIR="../../setup"
+DB_SETUPDIR="../../setup"
 
 if [ $RESET_POSTGRES_DB == "true" ]; then
     echo "Cleaning up DB"
 
-    $SETUPDIR/setup_meta.sh
+    $DB_SETUPDIR/setup_meta.sh
-    $SETUPDIR/add_cluster_user.sh postgres
+    $DB_SETUPDIR/add_cluster_user.sh postgres
 
     for node in $NODES; do
-        $SETUPDIR/setup_node.sh $node
+        $DB_SETUPDIR/setup_node.sh $node
-        $SETUPDIR/setup_kafka.sh $node
+        $DB_SETUPDIR/setup_kafka.sh $node
-        $SETUPDIR/add_node.sh $node $POSTGRES_HOST
+        $DB_SETUPDIR/add_node.sh $node $POSTGRES_HOST
     done
 
-    ./add_test_inputs.sh
+    ./setup/add_test_inputs.sh
-    ./add_test_outputs.sh
+    ./setup/add_test_outputs.sh
-
     psql -U $POSTGRES_USER -h $POSTGRES_HOST -v ON_ERROR_STOP=1 -d $INSTALL_DB -f ../../plpgunit/install/1.install-unit-test.sql
+
+    ./setup/add_test_utils.sh
+
 fi
 
 if [ "$#" -ne 0 ]; then
     tests="$@"
 else
-    tests=`ls -1 *_test.sql`
+    tests=`ls -1 tests/*_test.sql`
 fi
 
 for test in $tests; do
@@ -44,4 +46,4 @@ for test in $tests; do
     psql -U $POSTGRES_USER -h $POSTGRES_HOST -d $INSTALL_DB -v ON_ERROR_STOP=1 -f $test
 done
 
-psql -U $POSTGRES_USER -h $POSTGRES_HOST -d $INSTALL_DB -v ON_ERROR_STOP=1 -f ./start_tests.sql
+psql -U $POSTGRES_USER -h $POSTGRES_HOST -d $INSTALL_DB -v ON_ERROR_STOP=1 -f ./setup/start_tests.sql

sql/tests/unit/setup/add_test_inputs.sh

@@ -21,7 +21,7 @@ NAMESPACE="33_testNs"
 psql -U $POSTGRES_USER -h $POSTGRES_HOST -d $INSTALL_DB_META -v ON_ERROR_STOP=1  <<EOF
     SELECT add_namespace('$NAMESPACE' :: NAME);
     SELECT add_field('$NAMESPACE' :: NAME, 'device_id', 'text', TRUE, TRUE, ARRAY ['VALUE-TIME'] :: field_index_type []);
-    SELECT add_field('$NAMESPACE' :: NAME, 'num_1', 'double precision', FALSE, FALSE, ARRAY ['VALUE-TIME'] :: field_index_type []);
+    SELECT add_field('$NAMESPACE' :: NAME, 'nUm_1', 'double precision', FALSE, FALSE, ARRAY ['VALUE-TIME'] :: field_index_type []);
     SELECT add_field('$NAMESPACE' :: NAME, 'num_2', 'double precision', FALSE, FALSE, ARRAY ['VALUE-TIME'] :: field_index_type []);
     SELECT add_field('$NAMESPACE' :: NAME, 'bool_1', 'boolean', FALSE, FALSE, ARRAY ['VALUE-TIME'] :: field_index_type []);
     SELECT add_field('$NAMESPACE' :: NAME, 'string_1', 'text', FALSE, FALSE, ARRAY ['VALUE-TIME'] :: field_index_type []);
@@ -30,7 +30,7 @@ psql -U $POSTGRES_USER -h $POSTGRES_HOST -d $INSTALL_DB_META -v ON_ERROR_STOP=1
     SELECT add_field('$NAMESPACE' :: NAME, 'field_only_dev2', 'double precision', FALSE, FALSE, ARRAY ['VALUE-TIME'] :: field_index_type []);
 EOF
 
-INPUT_DATA_DIR="input_data" 
+INPUT_DATA_DIR="../import_data" 
 FILE_SUFFIX=".tsv"
 DATASETS=`ls $INPUT_DATA_DIR/*$FILE_SUFFIX`
 TEMPTABLENAME="copy_t"
@@ -46,8 +46,8 @@ psql -U $POSTGRES_USER -h $POSTGRES_HOST -d $INSTALL_DB_MAIN -v ON_ERROR_STOP=1
         SELECT *
         FROM create_temp_copy_table_one_partition('copy_t'::text, get_partition_for_key('$PARTITION_KEY'::text, 10 :: SMALLINT), 10 :: SMALLINT);
         \COPY $TEMPTABLENAME FROM '$DS_PATH';
-        DROP TABLE IF EXISTS $DATASET;
         CREATE SCHEMA IF NOT EXISTS test_input_data;
+        DROP TABLE IF EXISTS test_input_data.$DATASET;
         CREATE TABLE test_input_data.$DATASET AS SELECT * FROM $TEMPTABLENAME; 
         COMMIT; 
 EOF

sql/tests/unit/setup/add_test_outputs.sh

@@ -16,7 +16,7 @@ echo "Connecting to $POSTGRES_HOST as user $POSTGRES_USER and with meta db $INST
 
 cd $DIR
 
-OUTPUT_DATA_DIR="expected_outputs"
+OUTPUT_DATA_DIR="../expected_outputs"
 FILE_SUFFIX="csv"
 DATAFILES=`ls $OUTPUT_DATA_DIR/*.$FILE_SUFFIX`
 SCHEMA_NAME="test_outputs"

sql/tests/unit/setup/add_test_utils.sh

@@ -0,0 +1,17 @@
+#!/bin/bash
+set -u
+set -e
+
+PWD=`pwd`
+DIR=`dirname $0`
+
+POSTGRES_HOST=${POSTGRES_HOST:-localhost}
+POSTGRES_USER=${POSTGRES_USER:-postgres}
+INSTALL_DB=${INSTALL_DB:-Test1}
+
+echo "Connecting to $POSTGRES_HOST as user $POSTGRES_USER and with db $INSTALL_DB"
+
+cd $DIR
+psql -U $POSTGRES_USER -h $POSTGRES_HOST -d $INSTALL_DB -v ON_ERROR_STOP=1 -f test_utils.sql
+
+cd $PWD

sql/tests/unit/setup/test_utils.sql

@@ -0,0 +1,47 @@
+CREATE SCHEMA IF NOT EXISTS test_utils;
+
+CREATE OR REPLACE FUNCTION test_utils.test_query(query ioql_query, expected_table_schema text, expected_table text)
+RETURNS void
+AS 
+$$
+DECLARE
+cursor REFCURSOR;
+expected_record record;
+returned_record record;
+BEGIN
+    SELECT ioql_exec_query_record_cursor(query, 'cursor') into cursor;
+
+    FOR expected_record in EXECUTE format('SELECT * FROM %I.%I', expected_table_schema, expected_table)
+    LOOP    
+        FETCH cursor INTO returned_record;
+        IF FOUND = FALSE THEN 
+            raise exception 'Expected row: %v got nothing', to_jsonb(expected_record);
+            EXIT;
+        END IF;
+
+        -- Record comparison fails on different types of columns
+        IF expected_record != returned_record THEN
+            raise exception 'Expected row: %v got: %v', to_jsonb(expected_record), to_jsonb(returned_record);
+            EXIT;
+        END IF;
+    END LOOP;
+
+    CLOSE cursor;
+END
+$$
+LANGUAGE plpgsql;
+
+CREATE OR REPLACE FUNCTION test_utils.query_to_table(query ioql_query, output_table text)
+RETURNS boolean
+AS 
+$$
+DECLARE
+prepared_query text;
+BEGIN
+    SELECT ioql_exec_query_record_sql(query) into prepared_query;
+    EXECUTE format('DROP TABLE IF EXISTS %I', output_table);
+    EXECUTE format('CREATE TABLE %I AS %s WITH DATA', output_table, prepared_query); 
+    RETURN TRUE;
+END
+$$
+LANGUAGE plpgsql;

sql/tests/unit/tests/ioql_empty_test.sql

@@ -0,0 +1,42 @@
+CREATE OR REPLACE FUNCTION unit_tests.test_ioql_queries_empty()
+RETURNS test_result
+AS
+$$
+DECLARE
+message test_result;
+success boolean;
+BEGIN
+    
+	PERFORM test_utils.test_query(new_ioql_query(namespace_name => '33_testNs', 
+	select_items => ARRAY[new_select_item('nUm_1'::text, 
+	NULL::aggregate_function_type)]), 'test_outputs', 'empty_result');
+	
+	PERFORM test_utils.test_query(new_ioql_query(namespace_name => '33_testNs', 
+	select_items => ARRAY[new_select_item('nUm_1'::text, 'SUM'::aggregate_function_type)], 
+	aggregate => new_aggregate(2592000000000000, NULL)), 'test_outputs', 'empty_result');
+
+	PERFORM test_utils.test_query(new_ioql_query(namespace_name => '33_testNs', 
+	select_items => ARRAY[new_select_item('nUm_1'::text, 'AVG'::aggregate_function_type)], 
+	aggregate => new_aggregate(2592000000000000, NULL)), 'test_outputs', 'empty_result');
+
+	PERFORM test_utils.test_query(new_ioql_query(namespace_name => '33_testNs', 
+	select_items => ARRAY[new_select_item('nUm_1'::text, 'MAX'::aggregate_function_type)], 
+	aggregate => new_aggregate(2592000000000000, NULL)), 'test_outputs', 'empty_result');
+
+	PERFORM test_utils.test_query(new_ioql_query(namespace_name => '33_testNs', 
+	select_items => ARRAY[new_select_item('nUm_1'::text, 'MIN'::aggregate_function_type)], 
+	aggregate => new_aggregate(2592000000000000, NULL)), 'test_outputs', 'empty_result');
+	
+	PERFORM test_utils.test_query(new_ioql_query(namespace_name => '33_testNs', 
+	select_items => ARRAY[new_select_item('nUm_1'::text, 'COUNT'::aggregate_function_type)], 
+	aggregate => new_aggregate(2592000000000000, NULL)), 'test_outputs', 'empty_result');
+	
+	PERFORM test_utils.test_query(new_ioql_query(namespace_name => '33_testNs', 
+	select_items => ARRAY[new_select_item('nUm_1'::text, 'SUM'::aggregate_function_type)], 
+	aggregate => new_aggregate(2592000000000000, 'device_id')), 'test_outputs', 'empty_result');	
+
+	SELECT assert.ok('End of test.') INTO message;
+	RETURN message;
+END
+$$
+LANGUAGE plpgsql;

sql/tests/unit/tests/names_test.sql

@@ -1,5 +1,3 @@
-\ir ../../main/names.sql
-
 DROP FUNCTION IF EXISTS unit_tests.names_tests_start_set();
 CREATE FUNCTION unit_tests.names_tests_start_set()
 RETURNS test_result

sql/tests/unit/utils/generate_expected_csv.sh

@@ -0,0 +1,41 @@
+#!/bin/bash
+
+#
+# NOTE ! This script will not populate the db. You have to make sure the right data is imported before the query
+#
+set -u
+set -e
+
+PWD=`pwd`
+DIR=`dirname $0`
+
+POSTGRES_HOST=${POSTGRES_HOST:-localhost}
+POSTGRES_USER=${POSTGRES_USER:-postgres}
+QUERY_DB=${QUERY_DB:-Test1}
+
+if [ "$#" -ne 2 ] ; then
+    echo "usage: $0 query outputfile"
+    echo "ex: $0 \"new_ioql_query(namespace_name => '33_testNs')\" result.csv "
+    echo "NOTE ! This script will not populate the db. You have to make sure the right data is imported before the query"
+    exit 1
+fi
+
+QUERY=$1
+OUTPUTFILE=$2
+TMPFILE="output.tmp"
+
+echo "Connecting to $POSTGRES_HOST as user $POSTGRES_USER and with db $QUERY_DB"
+psql -U $POSTGRES_USER -h $POSTGRES_HOST -d $QUERY_DB -v ON_ERROR_STOP=1  <<EOF
+CREATE SCHEMA IF NOT EXISTS tmp;
+DROP TABLE IF EXISTS tmptable;
+select test_utils.query_to_table($QUERY, 'tmptable');
+\copy (Select * From tmptable) To '$TMPFILE' With CSV DELIMITER ';' NULL 'null';
+EOF
+echo "select test_utils.query_to_table($QUERY, 'tmptable');"
+psql -U $POSTGRES_USER -h $POSTGRES_HOST -d $QUERY_DB -v \
+ON_ERROR_STOP=1 -c "\d tmptable" | tail -n +4 | sed "s/|//;s/|/;/;s/  */ /g;s/ ; /;/g" \
+| tr -d '\n' | sed "s/.$//;s/^ //" > $OUTPUTFILE
+cat $TMPFILE >> $OUTPUTFILE
+#rm $TMPFILE
+
+cd $PWD

sql/tests/unit/utils/import_batch.sh

@@ -0,0 +1,28 @@
+ #!/bin/bash
+
+# To avoid pw writing, add localhost:5432:*:postgres:test to ~/.pgpass
+set -u
+set -e
+
+PWD=`pwd`
+DIR=`dirname $0`
+
+if [ "$#" -ne 2 ] ; then
+    echo "usage: $0 batch_name partition_value"
+    echo "ex: $0 test_input_data.batch1_dev1 dev1"
+    exit 1
+fi
+
+POSTGRES_HOST=${POSTGRES_HOST:-localhost}
+POSTGRES_USER=${POSTGRES_USER:-postgres}
+INSTALL_DB_MAIN=${INSTALL_DB_MAIN:-Test1}
+
+echo "Connecting to $POSTGRES_HOST as user $POSTGRES_USER and with db $INSTALL_DB_MAIN"
+
+cd $DIR
+psql -U $POSTGRES_USER -h $POSTGRES_HOST -d $INSTALL_DB_MAIN -v ON_ERROR_STOP=1  <<EOF
+    select insert_data_one_partition('$1', get_partition_for_key('$2'::text, 10 :: SMALLINT), 10 :: SMALLINT);
+EOF
+cd $PWD
+ 
+