q version 3.1.0-beta - automatic immutable caching and direct queries on sqlite databases (#280) + New Packaging Workflowv3.1.0-beta

8 files changed, 5704 insertions, 2961 deletions

diff --git a/test/__init__.py b/test/__init__.py
new file mode 100644
index 0000000..e69de29
--- /dev/null
+++ b/test/__init__.py
diff --git a/test/benchmark-config.sh b/test/benchmark-config.sh
deleted file mode 100644
index 52cf71e..0000000
--- a/test/benchmark-config.sh
+++ /dev/null
@@ -1,3 +0,0 @@
-#!/bin/bash
-
-BENCHMARK_PYTHON_VERSIONS=(2.7.18 3.6.4 3.7.9 3.8.5)
diff --git a/test/prepare-benchmark-env b/test/prepare-benchmark-env
deleted file mode 100755
index 397a290..0000000
--- a/test/prepare-benchmark-env
+++ /dev/null
@@ -1,44 +0,0 @@
-#!/bin/bash
-
-set -e
-
-eval "$(pyenv init -)"
-eval "$(pyenv virtualenv-init -)"
-
-source benchmark-config.sh
-
-if [ ! -f ./benchmark_data.tar.gz ];
-then
-	echo benchmark data not found. downloading it
-  curl "https://s3.amazonaws.com/harelba-q-public/benchmark_data.tar.gz" -o ./benchmark_data.tar.gz
-else
-  echo no need to download benchmark data
-fi
-
-if [ ! -d ./_benchmark_data ];
-then
-	echo extracting benchmark data
-  tar xvfz benchmark_data.tar.gz
-  echo benchmark data is ready
-else
-  echo no need to extract benchmark data
-fi
-
-for ver in "${BENCHMARK_PYTHON_VERSIONS[@]}"
-do
-  echo installing $ver 
-  pyenv install -s $ver
-
-  venv_name=q-benchmark-$ver
-  echo create venv $venv_name
-  pyenv virtualenv -f $ver $venv_name
-  echo activate venv $venv_name
-  pyenv activate $venv_name
-  pyenv version
-  echo installing requirements $venv_name
-  pip install -r ../requirements.txt
-  echo deactivating $venv_name
-  pyenv deactivate    
-done
-
-
diff --git a/test/run-benchmark b/test/run-benchmark
deleted file mode 100755
index a1c6ff2..0000000
--- a/test/run-benchmark
+++ /dev/null
@@ -1,77 +0,0 @@
-#!/bin/bash
-
-# Usage: ./run-benchmark.sh <benchmark-id> <q-executable>
-set -e
-
-get_abs_filename() {
-  # $1 : relative filename
-  echo "$(cd "$(dirname "$1")" && pwd)/$(basename "$1")"
-}
-
-eval "$(pyenv init -)"
-eval "$(pyenv virtualenv-init -)"
-
-if [ "x$1" == "x" ];
-then
-	echo Benchmark id must be provided as a parameter
-  exit 1
-fi
-Q_BENCHMARK_ID=$1
-
-if [ "x$2" == "x" ];
-then
-  EFFECTIVE_Q_EXECUTABLE="source-files-$(git rev-parse HEAD)"
-else
-  ABS_Q_EXECUTABLE="$(get_abs_filename $2)"
-  export Q_EXECUTABLE=$ABS_Q_EXECUTABLE
-	if [ ! -f $ABS_Q_EXECUTABLE ]
-	then
-		echo "q executable must exist ($ABS_Q_EXECUTABLE)"
-		exit 1
-	fi
-  EFFECTIVE_Q_EXECUTABLE="${ABS_Q_EXECUTABLE//\//__}"
-fi
-
-echo "Q executable to use is $EFFECTIVE_Q_EXECUTABLE"
-
-# Must be provided to the benchmark code so it knows where to write the results to
-export Q_BENCHMARK_RESULTS_FOLDER="./benchmark-results/${EFFECTIVE_Q_EXECUTABLE}/${Q_BENCHMARK_ID}/"
-echo Benchmark results folder is $Q_BENCHMARK_RESULTS_FOLDER
-mkdir -p $Q_BENCHMARK_RESULTS_FOLDER
-
-source benchmark-config.sh
-
-ALL_FILES=()
-
-for ver in "${BENCHMARK_PYTHON_VERSIONS[@]}"
-do
-venv_name=q-benchmark-$ver
-echo activating $venv_name
-pyenv activate $venv_name
-echo "==== testing inside $venv_name ==="
-./test-all BenchmarkTests.test_q_matrix -v
-RESULT_FILE="${Q_BENCHMARK_RESULTS_FOLDER}/$venv_name.benchmark-results"
-echo "==== Done. Results are in $RESULT_FILE"
-ALL_FILES[${#ALL_FILES[@]}]="$RESULT_FILE"
-echo "Deactivating"
-pyenv deactivate
-done
-
-echo "==== testing textql ==="
-./test-all BenchmarkTests.test_textql_matrix -v
-RESULT_FILE="textql*.benchmark-results"
-ALL_FILES[${#ALL_FILES[@]}]="${Q_BENCHMARK_RESULTS_FOLDER}/$RESULT_FILE"
-echo "Done. Results are in textql.benchmark-results"
-
-echo "==== testing octosql ==="
-./test-all BenchmarkTests.test_octosql_matrix -v
-RESULT_FILE="octosql*.benchmark-results"
-ALL_FILES[${#ALL_FILES[@]}]="${Q_BENCHMARK_RESULTS_FOLDER}/$RESULT_FILE"
-echo "Done. Results are in octosql.benchmark-results"
-
-summary_file="$Q_BENCHMARK_RESULTS_FOLDER/summary.benchmark-results"
-
-rm -vf $summary_file
-
-paste ${ALL_FILES[*]} > $summary_file
-echo "Done. final results file is $summary_file"
diff --git a/test/test-all b/test/test-all
deleted file mode 100755
index d3dcb02..0000000
--- a/test/test-all
+++ /dev/null
@@ -1,14 +0,0 @@
-#!/bin/bash
-
-set -e
-
-function return_to_original_folder() {
-	popd
-}
-trap return_to_original_folder EXIT
-
-pushd $(dirname $0)/
-
-./test-suite "$@"
-
-set +e
diff --git a/test/test-all.bat b/test/test-all.bat
deleted file mode 100644
index 3192c9e..0000000
--- a/test/test-all.bat
+++ /dev/null
@@ -1,4 +0,0 @@
-@echo off
-
-echo TBD
-
diff --git a/test/test-suite b/test/test-suite
deleted file mode 100755
index 5628e6c..0000000
--- a/test/test-suite
+++ /dev/null
@@ -1,2819 +0,0 @@
-#!/usr/bin/env python
-
-#
-# test suite for q.
-# 
-# Prefer end-to-end tests, running the actual q command and testing stdout/stderr, and the return code.
-# Some utilities are provided for making that easy, see other tests for examples.
-#
-# Don't forget to use the Q_EXECUTABLE instead of hardcoding the q command line. This will be used in the near future
-# in order to test the resulting binary executables as well, instead of just executing the q python source code.
-#
-
-from __future__ import print_function
-import unittest
-import random
-import json
-from json import JSONEncoder
-from subprocess import PIPE, Popen, STDOUT
-import sys
-import os
-import time
-from tempfile import NamedTemporaryFile
-import locale
-import pprint
-import six
-from six.moves import range
-import codecs
-import itertools
-
-sys.path.append(os.path.join(os.path.abspath(os.path.dirname(sys.argv[0])),'..','bin'))
-from q import QTextAsData,QOutput,QOutputPrinter,QInputParams
-
-# q uses this encoding as the default output encoding. Some of the tests use it in order to 
-# make sure that the output is correctly encoded
-SYSTEM_ENCODING = locale.getpreferredencoding()
-
-EXAMPLES = os.path.abspath(os.path.join(os.pardir, 'examples'))
-
-Q_EXECUTABLE = os.getenv('Q_EXECUTABLE', '../bin/q.py')
-
-if not os.path.exists(Q_EXECUTABLE):
-    raise Exception("q executable must reside in {}".format(Q_EXECUTABLE))
-
-DEBUG = False
-if len(sys.argv) > 2 and sys.argv[2] == '-v':
-    DEBUG = True
-
-
-def run_command(cmd_to_run):
-    global DEBUG
-    if DEBUG:
-        print("CMD: {}".format(cmd_to_run))
-
-    p = Popen(cmd_to_run, stdout=PIPE, stderr=PIPE, shell=True)
-    o, e = p.communicate()
-    # remove last newline
-    o = o.rstrip()
-    e = e.strip()
-    # split rows
-    if o != six.b(''):
-        o = o.split(six.b(os.linesep))
-    else:
-        o = []
-    if e != six.b(''):
-        e = e.split(six.b(os.linesep))
-    else:
-        e = []
-
-    res = (p.returncode, o, e)
-    if DEBUG:
-        print("RESULT:{}".format(res))
-    return res
-
-
-uneven_ls_output = six.b("""drwxr-xr-x   2 root     root      4096 Jun 11  2012 /selinux
-drwxr-xr-x   2 root     root      4096 Apr 19  2013 /mnt
-drwxr-xr-x   2 root     root      4096 Apr 24  2013 /srv
-drwx------   2 root     root     16384 Jun 21  2013 /lost+found
-lrwxrwxrwx   1 root     root        33 Jun 21  2013 /initrd.img.old -> /boot/initrd.img-3.8.0-19-generic
-drwxr-xr-x   2 root     root      4096 Jun 21  2013 /cdrom
-drwxr-xr-x   3 root     root      4096 Jun 21  2013 /home
-lrwxrwxrwx   1 root     root        29 Jun 21  2013 /vmlinuz -> boot/vmlinuz-3.8.0-19-generic
-lrwxrwxrwx   1 root     root        32 Jun 21  2013 /initrd.img -> boot/initrd.img-3.8.0-19-generic
-""")
-
-
-find_output = six.b("""8257537   32 drwxrwxrwt 218 root     root        28672 Mar  1 11:00 /tmp
-8299123    4 drwxrwxr-x   2 harel    harel        4096 Feb 27 10:06 /tmp/1628a3fd-b9fe-4dd1-bcdc-7eb869fe7461/supervisor/stormdist/testTopology3fad644a-54c0-4def-b19e-77ca97941595-1-1393513576
-8263229  964 -rw-rw-r--   1 mapred   mapred      984569 Feb 27 10:06 /tmp/1628a3fd-b9fe-4dd1-bcdc-7eb869fe7461/supervisor/stormdist/testTopology3fad644a-54c0-4def-b19e-77ca97941595-1-1393513576/stormcode.ser
-8263230    4 -rw-rw-r--   1 harel    harel        1223 Feb 27 10:06 /tmp/1628a3fd-b9fe-4dd1-bcdc-7eb869fe7461/supervisor/stormdist/testTopology3fad644a-54c0-4def-b19e-77ca97941595-1-1393513576/stormconf.ser
-8299113    4 drwxrwxr-x   2 harel    harel        4096 Feb 27 10:16 /tmp/1628a3fd-b9fe-4dd1-bcdc-7eb869fe7461/supervisor/localstate
-8263406    4 -rw-rw-r--   1 harel    harel        2002 Feb 27 10:16 /tmp/1628a3fd-b9fe-4dd1-bcdc-7eb869fe7461/supervisor/localstate/1393514168746
-8263476    0 -rw-rw-r--   1 harel    harel           0 Feb 27 10:16 /tmp/1628a3fd-b9fe-4dd1-bcdc-7eb869fe7461/supervisor/localstate/1393514168746.version
-8263607    0 -rw-rw-r--   1 harel    harel           0 Feb 27 10:16 /tmp/1628a3fd-b9fe-4dd1-bcdc-7eb869fe7461/supervisor/localstate/1393514169735.version
-8263533    0 -rw-rw-r--   1 harel    harel           0 Feb 27 10:16 /tmp/1628a3fd-b9fe-4dd1-bcdc-7eb869fe7461/supervisor/localstate/1393514172733.version
-8263604    0 -rw-rw-r--   1 harel    harel           0 Feb 27 10:16 /tmp/1628a3fd-b9fe-4dd1-bcdc-7eb869fe7461/supervisor/localstate/1393514175754.version
-""")
-
-
-header_row = six.b('name,value1,value2')
-sample_data_rows = [six.b('a,1,0'), six.b('b,2,0'), six.b('c,,0')]
-sample_data_rows_with_empty_string = [six.b('a,aaa,0'), six.b('b,bbb,0'), six.b('c,,0')]
-sample_data_no_header = six.b("\n").join(sample_data_rows) + six.b("\n")
-sample_data_with_empty_string_no_header = six.b("\n").join(
-    sample_data_rows_with_empty_string) + six.b("\n")
-sample_data_with_header = header_row + six.b("\n") + sample_data_no_header
-sample_data_with_missing_header_names = six.b("name,value1\n") + sample_data_no_header
-
-def generate_sample_data_with_header(header):
-    return header + six.b("\n") + sample_data_no_header
-
-sample_quoted_data = six.b('''non_quoted regular_double_quoted double_double_quoted escaped_double_quoted multiline_double_double_quoted multiline_escaped_double_quoted
-control-value-1 "control-value-2" control-value-3 "control-value-4" control-value-5 "control-value-6"
-non-quoted-value "this is a quoted value" "this is a ""double double"" quoted value" "this is an escaped \\"quoted value\\"" "this is a double double quoted ""multiline
-  value""." "this is an escaped \\"multiline
-  value\\"."
-control-value-1 "control-value-2" control-value-3 "control-value-4" control-value-5 "control-value-6"
-''')
-
-double_double_quoted_data = six.b('''regular_double_quoted double_double_quoted
-"this is a quoted value" "this is a quoted value with ""double double quotes"""
-''')
-
-escaped_double_quoted_data = six.b('''regular_double_quoted escaped_double_quoted
-"this is a quoted value" "this is a quoted value with \\"escaped double quotes\\""
-''')
-
-combined_quoted_data = six.b('''regular_double_quoted double_double_quoted escaped_double_quoted
-"this is a quoted value" "this is a quoted value with ""double double quotes""" "this is a quoted value with \\"escaped double quotes\\""
-''')
-
-sample_quoted_data2 = six.b('"quoted data" 23\nunquoted-data 54')
-
-sample_quoted_data2_with_newline = six.b('"quoted data with\na new line inside it":23\nunquoted-data:54')
-
-one_column_data = six.b('''data without commas 1
-data without commas 2
-''')
-
-# Values with leading whitespace
-sample_data_rows_with_spaces = [six.b('a,1,0'), six.b('   b,   2,0'), six.b('c,,0')]
-sample_data_with_spaces_no_header = six.b("\n").join(
-    sample_data_rows_with_spaces) + six.b("\n")
-
-header_row_with_spaces = six.b('name,value 1,value2')
-sample_data_with_spaces_with_header = header_row_with_spaces + \
-    six.b("\n") + sample_data_with_spaces_no_header
-
-long_value1 = "23683289372328372328373"
-int_value = "2328372328373"
-sample_data_with_long_values = "%s\n%s\n%s" % (long_value1,int_value,int_value)
-
-
-def one_column_warning(e):
-    return e[0].startswith(six.b('Warning: column count is one'))
-
-
-class AbstractQTestCase(unittest.TestCase):
-
-    def create_file_with_data(self, data, encoding=None):
-        if encoding is not None:
-            raise Exception('Deprecated: Encoding must be none')
-        tmpfile = NamedTemporaryFile(delete=False)
-        tmpfile.write(data)
-        tmpfile.close()
-        return tmpfile
-
-    def cleanup(self, tmpfile):
-        global DEBUG
-        if not DEBUG:
-            os.remove(tmpfile.name)
-
-    def random_tmp_filename(self,prefix,postfix):
-        # TODO Use more robust method for this
-        path = '/var/tmp'
-        return '%s/%s-%s.%s' % (path,prefix,random.randint(0,1000000000),postfix)
-
-
-class SaveDbToDiskTests(AbstractQTestCase):
-
-    def test_store_to_disk(self):
-        db_filename = self.random_tmp_filename('store-to-disk','db')
-        self.assertFalse(os.path.exists(db_filename))
-
-        retcode, o, e = run_command('seq 1 1000 | ' + Q_EXECUTABLE + ' "select count(*) from -" -c 1 -S %s' % db_filename)
-
-        self.assertTrue(retcode == 0)
-        self.assertTrue(len(o) == 0)
-        self.assertTrue(len(e) == 5)
-        self.assertTrue(e[0].startswith(six.b('Going to save data')))
-        self.assertTrue(db_filename.encode(sys.stdout.encoding or 'utf-8') in e[0])
-        self.assertTrue(e[1].startswith(six.b('Data has been loaded in')))
-        self.assertTrue(e[2].startswith(six.b('Saving data to db file')))
-        self.assertTrue(e[3].startswith(six.b('Data has been saved into')))
-        self.assertTrue(e[4] == six.b('Query to run on the database: select count(*) from `-`;'))
-
-        self.assertTrue(os.path.exists(db_filename))
-
-        sqlite_command = """echo 'select * from `-`;' | sqlite3 %s""" % db_filename
-        sqlite_retcode,sqlite_o,sqlite_e = run_command(sqlite_command)
-        self.assertTrue(sqlite_retcode == 0)
-        self.assertTrue(len(sqlite_o) == 1000)
-        self.assertTrue(len(sqlite_e) == 0)
-
-        os.remove(db_filename)
-
-    def test_preventing_db_overwrite(self):
-        db_filename = self.random_tmp_filename('store-to-disk', 'db')
-        self.assertFalse(os.path.exists(db_filename))
-
-        retcode, o, e = run_command('seq 1 1000 | ' + Q_EXECUTABLE + ' "select count(*) from -" -c 1 -S %s' % db_filename)
-
-        self.assertTrue(retcode == 0)
-        self.assertTrue(os.path.exists(db_filename))
-
-        retcode2, o2, e2 = run_command('seq 1 1000 | ' + Q_EXECUTABLE + ' "select count(*) from -" -c 1 -S %s' % db_filename)
-        self.assertTrue(retcode2 != 0)
-        self.assertTrue(e2[0].startswith(six.b('Going to save data into a disk database')))
-        self.assertTrue(e2[1] == six.b('Disk database file {} already exists.'.format(db_filename)))
-
-        os.remove(db_filename)
-
-
-class BasicTests(AbstractQTestCase):
-
-    def test_basic_aggregation(self):
-        retcode, o, e = run_command(
-            'seq 1 10 | ' + Q_EXECUTABLE + ' "select sum(c1),avg(c1) from -"')
-        self.assertTrue(retcode == 0)
-        self.assertTrue(len(o) == 1)
-        self.assertTrue(len(e) == 1)
-
-        s = sum(range(1, 11))
-        self.assertTrue(o[0] == six.b('%s %s' % (s, s / 10.0)))
-        self.assertTrue(one_column_warning(e))
-
-    def test_gzipped_file(self):
-        tmpfile = self.create_file_with_data(
-            six.b('\x1f\x8b\x08\x08\xf2\x18\x12S\x00\x03xxxxxx\x003\xe42\xe22\xe62\xe12\xe52\xe32\xe7\xb2\xe0\xb2\xe424\xe0\x02\x00\xeb\xbf\x8a\x13\x15\x00\x00\x00'))
-
-        cmd = Q_EXECUTABLE + ' -z "select sum(c1),avg(c1) from %s"' % tmpfile.name
-
-        retcode, o, e = run_command(cmd)
-        self.assertTrue(retcode == 0)
-        self.assertTrue(len(o) == 1)
-        self.assertTrue(len(e) == 1)
-
-        s = sum(range(1, 11))
-        self.assertTrue(o[0] == six.b('%s %s' % (s, s / 10.0)))
-        self.assertTrue(one_column_warning(e))
-
-        self.cleanup(tmpfile)
-
-    def test_attempt_to_unzip_stdin(self):
-        tmpfile = self.create_file_with_data(
-            six.b('\x1f\x8b\x08\x08\xf2\x18\x12S\x00\x03xxxxxx\x003\xe42\xe22\xe62\xe12\xe52\xe32\xe7\xb2\xe0\xb2\xe424\xe0\x02\x00\xeb\xbf\x8a\x13\x15\x00\x00\x00'))
-
-        cmd = 'cat %s | ' % tmpfile.name + Q_EXECUTABLE + ' -z "select sum(c1),avg(c1) from -"'
-
-        retcode, o, e = run_command(cmd)
-        self.assertTrue(retcode != 0)
-        self.assertTrue(len(o) == 0)
-        self.assertTrue(len(e) == 1)
-
-        self.assertEqual(e[0],six.b('Cannot decompress standard input. Pipe the input through zcat in order to decompress.'))
-
-        self.cleanup(tmpfile)
-
-    def test_delimition_mistake_with_header(self):
-        tmpfile = self.create_file_with_data(sample_data_no_header)
-
-        cmd = Q_EXECUTABLE + ' -d " " "select * from %s" -H' % tmpfile.name
-        retcode, o, e = run_command(cmd)
-
-        self.assertNotEqual(retcode, 0)
-        self.assertEqual(len(o), 0)
-        self.assertEqual(len(e), 3)
-
-        self.assertTrue(e[0].startswith(
-            six.b("Warning: column count is one - did you provide the correct delimiter")))
-        self.assertTrue(e[1].startswith(six.b("Bad header row")))
-        self.assertTrue(six.b("Column name cannot contain commas") in e[2])
-
-        self.cleanup(tmpfile)
-
-    def test_select_one_column(self):
-        tmpfile = self.create_file_with_data(sample_data_no_header)
-
-        cmd = Q_EXECUTABLE + ' -d , "select c1 from %s"' % tmpfile.name
-        retcode, o, e = run_command(cmd)
-
-        self.assertEqual(retcode, 0)
-        self.assertEqual(len(o), 3)
-        self.assertEqual(len(e), 0)
-
-        self.assertEqual(six.b(" ").join(o), six.b('a b c'))
-
-        self.cleanup(tmpfile)
-
-    def test_tab_delimition_parameter(self):
-        tmpfile = self.create_file_with_data(
-            sample_data_no_header.replace(six.b(","), six.b("\t")))
-        cmd = Q_EXECUTABLE + ' -t "select c1,c2,c3 from %s"' % tmpfile.name
-        retcode, o, e = run_command(cmd)
-
-        self.assertEqual(retcode, 0)
-        self.assertEqual(len(o), 3)
-        self.assertEqual(len(e), 0)
-        self.assertEqual(o[0], sample_data_rows[0].replace(six.b(","), six.b("\t")))
-        self.assertEqual(o[1], sample_data_rows[1].replace(six.b(","), six.b("\t")))
-        self.assertEqual(o[2], sample_data_rows[2].replace(six.b(","), six.b("\t")))
-
-        self.cleanup(tmpfile)
-
-    def test_pipe_delimition_parameter(self):
-        tmpfile = self.create_file_with_data(
-            sample_data_no_header.replace(six.b(","), six.b("|")))
-        cmd = Q_EXECUTABLE + ' -p "select c1,c2,c3 from %s"' % tmpfile.name
-        retcode, o, e = run_command(cmd)
-
-        self.assertEqual(retcode, 0)
-        self.assertEqual(len(o), 3)
-        self.assertEqual(len(e), 0)
-        self.assertEqual(o[0], sample_data_rows[0].replace(six.b(","), six.b("|")))
-        self.assertEqual(o[1], sample_data_rows[1].replace(six.b(","), six.b("|")))
-        self.assertEqual(o[2], sample_data_rows[2].replace(six.b(","), six.b("|")))
-
-        self.cleanup(tmpfile)
-
-    def test_tab_delimition_parameter__with_manual_override_attempt(self):
-        tmpfile = self.create_file_with_data(
-            sample_data_no_header.replace(six.b(","), six.b("\t")))
-        cmd = Q_EXECUTABLE + ' -t -d , "select c1,c2,c3 from %s"' % tmpfile.name
-        retcode, o, e = run_command(cmd)
-
-        self.assertEqual(retcode, 0)
-        self.assertEqual(len(o), 3)
-        self.assertEqual(len(e), 1)
-        self.assertEqual(o[0], sample_data_rows[0].replace(six.b(","), six.b("\t")))
-        self.assertEqual(o[1], sample_data_rows[1].replace(six.b(","), six.b("\t")))
-        self.assertEqual(o[2], sample_data_rows[2].replace(six.b(","), six.b("\t")))
-        self.assertEqual(e[0],six.b('Warning: -t parameter overrides -d parameter (,)'))
-
-        self.cleanup(tmpfile)
-
-    def test_pipe_delimition_parameter__with_manual_override_attempt(self):
-        tmpfile = self.create_file_with_data(
-            sample_data_no_header.replace(six.b(","), six.b("|")))
-        cmd = Q_EXECUTABLE + ' -p -d , "select c1,c2,c3 from %s"' % tmpfile.name
-        retcode, o, e = run_command(cmd)
-
-        self.assertEqual(retcode, 0)
-        self.assertEqual(len(o), 3)
-        self.assertEqual(len(e), 1)
-        self.assertEqual(o[0], sample_data_rows[0].replace(six.b(","), six.b("|")))
-        self.assertEqual(o[1], sample_data_rows[1].replace(six.b(","), six.b("|")))
-        self.assertEqual(o[2], sample_data_rows[2].replace(six.b(","), six.b("|")))
-        self.assertEqual(e[0],six.b('Warning: -p parameter overrides -d parameter (,)'))
-
-        self.cleanup(tmpfile)
-
-    def test_output_delimiter(self):
-        tmpfile = self.create_file_with_data(sample_data_no_header)
-        cmd = Q_EXECUTABLE + ' -d , -D "|" "select c1,c2,c3 from %s"' % tmpfile.name
-        retcode, o, e = run_command(cmd)
-
-        self.assertEqual(retcode, 0)
-        self.assertEqual(len(o), 3)
-        self.assertEqual(len(e), 0)
-
-        self.assertEqual(o[0], sample_data_rows[0].replace(six.b(","), six.b("|")))
-        self.assertEqual(o[1], sample_data_rows[1].replace(six.b(","), six.b("|")))
-        self.assertEqual(o[2], sample_data_rows[2].replace(six.b(","), six.b("|")))
-
-        self.cleanup(tmpfile)
-
-    def test_output_delimiter_tab_parameter(self):
-        tmpfile = self.create_file_with_data(sample_data_no_header)
-        cmd = Q_EXECUTABLE + ' -d , -T "select c1,c2,c3 from %s"' % tmpfile.name
-        retcode, o, e = run_command(cmd)
-
-        self.assertEqual(retcode, 0)
-        self.assertEqual(len(o), 3)
-        self.assertEqual(len(e), 0)
-
-        self.assertEqual(o[0], sample_data_rows[0].replace(six.b(","), six.b("\t")))
-        self.assertEqual(o[1], sample_data_rows[1].replace(six.b(","), six.b("\t")))
-        self.assertEqual(o[2], sample_data_rows[2].replace(six.b(","), six.b("\t")))
-
-        self.cleanup(tmpfile)
-
-    def test_output_delimiter_pipe_parameter(self):
-        tmpfile = self.create_file_with_data(sample_data_no_header)
-        cmd = Q_EXECUTABLE + ' -d , -P "select c1,c2,c3 from %s"' % tmpfile.name
-        retcode, o, e = run_command(cmd)
-
-        self.assertEqual(retcode, 0)
-        self.assertEqual(len(o), 3)
-        self.assertEqual(len(e), 0)
-
-        self.assertEqual(o[0], sample_data_rows[0].replace(six.b(","), six.b("|")))
-        self.assertEqual(o[1], sample_data_rows[1].replace(six.b(","), six.b("|")))
-        self.assertEqual(o[2], sample_data_rows[2].replace(six.b(","), six.b("|")))
-
-        self.cleanup(tmpfile)
-
-    def test_output_delimiter_tab_parameter__with_manual_override_attempt(self):
-        tmpfile = self.create_file_with_data(sample_data_no_header)
-        cmd = Q_EXECUTABLE + ' -d , -T -D "|" "select c1,c2,c3 from %s"' % tmpfile.name
-        retcode, o, e = run_command(cmd)
-
-        self.assertEqual(retcode, 0)
-        self.assertEqual(len(o), 3)
-        self.assertEqual(len(e), 1)
-
-        self.assertEqual(o[0], sample_data_rows[0].replace(six.b(","), six.b("\t")))
-        self.assertEqual(o[1], sample_data_rows[1].replace(six.b(","), six.b("\t")))
-        self.assertEqual(o[2], sample_data_rows[2].replace(six.b(","), six.b("\t")))
-        self.assertEqual(e[0], six.b('Warning: -T parameter overrides -D parameter (|)'))
-
-        self.cleanup(tmpfile)
-
-    def test_output_delimiter_pipe_parameter__with_manual_override_attempt(self):
-        tmpfile = self.create_file_with_data(sample_data_no_header)
-        cmd = Q_EXECUTABLE + ' -d , -P -D ":" "select c1,c2,c3 from %s"' % tmpfile.name
-        retcode, o, e = run_command(cmd)
-
-        self.assertEqual(retcode, 0)
-        self.assertEqual(len(o), 3)
-        self.assertEqual(len(e), 1)
-
-        self.assertEqual(o[0], sample_data_rows[0].replace(six.b(","), six.b("|")))
-        self.assertEqual(o[1], sample_data_rows[1].replace(six.b(","), six.b("|")))
-        self.assertEqual(o[2], sample_data_rows[2].replace(six.b(","), six.b("|")))
-        self.assertEqual(e[0],six.b('Warning: -P parameter overrides -D parameter (:)'))
-
-        self.cleanup(tmpfile)
-
-    def test_stdin_input(self):
-        cmd = six.b('printf "%s" | ' + Q_EXECUTABLE + ' -d , "select c1,c2,c3 from -"') % sample_data_no_header
-        retcode, o, e = run_command(cmd)
-
-        self.assertEqual(retcode, 0)
-        self.assertEqual(len(o), 3)
-        self.assertEqual(len(e), 0)
-
-        self.assertEqual(o[0], sample_data_rows[0])
-        self.assertEqual(o[1], sample_data_rows[1])
-        self.assertEqual(o[2], sample_data_rows[2])
-
-    def test_column_separation(self):
-        tmpfile = self.create_file_with_data(sample_data_no_header)
-        cmd = Q_EXECUTABLE + ' -d , "select c1,c2,c3 from %s"' % tmpfile.name
-        retcode, o, e = run_command(cmd)
-
-        self.assertEqual(retcode, 0)
-        self.assertEqual(len(o), 3)
-        self.assertEqual(len(e), 0)
-
-        self.assertEqual(o[0], sample_data_rows[0])
-        self.assertEqual(o[1], sample_data_rows[1])
-        self.assertEqual(o[2], sample_data_rows[2])
-
-        self.cleanup(tmpfile)
-
-    def test_column_analysis(self):
-        tmpfile = self.create_file_with_data(sample_data_no_header)
-
-        cmd = Q_EXECUTABLE + ' -d , "select c1 from %s" -A' % tmpfile.name
-        retcode, o, e = run_command(cmd)
-
-        self.assertEqual(retcode, 0)
-        self.assertEqual(o[0], six.b('Table for file: %s' % tmpfile.name))
-        self.assertEqual(o[1].strip(), six.b('`c1` - text'))
-        self.assertEqual(o[2].strip(), six.b('`c2` - int'))
-        self.assertEqual(o[3].strip(), six.b('`c3` - int'))
-
-        self.cleanup(tmpfile)
-
-    def test_column_analysis_no_header(self):
-        tmpfile = self.create_file_with_data(sample_data_no_header)
-
-        cmd = Q_EXECUTABLE + ' -d , "select c1 from %s" -A' % tmpfile.name
-        retcode, o, e = run_command(cmd)
-
-        self.assertEqual(retcode, 0)
-        self.assertEqual(o[0], six.b('Table for file: %s' % tmpfile.name))
-        self.assertEqual(o[1].strip(), six.b('`c1` - text'))
-        self.assertEqual(o[2].strip(), six.b('`c2` - int'))
-        self.assertEqual(o[3].strip(), six.b('`c3` - int'))
-
-    def test_header_exception_on_numeric_header_data(self):
-        tmpfile = self.create_file_with_data(sample_data_no_header)
-        cmd = Q_EXECUTABLE + ' -d , "select * from %s" -A -H' % tmpfile.name
-        retcode, o, e = run_command(cmd)
-
-        self.assertNotEqual(retcode, 0)
-        self.assertEqual(len(o), 0)
-        self.assertEqual(len(e), 3)
-        self.assertTrue(
-            six.b('Bad header row: Header must contain only strings') in e[0])
-        self.assertTrue(six.b("Column name must be a string") in e[1])
-        self.assertTrue(six.b("Column name must be a string") in e[2])
-
-        self.cleanup(tmpfile)
-
-    def test_column_analysis_with_header(self):
-        tmpfile = self.create_file_with_data(sample_data_with_header)
-        cmd = Q_EXECUTABLE + ' -d , "select c1 from %s" -A -H' % tmpfile.name
-        retcode, o, e = run_command(cmd)
-
-        self.assertNotEqual(retcode, 0)
-        self.assertEqual(len(o),4)
-        self.assertEqual(len(e),2)
-        self.assertEqual(o[0], six.b('Table for file: %s' % tmpfile.name))
-        self.assertEqual(o[1].strip(), six.b('`name` - text'))
-        self.assertEqual(o[2].strip(), six.b('`value1` - int'))
-        self.assertEqual(o[3].strip(), six.b('`value2` - int'))
-        self.assertEqual(e[0].strip(),six.b('query error: no such column: c1'))
-        self.assertTrue(e[1].startswith(six.b('Warning - There seems to be a ')))
-
-        self.cleanup(tmpfile)
-
-    def test_data_with_header(self):
-        tmpfile = self.create_file_with_data(sample_data_with_header)
-        cmd = Q_EXECUTABLE + ' -d , "select name from %s" -H' % tmpfile.name
-        retcode, o, e = run_command(cmd)
-
-        self.assertEqual(retcode, 0)
-        self.assertEqual(len(o), 3)
-        self.assertEqual(six.b(" ").join(o), six.b("a b c"))
-
-        self.cleanup(tmpfile)
-
-    def test_output_header_when_input_header_exists(self):
-        tmpfile = self.create_file_with_data(sample_data_with_header)
-        cmd = Q_EXECUTABLE + ' -d , "select name from %s" -H -O' % tmpfile.name
-        retcode, o, e = run_command(cmd)
-
-        self.assertEqual(retcode, 0)
-        self.assertEqual(len(o), 4)
-        self.assertEqual(o[0],six.b('name'))
-        self.assertEqual(o[1],six.b('a'))
-        self.assertEqual(o[2],six.b('b'))
-        self.assertEqual(o[3],six.b('c'))
-
-        self.cleanup(tmpfile)
-
-    def test_generated_column_name_warning_when_header_line_exists(self):
-        tmpfile = self.create_file_with_data(sample_data_with_header)
-        cmd = Q_EXECUTABLE + ' -d , "select c3 from %s" -H' % tmpfile.name
-
-        retcode, o, e = run_command(cmd)
-
-        self.assertNotEqual(retcode, 0)
-        self.assertEqual(len(o), 0)
-        self.assertEqual(len(e), 2)
-        self.assertTrue(six.b('no such column: c3') in e[0])
-        self.assertTrue(
-            e[1].startswith(six.b('Warning - There seems to be a "no such column" error, and -H (header line) exists. Please make sure that you are using the column names from the header line and not the default (cXX) column names')))
-
-        self.cleanup(tmpfile)
-
-    def test_column_analysis_with_unexpected_header(self):
-        tmpfile = self.create_file_with_data(sample_data_with_header)
-        cmd = Q_EXECUTABLE + ' -d , "select c1 from %s" -A' % tmpfile.name
-        retcode, o, e = run_command(cmd)
-
-        self.assertEqual(retcode, 0)
-        self.assertEqual(len(o), 4)
-        self.assertEqual(len(e), 1)
-
-        self.assertEqual(o[0], six.b('Table for file: %s' % tmpfile.name))
-        self.assertEqual(o[1].strip(), six.b('`c1` - text'))
-        self.assertEqual(o[2].strip(), six.b('`c2` - text'))
-        self.assertEqual(o[3].strip(), six.b('`c3` - text'))
-
-        self.assertEqual(
-            e[0], six.b('Warning - There seems to be header line in the file, but -H has not been specified. All fields will be detected as text fields, and the header line will appear as part of the data'))
-
-        self.cleanup(tmpfile)
-
-    def test_empty_data(self):
-        tmpfile = self.create_file_with_data(six.b(''))
-        cmd = Q_EXECUTABLE + ' -d , "select c1 from %s"' % tmpfile.name
-        retcode, o, e = run_command(cmd)
-
-        self.assertEqual(retcode, 0)
-        self.assertEqual(len(o), 0)
-        self.assertEqual(len(e), 1)
-
-        self.assertTrue(six.b('Warning - data is empty') in e[0])
-
-        self.cleanup(tmpfile)
-
-    def test_empty_data_with_header_param(self):
-        tmpfile = self.create_file_with_data(six.b(''))
-        cmd = Q_EXECUTABLE + ' -d , "select c1 from %s" -H' % tmpfile.name
-        retcode, o, e = run_command(cmd)
-
-        self.assertNotEqual(retcode, 0)
-        self.assertEqual(len(o), 0)
-        self.assertEqual(len(e), 1)
-
-        m = six.b("Header line is expected but missing in file %s" % tmpfile.name)
-        self.assertTrue(m in e[0])
-
-        self.cleanup(tmpfile)
-
-    def test_one_row_of_data_without_header_param(self):
-        tmpfile = self.create_file_with_data(header_row)
-        cmd = Q_EXECUTABLE + ' -d , "select c2 from %s"' % tmpfile.name
-        retcode, o, e = run_command(cmd)
-
-        self.assertEqual(retcode, 0)
-        self.assertEqual(len(o), 1)
-        self.assertEqual(len(e), 0)
-
-        self.assertEqual(o[0], six.b('value1'))
-
-        self.cleanup(tmpfile)
-
-    def test_one_row_of_data_with_header_param(self):
-        tmpfile = self.create_file_with_data(header_row)
-        cmd = Q_EXECUTABLE + ' -d , "select c2 from %s" -H' % tmpfile.name
-        retcode, o, e = run_command(cmd)
-
-        self.assertEqual(retcode, 0)
-        self.assertEqual(len(o), 0)
-        self.assertEqual(len(e), 1)
-
-        self.assertTrue(six.b('Warning - data is empty') in e[0])
-
-        self.cleanup(tmpfile)
-
-    def test_dont_leading_keep_whitespace_in_values(self):
-        tmpfile = self.create_file_with_data(sample_data_with_spaces_no_header)
-        cmd = Q_EXECUTABLE + ' -d , "select c1 from %s"' % tmpfile.name
-        retcode, o, e = run_command(cmd)
-
-        self.assertEqual(retcode, 0)
-        self.assertEqual(len(e), 0)
-        self.assertEqual(len(o), 3)
-
-        self.assertEqual(o[0], six.b('a'))
-        self.assertEqual(o[1], six.b('b'))
-        self.assertEqual(o[2], six.b('c'))
-
-        self.cleanup(tmpfile)
-
-    def test_keep_leading_whitespace_in_values(self):
-        tmpfile = self.create_file_with_data(sample_data_with_spaces_no_header)
-        cmd = Q_EXECUTABLE + ' -d , "select c1 from %s" -k' % tmpfile.name
-        retcode, o, e = run_command(cmd)
-
-        self.assertEqual(retcode, 0)
-        self.assertEqual(len(e), 0)
-        self.assertEqual(len(o), 3)
-
-        self.assertEqual(o[0], six.b('a'))
-        self.assertEqual(o[1], six.b('   b'))
-        self.assertEqual(o[2], six.b('c'))
-
-        self.cleanup(tmpfile)
-
-    def test_no_impact_of_keeping_leading_whitespace_on_integers(self):
-        tmpfile = self.create_file_with_data(sample_data_with_spaces_no_header)
-        cmd = Q_EXECUTABLE + ' -d , "select c2 from %s" -k -A' % tmpfile.name
-        retcode, o, e = run_command(cmd)
-
-        self.assertEqual(retcode, 0)
-        self.assertEqual(len(e), 0)
-        self.assertEqual(len(o), 4)
-
-        self.assertEqual(o[0], six.b('Table for file: %s' % tmpfile.name))
-        self.assertEqual(o[1].strip(), six.b('`c1` - text'))
-        self.assertEqual(o[2].strip(), six.b('`c2` - int'))
-        self.assertEqual(o[3].strip(), six.b('`c3` - int'))
-
-        self.cleanup(tmpfile)
-
-    def test_spaces_in_header_row(self):
-        tmpfile = self.create_file_with_data(
-            header_row_with_spaces + six.b("\n") + sample_data_no_header)
-        cmd = Q_EXECUTABLE + ' -d , "select name,\`value 1\` from %s" -H' % tmpfile.name
-        retcode, o, e = run_command(cmd)
-
-        self.assertEqual(retcode, 0)
-        self.assertEqual(len(e), 0)
-        self.assertEqual(len(o), 3)
-
-        self.assertEqual(o[0], six.b('a,1'))
-        self.assertEqual(o[1], six.b('b,2'))
-        self.assertEqual(o[2], six.b('c,'))
-
-        self.cleanup(tmpfile)
-
-    def test_column_analysis_for_spaces_in_header_row(self):
-        tmpfile = self.create_file_with_data(
-            header_row_with_spaces + six.b("\n") + sample_data_no_header)
-        cmd = Q_EXECUTABLE + ' -d , "select name,\`value 1\` from %s" -H -A' % tmpfile.name
-        retcode, o, e = run_command(cmd)
-
-        self.assertEqual(retcode, 0)
-        self.assertEqual(len(e), 0)
-        self.assertEqual(len(o), 4)
-
-        self.assertEqual(o[0], six.b('Table for file: %s' % tmpfile.name))
-        self.assertEqual(o[1].strip(), six.b('`name` - text'))
-        self.assertEqual(o[2].strip(), six.b('`value 1` - int'))
-        self.assertEqual(