wip

9 files changed, 512 insertions, 233 deletions

diff --git a/.travis.yml b/.travis.yml
index 402d9e1..e8186f1 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -1,14 +1,137 @@
-language: python
-python:
-  - "2.7"
-  - "3.6"
-matrix:
+sudo: false
+
+stages:
+  - integration
+  - release
+
+env:
+  global:
+    - CACHE_NAME=${TRAVIS_JOB_NAME}
+
+
+_commands_provider:
+
+  _test: &_test make test
+
+  _lint: &_lint make lint
+
+  _release: &_release make local-release
+
+  _install_requirements: &_install_requirements make dep
+
+  # https://ttcshelbyville.wordpress.com/2012/12/19/disable-remote-differential-compression-form-the-command-line/
+  _disable_windows_compression: &_disable_windows_compression "powershell Disable-WindowsOptionalFeature -Online -FeatureName MSRDC-Infrastructure"
+
+  # https://travis-ci.community/t/yarn-network-troubles/333/7
+  _disable_windows_defender: &_disable_windows_defender "powershell Set-MpPreference -DisableRealtimeMonitoring \\$true"
+
+
+_steps_provider:
+
+  _test: &_step_test
+
+    install:
+      - *_install_requirements
+    before_script: *_lint
+    script: *_test
+
+  _release: &_step_release
+
+    install: *_install_requirements
+    script: *_release
+
+
+
+jobs:
   include:
-    - python: "3.7"
-      dist: xenial  # Need for python 3.7
-  allow_failures:
-    - python: "3.6"
-    - python: "3.7"
-install: pip install -r requirements.txt
-before_script: flake8 ./bin/q ./test/test-suite --count --select=E901,E999,F821,F822,F823 --show-source --statistics
-script: test/test-all
+
+    - stage: integration
+      name: py27-macos
+      os: osx
+      language: generic
+      osx_image: xcode7.3
+      env:
+        - PYENV_VERSION=2.7.14
+      before_install: source setup-pyenv.sh
+      <<: *_step_test
+      cache:
+        directories:
+          - ${HOME}/.pyenv_cache
+
+    - stage: integration
+      name: py36-macos
+      os: osx
+      language: generic
+      osx_image: xcode7.3
+      env:
+        - PYENV_VERSION=3.6.4
+      before_install: source setup-pyenv.sh
+      <<: *_step_test
+      cache:
+        directories:
+          - ${HOME}/.pyenv_cache
+
+    - stage: integration
+      name: py37-macos
+      os: osx
+      language: generic
+      osx_image: xcode7.3
+      env:
+        - PYENV_VERSION=3.7.3
+      before_install: source setup-pyenv.sh
+      <<: *_step_test
+      cache:
+        directories:
+          - ${HOME}/.pyenv_cache
+
+    - stage: integration
+      name: py27-linux
+      language: python
+      python: "2.7"
+      <<: *_step_test
+
+    - stage: integration
+      name: py36-linux
+      language: python
+      python: "3.6"
+      <<: *_step_test
+
+    - stage: integration
+      name: py37-linux
+      language: python
+      dist: xenial
+      python: "3.7"
+      <<: *_step_test
+
+    - stage: release
+      name: macos
+      os: osx
+      language: generic
+      osx_image: xcode7.3
+      env:
+        - PYENV_VERSION=3.7.3
+      before_install: source setup-pyenv.sh
+      <<: *_step_release
+      cache:
+        directories:
+          - ${HOME}/.pyenv_cache
+
+    - stage: release
+      name: linux
+      language: python
+      dist: xenial
+      python: "3.7"
+      <<: *_step_release
+
+    - stage: release
+      name: windows
+      os: windows
+      language: shell
+      env:
+        - PATH=/c/Python37:/c/Python37/Scripts:$PATH
+      before_install:
+        - *_disable_windows_compression
+        - *_disable_windows_defender
+        - choco install make
+        - choco install python --version 3.7.3
+      <<: *_step_release
diff --git a/Makefile b/Makefile
new file mode 100644
index 0000000..1c7b971
--- /dev/null
+++ b/Makefile
@@ -0,0 +1,35 @@
+SHELL := /bin/bash
+
+PROJECT_NAME=$(shell dirname "$0")
+ROOT_DIR:=$(shell dirname $(realpath $(lastword $(MAKEFILE_LIST))))
+
+.PHONY: test help
+.DEFAULT_GOAL := ci
+
+ci: lint test ## Equivelant to 'make lint test'
+
+help: ## Show this help message.
+
+	@grep -E '^[a-zA-Z_-]+:.*?## .*$$' $(MAKEFILE_LIST) | awk 'BEGIN {FS = ":.*?## "}; {printf "\033[36m%-30s\033[0m %s\n", $$1, $$2}'
+
+dep: ## Install the dependent libraries.
+
+	pip install -r test-requirements.txt
+	pip install -e .
+
+lint: dep ## Run lint validations.
+
+	flake8 q/ --count --select=E901,E999,F821,F822,F823 --show-source --statistics
+
+test: dep ## Run the unit tests.
+
+	py.test -rs -c pytest.ini -s -v q/tests/suite.py --rootdir . 
+
+release: ## Run release
+	pip install py-ci
+	pyci release --no-wheel-publish --wheel-universal
+
+local-release:
+	pip install py-ci
+	./do-manual-release.sh
+	
diff --git a/bin/q b/bin/q
index 56c6fe0..a4004dc 100755
--- a/bin/q
+++ b/bin/q
@@ -31,7 +31,7 @@ from __future__ import absolute_import
 from __future__ import division
 from __future__ import print_function
 
-q_version = "2.0.6"
+q_version = "2.0.9"
 
 __all__ = [ 'QTextAsData' ]
 
@@ -1143,7 +1143,8 @@ def determine_max_col_lengths(m,output_field_quoting_func,output_delimiter):
 
 def print_credentials():
     print("q version %s" % q_version, file=sys.stderr)
-    print("Copyright (C) 2012-2017 Harel Ben-Attia (harelba@gmail.com, @harelba on twitter)", file=sys.stderr)
+    print("Python: %s" % (sys.version.replace("\n","//")), file=sys.stderr)
+    print("Copyright (C) 2012-2019 Harel Ben-Attia (harelba@gmail.com, @harelba on twitter)", file=sys.stderr)
     print("http://harelba.github.io/q/", file=sys.stderr)
     print(file=sys.stderr)
 
diff --git a/do-manual-release.sh b/do-manual-release.sh
new file mode 100755
index 0000000..56b302c
--- /dev/null
+++ b/do-manual-release.sh
@@ -0,0 +1,29 @@
+#!/bin/bash -x
+
+set -e
+
+VERSION=2.0.9
+
+echo "Packing binary for $TRAVIS_OS_NAME"
+
+if [[ "$TRAVIS_OS_NAME" == "osx" || "$TRAVIS_OS_NAME" == "linux" ]]
+then
+	echo "Packing $TRAVIS_OS_NAME installer - packing binary"
+	pyci pack binary
+	echo "Packing $TRAVIS_OS_NAME installer - uploading"
+	pyci github upload-asset --asset q-$(uname -m)-$(uname -s) --release $VERSION
+else
+	echo "Packing windows installer - packing binary"
+	pyci pack binary
+	echo "Packing windows installer - listing files"
+	find `pwd` -ls | grep -v \.git/
+	echo "Packing windows installer - packing nsis"
+	BINARY_LOCATION="c:\\Users\\travis\\build\\harelba\\q\\q-AMD64-Windows.exe"
+	pyci pack nsis --program-files-dir q-TextAsData --binary-path $BINARY_LOCATION --version ${VERSION}.0
+	echo "Packing windows installer - uploading"
+	pyci github upload-asset --asset $BINARY_LOCATION --release $VERSION
+	SETUP_LOCATION="c:\\Users\\travis\\build\\harelba\\q\\q-AMD64-Windows-installer.exe"
+	pyci github upload-asset --asset $SETUP_LOCATION --release $VERSION
+fi
+
+echo "done"
diff --git a/pytest.ini b/pytest.ini
new file mode 100644
index 0000000..9d60ede
--- /dev/null
+++ b/pytest.ini
@@ -0,0 +1,2 @@
+[pytest]
+log_print = True
diff --git a/setup-pyenv.sh b/setup-pyenv.sh
new file mode 100644
index 0000000..6b29d86
--- /dev/null
+++ b/setup-pyenv.sh
@@ -0,0 +1,134 @@
+#!/usr/bin/env bash
+# NOTE: This script needs to be sourced so it can modify the environment.
+#
+# Environment variables that can be set:
+# - PYENV_VERSION
+#     Python to install [required]
+# - PYENV_VERSION_STRING
+#     String to `grep -F` against the output of `python --version` to validate
+#     that the correct Python was installed (recommended) [default: none]
+# - PYENV_ROOT
+#     Directory in which to install pyenv [default: ~/.travis-pyenv]
+# - PYENV_RELEASE
+#     Release tag of pyenv to download [default: clone from master]
+# - PYENV_CACHE_PATH
+#     Directory where full Python builds are cached (i.e., for Travis)
+
+# PYENV_ROOT is exported because pyenv uses it
+export PYENV_ROOT="${PYENV_ROOT:-$HOME/.travis-pyenv}"
+export PYTHON_CONFIGURE_OPTS="--enable-shared"
+PYENV_CACHE_PATH="${PYENV_CACHE_PATH:-$HOME/.pyenv_cache}"
+version_cache_path="$PYENV_CACHE_PATH/$PYENV_VERSION"
+version_pyenv_path="$PYENV_ROOT/versions/$PYENV_VERSION"
+
+# Functions
+#
+# verify_python -- attempts to call the Python command or binary
+# supplied in the first argument with the --version flag. If
+# PYENV_VERSION_STRING is set, then it validates the returned version string
+# as well (using grep -F). Returns whatever status code the command returns.
+verify_python() {
+  local python_bin="$1"; shift
+
+  if [[ -n "$PYENV_VERSION_STRING" ]]; then
+    "$python_bin" --version 2>&1 | grep -F "$PYENV_VERSION_STRING" &>/dev/null
+  else
+    "$python_bin" --version &>/dev/null
+  fi
+}
+
+# use_cached_python -- Tries symlinking to the cached PYENV_VERSION and
+# verifying that it's a working build. Returns 0 if it's found and it
+# verifies, otherwise returns 1.
+use_cached_python() {
+  if [[ -d "$version_cache_path" ]]; then
+    printf "Cached python found, %s. Verifying..." "$PYENV_VERSION"
+    ln -s "$version_cache_path" "$version_pyenv_path"
+    if verify_python "$version_pyenv_path/bin/python"; then
+      printf "success!\n"
+      return 0
+    else
+      printf "FAILED.\nClearing cached version..."
+      rm -f "$version_pyenv_path"
+      rm -rf "$version_cache_path"
+      printf "done.\n"
+      return 1
+    fi
+  else
+    echo "No cached python found."
+    return 1
+  fi
+}
+
+# output_debugging_info -- Outputs useful debugging information
+output_debugging_info() {
+  echo "**** Debugging information"
+  printf "PYENV_VERSION\n%s\n" "$PYENV_VERSION"
+  printf "PYENV_VERSION_STRING\n%s\n" "$PYENV_VERSION_STRING"
+  printf "PYENV_CACHE_PATH\n%s\n" "$PYENV_CACHE_PATH"
+  set -x
+  python --version
+  "$version_cache_path/bin/python" --version
+  which python
+  pyenv which python
+  set +x
+}
+
+# Main script begins.
+
+if [[ -z "$PYENV_VERSION" ]]; then
+  echo "PYENV_VERSION is not set. Not installing a pyenv."
+  return 0
+fi
+
+# Get out of the virtualenv we're in (if we're in one).
+[[ -z "$VIRTUAL_ENV" ]] || deactivate
+
+# Install pyenv
+echo "**** Installing pyenv."
+if [[ -n "$PYENV_RELEASE" ]]; then
+  # Fetch the release archive from Github (slightly faster than cloning)
+  mkdir "$PYENV_ROOT"
+  curl -fsSL "https://github.com/yyuu/pyenv/archive/$PYENV_RELEASE.tar.gz" \
+    | tar -xz -C "$PYENV_ROOT" --strip-components 1
+else
+  # Don't have a release to fetch, so just clone directly
+  git clone --depth 1 https://github.com/yyuu/pyenv.git "$PYENV_ROOT"
+fi
+
+export PATH="$PYENV_ROOT/bin:$PATH"
+eval "$(pyenv init -)"
+
+# Make sure the cache directory exists
+mkdir -p "$PYENV_CACHE_PATH"
+
+# Try using an already cached PYENV_VERSION. If it fails or is not found,
+# then install from scratch.
+echo "**** Trying to find and use cached python $PYENV_VERSION."
+if ! use_cached_python; then
+  echo "**** Installing python $PYENV_VERSION with pyenv now."
+  if pyenv install "$PYENV_VERSION"; then
+    if mv "$version_pyenv_path" "$PYENV_CACHE_PATH"; then
+      echo "Python was successfully built and moved to cache."
+      echo "**** Trying to find and use cached python $PYENV_VERSION."
+      if ! use_cached_python; then
+        echo "Python version $PYENV_VERSION was apparently successfully built"
+        echo "with pyenv, but, once cached, it could not be verified."
+        output_debugging_info
+        return 1
+      fi
+    else
+      echo "**** Warning: Python was succesfully built, but moving to cache"
+      echo "failed. Proceeding anyway without caching."
+    fi
+  else
+    echo "Python version $PYENV_VERSION build FAILED."
+    return 1
+  fi
+fi
+
+# Now we have to reinitialize pyenv, as we need the shims etc to be created so
+# the pyenv activates correctly.
+echo "**** Activating python $PYENV_VERSION and generating new virtualenv."
+eval "$(pyenv init -)"
+pyenv global "$PYENV_VERSION"
diff --git a/setup.py b/setup.py
new file mode 100644
index 0000000..d78de8e
--- /dev/null
+++ b/setup.py
@@ -0,0 +1,24 @@
+#!/usr/bin/env python
+
+from setuptools import setup
+
+setup(
+    name='q',
+    url='https://github.com/harelba/q',
+    license='LICENSE',
+    version='2.0.9',
+    author='Harel Ben-Attia',
+    description="Run SQL directly on CSV or TSV files",
+    author_email='harelba@gmail.com',
+    install_requires=[
+        'six==1.11.0'
+    ],
+    packages=[
+        'q'
+    ],
+    entry_points={
+        'console_scripts': [
+            'q = bin.q:run_standalone'
+        ]
+    }
+)
diff --git a/test-requirements.txt b/test-requirements.txt
new file mode 100644
index 0000000..a89474c
--- /dev/null
+++ b/test-requirements.txt
@@ -0,0 +1,2 @@
+pytest==4.6.2
+flake8==3.6.0
\ No newline at end of file
diff --git a/test/test-suite b/test/test-suite
index bc7fc37..f5f0b29 100755
--- a/test/test-suite
+++ b/test/test-suite
@@ -1,12 +1,13 @@
 #!/usr/bin/env python
 
 #
+# test suite for q.
+# 
+# All tests must be end-to-end tests, running the actual q command and testing stdout/stderr, and the return code.
+# Some utilities are provided for making that easy, see other tests for examples.
 #
-# Simplistic test suite for q.
-#
-# Currently takes into account the project folder structure for running, so it needs
-# to be executed from the current folder
-#
+# Don't forget to use the Q_EXECUTABLE instead of hardcoding the q command line. This will be used in the near future
+# in order to test the resulting binary executables as well, instead of just executing the q python source code.
 #
 
 import unittest
@@ -24,18 +25,22 @@ import six
 from six.moves import range
 import codecs
 
-sys.path.append(os.path.join(os.path.abspath(os.path.dirname(sys.argv[0])),'..','bin'))
-from qtextasdata import QTextAsData,QOutput,QOutputPrinter,QInputParams
+
+import QTextAsData,QOutput,QOutputPrinter,QInputParams
 
 # q uses this encoding as the default output encoding. Some of the tests use it in order to 
 # make sure that the output is correctly encoded
 SYSTEM_ENCODING = locale.getpreferredencoding()
 
+EXAMPLES = os.path.abspath(os.path.join(q.__file__, os.pardir, os.pardir, 'examples'))
+
+Q_EXECUTABLE = os.getenv('Q_EXECUTABLE', '../bin/q')
 
 DEBUG = False
 if len(sys.argv) > 2 and sys.argv[2] == '-v':
     DEBUG = True
 
+
 def run_command(cmd_to_run):
     global DEBUG
     if DEBUG:
@@ -61,6 +66,7 @@ def run_command(cmd_to_run):
         print("RESULT:{}".format(res))
     return res
 
+
 uneven_ls_output = six.b("""drwxr-xr-x   2 root     root      4096 Jun 11  2012 /selinux
 drwxr-xr-x   2 root     root      4096 Apr 19  2013 /mnt
 drwxr-xr-x   2 root     root      4096 Apr 24  2013 /srv
@@ -72,6 +78,7 @@ lrwxrwxrwx   1 root     root        29 Jun 21  2013 /vmlinuz -> boot/vmlinuz-3.8
 lrwxrwxrwx   1 root     root        32 Jun 21  2013 /initrd.img -> boot/initrd.img-3.8.0-19-generic
 """)
 
+
 find_output = six.b("""8257537   32 drwxrwxrwt 218 root     root        28672 Mar  1 11:00 /tmp
 8299123    4 drwxrwxr-x   2 harel    harel        4096 Feb 27 10:06 /tmp/1628a3fd-b9fe-4dd1-bcdc-7eb869fe7461/supervisor/stormdist/testTopology3fad644a-54c0-4def-b19e-77ca97941595-1-1393513576
 8263229  964 -rw-rw-r--   1 mapred   mapred      984569 Feb 27 10:06 /tmp/1628a3fd-b9fe-4dd1-bcdc-7eb869fe7461/supervisor/stormdist/testTopology3fad644a-54c0-4def-b19e-77ca97941595-1-1393513576/stormcode.ser
@@ -84,6 +91,7 @@ find_output = six.b("""8257537   32 drwxrwxrwt 218 root     root        28672 Ma
 8263604    0 -rw-rw-r--   1 harel    harel           0 Feb 27 10:16 /tmp/1628a3fd-b9fe-4dd1-bcdc-7eb869fe7461/supervisor/localstate/1393514175754.version
 """)
 
+
 header_row = six.b('name,value1,value2')
 sample_data_rows = [six.b('a,1,0'), six.b('b,2,0'), six.b('c,,0')]
 sample_data_rows_with_empty_string = [six.b('a,aaa,0'), six.b('b,bbb,0'), six.b('c,,0')]
@@ -135,6 +143,7 @@ long_value1 = "23683289372328372328373"
 int_value = "2328372328373"
 sample_data_with_long_values = "%s\n%s\n%s" % (long_value1,int_value,int_value)
 
+
 def one_column_warning(e):
     return e[0].startswith(six.b('Warning: column count is one'))
 
@@ -159,19 +168,20 @@ class AbstractQTestCase(unittest.TestCase):
         path = '/var/tmp'
         return '%s/%s-%s.%s' % (path,prefix,random.randint(0,1000000000),postfix)
 
+
 class SaveDbToDiskTests(AbstractQTestCase):
 
     def test_store_to_disk(self):
         db_filename = self.random_tmp_filename('store-to-disk','db')
         self.assertFalse(os.path.exists(db_filename))
 
-        retcode, o, e = run_command('seq 1 1000 | ../bin/q "select count(*) from -" -c 1 -S %s' % db_filename)
+        retcode, o, e = run_command('seq 1 1000 | ' + Q_EXECUTABLE + ' "select count(*) from -" -c 1 -S %s' % db_filename)
 
         self.assertTrue(retcode == 0)
         self.assertTrue(len(o) == 0)
         self.assertTrue(len(e) == 5)
         self.assertTrue(e[0].startswith(six.b('Going to save data')))
-        self.assertTrue(db_filename.encode(sys.stdout.encoding) in e[0])
+        self.assertTrue(db_filename.encode(sys.stdout.encoding or 'utf-8') in e[0])
         self.assertTrue(e[1].startswith(six.b('Data has been loaded in')))
         self.assertTrue(e[2].startswith(six.b('Saving data to db file')))
         self.assertTrue(e[3].startswith(six.b('Data has been saved into')))
@@ -191,12 +201,12 @@ class SaveDbToDiskTests(AbstractQTestCase):
         db_filename = self.random_tmp_filename('store-to-disk', 'db')
         self.assertFalse(os.path.exists(db_filename))
 
-        retcode, o, e = run_command('seq 1 1000 | ../bin/q "select count(*) from -" -c 1 -S %s' % db_filename)
+        retcode, o, e = run_command('seq 1 1000 | ' + Q_EXECUTABLE + ' "select count(*) from -" -c 1 -S %s' % db_filename)
 
         self.assertTrue(retcode == 0)
         self.assertTrue(os.path.exists(db_filename))
 
-        retcode2, o2, e2 = run_command('seq 1 1000 | ../bin/q "select count(*) from -" -c 1 -S %s' % db_filename)
+        retcode2, o2, e2 = run_command('seq 1 1000 | ' + Q_EXECUTABLE + ' "select count(*) from -" -c 1 -S %s' % db_filename)
         self.assertTrue(retcode2 != 0)
         self.assertTrue(e2[0].startswith(six.b('Going to save data into a disk database')))
         self.assertTrue(e2[1] == six.b('Disk database file {} already exists.'.format(db_filename)))
@@ -208,7 +218,7 @@ class BasicTests(AbstractQTestCase):
 
     def test_basic_aggregation(self):
         retcode, o, e = run_command(
-            'seq 1 10 | ../bin/q "select sum(c1),avg(c1) from -"')
+            'seq 1 10 | ' + Q_EXECUTABLE + ' "select sum(c1),avg(c1) from -"')
         self.assertTrue(retcode == 0)
         self.assertTrue(len(o) == 1)
         self.assertTrue(len(e) == 1)
@@ -221,7 +231,7 @@ class BasicTests(AbstractQTestCase):
         tmpfile = self.create_file_with_data(
             six.b('\x1f\x8b\x08\x08\xf2\x18\x12S\x00\x03xxxxxx\x003\xe42\xe22\xe62\xe12\xe52\xe32\xe7\xb2\xe0\xb2\xe424\xe0\x02\x00\xeb\xbf\x8a\x13\x15\x00\x00\x00'))
 
-        cmd = '../bin/q -z "select sum(c1),avg(c1) from %s"' % tmpfile.name
+        cmd = Q_EXECUTABLE + ' -z "select sum(c1),avg(c1) from %s"' % tmpfile.name
 
         retcode, o, e = run_command(cmd)
         self.assertTrue(retcode == 0)
@@ -238,7 +248,7 @@ class BasicTests(AbstractQTestCase):
         tmpfile = self.create_file_with_data(
             six.b('\x1f\x8b\x08\x08\xf2\x18\x12S\x00\x03xxxxxx\x003\xe42\xe22\xe62\xe12\xe52\xe32\xe7\xb2\xe0\xb2\xe424\xe0\x02\x00\xeb\xbf\x8a\x13\x15\x00\x00\x00'))
 
-        cmd = 'cat %s | ../bin/q -z "select sum(c1),avg(c1) from -"' % tmpfile.name
+        cmd = 'cat %s | ' % tmpfile.name + Q_EXECUTABLE + ' -z "select sum(c1),avg(c1) from -"'
 
         retcode, o, e = run_command(cmd)
         self.assertTrue(retcode != 0)
@@ -252,7 +262,7 @@ class BasicTests(AbstractQTestCase):
     def test_delimition_mistake_with_header(self):
         tmpfile = self.create_file_with_data(sample_data_no_header)
 
-        cmd = '../bin/q -d " " "select * from %s" -H' % tmpfile.name
+        cmd = Q_EXECUTABLE + ' -d " " "select * from %s" -H' % tmpfile.name
         retcode, o, e = run_command(cmd)
 
         self.assertNotEqual(retcode, 0)
@@ -269,7 +279,7 @@ class BasicTests(AbstractQTestCase):
     def test_regexp_int_data_handling(self):
         tmpfile = self.create_file_with_data(sample_data_no_header)
 
-        cmd = '../bin/q -d , "select c2 from %s where regexp(\'^1\',c2)"' % tmpfile.name
+        cmd = Q_EXECUTABLE + ' -d , "select c2 from %s where regexp(\'^1\',c2)"' % tmpfile.name
         retcode, o, e = run_command(cmd)
 
         self.assertEqual(retcode, 0)
@@ -283,7 +293,7 @@ class BasicTests(AbstractQTestCase):
     def test_regexp_null_data_handling(self):
         tmpfile = self.create_file_with_data(sample_data_no_header)
 
-        cmd = '../bin/q -d , "select count(*) from %s where regexp(\'^\',c2)"' % tmpfile.name
+        cmd = Q_EXECUTABLE + ' -d , "select count(*) from %s where regexp(\'^\',c2)"' % tmpfile.name
         retcode, o, e = run_command(cmd)
 
         self.assertEqual(retcode, 0)
@@ -297,7 +307,7 @@ class BasicTests(AbstractQTestCase):
     def test_select_one_column(self):
         tmpfile = self.create_file_with_data(sample_data_no_header)
 
-        cmd = '../bin/q -d , "select c1 from %s"' % tmpfile.name
+        cmd = Q_EXECUTABLE + ' -d , "select c1 from %s"' % tmpfile.name
         retcode, o, e = run_command(cmd)
 
         self.assertEqual(retcode, 0)
@@ -311,7 +321,7 @@ class BasicTests(AbstractQTestCase):
     def test_tab_delimition_parameter(self):
         tmpfile = self.create_file_with_data(
             sample_data_no_header.replace(six.b(","), six.b("\t")))
-        cmd = '../bin/q -t "select c1,c2,c3 from %s"' % tmpfile.name
+        cmd = Q_EXECUTABLE + ' -t "select c1,c2,c3 from %s"' % tmpfile.name
         retcode, o, e = run_command(cmd)
 
         self.assertEqual(retcode, 0)
@@ -326,7 +336,7 @@ class BasicTests(AbstractQTestCase):
     def test_tab_delimition_parameter__with_manual_override_attempt(self):
         tmpfile = self.create_file_with_data(
             sample_data_no_header.replace(six.b(","), six.b("\t")))
-        cmd = '../bin/q -t -d , "select c1,c2,c3 from %s"' % tmpfile.name
+        cmd = Q_EXECUTABLE + ' -t -d , "select c1,c2,c3 from %s"' % tmpfile.name
         retcode, o, e = run_command(cmd)
 
         self.assertEqual(retcode, 0)
@@ -340,7 +350,7 @@ class BasicTests(AbstractQTestCase):
 
     def test_output_delimiter(self):
         tmpfile = self.create_file_with_data(sample_data_no_header)
-        cmd = '../bin/q -d , -D "|" "select c1,c2,c3 from %s"' % tmpfile.name
+        cmd = Q_EXECUTABLE + ' -d , -D "|" "select c1,c2,c3 from %s"' % tmpfile.name
         retcode, o, e = run_command(cmd)
 
         self.assertEqual(retcode, 0)
@@ -355,7 +365,7 @@ class BasicTests(AbstractQTestCase):
 
     def test_output_delimiter_tab_parameter(self):
         tmpfile = self.create_file_with_data(sample_data_no_header)
-        cmd = '../bin/q -d , -T "select c1,c2,c3 from %s"' % tmpfile.name
+        cmd = Q_EXECUTABLE + ' -d , -T "select c1,c2,c3 from %s"' % tmpfile.name
         retcode, o, e = run_command(cmd)
 
         self.assertEqual(retcode, 0)
@@ -370,7 +380,7 @@ class BasicTests(AbstractQTestCase):
 
     def test_output_delimiter_tab_parameter__with_manual_override_attempt(self):
         tmpfile = self.create_file_with_data(sample_data_no_header)
-        cmd = '../bin/q -d , -T -D "|" "select c1,c2,c3 from %s"' % tmpfile.name
+        cmd = Q_EXECUTABLE + ' -d , -T -D "|" "select c1,c2,c3 from %s"' % tmpfile.name
         retcode, o, e = run_command(cmd)
 
         self.assertEqual(retcode, 0)
@@ -384,7 +394,7 @@ class BasicTests(AbstractQTestCase):
         self.cleanup(tmpfile)
 
     def test_stdin_input(self):
-        cmd = six.b('printf "%s" | ../bin/q -d , "select c1,c2,c3 from -"') % sample_data_no_header
+        cmd = six.b('printf "%s" | ' + Q_EXECUTABLE + ' -d , "select c1,c2,c3 from -"') % sample_data_no_header
         retcode, o, e = run_command(cmd)
 
         self.assertEqual(retcode, 0)
@@ -397,7 +407,7 @@ class BasicTests(AbstractQTestCase):
 
     def test_column_separation(self):
         tmpfile = self.create_file_with_data(sample_data_no_header)
-        cmd = '../bin/q -d , "select c1,c2,c3 from %s"' % tmpfile.name
+        cmd = Q_EXECUTABLE + ' -d , "select c1,c2,c3 from %s"' % tmpfile.name
         retcode, o, e = run_command(cmd)
 
         self.assertEqual(retcode, 0)
@@ -413,7 +423,7 @@ class BasicTests(AbstractQTestCase):
     def test_column_analysis(self):
         tmpfile = self.create_file_with_data(sample_data_no_header)
 
-        cmd = '../bin/q -d , "select c1 from %s" -A' % tmpfile.name
+        cmd = Q_EXECUTABLE + ' -d , "select c1 from %s" -A' % tmpfile.name
         retcode, o, e = run_command(cmd)
 
         self.assertEqual(retcode, 0)
@@ -427,7 +437,7 @@ class BasicTests(AbstractQTestCase):
     def test_column_analysis_no_header(self):
         tmpfile = self.create_file_with_data(sample_data_no_header)
 
-        cmd = '../bin/q -d , "select c1 from %s" -A' % tmpfile.name
+        cmd = Q_EXECUTABLE + ' -d , "select c1 from %s" -A' % tmpfile.name
         retcode, o, e = run_command(cmd)
 
         self.assertEqual(retcode, 0)
@@ -438,7 +448,7 @@ class BasicTests(AbstractQTestCase):
 
     def test_header_exception_on_numeric_header_data(self):
         tmpfile = self.create_file_with_data(sample_data_no_header)
-        cmd = '../bin/q -d , "select * from %s" -A -H' % tmpfile.name
+        cmd = Q_EXECUTABLE + ' -d , "select * from %s" -A -H' % tmpfile.name
         retcode, o, e = run_command(cmd)
 
         self.assertNotEqual(retcode, 0)
@@ -453,7 +463,7 @@ class BasicTests(AbstractQTestCase):
 
     def test_column_analysis_with_header(self):
         tmpfile = self.create_file_with_data(sample_data_with_header)
-        cmd = '../bin/q -d , "select c1 from %s" -A -H' % tmpfile.name
+        cmd = Q_EXECUTABLE + ' -d , "select c1 from %s" -A -H' % tmpfile.name
         retcode, o, e = run_command(cmd)
 
         self.assertNotEqual(retcode, 0)
@@ -470,7 +480,7 @@ class BasicTests(AbstractQTestCase):
 
     def test_data_with_header(self):
         tmpfile = self.create_file_with_data(sample_data_with_header)
-        cmd = '../bin/q -d , "select name from %s" -H' % tmpfile.name
+        cmd = Q_EXECUTABLE + ' -d , "select name from %s" -H' % tmpfile.name
         retcode, o, e = run_command(cmd)
 
         self.assertEqual(retcode, 0)
@@ -481,7 +491,7 @@ class BasicTests(AbstractQTestCase):
 
     def test_output_header_when_input_header_exists(self):
         tmpfile = self.create_file_with_data(sample_data_with_header)
-        cmd = '../bin/q -d , "select name from %s" -H -O' % tmpfile.name
+        cmd = Q_EXECUTABLE + ' -d , "select name from %s" -H -O' % tmpfile.name
         retcode, o, e = run_command(cmd)
 
         self.assertEqual(retcode, 0)
@@ -495,7 +505,7 @@ class BasicTests(AbstractQTestCase):
 
     def test_generated_column_name_warning_when_header_line_exists(self):
         tmpfile = self.create_file_with_data(sample_data_with_header)
-        cmd = '../bin/q -d , "select c3 from %s" -H' % tmpfile.name
+        cmd = Q_EXECUTABLE + ' -d , "select c3 from %s" -H' % tmpfile.name
 
         retcode, o, e = run_command(cmd)
 
@@ -510,7 +520,7 @@ class BasicTests(AbstractQTestCase):
 
     def test_column_analysis_with_unexpected_header(self):
         tmpfile = self.create_file_with_data(sample_data_with_header)
-        cmd = '../bin/q -d , "select c1 from %s" -A' % tmpfile.name
+        cmd = Q_EXECUTABLE + ' -d , "select c1 from %s" -A' % tmpfile.name
         retcode, o, e = run_command(cmd)
 
         self.assertEqual(retcode, 0)
@@ -529,7 +539,7 @@ class BasicTests(AbstractQTestCase):
 
     def test_empty_data(self):
         tmpfile = self.create_file_with_data(six.b(''))
-        cmd = '../bin/q -d , "select c1 from %s"' % tmpfile.name
+        cmd = Q_EXECUTABLE + ' -d , "select c1 from %s"' % tmpfile.name
         retcode, o, e = run_command(cmd)
 
         self.assertEqual(retcode, 0)
@@ -542,7 +552,7 @@ class BasicTests(AbstractQTestCase):
 
     def test_empty_data_with_header_param(self):
         tmpfile = self.create_file_with_data(six.b(''))
-        cmd = '../bin/q -d , "select c1 from %s" -H' % tmpfile.name
+        cmd = Q_EXECUTABLE + ' -d , "select c1 from %s" -H' % tmpfile.name
         retcode, o, e = run_command(cmd)
 
         self.assertNotEqual(retcode, 0)
@@ -556,7 +566,7 @@ class BasicTests(AbstractQTestCase):
 
     def test_one_row_of_data_without_header_param(self):
         tmpfile = self.create_file_with_data(header_row)
-        cmd = '../bin/q -d , "select c2 from %s"' % tmpfile.name
+        cmd = Q_EXECUTABLE + ' -d , "select c2 from %s"' % tmpfile.name
         retcode, o, e = run_command(cmd)
 
         self.assertEqual(retcode, 0)
@@ -569,7 +579,7 @@ class BasicTests(AbstractQTestCase):
 
     def test_one_row_of_data_with_header_param(self):
         tmpfile = self.create_file_with_data(header_row)
-        cmd = '../bin/q -d , "select c2 from %s" -H' % tmpfile.name
+        cmd = Q_EXECUTABLE + ' -d , "select c2 from %s" -H' % tmpfile.name
         retcode, o, e = run_command(cmd)
 
         self.assertEqual(retcode, 0)
@@ -582,7 +592,7 @@ class BasicTests(AbstractQTestCase):
 
     def test_dont_leading_keep_whitespace_in_values(self):
         tmpfile = self.create_file_with_data(sample_data_with_spaces_no_header)
-        cmd = '../bin/q -d , "select c1 from %s"' % tmpfile.name
+        cmd = Q_EXECUTABLE + ' -d , "select c1 from %s"' % tmpfile.name
         retcode, o, e = run_command(cmd)
 
         self.assertEqual(retcode, 0)
@@ -597,7 +607,7 @@ class BasicTests(AbstractQTestCase):
 
     def test_keep_leading_whitespace_in_values(self):
         tmpfile = self.create_file_with_data(sample_data_with_spaces_no_header)
-        cmd = '../bin/q -d , "select c1 from %s" -k' % tmpfile.name
+        cmd = Q_EXECUTABLE + ' -d , "select c1 from %s" -k' % tmpfile.name
         retcode, o, e = run_command(cmd)
 
         self.assertEqual(retcode, 0)
@@ -612,7 +622,7 @@ class BasicTests(AbstractQTestCase):
 
     def test_no_impact_of_keeping_leading_whitespace_on_integers(self):
         tmpfile = self.create_file_with_data(sample_data_with_spaces_no_header)
-        cmd = '../bin/q -d , "select c2 from %s" -k -A' % tmpfile.name
+        cmd = Q_EXECUTABLE + ' -d , "select c2 from %s" -k -A' % tmpfile.name
         retcode, o, e = run_command(cmd)
 
         self.assertEqual(retcode, 0)
@@ -629,7 +639,7 @@ class BasicTests(AbstractQTestCase):
     def test_spaces_in_header_row(self):
         tmpfile = self.create_file_with_data(
             header_row_with_spaces + six.b("\n") + sample_data_no_header)
-        cmd = '../bin/q -d , "select name,\`value 1\` from %s" -H' % tmpfile.name
+        cmd = Q_EXECUTABLE + ' -d , "select name,\`value 1\` from %s" -H' % tmpfile.name
         retcode, o, e = run_command(cmd)
 
         self.assertEqual(retcode, 0)
@@ -645,7 +655,7 @@ class BasicTests(AbstractQTestCase):
     def test_column_analysis_for_spaces_in_header_row(self):
         tmpfile = self.create_file_with_data(
             header_row_with_spaces + six.b("\n") + sample_data_no_header)
-        cmd = '../bin/q -d , "select name,\`value 1\` from %s" -H -A' % tmpfile.name
+        cmd = Q_EXECUTABLE + ' -d , "select name,\`value 1\` from %s" -H -A' % tmpfile.name
         retcode, o, e = run_command(cmd)
 
         self.assertEqual(retcode, 0)
@@ -660,7 +670,7 @@ class BasicTests(AbstractQTestCase):
         self.cleanup(tmpfile)
 
     def test_no_query_in_command_line(self):
-        cmd = '../bin/q -d , ""'
+        cmd = Q_EXECUTABLE + ' -d , ""'
         retcode, o, e = run_command(cmd)
 
         self.assertEqual(retcode, 1)
@@ -670,7 +680,7 @@ class BasicTests(AbstractQTestCase):
         self.assertEqual(e[0],six.b('Query cannot be empty (query number 1)'))
 
     def test_empty_query_in_command_line(self):
-        cmd = '../bin/q -d , "  "'
+        cmd = Q_EXECUTABLE + ' -d , "  "'
         retcode, o, e = run_command(cmd)
 
         self.assertEqual(retcode, 1)
@@ -680,7 +690,7 @@ class BasicTests(AbstractQTestCase):
         self.assertEqual(e[0],six.b('Query cannot be empty (query number 1)'))
 
     def test_failure_in_query_stops_processing_queries(self):
-        cmd = '../bin/q -d , "select 500" "select 300" "wrong-query" "select 8000"'
+        cmd = Q_EXECUTABLE + ' -d , "select 500" "select 300" "wrong-query" "select 8000"'
         retcode, o, e = run_command(cmd)
 
         self.assertEqual(retcode, 1)
@@ -690,7 +700,7 @@ class BasicTests(AbstractQTestCase):
         self.assertEqual(o[1],six.b('300'))
 
     def test_multiple_queries_in_command_line(self):
-        cmd = '../bin/q -d , "select 500" "select 300+100" "select 300" "select 200"'
+        cmd = Q_EXECUTABLE + ' -d , "select 500" "select 300+100" "select 300" "select 200"'
         retcode, o, e = run_command(cmd)
 
         self.assertEqual(retcode, 0)
@@ -703,7 +713,7 @@ class BasicTests(AbstractQTestCase):
         self.assertEqual(o[3],six.b('200'))
 
     def test_literal_calculation_query(self):
-        cmd = '../bin/q -d , "select 1+40/6"'
+        cmd = Q_EXECUTABLE + ' -d , "select 1+40/6"'
         retcode, o, e = run_command(cmd)
 
         self.assertEqual(retcode, 0)
@@ -713,7 +723,7 @@ class BasicTests(AbstractQTestCase):
         self.assertEqual(o[0],six.b('7'))
 
     def test_literal_calculation_query_float_result(self):
-        cmd = '../bin/q -d , "select 1+40/6.0"'
+        cmd = Q_EXECUTABLE + ' -d , "select 1+40/6.0"'
         retcode, o, e = run_command(cmd)
 
         self.assertEqual(retcode, 0)
@@ -726,7 +736,7 @@ class BasicTests(Abs