q version 3.1.0-beta - automatic immutable caching and direct queries on sqlite databases (#280) + New Packaging Workflowv3.1.0-beta

48 files changed, 8840 insertions, 4984 deletions

diff --git a/.github/workflows/build-and-package.yaml b/.github/workflows/build-and-package.yaml
new file mode 100644
index 0000000..f723786
--- /dev/null
+++ b/.github/workflows/build-and-package.yaml
@@ -0,0 +1,543 @@
+name: BuildAndPackage
+
+on:
+  push:
+    branches: master
+    paths-ignore:
+      - "*.md"
+      - "*.markdown"
+      - "mkdocs/**/*"
+  # Remove comment in order to pre-release on a PR, to validate packaging flow
+  pull_request:
+    branches: master
+    paths-ignore:
+      - "*.md"
+      - "*.markdown"
+      - "mkdocs/**/*"
+
+jobs:
+  create-man:
+    runs-on: ubuntu-18.04
+    steps:
+    - name: Checkout
+      uses: actions/checkout@v2
+    - name: Install Ruby
+      uses: ruby/setup-ruby@v1
+      with:
+        ruby-version: '2.6'
+    - name: Create man page
+      run: |
+        set -x -e
+        gem install ronn
+
+        ronn doc/USAGE.markdown
+        # Must be gzipped, otherwise debian does not install it
+        gzip doc/USAGE
+    - name: Upload man page
+      uses: actions/upload-artifact@v1.0.0
+      with:
+        name: q-man-page
+        path: doc/USAGE.gz
+
+  build-linux:
+    runs-on: ubuntu-18.04
+    steps:
+    - name: Checkout
+      uses: actions/checkout@v2
+    - name: Cache pyox
+      uses: actions/cache@v2
+      with:
+        path: |
+          ~/.cache/pyoxidizer
+        key: ${{ runner.os }}-pyox
+    - name: Install pyoxidizer
+      run: |
+        set -e -x
+
+        sudo apt-get update
+        sudo apt-get install -y zip sqlite3 rpm
+
+        curl -o pyoxidizer.zip -L "https://github.com/indygreg/PyOxidizer/releases/download/pyoxidizer%2F0.17/pyoxidizer-0.17.0-linux_x86_64.zip"
+        unzip pyoxidizer.zip
+        chmod +x ./pyoxidizer
+    - name: Create Q Executable - Linux
+      run: |
+        set -e -x
+
+        ./pyoxidizer build --release
+
+        export Q_EXECUTABLE=./build/x86_64-unknown-linux-gnu/release/install/q
+        chmod 755 $Q_EXECUTABLE
+
+        seq 1 100 | $Q_EXECUTABLE -c 1 "select sum(c1),count(*) from -" -S test.sqlite
+
+        mkdir -p packages/linux/
+        cp $Q_EXECUTABLE packages/linux/linux-q
+    - name: Upload Linux Executable
+      uses: actions/upload-artifact@v1.0.0
+      with:
+        name: linux-q
+        path: packages/linux/linux-q
+
+  test-linux:
+    needs: build-linux
+    runs-on: ubuntu-18.04
+    steps:
+    - name: Checkout
+      uses: actions/checkout@v2
+    - name: Install Python for Testing
+      uses: actions/setup-python@v2
+      with:
+        python-version: '3.8.12'
+        architecture: 'x64'
+    - name: Prepare Testing
+      run: |
+        set -e -x
+
+        pip3 install -r test-requirements.txt
+    - name: Download Linux Executable
+      uses: actions/download-artifact@v2
+      with:
+        name: linux-q
+    - name: Run Tests on Linux Executable
+      run: |
+        set -x -e
+
+        find ./ -ls
+
+        chmod 755 ./linux-q
+
+        Q_EXECUTABLE=`pwd`/linux-q Q_SKIP_EXECUTABLE_VALIDATION=true ./run-tests.sh -v
+
+  package-linux-deb:
+    needs: [test-linux, create-man]
+    runs-on: ubuntu-18.04
+    steps:
+    - name: Checkout
+      uses: actions/checkout@v2
+    - name: Install Ruby
+      uses: ruby/setup-ruby@v1
+      with:
+        ruby-version: '2.6'
+    - name: Downoad man page
+      uses: actions/download-artifact@v2
+      with:
+        name: q-man-page
+    - name: Download Linux Executable
+      uses: actions/download-artifact@v2
+      with:
+        name: linux-q
+    - name: Build DEB Package
+      run: |
+        set -e -x
+
+        mkdir -p packages/linux/
+
+        find ./ -ls
+
+        chmod 755 ./linux-q
+
+        gem install fpm
+        cp dist/fpm-config ~/.fpm
+        fpm -s dir -t deb --deb-use-file-permissions -p packages/linux/q-text-as-data-3.1.0-beta-1.x86_64.deb --version 3.1.0-beta ./linux-q=/usr/bin/q USAGE.gz=/usr/share/man/man1/q.1.gz
+    - name: Upload DEB Package
+      uses: actions/upload-artifact@v1.0.0
+      with:
+        name: q-text-as-data-3.1.0-beta-1.x86_64.deb
+        path: packages/linux/q-text-as-data-3.1.0-beta-1.x86_64.deb
+
+  test-deb-packaging:
+    runs-on: ubuntu-18.04
+    needs: package-linux-deb
+    steps:
+    - name: Checkout
+      uses: actions/checkout@v2
+    - name: Download DEB
+      uses: actions/download-artifact@v2
+      with:
+        name: q-text-as-data-3.1.0-beta-1.x86_64.deb
+    - name: Install Python for Testing
+      uses: actions/setup-python@v2
+      with:
+        python-version: '3.8.12'
+        architecture: 'x64'
+    - name: Prepare Testing
+      run: |
+        set -e -x
+
+        pip3 install -r test-requirements.txt
+    - name: Test DEB Package Installation
+      run: ./dist/test-using-deb.sh ./q-text-as-data-3.1.0-beta-1.x86_64.deb
+
+  package-linux-rpm:
+    needs: [test-linux, create-man]
+    runs-on: ubuntu-18.04
+    steps:
+    - name: Checkout
+      uses: actions/checkout@v2
+    - name: Install Ruby
+      uses: ruby/setup-ruby@v1
+      with:
+        ruby-version: '2.6'
+    - name: Download man page
+      uses: actions/download-artifact@v2
+      with:
+        name: q-man-page
+    - name: Download Linux Executable
+      uses: actions/download-artifact@v2
+      with:
+        name: linux-q
+    - name: Build RPM Package
+      run: |
+        set -e -x
+
+        mkdir -p packages/linux
+
+        find ./ -ls
+
+        chmod 755 ./linux-q
+
+        gem install fpm
+        cp dist/fpm-config ~/.fpm
+        fpm -s dir -t rpm --rpm-use-file-permissions -p packages/linux/q-text-as-data-3.1.0-beta.x86_64.rpm --version 3.1.0-beta ./linux-q=/usr/bin/q USAGE.gz=/usr/share/man/man1/q.1.gz
+    - name: Upload RPM Package
+      uses: actions/upload-artifact@v1.0.0
+      with:
+        name: q-text-as-data-3.1.0-beta.x86_64.rpm
+        path: packages/linux/q-text-as-data-3.1.0-beta.x86_64.rpm
+
+  test-rpm-packaging:
+    runs-on: ubuntu-18.04
+    needs: package-linux-rpm
+    steps:
+    - name: Checkout
+      uses: actions/checkout@v2
+    - name: Download RPM
+      uses: actions/download-artifact@v2
+      with:
+        name: q-text-as-data-3.1.0-beta.x86_64.rpm
+    - name: Retest using RPM
+      run: ./dist/test-using-rpm.sh ./q-text-as-data-3.1.0-beta.x86_64.rpm
+
+  build-mac:
+    runs-on: macos-11
+    steps:
+    - name: Checkout
+      uses: actions/checkout@v2
+    - name: Cache pyox
+      uses: actions/cache@v2
+      with:
+        path: |
+          ~/.cache/pyoxidizer
+        key: ${{ runner.os }}-pyox
+    - name: Install pyoxidizer
+      run: |
+        set -e -x
+
+        curl -o  pyoxidizer.zip -L "https://github.com/indygreg/PyOxidizer/releases/download/pyoxidizer%2F0.17/pyoxidizer-0.17.0-macos-universal.zip"
+        unzip pyoxidizer.zip
+        mv macos-universal/pyoxidizer ./pyoxidizer
+
+        chmod +x ./pyoxidizer
+    - name: Create Q Executable - Mac
+      run: |
+        set -e -x
+
+        ./pyoxidizer build --release
+
+        export Q_EXECUTABLE=./build/x86_64-apple-darwin/release/install/q
+        chmod 755 $Q_EXECUTABLE
+
+        seq 1 100 | $Q_EXECUTABLE -c 1 "select sum(c1),count(*) from -" -S test.sqlite
+
+        mkdir -p packages/macos/
+        cp $Q_EXECUTABLE packages/macos/macos-q
+    - name: Upload MacOS Executable
+      uses: actions/upload-artifact@v1.0.0
+      with:
+        name: macos-q
+        path: packages/macos/macos-q
+
+  test-mac:
+    needs: build-mac
+    runs-on: macos-11
+    steps:
+    - name: Checkout
+      uses: actions/checkout@v2
+    - name: Install Python for Testing
+      uses: actions/setup-python@v2
+      with:
+        python-version: '3.8.12'
+        architecture: 'x64'
+    - name: Prepare Testing
+      run: |
+        set -e -x
+
+        pip3 install wheel
+
+        pip3 install -r test-requirements.txt
+    - name: Download MacOS Executable
+      uses: actions/download-artifact@v2
+      with:
+        name: macos-q
+    - name: Run Tests on MacOS Executable
+      run: |
+        set -e -x
+
+        find ./ -ls
+
+        chmod 755 ./macos-q
+
+        Q_EXECUTABLE=`pwd`/macos-q Q_SKIP_EXECUTABLE_VALIDATION=true ./run-tests.sh -v
+
+  package-mac:
+    # create-man is not needed, as it's generated inside the brew formula independently
+    needs: [test-mac]
+    runs-on: macos-11
+    steps:
+    - name: Checkout
+      uses: actions/checkout@v2
+    - name: Create homebrew formula and install it
+      run: |
+        set -x -e
+        set -o pipefail
+
+        mkdir brew
+
+        # TODO temp, since template rendering action doesn't work in mac
+        cat .github/workflows/q.rb.brew-formula-template | sed 's/{{ .Q_VERSION }}/3.1.0-beta/g' > ./brew/q.rb
+
+        brew install --display-times --formula --build-bottle --verbose ./brew/q.rb
+        brew test ./brew/q.rb
+
+    - name: Create q bottle
+      run: |
+        brew bottle --force-core-tap --no-rebuild ./brew/q.rb
+    - name: Upload  Executable
+      uses: actions/upload-artifact@v1.0.0
+      with:
+        name: q--3.1.0-beta_1.big_sur.bottle.tar.gz
+        path: ./q--3.1.0-beta_1.big_sur.bottle.tar.gz
+
+# TODO auto-create PR to main homebrew-core
+#        git clone https://github.com/harelba/homebrew-core.git
+#        cd homebrew-core
+#
+#        git checkout -b new-q-version
+
+  test-mac-packaging:
+    needs: package-mac
+    runs-on: macos-11
+    steps:
+    - name: Checkout
+      uses: actions/checkout@v2
+    - name: Download q bottle
+      uses: actions/download-artifact@v2
+      with:
+        name: q--3.1.0-beta_1.big_sur.bottle.tar.gz
+    - name: Test the created bottle
+      run: |
+        set -x -e
+        set -o pipefail
+
+        WD=$(pwd)
+
+        pushd /usr/local/Cellar
+        tar xvfz ${WD}/q--3.1.0-beta_1.big_sur.bottle.tar.gz
+        popd
+
+        brew link q
+
+        seq 1 100 | q -c 1 "select sum(c1),count(*) from -" -S test.sqlite
+
+        echo "select sum(c1),count(*) from data_stream_stdin" | sqlite3 test.sqlite
+
+  # TODO Windows build/test/package flow is running, but q executable is still not running well, due to pyox+sqlite3 issue
+  build-windows:
+    runs-on: windows-latest
+    if: ${{ false }}
+    steps:
+    - name: Checkout
+      uses: actions/checkout@v2
+    - name: Install MSVC build tools
+      uses: ilammy/msvc-dev-cmd@v1
+    - name: Install Python
+      uses: actions/setup-python@v2
+      with:
+        python-version: '3.8.10'
+        architecture: 'x64'
+    - name: Install pyoxidizer
+      shell: bash
+      run: |
+        set -x -e
+
+        python3 -V
+        pip3 -V
+
+        pip3 install pyoxidizer
+    - name: Create Q Executable - Windows
+      shell: bash
+      run: |
+        set -e -x
+
+        # Hack to overcome the fact that apsw doesn't have a registered wheel for Windows
+        cp requirements-win-x86_64.txt requirements.txt
+
+        pyoxidizer build --release
+
+        export Q_EXECUTABLE=./build/x86_64-pc-windows-msvc/release/install/q
+        chmod 755 $Q_EXECUTABLE
+
+        seq 1 100 | $Q_EXECUTABLE -c 1 "select sum(c1),count(*) from -" -S test.sqlite
+
+        mkdir -p packages/windows/
+        cp $Q_EXECUTABLE packages/windows/win-q.exe
+
+        find ./ -ls
+    - name: Upload Linux Executable
+      uses: actions/upload-artifact@v1.0.0
+      with:
+        name: win-q
+        path: packages/windows/win-q.exe
+
+  test-windows:
+    needs: build-windows
+    runs-on: windows-latest
+    steps:
+    - name: Checkout
+      uses: actions/checkout@v2
+    - name: Fail deliberately - No tests on Windows
+      continue-on-error: true
+      run: |
+        echo "Tests are not actually running on the Windows Executable itself. Only the packaging is later tested"
+        exit 1
+
+  package-windows:
+    needs: [create-man, test-windows]
+    runs-on: windows-latest
+    steps:
+    - name: Checkout
+      uses: actions/checkout@v2
+    - name: Install MSVC build tools
+      uses: ilammy/msvc-dev-cmd@v1
+    - name: Install Python
+      uses: actions/setup-python@v2
+      with:
+        python-version: '3.8.10'
+        architecture: 'x64'
+    - name: Install pyoxidizer
+      shell: bash
+      run: |
+        set -x -e
+
+        python3 -V
+        pip3 -V
+
+        pip3 install pyoxidizer
+    - name: Create Q MSI - Windows
+      shell: bash
+      run: |
+        set -e -x
+
+        # Hack to overcome the fact that apsw doesn't have a registered wheel for Windows
+        cp requirements-win-x86_64.txt requirements.txt
+
+        pyoxidizer build --release msi_installer
+
+        find ./ -ls
+
+        export Q_MSI=./build/x86_64-pc-windows-msvc/release/msi_installer/q-text-as-data-3.1.0-beta.msi
+        chmod 755 $Q_MSI
+
+        mkdir -p packages/windows/
+        cp $Q_MSI packages/windows/q-text-as-data-3.1.0-beta.msi
+
+    - name: Upload Windows MSI
+      uses: actions/upload-artifact@v1.0.0
+      with:
+        name: q-text-as-data-3.1.0-beta.msi
+        path: packages/windows/q-text-as-data-3.1.0-beta.msi
+
+  test-windows-packaging:
+    needs: package-windows
+    runs-on: windows-latest
+    steps:
+    - name: Checkout
+      uses: actions/checkout@v2
+    - name: Download Windows Package
+      uses: actions/download-artifact@v2
+      with:
+        name: q-text-as-data-3.1.0-beta.msi
+    - name: Test Install of MSI
+      continue-on-error: true
+      shell: powershell
+      run: |
+        $process = Start-Process msiexec.exe -ArgumentList "/i q-text-as-data-3.1.0-beta.msi -l* msi-install.log /norestart /quiet" -PassThru -Wait
+        $process.ExitCode
+        gc msi-install.log
+
+        exit $process.ExitCode
+    - name: Test Uninstall of MSI
+      continue-on-error: true
+      shell: powershell
+      run: |
+        $process = Start-Process msiexec.exe -ArgumentList "/u q-text-as-data-3.1.0-beta.msi /norestart /quiet" -PassThru -Wait
+        $process.ExitCode
+        exit $process.ExitCode
+
+  perform-prerelease:
+    # We'd like artifacts to be uploaded regardless of tests succeeded or not,
+    # this is why the dependency here is not on test-X-packaging jobs
+    needs: [package-linux-deb, package-linux-rpm, package-mac]
+    runs-on: ubuntu-latest
+    # TODO Push to master will now pre-release as well, until things stabilize
+    # if: ${{ github.event_name == 'pull_request' }}
+    steps:
+    - name: Download All Artifacts
+      uses: actions/download-artifact@v2
+      with:
+        path: artifacts/
+    - name: Timestamp pre-release
+      run: |
+        set -e -x
+
+        echo "Workflow finished at $(date)" >> artifacts/workflow-finish-time.txt
+    - name: Create pre-release
+      uses: "marvinpinto/action-automatic-releases@v1.2.1"
+      with:
+        repo_token: "${{ secrets.GITHUB_TOKEN }}"
+        automatic_release_tag: "latest"
+        prerelease: true
+        title: "Next Release Development Build"
+        files: |
+          artifacts/**/*
+
+  perform-release:
+    # TODO Windows is not here so users won't be confused by seeing an MSI (it's still not production-grade, you need to have sqlite3 dll in the path)
+    needs: [test-mac-packaging, test-deb-packaging, test-rpm-packaging]
+    runs-on: ubuntu-latest
+    # Disabled on purpose for now - Changing the beta release to a real one will be done manually until everything stabilizes
+    # and then this will be reinstated
+    # if: ${{ github.event_name == 'push' && github.ref == 'refs/heads/master' }}
+    if: ${{ false }}
+    steps:
+    - name: Download All Artifacts
+      uses: actions/download-artifact@v2
+      with:
+        path: artifacts/
+    - name: Delete Windows Artifacts so they're not part of the release for now
+      run: |
+        set -x -e
+
+        echo "Deleting windows artifacts so they're not part of the release - windows is not fully ready"
+
+        set +e
+        rm -vf artifacts/*.msi
+        rm -vf artifacts/win-q.exe
+        set -e
+    - uses: "marvinpinto/action-automatic-releases@v1.2.1"
+      with:
+        repo_token: "${{ secrets.GITHUB_TOKEN }}"
+        prerelease: false
+        files: |
+          artifacts/**/*
diff --git a/.github/workflows/q.rb.brew-formula-template b/.github/workflows/q.rb.brew-formula-template
new file mode 100644
index 0000000..c60fb43
--- /dev/null
+++ b/.github/workflows/q.rb.brew-formula-template
@@ -0,0 +1,35 @@
+# frozen_string_literal: true
+
+# Formula for q
+class Q < Formula
+  desc "Run SQL directly on CSV or TSV files"
+  homepage "https://harelba.github.io/q/"
+  # Building directly from master for now, eventually it will be tag-based so the version tag will be downloaded
+  url "https://github.com/harelba/q/archive/master.tar.gz"
+  version "{{ .Q_VERSION }}"
+
+  # Removed for now, until everything is finalized
+  # sha256 "0844aed6658d0347a299b84bee978c88724d45093e8cbd7b05506ecc0b93c98c"
+
+  license "GPL-3.0-or-later"
+  revision 1
+
+  depends_on "pyoxidizer" => :build
+  depends_on "python@3.8" => :build
+  depends_on "ronn" => :build
+  depends_on xcode: ["12.4", :build]
+
+  def install
+    system "pyoxidizer", "build", "--release"
+    bin.install "./build/x86_64-apple-darwin/release/install/q"
+
+    system "ronn", "--roff", "--section=1", "doc/USAGE.markdown"
+    man1.install "doc/USAGE.1" => "q.1"
+  end
+
+  test do
+    seq = (1..100).map(&:to_s).join("\n")
+    output = pipe_output("#{bin}/q -c 1 'select sum(c1) from -'", seq)
+    assert_equal "5050\n", output
+  end
+end
diff --git a/.gitignore b/.gitignore
index 2d4ca0f..eb3a2d2 100644
--- a/.gitignore
+++ b/.gitignore
@@ -15,3 +15,11 @@ generated-site/
 benchmark_data.tar.gz
 _benchmark_data/
 q.egg-info/
+.pytest_cache/
+*.qsql
+htmlcov/
+*.sqlite
+*.tar.gz
+.coverage
+.DS_Store
+*.egg
diff --git a/.travis.yml b/.travis.yml
deleted file mode 100644
index e8186f1..0000000
--- a/.travis.yml
+++ /dev/null
@@ -1,137 +0,0 @@
-sudo: false
-
-stages:
-  - integration
-  - release
-
-env:
-  global:
-    - CACHE_NAME=${TRAVIS_JOB_NAME}
-
-
-_commands_provider:
-
-  _test: &_test make test
-
-  _lint: &_lint make lint
-
-  _release: &_release make local-release
-
-  _install_requirements: &_install_requirements make dep
-
-  # https://ttcshelbyville.wordpress.com/2012/12/19/disable-remote-differential-compression-form-the-command-line/
-  _disable_windows_compression: &_disable_windows_compression "powershell Disable-WindowsOptionalFeature -Online -FeatureName MSRDC-Infrastructure"
-
-  # https://travis-ci.community/t/yarn-network-troubles/333/7
-  _disable_windows_defender: &_disable_windows_defender "powershell Set-MpPreference -DisableRealtimeMonitoring \\$true"
-
-
-_steps_provider:
-
-  _test: &_step_test
-
-    install:
-      - *_install_requirements
-    before_script: *_lint
-    script: *_test
-
-  _release: &_step_release
-
-    install: *_install_requirements
-    script: *_release
-
-
-
-jobs:
-  include:
-
-    - stage: integration
-      name: py27-macos
-      os: osx
-      language: generic
-      osx_image: xcode7.3
-      env:
-        - PYENV_VERSION=2.7.14
-      before_install: source setup-pyenv.sh
-      <<: *_step_test
-      cache:
-        directories:
-          - ${HOME}/.pyenv_cache
-
-    - stage: integration
-      name: py36-macos
-      os: osx
-      language: generic
-      osx_image: xcode7.3
-      env:
-        - PYENV_VERSION=3.6.4
-      before_install: source setup-pyenv.sh
-      <<: *_step_test
-      cache:
-        directories:
-          - ${HOME}/.pyenv_cache
-
-    - stage: integration
-      name: py37-macos
-      os: osx
-      language: generic
-      osx_image: xcode7.3
-      env:
-        - PYENV_VERSION=3.7.3
-      before_install: source setup-pyenv.sh
-      <<: *_step_test
-      cache:
-        directories:
-          - ${HOME}/.pyenv_cache
-
-    - stage: integration
-      name: py27-linux
-      language: python
-      python: "2.7"
-      <<: *_step_test
-
-    - stage: integration
-      name: py36-linux
-      language: python
-      python: "3.6"
-      <<: *_step_test
-
-    - stage: integration
-      name: py37-linux
-      language: python
-      dist: xenial
-      python: "3.7"
-      <<: *_step_test
-
-    - stage: release
-      name: macos
-      os: osx
-      language: generic
-      osx_image: xcode7.3
-      env:
-        - PYENV_VERSION=3.7.3
-      before_install: source setup-pyenv.sh
-      <<: *_step_release
-      cache:
-        directories:
-          - ${HOME}/.pyenv_cache
-
-    - stage: release
-      name: linux
-      language: python
-      dist: xenial
-      python: "3.7"
-      <<: *_step_release
-
-    - stage: release
-      name: windows
-      os: windows
-      language: shell
-      env:
-        - PATH=/c/Python37:/c/Python37/Scripts:$PATH
-      before_install:
-        - *_disable_windows_compression
-        - *_disable_windows_defender
-        - choco install make
-        - choco install python --version 3.7.3
-      <<: *_step_release
diff --git a/Makefile b/Makefile
deleted file mode 100644
index e612a54..0000000
--- a/Makefile
+++ /dev/null
@@ -1,37 +0,0 @@
-SHELL := /bin/bash
-
-PROJECT_NAME=$(shell dirname "$0")
-ROOT_DIR:=$(shell dirname $(realpath $(lastword $(MAKEFILE_LIST))))
-
-.PHONY: test help
-.DEFAULT_GOAL := ci
-
-ci: lint test ## Equivelant to 'make lint test'
-
-help: ## Show this help message.
-
-	@grep -E '^[a-zA-Z_-]+:.*?## .*$$' $(MAKEFILE_LIST) | awk 'BEGIN {FS = ":.*?## "}; {printf "\033[36m%-30s\033[0m %s\n", $$1, $$2}'
-
-dep: ## Install the dependent libraries.
-
-	pip install -r test-requirements.txt
-	pip install -e .
-
-lint: dep ## Run lint validations.
-
-	flake8 q/ --count --select=E901,E999,F821,F822,F823 --show-source --statistics
-
-test: dep ## Run the unit tests.
-
-	test/test-all
-	## TODO Bring back pytest
-	## py.test -rs -c pytest.ini -s -v q/tests/suite.py --rootdir . 
-
-release: ## Run release
-	pip install py-ci
-	pyci release --no-wheel-publish --wheel-universal
-
-local-release:
-	pip install py-ci
-	./do-manual-release.sh
-	
diff --git a/QSQL-NOTES.md b/QSQL-NOTES.md
new file mode 100644
index 0000000..1bb1e43
--- /dev/null
+++ b/QSQL-NOTES.md
@@ -0,0 +1,99 @@
+
+# New beta version 3.1.0-beta is available, which contains the following major changes/additions:
+
+The following sections provide the details of each of the new functionality in this major version.
+
+## Automatic caching of data files
+Speeding up subsequent reads from the same file by several orders of magnitude by automatically creating an immutable cache file for each tabular text file.  
+
+For example, reading a 0.9GB file with 1M rows and 100 columns without caching takes ~50 seconds. When the cache exists, querying the same file will take less than 1 second. Obviously, the cache can be used in ord