remove sqlitebck dependency entirely, and check if windows can build properly without it

7 files changed, 10 insertions, 13 deletions

diff --git a/.github/workflows/build-and-package.yaml b/.github/workflows/build-and-package.yaml
index f723786..acf2b56 100644
--- a/.github/workflows/build-and-package.yaml
+++ b/.github/workflows/build-and-package.yaml
@@ -304,8 +304,10 @@ jobs:
 
         mkdir brew
 
+        export BRANCH_NAME=$(echo ${GITHUB_REF#refs/heads/})
+
         # TODO temp, since template rendering action doesn't work in mac
-        cat .github/workflows/q.rb.brew-formula-template | sed 's/{{ .Q_VERSION }}/3.1.0-beta/g' > ./brew/q.rb
+        cat .github/workflows/q.rb.brew-formula-template | sed 's/{{ .Q_VERSION }}/3.1.0-beta/g' | sed "s/{{ .Q_BRANCH_NAME }}/${BRANCH_NAME}/g" > ./brew/q.rb
 
         brew install --display-times --formula --build-bottle --verbose ./brew/q.rb
         brew test ./brew/q.rb
@@ -352,10 +354,8 @@ jobs:
 
         echo "select sum(c1),count(*) from data_stream_stdin" | sqlite3 test.sqlite
 
-  # TODO Windows build/test/package flow is running, but q executable is still not running well, due to pyox+sqlite3 issue
   build-windows:
     runs-on: windows-latest
-    if: ${{ false }}
     steps:
     - name: Checkout
       uses: actions/checkout@v2
@@ -380,7 +380,7 @@ jobs:
       run: |
         set -e -x
 
-        # Hack to overcome the fact that apsw doesn't have a registered wheel for Windows
+        # TODO win requirements are now the same, so this can go away hopefully
         cp requirements-win-x86_64.txt requirements.txt
 
         pyoxidizer build --release
@@ -488,7 +488,7 @@ jobs:
   perform-prerelease:
     # We'd like artifacts to be uploaded regardless of tests succeeded or not,
     # this is why the dependency here is not on test-X-packaging jobs
-    needs: [package-linux-deb, package-linux-rpm, package-mac]
+    needs: [package-linux-deb, package-linux-rpm, package-mac, package-windows]
     runs-on: ubuntu-latest
     # TODO Push to master will now pre-release as well, until things stabilize
     # if: ${{ github.event_name == 'pull_request' }}
diff --git a/.github/workflows/q.rb.brew-formula-template b/.github/workflows/q.rb.brew-formula-template
index c60fb43..15b77d0 100644
--- a/.github/workflows/q.rb.brew-formula-template
+++ b/.github/workflows/q.rb.brew-formula-template
@@ -4,8 +4,8 @@
 class Q < Formula
   desc "Run SQL directly on CSV or TSV files"
   homepage "https://harelba.github.io/q/"
-  # Building directly from master for now, eventually it will be tag-based so the version tag will be downloaded
-  url "https://github.com/harelba/q/archive/master.tar.gz"
+  # Using branch name for pre-releases, for tagged releases this would be the version tag, and not "version" part will be needed
+  url "https://github.com/harelba/q/archive/{{ .Q_BRANCH_NAME }}.tar.gz"
   version "{{ .Q_VERSION }}"
 
   # Removed for now, until everything is finalized
diff --git a/QSQL-NOTES.md b/QSQL-NOTES.md
index 48fa328..0334fe6 100644
--- a/QSQL-NOTES.md
+++ b/QSQL-NOTES.md
@@ -20,7 +20,7 @@ time q -c 1 "select sum(c1),count(*) from myfile.csv" -C readwrite
 500000500000 1000000
 q -c 1 "select sum(c1),count(*) from myfile.csv" -C readwrite  3.96s user 0.08s system 99% cpu 4.057 total
 
-# Now run with `-C read`. The query will run from the cache file and not the original. Change the query and run it several times, to notice the difference in speed.
+# Now run with `-C read`. The query will run from the cache file and not the original. Change the query and run it several times, to notice the difference in speed. As the file gets bigger, the difference will be much more noticable
 $ time q -c 1 "select sum(c1),count(*) from myfile.csv" -C read
 500000500000 1000000
 q -c 1 "select sum(c1),count(*) from myfile.csv" -C read  0.17s user 0.05s system 94% cpu 0.229 total
diff --git a/bin/q.py b/bin/q.py
index a71fec4..197f007 100755
--- a/bin/q.py
+++ b/bin/q.py
@@ -56,7 +56,6 @@ import math
 import six
 import io
 import json
-import sqlitebck
 import datetime
 import hashlib
 
@@ -1699,7 +1698,8 @@ class MaterializedDelimitedFileState(MaterializedState):
     def _store_qsql(self, source_sqlite_db, disk_db_filename):
         xprint("Storing data as disk db")
         disk_db_conn = sqlite3.connect(disk_db_filename)
-        sqlitebck.copy(source_sqlite_db.conn,disk_db_conn)
+        with disk_db_conn:
+            source_sqlite_db.conn.backup(disk_db_conn)
         xprint("Written db to disk: disk db filename %s" % (disk_db_filename))
         disk_db_conn.close()
 
diff --git a/requirements-win-x86_64.txt b/requirements-win-x86_64.txt
index 69d4961..3ad7d2b 100644
--- a/requirements-win-x86_64.txt
+++ b/requirements-win-x86_64.txt
@@ -1,4 +1,3 @@
 six==1.11.0
 flake8==3.6.0
 setuptools<45.0.0
-# Turns out it would not be possible to use apsw without a major change all around, so we'd need to be able to compile sqlitebck on Windows as well
diff --git a/requirements.txt b/requirements.txt
index 9661caf..3ad7d2b 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -1,4 +1,3 @@
 six==1.11.0
 flake8==3.6.0
 setuptools<45.0.0
-sqlitebck
diff --git a/test-requirements.txt b/test-requirements.txt
index bb72fa5..7e995ba 100644
--- a/test-requirements.txt
+++ b/test-requirements.txt
@@ -1,4 +1,3 @@
 pytest==6.2.2
 flake8==3.6.0
 six==1.11.0
-sqlitebck