wip

2 files changed, 50 insertions, 0 deletions

diff --git a/test/BENCHMARK.md b/test/BENCHMARK.md
index 8a7f0a4..fc225d3 100644
--- a/test/BENCHMARK.md
+++ b/test/BENCHMARK.md
@@ -26,6 +26,8 @@ OSX Sierra on a 15" Macbook Pro from Mid 2015, with 16GB of RAM, and an internal
 
 ## Running the benchmark
 
+Please note that the initial run generates big files, so you'd need more than 3GB of free space available. This also means that the first run will take much longer than additional runs. This is typical, and does not affect the benchmark results. All the generated files reside in the `_benchmark_data/` folder.
+
 * Create and activate a python 2.7 virtual environment called `py2-q`, and `pip install -r requirements.txt`
 * $ `./test-all BenchmarkTests.test_q_matrix`
 * Create and avtivate a python 3.x virtual environment called `py3-q`, and `pip install -r requirements.txt`
diff --git a/test/results/benchmark-results-2018-12-17 b/test/results/benchmark-results-2018-12-17
new file mode 100644
index 0000000..8d40754
--- /dev/null
+++ b/test/results/benchmark-results-2018-12-17
@@ -0,0 +1,48 @@
+lines	columns	py2-q_mean	py2-q_stddev	lines	columns	py3-q_mean	py3-q_stddev	lines	columns	textql_mean	textql_stddev
+1	1	0.06731581688	0.005270230559	1	1	0.09322199821	0.008088911233	1	1	0.01541593075	0.00846248027
+10	1	0.06453447342	0.003110529879	10	1	0.0952757597	0.01068078746	10	1	0.01273214817	0.001517273708
+100	1	0.06692070961	0.004081653457	100	1	0.09462814331	0.00550010348	100	1	0.01279251575	0.0007315880067
+1000	1	0.0703766346	0.002271640626	1000	1	0.09908235073	0.0085850761	1000	1	0.01575729847	0.001170010368
+10000	1	0.1229094744	0.005485221564	10000	1	0.1375562668	0.009702295105	10000	1	0.04378418922	0.001448525422
+100000	1	0.598156023	0.01721054649	100000	1	0.522838521	0.01662262184	100000	1	0.3162255287	0.01030908105
+1000000	1	5.372911286	0.0425664739	1000000	1	4.312362194	0.04878944441	1000000	1	3.042521834	0.02222183573
+lines	columns	py2-q_mean	py2-q_stddev	lines	columns	py3-q_mean	py3-q_stddev	lines	columns	textql_mean	textql_stddev
+1	5	0.06542704105	0.001973147455	1	5	0.09278903008	0.007920553711	1	5	0.01264638901	0.0009375946825
+10	5	0.06713621616	0.003302711249	10	5	0.09266264439	0.006464956796	10	5	0.01264002323	0.0005921679139
+100	5	0.07043097019	0.003513428229	100	5	0.09614286423	0.006232406135	100	5	0.01298532486	0.001484074702
+1000	5	0.07853364944	0.002677513043	1000	5	0.1007899046	0.009419248049	1000	5	0.01899263859	0.0005582728364
+10000	5	0.1847445965	0.006918806414	10000	5	0.151746726	0.007045195955	10000	5	0.07659320831	0.00297289199
+100000	5	1.206378174	0.01569912364	100000	5	0.6551784992	0.02468335852	100000	5	0.6256412745	0.009538934388
+1000000	5	11.4774132	0.2737370571	1000000	5	5.54825387	0.06392730387	1000000	5	6.174384165	0.0396257937
+lines	columns	py2-q_mean	py2-q_stddev	lines	columns	py3-q_mean	py3-q_stddev	lines	columns	textql_mean	textql_stddev
+1	10	0.06635277271	0.003224367089	1	10	0.09342534542	0.003372803039	1	10	0.01265852451	0.00115658081
+10	10	0.06949725151	0.004236749478	10	10	0.09139561653	0.00361962951	10	10	0.01304826736	0.0009077163448
+100	10	0.07332832813	0.003211229764	100	10	0.09613847733	0.002976111632	100	10	0.01362993717	0.0003077883843
+1000	10	0.09426920414	0.004147375078	1000	10	0.10503757	0.004323166227	1000	10	0.02448859215	0.001551123656
+10000	10	0.26318748	0.007391059562	10000	10	0.1713474512	0.004400747258	10000	10	0.1165221453	0.004626763279
+100000	10	1.939086366	0.01711379803	100000	10	0.8509856939	0.01451489164	100000	10	1.03131845	0.0154166
+1000000	10	19.16211414	0.3417997674	1000000	10	7.636127377	0.06577367856	1000000	10	10.22023973	0.0443451077
+lines	columns	py2-q_mean	py2-q_stddev	lines	columns	py3-q_mean	py3-q_stddev	lines	columns	textql_mean	textql_stddev
+1	20	0.06688520908	0.003686408801	1	20	0.0937997818	0.00504618112	1	20	0.01299088001	0.00130498302
+10	20	0.06709973812	0.003909120415	10	20	0.09303014278	0.004256698801	10	20	0.01291837692	0.001043654863
+100	20	0.0813845396	0.005158197903	100	20	0.1016526461	0.004238640414	100	20	0.01500227451	0.001216417242
+1000	20	0.1107584953	0.006723338286	1000	20	0.1139468193	0.005867712372	1000	20	0.03420743942	0.003094073019
+10000	20	0.4188146114	0.01474904378	10000	20	0.2173264027	0.005747071741	10000	20	0.1986592293	0.006588276071
+100000	20	3.461091924	0.1043205869	100000	20	1.287664986	0.0221862172	100000	20	1.829260516	0.01414616471
+1000000	20	33.20876031	0.3190789024	1000000	20	11.84579525	0.1406809832	1000000	20	18.15644448	0.1474355796
+lines	columns	py2-q_mean	py2-q_stddev	lines	columns	py3-q_mean	py3-q_stddev	lines	columns	textql_mean	textql_stddev
+1	50	0.06706497669	0.003487010206	1	50	0.09036362171	0.00392337182	1	50	0.0134802103	0.001043321639
+10	50	0.0721385479	0.00526657204	10	50	0.09356541634	0.003705587568	10	50	0.01397790909	0.001008071038
+100	50	0.1015130758	0.003524910234	100	50	0.1168865919	0.002810940717	100	50	0.01766057014	0.0008818513382
+1000	50	0.1666964769	0.006661858999	1000	50	0.1373265505	0.004538848823	1000	50	0.05760366917	0.003787637225
+10000	50	0.8726647139	0.04817920962	10000	50	0.3499189854	0.006489403179	10000	50	0.4113406658	0.00551681222
+100000	50	7.659929824	0.1190133198	100000	50	2.486357236	0.04149367418	100000	50	4.023236489	0.02935989293
+1000000	50	75.64912643	1.036366669	1000000	50	23.88283024	0.4251339799	1000000	50	40.02736287	0.3879349969
+lines	columns	py2-q_mean	py2-q_stddev	lines	columns	py3-q_mean	py3-q_stddev	lines	columns	textql_mean	textql_stddev
+1	100	0.06666021347	0.001720503522	1	100	0.09272692204	0.005532725603	1	100	0.01451745033	0.0009589603269
+10	100	0.0746655941	0.004541222011	10	100	0.09874138832	0.007172096503	10	100	0.0155831337	0.001020332488
+100	100	0.1330797672	0.004335602846	100	100	0.1412571669	0.008253862291	100	100	0.02391133308	0.001714142787
+1000	100	0.2642062426	0.01022737492	1000	100	0.1779050112	0.006555498616	1000	100	0.09285030365	0.002734967858
+10000	100	1.570353174	0.01475258288	10000	100	0.5818499565	0.01616512044	10000	100	0.779653573	0.01021001276
+100000	100	14.70140581	0.3328709764	100000	100	4.601756811	0.05434568891	100000	100	7.700500083	0.06577229359
+1000000	100	148.4634018	7.316550329	1000000	100	44.62859902	0.4333388333	1000000	100	77.977897	0.7301257528