Benchmark with hyperfine on commit history

3 files changed, 52 insertions, 0 deletions

diff --git a/Makefile b/Makefile
index 0668eeae..52efd618 100644
--- a/Makefile
+++ b/Makefile
@@ -10,3 +10,8 @@ test:
                                           --file-style plain \
                                           --hunk-style plain \
                                   | ansifilter | cut -c 2-)"
+
+hyperfine:
+	./scripts/build-commits.sh master
+	git log -p 23c292d3f25c67082a2ba315a187268be1a9b0ab > /tmp/input.gitdiff
+	./scripts/hyperfine-commits.py /tmp/input.gitdiff /tmp/hyperfine.json /tmp/delta
diff --git a/scripts/build-commits.sh b/scripts/build-commits.sh
new file mode 100755
index 00000000..291cb474
--- /dev/null
+++ b/scripts/build-commits.sh
@@ -0,0 +1,22 @@
+#!/bin/bash
+
+set -e
+
+build () {
+    local commit=$1
+    local i=$2
+    local outfile=$outdir/$(printf "%04s" $i)-$commit
+    [ -e $outfile ] && return
+    echo $outfile
+    git checkout --quiet $commit
+    cargo build --release 2>&1 > /dev/null && \
+        mv target/release/delta $outfile || \
+        echo build failed: $commit
+}
+
+original_head=$(git rev-parse --abbrev-ref HEAD)
+outdir=/tmp/delta
+mkdir -p $outdir
+i=1
+git rev-list $@ | tac | while read commit; do build $commit $i; ((i+=1)); done
+git checkout $original_head
diff --git a/scripts/hyperfine-commits.py b/scripts/hyperfine-commits.py
new file mode 100755
index 00000000..a7a85ec9
--- /dev/null
+++ b/scripts/hyperfine-commits.py
@@ -0,0 +1,25 @@
+#!/usr/bin/env python
+import argparse
+import os
+import subprocess
+
+parser = argparse.ArgumentParser()
+parser.add_argument("input_file", help="Input file for each command")
+parser.add_argument("output_file", help="File to receive JSON output")
+parser.add_argument("executables_dir", help="Directory of executables to be benchmarked")
+args = parser.parse_args()
+
+runs = 5
+hyperfine_args = [
+    "hyperfine",
+    "--runs",
+    f"{runs}",
+    "--export-json",
+    f"{args.output_file}",
+    "--ignore-failure",
+]
+hyperfine_args.extend(
+    f"{args.executables_dir}/{executable} < {args.input_file} > /dev/null"
+    for executable in sorted(os.listdir(args.executables_dir))
+)
+subprocess.check_call(hyperfine_args)