nixpkgs: afl 1.58b -> 1.63b

  - Adds new LLVM-based instrumentation support via afl-clang-fast.
  - Experimental support for cgroup management via afl-cgroup, to
    mitigate OOM issues when using afl with address sanitizer.

Signed-off-by: Austin Seipp <aseipp@pobox.com>

5 files changed, 40 insertions, 12 deletions

diff --git a/pkgs/tools/security/afl/README.md b/pkgs/tools/security/afl/README.md
index 7d9544617739..0b0d4e80fbca 100644
--- a/pkgs/tools/security/afl/README.md
+++ b/pkgs/tools/security/afl/README.md
@@ -11,9 +11,9 @@ right QEMU version and options in `qemu.nix`:
 
 https://github.com/mirrorer/afl/blob/master/qemu_mode/build_qemu_support.sh
 
-`afl-config.h` and `afl-qemu-cpu-inl.h` are part of the afl source
-code, and copied from `config.h` and `afl-qemu-cpu-inl.h`
-appropriately. The QEMU patches need to be slightly adjusted to
-`#include` these files (the patches try to otherwise include files
-like `../../config.h` which causes the build to fail). See `qemu.nix`
-for details.
+`afl-config.h`, `afl-types.h`, and `afl-qemu-cpu-inl.h` are part of
+the afl source code, and copied from `config.h`, `types.h` and
+`afl-qemu-cpu-inl.h` appropriately. These files and the QEMU patches
+need to be slightly adjusted to fix their `#include`s (the patches
+try to otherwise include files like `../../config.h` which causes the
+build to fail).
diff --git a/pkgs/tools/security/afl/default.nix b/pkgs/tools/security/afl/default.nix
index 93c5988b55d4..ef9fa1f80221 100644
--- a/pkgs/tools/security/afl/default.nix
+++ b/pkgs/tools/security/afl/default.nix
@@ -1,4 +1,5 @@
-{ stdenv, fetchurl, bash, callPackage, makeWrapper }:
+{ stdenv, fetchurl, bash, callPackage, makeWrapper
+, clang, llvm, which, libcgroup }:
 
 let
   afl-qemu = callPackage ./qemu.nix {};
@@ -8,16 +9,21 @@ let
 in
 stdenv.mkDerivation rec {
   name    = "afl-${version}";
-  version = "1.58b";
+  version = "1.63b";
 
   src = fetchurl {
     url    = "http://lcamtuf.coredump.cx/afl/releases/${name}.tgz";
-    sha256 = "1szggm4x9i9bsrcb99s5vbgncagp7jvhz8cg9amkx7p6mp2x4pld";
+    sha256 = "1v3py0g52j687qacwhri8jbz2h0ggh3zqknp011z5ijf820vc09g";
   };
 
-  buildInputs  = [ makeWrapper ];
+  # Note: libcgroup isn't needed for building, just for the afl-cgroup
+  # script.
+  buildInputs  = [ makeWrapper clang llvm which ];
 
-  buildPhase   = "make PREFIX=$out";
+  buildPhase   = ''
+    make PREFIX=$out
+    cd llvm_mode && make && cd ..
+  '';
   installPhase = ''
     # Do the normal installation
     make install PREFIX=$out
@@ -25,6 +31,17 @@ stdenv.mkDerivation rec {
     # Install the custom QEMU emulator for binary blob fuzzing.
     cp ${afl-qemu}/bin/${qemu-exe-name} $out/bin/afl-qemu-trace
 
+    # Install the cgroups wrapper for asan-based fuzzing.
+    cp experimental/asan_cgroups/limit_memory.sh $out/bin/afl-cgroup
+    chmod +x $out/bin/afl-cgroup
+    substituteInPlace $out/bin/afl-cgroup \
+      --replace "cgcreate" "${libcgroup}/bin/cgcreate" \
+      --replace "cgexec"   "${libcgroup}/bin/cgexec" \
+      --replace "cgdelete" "${libcgroup}/bin/cgdelete"
+
+    # Patch shebangs before wrapping
+    patchShebangs $out/bin
+
     # Wrap every program with a custom $AFL_PATH; I believe there is a
     # bug in afl which causes it to fail to find `afl-qemu-trace`
     # relative to `afl-fuzz` or `afl-showmap`, so we instead set
diff --git a/pkgs/tools/security/afl/qemu-patches/afl-config.h b/pkgs/tools/security/afl/qemu-patches/afl-config.h
index 051b38ffbca4..0017f9d83f4b 100644
--- a/pkgs/tools/security/afl/qemu-patches/afl-config.h
+++ b/pkgs/tools/security/afl/qemu-patches/afl-config.h
@@ -138,6 +138,11 @@
 
 #define TMIN_MAX_FILE       (10 * 1024 * 1024)
 
+/* Block normalization steps for afl-tmin: */
+
+#define TMIN_SET_MIN_SIZE   4
+#define TMIN_SET_STEPS      128
+
 /* Maximum dictionary token size (-x), in bytes: */
 
 #define MAX_DICT_FILE       128
diff --git a/pkgs/tools/security/afl/qemu-patches/afl-qemu-cpu-inl.h b/pkgs/tools/security/afl/qemu-patches/afl-qemu-cpu-inl.h
index c6ebc873ae09..e4a470b55230 100644
--- a/pkgs/tools/security/afl/qemu-patches/afl-qemu-cpu-inl.h
+++ b/pkgs/tools/security/afl/qemu-patches/afl-qemu-cpu-inl.h
@@ -134,6 +134,12 @@ static void afl_setup(void) {
 
     if (afl_area_ptr == (void*)-1) exit(1);
 
+    /* With AFL_INST_RATIO set to a low value, we want to touch the bitmap
+       so that the parent doesn't give up on us. */
+
+    if (inst_r) afl_area_ptr[0] = 1;
+
+
   }
 
   if (getenv("AFL_INST_LIBS")) {
diff --git a/pkgs/tools/security/afl/qemu.nix b/pkgs/tools/security/afl/qemu.nix
index 441d64415cbd..5b69811d08d6 100644
--- a/pkgs/tools/security/afl/qemu.nix
+++ b/pkgs/tools/security/afl/qemu.nix
@@ -65,7 +65,7 @@ stdenv.mkDerivation rec {
 
   meta = with stdenv.lib; {
     homepage = http://www.qemu.org/;
-    description = "Fork of QEMU with American Fuzzy Lop instrumentation support";
+    description = "Fork of QEMU with AFL instrumentation support";
     license = licenses.gpl2Plus;
     maintainers = with maintainers; [ thoughtpolice ];
     platforms = platforms.linux;