setcap-wrapper: Merging with upstream master and resolving conflicts

44 files changed, 774 insertions, 348 deletions

diff --git a/pkgs/build-support/build-fhs-userenv/env.nix b/pkgs/build-support/build-fhs-userenv/env.nix
index f69338cb16cc..b30e1362aba9 100644
--- a/pkgs/build-support/build-fhs-userenv/env.nix
+++ b/pkgs/build-support/build-fhs-userenv/env.nix
@@ -89,6 +89,7 @@ let
 
       # symlink other core stuff
       ln -s /host/etc/localtime localtime
+      ln -s /host/etc/zoneinfo zoneinfo
       ln -s /host/etc/machine-id machine-id
       ln -s /host/etc/os-release os-release
 
diff --git a/pkgs/build-support/cc-wrapper/cc-wrapper.sh b/pkgs/build-support/cc-wrapper/cc-wrapper.sh
index 03f068d8298e..3ccdc34db5b2 100644
--- a/pkgs/build-support/cc-wrapper/cc-wrapper.sh
+++ b/pkgs/build-support/cc-wrapper/cc-wrapper.sh
@@ -24,7 +24,7 @@ nonFlagArgs=0
 [[ "@prog@" = *++ ]] && isCpp=1 || isCpp=0
 cppInclude=1
 
-params=("$@")
+expandResponseParams "$@"
 n=0
 while [ $n -lt ${#params[*]} ]; do
     p=${params[n]}
diff --git a/pkgs/build-support/cc-wrapper/default.nix b/pkgs/build-support/cc-wrapper/default.nix
index 8a746ea016ef..c8e3d8b4cc82 100644
--- a/pkgs/build-support/cc-wrapper/default.nix
+++ b/pkgs/build-support/cc-wrapper/default.nix
@@ -46,7 +46,20 @@ stdenv.mkDerivation {
   inherit cc shell libc_bin libc_dev libc_lib binutils_bin coreutils_bin;
   gnugrep_bin = if nativeTools then "" else gnugrep;
 
-  passthru = { inherit libc nativeTools nativeLibc nativePrefix isGNU isClang; };
+  passthru = {
+    inherit libc nativeTools nativeLibc nativePrefix isGNU isClang;
+
+    emacsBufferSetup = pkgs: ''
+      ; We should handle propagation here too
+      (mapc (lambda (arg)
+        (when (file-directory-p (concat arg "/include"))
+          (setenv "NIX_CFLAGS_COMPILE" (concat (getenv "NIX_CFLAGS_COMPILE") " -isystem " arg "/include")))
+        (when (file-directory-p (concat arg "/lib"))
+          (setenv "NIX_LDFLAGS" (concat (getenv "NIX_LDFLAGS") " -L" arg "/lib")))
+        (when (file-directory-p (concat arg "/lib64"))
+          (setenv "NIX_LDFLAGS" (concat (getenv "NIX_LDFLAGS") " -L" arg "/lib64")))) '(${concatStringsSep " " (map (pkg: "\"${pkg}\"") pkgs)}))
+    '';
+  };
 
   buildCommand =
     ''
@@ -239,10 +252,10 @@ stdenv.mkDerivation {
 
       # some linkers on some platforms don't support specific -z flags
       hardening_unsupported_flags=""
-      if [[ "$($ldPath/ld -z now 2>&1 || true)" =~ "unknown option" ]]; then
+      if [[ "$($ldPath/ld -z now 2>&1 || true)" =~ un(recognized|known)\ option ]]; then
         hardening_unsupported_flags+=" bindnow"
       fi
-      if [[ "$($ldPath/ld -z relro 2>&1 || true)" =~ "unknown option" ]]; then
+      if [[ "$($ldPath/ld -z relro 2>&1 || true)" =~ un(recognized|known)\ option ]]; then
         hardening_unsupported_flags+=" relro"
       fi
 
diff --git a/pkgs/build-support/cc-wrapper/ld-solaris-wrapper.sh b/pkgs/build-support/cc-wrapper/ld-solaris-wrapper.sh
index 5a7b92b5ad7d..263ea5408e9a 100644..100755
--- a/pkgs/build-support/cc-wrapper/ld-solaris-wrapper.sh
+++ b/pkgs/build-support/cc-wrapper/ld-solaris-wrapper.sh
@@ -1,4 +1,4 @@
-#!/bin/bash
+#!@shell@
 
 set -e
 set -u
diff --git a/pkgs/build-support/cc-wrapper/ld-wrapper.sh b/pkgs/build-support/cc-wrapper/ld-wrapper.sh
index 44d9a047936a..056cfa920535 100644
--- a/pkgs/build-support/cc-wrapper/ld-wrapper.sh
+++ b/pkgs/build-support/cc-wrapper/ld-wrapper.sh
@@ -16,7 +16,7 @@ source @out@/nix-support/utils.sh
 
 
 # Optionally filter out paths not refering to the store.
-params=("$@")
+expandResponseParams "$@"
 if [ "$NIX_ENFORCE_PURITY" = 1 -a -n "$NIX_STORE" \
         -a \( -z "$NIX_IGNORE_LD_THROUGH_GCC" -o -z "$NIX_LDFLAGS_SET" \) ]; then
     rest=()
diff --git a/pkgs/build-support/cc-wrapper/utils.sh b/pkgs/build-support/cc-wrapper/utils.sh
index 3ab512d85c4e..aba5f3295a98 100644
--- a/pkgs/build-support/cc-wrapper/utils.sh
+++ b/pkgs/build-support/cc-wrapper/utils.sh
@@ -22,3 +22,27 @@ badPath() {
         "${p:0:4}" != "/tmp" -a \
         "${p:0:${#NIX_BUILD_TOP}}" != "$NIX_BUILD_TOP"
 }
+
+expandResponseParams() {
+    local inparams=("$@")
+    local n=0
+    local p
+    params=()
+    while [ $n -lt ${#inparams[*]} ]; do
+        p=${inparams[n]}
+        case $p in
+            @*)
+                if [ -e "${p:1}" ]; then
+                    args=$(<"${p:1}")
+                    eval 'for arg in '${args//$/\\$}'; do params+=("$arg"); done'
+                else
+                    params+=("$p")
+                fi
+                ;;
+            *)
+                params+=("$p")
+                ;;
+        esac
+        n=$((n + 1))
+    done
+}
diff --git a/pkgs/build-support/docker/default.nix b/pkgs/build-support/docker/default.nix
index 4c5378ea73f3..27575053954f 100644
--- a/pkgs/build-support/docker/default.nix
+++ b/pkgs/build-support/docker/default.nix
@@ -1,12 +1,36 @@
-{ stdenv, lib, callPackage, runCommand, writeReferencesToFile, writeText, vmTools, writeScript
-, docker, shadow, utillinux, coreutils, jshon, e2fsprogs, go, pigz }:
+{
+  callPackage,
+  coreutils,
+  docker,
+  e2fsprogs,
+  findutils,
+  go,
+  jshon,
+  lib,
+  pkgs,
+  pigz,
+  runCommand,
+  rsync,
+  shadow,
+  stdenv,
+  storeDir ? builtins.storeDir,
+  utillinux,
+  vmTools,
+  writeReferencesToFile,
+  writeScript,
+  writeText,
+}:
 
 # WARNING: this API is unstable and may be subject to backwards-incompatible changes in the future.
-  
+
 rec {
 
+  examples = import ./examples.nix {
+    inherit pkgs buildImage pullImage shadowSetup;
+  };
+
   pullImage = callPackage ./pull.nix {};
-  
+
   # We need to sum layer.tar, not a directory, hence tarsum instead of nix-hash.
   # And we cannot untar it, because then we cannot preserve permissions ecc.
   tarsum = runCommand "tarsum" {
@@ -23,110 +47,138 @@ rec {
 
     cp tarsum $out
   '';
-  
+
   # buildEnv creates symlinks to dirs, which is hard to edit inside the overlay VM
-  mergeDrvs = { drvs, onlyDeps ? false }:
+  mergeDrvs = {
+    derivations,
+    onlyDeps ? false
+  }:
     runCommand "merge-drvs" {
-      inherit drvs onlyDeps;
+      inherit derivations onlyDeps;
     } ''
-      if [ -n "$onlyDeps" ]; then
-        echo $drvs > $out
+      if [[ -n "$onlyDeps" ]]; then
+        echo $derivations > $out
         exit 0
       fi
-        
+
       mkdir $out
-      for drv in $drvs; do
-        echo Merging $drv
-        if [ -d "$drv" ]; then
-          cp -drf --preserve=mode -f $drv/* $out/
+      for derivation in $derivations; do
+        echo "Merging $derivation..."
+        if [[ -d "$derivation" ]]; then
+          # If it's a directory, copy all of its contents into $out.
+          cp -drf --preserve=mode -f $derivation/* $out/
         else
+          # Otherwise treat the derivation as a tarball and extract it
+          # into $out.
           tar -C $out -xpf $drv || true
         fi
       done
     '';
-  
-  shellScript = text:
-    writeScript "script.sh" ''
-      #!${stdenv.shell}
-      set -e
-      export PATH=${coreutils}/bin:/bin
-
-      ${text}
-    '';
 
+  # Helper for setting up the base files for managing users and
+  # groups, only if such files don't exist already. It is suitable for
+  # being used in a runAsRoot script.
   shadowSetup = ''
     export PATH=${shadow}/bin:$PATH
     mkdir -p /etc/pam.d
-    if [ ! -f /etc/passwd ]; then
+    if [[ ! -f /etc/passwd ]]; then
       echo "root:x:0:0::/root:/bin/sh" > /etc/passwd
       echo "root:!x:::::::" > /etc/shadow
     fi
-    if [ ! -f /etc/group ]; then
+    if [[ ! -f /etc/group ]]; then
       echo "root:x:0:" > /etc/group
       echo "root:x::" > /etc/gshadow
     fi
-    if [ ! -f /etc/pam.d/other ]; then
+    if [[ ! -f /etc/pam.d/other ]]; then
       cat > /etc/pam.d/other <<EOF
-account sufficient pam_unix.so
-auth sufficient pam_rootok.so
-password requisite pam_unix.so nullok sha512
-session required pam_unix.so
-EOF
+    account sufficient pam_unix.so
+    auth sufficient pam_rootok.so
+    password requisite pam_unix.so nullok sha512
+    session required pam_unix.so
+    EOF
     fi
-    if [ ! -f /etc/login.defs ]; then
+    if [[ ! -f /etc/login.defs ]]; then
       touch /etc/login.defs
     fi
   '';
 
-  runWithOverlay = { name , fromImage ? null, fromImageName ? null, fromImageTag ? null
-                   , diskSize ? 1024, preMount ? "", postMount ? "", postUmount ? "" }:
+  # Run commands in a virtual machine.
+  runWithOverlay = {
+    name,
+    fromImage ? null,
+    fromImageName ? null,
+    fromImageTag ? null,
+    diskSize ? 1024,
+    preMount ? "",
+    postMount ? "",
+    postUmount ? ""
+  }:
     vmTools.runInLinuxVM (
       runCommand name {
-        preVM = vmTools.createEmptyImage { size = diskSize; fullName = "docker-run-disk"; };
-
+        preVM = vmTools.createEmptyImage {
+          size = diskSize;
+          fullName = "docker-run-disk";
+        };
         inherit fromImage fromImageName fromImageTag;
-        
-        buildInputs = [ utillinux e2fsprogs jshon ];
+
+        buildInputs = [ utillinux e2fsprogs jshon rsync ];
       } ''
       rm -rf $out
-      
+
       mkdir disk
       mkfs /dev/${vmTools.hd}
       mount /dev/${vmTools.hd} disk
       cd disk
 
-      if [ -n "$fromImage" ]; then
-        echo Unpacking base image
+      if [[ -n "$fromImage" ]]; then
+        echo "Unpacking base image..."
         mkdir image
         tar -C image -xpf "$fromImage"
 
-        if [ -z "$fromImageName" ]; then
-          fromImageName=$(jshon -k < image/repositories|head -n1)
+        # If the image name isn't set, read it from the image repository json.
+        if [[ -z "$fromImageName" ]]; then
+          fromImageName=$(jshon -k < image/repositories | head -n 1)
+          echo "From-image name wasn't set. Read $fromImageName."
         fi
-        if [ -z "$fromImageTag" ]; then
-          fromImageTag=$(jshon -e $fromImageName -k < image/repositories|head -n1)
+
+        # If the tag isn't set, use the name as an index into the json
+        # and read the first key found.
+        if [[ -z "$fromImageTag" ]]; then
+          fromImageTag=$(jshon -e $fromImageName -k < image/repositories \
+                         | head -n1)
+          echo "From-image tag wasn't set. Read $fromImageTag."
         fi
-        parentID=$(jshon -e $fromImageName -e $fromImageTag -u < image/repositories)
+
+        # Use the name and tag to get the parent ID field.
+        parentID=$(jshon -e $fromImageName -e $fromImageTag -u \
+                   < image/repositories)
       fi
 
+      # Unpack all of the parent layers into the image.
       lowerdir=""
-      while [ -n "$parentID" ]; do
-        echo Unpacking layer $parentID
+      while [[ -n "$parentID" ]]; do
+        echo "Unpacking layer $parentID"
         mkdir -p image/$parentID/layer
         tar -C image/$parentID/layer -xpf image/$parentID/layer.tar
         rm image/$parentID/layer.tar
 
         find image/$parentID/layer -name ".wh.*" -exec bash -c 'name="$(basename {}|sed "s/^.wh.//")"; mknod "$(dirname {})/$name" c 0 0; rm {}' \;
 
+        # Get the next lower directory and continue the loop.
         lowerdir=$lowerdir''${lowerdir:+:}image/$parentID/layer
-        parentID=$(cat image/$parentID/json|(jshon -e parent -u 2>/dev/null || true))
+        parentID=$(cat image/$parentID/json \
+                  | (jshon -e parent -u 2>/dev/null || true))
       done
 
       mkdir work
       mkdir layer
       mkdir mnt
 
-      ${preMount}
+      ${lib.optionalString (preMount != "") ''
+        # Execute pre-mount steps
+        echo "Executing pre-mount steps..."
+        ${preMount}
+      ''}
 
       if [ -n "$lowerdir" ]; then
         mount -t overlay overlay -olowerdir=$lowerdir,workdir=work,upperdir=layer mnt
@@ -134,13 +186,19 @@ EOF
         mount --bind layer mnt
       fi
 
-      ${postMount}
- 
+      ${lib.optionalString (postMount != "") ''
+        # Execute post-mount steps
+        echo "Executing post-mount steps..."
+        ${postMount}
+      ''}
+
       umount mnt
 
-      pushd layer
-      find . -type c -exec bash -c 'name="$(basename {})"; touch "$(dirname {})/.wh.$name"; rm "{}"' \;
-      popd
+      (
+        cd layer
+        cmd='name="$(basename {})"; touch "$(dirname {})/.wh.$name"; rm "{}"'
+        find . -type c -exec bash -c "$cmd" \;
+      )
 
       ${postUmount}
       '');
@@ -150,76 +208,148 @@ EOF
       inherit name fromImage fromImageName fromImageTag diskSize;
 
       postMount = ''
-        echo Packing raw image
+        echo "Packing raw image..."
         tar -C mnt --mtime=0 -cf $out .
       '';
     };
-    
-  mkPureLayer = { baseJson, contents ? null, extraCommands ? "" }:
-    runCommand "docker-layer" {
+
+
+  # Create an executable shell script which has the coreutils in its
+  # PATH. Since root scripts are executed in a blank environment, even
+  # things like `ls` or `echo` will be missing.
+  shellScript = name: text:
+    writeScript name ''
+      #!${stdenv.shell}
+      set -e
+      export PATH=${coreutils}/bin:/bin
+      ${text}
+    '';
+
+  # Create a "layer" (set of files).
+  mkPureLayer = {
+    # Name of the layer
+    name,
+    # JSON containing configuration and metadata for this layer.
+    baseJson,
+    # Files to add to the layer.
+    contents ? null,
+    # Additional commands to run on the layer before it is tar'd up.
+    extraCommands ? ""
+  }:
+    runCommand "docker-layer-${name}" {
       inherit baseJson contents extraCommands;
 
-      buildInputs = [ jshon ];
-    } ''
+      buildInputs = [ jshon rsync ];
+    }
+    ''
       mkdir layer
-      if [ -n "$contents" ]; then
-        echo Adding contents
-        for c in $contents; do
-          cp -drf $c/* layer/
-          chmod -R ug+w layer/
+      if [[ -n "$contents" ]]; then
+        echo "Adding contents..."
+        for item in $contents; do
+          echo "Adding $item"
+          rsync -ak $item/ layer/
         done
+      else
+        echo "No contents to add to layer."
+      fi
+
+      if [[ -n $extraCommands ]]; then
+        (cd layer; eval "$extraCommands")
       fi
 
-      pushd layer
-      ${extraCommands}
-      popd
-      
-      echo Packing layer
+      # Tar up the layer and throw it into 'layer.tar'.
+      echo "Packing layer..."
       mkdir $out
       tar -C layer --mtime=0 -cf $out/layer.tar .
-      ts=$(${tarsum} < $out/layer.tar)
-      cat ${baseJson} | jshon -s "$ts" -i checksum > $out/json
+
+      # Compute a checksum of the tarball.
+      echo "Computing layer checksum..."
+      tarsum=$(${tarsum} < $out/layer.tar)
+
+      # Add a 'checksum' field to the JSON, with the value set to the
+      # checksum of the tarball.
+      cat ${baseJson} | jshon -s "$tarsum" -i checksum > $out/json
+
+      # Indicate to docker that we're using schema version 1.0.
       echo -n "1.0" > $out/VERSION
+
+      echo "Finished building layer '${name}'"
     '';
 
-  mkRootLayer = { runAsRoot, baseJson, fromImage ? null, fromImageName ? null, fromImageTag ? null
-                , diskSize ? 1024, contents ? null, extraCommands ? "" }:
-    let runAsRootScript = writeScript "run-as-root.sh" runAsRoot;
+  # Make a "root" layer; required if we need to execute commands as a
+  # privileged user on the image. The commands themselves will be
+  # performed in a virtual machine sandbox.
+  mkRootLayer = {
+    # Name of the image.
+    name,
+    # Script to run as root. Bash.
+    runAsRoot,
+    # Files to add to the layer. If null, an empty layer will be created.
+    contents ? null,
+    # JSON containing configuration and metadata for this layer.
+    baseJson,
+    # Existing image onto which to append the new layer.
+    fromImage ? null,
+    # Name of the image we're appending onto.
+    fromImageName ? null,
+    # Tag of the image we're appending onto.
+    fromImageTag ? null,
+    # How much disk to allocate for the temporary virtual machine.
+    diskSize ? 1024,
+    # Commands (bash) to run on the layer; these do not require sudo.
+    extraCommands ? ""
+  }:
+    # Generate an executable script from the `runAsRoot` text.
+    let runAsRootScript = shellScript "run-as-root.sh" runAsRoot;
     in runWithOverlay {
-      name = "docker-layer";
-      
+      name = "docker-layer-${name}";
+
       inherit fromImage fromImageName fromImageTag diskSize;
 
-      preMount = lib.optionalString (contents != null) ''
-        echo Adding contents
-        for c in ${builtins.toString contents}; do
-          cp -drf $c/* layer/
-          chmod -R ug+w layer/
+      preMount = lib.optionalString (contents != null && contents != []) ''
+        echo "Adding contents..."
+        for item in ${toString contents}; do
+          echo "Adding $item..."
+          rsync -ak $item/ layer/
         done
       '';
 
       postMount = ''
-        mkdir -p mnt/{dev,proc,sys,nix/store}
+        mkdir -p mnt/{dev,proc,sys} mnt${storeDir}
+
+        # Mount /dev, /sys and the nix store as shared folders.
         mount --rbind /dev mnt/dev
         mount --rbind /sys mnt/sys
-        mount --rbind /nix/store mnt/nix/store
+        mount --rbind ${storeDir} mnt${storeDir}
 
+        # Execute the run as root script. See 'man unshare' for
+        # details on what's going on here; basically this command
+        # means that the runAsRootScript will be executed in a nearly
+        # completely isolated environment.
         unshare -imnpuf --mount-proc chroot mnt ${runAsRootScript}
-        umount -R mnt/dev mnt/sys mnt/nix/store
-        rmdir --ignore-fail-on-non-empty mnt/dev mnt/proc mnt/sys mnt/nix/store mnt/nix
+
+        # Unmount directories and remove them.
+        umount -R mnt/dev mnt/sys mnt${storeDir}
+        rmdir --ignore-fail-on-non-empty \
+          mnt/dev mnt/proc mnt/sys mnt${storeDir} \
+          mnt$(dirname ${storeDir})
       '';
- 
+
       postUmount = ''
-        pushd layer
-        ${extraCommands}
-        popd
+        (cd layer; eval "${extraCommands}")
 
-        echo Packing layer
+        echo "Packing layer..."
         mkdir $out
         tar -C layer --mtime=0 -cf $out/layer.tar .
+
+        # Compute the tar checksum and add it to the output json.
+        echo "Computing checksum..."
         ts=$(${tarsum} < $out/layer.tar)
         cat ${baseJson} | jshon -s "$ts" -i checksum > $out/json
+        # Indicate to docker that we're using schema version 1.0.
         echo -n "1.0" > $out/VERSION
+
+        echo "Finished building layer '${name}'"
       '';
     };
 
@@ -229,105 +359,148 @@ EOF
   # 4. compute the layer id
   # 5. put the layer in the image
   # 6. repack the image
-  buildImage = args@{ name, tag ? "latest"
-               , fromImage ? null, fromImageName ? null, fromImageTag ? null
-               , contents ? null, config ? null, runAsRoot ? null
-               , diskSize ? 1024, extraCommands ? "" }:
+  buildImage = args@{
+    # Image name.
+    name,
+    # Image tag.
+    tag ? "latest",
+    # Parent image, to append to.
+    fromImage ? null,
+    # Name of the parent image; will be read from the image otherwise.
+    fromImageName ? null,
+    # Tag of the parent image; will be read from the image otherwise.
+    fromImageTag ? null,
+    # Files to put on the image (a nix store path or list of paths).
+    contents ? null,
+    # Docker config; e.g. what command to run on the container.
+    config ? null,
+    # Optional bash script to run on the files prior to fixturizing the layer.
+    extraCommands ? "",
+    # Optional bash script to run as root on the image when provisioning.
+    runAsRoot ? null,
+    # Size of the virtual machine disk to provision when building the image.
+    diskSize ? 1024,
+  }:
 
     let
-
       baseName = baseNameOf name;
 
+      # Create a JSON blob of the configuration. Set the date to unix zero.
       baseJson = writeText "${baseName}-config.json" (builtins.toJSON {
-          created = "1970-01-01T00:00:01Z";
-          architecture = "amd64";
-          os = "linux";
-          config = config;
+        created = "1970-01-01T00:00:01Z";
+        architecture = "amd64";
+        os = "linux";
+        config = config;
       });
 
-      layer = (if runAsRoot == null
-               then mkPureLayer { inherit baseJson contents extraCommands; }
-               else mkRootLayer { inherit baseJson fromImage fromImageName fromImageTag contents runAsRoot diskSize extraCommands; });
-      result = runCommand "${baseName}.tar.gz" {
-        buildInputs = [ jshon pigz ];
-
+      layer =
+        if runAsRoot == null
+        then mkPureLayer {
+          name = baseName;
+          inherit baseJson contents extraCommands;
+        } else mkRootLayer {
+          name = baseName;
+          inherit baseJson fromImage fromImageName fromImageTag
+                  contents runAsRoot diskSize extraCommands;
+        };
+      result = runCommand "docker-image-${baseName}.tar.gz" {
+        buildInputs = [ jshon pigz coreutils findutils ];
         imageName = name;
         imageTag = tag;
         inherit fromImage baseJson;
-
         layerClosure = writeReferencesToFile layer;
-
-        passthru = {
-          buildArgs = args;
-        };
+        passthru.buildArgs = args;
+        passthru.layer = layer;
       } ''
+        # Print tar contents:
+        # 1: Interpreted as relative to the root directory
+        # 2: With no trailing sla