Merge branch 'improved-make-overridable' of git://github.com/ElvishJerricco/nixpkgs

9 files changed, 206 insertions, 113 deletions

diff --git a/lib/customisation.nix b/lib/customisation.nix
index 483ef6fd4866..217daada7818 100644
--- a/lib/customisation.nix
+++ b/lib/customisation.nix
@@ -50,10 +50,50 @@ rec {
        }
        else { }));
 
+  # A more powerful version of `makeOverridable` with features similar
+  # to `makeExtensibleWithInterface`.
+  makeOverridableWithInterface = interface: f: origArgs: let
+
+    addOverrideFuncs = {val, args, ...}: overridePackage:
+      (lib.optionalAttrs (builtins.isAttrs val) (val // {
+        extend = f: overridePackage (_: self: super: {
+          val = super.val // f self.val super.val;
+        });
+
+        overrideDerivation = newArgs: overridePackage (_: self: super: {
+          val = lib.overrideDerivation super.val newArgs;
+        });
+
+        ${if val ? overrideAttrs then "overrideAttrs" else null} = fdrv:
+          overridePackage (_: self: super: {
+            val = super.val.overrideAttrs fdrv;
+          });
+      })) // (lib.optionalAttrs (builtins.isFunction val) {
+        __functor = _: val;
+        extend = throw "extend not yet supported for functors";
+        overrideDerivation = throw "overrideDerivation not yet supported for functors";
+      }) // {
+        inherit overridePackage;
+
+        override = newArgs: overridePackage (_: self: super: {
+          args = super.args //
+            (if builtins.isFunction newArgs then newArgs super.args else newArgs);
+        });
+      };
+
+  in lib.makeExtensibleWithInterface (x: o: interface (addOverrideFuncs x o) o) (output: self: {
+    args = origArgs;
+    val = f output self.args self.val;
+  });
+
 
-  /* `makeOverridable` takes a function from attribute set to attribute set and
-     injects `override` attibute which can be used to override arguments of
-     the function.
+  /* `makeOverridable` takes a function from attribute set to
+     attribute set and injects 4 attributes which can be used to
+     override arguments and return values of the function.
+
+
+     1. `override` allows you to change what arguments were passed to
+     the function and acquire the new result.
 
        nix-repl> x = {a, b}: { result = a + b; }
 
@@ -65,28 +105,75 @@ rec {
        nix-repl> y.override { a = 10; }
        { override = «lambda»; overrideDerivation = «lambda»; result = 12; }
 
-     Please refer to "Nixpkgs Contributors Guide" section
-     "<pkg>.overrideDerivation" to learn about `overrideDerivation` and caveats
-     related to its use.
+
+     2. `extend` changes the results of the function, giving you a
+     view of the original result and a view of the eventual final
+     result. It is meant to do the same thing as
+     `makeExtensible`. That is, it lets you add to or change the
+     return value, such that previous extensions are consistent with
+     the final view, rather than being based on outdated
+     values. "Outdated" values come from the `super` argument, which
+     must be used when you are attempting to modify and old value. And
+     the final values come from the `self` argument, which recursively
+     refers to what all extensions combined return.
+
+       nix-repl> obj = makeOverridable (args: { }) { }
+
+       nix-repl> obj = obj.extend (self: super: { foo = "foo"; })
+
+       nix-repl> obj.foo
+       "foo"
+
+       nix-repl> obj = obj.extend (self: super: { foo = super.foo + " + "; bar = "bar"; foobar = self.foo + self.bar; })
+
+       nix-repl> obj
+       { bar = "bar"; foo = "foo + "; foobar = "foo + bar"; ... } # Excess omitted
+
+
+     3. `overrideDerivation`: Please refer to "Nixpkgs Contributors
+     Guide" section "<pkg>.overrideDerivation" to learn about
+     `overrideDerivation` and caveats related to its use.
+
+
+     4. `overridePackage` is by far the most powerful of the four, as
+     it exposes a deeper structure. It provides `self` and `super`
+     views of both the arguments and return value of the function,
+     allowing you to change both in one override; you can even have
+     overrides for one based on overrides for the other. It also
+     provides the `output` view, which is the view of `self` after
+     passing it through the `makeOverridable` interface and adding all
+     the `overrideX` functions. `output` is necessary when your
+     overrides depend on the overridable structure of `output`.
+
+       nix-repl> obj = makeOverridable ({a, b}: {inherit a b;}) {a = 1; b = 3;}
+
+       nix-repl> obj = obj.overridePackage (output: self: super: { args = super.args // {b = self.val.a;}; })
+
+       nix-repl> obj.b
+       1
+
+       nix-repl> obj = obj.overridePackage (output: self: super: { val = super.val // {a = self.args.a + 10;}; })
+
+       nix-repl> obj.b
+       11
+
   */
-  makeOverridable = f: origArgs:
+  makeOverridable = fn: makeOverridableWithInterface (x: _: x) (_: args: _: fn args);
+
+  callPackageCommon = functionArgs: scope: f: args:
     let
-      ff = f origArgs;
-      overrideWith = newArgs: origArgs // (if builtins.isFunction newArgs then newArgs origArgs else newArgs);
-    in
-      if builtins.isAttrs ff then (ff // {
-        override = newArgs: makeOverridable f (overrideWith newArgs);
-        overrideDerivation = fdrv:
-          makeOverridable (args: overrideDerivation (f args) fdrv) origArgs;
-        ${if ff ? overrideAttrs then "overrideAttrs" else null} = fdrv:
-          makeOverridable (args: (f args).overrideAttrs fdrv) origArgs;
-      })
-      else if builtins.isFunction ff then {
-        override = newArgs: makeOverridable f (overrideWith newArgs);
-        __functor = self: ff;
-        overrideDerivation = throw "overrideDerivation not yet supported for functors";
-      }
-      else ff;
+      intersect = builtins.intersectAttrs functionArgs;
+      interface = val: overridePackage: val // {
+        overrideScope = newScope: overridePackage (_: self: super: {
+          scope = super.scope.extend newScope;
+        });
+      };
+    in (makeOverridableWithInterface interface f (intersect scope // args))
+      .overridePackage (output: self: super: {
+        inherit scope;
+        # Don't use super.args because that contains the original scope.
+        args = intersect self.scope  // args;
+      });
 
 
   /* Call the package function in the file `fn' with the required
@@ -109,12 +196,35 @@ rec {
         libfoo = null;
         enableX11 = true;
       };
+
+    On top of the additions from `makeOverridable`, an `overrideScope`
+    function is also added to the result. It is similar to `override`,
+    except that it provides `self` and `super` views to the
+    scope. This can't be done in `makeOverridable` because the scope
+    is filtered to just the arguments needed by the function before
+    entering `makeOverridable`. It is useful to have a view of the
+    scope before restriction; for example, to change versions for a
+    particular dependency.
+
+      foo.overrideScope (self: super: {
+        llvm = self.llvm_37;
+      })
+
+    `llvm_37` would not exist in the scope after restriction.
+
   */
   callPackageWith = autoArgs: fn: args:
-    let
-      f = if builtins.isFunction fn then fn else import fn;
-      auto = builtins.intersectAttrs (builtins.functionArgs f) autoArgs;
-    in makeOverridable f (auto // args);
+    let f = if builtins.isFunction fn then fn else import fn;
+    in callPackageCommon (builtins.functionArgs f) autoArgs (output: x: _: f x) args;
+
+
+  # Like `callPackageWith`, but provides the function with a `self`
+  # view of the output, which has the override functions
+  # injected. `fn` is called with the new output whenever an override
+  # or extension is added.
+  callPackageWithOutputWith = autoArgs: fn: args:
+    let f = if builtins.isFunction fn then fn else import fn;
+    in callPackageCommon (builtins.functionArgs f) autoArgs (output: args: _: f args output ) args;
 
 
   /* Like callPackage, but for a function that returns an attribute
diff --git a/lib/fixed-points.nix b/lib/fixed-points.nix
index 13e053b5aa7d..2526360c111d 100644
--- a/lib/fixed-points.nix
+++ b/lib/fixed-points.nix
@@ -72,8 +72,34 @@ rec {
 
   # Same as `makeExtensible` but the name of the extending attribute is
   # customized.
-  makeExtensibleWithCustomName = extenderName: rattrs:
-    fix' rattrs // {
-      ${extenderName} = f: makeExtensibleWithCustomName extenderName (extends f rattrs);
-   };
+  makeExtensibleWithCustomName = extenderName: f: makeExtensibleWithInterface
+    (fixedPoint: extend: fixedPoint // { ${extenderName} = ext: extend (_: ext); })
+    (_: f);
+
+  # A version of `makeExtensible` that allows the function being fixed
+  # to return a different interface than the interface returned to the
+  # user. Along with `self` and `super` views of the internal
+  # interface, a `self` view of the output interface is also
+  # provided. `extend` is not added to the output by default. This is
+  # the job of the interface.
+  #
+  #     nix-repl> foo = {a, b}: {c = a + b;}
+  #
+  #     nix-repl> interface = {args, val, ...}: extend: val // {inherit extend;}
+  #
+  #     nix-repl> obj = makeExtensibleWithInterface interface (output: self: { args = {a = 1; b = 2;}; val = foo self.args; })
+  #
+  #     nix-repl> obj.c
+  #     3
+  #
+  #     nix-repl> obj = obj.extend (output: self: super: { args = super.args // { b = output.d; }; })
+  #
+  #     nix-repl> obj = obj.extend (output: self: super: { val = super.val // { d = 10; }; })
+  #
+  #     nix-repl> { inherit (obj) c d; }
+  #     { c = 11; d = 10; }
+  makeExtensibleWithInterface = interface: f: let i = interface
+    (fix' (f i))
+    (fext: makeExtensibleWithInterface interface (i': (extends (fext i') (f i'))));
+  in i;
 }
diff --git a/pkgs/development/beam-modules/default.nix b/pkgs/development/beam-modules/default.nix
index 1d4cef685148..95fe683cd1e5 100644
--- a/pkgs/development/beam-modules/default.nix
+++ b/pkgs/development/beam-modules/default.nix
@@ -5,14 +5,9 @@ let
 
   lib = pkgs.callPackage ./lib.nix {};
 
-  # FIXME: add support for overrideScope
-  callPackageWithScope = scope: drv: args: stdenv.lib.callPackageWith scope drv args;
-  mkScope = scope: pkgs // scope;
-
   packages = self:
     let
-      defaultScope = mkScope self;
-      callPackage = drv: args: callPackageWithScope defaultScope drv args;
+      callPackage = stdenv.lib.callPackageWith (pkgs // self);
     in
       import ./hex-packages.nix {
         inherit pkgs stdenv callPackage;
diff --git a/pkgs/development/haskell-modules/default.nix b/pkgs/development/haskell-modules/default.nix
index 1658ce793936..9eeae0eddc76 100644
--- a/pkgs/development/haskell-modules/default.nix
+++ b/pkgs/development/haskell-modules/default.nix
@@ -7,6 +7,8 @@
 , configurationNix ? import ./configuration-nix.nix
 }:
 
+self: # Provided by `callPackageWithOutput`
+
 let
 
   inherit (lib) extends makeExtensible;
@@ -14,19 +16,15 @@ let
 
   haskellPackages = pkgs.callPackage makePackageSet {
     package-set = initialPackages;
-    inherit stdenv haskellLib ghc extensible-self;
+    extensible-self = self;
+    inherit stdenv haskellLib ghc;
   };
 
   commonConfiguration = configurationCommon { inherit pkgs haskellLib; };
   nixConfiguration = configurationNix { inherit pkgs haskellLib; };
 
-  extensible-self = makeExtensible
-    (extends overrides
-      (extends packageSetConfig
-        (extends compilerConfig
-          (extends commonConfiguration
-            (extends nixConfiguration haskellPackages)))));
-
-in
-
-  extensible-self
+in (extends overrides
+     (extends packageSetConfig
+       (extends compilerConfig
+         (extends commonConfiguration
+           (extends nixConfiguration haskellPackages))))) self
diff --git a/pkgs/development/haskell-modules/make-package-set.nix b/pkgs/development/haskell-modules/make-package-set.nix
index ff5be894b926..b6bd3fdd30b7 100644
--- a/pkgs/development/haskell-modules/make-package-set.nix
+++ b/pkgs/development/haskell-modules/make-package-set.nix
@@ -29,7 +29,7 @@ self:
 
 let
 
-  inherit (stdenv.lib) fix' extends makeOverridable;
+  inherit (stdenv.lib) fix' extends makeOverridable callPackageWith;
   inherit (haskellLib) overrideCabal;
 
   mkDerivationImpl = pkgs.callPackage ./generic-builder.nix {
@@ -61,39 +61,9 @@ let
 
   mkDerivation = makeOverridable mkDerivationImpl;
 
-  # manualArgs are the arguments that were explictly passed to `callPackage`, like:
-  #
-  # callPackage foo { bar = null; };
-  #
-  # here `bar` is a manual argument.
-  callPackageWithScope = scope: fn: manualArgs:
-    let
-      # this code is copied from callPackage in lib/customisation.nix
-      #
-      # we cannot use `callPackage` here because we want to call `makeOverridable`
-      # on `drvScope` (we cannot add `overrideScope` after calling `callPackage` because then it is
-      # lost on `.override`) but determine the auto-args based on `drv` (the problem here
-      # is that nix has no way to "passthrough" args while preserving the reflection
-      # info that callPackage uses to determine the arguments).
-      drv = if builtins.isFunction fn then fn else import fn;
-      auto = builtins.intersectAttrs (builtins.functionArgs drv) scope;
-
-      # this wraps the `drv` function to add a `overrideScope` function to the result.
-      drvScope = allArgs: drv allArgs // {
-        overrideScope = f:
-          let newScope = mkScope (fix' (extends f scope.__unfix__));
-          # note that we have to be careful here: `allArgs` includes the auto-arguments that
-          # weren't manually specified. If we would just pass `allArgs` to the recursive call here,
-          # then we wouldn't look up any packages in the scope in the next interation, because it
-          # appears as if all arguments were already manually passed, so the scope change would do
-          # nothing.
-          in callPackageWithScope newScope drv manualArgs;
-      };
-    in stdenv.lib.makeOverridable drvScope (auto // manualArgs);
-
   mkScope = scope: pkgs // pkgs.xorg // pkgs.gnome2 // { inherit stdenv; } // scope;
   defaultScope = mkScope self;
-  callPackage = drv: args: callPackageWithScope defaultScope drv args;
+  callPackage = drv: args: callPackageWith defaultScope drv args;
 
   withPackages = packages: callPackage ./with-packages-wrapper.nix {
     inherit (self) llvmPackages;
diff --git a/pkgs/development/idris-modules/default.nix b/pkgs/development/idris-modules/default.nix
index 4d7c4928283a..cb8f46d0c2e7 100644
--- a/pkgs/development/idris-modules/default.nix
+++ b/pkgs/development/idris-modules/default.nix
@@ -1,17 +1,8 @@
 { pkgs, idris, overrides ? (self: super: {}) }: let
   inherit (pkgs.lib) callPackageWith fix' extends;
 
-  /* Taken from haskell-modules/default.nix, should probably abstract this away */
-  callPackageWithScope = scope: drv: args: (callPackageWith scope drv args) // {
-    overrideScope = f: callPackageWithScope (mkScope (fix' (extends f scope.__unfix__))) drv args;
-  };
-
-  mkScope = scope : pkgs // pkgs.xorg // pkgs.gnome2 // scope;
-
   idrisPackages = self: let
-    defaultScope = mkScope self;
-
-    callPackage = callPackageWithScope defaultScope;
+    callPackage = callPackageWith (pkgs // pkgs.xorg // pkgs.gnome2 // self);
 
     builtins_ = pkgs.lib.mapAttrs self.build-builtin-package {
       prelude = [];
diff --git a/pkgs/top-level/all-packages.nix b/pkgs/top-level/all-packages.nix
index 083e76ee0efb..712ef39aed3b 100644
--- a/pkgs/top-level/all-packages.nix
+++ b/pkgs/top-level/all-packages.nix
@@ -5708,9 +5708,7 @@ with pkgs;
 
   haskell = callPackage ./haskell-packages.nix { };
 
-  haskellPackages = haskell.packages.ghc802.override {
-    overrides = config.haskellPackageOverrides or (self: super: {});
-  };
+  haskellPackages = haskell.packages.ghc802.extend (config.haskellPackageOverrides or (self: super: {}));
 
   inherit (haskellPackages) ghc;
 
diff --git a/pkgs/top-level/haskell-packages.nix b/pkgs/top-level/haskell-packages.nix
index b3bbb0d5b643..a1f9b2e81c3c 100644
--- a/pkgs/top-level/haskell-packages.nix
+++ b/pkgs/top-level/haskell-packages.nix
@@ -1,4 +1,4 @@
-{ pkgs, lib, newScope, stdenv, buildPlatform, targetPlatform }:
+{ pkgs, lib, stdenv, buildPlatform, targetPlatform }:
 
 let
   # These are attributes in compiler and packages that don't support integer-simple.
@@ -23,7 +23,8 @@ let
     inherit pkgs;
   };
 
-  callPackage = newScope { inherit haskellLib; };
+  callPackage = lib.callPackageWith (pkgs // { inherit haskellLib; });
+  callPackageWithOutput = lib.callPackageWithOutputWith (pkgs // { inherit haskellLib; });
 
 in rec {
   lib = haskellLib;
@@ -121,75 +122,75 @@ in rec {
   packages = {
 
     # Support for this compiler is broken, because it can't deal with directory-based package databases.
-    # ghc6104 = callPackage ../development/haskell-modules { ghc = compiler.ghc6104; };
-    ghc6123 = callPackage ../development/haskell-modules {
+    # ghc6104 = callPackageWithOutput ../development/haskell-modules { ghc = compiler.ghc6104; };
+    ghc6123 = callPackageWithOutput ../development/haskell-modules {
       ghc = compiler.ghc6123;
       compilerConfig = callPackage ../development/haskell-modules/configuration-ghc-6.12.x.nix { };
     };
-    ghc704 = callPackage ../development/haskell-modules {
+    ghc704 = callPackageWithOutput ../development/haskell-modules {
       ghc = compiler.ghc704;
       compilerConfig = callPackage ../development/haskell-modules/configuration-ghc-7.0.x.nix { };
     };
-    ghc722 = callPackage ../development/haskell-modules {
+    ghc722 = callPackageWithOutput ../development/haskell-modules {
       ghc = compiler.ghc722;
       compilerConfig = callPackage ../development/haskell-modules/configuration-ghc-7.2.x.nix { };
     };
-    ghc742 = callPackage ../development/haskell-modules {
+    ghc742 = callPackageWithOutput ../development/haskell-modules {
       ghc = compiler.ghc742;
       compilerConfig = callPackage ../development/haskell-modules/configuration-ghc-7.4.x.nix { };
     };
-    ghc763 = callPackage ../development/haskell-modules {
+    ghc763 = callPackageWithOutput ../development/haskell-modules {
       ghc = compiler.ghc763;
       compilerConfig = callPackage ../development/haskell-modules/configuration-ghc-7.6.x.nix { };
     };
-    ghc783 = callPackage ../development/haskell-modules {
+    ghc783 = callPackageWithOutput ../development/haskell-modules {
       ghc = compiler.ghc783;
       compilerConfig = callPackage ../development/haskell-modules/configuration-ghc-7.8.x.nix { };
     };
-    ghc784 = callPackage ../development/haskell-modules {
+    ghc784 = callPackageWithOutput ../development/haskell-modules {
       ghc = compiler.ghc784;
       compilerConfig = callPackage ../development/haskell-modules/configuration-ghc-7.8.x.nix { };
     };
-    ghc7102 = callPackage ../development/haskell-modules {
+    ghc7102 = callPackageWithOutput ../development/haskell-modules {
       ghc = compiler.ghc7102;
       compilerConfig = callPackage ../development/haskell-modules/configuration-ghc-7.10.x.nix { };
     };
-    ghc7103 = callPackage ../development/haskell-modules {
+    ghc7103 = callPackageWithOutput ../development/haskell-modules {
       ghc = compiler.ghc7103;
       compilerConfig = callPackage ../development/haskell-modules/configuration-ghc-7.10.x.nix { };
     };
-    ghc802 = callPackage ../development/haskell-modules {
+    ghc802 = callPackageWithOutput ../development/haskell-modules {
       ghc = compiler.ghc802;
       compilerConfig = callPackage ../development/haskell-modules/configuration-ghc-8.0.x.nix { };
     };
-    ghc821 = callPackage ../development/haskell-modules {
+    ghc821 = callPackageWithOutput ../development/haskell-modules {
       ghc = compiler.ghc821;
       compilerConfig = callPackage ../development/haskell-modules/configuration-ghc-8.2.x.nix { };
     };
-    ghcHEAD = callPackage ../development/haskell-modules {
+    ghcHEAD = callPackageWithOutput ../development/haskell-modules {
       ghc = compiler.ghcHEAD;
       compilerConfig = callPackage ../development/haskell-modules/configuration-ghc-head.nix { };
     };
     # TODO Support for multiple variants here
-    ghcCross = callPackage ../development/haskell-modules {
+    ghcCross = callPackageWithOutput ../development/haskell-modules {
       ghc = compiler.ghcHEAD.crossCompiler;
       compilerConfig = callPackage ../development/haskell-modules/configuration-ghc-head.nix { };
     };
-    ghcCross821 = callPackage ../development/haskell-modules {
+    ghcCross821 = callPackageWithOutput ../development/haskell-modules {
       ghc = compiler.ghc821.crossCompiler;
       compilerConfig = callPackage ../development/haskell-modules/configuration-ghc-8.2.x.nix { };
     };
-    ghcjs = callPackage ../development/haskell-modules {
+    ghcjs = callPackageWithOutput ../development/haskell-modules {
       ghc = compiler.ghcjs;
       compilerConfig = callPackage ../development/haskell-modules/configuration-ghc-7.10.x.nix { };
       packageSetConfig = callPackage ../development/haskell-modules/configuration-ghcjs.nix { };
     };
-    ghcjsHEAD = callPackage ../development/haskell-modules {
+    ghcjsHEAD = callPackageWithOutput ../development/haskell-modules {
       ghc = compiler.ghcjsHEAD;
       compilerConfig = callPackage ../development/haskell-modules/configuration-ghc-8.0.x.nix { };
       packageSetConfig = callPackage ../development/haskell-modules/configuration-ghcjs.nix { };
     };
-    ghcHaLVM240 = callPackage ../development/haskell-modules {
+    ghcHaLVM240 = callPackageWithOutput ../development/haskell-modules {
       ghc = compiler.ghcHaLVM240;
       compilerConfig = callPackage ../development/haskell-modules/configuration-halvm-2.4.0.nix { };
     };
diff --git a/pkgs/top-level/splice.nix b/pkgs/top-level/splice.nix
index 44a46b7b6929..d6498be949b3 100644
--- a/pkgs/top-level/splice.nix
+++ b/pkgs/top-level/splice.nix
@@ -80,7 +80,11 @@ in
   # `newScope' for sets of packages in `pkgs' (see e.g. `gnome' below).
   callPackage = pkgs.newScope {};
 
+  callPackageWithOutput = pkgs.newScopeWithOutput {};
+
   callPackages = lib.callPackagesWith splicedPackages;
 
   newScope = extra: lib.callPackageWith (splicedPackages // extra);
+
+  newScopeWithOutput = extra: lib.callPackageWithOutputWith (splicedPackages // extra);
 }