Merge branch 'master' into fix/man-in-outputsToInstall

4 files changed, 345 insertions, 149 deletions

diff --git a/pkgs/stdenv/generic/check-meta.nix b/pkgs/stdenv/generic/check-meta.nix
index 93d0f4cc9805..fe0c8cfad919 100644
--- a/pkgs/stdenv/generic/check-meta.nix
+++ b/pkgs/stdenv/generic/check-meta.nix
@@ -1,16 +1,19 @@
 # Checks derivation meta and attrs for problems (like brokenness,
 # licenses, etc).
 
-{ lib, config, system, meta, derivationArg, mkDerivationArg }:
+{ lib, config, hostPlatform, meta }:
 
 let
-  attrs = mkDerivationArg; # TODO: probably get rid of passing this one
+  # If we're in hydra, we can dispense with the more verbose error
+  # messages and make problems easier to spot.
+  inHydra = config.inHydra or false;
 
   # See discussion at https://github.com/NixOS/nixpkgs/pull/25304#issuecomment-298385426
   # for why this defaults to false, but I (@copumpkin) want to default it to true soon.
   shouldCheckMeta = config.checkMeta or false;
 
-  allowUnfree = config.allowUnfree or false || builtins.getEnv "NIXPKGS_ALLOW_UNFREE" == "1";
+  allowUnfree = config.allowUnfree or false
+    || builtins.getEnv "NIXPKGS_ALLOW_UNFREE" == "1";
 
   whitelist = config.whitelistedLicenses or [];
   blacklist = config.blacklistedLicenses or [];
@@ -37,12 +40,13 @@ let
   hasBlacklistedLicense = assert areLicenseListsValid; attrs:
     hasLicense attrs && builtins.elem attrs.meta.license blacklist;
 
-  allowBroken = config.allowBroken or false || builtins.getEnv "NIXPKGS_ALLOW_BROKEN" == "1";
+  allowBroken = config.allowBroken or false
+    || builtins.getEnv "NIXPKGS_ALLOW_BROKEN" == "1";
 
-  allowUnsupportedSystem = config.allowUnsupportedSystem or false;
+  allowUnsupportedSystem = config.allowUnsupportedSystem or false
+    || builtins.getEnv "NIXPKGS_ALLOW_UNSUPPORTED_SYSTEM" == "1";
 
-  isUnfree = licenses: lib.lists.any (l:
-    !l.free or true || l == "unfree" || l == "unfree-redistributable") licenses;
+  isUnfree = licenses: lib.lists.any (l: !l.free or true) licenses;
 
   # Alow granular checks to allow only some unfree packages
   # Example:
@@ -55,7 +59,7 @@ let
 
   # Check whether unfree packages are allowed and if not, whether the
   # package has an unfree license and is not explicitely allowed by the
-  # `allowUNfreePredicate` function.
+  # `allowUnfreePredicate` function.
   hasDeniedUnfreeLicense = attrs:
     !allowUnfree &&
     hasLicense attrs &&
@@ -77,8 +81,10 @@ let
   remediation = {
     unfree = remediate_whitelist "Unfree";
     broken = remediate_whitelist "Broken";
+    unsupported = remediate_whitelist "UnsupportedSystem";
     blacklisted = x: "";
     insecure = remediate_insecure;
+    broken-outputs = remediateOutputsToInstall;
     unknown-meta = x: "";
   };
   remediate_whitelist = allow_attr: attrs:
@@ -123,12 +129,28 @@ let
 
       '';
 
-  handleEvalIssue = { reason , errormsg ? "" }:
+  remediateOutputsToInstall = attrs: let
+      expectedOutputs = attrs.meta.outputsToInstall or [];
+      actualOutputs = attrs.outputs or [ "out" ];
+      missingOutputs = builtins.filter (output: ! builtins.elem output actualOutputs) expectedOutputs;
+    in ''
+      The package ${attrs.name} has set meta.outputsToInstall to: ${builtins.concatStringsSep ", " expectedOutputs}
+
+      however ${attrs.name} only has the outputs: ${builtins.concatStringsSep ", " actualOutputs}
+
+      and is missing the following ouputs:
+
+      ${lib.concatStrings (builtins.map (output: "  - ${output}\n") missingOutputs)}
+    '';
+
+  handleEvalIssue = attrs: { reason , errormsg ? "" }:
     let
-      msg = ''
-        Package ‘${attrs.name or "«name-missing»"}’ in ${pos_str} ${errormsg}, refusing to evaluate.
+      msg = if inHydra
+        then "Failed to evaluate ${attrs.name or "«name-missing»"}: «${reason}»: ${errormsg}"
+        else ''
+          Package ‘${attrs.name or "«name-missing»"}’ in ${pos_str} ${errormsg}, refusing to evaluate.
 
-      '' + (builtins.getAttr reason remediation) attrs;
+        '' + (builtins.getAttr reason remediation) attrs;
 
       handler = if config ? "handleEvalIssue"
         then config.handleEvalIssue reason
@@ -144,11 +166,22 @@ let
     homepage = either (listOf str) str;
     downloadPage = str;
     license = either (listOf lib.types.attrs) (either lib.types.attrs str);
-    maintainers = listOf str;
+    maintainers = listOf (attrsOf str);
     priority = int;
-    platforms = listOf str;
+    platforms = listOf (either str lib.systems.parsedPlatform.types.system);
     hydraPlatforms = listOf str;
     broken = bool;
+    # TODO: refactor once something like Profpatsch's types-simple will land
+    # This is currently dead code due to https://github.com/NixOS/nix/issues/2532
+    tests = attrsOf (mkOptionType {
+      name = "test";
+      check = x: x == {} || ( # Accept {} for tests that are unsupported
+        isDerivation x &&
+        x ? meta.timeout
+      );
+      merge = lib.options.mergeOneOption;
+    });
+    timeout = int;
 
     # Weirder stuff that doesn't appear in the documentation?
     knownVulnerabilities = listOf str;
@@ -167,6 +200,7 @@ let
     isFcitxEngine = bool;
     isIbusEngine = bool;
     isGutenprint = bool;
+    badPlatforms = platforms;
   };
 
   checkMetaAttr = k: v:
@@ -175,6 +209,19 @@ let
     else "key '${k}' is unrecognized; expected one of: \n\t      [${lib.concatMapStringsSep ", " (x: "'${x}'") (lib.attrNames metaTypes)}]";
   checkMeta = meta: if shouldCheckMeta then lib.remove null (lib.mapAttrsToList checkMetaAttr meta) else [];
 
+  checkPlatform = attrs: let
+      anyMatch = lib.any (lib.meta.platformMatch hostPlatform);
+    in  anyMatch (attrs.meta.platforms or lib.platforms.all) &&
+      ! anyMatch (attrs.meta.badPlatforms or []);
+
+  checkOutputsToInstall = attrs: let
+      expectedOutputs = attrs.meta.outputsToInstall or [];
+      actualOutputs = attrs.outputs or [ "out" ];
+      missingOutputs = builtins.filter (output: ! builtins.elem output actualOutputs) expectedOutputs;
+    in if shouldCheckMeta
+       then builtins.length missingOutputs > 0
+       else false;
+
   # Check if a derivation is valid, that is whether it passes checks for
   # e.g brokenness or license.
   #
@@ -188,19 +235,23 @@ let
       { valid = false; reason = "blacklisted"; errormsg = "has a blacklisted license (‘${showLicense attrs.meta.license}’)"; }
     else if !allowBroken && attrs.meta.broken or false then
       { valid = false; reason = "broken"; errormsg = "is marked as broken"; }
-    else if !allowUnsupportedSystem && !allowBroken && attrs.meta.platforms or null != null && !lib.lists.elem system attrs.meta.platforms then
-      { valid = false; reason = "broken"; errormsg = "is not supported on ‘${system}’"; }
+    else if !allowUnsupportedSystem && !(checkPlatform attrs) then
+      { valid = false; reason = "unsupported"; errormsg = "is not supported on ‘${hostPlatform.config}’"; }
     else if !(hasAllowedInsecure attrs) then
       { valid = false; reason = "insecure"; errormsg = "is marked as insecure"; }
+    else if checkOutputsToInstall attrs then
+      { valid = false; reason = "broken-outputs"; errormsg = "has invalid meta.outputsToInstall"; }
     else let res = checkMeta (attrs.meta or {}); in if res != [] then
       { valid = false; reason = "unknown-meta"; errormsg = "has an invalid meta attrset:${lib.concatMapStrings (x: "\n\t - " + x) res}"; }
     else { valid = true; };
 
-   validity = checkValidity attrs;
+  assertValidity = attrs: let
+      validity = checkValidity attrs;
+    in validity // {
+      # Throw an error if trying to evaluate an non-valid derivation
+      handled = if !validity.valid
+        then handleEvalIssue attrs (removeAttrs validity ["valid"])
+        else true;
+  };
 
-in validity // {
-  # Throw an error if trying to evaluate an non-valid derivation
-  handled = if !validity.valid
-    then handleEvalIssue (removeAttrs validity ["valid"])
-    else true;
-}
+in assertValidity
diff --git a/pkgs/stdenv/generic/default.nix b/pkgs/stdenv/generic/default.nix
index 42288e6245e4..f77f43aea886 100644
--- a/pkgs/stdenv/generic/default.nix
+++ b/pkgs/stdenv/generic/default.nix
@@ -90,9 +90,14 @@ let
       '' + lib.optionalString hostPlatform.isDarwin ''
         export NIX_DONT_SET_RPATH=1
         export NIX_NO_SELF_RPATH=1
-      '' + lib.optionalString targetPlatform.isDarwin ''
-        export NIX_TARGET_DONT_SET_RPATH=1
-      '';
+      ''
+      # TODO this should be uncommented, but it causes stupid mass rebuilds. I
+      # think the best solution would just be to fixup linux RPATHs so we don't
+      # need to set `-rpath` anywhere.
+      # + lib.optionalString targetPlatform.isDarwin ''
+      #   export NIX_TARGET_DONT_SET_RPATH=1
+      # ''
+      ;
 
       inherit initialPath shell
         defaultNativeBuildInputs defaultBuildInputs;
@@ -116,11 +121,14 @@ let
 
       # Utility flags to test the type of platform.
       inherit (hostPlatform)
-        isDarwin isLinux isSunOS isHurd isCygwin isFreeBSD isOpenBSD
-        isi686 isx86_64 is64bit isArm isAarch64 isMips isBigEndian;
+        isDarwin isLinux isSunOS isCygwin isFreeBSD isOpenBSD
+        isi686 isx86_64 is64bit isAarch32 isAarch64 isMips isBigEndian;
+      isArm = lib.warn
+        "`stdenv.isArm` is deprecated after 18.03. Please use `stdenv.isAarch32` instead"
+        hostPlatform.isAarch32;
 
-      # Whether we should run paxctl to pax-mark binaries.
-      needsPax = isLinux;
+      # The derivation's `system` is `buildPlatform.system`.
+      inherit (buildPlatform) system;
 
       inherit (import ./make-derivation.nix {
         inherit lib config stdenv;
@@ -135,8 +143,6 @@ let
       inherit overrides;
 
       inherit cc;
-
-      isCross = targetPlatform != buildPlatform;
     }
 
     # Propagate any extra attributes.  For instance, we use this to
diff --git a/pkgs/stdenv/generic/make-derivation.nix b/pkgs/stdenv/generic/make-derivation.nix
index 432a7e338944..96f8e91412cd 100644
--- a/pkgs/stdenv/generic/make-derivation.nix
+++ b/pkgs/stdenv/generic/make-derivation.nix
@@ -36,8 +36,12 @@ rec {
     , depsTargetTarget            ? [] #  1 ->  1
     , depsTargetTargetPropagated  ? [] #  1 ->  1
 
+    , checkInputs                 ? []
+    , installCheckInputs          ? []
+
     # Configure Phase
     , configureFlags ? []
+    , cmakeFlags ? []
     , # Target is not included by default because most programs don't care.
       # Including it then would cause needless mass rebuilds.
       #
@@ -46,18 +50,23 @@ rec {
         (stdenv.hostPlatform != stdenv.buildPlatform)
         [ "build" "host" ]
 
+    # TODO(@Ericson2314): Make unconditional / resolve #33599
     # Check phase
-    , doCheck ? false
+    , doCheck ? config.doCheckByDefault or false
 
+    # TODO(@Ericson2314): Make unconditional / resolve #33599
     # InstallCheck phase
-    , doInstallCheck ? false
+    , doInstallCheck ? config.doCheckByDefault or false
 
-    , crossConfig ? null
+    , # TODO(@Ericson2314): Make always true and remove
+      strictDeps ? stdenv.hostPlatform != stdenv.buildPlatform
     , meta ? {}
     , passthru ? {}
     , pos ? # position used in error messages and for meta.position
         (if attrs.meta.description or null != null
           then builtins.unsafeGetAttrPos "description" attrs.meta
+          else if attrs.version or null != null
+          then builtins.unsafeGetAttrPos "version" attrs
           else builtins.unsafeGetAttrPos "name" attrs)
     , separateDebugInfo ? false
     , outputs ? [ "out" ]
@@ -71,9 +80,31 @@ rec {
 
     , ... } @ attrs:
 
-    # TODO(@Ericson2314): Make this more modular, and not O(n^2).
     let
+      computedName = if name != "" then name else "${attrs.pname}-${attrs.version}";
+
+      # TODO(@oxij, @Ericson2314): This is here to keep the old semantics, remove when
+      # no package has `doCheck = true`.
+      doCheck' = doCheck && stdenv.hostPlatform == stdenv.buildPlatform;
+      doInstallCheck' = doInstallCheck && stdenv.hostPlatform == stdenv.buildPlatform;
+
+      separateDebugInfo' = separateDebugInfo && stdenv.hostPlatform.isLinux;
+      outputs' = outputs ++ lib.optional separateDebugInfo' "debug";
+
+      fixedOutputDrv = attrs ? outputHash;
+      noNonNativeDeps = builtins.length (depsBuildTarget ++ depsBuildTargetPropagated
+                                      ++ depsHostHost ++ depsHostHostPropagated
+                                      ++ buildInputs ++ propagatedBuildInputs
+                                      ++ depsTargetTarget ++ depsTargetTargetPropagated) == 0;
+      dontAddHostSuffix = attrs ? outputHash && !noNonNativeDeps || stdenv.cc == null;
       supportedHardeningFlags = [ "fortify" "stackprotector" "pie" "pic" "strictoverflow" "format" "relro" "bindnow" ];
+      defaultHardeningFlags = if stdenv.hostPlatform.isMusl
+                              then supportedHardeningFlags
+                              else lib.remove "pie" supportedHardeningFlags;
+      enabledHardeningOptions =
+        if builtins.elem "all" hardeningDisable
+        then []
+        else lib.subtractLists hardeningDisable (defaultHardeningFlags ++ hardeningEnable);
       # hardeningDisable additionally supports "all".
       erroneousHardeningFlags = lib.subtractLists supportedHardeningFlags (hardeningEnable ++ lib.remove "all" hardeningDisable);
     in if builtins.length erroneousHardeningFlags != 0
@@ -81,6 +112,11 @@ rec {
       inherit erroneousHardeningFlags hardeningDisable hardeningEnable supportedHardeningFlags;
     })
     else let
+      doCheck = doCheck';
+      doInstallCheck = doInstallCheck';
+
+      outputs = outputs';
+
       references = nativeBuildInputs ++ buildInputs
                 ++ propagatedNativeBuildInputs ++ propagatedBuildInputs;
 
@@ -88,13 +124,15 @@ rec {
         [
           (map (drv: drv.__spliced.buildBuild or drv) depsBuildBuild)
           (map (drv: drv.nativeDrv or drv) nativeBuildInputs
-             ++ lib.optional separateDebugInfo ../../build-support/setup-hooks/separate-debug-info.sh
+             ++ lib.optional separateDebugInfo' ../../build-support/setup-hooks/separate-debug-info.sh
              ++ lib.optional stdenv.hostPlatform.isWindows ../../build-support/setup-hooks/win-dll-link.sh)
           (map (drv: drv.__spliced.buildTarget or drv) depsBuildTarget)
         ]
         [
           (map (drv: drv.__spliced.hostHost or drv) depsHostHost)
-          (map (drv: drv.crossDrv or drv) buildInputs)
+          (map (drv: drv.crossDrv or drv) (buildInputs
+             ++ lib.optionals doCheck checkInputs
+             ++ lib.optionals doInstallCheck' installCheckInputs))
         ]
         [
           (map (drv: drv.__spliced.targetTarget or drv) depsTargetTarget)
@@ -115,58 +153,71 @@ rec {
         ]
       ];
 
-      outputs' =
-        outputs ++
-        (if separateDebugInfo then assert stdenv.hostPlatform.isLinux; [ "debug" ] else []);
+      computedSandboxProfile =
+        lib.concatMap (input: input.__propagatedSandboxProfile or [])
+          (stdenv.extraNativeBuildInputs
+           ++ stdenv.extraBuildInputs
+           ++ lib.concatLists dependencies);
+
+      computedPropagatedSandboxProfile =
+        lib.concatMap (input: input.__propagatedSandboxProfile or [])
+          (lib.concatLists propagatedDependencies);
+
+      computedImpureHostDeps =
+        lib.unique (lib.concatMap (input: input.__propagatedImpureHostDeps or [])
+          (stdenv.extraNativeBuildInputs
+           ++ stdenv.extraBuildInputs
+           ++ lib.concatLists dependencies));
+
+      computedPropagatedImpureHostDeps =
+        lib.unique (lib.concatMap (input: input.__propagatedImpureHostDeps or [])
+          (lib.concatLists propagatedDependencies));
 
       derivationArg =
         (removeAttrs attrs
-          ["meta" "passthru" "crossAttrs" "pos"
+          ["meta" "passthru" "pos"
+           "checkInputs" "installCheckInputs"
            "__impureHostDeps" "__propagatedImpureHostDeps"
            "sandboxProfile" "propagatedSandboxProfile"])
-        // (let
-          computedSandboxProfile =
-            lib.concatMap (input: input.__propagatedSandboxProfile or [])
-              (stdenv.extraNativeBuildInputs
-               ++ stdenv.extraBuildInputs
-               ++ lib.concatLists dependencies);
-          computedPropagatedSandboxProfile =
-            lib.concatMap (input: input.__propagatedSandboxProfile or [])
-              (lib.concatLists propagatedDependencies);
-          computedImpureHostDeps =
-            lib.unique (lib.concatMap (input: input.__propagatedImpureHostDeps or [])
-              (stdenv.extraNativeBuildInputs
-               ++ stdenv.extraBuildInputs
-               ++ lib.concatLists dependencies));
-          computedPropagatedImpureHostDeps =
-            lib.unique (lib.concatMap (input: input.__propagatedImpureHostDeps or [])
-              (lib.concatLists propagatedDependencies));
-        in
-        {
+        // {
           # A hack to make `nix-env -qa` and `nix search` ignore broken packages.
           # TODO(@oxij): remove this assert when something like NixOS/nix#1771 gets merged into nix.
-          name = assert validity.handled; name + lib.optionalString
-            (stdenv.hostPlatform != stdenv.buildPlatform)
+          name = assert validity.handled; computedName + lib.optionalString
+            # Fixed-output derivations like source tarballs shouldn't get a host
+            # suffix. But we have some weird ones with run-time deps that are
+            # just used for their side-affects. Those might as well since the
+            # hash can't be the same. See #32986.
+            (stdenv.hostPlatform != stdenv.buildPlatform && !dontAddHostSuffix)
             ("-" + stdenv.hostPlatform.config);
 
           builder = attrs.realBuilder or stdenv.shell;
           args = attrs.args or ["-e" (attrs.builder or ./default-builder.sh)];
           inherit stdenv;
-          inherit (stdenv) system;
+
+          # The `system` attribute of a derivation has special meaning to Nix.
+          # Derivations set it to choose what sort of machine could be used to
+          # execute the build, The build platform entirely determines this,
+          # indeed more finely than Nix knows or cares about. The `system`
+          # attribute of `buildPlatfom` matches Nix's degree of specificity.
+          # exactly.
+          inherit (stdenv.buildPlatform) system;
+
           userHook = config.stdenv.userHook or null;
           __ignoreNulls = true;
 
+          inherit strictDeps;
+
           depsBuildBuild              = lib.elemAt (lib.elemAt dependencies 0) 0;
           nativeBuildInputs           = lib.elemAt (lib.elemAt dependencies 0) 1;
           depsBuildTarget             = lib.elemAt (lib.elemAt dependencies 0) 2;
-          depsHostBuild               = lib.elemAt (lib.elemAt dependencies 1) 0;
+          depsHostHost                = lib.elemAt (lib.elemAt dependencies 1) 0;
           buildInputs                 = lib.elemAt (lib.elemAt dependencies 1) 1;
           depsTargetTarget            = lib.elemAt (lib.elemAt dependencies 2) 0;
 
           depsBuildBuildPropagated    = lib.elemAt (lib.elemAt propagatedDependencies 0) 0;
           propagatedNativeBuildInputs = lib.elemAt (lib.elemAt propagatedDependencies 0) 1;
           depsBuildTargetPropagated   = lib.elemAt (lib.elemAt propagatedDependencies 0) 2;
-          depsHostBuildPropagated     = lib.elemAt (lib.elemAt propagatedDependencies 1) 0;
+          depsHostHostPropagated      = lib.elemAt (lib.elemAt propagatedDependencies 1) 0;
           propagatedBuildInputs       = lib.elemAt (lib.elemAt propagatedDependencies 1) 1;
           depsTargetTargetPropagated  = lib.elemAt (lib.elemAt propagatedDependencies 2) 0;
 
@@ -179,6 +230,24 @@ rec {
             ++ optional (elem "host"   configurePlatforms) "--host=${stdenv.hostPlatform.config}"
             ++ optional (elem "target" configurePlatforms) "--target=${stdenv.targetPlatform.config}";
 
+          inherit doCheck doInstallCheck;
+
+          inherit outputs;
+        } // lib.optionalAttrs (stdenv.hostPlatform != stdenv.buildPlatform) {
+          cmakeFlags =
+            (/**/ if lib.isString cmakeFlags then [cmakeFlags]
+             else if cmakeFlags == null      then []
+             else                                     cmakeFlags)
+          ++ lib.optional (stdenv.hostPlatform.uname.system != null) "-DCMAKE_SYSTEM_NAME=${stdenv.hostPlatform.uname.system}"
+          ++ lib.optional (stdenv.hostPlatform.uname.processor != null) "-DCMAKE_SYSTEM_PROCESSOR=${stdenv.hostPlatform.uname.processor}"
+          ++ lib.optional (stdenv.hostPlatform.uname.release != null) "-DCMAKE_SYSTEM_VERSION=${stdenv.hostPlatform.release}"
+          ++ lib.optional (stdenv.buildPlatform.uname.system != null) "-DCMAKE_HOST_SYSTEM_NAME=${stdenv.buildPlatform.uname.system}"
+          ++ lib.optional (stdenv.buildPlatform.uname.processor != null) "-DCMAKE_HOST_SYSTEM_PROCESSOR=${stdenv.buildPlatform.uname.processor}"
+          ++ lib.optional (stdenv.buildPlatform.uname.release != null) "-DCMAKE_HOST_SYSTEM_VERSION=${stdenv.buildPlatform.uname.release}";
+        } // lib.optionalAttrs (attrs.enableParallelBuilding or false) {
+          enableParallelChecking = attrs.enableParallelChecking or true;
+        } // lib.optionalAttrs (hardeningDisable != [] || hardeningEnable != []) {
+          NIX_HARDENING_ENABLE = enabledHardeningOptions;
         } // lib.optionalAttrs (stdenv.buildPlatform.isDarwin) {
           # TODO: remove lib.unique once nix has a list canonicalization primitive
           __sandboxProfile =
@@ -193,23 +262,14 @@ rec {
             "/bin/sh"
           ];
           __propagatedImpureHostDeps = computedPropagatedImpureHostDeps ++ __propagatedImpureHostDeps;
-        } // lib.optionalAttrs (outputs' != [ "out" ]) {
-          outputs = outputs';
-        } // lib.optionalAttrs (attrs ? doCheck) {
-          # TODO(@Ericson2314): Make unconditional / resolve #33599
-          doCheck = doCheck && (stdenv.hostPlatform == stdenv.buildPlatform);
-        } // lib.optionalAttrs (attrs ? doInstallCheck) {
-          # TODO(@Ericson2314): Make unconditional / resolve #33599
-          doInstallCheck = doInstallCheck && (stdenv.hostPlatform == stdenv.buildPlatform);
-        });
+        };
 
       validity = import ./check-meta.nix {
-        inherit lib config meta derivationArg;
-        mkDerivationArg = attrs;
+        inherit lib config meta;
         # Nix itself uses the `system` field of a derivation to decide where
         # to build it. This is a bit confusing for cross compilation.
-        inherit (stdenv) system;
-      };
+        inherit (stdenv) hostPlatform;
+      } attrs;
 
       # The meta attribute is passed in the resulting attribute set,
       # but it's not part of the actual derivation, i.e., it's not
@@ -218,7 +278,7 @@ rec {
       meta = {
           # `name` above includes cross-compilation cruft (and is under assert),
           # lets have a clean always accessible version here.
-          inherit name;
+          name = computedName;
 
           # If the packager hasn't specified `outputsToInstall`, choose a default,
           # which is the name of `p.bin or p.out or p`;
@@ -228,10 +288,9 @@ rec {
           #   unless they are comfortable with this default.
           outputsToInstall =
             let
-              outs = outputs'; # the value passed to derivation primitive
-              hasOutput = out: builtins.elem out outs;
-            in [( lib.findFirst hasOutput null (["bin" "out"] ++ outs) )]
-            ++ lib.optional (hasOutput "man") "man";
+              hasOutput = out: builtins.elem out outputs;
+            in [( lib.findFirst hasOutput null (["bin" "out"] ++ outputs) )]
+              ++ lib.optional (hasOutput "man") "man";
         }
         // attrs.meta or {}
         # Fill `meta.position` to identify the source location of the package.
diff --git a/pkgs/stdenv/generic/setup.sh b/pkgs/stdenv/generic/setup.sh
index d7a4781448ae..03fa589138d5 100644
--- a/pkgs/stdenv/generic/setup.sh
+++ b/pkgs/stdenv/generic/setup.sh
@@ -211,7 +211,7 @@ isELF() {
     exec {fd}< "$fn"
     read -r -n 4 -u "$fd" magic
     exec {fd}<&-
-    if [[ "$magic" =~ ELF ]]; then return 0; else return 1; fi
+    if [ "$magic" = $'\177ELF' ]; then return 0; else return 1; fi
 }
 
 # Return success if the specified file is a script (i.e. starts with
@@ -220,7 +220,6 @@ isScript() {
     local fn="$1"
     local fd
     local magic
-    if ! [ -x /bin/sh ]; then return 0; fi
     exec {fd}< "$fn"
     read -r -n 2 -u "$fd" magic
     exec {fd}<&-
@@ -258,9 +257,17 @@ shopt -s nullglob
 
 # Set up the initial path.
 PATH=
+HOST_PATH=
 for i in $initialPath; do
     if [ "$i" = / ]; then i=; fi
     addToSearchPath PATH "$i/bin"
+
+    # For backward compatibility, we add initial path to HOST_PATH so
+    # it can be used in auto patch-shebangs. Unfortunately this will
+    # not work with cross compilation.
+    if [ -z "${strictDeps-}" ]; then
+        addToSearchPath HOST_PATH "$i/bin"
+    fi
 done
 
 if (( "${NIX_DEBUG:-0}" >= 1 )); then
@@ -274,11 +281,6 @@ BASH="$SHELL"
 export CONFIG_SHELL="$SHELL"
 
 
-# Dummy implementation of the paxmark function. On Linux, this is
-# overwritten by paxctl's setup hook.
-paxmark() { true; }
-
-
 # Execute the pre-hook.
 if [ -z "${shell:-}" ]; then export shell="$SHELL"; fi
 runHook preHook
@@ -501,10 +503,14 @@ activatePackage() {
     # the transition, we do include everything in thatcase.
     #
     # TODO(@Ericson2314): Don't special-case native compilation
-    if [[ ( -z "${crossConfig-}" ||  "$hostOffset" -le -1 ) && -d "$pkg/bin" ]]; then
+    if [[ ( -z "${strictDeps-}" ||  "$hostOffset" -le -1 ) && -d "$pkg/bin" ]]; then
         addToSearchPath _PATH "$pkg/bin"
     fi
 
+    if [[ "$hostOffset" -eq 0 && -d "$pkg/bin" ]]; then
+        addToSearchPath HOST_PATH "$pkg/bin"
+    fi
+
     if [[ -f "$pkg/nix-support/setup-hook" ]]; then
         local oldOpts="$(shopt -po nounset)"
         set +u
@@ -551,7 +557,7 @@ _addToEnv() {
         for depTargetOffset in "${allPlatOffsets[@]}"; do
             (( "$depHostOffset" <= "$depTargetOffset" )) || continue
             local hookRef="${hookVar}[$depTargetOffset - $depHostOffset]"
-            if [[ -z "${crossConfig-}" ]]; then
+            if [[ -z "${strictDeps-}" ]]; then
                 # Apply environment hooks to all packages during native
                 # compilation to ease the transition.
                 #
@@ -641,29 +647,25 @@ fi
 # Textual substitution functions.
 
 
-substitute() {
-    local input="$1"
-    local output="$2"
+substituteStream() {
+    local var=$1
+    local description=$2
     shift 2
 
-    if [ ! -f "$input" ]; then
-      echo "substitute(): ERROR: file '$input' does not exist" >&2
-      return 1
-    fi
-
-    local content
-    # read returns non-0 on EOF, so we want read to fail
-    if IFS='' read -r -N 0 content < "$input"; then
-        echo "substitute(): ERROR: File \"$input\" has null bytes, won't process" >&2
-        return 1
-    fi
-
     while (( "$#" )); do
         case "$1" in
             --replace)
                 pattern="$2"
                 replacement="$3"
                 shift 3
+                local savedvar
+                savedvar="${!var}"
+                eval "$var"'=${'"$var"'//"$pattern"/"$replacement"}'
+                if [ "$pattern" != "$replacement" ]; then
+                    if [ "${!var}" == "$savedvar" ]; then
+                        echo "substituteStream(): WARNING: pattern '$pattern' doesn't match anything in $description" >&2
+                    fi
+                fi
                 ;;
 
             --subst-var)
@@ -671,32 +673,59 @@ substitute() {
                 shift 2
                 # check if the used nix attribute name is a valid bash name
                 if ! [[ "$varName" =~ ^[a-zA-Z_][a-zA-Z0-9_]*$ ]]; then
-                    echo "substitute(): ERROR: substitution variables must be valid Bash names, \"$varName\" isn't." >&2
+                    echo "substituteStream(): ERROR: substitution variables must be valid Bash names, \"$varName\" isn't." >&2
+                    return 1
+                fi
+                if [ -z ${!varName+x} ]; then
+                    echo "substituteStream(): ERROR: variable \$$varName is unset" >&2
                     return 1
                 fi
                 pattern="@$varName@"
                 replacement="${!varName}"
+                eval "$var"'=${'"$var"'//"$pattern"/"$replacement"}'
                 ;;
 
             --subst-var-by)
                 pattern="@$2@"
                 replacement="$3"
+                eval "$var"'=${'"$var"'//"$pattern"/"$replacement"}'
                 shift 3
                 ;;
 
             *)
-                echo "substitute(): ERROR: Invalid command line argument: $1" >&2
+                echo "substituteStream(): ERROR: Invalid command line argument: $1" >&2
                 return 1
                 ;;
         esac
-
-        content="${content//"$pattern"/$replacement}"
     done
 
-    if [ -e "$output" ]; then chmod +w "$output"; fi
-    printf "%s" "$content" > "$output"
+    printf "%s" "${!var}"
+}
+
+consumeEntire() {
+    # read returns non-0 on EOF, so we want read to fail
+    if IFS='' read -r -N 0 $1; then
+        echo "consumeEntire(): ERROR: Input null bytes, won't process" >&2
+        return 1
+    fi
 }
 
+substitute() {
+    local input="$1"
+    local output="$2"
+    shift 2
+
+    if [ ! -f "$input" ]; then
+        echo "substitute(): ERROR: file '$input' does not exist" >&2
+        return 1
+    fi
+
+    local content
+    consumeEntire content < "$input"
+
+    if [ -e "$output" ]; then chmod +w "$output"; fi
+    substituteStream content "file '$input'" "$@" > "$output"
+}
 
 substituteInPlace() {
     local fileName="$1"
@@ -704,20 +733,30 @@ substituteInPlace() {
     substitute "$fileName" "$fileName" "$@"
 }
 
+_allFlags() {
+    for varName in $(awk 'BEGIN { for (v in ENVIRON) if (v ~ /^[a-z][a-zA-Z0-9_]*$/) print v }'); do
+        if (( "${NIX_DEBUG:-0}" >= 1 )); then
+            printf "@%s@ -> %q\n" "${varName}" "${!varName}"
+        fi
+        args+=("--subst-var" "$varName")
+    done
+}
+
+substituteAllStream() {
+    local -a args=()
+    _allFlags
+
+    substituteStream "$1" "$2" "${args[@]}"
+}
 
 # Substitute all environment variables that start with a lowercase character and
 # are valid Bash names.
 substituteAll() {
     local input="$1"
     local output="$2"
-    local -a args=()
 
-    for varName in $(awk 'BEGIN { for (v in ENVIRON) if (v ~ /^[a-z][a-zA-Z0-9_]*$/) print v }'); do
-        if (( "${NIX_DEBUG:-0}" >= 1 )); then
-            printf "@%s@ -> %q\n" "${varName}" "${!varName}"
-        fi
-        args+=("--subst-var" "$varName")
-    done
+    local -a args=()
+    _allFlags
 
     substitute "$input" "$output" "${args[@]}"
 }
@@ -774,11 +813,11 @@ _defaultUnpack() {
     else
 
         case "$fn" in
-            *.tar.xz | *.tar.lzma)
+            *.tar.xz | *.tar.lzma | *.txz)
                 # Don't rely on tar knowing about .xz.
                 xz -d < "$fn" | tar xf -
                 ;;
-            *.tar | *.tar.* | *.tgz | *.tbz2)
+            *.tar | *.tar.* | *.tgz | *.tbz2 | *.tbz)
                 # GNU tar can automatically select the decompression method
                 # (info "(tar) gzip").
                 tar xf "$fn"
@@ -968,9 +1007,11 @@ buildPhase() {
     # set to empty if unset
     : ${makeFlags=}
 
-    if [[ -z "$makeFlags" && ! ( -n "${makefile:-}" || -e Makefile || -e makefile || -e GNUmakefile ) ]]; then
+    if [[ -z "$makeFlags" && -z "${makefile:-}" && ! ( -e Makefile || -e makefile || -e GNUmakefile ) ]]; then
         echo "no Makefile, doing nothing"
     else
+        foundMakefile=1
+
         # See https://github.com/NixOS/nixpkgs/pull/1354#issuecomment-31260409
         makeFlags="SHELL=$SHELL $makeFlags"
 
@@ -994,18 +1035,38 @@ buildPhase() {
 checkPhase() {
     runHook preCheck
 
-    # Old bash empty array hack
-    # shellcheck disable=SC2086
-    local flagsArray=(
-        ${enableParallelBuilding:+-j${NIX_BUILD_CORES} -l${NIX_BUILD_CORES}}
-        $makeFlags ${makeFlagsArray+"${makeFlagsArray[@]}"}
-        ${checkFlags:-VERBOSE=y} ${checkFlagsArray+"${checkFlagsArray[@]}"}
-        ${checkTarget:-check}
-    )
+    if [[ -z "${foundMakefile:-}" ]]; then
+        echo "no Makefile or custom buildPhase, doing nothing"
+        runHook postCheck
+        return
+    fi
 
-    echoCmd 'check flags' "${flagsArray[@]}"
-    make ${makefile:+-f $makefile} "${flagsArray[@]}"
-    unset flagsArray
+    if [[ -z "${checkTarget:-}" ]]; then
+        #TODO(@oxij): should flagsArray influence make -n?
+        if make -n ${makefile:+-f $makefile} check >/dev/null 2>&1; then
+            checkTarget=check
+        elif make -n ${makefile:+-f $makefile} test >/dev/null 2>&1; then
+            checkTarget=test
+        fi
+    fi
+
+    if [[ -z "${checkTarget:-}" ]]; then
+        echo "no check/test target in ${makefile:-Makefile}, doing nothing"
+    else
+        # Old bash empty array hack
+        # shellcheck disable=SC2086
+        local flagsArray=(
+            ${enableParallelChecking:+-j${NIX_BUILD_CORES} -l${NIX_BUILD_CORES}}
+            $makeFlags ${makeFlagsArray+"${makeFlagsArray[@]}"}
+            ${checkFlags:-VERBOSE=y} ${checkFlagsArray+"${checkFlagsArray[@]}"}
+            ${checkTarget}
+        )
+
+        echoCmd 'check flags' "${flagsArray[@]}"
+        make ${makefile:+-f $makefile} "${flagsArray[@]}"
+
+        unset flagsArray
+    fi
 
     runHook postCheck
 }
@@ -1018,14 +1079,12 @@ installPhase() {
         mkdir -p "$prefix"
     fi
 
-    installTargets="${installTargets:-install}"
-
     # Old bash empty array hack
     # shellcheck disable=SC2086
     local flagsArray=(
-        $installTargets
         $makeFlags ${makeFlagsArray+"${makeFlagsArray[@]}"}
         $installFlags ${installFlagsArray+"${installFlagsArray[@]}"}
+        ${installTargets:-install}
     )
 
     echoCmd 'install flags' "${flagsArray[@]}"
@@ -1091,6 +1150,19 @@ fixupPhase() {
         substituteAll "$setupHook" "${!outputDev}/nix-support/setup-hook"
     fi
 
+    # TODO(@Ericson2314): Remove after https://github.com/NixOS/nixpkgs/pull/31414
+    if [