darwin/stdenv: refactoring

Build the llvm support libraries (libcxx, libcxxabi) from scratch
without using the existing llvm libraries. This is the same spirit and
similar implementation as the "useLLVM" bootstrap in llvm package
sets. Critically it avoids having libcxxabi provided by the cc-wrapper
when building libcxx, which otherwise results in two libcxxabi
instances.

$ otool -L /nix/store/vd4vvgs9xngqbjzpg3qc41wl6jh42s9i-libc++-7.1.0/lib/libc++.dylib
/nix/store/vd4vvgs9xngqbjzpg3qc41wl6jh42s9i-libc++-7.1.0/lib/libc++.dylib:
        /nix/store/vd4vvgs9xngqbjzpg3qc41wl6jh42s9i-libc++-7.1.0/lib/libc++.1.0.dylib (compatibility version 1.0.0, current version 1.0.0)
        /nix/store/gmpwk5fyp3iasppqrrdpswxvid6kcp8r-libc++abi-7.1.0/lib/libc++abi.dylib (compatibility version 1.0.0, current version 1.0.0)
        /nix/store/3hn7azynqgp2pm5gpdg45gpq0ia72skg-libc++abi-7.1.0/lib/libc++abi.dylib (compatibility version 1.0.0, current version 1.0.0)
        /nix/store/1nq94scbxs6bk7pimqhvz76q6cfmbv97-Libsystem-osx-10.12.6/lib/libSystem.B.dylib (compatibility version 1.0.0, current version 1226.10.1)

Additionally move some utilities (clang, binutils, coreutils, gnugrep)
to the stage layers so they can be replaced before the final
stdenv. This should cause most of stage4 to be built from the
toolchain assembled as of stage3 instead of the bootstrap toolchain.

1 files changed, 91 insertions, 43 deletions

diff --git a/pkgs/stdenv/darwin/default.nix b/pkgs/stdenv/darwin/default.nix
index 5386213f2ad7..fbd5f1cf3f01 100644
--- a/pkgs/stdenv/darwin/default.nix
+++ b/pkgs/stdenv/darwin/default.nix
@@ -74,44 +74,48 @@ in rec {
         inherit (last) stdenv;
       };
 
-      coreutils = { name = "${name}-coreutils"; outPath = bootstrapTools; };
-      gnugrep   = { name = "${name}-gnugrep";   outPath = bootstrapTools; };
-
-      bintools = import ../../build-support/bintools-wrapper {
-        inherit shell;
-        inherit (last) stdenvNoCC;
-
-        nativeTools  = false;
-        nativeLibc   = false;
-        inherit buildPackages coreutils gnugrep;
-        libc         = last.pkgs.darwin.Libsystem;
-        bintools     = { name = "${name}-binutils"; outPath = bootstrapTools; };
-      };
-
-      cc = if last == null then "/dev/null" else import ../../build-support/cc-wrapper {
-        inherit shell;
-        inherit (last) stdenvNoCC;
-
+      mkExtraBuildCommands = cc: ''
+        rsrc="$out/resource-root"
+        mkdir "$rsrc"
+        ln -s "${cc}/lib/clang/${cc.version}/include" "$rsrc"
+        ln -s "${last.pkgs.llvmPackages_7.compiler-rt.out}/lib" "$rsrc/lib"
+        echo "-resource-dir=$rsrc" >> $out/nix-support/cc-cflags
+      '';
+
+      mkCC = overrides: import ../../build-support/cc-wrapper (
+        let args = {
+          inherit shell;
+          inherit (last) stdenvNoCC;
+
+          nativeTools  = false;
+          nativeLibc   = false;
+          inherit buildPackages libcxx;
+          inherit (last.pkgs) coreutils gnugrep;
+          bintools     = last.pkgs.darwin.binutils;
+          libc         = last.pkgs.darwin.Libsystem;
+          isClang      = true;
+          cc           = last.pkgs.llvmPackages_7.clang-unwrapped;
+        }; in args // (overrides args));
+
+      cc = if last == null then "/dev/null" else mkCC ({ cc, ... }: {
         extraPackages = [
-          # last.pkgs.llvmPackages_7.libcxxabi # TODO: is this required? if not, why not?
+          last.pkgs.llvmPackages_7.libcxxabi
           last.pkgs.llvmPackages_7.compiler-rt
         ];
+        extraBuildCommands = mkExtraBuildCommands cc;
+      });
 
+      ccNoLibcxx = if last == null then "/dev/null" else mkCC ({ cc, ... }: {
+        libcxx = null;
+        extraPackages = [
+          last.pkgs.llvmPackages_7.compiler-rt
+        ];
         extraBuildCommands = ''
-          rsrc="$out/resource-root"
-          mkdir "$rsrc"
-          ln -s "${bootstrapTools}/lib/clang/${bootstrapClangVersion}/include" "$rsrc"
-          ln -s "${last.pkgs.llvmPackages_7.compiler-rt.out}/lib" "$rsrc/lib"
-          echo "-resource-dir=$rsrc" >> $out/nix-support/cc-cflags
-        '';
-
-        nativeTools  = false;
-        nativeLibc   = false;
-        inherit buildPackages coreutils gnugrep bintools libcxx;
-        libc         = last.pkgs.darwin.Libsystem;
-        isClang      = true;
-        cc           = { name = "${name}-clang"; outPath = bootstrapTools; };
-      };
+          echo "-rtlib=compiler-rt" >> $out/nix-support/cc-cflags
+          echo "-B${last.pkgs.llvmPackages_7.compiler-rt}/lib" >> $out/nix-support/cc-cflags
+          echo "-nostdlib++" >> $out/nix-support/cc-cflags
+        '' + mkExtraBuildCommands cc;
+      });
 
       thisStdenv = import ../generic {
         name = "${name}-stdenv-darwin";
@@ -150,7 +154,10 @@ in rec {
         extraAttrs = {
           inherit macosVersionMin appleSdkVersion platform;
         };
-        overrides  = self: super: (overrides self super) // { fetchurl = thisStdenv.fetchurlBoot; };
+        overrides  = self: super: (overrides self super) // {
+          inherit ccNoLibcxx;
+          fetchurl = thisStdenv.fetchurlBoot;
+        };
       };
 
     in {
@@ -160,6 +167,9 @@ in rec {
 
   stage0 = stageFun 0 null {
     overrides = self: super: with stage0; {
+      coreutils = { name = "bootstrap-stage0-coreutils"; outPath = bootstrapTools; };
+      gnugrep   = { name = "bootstrap-stage0-gnugrep";   outPath = bootstrapTools; };
+
       darwin = super.darwin // {
         Libsystem = stdenv.mkDerivation {
           name = "bootstrap-stage0-Libsystem";
@@ -170,9 +180,26 @@ in rec {
           '';
         };
         dyld = bootstrapTools;
+
+        binutils = lib.makeOverridable (import ../../build-support/bintools-wrapper) {
+          shell = "${bootstrapTools}/bin/bash";
+          inherit (self) stdenvNoCC;
+
+          nativeTools  = false;
+          nativeLibc   = false;
+          inherit (self) buildPackages coreutils gnugrep;
+          libc         = self.pkgs.darwin.Libsystem;
+          bintools     = { name = "bootstrap-stage0-binutils"; outPath = bootstrapTools; };
+        };
       };
 
       llvmPackages_7 = {
+        clang-unwrapped = {
+          name = "bootstrap-stage0-clang";
+          outPath = bootstrapTools;
+          version = bootstrapClangVersion;
+        };
+
         libcxx = stdenv.mkDerivation {
           name = "bootstrap-stage0-libcxx";
           phases = [ "installPhase" "fixupPhase" ];
@@ -219,10 +246,19 @@ in rec {
       ninja = super.ninja.override { buildDocs = false; };
 
       llvmPackages_7 = super.llvmPackages_7 // (let
+        tools = super.llvmPackages_7.tools.extend (_: _: {
+          inherit (llvmPackages_7) clang-unwrapped;
+        });
         libraries = super.llvmPackages_7.libraries.extend (_: _: {
-          inherit (llvmPackages_7) compiler-rt;
+          inherit (llvmPackages_7) compiler-rt libcxx libcxxabi;
         });
-      in { inherit libraries; } // libraries);
+      in { inherit tools libraries; } // tools // libraries);
+
+      darwin = super.darwin // {
+        binutils = darwin.binutils.override {
+          libc = self.darwin.Libsystem;
+        };
+      };
     };
   in with prevStage; stageFun 1 prevStage {
     extraPreHook = "export NIX_CFLAGS_COMPILE+=\" -F${bootstrapTools}/Library/Frameworks\"";
@@ -248,14 +284,24 @@ in rec {
         libssh2 nghttp2 libkrb5 ninja;
 
       llvmPackages_7 = super.llvmPackages_7 // (let
-        libraries = super.llvmPackages_7.libraries.extend (_: _: {
+        tools = super.llvmPackages_7.tools.extend (_: _: {
+          inherit (llvmPackages_7) clang-unwrapped;
+        });
+        libraries = super.llvmPackages_7.libraries.extend (_: libSuper: {
           inherit (llvmPackages_7) compiler-rt;
+          libcxx = libSuper.libcxx.override {
+            stdenv = overrideCC self.stdenv self.ccNoLibcxx;
+          };
+          libcxxabi = libSuper.libcxxabi.override {
+            stdenv = overrideCC self.stdenv self.ccNoLibcxx;
+            standalone = true;
+          };
         });
-      in { inherit libraries; } // libraries);
+      in { inherit tools libraries; } // tools // libraries);
 
       darwin = super.darwin // {
         inherit (darwin)
-          dyld Libsystem xnu configd ICU libdispatch libclosure launchd CF;
+          binutils dyld Libsystem xnu configd ICU libdispatch libclosure launchd CF;
       };
     };
   in with prevStage; stageFun 2 prevStage {
@@ -270,8 +316,9 @@ in rec {
     allowedRequisites =
       [ bootstrapTools ] ++
       (with pkgs; [
-        xz.bin xz.out libcxx libcxxabi llvmPackages_7.compiler-rt zlib
-        libxml2.out curl.out openssl.out libssh2.out nghttp2.lib libkrb5
+        xz.bin xz.out libcxx libcxxabi llvmPackages_7.compiler-rt
+        zlib libxml2.out curl.out openssl.out libssh2.out
+        nghttp2.lib libkrb5 coreutils gnugrep pcre.out gmp libiconv
       ]) ++
       (with pkgs.darwin; [ dyld Libsystem CF ICU locale ]);
 
@@ -320,8 +367,9 @@ in rec {
     allowedRequisites =
       [ bootstrapTools ] ++
       (with pkgs; [
-        xz.bin xz.out bash libcxx libcxxabi llvmPackages_7.compiler-rt zlib
-        libxml2.out curl.out openssl.out libssh2.out nghttp2.lib libkrb5
+        xz.bin xz.out bash libcxx libcxxabi llvmPackages_7.compiler-rt
+        zlib libxml2.out curl.out openssl.out libssh2.out
+        nghttp2.lib libkrb5 coreutils gnugrep pcre.out gmp libiconv
       ]) ++
       (with pkgs.darwin; [ dyld ICU Libsystem locale ]);