stdenv/darwin: integrate a new CoreFoundation

This also updates the bootstrap tool builder to LLVM 5, but not the ones
we actually use for bootstrap. I'll make that change in a subsequent commit
so as to provide traceable provenance of the bootstrap tools.

17 files changed, 148 insertions, 189 deletions

diff --git a/pkgs/development/interpreters/python/cpython/2.7/boot.nix b/pkgs/development/interpreters/python/cpython/2.7/boot.nix
index 7d6f2541d3d1..976d30819dbe 100644
--- a/pkgs/development/interpreters/python/cpython/2.7/boot.nix
+++ b/pkgs/development/interpreters/python/cpython/2.7/boot.nix
@@ -43,6 +43,15 @@ stdenv.mkDerivation rec {
       ./deterministic-build.patch
     ];
 
+  # Hack hack hack to stop shit from failing from a missing _scproxy on Darwin. Since
+  # we only use this python for bootstrappy things, it doesn't really matter if it
+  # doesn't have perfect proxy support in urllib :) this just makes it fall back on env
+  # vars instead of attempting to read the proxy configuration automatically, so not a
+  # huge loss even if for whatever reason we did want proxy support.
+  postPatch = ''
+    substituteInPlace Lib/urllib.py --replace "if sys.platform == 'darwin'" "if False"
+  '';
+
   DETERMINISTIC_BUILD = 1;
 
   preConfigure = ''
diff --git a/pkgs/development/libraries/libunistring/default.nix b/pkgs/development/libraries/libunistring/default.nix
index 24da3a8e2c7d..312a5c118103 100644
--- a/pkgs/development/libraries/libunistring/default.nix
+++ b/pkgs/development/libraries/libunistring/default.nix
@@ -19,7 +19,19 @@ stdenv.mkDerivation rec {
 
   doCheck = true;
 
-  enableParallelBuilding = true;
+  /* This seems to cause several random failures like these, which I assume
+     is because of bad or missing target dependencies in their build system:
+
+        ./unistdio/test-u16-vasnprintf2.sh: line 16: ./test-u16-vasnprintf1: No such file or directory
+        FAIL unistdio/test-u16-vasnprintf2.sh (exit status: 1)
+
+        FAIL: unistdio/test-u16-vasnprintf3.sh
+        ======================================
+
+        ./unistdio/test-u16-vasnprintf3.sh: line 16: ./test-u16-vasnprintf1: No such file or directory
+        FAIL unistdio/test-u16-vasnprintf3.sh (exit status: 1)
+  */
+  enableParallelBuilding = false;
 
   meta = {
     homepage = http://www.gnu.org/software/libunistring/;
diff --git a/pkgs/development/libraries/libuv/default.nix b/pkgs/development/libraries/libuv/default.nix
index 9ee934cd78bb..f75ce4396a6d 100644
--- a/pkgs/development/libraries/libuv/default.nix
+++ b/pkgs/development/libraries/libuv/default.nix
@@ -40,7 +40,7 @@ stdenv.mkDerivation rec {
         "tcp_open" "tcp_write_queue_order" "tcp_try_write" "tcp_writealot"
         "multiple_listen" "delayed_accept"
         "shutdown_close_tcp" "shutdown_eof" "shutdown_twice" "callback_stack"
-        "tty_pty"
+        "tty_pty" "condvar_5"
       ];
     tdRegexp = lib.concatStringsSep "\\|" toDisable;
     in lib.optionalString doCheck ''
diff --git a/pkgs/development/tools/build-managers/ninja/default.nix b/pkgs/development/tools/build-managers/ninja/default.nix
index b1df54f9bd5c..e1637e2f2cab 100644
--- a/pkgs/development/tools/build-managers/ninja/default.nix
+++ b/pkgs/development/tools/build-managers/ninja/default.nix
@@ -1,4 +1,6 @@
-{ stdenv, fetchFromGitHub, python, asciidoc, docbook_xml_dtd_45, docbook_xsl, libxslt, re2c }:
+{ stdenv, fetchFromGitHub, python, buildDocs ? true, asciidoc, docbook_xml_dtd_45, docbook_xsl, libxslt, re2c }:
+
+with stdenv.lib;
 
 stdenv.mkDerivation rec {
   name = "ninja-${version}";
@@ -11,25 +13,27 @@ stdenv.mkDerivation rec {
     sha256 = "16scq9hcq6c5ap6sy8j4qi75qps1zvrf3p79j1vbrvnqzp928i5f";
   };
 
-  nativeBuildInputs = [ python asciidoc docbook_xml_dtd_45 docbook_xsl libxslt.bin re2c ];
+  nativeBuildInputs = [ python ] ++ optionals buildDocs [ asciidoc docbook_xml_dtd_45 docbook_xsl libxslt.bin re2c ];
 
   buildPhase = ''
     python configure.py --bootstrap
     # "./ninja -vn manual" output copied here to support cross compilation.
+  '' + optionalString buildDocs ''
     asciidoc -b docbook -d book -o build/manual.xml doc/manual.asciidoc
     xsltproc --nonet doc/docbook.xsl build/manual.xml > doc/manual.html
   '';
 
   installPhase = ''
     install -Dm555 -t $out/bin ninja
-    install -Dm444 -t $out/share/doc/ninja doc/manual.asciidoc doc/manual.html
     install -Dm444 misc/bash-completion $out/share/bash-completion/completions/ninja
     install -Dm444 misc/zsh-completion $out/share/zsh/site-functions/_ninja
+  '' + optionalString buildDocs ''
+    install -Dm444 -t $out/share/doc/ninja doc/manual.asciidoc doc/manual.html
   '';
 
   setupHook = ./setup-hook.sh;
 
-  meta = with stdenv.lib; {
+  meta = {
     description = "Small build system with a focus on speed";
     longDescription = ''
       Ninja is a small build system with a focus on speed. It differs from
diff --git a/pkgs/os-specific/darwin/apple-source-releases/CF/add-cfmachport.patch b/pkgs/os-specific/darwin/apple-source-releases/CF/add-cfmachport.patch
deleted file mode 100644
index a1018d389c14..000000000000
--- a/pkgs/os-specific/darwin/apple-source-releases/CF/add-cfmachport.patch
+++ /dev/null
@@ -1,22 +0,0 @@
---- CF-855.17/CoreFoundation.h  2015-01-03 00:17:41.000000000 -0500
-+++ CF-855.17/CoreFoundation.h.new  2015-01-03 00:18:35.000000000 -0500
-@@ -72,6 +72,7 @@
- #include <CoreFoundation/CFDictionary.h>
- #include <CoreFoundation/CFError.h>
- #include <CoreFoundation/CFLocale.h>
-+#include <CoreFoundation/CFMachPort.h>
- #include <CoreFoundation/CFNumber.h>
- #include <CoreFoundation/CFNumberFormatter.h>
- #include <CoreFoundation/CFPreferences.h>
-
---- CF-855.17/Makefile  2015-01-03 00:32:52.000000000 -0500
-+++ CF-855.17/Makefile.new  2015-01-03 00:33:07.000000000 -0500
-@@ -9,7 +9,7 @@
- HFILES = $(wildcard *.h)
- INTERMEDIATE_HFILES = $(addprefix $(OBJBASE)/CoreFoundation/,$(HFILES))
- 
--PUBLIC_HEADERS=CFArray.h CFBag.h CFBase.h CFBinaryHeap.h CFBitVector.h CFBundle.h CFByteOrder.h CFCalendar.h CFCharacterSet.h CFData.h CFDate.h CFDateFormatter.h CFDictionary.h CFError.h CFLocale.h CFMessagePort.h CFNumber.h CFNumberFormatter.h CFPlugIn.h CFPlugInCOM.h CFPreferences.h CFPropertyList.h CFRunLoop.h CFSet.h CFSocket.h CFStream.h CFString.h CFStringEncodingExt.h CFTimeZone.h CFTree.h CFURL.h CFURLAccess.h CFUUID.h CFUserNotification.h CFXMLNode.h CFXMLParser.h CFAvailability.h CFUtilities.h CoreFoundation.h
-+PUBLIC_HEADERS=CFArray.h CFBag.h CFBase.h CFBinaryHeap.h CFBitVector.h CFBundle.h CFByteOrder.h CFCalendar.h CFCharacterSet.h CFData.h CFDate.h CFDateFormatter.h CFDictionary.h CFError.h CFLocale.h CFMachPort.h CFMessagePort.h CFNumber.h CFNumberFormatter.h CFPlugIn.h CFPlugInCOM.h CFPreferences.h CFPropertyList.h CFRunLoop.h CFSet.h CFSocket.h CFStream.h CFString.h CFStringEncodingExt.h CFTimeZone.h CFTree.h CFURL.h CFURLAccess.h CFUUID.h CFUserNotification.h CFXMLNode.h CFXMLParser.h CFAvailability.h CFUtilities.h CoreFoundation.h
- 
- PRIVATE_HEADERS=CFBundlePriv.h CFCharacterSetPriv.h CFError_Private.h CFLogUtilities.h CFPriv.h CFRuntime.h CFStorage.h CFStreamAbstract.h CFStreamPriv.h CFStreamInternal.h CFStringDefaultEncoding.h CFStringEncodingConverter.h CFStringEncodingConverterExt.h CFUniChar.h CFUnicodeDecomposition.h CFUnicodePrecomposition.h ForFoundationOnly.h CFBurstTrie.h CFICULogging.h
- 
diff --git a/pkgs/os-specific/darwin/apple-source-releases/CF/cf-bridging.patch b/pkgs/os-specific/darwin/apple-source-releases/CF/cf-bridging.patch
deleted file mode 100644
index 068a6311a9cb..000000000000
--- a/pkgs/os-specific/darwin/apple-source-releases/CF/cf-bridging.patch
+++ /dev/null
@@ -1,39 +0,0 @@
-diff --git a/CFBase.h b/CFBase.h
-index ffddd2b..e5a926b 100644
---- a/CFBase.h
-+++ b/CFBase.h
-@@ -249,6 +249,33 @@ CF_EXTERN_C_BEGIN
- #endif
- #endif
- 
-+#if __has_attribute(objc_bridge) && __has_feature(objc_bridge_id) && __has_feature(objc_bridge_id_on_typedefs)
-+
-+#ifdef __OBJC__
-+@class NSArray;
-+@class NSAttributedString;
-+@class NSString;
-+@class NSNull;
-+@class NSCharacterSet;
-+@class NSData;
-+@class NSDate;
-+@class NSTimeZone;
-+@class NSDictionary;
-+@class NSError;
-+@class NSLocale;
-+@class NSNumber;
-+@class NSSet;
-+@class NSURL;
-+#endif
-+
-+#define CF_BRIDGED_TYPE(T)		__attribute__((objc_bridge(T)))
-+#define CF_BRIDGED_MUTABLE_TYPE(T)	__attribute__((objc_bridge_mutable(T)))
-+#define CF_RELATED_TYPE(T,C,I)		__attribute__((objc_bridge_related(T,C,I)))
-+#else
-+#define CF_BRIDGED_TYPE(T)
-+#define CF_BRIDGED_MUTABLE_TYPE(T)
-+#define CF_RELATED_TYPE(T,C,I)
-+#endif
- 
- CF_EXPORT double kCFCoreFoundationVersionNumber;
- 
-
diff --git a/pkgs/os-specific/darwin/apple-source-releases/CF/default.nix b/pkgs/os-specific/darwin/apple-source-releases/CF/default.nix
deleted file mode 100644
index 5589d1592f46..000000000000
--- a/pkgs/os-specific/darwin/apple-source-releases/CF/default.nix
+++ /dev/null
@@ -1,51 +0,0 @@
-{ stdenv, appleDerivation, ICU, dyld, libdispatch, libplatform, launchd, libclosure }:
-
-# this project uses blocks, a clang-only extension
-assert stdenv.cc.isClang;
-
-appleDerivation {
-  buildInputs = [ dyld ICU libdispatch libplatform launchd libclosure ];
-
-  patches = [ ./add-cfmachport.patch ./cf-bridging.patch ./remove-xpc.patch ];
-
-  __propagatedImpureHostDeps = [ "/System/Library/Frameworks/CoreFoundation.framework/CoreFoundation" ];
-
-  preBuild = ''
-    substituteInPlace Makefile \
-      --replace "/usr/bin/clang" "clang" \
-      --replace "-arch i386 " "" \
-      --replace "/usr/bin/" "" \
-      --replace "/usr/sbin/" "" \
-      --replace "/bin/" "" \
-      --replace "INSTALLNAME=/System" "INSTALLNAME=$out" \
-      --replace "install_name_tool -id /System/Library/Frameworks" "install_name_tool -id @rpath" \
-      --replace 'chown -RH -f root:wheel $(DSTBASE)/CoreFoundation.framework' "" \
-      --replace 'chmod -RH' 'chmod -R'
-
-    # with this file present, CoreFoundation gets a _main symbol defined, which can
-    # interfere with linking other programs
-    rm plconvert.c
-
-    replacement=''$'#define __PTK_FRAMEWORK_COREFOUNDATION_KEY5 55\n#define _pthread_getspecific_direct(key) pthread_getspecific((key))\n#define _pthread_setspecific_direct(key, val) pthread_setspecific((key), (val))'
-
-    substituteInPlace CFPlatform.c --replace "#include <pthread/tsd_private.h>" "$replacement"
-
-    substituteInPlace CFRunLoop.c --replace "#include <pthread/private.h>" ""
-
-    substituteInPlace CFURLPriv.h \
-      --replace "#include <CoreFoundation/CFFileSecurity.h>" "" \
-      --replace "#include <CoreFoundation/CFURLEnumerator.h>" "" \
-      --replace "CFFileSecurityRef" "void *" \
-      --replace "CFURLEnumeratorResult" "void *" \
-      --replace "CFURLEnumeratorRef" "void *"
-
-    export DSTROOT=$out
-  '';
-
-  postInstall = ''
-    mv $out/System/* $out
-    rmdir $out/System
-    mv $out/Library/Frameworks/CoreFoundation.framework/Versions/A/PrivateHeaders/* \
-       $out/Library/Frameworks/CoreFoundation.framework/Versions/A/Headers
-  '';
-}
diff --git a/pkgs/os-specific/darwin/apple-source-releases/CF/remove-xpc.patch b/pkgs/os-specific/darwin/apple-source-releases/CF/remove-xpc.patch
deleted file mode 100644
index a7b9fe486434..000000000000
--- a/pkgs/os-specific/darwin/apple-source-releases/CF/remove-xpc.patch
+++ /dev/null
@@ -1,17 +0,0 @@
-diff --git a/CFBundlePriv.h b/CFBundlePriv.h
-index d4feb5f..e7b52e8 100644
---- a/CFBundlePriv.h
-+++ b/CFBundlePriv.h
-@@ -254,12 +254,6 @@ Boolean _CFBundleGetStringsFilesShared(CFBundleRef bundle);
- CF_EXPORT
- CFURLRef _CFBundleCopyFrameworkURLForExecutablePath(CFStringRef executablePath);
- 
--#if (TARGET_OS_MAC && !(TARGET_OS_EMBEDDED || TARGET_OS_IPHONE)) || (TARGET_OS_EMBEDDED || TARGET_OS_IPHONE)
--#include <xpc/xpc.h>
--CF_EXPORT
--void _CFBundleSetupXPCBootstrap(xpc_object_t bootstrap) CF_AVAILABLE(10_10, 8_0);
--#endif
--
- /* Functions deprecated as SPI */
- 
- CF_EXPORT
diff --git a/pkgs/os-specific/darwin/apple-source-releases/default.nix b/pkgs/os-specific/darwin/apple-source-releases/default.nix
index e2d62cef1ba4..d490048c4df3 100644
--- a/pkgs/os-specific/darwin/apple-source-releases/default.nix
+++ b/pkgs/os-specific/darwin/apple-source-releases/default.nix
@@ -204,7 +204,6 @@ let
     bootstrap_cmds  = applePackage "bootstrap_cmds"    "dev-tools-7.0"   "1v5dv2q3af1xwj5kz0a5g54fd5dm6j4c9dd2g66n4kc44ixyrhp3" {};
     bsdmake         = applePackage "bsdmake"           "dev-tools-3.2.6" "11a9kkhz5bfgi1i8kpdkis78lhc6b5vxmhd598fcdgra1jw4iac2" {};
     CarbonHeaders   = applePackage "CarbonHeaders"     "osx-10.6.2"      "1zam29847cxr6y9rnl76zqmkbac53nx0szmqm9w5p469a6wzjqar" {};
-    CF              = applePackage "CF"                "osx-10.10.5"     "07f5psjxi7wyd13ci4x83ya5hy6p69sjfqcpp2mmxdlhd8yzkf74" {};
     CommonCrypto    = applePackage "CommonCrypto"      "osx-10.11.6"     "0vllfpb8f4f97wj2vpdd7w5k9ibnsbr6ff1zslpp6q323h01n25y" {};
     configd         = applePackage "configd"           "osx-10.8.5"      "1gxakahk8gallf16xmhxhprdxkh3prrmzxnmxfvj0slr0939mmr2" {};
     copyfile        = applePackage "copyfile"          "osx-10.11.6"     "1rkf3iaxmjz5ycgrmf0g971kh90jb2z1zqxg5vlqz001s4y457gs" {};
diff --git a/pkgs/os-specific/darwin/apple-source-releases/libsecurity_generic/default.nix b/pkgs/os-specific/darwin/apple-source-releases/libsecurity_generic/default.nix
index fe68ee78d1f0..714524e8da58 100644
--- a/pkgs/os-specific/darwin/apple-source-releases/libsecurity_generic/default.nix
+++ b/pkgs/os-specific/darwin/apple-source-releases/libsecurity_generic/default.nix
@@ -33,7 +33,6 @@ name: version: sha256: args: let
         pkgs.gnustep.make
         pkgs.darwin.apple_sdk.frameworks.AppKit
         pkgs.darwin.apple_sdk.frameworks.Foundation
-        pkgs.darwin.cf-private
       ];
       makeFlags = [
         "-f${makeFile}"
diff --git a/pkgs/os-specific/darwin/apple-source-releases/libsecurity_keychain/default.nix b/pkgs/os-specific/darwin/apple-source-releases/libsecurity_keychain/default.nix
index 609d07fda100..724c4788b6cc 100644
--- a/pkgs/os-specific/darwin/apple-source-releases/libsecurity_keychain/default.nix
+++ b/pkgs/os-specific/darwin/apple-source-releases/libsecurity_keychain/default.nix
@@ -39,9 +39,5 @@ appleDerivation {
       --replace 'return mLoginDLDbIdentifier;' 'return mLoginDLDbIdentifier; }' \
       --replace '_xpc_runtime_is_app_sandboxed()' 'false'
       # hope that doesn't hurt anything
-
-    substituteInPlace lib/KCEventNotifier.h --replace \
-      'CoreFoundation/CFNotificationCenter.h' \
-      '${apple_sdk.sdk.out}/Library/Frameworks/CoreFoundation.framework/Versions/A/Headers/CFNotificationCenter.h'
   '';
 }
diff --git a/pkgs/os-specific/darwin/cf-private/default.nix b/pkgs/os-specific/darwin/cf-private/default.nix
index 603c0f652b01..3fac20d23c78 100644
--- a/pkgs/os-specific/darwin/cf-private/default.nix
+++ b/pkgs/os-specific/darwin/cf-private/default.nix
@@ -1,21 +1,59 @@
-{ stdenv, osx_private_sdk, CF }:
+{ CF, apple_sdk }:
 
-stdenv.mkDerivation {
-  name = "${CF.name}-private";
-  phases = [ "installPhase" "fixupPhase" ];
-  installPhase = ''
-    dest=$out/Library/Frameworks/CoreFoundation.framework/Headers
-    mkdir -p $dest
-    pushd $dest
-      for file in ${CF}/Library/Frameworks/CoreFoundation.framework/Headers/*; do
-        ln -sf $file
-      done
-
-      # Copy or overwrite private headers, some of these might already
-      # exist in CF but the private versions have more information.
-      cp -Lfv ${osx_private_sdk}/include/CoreFoundationPrivateHeaders/* $dest
-    popd
-  '';
+# cf-private is a bit weird, but boils down to CF with a weird setup-hook that
+# makes a build link against the system CoreFoundation rather than our pure one.
+# The reason it exists is that although our CF headers and build are pretty legit
+# now, the underlying runtime is quite different. Apple's in a bit of flux around CF
+# right now, and support three different backends for it: swift, "C", and an ObjC
+# one. The former two can be built from public sources, but the ObjC one isn't really
+# public. Unfortunately, it's also one of the core underpinnings of a lot of Mac-
+# specific behavior, and defines a lot of symbols that some Objective C apps depend
+# on, even though one might expect those symbols to derive from Foundation. So if
+# your app relies on NSArray and several other basic ObjC types, it turns out that
+# because of their magic "toll-free bridging" support, the symbols for those types
+# live in CoreFoundation with an ObjC runtime. And because that isn't public, we have
+# this hack in place to let people link properly anyway. Phew!
+# 
+# This can be revisited if Apple ever decide to release the ObjC backend in a publicly
+# buildable form.
+# 
+# This doesn't really need to rebuild CF, but it's cheap, and adding a setup hook to
+# an existing package was annoying. We need a buildEnv that knows how to add those
+CF.overrideAttrs (orig: {
+  # PLEASE if you add things to this derivation, explain in reasonable detail why
+  # you're adding them and when the workaround can go away. This whole derivation is
+  # a workaround and if you don't explain what you're working around, it makes it
+  # very hard for people to clean it up later.
 
+  name = "${orig.name}-private";
   setupHook = ./setup-hook.sh;
-}
+
+  # TODO: consider re-adding https://github.com/NixOS/nixpkgs/blob/master/pkgs/os-specific/darwin/apple-source-releases/CF/cf-bridging.patch
+  # once the missing headers are in and see if that fixes all need for this.
+
+  # This can go away once https://bugs.swift.org/browse/SR-8741 happens, which is
+  # looking more likely these days with the friendly people at Apple! We only need
+  # the header because the setup hook takes care of linking us against a version
+  # of the framework with the functionality built into it. The main user I know of
+  # this is watchman, who can almost certainly switch to the pure CF once the header
+  # and functionality is merged in.
+  installPhase = orig.installPhase + ''
+    basepath="Library/Frameworks/CoreFoundation.framework/Headers"
+    path="$basepath/CFFileDescriptor.h"
+
+    # Append the include at top level or nobody will notice the header we're about to add
+    sed -i '/CFNotificationCenter.h/a #include <CoreFoundation/CFFileDescriptor.h>' \
+      "$out/$basepath/CoreFoundation.h"
+
+    cp ${apple_sdk.frameworks.CoreFoundation}/$path $out/$path
+  '' +
+  # This one is less likely to go away, but I'll mention it anyway. The issue is at
+  # https://bugs.swift.org/browse/SR-8744, and the main user I know of is qtbase
+  ''
+    path="$basepath/CFURLEnumerator.h"    
+    sed -i '/CFNotificationCenter.h/a #include <CoreFoundation/CFURLEnumerator.h>' \
+      "$out/$basepath/CoreFoundation.h"
+
+    cp ${apple_sdk.frameworks.CoreFoundation}/$path $out/$path
+  '';
+})
\ No newline at end of file
diff --git a/pkgs/os-specific/darwin/swift-corelibs/corefoundation.nix b/pkgs/os-specific/darwin/swift-corelibs/corefoundation.nix
index 1dea55cccc9e..f819429f4deb 100644
--- a/pkgs/os-specific/darwin/swift-corelibs/corefoundation.nix
+++ b/pkgs/os-specific/darwin/swift-corelibs/corefoundation.nix
@@ -14,11 +14,12 @@ in stdenv.mkDerivation {
   src = fetchFromGitHub {
     owner  = "apple";
     repo   = "swift-corelibs-foundation";
-    rev    = "85c640e7ce50e6ca61a134c72270e214bc63fdba"; # https://github.com/apple/swift-corelibs-foundation/pull/1686
-    sha256 = "0z2v278wy7jh0c92g1dszd8hj8naxari660sqx6yab5dwapd46qc";
+    rev    = "71aaba20e1450a82c516af1342fe23268e15de0a";
+    sha256 = "17kpql0f27xxz4jjw84vpas5f5sn4vdqwv10g151rc3rswbwln1z";
   };
 
-  buildInputs = [ ninja python libxml2 objc4 ICU curl ];
+  nativeBuildInputs = [ ninja python ];
+  buildInputs = [ libxml2 objc4 ICU curl ];
 
   sourceRoot = "source/CoreFoundation";
 
@@ -31,8 +32,7 @@ in stdenv.mkDerivation {
     # 3. Use the legit CoreFoundation.h, not the one telling you not to use it because of Swift
     substituteInPlace build.py \
       --replace "cf.CFLAGS += '-DDEPLOYMENT" '#' \
-      --replace "cf.LDFLAGS += '-ldispatch" '#' \
-      --replace "Base.subproj/SwiftRuntime/CoreFoundation.h" 'Base.subproj/CoreFoundation.h'
+      --replace "cf.LDFLAGS += '-ldispatch" '#'
 
     # Includes xpc for some initialization routine that they don't define anyway, so no harm here
     substituteInPlace PlugIn.subproj/CFBundlePriv.h \
@@ -53,8 +53,11 @@ in stdenv.mkDerivation {
 
   BUILD_DIR = "./Build";
   CFLAGS = "-DINCLUDE_OBJC -I${libxml2.dev}/include/libxml2"; # They seem to assume we include objc in some places and not in others, make a PR; also not sure why but libxml2 include path isn't getting picked up from buildInputs
-  LDFLAGS = "-install_name ${placeholder "out"}/Frameworks/CoreFoundation.framework/CoreFoundation -current_version 1234.56.7 -compatibility_version 150.0.0 -init ___CFInitialize";
-  configurePhase = "../configure --sysroot unused";
+  
+  # I'm guessing at the version here. https://github.com/apple/swift-corelibs-foundation/commit/df3ec55fe6c162d590a7653d89ad669c2b9716b1 imported "high sierra"
+  # and this version is a version from there. No idea how accurate it is.
+  LDFLAGS = "-current_version 1454.90.0 -compatibility_version 150.0.0 -init ___CFInitialize";
+  configurePhase = "../configure release --sysroot UNUSED";
 
   enableParallelBuilding = true;
   buildPhase = "ninja -j $NIX_BUILD_CORES";
@@ -66,6 +69,12 @@ in stdenv.mkDerivation {
     mkdir -p $base/Versions/A/{Headers,PrivateHeaders,Modules}
 
     cp ./Build/CoreFoundation/libCoreFoundation.dylib $base/Versions/A/CoreFoundation
+
+    # Note that this could easily live in the ldflags above as `-install_name @rpath/...` but
+    # https://github.com/NixOS/nixpkgs/issues/46434 thwarts that, so for now I'm hacking it up
+    # after the fact.
+    install_name_tool -id '@rpath/CoreFoundation.framework/Versions/A/CoreFoundation' $base/Versions/A/CoreFoundation
+
     cp ./Build/CoreFoundation/usr/include/CoreFoundation/*.h $base/Versions/A/Headers
     cp ./Build/CoreFoundation/usr/include/CoreFoundation/module.modulemap $base/Versions/A/Modules
 
diff --git a/pkgs/os-specific/darwin/swift-corelibs/default.nix b/pkgs/os-specific/darwin/swift-corelibs/default.nix
deleted file mode 100644
index 0d96b8fd008e..000000000000
--- a/pkgs/os-specific/darwin/swift-corelibs/default.nix
+++ /dev/null
@@ -1,8 +0,0 @@
-{ callPackage, darwin }:
-
-rec {
-  corefoundation = callPackage ./corefoundation.nix { inherit (darwin) objc4 ICU; };
-  libdispatch = callPackage ./libdispatch.nix {
-   inherit (darwin) apple_sdk_sierra xnu;
-  };
-}
diff --git a/pkgs/stdenv/darwin/default.nix b/pkgs/stdenv/darwin/default.nix
index 6d224e4cc44f..d287517e8868 100644
--- a/pkgs/stdenv/darwin/default.nix
+++ b/pkgs/stdenv/darwin/default.nix
@@ -198,6 +198,9 @@ in rec {
         CF = null;  # use CoreFoundation from bootstrap-tools
         configd = null;
       };
+      python2 = self.python;
+
+      ninja = super.ninja.override { buildDocs = false; };
     };
   in with prevStage; stageFun 1 prevStage {
     extraPreHook = "export NIX_CFLAGS_COMPILE+=\" -F${bootstrapTools}/Library/Frameworks\"";
@@ -217,11 +220,12 @@ in rec {
         zlib patchutils m4 scons flex perl bison unifdef unzip openssl python
         libxml2 gettext sharutils gmp libarchive ncurses pkg-config libedit groff
         openssh sqlite sed serf openldap db cyrus-sasl expat apr-util subversion xz
-        findfreetype libssh curl cmake autoconf automake libtool ed cpio coreutils;
+        findfreetype libssh curl cmake autoconf automake libtool ed cpio coreutils
+        libssh2 nghttp2 libkrb5 python2 ninja;
 
       darwin = super.darwin // {
         inherit (darwin)
-          dyld Libsystem xnu configd ICU libdispatch libclosure launchd;
+          dyld Libsystem xnu configd ICU libdispatch libclosure launchd CF;
       };
     };
   in with prevStage; stageFun 2 prevStage {
@@ -235,7 +239,10 @@ in rec {
 
     allowedRequisites =
       [ bootstrapTools ] ++
-      (with pkgs; [ xz.bin xz.out libcxx libcxxabi ]) ++
+      (with pkgs; [
+        xz.bin xz.out libcxx libcxxabi zlib libxml2.out curl.out openssl.out libssh2.out
+        nghttp2.lib libkrb5
+      ]) ++
       (with pkgs.darwin; [ dyld Libsystem CF ICU locale ]);
 
     overrides = persistent;
@@ -247,9 +254,10 @@ in rec {
         patchutils m4 scons flex perl bison unifdef unzip openssl python
         gettext sharutils libarchive pkg-config groff bash subversion
         openssh sqlite sed serf openldap db cyrus-sasl expat apr-util
-        findfreetype libssh curl cmake autoconf automake libtool cpio;
+        findfreetype libssh curl cmake autoconf automake libtool cpio
+        libssh2 nghttp2 libkrb5 python2 ninja;
 
-      # Avoid pulling in a full python and it's extra dependencies for the llvm/clang builds.
+      # Avoid pulling in a full python and its extra dependencies for the llvm/clang builds.
       libxml2 = super.libxml2.override { pythonSupport = false; };
 
       llvmPackages_5 = super.llvmPackages_5 // (let
@@ -281,7 +289,10 @@ in rec {
 
     allowedRequisites =
       [ bootstrapTools ] ++
-      (with pkgs; [ xz.bin xz.out bash libcxx libcxxabi ]) ++
+      (with pkgs; [
+        xz.bin xz.out bash libcxx libcxxabi zlib libxml2.out curl.out openssl.out libssh2.out
+        nghttp2.lib libkrb5
+      ]) ++
       (with pkgs.darwin; [ dyld ICU Libsystem locale ]);
 
     overrides = persistent;
@@ -292,7 +303,7 @@ in rec {
       inherit
         gnumake gzip gnused bzip2 gawk ed xz patch bash
         ncurses libffi zlib gmp pcre gnugrep
-        coreutils findutils diffutils patchutils;
+        coreutils findutils diffutils patchutils ninja;
 
       # Hack to make sure we don't link ncurses in bootstrap tools. The proper
       # solution is to avoid passing -L/nix-store/...-bootstrap-tools/lib,
@@ -312,8 +323,14 @@ in rec {
         });
       in { inherit tools libraries; } // tools // libraries);
 
-      darwin = super.darwin // {
+      darwin = super.darwin // rec {
         inherit (darwin) dyld Libsystem libiconv locale;
+
+        libxml2-nopython = super.libxml2.override { pythonSupport = false; };
+        CF = super.darwin.CF.override {
+          libxml2 = libxml2-nopython;
+          python = prevStage.python;
+        };
       };
     };
   in with prevStage; stageFun 4 prevStage {
@@ -345,6 +362,17 @@ in rec {
         });
       in { inherit tools libraries; } // tools // libraries);
 
+      # N.B: the important thing here is to ensure that python == python2
+      # == python27 or you get weird issues with inconsistent package sets.
+      # In a particularly subtle bug, I overrode python2 instead of python27
+      # here, and it caused gnome-doc-utils to complain about:
+      # "PyThreadState_Get: no current thread". This is because Python gets
+      # really unhappy if you have Python A which loads a native python lib
+      # which was linked against Python B, which in our case was happening
+      # because we didn't override python "deeply enough". Anyway, this works
+      # and I'm just leaving this blurb here so people realize why it matters
+      python27 = super.python27.override { CF = prevStage.darwin.CF; };
+
       darwin = super.darwin // {
         inherit (darwin) dyld ICU Libsystem libiconv;
       } // lib.optionalAttrs (super.stdenv.targetPlatform == localSystem) {
@@ -398,9 +426,10 @@ in rec {
       gzip ncurses.out ncurses.dev ncurses.man gnused bash gawk
       gnugrep llvmPackages.clang-unwrapped llvmPackages.clang-unwrapped.lib patch pcre.out gettext
       binutils.bintools darwin.binutils darwin.binutils.bintools
+      curl.out openssl.out libssh2.out nghttp2.lib libkrb5
       cc.expand-response-params
     ]) ++ (with pkgs.darwin; [
-      dyld Libsystem CF cctools ICU libiconv locale
+      dyld Libsystem CF cctools ICU libiconv locale libxml2-nopython.out
     ]);
 
     overrides = lib.composeExtensions persistent (self: super: {
diff --git a/pkgs/stdenv/darwin/make-bootstrap-tools.nix b/pkgs/stdenv/darwin/make-bootstrap-tools.nix
index 66c5f419f2f6..d128be7019bd 100644
--- a/pkgs/stdenv/darwin/make-bootstrap-tools.nix
+++ b/pkgs/stdenv/darwin/make-bootstrap-tools.nix
@@ -3,7 +3,7 @@
 with import pkgspath { inherit system; };
 
 let
-  llvmPackages = llvmPackages_4;
+  llvmPackages = llvmPackages_5;
 in rec {
   coreutils_ = coreutils.override (args: {
     # We want coreutils without ACL support.
@@ -73,6 +73,7 @@ in rec {
       cp -d ${gettext}/lib/libintl*.dylib $out/lib
       chmod +x $out/lib/libintl*.dylib
       cp -d ${ncurses.out}/lib/libncurses*.dylib $out/lib
+      cp -d ${libxml2.out}/lib/libxml2*.dylib $out/lib
 
       # Copy what we need of clang
       cp -d ${llvmPackages.clang-unwrapped}/bin/clang $out/bin
diff --git a/pkgs/top-level/darwin-packages.nix b/pkgs/top-level/darwin-packages.nix
index b0c82508c114..3bf7c31b7008 100644
--- a/pkgs/top-level/darwin-packages.nix
+++ b/pkgs/top-level/darwin-packages.nix
@@ -31,10 +31,7 @@ in
     libcxxabi = pkgs.libcxxabi;
   };
 
-  cf-private = callPackage ../os-specific/darwin/cf-private {
-    inherit (apple-source-releases) CF;
-    inherit (darwin) osx_private_sdk;
-  };
+  cf-private = callPackage ../os-specific/darwin/cf-private { inherit (darwin) CF apple_sdk; };
 
   DarwinTools = callPackage ../os-specific/darwin/DarwinTools { };
 
@@ -76,7 +73,10 @@ in
 
   CoreSymbolication = callPackage ../os-specific/darwin/CoreSymbolication { };
 
-  swift-corelibs = callPackages ../os-specific/darwin/swift-corelibs { };
+  CF = callPackage ../os-specific/darwin/swift-corelibs/corefoundation.nix { inherit (darwin) objc4 ICU; };
+  
+  # As the name says, this is broken, but I don't want to lose it since it's a direction we want to go in
+  # libdispatch-broken = callPackage ../os-specific/darwin/swift-corelibs/libdispatch.nix { inherit (darwin) apple_sdk_sierra xnu; };
 
   darling = callPackage ../os-specific/darwin/darling/default.nix { };