Merge pull request #23026 from copumpkin/nixos-install-wip

Refactor nixos-install to separate out filesystem build logic

4 files changed, 161 insertions, 128 deletions

diff --git a/nixos/modules/installer/tools/nixos-install.sh b/nixos/modules/installer/tools/nixos-install.sh
index 57bc249360e7..e2ae2ee9fdf8 100644
--- a/nixos/modules/installer/tools/nixos-install.sh
+++ b/nixos/modules/installer/tools/nixos-install.sh
@@ -87,38 +87,6 @@ if ! test -e "$mountPoint"; then
     exit 1
 fi
 
-
-# Mount some stuff in the target root directory.
-mkdir -m 0755 -p $mountPoint/dev $mountPoint/proc $mountPoint/sys $mountPoint/etc $mountPoint/run $mountPoint/home
-mkdir -m 01777 -p $mountPoint/tmp
-mkdir -m 0755 -p $mountPoint/tmp/root
-mkdir -m 0755 -p $mountPoint/var
-mkdir -m 0700 -p $mountPoint/root
-mount --rbind /dev $mountPoint/dev
-mount --rbind /proc $mountPoint/proc
-mount --rbind /sys $mountPoint/sys
-mount --rbind / $mountPoint/tmp/root
-mount -t tmpfs -o "mode=0755" none $mountPoint/run
-rm -rf $mountPoint/var/run
-ln -s /run $mountPoint/var/run
-for f in /etc/resolv.conf /etc/hosts; do rm -f $mountPoint/$f; [ -f "$f" ] && cp -Lf $f $mountPoint/etc/; done
-for f in /etc/passwd /etc/group;      do touch $mountPoint/$f; [ -f "$f" ] && mount --rbind -o ro $f $mountPoint/$f; done
-
-cp -Lf "@cacert@" "$mountPoint/tmp/ca-cert.crt"
-export SSL_CERT_FILE=/tmp/ca-cert.crt
-# For Nix 1.7
-export CURL_CA_BUNDLE=/tmp/ca-cert.crt
-
-if [ -n "$runChroot" ]; then
-    if ! [ -L $mountPoint/nix/var/nix/profiles/system ]; then
-        echo "$0: installation not finished; cannot chroot into installation directory"
-        exit 1
-    fi
-    ln -s /nix/var/nix/profiles/system $mountPoint/run/current-system
-    exec chroot $mountPoint "${chrootCommand[@]}"
-fi
-
-
 # Get the path of the NixOS configuration file.
 if test -z "$NIXOS_CONFIG"; then
     NIXOS_CONFIG=/etc/nixos/configuration.nix
@@ -130,121 +98,60 @@ if [ ! -e "$mountPoint/$NIXOS_CONFIG" ] && [ -z "$closure" ]; then
 fi
 
 
-# Create the necessary Nix directories on the target device, if they
-# don't already exist.
-mkdir -m 0755 -p \
-    $mountPoint/nix/var/nix/gcroots \
-    $mountPoint/nix/var/nix/temproots \
-    $mountPoint/nix/var/nix/userpool \
-    $mountPoint/nix/var/nix/profiles \
-    $mountPoint/nix/var/nix/db \
-    $mountPoint/nix/var/log/nix/drvs
-
-mkdir -m 1775 -p $mountPoint/nix/store
-chown @root_uid@:@nixbld_gid@ $mountPoint/nix/store
-
-
-# There is no daemon in the chroot.
-unset NIX_REMOTE
-
-
-# We don't have locale-archive in the chroot, so clear $LANG.
-export LANG=
-export LC_ALL=
-export LC_TIME=
-
-
 # Builds will use users that are members of this group
 extraBuildFlags+=(--option "build-users-group" "$buildUsersGroup")
 
-
 # Inherit binary caches from the host
+# TODO: will this still work with Nix 1.12 now that it has no perl? Probably not... 
 binary_caches="$(@perl@/bin/perl -I @nix@/lib/perl5/site_perl/*/* -e 'use Nix::Config; Nix::Config::readConfig; print $Nix::Config::config{"binary-caches"};')"
 extraBuildFlags+=(--option "binary-caches" "$binary_caches")
 
+nixpkgs="$(readlink -f "$(nix-instantiate --find-file nixpkgs)")"
+export NIX_PATH="nixpkgs=$nixpkgs:nixos-config=$mountPoint/$NIXOS_CONFIG"
+unset NIXOS_CONFIG
 
-# Copy Nix to the Nix store on the target device, unless it's already there.
-if ! NIX_DB_DIR=$mountPoint/nix/var/nix/db nix-store --check-validity @nix@ 2> /dev/null; then
-    echo "copying Nix to $mountPoint...."
-    for i in $(@perl@/bin/perl @pathsFromGraph@ @nixClosure@); do
-        echo "  $i"
-        chattr -R -i $mountPoint/$i 2> /dev/null || true # clear immutable bit
-        @rsync@/bin/rsync -a $i $mountPoint/nix/store/
-    done
-
-    # Register the paths in the Nix closure as valid.  This is necessary
-    # to prevent them from being deleted the first time we install
-    # something.  (I.e., Nix will see that, e.g., the glibc path is not
-    # valid, delete it to get it out of the way, but as a result nothing
-    # will work anymore.)
-    chroot $mountPoint @nix@/bin/nix-store --register-validity < @nixClosure@
-fi
-
-
-# Create the required /bin/sh symlink; otherwise lots of things
-# (notably the system() function) won't work.
-mkdir -m 0755 -p $mountPoint/bin
-# !!! assuming that @shell@ is in the closure
-ln -sf @shell@ $mountPoint/bin/sh
+# TODO: do I need to set NIX_SUBSTITUTERS here or is the --option binary-caches above enough?
 
 
-# Build hooks likely won't function correctly in the minimal chroot; just disable them.
-unset NIX_BUILD_HOOK
+# A place to drop temporary closures
+trap "rm -rf $tmpdir" EXIT
+tmpdir="$(mktemp -d)"
 
-# Make the build below copy paths from the CD if possible.  Note that
-# /tmp/root in the chroot is the root of the CD.
-export NIX_OTHER_STORES=/tmp/root/nix:$NIX_OTHER_STORES
-
-p=@nix@/libexec/nix/substituters
-export NIX_SUBSTITUTERS=$p/copy-from-other-stores.pl:$p/download-from-binary-cache.pl
+# Build a closure (on the host; we then copy it into the guest)
+function closure() {
+    nix-build "${extraBuildFlags[@]}" --no-out-link -E "with import <nixpkgs> {}; runCommand \"closure\" { exportReferencesGraph = [ \"x\" (buildEnv { name = \"env\"; paths = [ ($1) stdenv ]; }) ]; } \"cp x \$out\""
+}
 
+system_closure="$tmpdir/system.closure"
 
 if [ -z "$closure" ]; then
-    # Get the absolute path to the NixOS/Nixpkgs sources.
-    nixpkgs="$(readlink -f $(nix-instantiate --find-file nixpkgs))"
-
-    nixEnvAction="-f <nixpkgs/nixos> --set -A system"
+    expr="(import <nixpkgs/nixos> {}).system"
+    system_root="$(nix-build -E "$expr")"
+    system_closure="$(closure "$expr")"
 else
-    nixpkgs=""
-    nixEnvAction="--set $closure"
+    system_root=$closure
+    # Create a temporary file ending in .closure (so nixos-prepare-root knows to --import it) to transport the store closure
+    # to the filesytem we're preparing. Also delete it on exit!
+    nix-store --export $(nix-store -qR $closure) > $system_closure
 fi
 
-# Build the specified Nix expression in the target store and install
-# it into the system configuration profile.
-echo "building the system configuration..."
-NIX_PATH="nixpkgs=/tmp/root/$nixpkgs:nixos-config=$NIXOS_CONFIG" NIXOS_CONFIG= \
-    chroot $mountPoint @nix@/bin/nix-env \
-    "${extraBuildFlags[@]}" -p /nix/var/nix/profiles/system $nixEnvAction
+channel_root="$(nix-env -p /nix/var/nix/profiles/per-user/root/channels -q nixos --no-name --out-path 2>/dev/null || echo -n "")"
+channel_closure="$tmpdir/channel.closure"
+nix-store --export $channel_root > $channel_closure
 
+# Populate the target root directory with the basics
+@prepare_root@/bin/nixos-prepare-root $mountPoint $channel_root $system_root @nixClosure@ $system_closure $channel_closure
 
-# Copy the NixOS/Nixpkgs sources to the target as the initial contents
-# of the NixOS channel.
-mkdir -m 0755 -p $mountPoint/nix/var/nix/profiles
-mkdir -m 1777 -p $mountPoint/nix/var/nix/profiles/per-user
-mkdir -m 0755 -p $mountPoint/nix/var/nix/profiles/per-user/root
-srcs=$(nix-env "${extraBuildFlags[@]}" -p /nix/var/nix/profiles/per-user/root/channels -q nixos --no-name --out-path 2>/dev/null || echo -n "")
-if [ -z "$noChannelCopy" ] && [ -n "$srcs" ]; then
-    echo "copying NixOS/Nixpkgs sources..."
-    chroot $mountPoint @nix@/bin/nix-env \
-        "${extraBuildFlags[@]}" -p /nix/var/nix/profiles/per-user/root/channels -i "$srcs" --quiet
-fi
-mkdir -m 0700 -p $mountPoint/root/.nix-defexpr
-ln -sfn /nix/var/nix/profiles/per-user/root/channels $mountPoint/root/.nix-defexpr/channels
-
-
-# Get rid of the /etc bind mounts.
-for f in /etc/passwd /etc/group; do [ -f "$f" ] && umount $mountPoint/$f; done
+# nixos-prepare-root doesn't currently do anything with file ownership, so we set it up here instead
+chown @root_uid@:@nixbld_gid@ $mountPoint/nix/store
 
+mount --rbind /dev $mountPoint/dev
+mount --rbind /proc $mountPoint/proc
+mount --rbind /sys $mountPoint/sys
 
 # Grub needs an mtab.
 ln -sfn /proc/mounts $mountPoint/etc/mtab
 
-
-# Mark the target as a NixOS installation, otherwise
-# switch-to-configuration will chicken out.
-touch $mountPoint/etc/NIXOS
-
-
 # Switch to the new system configuration.  This will install Grub with
 # a menu default pointing at the kernel/initrd/etc of the new
 # configuration.
diff --git a/nixos/modules/installer/tools/nixos-prepare-root.sh b/nixos/modules/installer/tools/nixos-prepare-root.sh
new file mode 100644
index 000000000000..c374330f8464
--- /dev/null
+++ b/nixos/modules/installer/tools/nixos-prepare-root.sh
@@ -0,0 +1,105 @@
+#! @shell@
+
+# This script's goal is to perform all "static" setup of a filesystem structure from pre-built store paths. Everything
+# in here should run in a non-root context and inside a Nix builder. It's designed primarily to be called from image-
+# building scripts and from nixos-install, but because it makes very few assumptions about the context in which it runs,
+# it could be useful in other contexts as well.
+#
+# Current behavior:
+#  - set up basic filesystem structure
+#  - make Nix store etc.
+#  - copy Nix, system, channel, and misceallaneous closures to target Nix store
+#  - register validity of all paths in the target store
+#  - set up channel and system profiles
+
+# Ensure a consistent umask.
+umask 0022
+
+set -e
+
+mountPoint="$1"
+channel="$2"
+system="$3"
+shift 3
+closures="$@"
+
+PATH="@coreutils@/bin:@nix@/bin:@perl@/bin:@utillinux@/bin:@rsync@/bin"
+
+if ! test -e "$mountPoint"; then
+    echo "mount point $mountPoint doesn't exist"
+    exit 1
+fi
+
+# Create a few of the standard directories in the target root directory.
+mkdir -m 0755 -p $mountPoint/dev $mountPoint/proc $mountPoint/sys $mountPoint/etc $mountPoint/run $mountPoint/home
+mkdir -m 01777 -p $mountPoint/tmp
+mkdir -m 0755 -p $mountPoint/tmp/root
+mkdir -m 0755 -p $mountPoint/var
+mkdir -m 0700 -p $mountPoint/root
+
+ln -s /run $mountPoint/var/run
+
+# Create the necessary Nix directories on the target device
+mkdir -m 0755 -p \
+    $mountPoint/nix/var/nix/gcroots \
+    $mountPoint/nix/var/nix/temproots \
+    $mountPoint/nix/var/nix/userpool \
+    $mountPoint/nix/var/nix/profiles \
+    $mountPoint/nix/var/nix/db \
+    $mountPoint/nix/var/log/nix/drvs
+
+mkdir -m 1775 -p $mountPoint/nix/store
+
+# All Nix operations below should operate on our target store, not /nix/store.
+# N.B: this relies on Nix 1.12 or higher
+export NIX_REMOTE=local?root=$mountPoint
+
+# Copy our closures to the Nix store on the target mount point, unless they're already there.
+for i in $closures; do
+    # We support closures both in the format produced by `nix-store --export` and by `exportReferencesGraph`,
+    # mostly because there doesn't seem to be a single format that can be produced outside of a nix build and
+    # inside one. See https://github.com/NixOS/nix/issues/1242 for more discussion.
+    if [[ "$i" =~ \.closure$ ]]; then
+        echo "importing serialized closure $i to $mountPoint..."
+        nix-store --import < $i
+    else
+        # There has to be a better way to do this, right?
+        echo "copying closure $i to $mountPoint..."
+        for j in $(perl @pathsFromGraph@ $i); do
+            echo "  $j... "
+            rsync -a $j $mountPoint/nix/store/
+        done
+
+        nix-store --register-validity < $i
+    fi
+done
+
+# Create the required /bin/sh symlink; otherwise lots of things
+# (notably the system() function) won't work.
+if [ ! -x $mountPoint/@shell@ ]; then
+    echo "Error: @shell@ wasn't included in the closure" >&2
+    exit 1
+fi
+mkdir -m 0755 -p $mountPoint/bin
+ln -sf @shell@ $mountPoint/bin/sh
+
+echo "setting the system closure to '$system'..."
+nix-env "${extraBuildFlags[@]}" -p $mountPoint/nix/var/nix/profiles/system --set "$system"
+
+ln -sfn /nix/var/nix/profiles/system $mountPoint/run/current-system
+
+# Copy the NixOS/Nixpkgs sources to the target as the initial contents of the NixOS channel.
+mkdir -m 0755 -p $mountPoint/nix/var/nix/profiles
+mkdir -m 1777 -p $mountPoint/nix/var/nix/profiles/per-user
+mkdir -m 0755 -p $mountPoint/nix/var/nix/profiles/per-user/root
+
+if [ -z "$noChannelCopy" ] && [ -n "$channel" ]; then
+    echo "copying channel..."
+    nix-env --option build-use-substitutes false "${extraBuildFlags[@]}" -p $mountPoint/nix/var/nix/profiles/per-user/root/channels --set "$channel" --quiet
+fi
+mkdir -m 0700 -p $mountPoint/root/.nix-defexpr
+ln -sfn /nix/var/nix/profiles/per-user/root/channels $mountPoint/root/.nix-defexpr/channels
+
+# Mark the target as a NixOS installation, otherwise switch-to-configuration will chicken out.
+touch $mountPoint/etc/NIXOS
+
diff --git a/nixos/modules/installer/tools/tools.nix b/nixos/modules/installer/tools/tools.nix
index a35f6ad8ae54..a3bae78c0ffc 100644
--- a/nixos/modules/installer/tools/tools.nix
+++ b/nixos/modules/installer/tools/tools.nix
@@ -4,7 +4,6 @@
 { config, pkgs, modulesPath, ... }:
 
 let
-
   cfg = config.installer;
 
   makeProg = args: pkgs.substituteAll (args // {
@@ -17,6 +16,14 @@ let
     src = ./nixos-build-vms/nixos-build-vms.sh;
   };
 
+  nixos-prepare-root = makeProg {
+    name = "nixos-prepare-root";
+    src = ./nixos-prepare-root.sh;
+
+    nix = pkgs.nixUnstable;
+    inherit (pkgs) perl pathsFromGraph rsync utillinux coreutils;
+  };
+
   nixos-install = makeProg {
     name = "nixos-install";
     src = ./nixos-install.sh;
@@ -26,6 +33,7 @@ let
     cacert = "${pkgs.cacert}/etc/ssl/certs/ca-bundle.crt";
     root_uid = config.ids.uids.root;
     nixbld_gid = config.ids.gids.nixbld;
+    prepare_root = nixos-prepare-root;
 
     nixClosure = pkgs.runCommand "closure"
       { exportReferencesGraph = ["refs" config.nix.package.out]; }
@@ -69,6 +77,7 @@ in
 
     environment.systemPackages =
       [ nixos-build-vms
+        nixos-prepare-root
         nixos-install
         nixos-rebuild
         nixos-generate-config
@@ -77,7 +86,7 @@ in
       ];
 
     system.build = {
-      inherit nixos-install nixos-generate-config nixos-option nixos-rebuild;
+      inherit nixos-install nixos-prepare-root nixos-generate-config nixos-option nixos-rebuild;
     };
 
   };
diff --git a/nixos/tests/installer.nix b/nixos/tests/installer.nix
index 35dd00fe630f..3ab3c1bac48a 100644
--- a/nixos/tests/installer.nix
+++ b/nixos/tests/installer.nix
@@ -34,6 +34,12 @@ let
           boot.loader.systemd-boot.enable = true;
         ''}
 
+        users.extraUsers.alice = {
+          isNormalUser = true;
+          home = "/home/alice";
+          description = "Alice Foobar";
+        };
+
         hardware.enableAllFirmware = lib.mkForce false;
 
         ${replaceChars ["\n"] ["\n  "] extraConfig}
@@ -96,7 +102,7 @@ let
       $machine->shutdown;
 
       # Now see if we can boot the installation.
-      $machine = createMachine({ ${hdFlags} qemuFlags => "${qemuFlags}" });
+      $machine = createMachine({ ${hdFlags} qemuFlags => "${qemuFlags}", name => "boot-after-install" });
 
       # For example to enter LUKS passphrase.
       ${preBootCommands}
@@ -118,11 +124,17 @@ let
       $machine->waitForUnit("swap.target");
       $machine->succeed("cat /proc/swaps | grep -q /dev");
 
+      # Check that the store is in good shape
+      $machine->succeed("nix-store --verify --check-contents >&2");
+
       # Check whether the channel works.
       $machine->succeed("nix-env -iA nixos.procps >&2");
       $machine->succeed("type -tP ps | tee /dev/stderr") =~ /.nix-profile/
           or die "nix-env failed";
 
+      # Check that the daemon works, and that non-root users can run builds (this will build a new profile generation through the daemon)
+      $machine->succeed("su alice -l -c 'nix-env -iA nixos.procps' >&2");
+
       # We need to a writable nix-store on next boot.
       $machine->copyFileFromHost(
           "${ makeConfig { inherit bootLoader grubVersion grubDevice grubIdentifier extraConfig; forceGrubReinstallCount = 1; } }",
@@ -139,7 +151,7 @@ let
       $machine->shutdown;
 
       # Check whether a writable store build works
-      $machine = createMachine({ ${hdFlags} qemuFlags => "${qemuFlags}" });
+      $machine = createMachine({ ${hdFlags} qemuFlags => "${qemuFlags}", name => "rebuild-switch" });
       ${preBootCommands}
       $machine->waitForUnit("multi-user.target");
       $machine->copyFileFromHost(
@@ -150,7 +162,7 @@ let
 
       # And just to be sure, check that the machine still boots after
       # "nixos-rebuild switch".
-      $machine = createMachine({ ${hdFlags} qemuFlags => "${qemuFlags}" });
+      $machine = createMachine({ ${hdFlags} qemuFlags => "${qemuFlags}", "boot-after-rebuild-switch" });
       ${preBootCommands}
       $machine->waitForUnit("network.target");
       $machine->shutdown;