Merge remote-tracking branch 'origin/master' into staging-next

55 files changed, 1465 insertions, 556 deletions

diff --git a/nixos/doc/manual/default.nix b/nixos/doc/manual/default.nix
index d61bbaddf764..ecd62eb4e848 100644
--- a/nixos/doc/manual/default.nix
+++ b/nixos/doc/manual/default.nix
@@ -13,6 +13,8 @@
 with pkgs;
 
 let
+  inherit (lib) hasPrefix removePrefix;
+
   lib = pkgs.lib;
 
   docbook_xsl_ns = pkgs.docbook-xsl-ns.override {
@@ -36,6 +38,33 @@ let
     };
   };
 
+  nixos-lib = import ../../lib { };
+
+  testOptionsDoc = let
+      eval = nixos-lib.evalTest {
+        # Avoid evaluating a NixOS config prototype.
+        config.node.type = lib.types.deferredModule;
+        options._module.args = lib.mkOption { internal = true; };
+      };
+    in buildPackages.nixosOptionsDoc {
+      inherit (eval) options;
+      inherit (revision);
+      transformOptions = opt: opt // {
+        # Clean up declaration sites to not refer to the NixOS source tree.
+        declarations =
+          map
+            (decl:
+              if hasPrefix (toString ../../..) (toString decl)
+              then
+                let subpath = removePrefix "/" (removePrefix (toString ../../..) (toString decl));
+                in { url = "https://github.com/NixOS/nixpkgs/blob/master/${subpath}"; name = subpath; }
+              else decl)
+            opt.declarations;
+      };
+      documentType = "none";
+      variablelistId = "test-options-list";
+    };
+
   sources = lib.sourceFilesBySuffices ./. [".xml"];
 
   modulesDoc = builtins.toFile "modules.xml" ''
@@ -50,6 +79,7 @@ let
     mkdir $out
     ln -s ${modulesDoc} $out/modules.xml
     ln -s ${optionsDoc.optionsDocBook} $out/options-db.xml
+    ln -s ${testOptionsDoc.optionsDocBook} $out/test-options-db.xml
     printf "%s" "${version}" > $out/version
   '';
 
diff --git a/nixos/doc/manual/development/running-nixos-tests-interactively.section.md b/nixos/doc/manual/development/running-nixos-tests-interactively.section.md
index a1431859ff59..d9c316f4b139 100644
--- a/nixos/doc/manual/development/running-nixos-tests-interactively.section.md
+++ b/nixos/doc/manual/development/running-nixos-tests-interactively.section.md
@@ -24,6 +24,8 @@ back into the test driver command line upon its completion. This allows
 you to inspect the state of the VMs after the test (e.g. to debug the
 test script).
 
+## Reuse VM state {#sec-nixos-test-reuse-vm-state}
+
 You can re-use the VM states coming from a previous run by setting the
 `--keep-vm-state` flag.
 
@@ -33,3 +35,15 @@ $ ./result/bin/nixos-test-driver --keep-vm-state
 
 The machine state is stored in the `$TMPDIR/vm-state-machinename`
 directory.
+
+## Interactive-only test configuration {#sec-nixos-test-interactive-configuration}
+
+The `.driverInteractive` attribute combines the regular test configuration with
+definitions from the [`interactive` submodule](#opt-interactive). This gives you
+a more usable, graphical, but slightly different configuration.
+
+You can add your own interactive-only test configuration by adding extra
+configuration to the [`interactive` submodule](#opt-interactive).
+
+To interactively run only the regular configuration, build the `<test>.driver` attribute
+instead, and call it with the flag `result/bin/nixos-test-driver --interactive`.
diff --git a/nixos/doc/manual/development/running-nixos-tests.section.md b/nixos/doc/manual/development/running-nixos-tests.section.md
index 1bec023b613a..33076f5dc2a7 100644
--- a/nixos/doc/manual/development/running-nixos-tests.section.md
+++ b/nixos/doc/manual/development/running-nixos-tests.section.md
@@ -2,22 +2,11 @@
 
 You can run tests using `nix-build`. For example, to run the test
 [`login.nix`](https://github.com/NixOS/nixpkgs/blob/master/nixos/tests/login.nix),
-you just do:
+you do:
 
 ```ShellSession
-$ nix-build '<nixpkgs/nixos/tests/login.nix>'
-```
-
-or, if you don't want to rely on `NIX_PATH`:
-
-```ShellSession
-$ cd /my/nixpkgs/nixos/tests
-$ nix-build login.nix
-…
-running the VM test script
-machine: QEMU running (pid 8841)
-…
-6 out of 6 tests succeeded
+$ cd /my/git/clone/of/nixpkgs
+$ nix-build -A nixosTests.login
 ```
 
 After building/downloading all required dependencies, this will perform
diff --git a/nixos/doc/manual/development/writing-nixos-tests.section.md b/nixos/doc/manual/development/writing-nixos-tests.section.md
index 6934bb0face7..99704ec3c141 100644
--- a/nixos/doc/manual/development/writing-nixos-tests.section.md
+++ b/nixos/doc/manual/development/writing-nixos-tests.section.md
@@ -1,9 +1,9 @@
 # Writing Tests {#sec-writing-nixos-tests}
 
-A NixOS test is a Nix expression that has the following structure:
+A NixOS test is a module that has the following structure:
 
 ```nix
-import ./make-test-python.nix {
+{
 
   # One or more machines:
   nodes =
@@ -21,10 +21,13 @@ import ./make-test-python.nix {
 }
 ```
 
-The attribute `testScript` is a bit of Python code that executes the
+We refer to the whole test above as a test module, whereas the values
+in [`nodes.<name>`](#opt-nodes) are NixOS modules themselves.
+
+The option [`testScript`](#opt-testScript) is a piece of Python code that executes the
 test (described below). During the test, it will start one or more
 virtual machines, the configuration of which is described by
-the attribute `nodes`.
+the option [`nodes`](#opt-nodes).
 
 An example of a single-node test is
 [`login.nix`](https://github.com/NixOS/nixpkgs/blob/master/nixos/tests/login.nix).
@@ -34,7 +37,54 @@ when switching between consoles, and so on. An interesting multi-node test is
 [`nfs/simple.nix`](https://github.com/NixOS/nixpkgs/blob/master/nixos/tests/nfs/simple.nix).
 It uses two client nodes to test correct locking across server crashes.
 
-There are a few special NixOS configuration options for test VMs:
+## Calling a test {#sec-calling-nixos-tests}
+
+Tests are invoked differently depending on whether the test is part of NixOS or lives in a different project.
+
+### Testing within NixOS {#sec-call-nixos-test-in-nixos}
+
+Tests that are part of NixOS are added to [`nixos/tests/all-tests.nix`](https://github.com/NixOS/nixpkgs/blob/master/nixos/tests/all-tests.nix).
+
+```nix
+  hostname = runTest ./hostname.nix;
+```
+
+Overrides can be added by defining an anonymous module in `all-tests.nix`.
+
+```nix
+  hostname = runTest {
+    imports = [ ./hostname.nix ];
+    defaults.networking.firewall.enable = false;
+  };
+```
+
+You can run a test with attribute name `hostname` in `nixos/tests/all-tests.nix` by invoking:
+
+```shell
+cd /my/git/clone/of/nixpkgs
+nix-build -A nixosTests.hostname
+```
+
+### Testing outside the NixOS project {#sec-call-nixos-test-outside-nixos}
+
+Outside the `nixpkgs` repository, you can instantiate the test by first importing the NixOS library,
+
+```nix
+let nixos-lib = import (nixpkgs + "/nixos/lib") { };
+in
+
+nixos-lib.runTest {
+  imports = [ ./test.nix ];
+  hostPkgs = pkgs;  # the Nixpkgs package set used outside the VMs
+  defaults.services.foo.package = mypkg;
+}
+```
+
+`runTest` returns a derivation that runs the test.
+
+## Configuring the nodes {#sec-nixos-test-nodes}
+
+There are a few special NixOS options for test VMs:
 
 `virtualisation.memorySize`
 
@@ -121,7 +171,7 @@ The following methods are available on machine objects:
     least one will be returned.
 
     ::: {.note}
-    This requires passing `enableOCR` to the test attribute set.
+    This requires [`enableOCR`](#opt-enableOCR) to be set to `true`.
     :::
 
 `get_screen_text`
@@ -130,7 +180,7 @@ The following methods are available on machine objects:
     machine\'s screen using optical character recognition.
 
     ::: {.note}
-    This requires passing `enableOCR` to the test attribute set.
+    This requires [`enableOCR`](#opt-enableOCR) to be set to `true`.
     :::
 
 `send_monitor_command`
@@ -241,7 +291,7 @@ The following methods are available on machine objects:
     `get_screen_text` and `get_screen_text_variants`).
 
     ::: {.note}
-    This requires passing `enableOCR` to the test attribute set.
+    This requires [`enableOCR`](#opt-enableOCR) to be set to `true`.
     :::
 
 `wait_for_console_text`
@@ -304,7 +354,7 @@ For faster dev cycles it\'s also possible to disable the code-linters
 (this shouldn\'t be commited though):
 
 ```nix
-import ./make-test-python.nix {
+{
   skipLint = true;
   nodes.machine =
     { config, pkgs, ... }:
@@ -336,7 +386,7 @@ Similarly, the type checking of test scripts can be disabled in the following
 way:
 
 ```nix
-import ./make-test-python.nix {
+{
   skipTypeCheck = true;
   nodes.machine =
     { config, pkgs, ... }:
@@ -400,7 +450,6 @@ added using the parameter `extraPythonPackages`. For example, you could add
 `numpy` like this:
 
 ```nix
-import ./make-test-python.nix
 {
   extraPythonPackages = p: [ p.numpy ];
 
@@ -417,3 +466,11 @@ import ./make-test-python.nix
 ```
 
 In that case, `numpy` is chosen from the generic `python3Packages`.
+
+## Test Options Reference {#sec-test-options-reference}
+
+The following options can be used when writing tests.
+
+```{=docbook}
+<xi:include href="../../generated/test-options-db.xml" xpointer="test-options-list"/>
+```
diff --git a/nixos/doc/manual/from_md/development/running-nixos-tests-interactively.section.xml b/nixos/doc/manual/from_md/development/running-nixos-tests-interactively.section.xml
index 0e47350a0d24..35d9bbd1c1fe 100644
--- a/nixos/doc/manual/from_md/development/running-nixos-tests-interactively.section.xml
+++ b/nixos/doc/manual/from_md/development/running-nixos-tests-interactively.section.xml
@@ -25,15 +25,40 @@ $ ./result/bin/nixos-test-driver
     completion. This allows you to inspect the state of the VMs after
     the test (e.g. to debug the test script).
   </para>
-  <para>
-    You can re-use the VM states coming from a previous run by setting
-    the <literal>--keep-vm-state</literal> flag.
-  </para>
-  <programlisting>
+  <section xml:id="sec-nixos-test-reuse-vm-state">
+    <title>Reuse VM state</title>
+    <para>
+      You can re-use the VM states coming from a previous run by setting
+      the <literal>--keep-vm-state</literal> flag.
+    </para>
+    <programlisting>
 $ ./result/bin/nixos-test-driver --keep-vm-state
 </programlisting>
-  <para>
-    The machine state is stored in the
-    <literal>$TMPDIR/vm-state-machinename</literal> directory.
-  </para>
+    <para>
+      The machine state is stored in the
+      <literal>$TMPDIR/vm-state-machinename</literal> directory.
+    </para>
+  </section>
+  <section xml:id="sec-nixos-test-interactive-configuration">
+    <title>Interactive-only test configuration</title>
+    <para>
+      The <literal>.driverInteractive</literal> attribute combines the
+      regular test configuration with definitions from the
+      <link linkend="opt-interactive"><literal>interactive</literal>
+      submodule</link>. This gives you a more usable, graphical, but
+      slightly different configuration.
+    </para>
+    <para>
+      You can add your own interactive-only test configuration by adding
+      extra configuration to the
+      <link linkend="opt-interactive"><literal>interactive</literal>
+      submodule</link>.
+    </para>
+    <para>
+      To interactively run only the regular configuration, build the
+      <literal>&lt;test&gt;.driver</literal> attribute instead, and call
+      it with the flag
+      <literal>result/bin/nixos-test-driver --interactive</literal>.
+    </para>
+  </section>
 </section>
diff --git a/nixos/doc/manual/from_md/development/running-nixos-tests.section.xml b/nixos/doc/manual/from_md/development/running-nixos-tests.section.xml
index da2e5076c956..23abb546899f 100644
--- a/nixos/doc/manual/from_md/development/running-nixos-tests.section.xml
+++ b/nixos/doc/manual/from_md/development/running-nixos-tests.section.xml
@@ -4,22 +4,11 @@
     You can run tests using <literal>nix-build</literal>. For example,
     to run the test
     <link xlink:href="https://github.com/NixOS/nixpkgs/blob/master/nixos/tests/login.nix"><literal>login.nix</literal></link>,
-    you just do:
+    you do:
   </para>
   <programlisting>
-$ nix-build '&lt;nixpkgs/nixos/tests/login.nix&gt;'
-</programlisting>
-  <para>
-    or, if you don’t want to rely on <literal>NIX_PATH</literal>:
-  </para>
-  <programlisting>
-$ cd /my/nixpkgs/nixos/tests
-$ nix-build login.nix
-…
-running the VM test script
-machine: QEMU running (pid 8841)
-…
-6 out of 6 tests succeeded
+$ cd /my/git/clone/of/nixpkgs
+$ nix-build -A nixosTests.login
 </programlisting>
   <para>
     After building/downloading all required dependencies, this will
diff --git a/nixos/doc/manual/from_md/development/writing-nixos-tests.section.xml b/nixos/doc/manual/from_md/development/writing-nixos-tests.section.xml
index d6f4f61c0645..32f5fdb77f50 100644
--- a/nixos/doc/manual/from_md/development/writing-nixos-tests.section.xml
+++ b/nixos/doc/manual/from_md/development/writing-nixos-tests.section.xml
@@ -1,10 +1,10 @@
-<section xmlns="http://docbook.org/ns/docbook" xmlns:xlink="http://www.w3.org/1999/xlink" xml:id="sec-writing-nixos-tests">
+<section xmlns="http://docbook.org/ns/docbook"  xmlns:xlink="http://www.w3.org/1999/xlink" xmlns:xi="http://www.w3.org/2001/XInclude" xml:id="sec-writing-nixos-tests">
   <title>Writing Tests</title>
   <para>
-    A NixOS test is a Nix expression that has the following structure:
+    A NixOS test is a module that has the following structure:
   </para>
   <programlisting language="bash">
-import ./make-test-python.nix {
+{
 
   # One or more machines:
   nodes =
@@ -22,10 +22,18 @@ import ./make-test-python.nix {
 }
 </programlisting>
   <para>
-    The attribute <literal>testScript</literal> is a bit of Python code
-    that executes the test (described below). During the test, it will
-    start one or more virtual machines, the configuration of which is
-    described by the attribute <literal>nodes</literal>.
+    We refer to the whole test above as a test module, whereas the
+    values in
+    <link linkend="opt-nodes"><literal>nodes.&lt;name&gt;</literal></link>
+    are NixOS modules themselves.
+  </para>
+  <para>
+    The option
+    <link linkend="opt-testScript"><literal>testScript</literal></link>
+    is a piece of Python code that executes the test (described below).
+    During the test, it will start one or more virtual machines, the
+    configuration of which is described by the option
+    <link linkend="opt-nodes"><literal>nodes</literal></link>.
   </para>
   <para>
     An example of a single-node test is
@@ -38,78 +46,138 @@ import ./make-test-python.nix {
     It uses two client nodes to test correct locking across server
     crashes.
   </para>
-  <para>
-    There are a few special NixOS configuration options for test VMs:
-  </para>
-  <variablelist>
-    <varlistentry>
-      <term>
-        <literal>virtualisation.memorySize</literal>
-      </term>
-      <listitem>
-        <para>
-          The memory of the VM in megabytes.
-        </para>
-      </listitem>
-    </varlistentry>
-    <varlistentry>
-      <term>
-        <literal>virtualisation.vlans</literal>
-      </term>
-      <listitem>
-        <para>
-          The virtual networks to which the VM is connected. See
-          <link xlink:href="https://github.com/NixOS/nixpkgs/blob/master/nixos/tests/nat.nix"><literal>nat.nix</literal></link>
-          for an example.
-        </para>
-      </listitem>
-    </varlistentry>
-    <varlistentry>
-      <term>
-        <literal>virtualisation.writableStore</literal>
-      </term>
-      <listitem>
-        <para>
-          By default, the Nix store in the VM is not writable. If you
-          enable this option, a writable union file system is mounted on
-          top of the Nix store to make it appear writable. This is
-          necessary for tests that run Nix operations that modify the
-          store.
-        </para>
-      </listitem>
-    </varlistentry>
-  </variablelist>
-  <para>
-    For more options, see the module
-    <link xlink:href="https://github.com/NixOS/nixpkgs/blob/master/nixos/modules/virtualisation/qemu-vm.nix"><literal>qemu-vm.nix</literal></link>.
-  </para>
-  <para>
-    The test script is a sequence of Python statements that perform
-    various actions, such as starting VMs, executing commands in the
-    VMs, and so on. Each virtual machine is represented as an object
-    stored in the variable <literal>name</literal> if this is also the
-    identifier of the machine in the declarative config. If you
-    specified a node <literal>nodes.machine</literal>, the following
-    example starts the machine, waits until it has finished booting,
-    then executes a command and checks that the output is more-or-less
-    correct:
-  </para>
-  <programlisting language="python">
+  <section xml:id="sec-calling-nixos-tests">
+    <title>Calling a test</title>
+    <para>
+      Tests are invoked differently depending on whether the test is
+      part of NixOS or lives in a different project.
+    </para>
+    <section xml:id="sec-call-nixos-test-in-nixos">
+      <title>Testing within NixOS</title>
+      <para>
+        Tests that are part of NixOS are added to
+        <link xlink:href="https://github.com/NixOS/nixpkgs/blob/master/nixos/tests/all-tests.nix"><literal>nixos/tests/all-tests.nix</literal></link>.
+      </para>
+      <programlisting language="bash">
+  hostname = runTest ./hostname.nix;
+</programlisting>
+      <para>
+        Overrides can be added by defining an anonymous module in
+        <literal>all-tests.nix</literal>.
+      </para>
+      <programlisting language="bash">
+  hostname = runTest {
+    imports = [ ./hostname.nix ];
+    defaults.networking.firewall.enable = false;
+  };
+</programlisting>
+      <para>
+        You can run a test with attribute name
+        <literal>hostname</literal> in
+        <literal>nixos/tests/all-tests.nix</literal> by invoking:
+      </para>
+      <programlisting>
+cd /my/git/clone/of/nixpkgs
+nix-build -A nixosTests.hostname
+</programlisting>
+    </section>
+    <section xml:id="sec-call-nixos-test-outside-nixos">
+      <title>Testing outside the NixOS project</title>
+      <para>
+        Outside the <literal>nixpkgs</literal> repository, you can
+        instantiate the test by first importing the NixOS library,
+      </para>
+      <programlisting language="bash">
+let nixos-lib = import (nixpkgs + &quot;/nixos/lib&quot;) { };
+in
+
+nixos-lib.runTest {
+  imports = [ ./test.nix ];
+  hostPkgs = pkgs;  # the Nixpkgs package set used outside the VMs
+  defaults.services.foo.package = mypkg;
+}
+</programlisting>
+      <para>
+        <literal>runTest</literal> returns a derivation that runs the
+        test.
+      </para>
+    </section>
+  </section>
+  <section xml:id="sec-nixos-test-nodes">
+    <title>Configuring the nodes</title>
+    <para>
+      There are a few special NixOS options for test VMs:
+    </para>
+    <variablelist>
+      <varlistentry>
+        <term>
+          <literal>virtualisation.memory