hydra: 2020-02-06 -> 2020-03-{24,27}

Upgrades Hydra to the latest master/flake branch. To perform this
upgrade, it's needed to do a non-trivial db-migration which provides a
massive performance-improvement[1].

The basic ideas behind multi-step upgrades of services between NixOS versions
have been gathered already[2]. For further context it's recommended to
read this first.

Basically, the following steps are needed:

* Upgrade to a non-breaking version of Hydra with the db-changes
  (columns are still nullable here). If `system.stateVersion` is set to
  something older than 20.03, the package will be selected
  automatically, otherwise `pkgs.hydra-migration` needs to be used.

* Run `hydra-backfill-ids` on the server.

* Deploy either `pkgs.hydra-unstable` (for Hydra master) or
  `pkgs.hydra-flakes` (for flakes-support) to activate the optimization.

The steps are also documented in the release-notes and in the module
using `warnings`.

`pkgs.hydra` has been removed as latest Hydra doesn't compile with
`pkgs.nixStable` and to ensure a graceful migration using the newly
introduced packages.

To verify the approach, a simple vm-test has been added which verifies
the migration steps.

[1] https://github.com/NixOS/hydra/pull/711
[2] https://github.com/NixOS/nixpkgs/pull/82353#issuecomment-598269471

6 files changed, 271 insertions, 93 deletions

diff --git a/nixos/doc/manual/release-notes/rl-2003.xml b/nixos/doc/manual/release-notes/rl-2003.xml
index 7674b0a5c0de..76b12af362f8 100644
--- a/nixos/doc/manual/release-notes/rl-2003.xml
+++ b/nixos/doc/manual/release-notes/rl-2003.xml
@@ -697,6 +697,66 @@ auth required pam_succeed_if.so uid >= 1000 quiet
      </para>
     </warning>
    </listitem>
+   <listitem>
+    <para>
+     <package>Hydra</package> has gained a massive performance improvement due to
+     <link xlink:href="https://github.com/NixOS/hydra/pull/710">some database schema
+     changes</link> by adding several IDs and better indexing. However, it's necessary
+     to upgrade Hydra in multiple steps:
+     <itemizedlist>
+      <listitem>
+       <para>
+        At first, an older version of Hydra needs to be deployed which adds those
+        (nullable) columns. When having set <link linkend="opt-system.stateVersion">stateVersion
+        </link> to a value older than <literal>20.03</literal>, this package will be selected
+        by default from the module when upgrading. Otherwise, the package can be deployed using
+        the following config:
+<programlisting>{ pkgs, ... }: {
+  <link linkend="opt-services.hydra.package">services.hydra.package</link> = pkgs.hydra-migration;
+}</programlisting>
+       </para>
+      </listitem>
+      <listitem>
+       <para>
+        Automatically fill the newly added ID columns on the server by running the following
+        command:
+<screen>
+<prompt>$ </prompt>hydra-backfill-ids
+</screen>
+        <warning>
+         <para>Please note that this process can take a while depending on your database-size!</para>
+        </warning>
+       </para>
+      </listitem>
+      <listitem>
+       <para>
+        Deploy a newer version of Hydra to activate the DB optimizations. You can choose from
+        either <package>hydra-unstable</package> (latest <literal>master</literal> compiled
+        against <package>nixUnstable</package>) and <package>hydra-flakes</package> (latest
+        version with flake-support).
+        <warning>
+         <para>
+          If your <link linkend="opt-system.stateVersion">stateVersion</link> is set to
+          <literal>20.03</literal> or greater, <package>hydra-unstable</package> will be used
+          automatically! This will break your setup if you didn't run the migration.
+         </para>
+        </warning>
+        Please note that Hydra is currently not available with <package>nixStable</package>
+        as this doesn't compile anymore.
+       </para>
+      </listitem>
+     </itemizedlist>
+     <warning>
+      <para>
+       <package>pkgs.hydra</package> has been removed to ensure a graceful database-migration
+       using the dedicated package-attributes. If you still have <package>pkgs.hydra</package>
+       defined in e.g. an overlay, an assertion error will be thrown. To circumvent this,
+       you need to set <xref linkend="opt-services.hydra.package" /> to <package>pkgs.hydra</package>
+       explicitly and make sure you know what you're doing!
+      </para>
+     </warning>
+    </para>
+   </listitem>
   </itemizedlist>
  </section>
 
diff --git a/nixos/modules/services/continuous-integration/hydra/default.nix b/nixos/modules/services/continuous-integration/hydra/default.nix
index 8b56207590a1..0c335f14f785 100644
--- a/nixos/modules/services/continuous-integration/hydra/default.nix
+++ b/nixos/modules/services/continuous-integration/hydra/default.nix
@@ -37,6 +37,8 @@ let
 
   haveLocalDB = cfg.dbi == localDB;
 
+  inherit (config.system) stateVersion;
+
 in
 
 {
@@ -63,8 +65,7 @@ in
       };
 
       package = mkOption {
-        type = types.path;
-        default = pkgs.hydra;
+        type = types.package;
         defaultText = "pkgs.hydra";
         description = "The Hydra package.";
       };
@@ -194,6 +195,34 @@ in
 
   config = mkIf cfg.enable {
 
+    warnings = optional (cfg.package.migration or false) ''
+      You're currently deploying an older version of Hydra which is needed to
+      make some required database changes[1]. As soon as this is done, it's recommended
+      to run `hydra-backfill-ids` and set `services.hydra.package` to either `pkgs.hydra-unstable`
+      or `pkgs.hydra-flakes` after that.
+
+      [1] https://github.com/NixOS/hydra/pull/711
+    '';
+
+    services.hydra.package = with pkgs;
+      mkDefault (
+        if pkgs ? hydra
+          then throw ''
+            The Hydra package doesn't exist anymore in `nixpkgs`! It probably exists
+            due to an overlay. To upgrade Hydra, you need to take two steps as some
+            bigger changes in the database schema were implemented recently[1]. You first
+            need to deploy `pkgs.hydra-migration`, run `hydra-backfill-ids` on the server
+            and then deploy either `pkgs.hydra-unstable` or `pkgs.hydra-flakes`.
+
+            If you want to use `pkgs.hydra` from your overlay, please set `services.hydra.package`
+            explicitly to `pkgs.hydra` and make sure you know what you're doing.
+
+            [1] https://github.com/NixOS/hydra/pull/711
+          ''
+        else if versionOlder stateVersion "20.03" then hydra-migration
+        else hydra-unstable
+      );
+
     users.groups.hydra = {
       gid = config.ids.gids.hydra;
     };
diff --git a/nixos/tests/all-tests.nix b/nixos/tests/all-tests.nix
index 3501c551625d..265c93298f3c 100644
--- a/nixos/tests/all-tests.nix
+++ b/nixos/tests/all-tests.nix
@@ -127,6 +127,7 @@ in
   home-assistant = handleTest ./home-assistant.nix {};
   hound = handleTest ./hound.nix {};
   hydra = handleTest ./hydra {};
+  hydra-db-migration = handleTest ./hydra/db-migration.nix {};
   i3wm = handleTest ./i3wm.nix {};
   icingaweb2 = handleTest ./icingaweb2.nix {};
   iftop = handleTest ./iftop.nix {};
diff --git a/nixos/tests/hydra/common.nix b/nixos/tests/hydra/common.nix
new file mode 100644
index 000000000000..f612717dc968
--- /dev/null
+++ b/nixos/tests/hydra/common.nix
@@ -0,0 +1,47 @@
+{ system, ... }:
+{
+  baseConfig = { pkgs, ... }: let
+    trivialJob = pkgs.writeTextDir "trivial.nix" ''
+     { trivial = builtins.derivation {
+         name = "trivial";
+         system = "${system}";
+         builder = "/bin/sh";
+         allowSubstitutes = false;
+         preferLocalBuild = true;
+         args = ["-c" "echo success > $out; exit 0"];
+       };
+     }
+    '';
+
+    createTrivialProject = pkgs.stdenv.mkDerivation {
+      name = "create-trivial-project";
+      dontUnpack = true;
+      buildInputs = [ pkgs.makeWrapper ];
+      installPhase = "install -m755 -D ${./create-trivial-project.sh} $out/bin/create-trivial-project.sh";
+      postFixup = ''
+        wrapProgram "$out/bin/create-trivial-project.sh" --prefix PATH ":" ${pkgs.stdenv.lib.makeBinPath [ pkgs.curl ]} --set EXPR_PATH ${trivialJob}
+      '';
+    };
+  in {
+    virtualisation.memorySize = 2048;
+    time.timeZone = "UTC";
+    environment.systemPackages = [ createTrivialProject pkgs.jq ];
+    services.hydra = {
+      enable = true;
+      # Hydra needs those settings to start up, so we add something not harmfull.
+      hydraURL = "example.com";
+      notificationSender = "example@example.com";
+      extraConfig = ''
+        email_notification = 1
+      '';
+    };
+    services.postfix.enable = true;
+    nix = {
+      buildMachines = [{
+        hostName = "localhost";
+        systems = [ system ];
+      }];
+      binaryCaches = [];
+    };
+  };
+}
diff --git a/nixos/tests/hydra/db-migration.nix b/nixos/tests/hydra/db-migration.nix
new file mode 100644
index 000000000000..aa1c81c9e775
--- /dev/null
+++ b/nixos/tests/hydra/db-migration.nix
@@ -0,0 +1,86 @@
+{ system ? builtins.currentSystem, ... }:
+
+let inherit (import ./common.nix { inherit system; }) baseConfig; in
+
+{ mig = import ../make-test-python.nix ({ pkgs, lib, ... }: {
+    name = "hydra-db-migration";
+    meta = with pkgs.stdenv.lib.maintainers; {
+      maintainers = [ ma27 ];
+    };
+
+    nodes = {
+      original = { pkgs, lib, ... }: {
+        imports = [ baseConfig ];
+
+        # An older version of Hydra before the db change
+        # for testing purposes.
+        services.hydra.package = pkgs.hydra-migration.overrideAttrs (old: {
+          inherit (old) pname;
+          version = "2020-02-06";
+          src = pkgs.fetchFromGitHub {
+            owner = "NixOS";
+            repo = "hydra";
+            rev = "2b4f14963b16b21ebfcd6b6bfa7832842e9b2afc";
+            sha256 = "16q0cffcsfx5pqd91n9k19850c1nbh4vvbd9h8yi64ihn7v8bick";
+          };
+        });
+      };
+
+      migration_phase1 = { pkgs, lib, ... }: {
+        imports = [ baseConfig ];
+        services.hydra.package = pkgs.hydra-migration;
+      };
+
+      finished = { pkgs, lib, ... }: {
+        imports = [ baseConfig ];
+        services.hydra.package = pkgs.hydra-unstable;
+      };
+    };
+
+    testScript = { nodes, ... }: let
+      next = nodes.migration_phase1.config.system.build.toplevel;
+      finished = nodes.finished.config.system.build.toplevel;
+    in ''
+      original.start()
+      original.wait_for_unit("multi-user.target")
+      original.wait_for_unit("postgresql.service")
+      original.wait_for_unit("hydra-init.service")
+      original.require_unit_state("hydra-queue-runner.service")
+      original.require_unit_state("hydra-evaluator.service")
+      original.require_unit_state("hydra-notify.service")
+      original.succeed("hydra-create-user admin --role admin --password admin")
+      original.wait_for_open_port(3000)
+      original.succeed("create-trivial-project.sh")
+      original.wait_until_succeeds(
+          'curl -L -s http://localhost:3000/build/1 -H "Accept: application/json" |  jq .buildstatus | xargs test 0 -eq'
+      )
+
+      out = original.succeed("su -l postgres -c 'psql -d hydra <<< \"\\d+ jobs\" -A'")
+      assert "jobset_id" not in out
+
+      original.succeed(
+          "${next}/bin/switch-to-configuration test >&2"
+      )
+      original.wait_for_unit("hydra-init.service")
+
+      out = original.succeed("su -l postgres -c 'psql -d hydra <<< \"\\d+ jobs\" -A'")
+      assert "jobset_id|integer|||" in out
+
+      original.succeed("hydra-backfill-ids")
+
+      original.succeed(
+          "${finished}/bin/switch-to-configuration test >&2"
+      )
+      original.wait_for_unit("hydra-init.service")
+
+      out = original.succeed("su -l postgres -c 'psql -d hydra <<< \"\\d+ jobs\" -A'")
+      assert "jobset_id|integer||not null|" in out
+
+      original.wait_until_succeeds(
+          'curl -L -s http://localhost:3000/build/1 -H "Accept: application/json" |  jq .buildstatus | xargs test 0 -eq'
+      )
+
+      original.shutdown()
+    '';
+  });
+}
diff --git a/nixos/tests/hydra/default.nix b/nixos/tests/hydra/default.nix
index 1c0ed3369b1c..5d94eb91bf5b 100644
--- a/nixos/tests/hydra/default.nix
+++ b/nixos/tests/hydra/default.nix
@@ -3,102 +3,57 @@
 , pkgs ? import ../../.. { inherit system config; }
 }:
 
-let
-
-  trivialJob = pkgs.writeTextDir "trivial.nix" ''
-   { trivial = builtins.derivation {
-       name = "trivial";
-       system = "${system}";
-       builder = "/bin/sh";
-       allowSubstitutes = false;
-       preferLocalBuild = true;
-       args = ["-c" "echo success > $out; exit 0"];
-     };
-   }
-  '';
+with import ../../lib/testing-python.nix { inherit system pkgs; };
+with pkgs.lib;
 
-  createTrivialProject = pkgs.stdenv.mkDerivation {
-    name = "create-trivial-project";
-    dontUnpack = true;
-    buildInputs = [ pkgs.makeWrapper ];
-    installPhase = "install -m755 -D ${./create-trivial-project.sh} $out/bin/create-trivial-project.sh";
-    postFixup = ''
-      wrapProgram "$out/bin/create-trivial-project.sh" --prefix PATH ":" ${pkgs.stdenv.lib.makeBinPath [ pkgs.curl ]} --set EXPR_PATH ${trivialJob}
-    '';
-  };
+let
 
-  callTest = f: f { inherit system pkgs; };
+  inherit (import ./common.nix { inherit system; }) baseConfig;
 
   hydraPkgs = {
-    inherit (pkgs) nixStable nixUnstable nixFlakes;
+    inherit (pkgs) hydra-migration hydra-unstable hydra-flakes;
   };
 
-  tests = pkgs.lib.flip pkgs.lib.mapAttrs hydraPkgs (name: nix:
-    callTest (import ../make-test-python.nix ({ pkgs, lib, ... }:
-      {
-        name = "hydra-with-${name}";
-        meta = with pkgs.stdenv.lib.maintainers; {
-          maintainers = [ pstn lewo ma27 ];
-        };
-
-        machine = { pkgs, ... }:
-          {
-            virtualisation.memorySize = 1024;
-            time.timeZone = "UTC";
-
-            environment.systemPackages = [ createTrivialProject pkgs.jq ];
-            services.hydra = {
-              enable = true;
-
-              #Hydra needs those settings to start up, so we add something not harmfull.
-              hydraURL = "example.com";
-              notificationSender = "example@example.com";
-
-              package = pkgs.hydra.override { inherit nix; };
-
-              extraConfig = ''
-                email_notification = 1
-              '';
-            };
-            services.postfix.enable = true;
-            nix = {
-              buildMachines = [{
-                hostName = "localhost";
-                systems = [ system ];
-              }];
-
-              binaryCaches = [];
-            };
-          };
-
-        testScript = ''
-          # let the system boot up
-          machine.wait_for_unit("multi-user.target")
-          # test whether the database is running
-          machine.wait_for_unit("postgresql.service")
-          # test whether the actual hydra daemons are running
-          machine.wait_for_unit("hydra-init.service")
-          machine.require_unit_state("hydra-queue-runner.service")
-          machine.require_unit_state("hydra-evaluator.service")
-          machine.require_unit_state("hydra-notify.service")
-
-          machine.succeed("hydra-create-user admin --role admin --password admin")
-
-          # create a project with a trivial job
-          machine.wait_for_open_port(3000)
-
-          # make sure the build as been successfully built
-          machine.succeed("create-trivial-project.sh")
-
-          machine.wait_until_succeeds(
-              'curl -L -s http://localhost:3000/build/1 -H "Accept: application/json" |  jq .buildstatus | xargs test 0 -eq'
-          )
-
-          machine.wait_until_succeeds(
-              'journalctl -eu hydra-notify.service -o cat | grep -q "sending mail notification to hydra@localhost"'
-          )
-        '';
-      })));
+  makeHydraTest = with pkgs.lib; name: package: makeTest {
+    name = "hydra-${name}";
+    meta = with pkgs.stdenv.lib.maintainers; {
+      maintainers = [ pstn lewo ma27 ];
+    };
+
+    machine = { pkgs, lib, ... }: {
+      imports = [ baseConfig ];
+      services.hydra = { inherit package; };
+    };
+
+    testScript = ''
+      # let the system boot up
+      machine.wait_for_unit("multi-user.target")
+      # test whether the database is running
+      machine.wait_for_unit("postgresql.service")
+      # test whether the actual hydra daemons are running
+      machine.wait_for_unit("hydra-init.service")
+      machine.require_unit_state("hydra-queue-runner.service")
+      machine.require_unit_state("hydra-evaluator.service")
+      machine.require_unit_state("hydra-notify.service")
+
+      machine.succeed("hydra-create-user admin --role admin --password admin")
+
+      # create a project with a trivial job
+      machine.wait_for_open_port(3000)
+
+      # make sure the build as been successfully built
+      machine.succeed("create-trivial-project.sh")
+
+      machine.wait_until_succeeds(
+          'curl -L -s http://localhost:3000/build/1 -H "Accept: application/json" |  jq .buildstatus | xargs test 0 -eq'
+      )
+
+      machine.wait_until_succeeds(
+          'journalctl -eu hydra-notify.service -o cat | grep -q "sending mail notification to hydra@localhost"'
+      )
+    '';
+  };
 
 in
-  tests
+
+mapAttrs makeHydraTest hydraPkgs