hydra: 2020-02-06 -> 2020-03-{24,27}

Upgrades Hydra to the latest master/flake branch. To perform this
upgrade, it's needed to do a non-trivial db-migration which provides a
massive performance-improvement[1].

The basic ideas behind multi-step upgrades of services between NixOS versions
have been gathered already[2]. For further context it's recommended to
read this first.

Basically, the following steps are needed:

* Upgrade to a non-breaking version of Hydra with the db-changes
  (columns are still nullable here). If `system.stateVersion` is set to
  something older than 20.03, the package will be selected
  automatically, otherwise `pkgs.hydra-migration` needs to be used.

* Run `hydra-backfill-ids` on the server.

* Deploy either `pkgs.hydra-unstable` (for Hydra master) or
  `pkgs.hydra-flakes` (for flakes-support) to activate the optimization.

The steps are also documented in the release-notes and in the module
using `warnings`.

`pkgs.hydra` has been removed as latest Hydra doesn't compile with
`pkgs.nixStable` and to ensure a graceful migration using the newly
introduced packages.

To verify the approach, a simple vm-test has been added which verifies
the migration steps.

[1] https://github.com/NixOS/hydra/pull/711
[2] https://github.com/NixOS/nixpkgs/pull/82353#issuecomment-598269471

3 files changed, 179 insertions, 91 deletions

diff --git a/nixos/tests/hydra/common.nix b/nixos/tests/hydra/common.nix
new file mode 100644
index 000000000000..f612717dc968
--- /dev/null
+++ b/nixos/tests/hydra/common.nix
@@ -0,0 +1,47 @@
+{ system, ... }:
+{
+  baseConfig = { pkgs, ... }: let
+    trivialJob = pkgs.writeTextDir "trivial.nix" ''
+     { trivial = builtins.derivation {
+         name = "trivial";
+         system = "${system}";
+         builder = "/bin/sh";
+         allowSubstitutes = false;
+         preferLocalBuild = true;
+         args = ["-c" "echo success > $out; exit 0"];
+       };
+     }
+    '';
+
+    createTrivialProject = pkgs.stdenv.mkDerivation {
+      name = "create-trivial-project";
+      dontUnpack = true;
+      buildInputs = [ pkgs.makeWrapper ];
+      installPhase = "install -m755 -D ${./create-trivial-project.sh} $out/bin/create-trivial-project.sh";
+      postFixup = ''
+        wrapProgram "$out/bin/create-trivial-project.sh" --prefix PATH ":" ${pkgs.stdenv.lib.makeBinPath [ pkgs.curl ]} --set EXPR_PATH ${trivialJob}
+      '';
+    };
+  in {
+    virtualisation.memorySize = 2048;
+    time.timeZone = "UTC";
+    environment.systemPackages = [ createTrivialProject pkgs.jq ];
+    services.hydra = {
+      enable = true;
+      # Hydra needs those settings to start up, so we add something not harmfull.
+      hydraURL = "example.com";
+      notificationSender = "example@example.com";
+      extraConfig = ''
+        email_notification = 1
+      '';
+    };
+    services.postfix.enable = true;
+    nix = {
+      buildMachines = [{
+        hostName = "localhost";
+        systems = [ system ];
+      }];
+      binaryCaches = [];
+    };
+  };
+}
diff --git a/nixos/tests/hydra/db-migration.nix b/nixos/tests/hydra/db-migration.nix
new file mode 100644
index 000000000000..aa1c81c9e775
--- /dev/null
+++ b/nixos/tests/hydra/db-migration.nix
@@ -0,0 +1,86 @@
+{ system ? builtins.currentSystem, ... }:
+
+let inherit (import ./common.nix { inherit system; }) baseConfig; in
+
+{ mig = import ../make-test-python.nix ({ pkgs, lib, ... }: {
+    name = "hydra-db-migration";
+    meta = with pkgs.stdenv.lib.maintainers; {
+      maintainers = [ ma27 ];
+    };
+
+    nodes = {
+      original = { pkgs, lib, ... }: {
+        imports = [ baseConfig ];
+
+        # An older version of Hydra before the db change
+        # for testing purposes.
+        services.hydra.package = pkgs.hydra-migration.overrideAttrs (old: {
+          inherit (old) pname;
+          version = "2020-02-06";
+          src = pkgs.fetchFromGitHub {
+            owner = "NixOS";
+            repo = "hydra";
+            rev = "2b4f14963b16b21ebfcd6b6bfa7832842e9b2afc";
+            sha256 = "16q0cffcsfx5pqd91n9k19850c1nbh4vvbd9h8yi64ihn7v8bick";
+          };
+        });
+      };
+
+      migration_phase1 = { pkgs, lib, ... }: {
+        imports = [ baseConfig ];
+        services.hydra.package = pkgs.hydra-migration;
+      };
+
+      finished = { pkgs, lib, ... }: {
+        imports = [ baseConfig ];
+        services.hydra.package = pkgs.hydra-unstable;
+      };
+    };
+
+    testScript = { nodes, ... }: let
+      next = nodes.migration_phase1.config.system.build.toplevel;
+      finished = nodes.finished.config.system.build.toplevel;
+    in ''
+      original.start()
+      original.wait_for_unit("multi-user.target")
+      original.wait_for_unit("postgresql.service")
+      original.wait_for_unit("hydra-init.service")
+      original.require_unit_state("hydra-queue-runner.service")
+      original.require_unit_state("hydra-evaluator.service")
+      original.require_unit_state("hydra-notify.service")
+      original.succeed("hydra-create-user admin --role admin --password admin")
+      original.wait_for_open_port(3000)
+      original.succeed("create-trivial-project.sh")
+      original.wait_until_succeeds(
+          'curl -L -s http://localhost:3000/build/1 -H "Accept: application/json" |  jq .buildstatus | xargs test 0 -eq'
+      )
+
+      out = original.succeed("su -l postgres -c 'psql -d hydra <<< \"\\d+ jobs\" -A'")
+      assert "jobset_id" not in out
+
+      original.succeed(
+          "${next}/bin/switch-to-configuration test >&2"
+      )
+      original.wait_for_unit("hydra-init.service")
+
+      out = original.succeed("su -l postgres -c 'psql -d hydra <<< \"\\d+ jobs\" -A'")
+      assert "jobset_id|integer|||" in out
+
+      original.succeed("hydra-backfill-ids")
+
+      original.succeed(
+          "${finished}/bin/switch-to-configuration test >&2"
+      )
+      original.wait_for_unit("hydra-init.service")
+
+      out = original.succeed("su -l postgres -c 'psql -d hydra <<< \"\\d+ jobs\" -A'")
+      assert "jobset_id|integer||not null|" in out
+
+      original.wait_until_succeeds(
+          'curl -L -s http://localhost:3000/build/1 -H "Accept: application/json" |  jq .buildstatus | xargs test 0 -eq'
+      )
+
+      original.shutdown()
+    '';
+  });
+}
diff --git a/nixos/tests/hydra/default.nix b/nixos/tests/hydra/default.nix
index 1c0ed3369b1c..5d94eb91bf5b 100644
--- a/nixos/tests/hydra/default.nix
+++ b/nixos/tests/hydra/default.nix
@@ -3,102 +3,57 @@
 , pkgs ? import ../../.. { inherit system config; }
 }:
 
-let
-
-  trivialJob = pkgs.writeTextDir "trivial.nix" ''
-   { trivial = builtins.derivation {
-       name = "trivial";
-       system = "${system}";
-       builder = "/bin/sh";
-       allowSubstitutes = false;
-       preferLocalBuild = true;
-       args = ["-c" "echo success > $out; exit 0"];
-     };
-   }
-  '';
+with import ../../lib/testing-python.nix { inherit system pkgs; };
+with pkgs.lib;
 
-  createTrivialProject = pkgs.stdenv.mkDerivation {
-    name = "create-trivial-project";
-    dontUnpack = true;
-    buildInputs = [ pkgs.makeWrapper ];
-    installPhase = "install -m755 -D ${./create-trivial-project.sh} $out/bin/create-trivial-project.sh";
-    postFixup = ''
-      wrapProgram "$out/bin/create-trivial-project.sh" --prefix PATH ":" ${pkgs.stdenv.lib.makeBinPath [ pkgs.curl ]} --set EXPR_PATH ${trivialJob}
-    '';
-  };
+let
 
-  callTest = f: f { inherit system pkgs; };
+  inherit (import ./common.nix { inherit system; }) baseConfig;
 
   hydraPkgs = {
-    inherit (pkgs) nixStable nixUnstable nixFlakes;
+    inherit (pkgs) hydra-migration hydra-unstable hydra-flakes;
   };
 
-  tests = pkgs.lib.flip pkgs.lib.mapAttrs hydraPkgs (name: nix:
-    callTest (import ../make-test-python.nix ({ pkgs, lib, ... }:
-      {
-        name = "hydra-with-${name}";
-        meta = with pkgs.stdenv.lib.maintainers; {
-          maintainers = [ pstn lewo ma27 ];
-        };
-
-        machine = { pkgs, ... }:
-          {
-            virtualisation.memorySize = 1024;
-            time.timeZone = "UTC";
-
-            environment.systemPackages = [ createTrivialProject pkgs.jq ];
-            services.hydra = {
-              enable = true;
-
-              #Hydra needs those settings to start up, so we add something not harmfull.
-              hydraURL = "example.com";
-              notificationSender = "example@example.com";
-
-              package = pkgs.hydra.override { inherit nix; };
-
-              extraConfig = ''
-                email_notification = 1
-              '';
-            };
-            services.postfix.enable = true;
-            nix = {
-              buildMachines = [{
-                hostName = "localhost";
-                systems = [ system ];
-              }];
-
-              binaryCaches = [];
-            };
-          };
-
-        testScript = ''
-          # let the system boot up
-          machine.wait_for_unit("multi-user.target")
-          # test whether the database is running
-          machine.wait_for_unit("postgresql.service")
-          # test whether the actual hydra daemons are running
-          machine.wait_for_unit("hydra-init.service")
-          machine.require_unit_state("hydra-queue-runner.service")
-          machine.require_unit_state("hydra-evaluator.service")
-          machine.require_unit_state("hydra-notify.service")
-
-          machine.succeed("hydra-create-user admin --role admin --password admin")
-
-          # create a project with a trivial job
-          machine.wait_for_open_port(3000)
-
-          # make sure the build as been successfully built
-          machine.succeed("create-trivial-project.sh")
-
-          machine.wait_until_succeeds(
-              'curl -L -s http://localhost:3000/build/1 -H "Accept: application/json" |  jq .buildstatus | xargs test 0 -eq'
-          )
-
-          machine.wait_until_succeeds(
-              'journalctl -eu hydra-notify.service -o cat | grep -q "sending mail notification to hydra@localhost"'
-          )
-        '';
-      })));
+  makeHydraTest = with pkgs.lib; name: package: makeTest {
+    name = "hydra-${name}";
+    meta = with pkgs.stdenv.lib.maintainers; {
+      maintainers = [ pstn lewo ma27 ];
+    };
+
+    machine = { pkgs, lib, ... }: {
+      imports = [ baseConfig ];
+      services.hydra = { inherit package; };
+    };
+
+    testScript = ''
+      # let the system boot up
+      machine.wait_for_unit("multi-user.target")
+      # test whether the database is running
+      machine.wait_for_unit("postgresql.service")
+      # test whether the actual hydra daemons are running
+      machine.wait_for_unit("hydra-init.service")
+      machine.require_unit_state("hydra-queue-runner.service")
+      machine.require_unit_state("hydra-evaluator.service")
+      machine.require_unit_state("hydra-notify.service")
+
+      machine.succeed("hydra-create-user admin --role admin --password admin")
+
+      # create a project with a trivial job
+      machine.wait_for_open_port(3000)
+
+      # make sure the build as been successfully built
+      machine.succeed("create-trivial-project.sh")
+
+      machine.wait_until_succeeds(
+          'curl -L -s http://localhost:3000/build/1 -H "Accept: application/json" |  jq .buildstatus | xargs test 0 -eq'
+      )
+
+      machine.wait_until_succeeds(
+          'journalctl -eu hydra-notify.service -o cat | grep -q "sending mail notification to hydra@localhost"'
+      )
+    '';
+  };
 
 in
-  tests
+
+mapAttrs makeHydraTest hydraPkgs