nixos/tests/letsencrypt: use Pebble instead of Boulder

Let's encrypt bumped ACME to V2. We need to update our nixos test to
be compatible with this new protocol version.

We decided to drop the Boulder ACME server in favor of the more
integration test friendly Pebble.

- overriding cacert not necessary
- this avoids rebuilding lots of packages needlessly
- nixos/tests/acme: use pebble's ca for client tests
- pebble always generates its own ca which has to be fetched

TODO: write proper commit msg :)

7 files changed, 313 insertions, 592 deletions

diff --git a/nixos/modules/security/acme.nix b/nixos/modules/security/acme.nix
index e35ea0c7b927..e08c7e965eef 100644
--- a/nixos/modules/security/acme.nix
+++ b/nixos/modules/security/acme.nix
@@ -203,6 +203,11 @@ in
                   description = "Renew ACME Certificate for ${cert}";
                   after = [ "network.target" "network-online.target" ];
                   wants = [ "network-online.target" ];
+                  # simp_le uses requests, which uses certifi under the hood,
+                  # which doesn't respect the system trust store.
+                  # At least in the acme test, we provision a fake CA, impersonating the LE endpoint.
+                  # REQUESTS_CA_BUNDLE is a way to teach python requests to use something else
+                  environment.REQUESTS_CA_BUNDLE = "/etc/ssl/certs/ca-certificates.crt";
                   serviceConfig = {
                     Type = "oneshot";
                     SuccessExitStatus = [ "0" "1" ];
diff --git a/nixos/tests/acme.nix b/nixos/tests/acme.nix
index 8cfdea4a16ef..85d32d109442 100644
--- a/nixos/tests/acme.nix
+++ b/nixos/tests/acme.nix
@@ -80,7 +80,7 @@ in import ./make-test.nix {
     client = commonConfig;
   };
 
-  testScript = {nodes, ...}: 
+  testScript = {nodes, ...}:
     let
       newServerSystem = nodes.webserver2.config.system.build.toplevel;
       switchToNewServer = "${newServerSystem}/bin/switch-to-configuration test";
@@ -91,9 +91,12 @@ in import ./make-test.nix {
     # get pulled in by the oneshot units. The target units linger after activation, and hence we
     # can use them to probe that a oneshot fired. It is a bit ugly, but it is the best we can do
     ''
-      $client->waitForUnit("default.target");
+      $client->start;
+      $letsencrypt->start;
+      $acmeStandalone->start;
+
       $letsencrypt->waitForUnit("default.target");
-      $letsencrypt->waitForUnit("boulder.service");
+      $letsencrypt->waitForUnit("pebble.service");
 
       subtest "can request certificate with HTTPS-01 challenge", sub {
         $acmeStandalone->waitForUnit("default.target");
@@ -101,15 +104,20 @@ in import ./make-test.nix {
         $acmeStandalone->waitForUnit("acme-finished-standalone.com.target");
       };
 
+      $client->waitForUnit("default.target");
+
+      $client->succeed('curl https://acme-v02.api.letsencrypt.org:15000/roots/0 > /tmp/ca.crt');
+      $client->succeed('curl https://acme-v02.api.letsencrypt.org:15000/intermediate-keys/0 >> /tmp/ca.crt');
+
       subtest "Can request certificate for nginx service", sub {
         $webserver->waitForUnit("acme-finished-a.example.com.target");
-        $client->succeed('curl https://a.example.com/ | grep -qF "hello world"');
+        $client->succeed('curl --cacert /tmp/ca.crt https://a.example.com/ | grep -qF "hello world"');
       };
 
       subtest "Can add another certificate for nginx service", sub {
         $webserver->succeed("/run/current-system/fine-tune/child-1/bin/switch-to-configuration test");
         $webserver->waitForUnit("acme-finished-b.example.com.target");
-        $client->succeed('curl https://b.example.com/ | grep -qF "hello world"');
+        $client->succeed('curl --cacert /tmp/ca.crt https://b.example.com/ | grep -qF "hello world"');
       };
     '';
 }
diff --git a/nixos/tests/common/letsencrypt/0001-Change-ACME-directory-endpoint-to-directory.patch b/nixos/tests/common/letsencrypt/0001-Change-ACME-directory-endpoint-to-directory.patch
new file mode 100644
index 000000000000..9d4a483dd889
--- /dev/null
+++ b/nixos/tests/common/letsencrypt/0001-Change-ACME-directory-endpoint-to-directory.patch
@@ -0,0 +1,25 @@
+From c3b4004386074342d22cab5e129c1f7e623f4272 Mon Sep 17 00:00:00 2001
+From: =?UTF-8?q?F=C3=A9lix=20Baylac-Jacqu=C3=A9?= <felix@alternativebit.fr>
+Date: Mon, 21 Oct 2019 10:56:13 +0200
+Subject: [PATCH] Change ACME directory endpoint to /directory
+
+---
+ wfe/wfe.go | 2 +-
+ 1 file changed, 1 insertion(+), 1 deletion(-)
+
+diff --git a/wfe/wfe.go b/wfe/wfe.go
+index e24797f..10d29fb 100644
+--- a/wfe/wfe.go
++++ b/wfe/wfe.go
+@@ -39,7 +39,7 @@ const (
+ 	// Note: We deliberately pick endpoint paths that differ from Boulder to
+ 	// exercise clients processing of the /directory response
+ 	// We export the DirectoryPath so that the pebble binary can reference it
+-	DirectoryPath     = "/dir"
++	DirectoryPath     = "/directory"
+ 	noncePath         = "/nonce-plz"
+ 	newAccountPath    = "/sign-me-up"
+ 	acctPath          = "/my-account/"
+-- 
+2.23.0
+
diff --git a/nixos/tests/common/letsencrypt/common.nix b/nixos/tests/common/letsencrypt/common.nix
index 798a749f7f9b..c530de817bf2 100644
--- a/nixos/tests/common/letsencrypt/common.nix
+++ b/nixos/tests/common/letsencrypt/common.nix
@@ -1,27 +1,9 @@
-{ lib, nodes, ... }: {
+{ lib, nodes, pkgs, ... }: let
+  letsencrypt-ca = nodes.letsencrypt.config.test-support.letsencrypt.caCert;
+in {
   networking.nameservers = [
     nodes.letsencrypt.config.networking.primaryIPAddress
   ];
 
-  nixpkgs.overlays = lib.singleton (self: super: {
-    cacert = super.cacert.overrideDerivation (drv: {
-      installPhase = (drv.installPhase or "") + ''
-        cat "${nodes.letsencrypt.config.test-support.letsencrypt.caCert}" \
-          >> "$out/etc/ssl/certs/ca-bundle.crt"
-      '';
-    });
-
-    # Override certifi so that it accepts fake certificate for Let's Encrypt
-    # Need to override the attribute used by simp_le, which is python3Packages
-    python3Packages = (super.python3.override {
-      packageOverrides = lib.const (pysuper: {
-        certifi = pysuper.certifi.overridePythonAttrs (attrs: {
-          postPatch = (attrs.postPatch or "") + ''
-            cat "${self.cacert}/etc/ssl/certs/ca-bundle.crt" \
-              > certifi/cacert.pem
-          '';
-        });
-      });
-    }).pkgs;
-  });
+  security.pki.certificateFiles = [ letsencrypt-ca ];
 }
diff --git a/nixos/tests/common/letsencrypt/default.nix b/nixos/tests/common/letsencrypt/default.nix
index 58d87c64e344..aaf2896f21cb 100644
--- a/nixos/tests/common/letsencrypt/default.nix
+++ b/nixos/tests/common/letsencrypt/default.nix
@@ -1,6 +1,3 @@
-# Fully pluggable module to have Letsencrypt's Boulder ACME service running in
-# a test environment.
-#
 # The certificate for the ACME service is exported as:
 #
 #   config.test-support.letsencrypt.caCert
@@ -54,277 +51,45 @@
 # that it has to be started _before_ the ACME service.
 { config, pkgs, lib, ... }:
 
-let
-  softhsm = pkgs.stdenv.mkDerivation rec {
-    pname = "softhsm";
-    version = "1.3.8";
-
-    src = pkgs.fetchurl {
-      url = "https://dist.opendnssec.org/source/${pname}-${version}.tar.gz";
-      sha256 = "0flmnpkgp65ym7w3qyg78d3fbmvq3aznmi66rgd420n33shf7aif";
-    };
-
-    configureFlags = [ "--with-botan=${pkgs.botan}" ];
-    buildInputs = [ pkgs.sqlite ];
-  };
-
-  pkcs11-proxy = pkgs.stdenv.mkDerivation {
-    name = "pkcs11-proxy";
-
-    src = pkgs.fetchFromGitHub {
-      owner = "SUNET";
-      repo = "pkcs11-proxy";
-      rev = "944684f78bca0c8da6cabe3fa273fed3db44a890";
-      sha256 = "1nxgd29y9wmifm11pjcdpd2y293p0dgi0x5ycis55miy97n0f5zy";
-    };
-
-    postPatch = "patchShebangs mksyscalls.sh";
-
-    nativeBuildInputs = [ pkgs.cmake ];
-    buildInputs = [ pkgs.openssl pkgs.libseccomp ];
-  };
-
-  mkGoDep = { goPackagePath, url ? "https://${goPackagePath}", rev, sha256 }: {
-    inherit goPackagePath;
-    src = pkgs.fetchgit { inherit url rev sha256; };
-  };
-
-  goose = let
-    owner = "liamstask";
-    repo = "goose";
-    rev = "8488cc47d90c8a502b1c41a462a6d9cc8ee0a895";
-    version = "20150116";
-
-  in pkgs.buildGoPackage rec {
-    name = "${repo}-${version}";
-
-    src = pkgs.fetchFromBitbucket {
-      name = "${name}-src";
-      inherit rev owner repo;
-      sha256 = "1jy0pscxjnxjdg3hj111w21g8079rq9ah2ix5ycxxhbbi3f0wdhs";
-    };
-
-    goPackagePath = "bitbucket.org/${owner}/${repo}";
-    subPackages = [ "cmd/goose" ];
-    extraSrcs = map mkGoDep [
-      { goPackagePath = "github.com/go-sql-driver/mysql";
-        rev = "2e00b5cd70399450106cec6431c2e2ce3cae5034";
-        sha256 = "085g48jq9hzmlcxg122n0c4pi41sc1nn2qpx1vrl2jfa8crsppa5";
-      }
-      { goPackagePath = "github.com/kylelemons/go-gypsy";
-        rev = "08cad365cd28a7fba23bb1e57aa43c5e18ad8bb8";
-        sha256 = "1djv7nii3hy451n5jlslk0dblqzb1hia1cbqpdwhnps1g8hqjy8q";
-      }
-      { goPackagePath = "github.com/lib/pq";
-        rev = "ba5d4f7a35561e22fbdf7a39aa0070f4d460cfc0";
-        sha256 = "1mfbqw9g00bk24bfmf53wri5c2wqmgl0qh4sh1qv2da13a7cwwg3";
-      }
-      { goPackagePath = "github.com/mattn/go-sqlite3";
-        rev = "2acfafad5870400156f6fceb12852c281cbba4d5";
-        sha256 = "1rpgil3w4hh1cibidskv1js898hwz83ps06gh0hm3mym7ki8d5h7";
-      }
-      { goPackagePath = "github.com/ziutek/mymysql";
-        rev = "0582bcf675f52c0c2045c027fd135bd726048f45";
-        sha256 = "0bkc9x8sgqbzgdimsmsnhb0qrzlzfv33fgajmmjxl4hcb21qz3rf";
-      }
-      { goPackagePath = "golang.org/x/net";
-        url = "https://go.googlesource.com/net";
-        rev = "10c134ea0df15f7e34d789338c7a2d76cc7a3ab9";
-        sha256 = "14cbr2shl08gyg85n5gj7nbjhrhhgrd52h073qd14j97qcxsakcz";
-      }
-    ];
-  };
-
-  boulder = let
-    owner = "letsencrypt";
-    repo = "boulder";
-    rev = "9c6a1f2adc4c26d925588f5ae366cfd4efb7813a";
-    version = "20180129";
-
-  in pkgs.buildGoPackage rec {
-    name = "${repo}-${version}";
-
-    src = pkgs.fetchFromGitHub {
-      name = "${name}-src";
-      inherit rev owner repo;
-      sha256 = "09kszswrifm9rc6idfaq0p1mz5w21as2qbc8gd5pphrq9cf9pn55";
-    };
-
-    postPatch = ''
-      # compat for go < 1.8
-      sed -i -e 's/time\.Until(\([^)]\+\))/\1.Sub(time.Now())/' \
-        test/ocsp/helper/helper.go
-
-      find test -type f -exec sed -i -e '/libpkcs11-proxy.so/ {
-        s,/usr/local,${pkcs11-proxy},
-      }' {} +
-
-      sed -i -r \
-        -e '/^def +install/a \    return True' \
-        -e 's,exec \./bin/,,' \
-        test/startservers.py
-
-      cat ${lib.escapeShellArg snakeOilCerts.ca.key} > test/test-ca.key
-      cat ${lib.escapeShellArg snakeOilCerts.ca.cert} > test/test-ca.pem
-    '';
-
-    # Until vendored pkcs11 is go 1.9 compatible
-    preBuild = ''
-      rm -r go/src/github.com/letsencrypt/boulder/vendor/github.com/miekg/pkcs11
-    '';
-
-    # XXX: Temporarily brought back putting the source code in the output,
-    # since e95f17e2720e67e2eabd59d7754c814d3e27a0b2 was removing that from
-    # buildGoPackage.
-    preInstall = ''
-      mkdir -p $out
-      pushd "$NIX_BUILD_TOP/go"
-      while read f; do
-        echo "$f" | grep -q '^./\(src\|pkg/[^/]*\)/${goPackagePath}' \
-          || continue
-        mkdir -p "$(dirname "$out/share/go/$f")"
-        cp "$NIX_BUILD_TOP/go/$f" "$out/share/go/$f"
-      done < <(find . -type f)
-      popd
-    '';
-
-    extraSrcs = map mkGoDep [
-      { goPackagePath = "github.com/miekg/pkcs11";
-        rev           = "6dbd569b952ec150d1425722dbbe80f2c6193f83";
-        sha256        = "1m8g6fx7df6hf6q6zsbyw1icjmm52dmsx28rgb0h930wagvngfwb";
-      }
-    ];
-
-    goPackagePath = "github.com/${owner}/${repo}";
-    buildInputs = [ pkgs.libtool ];
-  };
-
-  boulderSource = "${boulder.out}/share/go/src/${boulder.goPackagePath}";
-
-  softHsmConf = pkgs.writeText "softhsm.conf" ''
-    0:/var/lib/softhsm/slot0.db
-    1:/var/lib/softhsm/slot1.db
-  '';
 
+let
   snakeOilCerts = import ./snakeoil-certs.nix;
 
-  wfeDomain = "acme-v01.api.letsencrypt.org";
+  wfeDomain = "acme-v02.api.letsencrypt.org";
   wfeCertFile = snakeOilCerts.${wfeDomain}.cert;
   wfeKeyFile = snakeOilCerts.${wfeDomain}.key;
 
   siteDomain = "letsencrypt.org";
   siteCertFile = snakeOilCerts.${siteDomain}.cert;
   siteKeyFile = snakeOilCerts.${siteDomain}.key;
-
-  # Retrieved via:
-  # curl -s -I https://acme-v01.api.letsencrypt.org/terms \
-  #   | sed -ne 's/^[Ll]ocation: *//p'
-  tosUrl = "https://letsencrypt.org/documents/2017.11.15-LE-SA-v1.2.pdf";
-  tosPath = builtins.head (builtins.match "https?://[^/]+(.*)" tosUrl);
-
-  tosFile = pkgs.fetchurl {
-    url = tosUrl;
-    sha256 = "0yvyckqzj0b1xi61sypcha82nanizzlm8yqy828h2jbza7cxi26c";
-  };
+  pebble = pkgs.pebble.overrideAttrs (attrs: {
+    # The pebble directory endpoint is /dir when the bouder (official
+    # ACME server) is /directory. Sadly, this endpoint is hardcoded,
+    # we have to patch it.
+    #
+    # Tried to upstream, that said upstream maintainers rather keep
+    # this custom endpoint to test ACME clients robustness. See
+    # https://github.com/letsencrypt/pebble/issues/283#issuecomment-545123242
+    patches = [ ./0001-Change-ACME-directory-endpoint-to-directory.patch ];
+  });
 
   resolver = let
     message = "You need to define a resolver for the letsencrypt test module.";
     firstNS = lib.head config.networking.nameservers;
   in if config.networking.nameservers == [] then throw message else firstNS;
 
-  cfgDir = pkgs.stdenv.mkDerivation {
-    name = "boulder-config";
-    src = "${boulderSource}/test/config";
-    nativeBuildInputs = [ pkgs.jq ];
-    phases = [ "unpackPhase" "patchPhase" "installPhase" ];
-    postPatch = ''
-      sed -i -e 's/5002/80/' -e 's/5002/443/' va.json
-      sed -i -e '/listenAddress/s/:4000/:80/' wfe.json
-      sed -i -r \
-        -e ${lib.escapeShellArg "s,http://boulder:4000/terms/v1,${tosUrl},g"} \
-        -e 's,http://(boulder|127\.0\.0\.1):4000,https://${wfeDomain},g' \
-        -e '/dnsResolver/s/127\.0\.0\.1:8053/${resolver}:53/' \
-        *.json
-      if grep 4000 *.json; then exit 1; fi
-
-      # Change all ports from 1909X to 909X, because the 1909X range of ports is
-      # allocated by startservers.py in order to intercept gRPC communication.
-      sed -i -e 's/\<1\(909[0-9]\)\>/\1/' *.json
-
-      # Patch out all additional issuer certs
-      jq '. + {ca: (.ca + {Issuers:
-        [.ca.Issuers[] | select(.CertFile == "test/test-ca.pem")]
-      })}' ca.json > tmp
-      mv tmp ca.json
-    '';
-    installPhase = "cp -r . \"$out\"";
-  };
-
-  components = {
-    gsb-test-srv.args = "-apikey my-voice-is-my-passport";
-    gsb-test-srv.waitForPort = 6000;
-    gsb-test-srv.first = true;
-    boulder-sa.args = "--config ${cfgDir}/sa.json";
-    boulder-wfe.args = "--config ${cfgDir}/wfe.json";
-    boulder-ra.args = "--config ${cfgDir}/ra.json";
-    boulder-ca.args = "--config ${cfgDir}/ca.json";
-    boulder-va.args = "--config ${cfgDir}/va.json";
-    boulder-publisher.args = "--config ${cfgDir}/publisher.json";
-    boulder-publisher.waitForPort = 9091;
-    ocsp-updater.args = "--config ${cfgDir}/ocsp-updater.json";
-    ocsp-updater.after = [ "boulder-publisher" ];
-    ocsp-responder.args = "--config ${cfgDir}/ocsp-responder.json";
-    ct-test-srv = {};
-    mail-test-srv.args = let
-      key = "${boulderSource}/test/mail-test-srv/minica-key.pem";
-      crt = "${boulderSource}/test/mail-test-srv/minica.pem";
-     in
-      "--closeFirst 5 --cert ${crt} --key ${key}";
+  pebbleConf.pebble = {
+    listenAddress = "0.0.0.0:443";
+    managementListenAddress = "0.0.0.0:15000";
+    certificate = snakeOilCerts.${wfeDomain}.cert;
+    privateKey = snakeOilCerts.${wfeDomain}.key;
+    httpPort = 80;
+    tlsPort = 443;
+    ocspResponderURL = "http://0.0.0.0:4002";
   };
 
-  commonPath = [ softhsm pkgs.mariadb goose boulder ];
-
-  mkServices = a: b: with lib; listToAttrs (concatLists (mapAttrsToList a b));
-
-  componentServices = mkServices (name: attrs: let
-    mkSrvName = n: "boulder-${n}.service";
-    firsts = lib.filterAttrs (lib.const (c: c.first or false)) components;
-    firstServices = map mkSrvName (lib.attrNames firsts);
-    firstServicesNoSelf = lib.remove "boulder-${name}.service" firstServices;
-    additionalAfter = firstServicesNoSelf ++ map mkSrvName (attrs.after or []);
-    needsPort = attrs ? waitForPort;
-    inits = map (n: "boulder-init-${n}.service") [ "mysql" "softhsm" ];
-    portWaiter = {
-      name = "boulder-${name}";
-      value = {
-        description = "Wait For Port ${toString attrs.waitForPort} (${name})";
-        after = [ "boulder-real-${name}.service" "bind.service" ];
-        requires = [ "boulder-real-${name}.service" ];
-        requiredBy = [ "boulder.service" ];
-        serviceConfig.Type = "oneshot";
-        serviceConfig.RemainAfterExit = true;
-        script = let
-          netcat = "${pkgs.libressl.nc}/bin/nc";
-          portCheck = "${netcat} -z 127.0.0.1 ${toString attrs.waitForPort}";
-        in "while ! ${portCheck}; do :; done";
-      };
-    };
-  in lib.optional needsPort portWaiter ++ lib.singleton {
-    name = if needsPort then "boulder-real-${name}" else "boulder-${name}";
-    value = {
-      description = "Boulder ACME Component (${name})";
-      after = inits ++ additionalAfter;
-      requires = inits;
-      requiredBy = [ "boulder.service" ];
-      path = commonPath;
-      environment.GORACE = "halt_on_error=1";
-      environment.SOFTHSM_CONF = softHsmConf;
-      environment.PKCS11_PROXY_SOCKET = "tcp://127.0.0.1:5657";
-      serviceConfig.WorkingDirectory = boulderSource;
-      serviceConfig.ExecStart = "${boulder}/bin/${name} ${attrs.args or ""}";
-      serviceConfig.Restart = "on-failure";
-    };
-  }) components;
+  pebbleConfFile = pkgs.writeText "pebble.conf" (builtins.toJSON pebbleConf);
+  pebbleDataDir = "/root/pebble";
 
 in {
   imports = [ ../resolver.nix ];
@@ -352,94 +117,29 @@ in {
     networking.firewall.enable = false;
 
     networking.extraHosts = ''
-      127.0.0.1 ${toString [
-        "sa.boulder" "ra.boulder" "wfe.boulder" "ca.boulder" "va.boulder"
-        "publisher.boulder" "ocsp-updater.boulder" "admin-revoker.boulder"
-        "boulder" "boulder-mysql" wfeDomain
-      ]}
+      127.0.0.1 ${wfeDomain}
       ${config.networking.primaryIPAddress} ${wfeDomain} ${siteDomain}
     '';
 
-    services.mysql.enable = true;
-    services.mysql.package = pkgs.mariadb;
-
-    services.nginx.enable = true;
-    services.nginx.recommendedProxySettings = true;
-    # This fixes the test on i686
-    services.nginx.commonHttpConfig = ''
-      server_names_hash_bucket_size 64;
-    '';
-    services.nginx.virtualHosts.${wfeDomain} = {
-      onlySSL = true;
-      enableACME = false;
-      sslCertificate = wfeCertFile;
-      sslCertificateKey = wfeKeyFile;
-      locations."/".proxyPass = "http://127.0.0.1:80";
-    };
-    services.nginx.virtualHosts.${siteDomain} = {
-      onlySSL = true;
-      enableACME = false;
-      sslCertificate = siteCertFile;
-      sslCertificateKey = siteKeyFile;
-      locations."= ${tosPath}".alias = tosFile;
-    };
-
     systemd.services = {
-      pkcs11-daemon = {
-        description = "PKCS11 Daemon";
-        after = [ "boulder-init-softhsm.service" ];
-        before = map (n: "${n}.service") (lib.attrNames componentServices);
-        wantedBy = [ "multi-user.target" ];
-        environment.SOFTHSM_CONF = softHsmConf;
-        environment.PKCS11_DAEMON_SOCKET = "tcp://127.0.0.1:5657";
-        serviceConfig.ExecStart = let
-          softhsmLib = "${softhsm}/lib/softhsm/libsofthsm.so";
-        in "${pkcs11-proxy}/bin/pkcs11-daemon ${softhsmLib}";
-      };
-
-      boulder-init-mysql = {
-        description = "Boulder ACME Init (MySQL)";
-        after = [ "mysql.service" ];
-        serviceConfig.Type = "oneshot";
-        serviceConfig.RemainAfterExit = true;
-        serviceConfig.WorkingDirectory = boulderSource;
-        path = commonPath;
-        script = "${pkgs.bash}/bin/sh test/create_db.sh";
-      };
-
-      boulder-init-softhsm = {
-        description = "Boulder ACME Init (SoftHSM)";
-        environment.SOFTHSM_CONF = softHsmConf;
-        serviceConfig.Type = "oneshot";
-        serviceConfig.RemainAfterExit = true;
-        serviceConfig.WorkingDirectory = boulderSource;
-        preStart = "mkdir -p /var/lib/softhsm";
-        path = commonPath;
+      pebble = {
+        enable = true;
+        description = "Pebble ACME server";
+        requires = [ ];
+        wantedBy = [ "network.target" ];
+        preStart = ''
+          mkdir ${pebbleDataDir}
+        '';
         script = ''
-          softhsm --slot 0 --init-token \
-            --label intermediate --pin 5678 --so-pin 1234
-          softhsm --slot 0 --import test/test-ca.key \
-            --label intermediate_key --pin 5678 --id FB
-          softhsm --slot 1 --init-token \
-            --label root --pin 5678 --so-pin 1234
-          softhsm --slot 1 --import test/test-root.key \
-            --label root_key --pin 5678 --id FA
+          cd ${pebbleDataDir}
+          ${pebble}/bin/pebble -config ${pebbleConfFile}
         '';
+        serviceConfig = {
+          # Required to bind on privileged ports.
+          User = "root";
+          Group = "root";
+        };
       };
-
-      boulder = {
-        description = "Boulder ACME Server";
-        after = map (n: "${n}.service") (lib.attrNames componentServices);
-        wantedBy = [ "multi-user.target" ];
-        serviceConfig.Type = "oneshot";
-        serviceConfig.RemainAfterExit = true;
-        script = let
-          ports = lib.range 8000 8005 ++ lib.singleton 80;
-          netcat = "${pkgs.libressl.nc}/bin/nc";
-          mkPortCheck = port: "${netcat} -z 127.0.0.1 ${toString port}";
-          checks = "(${lib.concatMapStringsSep " && " mkPortCheck ports})";
-        in "while ! ${checks}; do :; done";
-      };
-    } // componentServices;
+    };
   };
 }
diff --git a/nixos/tests/common/letsencrypt/mkcerts.nix b/nixos/tests/common/letsencrypt/mkcerts.nix
index 3b4a589e4142..e7ac2bae46bd 100644
--- a/nixos/tests/common/letsencrypt/mkcerts.nix
+++ b/nixos/tests/common/letsencrypt/mkcerts.nix
@@ -1,7 +1,7 @@
 { pkgs ? import <nixpkgs> {}
 , lib ? pkgs.lib
 
-, domains ? [ "acme-v01.api.letsencrypt.org" "letsencrypt.org" ]
+, domains ? [ "acme-v02.api.letsencrypt.org" "letsencrypt.org" ]
 }:
 
 pkgs.runCommand "letsencrypt-snakeoil-ca" {
diff --git a/nixos/tests/common/letsencrypt/snakeoil-certs.nix b/nixos/tests/common/letsencrypt/snakeoil-certs.nix
index c3d29ab8f163..ca4f71ae688a 100644
--- a/nixos/tests/common/letsencrypt/snakeoil-certs.nix
+++ b/nixos/tests/common/letsencrypt/snakeoil-certs.nix
@@ -2,252 +2,253 @@
 {
   ca.key = builtins.toFile "ca.key" ''
     -----BEGIN PRIVATE KEY-----
-    MIIJQwIBADANBgkqhkiG9w0BAQEFAASCCS0wggkpAgEAAoICAQDfdVxC/4HwhuzD
-    9or9CDDu3TBQE5lirJI5KYmfMZtfgdzEjgOzmR9AVSkn2rQeCqzM5m+YCzPO+2y7
-    0Fdk7vDORi1OdhYfUQIW6/TZ27xEjx4t82j9i705yUqTJZKjMbD830geXImJ6VGj
-    Nv/WisTHmwBspWKefYQPN68ZvYNCn0d5rYJg9uROZPJHSI0MYj9iERWIPN+xhZoS
-    xN74ILJ0rEOQfx2GHDhTr99vZYAFqbAIfh35fYulRWarUSekI+rDxa83FD8q9cMg
-    OP84KkLep2dRXXTbUWErGUOpHP55M9M7ws0RVNdl9PUSbDgChl7yYlHCde3261q/
-    zGp5dMV/t/jXXNUgRurvXc4gUKKjS4Sffvg0XVnPs3sMlZ4JNmycK9klgISVmbTK
-    VcjRRJv8Bva2NQVsJ9TIryV0QEk94DucgsC3LbhQfQdmnWVcEdzwrZHNpk9az5mn
-    w42RuvZW9L19T7xpIrdLSHaOis4VEquZjkWIhfIz0DVMeXtYEQmwqFG23Ww0utcp
-    mCW4FPvpyYs5GAPmGWfrlMxsLD/7eteot3AheC+56ZBoVBnI8FFvIX2qci+gfVDu
-    CjvDmbyS/0NvxLGqvSC1GUPmWP3TR5Fb1H8Rp+39zJHRmH+qYWlhcv6p7FlY2/6d
-    9Rkw8WKRTSCB7yeUdNNPiPopk6N4NwIDAQABAoICAQCzV0ei5dntpvwjEp3eElLj
-    glYiDnjOPt5kTjgLsg6XCmyau7ewzrXMNgz/1YE1ky+4i0EI8AS2nAdafQ2HDlXp
-    11zJWfDLVYKtztYGe1qQU6TPEEo1I4/M7waRLliP7XO0n6cL5wzjyIQi0CNolprz
-    8CzZBasutGHmrLQ1nmnYcGk2+NBo7f2yBUaFe27of3mLRVbYrrKBkU5kveiNkABp
-    r0/SipKxbbivQbm7d+TVpqiHSGDaOa54CEksOcfs7n6efOvw8qj326KtG9GJzDE6
-    7XP4U19UHe40XuR0t7Zso/FmRyO6QzNUutJt5LjXHezZ75razTcdMyr0QCU8MUHH
-    jXZxQCsbt+9AmdxUMBm1SMNVBdHYM8oiNHynlgsEj9eM6jxDEss/Uc3FeKoHl+XL
-    L6m28guIB8NivqjVzZcwhxvdiQCzYxjyqMC+/eX7aaK4NIlX2QRMoDL6mJ58Bz/8
-    V2Qxp2UNVwKJFWAmpgXC+sq6XV/TP3HkOvd0OK82Nid2QxEvfE/EmOhU63qAjgUR
-    QnteLEcJ3MkGGurs05pYBDE7ejKVz6uu2tHahFMOv+yanGP2gfivnT9a323/nTqH
-    oR5ffMEI1u/ufpWU7sWXZfL/mH1L47x87k+9wwXHCPeSigcy+hFI7t1+rYsdCmz9
-    V6QtmxZHMLanwzh5R0ipcQKCAQEA8kuZIz9JyYP6L+5qmIUxiWESihVlRCSKIqLB
-    fJ5sQ06aDBV2sqS4XnoWsHuJWUd39rulks8cg8WIQu8oJwVkFI9EpARt/+a1fRP0
-    Ncc9qiBdP6VctQGgKfe5KyOfMzIBUl3zj2cAmU6q+CW1OgdhnEl4QhgBe5XQGquZ
-    Alrd2P2jhJbMO3sNFgzTy7xPEr3KqUy+L4gtRnGOegKIh8EllmsyMRO4eIrZV2z3
-    XI+S2ZLyUn3WHYkaJqvUFrbfekgBBmbk5Ead6ImlsLsBla6MolKrVYV1kN6KT+Y+
-    plcxNpWY8bnWfw5058OWPLPa9LPfReu9rxAeGT2ZLmAhSkjGxQKCAQEA7BkBzT3m
-    SIzop9RKl5VzYbVysCYDjFU9KYMW5kBIw5ghSMnRmU7kXIZUkc6C1L/v9cTNFFLw
-    ZSF4vCHLdYLmDysW2d4DU8fS4qdlDlco5A00g8T1FS7nD9CzdkVN/oix6ujw7RuI
-    7pE1K3JELUYFBc8AZ7mIGGbddeCwnM+NdPIlhWzk5s4x4/r31cdk0gzor0kE4e+d
-    5m0s1T4O/Iak6rc0MGDeTejZQg04p1eAJFYQ6OY23tJhH/kO8CMYnQ4fidfCkf8v
-    85v4EC1MCorFR7J65uSj8MiaL7LTXPvLAkgFls1c3ijQ2tJ8qXvqmfo0by33T1OF
-    ZGyaOP9/1WQSywKCAQB47m6CfyYO5EZNAgxGD8SHsuGT9dXTSwF/BAjacB/NAEA2
-    48eYpko3LWyBrUcCPn+LsGCVg7XRtxepgMBjqXcoI9G4o1VbsgTHZtwus0D91qV0
-    DM7WsPcFu1S6SU8+OCkcuTPFUT2lRvRiYj+vtNttK+ZP5rdmvYFermLyH/Q2R3ID
-    zVgmH+aKKODVASneSsgJ8/nAs5EVZbwc/YKzbx2Zk+s7P4KE95g+4G4dzrMW0RcN
-    QS1LFJDu2DhFFgU4fRO15Ek9/lj2JS2DpfLGiJY8tlI5nyDsq4YRFvQSBdbUTZpG
-    m+CJDegffSlRJtuT4ur/dQf5hmvfYTVBRk2XS/eZAoIBAB143a22PWnvFRfmO02C
-    3X1j/iYZCLZa6aCl+ZTSj4LDGdyRPPXrUDxwlFwDMHfIYfcHEyanV9T4Aa9SdKh9
-    p6RbF6YovbeWqS+b/9RzcupM77JHQuTbDwL9ZXmtGxhcDgGqBHFEz6ogPEfpIrOY
-    GwZnmcBY+7E4HgsZ+lII4rqng6GNP2HEeZvg91Eba+2AqQdAkTh3Bfn+xOr1rT8+
-    u5WFOyGS5g1JtN0280yIcrmWeNPp8Q2Nq4wnNgMqDmeEnNFDOsmo1l6NqMC0NtrW
-    CdxyXj82aXSkRgMQSqw/zk7BmNkDV8VvyOqX/fHWQynnfuYmEco4Pd2UZQgadOW5
-    cVMCggEBANGz1fC+QQaangUzsVNOJwg2+CsUFYlAKYA3pRKZPIyMob2CBXk3Oln/
-    YqOq6j373kG2AX74EZT07JFn28F27JF3r+zpyS/TYrfZyO1lz/5ZejPtDTmqBiVd
-    qa2coaPKwCOz64s77A9KSPyvpvyuTfRVa8UoArHcrQsPXMHgEhnFRsbxgmdP582A
-    kfYfoJBSse6dQtS9ZnREJtyWJlBNIBvsuKwzicuIgtE3oCBcIUZpEa6rBSN7Om2d
-    ex8ejCcS7qpHeULYspXbm5ZcwE4glKlQbJDTKaJ9mjiMdvuNFUZnv1BdMQ3Tb8zf
-    Gvfq54FbDuB10XP8JdLrsy9Z6GEsmoE=
+    MIIJQgIBADANBgkqhkiG9w0BAQEFAASCCSwwggkoAgEAAoICAQDQ0b23I1srJZwR
+    2MMdvSJK5pcwLfrXU+4gEZEnWNyT8yeVweya+8vmNNOlvK3zxf+ZiY/7aQ0RZJMO
+    h2+VdlgHmr2QKhQTf1HwfZA/06FolD3/DcS+DMJMSTVr179/XLndeVVZUqU7tjvB
+    AWKSIS8H2hSF1UOPi9gBDR8MwCP6Qgj8WYhbkt9q47/lO96qAmm6U1F+Q7RYM9ZQ
+    IWI81N0Ms5wJocg7n6S19iV66ePh7APapZFYup61gFGWfahmA217ELIZd56n8yjO
+    F0epb9sC0XpYCDRrYKBWLqPiv+6wvdZtZvALItyIv08ZwXlBkFg3LbAAhPnf0Vxz
+    pYysQmyyyzkgy252n+Sie0kx+B4qm6fOkpfgYlPSVTb2dXx/be/SE08u0a9FO0fZ
+    pkByWEZJUUwngsJgLUa7MorQf3avxozfC25XqvzbieZfSXlA7mOUclZbC/WUFpyj
+    MlyJU2eCQ8wSwsPXl91oxcYlOkuVLgd41gr9pGXQSuKIkrgbfkftjg2tDC+7g7O8
+    qrdF42FjbZjIx/74AasmsGh4GTQtiSkvEnTstioC6aCV44DlJWbBIMvkyawubjUl
+    Ppij0H66Y9Q4tEc/ktc7oGQfqqluyLb43TeobTPHALsNeAYb39rMtBo5DDCUc81s
+    fuDMhMr/oYXKrFstUsg5AY6mJaRG0QIDAQABAoICAF5ZVfmoPOoKzTB3GvmV2iez
+    dj4rmDmwT1gn98iqasdiRtFwVGJWQHNcDQDGdmY9YNZThD2Y4nGoWpVm9jC2zuFo
+    thusF3QTw8cARKvCCBzDVhumce1YwHVNYpi+W2TFValOyBRathN7rBXxdUMHQUOv
+    8jPh/uudyNP4xL2zFs5dBchW/7g4bT/TdYGyglGYU4L/YEPHfXWYvk1oOAW6O8Ig
+    aPElKt5drEMW2yplATSzua4RvtEzSMBDIRn43pxxEgdXrNC67nF9+ULc2+Efi/oD
+    Ad9CncSiXO9zlVK/W655p6e4qd6uOqyCm8/MTegkuub7eplRe8D3zGjoNN4kCQ4S
+    rckVvIDDb6vZk7PKx9F7GWIqaG/YvFFFKO1MrAZg7SguFA6PtGOYAFocT03P6KXT
+    l2SnZQWKyxUAlh4tOBGlRFgGCx/krRIKbgNYn/qk/ezcRl8c7GpOPh+b7Icoq7u3
+    l4tIVBBHqS8uGgtyi+YwuJeht2MV1aEcSkykKLh2ipp8tb6spORJUkhjawDjvxeQ
+    GztN30Xh2riTXYZ0HExVTtJa8jyvFyp/97ptPIJXaVt2A2KIS3sBFHKnpY+/OrQg
+    uUauYgi13WFHsKOxZL9GYGk7Ujd8bw4CEcJFxKY7bhpGVI6Du7NRkUDWN0+0yusI
+    2szCJ7+ZqJkrc1+GrI/RAoIBAQDseAEggOLYZkpU2Pht15ZbxjM9ayT2ANq1+RTu
+    LjJx4gv2/o/XJCfMZCL0b9TJqtYeH+N6G9oDRJ99VIhUPedhWSYdj9Qj+rPd++TS
+    bp+MoSjmfUfxLTDrmFHL7ppquAE65aDy3B5c+OCb0I4X6CILUf0LynBzgl4kdrzN
+    U6BG3Mt0RiGPojlPV82B9ZUF/09YAz7BIz9X3KMhze1Gps5OeGuUnc9O2IAJYkrj
+    ur9H2YlNS4w+IjRLAXSXUqC8bqPZp6WTo1G/rlyAkIRXCGN90uk5JQvXoj9immFO
+    WaylbdcNG3YcGutreYeZL/UIWF6zCdc6pYG0cCBJS6S/RN7FAoIBAQDiERrLuUbV
+    3fx/a8uMeZop6hXtQpF7jlFxqUmza7QSvBuwks4QVJF+qMSiSvKDkCKqZD4qVf4N
+    TMxEj5vNR0PbnmDshyKJNGVjEauKJSb65CFDUcL1eR/A/oJvxiIdN1Z4cPrpnRux
+    /zIfPuYfYHpdz52buxxmlD7bfwYmVKVpnzjB9z0I1CasZ5uqB0Z8H0OLyUu8S4ju
+    RfkKBDMgVl2q96i8ZvX4C1b7XuimIUqv4WHq5+ejcYirgrYtUbBIaDU3/LORcJdy
+    /K76L1/up70RTDUYYm/HKaRy+vMTpUsZJ7Qbh0hrvQkUvNQ1HXjprW2AePIYi33N
+    h3mb1ulqw4idAoIBAQCsn0YjVjNDShkFK4bfmLv4rw2Ezoyi0SjYIsb2wN6uaBfX
+    7SlQIuKywH8L9f9eYMoCH8FNyLs0G4paUbVb2fzpAc1jUzXINiHL8TCvtXXfkV5s
+    NBSqqRTHR+CegMZVFZJATpVZ9PptYHmHBY5VQW5o2SdizhudFxRmhg95zIx6boBP
+    l0q0sfYoR66MKpzpTeG8HFJZZ8O7/iNQcCXAp9B/VEUkrrdBlaaSMyD8cb1lVBZ5
+    SKdOTGXkQ2G7feQ86n/OSiYDSvxIc56vc9BIQKVwmuEKiFLGzXh8ILrcGXaBJVgS
+    B3QHPFeTk5o7Z9j2iJxJEuv9sginkhrfpsrTnhEJAoIBACkrUkTtjd/e2F/gIqaH
+    crLVZX7a06G7rktTuA9LuvR6e1Rxt8Mzk3eMhprDqVyaQCXlsYiGNoj3hm+p84az
+    xsDVG/OXPIveFeSv0ByNXYbtSr12w1lu4ICGGP0ACTBm5oFymc83hFarEdas3r2y
+    FTbGW36D2c04jCXvARCz85fDnlN8kgnskMpu5+NUBdsO2n83fmphGyPBbHQNhb4K
+    3G4JQhplab/tWL7YbufqQi67jdh4uS+Duo75c/HW4ZKeH6r9gzomVf5j0/3N6NuO
+    gpkG1tiE/LQ5ejBSUTgvrvh6yYsF3QN53pB/PuoZXu63Xay62ePsa1GlrVjbD5EY
+    4OUCggEAJFr7F7AQLMJTAxHFLCsZZ0ZZ+tXYclBC4eHPkZ6sD5jvL3KIpW3Q7jXk
+    oIoD/XEX4B+Qe5M3jQJ/Y5ZJETHcgfcHZbDpCKN2WHQgldQbAJiFd4GY1OegdVsr
+    7TC8jh3Q2eYjzL8u4z7LSNI6aQSv1eWE7S1Q5j/sX/YYDR4W3CBMeIUpqoDWpn87
+    czbIRyA/4L0Y/HLpg/ZCbvtJZbsQwYXhyqfbjlm4BRQ6JiC5uEBKvuDRUXToBJta
+    JU8XMm+Ae5Ogrw7P6hg68dWpagfjb7UZ7Zxv+VDsbrU6KsDcyGCAwrrRZou/6KUG
+    Eq4OVTSu/s8gmY94tgbjeOaLUPEPmg==
     -----END PRIVATE KEY-----
   '';
   ca.cert = builtins.toFile "ca.cert" ''
     -----BEGIN CERTIFICATE-----
-    MIIFATCCAumgAwIBAgIJANydi4uFZr0LMA0GCSqGSIb3DQEBCwUAMBYxFDASBgNV
-    BAMMC1NuYWtlb2lsIENBMCAXDTE4MDcxMjAwMjIxNloYDzIxMTgwNjE4MDAyMjE2
-    WjAWMRQwEgYDVQQDDAtTbmFrZW9pbCBDQTCCAiIwDQYJKoZIhvcNAQEBBQADggIP
-    ADCCAgoCggIBAN91XEL/gfCG7MP2iv0IMO7dMFATmWKskjkpiZ8xm1+B3MSOA7OZ
-    H0BVKSfatB4KrMzmb5gLM877bLvQV2Tu8M5GLU52Fh9RAhbr9NnbvESPHi3zaP2L
-    vTnJSpMlkqMxsPzfSB5ciYnpUaM2/9aKxMebAGylYp59hA83rxm9g0KfR3mtgmD2
-    5E5k8kdIjQxiP2IRFYg837GFmhLE3vggsnSsQ5B/HYYcOFOv329lgAWpsAh+Hfl9
-    i6VFZqtRJ6Qj6sPFrzcUPyr1wyA4/zgqQt6nZ1FddNtRYSsZQ6kc/nkz0zvCzRFU
-    12X09RJsOAKGXvJiUcJ17fbrWr/Manl0xX+3+Ndc1SBG6u9dziBQoqNLhJ9++DRd
-    Wc+zewyVngk2bJwr2SWAhJWZtMpVyNFEm/wG9rY1BWwn1MivJXRAST3gO5yCwLct
-    uFB9B2adZVwR3PCtkc2mT1rPmafDjZG69lb0vX1PvGkit0tIdo6KzhUSq5mORYiF
-    8jPQNUx5e1gRCbCoUbbdbDS61ymYJbgU++nJizkYA+YZZ+uUzGwsP/t616i3cCF4
-    L7npkGhUGcjwUW8hfapyL6B9UO4KO8OZvJL/Q2/Esaq9ILUZQ+ZY/dNHkVvUfxGn
-    7f3MkdGYf6phaWFy/qnsWVjb/p31GTDxYpFNIIHvJ5R000+I+imTo3g3AgMBAAGj
-    UDBOMB0GA1UdDgQWBBQ3vPWzjLmu5krbSpfhBAht9KL3czAfBgNVHSMEGDAWgBQ3
-    vPWzjLmu5krbSpfhBAht9KL3czAMBgNVHRMEBTADAQH/MA0GCSqGSIb3DQEBCwUA
-    A4ICAQDF9HyC1ZFN3Ob+JA9Dj5+Rcobi7JIA5F8uW3Q92LfPoVaUGEkBrwJSiTFX
-    47zvP/ySBJIpZ9rzHMbJ+1L+eJgczF1uQ91inthCKo1THTPo5TgBrpJj0YAIunsj
-    9eH1tBnfWFYdVIDZoTSiwPtgIvglpyuK/eJXEe+FRzubhtdc9w1Hlzox1sd0TQuy
-    Pl9KFHg7BlFZfCPig1mkB8pfwjBDgVhv5DKJ9cJXh3R5zSoiyuS2b+qYSvw8YTHq
-    0WNKWUthb7BVAYE3OmcbOHgUAUjtJ6EIGIB9z/SoLe90CofXLXFR5dppuVLKCMBA
-    kgL4luBIu7t8mcnN2yzobvcGHy8RVY6F5abCCy6gackLzjOzvH1SYOxP8yN74aKB
-    ANgcqdWspb8JYoU8lEbA8dhBVrsgBf7XeJlrZvMdcUENlJ2PI0JWr9WvlRAM9rYY
-    EY1alJqBCp6530Ggd6/f0V64cEqptejUdmN9L0zboxKjQf4LjpUNraGvg8tw/xkY
-    4dT1U2HlVnhOyBVkx/tE6zIK/RU16oMqwpjCdfbK/TuWCNc/emJz5PMlp81zm83+
-    dExpWwuV4rt6OQbZ/GSatNLJXOw+pkLjaEhnHgrsgI+HqAUXg3ByKol+1e76wN51
-    k1ZKpB6mk4kejySGPYBHiJwED0IyXu9gUfalSczXFO4ySAvhCg==
+    MIIFDzCCAvegAwIBAgIUU9rbCLTuvaI6gjSsFsJJjfLWIX8wDQYJKoZIhvcNAQEL
+    BQAwFjEUMBIGA1UEAwwLU25ha2VvaWwgQ0EwIBcNMTkxMDE4MDc1NDEyWhgPMjEx
+    OTA5MjQwNzU0MTJaMBYxFDASBgNVBAMMC1NuYWtlb2lsIENBMIICIjANBgkqhkiG
+    9w0BAQEFAAOCAg8AMIICCgKCAgEA0NG9tyNbKyWcEdjDHb0iSuaXMC3611PuIBGR
+    J1jck/MnlcHsmvvL5jTTpbyt88X/mYmP+2kNEWSTDodvlXZYB5q9kCoUE39R8H2Q
+    P9OhaJQ9/w3EvgzCTEk1a9e/f1y53XlVWVKlO7Y7wQFikiEvB9oUhdVDj4vYAQ0f
+    DMAj+kII/FmIW5LfauO/5TveqgJpulNRfkO0WDPWUCFiPNTdDLOcCaHIO5+ktfYl
+    eunj4ewD2qWRWLqetYBRln2oZgNtexCyGXeep/MozhdHqW/bAtF6WAg0a2CgVi6j
+    4r/usL3WbWbwCyLciL9PGcF5QZBYNy2wAIT539Fcc6WMrEJssss5IMtudp/kontJ
+    MfgeKpunzpKX4GJT0lU29nV8f23v0hNPLtGvRTtH2aZAclhGSVFMJ4LCYC1GuzKK
+    0H92r8aM3wtuV6r824nmX0l5QO5jlHJWWwv1lBacozJciVNngkPMEsLD15fdaMXG
+    JTpLlS4HeNYK/aRl0EriiJK4G35H7Y4NrQwvu4OzvKq3ReNhY22YyMf++AGrJrBo
+    eBk0LYkpLxJ07LYqAumgleOA5SVmwSDL5MmsLm41JT6Yo9B+umPUOLRHP5LXO6Bk
+    H6qpbsi2+N03qG0zxwC7DXgGG9/azLQaOQwwlHPNbH7gzITK/6GFyqxbLVLIOQGO
+    piWkRtECAwEAAaNTMFEwHQYDVR0OBBYEFAZcEiVphGxBT4OWXbM6lKu96dvbMB8G
+    A1UdIwQYMBaAFAZcEiVphGxBT4OWXbM6lKu96dvbMA8GA1UdEwEB/wQFMAMBAf8w
+    DQYJKoZIhvcNAQELBQADggIBAGJ5Jnxq1IQ++IRYxCE7r7BqzzF+HTx0EWKkSOmt
+    eSPqeOdhC26hJlclgGZXAF/Xosmn8vkSQMHhj/jr4HI0VF9IyvDUJm8AKsnOgu/7
+    DUey3lEUdOtJpTG9NyTOcrzxToMJ+hWlFLZKxx2dk4FLIvTLjmo1VHM97Bat7XYW
+    IrL9RRIZ25V+eCYtlR7XYjceGFQ0rCdp8SFIQwC6C/AH2tV3b1AJFsND9PcoLu7c
+    //fH+WUQCcD/N0grdC/QCX7AFWzd4rKQ8gjfND4TSYFTSDwW10Mud4kAVhY2P1sY
+    Y3ZpnxWrCHbIZMbszlbMyD+cjsCBnNvOtYGm7pDut/371rllVcB/uOWYWMCtKPoj
+    0elPrwNMrK+P+wceNBCRQO+9gwzB589F2morFTtsob/qtpAygW8Sfl8M+iLWXeYS
+    c3LBLnj0TpgXKRWg7wgIWKSZx9v6pgy70U0qvkjNS1XseUCPf7hfAbxT3xF+37Dw
+    zZRwF4WAWqdnJoOey21mgc+a2DQzqtykA6KfHgCqNFfDbQXPXvNy25DDThbk+paX
+    G2M2EWtr+Nv9s/zm7Xv/pOXlgMFavaj+ikqZ4wfJf6c/sMOdZJtMA4TsYtAJgbc8
+    ts+0eymTq4v5S8/fW51Lbjw6hc1Kcm8k7NbHSi9sEjBfxFLTZNQ5eb4NGr9Od3sU
+    kgwJ
     -----END CERTIFICATE-----
   '';
-  "acme-v01.api.letsencrypt.org".key = builtins.toFile "acme-v01.api.letsencrypt.org.key" ''
+  "acme-v02.api.letsencrypt.org".key = builtins.toFile "acme-v02.api.letsencrypt.org.key" ''
     -----BEGIN RSA PRIVATE KEY-----
-    MIIJKQIBAAKCAgEAvG+sL4q0VkgSClBTn4NkPiUrtXx5oLyZ+CCM1jrQx/xotUt5
-    X2S4/7vMnAK/yRLsR7R2PhXO8CZPqJ7B6OfAgaDTgvipJkZYPZQSMP3KOinM3WJL
-    ssqKh7/HOxZIf0iyUXewrnX5eTAo/CLsUnhBjBD7E99nmQz/leLWSl82sSYDkO3n
-    Uk3/1qJZA8iddb4uH0IEQWcNKev3WoQQzwiVrXBiftlRQOJy5JJXm5m8229MCpMA
-    1AUWmpdu6sl3/gFFdsDhUFq/a7LFrVyaUCMRIHg9szAB7ZFkixr9umQs8jKwuo98
-    3JHB11h2SirwgfIzHHmyhaWhCt22ucTwEXGhq63LtrzZvLsfP8Ql5S+AuqGTH0v8
-    meuc784leAjulBZjkpuIFwDnVv9+YeUEbqJeo1hSHrILddora3nkH4E2dJWmLpqp
-    iPr++GRi+BNgYKW/BQLTJ7C6v+vUs+kdPgYJH5z7oP6f0YZkT0Wkubp/UEz7UV2d
-    fjz57d77DYx5rFWGYzJriWR/xltgL1zDpjwjwG1FDpRqwlyYbBFpjQhxI+X0aT98
-    m6fCzBDQHDb/+JgvsjTHh6OZatahFAwzFIEfrceDv1BG8sBWIaZGhLzYiWQxafl8
-    oXbWv1T6I1jpsTlCdCSkWzaJb4ZjxI9Ga1ynVu8F16+GR2a71wKWu7UbZQsCAwEA
-    AQKCAgBYvrs4FLoD3KNqahRIDqhaQEVKjtn1Yn2dBy9tAXwsg2qI34fE7nnWLwsY
-    +o56U0gmKQ57BOhV36Uqg8JNP0BBjI2wpA19simCrsa2fgAMznzmUpHWHV+KuT5K
-    TJ9OGt2oUpdKQtOASLc0r/neiTZNkf29iTyQLzf7zj4f/qGSYpXRXsnP0F5KJmGH
-    z6agujWckQnSB4eCk9gFsCb+akubyE8K8Kw8w6lajrVl2czBB7SnUj5UnCTeH62k
-    M8goP08Is6QppON8BFDm6bLfRPSe9yIPzu9JhGz2unp+mwkz872Zz1P9yUOieM4U
-    9g4ZFQkPQx1ZpfynUm3pJZ/uhzadBabnIvMe/1qwDAEDifh/WzEM76/2kBpQkHtS
-    qcjwjAElfWnP8aBr1Pj42/cVJy3dbDqb0OawFHx/8xSO2CkY4Gq2h3OYv1XpPv3g
-    S9qqKhvuaT+aD0YjKhP4FYc2vvQSJwdZL8vqOyma8JGmc+r7jakIPCyOx3oPVqnS
-    L2P7DuJ1FcGIZyYOU3UUSzKndDU9fVC8YoLWvHDlwm4RK9UPtdsBY8mEu6BlaAwL
-    zEQG+fbcFnEkHPiJeAohYUCHiqCihLt0pqGwZi+QrudPQE6C47YijGZWJu4VVLjB
-    B2L9iDQKsN4FnBJ9egJIwWBLX3XXQfjC43UGm1A5sBvD+ScsCQKCAQEA7GxU7/SW
-    4YJ+wBXrp7Z3vzlc5mTT5U4L2muWZLhIjT/jmpHpZ4c9a5DY/K9OYcu8XJ+7kx2B
-    N40cU3ZkT2ZbB5/BUCEmi3Wzy3R/KZshHDzvvSZHcXJqVBtv+HGJgR5ssFqAw8c6
-    gJtDls+JE9Sz+nhLk0ZZ4658vbTQfG1lmtzrbC3Kz2xK8RPTdOU5Or7fayeaEKEW
-    ECBJPE41ME2UTdB/E85vyYoee0MBijjAs19QKqvoNbyrsZ5bihcIDYsrvjCmkdW1
-    20IUrSF3ZYJ9bb+CxHeRyNqwvRxPYSkzdMjZHx+xEAvJgw51QqmIi2QQf/qB+ych
-    cSbE/0Jhx4QbDQKCAQEAzAoenEOgmZvUegFU