k3s: streamline HA setup

- Replace misleading docs.
- Add new assertions to let configurations make more sense.
- Add clusterInit flag.
- Add some more docs about HA and non-HA modes setup.
- Improve multi-node tests for HA mode.

Fix https://github.com/NixOS/nixpkgs/issues/182085

1 files changed, 52 insertions, 4 deletions

diff --git a/nixos/modules/services/cluster/k3s/default.nix b/nixos/modules/services/cluster/k3s/default.nix
index a1f6d4ecdfad..d0269fde8936 100644
--- a/nixos/modules/services/cluster/k3s/default.nix
+++ b/nixos/modules/services/cluster/k3s/default.nix
@@ -25,7 +25,17 @@ in
     role = mkOption {
       description = lib.mdDoc ''
         Whether k3s should run as a server or agent.
-        Note that the server, by default, also runs as an agent.
+
+        If it's a server:
+
+        - By default it also runs workloads as an agent.
+        - Starts by default as a standalone server using an embedded sqlite datastore.
+        - Configure `clusterInit = true` to switch over to embedded etcd datastore and enable HA mode.
+        - Configure `serverAddr` to join an already-initialized HA cluster.
+
+        If it's an agent:
+
+        - `serverAddr` is required.
       '';
       default = "server";
       type = types.enum [ "server" "agent" ];
@@ -33,15 +43,44 @@ in
 
     serverAddr = mkOption {
       type = types.str;
-      description = lib.mdDoc "The k3s server to connect to. This option only makes sense for an agent.";
+      description = lib.mdDoc ''
+        The k3s server to connect to.
+
+        Servers and agents need to communicate each other. Read
+        [the networking docs](https://rancher.com/docs/k3s/latest/en/installation/installation-requirements/#networking)
+        to know how to configure the firewall.
+      '';
       example = "https://10.0.0.10:6443";
       default = "";
     };
 
+    clusterInit = mkOption {
+      type = types.bool;
+      default = false;
+      description = lib.mdDoc ''
+        Initialize HA cluster using an embedded etcd datastore.
+
+        If this option is `false` and `role` is `server`
+
+        On a server that was using the default embedded sqlite backend,
+        enabling this option will migrate to an embedded etcd DB.
+
+        If an HA cluster using the embedded etcd datastore was already initialized,
+        this option has no effect.
+
+        This option only makes sense in a server that is not connecting to another server.
+
+        If you are configuring an HA cluster with an embedded etcd,
+        the 1st server must have `clusterInit = true`
+        and other servers must connect to it using `serverAddr`.
+      '';
+    };
+
     token = mkOption {
       type = types.str;
       description = lib.mdDoc ''
-        The k3s token to use when connecting to the server. This option only makes sense for an agent.
+        The k3s token to use when connecting to a server.
+
         WARNING: This option will expose store your token unencrypted world-readable in the nix store.
         If this is undesired use the tokenFile option instead.
       '';
@@ -50,7 +89,7 @@ in
 
     tokenFile = mkOption {
       type = types.nullOr types.path;
-      description = lib.mdDoc "File path containing k3s token to use when connecting to the server. This option only makes sense for an agent.";
+      description = lib.mdDoc "File path containing k3s token to use when connecting to the server.";
       default = null;
     };
 
@@ -86,6 +125,14 @@ in
         assertion = cfg.role == "agent" -> cfg.configPath != null || cfg.tokenFile != null || cfg.token != "";
         message = "token or tokenFile or configPath (with 'token' or 'token-file' keys) should be set if role is 'agent'";
       }
+      {
+        assertion = cfg.role == "agent" -> !cfg.disableAgent;
+        message = "disableAgent must be false if role is 'agent'";
+      }
+      {
+        assertion = cfg.role == "agent" -> !cfg.clusterInit;
+        message = "clusterInit must be false if role is 'agent'";
+      }
     ];
 
     environment.systemPackages = [ config.services.k3s.package ];
@@ -111,6 +158,7 @@ in
           [
             "${cfg.package}/bin/k3s ${cfg.role}"
           ]
+          ++ (optional cfg.clusterInit "--cluster-init")
           ++ (optional cfg.disableAgent "--disable-agent")
           ++ (optional (cfg.serverAddr != "") "--server ${cfg.serverAddr}")
           ++ (optional (cfg.token != "") "--token ${cfg.token}")