diff options
| -rw-r--r-- | nixos/modules/services/network-filesystems/ceph.nix | 19 | ||||
| -rw-r--r-- | nixos/tests/ceph-multi-node.nix | 26 | ||||
| -rw-r--r-- | nixos/tests/ceph-single-node.nix | 14 |
3 files changed, 27 insertions, 32 deletions
diff --git a/nixos/modules/services/network-filesystems/ceph.nix b/nixos/modules/services/network-filesystems/ceph.nix index ef0d2d90a1a1..543a7b25d5d6 100644 --- a/nixos/modules/services/network-filesystems/ceph.nix +++ b/nixos/modules/services/network-filesystems/ceph.nix @@ -14,7 +14,9 @@ let { "ceph-${daemonType}-${daemonId}" = makeService daemonType daemonId cfg.global.clusterName pkgs.ceph; }) daemonIds)); - makeService = (daemonType: daemonId: clusterName: ceph: { + makeService = (daemonType: daemonId: clusterName: ceph: + let + stateDirectory = "ceph/${if daemonType == "rgw" then "radosgw" else daemonType}/${clusterName}-${daemonId}"; in { enable = true; description = "Ceph ${builtins.replaceStrings lowerChars upperChars daemonType} daemon ${daemonId}"; after = [ "network-online.target" "time-sync.target" ] ++ optional (daemonType == "osd") "ceph-mon.target"; @@ -24,6 +26,9 @@ let path = [ pkgs.getopt ]; + # Don't start services that are not yet initialized + unitConfig.ConditionPathExists = "/var/lib/${stateDirectory}/keyring"; + serviceConfig = { LimitNOFILE = 1048576; LimitNPROC = 1048576; @@ -36,7 +41,7 @@ let Restart = "on-failure"; StartLimitBurst = "5"; StartLimitInterval = "30min"; - StateDirectory = "ceph/${if daemonType == "rgw" then "radosgw" else daemonType}/${clusterName}-${daemonId}"; + StateDirectory = stateDirectory; User = "ceph"; Group = if daemonType == "osd" then "disk" else "ceph"; ExecStart = ''${ceph.out}/bin/${if daemonType == "rgw" then "radosgw" else "ceph-${daemonType}"} \ @@ -60,6 +65,7 @@ let partOf = [ "ceph.target" ]; wantedBy = [ "ceph.target" ]; before = [ "ceph.target" ]; + unitConfig.StopWhenUnneeded = true; }; } ); @@ -389,9 +395,12 @@ in systemd.targets = let targets = [ - { ceph = { description = "Ceph target allowing to start/stop all ceph service instances at once"; - wantedBy = [ "multi-user.target" ]; }; } - ] ++ optional cfg.mon.enable (makeTarget "mon") + { ceph = { + description = "Ceph target allowing to start/stop all ceph service instances at once"; + wantedBy = [ "multi-user.target" ]; + unitConfig.StopWhenUnneeded = true; + }; } ] + ++ optional cfg.mon.enable (makeTarget "mon") ++ optional cfg.mds.enable (makeTarget "mds") ++ optional cfg.osd.enable (makeTarget "osd") ++ optional cfg.rgw.enable (makeTarget "rgw") diff --git a/nixos/tests/ceph-multi-node.nix b/nixos/tests/ceph-multi-node.nix index c34b134de531..09a81ae25e3d 100644 --- a/nixos/tests/ceph-multi-node.nix +++ b/nixos/tests/ceph-multi-node.nix @@ -49,9 +49,6 @@ let boot.kernelModules = [ "xfs" ]; services.ceph = cephConfig; - - # So that we don't have to battle systemd when bootstraping - systemd.targets.ceph.wantedBy = lib.mkForce []; }; networkMonA = { @@ -191,22 +188,17 @@ let "ceph osd pool delete multi-node-other-test multi-node-other-test --yes-i-really-really-mean-it" ); - # As we disable the target in the config, we still want to test that it works as intended - $osd0->mustSucceed("systemctl stop ceph-osd-${cfg.osd0.name}"); - $osd1->mustSucceed("systemctl stop ceph-osd-${cfg.osd1.name}"); - $monA->mustSucceed( - "systemctl stop ceph-mgr-${cfg.monA.name}", - "systemctl stop ceph-mon-${cfg.monA.name}" - ); + # Shut down ceph on all machines in a very unpolite way + $monA->crash; + $osd0->crash; + $osd1->crash; - $monA->succeed("systemctl start ceph.target"); - $monA->waitForUnit("ceph-mon-${cfg.monA.name}"); - $monA->waitForUnit("ceph-mgr-${cfg.monA.name}"); - $osd0->succeed("systemctl start ceph.target"); - $osd0->waitForUnit("ceph-osd-${cfg.osd0.name}"); - $osd1->succeed("systemctl start ceph.target"); - $osd1->waitForUnit("ceph-osd-${cfg.osd1.name}"); + # Start it up + $osd0->start; + $osd1->start; + $monA->start; + # Ensure the cluster comes back up again $monA->succeed("ceph -s | grep 'mon: 1 daemons'"); $monA->waitUntilSucceeds("ceph -s | grep 'quorum ${cfg.monA.name}'"); $monA->waitUntilSucceeds("ceph osd stat | grep -e '2 osds: 2 up[^,]*, 2 in'"); diff --git a/nixos/tests/ceph-single-node.nix b/nixos/tests/ceph-single-node.nix index 5938300ebb4a..3278d76769a4 100644 --- a/nixos/tests/ceph-single-node.nix +++ b/nixos/tests/ceph-single-node.nix @@ -46,9 +46,6 @@ let boot.kernelModules = [ "xfs" ]; services.ceph = cephConfig; - - # So that we don't have to battle systemd when bootstraping - systemd.targets.ceph.wantedBy = lib.mkForce []; }; networkMonA = { @@ -151,20 +148,17 @@ let "ceph osd pool delete single-node-other-test single-node-other-test --yes-i-really-really-mean-it" ); - # As we disable the target in the config, we still want to test that it works as intended - $monA->mustSucceed( - "systemctl stop ceph-osd-${cfg.osd0.name}", - "systemctl stop ceph-osd-${cfg.osd1.name}", - "systemctl stop ceph-mgr-${cfg.monA.name}", - "systemctl stop ceph-mon-${cfg.monA.name}" - ); + # Shut down ceph by stopping ceph.target. + $monA->mustSucceed("systemctl stop ceph.target"); + # Start it up $monA->succeed("systemctl start ceph.target"); $monA->waitForUnit("ceph-mon-${cfg.monA.name}"); $monA->waitForUnit("ceph-mgr-${cfg.monA.name}"); $monA->waitForUnit("ceph-osd-${cfg.osd0.name}"); $monA->waitForUnit("ceph-osd-${cfg.osd1.name}"); + # Ensure the cluster comes back up again $monA->succeed("ceph -s | grep 'mon: 1 daemons'"); $monA->waitUntilSucceeds("ceph -s | grep 'quorum ${cfg.monA.name}'"); $monA->waitUntilSucceeds("ceph osd stat | grep -e '2 osds: 2 up[^,]*, 2 in'"); |