diff options
author | Aaron Andersen <aaron@fosslib.net> | 2019-06-13 20:37:10 -0400 |
---|---|---|
committer | GitHub <noreply@github.com> | 2019-06-13 20:37:10 -0400 |
commit | fadceeb07589c901e96cd565a87a7bc68fa9e891 (patch) | |
tree | 38b52374a62fde1c47eb79034ca1a6a07564c27c /nixos | |
parent | 7815c86c104a99417db844791dcda34fe7a7965f (diff) | |
parent | 03503121da3305b03b17d9f4fbfe99258cfd195f (diff) |
Merge pull request #59179 from JohnAZoidberg/cassandra-module
Fix Cassandra, improve config and tests
Diffstat (limited to 'nixos')
-rw-r--r-- | nixos/modules/services/databases/cassandra.nix | 210 | ||||
-rw-r--r-- | nixos/tests/all-tests.nix | 1 | ||||
-rw-r--r-- | nixos/tests/cassandra.nix | 96 |
3 files changed, 261 insertions, 46 deletions
diff --git a/nixos/modules/services/databases/cassandra.nix b/nixos/modules/services/databases/cassandra.nix index 688938868020..fc5812c49df0 100644 --- a/nixos/modules/services/databases/cassandra.nix +++ b/nixos/modules/services/databases/cassandra.nix @@ -8,18 +8,21 @@ let cassandraConfig = flip recursiveUpdate cfg.extraConfig ({ commitlog_sync = "batch"; commitlog_sync_batch_window_in_ms = 2; + start_native_transport = cfg.allowClients; + cluster_name = cfg.clusterName; partitioner = "org.apache.cassandra.dht.Murmur3Partitioner"; endpoint_snitch = "SimpleSnitch"; - seed_provider = - [{ class_name = "org.apache.cassandra.locator.SimpleSeedProvider"; - parameters = [ { seeds = "127.0.0.1"; } ]; - }]; data_file_directories = [ "${cfg.homeDir}/data" ]; commitlog_directory = "${cfg.homeDir}/commitlog"; saved_caches_directory = "${cfg.homeDir}/saved_caches"; - } // (if builtins.compareVersions cfg.package.version "3" >= 0 - then { hints_directory = "${cfg.homeDir}/hints"; } - else {}) + } // (lib.optionalAttrs (cfg.seedAddresses != []) { + seed_provider = [{ + class_name = "org.apache.cassandra.locator.SimpleSeedProvider"; + parameters = [ { seeds = concatStringsSep "," cfg.seedAddresses; } ]; + }]; + }) // (lib.optionalAttrs (lib.versionAtLeast cfg.package.version "3") { + hints_directory = "${cfg.homeDir}/hints"; + }) ); cassandraConfigWithAddresses = cassandraConfig // ( if cfg.listenAddress == null @@ -39,15 +42,42 @@ let mkdir -p "$out" echo "$cassandraYaml" > "$out/cassandra.yaml" - ln -s "$cassandraEnvPkg" "$out/cassandra-env.sh" ln -s "$cassandraLogbackConfig" "$out/logback.xml" + + cp "$cassandraEnvPkg" "$out/cassandra-env.sh" + + # Delete default JMX Port, otherwise we can't set it using env variable + sed -i '/JMX_PORT="7199"/d' "$out/cassandra-env.sh" + + # Delete default password file + sed -i '/-Dcom.sun.management.jmxremote.password.file=\/etc\/cassandra\/jmxremote.password/d' "$out/cassandra-env.sh" ''; }; + defaultJmxRolesFile = builtins.foldl' + (left: right: left + right) "" + (map (role: "${role.username} ${role.password}") cfg.jmxRoles); + fullJvmOptions = cfg.jvmOpts + ++ lib.optionals (cfg.jmxRoles != []) [ + "-Dcom.sun.management.jmxremote.authenticate=true" + "-Dcom.sun.management.jmxremote.password.file=${cfg.jmxRolesFile}" + ] + ++ lib.optionals cfg.remoteJmx [ + "-Djava.rmi.server.hostname=${cfg.rpcAddress}" + ]; in { options.services.cassandra = { enable = mkEnableOption '' Apache Cassandra – Scalable and highly available database. ''; + clusterName = mkOption { + type = types.str; + default = "NixOS Test Cluster"; + description = '' + The name of the cluster. + This setting prevents nodes in one logical cluster from joining + another. All nodes in a cluster must have the same value. + ''; + }; user = mkOption { type = types.str; default = defaultUser; @@ -162,6 +192,28 @@ in { XML logback configuration for cassandra ''; }; + seedAddresses = mkOption { + type = types.listOf types.str; + default = [ "127.0.0.1" ]; + description = '' + The addresses of hosts designated as contact points in the cluster. A + joining node contacts one of the nodes in the seeds list to learn the + topology of the ring. + Set to 127.0.0.1 for a single node cluster. + ''; + }; + allowClients = mkOption { + type = types.bool; + default = true; + description = '' + Enables or disables the native transport server (CQL binary protocol). + This server uses the same address as the <literal>rpcAddress</literal>, + but the port it uses is not <literal>rpc_port</literal> but + <literal>native_transport_port</literal>. See the official Cassandra + docs for more information on these variables and set them using + <literal>extraConfig</literal>. + ''; + }; extraConfig = mkOption { type = types.attrs; default = {}; @@ -178,11 +230,11 @@ in { example = literalExample "null"; description = '' Set the interval how often full repairs are run, i.e. - `nodetool repair --full` is executed. See + <literal>nodetool repair --full</literal> is executed. See https://cassandra.apache.org/doc/latest/operating/repair.html for more information. - Set to `null` to disable full repairs. + Set to <literal>null</literal> to disable full repairs. ''; }; fullRepairOptions = mkOption { @@ -199,11 +251,11 @@ in { example = literalExample "null"; description = '' Set the interval how often incremental repairs are run, i.e. - `nodetool repair` is executed. See + <literal>nodetool repair<literal> is executed. See https://cassandra.apache.org/doc/latest/operating/repair.html for more information. - Set to `null` to disable incremental repairs. + Set to <literal>null</literal> to disable incremental repairs. ''; }; incrementalRepairOptions = mkOption { @@ -214,20 +266,135 @@ in { Options passed through to the incremental repair command. ''; }; + maxHeapSize = mkOption { + type = types.nullOr types.string; + default = null; + example = "4G"; + description = '' + Must be left blank or set together with heapNewSize. + If left blank a sensible value for the available amount of RAM and CPU + cores is calculated. + + Override to set the amount of memory to allocate to the JVM at + start-up. For production use you may wish to adjust this for your + environment. MAX_HEAP_SIZE is the total amount of memory dedicated + to the Java heap. HEAP_NEWSIZE refers to the size of the young + generation. + + The main trade-off for the young generation is that the larger it + is, the longer GC pause times will be. The shorter it is, the more + expensive GC will be (usually). + ''; + }; + heapNewSize = mkOption { + type = types.nullOr types.string; + default = null; + example = "800M"; + description = '' + Must be left blank or set together with heapNewSize. + If left blank a sensible value for the available amount of RAM and CPU + cores is calculated. + + Override to set the amount of memory to allocate to the JVM at + start-up. For production use you may wish to adjust this for your + environment. HEAP_NEWSIZE refers to the size of the young + generation. + + The main trade-off for the young generation is that the larger it + is, the longer GC pause times will be. The shorter it is, the more + expensive GC will be (usually). + + The example HEAP_NEWSIZE assumes a modern 8-core+ machine for decent pause + times. If in doubt, and if you do not particularly want to tweak, go with + 100 MB per physical CPU core. + ''; + }; + mallocArenaMax = mkOption { + type = types.nullOr types.int; + default = null; + example = 4; + description = '' + Set this to control the amount of arenas per-thread in glibc. + ''; + }; + remoteJmx = mkOption { + type = types.bool; + default = false; + description = '' + Cassandra ships with JMX accessible *only* from localhost. + To enable remote JMX connections set to true. + + Be sure to also enable authentication and/or TLS. + See: https://wiki.apache.org/cassandra/JmxSecurity + ''; + }; + jmxPort = mkOption { + type = types.int; + default = 7199; + description = '' + Specifies the default port over which Cassandra will be available for + JMX connections. + For security reasons, you should not expose this port to the internet. + Firewall it if needed. + ''; + }; + jmxRoles = mkOption { + default = []; + description = '' + Roles that are allowed to access the JMX (e.g. nodetool) + BEWARE: The passwords will be stored world readable in the nix-store. + It's recommended to use your own protected file using + <literal>jmxRolesFile</literal> + + Doesn't work in versions older than 3.11 because they don't like that + it's world readable. + ''; + type = types.listOf (types.submodule { + options = { + username = mkOption { + type = types.string; + description = "Username for JMX"; + }; + password = mkOption { + type = types.string; + description = "Password for JMX"; + }; + }; + }); + }; + jmxRolesFile = mkOption { + type = types.nullOr types.path; + default = if (lib.versionAtLeast cfg.package.version "3.11") + then pkgs.writeText "jmx-roles-file" defaultJmxRolesFile + else null; + example = "/var/lib/cassandra/jmx.password"; + description = '' + Specify your own jmx roles file. + + Make sure the permissions forbid "others" from reading the file if + you're using Cassandra below version 3.11. + ''; + }; }; config = mkIf cfg.enable { assertions = - [ { assertion = - (cfg.listenAddress == null || cfg.listenInterface == null) - && !(cfg.listenAddress == null && cfg.listenInterface == null); + [ { assertion = (cfg.listenAddress == null) != (cfg.listenInterface == null); message = "You have to set either listenAddress or listenInterface"; } - { assertion = - (cfg.rpcAddress == null || cfg.rpcInterface == null) - && !(cfg.rpcAddress == null && cfg.rpcInterface == null); + { assertion = (cfg.rpcAddress == null) != (cfg.rpcInterface == null); message = "You have to set either rpcAddress or rpcInterface"; } + { assertion = (cfg.maxHeapSize == null) == (cfg.heapNewSize == null); + message = "If you set either of maxHeapSize or heapNewSize you have to set both"; + } + { assertion = cfg.remoteJmx -> cfg.jmxRolesFile != null; + message = '' + If you want JMX available remotely you need to set a password using + <literal>jmxRoles</literal> or <literal>jmxRolesFile</literal> if + using Cassandra older than v3.11. + ''; + } ]; users = mkIf (cfg.user == defaultUser) { extraUsers."${defaultUser}" = @@ -245,7 +412,12 @@ in { after = [ "network.target" ]; environment = { CASSANDRA_CONF = "${cassandraEtc}"; - JVM_OPTS = builtins.concatStringsSep " " cfg.jvmOpts; + JVM_OPTS = builtins.concatStringsSep " " fullJvmOptions; + MAX_HEAP_SIZE = toString cfg.maxHeapSize; + HEAP_NEWSIZE = toString cfg.heapNewSize; + MALLOC_ARENA_MAX = toString cfg.mallocArenaMax; + LOCAL_JMX = if cfg.remoteJmx then "no" else "yes"; + JMX_PORT = toString cfg.jmxPort; }; wantedBy = [ "multi-user.target" ]; serviceConfig = diff --git a/nixos/tests/all-tests.nix b/nixos/tests/all-tests.nix index 8b38e802e62e..912e4f552313 100644 --- a/nixos/tests/all-tests.nix +++ b/nixos/tests/all-tests.nix @@ -36,6 +36,7 @@ in borgbackup = handleTest ./borgbackup.nix {}; buildbot = handleTest ./buildbot.nix {}; cadvisor = handleTestOn ["x86_64-linux"] ./cadvisor.nix {}; + cassandra = handleTest ./cassandra.nix {}; ceph = handleTestOn ["x86_64-linux"] ./ceph.nix {}; certmgr = handleTest ./certmgr.nix {}; cfssl = handleTestOn ["x86_64-linux"] ./cfssl.nix {}; diff --git a/nixos/tests/cassandra.nix b/nixos/tests/cassandra.nix index 60d0c6d76068..aea4fa4d1c95 100644 --- a/nixos/tests/cassandra.nix +++ b/nixos/tests/cassandra.nix @@ -1,26 +1,43 @@ -import ./make-test.nix ({ pkgs, ...}: +import ./make-test.nix ({ pkgs, lib, ... }: let # Change this to test a different version of Cassandra: testPackage = pkgs.cassandra; - cassandraCfg = + clusterName = "NixOS Automated-Test Cluster"; + + testRemoteAuth = lib.versionAtLeast testPackage.version "3.11"; + jmxRoles = [{ username = "me"; password = "password"; }]; + jmxRolesFile = ./cassandra-jmx-roles; + jmxAuthArgs = "-u ${(builtins.elemAt jmxRoles 0).username} -pw ${(builtins.elemAt jmxRoles 0).password}"; + + # Would usually be assigned to 512M + numMaxHeapSize = "400"; + getHeapLimitCommand = '' + nodetool info | grep "^Heap Memory" | awk \'{print $NF}\' + ''; + checkHeapLimitCommand = '' + [ 1 -eq "$(echo "$(${getHeapLimitCommand}) < ${numMaxHeapSize}" | ${pkgs.bc}/bin/bc)" ] + ''; + + cassandraCfg = ipAddress: { enable = true; - listenAddress = null; - listenInterface = "eth1"; - rpcAddress = null; - rpcInterface = "eth1"; - extraConfig = - { start_native_transport = true; - seed_provider = - [{ class_name = "org.apache.cassandra.locator.SimpleSeedProvider"; - parameters = [ { seeds = "cass0"; } ]; - }]; - }; + inherit clusterName; + listenAddress = ipAddress; + rpcAddress = ipAddress; + seedAddresses = [ "192.168.1.1" ]; package = testPackage; + maxHeapSize = "${numMaxHeapSize}M"; + heapNewSize = "100M"; }; - nodeCfg = extra: {pkgs, config, ...}: + nodeCfg = ipAddress: extra: {pkgs, config, ...}: { environment.systemPackages = [ testPackage ]; - networking.firewall.enable = false; - services.cassandra = cassandraCfg // extra; + networking = { + firewall.allowedTCPPorts = [ 7000 7199 9042 ]; + useDHCP = false; + interfaces.eth1.ipv4.addresses = pkgs.lib.mkOverride 0 [ + { address = ipAddress; prefixLength = 24; } + ]; + }; + services.cassandra = cassandraCfg ipAddress // extra; virtualisation.memorySize = 1024; }; in @@ -28,40 +45,65 @@ in name = "cassandra-ci"; nodes = { - cass0 = nodeCfg {}; - cass1 = nodeCfg {}; - cass2 = nodeCfg { jvmOpts = [ "-Dcassandra.replace_address=cass1" ]; }; + cass0 = nodeCfg "192.168.1.1" {}; + cass1 = nodeCfg "192.168.1.2" (lib.optionalAttrs testRemoteAuth { inherit jmxRoles; remoteJmx = true; }); + cass2 = nodeCfg "192.168.1.3" { jvmOpts = [ "-Dcassandra.replace_address=cass1" ]; }; }; testScript = '' - subtest "timers exist", sub { + # Check configuration + subtest "Timers exist", sub { $cass0->succeed("systemctl list-timers | grep cassandra-full-repair.timer"); $cass0->succeed("systemctl list-timers | grep cassandra-incremental-repair.timer"); }; - subtest "can connect via cqlsh", sub { + subtest "Can connect via cqlsh", sub { $cass0->waitForUnit("cassandra.service"); $cass0->waitUntilSucceeds("nc -z cass0 9042"); $cass0->succeed("echo 'show version;' | cqlsh cass0"); }; - subtest "nodetool is operational", sub { + subtest "Nodetool is operational", sub { $cass0->waitForUnit("cassandra.service"); $cass0->waitUntilSucceeds("nc -z localhost 7199"); $cass0->succeed("nodetool status --resolve-ip | egrep '^UN[[:space:]]+cass0'"); }; - subtest "bring up cluster", sub { + subtest "Cluster name was set", sub { + $cass0->waitForUnit("cassandra.service"); + $cass0->waitUntilSucceeds("nc -z localhost 7199"); + $cass0->waitUntilSucceeds("nodetool describecluster | grep 'Name: ${clusterName}'"); + }; + subtest "Heap limit set correctly", sub { + # Nodetool takes a while until it can display info + $cass0->waitUntilSucceeds('nodetool info'); + $cass0->succeed('${checkHeapLimitCommand}'); + }; + + # Check cluster interaction + subtest "Bring up cluster", sub { $cass1->waitForUnit("cassandra.service"); - $cass1->waitUntilSucceeds("nodetool status | egrep -c '^UN' | grep 2"); + $cass1->waitUntilSucceeds("nodetool ${jmxAuthArgs} status | egrep -c '^UN' | grep 2"); $cass0->succeed("nodetool status --resolve-ip | egrep '^UN[[:space:]]+cass1'"); }; - subtest "break and fix node", sub { + '' + lib.optionalString testRemoteAuth '' + subtest "Remote authenticated jmx", sub { + # Doesn't work if not enabled + $cass0->waitUntilSucceeds("nc -z localhost 7199"); + $cass1->fail("nc -z 192.168.1.1 7199"); + $cass1->fail("nodetool -h 192.168.1.1 status"); + + # Works if enabled + $cass1->waitUntilSucceeds("nc -z localhost 7199"); + $cass0->succeed("nodetool -h 192.168.1.2 ${jmxAuthArgs} status"); + }; + '' + '' + subtest "Break and fix node", sub { $cass1->block; $cass0->waitUntilSucceeds("nodetool status --resolve-ip | egrep -c '^DN[[:space:]]+cass1'"); $cass0->succeed("nodetool status | egrep -c '^UN' | grep 1"); $cass1->unblock; - $cass1->waitUntilSucceeds("nodetool status | egrep -c '^UN' | grep 2"); + $cass1->waitUntilSucceeds("nodetool ${jmxAuthArgs} status | egrep -c '^UN' | grep 2"); $cass0->succeed("nodetool status | egrep -c '^UN' | grep 2"); }; - subtest "replace crashed node", sub { + subtest "Replace crashed node", sub { $cass1->crash; $cass2->waitForUnit("cassandra.service"); $cass0->waitUntilFails("nodetool status --resolve-ip | egrep '^UN[[:space:]]+cass1'"); |