summaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
Diffstat
-rw-r--r--nixos/doc/manual/release-notes/rl-1909.xml9
-rw-r--r--nixos/modules/rename.nix3
-rw-r--r--nixos/modules/services/monitoring/prometheus/default.nix544
-rw-r--r--nixos/modules/services/monitoring/thanos.nix16
-rw-r--r--nixos/tests/all-tests.nix1
-rw-r--r--nixos/tests/prometheus-2.nix239
-rw-r--r--nixos/tests/prometheus.nix255
-rw-r--r--pkgs/servers/monitoring/prometheus/default.nix82
-rw-r--r--pkgs/top-level/all-packages.nix7
9 files changed, 434 insertions, 722 deletions
diff --git a/nixos/doc/manual/release-notes/rl-1909.xml b/nixos/doc/manual/release-notes/rl-1909.xml
index e38dd8b12853..ee3b03581529 100644
--- a/nixos/doc/manual/release-notes/rl-1909.xml
+++ b/nixos/doc/manual/release-notes/rl-1909.xml
@@ -464,7 +464,14 @@
<literal>packetbeat5</literal>) of the ELK-stack and Elastic beats have been removed.
</para>
</listitem>
-
+ <listitem>
+ <para>
+ For NixOS 19.03, both Prometheus 1 and 2 were available to allow for
+ a seamless transition from version 1 to 2 with existing setups.
+ Because Prometheus 1 is no longer developed, it was removed.
+ Prometheus 2 is now configured with <literal>services.prometheus</literal>.
+ </para>
+ </listitem>
</itemizedlist>
</section>
diff --git a/nixos/modules/rename.nix b/nixos/modules/rename.nix
index 1fa91f05030d..d1303f90ad8d 100644
--- a/nixos/modules/rename.nix
+++ b/nixos/modules/rename.nix
@@ -52,10 +52,11 @@ with lib;
(mkRemovedOptionModule [ "services" "misc" "nzbget" "openFirewall" ] "The port used by nzbget is managed through the web interface so you should adjust your firewall rules accordingly.")
(mkRemovedOptionModule [ "services" "prometheus" "alertmanager" "user" ] "The alertmanager service is now using systemd's DynamicUser mechanism which obviates a user setting.")
(mkRemovedOptionModule [ "services" "prometheus" "alertmanager" "group" ] "The alertmanager service is now using systemd's DynamicUser mechanism which obviates a group setting.")
- (mkRemovedOptionModule [ "services" "prometheus2" "alertmanagerURL" ] ''
+ (mkRemovedOptionModule [ "services" "prometheus" "alertmanagerURL" ] ''
Due to incompatibility, the alertmanagerURL option has been removed,
please use 'services.prometheus2.alertmanagers' instead.
'')
+ (mkRenamedOptionModule [ "services" "prometheus2" ] [ "services" "prometheus" ])
(mkRenamedOptionModule [ "services" "tor" "relay" "portSpec" ] [ "services" "tor" "relay" "port" ])
(mkRenamedOptionModule [ "services" "vmwareGuest" ] [ "virtualisation" "vmware" "guest" ])
(mkRenamedOptionModule [ "jobs" ] [ "systemd" "services" ])
diff --git a/nixos/modules/services/monitoring/prometheus/default.nix b/nixos/modules/services/monitoring/prometheus/default.nix
index 647d67533b89..191c0bff9c84 100644
--- a/nixos/modules/services/monitoring/prometheus/default.nix
+++ b/nixos/modules/services/monitoring/prometheus/default.nix
@@ -4,37 +4,14 @@ with lib;
let
cfg = config.services.prometheus;
- cfg2 = config.services.prometheus2;
- promUser = "prometheus";
- promGroup = "prometheus";
-
- stateDir =
- if cfg.stateDir != null
- then cfg.stateDir
- else
- if cfg.dataDir != null
- then
- # This assumes /var/lib/ is a prefix of cfg.dataDir.
- # This is checked as an assertion below.
- removePrefix stateDirBase cfg.dataDir
- else "prometheus";
- stateDirBase = "/var/lib/";
- workingDir = stateDirBase + stateDir;
- workingDir2 = stateDirBase + cfg2.stateDir;
- # a wrapper that verifies that the configuration is valid
- promtoolCheck = what: name: file: pkgs.runCommand "${name}-${what}-checked"
- { buildInputs = [ cfg.package ]; } ''
- ln -s ${file} $out
- promtool ${what} $out
- '';
+ workingDir = "/var/lib/" + cfg.stateDir;
- # a wrapper that verifies that the configuration is valid for
- # prometheus 2
- prom2toolCheck = what: name: file:
+ # a wrapper that verifies that the configuration is valid
+ promtoolCheck = what: name: file:
pkgs.runCommand
"${name}-${replaceStrings [" "] [""] what}-checked"
- { buildInputs = [ cfg2.package ]; } ''
+ { buildInputs = [ cfg.package ]; } ''
ln -s ${file} $out
promtool ${what} $out
'';
@@ -45,61 +22,34 @@ let
echo '${builtins.toJSON x}' | ${pkgs.jq}/bin/jq . > $out
'';
- # This becomes the main config file for Prometheus 1
+ generatedPrometheusYml = writePrettyJSON "prometheus.yml" promConfig;
+
+ # This becomes the main config file for Prometheus
promConfig = {
global = filterValidPrometheus cfg.globalConfig;
- rule_files = map (promtoolCheck "check-rules" "rules") (cfg.ruleFiles ++ [
+ rule_files = map (promtoolCheck "check rules" "rules") (cfg.ruleFiles ++ [
(pkgs.writeText "prometheus.rules" (concatStringsSep "\n" cfg.rules))
]);
scrape_configs = filterValidPrometheus cfg.scrapeConfigs;
+ alerting = {
+ inherit (cfg) alertmanagers;
+ };
};
- generatedPrometheusYml = writePrettyJSON "prometheus.yml" promConfig;
-
prometheusYml = let
yml = if cfg.configText != null then
pkgs.writeText "prometheus.yml" cfg.configText
else generatedPrometheusYml;
- in promtoolCheck "check-config" "prometheus.yml" yml;
+ in promtoolCheck "check config" "prometheus.yml" yml;
cmdlineArgs = cfg.extraFlags ++ [
- "-storage.local.path=${workingDir}/metrics"
- "-config.file=${prometheusYml}"
- "-web.listen-address=${cfg.listenAddress}"
- "-alertmanager.notification-queue-capacity=${toString cfg.alertmanagerNotificationQueueCapacity}"
- "-alertmanager.timeout=${toString cfg.alertmanagerTimeout}s"
+ "--storage.tsdb.path=${workingDir}/data/"
+ "--config.file=${prometheusYml}"
+ "--web.listen-address=${cfg.listenAddress}"
+ "--alertmanager.notification-queue-capacity=${toString cfg.alertmanagerNotificationQueueCapacity}"
+ "--alertmanager.timeout=${toString cfg.alertmanagerTimeout}s"
] ++
- optional (cfg.alertmanagerURL != []) "-alertmanager.url=${concatStringsSep "," cfg.alertmanagerURL}" ++
- optional (cfg.webExternalUrl != null) "-web.external-url=${cfg.webExternalUrl}";
-
- # This becomes the main config file for Prometheus 2
- promConfig2 = {
- global = filterValidPrometheus cfg2.globalConfig;
- rule_files = map (prom2toolCheck "check rules" "rules") (cfg2.ruleFiles ++ [
- (pkgs.writeText "prometheus.rules" (concatStringsSep "\n" cfg2.rules))
- ]);
- scrape_configs = filterValidPrometheus cfg2.scrapeConfigs;
- alerting = {
- inherit (cfg2) alertmanagers;
- };
- };
-
- generatedPrometheus2Yml = writePrettyJSON "prometheus.yml" promConfig2;
-
- prometheus2Yml = let
- yml = if cfg2.configText != null then
- pkgs.writeText "prometheus.yml" cfg2.configText
- else generatedPrometheus2Yml;
- in prom2toolCheck "check config" "prometheus.yml" yml;
-
- cmdlineArgs2 = cfg2.extraFlags ++ [
- "--storage.tsdb.path=${workingDir2}/data/"
- "--config.file=${prometheus2Yml}"
- "--web.listen-address=${cfg2.listenAddress}"
- "--alertmanager.notification-queue-capacity=${toString cfg2.alertmanagerNotificationQueueCapacity}"
- "--alertmanager.timeout=${toString cfg2.alertmanagerTimeout}s"
- ] ++
- optional (cfg2.webExternalUrl != null) "--web.external-url=${cfg2.webExternalUrl}";
+ optional (cfg.webExternalUrl != null) "--web.external-url=${cfg.webExternalUrl}";
filterValidPrometheus = filterAttrsListRecursive (n: v: !(n == "_module" || v == null));
filterAttrsListRecursive = pred: x:
@@ -514,343 +464,159 @@ let
};
in {
- options = {
- services.prometheus = {
-
- enable = mkOption {
- type = types.bool;
- default = false;
- description = ''
- Enable the Prometheus monitoring daemon.
- '';
- };
-
- package = mkOption {
- type = types.package;
- default = pkgs.prometheus;
- defaultText = "pkgs.prometheus";
- description = ''
- The prometheus package that should be used.
- '';
- };
-
- listenAddress = mkOption {
- type = types.str;
- default = "0.0.0.0:9090";
- description = ''
- Address to listen on for the web interface, API, and telemetry.
- '';
- };
-
- dataDir = mkOption {
- type = types.nullOr types.path;
- default = null;
- description = ''
- Directory to store Prometheus metrics data.
- This option is deprecated, please use <option>services.prometheus.stateDir</option>.
- '';
- };
-
- stateDir = mkOption {
- type = types.nullOr types.str;
- default = null;
- description = ''
- Directory below <literal>${stateDirBase}</literal> to store Prometheus metrics data.
- This directory will be created automatically using systemd's StateDirectory mechanism.
- Defaults to <literal>prometheus</literal>.
- '';
- };
-
- extraFlags = mkOption {
- type = types.listOf types.str;
- default = [];
- description = ''
- Extra commandline options when launching Prometheus.
- '';
- };
-
- configText = mkOption {
- type = types.nullOr types.lines;
- default = null;
- description = ''
- If non-null, this option defines the text that is written to
- prometheus.yml. If null, the contents of prometheus.yml is generated
- from the structured config options.
- '';
- };
-
- globalConfig = mkOption {
- type = promTypes.globalConfig;
- default = {};
- description = ''
- Parameters that are valid in all configuration contexts. They
- also serve as defaults for other configuration sections
- '';
- };
-
- rules = mkOption {
- type = types.listOf types.str;
- default = [];
- description = ''
- Alerting and/or Recording rules to evaluate at runtime.
- '';
- };
-
- ruleFiles = mkOption {
- type = types.listOf types.path;
- default = [];
- description = ''
- Any additional rules files to include in this configuration.
- '';
- };
+ options.services.prometheus = {
- scrapeConfigs = mkOption {
- type = types.listOf promTypes.scrape_config;
- default = [];
- description = ''
- A list of scrape configurations.
- '';
- };
-
- alertmanagerURL = mkOption {
- type = types.listOf types.str;
- default = [];
- description = ''
- List of Alertmanager URLs to send notifications to.
- '';
- };
-
- alertmanagerNotificationQueueCapacity = mkOption {
- type = types.int;
- default = 10000;
- description = ''
- The capacity of the queue for pending alert manager notifications.
- '';
- };
-
- alertmanagerTimeout = mkOption {
- type = types.int;
- default = 10;
- description = ''
- Alert manager HTTP API timeout (in seconds).
- '';
- };
-
- webExternalUrl = mkOption {
- type = types.nullOr types.str;
- default = null;
- example = "https://example.com/";
- description = ''
- The URL under which Prometheus is externally reachable (for example,
- if Prometheus is served via a reverse proxy).
- '';
- };
+ enable = mkOption {
+ type = types.bool;
+ default = false;
+ description = ''
+ Enable the Prometheus monitoring daemon.
+ '';
};
- services.prometheus2 = {
- enable = mkOption {
- type = types.bool;
- default = false;
- description = ''
- Enable the Prometheus 2 monitoring daemon.
- '';
- };
+ package = mkOption {
+ type = types.package;
+ default = pkgs.prometheus;
+ defaultText = "pkgs.prometheus";
+ description = ''
+ The prometheus package that should be used.
+ '';
+ };
- package = mkOption {
- type = types.package;
- default = pkgs.prometheus_2;
- defaultText = "pkgs.prometheus_2";
- description = ''
- The prometheus2 package that should be used.
- '';
- };
+ listenAddress = mkOption {
+ type = types.str;
+ default = "0.0.0.0:9090";
+ description = ''
+ Address to listen on for the web interface, API, and telemetry.
+ '';
+ };
- listenAddress = mkOption {
- type = types.str;
- default = "0.0.0.0:9090";
- description = ''
- Address to listen on for the web interface, API, and telemetry.
- '';
- };
+ stateDir = mkOption {
+ type = types.str;
+ default = "prometheus2";
+ description = ''
+ Directory below <literal>/var/lib</literal> to store Prometheus metrics data.
+ This directory will be created automatically using systemd's StateDirectory mechanism.
+ '';
+ };
- stateDir = mkOption {
- type = types.str;
- default = "prometheus2";
- description = ''
- Directory below <literal>${stateDirBase}</literal> to store Prometheus metrics data.
- This directory will be created automatically using systemd's StateDirectory mechanism.
- Defaults to <literal>prometheus2</literal>.
- '';
- };
+ extraFlags = mkOption {
+ type = types.listOf types.str;
+ default = [];
+ description = ''
+ Extra commandline options when launching Prometheus.
+ '';
+ };
- extraFlags = mkOption {
- type = types.listOf types.str;
- default = [];
- description = ''
- Extra commandline options when launching Prometheus 2.
- '';
- };
+ configText = mkOption {
+ type = types.nullOr types.lines;
+ default = null;
+ description = ''
+ If non-null, this option defines the text that is written to
+ prometheus.yml. If null, the contents of prometheus.yml is generated
+ from the structured config options.
+ '';
+ };
- configText = mkOption {
- type = types.nullOr types.lines;
- default = null;
- description = ''
- If non-null, this option defines the text that is written to
- prometheus.yml. If null, the contents of prometheus.yml is generated
- from the structured config options.
- '';
- };
+ globalConfig = mkOption {
+ type = promTypes.globalConfig;
+ default = {};
+ description = ''
+ Parameters that are valid in all configuration contexts. They
+ also serve as defaults for other configuration sections
+ '';
+ };
- globalConfig = mkOption {
- type = promTypes.globalConfig;
- default = {};
- description = ''
- Parameters that are valid in all configuration contexts. They
- also serve as defaults for other configuration sections
- '';
- };
+ rules = mkOption {
+ type = types.listOf types.str;
+ default = [];
+ description = ''
+ Alerting and/or Recording rules to evaluate at runtime.
+ '';
+ };
- rules = mkOption {
- type = types.listOf types.str;
- default = [];
- description = ''
- Alerting and/or Recording rules to evaluate at runtime.
- '';
- };
+ ruleFiles = mkOption {
+ type = types.listOf types.path;
+ default = [];
+ description = ''
+ Any additional rules files to include in this configuration.
+ '';
+ };
- ruleFiles = mkOption {
- type = types.listOf types.path;
- default = [];
- description = ''
- Any additional rules files to include in this configuration.
- '';
- };
+ scrapeConfigs = mkOption {
+ type = types.listOf promTypes.scrape_config;
+ default = [];
+ description = ''
+ A list of scrape configurations.
+ '';
+ };
- scrapeConfigs = mkOption {
- type = types.listOf promTypes.scrape_config;
- default = [];
- description = ''
- A list of scrape configurations.
- '';
- };
+ alertmanagers = mkOption {
+ type = types.listOf types.attrs;
+ example = literalExample ''
+ [ {
+ scheme = "https";
+ path_prefix = "/alertmanager";
+ static_configs = [ {
+ targets = [
+ "prometheus.domain.tld"
+ ];
+ } ];
+ } ]
+ '';
+ default = [];
+ description = ''
+ A list of alertmanagers to send alerts to.
+ See <link xlink:href="https://prometheus.io/docs/prometheus/latest/configuration/configuration/#alertmanager_config">the official documentation</link> for more information.
+ '';
+ };
- alertmanagers = mkOption {
- type = types.listOf types.attrs;
- example = literalExample ''
- [ {
- scheme = "https";
- path_prefix = "/alertmanager";
- static_configs = [ {
- targets = [
- "prometheus.domain.tld"
- ];
- } ];
- } ]
- '';
- default = [];
- description = ''
- A list of alertmanagers to send alerts to.
- See <link xlink:href="https://prometheus.io/docs/prometheus/latest/configuration/configuration/#alertmanager_config">the official documentation</link> for more information.
- '';
- };
+ alertmanagerNotificationQueueCapacity = mkOption {
+ type = types.int;
+ default = 10000;
+ description = ''
+ The capacity of the queue for pending alert manager notifications.
+ '';
+ };
- alertmanagerNotificationQueueCapacity = mkOption {
- type = types.int;
- default = 10000;
- description = ''
- The capacity of the queue for pending alert manager notifications.
- '';
- };
+ alertmanagerTimeout = mkOption {
+ type = types.int;
+ default = 10;
+ description = ''
+ Alert manager HTTP API timeout (in seconds).
+ '';
+ };
- alertmanagerTimeout = mkOption {
- type = types.int;
- default = 10;
- description = ''
- Alert manager HTTP API timeout (in seconds).
- '';
- };
+ webExternalUrl = mkOption {
+ type = types.nullOr types.str;
+ default = null;
+ example = "https://example.com/";
+ description = ''
+ The URL under which Prometheus is externally reachable (for example,
+ if Prometheus is served via a reverse proxy).
+ '';
+ };
+ };
- webExternalUrl = mkOption {
- type = types.nullOr types.str;
- default = null;
- example = "https://example.com/";
- description = ''
- The URL under which Prometheus is externally reachable (for example,
- if Prometheus is served via a reverse proxy).
- '';
- };
+ config = mkIf cfg.enable {
+ users.groups.prometheus.gid = config.ids.gids.prometheus;
+ users.users.prometheus = {
+ description = "Prometheus daemon user";
+ uid = config.ids.uids.prometheus;
+ group = "prometheus";
};
- };
-
- config = mkMerge [
- (mkIf (cfg.enable || cfg2.enable) {
- users.groups.${promGroup}.gid = config.ids.gids.prometheus;
- users.users.${promUser} = {
- description = "Prometheus daemon user";
- uid = config.ids.uids.prometheus;
- group = promGroup;
+ systemd.services.prometheus = {
+ wantedBy = [ "multi-user.target" ];
+ after = [ "network.target" ];
+ serviceConfig = {
+ ExecStart = "${cfg.package}/bin/prometheus" +
+ optionalString (length cmdlineArgs != 0) (" \\\n " +
+ concatStringsSep " \\\n " cmdlineArgs);
+ User = "prometheus";
+ Restart = "always";
+ WorkingDirectory = workingDir;
+ StateDirectory = cfg.stateDir;
};
- })
- (mkIf cfg.enable {
- warnings =
- optional (cfg.dataDir != null) ''
- The option services.prometheus.dataDir is deprecated, please use
- services.prometheus.stateDir.
- '';
- assertions = [
- {
- assertion = !(cfg.dataDir != null && cfg.stateDir != null);
- message =
- "The options services.prometheus.dataDir and services.prometheus.stateDir" +
- " can't both be set at the same time! It's recommended to only set the latter" +
- " since the former is deprecated.";
- }
- {
- assertion = cfg.dataDir != null -> hasPrefix stateDirBase cfg.dataDir;
- message =
- "The option services.prometheus.dataDir should have ${stateDirBase} as a prefix!";
- }
- {
- assertion = cfg.stateDir != null -> !hasPrefix "/" cfg.stateDir;
- message =
- "The option services.prometheus.stateDir shouldn't be an absolute directory." +
- " It should be a directory relative to ${stateDirBase}.";
- }
- {
- assertion = cfg2.stateDir != null -> !hasPrefix "/" cfg2.stateDir;
- message =
- "The option services.prometheus2.stateDir shouldn't be an absolute directory." +
- " It should be a directory relative to ${stateDirBase}.";
- }
- ];
- systemd.services.prometheus = {
- wantedBy = [ "multi-user.target" ];
- after = [ "network.target" ];
- serviceConfig = {
- ExecStart = "${cfg.package}/bin/prometheus" +
- optionalString (length cmdlineArgs != 0) (" \\\n " +
- concatStringsSep " \\\n " cmdlineArgs);
- User = promUser;
- Restart = "always";
- WorkingDirectory = workingDir;
- StateDirectory = stateDir;
- };
- };
- })
- (mkIf cfg2.enable {
- systemd.services.prometheus2 = {
- wantedBy = [ "multi-user.target" ];
- after = [ "network.target" ];
- serviceConfig = {
- ExecStart = "${cfg2.package}/bin/prometheus" +
- optionalString (length cmdlineArgs2 != 0) (" \\\n " +
- concatStringsSep " \\\n " cmdlineArgs2);
- User = promUser;
- Restart = "always";
- WorkingDirectory = workingDir2;
- StateDirectory = cfg2.stateDir;
- };
- };
- })
- ];
+ };
+ };
}
diff --git a/nixos/modules/services/monitoring/thanos.nix b/nixos/modules/services/monitoring/thanos.nix
index b41e99b76477..4659a0da414b 100644
--- a/nixos/modules/services/monitoring/thanos.nix
+++ b/nixos/modules/services/monitoring/thanos.nix
@@ -218,8 +218,8 @@ let
toArgs = optionToArgs;
option = mkOption {
type = types.str;
- default = "/var/lib/${config.services.prometheus2.stateDir}/data";
- defaultText = "/var/lib/\${config.services.prometheus2.stateDir}/data";
+ default = "/var/lib/${config.services.prometheus.stateDir}/data";
+ defaultText = "/var/lib/\${config.services.prometheus.stateDir}/data";
description = ''
Data directory of TSDB.
'';
@@ -679,22 +679,22 @@ in {
(mkIf cfg.sidecar.enable {
assertions = [
{
- assertion = config.services.prometheus2.enable;
+ assertion = config.services.prometheus.enable;
message =
- "Please enable services.prometheus2 when enabling services.thanos.sidecar.";
+ "Please enable services.prometheus when enabling services.thanos.sidecar.";
}
{
- assertion = !(config.services.prometheus2.globalConfig.external_labels == null ||
- config.services.prometheus2.globalConfig.external_labels == {});
+ assertion = !(config.services.prometheus.globalConfig.external_labels == null ||
+ config.services.prometheus.globalConfig.external_labels == {});
message =
"services.thanos.sidecar requires uniquely identifying external labels " +
"to be configured in the Prometheus server. " +
- "Please set services.prometheus2.globalConfig.external_labels.";
+ "Please set services.prometheus.globalConfig.external_labels.";
}
];
systemd.services.thanos-sidecar = {
wantedBy = [ "multi-user.target" ];
- after = [ "network.target" "prometheus2.service" ];
+ after = [ "network.target" "prometheus.service" ];
serviceConfig = {
User = "prometheus";
Restart = "always";
diff --git a/nixos/tests/all-tests.nix b/nixos/tests/all-tests.nix
index 8ee4dfbf13bc..5eb8111aa6d3 100644
--- a/nixos/tests/all-tests.nix
+++ b/nixos/tests/all-tests.nix
@@ -224,7 +224,6 @@ in
predictable-interface-names = handleTest ./predictable-interface-names.nix {};
printing = handleTest ./printing.nix {};
prometheus = handleTest ./prometheus.nix {};
- prometheus2 = handleTest ./prometheus-2.nix {};
prometheus-exporters = handleTest ./prometheus-exporters.nix {};
prosody = handleTest ./xmpp/prosody.nix {};
prosodyMysql = handleTest ./xmpp/prosody-mysql.nix {};
diff --git a/nixos/tests/prometheus-2.nix b/nixos/tests/prometheus-2.nix
deleted file mode 100644
index 219c47c73d95..000000000000
--- a/nixos/tests/prometheus-2.nix
+++ /dev/null
@@ -1,239 +0,0 @@
-let
- grpcPort = 19090;
- queryPort = 9090;
- minioPort = 9000;
- pushgwPort = 9091;
-
- s3 = {
- accessKey = "BKIKJAA5BMMU2RHO6IBB";
- secretKey = "V7f1CwQqAcwo80UEIJEjc5gVQUSSx5ohQ9GSrr12";
- };
-
- objstore.config = {
- type = "S3";
- config = {
- bucket = "thanos-bucket";
- endpoint = "s3:${toString minioPort}";
- region = "us-east-1";
- access_key = s3.accessKey;
- secret_key = s3.secretKey;
- insecure = true;
- signature_version2 = false;
- encrypt_sse = false;
- put_user_metadata = {};
- http_config = {
- idle_conn_timeout = "0s";
- insecure_skip_verify = false;
- };
- trace = {
- enable = false;
- };
- };
- };
-
-in import ./make-test.nix {
- name = "prometheus-2";
-
- nodes = {
- prometheus = { pkgs, ... }: {
- virtualisation.diskSize = 2 * 1024;
- environment.systemPackages = [ pkgs.jq ];
- networking.firewall.allowedTCPPorts = [ grpcPort ];
- services.prometheus2 = {
- enable = true;
- scrapeConfigs = [
- {
- job_name = "prometheus";
- static_configs = [
- {
- targets = [ "127.0.0.1:${toString queryPort}" ];
- labels = { instance = "localhost"; };
- }
- ];
- }
- {
- job_name = "pushgateway";
- scrape_interval = "1s";
- static_configs = [
- {
- targets = [ "127.0.0.1:${toString pushgwPort}" ];
- }
- ];
- }
- ];
- rules = [
- ''
- groups:
- - name: test
- rules:
- - record: testrule
- expr: count(up{job="prometheus"})
- ''
- ];
- globalConfig = {
- external_labels = {
- some_label = "required by thanos";
- };
- };
- extraFlags = [
- # Required by thanos
- "--storage.tsdb.min-block-duration=5s"
- "--storage.tsdb.max-block-duration=5s"
- ];
- };
- services.prometheus.pushgateway = {
- enable = true;
- web.listen-address = ":${toString pushgwPort}";
- persistMetrics = true;
- persistence.interval = "1s";
- stateDir = "prometheus-pushgateway";
- };
- services.thanos = {
- sidecar = {
- enable = true;
- grpc-address = "0.0.0.0:${toString grpcPort}";
- inherit objstore;
- };
-
- # TODO: Add some tests for these services:
- #rule = {
- # enable = true;
- # http-address = "0.0.0.0:19194";
- # grpc-address = "0.0.0.0:19193";
- # query.addresses = [
- # "localhost:19191"
- # ];
- # labels = {
- # just = "some";
- # nice = "labels";
- # };
- #};
- #
- #receive = {
- # http-address = "0.0.0.0:19195";
- # enable = true;
- # labels = {
- # just = "some";
- # nice = "labels";
- # };
- #};
- };
- };
-
- query = { pkgs, ... }: {
- environment.systemPackages = [ pkgs.jq ];
- services.thanos.query = {
- enable = true;
- http-address = "0.0.0.0:${toString queryPort}";
- store.addresses = [
- "prometheus:${toString grpcPort}"
- ];
- };
- };
-
- store = { pkgs, ... }: {
- virtualisation.diskSize = 2 * 1024;
- environment.systemPackages = with pkgs; [ jq thanos ];
- services.thanos.store = {
- enable = true;
- http-address = "0.0.0.0:10902";
- grpc-address = "0.0.0.0:${toString grpcPort}";
- inherit objstore;
- sync-block-duration = "1s";
- };
- services.thanos.compact = {
- enable = true;
- http-address = "0.0.0.0:10903";
- inherit objstore;
- consistency-delay = "5s";
- };
- services.thanos.query = {
- enable = true;
- http-address = "0.0.0.0:${toString queryPort}";
- store.addresses = [
- "localhost:${toString grpcPort}"
- ];
- };
- };
-
- s3 = { pkgs, ... } : {
- # Minio requires at least 1GiB of free disk space to run.
- virtualisation.diskSize = 2 * 1024;
- networking.firewall.allowedTCPPorts = [ minioPort ];
-
- services.minio = {
- enable = true;
- inherit (s3) accessKey secretKey;
- };
-
- environment.systemPackages = [ pkgs.minio-client ];
- };
- };
-
- testScript = { nodes, ... } : ''
- # Before starting the other machines we first make sure that our S3 service is online
- # and has a bucket added for thanos:
- $s3->start;
- $s3->waitForUnit("minio.service");
- $s3->waitForOpenPort(${toString minioPort});
- $s3->succeed(
- "mc config host add minio " .
- "http://localhost:${toString minioPort} ${s3.accessKey} ${s3.secretKey} S3v4");
- $s3->succeed("mc mb minio/thanos-bucket");
-
- # Now that s3 has started we can start the other machines:
- $prometheus->start;
- $query->start;
- $store->start;
-
- # Check if prometheus responds to requests:
- $prometheus->waitForUnit("prometheus2.service");
- $prometheus->waitForOpenPort(${toString queryPort});
- $prometheus->succeed("curl -s http://127.0.0.1:${toString queryPort}/metrics");
-
- # Let's test if pushing a metric to the pushgateway succeeds:
- $prometheus->waitForUnit("pushgateway.service");
- $prometheus->succeed(
- "echo 'some_metric 3.14' | " .
- "curl --data-binary \@- http://127.0.0.1:${toString pushgwPort}/metrics/job/some_job");
-
- # Now check whether that metric gets ingested by prometheus.
- # Since we'll check for the metric several times on different machines
- # we abstract the test using the following function:
-
- # Function to check if the metric "some_metric" has been received and returns the correct value.
- local *Machine::waitForMetric = sub {
- my ($self) = @_;
- $self->waitUntilSucceeds(
- "curl -sf 'http://127.0.0.1:${toString queryPort}/api/v1/query?query=some_metric' " .
- "| jq '.data.result[0].value[1]' | grep '\"3.14\"'");
- };
-
- $prometheus->waitForMetric;
-
- # Let's test if the pushgateway persists metrics to the configured location.
- $prometheus->waitUntilSucceeds("test -e /var/lib/prometheus-pushgateway/metrics");
-
- # Test thanos
- $prometheus->waitForUnit("thanos-sidecar.service");
-
- # Test if the Thanos query service can correctly retrieve the metric that was send above.
- $query->waitForUnit("thanos-query.service");
- $query->waitForMetric;
-
- # Test if the Thanos sidecar has correctly uploaded its TSDB to S3, if the
- # Thanos storage service has correctly downloaded it from S3 and if the Thanos
- # query service running on $store can correctly retrieve the metric:
- $store->waitForUnit("thanos-store.service");
- $store->waitForMetric;
-
- $store->waitForUnit("thanos-compact.service");
-
- # Test if the Thanos bucket command is able to retrieve blocks from the S3 bucket
- # and check if the blocks have the correct labels:
- $store->succeed(
- "thanos bucket ls" .
- " --objstore.config-file=${nodes.store.config.services.thanos.store.objstore.config-file}" .
- " --output=json | jq .thanos.labels.some_label | grep 'required by thanos'");
- '';
-}
diff --git a/nixos/tests/prometheus.nix b/nixos/tests/prometheus.nix
index f1b20a33d71e..52f61046be39 100644
--- a/nixos/tests/prometheus.nix
+++ b/