elk7: 7.11.1 -> 7.16.1, 6.8.3 -> 6.8.21 + add filebeat module and tests (#150879)

* elk7: 7.11.1 -> 7.16.1

* nixosTests.elk: Improve reliability and compatibility with ELK 7.x

- Use comparisons in jq instead of grepping
- Match for `.hits.total.value` if version >= 7, otherwise it always
  passes
- Make curl fail if requests fails

* nixos/filebeat: Add initial module and test

Filebeat is an open source file harvester, mostly used to fetch logs
files and feed them into logstash.

This module can be used instead of journalbeat if used with
`filebeat7` and configured with the `journald` input.

* python3Packages.parsedmarc.tests: Fix breakage

- Don't use the deprecated elasticsearch7-oss package
- Improve jq query robustness and add tracing

* rl-2205: Note the addition of the filebeat service

* elk6: 6.8.3 -> 6.8.21

The latest version includes a fix for CVE-2021-44228.

* nixos/journalbeat: Add a loose dependency on elasticsearch

Avoid unnecssary back-off when elasticsearch is running on the same
host.

2 files changed, 11 insertions, 15 deletions

diff --git a/pkgs/misc/logging/beats/6.x.nix b/pkgs/misc/logging/beats/6.x.nix
index 1808197498b5..f5e31924791b 100644
--- a/pkgs/misc/logging/beats/6.x.nix
+++ b/pkgs/misc/logging/beats/6.x.nix
@@ -8,7 +8,7 @@ let beat = package : extraArgs : buildGoPackage (rec {
         owner = "elastic";
         repo = "beats";
         rev = "v${version}";
-        sha256 = "0jkiz5dfdi9zsji04ipcmcj7pml9294v455y7s2c22k24gyzbaw8";
+        sha256 = "1vnw9clsc10cfpjf6vxvc6m507b2q17sgsl079iwqbp4v0286il7";
       };
 
       goPackagePath = "github.com/elastic/beats";
diff --git a/pkgs/misc/logging/beats/7.x.nix b/pkgs/misc/logging/beats/7.x.nix
index b8b82ed4b308..9f5e550d9720 100644
--- a/pkgs/misc/logging/beats/7.x.nix
+++ b/pkgs/misc/logging/beats/7.x.nix
@@ -8,10 +8,10 @@ let beat = package: extraArgs: buildGoModule (rec {
     owner = "elastic";
     repo = "beats";
     rev = "v${version}";
-    sha256 = "0gjyzprgj9nskvlkm2bf125b7qn3608llz4kh1fyzsvrw6zb7sm8";
+    sha256 = "sha256-9Jl5Xo1iKdOY9ZE5JXKSL4ee+NdsN3KCY2dDYuxlzPI=";
   };
 
-  vendorSha256 = "04cwf96fh60ld3ndjzzssgirc9ssb53yq71j6ksx36m3y1x7fq9c";
+  vendorSha256 = "sha256-tyxyM7RsTHTVVxc9gagPsSvFRaWGTmobKzyv9RODXBk=";
 
   subPackages = [ package ];
 
@@ -24,7 +24,14 @@ let beat = package: extraArgs: buildGoModule (rec {
 } // extraArgs);
 in
 rec {
-  filebeat7 = beat "filebeat" { meta.description = "Lightweight shipper for logfiles"; };
+  filebeat7 = beat "filebeat" {
+    meta.description = "Lightweight shipper for logfiles";
+    buildInputs = [ systemd ];
+    tags = [ "withjournald" ];
+    postFixup = ''
+      patchelf --set-rpath ${lib.makeLibraryPath [ (lib.getLib systemd) ]} "$out/bin/filebeat"
+    '';
+  };
   heartbeat7 = beat "heartbeat" { meta.description = "Lightweight shipper for uptime monitoring"; };
   metricbeat7 = beat "metricbeat" {
     meta.description = "Lightweight shipper for metrics";
@@ -47,15 +54,4 @@ rec {
       PostgreSQL, Redis or Thrift and correlate the messages into transactions.
     '';
   };
-  journalbeat7 = beat "journalbeat" {
-    meta.description = ''
-      Journalbeat is an open source data collector to read and forward
-      journal entries from Linuxes with systemd.
-    '';
-    buildInputs = [ systemd.dev ];
-    postFixup = let libPath = lib.makeLibraryPath [ (lib.getLib systemd) ]; in
-      ''
-        patchelf --set-rpath ${libPath} "$out/bin/journalbeat"
-      '';
-  };
 }