diff options
| -rw-r--r-- | lib/licenses.nix | 6 | ||||
| -rw-r--r-- | nixos/modules/services/search/elasticsearch.nix | 42 | ||||
| -rw-r--r-- | nixos/tests/elk.nix | 19 | ||||
| -rw-r--r-- | pkgs/development/tools/misc/kibana/6.x.nix | 47 | ||||
| -rw-r--r-- | pkgs/misc/logging/beats/6.x.nix | 2 | ||||
| -rw-r--r-- | pkgs/servers/search/elasticsearch/6.x.nix | 36 | ||||
| -rw-r--r-- | pkgs/tools/misc/logstash/6.x.nix | 19 | ||||
| -rw-r--r-- | pkgs/top-level/all-packages.nix | 11 |
8 files changed, 145 insertions, 37 deletions
diff --git a/lib/licenses.nix b/lib/licenses.nix index 1c953a2ba844..79124855f7fa 100644 --- a/lib/licenses.nix +++ b/lib/licenses.nix @@ -231,6 +231,12 @@ lib.mapAttrs (n: v: v // { shortName = n; }) rec { fullName = "Eiffel Forum License v2.0"; }; + elastic = { + fullName = "ELASTIC LICENSE"; + url = https://github.com/elastic/elasticsearch/blob/master/licenses/ELASTIC-LICENSE.txt; + free = false; + }; + epl10 = spdx { spdxId = "EPL-1.0"; fullName = "Eclipse Public License 1.0"; diff --git a/nixos/modules/services/search/elasticsearch.nix b/nixos/modules/services/search/elasticsearch.nix index d61f588205af..b0831dcd1ca8 100644 --- a/nixos/modules/services/search/elasticsearch.nix +++ b/nixos/modules/services/search/elasticsearch.nix @@ -25,18 +25,17 @@ let ${cfg.extraConf} ''; - configDir = pkgs.buildEnv { - name = "elasticsearch-config"; - paths = [ - (pkgs.writeTextDir "elasticsearch.yml" esConfig) - (if es5 then (pkgs.writeTextDir "log4j2.properties" cfg.logging) - else (pkgs.writeTextDir "logging.yml" cfg.logging)) - ]; - postBuild = concatStringsSep "\n" (concatLists [ - # Elasticsearch 5.x won't start when the scripts directory does not exist - (optional es5 "${pkgs.coreutils}/bin/mkdir -p $out/scripts") - (optional es6 "ln -s ${cfg.package}/config/jvm.options $out/jvm.options") - ]); + configDir = cfg.dataDir + "/config"; + + elasticsearchYml = pkgs.writeTextFile { + name = "elasticsearch.yml"; + text = esConfig; + }; + + loggingConfigFilename = if es5 then "log4j2.properties" else "logging.yml"; + loggingConfigFile = pkgs.writeTextFile { + name = loggingConfigFilename; + text = cfg.logging; }; esPlugins = pkgs.buildEnv { @@ -193,7 +192,24 @@ in { ln -sfT ${esPlugins}/plugins ${cfg.dataDir}/plugins ln -sfT ${cfg.package}/lib ${cfg.dataDir}/lib ln -sfT ${cfg.package}/modules ${cfg.dataDir}/modules - if [ "$(id -u)" = 0 ]; then chown -R elasticsearch ${cfg.dataDir}; fi + + # elasticsearch needs to create the elasticsearch.keystore in the config directory + # so this directory needs to be writable. + mkdir -m 0700 -p ${configDir} + + # Note that we copy config files from the nix store instead of symbolically linking them + # because otherwise X-Pack Security will raise the following exception: + # java.security.AccessControlException: + # access denied ("java.io.FilePermission" "/var/lib/elasticsearch/config/elasticsearch.yml" "read") + + cp ${elasticsearchYml} ${configDir}/elasticsearch.yml + # Make sure the logging configuration for old elasticsearch versions is removed: + rm -f ${if es5 then "${configDir}/logging.yml" else "${configDir}/log4j2.properties"} + cp ${loggingConfigFile} ${configDir}/${loggingConfigFilename} + ${optionalString es5 "mkdir -p ${configDir}/scripts"} + ${optionalString es6 "cp ${cfg.package}/config/jvm.options ${configDir}/jvm.options"} + + if [ "$(id -u)" = 0 ]; then chown -R elasticsearch:elasticsearch ${cfg.dataDir}; fi ''; }; diff --git a/nixos/tests/elk.nix b/nixos/tests/elk.nix index 8dba7a905fa6..4c5c441ca265 100644 --- a/nixos/tests/elk.nix +++ b/nixos/tests/elk.nix @@ -1,4 +1,4 @@ -{ system ? builtins.currentSystem }: +{ system ? builtins.currentSystem, enableUnfree ? false }: with import ../lib/testing.nix { inherit system; }; with pkgs.lib; let @@ -99,9 +99,16 @@ in mapAttrs mkElkTest { logstash = pkgs.logstash5; kibana = pkgs.kibana5; }; - "ELK-6" = { - elasticsearch = pkgs.elasticsearch6; - logstash = pkgs.logstash6; - kibana = pkgs.kibana6; - }; + "ELK-6" = + if enableUnfree + then { + elasticsearch = pkgs.elasticsearch6; + logstash = pkgs.logstash6; + kibana = pkgs.kibana6; + } + else { + elasticsearch = pkgs.elasticsearch6-oss; + logstash = pkgs.logstash6-oss; + kibana = pkgs.kibana6-oss; + }; } diff --git a/pkgs/development/tools/misc/kibana/6.x.nix b/pkgs/development/tools/misc/kibana/6.x.nix index 17620c9df6c8..1fd52f435574 100644 --- a/pkgs/development/tools/misc/kibana/6.x.nix +++ b/pkgs/development/tools/misc/kibana/6.x.nix @@ -1,4 +1,13 @@ -{ stdenv, makeWrapper, fetchurl, elk6Version, nodejs, coreutils, which }: +{ elk6Version +, enableUnfree ? true +, stdenv +, makeWrapper +, fetchzip +, fetchurl +, nodejs +, coreutils +, which +}: with stdenv.lib; let @@ -6,12 +15,31 @@ let info = splitString "-" stdenv.system; arch = elemAt info 0; plat = elemAt info 1; - shas = { - "x86_64-linux" = "1br9nvwa3i5sfcbnrxp2x3dxxnsbs9iavz6zwgw0jlh5ngf5vysk"; - "x86_64-darwin" = "1w4dck02i0rrl8m18kvy2zz02cb7bb9a2pdhkd1jfy1qz4ssnhii"; + shas = + if enableUnfree + then { + "x86_64-linux" = "1kk97ggpzmblhqm6cfd2sv5940f58h323xcyg6rba1njj7lzanv0"; + "x86_64-darwin" = "1xvwffk8d8br92h0laf4b1m76kvki6cj0pbgcvirfcj1r70vk6c3"; + } + else { + "x86_64-linux" = "0m81ki1v61gpwb3s6zf84azqrirlm9pdfx65g3xmvdp3d3wii5ly"; + "x86_64-darwin" = "0zh9p6vsq1d0gh6ks7z6bh8sbhn6rm4jshjcfp3c9k7n2qa8vv9b"; + }; + + # For the correct phantomjs version see: + # https://github.com/elastic/kibana/blob/master/x-pack/plugins/reporting/server/browsers/phantom/paths.js + phantomjs = rec { + name = "phantomjs-${version}-linux-x86_64"; + version = "2.1.1"; + src = fetchzip { + inherit name; + url = "https://github.com/Medium/phantomjs/releases/download/v${version}/${name}.tar.bz2"; + sha256 = "0g2dqjzr2daz6rkd6shj6rrlw55z4167vqh7bxadl8jl6jk7zbfv"; + }; }; + in stdenv.mkDerivation rec { - name = "kibana-${version}"; + name = "kibana-${optionalString (!enableUnfree) "oss-"}${version}"; version = elk6Version; src = fetchurl { @@ -28,12 +56,19 @@ in stdenv.mkDerivation rec { makeWrapper $out/libexec/kibana/bin/kibana $out/bin/kibana \ --prefix PATH : "${stdenv.lib.makeBinPath [ nodejs coreutils which ]}" sed -i 's@NODE=.*@NODE=${nodejs}/bin/node@' $out/libexec/kibana/bin/kibana + '' + + # phantomjs is needed in the unfree version. When phantomjs doesn't exist in + # $out/libexec/kibana/data kibana will try to download and unpack it during + # runtime which will fail because the nix store is read-only. So we make sure + # it already exist in the nix store. + optionalString enableUnfree '' + ln -s ${phantomjs.src} $out/libexec/kibana/data/${phantomjs.name} ''; meta = { description = "Visualize logs and time-stamped data"; homepage = http://www.elasticsearch.org/overview/kibana; - license = licenses.asl20; + license = if enableUnfree then licenses.elastic else licenses.asl20; maintainers = with maintainers; [ offline rickynils basvandijk ]; platforms = with platforms; unix; }; diff --git a/pkgs/misc/logging/beats/6.x.nix b/pkgs/misc/logging/beats/6.x.nix index ce911ae40d5e..111bc449d888 100644 --- a/pkgs/misc/logging/beats/6.x.nix +++ b/pkgs/misc/logging/beats/6.x.nix @@ -8,7 +8,7 @@ let beat = package : extraArgs : buildGoPackage (rec { owner = "elastic"; repo = "beats"; rev = "v${version}"; - sha256 = "194z3j9zwlbc6j97iy1m1cl0xqks3ws2bjp2xrgy8cwpi7fclaw2"; + sha256 = "0ymg6y6v0mdhs1rs11fn33xdp3r6v85563z0f4p7s22j1kd3nd6r"; }; goPackagePath = "github.com/elastic/beats"; diff --git a/pkgs/servers/search/elasticsearch/6.x.nix b/pkgs/servers/search/elasticsearch/6.x.nix index 06caf191587c..84872649c492 100644 --- a/pkgs/servers/search/elasticsearch/6.x.nix +++ b/pkgs/servers/search/elasticsearch/6.x.nix @@ -1,14 +1,26 @@ -{ stdenv, fetchurl, elk6Version, makeWrapper, jre_headless, utillinux }: +{ elk6Version +, enableUnfree ? true +, stdenv +, fetchurl +, makeWrapper +, jre_headless +, utillinux +, autoPatchelfHook +, zlib +}: with stdenv.lib; -stdenv.mkDerivation rec { +stdenv.mkDerivation (rec { version = elk6Version; - name = "elasticsearch-${version}"; + name = "elasticsearch-${optionalString (!enableUnfree) "oss-"}${version}"; src = fetchurl { url = "https://artifacts.elastic.co/downloads/elasticsearch/${name}.tar.gz"; - sha256 = "13hf00khq33yw6zv022vcrsf6vm43isx40x7ww8r1lqx3vmg3rli"; + sha256 = + if enableUnfree + then "0960ak602pm95p2mha9cb1mrwdky8pfw3y89r2v4zpr5n730hmnh" + else "1i4i1ai75bf8k0zd1qf8x0bavrm8rcw13xdim443zza09w95ypk4"; }; patches = [ ./es-home-6.x.patch ]; @@ -32,10 +44,22 @@ stdenv.mkDerivation rec { wrapProgram $out/bin/elasticsearch-plugin --set JAVA_HOME "${jre_headless}" ''; + passthru = { inherit enableUnfree; }; + meta = { description = "Open Source, Distributed, RESTful Search Engine"; - license = licenses.asl20; + license = if enableUnfree then licenses.elastic else licenses.asl20; platforms = platforms.unix; maintainers = with maintainers; [ apeschar basvandijk ]; }; -} +} // optionalAttrs enableUnfree { + dontPatchELF = true; + nativeBuildInputs = [ autoPatchelfHook ]; + runtimeDependencies = [ zlib ]; + postFixup = '' + for exe in $(find $out/modules/x-pack/x-pack-ml/platform/linux-x86_64/bin -executable -type f); do + echo "patching $exe..." + patchelf --set-interpreter "$(cat $NIX_CC/nix-support/dynamic-linker)" "$exe" + done + ''; +}) diff --git a/pkgs/tools/misc/logstash/6.x.nix b/pkgs/tools/misc/logstash/6.x.nix index 6aa3fd836965..4f15ba90d39d 100644 --- a/pkgs/tools/misc/logstash/6.x.nix +++ b/pkgs/tools/misc/logstash/6.x.nix @@ -1,12 +1,23 @@ -{ stdenv, fetchurl, elk6Version, makeWrapper, jre }: +{ elk6Version +, enableUnfree ? true +, stdenv +, fetchurl +, makeWrapper +, jre +}: + +with stdenv.lib; stdenv.mkDerivation rec { version = elk6Version; - name = "logstash-${version}"; + name = "logstash-${optionalString (!enableUnfree) "oss-"}${version}"; src = fetchurl { url = "https://artifacts.elastic.co/downloads/logstash/${name}.tar.gz"; - sha256 = "07j3jjg5ik4gjgvcx15qqqas9p1m3815jml82a5r1ip9l6vc4h20"; + sha256 = + if enableUnfree + then "0yx9hpiav4d5z1b52x2h5i0iknqs9lmxy8vmz0wkb23mjiz8njdr" + else "1ir8pbq706mxr56k5cgc9ajn2jp603zrqj66dimx6xxf2nfamw0w"; }; dontBuild = true; @@ -35,7 +46,7 @@ stdenv.mkDerivation rec { meta = with stdenv.lib; { description = "Logstash is a data pipeline that helps you process logs and other event data from a variety of systems"; homepage = https://www.elastic.co/products/logstash; - license = licenses.asl20; + license = if enableUnfree then licenses.elastic else licenses.asl20; platforms = platforms.unix; maintainers = with maintainers; [ wjlroe offline basvandijk ]; }; diff --git a/pkgs/top-level/all-packages.nix b/pkgs/top-level/all-packages.nix index c9f6c16e9928..6d2f490136c6 100644 --- a/pkgs/top-level/all-packages.nix +++ b/pkgs/top-level/all-packages.nix @@ -2254,12 +2254,15 @@ with pkgs; # The latest version used by elasticsearch, logstash, kibana and the the beats from elastic. elk5Version = "5.6.9"; - elk6Version = "6.2.4"; + elk6Version = "6.3.2"; elasticsearch = callPackage ../servers/search/elasticsearch { }; elasticsearch2 = callPackage ../servers/search/elasticsearch/2.x.nix { }; elasticsearch5 = callPackage ../servers/search/elasticsearch/5.x.nix { }; elasticsearch6 = callPackage ../servers/search/elasticsearch/6.x.nix { }; + elasticsearch6-oss = callPackage ../servers/search/elasticsearch/6.x.nix { + enableUnfree = false; + }; elasticsearchPlugins = recurseIntoAttrs ( callPackage ../servers/search/elasticsearch/plugins.nix { } @@ -3364,6 +3367,9 @@ with pkgs; kibana = callPackage ../development/tools/misc/kibana { }; kibana5 = callPackage ../development/tools/misc/kibana/5.x.nix { }; kibana6 = callPackage ../development/tools/misc/kibana/6.x.nix { }; + kibana6-oss = callPackage ../development/tools/misc/kibana/6.x.nix { + enableUnfree = false; + }; kismet = callPackage ../applications/networking/sniffers/kismet { }; @@ -3443,6 +3449,9 @@ with pkgs; logstash = callPackage ../tools/misc/logstash { }; logstash5 = callPackage ../tools/misc/logstash/5.x.nix { }; logstash6 = callPackage ../tools/misc/logstash/6.x.nix { }; + logstash6-oss = callPackage ../tools/misc/logstash/6.x.nix { + enableUnfree = false; + }; logstash-contrib = callPackage ../tools/misc/logstash/contrib.nix { }; |