ceph: 13.2.4 -> 14.2.1

* remove kinetic
* release note
* add johanot as maintainer

nixos/ceph: create option for mgr_module_path
  - since the upstream default is no longer correct in v14

* fix module, default location for libexec has changed
* ceph: fix test

7 files changed, 108 insertions, 84 deletions

diff --git a/nixos/doc/manual/release-notes/rl-1909.xml b/nixos/doc/manual/release-notes/rl-1909.xml
index c0b7cc48a461..78e911fa5d93 100644
--- a/nixos/doc/manual/release-notes/rl-1909.xml
+++ b/nixos/doc/manual/release-notes/rl-1909.xml
@@ -710,6 +710,13 @@
        <literal>nix-shell -p altcoins.dogecoin</literal>, etc.
      </para>
    </listitem>
+   <listitem>
+     Ceph has been upgraded to v14.2.1.
+     See the <link xlink:href="https://ceph.com/releases/v14-2-0-nautilus-released/">release notes</link> for details.
+     The mgr dashboard as well as osds backed by loop-devices is no longer explicitly supported by the package and module.
+     Note: There's been some issues with python-cherrypy, which is used by the dashboard
+     and prometheus mgr modules (and possibly others), hence 0000-dont-check-cherrypy-version.patch.
+   </listitem>
   </itemizedlist>
  </section>
 </section>
diff --git a/nixos/modules/services/network-filesystems/ceph.nix b/nixos/modules/services/network-filesystems/ceph.nix
index 58922897a181..aef284c31a5a 100644
--- a/nixos/modules/services/network-filesystems/ceph.nix
+++ b/nixos/modules/services/network-filesystems/ceph.nix
@@ -38,7 +38,7 @@ let
                     -f --cluster ${clusterName} --id ${daemonId} --setuser ceph \
                     --setgroup ${if daemonType == "osd" then "disk" else "ceph"}'';
     } // extraServiceConfig
-      // optionalAttrs (daemonType == "osd") { ExecStartPre = ''${ceph.out}/libexec/ceph/ceph-osd-prestart.sh \
+      // optionalAttrs (daemonType == "osd") { ExecStartPre = ''${ceph.lib}/libexec/ceph/ceph-osd-prestart.sh \
                                                               --id ${daemonId} --cluster ${clusterName}''; };
     } // optionalAttrs (builtins.elem daemonType [ "mds" "mon" "rgw" "mgr" ]) {
       preStart = ''
@@ -88,6 +88,14 @@ in
         '';
       };
 
+      mgrModulePath = mkOption {
+        type = types.path;
+        default = "${pkgs.ceph.lib}/lib/ceph/mgr";
+        description = ''
+          Path at which to find ceph-mgr modules.
+        '';
+      };
+
       monInitialMembers = mkOption {
         type = with types; nullOr commas;
         default = null;
diff --git a/nixos/tests/ceph.nix b/nixos/tests/ceph.nix
index a80f63da6655..7a6e238d9157 100644
--- a/nixos/tests/ceph.nix
+++ b/nixos/tests/ceph.nix
@@ -7,6 +7,7 @@ import ./make-test.nix ({pkgs, lib, ...}: rec {
   nodes = {
     aio = { pkgs, ... }: {
       virtualisation = {
+        memorySize = 1536;
         emptyDiskImages = [ 20480 20480 ];
         vlans = [ 1 ];
       };
@@ -24,9 +25,6 @@ import ./make-test.nix ({pkgs, lib, ...}: rec {
         ceph
         xfsprogs
       ];
-      nixpkgs.config.packageOverrides = super: {
-        ceph = super.ceph.override({ nss = super.nss; libxfs = super.libxfs; libaio = super.libaio; jemalloc = super.jemalloc; });
-      };
 
       boot.kernelModules = [ "xfs" ];
 
@@ -40,9 +38,6 @@ import ./make-test.nix ({pkgs, lib, ...}: rec {
       services.ceph.mon = {
         enable = true;
         daemons = [ "aio" ];
-        extraConfig = {
-          "mgr initial modules" = "dashboard";
-        };
       };
 
       services.ceph.mgr = {
@@ -67,23 +62,26 @@ import ./make-test.nix ({pkgs, lib, ...}: rec {
 
     # Create the ceph-related directories
     $aio->mustSucceed(
-      "mkdir -p /var/lib/ceph/mgr/ceph-aio/",
-      "mkdir -p /var/lib/ceph/mon/ceph-aio/",
-      "mkdir -p /var/lib/ceph/osd/ceph-{0..1}/",
-      "chown ceph:ceph -R /var/lib/ceph/"
+      "mkdir -p /var/lib/ceph/mgr/ceph-aio",
+      "mkdir -p /var/lib/ceph/mon/ceph-aio",
+      "mkdir -p /var/lib/ceph/osd/ceph-{0,1}",
+      "chown ceph:ceph -R /var/lib/ceph/",
+      "mkdir -p /etc/ceph",
+      "chown ceph:ceph -R /etc/ceph"
     );
 
     # Bootstrap ceph-mon daemon
     $aio->mustSucceed(
       "sudo -u ceph ceph-authtool --create-keyring /tmp/ceph.mon.keyring --gen-key -n mon. --cap mon 'allow *'",
-      "ceph-authtool --create-keyring /etc/ceph/ceph.client.admin.keyring --gen-key -n client.admin --set-uid=0 --cap mon 'allow *' --cap osd 'allow *' --cap mds 'allow *' --cap mgr 'allow *'",
-      "ceph-authtool /tmp/ceph.mon.keyring --import-keyring /etc/ceph/ceph.client.admin.keyring",
-            "monmaptool --create --add aio 192.168.1.1 --fsid 066ae264-2a5d-4729-8001-6ad265f50b03 /tmp/monmap",
+      "sudo -u ceph ceph-authtool --create-keyring /etc/ceph/ceph.client.admin.keyring --gen-key -n client.admin --cap mon 'allow *' --cap osd 'allow *' --cap mds 'allow *' --cap mgr 'allow *'",
+      "sudo -u ceph ceph-authtool /tmp/ceph.mon.keyring --import-keyring /etc/ceph/ceph.client.admin.keyring",
+      "monmaptool --create --add aio 192.168.1.1 --fsid 066ae264-2a5d-4729-8001-6ad265f50b03 /tmp/monmap",
       "sudo -u ceph ceph-mon --mkfs -i aio --monmap /tmp/monmap --keyring /tmp/ceph.mon.keyring",
-      "touch /var/lib/ceph/mon/ceph-aio/done",
+      "sudo -u ceph touch /var/lib/ceph/mon/ceph-aio/done",
       "systemctl start ceph-mon-aio"
     );
     $aio->waitForUnit("ceph-mon-aio");
+    $aio->mustSucceed("ceph mon enable-msgr2");
 
     # Can't check ceph status until a mon is up
     $aio->succeed("ceph -s | grep 'mon: 1 daemons'");
@@ -95,7 +93,7 @@ import ./make-test.nix ({pkgs, lib, ...}: rec {
     );
     $aio->waitForUnit("ceph-mgr-aio");
     $aio->waitUntilSucceeds("ceph -s | grep 'quorum aio'");
-    $aio->waitUntilSucceeds("ceph -s | grep 'mgr: aio(active)'");
+    $aio->waitUntilSucceeds("ceph -s | grep 'mgr: aio(active,'");
 
     # Bootstrap both OSDs
     $aio->mustSucceed(
@@ -118,8 +116,8 @@ import ./make-test.nix ({pkgs, lib, ...}: rec {
       "systemctl start ceph-osd-1"
     );
 
-    $aio->waitUntilSucceeds("ceph osd stat | grep '2 osds: 2 up, 2 in'");
-    $aio->waitUntilSucceeds("ceph -s | grep 'mgr: aio(active)'");
+    $aio->waitUntilSucceeds("ceph osd stat | grep -e '2 osds: 2 up[^,]*, 2 in'");
+    $aio->waitUntilSucceeds("ceph -s | grep 'mgr: aio(active,'");
     $aio->waitUntilSucceeds("ceph -s | grep 'HEALTH_OK'");
 
     $aio->mustSucceed(
@@ -156,8 +154,8 @@ import ./make-test.nix ({pkgs, lib, ...}: rec {
     $aio->waitForUnit("ceph-osd-1");
     $aio->succeed("ceph -s | grep 'mon: 1 daemons'");
     $aio->waitUntilSucceeds("ceph -s | grep 'quorum aio'");
-    $aio->waitUntilSucceeds("ceph osd stat | grep '2 osds: 2 up, 2 in'");
-    $aio->waitUntilSucceeds("ceph -s | grep 'mgr: aio(active)'");
+    $aio->waitUntilSucceeds("ceph osd stat | grep -e '2 osds: 2 up[^,]*, 2 in'");
+    $aio->waitUntilSucceeds("ceph -s | grep 'mgr: aio(active,'");
     $aio->waitUntilSucceeds("ceph -s | grep 'HEALTH_OK'");
   '';
 })
diff --git a/pkgs/tools/filesystems/ceph/0000-ceph-volume-allow-loop.patch b/pkgs/tools/filesystems/ceph/0000-ceph-volume-allow-loop.patch
deleted file mode 100644
index d103c42b5189..000000000000
--- a/pkgs/tools/filesystems/ceph/0000-ceph-volume-allow-loop.patch
+++ /dev/null
@@ -1,35 +0,0 @@
-diff --git a/src/ceph-disk/ceph_disk/main.py b/src/ceph-disk/ceph_disk/main.py
-index f8ef35503e..2cc8a75d2b 100644
---- a/src/ceph-disk/ceph_disk/main.py
-+++ b/src/ceph-disk/ceph_disk/main.py
-@@ -881,13 +881,14 @@ def is_partition(dev):
-     if not stmode_is_diskdevice(st.st_mode):
-         raise Error('not a block device', dev)
- 
-+    major = os.major(st.st_rdev)
-+    minor = os.minor(st.st_rdev)
-+
-     name = get_dev_name(dev)
--    if os.path.exists(os.path.join(BLOCKDIR, name)):
-+    if os.path.exists(os.path.join(BLOCKDIR, name)) or os.path.exists(os.path.join(BLOCKDIR, "loop%d" % minor)):
-         return False
- 
-     # make sure it is a partition of something else
--    major = os.major(st.st_rdev)
--    minor = os.minor(st.st_rdev)
-     if os.path.exists('/sys/dev/block/%d:%d/partition' % (major, minor)):
-         return True
- 
-diff --git a/src/ceph-volume/ceph_volume/util/disk.py b/src/ceph-volume/ceph_volume/util/disk.py
-index c3a7915400..9a708b05f2 100644
---- a/src/ceph-volume/ceph_volume/util/disk.py
-+++ b/src/ceph-volume/ceph_volume/util/disk.py
-@@ -203,7 +203,7 @@ def is_device(dev):
-     # use lsblk first, fall back to using stat
-     TYPE = lsblk(dev).get('TYPE')
-     if TYPE:
--        return TYPE == 'disk'
-+        return TYPE == 'disk' or TYPE == 'loop'
- 
-     # fallback to stat
-     return _stat_is_device(os.lstat(dev).st_mode)
diff --git a/pkgs/tools/filesystems/ceph/0000-dont-check-cherrypy-version.patch b/pkgs/tools/filesystems/ceph/0000-dont-check-cherrypy-version.patch
index 154e90d91467..9604da87fb50 100644
--- a/pkgs/tools/filesystems/ceph/0000-dont-check-cherrypy-version.patch
+++ b/pkgs/tools/filesystems/ceph/0000-dont-check-cherrypy-version.patch
@@ -1,12 +1,11 @@
 diff --git a/src/pybind/mgr/dashboard/module.py b/src/pybind/mgr/dashboard/module.py
-index 0b53743d74..2e17ce1168 100644
+index a8a3ec07c1..bcc9b86c37 100644
 --- a/src/pybind/mgr/dashboard/module.py
 +++ b/src/pybind/mgr/dashboard/module.py
-@@ -28,28 +28,6 @@ except ImportError:
-     # To be picked up and reported by .can_run()
-     cherrypy = None
+@@ -25,40 +25,6 @@ except ImportError:
+ 
+ from .services.sso import load_sso_db
  
--
 -# The SSL code in CherryPy 3.5.0 is buggy.  It was fixed long ago,
 -# but 3.5.0 is still shipping in major linux distributions
 -# (Fedora 27, Ubuntu Xenial), so we must monkey patch it to get SSL working.
@@ -14,7 +13,7 @@ index 0b53743d74..2e17ce1168 100644
 -    v = StrictVersion(cherrypy.__version__)
 -    # It was fixed in 3.7.0.  Exact lower bound version is probably earlier,
 -    # but 3.5.0 is what this monkey patch is tested on.
--    if v >= StrictVersion("3.5.0") and v < StrictVersion("3.7.0"):
+-    if StrictVersion("3.5.0") <= v < StrictVersion("3.7.0"):
 -        from cherrypy.wsgiserver.wsgiserver2 import HTTPConnection,\
 -                                                    CP_fileobject
 -
@@ -27,7 +26,45 @@ index 0b53743d74..2e17ce1168 100644
 -
 -        HTTPConnection.__init__ = fixed_init
 -
+-# When the CherryPy server in 3.2.2 (and later) starts it attempts to verify
+-# that the ports its listening on are in fact bound. When using the any address
+-# "::" it tries both ipv4 and ipv6, and in some environments (e.g. kubernetes)
+-# ipv6 isn't yet configured / supported and CherryPy throws an uncaught
+-# exception.
+-if cherrypy is not None:
+-    v = StrictVersion(cherrypy.__version__)
+-    # the issue was fixed in 3.2.3. it's present in 3.2.2 (current version on
+-    # centos:7) and back to at least 3.0.0.
+-    if StrictVersion("3.1.2") <= v < StrictVersion("3.2.3"):
+-        # https://github.com/cherrypy/cherrypy/issues/1100
+-        from cherrypy.process import servers
+-        servers.wait_for_occupied_port = lambda host, port: None
 -
  if 'COVERAGE_ENABLED' in os.environ:
      import coverage
-     _cov = coverage.Coverage(config_file="{}/.coveragerc".format(os.path.dirname(__file__)))
+     __cov = coverage.Coverage(config_file="{}/.coveragerc".format(os.path.dirname(__file__)),
+diff --git a/src/pybind/mgr/prometheus/module.py b/src/pybind/mgr/prometheus/module.py
+index b7fecf8d85..dfd4160591 100644
+--- a/src/pybind/mgr/prometheus/module.py
++++ b/src/pybind/mgr/prometheus/module.py
+@@ -18,20 +18,6 @@ from rbd import RBD
+ DEFAULT_ADDR = '::'
+ DEFAULT_PORT = 9283
+ 
+-# When the CherryPy server in 3.2.2 (and later) starts it attempts to verify
+-# that the ports its listening on are in fact bound. When using the any address
+-# "::" it tries both ipv4 and ipv6, and in some environments (e.g. kubernetes)
+-# ipv6 isn't yet configured / supported and CherryPy throws an uncaught
+-# exception.
+-if cherrypy is not None:
+-    v = StrictVersion(cherrypy.__version__)
+-    # the issue was fixed in 3.2.3. it's present in 3.2.2 (current version on
+-    # centos:7) and back to at least 3.0.0.
+-    if StrictVersion("3.1.2") <= v < StrictVersion("3.2.3"):
+-        # https://github.com/cherrypy/cherrypy/issues/1100
+-        from cherrypy.process import servers
+-        servers.wait_for_occupied_port = lambda host, port: None
+-
+ # cherrypy likes to sys.exit on error.  don't let it take us down too!
+ def os_exit_noop(*args, **kwargs):
+     pass
diff --git a/pkgs/tools/filesystems/ceph/default.nix b/pkgs/tools/filesystems/ceph/default.nix
index b9c71c49e807..efb292687ab4 100644
--- a/pkgs/tools/filesystems/ceph/default.nix
+++ b/pkgs/tools/filesystems/ceph/default.nix
@@ -2,18 +2,18 @@
 , ensureNewerSourcesHook
 , cmake, pkgconfig
 , which, git
-, boost, python2Packages
+, boost, python3Packages
 , libxml2, zlib, lz4
 , openldap, lttng-ust
 , babeltrace, gperf
 , cunit, snappy
 , rocksdb, makeWrapper
-, leveldb, oathToolkit
+, leveldb, oathToolkit, removeReferencesTo
 
 # Optional Dependencies
 , yasm ? null, fcgi ? null, expat ? null
 , curl ? null, fuse ? null
-, libedit ? null, libatomic_ops ? null, kinetic-cpp-client ? null
+, libedit ? null, libatomic_ops ? null
 , libs3 ? null
 
 # Mallocs
@@ -24,7 +24,7 @@
 , nss ? null, nspr ? null
 
 # Linux Only Dependencies
-, linuxHeaders, utillinux, libuuid, udev, keyutils
+, linuxHeaders, utillinux, libuuid, udev, keyutils, rdma-core, rabbitmq-c
 , libaio ? null, libxfs ? null, zfs ? null
 , ...
 }:
@@ -43,7 +43,6 @@ let
   optFuse = shouldUsePkg fuse;
   optLibedit = shouldUsePkg libedit;
   optLibatomic_ops = shouldUsePkg libatomic_ops;
-  optKinetic-cpp-client = shouldUsePkg kinetic-cpp-client;
   optLibs3 = shouldUsePkg libs3;
 
   optJemalloc = shouldUsePkg jemalloc;
@@ -73,7 +72,7 @@ let
     none = [ ];
   };
 
-  ceph-python-env = python2Packages.python.withPackages (ps: [
+  ceph-python-env = python3Packages.python.withPackages (ps: [
     ps.sphinx
     ps.flask
     ps.cython
@@ -84,30 +83,30 @@ let
     ps.cherrypy
     ps.pecan
     ps.prettytable
+    ps.pyjwt
     ps.webob
     ps.bcrypt
+    ps.six
   ]);
 
-  version = "13.2.4";
+  version = "14.2.1";
 in rec {
   ceph = stdenv.mkDerivation {
     name="ceph-${version}";
 
     src = fetchurl {
       url = "http://download.ceph.com/tarballs/ceph-${version}.tar.gz";
-      sha256 = "0g2mc6rp84ia44vz8kl449820m9hmfavzfmwn8fy6li14xr8a00w";
+      sha256 = "0qa9p8xp26d45h3jfj1rbwhmqv44f9n1mvccmpzaf2i05v42kmzb";
     };
 
     patches = [
       ./0000-fix-SPDK-build-env.patch
-      # TODO: remove when https://github.com/ceph/ceph/pull/21289 is merged
-      ./0000-ceph-volume-allow-loop.patch
       ./0000-dont-check-cherrypy-version.patch
     ];
 
     nativeBuildInputs = [
       cmake
-      pkgconfig which git python2Packages.wrapPython makeWrapper
+      pkgconfig which git python3Packages.wrapPython makeWrapper
       (ensureNewerSourcesHook { year = "1980"; })
     ];
 
@@ -115,9 +114,11 @@ in rec {
       boost ceph-python-env libxml2 optYasm optLibatomic_ops optLibs3
       malloc zlib openldap lttng-ust babeltrace gperf cunit
       snappy rocksdb lz4 oathToolkit leveldb
-      optKinetic-cpp-client
+      removeReferencesTo
     ] ++ optionals stdenv.isLinux [
       linuxHeaders utillinux libuuid udev keyutils optLibaio optLibxfs optZfs
+      # ceph 14
+      rdma-core rabbitmq-c
     ] ++ optionals hasRadosgw [
       optFcgi optExpat optCurl optFuse optLibedit
     ];
@@ -126,20 +127,21 @@ in rec {
       substituteInPlace src/common/module.c --replace "/sbin/modinfo"  "modinfo"
       substituteInPlace src/common/module.c --replace "/sbin/modprobe" "modprobe"
       # Since Boost 1.67 this seems to have changed
-      substituteInPlace CMakeLists.txt --replace "list(APPEND BOOST_COMPONENTS python)" "list(APPEND BOOST_COMPONENTS python27)"
-      substituteInPlace src/CMakeLists.txt --replace "Boost::python " "Boost::python27 "
+      substituteInPlace CMakeLists.txt --replace "list(APPEND BOOST_COMPONENTS python)" "list(APPEND BOOST_COMPONENTS python37)"
+      substituteInPlace src/CMakeLists.txt --replace "Boost::python " "Boost::python37 "
 
       # for pybind/rgw to find internal dep
       export LD_LIBRARY_PATH="$PWD/build/lib:$LD_LIBRARY_PATH"
       # install target needs to be in PYTHONPATH for "*.pth support" check to succeed
-      export PYTHONPATH=$lib/lib/python2.7/site-packages/:$out/lib/python2.7/site-packages/
+      export PYTHONPATH=${ceph-python-env}/lib/python3.7/site-packages:$lib/lib/python3.7/site-packages/:$out/lib/python3.7/site-packages/
 
       patchShebangs src/spdk
     '';
 
     cmakeFlags = [
-      "-DWITH_SYSTEM_ROCKSDB=ON"
-      "-DROCKSDB_INCLUDE_DIR=${rocksdb}/include/rocksdb"
+      "-DWITH_PYTHON3=ON"
+      "-DWITH_SYSTEM_ROCKSDB=OFF"
+
       "-DWITH_SYSTEM_BOOST=ON"
       "-DWITH_SYSTEMD=OFF"
       "-DWITH_TESTS=OFF"
@@ -147,9 +149,16 @@ in rec {
       "-DWITH_MGR_DASHBOARD_FRONTEND=OFF"
     ];
 
+    preFixup = ''
+      find $lib -type f -exec remove-references-to -t $out '{}' +
+      mv $out/share/ceph/mgr $lib/lib/ceph/
+    '';
+
     postFixup = ''
+      export PYTHONPATH="${ceph-python-env}/lib/python3.7/site-packages:$lib/lib/ceph/mgr:$out/lib/python3.7/site-packages/"
       wrapPythonPrograms
-      wrapProgram $out/bin/ceph-mgr --prefix PYTHONPATH ":" "$lib/lib/ceph/mgr:$out/lib/python2.7/site-packages/"
+      wrapProgram $out/bin/ceph-mgr --prefix PYTHONPATH ":" "${ceph-python-env}/lib/python3.7/site-packages:$lib/lib/ceph/mgr:$out/lib/python3.7/site-packages/"
+      wrapProgram $out/bin/ceph-volume --prefix PYTHONPATH ":" "${ceph-python-env}/lib/python3.7/site-packages:$lib/lib/ceph/mgr:$out/lib/python3.7/site-packages/"
     '';
 
     enableParallelBuilding = true;
@@ -160,7 +169,7 @@ in rec {
       homepage = https://ceph.com/;
       description = "Distributed storage system";
       license = with licenses; [ lgpl21 gpl2 bsd3 mit publicDomain ];
-      maintainers = with maintainers; [ adev ak krav ];
+      maintainers = with maintainers; [ adev ak krav johanot ];
       platforms = platforms.unix;
     };
 
@@ -176,11 +185,11 @@ in rec {
         platforms = platforms.unix;
       };
     } ''
-      mkdir -p $out/{bin,etc,lib/python2.7/site-packages}
+      mkdir -p $out/{bin,etc,lib/python3.7/site-packages}
       cp -r ${ceph}/bin/{ceph,.ceph-wrapped,rados,rbd,rbdmap} $out/bin
       cp -r ${ceph}/bin/ceph-{authtool,conf,dencoder,rbdnamer,syn} $out/bin
       cp -r ${ceph}/bin/rbd-replay* $out/bin
-      cp -r ${ceph}/lib/python2.7/site-packages $out/lib/python2.7/
+      cp -r ${ceph}/lib/python3.7/site-packages $out/lib/python3.7/
       cp -r ${ceph}/etc/bash_completion.d $out/etc
       # wrapPythonPrograms modifies .ceph-wrapped, so lets just update its paths
       substituteInPlace $out/bin/ceph          --replace ${ceph} $out
diff --git a/pkgs/top-level/all-packages.nix b/pkgs/top-level/all-packages.nix
index 85f7a786a6c3..21f5f14a2dcb 100644
--- a/pkgs/top-level/all-packages.nix
+++ b/pkgs/top-level/all-packages.nix
@@ -2194,7 +2194,7 @@ in
 
   libceph = ceph.lib;
   inherit (callPackages ../tools/filesystems/ceph {
-    boost = boost167.override { enablePython = true; };
+    boost = boost167.override { enablePython = true; python = python37; };
   })
     ceph
     ceph-client;