diff options
author | Markus Kowalewski <markus.kowalewski@gmail.com> | 2021-01-10 13:40:19 +0100 |
---|---|---|
committer | Markus Kowalewski <markus.kowalewski@gmail.com> | 2021-01-23 12:15:13 +0100 |
commit | 6dba41fbcb4239a628ac5bdf0035882a679b8648 (patch) | |
tree | a11ed2cef0b00b88ba8e99a8d30f8890aec90781 /pkgs/development | |
parent | f6a583eeece936a1d917de67194fec4b6c74cf1f (diff) |
mpi: use mpi attribute consistently as the default MPI implementations
Use the attribute mpi to provide a system wide default MPI
implementation. The default is openmpi (as before).
This now allows for overriding the MPI implentation by using
the overlay mechanism. Build all packages with mpich instead
of the default openmpi can now be achived like this:
self: super:
{
mpi = super.mpich;
}
All derivations that have been using "mpi ? null" to provide optional
building with MPI have been change in the following way to allow for
optional builds with MPI:
{ ...
, mpi
, useMpi ? false
}
Diffstat (limited to 'pkgs/development')
8 files changed, 25 insertions, 26 deletions
diff --git a/pkgs/development/libraries/boost/generic.nix b/pkgs/development/libraries/boost/generic.nix index 5a2b437d41dc..abff1268bfae 100644 --- a/pkgs/development/libraries/boost/generic.nix +++ b/pkgs/development/libraries/boost/generic.nix @@ -14,7 +14,8 @@ , enableNumpy ? false , taggedLayout ? ((enableRelease && enableDebug) || (enableSingleThreaded && enableMultiThreaded) || (enableShared && enableStatic)) , patches ? [] -, mpi ? null +, useMpi ? false +, mpi , extraB2Args ? [] # Attributes inherit from specific versions @@ -94,7 +95,7 @@ let ++ optional (variant == "release") "debug-symbols=off" ++ optional (toolset != null) "toolset=${toolset}" ++ optional (!enablePython) "--without-python" - ++ optional (mpi != null || stdenv.hostPlatform != stdenv.buildPlatform) "--user-config=user-config.jam" + ++ optional (useMpi || stdenv.hostPlatform != stdenv.buildPlatform) "--user-config=user-config.jam" ++ optionals (stdenv.hostPlatform.libc == "msvcrt") [ "threadapi=win32" ] ++ extraB2Args @@ -140,7 +141,7 @@ stdenv.mkDerivation { substituteInPlace tools/build/src/tools/clang-darwin.jam \ --replace '@rpath/$(<[1]:D=)' "$out/lib/\$(<[1]:D=)"; fi; - '' + optionalString (mpi != null) '' + '' + optionalString useMpi '' cat << EOF >> user-config.jam using mpi : ${mpi}/bin/mpiCC ; EOF diff --git a/pkgs/development/libraries/globalarrays/default.nix b/pkgs/development/libraries/globalarrays/default.nix index 049e262bb17b..1f70946cbcec 100644 --- a/pkgs/development/libraries/globalarrays/default.nix +++ b/pkgs/development/libraries/globalarrays/default.nix @@ -1,5 +1,5 @@ { lib, stdenv, fetchpatch, fetchFromGitHub, autoreconfHook -, blas, gfortran, openssh, openmpi +, blas, gfortran, openssh, mpi } : let @@ -17,7 +17,7 @@ in stdenv.mkDerivation { }; nativeBuildInputs = [ autoreconfHook ]; - buildInputs = [ openmpi blas gfortran openssh ]; + buildInputs = [ mpi blas gfortran openssh ]; preConfigure = '' configureFlagsArray+=( "--enable-i8" \ diff --git a/pkgs/development/libraries/precice/default.nix b/pkgs/development/libraries/precice/default.nix index e9e67737d328..10a80ebb91e8 100644 --- a/pkgs/development/libraries/precice/default.nix +++ b/pkgs/development/libraries/precice/default.nix @@ -1,4 +1,4 @@ -{ lib, stdenv, fetchFromGitHub, cmake, gcc, boost, eigen, libxml2, openmpi, python3, petsc }: +{ lib, stdenv, fetchFromGitHub, cmake, gcc, boost, eigen, libxml2, mpi, python3, petsc }: stdenv.mkDerivation rec { pname = "precice"; @@ -21,7 +21,7 @@ stdenv.mkDerivation rec { NIX_CFLAGS_COMPILE = lib.optional stdenv.isDarwin [ "-D_GNU_SOURCE" ]; nativeBuildInputs = [ cmake gcc ]; - buildInputs = [ boost eigen libxml2 openmpi python3 python3.pkgs.numpy ]; + buildInputs = [ boost eigen libxml2 mpi python3 python3.pkgs.numpy ]; meta = { description = "preCICE stands for Precise Code Interaction Coupling Environment"; diff --git a/pkgs/development/python-modules/cntk/default.nix b/pkgs/development/python-modules/cntk/default.nix index ce388bb1d5d2..b1bba8cf1e0f 100644 --- a/pkgs/development/python-modules/cntk/default.nix +++ b/pkgs/development/python-modules/cntk/default.nix @@ -3,7 +3,7 @@ , pkgs , numpy , scipy -, openmpi +, mpi , enum34 , protobuf , pip @@ -17,8 +17,8 @@ in buildPythonPackage { inherit (cntk) name version src; - nativeBuildInputs = [ swig openmpi ]; - buildInputs = [ cntk openmpi ]; + nativeBuildInputs = [ swig mpi ]; + buildInputs = [ cntk mpi ]; propagatedBuildInputs = [ numpy scipy enum34 protobuf pip ]; CNTK_LIB_PATH = "${cntk}/lib"; @@ -28,7 +28,7 @@ buildPythonPackage { postPatch = '' cd bindings/python - sed -i 's,"libmpi.so.12","${openmpi}/lib/libmpi.so",g' cntk/train/distributed.py + sed -i 's,"libmpi.so.12","${mpi}/lib/libmpi.so",g' cntk/train/distributed.py # Remove distro and libs checks; they aren't compatible with NixOS and besides we guarantee # compatibility by providing a package. diff --git a/pkgs/development/python-modules/pytorch/default.nix b/pkgs/development/python-modules/pytorch/default.nix index f072972937a9..822586bf190c 100644 --- a/pkgs/development/python-modules/pytorch/default.nix +++ b/pkgs/development/python-modules/pytorch/default.nix @@ -1,7 +1,7 @@ { stdenv, lib, fetchFromGitHub, fetchpatch, buildPythonPackage, python, cudaSupport ? false, cudatoolkit ? null, cudnn ? null, nccl ? null, magma ? null, mklDnnSupport ? true, useSystemNccl ? true, - openMPISupport ? false, openmpi ? null, + MPISupport ? false, mpi, buildDocs ? false, cudaArchList ? null, @@ -29,8 +29,6 @@ isPy3k, pythonOlder }: -assert !openMPISupport || openmpi != null; - # assert that everything needed for cuda is present and that the correct cuda versions are used assert !cudaSupport || cudatoolkit != null; assert cudnn == null || cudatoolkit != null; @@ -38,7 +36,7 @@ assert !cudaSupport || (let majorIs = lib.versions.major cudatoolkit.version; in majorIs == "9" || majorIs == "10" || majorIs == "11"); # confirm that cudatoolkits are sync'd across dependencies -assert !(openMPISupport && cudaSupport) || openmpi.cudatoolkit == cudatoolkit; +assert !(MPISupport && cudaSupport) || mpi.cudatoolkit == cudatoolkit; assert !cudaSupport || magma.cudatoolkit == cudatoolkit; let @@ -224,7 +222,7 @@ in buildPythonPackage rec { typing-extensions # the following are required for tensorboard support pillow six future tensorflow-tensorboard protobuf - ] ++ lib.optionals openMPISupport [ openmpi ] + ] ++ lib.optionals MPISupport [ mpi ] ++ lib.optionals (pythonOlder "3.7") [ dataclasses ]; checkInputs = [ hypothesis ninja psutil ]; diff --git a/pkgs/development/python-modules/tensorflow/default.nix b/pkgs/development/python-modules/tensorflow/default.nix index 342a87a6e8d5..9f64a689e2b8 100644 --- a/pkgs/development/python-modules/tensorflow/default.nix +++ b/pkgs/development/python-modules/tensorflow/default.nix @@ -11,7 +11,7 @@ # Common deps , git, pybind11, which, binutils, glibcLocales, cython, perl # Common libraries -, jemalloc, openmpi, gast, grpc, sqlite, boringssl, jsoncpp +, jemalloc, mpi, gast, grpc, sqlite, boringssl, jsoncpp , curl, snappy, flatbuffers-core, lmdb-core, icu, double-conversion, libpng, libjpeg_turbo, giflib # Upsteam by default includes cuda support since tensorflow 1.15. We could do # that in nix as well. It would make some things easier and less confusing, but @@ -129,7 +129,7 @@ let buildInputs = [ jemalloc - openmpi + mpi glibcLocales git diff --git a/pkgs/development/r-modules/default.nix b/pkgs/development/r-modules/default.nix index e79504bdaab3..e827c9b807fd 100644 --- a/pkgs/development/r-modules/default.nix +++ b/pkgs/development/r-modules/default.nix @@ -237,7 +237,7 @@ let BayesSAE = [ pkgs.gsl_1 ]; BayesVarSel = [ pkgs.gsl_1 ]; BayesXsrc = [ pkgs.readline.dev pkgs.ncurses ]; - bigGP = [ pkgs.openmpi ]; + bigGP = [ pkgs.mpi ]; bio3d = [ pkgs.zlib ]; BiocCheck = [ pkgs.which ]; Biostrings = [ pkgs.zlib ]; @@ -284,8 +284,8 @@ let n1qn1 = [ pkgs.gfortran ]; odbc = [ pkgs.unixODBC ]; pander = [ pkgs.pandoc pkgs.which ]; - pbdMPI = [ pkgs.openmpi ]; - pbdPROF = [ pkgs.openmpi ]; + pbdMPI = [ pkgs.mpi ]; + pbdPROF = [ pkgs.mpi ]; pbdZMQ = lib.optionals stdenv.isDarwin [ pkgs.which ]; pdftools = [ pkgs.poppler.dev ]; phytools = [ pkgs.which ]; @@ -309,14 +309,14 @@ let RGtk2 = [ pkgs.gtk2.dev ]; rhdf5 = [ pkgs.zlib ]; Rhdf5lib = [ pkgs.zlib ]; - Rhpc = [ pkgs.zlib pkgs.bzip2.dev pkgs.icu pkgs.lzma.dev pkgs.openmpi pkgs.pcre.dev ]; + Rhpc = [ pkgs.zlib pkgs.bzip2.dev pkgs.icu pkgs.lzma.dev pkgs.mpi pkgs.pcre.dev ]; Rhtslib = [ pkgs.zlib.dev pkgs.automake pkgs.autoconf pkgs.bzip2.dev pkgs.lzma.dev pkgs.curl.dev ]; rjags = [ pkgs.jags ]; rJava = [ pkgs.zlib pkgs.bzip2.dev pkgs.icu pkgs.lzma.dev pkgs.pcre.dev pkgs.jdk pkgs.libzip ]; Rlibeemd = [ pkgs.gsl_1 ]; rmatio = [ pkgs.zlib.dev ]; Rmpfr = [ pkgs.gmp pkgs.mpfr.dev ]; - Rmpi = [ pkgs.openmpi ]; + Rmpi = [ pkgs.mpi ]; RMySQL = [ pkgs.zlib pkgs.libmysqlclient pkgs.openssl.dev ]; RNetCDF = [ pkgs.netcdf pkgs.udunits ]; RODBC = [ pkgs.libiodbc ]; diff --git a/pkgs/development/tools/poetry2nix/poetry2nix/overrides.nix b/pkgs/development/tools/poetry2nix/poetry2nix/overrides.nix index 310657589141..c3cea53189ba 100644 --- a/pkgs/development/tools/poetry2nix/poetry2nix/overrides.nix +++ b/pkgs/development/tools/poetry2nix/poetry2nix/overrides.nix @@ -245,7 +245,7 @@ self: super: horovod = super.horovod.overridePythonAttrs ( old: { - propagatedBuildInputs = old.propagatedBuildInputs ++ [ pkgs.openmpi ]; + propagatedBuildInputs = old.propagatedBuildInputs ++ [ pkgs.mpi ]; } ); @@ -528,14 +528,14 @@ self: super: { } { mpi = { - mpicc = "${pkgs.openmpi.outPath}/bin/mpicc"; + mpicc = "${pkgs.mpi.outPath}/bin/mpicc"; }; } ); }; in { - propagatedBuildInputs = old.propagatedBuildInputs ++ [ pkgs.openmpi ]; + propagatedBuildInputs = old.propagatedBuildInputs ++ [ pkgs.mpi ]; enableParallelBuilding = true; preBuild = '' ln -sf ${cfg} mpi.cfg |