diff options
author | Frederik Rietdijk <fridh@fridh.nl> | 2022-04-03 11:19:04 +0200 |
---|---|---|
committer | Frederik Rietdijk <freddyrietdijk@fridh.nl> | 2022-04-09 08:50:22 +0200 |
commit | 1d63f89caaf140ac33f1796dc0aaeb20f4ac4e55 (patch) | |
tree | 82668e186706e393a7f43c029be21ea4abfc44e9 | |
parent | 934190fa06dc0de1fe7eecc543583d91657fb5fa (diff) |
cudaPackages: overhaul of how we package cuda packages
There are many different versions of the `cudatoolkit` and related
cuda packages, and it can be tricky to ensure they remain compatible.
- `cudaPackages` is now a package set with `cudatoolkit`, `cudnn`, `cutensor`, `nccl`, as well as `cudatoolkit` split into smaller packages ("redist");
- expressions should now use `cudaPackages` as parameter instead of the individual cuda packages;
- `makeScope` is now used, so it is possible to use `.overrideScope'` to set e.g. a different `cudnn` version;
- `release-cuda.nix` is introduced to easily evaluate cuda packages using hydra.
50 files changed, 3352 insertions, 523 deletions
diff --git a/doc/languages-frameworks/cuda.section.md b/doc/languages-frameworks/cuda.section.md new file mode 100644 index 000000000000..fccf66bf79d2 --- /dev/null +++ b/doc/languages-frameworks/cuda.section.md @@ -0,0 +1,34 @@ +# CUDA {#cuda} + +CUDA-only packages are stored in the `cudaPackages` packages set. This set +includes the `cudatoolkit`, portions of the toolkit in separate derivations, +`cudnn`, `cutensor` and `nccl`. + +A package set is available for each CUDA version, so for example +`cudaPackages_11_6`. Within each set is a matching version of the above listed +packages. Additionally, other versions of the packages that are packaged and +compatible are available as well. For example, there can be a +`cudaPackages.cudnn_8_3_2` package. + +To use one or more CUDA packages in an expression, give the expression a `cudaPackages` parameter, and in case CUDA is optional +```nix +cudaSupport ? false +cudaPackages ? {} +``` + +When using `callPackage`, you can choose to pass in a different variant, e.g. +when a different version of the toolkit suffices +```nix +mypkg = callPackage { cudaPackages = cudaPackages_11_5; } +``` + +If another version of say `cudnn` or `cutensor` is needed, you can override the +package set to make it the default. This guarantees you get a consistent package +set. +```nix +mypkg = let + cudaPackages = cudaPackages_11_5.overrideScope' (final: prev { + cudnn = prev.cudnn_8_3_2; + }}); +in callPackage { inherit cudaPackages; }; +``` diff --git a/doc/languages-frameworks/index.xml b/doc/languages-frameworks/index.xml index f221693e764c..144fa96bac12 100644 --- a/doc/languages-frameworks/index.xml +++ b/doc/languages-frameworks/index.xml @@ -11,6 +11,7 @@ <xi:include href="bower.section.xml" /> <xi:include href="coq.section.xml" /> <xi:include href="crystal.section.xml" /> + <xi:include href="cuda.section.xml" /> <xi:include href="dhall.section.xml" /> <xi:include href="dotnet.section.xml" /> <xi:include href="emscripten.section.xml" /> diff --git a/nixos/modules/services/misc/ethminer.nix b/nixos/modules/services/misc/ethminer.nix index 253476d1a23e..223634669828 100644 --- a/nixos/modules/services/misc/ethminer.nix +++ b/nixos/modules/services/misc/ethminer.nix @@ -85,7 +85,7 @@ in config = mkIf cfg.enable { systemd.services.ethminer = { - path = optional (cfg.toolkit == "cuda") [ pkgs.cudatoolkit ]; + path = optional (cfg.toolkit == "cuda") [ pkgs.cudaPackages.cudatoolkit ]; description = "ethminer ethereum mining service"; wantedBy = [ "multi-user.target" ]; after = [ "network.target" ]; diff --git a/pkgs/applications/misc/blender/default.nix b/pkgs/applications/misc/blender/default.nix index a0467b817142..2e3ff53550a9 100644 --- a/pkgs/applications/misc/blender/default.nix +++ b/pkgs/applications/misc/blender/default.nix @@ -5,7 +5,7 @@ , openvdb, libXxf86vm, tbb, alembic , zlib, zstd, fftw, opensubdiv, freetype, jemalloc, ocl-icd, addOpenGLRunpath , jackaudioSupport ? false, libjack2 -, cudaSupport ? config.cudaSupport or false, cudatoolkit_11 +, cudaSupport ? config.cudaSupport or false, cudaPackages ? {} , colladaSupport ? true, opencollada , spaceNavSupport ? stdenv.isLinux, libspnav , makeWrapper @@ -63,7 +63,7 @@ stdenv.mkDerivation rec { llvmPackages.openmp SDL Cocoa CoreGraphics ForceFeedback OpenAL OpenGL ]) ++ optional jackaudioSupport libjack2 - ++ optional cudaSupport cudatoolkit_11 + ++ optional cudaSupport cudaPackages.cudatoolkit ++ optional colladaSupport opencollada ++ optional spaceNavSupport libspnav; pythonPath = with python310Packages; [ numpy requests ]; diff --git a/pkgs/applications/misc/firestarter/default.nix b/pkgs/applications/misc/firestarter/default.nix index 92c517835d83..dca8e1256919 100644 --- a/pkgs/applications/misc/firestarter/default.nix +++ b/pkgs/applications/misc/firestarter/default.nix @@ -8,11 +8,13 @@ , glibc , git , pkg-config -, cudatoolkit +, cudaPackages ? {} , withCuda ? false }: let + inherit (cudaPackages) cudatoolkit; + hwloc = stdenv.mkDerivation rec { pname = "hwloc"; version = "2.2.0"; diff --git a/pkgs/applications/science/math/caffe/default.nix b/pkgs/applications/science/math/caffe/default.nix index db2e74f24523..d6c95633dd52 100644 --- a/pkgs/applications/science/math/caffe/default.nix +++ b/pkgs/applications/science/math/caffe/default.nix @@ -13,13 +13,17 @@ , Accelerate, CoreGraphics, CoreVideo , lmdbSupport ? true, lmdb , leveldbSupport ? true, leveldb, snappy -, cudaSupport ? config.cudaSupport or false, cudatoolkit -, cudnnSupport ? cudaSupport, cudnn ? null -, ncclSupport ? false, nccl ? null +, cudaSupport ? config.cudaSupport or false, cudaPackages ? {} +, cudnnSupport ? cudaSupport +, ncclSupport ? false , pythonSupport ? false, python ? null, numpy ? null , substituteAll }: +let + inherit (cudaPackages) cudatoolkit cudnn nccl; +in + assert leveldbSupport -> (leveldb != null && snappy != null); assert cudnnSupport -> cudaSupport; assert ncclSupport -> cudaSupport; diff --git a/pkgs/applications/science/math/cntk/default.nix b/pkgs/applications/science/math/cntk/default.nix index eb14982575a2..d39c04e8feec 100644 --- a/pkgs/applications/science/math/cntk/default.nix +++ b/pkgs/applications/science/math/cntk/default.nix @@ -2,10 +2,14 @@ , fetchpatch , openblas, blas, lapack, opencv3, libzip, boost, protobuf, mpi , onebitSGDSupport ? false -, cudaSupport ? false, addOpenGLRunpath, cudatoolkit, nvidia_x11 -, cudnnSupport ? cudaSupport, cudnn +, cudaSupport ? false, cudaPackages ? {}, addOpenGLRunpath, cudatoolkit, nvidia_x11 +, cudnnSupport ? cudaSupport }: +let + inherit (cudaPackages) cudatoolkit cudnn; +in + assert cudnnSupport -> cudaSupport; assert blas.implementation == "openblas" && lapack.implementation == "openblas"; diff --git a/pkgs/applications/science/math/mxnet/default.nix b/pkgs/applications/science/math/mxnet/default.nix index 8e5e1ebd7e80..8353226cefed 100644 --- a/pkgs/applications/science/math/mxnet/default.nix +++ b/pkgs/applications/science/math/mxnet/default.nix @@ -1,10 +1,14 @@ { config, stdenv, lib, fetchurl, fetchpatch, bash, cmake , opencv3, gtest, blas, gomp, llvmPackages, perl -, cudaSupport ? config.cudaSupport or false, cudatoolkit, nvidia_x11 -, cudnnSupport ? cudaSupport, cudnn +, cudaSupport ? config.cudaSupport or false, cudaPackages ? {}, nvidia_x11 +, cudnnSupport ? cudaSupport , cudaCapabilities ? [ "3.7" "5.0" "6.0" "7.0" "7.5" "8.0" "8.6" ] }: +let + inherit (cudaPackages) cudatoolkit cudnn; +in + assert cudnnSupport -> cudaSupport; stdenv.mkDerivation rec { diff --git a/pkgs/development/compilers/cudatoolkit/auto-add-opengl-runpath-hook.sh b/pkgs/development/compilers/cudatoolkit/auto-add-opengl-runpath-hook.sh new file mode 100644 index 000000000000..0b0da336514a --- /dev/null +++ b/pkgs/development/compilers/cudatoolkit/auto-add-opengl-runpath-hook.sh @@ -0,0 +1,14 @@ +# Run autoOpenGLRunpath on all files +echo "Sourcing auto-add-opengl-runpath-hook" + +autoAddOpenGLRunpathPhase () { + # TODO: support multiple outputs + for file in $(find ${out,lib,bin} -type f); do + addOpenGLRunpath $file + done +} + +if [ -z "${dontUseAutoAddOpenGLRunpath-}" ]; then + echo "Using autoAddOpenGLRunpathPhase" + postFixupHooks+=(autoAddOpenGLRunpathPhase) +fi diff --git a/pkgs/development/compilers/cudatoolkit/common.nix b/pkgs/development/compilers/cudatoolkit/common.nix index 46832e1cc2de..6375c513f411 100644 --- a/pkgs/development/compilers/cudatoolkit/common.nix +++ b/pkgs/development/compilers/cudatoolkit/common.nix @@ -254,6 +254,7 @@ stdenv.mkDerivation rec { ''; passthru = { cc = gcc; + majorMinorVersion = lib.versions.majorMinor version; majorVersion = lib.versions.majorMinor version; }; diff --git a/pkgs/development/compilers/cudatoolkit/default.nix b/pkgs/development/compilers/cudatoolkit/default.nix deleted file mode 100644 index d7fa6ee59f0a..000000000000 --- a/pkgs/development/compilers/cudatoolkit/default.nix +++ /dev/null @@ -1,94 +0,0 @@ -{ callPackage -, fetchurl -, gcc7 -, gcc9 -, gcc10 -, lib -}: - -let - common = callPackage ./common.nix; -in -rec { - cudatoolkit_10_0 = common { - version = "10.0.130"; - url = "https://developer.nvidia.com/compute/cuda/10.0/Prod/local_installers/cuda_10.0.130_410.48_linux"; - sha256 = "16p3bv1lwmyqpxil8r951h385sy9asc578afrc7lssa68c71ydcj"; - - gcc = gcc7; - }; - - cudatoolkit_10_1 = common { - version = "10.1.243"; - url = "https://developer.download.nvidia.com/compute/cuda/10.1/Prod/local_installers/cuda_10.1.243_418.87.00_linux.run"; - sha256 = "0caxhlv2bdq863dfp6wj7nad66ml81vasq2ayf11psvq2b12vhp7"; - - gcc = gcc7; - }; - - cudatoolkit_10_2 = common { - version = "10.2.89"; - url = "http://developer.download.nvidia.com/compute/cuda/10.2/Prod/local_installers/cuda_10.2.89_440.33.01_linux.run"; - sha256 = "04fasl9sjkb1jvchvqgaqxprnprcz7a8r52249zp2ijarzyhf3an"; - - gcc = gcc7; - }; - - cudatoolkit_10 = cudatoolkit_10_2; - - cudatoolkit_11_0 = common { - version = "11.0.3"; - url = "https://developer.download.nvidia.com/compute/cuda/11.0.3/local_installers/cuda_11.0.3_450.51.06_linux.run"; - sha256 = "1h4c69nfrgm09jzv8xjnjcvpq8n4gnlii17v3wzqry5d13jc8ydh"; - - gcc = gcc9; - }; - - cudatoolkit_11_1 = common { - version = "11.1.1"; - url = "https://developer.download.nvidia.com/compute/cuda/11.1.1/local_installers/cuda_11.1.1_455.32.00_linux.run"; - sha256 = "13yxv2fgvdnqqbwh1zb80x4xhyfkbajfkwyfpdg9493010kngbiy"; - gcc = gcc9; - }; - - cudatoolkit_11_2 = common { - version = "11.2.1"; - url = "https://developer.download.nvidia.com/compute/cuda/11.2.1/local_installers/cuda_11.2.1_460.32.03_linux.run"; - sha256 = "sha256-HamMuJfMX1inRFpKZspPaSaGdwbLOvWKZpzc2Nw9F8g="; - gcc = gcc9; - }; - - cudatoolkit_11_3 = common { - version = "11.3.1"; - url = "https://developer.download.nvidia.com/compute/cuda/11.3.1/local_installers/cuda_11.3.1_465.19.01_linux.run"; - sha256 = "0d19pwcqin76scbw1s5kgj8n0z1p4v1hyfldqmamilyfxycfm4xd"; - gcc = gcc9; - }; - - cudatoolkit_11_4 = common { - version = "11.4.2"; - url = "https://developer.download.nvidia.com/compute/cuda/11.4.2/local_installers/cuda_11.4.2_470.57.02_linux.run"; - sha256 = "sha256-u9h8oOkT+DdFSnljZ0c1E83e9VUILk2G7Zo4ZZzIHwo="; - gcc = gcc10; # can bump to 11 along with stdenv.cc - }; - - cudatoolkit_11_5 = common { - version = "11.5.0"; - url = "https://developer.download.nvidia.com/compute/cuda/11.5.0/local_installers/cuda_11.5.0_495.29.05_linux.run"; - sha256 = "sha256-rgoWk9lJfPPYHmlIlD43lGNpANtxyY1Y7v2sr38aHkw="; - gcc = gcc10; # can bump to 11 along with stdenv.cc - }; - - cudatoolkit_11_6 = common { - version = "11.6.1"; - url = "https://developer.download.nvidia.com/compute/cuda/11.6.1/local_installers/cuda_11.6.1_510.47.03_linux.run"; - sha256 = "sha256-qyGa/OALdCABEyaYZvv/derQN7z8I1UagzjCaEyYTX4="; - gcc = gcc10; # can bump to 11 along with stdenv.cc - }; - - # Make sure to only ever update this to a version that is compatible with the - # latest cudnn, nccl, cutensor, etc! It sometimes happens that CUDA versions - # are released prior to compatibility with the rest of the ecosystem. And - # don't forget to request a review from @NixOS/cuda-maintainers! - cudatoolkit_11 = cudatoolkit_11_5; # update me to 11_6 when cudnn>=8.3.3 -} diff --git a/pkgs/development/compilers/cudatoolkit/extension.nix b/pkgs/development/compilers/cudatoolkit/extension.nix new file mode 100644 index 000000000000..862c83167992 --- /dev/null +++ b/pkgs/development/compilers/cudatoolkit/extension.nix @@ -0,0 +1,15 @@ +final: prev: let + ### Cuda Toolkit + + # Function to build the class cudatoolkit package + buildCudaToolkitPackage = final.callPackage ./common.nix; + + # Version info for the classic cudatoolkit packages that contain everything that is in redist. + cudatoolkitVersions = final.lib.importTOML ./versions.toml; + + ### Add classic cudatoolkit package + cudatoolkit = buildCudaToolkitPackage ((attrs: attrs // { gcc = prev.pkgs.${attrs.gcc}; }) cudatoolkitVersions.${final.cudaVersion}); + +in { + inherit cudatoolkit; +} diff --git a/pkgs/development/compilers/cudatoolkit/redist/build-cuda-redist-package.nix b/pkgs/development/compilers/cudatoolkit/redist/build-cuda-redist-package.nix new file mode 100644 index 000000000000..9bbd7ea1da11 --- /dev/null +++ b/pkgs/development/compilers/cudatoolkit/redist/build-cuda-redist-package.nix @@ -0,0 +1,51 @@ +{ lib +, stdenv +, fetchurl +, autoPatchelfHook +, autoAddOpenGLRunpathHook +}: + +pname: +attrs: + +let + arch = "linux-x86_64"; +in stdenv.mkDerivation { + inherit pname; + inherit (attrs) version; + + src = assert (lib.hasAttr arch attrs); fetchurl { + url = "https://developer.download.nvidia.com/compute/cuda/redist/${attrs.${arch}.relative_path}"; + inherit (attrs.${arch}) sha256; + }; + + nativeBuildInputs = [ + autoPatchelfHook + # This hook will make sure libcuda can be found + # in typically /lib/opengl-driver by adding that + # directory to the rpath of all ELF binaries. + # Check e.g. with `patchelf --print-rpath path/to/my/binary + autoAddOpenGLRunpathHook + ]; + + buildInputs = [ + stdenv.cc.cc.lib + ]; + + dontBuild = true; + + # TODO: choose whether to install static/dynamic libs + installPhase = '' + runHook preInstall + rm LICENSE + mkdir -p $out + mv * $out + runHook postInstall + ''; + + meta = { + description = attrs.name; + license = lib.licenses.unfree; + platforms = lib.optionals (lib.hasAttr arch attrs) [ "x86_64-linux" ]; + }; +} diff --git a/pkgs/development/compilers/cudatoolkit/redist/extension.nix b/pkgs/development/compilers/cudatoolkit/redist/extension.nix new file mode 100644 index 000000000000..65057b90a03c --- /dev/null +++ b/pkgs/development/compilers/cudatoolkit/redist/extension.nix @@ -0,0 +1,32 @@ +final: prev: let + + inherit (final) callPackage; + inherit (prev) cudaVersion lib pkgs; + + ### Cuda Toolkit Redist + + # Manifest files for redist cudatoolkit. These can be found at + # https://developer.download.nvidia.com/compute/cuda/redist/ + cudaToolkitRedistManifests = { + "11.4" = ./manifests/redistrib_11.4.4.json; + "11.5" = ./manifests/redistrib_11.5.2.json; + "11.6" = ./manifests/redistrib_11.6.2.json; + }; + + # Function to build a single cudatoolkit redist package + buildCudaToolkitRedistPackage = callPackage ./build-cuda-redist-package.nix { }; + + # Function that builds all cudatoolkit redist packages given a cuda version and manifest file + buildCudaToolkitRedistPackages = { version, manifest }: let + attrs = lib.filterAttrs (key: value: key != "release_date") (lib.importJSON manifest); + in lib.mapAttrs buildCudaToolkitRedistPackage attrs; + + redistExists = cudaToolkitRedistManifests ? "${cudaVersion}"; + + # All cudatoolkit redist packages for the current cuda version + cudaToolkitRedistPackages = if + lib.hasAttr cudaVersion cudaToolkitRedistManifests + then buildCudaToolkitRedistPackages { version = cudaVersion; manifest = cudaToolkitRedistManifests.${cudaVersion}; } + else {}; + +in cudaToolkitRedistPackages diff --git a/pkgs/development/compilers/cudatoolkit/redist/manifests/redistrib_11.4.4.json b/pkgs/development/compilers/cudatoolkit/redist/manifests/redistrib_11.4.4.json new file mode 100644 index 000000000000..374079b2bcb3 --- /dev/null +++ b/pkgs/development/compilers/cudatoolkit/redist/manifests/redistrib_11.4.4.json @@ -0,0 +1,862 @@ +{ + "release_date": "2022-02-01", + "cuda_cccl": { + "name": "CXX Core Compute Libraries", + "license": "CUDA Toolkit", + "version": "11.4.122", + "linux-x86_64": { + "relative_path": "cuda_cccl/linux-x86_64/cuda_cccl-linux-x86_64-11.4.122-archive.tar.xz", + "sha256": "150b163bde4975154d6238ae1272c74df81b303dccca1f7637bd91027b65afc5", + "md5": "e0e41040716a4d6113410d7cb25c7906", + "size": "951832" + }, + "linux-ppc64le": { + "relative_path": "cuda_cccl/linux-ppc64le/cuda_cccl-linux-ppc64le-11.4.122-archive.tar.xz", + "sha256": "0d1d729ac3f52bdd04f197a6cef295b3463c4975d70dbd97f4e1b08a4857f488", + "md5": "f615eccb3b8d694f4c7aa864c93959d7", + "size": "951780" + }, + "linux-sbsa": { + "relative_path": "cuda_cccl/linux-sbsa/cuda_cccl-linux-sbsa-11.4.122-archive.tar.xz", + "sha256": "98f495b99c68fae54fe728e72aa3b8e2e5b1d77c9fd4946370cec3b063a7fcf3", + "md5": "28692dd487b5936dbfac70ac56b65561", + "size": "951340" + }, + "windows-x86_64": { + "relative_path": "cuda_cccl/windows-x86_64/cuda_cccl-windows-x86_64-11.4.122-archive.zip", + "sha256": "d721c4bcc5bf8054eb944b7c3203ce5f593523f798559f3f4c1effdd8ed36f48", + "md5": "fa6bbf1291ebdd8f63f4f5176be4ae9c", + "size": "2422426" + } + }, + "cuda_cudart": { + "name": "CUDA Runtime (cudart)", + "license": "CUDA Toolkit", + "version": "11.4.148", + "linux-x86_64": { + "relative_path": "cuda_cudart/linux-x86_64/cuda_cudart-linux-x86_64-11.4.148-archive.tar.xz", + "sha256": "36bd6f1ea8711f69405ebd0b26d1f791a6ef6eb01cc3f01ba4231eb8ec09c208", + "md5": "b98036aa5e633eb19b7ef2e32e8811a7", + "size": "828732" + }, + "linux-ppc64le": { + "relative_path": "cuda_cudart/linux-ppc64le/cuda_cudart-linux-ppc64le-11.4.148-archive.tar.xz", + "sha256": "fb70c6477dce2cfc3ee13d361b76453ad92671be36f98fab50e2510f657eb00e", + "md5": "594855b91ad5e9aa674d3a93d741431e", + "size": "777952" + }, + "linux-sbsa": { + "relative_path": "cuda_cudart/linux-sbsa/cuda_cudart-linux-sbsa-11.4.148-archive.tar.xz", + "sha256": "d013448f76c22614e458de94b5289807d2d0934b3a5deb073b621401a222c16f", + "md5": "2bcac7ef368ebeb804e33c4c7920ff78", + "size": "783344" + }, + "windows-x86_64": { + "relative_path": "cuda_cudart/windows-x86_64/cuda_cudart-windows-x86_64-11.4.148-archive.zip", + "sha256": "e889e38929a95845409ed7dcdb0e6189343af5d17442a3b8c9f2ba297f0d262e", + "md5": "c2fcccb16eec0cf4ed5807fcba2bd745", + "size": "2898471" + } + }, + "cuda_cuobjdump": { + "name": "cuobjdump", + "license": "CUDA Toolkit", + "version": "11.4.120", + "linux-x86_64": { + "relative_path": "cuda_cuobjdump/linux-x86_64/cuda_cuobjdump-linux-x86_64-11.4.120-archive.tar.xz", + "sha256": "dd9d51ca6d1772545ba77640f74bd1162ea4ffe223e17c8faa6e0a11ec9b5e2a", + "md5": "0a28c3ee87dbe294afc73b90ff0c8101", + "size": "110008" + }, + "linux-ppc64le": { + "relative_path": "cuda_cuobjdump/linux-ppc64le/cuda_cuobjdump-linux-ppc64le-11.4.120-archive.tar.xz", + "sha256": "3f5013ecd3ad3ef1f90f981189826ae9e5aadb7549a99e834c0f6d77c897a7ff", + "md5": "56cabd1ef862f62c10a6fc9e47186797", + |