diff options
author | Frederik Rietdijk <fridh@fridh.nl> | 2019-09-06 22:47:40 +0200 |
---|---|---|
committer | Frederik Rietdijk <fridh@fridh.nl> | 2019-09-06 22:47:40 +0200 |
commit | 9894a70299497ee1d2bb746374d986e8d0f771e3 (patch) | |
tree | dd3c241c075e47283065a5d70d3510eaa5c6ae5f | |
parent | 66bc7fc1b3e7a85a2cfde9bb121498a181d411a4 (diff) | |
parent | 3263254cfb3163ec5b036d135b7d516f175014da (diff) |
Merge staging into staging-next
66 files changed, 1154 insertions, 552 deletions
diff --git a/doc/languages-frameworks/python.section.md b/doc/languages-frameworks/python.section.md index 4963c97a6c9a..88dc42ebc6c2 100644 --- a/doc/languages-frameworks/python.section.md +++ b/doc/languages-frameworks/python.section.md @@ -540,7 +540,8 @@ and the aliases #### `buildPythonPackage` function The `buildPythonPackage` function is implemented in -`pkgs/development/interpreters/python/build-python-package.nix` +`pkgs/development/interpreters/python/mk-python-derivation` +using setup hooks. The following is an example: ```nix @@ -797,6 +798,22 @@ such as `ignoreCollisions = true` or `postBuild`. If you need them, you have to Python 2 namespace packages may provide `__init__.py` that collide. In that case `python.buildEnv` should be used with `ignoreCollisions = true`. +#### Setup hooks + +The following are setup hooks specifically for Python packages. Most of these are +used in `buildPythonPackage`. + +- `flitBuildHook` to build a wheel using `flit`. +- `pipBuildHook` to build a wheel using `pip` and PEP 517. Note a build system (e.g. `setuptools` or `flit`) should still be added as `nativeBuildInput`. +- `pipInstallHook` to install wheels. +- `pytestCheckHook` to run tests with `pytest`. +- `pythonCatchConflictsHook` to check whether a Python package is not already existing. +- `pythonImportsCheckHook` to check whether importing the listed modules works. +- `pythonRemoveBinBytecode` to remove bytecode from the `/bin` folder. +- `setuptoolsBuildHook` to build a wheel using `setuptools`. +- `setuptoolsCheckHook` to run tests with `python setup.py test`. +- `wheelUnpackHook` to move a wheel to the correct folder so it can be installed with the `pipInstallHook`. + ### Development mode Development or editable mode is supported. To develop Python packages diff --git a/nixos/tests/all-tests.nix b/nixos/tests/all-tests.nix index 5eb8111aa6d3..60762de76d33 100644 --- a/nixos/tests/all-tests.nix +++ b/nixos/tests/all-tests.nix @@ -95,6 +95,7 @@ in gitlab = handleTest ./gitlab.nix {}; gitolite = handleTest ./gitolite.nix {}; gjs = handleTest ./gjs.nix {}; + glib-networking = handleTest ./glib-networking.nix {}; glusterfs = handleTest ./glusterfs.nix {}; gnome3-xorg = handleTest ./gnome3-xorg.nix {}; gnome3 = handleTest ./gnome3.nix {}; @@ -144,6 +145,7 @@ in latestKernel.login = handleTest ./login.nix { latestKernel = true; }; ldap = handleTest ./ldap.nix {}; leaps = handleTest ./leaps.nix {}; + libgdata = handleTest ./libgdata.nix {}; libxmlb = handleTest ./libxmlb.nix {}; lidarr = handleTest ./lidarr.nix {}; lightdm = handleTest ./lightdm.nix {}; diff --git a/nixos/tests/glib-networking.nix b/nixos/tests/glib-networking.nix new file mode 100644 index 000000000000..c0bbb2b3554b --- /dev/null +++ b/nixos/tests/glib-networking.nix @@ -0,0 +1,17 @@ +# run installed tests +import ./make-test.nix ({ pkgs, ... }: + +{ + name = "glib-networking"; + meta = { + maintainers = pkgs.glib-networking.meta.maintainers; + }; + + machine = { pkgs, ... }: { + environment.systemPackages = with pkgs; [ gnome-desktop-testing ]; + }; + + testScript = '' + $machine->succeed("gnome-desktop-testing-runner -d '${pkgs.glib-networking.installedTests}/share'"); + ''; +}) diff --git a/nixos/tests/libgdata.nix b/nixos/tests/libgdata.nix new file mode 100644 index 000000000000..10a3ca97dd22 --- /dev/null +++ b/nixos/tests/libgdata.nix @@ -0,0 +1,21 @@ +# run installed tests +import ./make-test.nix ({ pkgs, ... }: + +{ + name = "libgdata"; + + meta = { + maintainers = pkgs.libgdata.meta.maintainers; + }; + + machine = { pkgs, ... }: { + environment.systemPackages = with pkgs; [ gnome-desktop-testing ]; + # # GLib-GIO-DEBUG: _g_io_module_get_default: Found default implementation dummy (GDummyTlsBackend) for ‘gio-tls-backend’ + # Bail out! libgdata:ERROR:../gdata/tests/common.c:134:gdata_test_init: assertion failed (child_error == NULL): TLS support is not available (g-tls-error-quark, 0) + services.gnome3.glib-networking.enable = true; + }; + + testScript = '' + $machine->succeed("gnome-desktop-testing-runner -d '${pkgs.libgdata.installedTests}/share'"); + ''; +}) diff --git a/pkgs/development/interpreters/python/build-python-package-common.nix b/pkgs/development/interpreters/python/build-python-package-common.nix deleted file mode 100644 index 0f8e088d434a..000000000000 --- a/pkgs/development/interpreters/python/build-python-package-common.nix +++ /dev/null @@ -1,31 +0,0 @@ -# This function provides generic bits to install a Python wheel. - -{ python -}: - -{ buildInputs ? [] -# Additional flags to pass to "pip install". -, installFlags ? [] -, ... } @ attrs: - -attrs // { - buildInputs = buildInputs ++ [ python.pythonForBuild.pkgs.bootstrapped-pip ]; - - configurePhase = attrs.configurePhase or '' - runHook preConfigure - runHook postConfigure - ''; - - installPhase = attrs.installPhase or '' - runHook preInstall - - mkdir -p "$out/${python.sitePackages}" - export PYTHONPATH="$out/${python.sitePackages}:$PYTHONPATH" - - pushd dist - ${python.pythonForBuild.pkgs.bootstrapped-pip}/bin/pip install *.whl --no-index --prefix=$out --no-cache ${toString installFlags} --build tmpbuild - popd - - runHook postInstall - ''; -} diff --git a/pkgs/development/interpreters/python/build-python-package-flit.nix b/pkgs/development/interpreters/python/build-python-package-flit.nix deleted file mode 100644 index b0f9e0380211..000000000000 --- a/pkgs/development/interpreters/python/build-python-package-flit.nix +++ /dev/null @@ -1,22 +0,0 @@ -# This function provides specific bits for building a flit-based Python package. - -{ python -, flit -}: - -{ ... } @ attrs: - -attrs // { - nativeBuildInputs = [ flit ]; - buildPhase = attrs.buildPhase or '' - runHook preBuild - flit build --format wheel - runHook postBuild - ''; - - # Flit packages, like setuptools packages, might have tests. - installCheckPhase = attrs.checkPhase or '' - ${python.interpreter} -m unittest discover - ''; - doCheck = attrs.doCheck or true; -} diff --git a/pkgs/development/interpreters/python/build-python-package-pyproject.nix b/pkgs/development/interpreters/python/build-python-package-pyproject.nix deleted file mode 100644 index 085db44f3e82..000000000000 --- a/pkgs/development/interpreters/python/build-python-package-pyproject.nix +++ /dev/null @@ -1,56 +0,0 @@ -# This function provides specific bits for building a setuptools-based Python package. - -{ lib -, python -}: - -{ -# Global options passed to "python setup.py" - setupPyGlobalFlags ? [] -# Build options passed to "build_ext" -# https://github.com/pypa/pip/issues/881 -# Rename to `buildOptions` because it is not setuptools specific? -, setupPyBuildFlags ? [] -# Execute before shell hook -, preShellHook ? "" -# Execute after shell hook -, postShellHook ? "" -, ... } @ attrs: - -let - pipGlobalFlagsString = lib.concatMapStringsSep " " (option: "--global-option ${option}") setupPyGlobalFlags; - pipBuildFlagsString = lib.concatMapStringsSep " " (option: "--build-option ${option}") setupPyBuildFlags; -in attrs // { - buildPhase = attrs.buildPhase or '' - runHook preBuild - mkdir -p dist - echo "Creating a wheel..." - ${python.pythonForBuild.interpreter} -m pip wheel --no-index --no-deps --no-clean --no-build-isolation --wheel-dir dist ${pipGlobalFlagsString} ${pipBuildFlagsString} . - echo "Finished creating a wheel..." - runHook postBuild - ''; - - installCheckPhase = '' - runHook preCheck - echo "No checkPhase defined. Either provide a checkPhase or disable tests in case tests are not available."; exit 1 - runHook postCheck - ''; - - # With Python it's a common idiom to run the tests - # after the software has been installed. - doCheck = attrs.doCheck or true; - - shellHook = attrs.shellHook or '' - ${preShellHook} - # Long-term setup.py should be dropped. - if [ -e pyproject.toml ]; then - tmp_path=$(mktemp -d) - export PATH="$tmp_path/bin:$PATH" - export PYTHONPATH="$tmp_path/${python.pythonForBuild.sitePackages}:$PYTHONPATH" - mkdir -p $tmp_path/${python.pythonForBuild.sitePackages} - ${python.pythonForBuild.pkgs.bootstrapped-pip}/bin/pip install -e . --prefix $tmp_path >&2 - fi - ${postShellHook} - ''; - -} diff --git a/pkgs/development/interpreters/python/build-python-package-setuptools.nix b/pkgs/development/interpreters/python/build-python-package-setuptools.nix deleted file mode 100644 index 7738ea2f66a5..000000000000 --- a/pkgs/development/interpreters/python/build-python-package-setuptools.nix +++ /dev/null @@ -1,60 +0,0 @@ -# This function provides specific bits for building a setuptools-based Python package. - -{ lib -, python -}: - -{ -# Global options passed to "python setup.py" - setupPyGlobalFlags ? [] -# Build options passed to "python setup.py build_ext" -# https://github.com/pypa/pip/issues/881 -, setupPyBuildFlags ? [] -# Execute before shell hook -, preShellHook ? "" -# Execute after shell hook -, postShellHook ? "" -, ... } @ attrs: - -let - # use setuptools shim (so that setuptools is imported before distutils) - # pip does the same thing: https://github.com/pypa/pip/pull/3265 - setuppy = ./run_setup.py; - - setupPyGlobalFlagsString = lib.concatStringsSep " " setupPyGlobalFlags; - setupPyBuildExtString = lib.optionalString (setupPyBuildFlags != []) ("build_ext " + (lib.concatStringsSep " " setupPyBuildFlags)); - -in attrs // { - # we copy nix_run_setup over so it's executed relative to the root of the source - # many project make that assumption - buildPhase = attrs.buildPhase or '' - runHook preBuild - cp ${setuppy} nix_run_setup - ${python.pythonForBuild.interpreter} nix_run_setup ${setupPyGlobalFlagsString} ${setupPyBuildExtString} bdist_wheel - runHook postBuild - ''; - - installCheckPhase = attrs.checkPhase or '' - runHook preCheck - ${python.pythonForBuild.interpreter} nix_run_setup test - runHook postCheck - ''; - - # Python packages that are installed with setuptools - # are typically distributed with tests. - # With Python it's a common idiom to run the tests - # after the software has been installed. - doCheck = attrs.doCheck or true; - - shellHook = attrs.shellHook or '' - ${preShellHook} - if test -e setup.py; then - tmp_path=$(mktemp -d) - export PATH="$tmp_path/bin:$PATH" - export PYTHONPATH="$tmp_path/${python.pythonForBuild.sitePackages}:$PYTHONPATH" - mkdir -p $tmp_path/${python.pythonForBuild.sitePackages} - ${python.pythonForBuild.pkgs.bootstrapped-pip}/bin/pip install -e . --prefix $tmp_path >&2 - fi - ${postShellHook} - ''; -} diff --git a/pkgs/development/interpreters/python/build-python-package-wheel.nix b/pkgs/development/interpreters/python/build-python-package-wheel.nix deleted file mode 100644 index e3c4e13c0e2d..000000000000 --- a/pkgs/development/interpreters/python/build-python-package-wheel.nix +++ /dev/null @@ -1,20 +0,0 @@ -# This function provides specific bits for building a wheel-based Python package. - -{ -}: - -{ ... } @ attrs: - -attrs // { - unpackPhase = '' - mkdir dist - cp "$src" "dist/$(stripHash "$src")" - ''; - - # Wheels are pre-compiled - buildPhase = attrs.buildPhase or ":"; - installCheckPhase = attrs.checkPhase or ":"; - - # Wheels don't have any checks to run - doCheck = attrs.doCheck or false; -}
\ No newline at end of file diff --git a/pkgs/development/interpreters/python/build-python-package.nix b/pkgs/development/interpreters/python/build-python-package.nix deleted file mode 100644 index 61c1186cef9e..000000000000 --- a/pkgs/development/interpreters/python/build-python-package.nix +++ /dev/null @@ -1,48 +0,0 @@ -# This function provides a generic Python package builder, -# and can build packages that use distutils, setuptools or flit. - -{ lib -, config -, python -, wrapPython -, setuptools -, unzip -, ensureNewerSourcesForZipFilesHook -, toPythonModule -, namePrefix -, flit -, writeScript -, update-python-libraries -}: - -let - setuptools-specific = import ./build-python-package-setuptools.nix { inherit lib python; }; - pyproject-specific = import ./build-python-package-pyproject.nix { inherit lib python; }; - flit-specific = import ./build-python-package-flit.nix { inherit python flit; }; - wheel-specific = import ./build-python-package-wheel.nix { }; - common = import ./build-python-package-common.nix { inherit python; }; - mkPythonDerivation = import ./mk-python-derivation.nix { - inherit lib config python wrapPython setuptools unzip ensureNewerSourcesForZipFilesHook; - inherit toPythonModule namePrefix update-python-libraries; - }; -in - -{ -# Several package formats are supported. -# "setuptools" : Install a common setuptools/distutils based package. This builds a wheel. -# "wheel" : Install from a pre-compiled wheel. -# "flit" : Install a flit package. This builds a wheel. -# "other" : Provide your own buildPhase and installPhase. -format ? "setuptools" -, ... } @ attrs: - -let - formatspecific = - if format == "pyproject" then common (pyproject-specific attrs) - else if format == "setuptools" then common (setuptools-specific attrs) - else if format == "flit" then common (flit-specific attrs) - else if format == "wheel" then common (wheel-specific attrs) - else if format == "other" then {} - else throw "Unsupported format ${format}"; - -in mkPythonDerivation ( attrs // formatspecific ) diff --git a/pkgs/development/interpreters/python/hooks/default.nix b/pkgs/development/interpreters/python/hooks/default.nix new file mode 100644 index 000000000000..9a7ec98ba178 --- /dev/null +++ b/pkgs/development/interpreters/python/hooks/default.nix @@ -0,0 +1,95 @@ +# Hooks for building Python packages. +{ python +, callPackage +, makeSetupHook +}: + +let + pythonInterpreter = python.pythonForBuild.interpreter; + pythonSitePackages = python.sitePackages; + pythonCheckInterpreter = python.interpreter; + setuppy = ../run_setup.py; +in rec { + + flitBuildHook = callPackage ({ flit }: + makeSetupHook { + name = "flit-build-hook"; + deps = [ flit ]; + substitutions = { + inherit pythonInterpreter; + }; + } ./flit-build-hook.sh) {}; + + pipBuildHook = callPackage ({ pip }: + makeSetupHook { + name = "pip-build-hook.sh"; + deps = [ pip ]; + substitutions = { + inherit pythonInterpreter pythonSitePackages; + }; + } ./pip-build-hook.sh) {}; + + pipInstallHook = callPackage ({ pip }: + makeSetupHook { + name = "pip-install-hook"; + deps = [ pip ]; + substitutions = { + inherit pythonInterpreter pythonSitePackages; + }; + } ./pip-install-hook.sh) {}; + + pytestCheckHook = callPackage ({ pytest }: + makeSetupHook { + name = "pytest-check-hook"; + deps = [ pytest ]; + substitutions = { + inherit pythonCheckInterpreter; + }; + } ./pytest-check-hook.sh) {}; + + pythonCatchConflictsHook = callPackage ({ setuptools }: + makeSetupHook { + name = "python-catch-conflicts-hook"; + deps = [ setuptools ]; + substitutions = { + inherit pythonInterpreter; + catchConflicts=../catch_conflicts/catch_conflicts.py; + }; + } ./python-catch-conflicts-hook.sh) {}; + + pythonImportsCheckHook = callPackage ({}: + makeSetupHook { + name = "python-imports-check-hook.sh"; + substitutions = { + inherit pythonCheckInterpreter; + }; + } ./python-imports-check-hook.sh) {}; + + pythonRemoveBinBytecodeHook = callPackage ({ }: + |