Merge staging into staging-next

66 files changed, 1154 insertions, 552 deletions

diff --git a/doc/languages-frameworks/python.section.md b/doc/languages-frameworks/python.section.md
index 4963c97a6c9a..88dc42ebc6c2 100644
--- a/doc/languages-frameworks/python.section.md
+++ b/doc/languages-frameworks/python.section.md
@@ -540,7 +540,8 @@ and the aliases
 #### `buildPythonPackage` function
 
 The `buildPythonPackage` function is implemented in
-`pkgs/development/interpreters/python/build-python-package.nix`
+`pkgs/development/interpreters/python/mk-python-derivation`
+using setup hooks.
 
 The following is an example:
 ```nix
@@ -797,6 +798,22 @@ such as `ignoreCollisions = true` or `postBuild`. If you need them, you have to
 Python 2 namespace packages may provide `__init__.py` that collide. In that case `python.buildEnv`
 should be used with `ignoreCollisions = true`.
 
+#### Setup hooks
+
+The following are setup hooks specifically for Python packages. Most of these are
+used in `buildPythonPackage`.
+
+- `flitBuildHook` to build a wheel using `flit`.
+- `pipBuildHook` to build a wheel using `pip` and PEP 517. Note a build system (e.g. `setuptools` or `flit`) should still be added as `nativeBuildInput`.
+- `pipInstallHook` to install wheels.
+- `pytestCheckHook` to run tests with `pytest`.
+- `pythonCatchConflictsHook` to check whether a Python package is not already existing.
+- `pythonImportsCheckHook` to check whether importing the listed modules works.
+- `pythonRemoveBinBytecode` to remove bytecode from the `/bin` folder.
+- `setuptoolsBuildHook` to build a wheel using `setuptools`.
+- `setuptoolsCheckHook` to run tests with `python setup.py test`.
+- `wheelUnpackHook` to move a wheel to the correct folder so it can be installed with the `pipInstallHook`.
+
 ### Development mode
 
 Development or editable mode is supported. To develop Python packages
diff --git a/nixos/tests/all-tests.nix b/nixos/tests/all-tests.nix
index 5eb8111aa6d3..60762de76d33 100644
--- a/nixos/tests/all-tests.nix
+++ b/nixos/tests/all-tests.nix
@@ -95,6 +95,7 @@ in
   gitlab = handleTest ./gitlab.nix {};
   gitolite = handleTest ./gitolite.nix {};
   gjs = handleTest ./gjs.nix {};
+  glib-networking = handleTest ./glib-networking.nix {};
   glusterfs = handleTest ./glusterfs.nix {};
   gnome3-xorg = handleTest ./gnome3-xorg.nix {};
   gnome3 = handleTest ./gnome3.nix {};
@@ -144,6 +145,7 @@ in
   latestKernel.login = handleTest ./login.nix { latestKernel = true; };
   ldap = handleTest ./ldap.nix {};
   leaps = handleTest ./leaps.nix {};
+  libgdata = handleTest ./libgdata.nix {};
   libxmlb = handleTest ./libxmlb.nix {};
   lidarr = handleTest ./lidarr.nix {};
   lightdm = handleTest ./lightdm.nix {};
diff --git a/nixos/tests/glib-networking.nix b/nixos/tests/glib-networking.nix
new file mode 100644
index 000000000000..c0bbb2b3554b
--- /dev/null
+++ b/nixos/tests/glib-networking.nix
@@ -0,0 +1,17 @@
+# run installed tests
+import ./make-test.nix ({ pkgs, ... }:
+
+{
+  name = "glib-networking";
+  meta = {
+    maintainers = pkgs.glib-networking.meta.maintainers;
+  };
+
+  machine = { pkgs, ... }: {
+    environment.systemPackages = with pkgs; [ gnome-desktop-testing ];
+  };
+
+  testScript = ''
+    $machine->succeed("gnome-desktop-testing-runner -d '${pkgs.glib-networking.installedTests}/share'");
+  '';
+})
diff --git a/nixos/tests/libgdata.nix b/nixos/tests/libgdata.nix
new file mode 100644
index 000000000000..10a3ca97dd22
--- /dev/null
+++ b/nixos/tests/libgdata.nix
@@ -0,0 +1,21 @@
+# run installed tests
+import ./make-test.nix ({ pkgs, ... }:
+
+{
+  name = "libgdata";
+
+  meta = {
+    maintainers = pkgs.libgdata.meta.maintainers;
+  };
+
+  machine = { pkgs, ... }: {
+    environment.systemPackages = with pkgs; [ gnome-desktop-testing ];
+    # # GLib-GIO-DEBUG: _g_io_module_get_default: Found default implementation dummy (GDummyTlsBackend) for ‘gio-tls-backend’
+    # Bail out! libgdata:ERROR:../gdata/tests/common.c:134:gdata_test_init: assertion failed (child_error == NULL): TLS support is not available (g-tls-error-quark, 0)
+    services.gnome3.glib-networking.enable = true;
+  };
+
+  testScript = ''
+    $machine->succeed("gnome-desktop-testing-runner -d '${pkgs.libgdata.installedTests}/share'");
+  '';
+})
diff --git a/pkgs/development/interpreters/python/build-python-package-common.nix b/pkgs/development/interpreters/python/build-python-package-common.nix
deleted file mode 100644
index 0f8e088d434a..000000000000
--- a/pkgs/development/interpreters/python/build-python-package-common.nix
+++ /dev/null
@@ -1,31 +0,0 @@
-# This function provides generic bits to install a Python wheel.
-
-{ python
-}:
-
-{ buildInputs ? []
-# Additional flags to pass to "pip install".
-, installFlags ? []
-, ... } @ attrs:
-
-attrs // {
-  buildInputs = buildInputs ++ [ python.pythonForBuild.pkgs.bootstrapped-pip ];
-
-  configurePhase = attrs.configurePhase or ''
-    runHook preConfigure
-    runHook postConfigure
-  '';
-
-  installPhase = attrs.installPhase or ''
-    runHook preInstall
-
-    mkdir -p "$out/${python.sitePackages}"
-    export PYTHONPATH="$out/${python.sitePackages}:$PYTHONPATH"
-
-    pushd dist
-    ${python.pythonForBuild.pkgs.bootstrapped-pip}/bin/pip install *.whl --no-index --prefix=$out --no-cache ${toString installFlags} --build tmpbuild
-    popd
-
-    runHook postInstall
-  '';
-}
diff --git a/pkgs/development/interpreters/python/build-python-package-flit.nix b/pkgs/development/interpreters/python/build-python-package-flit.nix
deleted file mode 100644
index b0f9e0380211..000000000000
--- a/pkgs/development/interpreters/python/build-python-package-flit.nix
+++ /dev/null
@@ -1,22 +0,0 @@
-# This function provides specific bits for building a flit-based Python package.
-
-{ python
-, flit
-}:
-
-{ ... } @ attrs:
-
-attrs // {
-  nativeBuildInputs = [ flit ];
-  buildPhase = attrs.buildPhase or ''
-    runHook preBuild
-    flit build --format wheel
-    runHook postBuild
-  '';
-
-  # Flit packages, like setuptools packages, might have tests.
-  installCheckPhase = attrs.checkPhase or ''
-    ${python.interpreter} -m unittest discover
-  '';
-  doCheck = attrs.doCheck or true;
-}
diff --git a/pkgs/development/interpreters/python/build-python-package-pyproject.nix b/pkgs/development/interpreters/python/build-python-package-pyproject.nix
deleted file mode 100644
index 085db44f3e82..000000000000
--- a/pkgs/development/interpreters/python/build-python-package-pyproject.nix
+++ /dev/null
@@ -1,56 +0,0 @@
-# This function provides specific bits for building a setuptools-based Python package.
-
-{ lib
-, python
-}:
-
-{
-# Global options passed to "python setup.py"
-  setupPyGlobalFlags ? []
-# Build options passed to "build_ext"
-# https://github.com/pypa/pip/issues/881
-# Rename to `buildOptions` because it is not setuptools specific?
-, setupPyBuildFlags ? []
-# Execute before shell hook
-, preShellHook ? ""
-# Execute after shell hook
-, postShellHook ? ""
-, ... } @ attrs:
-
-let
-  pipGlobalFlagsString = lib.concatMapStringsSep " " (option: "--global-option ${option}") setupPyGlobalFlags;
-  pipBuildFlagsString = lib.concatMapStringsSep " " (option: "--build-option ${option}") setupPyBuildFlags;
-in attrs // {
-  buildPhase = attrs.buildPhase or ''
-    runHook preBuild
-    mkdir -p dist
-    echo "Creating a wheel..."
-    ${python.pythonForBuild.interpreter} -m pip wheel --no-index --no-deps --no-clean --no-build-isolation --wheel-dir dist ${pipGlobalFlagsString} ${pipBuildFlagsString} .
-    echo "Finished creating a wheel..."
-    runHook postBuild
-  '';
-
-  installCheckPhase = ''
-    runHook preCheck
-    echo "No checkPhase defined. Either provide a checkPhase or disable tests in case tests are not available."; exit 1
-    runHook postCheck
-  '';
-
-  # With Python it's a common idiom to run the tests
-  # after the software has been installed.
-  doCheck = attrs.doCheck or true;
-
-  shellHook = attrs.shellHook or ''
-    ${preShellHook}
-    # Long-term setup.py should be dropped.
-    if [ -e pyproject.toml ]; then
-      tmp_path=$(mktemp -d)
-      export PATH="$tmp_path/bin:$PATH"
-      export PYTHONPATH="$tmp_path/${python.pythonForBuild.sitePackages}:$PYTHONPATH"
-      mkdir -p $tmp_path/${python.pythonForBuild.sitePackages}
-      ${python.pythonForBuild.pkgs.bootstrapped-pip}/bin/pip install -e . --prefix $tmp_path >&2
-    fi
-    ${postShellHook}
-  '';
-
-}
diff --git a/pkgs/development/interpreters/python/build-python-package-setuptools.nix b/pkgs/development/interpreters/python/build-python-package-setuptools.nix
deleted file mode 100644
index 7738ea2f66a5..000000000000
--- a/pkgs/development/interpreters/python/build-python-package-setuptools.nix
+++ /dev/null
@@ -1,60 +0,0 @@
-# This function provides specific bits for building a setuptools-based Python package.
-
-{ lib
-, python
-}:
-
-{
-# Global options passed to "python setup.py"
-  setupPyGlobalFlags ? []
-# Build options passed to "python setup.py build_ext"
-# https://github.com/pypa/pip/issues/881
-, setupPyBuildFlags ? []
-# Execute before shell hook
-, preShellHook ? ""
-# Execute after shell hook
-, postShellHook ? ""
-, ... } @ attrs:
-
-let
-  # use setuptools shim (so that setuptools is imported before distutils)
-  # pip does the same thing: https://github.com/pypa/pip/pull/3265
-  setuppy = ./run_setup.py;
-
-  setupPyGlobalFlagsString = lib.concatStringsSep " " setupPyGlobalFlags;
-  setupPyBuildExtString = lib.optionalString (setupPyBuildFlags != []) ("build_ext " + (lib.concatStringsSep " " setupPyBuildFlags));
-
-in attrs // {
-  # we copy nix_run_setup over so it's executed relative to the root of the source
-  # many project make that assumption
-  buildPhase = attrs.buildPhase or ''
-    runHook preBuild
-    cp ${setuppy} nix_run_setup
-    ${python.pythonForBuild.interpreter} nix_run_setup ${setupPyGlobalFlagsString} ${setupPyBuildExtString} bdist_wheel
-    runHook postBuild
-  '';
-
-  installCheckPhase = attrs.checkPhase or ''
-    runHook preCheck
-    ${python.pythonForBuild.interpreter} nix_run_setup test
-    runHook postCheck
-  '';
-
-  # Python packages that are installed with setuptools
-  # are typically distributed with tests.
-  # With Python it's a common idiom to run the tests
-  # after the software has been installed.
-  doCheck = attrs.doCheck or true;
-
-  shellHook = attrs.shellHook or ''
-    ${preShellHook}
-    if test -e setup.py; then
-      tmp_path=$(mktemp -d)
-      export PATH="$tmp_path/bin:$PATH"
-      export PYTHONPATH="$tmp_path/${python.pythonForBuild.sitePackages}:$PYTHONPATH"
-      mkdir -p $tmp_path/${python.pythonForBuild.sitePackages}
-      ${python.pythonForBuild.pkgs.bootstrapped-pip}/bin/pip install -e . --prefix $tmp_path >&2
-    fi
-    ${postShellHook}
-  '';
-}
diff --git a/pkgs/development/interpreters/python/build-python-package-wheel.nix b/pkgs/development/interpreters/python/build-python-package-wheel.nix
deleted file mode 100644
index e3c4e13c0e2d..000000000000
--- a/pkgs/development/interpreters/python/build-python-package-wheel.nix
+++ /dev/null
@@ -1,20 +0,0 @@
-# This function provides specific bits for building a wheel-based Python package.
-
-{
-}:
-
-{ ... } @ attrs:
-
-attrs // {
-  unpackPhase = ''
-    mkdir dist
-    cp "$src" "dist/$(stripHash "$src")"
-  '';
-
-  # Wheels are pre-compiled
-  buildPhase = attrs.buildPhase or ":";
-  installCheckPhase = attrs.checkPhase or ":";
-
-  # Wheels don't have any checks to run
-  doCheck = attrs.doCheck or false;
-}
\ No newline at end of file
diff --git a/pkgs/development/interpreters/python/build-python-package.nix b/pkgs/development/interpreters/python/build-python-package.nix
deleted file mode 100644
index 61c1186cef9e..000000000000
--- a/pkgs/development/interpreters/python/build-python-package.nix
+++ /dev/null
@@ -1,48 +0,0 @@
-# This function provides a generic Python package builder,
-# and can build packages that use distutils, setuptools or flit.
-
-{ lib
-, config
-, python
-, wrapPython
-, setuptools
-, unzip
-, ensureNewerSourcesForZipFilesHook
-, toPythonModule
-, namePrefix
-, flit
-, writeScript
-, update-python-libraries
-}:
-
-let
-  setuptools-specific = import ./build-python-package-setuptools.nix { inherit lib python; };
-  pyproject-specific = import ./build-python-package-pyproject.nix { inherit lib python; };
-  flit-specific = import ./build-python-package-flit.nix { inherit python flit; };
-  wheel-specific = import ./build-python-package-wheel.nix { };
-  common = import ./build-python-package-common.nix { inherit python; };
-  mkPythonDerivation = import ./mk-python-derivation.nix {
-    inherit lib config python wrapPython setuptools unzip ensureNewerSourcesForZipFilesHook;
-    inherit toPythonModule namePrefix update-python-libraries;
-  };
-in
-
-{
-# Several package formats are supported.
-# "setuptools" : Install a common setuptools/distutils based package. This builds a wheel.
-# "wheel" : Install from a pre-compiled wheel.
-# "flit" : Install a flit package. This builds a wheel.
-# "other" : Provide your own buildPhase and installPhase.
-format ? "setuptools"
-, ... } @ attrs:
-
-let
-  formatspecific =
-    if format == "pyproject" then common (pyproject-specific attrs)
-    else if format == "setuptools" then common (setuptools-specific attrs)
-    else if format == "flit" then common (flit-specific attrs)
-    else if format == "wheel" then common (wheel-specific attrs)
-    else if format == "other" then {}
-    else throw "Unsupported format ${format}";
-
-in mkPythonDerivation ( attrs // formatspecific )
diff --git a/pkgs/development/interpreters/python/hooks/default.nix b/pkgs/development/interpreters/python/hooks/default.nix
new file mode 100644
index 000000000000..9a7ec98ba178
--- /dev/null
+++ b/pkgs/development/interpreters/python/hooks/default.nix
@@ -0,0 +1,95 @@
+# Hooks for building Python packages.
+{ python
+, callPackage
+, makeSetupHook
+}:
+
+let
+  pythonInterpreter = python.pythonForBuild.interpreter;
+  pythonSitePackages = python.sitePackages;
+  pythonCheckInterpreter = python.interpreter;
+  setuppy = ../run_setup.py;
+in rec {
+
+  flitBuildHook = callPackage ({ flit }:
+    makeSetupHook {
+      name = "flit-build-hook";
+      deps = [ flit ];
+      substitutions = {
+        inherit pythonInterpreter;
+      };
+    } ./flit-build-hook.sh) {};
+
+  pipBuildHook = callPackage ({ pip }:
+    makeSetupHook {
+      name = "pip-build-hook.sh";
+      deps = [ pip ];
+      substitutions = {
+        inherit pythonInterpreter pythonSitePackages;
+      };
+    } ./pip-build-hook.sh) {};
+
+  pipInstallHook = callPackage ({ pip }:
+    makeSetupHook {
+      name = "pip-install-hook";
+      deps = [ pip ];
+      substitutions = {
+        inherit pythonInterpreter pythonSitePackages;
+      };
+    } ./pip-install-hook.sh) {};
+
+  pytestCheckHook = callPackage ({ pytest }:
+    makeSetupHook {
+      name = "pytest-check-hook";
+      deps = [ pytest ];
+      substitutions = {
+        inherit pythonCheckInterpreter;
+      };
+    } ./pytest-check-hook.sh) {};
+
+  pythonCatchConflictsHook = callPackage ({ setuptools }:
+    makeSetupHook {
+      name = "python-catch-conflicts-hook";
+      deps = [ setuptools ];
+      substitutions = {
+        inherit pythonInterpreter;
+        catchConflicts=../catch_conflicts/catch_conflicts.py;
+      };
+    } ./python-catch-conflicts-hook.sh) {};
+
+  pythonImportsCheckHook = callPackage ({}:
+    makeSetupHook {
+      name = "python-imports-check-hook.sh";
+      substitutions = {
+        inherit pythonCheckInterpreter;
+      };
+    } ./python-imports-check-hook.sh) {};
+
+  pythonRemoveBinBytecodeHook = callPackage ({ }:
+