Merge staging into staging-next

62 files changed, 1096 insertions, 551 deletions

diff --git a/pkgs/development/interpreters/python/build-python-package-common.nix b/pkgs/development/interpreters/python/build-python-package-common.nix
deleted file mode 100644
index 0f8e088d434a..000000000000
--- a/pkgs/development/interpreters/python/build-python-package-common.nix
+++ /dev/null
@@ -1,31 +0,0 @@
-# This function provides generic bits to install a Python wheel.
-
-{ python
-}:
-
-{ buildInputs ? []
-# Additional flags to pass to "pip install".
-, installFlags ? []
-, ... } @ attrs:
-
-attrs // {
-  buildInputs = buildInputs ++ [ python.pythonForBuild.pkgs.bootstrapped-pip ];
-
-  configurePhase = attrs.configurePhase or ''
-    runHook preConfigure
-    runHook postConfigure
-  '';
-
-  installPhase = attrs.installPhase or ''
-    runHook preInstall
-
-    mkdir -p "$out/${python.sitePackages}"
-    export PYTHONPATH="$out/${python.sitePackages}:$PYTHONPATH"
-
-    pushd dist
-    ${python.pythonForBuild.pkgs.bootstrapped-pip}/bin/pip install *.whl --no-index --prefix=$out --no-cache ${toString installFlags} --build tmpbuild
-    popd
-
-    runHook postInstall
-  '';
-}
diff --git a/pkgs/development/interpreters/python/build-python-package-flit.nix b/pkgs/development/interpreters/python/build-python-package-flit.nix
deleted file mode 100644
index b0f9e0380211..000000000000
--- a/pkgs/development/interpreters/python/build-python-package-flit.nix
+++ /dev/null
@@ -1,22 +0,0 @@
-# This function provides specific bits for building a flit-based Python package.
-
-{ python
-, flit
-}:
-
-{ ... } @ attrs:
-
-attrs // {
-  nativeBuildInputs = [ flit ];
-  buildPhase = attrs.buildPhase or ''
-    runHook preBuild
-    flit build --format wheel
-    runHook postBuild
-  '';
-
-  # Flit packages, like setuptools packages, might have tests.
-  installCheckPhase = attrs.checkPhase or ''
-    ${python.interpreter} -m unittest discover
-  '';
-  doCheck = attrs.doCheck or true;
-}
diff --git a/pkgs/development/interpreters/python/build-python-package-pyproject.nix b/pkgs/development/interpreters/python/build-python-package-pyproject.nix
deleted file mode 100644
index 085db44f3e82..000000000000
--- a/pkgs/development/interpreters/python/build-python-package-pyproject.nix
+++ /dev/null
@@ -1,56 +0,0 @@
-# This function provides specific bits for building a setuptools-based Python package.
-
-{ lib
-, python
-}:
-
-{
-# Global options passed to "python setup.py"
-  setupPyGlobalFlags ? []
-# Build options passed to "build_ext"
-# https://github.com/pypa/pip/issues/881
-# Rename to `buildOptions` because it is not setuptools specific?
-, setupPyBuildFlags ? []
-# Execute before shell hook
-, preShellHook ? ""
-# Execute after shell hook
-, postShellHook ? ""
-, ... } @ attrs:
-
-let
-  pipGlobalFlagsString = lib.concatMapStringsSep " " (option: "--global-option ${option}") setupPyGlobalFlags;
-  pipBuildFlagsString = lib.concatMapStringsSep " " (option: "--build-option ${option}") setupPyBuildFlags;
-in attrs // {
-  buildPhase = attrs.buildPhase or ''
-    runHook preBuild
-    mkdir -p dist
-    echo "Creating a wheel..."
-    ${python.pythonForBuild.interpreter} -m pip wheel --no-index --no-deps --no-clean --no-build-isolation --wheel-dir dist ${pipGlobalFlagsString} ${pipBuildFlagsString} .
-    echo "Finished creating a wheel..."
-    runHook postBuild
-  '';
-
-  installCheckPhase = ''
-    runHook preCheck
-    echo "No checkPhase defined. Either provide a checkPhase or disable tests in case tests are not available."; exit 1
-    runHook postCheck
-  '';
-
-  # With Python it's a common idiom to run the tests
-  # after the software has been installed.
-  doCheck = attrs.doCheck or true;
-
-  shellHook = attrs.shellHook or ''
-    ${preShellHook}
-    # Long-term setup.py should be dropped.
-    if [ -e pyproject.toml ]; then
-      tmp_path=$(mktemp -d)
-      export PATH="$tmp_path/bin:$PATH"
-      export PYTHONPATH="$tmp_path/${python.pythonForBuild.sitePackages}:$PYTHONPATH"
-      mkdir -p $tmp_path/${python.pythonForBuild.sitePackages}
-      ${python.pythonForBuild.pkgs.bootstrapped-pip}/bin/pip install -e . --prefix $tmp_path >&2
-    fi
-    ${postShellHook}
-  '';
-
-}
diff --git a/pkgs/development/interpreters/python/build-python-package-setuptools.nix b/pkgs/development/interpreters/python/build-python-package-setuptools.nix
deleted file mode 100644
index 7738ea2f66a5..000000000000
--- a/pkgs/development/interpreters/python/build-python-package-setuptools.nix
+++ /dev/null
@@ -1,60 +0,0 @@
-# This function provides specific bits for building a setuptools-based Python package.
-
-{ lib
-, python
-}:
-
-{
-# Global options passed to "python setup.py"
-  setupPyGlobalFlags ? []
-# Build options passed to "python setup.py build_ext"
-# https://github.com/pypa/pip/issues/881
-, setupPyBuildFlags ? []
-# Execute before shell hook
-, preShellHook ? ""
-# Execute after shell hook
-, postShellHook ? ""
-, ... } @ attrs:
-
-let
-  # use setuptools shim (so that setuptools is imported before distutils)
-  # pip does the same thing: https://github.com/pypa/pip/pull/3265
-  setuppy = ./run_setup.py;
-
-  setupPyGlobalFlagsString = lib.concatStringsSep " " setupPyGlobalFlags;
-  setupPyBuildExtString = lib.optionalString (setupPyBuildFlags != []) ("build_ext " + (lib.concatStringsSep " " setupPyBuildFlags));
-
-in attrs // {
-  # we copy nix_run_setup over so it's executed relative to the root of the source
-  # many project make that assumption
-  buildPhase = attrs.buildPhase or ''
-    runHook preBuild
-    cp ${setuppy} nix_run_setup
-    ${python.pythonForBuild.interpreter} nix_run_setup ${setupPyGlobalFlagsString} ${setupPyBuildExtString} bdist_wheel
-    runHook postBuild
-  '';
-
-  installCheckPhase = attrs.checkPhase or ''
-    runHook preCheck
-    ${python.pythonForBuild.interpreter} nix_run_setup test
-    runHook postCheck
-  '';
-
-  # Python packages that are installed with setuptools
-  # are typically distributed with tests.
-  # With Python it's a common idiom to run the tests
-  # after the software has been installed.
-  doCheck = attrs.doCheck or true;
-
-  shellHook = attrs.shellHook or ''
-    ${preShellHook}
-    if test -e setup.py; then
-      tmp_path=$(mktemp -d)
-      export PATH="$tmp_path/bin:$PATH"
-      export PYTHONPATH="$tmp_path/${python.pythonForBuild.sitePackages}:$PYTHONPATH"
-      mkdir -p $tmp_path/${python.pythonForBuild.sitePackages}
-      ${python.pythonForBuild.pkgs.bootstrapped-pip}/bin/pip install -e . --prefix $tmp_path >&2
-    fi
-    ${postShellHook}
-  '';
-}
diff --git a/pkgs/development/interpreters/python/build-python-package-wheel.nix b/pkgs/development/interpreters/python/build-python-package-wheel.nix
deleted file mode 100644
index e3c4e13c0e2d..000000000000
--- a/pkgs/development/interpreters/python/build-python-package-wheel.nix
+++ /dev/null
@@ -1,20 +0,0 @@
-# This function provides specific bits for building a wheel-based Python package.
-
-{
-}:
-
-{ ... } @ attrs:
-
-attrs // {
-  unpackPhase = ''
-    mkdir dist
-    cp "$src" "dist/$(stripHash "$src")"
-  '';
-
-  # Wheels are pre-compiled
-  buildPhase = attrs.buildPhase or ":";
-  installCheckPhase = attrs.checkPhase or ":";
-
-  # Wheels don't have any checks to run
-  doCheck = attrs.doCheck or false;
-}
\ No newline at end of file
diff --git a/pkgs/development/interpreters/python/build-python-package.nix b/pkgs/development/interpreters/python/build-python-package.nix
deleted file mode 100644
index 61c1186cef9e..000000000000
--- a/pkgs/development/interpreters/python/build-python-package.nix
+++ /dev/null
@@ -1,48 +0,0 @@
-# This function provides a generic Python package builder,
-# and can build packages that use distutils, setuptools or flit.
-
-{ lib
-, config
-, python
-, wrapPython
-, setuptools
-, unzip
-, ensureNewerSourcesForZipFilesHook
-, toPythonModule
-, namePrefix
-, flit
-, writeScript
-, update-python-libraries
-}:
-
-let
-  setuptools-specific = import ./build-python-package-setuptools.nix { inherit lib python; };
-  pyproject-specific = import ./build-python-package-pyproject.nix { inherit lib python; };
-  flit-specific = import ./build-python-package-flit.nix { inherit python flit; };
-  wheel-specific = import ./build-python-package-wheel.nix { };
-  common = import ./build-python-package-common.nix { inherit python; };
-  mkPythonDerivation = import ./mk-python-derivation.nix {
-    inherit lib config python wrapPython setuptools unzip ensureNewerSourcesForZipFilesHook;
-    inherit toPythonModule namePrefix update-python-libraries;
-  };
-in
-
-{
-# Several package formats are supported.
-# "setuptools" : Install a common setuptools/distutils based package. This builds a wheel.
-# "wheel" : Install from a pre-compiled wheel.
-# "flit" : Install a flit package. This builds a wheel.
-# "other" : Provide your own buildPhase and installPhase.
-format ? "setuptools"
-, ... } @ attrs:
-
-let
-  formatspecific =
-    if format == "pyproject" then common (pyproject-specific attrs)
-    else if format == "setuptools" then common (setuptools-specific attrs)
-    else if format == "flit" then common (flit-specific attrs)
-    else if format == "wheel" then common (wheel-specific attrs)
-    else if format == "other" then {}
-    else throw "Unsupported format ${format}";
-
-in mkPythonDerivation ( attrs // formatspecific )
diff --git a/pkgs/development/interpreters/python/hooks/default.nix b/pkgs/development/interpreters/python/hooks/default.nix
new file mode 100644
index 000000000000..9a7ec98ba178
--- /dev/null
+++ b/pkgs/development/interpreters/python/hooks/default.nix
@@ -0,0 +1,95 @@
+# Hooks for building Python packages.
+{ python
+, callPackage
+, makeSetupHook
+}:
+
+let
+  pythonInterpreter = python.pythonForBuild.interpreter;
+  pythonSitePackages = python.sitePackages;
+  pythonCheckInterpreter = python.interpreter;
+  setuppy = ../run_setup.py;
+in rec {
+
+  flitBuildHook = callPackage ({ flit }:
+    makeSetupHook {
+      name = "flit-build-hook";
+      deps = [ flit ];
+      substitutions = {
+        inherit pythonInterpreter;
+      };
+    } ./flit-build-hook.sh) {};
+
+  pipBuildHook = callPackage ({ pip }:
+    makeSetupHook {
+      name = "pip-build-hook.sh";
+      deps = [ pip ];
+      substitutions = {
+        inherit pythonInterpreter pythonSitePackages;
+      };
+    } ./pip-build-hook.sh) {};
+
+  pipInstallHook = callPackage ({ pip }:
+    makeSetupHook {
+      name = "pip-install-hook";
+      deps = [ pip ];
+      substitutions = {
+        inherit pythonInterpreter pythonSitePackages;
+      };
+    } ./pip-install-hook.sh) {};
+
+  pytestCheckHook = callPackage ({ pytest }:
+    makeSetupHook {
+      name = "pytest-check-hook";
+      deps = [ pytest ];
+      substitutions = {
+        inherit pythonCheckInterpreter;
+      };
+    } ./pytest-check-hook.sh) {};
+
+  pythonCatchConflictsHook = callPackage ({ setuptools }:
+    makeSetupHook {
+      name = "python-catch-conflicts-hook";
+      deps = [ setuptools ];
+      substitutions = {
+        inherit pythonInterpreter;
+        catchConflicts=../catch_conflicts/catch_conflicts.py;
+      };
+    } ./python-catch-conflicts-hook.sh) {};
+
+  pythonImportsCheckHook = callPackage ({}:
+    makeSetupHook {
+      name = "python-imports-check-hook.sh";
+      substitutions = {
+        inherit pythonCheckInterpreter;
+      };
+    } ./python-imports-check-hook.sh) {};
+
+  pythonRemoveBinBytecodeHook = callPackage ({ }:
+    makeSetupHook {
+      name = "python-remove-bin-bytecode-hook";
+    } ./python-remove-bin-bytecode-hook.sh) {};
+
+  setuptoolsBuildHook = callPackage ({ setuptools, wheel }:
+    makeSetupHook {
+      name = "setuptools-setup-hook";
+      deps = [ setuptools wheel ];
+      substitutions = {
+        inherit pythonInterpreter pythonSitePackages setuppy;
+      };
+    } ./setuptools-build-hook.sh) {};
+
+  setuptoolsCheckHook = callPackage ({ setuptools }:
+    makeSetupHook {
+      name = "setuptools-check-hook";
+      deps = [ setuptools ];
+      substitutions = {
+        inherit pythonCheckInterpreter setuppy;
+      };
+    } ./setuptools-check-hook.sh) {};
+
+  wheelUnpackHook = callPackage ({ }:
+    makeSetupHook {
+      name = "wheel-unpack-hook.sh";
+    } ./wheel-unpack-hook.sh) {};
+}
diff --git a/pkgs/development/interpreters/python/hooks/flit-build-hook.sh b/pkgs/development/interpreters/python/hooks/flit-build-hook.sh
new file mode 100644
index 000000000000..faa3f6e3075f
--- /dev/null
+++ b/pkgs/development/interpreters/python/hooks/flit-build-hook.sh
@@ -0,0 +1,15 @@
+# Setup hook for flit
+echo "Sourcing flit-build-hook"
+
+flitBuildPhase () {
+    echo "Executing flitBuildPhase"
+    preBuild
+    @pythonInterpreter@ -m flit build --format wheel
+    postBuild
+    echo "Finished executing flitBuildPhase"
+}
+
+if [ -z "$dontUseFlitBuild" ] && [ -z "$buildPhase" ]; then
+    echo "Using flitBuildPhase"
+    buildPhase=flitBuildPhase
+fi
diff --git a/pkgs/development/interpreters/python/hooks/pip-build-hook.sh b/pkgs/development/interpreters/python/hooks/pip-build-hook.sh
new file mode 100644
index 000000000000..6796d3efd0a1
--- /dev/null
+++ b/pkgs/development/interpreters/python/hooks/pip-build-hook.sh
@@ -0,0 +1,42 @@
+# Setup hook to use for pip projects
+echo "Sourcing pip-build-hook"
+
+pipBuildPhase() {
+    echo "Executing pipBuildPhase"
+    runHook preBuild
+
+    mkdir -p dist
+    echo "Creating a wheel..."
+    @pythonInterpreter@ -m pip wheel --no-index --no-deps --no-clean --no-build-isolation --wheel-dir dist "$options" .
+    echo "Finished creating a wheel..."
+
+    runHook postBuild
+    echo "Finished executing pipBuildPhase"
+}
+
+pipShellHook() {
+    echo "Executing pipShellHook"
+    runHook preShellHook
+
+    # Long-term setup.py should be dropped.
+    if [ -e pyproject.toml ]; then
+      tmp_path=$(mktemp -d)
+      export PATH="$tmp_path/bin:$PATH"
+      export PYTHONPATH="$tmp_path/@pythonSitePackages@:$PYTHONPATH"
+      mkdir -p "$tmp_path/@pythonSitePackages@"
+      @pythonInterpreter@ -m pip install -e . --prefix "$tmp_path" >&2
+    fi
+
+    runHook postShellHook
+    echo "Finished executing pipShellHook"
+}
+
+if [ -z "$dontUsePipBuild" ] && [ -z "$buildPhase" ]; then
+    echo "Using pipBuildPhase"
+    buildPhase=pipBuildPhase
+fi
+
+if [ -z "$shellHook" ]; then
+    echo "Using pipShellHook"
+    shellHook=pipShellHook
+fi
diff --git a/pkgs/development/interpreters/python/hooks/pip-install-hook.sh b/pkgs/development/interpreters/python/hooks/pip-install-hook.sh
new file mode 100644
index 000000000000..f528ec63cb8e
--- /dev/null
+++ b/pkgs/development/interpreters/python/hooks/pip-install-hook.sh
@@ -0,0 +1,24 @@
+# Setup hook for pip.
+echo "Sourcing pip-install-hook"
+
+declare -a pipInstallFlags
+
+pipInstallPhase() {
+    echo "Executing pipInstallPhase"
+    runHook preInstall
+
+    mkdir -p "$out/@pythonSitePackages@"
+    export PYTHONPATH="$out/@pythonSitePackages@:$PYTHONPATH"
+
+    pushd dist || return 1
+    @pythonInterpreter@ -m pip install ./*.whl --no-index --prefix="$out" --no-cache $pipInstallFlags --build tmpbuild
+    popd || return 1
+
+    runHook postInstall
+    echo "Finished executing pipInstallPhase"
+}
+
+if [ -z "$dontUsePipInstall" ] && [ -z "$installPhase" ]; then
+    echo "Using pipInstallPhase"
+    installPhase=pipInstallPhase
+fi
diff --git a/pkgs/development/interpreters/python/hooks/pytest-check-hook.sh b/pkgs/development/interpreters/python/hooks/pytest-check-hook.sh
new file mode 100644
index 000000000000..24510b9f9931
--- /dev/null
+++ b/pkgs/development/interpreters/python/hooks/pytest-check-hook.sh
@@ -0,0 +1,49 @@
+# Setup hook for pytest
+echo "Sourcing pytest-check-hook"
+
+declare -ar disabledTests
+
+function _concatSep {
+    local result
+    local sep="$1"
+    local -n arr=$2
+    for index in ${!arr[*]}; do
+        if [ $index -eq 0 ]; then
+            result="${arr[index]}"
+        else
+            result+=" $sep ${arr[index]}"
+        fi
+    done
+    echo "$result"
+}
+
+function _pytestComputeDisabledTestsString () {
+    declare -a t