Revert "Fix venv creation in Python environments"

This commit reverts all python changes from 234bb31f611f43f8b744b305ab82035de637aaca.

8 files changed, 71 insertions, 75 deletions

diff --git a/pkgs/development/interpreters/python/cpython/2.7/default.nix b/pkgs/development/interpreters/python/cpython/2.7/default.nix
index 86b09fa87768..dda254fca389 100644
--- a/pkgs/development/interpreters/python/cpython/2.7/default.nix
+++ b/pkgs/development/interpreters/python/cpython/2.7/default.nix
@@ -318,6 +318,8 @@ in with passthru; stdenv.mkDerivation ({
     inherit passthru;
 
     postFixup = ''
+      # Include a sitecustomize.py file. Note it causes an error when it's in postInstall with 2.7.
+      cp ${../../sitecustomize.py} $out/${sitePackages}/sitecustomize.py
     '' + lib.optionalString strip2to3 ''
       rm -R $out/bin/2to3 $out/lib/python*/lib2to3
     '' + lib.optionalString stripConfig ''
diff --git a/pkgs/development/interpreters/python/cpython/default.nix b/pkgs/development/interpreters/python/cpython/default.nix
index 96dcb6c25a13..301af7a29c9e 100644
--- a/pkgs/development/interpreters/python/cpython/default.nix
+++ b/pkgs/development/interpreters/python/cpython/default.nix
@@ -537,7 +537,8 @@ in with passthru; stdenv.mkDerivation (finalAttrs: {
     # Strip tests
     rm -R $out/lib/python*/test $out/lib/python*/**/test{,s}
     '' + optionalString includeSiteCustomize ''
-
+    # Include a sitecustomize.py file
+    cp ${../sitecustomize.py} $out/${sitePackages}/sitecustomize.py
     '' + optionalString stripBytecode ''
     # Determinism: deterministic bytecode
     # First we delete all old bytecode.
diff --git a/pkgs/development/interpreters/python/pypy/default.nix b/pkgs/development/interpreters/python/pypy/default.nix
index 5724f4944d9c..9b414944bba5 100644
--- a/pkgs/development/interpreters/python/pypy/default.nix
+++ b/pkgs/development/interpreters/python/pypy/default.nix
@@ -126,6 +126,9 @@ in with passthru; stdenv.mkDerivation rec {
     ln -s $out/${executable}-c/include $out/include/${libPrefix}
     ln -s $out/${executable}-c/lib-python/${if isPy3k then "3" else pythonVersion} $out/lib/${libPrefix}
 
+    # Include a sitecustomize.py file
+    cp ${../sitecustomize.py} $out/${if isPy38OrNewer then sitePackages else "lib/${libPrefix}/${sitePackages}"}/sitecustomize.py
+
     runHook postInstall
   '';
 
diff --git a/pkgs/development/interpreters/python/pypy/prebuilt.nix b/pkgs/development/interpreters/python/pypy/prebuilt.nix
index 70f8711ef086..4b47c642eca4 100644
--- a/pkgs/development/interpreters/python/pypy/prebuilt.nix
+++ b/pkgs/development/interpreters/python/pypy/prebuilt.nix
@@ -95,6 +95,9 @@ in with passthru; stdenv.mkDerivation {
     echo "Removing bytecode"
     find . -name "__pycache__" -type d -depth -delete
 
+    # Include a sitecustomize.py file
+    cp ${../sitecustomize.py} $out/${sitePackages}/sitecustomize.py
+
     runHook postInstall
   '';
 
diff --git a/pkgs/development/interpreters/python/pypy/prebuilt_2_7.nix b/pkgs/development/interpreters/python/pypy/prebuilt_2_7.nix
index f0b60c2333f5..37a06f9f61ed 100644
--- a/pkgs/development/interpreters/python/pypy/prebuilt_2_7.nix
+++ b/pkgs/development/interpreters/python/pypy/prebuilt_2_7.nix
@@ -96,6 +96,9 @@ in with passthru; stdenv.mkDerivation {
     echo "Removing bytecode"
     find . -name "__pycache__" -type d -depth -delete
 
+    # Include a sitecustomize.py file
+    cp ${../sitecustomize.py} $out/${sitePackages}/sitecustomize.py
+
     runHook postInstall
   '';
 
diff --git a/pkgs/development/interpreters/python/sitecustomize.py b/pkgs/development/interpreters/python/sitecustomize.py
new file mode 100644
index 000000000000..c6924a8e93f0
--- /dev/null
+++ b/pkgs/development/interpreters/python/sitecustomize.py
@@ -0,0 +1,39 @@
+"""
+This is a Nix-specific module for discovering modules built with Nix.
+
+The module recursively adds paths that are on `NIX_PYTHONPATH` to `sys.path`. In
+order to process possible `.pth` files `site.addsitedir` is used.
+
+The paths listed in `PYTHONPATH` are added to `sys.path` afterwards, but they
+will be added before the entries we add here and thus take precedence.
+
+Note the `NIX_PYTHONPATH` environment variable is unset in order to prevent leakage.
+
+Similarly, this module listens to the environment variable `NIX_PYTHONEXECUTABLE`
+and sets `sys.executable` to its value.
+"""
+import site
+import sys
+import os
+import functools
+
+paths = os.environ.pop('NIX_PYTHONPATH', None)
+if paths:
+    functools.reduce(lambda k, p: site.addsitedir(p, k), paths.split(':'), site._init_pathinfo())
+
+# Check whether we are in a venv or virtualenv.
+# For Python 3 we check whether our `base_prefix` is different from our current `prefix`.
+# For Python 2 we check whether the non-standard `real_prefix` is set.
+# https://stackoverflow.com/questions/1871549/determine-if-python-is-running-inside-virtualenv
+in_venv = (sys.version_info.major == 3 and sys.prefix != sys.base_prefix) or (sys.version_info.major == 2 and hasattr(sys, "real_prefix"))
+
+if not in_venv:
+    executable = os.environ.pop('NIX_PYTHONEXECUTABLE', None)
+    prefix = os.environ.pop('NIX_PYTHONPREFIX', None)
+
+    if 'PYTHONEXECUTABLE' not in os.environ and executable is not None:
+        sys.executable = executable
+    if prefix is not None:
+        # Sysconfig does not like it when sys.prefix is set to None
+        sys.prefix = sys.exec_prefix = prefix
+        site.PREFIXES.insert(0, prefix)
diff --git a/pkgs/development/interpreters/python/tests.nix b/pkgs/development/interpreters/python/tests.nix
index e989c92d5110..2cd29ca99032 100644
--- a/pkgs/development/interpreters/python/tests.nix
+++ b/pkgs/development/interpreters/python/tests.nix
@@ -39,25 +39,13 @@ let
         is_virtualenv = "False";
       };
     } // lib.optionalAttrs (!python.isPyPy && !stdenv.isDarwin) {
-      # Use virtualenv with symlinks from a Nix env.
+      # Use virtualenv from a Nix env.
       # Fails on darwin with
       #   virtualenv: error: argument dest: the destination . is not write-able at /nix/store
-      nixenv-virtualenv-links = rec {
-        env = runCommand "${python.name}-virtualenv-links" {} ''
-          ${pythonVirtualEnv.interpreter} -m virtualenv --system-site-packages --symlinks --no-seed $out
-        '';
-        interpreter = "${env}/bin/${python.executable}";
-        is_venv = "False";
-        is_nixenv = "True";
-        is_virtualenv = "True";
-      };
-    } // lib.optionalAttrs (!python.isPyPy && !stdenv.isDarwin) {
-      # Use virtualenv with copies from a Nix env.
-      # Fails on darwin with
-      #   virtualenv: error: argument dest: the destination . is not write-able at /nix/store
-      nixenv-virtualenv-copies = rec {
-        env = runCommand "${python.name}-virtualenv-copies" {} ''
-          ${pythonVirtualEnv.interpreter} -m virtualenv --system-site-packages --copies --no-seed $out
+      nixenv-virtualenv = rec {
+        env = runCommand "${python.name}-virtualenv" {} ''
+          ${pythonVirtualEnv.interpreter} -m virtualenv venv
+          mv venv $out
         '';
         interpreter = "${env}/bin/${python.executable}";
         is_venv = "False";
@@ -73,48 +61,27 @@ let
         is_nixenv = "True";
         is_virtualenv = "False";
       };
-    } // lib.optionalAttrs (python.pythonAtLeast "3.8" && (!python.isPyPy)) {
-      # Venv built using links to plain Python
-      # Python 2 does not support venv
-      # TODO: PyPy executable name is incorrect, it should be pypy-c or pypy-3c instead of pypy and pypy3.
-      plain-venv-links = rec {
-        env = runCommand "${python.name}-venv-links" {} ''
-          ${python.interpreter} -m venv --system-site-packages --symlinks --without-pip $out
-        '';
-        interpreter = "${env}/bin/${python.executable}";
-        is_venv = "True";
-        is_nixenv = "False";
-        is_virtualenv = "False";
-      };
-    } // lib.optionalAttrs (python.pythonAtLeast "3.8" && (!python.isPyPy)) {
-      # Venv built using copies from plain Python
+    } // lib.optionalAttrs (python.isPy3k && (!python.isPyPy)) {
+      # Venv built using plain Python
       # Python 2 does not support venv
       # TODO: PyPy executable name is incorrect, it should be pypy-c or pypy-3c instead of pypy and pypy3.
-      plain-venv-copies = rec {
-        env = runCommand "${python.name}-venv-copies" {} ''
-          ${python.interpreter} -m venv --system-site-packages --copies --without-pip $out
+      plain-venv = rec {
+        env = runCommand "${python.name}-venv" {} ''
+          ${python.interpreter} -m venv $out
         '';
         interpreter = "${env}/bin/${python.executable}";
         is_venv = "True";
         is_nixenv = "False";
         is_virtualenv = "False";
       };
+
     } // lib.optionalAttrs (python.pythonAtLeast "3.8") {
       # Venv built using Python Nix environment (python.buildEnv)
-      nixenv-venv-links = rec {
-        env = runCommand "${python.name}-venv-links" {} ''
-          ${pythonEnv.interpreter} -m venv --system-site-packages --symlinks --without-pip $out
-        '';
-        interpreter = "${env}/bin/${pythonEnv.executable}";
-        is_venv = "True";
-        is_nixenv = "True";
-        is_virtualenv = "False";
-      };
-    } // lib.optionalAttrs (python.pythonAtLeast "3.8") {
-      # Venv built using Python Nix environment (python.buildEnv)
-      nixenv-venv-copies = rec {
-        env = runCommand "${python.name}-venv-copies" {} ''
-          ${pythonEnv.interpreter} -m venv --system-site-packages --copies --without-pip $out
+      # TODO: Cannot create venv from a  nix env
+      # Error: Command '['/nix/store/ddc8nqx73pda86ibvhzdmvdsqmwnbjf7-python3-3.7.6-venv/bin/python3.7', '-Im', 'ensurepip', '--upgrade', '--default-pip']' returned non-zero exit status 1.
+      nixenv-venv = rec {
+        env = runCommand "${python.name}-venv" {} ''
+          ${pythonEnv.interpreter} -m venv $out
         '';
         interpreter = "${env}/bin/${pythonEnv.executable}";
         is_venv = "True";
@@ -126,33 +93,11 @@ let
     testfun = name: attrs: runCommand "${python.name}-tests-${name}" ({
       inherit (python) pythonVersion;
     } // attrs) ''
-      mkdir $out
-
-      # set up the test files
       cp -r ${./tests/test_environments} tests
       chmod -R +w tests
       substituteAllInPlace tests/test_python.py
-
-      # run the tests by invoking the interpreter via full path
-      echo "absolute path: ${attrs.interpreter}"
-      ${attrs.interpreter} -m unittest discover --verbose tests 2>&1 | tee "$out/full.txt"
-
-      # run the tests by invoking the interpreter via $PATH
-      export PATH="$(dirname ${attrs.interpreter}):$PATH"
-      echo "PATH: $(basename ${attrs.interpreter})"
-      "$(basename ${attrs.interpreter})" -m unittest discover --verbose tests 2>&1 | tee "$out/path.txt"
-
-      # make sure we get the right path when invoking through a result link
-      ln -s "${attrs.env}" result
-      relative="result/bin/$(basename ${attrs.interpreter})"
-      expected="$PWD/$relative"
-      actual="$(./$relative -c "import sys; print(sys.executable)" | tee "$out/result.txt")"
-      if [ "$actual" != "$expected" ]; then
-        echo "expected $expected, got $actual"
-        exit 1
-      fi
-
-      # if we got this far, the tests passed
+      ${attrs.interpreter} -m unittest discover --verbose tests #/test_python.py
+      mkdir $out
       touch $out/success
     '';
 
diff --git a/pkgs/development/interpreters/python/tests/test_environments/test_python.py b/pkgs/development/interpreters/python/tests/test_environments/test_python.py
index 538273f65dbc..0fc4b8a9e91c 100644
--- a/pkgs/development/interpreters/python/tests/test_environments/test_python.py
+++ b/pkgs/development/interpreters/python/tests/test_environments/test_python.py
@@ -38,7 +38,7 @@ class TestCasePython(unittest.TestCase):
 
     @unittest.skipIf(IS_PYPY or sys.version_info.major==2, "Python 2 does not have base_prefix")
     def test_base_prefix(self):
-        if IS_VENV or IS_VIRTUALENV:
+        if IS_VENV or IS_NIXENV or IS_VIRTUALENV:
             self.assertNotEqual(sys.prefix, sys.base_prefix)
         else:
             self.assertEqual(sys.prefix, sys.base_prefix)