Fix venv creation in Python environments

The way we build python environments is subtly broken. A python
environment should be semantically identical to a vanilla Python
installation in, say, /usr/local. The current implementation, however,
differs in two important ways. The first is that it's impossible to use
python packages from the environment in python virtual environments. The
second is that the nix-generated environment appears to be a venv, but
it's not.

This commit changes the way python environments are built:

  * When generating wrappers for python executables, we set argv0 to
    the full path of the wrapper. This causes python to initialize its
    configuration in the environment with all the correct paths.
  * We remove the sitecustomize.py file from the base python package.
    This file was used tweak the python configuration after it was
    incorrectly initialized. That's no longer necessary.
  * When building the environment's bin directory, we now detect wrapped
    python scripts from installed packages, and generate unwrapped
    copies with the environment's python executable as the interpreter.

The end result is that python environments no longer appear to be venvs,
and behave more like a vanilla python installation. In addition it's
possible to create a venv using an environment and use packages from
both the environment and the venv.

9 files changed, 10 insertions, 57 deletions

diff --git a/pkgs/development/interpreters/python/cpython/2.7/default.nix b/pkgs/development/interpreters/python/cpython/2.7/default.nix
index a77206ae3852..4b798e625866 100644
--- a/pkgs/development/interpreters/python/cpython/2.7/default.nix
+++ b/pkgs/development/interpreters/python/cpython/2.7/default.nix
@@ -312,8 +312,6 @@ in with passthru; stdenv.mkDerivation ({
     inherit passthru;
 
     postFixup = ''
-      # Include a sitecustomize.py file. Note it causes an error when it's in postInstall with 2.7.
-      cp ${../../sitecustomize.py} $out/${sitePackages}/sitecustomize.py
     '' + lib.optionalString strip2to3 ''
       rm -R $out/bin/2to3 $out/lib/python*/lib2to3
     '' + lib.optionalString stripConfig ''
diff --git a/pkgs/development/interpreters/python/cpython/default.nix b/pkgs/development/interpreters/python/cpython/default.nix
index 0191517aa9ef..a9ee50ceaa38 100644
--- a/pkgs/development/interpreters/python/cpython/default.nix
+++ b/pkgs/development/interpreters/python/cpython/default.nix
@@ -529,9 +529,6 @@ in with passthru; stdenv.mkDerivation (finalAttrs: {
     '' + optionalString stripTests ''
     # Strip tests
     rm -R $out/lib/python*/test $out/lib/python*/**/test{,s}
-    '' + optionalString includeSiteCustomize ''
-    # Include a sitecustomize.py file
-    cp ${../sitecustomize.py} $out/${sitePackages}/sitecustomize.py
     '' + optionalString stripBytecode ''
     # Determinism: deterministic bytecode
     # First we delete all old bytecode.
diff --git a/pkgs/development/interpreters/python/pypy/default.nix b/pkgs/development/interpreters/python/pypy/default.nix
index c64c65df350e..c11ca7330c2c 100644
--- a/pkgs/development/interpreters/python/pypy/default.nix
+++ b/pkgs/development/interpreters/python/pypy/default.nix
@@ -126,9 +126,6 @@ in with passthru; stdenv.mkDerivation rec {
     ln -s $out/${executable}-c/include $out/include/${libPrefix}
     ln -s $out/${executable}-c/lib-python/${if isPy3k then "3" else pythonVersion} $out/lib/${libPrefix}
 
-    # Include a sitecustomize.py file
-    cp ${../sitecustomize.py} $out/${if isPy38OrNewer then sitePackages else "lib/${libPrefix}/${sitePackages}"}/sitecustomize.py
-
     runHook postInstall
   '';
 
diff --git a/pkgs/development/interpreters/python/pypy/prebuilt.nix b/pkgs/development/interpreters/python/pypy/prebuilt.nix
index 4b47c642eca4..70f8711ef086 100644
--- a/pkgs/development/interpreters/python/pypy/prebuilt.nix
+++ b/pkgs/development/interpreters/python/pypy/prebuilt.nix
@@ -95,9 +95,6 @@ in with passthru; stdenv.mkDerivation {
     echo "Removing bytecode"
     find . -name "__pycache__" -type d -depth -delete
 
-    # Include a sitecustomize.py file
-    cp ${../sitecustomize.py} $out/${sitePackages}/sitecustomize.py
-
     runHook postInstall
   '';
 
diff --git a/pkgs/development/interpreters/python/pypy/prebuilt_2_7.nix b/pkgs/development/interpreters/python/pypy/prebuilt_2_7.nix
index 37a06f9f61ed..f0b60c2333f5 100644
--- a/pkgs/development/interpreters/python/pypy/prebuilt_2_7.nix
+++ b/pkgs/development/interpreters/python/pypy/prebuilt_2_7.nix
@@ -96,9 +96,6 @@ in with passthru; stdenv.mkDerivation {
     echo "Removing bytecode"
     find . -name "__pycache__" -type d -depth -delete
 
-    # Include a sitecustomize.py file
-    cp ${../sitecustomize.py} $out/${sitePackages}/sitecustomize.py
-
     runHook postInstall
   '';
 
diff --git a/pkgs/development/interpreters/python/sitecustomize.py b/pkgs/development/interpreters/python/sitecustomize.py
deleted file mode 100644
index d79a4696d8ea..000000000000
--- a/pkgs/development/interpreters/python/sitecustomize.py
+++ /dev/null
@@ -1,39 +0,0 @@
-"""
-This is a Nix-specific module for discovering modules built with Nix.
-
-The module recursively adds paths that are on `NIX_PYTHONPATH` to `sys.path`. In
-order to process possible `.pth` files `site.addsitedir` is used.
-
-The paths listed in `PYTHONPATH` are added to `sys.path` afterwards, but they
-will be added before the entries we add here and thus take precedence.
-
-Note the `NIX_PYTHONPATH` environment variable is unset in order to prevent leakage.
-
-Similarly, this module listens to the environment variable `NIX_PYTHONEXECUTABLE`
-and sets `sys.executable` to its value.
-"""
-import site
-import sys
-import os
-import functools
-
-paths = os.environ.pop('NIX_PYTHONPATH', None)
-if paths:
-    functools.reduce(lambda k, p: site.addsitedir(p, k), paths.split(':'), site._init_pathinfo())
-
-# Check whether we are in a venv or virtualenv. 
-# For Python 3 we check whether our `base_prefix` is different from our current `prefix`.
-# For Python 2 we check whether the non-standard `real_prefix` is set.
-# https://stackoverflow.com/questions/1871549/determine-if-python-is-running-inside-virtualenv
-in_venv = (sys.version_info.major == 3 and sys.prefix != sys.base_prefix) or (sys.version_info.major == 2 and hasattr(sys, "real_prefix"))
-
-if not in_venv:
-    executable = os.environ.pop('NIX_PYTHONEXECUTABLE', None)
-    prefix = os.environ.pop('NIX_PYTHONPREFIX', None)
-
-    if 'PYTHONEXECUTABLE' not in os.environ and executable is not None:
-        sys.executable = executable
-    if prefix is not None:
-        # Sysconfig does not like it when sys.prefix is set to None
-        sys.prefix = sys.exec_prefix = prefix
-        site.PREFIXES.insert(0, prefix)
diff --git a/pkgs/development/interpreters/python/tests.nix b/pkgs/development/interpreters/python/tests.nix
index df4484f9ec68..dd4dd9b56bfb 100644
--- a/pkgs/development/interpreters/python/tests.nix
+++ b/pkgs/development/interpreters/python/tests.nix
@@ -75,8 +75,6 @@ let
 
     } // lib.optionalAttrs (python.pythonAtLeast "3.8") {
       # Venv built using Python Nix environment (python.buildEnv)
-      # TODO: Cannot create venv from a  nix env
-      # Error: Command '['/nix/store/ddc8nqx73pda86ibvhzdmvdsqmwnbjf7-python3-3.7.6-venv/bin/python3.7', '-Im', 'ensurepip', '--upgrade', '--default-pip']' returned non-zero exit status 1.
       nixenv-venv = rec {
         env = runCommand "${python.name}-venv" {} ''
           ${pythonEnv.interpreter} -m venv $out
diff --git a/pkgs/development/interpreters/python/tests/test_environments/test_python.py b/pkgs/development/interpreters/python/tests/test_environments/test_python.py
index 0fc4b8a9e91c..538273f65dbc 100644
--- a/pkgs/development/interpreters/python/tests/test_environments/test_python.py
+++ b/pkgs/development/interpreters/python/tests/test_environments/test_python.py
@@ -38,7 +38,7 @@ class TestCasePython(unittest.TestCase):
 
     @unittest.skipIf(IS_PYPY or sys.version_info.major==2, "Python 2 does not have base_prefix")
     def test_base_prefix(self):
-        if IS_VENV or IS_NIXENV or IS_VIRTUALENV:
+        if IS_VENV or IS_VIRTUALENV:
             self.assertNotEqual(sys.prefix, sys.base_prefix)
         else:
             self.assertEqual(sys.prefix, sys.base_prefix)
diff --git a/pkgs/development/interpreters/python/wrapper.nix b/pkgs/development/interpreters/python/wrapper.nix
index f5f9b03e0fd3..7c419d320f50 100644
--- a/pkgs/development/interpreters/python/wrapper.nix
+++ b/pkgs/development/interpreters/python/wrapper.nix
@@ -35,6 +35,8 @@ let
       fi
       mkdir -p "$out/bin"
 
+      rm -f $out/bin/.*-wrapped
+
       for path in ${lib.concatStringsSep " " paths}; do
         if [ -d "$path/bin" ]; then
           cd "$path/bin"
@@ -42,7 +44,13 @@ let
             if [ -f "$prg" ]; then
               rm -f "$out/bin/$prg"
               if [ -x "$prg" ]; then
-                makeWrapper "$path/bin/$prg" "$out/bin/$prg" --set NIX_PYTHONPREFIX "$out" --set NIX_PYTHONEXECUTABLE ${pythonExecutable} --set NIX_PYTHONPATH ${pythonPath} ${lib.optionalString (!permitUserSite) ''--set PYTHONNOUSERSITE "true"''} ${lib.concatStringsSep " " makeWrapperArgs}
+                if [ -f ".$prg-wrapped" ]; then
+                  echo "#!${pythonExecutable}" > "$out/bin/$prg"
+                  sed -e '1d' -e '3d' ".$prg-wrapped" >> "$out/bin/$prg"
+                  chmod +x "$out/bin/$prg"
+                else
+                  makeWrapper "$path/bin/$prg" "$out/bin/$prg" --argv0 "$out/bin/$prg" ${lib.optionalString (!permitUserSite) ''--set PYTHONNOUSERSITE "true"''} ${lib.concatStringsSep " " makeWrapperArgs}
+                fi
               fi
             fi
           done