Merge pull request #4433 from python-poetry/revert-system-env-changes

Fix the detection of the system environment with custom installer

install-poetry.py

@@ -528,6 +528,10 @@ class Installer:
 
             virtualenv.cli_run([str(env_path), "--clear"])
 
+        # We add a special file so that Poetry can detect
+        # its own virtual environment
+        env_path.joinpath("poetry_env").touch()
+
         return env_path
 
     def make_bin(self, version: str) -> None:

poetry/console/commands/env/info.py

@@ -44,6 +44,9 @@ class EnvInfoCommand(Command):
             "<info>Path</info>:           <comment>{}</>".format(
                 env.path if env.is_venv() else "NA"
             ),
+            "<info>Executable</info>:     <comment>{}</>".format(
+                env.python if env.is_venv() else "NA"
+            ),
         ]
         if env.is_venv():
             listing.append(
@@ -55,13 +58,18 @@ class EnvInfoCommand(Command):
 
         self.line("")
 
+        system_env = env.parent_env
         self.line("<b>System</b>")
         self.line(
             "\n".join(
                 [
-                    "<info>Platform</info>: <comment>{}</>".format(env.platform),
-                    "<info>OS</info>:       <comment>{}</>".format(env.os),
-                    "<info>Python</info>:   <comment>{}</>".format(env.base),
+                    "<info>Platform</info>:   <comment>{}</>".format(env.platform),
+                    "<info>OS</info>:         <comment>{}</>".format(env.os),
+                    "<info>Python</info>:     <comment>{}</>".format(
+                        ".".join(str(v) for v in system_env.version_info[:3])
+                    ),
+                    "<info>Path</info>:       <comment>{}</>".format(system_env.path),
+                    "<info>Executable</info>: <comment>{}</>".format(system_env.python),
                 ]
             )
         )

poetry/utils/env.py

@@ -152,6 +152,38 @@ import sysconfig
 print(json.dumps(sysconfig.get_paths()))
 """
 
+GET_PATHS_FOR_GENERIC_ENVS = """\
+# We can't use sysconfig.get_paths() because
+# on some distributions it does not return the proper paths
+# (those used by pip for instance). We go through distutils
+# to get the proper ones.
+import json
+import site
+import sysconfig
+
+from distutils.command.install import SCHEME_KEYS  # noqa
+from distutils.core import Distribution
+
+d = Distribution()
+d.parse_config_files()
+obj = d.get_command_obj("install", create=True)
+obj.finalize_options()
+
+paths = sysconfig.get_paths().copy()
+for key in SCHEME_KEYS:
+    if key == "headers":
+        # headers is not a path returned by sysconfig.get_paths()
+        continue
+
+    paths[key] = getattr(obj, f"install_{key}")
+
+if site.check_enableusersite() and hasattr(obj, "install_usersite"):
+    paths["usersite"] = getattr(obj, "install_usersite")
+    paths["userbase"] = getattr(obj, "install_userbase")
+
+print(json.dumps(paths))
+"""
+
 
 class SitePackages:
     def __init__(
@@ -730,7 +762,7 @@ class EnvManager:
 
         self.remove_venv(venv)
 
-        return VirtualEnv(venv)
+        return VirtualEnv(venv, venv)
 
     def create_venv(
         self,
@@ -1010,15 +1042,21 @@ class EnvManager:
         (e.g. plugin installation or self update).
         """
         prefix, base_prefix = Path(sys.prefix), Path(cls.get_base_prefix())
+        env = SystemEnv(prefix)
         if not naive:
-            try:
-                Path(__file__).relative_to(prefix)
-            except ValueError:
-                pass
+            if prefix.joinpath("poetry_env").exists():
+                env = GenericEnv(base_prefix, child_env=env)
             else:
-                return GenericEnv(base_prefix)
+                from poetry.locations import data_dir
 
-        return SystemEnv(prefix)
+                try:
+                    prefix.relative_to(data_dir())
+                except ValueError:
+                    pass
+                else:
+                    env = GenericEnv(base_prefix, child_env=env)
+
+        return env
 
     @classmethod
     def get_base_prefix(cls) -> Path:
@@ -1056,6 +1094,11 @@ class Env:
         self._path = path
         self._bin_dir = self._path / bin_dir
 
+        self._executable = "python"
+        self._pip_executable = "pip"
+
+        self.find_executables()
+
         self._base = base or path
 
         self._marker_env = None
@@ -1090,7 +1133,7 @@ class Env:
         """
         Path to current python executable
         """
-        return self._bin("python")
+        return self._bin(self._executable)
 
     @property
     def marker_env(self) -> Dict[str, Any]:
@@ -1099,6 +1142,39 @@ class Env:
 
         return self._marker_env
 
+    @property
+    def parent_env(self) -> "GenericEnv":
+        return GenericEnv(self.base, child_env=self)
+
+    def find_executables(self) -> None:
+        python_executables = sorted(
+            [
+                p.name
+                for p in self._bin_dir.glob("python*")
+                if re.match(r"python(?:\d+(?:\.\d+)?)?(?:\.exe)?$", p.name)
+            ]
+        )
+        if python_executables:
+            executable = python_executables[0]
+            if executable.endswith(".exe"):
+                executable = executable[:-4]
+
+            self._executable = executable
+
+        pip_executables = sorted(
+            [
+                p.name
+                for p in self._bin_dir.glob("pip*")
+                if re.match(r"pip(?:\d+(?:\.\d+)?)?(?:\.exe)?$", p.name)
+            ]
+        )
+        if pip_executables:
+            pip_executable = pip_executables[0]
+            if pip_executable.endswith(".exe"):
+                pip_executable = pip_executable[:-4]
+
+            self._pip_executable = pip_executable
+
     def get_embedded_wheel(self, distribution):
         return get_embed_wheel(
             distribution, "{}.{}".format(self.version_info[0], self.version_info[1])
@@ -1116,7 +1192,7 @@ class Env:
         Path to current pip executable
         """
         # we do not use as_posix() here due to issues with windows pathlib2 implementation
-        path = self._bin("pip")
+        path = self._bin(self._pip_executable)
         if not Path(path).exists():
             return str(self.pip_embedded)
         return path
@@ -1262,7 +1338,7 @@ class Env:
         return self._run(cmd, **kwargs)
 
     def run_python_script(self, content: str, **kwargs: Any) -> str:
-        return self.run("python", "-W", "ignore", "-", input_=content, **kwargs)
+        return self.run(self._executable, "-W", "ignore", "-", input_=content, **kwargs)
 
     def _run(self, cmd: List[str], **kwargs: Any) -> Union[int, str]:
         """
@@ -1329,7 +1405,11 @@ class Env:
         """
         Return path to the given executable.
         """
-        bin_path = (self._bin_dir / bin).with_suffix(".exe" if self._is_windows else "")
+        if self._is_windows and not bin.endswith(".exe"):
+            bin_path = self._bin_dir / (bin + ".exe")
+        else:
+            bin_path = self._bin_dir / bin
+
         if not bin_path.exists():
             # On Windows, some executables can be in the base path
             # This is especially true when installing Python with
@@ -1340,7 +1420,11 @@ class Env:
             # that creates a fake virtual environment pointing to
             # a base Python install.
             if self._is_windows:
-                bin_path = (self._path / bin).with_suffix(".exe")
+                if not bin.endswith(".exe"):
+                    bin_path = self._bin_dir / (bin + ".exe")
+                else:
+                    bin_path = self._path / bin
+
                 if bin_path.exists():
                     return str(bin_path)
 
@@ -1484,7 +1568,10 @@ class VirtualEnv(Env):
     def get_pip_command(self, embedded: bool = False) -> List[str]:
         # We're in a virtualenv that is known to be sane,
         # so assume that we have a functional pip
-        return [self._bin("python"), self.pip_embedded if embedded else self.pip]
+        return [
+            self._bin(self._executable),
+            self.pip_embedded if embedded else self.pip,
+        ]
 
     def get_supported_tags(self) -> List[Tag]:
         file_path = Path(packaging.tags.__file__)
@@ -1585,6 +1672,90 @@ class VirtualEnv(Env):
 
 
 class GenericEnv(VirtualEnv):
+    def __init__(
+        self, path: Path, base: Optional[Path] = None, child_env: Optional["Env"] = None
+    ) -> None:
+        self._child_env = child_env
+
+        super().__init__(path, base=base)
+
+    def find_executables(self) -> None:
+        patterns = [("python*", "pip*")]
+
+        if self._child_env:
+            minor_version = "{}.{}".format(
+                self._child_env.version_info[0], self._child_env.version_info[1]
+            )
+            major_version = "{}".format(self._child_env.version_info[0])
+            patterns = [
+                ("python{}".format(minor_version), "pip{}".format(minor_version)),
+                ("python{}".format(major_version), "pip{}".format(major_version)),
+            ]
+
+        python_executable = None
+        pip_executable = None
+
+        for python_pattern, pip_pattern in patterns:
+            if python_executable and pip_executable:
+                break
+
+            if not python_executable:
+                python_executables = sorted(
+                    [
+                        p.name
+                        for p in self._bin_dir.glob(python_pattern)
+                        if re.match(r"python(?:\d+(?:\.\d+)?)?(?:\.exe)?$", p.name)
+                    ]
+                )
+
+                if python_executables:
+                    executable = python_executables[0]
+                    if executable.endswith(".exe"):
+                        executable = executable[:-4]
+
+                    python_executable = executable
+
+            if not pip_executable:
+                pip_executables = sorted(
+                    [
+                        p.name
+                        for p in self._bin_dir.glob(pip_pattern)
+                        if re.match(r"pip(?:\d+(?:\.\d+)?)?(?:\.exe)?$", p.name)
+                    ]
+                )
+                if pip_executables:
+                    pip_executable = pip_executables[0]
+                    if pip_executable.endswith(".exe"):
+                        pip_executable = pip_executable[:-4]
+
+                    pip_executable = pip_executable
+
+            if python_executable:
+                self._executable = python_executable
+
+            if pip_executable:
+                self._pip_executable = pip_executable
+
+    def get_paths(self) -> Dict[str, str]:
+        output = self.run_python_script(GET_PATHS_FOR_GENERIC_ENVS)
+
+        return json.loads(output)
+
+    def execute(self, bin: str, *args: str, **kwargs: Any) -> Optional[int]:
+        command = self.get_command_from_bin(bin) + list(args)
+        env = kwargs.pop("env", {k: v for k, v in os.environ.items()})
+
+        if not self._is_windows:
+            return os.execvpe(command[0], command, env=env)
+        else:
+            exe = subprocess.Popen([command[0]] + command[1:], env=env, **kwargs)
+            exe.communicate()
+
+            return exe.returncode
+
+    def _run(self, cmd: List[str], **kwargs: Any) -> Optional[int]:
+        return super(VirtualEnv, self)._run(cmd, **kwargs)
+
     def is_venv(self) -> bool:
         return self._path != self._base
 
@@ -1602,7 +1773,10 @@ class NullEnv(SystemEnv):
         self.executed = []
 
     def get_pip_command(self, embedded: bool = False) -> List[str]:
-        return [self._bin("python"), self.pip_embedded if embedded else self.pip]
+        return [
+            self._bin(self._executable),
+            self.pip_embedded if embedded else self.pip,
+        ]
 
     def _run(self, cmd: List[str], **kwargs: Any) -> int:
         self.executed.append(cmd)

tests/console/commands/env/test_info.py

+import sys
+
 from pathlib import Path
 
 import pytest
@@ -28,14 +30,21 @@ Virtualenv
 Python:         3.7.0
 Implementation: CPython
 Path:           {prefix}
+Executable:     {executable}
 Valid:          True
 
 System
-Platform: darwin
-OS:       posix
-Python:   {base_prefix}
+Platform:   darwin
+OS:         posix
+Python:     {base_version}
+Path:       {base_prefix}
+Executable: {base_executable}
 """.format(
-        prefix=str(Path("/prefix")), base_prefix=str(Path("/base/prefix"))
+        prefix=str(Path("/prefix")),
+        base_prefix=str(Path("/base/prefix")),
+        base_version=".".join(str(v) for v in sys.version_info[:3]),
+        executable=sys.executable,
+        base_executable="python",
     )
 
     assert expected == tester.io.fetch_output()

tests/utils/test_env.py

@@ -15,9 +15,11 @@ from cleo.io.null_io import NullIO
 from poetry.core.semver.version import Version
 from poetry.core.toml.file import TOMLFile
 from poetry.factory import Factory
+from poetry.utils._compat import WINDOWS
 from poetry.utils.env import GET_BASE_PREFIX
 from poetry.utils.env import EnvCommandError
 from poetry.utils.env import EnvManager
+from poetry.utils.env import GenericEnv
 from poetry.utils.env import NoCompatiblePythonVersionFound
 from poetry.utils.env import SystemEnv
 from poetry.utils.env import VirtualEnv
@@ -964,3 +966,130 @@ def test_env_system_packages(tmp_path, config):
         assert not venv_path.joinpath(
             "lib", "python2.7", "no-global-site-packages.txt"
         ).exists()
+
+
+def test_env_finds_the_correct_executables(tmp_dir, manager):
+    venv_path = Path(tmp_dir) / "Virtual Env"
+    manager.build_venv(str(venv_path), with_pip=True)
+    venv = VirtualEnv(venv_path)
+
+    default_executable = expected_executable = "python" + (".exe" if WINDOWS else "")
+    default_pip_executable = expected_pip_executable = "pip" + (
+        ".exe" if WINDOWS else ""
+    )
+    major_executable = "python{}{}".format(
+        sys.version_info[0], ".exe" if WINDOWS else ""
+    )
+    major_pip_executable = "pip{}{}".format(
+        sys.version_info[0], ".exe" if WINDOWS else ""
+    )
+
+    if (
+        venv._bin_dir.joinpath(default_executable).exists()
+        and venv._bin_dir.joinpath(major_executable).exists()
+    ):
+        venv._bin_dir.joinpath(default_executable).unlink()
+        expected_executable = major_executable
+
+    if (
+        venv._bin_dir.joinpath(default_pip_executable).exists()
+        and venv._bin_dir.joinpath(major_pip_executable).exists()
+    ):
+        venv._bin_dir.joinpath(default_pip_executable).unlink()
+        expected_pip_executable = major_pip_executable
+
+    venv = VirtualEnv(venv_path)
+
+    assert Path(venv.python).name == expected_executable
+    assert Path(venv.pip).name.startswith(expected_pip_executable.split(".")[0])
+
+
+def test_env_finds_the_correct_executables_for_generic_env(tmp_dir, manager):
+    venv_path = Path(tmp_dir) / "Virtual Env"
+    child_venv_path = Path(tmp_dir) / "Child Virtual Env"
+    manager.build_venv(str(venv_path), with_pip=True)
+    parent_venv = VirtualEnv(venv_path)
+    manager.build_venv(
+        str(child_venv_path), executable=parent_venv.python, with_pip=True
+    )
+    venv = GenericEnv(parent_venv.path, child_env=VirtualEnv(child_venv_path))
+
+    expected_executable = "python{}.{}{}".format(
+        sys.version_info[0], sys.version_info[1], ".exe" if WINDOWS else ""
+    )
+    expected_pip_executable = "pip{}.{}{}".format(
+        sys.version_info[0], sys.version_info[1], ".exe" if WINDOWS else ""
+    )
+
+    if WINDOWS:
+        expected_executable = "python.exe"
+        expected_pip_executable = "pip.exe"
+
+    assert Path(venv.python).name == expected_executable
+    assert Path(venv.pip).name == expected_pip_executable
+
+
+def test_env_finds_fallback_executables_for_generic_env(tmp_dir, manager):
+    venv_path = Path(tmp_dir) / "Virtual Env"
+    child_venv_path = Path(tmp_dir) / "Child Virtual Env"
+    manager.build_venv(str(venv_path), with_pip=True)
+    parent_venv = VirtualEnv(venv_path)
+    manager.build_venv(
+        str(child_venv_path), executable=parent_venv.python, with_pip=True
+    )
+    venv = GenericEnv(parent_venv.path, child_env=VirtualEnv(child_venv_path))
+
+    default_executable = "python" + (".exe" if WINDOWS else "")
+    major_executable = "python{}{}".format(
+        sys.version_info[0], ".exe" if WINDOWS else ""
+    )
+    minor_executable = "python{}.{}{}".format(
+        sys.version_info[0], sys.version_info[1], ".exe" if WINDOWS else ""
+    )
+    expected_executable = minor_executable
+    if (
+        venv._bin_dir.joinpath(expected_executable).exists()
+        and venv._bin_dir.joinpath(major_executable).exists()
+    ):
+        venv._bin_dir.joinpath(expected_executable).unlink()
+        expected_executable = major_executable
+
+    if (
+        venv._bin_dir.joinpath(expected_executable).exists()
+        and venv._bin_dir.joinpath(default_executable).exists()
+    ):
+        venv._bin_dir.joinpath(expected_executable).unlink()
+        expected_executable = default_executable
+
+    default_pip_executable = "pip" + (".exe" if WINDOWS else "")
+    major_pip_executable = "pip{}{}".format(
+        sys.version_info[0], ".exe" if WINDOWS else ""
+    )
+    minor_pip_executable = "pip{}.{}{}".format(
+        sys.version_info[0], sys.version_info[1], ".exe" if WINDOWS else ""
+    )
+    expected_pip_executable = minor_pip_executable
+    if (
+        venv._bin_dir.joinpath(expected_pip_executable).exists()
+        and venv._bin_dir.joinpath(major_pip_executable).exists()
+    ):
+        venv._bin_dir.joinpath(expected_pip_executable).unlink()
+        expected_pip_executable = major_pip_executable
+
+    if (
+        venv._bin_dir.joinpath(expected_pip_executable).exists()
+        and venv._bin_dir.joinpath(default_pip_executable).exists()
+    ):
+        venv._bin_dir.joinpath(expected_pip_executable).unlink()
+        expected_pip_executable = default_pip_executable
+
+    if not venv._bin_dir.joinpath(expected_executable).exists():
+        expected_executable = default_executable
+
+    if not venv._bin_dir.joinpath(expected_pip_executable).exists():
+        expected_pip_executable = default_pip_executable
+
+    venv = GenericEnv(parent_venv.path, child_env=VirtualEnv(child_venv_path))
+
+    assert Path(venv.python).name == expected_executable
+    assert Path(venv.pip).name == expected_pip_executable