Use isolated ephemeral envs for editable installs

6400f423 · Arun Babu Neelicattu · 9a68da31 · 6400f423 · 6400f423 · 6400f423
Commit 6400f423 authored Sep 29, 2020 by Arun Babu Neelicattu
10 changed files
--- a/poetry/inspection/info.py
+++ b/poetry/inspection/info.py
@@ -22,8 +22,7 @@ from poetry.core.utils.helpers import parse_requires
 from poetry.core.utils.helpers import temporary_directory
 from poetry.core.version.markers import InvalidMarker
 from poetry.utils.env import EnvCommandError
-from poetry.utils.env import EnvManager
+from poetry.utils.env import ephemeral_environment
-from poetry.utils.env import VirtualEnv
 from poetry.utils.setup_reader import SetupReader
@@ -451,13 +450,9 @@ class PackageInfo:
        except PackageInfoError:
            pass
-        with temporary_directory() as tmp_dir:
+        with ephemeral_environment(pip=True, wheel=True, setuptools=True) as venv:
            # TODO: cache PEP 517 build environment corresponding to each project venv
-            venv_dir = Path(tmp_dir) / ".venv"
+            dest_dir = venv.path.parent / "dist"
-            EnvManager.build_venv(venv_dir.as_posix(), with_pip=True)
-            venv = VirtualEnv(venv_dir, venv_dir)
-            dest_dir = Path(tmp_dir) / "dist"
            dest_dir.mkdir()
            try:

--- a/poetry/installation/executor.py
+++ b/poetry/installation/executor.py
@@ -22,6 +22,7 @@ from poetry.core.pyproject.toml import PyProjectTOML
 from poetry.utils._compat import decode
 from poetry.utils.env import EnvCommandError
 from poetry.utils.helpers import safe_rmtree
+from poetry.utils.pip import pip_editable_install
 from .authenticator import Authenticator
 from .chef import Chef
@@ -572,14 +573,14 @@ class Executor(object):
                with builder.setup_py():
                    if package.develop:
-                        args.append("-e")
+                        return pip_editable_install(req, self._env)
                    args.append(req)
                    return self.run_pip(*args)
        if package.develop:
-            args.append("-e")
+            return pip_editable_install(req, self._env)
        args.append(req)

--- a/poetry/installation/pip_installer.py
+++ b/poetry/installation/pip_installer.py
@@ -10,15 +10,15 @@ from typing import Union
 from cleo.io.io import IO
-from poetry.installation.base_installer import BaseInstaller
 from poetry.core.pyproject.toml import PyProjectTOML
+from poetry.installation.base_installer import BaseInstaller
 from poetry.repositories.pool import Pool
 from poetry.utils._compat import encode
 from poetry.utils.env import Env
-from poetry.utils.env import EnvManager
-from poetry.utils.env import VirtualEnv
 from poetry.utils.helpers import safe_rmtree
-from poetry.utils.helpers import temporary_directory
+from poetry.utils.pip import pip_editable_install
+from poetry.utils.pip import pip_install
 if TYPE_CHECKING:
    from poetry.core.packages.package import Package
@@ -190,12 +190,12 @@ class PipInstaller(BaseInstaller):
        from poetry.factory import Factory
+        req: Path
        if package.root_dir:
            req = (package.root_dir / package.source_url).as_posix()
        else:
-            req = os.path.realpath(package.source_url)
+            req = Path(package.source_url).resolve(strict=False)
-        args = ["install", "--no-deps", "-U", "--root", str(self._env.path)]
        pyproject = PyProjectTOML(os.path.join(req, "pyproject.toml"))
@@ -230,22 +230,16 @@ class PipInstaller(BaseInstaller):
                with builder.setup_py():
                    if package.develop:
-                        args.append("-e")
+                        return pip_editable_install(
+                            directory=req, environment=self._env
-                    args.append(req)
+                        )
+                    return pip_install(
-                    return self.run(*args)
+                        path=req, environment=self._env, deps=False, upgrade=True
+                    )
        if package.develop:
-            args.append("-e")
+            return pip_editable_install(directory=req, environment=self._env)
+        return pip_install(path=req, environment=self._env, deps=False, upgrade=True)
-        args.append(req)
-        with temporary_directory() as tmp_dir:
-            venv_dir = Path(tmp_dir) / ".venv"
-            EnvManager.build_venv(venv_dir.as_posix(), with_pip=True)
-            venv = VirtualEnv(venv_dir, venv_dir)
-            return venv.run("pip", *args)
    def install_git(self, package: "Package") -> None:
        from poetry.core.packages.package import Package

--- a/poetry/masonry/builders/editable.py
+++ b/poetry/masonry/builders/editable.py
@@ -16,6 +16,7 @@ from poetry.core.semver.version import Version
 from poetry.utils._compat import WINDOWS
 from poetry.utils._compat import decode
 from poetry.utils.helpers import is_dir_writable
+from poetry.utils.pip import pip_editable_install
 if TYPE_CHECKING:
@@ -56,7 +57,6 @@ class EditableBuilder(Builder):
                self._debug(
                    "  - <warning>Falling back on using a <b>setup.py</b></warning>"
                )
                return self._setup_build()
            self._run_build_script(self._package.build_script)
@@ -85,14 +85,14 @@ class EditableBuilder(Builder):
        try:
            if self._env.pip_version < Version(19, 0):
-                self._env.run_pip("install", "-e", str(self._path), "--no-deps")
+                pip_editable_install(self._path, self._env)
            else:
                # Temporarily rename pyproject.toml
                shutil.move(
                    str(self._poetry.file), str(self._poetry.file.with_suffix(".tmp"))
                )
                try:
-                    self._env.run_pip("install", "-e", str(self._path), "--no-deps")
+                    pip_editable_install(self._path, self._env)
                finally:
                    shutil.move(
                        str(self._poetry.file.with_suffix(".tmp")),

--- a/poetry/utils/env.py
+++ b/poetry/utils/env.py
@@ -43,6 +43,7 @@ from poetry.utils._compat import encode
 from poetry.utils._compat import list_to_shell_command
 from poetry.utils.helpers import is_dir_writable
 from poetry.utils.helpers import paths_csv
+from poetry.utils.helpers import temporary_directory
 GET_ENVIRONMENT_INFO = """\
@@ -812,6 +813,8 @@ class EnvManager(object):
        executable: Optional[Union[str, Path]] = None,
        flags: Dict[str, bool] = None,
        with_pip: bool = False,
+        with_wheel: bool = False,
+        with_setuptools: bool = False,
    ) -> virtualenv.run.session.Session:
        flags = flags or {}
@@ -826,7 +829,16 @@ class EnvManager(object):
        ]
        if not with_pip:
-            args.extend(["--no-pip", "--no-wheel", "--no-setuptools"])
+            args.append("--no-pip")
+        else:
+            if with_wheel is None:
+                with_wheel = True
+        if with_wheel is None or not with_wheel:
+            args.append("--no-wheel")
+        if with_setuptools is None or not with_setuptools:
+            args.append("--no-setuptools")
        for flag, value in flags.items():
            if value is True:
@@ -1429,6 +1441,21 @@ class NullEnv(SystemEnv):
        return bin
+@contextmanager
+def ephemeral_environment(executable=None, pip=False, wheel=None, setuptools=None):
+    with temporary_directory() as tmp_dir:
+        # TODO: cache PEP 517 build environment corresponding to each project venv
+        venv_dir = Path(tmp_dir) / ".venv"
+        EnvManager.build_venv(
+            path=venv_dir.as_posix(),
+            executable=executable,
+            with_pip=pip,
+            with_wheel=wheel,
+            with_setuptools=setuptools,
+        )
+        yield VirtualEnv(venv_dir, venv_dir)
 class MockEnv(NullEnv):
    def __init__(
        self,

--- a/poetry/utils/pip.py
+++ b/poetry/utils/pip.py
+from poetry.exceptions import PoetryException
+from poetry.utils._compat import Path
+from poetry.utils.env import Env
+from poetry.utils.env import ephemeral_environment
+def pip_install(
+    path, environment, editable=False, deps=False, upgrade=False
+):  # type: (Path, Env, bool, bool, bool) -> None
+    path = Path(path) if isinstance(path, str) else path
+    args = ["pip", "install", "--prefix", str(environment.path)]
+    if upgrade:
+        args.append("--upgrade")
+    if not deps:
+        args.append("--no-deps")
+    if editable:
+        if not path.is_dir():
+            raise PoetryException(
+                "Cannot install non directory dependencies in editable mode"
+            )
+        args.append("-e")
+    args.append(str(path))
+    with ephemeral_environment(
+        executable=environment.python, pip=True, setuptools=True
+    ) as env:
+        return env.run(*args)
+def pip_editable_install(directory, environment):  # type: (Path, Env) -> None
+    return pip_install(
+        path=directory, environment=environment, editable=True, deps=False, upgrade=True
+    )
--- a/tests/installation/test_executor.py
+++ b/tests/installation/test_executor.py
@@ -66,8 +66,12 @@ def mock_file_downloads(http):
 def test_execute_executes_a_batch_of_operations(
-    config, pool, io, tmp_dir, mock_file_downloads, env
+    mocker, config, pool, io, tmp_dir, mock_file_downloads, env
 ):
+    pip_editable_install = mocker.patch(
+        "poetry.installation.executor.pip_editable_install"
+    )
    config = Config()
    config.merge({"cache-dir": tmp_dir})
@@ -101,6 +105,7 @@ def test_execute_executes_a_batch_of_operations(
        source_type="git",
        source_reference="master",
        source_url="https://github.com/demo/demo.git",
+        develop=True,
    )
    return_code = executor.execute(
@@ -131,8 +136,9 @@ Package operations: 4 installs, 1 update, 1 removal
    expected = set(expected.splitlines())
    output = set(io.fetch_output().splitlines())
    assert expected == output
-    assert 6 == len(env.executed)
+    assert 4 == len(env.executed)
    assert 0 == return_code
+    pip_editable_install.assert_called_once()
 def test_execute_shows_skipped_operations_if_verbose(

--- a/tests/installation/test_installer.py
+++ b/tests/installation/test_installer.py
@@ -12,8 +12,8 @@ from cleo.io.io import IO
 from cleo.io.null_io import NullIO
 from cleo.io.outputs.buffered_output import BufferedOutput
 from cleo.io.outputs.output import Verbosity
 from deepdiff import DeepDiff
 from poetry.core.packages.project_package import ProjectPackage
 from poetry.core.toml.file import TOMLFile
 from poetry.factory import Factory
@@ -833,8 +833,7 @@ def test_installer_with_pypi_repository(package, locker, installed, config):
    installer.run()
    expected = fixture("with-pypi-repository")
+    assert not DeepDiff(expected, locker.written_data, ignore_order=True)
-    assert not DeepDiff(locker.written_data, expected, ignore_order=True)
 def test_run_installs_with_local_file(installer, locker, repo, package, fixture_dir):

--- a/tests/installation/test_installer_old.py
+++ b/tests/installation/test_installer_old.py
@@ -7,8 +7,8 @@ from pathlib import Path
 import pytest
 from cleo.io.null_io import NullIO
 from deepdiff import DeepDiff
 from poetry.core.packages.project_package import ProjectPackage
 from poetry.core.toml.file import TOMLFile
 from poetry.factory import Factory
@@ -738,8 +738,7 @@ def test_installer_with_pypi_repository(package, locker, installed, config):
    installer.run()
    expected = fixture("with-pypi-repository")
+    assert not DeepDiff(expected, locker.written_data, ignore_order=True)
-    assert not DeepDiff(locker.written_data, expected, ignore_order=True)
 def test_run_installs_with_local_file(installer, locker, repo, package, fixture_dir):

--- a/tests/masonry/builders/test_editable_builder.py
+++ b/tests/masonry/builders/test_editable_builder.py
@@ -179,16 +179,19 @@ if __name__ == '__main__':
 def test_builder_falls_back_on_setup_and_pip_for_packages_with_build_scripts(
-    extended_poetry, tmp_dir
+    mocker, extended_poetry, tmp_dir
 ):
+    pip_editable_install = mocker.patch(
+        "poetry.masonry.builders.editable.pip_editable_install"
+    )
    env = MockEnv(path=Path(tmp_dir) / "foo")
    builder = EditableBuilder(extended_poetry, env, NullIO())
    builder.build()
-    assert [
+    pip_editable_install.assert_called_once_with(
-        env.get_pip_command()
+        extended_poetry.pyproject.file.path.parent, env
-        + ["install", "-e", str(extended_poetry.file.parent), "--no-deps"]
+    )
-    ] == env.executed
+    assert [] == env.executed
 def test_builder_installs_proper_files_when_packages_configured(