Improve editable install for Poetry projects

poetry/console/commands/install.py

@@ -39,7 +39,6 @@ exist it will look for <comment>pyproject.toml</> and do the same.
     _loggers = ["poetry.repositories.pypi_repository"]
 
     def handle(self):
-        from clikit.io import NullIO
         from poetry.installation.installer import Installer
         from poetry.masonry.builders import EditableBuilder
         from poetry.core.masonry.utils.module import ModuleOrPackageNotFound
@@ -69,7 +68,7 @@ exist it will look for <comment>pyproject.toml</> and do the same.
             return 0
 
         try:
-            builder = EditableBuilder(self.poetry, self._env, NullIO())
+            builder = EditableBuilder(self.poetry, self._env, self._io)
         except ModuleOrPackageNotFound:
             # This is likely due to the fact that the project is an application
             # not following the structure expected by Poetry

poetry/installation/pip_installer.py

@@ -175,9 +175,10 @@ class PipInstaller(BaseInstaller):
         return name
 
     def install_directory(self, package):
-        from poetry.core.masonry.builder import SdistBuilder
         from poetry.factory import Factory
+        from poetry.io.null_io import NullIO
         from poetry.utils._compat import decode
+        from poetry.masonry.builders.editable import EditableBuilder
         from poetry.utils.toml_file import TomlFile
 
         if package.root_dir:
@@ -197,18 +198,36 @@ class PipInstaller(BaseInstaller):
                 "tool" in pyproject_content and "poetry" in pyproject_content["tool"]
             )
             # Even if there is a build system specified
-            # pip as of right now does not support it fully
-            # TODO: Check for pip version when proper PEP-517 support lands
-            # has_build_system = ("build-system" in pyproject_content)
+            # some versions of pip (< 19.0.0) don't understand it
+            # so we need to check the version of pip to know
+            # if we can rely on the build system
+            pip_version = self._env.pip_version
+            pip_version_with_build_system_support = pip_version.__class__(19, 0, 0)
+            has_build_system = (
+                "build-system" in pyproject_content
+                and pip_version >= pip_version_with_build_system_support
+            )
 
         setup = os.path.join(req, "setup.py")
         has_setup = os.path.exists(setup)
-        if not has_setup and has_poetry and (package.develop or not has_build_system):
-            # We actually need to rely on creating a temporary setup.py
-            # file since pip, as of this comment, does not support
-            # build-system for editable packages
+        if has_poetry and package.develop and not package.build_script:
+            # This is a Poetry package in editable mode
+            # we can use the EditableBuilder without going through pip
+            # to install it, unless it has a build script.
+            builder = EditableBuilder(
+                Factory().create_poetry(pyproject.parent), self._env, NullIO()
+            )
+            builder.build()
+
+            return
+        elif has_poetry and (not has_build_system or package.build_script):
+            from poetry.core.masonry.builders.sdist import SdistBuilder
+
+            # We need to rely on creating a temporary setup.py
+            # file since the version of pip does not support
+            # build-systems
             # We also need it for non-PEP-517 packages
-            builder = SdistBuilder(Factory().create_poetry(pyproject.parent),)
+            builder = SdistBuilder(Factory().create_poetry(pyproject.parent))
 
             with open(setup, "w", encoding="utf-8") as f:
                 f.write(decode(builder.build_setup()))

poetry/masonry/builders/editable.py

 from __future__ import unicode_literals
 
+import hashlib
 import os
 import shutil
 
-from collections import defaultdict
+from base64 import urlsafe_b64encode
 
 from poetry.core.masonry.builders.builder import Builder
 from poetry.core.masonry.builders.sdist import SdistBuilder
 from poetry.core.semver.version import Version
+from poetry.utils._compat import WINDOWS
+from poetry.utils._compat import Path
 from poetry.utils._compat import decode
 
 
+SCRIPT_TEMPLATE = """\
+#!{python}
+from {module} import {callable_}
+
+if __name__ == '__main__':
+    {callable_}()
+"""
+
+WINDOWS_CMD_TEMPLATE = """\
+@echo off\r\n"{python}" "%~dp0\\{script}" %*\r\n
+"""
+
+
 class EditableBuilder(Builder):
     def __init__(self, poetry, env, io):
         super(EditableBuilder, self).__init__(poetry)
@@ -19,7 +35,22 @@ class EditableBuilder(Builder):
         self._io = io
 
     def build(self):
-        return self._setup_build()
+        self._debug(
+            "  - Building package <c1>{}</c1> in <info>editable</info> mode".format(
+                self._package.name
+            )
+        )
+
+        if self._package.build_script:
+            self._debug(
+                "  - <warning>Falling back on using a <b>setup.py</b></warning>"
+            )
+            return self._setup_build()
+
+        added_files = []
+        added_files += self._add_pth()
+        added_files += self._add_scripts()
+        self._add_dist_info(added_files)
 
     def _setup_build(self):
         builder = SdistBuilder(self._poetry)
@@ -36,14 +67,14 @@ class EditableBuilder(Builder):
 
         try:
             if self._env.pip_version < Version(19, 0):
-                self._env.run_pip("install", "-e", str(self._path))
+                self._env.run_pip("install", "-e", str(self._path), "--no-deps")
             else:
                 # Temporarily rename pyproject.toml
                 shutil.move(
                     str(self._poetry.file), str(self._poetry.file.with_suffix(".tmp"))
                 )
                 try:
-                    self._env.run_pip("install", "-e", str(self._path))
+                    self._env.run_pip("install", "-e", str(self._path), "--no-deps")
                 finally:
                     shutil.move(
                         str(self._poetry.file.with_suffix(".tmp")),
@@ -53,71 +84,125 @@ class EditableBuilder(Builder):
             if not has_setup:
                 os.remove(str(setup))
 
-    def _build_egg_info(self):
-        egg_info = self._path / "{}.egg-info".format(
-            self._package.name.replace("-", "_")
+    def _add_pth(self):
+        pth = self._env.site_packages.joinpath(self._module.name).with_suffix(".pth")
+        self._debug(
+            "  - Adding <c2>{}</c2> to <b>{}</b> for {}".format(
+                pth.name, self._env.site_packages, self._poetry.file.parent
+            )
         )
-        egg_info.mkdir(exist_ok=True)
+        with pth.open("w", encoding="utf-8") as f:
+            f.write(decode(str(self._poetry.file.parent.resolve())))
+
+        return [pth]
+
+    def _add_scripts(self):
+        added = []
+        entry_points = self.convert_entry_points()
+        scripts_path = Path(self._env.paths["scripts"])
+
+        scripts = entry_points.get("console_scripts", [])
+        for script in scripts:
+            name, script = script.split(" = ")
+            module, callable_ = script.split(":")
+            script_file = scripts_path.joinpath(name)
+            self._debug(
+                "  - Adding the <c2>{}</c2> script to <b>{}</b>".format(
+                    name, scripts_path
+                )
+            )
+            with script_file.open("w", encoding="utf-8") as f:
+                f.write(
+                    decode(
+                        SCRIPT_TEMPLATE.format(
+                            python=self._env._bin("python"),
+                            module=module,
+                            callable_=callable_,
+                        )
+                    )
+                )
+
+            script_file.chmod(0o755)
+
+            added.append(script_file)
+
+            if WINDOWS:
+                cmd_script = script_file.with_suffix(".cmd")
+                cmd = WINDOWS_CMD_TEMPLATE.format(
+                    python=self._env._bin("python"), script=name
+                )
+                self._debug(
+                    "  - Adding the <c2>{}</c2> script wrapper to <b>{}</b>".format(
+                        cmd_script.name, scripts_path
+                    )
+                )
+
+                with cmd_script.open("w", encoding="utf-8") as f:
+                    f.write(decode(cmd))
+
+                added.append(cmd_script)
 
-        with egg_info.joinpath("PKG-INFO").open("w", encoding="utf-8") as f:
-            f.write(decode(self.get_metadata_content()))
+        return added
 
-        with egg_info.joinpath("entry_points.txt").open("w", encoding="utf-8") as f:
-            entry_points = self.convert_entry_points()
+    def _add_dist_info(self, added_files):
+        from poetry.core.masonry.builders.wheel import WheelBuilder
 
-            for group_name in sorted(entry_points):
-                f.write("[{}]\n".format(group_name))
-                for ep in sorted(entry_points[group_name]):
-                    f.write(ep.replace(" ", "") + "\n")
+        added_files = added_files[:]
 
-                f.write("\n")
+        builder = WheelBuilder(self._poetry)
+        dist_info = self._env.site_packages.joinpath(builder.dist_info)
+
+        self._debug(
+            "  - Adding the <c2>{}</c2> directory to <b>{}</b>".format(
+                dist_info.name, self._env.site_packages
+            )
+        )
 
-        with egg_info.joinpath("requires.txt").open("w", encoding="utf-8") as f:
-            f.write(self._generate_requires())
+        if dist_info.exists():
+            shutil.rmtree(str(dist_info))
 
-    def _build_egg_link(self):
-        egg_link = self._env.site_packages / "{}.egg-link".format(self._package.name)
-        with egg_link.open("w", encoding="utf-8") as f:
-            f.write(str(self._poetry.file.parent.resolve()) + "\n")
-            f.write(".")
+        dist_info.mkdir()
 
-    def _add_easy_install_entry(self):
-        easy_install_pth = self._env.site_packages / "easy-install.pth"
-        path = str(self._poetry.file.parent.resolve())
-        content = ""
-        if easy_install_pth.exists():
-            with easy_install_pth.open(encoding="utf-8") as f:
-                content = f.read()
+        with dist_info.joinpath("METADATA").open("w", encoding="utf-8") as f:
+            builder._write_metadata_file(f)
 
-        if path in content:
-            return
+        added_files.append(dist_info.joinpath("METADATA"))
 
-        content += "{}\n".format(path)
+        with dist_info.joinpath("INSTALLER").open("w", encoding="utf-8") as f:
+            f.write("poetry")
 
-        with easy_install_pth.open("w", encoding="utf-8") as f:
-            f.write(content)
+        added_files.append(dist_info.joinpath("INSTALLER"))
 
-    def _generate_requires(self):
-        extras = defaultdict(list)
+        if self.convert_entry_points():
+            with dist_info.joinpath("entry_points.txt").open(
+                "w", encoding="utf-8"
+            ) as f:
+                builder._write_entry_points(f)
 
-        requires = ""
-        for dep in sorted(self._package.requires, key=lambda d: d.name):
-            marker = dep.marker
-            if marker.is_any():
-                requires += "{}\n".format(dep.base_pep_508_name)
-                continue
+            added_files.append(dist_info.joinpath("entry_points.txt"))
 
-            extras[str(marker)].append(dep.base_pep_508_name)
+        with dist_info.joinpath("RECORD").open("w", encoding="utf-8") as f:
+            for path in added_files:
+                hash = self._get_file_hash(path)
+                size = path.stat().st_size
+                f.write("{},sha256={},{}\n".format(str(path), hash, size))
 
-        if extras:
-            requires += "\n"
+            # RECORD itself is recorded with no hash or size
+            f.write("{},,\n".format(dist_info.joinpath("RECORD")))
 
-            for marker, deps in sorted(extras.items()):
-                requires += "[:{}]\n".format(marker)
+    def _get_file_hash(self, filepath):
+        hashsum = hashlib.sha256()
+        with filepath.open("rb") as src:
+            while True:
+                buf = src.read(1024 * 8)
+                if not buf:
+                    break
+                hashsum.update(buf)
 
-                for dep in deps:
-                    requires += dep + "\n"
+            src.seek(0)
 
-                requires += "\n"
+        return urlsafe_b64encode(hashsum.digest()).decode("ascii").rstrip("=")
 
-        return requires
+    def _debug(self, msg):
+        if self._io.is_debug():
+            self._io.write_line(msg)

poetry/utils/env.py

@@ -1156,6 +1156,9 @@ class NullEnv(SystemEnv):
         self._execute = execute
         self.executed = []
 
+    def get_pip_command(self):  # type: () -> List[str]
+        return [self._bin("python"), "-m", "pip"]
+
     def _run(self, cmd, **kwargs):
         self.executed.append(cmd)
 

tests/fixtures/extended_project/README.rst

+My Package
+==========

tests/fixtures/extended_project/pyproject.toml

+[tool.poetry]
+name = "extended-project"
+version = "1.2.3"
+description = "Some description."
+authors = [
+    "Sébastien Eustace <sebastien@eustace.io>"
+]
+license = "MIT"
+
+readme = "README.rst"
+
+homepage = "https://python-poetry.org"
+repository = "https://github.com/python-poetry/poetry"
+documentation = "https://python-poetry.org/docs"
+
+keywords = ["packaging", "dependency", "poetry"]
+
+classifiers = [
+    "Topic :: Software Development :: Build Tools",
+    "Topic :: Software Development :: Libraries :: Python Modules"
+]
+
+build = "build.py"
+
+# Requirements
+[tool.poetry.dependencies]
+python = "~2.7 || ^3.4"
+
+[tool.poetry.scripts]
+foo = "foo:bar"

tests/fixtures/simple_project/pyproject.toml

@@ -23,3 +23,6 @@ classifiers = [
 # Requirements
 [tool.poetry.dependencies]
 python = "~2.7 || ^3.4"
+
+[tool.poetry.scripts]
+foo = "foo:bar"

tests/masonry/builders/test_editable_builder.py

+# -*- coding: utf-8 -*-
+from __future__ import unicode_literals
+
+import shutil
+
+import pytest
+
+from poetry.factory import Factory
+from poetry.io.null_io import NullIO
+from poetry.masonry.builders.editable import EditableBuilder
+from poetry.utils._compat import Path
+from poetry.utils.env import EnvManager
+from poetry.utils.env import MockEnv
+from poetry.utils.env import VirtualEnv
+
+
+@pytest.fixture()
+def simple_poetry():
+    poetry = Factory().create_poetry(
+        Path(__file__).parent.parent.parent / "fixtures" / "simple_project"
+    )
+
+    return poetry
+
+
+@pytest.fixture()
+def extended_poetry():
+    poetry = Factory().create_poetry(
+        Path(__file__).parent.parent.parent / "fixtures" / "extended_project"
+    )
+
+    return poetry
+
+
+@pytest.fixture()
+def env_manager(simple_poetry):
+    return EnvManager(simple_poetry)
+
+
+@pytest.fixture
+def tmp_venv(tmp_dir, env_manager):
+    venv_path = Path(tmp_dir) / "venv"
+
+    env_manager.build_venv(str(venv_path))
+
+    venv = VirtualEnv(venv_path)
+    yield venv
+
+    shutil.rmtree(str(venv.path))
+
+
+def test_builder_installs_proper_files_for_standard_packages(simple_poetry, tmp_venv):
+    builder = EditableBuilder(simple_poetry, tmp_venv, NullIO())
+
+    builder.build()
+
+    assert tmp_venv._bin_dir.joinpath("foo").exists()
+    assert tmp_venv.site_packages.joinpath("simple_project.pth").exists()
+    assert (
+        str(simple_poetry.file.parent.resolve())
+        == tmp_venv.site_packages.joinpath("simple_project.pth").read_text()
+    )
+
+    dist_info = tmp_venv.site_packages.joinpath("simple_project-1.2.3.dist-info")
+    assert dist_info.exists()
+    assert dist_info.joinpath("INSTALLER").exists()
+    assert dist_info.joinpath("METADATA").exists()
+    assert dist_info.joinpath("RECORD").exists()
+    assert dist_info.joinpath("entry_points.txt").exists()
+
+    assert "poetry" == dist_info.joinpath("INSTALLER").read_text()
+    assert (
+        "[console_scripts]\nfoo=foo:bar\n\n"
+        == dist_info.joinpath("entry_points.txt").read_text()
+    )
+
+    metadata = """\
+Metadata-Version: 2.1
+Name: simple-project
+Version: 1.2.3
+Summary: Some description.
+Home-page: https://python-poetry.org
+License: MIT
+Keywords: packaging,dependency,poetry
+Author: Sébastien Eustace
+Author-email: sebastien@eustace.io
+Requires-Python: >=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*
+Classifier: License :: OSI Approved :: MIT License
+Classifier: Programming Language :: Python :: 2
+Classifier: Programming Language :: Python :: 2.7
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3.4
+Classifier: Programming Language :: Python :: 3.5
+Classifier: Programming Language :: Python :: 3.6
+Classifier: Programming Language :: Python :: 3.7
+Classifier: Programming Language :: Python :: 3.8
+Classifier: Topic :: Software Development :: Build Tools
+Classifier: Topic :: Software Development :: Libraries :: Python Modules
+Project-URL: Documentation, https://python-poetry.org/docs
+Project-URL: Repository, https://github.com/python-poetry/poetry
+Description-Content-Type: text/x-rst
+
+My Package
+==========
+
+"""
+    assert metadata == dist_info.joinpath("METADATA").read_text(encoding="utf-8")
+
+    records = dist_info.joinpath("RECORD").read_text()
+    assert str(tmp_venv.site_packages.joinpath("simple_project.pth")) in records
+    assert str(tmp_venv._bin_dir.joinpath("foo")) in records
+    assert str(dist_info.joinpath("METADATA")) in records
+    assert str(dist_info.joinpath("INSTALLER")) in records
+    assert str(dist_info.joinpath("entry_points.txt")) in records
+    assert str(dist_info.joinpath("RECORD")) in records
+
+
+def test_builder_falls_back_on_setup_and_pip_for_packages_with_build_scripts(
+    extended_poetry,
+):
+    env = MockEnv(path=Path("/foo"))
+    builder = EditableBuilder(extended_poetry, env, NullIO())
+
+    builder.build()
+
+    assert [
+        [
+            "python",
+            "-m",
+            "pip",
+            "install",
+            "-e",
+            str(extended_poetry.file.parent),
+            "--no-deps",
+        ]
+    ] == env.executed