Skip to content

P
python-poetry
Commit c019b817 by Sébastien Eustace Committed by GitHub
Options

Merge pull request #2722 from abn/fix-installed-packages 

Improve detection of installed packages in an environment
parents f8123c84 e68797d7
Showing with 234 additions and 57 deletions
poetry/repositories/installed_repository.py
import itertools
from typing import Set from typing import Set
from typing import Union
from poetry.core.packages import Package from poetry.core.packages import Package
from poetry.core.utils.helpers import module_name
from poetry.utils._compat import Path from poetry.utils._compat import Path
from poetry.utils._compat import metadata from poetry.utils._compat import metadata
from poetry.utils.env import Env from poetry.utils.env import Env
...@@ -11,11 +15,17 @@ from .repository import Repository ...@@ -11,11 +15,17 @@ from .repository import Repository
_VENDORS = Path(__file__).parent.parent.joinpath("_vendor") _VENDORS = Path(__file__).parent.parent.joinpath("_vendor")
try:
FileNotFoundError
except NameError:
FileNotFoundError = OSError
class InstalledRepository(Repository): class InstalledRepository(Repository):
@classmethod @classmethod
def get_package_paths(cls, sitedir, name): # type: (Path, str) -> Set[Path] def get_package_paths(cls, env, name): # type: (Env, str) -> Set[Path]
""" """
Process a .pth file within the site-packages directory, and return any valid Process a .pth file within the site-packages directories, and return any valid
paths. We skip executable .pth files as there is no reliable means to do this paths. We skip executable .pth files as there is no reliable means to do this
without side-effects to current run-time. Mo check is made that the item refers without side-effects to current run-time. Mo check is made that the item refers
to a directory rather than a file, however, in order to maintain backwards to a directory rather than a file, however, in order to maintain backwards
...@@ -24,26 +34,73 @@ class InstalledRepository(Repository): ...@@ -24,26 +34,73 @@ class InstalledRepository(Repository):
Reference: https://docs.python.org/3.8/library/site.html Reference: https://docs.python.org/3.8/library/site.html
:param sitedir: The site-packages directory to search for .pth file. :param env: The environment to search for the .pth file in.
:param name: The name of the package to search .pth file for. :param name: The name of the package to search .pth file for.
:return: A `Set` of valid `Path` objects. :return: A `Set` of valid `Path` objects.
""" """
paths = set() paths = set()
pth_file = sitedir.joinpath("{}.pth".format(name)) # we identify the candidate pth files to check, this is done so to handle cases
if pth_file.exists(): # where the pth file for foo-bar might have been installed as either foo-bar.pth or
# foo_bar.pth (expected) in either pure or platform lib directories.
candidates = itertools.product(
{env.purelib, env.platlib}, {name, module_name(name)},
)
for lib, module in candidates:
pth_file = lib.joinpath(module).with_suffix(".pth")
if not pth_file.exists():
continue
with pth_file.open() as f: with pth_file.open() as f:
for line in f: for line in f:
line = line.strip() line = line.strip()
if line and not line.startswith(("#", "import ", "import\t")): if line and not line.startswith(("#", "import ", "import\t")):
path = Path(line) path = Path(line)
if not path.is_absolute(): if not path.is_absolute():
path = sitedir.joinpath(path) try:
path = lib.joinpath(path).resolve()
except FileNotFoundError:
# this is required to handle pathlib oddity on win32 python==3.5
path = lib.joinpath(path)
paths.add(path) paths.add(path)
return paths return paths
@classmethod @classmethod
def set_package_vcs_properties_from_path(
cls, src, package
): # type: (Path, Package) -> None
from poetry.core.vcs.git import Git
git = Git()
revision = git.rev_parse("HEAD", src).strip()
url = git.remote_url(src)
package.source_type = "git"
package.source_url = url
package.source_reference = revision
@classmethod
def set_package_vcs_properties(cls, package, env): # type: (Package, Env) -> None
src = env.path / "src" / package.name
cls.set_package_vcs_properties_from_path(src, package)
@classmethod
def is_vcs_package(cls, package, env): # type: (Union[Path, Package], Env) -> bool
# A VCS dependency should have been installed
# in the src directory.
src = env.path / "src"
if isinstance(package, Package):
return src.joinpath(package.name).is_dir()
try:
package.relative_to(env.path / "src")
except ValueError:
return False
else:
return True
@classmethod
def load(cls, env): # type: (Env) -> InstalledRepository def load(cls, env): # type: (Env) -> InstalledRepository
""" """
Load installed packages. Load installed packages.
...@@ -75,41 +132,26 @@ class InstalledRepository(Repository): ...@@ -75,41 +132,26 @@ class InstalledRepository(Repository):
repo.add_package(package) repo.add_package(package)
is_standard_package = True is_standard_package = env.is_path_relative_to_lib(path)
try:
path.relative_to(env.site_packages)
except ValueError:
is_standard_package = False
if is_standard_package: if is_standard_package:
if path.name.endswith(".dist-info"): if path.name.endswith(".dist-info"):
paths = cls.get_package_paths( paths = cls.get_package_paths(env=env, name=package.pretty_name)
sitedir=env.site_packages, name=package.pretty_name
)
if paths: if paths:
# TODO: handle multiple source directories? for src in paths:
package.source_type = "directory" if cls.is_vcs_package(src, env):
package.source_url = paths.pop().as_posix() cls.set_package_vcs_properties(package, env)
break
else:
# TODO: handle multiple source directories?
package.source_type = "directory"
package.source_url = paths.pop().as_posix()
continue continue
src_path = env.path / "src" if cls.is_vcs_package(path, env):
cls.set_package_vcs_properties(package, env)
# A VCS dependency should have been installed else:
# in the src directory. If not, it's a path dependency # If not, it's a path dependency
try:
path.relative_to(src_path)
from poetry.core.vcs.git import Git
git = Git()
revision = git.rev_parse("HEAD", src_path / package.name).strip()
url = git.remote_url(src_path / package.name)
package.source_type = "git"
package.source_url = url
package.source_reference = revision
except ValueError:
package.source_type = "directory" package.source_type = "directory"
package.source_url = str(path.parent) package.source_url = str(path.parent)
......
poetry/utils/env.py
...@@ -754,6 +754,8 @@ class Env(object): ...@@ -754,6 +754,8 @@ class Env(object):
self._site_packages = None self._site_packages = None
self._paths = None self._paths = None
self._supported_tags = None self._supported_tags = None
self._purelib = None
self._platlib = None
@property @property
def path(self): # type: () -> Path def path(self): # type: () -> Path
...@@ -810,11 +812,38 @@ class Env(object): ...@@ -810,11 +812,38 @@ class Env(object):
@property @property
def site_packages(self): # type: () -> Path def site_packages(self): # type: () -> Path
if self._site_packages is None: if self._site_packages is None:
self._site_packages = Path(self.paths["purelib"]) self._site_packages = Path(self.purelib)
return self._site_packages return self._site_packages
@property @property
def purelib(self): # type: () -> Path
if self._purelib is None:
self._purelib = Path(self.paths["purelib"])
return self._purelib
@property
def platlib(self): # type: () -> Path
if self._platlib is None:
if "platlib" in self.paths:
self._platlib = Path(self.paths["platlib"])
else:
self._platlib = self.purelib
return self._platlib
def is_path_relative_to_lib(self, path): # type: (Path) -> bool
for lib_path in [self.purelib, self.platlib]:
try:
path.relative_to(lib_path)
return True
except ValueError:
pass
return False
@property
def sys_path(self): # type: () -> List[str] def sys_path(self): # type: () -> List[str]
raise NotImplementedError() raise NotImplementedError()
......
tests/fixtures/complete.toml
...@@ -22,7 +22,7 @@ toml = "^0.9" ...@@ -22,7 +22,7 @@ toml = "^0.9"
# Dependencies with extras # Dependencies with extras
requests = { version = "^2.13", extras = [ "security" ] } requests = { version = "^2.13", extras = [ "security" ] }
# Python specific dependencies with prereleases allowed # Python specific dependencies with prereleases allowed
pathlib2 = { version = "^2.2", python = "~2.7", allows-prereleases = true } pathlib2 = { version = "^2.2", python = "~2.7", allow-prereleases = true }
# Git dependencies # Git dependencies
cleo = { git = "https://github.com/sdispater/cleo.git", branch = "master" } cleo = { git = "https://github.com/sdispater/cleo.git", branch = "master" }
......
tests/installation/test_installer.py
...@@ -890,7 +890,7 @@ def test_run_with_prereleases(installer, locker, repo, package): ...@@ -890,7 +890,7 @@ def test_run_with_prereleases(installer, locker, repo, package):
repo.add_package(package_a) repo.add_package(package_a)
repo.add_package(package_b) repo.add_package(package_b)
package.add_dependency("A", {"version": "*", "allows-prereleases": True}) package.add_dependency("A", {"version": "*", "allow-prereleases": True})
package.add_dependency("B", "^1.1") package.add_dependency("B", "^1.1")
installer.update(True) installer.update(True)
......
tests/installation/test_installer_old.py
...@@ -861,7 +861,7 @@ def test_run_with_prereleases(installer, locker, repo, package): ...@@ -861,7 +861,7 @@ def test_run_with_prereleases(installer, locker, repo, package):
repo.add_package(package_a) repo.add_package(package_a)
repo.add_package(package_b) repo.add_package(package_b)
package.add_dependency("A", {"version": "*", "allows-prereleases": True}) package.add_dependency("A", {"version": "*", "allow-prereleases": True})
package.add_dependency("B", "^1.1") package.add_dependency("B", "^1.1")
installer.update(True) installer.update(True)
......
tests/repositories/fixtures/installed/lib64/python3.7/site-packages/bender-2.0.5.dist-info/METADATA 0 → 100644
Metadata-Version: 2.1
Name: bender
Version: 2.0.5
Summary: Python datetimes made easy
License: MIT
Keywords: cli,commands
Author: Leela
Author-email: leela@planetexpress.com
Requires-Python: >=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*
Classifier: License :: OSI Approved :: MIT License
Classifier: Programming Language :: Python :: 2
Classifier: Programming Language :: Python :: 2.7
Classifier: Programming Language :: Python :: 3
Classifier: Programming Language :: Python :: 3.4
Classifier: Programming Language :: Python :: 3.5
Classifier: Programming Language :: Python :: 3.6
Classifier: Programming Language :: Python :: 3.7
Classifier: Programming Language :: Python :: 3.8
Description-Content-Type: text/x-rst
tests/repositories/fixtures/installed/lib64/python3.7/site-packages/lib64-2.3.4.dist-info/METADATA 0 → 100644
Metadata-Version: 2.1
Name: lib64
Version: 2.3.4
Summary: lib64 description.
License: MIT
Keywords: cli,commands
Author: Foo Bar
Author-email: foo@bar.com
Requires-Python: >=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*
Classifier: License :: OSI Approved :: MIT License
Classifier: Programming Language :: Python :: 2
Classifier: Programming Language :: Python :: 2.7
Classifier: Programming Language :: Python :: 3
Classifier: Programming Language :: Python :: 3.4
Classifier: Programming Language :: Python :: 3.5
Classifier: Programming Language :: Python :: 3.6
Classifier: Programming Language :: Python :: 3.7
Classifier: Programming Language :: Python :: 3.8
Description-Content-Type: text/x-rst
lib64
####
tests/repositories/fixtures/installed/src/bender/bender.egg-info/PKG-INFO 0 → 100644
Metadata-Version: 2.1
Name: bender
Version: 2.0.5
Summary: Python datetimes made easy
License: MIT
Keywords: cli,commands
Author: Leela
Author-email: leela@planetexpress.com
Requires-Python: >=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*
Classifier: License :: OSI Approved :: MIT License
Classifier: Programming Language :: Python :: 2
Classifier: Programming Language :: Python :: 2.7
Classifier: Programming Language :: Python :: 3
Classifier: Programming Language :: Python :: 3.4
Classifier: Programming Language :: Python :: 3.5
Classifier: Programming Language :: Python :: 3.6
Classifier: Programming Language :: Python :: 3.7
Classifier: Programming Language :: Python :: 3.8
Description-Content-Type: text/x-rst
tests/repositories/test_installed_repository.py
from typing import Optional
import pytest import pytest
from poetry.core.packages import Package
from poetry.repositories.installed_repository import InstalledRepository from poetry.repositories.installed_repository import InstalledRepository
from poetry.utils._compat import PY36 from poetry.utils._compat import PY36
from poetry.utils._compat import Path from poetry.utils._compat import Path
...@@ -11,25 +14,35 @@ from pytest_mock.plugin import MockFixture ...@@ -11,25 +14,35 @@ from pytest_mock.plugin import MockFixture
FIXTURES_DIR = Path(__file__).parent / "fixtures" FIXTURES_DIR = Path(__file__).parent / "fixtures"
ENV_DIR = (FIXTURES_DIR / "installed").resolve() ENV_DIR = (FIXTURES_DIR / "installed").resolve()
SITE_PACKAGES = ENV_DIR / "lib" / "python3.7" / "site-packages" SITE_PURELIB = ENV_DIR / "lib" / "python3.7" / "site-packages"
SITE_PLATLIB = ENV_DIR / "lib64" / "python3.7" / "site-packages"
SRC = ENV_DIR / "src" SRC = ENV_DIR / "src"
VENDOR_DIR = ENV_DIR / "vendor" / "py3.7" VENDOR_DIR = ENV_DIR / "vendor" / "py3.7"
INSTALLED_RESULTS = [ INSTALLED_RESULTS = [
metadata.PathDistribution(SITE_PACKAGES / "cleo-0.7.6.dist-info"), metadata.PathDistribution(SITE_PURELIB / "cleo-0.7.6.dist-info"),
metadata.PathDistribution(SRC / "pendulum" / "pendulum.egg-info"), metadata.PathDistribution(SRC / "pendulum" / "pendulum.egg-info"),
metadata.PathDistribution( metadata.PathDistribution(
zipp.Path(str(SITE_PACKAGES / "foo-0.1.0-py3.8.egg"), "EGG-INFO") zipp.Path(str(SITE_PURELIB / "foo-0.1.0-py3.8.egg"), "EGG-INFO")
), ),
metadata.PathDistribution(VENDOR_DIR / "attrs-19.3.0.dist-info"), metadata.PathDistribution(VENDOR_DIR / "attrs-19.3.0.dist-info"),
metadata.PathDistribution(SITE_PACKAGES / "editable-2.3.4.dist-info"), metadata.PathDistribution(SITE_PURELIB / "editable-2.3.4.dist-info"),
metadata.PathDistribution(SITE_PACKAGES / "editable-with-import-2.3.4.dist-info"), metadata.PathDistribution(SITE_PURELIB / "editable-with-import-2.3.4.dist-info"),
metadata.PathDistribution(SITE_PLATLIB / "lib64-2.3.4.dist-info"),
metadata.PathDistribution(SITE_PLATLIB / "bender-2.0.5.dist-info"),
] ]
class MockEnv(BaseMockEnv): class MockEnv(BaseMockEnv):
@property @property
def site_packages(self): # type: () -> Path def paths(self):
return SITE_PACKAGES return {
"purelib": SITE_PURELIB,
"platlib": SITE_PLATLIB,
}
@property
def sys_path(self):
return [ENV_DIR, SITE_PLATLIB, SITE_PURELIB]
@pytest.fixture @pytest.fixture
...@@ -58,17 +71,27 @@ def repository(mocker, env): # type: (MockFixture, MockEnv) -> InstalledReposit ...@@ -58,17 +71,27 @@ def repository(mocker, env): # type: (MockFixture, MockEnv) -> InstalledReposit
return InstalledRepository.load(env) return InstalledRepository.load(env)
def get_package_from_repository(
name, repository
): # type: (str, InstalledRepository) -> Optional[Package]
for pkg in repository.packages:
if pkg.name == name:
return pkg
return None
def test_load_successful(repository): def test_load_successful(repository):
assert len(repository.packages) == 5 assert len(repository.packages) == len(INSTALLED_RESULTS) - 1
def test_load_ensure_isolation(repository): def test_load_ensure_isolation(repository):
for pkg in repository.packages: package = get_package_from_repository("attrs", repository)
assert pkg.name != "attrs" assert package is None
def test_load_standard_package(repository): def test_load_standard_package(repository):
cleo = repository.packages[0] cleo = get_package_from_repository("cleo", repository)
assert cleo is not None
assert cleo.name == "cleo" assert cleo.name == "cleo"
assert cleo.version.text == "0.7.6" assert cleo.version.text == "0.7.6"
assert ( assert (
...@@ -76,27 +99,47 @@ def test_load_standard_package(repository): ...@@ -76,27 +99,47 @@ def test_load_standard_package(repository):
== "Cleo allows you to create beautiful and testable command-line interfaces." == "Cleo allows you to create beautiful and testable command-line interfaces."
) )
foo = repository.packages[3] foo = get_package_from_repository("foo", repository)
assert foo.name == "foo" assert foo is not None
assert foo.version.text == "0.1.0" assert foo.version.text == "0.1.0"
def test_load_git_package(repository): def test_load_git_package(repository):
pendulum = repository.packages[4] pendulum = get_package_from_repository("pendulum", repository)
assert pendulum is not None
assert pendulum.name == "pendulum" assert pendulum.name == "pendulum"
assert pendulum.version.text == "2.0.5" assert pendulum.version.text == "2.0.5"
assert pendulum.description == "Python datetimes made easy" assert pendulum.description == "Python datetimes made easy"
assert pendulum.source_type == "git" assert pendulum.source_type == "git"
assert pendulum.source_url == "https://github.com/sdispater/pendulum.git" assert pendulum.source_url in [
"git@github.com:sdispater/pendulum.git",
"https://github.com/sdispater/pendulum.git",
]
assert pendulum.source_reference == "bb058f6b78b2d28ef5d9a5e759cfa179a1a713d6" assert pendulum.source_reference == "bb058f6b78b2d28ef5d9a5e759cfa179a1a713d6"
def test_load_git_package_pth(repository):
bender = get_package_from_repository("bender", repository)
assert bender is not None
assert bender.name == "bender"
assert bender.version.text == "2.0.5"
assert bender.source_type == "git"
def test_load_platlib_package(repository):
lib64 = get_package_from_repository("lib64", repository)
assert lib64 is not None
assert lib64.name == "lib64"
assert lib64.version.text == "2.3.4"
@pytest.mark.skipif( @pytest.mark.skipif(
not PY36, reason="pathlib.resolve() does not support strict argument" not PY36, reason="pathlib.resolve() does not support strict argument"
) )
def test_load_editable_package(repository): def test_load_editable_package(repository):
# test editable package with text .pth file # test editable package with text .pth file
editable = repository.packages[1] editable = get_package_from_repository("editable", repository)
assert editable is not None
assert editable.name == "editable" assert editable.name == "editable"
assert editable.version.text == "2.3.4" assert editable.version.text == "2.3.4"
assert editable.source_type == "directory" assert editable.source_type == "directory"
...@@ -108,7 +151,8 @@ def test_load_editable_package(repository): ...@@ -108,7 +151,8 @@ def test_load_editable_package(repository):
def test_load_editable_with_import_package(repository): def test_load_editable_with_import_package(repository):
# test editable package with executable .pth file # test editable package with executable .pth file
editable = repository.packages[2] editable = get_package_from_repository("editable-with-import", repository)
assert editable is not None
assert editable.name == "editable-with-import" assert editable.name == "editable-with-import"
assert editable.version.text == "2.3.4" assert editable.version.text == "2.3.4"
assert editable.source_type == "" assert editable.source_type == ""
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment