Skip to content

P
python-poetry
Commit c019b817 by Sébastien Eustace Committed by GitHub
Options

Merge pull request #2722 from abn/fix-installed-packages 

Improve detection of installed packages in an environment
parents f8123c84 e68797d7
Showing with 231 additions and 54 deletions
poetry/repositories/installed_repository.py
import itertools
from typing import Set
from typing import Union
from poetry.core.packages import Package
from poetry.core.utils.helpers import module_name
from poetry.utils._compat import Path
from poetry.utils._compat import metadata
from poetry.utils.env import Env
......@@ -11,11 +15,17 @@ from .repository import Repository
_VENDORS = Path(__file__).parent.parent.joinpath("_vendor")
try:
FileNotFoundError
except NameError:
FileNotFoundError = OSError
class InstalledRepository(Repository):
@classmethod
def get_package_paths(cls, sitedir, name): # type: (Path, str) -> Set[Path]
def get_package_paths(cls, env, name): # type: (Env, str) -> Set[Path]
"""
Process a .pth file within the site-packages directory, and return any valid
Process a .pth file within the site-packages directories, and return any valid
paths. We skip executable .pth files as there is no reliable means to do this
without side-effects to current run-time. Mo check is made that the item refers
to a directory rather than a file, however, in order to maintain backwards
......@@ -24,26 +34,73 @@ class InstalledRepository(Repository):
Reference: https://docs.python.org/3.8/library/site.html
:param sitedir: The site-packages directory to search for .pth file.
:param env: The environment to search for the .pth file in.
:param name: The name of the package to search .pth file for.
:return: A `Set` of valid `Path` objects.
"""
paths = set()
pth_file = sitedir.joinpath("{}.pth".format(name))
if pth_file.exists():
# we identify the candidate pth files to check, this is done so to handle cases
# where the pth file for foo-bar might have been installed as either foo-bar.pth or
# foo_bar.pth (expected) in either pure or platform lib directories.
candidates = itertools.product(
{env.purelib, env.platlib}, {name, module_name(name)},
)
for lib, module in candidates:
pth_file = lib.joinpath(module).with_suffix(".pth")
if not pth_file.exists():
continue
with pth_file.open() as f:
for line in f:
line = line.strip()
if line and not line.startswith(("#", "import ", "import\t")):
path = Path(line)
if not path.is_absolute():
path = sitedir.joinpath(path)
try:
path = lib.joinpath(path).resolve()
except FileNotFoundError:
# this is required to handle pathlib oddity on win32 python==3.5
path = lib.joinpath(path)
paths.add(path)
return paths
@classmethod
def set_package_vcs_properties_from_path(
cls, src, package
): # type: (Path, Package) -> None
from poetry.core.vcs.git import Git
git = Git()
revision = git.rev_parse("HEAD", src).strip()
url = git.remote_url(src)
package.source_type = "git"
package.source_url = url
package.source_reference = revision
@classmethod
def set_package_vcs_properties(cls, package, env): # type: (Package, Env) -> None
src = env.path / "src" / package.name
cls.set_package_vcs_properties_from_path(src, package)
@classmethod
def is_vcs_package(cls, package, env): # type: (Union[Path, Package], Env) -> bool
# A VCS dependency should have been installed
# in the src directory.
src = env.path / "src"
if isinstance(package, Package):
return src.joinpath(package.name).is_dir()
try:
package.relative_to(env.path / "src")
except ValueError:
return False
else:
return True
@classmethod
def load(cls, env): # type: (Env) -> InstalledRepository
"""
Load installed packages.
......@@ -75,41 +132,26 @@ class InstalledRepository(Repository):
repo.add_package(package)
is_standard_package = True
try:
path.relative_to(env.site_packages)
except ValueError:
is_standard_package = False
is_standard_package = env.is_path_relative_to_lib(path)
if is_standard_package:
if path.name.endswith(".dist-info"):
paths = cls.get_package_paths(
sitedir=env.site_packages, name=package.pretty_name
)
paths = cls.get_package_paths(env=env, name=package.pretty_name)
if paths:
for src in paths:
if cls.is_vcs_package(src, env):
cls.set_package_vcs_properties(package, env)
break
else:
# TODO: handle multiple source directories?
package.source_type = "directory"
package.source_url = paths.pop().as_posix()
continue
src_path = env.path / "src"
# A VCS dependency should have been installed
# in the src directory. If not, it's a path dependency
try:
path.relative_to(src_path)
from poetry.core.vcs.git import Git
git = Git()
revision = git.rev_parse("HEAD", src_path / package.name).strip()
url = git.remote_url(src_path / package.name)
package.source_type = "git"
package.source_url = url
package.source_reference = revision
except ValueError:
if cls.is_vcs_package(path, env):
cls.set_package_vcs_properties(package, env)
else:
# If not, it's a path dependency
package.source_type = "directory"
package.source_url = str(path.parent)
......
poetry/utils/env.py
......@@ -754,6 +754,8 @@ class Env(object):
self._site_packages = None
self._paths = None
self._supported_tags = None
self._purelib = None
self._platlib = None
@property
def path(self): # type: () -> Path
......@@ -810,11 +812,38 @@ class Env(object):
@property
def site_packages(self): # type: () -> Path
if self._site_packages is None:
self._site_packages = Path(self.paths["purelib"])
self._site_packages = Path(self.purelib)
return self._site_packages
@property
def purelib(self): # type: () -> Path
if self._purelib is None:
self._purelib = Path(self.paths["purelib"])
return self._purelib
@property
def platlib(self): # type: () -> Path
if self._platlib is None:
if "platlib" in self.paths:
self._platlib = Path(self.paths["platlib"])
else:
self._platlib = self.purelib
return self._platlib
def is_path_relative_to_lib(self, path): # type: (Path) -> bool
for lib_path in [self.purelib, self.platlib]:
try:
path.relative_to(lib_path)
return True
except ValueError:
pass
return False
@property
def sys_path(self): # type: () -> List[str]
raise NotImplementedError()
......
tests/fixtures/complete.toml
......@@ -22,7 +22,7 @@ toml = "^0.9"
# Dependencies with extras
requests = { version = "^2.13", extras = [ "security" ] }
# Python specific dependencies with prereleases allowed
pathlib2 = { version = "^2.2", python = "~2.7", allows-prereleases = true }
pathlib2 = { version = "^2.2", python = "~2.7", allow-prereleases = true }
# Git dependencies
cleo = { git = "https://github.com/sdispater/cleo.git", branch = "master" }
......
tests/installation/test_installer.py
......@@ -890,7 +890,7 @@ def test_run_with_prereleases(installer, locker, repo, package):
repo.add_package(package_a)
repo.add_package(package_b)
package.add_dependency("A", {"version": "*", "allows-prereleases": True})
package.add_dependency("A", {"version": "*", "allow-prereleases": True})
package.add_dependency("B", "^1.1")
installer.update(True)
......
tests/installation/test_installer_old.py
......@@ -861,7 +861,7 @@ def test_run_with_prereleases(installer, locker, repo, package):
repo.add_package(package_a)
repo.add_package(package_b)
package.add_dependency("A", {"version": "*", "allows-prereleases": True})
package.add_dependency("A", {"version": "*", "allow-prereleases": True})
package.add_dependency("B", "^1.1")
installer.update(True)
......
tests/repositories/fixtures/installed/lib64/python3.7/site-packages/bender-2.0.5.dist-info/METADATA 0 → 100644
Metadata-Version: 2.1
Name: bender
Version: 2.0.5
Summary: Python datetimes made easy
License: MIT
Keywords: cli,commands
Author: Leela
Author-email: leela@planetexpress.com
Requires-Python: >=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*
Classifier: License :: OSI Approved :: MIT License
Classifier: Programming Language :: Python :: 2
Classifier: Programming Language :: Python :: 2.7
Classifier: Programming Language :: Python :: 3
Classifier: Programming Language :: Python :: 3.4
Classifier: Programming Language :: Python :: 3.5
Classifier: Programming Language :: Python :: 3.6
Classifier: Programming Language :: Python :: 3.7
Classifier: Programming Language :: Python :: 3.8
Description-Content-Type: text/x-rst
tests/repositories/fixtures/installed/lib64/python3.7/site-packages/lib64-2.3.4.dist-info/METADATA 0 → 100644
Metadata-Version: 2.1
Name: lib64
Version: 2.3.4
Summary: lib64 description.
License: MIT
Keywords: cli,commands
Author: Foo Bar
Author-email: foo@bar.com
Requires-Python: >=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*
Classifier: License :: OSI Approved :: MIT License
Classifier: Programming Language :: Python :: 2
Classifier: Programming Language :: Python :: 2.7
Classifier: Programming Language :: Python :: 3
Classifier: Programming Language :: Python :: 3.4
Classifier: Programming Language :: Python :: 3.5
Classifier: Programming Language :: Python :: 3.6
Classifier: Programming Language :: Python :: 3.7
Classifier: Programming Language :: Python :: 3.8
Description-Content-Type: text/x-rst
lib64
####
tests/repositories/fixtures/installed/src/bender/bender.egg-info/PKG-INFO 0 → 100644
Metadata-Version: 2.1
Name: bender
Version: 2.0.5
Summary: Python datetimes made easy
License: MIT
Keywords: cli,commands
Author: Leela
Author-email: leela@planetexpress.com
Requires-Python: >=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*
Classifier: License :: OSI Approved :: MIT License
Classifier: Programming Language :: Python :: 2
Classifier: Programming Language :: Python :: 2.7
Classifier: Programming Language :: Python :: 3
Classifier: Programming Language :: Python :: 3.4
Classifier: Programming Language :: Python :: 3.5
Classifier: Programming Language :: Python :: 3.6
Classifier: Programming Language :: Python :: 3.7
Classifier: Programming Language :: Python :: 3.8
Description-Content-Type: text/x-rst
tests/repositories/test_installed_repository.py
from typing import Optional
import pytest
from poetry.core.packages import Package
from poetry.repositories.installed_repository import InstalledRepository
from poetry.utils._compat import PY36
from poetry.utils._compat import Path
......@@ -11,25 +14,35 @@ from pytest_mock.plugin import MockFixture
FIXTURES_DIR = Path(__file__).parent / "fixtures"
ENV_DIR = (FIXTURES_DIR / "installed").resolve()
SITE_PACKAGES = ENV_DIR / "lib" / "python3.7" / "site-packages"
SITE_PURELIB = ENV_DIR / "lib" / "python3.7" / "site-packages"
SITE_PLATLIB = ENV_DIR / "lib64" / "python3.7" / "site-packages"
SRC = ENV_DIR / "src"
VENDOR_DIR = ENV_DIR / "vendor" / "py3.7"
INSTALLED_RESULTS = [
metadata.PathDistribution(SITE_PACKAGES / "cleo-0.7.6.dist-info"),
metadata.PathDistribution(SITE_PURELIB / "cleo-0.7.6.dist-info"),
metadata.PathDistribution(SRC / "pendulum" / "pendulum.egg-info"),
metadata.PathDistribution(
zipp.Path(str(SITE_PACKAGES / "foo-0.1.0-py3.8.egg"), "EGG-INFO")
zipp.Path(str(SITE_PURELIB / "foo-0.1.0-py3.8.egg"), "EGG-INFO")
),
metadata.PathDistribution(VENDOR_DIR / "attrs-19.3.0.dist-info"),
metadata.PathDistribution(SITE_PACKAGES / "editable-2.3.4.dist-info"),
metadata.PathDistribution(SITE_PACKAGES / "editable-with-import-2.3.4.dist-info"),
metadata.PathDistribution(SITE_PURELIB / "editable-2.3.4.dist-info"),
metadata.PathDistribution(SITE_PURELIB / "editable-with-import-2.3.4.dist-info"),
metadata.PathDistribution(SITE_PLATLIB / "lib64-2.3.4.dist-info"),
metadata.PathDistribution(SITE_PLATLIB / "bender-2.0.5.dist-info"),
]
class MockEnv(BaseMockEnv):
@property
def site_packages(self): # type: () -> Path
return SITE_PACKAGES
def paths(self):
return {
"purelib": SITE_PURELIB,
"platlib": SITE_PLATLIB,
}
@property
def sys_path(self):
return [ENV_DIR, SITE_PLATLIB, SITE_PURELIB]
@pytest.fixture
......@@ -58,17 +71,27 @@ def repository(mocker, env): # type: (MockFixture, MockEnv) -> InstalledReposit
return InstalledRepository.load(env)
def get_package_from_repository(
name, repository
): # type: (str, InstalledRepository) -> Optional[Package]
for pkg in repository.packages:
if pkg.name == name:
return pkg
return None
def test_load_successful(repository):
assert len(repository.packages) == 5
assert len(repository.packages) == len(INSTALLED_RESULTS) - 1
def test_load_ensure_isolation(repository):
for pkg in repository.packages:
assert pkg.name != "attrs"
package = get_package_from_repository("attrs", repository)
assert package is None
def test_load_standard_package(repository):
cleo = repository.packages[0]
cleo = get_package_from_repository("cleo", repository)
assert cleo is not None
assert cleo.name == "cleo"
assert cleo.version.text == "0.7.6"
assert (
......@@ -76,27 +99,47 @@ def test_load_standard_package(repository):
== "Cleo allows you to create beautiful and testable command-line interfaces."
)
foo = repository.packages[3]
assert foo.name == "foo"
foo = get_package_from_repository("foo", repository)
assert foo is not None
assert foo.version.text == "0.1.0"
def test_load_git_package(repository):
pendulum = repository.packages[4]
pendulum = get_package_from_repository("pendulum", repository)
assert pendulum is not None
assert pendulum.name == "pendulum"
assert pendulum.version.text == "2.0.5"
assert pendulum.description == "Python datetimes made easy"
assert pendulum.source_type == "git"
assert pendulum.source_url == "https://github.com/sdispater/pendulum.git"
assert pendulum.source_url in [
"git@github.com:sdispater/pendulum.git",
"https://github.com/sdispater/pendulum.git",
]
assert pendulum.source_reference == "bb058f6b78b2d28ef5d9a5e759cfa179a1a713d6"
def test_load_git_package_pth(repository):
bender = get_package_from_repository("bender", repository)
assert bender is not None
assert bender.name == "bender"
assert bender.version.text == "2.0.5"
assert bender.source_type == "git"
def test_load_platlib_package(repository):
lib64 = get_package_from_repository("lib64", repository)
assert lib64 is not None
assert lib64.name == "lib64"
assert lib64.version.text == "2.3.4"
@pytest.mark.skipif(
not PY36, reason="pathlib.resolve() does not support strict argument"
)
def test_load_editable_package(repository):
# test editable package with text .pth file
editable = repository.packages[1]
editable = get_package_from_repository("editable", repository)
assert editable is not None
assert editable.name == "editable"
assert editable.version.text == "2.3.4"
assert editable.source_type == "directory"
......@@ -108,7 +151,8 @@ def test_load_editable_package(repository):
def test_load_editable_with_import_package(repository):
# test editable package with executable .pth file
editable = repository.packages[2]
editable = get_package_from_repository("editable-with-import", repository)
assert editable is not None
assert editable.name == "editable-with-import"
assert editable.version.text == "2.3.4"
assert editable.source_type == ""
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment