Skip to content

P
python-poetry
Commit 10b8eab5 by Arun Babu Neelicattu
Options

unify package inspection logic 

This change brings together logic repeated in various places to
discover package information.
parent f20043af
Showing with 69 additions and 575 deletions
poetry/puzzle/provider.py
import glob
import logging
import os
import re
......@@ -10,8 +9,6 @@ from typing import Any
from typing import List
from typing import Optional
import pkginfo
from clikit.ui.components import ProgressIndicator
from poetry.core.packages import Dependency
......@@ -20,34 +17,25 @@ from poetry.core.packages import FileDependency
from poetry.core.packages import Package
from poetry.core.packages import URLDependency
from poetry.core.packages import VCSDependency
from poetry.core.packages import dependency_from_pep_508
from poetry.core.packages.utils.utils import get_python_constraint_from_marker
from poetry.core.utils.helpers import parse_requires
from poetry.core.vcs.git import Git
from poetry.core.version.markers import MarkerUnion
from poetry.factory import Factory
from poetry.inspection.info import PackageInfo
from poetry.inspection.info import PackageInfoError
from poetry.mixology.incompatibility import Incompatibility
from poetry.mixology.incompatibility_cause import DependencyCause
from poetry.mixology.incompatibility_cause import PythonCause
from poetry.mixology.term import Term
from poetry.packages import DependencyPackage
from poetry.packages.package_collection import PackageCollection
from poetry.puzzle.exceptions import OverrideNeeded
from poetry.repositories import Pool
from poetry.utils._compat import PY35
from poetry.utils._compat import OrderedDict
from poetry.utils._compat import Path
from poetry.utils._compat import urlparse
from poetry.utils.env import EnvCommandError
from poetry.utils.env import EnvManager
from poetry.utils.env import VirtualEnv
from poetry.utils.helpers import download_file
from poetry.utils.helpers import safe_rmtree
from poetry.utils.helpers import temporary_directory
from poetry.utils.inspector import Inspector
from poetry.utils.setup_reader import SetupReader
from poetry.utils.toml_file import TomlFile
from .exceptions import OverrideNeeded
logger = logging.getLogger(__name__)
......@@ -68,7 +56,6 @@ class Provider:
self._package = package
self._pool = pool
self._io = io
self._inspector = Inspector()
self._python_constraint = package.python_constraint
self._search_for = {}
self._is_debugging = self._io.is_debug() or self._io.is_very_verbose()
......@@ -245,31 +232,18 @@ class Provider:
@classmethod
def get_package_from_file(cls, file_path): # type: (Path) -> Package
info = Inspector().inspect(file_path)
if not info["name"]:
try:
package = PackageInfo.from_path(path=file_path).to_package(
root_dir=file_path
)
except PackageInfoError:
raise RuntimeError(
"Unable to determine the package name of {}".format(file_path)
"Unable to determine package info from path: {}".format(file_path)
)
package = Package(info["name"], info["version"])
package.source_type = "file"
package.source_url = file_path.as_posix()
package.description = info["summary"]
for req in info["requires_dist"]:
dep = dependency_from_pep_508(req)
for extra in dep.in_extras:
if extra not in package.extras:
package.extras[extra] = []
package.extras[extra].append(dep)
if not dep.is_optional():
package.requires.append(dep)
if info["requires_python"]:
package.python_versions = info["requires_python"]
return package
def search_for_directory(
......@@ -298,136 +272,9 @@ class Provider:
def get_package_from_directory(
cls, directory, name=None
): # type: (Path, Optional[str]) -> Package
supports_poetry = False
pyproject = directory.joinpath("pyproject.toml")
if pyproject.exists():
pyproject = TomlFile(pyproject)
pyproject_content = pyproject.read()
supports_poetry = (
"tool" in pyproject_content and "poetry" in pyproject_content["tool"]
)
if supports_poetry:
poetry = Factory().create_poetry(directory)
pkg = poetry.package
package = Package(pkg.name, pkg.version)
for dep in pkg.requires:
if not dep.is_optional():
package.requires.append(dep)
for extra, deps in pkg.extras.items():
if extra not in package.extras:
package.extras[extra] = []
for dep in deps:
package.extras[extra].append(dep)
package.python_versions = pkg.python_versions
else:
# Execute egg_info
current_dir = os.getcwd()
os.chdir(str(directory))
try:
with temporary_directory() as tmp_dir:
EnvManager.build_venv(tmp_dir)
venv = VirtualEnv(Path(tmp_dir), Path(tmp_dir))
venv.run("python", "setup.py", "egg_info")
except EnvCommandError:
result = SetupReader.read_from_directory(directory)
if not result["name"]:
# The name could not be determined
# We use the dependency name
result["name"] = name
if not result["version"]:
# The version could not be determined
# so we raise an error since it is mandatory
raise RuntimeError(
"Unable to retrieve the package version for {}".format(
directory
)
)
package_name = result["name"]
package_version = result["version"]
python_requires = result["python_requires"]
if python_requires is None:
python_requires = "*"
package_summary = ""
requires = ""
for dep in result["install_requires"]:
requires += dep + "\n"
if result["extras_require"]:
requires += "\n"
for extra_name, deps in result["extras_require"].items():
requires += "[{}]\n".format(extra_name)
for dep in deps:
requires += dep + "\n"
requires += "\n"
reqs = parse_requires(requires)
else:
os.chdir(current_dir)
# Sometimes pathlib will fail on recursive
# symbolic links, so we need to workaround it
# and use the glob module instead.
# Note that this does not happen with pathlib2
# so it's safe to use it for Python < 3.4.
if PY35:
egg_info = next(
Path(p)
for p in glob.glob(
os.path.join(str(directory), "**", "*.egg-info"),
recursive=True,
)
)
else:
egg_info = next(directory.glob("**/*.egg-info"))
meta = pkginfo.UnpackedSDist(str(egg_info))
package_name = meta.name
package_version = meta.version
package_summary = meta.summary
python_requires = meta.requires_python
if meta.requires_dist:
reqs = list(meta.requires_dist)
else:
reqs = []
requires = egg_info / "requires.txt"
if requires.exists():
with requires.open(encoding="utf-8") as f:
reqs = parse_requires(f.read())
finally:
os.chdir(current_dir)
package = Package(package_name, package_version)
package.description = package_summary
for req in reqs:
dep = dependency_from_pep_508(req)
if dep.in_extras:
for extra in dep.in_extras:
if extra not in package.extras:
package.extras[extra] = []
package.extras[extra].append(dep)
if not dep.is_optional():
package.requires.append(dep)
if python_requires:
package.python_versions = python_requires
package = PackageInfo.from_directory(
path=directory, allow_build=True
).to_package(root_dir=directory)
if name and name != package.name:
# For now, the dependency's name must match the actual package's name
raise RuntimeError(
......
poetry/repositories/legacy_repository.py
......@@ -14,19 +14,17 @@ from cachecontrol.caches.file_cache import FileCache
from cachy import CacheManager
from poetry.core.packages import Package
from poetry.core.packages import dependency_from_pep_508
from poetry.core.packages.utils.link import Link
from poetry.core.semver import Version
from poetry.core.semver import VersionConstraint
from poetry.core.semver import VersionRange
from poetry.core.semver import parse_constraint
from poetry.core.version.markers import InvalidMarker
from poetry.locations import REPOSITORY_CACHE_DIR
from poetry.utils._compat import Path
from poetry.utils.helpers import canonicalize_name
from poetry.utils.inspector import Inspector
from poetry.utils.patterns import wheel_file_re
from ..inspection.info import PackageInfo
from .auth import Auth
from .exceptions import PackageNotFound
from .pypi_repository import PyPiRepository
......@@ -171,7 +169,6 @@ class LegacyRepository(PyPiRepository):
self._auth = auth
self._client_cert = client_cert
self._cert = cert
self._inspector = Inspector()
self._cache_dir = REPOSITORY_CACHE_DIR / name
self._cache = CacheManager(
{
......@@ -298,63 +295,8 @@ class LegacyRepository(PyPiRepository):
return self._packages[index]
except ValueError:
if extras is None:
extras = []
release_info = self.get_release_info(name, version)
package = Package(name, version, version)
if release_info["requires_python"]:
package.python_versions = release_info["requires_python"]
package = super(LegacyRepository, self).package(name, version, extras)
package.source_url = self._url
package.source_reference = self.name
requires_dist = release_info["requires_dist"] or []
for req in requires_dist:
try:
dependency = dependency_from_pep_508(req)
except InvalidMarker:
# Invalid marker
# We strip the markers hoping for the best
req = req.split(";")[0]
dependency = dependency_from_pep_508(req)
except ValueError:
# Likely unable to parse constraint so we skip it
self._log(
"Invalid constraint ({}) found in {}-{} dependencies, "
"skipping".format(req, package.name, package.version),
level="debug",
)
continue
if dependency.in_extras:
for extra in dependency.in_extras:
if extra not in package.extras:
package.extras[extra] = []
package.extras[extra].append(dependency)
if not dependency.is_optional():
package.requires.append(dependency)
# Adding description
package.description = release_info.get("summary", "")
# Adding hashes information
package.files = release_info["files"]
# Activate extra dependencies
for extra in extras:
if extra in package.extras:
for dep in package.extras[extra]:
dep.activate()
package.requires += package.extras[extra]
self._packages.append(package)
return package
def _get_release_info(self, name, version): # type: (str, str) -> dict
......@@ -362,15 +304,16 @@ class LegacyRepository(PyPiRepository):
if page is None:
raise PackageNotFound('No package named "{}"'.format(name))
data = {
"name": name,
"version": version,
"summary": "",
"requires_dist": [],
"requires_python": None,
"files": [],
"_cache_version": str(self.CACHE_VERSION),
}
data = PackageInfo(
name=name,
version=version,
summary="",
platform=None,
requires_dist=[],
requires_python=None,
files=[],
cache_version=str(self.CACHE_VERSION),
)
links = list(page.links_for_version(Version.parse(version)))
if not links:
......@@ -394,15 +337,15 @@ class LegacyRepository(PyPiRepository):
h = link.hash_name + ":" + link.hash
files.append({"file": link.filename, "hash": h})
data["files"] = files
data.files = files
info = self._get_info_from_urls(urls)
data["summary"] = info["summary"]
data["requires_dist"] = info["requires_dist"]
data["requires_python"] = info["requires_python"]
data.summary = info.summary
data.requires_dist = info.requires_dist
data.requires_python = info.requires_python
return data
return data.asdict()
def _get(self, endpoint): # type: (str) -> Union[Page, None]
url = self._url + endpoint
......
poetry/utils/inspector.py deleted 100644 → 0
import logging
import os
import tarfile
import zipfile
from typing import Dict
from typing import List
from typing import Union
import pkginfo
from requests import get
from ._compat import Path
from .helpers import parse_requires
from .setup_reader import SetupReader
from .toml_file import TomlFile
logger = logging.getLogger(__name__)
class Inspector:
"""
A class to download and inspect remote packages.
"""
@classmethod
def download(cls, url, dest): # type: (str, Path) -> None
r = get(url, stream=True)
r.raise_for_status()
with open(str(dest), "wb") as f:
for chunk in r.iter_content(chunk_size=1024):
if chunk:
f.write(chunk)
def inspect(self, file_path): # type: (Path) -> Dict[str, Union[str, List[str]]]
if file_path.suffix == ".whl":
return self.inspect_wheel(file_path)
return self.inspect_sdist(file_path)
def inspect_wheel(
self, file_path
): # type: (Path) -> Dict[str, Union[str, List[str]]]
info = {
"name": "",
"version": "",
"summary": "",
"requires_python": None,
"requires_dist": [],
}
try:
meta = pkginfo.Wheel(str(file_path))
except ValueError:
# Unable to determine dependencies
# Assume none
return info
if meta.name:
info["name"] = meta.name
if meta.version:
info["version"] = meta.version
if meta.summary:
info["summary"] = meta.summary or ""
info["requires_python"] = meta.requires_python
if meta.requires_dist:
info["requires_dist"] = meta.requires_dist
return info
def inspect_sdist(
self, file_path
): # type: (Path) -> Dict[str, Union[str, List[str]]]
info = {
"name": "",
"version": "",
"summary": "",
"requires_python": None,
"requires_dist": None,
}
try:
meta = pkginfo.SDist(str(file_path))
if meta.name:
info["name"] = meta.name
if meta.version:
info["version"] = meta.version
if meta.summary:
info["summary"] = meta.summary
if meta.requires_python:
info["requires_python"] = meta.requires_python
if meta.requires_dist:
info["requires_dist"] = list(meta.requires_dist)
return info
except ValueError:
# Unable to determine dependencies
# We pass and go deeper
pass
# Still not dependencies found
# So, we unpack and introspect
suffix = file_path.suffix
if suffix == ".zip":
tar = zipfile.ZipFile(str(file_path))
else:
if suffix == ".bz2":
suffixes = file_path.suffixes
if len(suffixes) > 1 and suffixes[-2] == ".tar":
suffix = ".tar.bz2"
else:
suffix = ".tar.gz"
tar = tarfile.open(str(file_path))
try:
tar.extractall(os.path.join(str(file_path.parent), "unpacked"))
finally:
tar.close()
unpacked = file_path.parent / "unpacked"
elements = list(unpacked.glob("*"))
if len(elements) == 1 and elements[0].is_dir():
sdist_dir = elements[0]
else:
sdist_dir = unpacked / file_path.name.rstrip(suffix)
pyproject = TomlFile(sdist_dir / "pyproject.toml")
if pyproject.exists():
from poetry.factory import Factory
pyproject_content = pyproject.read()
if "tool" in pyproject_content and "poetry" in pyproject_content["tool"]:
package = Factory().create_poetry(sdist_dir).package
return {
"name": package.name,
"version": package.version.text,
"summary": package.description,
"requires_dist": [dep.to_pep_508() for dep in package.requires],
"requires_python": package.python_versions,
}
# Checking for .egg-info at root
eggs = list(sdist_dir.glob("*.egg-info"))
if eggs:
egg_info = eggs[0]
requires = egg_info / "requires.txt"
if requires.exists():
with requires.open(encoding="utf-8") as f:
info["requires_dist"] = parse_requires(f.read())
return info
# Searching for .egg-info in sub directories
eggs = list(sdist_dir.glob("**/*.egg-info"))
if eggs:
egg_info = eggs[0]
requires = egg_info / "requires.txt"
if requires.exists():
with requires.open(encoding="utf-8") as f:
info["requires_dist"] = parse_requires(f.read())
return info
# Still nothing, try reading (without executing it)
# the setup.py file.
try:
setup_info = self._inspect_sdist_with_setup(sdist_dir)
for key, value in info.items():
if value:
continue
info[key] = setup_info[key]
return info
except Exception as e:
logger.warning(
"An error occurred when reading setup.py or setup.cfg: {}".format(
str(e)
)
)
return info
def _inspect_sdist_with_setup(
self, sdist_dir
): # type: (Path) -> Dict[str, Union[str, List[str]]]
info = {
"name": None,
"version": None,
"summary": "",
"requires_python": None,
"requires_dist": None,
}
result = SetupReader.read_from_directory(sdist_dir)
requires = ""
for dep in result["install_requires"]:
requires += dep + "\n"
if result["extras_require"]:
requires += "\n"
for extra_name, deps in result["extras_require"].items():
requires += "[{}]\n".format(extra_name)
for dep in deps:
requires += dep + "\n"
requires += "\n"
info["name"] = result["name"]
info["version"] = result["version"]
info["requires_dist"] = parse_requires(requires)
info["requires_python"] = result["python_requires"]
return info
tests/conftest.py
......@@ -71,7 +71,6 @@ def config(config_source, auth_config_source, mocker):
@pytest.fixture(autouse=True)
def download_mock(mocker):
# Patch download to not download anything but to just copy from fixtures
mocker.patch("poetry.utils.inspector.Inspector.download", new=mock_download)
mocker.patch("poetry.utils.helpers.download_file", new=mock_download)
mocker.patch("poetry.puzzle.provider.download_file", new=mock_download)
mocker.patch("poetry.repositories.pypi_repository.download_file", new=mock_download)
......
tests/console/conftest.py
......@@ -16,7 +16,6 @@ from poetry.utils._compat import Path
from poetry.utils.env import MockEnv
from poetry.utils.toml_file import TomlFile
from tests.helpers import mock_clone
from tests.helpers import mock_download
@pytest.fixture()
......@@ -54,9 +53,6 @@ def setup(mocker, installer, installed, config, env):
p = mocker.patch("poetry.core.vcs.git.Git.rev_parse")
p.return_value = "9cf87a285a2d3fbb0b9fa621997b3acc3631ed24"
# Patch download to not download anything but to just copy from fixtures
mocker.patch("poetry.utils.inspector.Inspector.download", new=mock_download)
# Patch the virtual environment creation do actually do nothing
mocker.patch("poetry.utils.env.EnvManager.create_venv", return_value=env)
......
tests/installation/fixtures/with-directory-dependency-poetry-transitive.test
[[package]]
category = "main"
description = ""
description = "This is a description"
develop = true
name = "project-with-extras"
optional = false
......@@ -18,7 +18,7 @@ url = "tests/fixtures/directory/project_with_transitive_directory_dependencies/.
[[package]]
category = "main"
description = ""
description = "This is a description"
develop = true
name = "project-with-transitive-directory-dependencies"
optional = false
......
tests/installation/fixtures/with-directory-dependency-poetry.test
......@@ -8,7 +8,7 @@ version = "1.4.4"
[[package]]
category = "main"
description = ""
description = "This is a description"
develop = true
name = "project-with-extras"
optional = false
......
tests/installation/fixtures/with-file-dependency-transitive.test
......@@ -28,7 +28,7 @@ version = "1.4.4"
[[package]]
category = "main"
description = ""
description = "This is a description"
develop = true
name = "project-with-transitive-file-dependencies"
optional = false
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment