Skip to content

P
python-poetry
Commit 10b8eab5 by Arun Babu Neelicattu
Options

unify package inspection logic 

This change brings together logic repeated in various places to
discover package information.
parent f20043af
Showing with 568 additions and 575 deletions
poetry/inspection/info.py 0 → 100644
import glob
import logging
import os
import tarfile
import zipfile
from typing import Dict
from typing import Iterator
from typing import List
from typing import Optional
from typing import Union
import pkginfo
from poetry.core.factory import Factory
from poetry.core.packages import Package
from poetry.core.packages import ProjectPackage
from poetry.core.packages import dependency_from_pep_508
from poetry.core.utils._compat import PY35
from poetry.core.utils._compat import Path
from poetry.core.utils.helpers import parse_requires
from poetry.core.utils.helpers import temporary_directory
from poetry.core.version.markers import InvalidMarker
from poetry.utils.env import EnvCommandError
from poetry.utils.env import EnvManager
from poetry.utils.env import VirtualEnv
from poetry.utils.setup_reader import SetupReader
from poetry.utils.toml_file import TomlFile
logger = logging.getLogger(__name__)
class PackageInfoError(ValueError):
def __init__(self, path): # type: (Union[Path, str]) -> None
super(PackageInfoError, self).__init__(
"Unable to determine package info for path: {}".format(str(path))
)
class PackageInfo:
def __init__(
self,
name=None, # type: Optional[str]
version=None, # type: Optional[str]
summary=None, # type: Optional[str]
platform=None, # type: Optional[str]
requires_dist=None, # type: Optional[List[str]]
requires_python=None, # type: Optional[str]
files=None, # type: Optional[List[str]]
cache_version=None, # type: Optional[str]
):
self.name = name
self.version = version
self.summary = summary
self.platform = platform
self.requires_dist = requires_dist
self.requires_python = requires_python
self.files = files or []
self._cache_version = cache_version
@property
def cache_version(self): # type: () -> Optional[str]
return self._cache_version
def update(self, other): # type: (PackageInfo) -> PackageInfo
self.name = other.name or self.name
self.version = other.version or self.version
self.summary = other.summary or self.summary
self.platform = other.platform or self.platform
self.requires_dist = other.requires_dist or self.requires_dist
self.requires_python = other.requires_python or self.requires_python
self.files = other.files or self.files
self._cache_version = other.cache_version or self._cache_version
return self
def asdict(self): # type: () -> Dict[str, Optional[Union[str, List[str]]]]
"""
Helper method to convert package info into a dictionary used for caching.
"""
return {
"name": self.name,
"version": self.version,
"summary": self.summary,
"platform": self.platform,
"requires_dist": self.requires_dist,
"requires_python": self.requires_python,
"files": self.files,
"_cache_version": self._cache_version,
}
@classmethod
def load(
cls, data
): # type: (Dict[str, Optional[Union[str, List[str]]]]) -> PackageInfo
"""
Helper method to load data from a dictionary produced by `PackageInfo.asdict()`.
:param data: Data to load. This is expected to be a `dict` object output by `asdict()`.
"""
cache_version = data.pop("_cache_version", None)
return cls(cache_version=cache_version, **data)
@classmethod
def _log(cls, msg, level="info"):
"""Internal helper method to log information."""
getattr(logger, level)("<debug>{}:</debug> {}".format(cls.__name__, msg))
def to_package(
self, name=None, extras=None, root_dir=None
): # type: (Optional[str], Optional[List[str]], Optional[Path]) -> Package
"""
Create a new `poetry.core.packages.package.Package` instance using metadata from this instance.
:param name: Name to use for the package, if not specified name from this instance is used.
:param extras: Extras to activate for this package.
:param root_dir: Optional root directory to use for the package. If set, dependency strings
will be parsed relative to this directory.
"""
name = name or self.name
if not self.version:
# The version could not be determined, so we raise an error since it is mandatory.
raise RuntimeError(
"Unable to retrieve the package version for {}".format(name)
)
package = Package(name=name, version=self.version)
package.description = self.summary
package.root_dir = root_dir
package.python_versions = self.requires_python or "*"
package.files = self.files
for req in self.requires_dist or []:
try:
# Attempt to parse the PEP-508 requirement string
dependency = dependency_from_pep_508(req, relative_to=root_dir)
except InvalidMarker:
# Invalid marker, We strip the markers hoping for the best
req = req.split(";")[0]
dependency = dependency_from_pep_508(req, relative_to=root_dir)
except ValueError:
# Likely unable to parse constraint so we skip it
self._log(
"Invalid constraint ({}) found in {}-{} dependencies, "
"skipping".format(req, package.name, package.version),
level="debug",
)
continue
if dependency.in_extras:
# this dependency is required by an extra package
for extra in dependency.in_extras:
if extra not in package.extras:
# this is the first time we encounter this extra for this package
package.extras[extra] = []
# Activate extra dependencies if specified
if extras and extra in extras:
dependency.activate()
package.extras[extra].append(dependency)
if not dependency.is_optional() or dependency.is_activated():
# we skip add only if the dependency is option and was not activated as part of an extra
package.requires.append(dependency)
return package
@classmethod
def _from_distribution(
cls, dist
): # type: (Union[pkginfo.BDist, pkginfo.SDist, pkginfo.Wheel]) -> PackageInfo
"""
Helper method to parse package information from a `pkginfo.Distribution` instance.
:param dist: The distribution instance to parse information from.
"""
if dist.requires_dist:
requirements = list(dist.requires_dist)
else:
requirements = []
requires = Path(dist.filename) / "requires.txt"
if requires.exists():
with requires.open(encoding="utf-8") as f:
requirements = parse_requires(f.read())
return cls(
name=dist.name,
version=dist.version,
summary=dist.summary,
platform=dist.supported_platforms,
requires_dist=requirements or None,
requires_python=dist.requires_python,
)
@classmethod
def _from_sdist_file(cls, path): # type: (Path) -> PackageInfo
"""
Helper method to parse package information from an sdist file. We attempt to first inspect the
file using `pkginfo.SDist`. If this does not provide us with package requirements, we extract the
source and handle it as a directory.
:param path: The sdist file to parse information from.
"""
info = None
try:
info = cls._from_distribution(pkginfo.SDist(str(path)))
except ValueError:
# Unable to determine dependencies
# We pass and go deeper
pass
else:
if info.requires_dist is not None:
# we successfully retrieved dependencies from sdist metadata
return info
# Still not dependencies found
# So, we unpack and introspect
suffix = path.suffix
if suffix == ".zip":
context = zipfile.ZipFile
else:
if suffix == ".bz2":
suffixes = path.suffixes
if len(suffixes) > 1 and suffixes[-2] == ".tar":
suffix = ".tar.bz2"
else:
suffix = ".tar.gz"
context = tarfile.open
with temporary_directory() as tmp:
tmp = Path(tmp)
with context(str(path)) as archive:
archive.extractall(str(tmp))
# a little bit of guess work to determine the directory we care about
elements = list(tmp.glob("*"))
if len(elements) == 1 and elements[0].is_dir():
sdist_dir = elements[0]
else:
sdist_dir = tmp / path.name.rstrip(suffix)
# now this is an unpacked directory we know how to deal with
new_info = cls.from_directory(path=sdist_dir)
if not info:
return new_info
return info.update(new_info)
@classmethod
def from_setup_py(cls, path): # type: (Union[str, Path]) -> PackageInfo
"""
Mechanism to parse package information from a `setup.py` file. This uses the implentation
at `poetry.utils.setup_reader.SetupReader` in order to parse the file. This is not reliable for
complex setup files and should only attempted as a fallback.
:param path: Path to `setup.py` file
:return:
"""
result = SetupReader.read_from_directory(Path(path))
python_requires = result["python_requires"]
if python_requires is None:
python_requires = "*"
requires = ""
for dep in result["install_requires"]:
requires += dep + "\n"
if result["extras_require"]:
requires += "\n"
for extra_name, deps in result["extras_require"].items():
requires += "[{}]\n".format(extra_name)
for dep in deps:
requires += dep + "\n"
requires += "\n"
requirements = parse_requires(requires)
return cls(
name=result.get("name"),
version=result.get("version"),
summary=result.get("description", ""),
requires_dist=requirements or None,
requires_python=python_requires,
)
@staticmethod
def _find_dist_info(path): # type: (Path) -> Iterator[Path]
"""
Discover all `*.*-info` directories in a given path.
:param path: Path to search.
"""
pattern = "**/*.*-info"
if PY35:
# Sometimes pathlib will fail on recursive symbolic links, so we need to workaround it
# and use the glob module instead. Note that this does not happen with pathlib2
# so it's safe to use it for Python < 3.4.
directories = glob.iglob(Path(path, pattern).as_posix(), recursive=True)
else:
directories = path.glob(pattern)
for d in directories:
yield Path(d)
@classmethod
def from_metadata(cls, path): # type: (Union[str, Path]) -> PackageInfo
"""
Helper method to parse package information from an unpacked metadata directory.
:param path: The metadata directory to parse information from.
"""
path = Path(path)
if path.suffix in {".dist-info", ".egg-info"}:
directories = [path.suffix]
else:
directories = cls._find_dist_info(path=path)
for directory in directories:
try:
if directory.suffix == ".egg-info":
dist = pkginfo.UnpackedSDist(str(directory))
elif directory.suffix == ".dist-info":
dist = pkginfo.Wheel(str(directory))
else:
continue
info = cls._from_distribution(dist=dist)
if info:
return info
except ValueError:
pass
@classmethod
def from_package(cls, package): # type: (Package) -> PackageInfo
"""
Helper method to inspect a `Package` object, in order to generate package info.
:param package: This must be a poetry package instance.
"""
requires = {dependency.to_pep_508() for dependency in package.requires}
for extra_requires in package.extras.values():
for dependency in extra_requires:
requires.add(dependency.to_pep_508())
return cls(
name=package.name,
version=package.version,
summary=package.description,
platform=package.platform,
requires_dist=list(requires),
requires_python=package.python_versions,
files=package.files,
)
@staticmethod
def _get_poetry_package(path): # type: (Path) -> Optional[ProjectPackage]
pyproject = path.joinpath("pyproject.toml")
try:
# Note: we ignore any setup.py file at this step
if pyproject.exists():
# TODO: add support for handling non-poetry PEP-517 builds
pyproject = TomlFile(pyproject)
pyproject_content = pyproject.read()
supports_poetry = (
"tool" in pyproject_content
and "poetry" in pyproject_content["tool"]
)
if supports_poetry:
return Factory().create_poetry(path).package
except RuntimeError:
pass
@classmethod
def from_directory(
cls, path, allow_build=False
): # type: (Union[str, Path], bool) -> PackageInfo
"""
Generate package information from a package source directory. When `allow_build` is enabled and
introspection of all available metadata fails, the package is attempted to be build in an isolated
environment so as to generate required metadata.
:param path: Path to generate package information from.
:param allow_build: If enabled, as a fallback, build the project to gather metadata.
"""
path = Path(path)
setup_py = path.joinpath("setup.py")
project_package = cls._get_poetry_package(path)
if project_package:
return cls.from_package(project_package)
if not setup_py.exists():
# this means we cannot do anything else here
raise PackageInfoError(path)
current_dir = os.getcwd()
info = cls.from_metadata(path)
if info and info.requires_dist is not None:
# return only if requirements are discovered
return info
if not allow_build:
return cls.from_setup_py(path=path)
try:
# TODO: replace with PEP517
# we need to switch to the correct path in order for egg_info command to work
os.chdir(str(path))
# Execute egg_info
with temporary_directory() as tmp_dir:
EnvManager.build_venv(tmp_dir)
venv = VirtualEnv(Path(tmp_dir), Path(tmp_dir))
venv.run("python", "setup.py", "egg_info")
except EnvCommandError:
cls._log(
"Falling back to parsing setup.py file for {}".format(path), "debug"
)
# egg_info could not be generated, we fallback to ast parser
return cls.from_setup_py(path=path)
else:
info = cls.from_metadata(path)
if info:
return info
finally:
os.chdir(current_dir)
# if we reach here, everything has failed and all hope is lost
raise PackageInfoError(path)
@classmethod
def from_sdist(cls, path): # type: (Union[Path, pkginfo.SDist]) -> PackageInfo
"""
Gather package information from an sdist file, packed or unpacked.
:param path: Path to an sdist file or unpacked directory.
"""
if path.is_file():
return cls._from_sdist_file(path=path)
# if we get here then it is neither an sdist instance nor a file
# so, we assume this is an directory
return cls.from_directory(path=path)
@classmethod
def from_wheel(cls, path): # type: (Path) -> PackageInfo
"""
Gather package information from a wheel.
:param path: Path to wheel.
"""
try:
return cls._from_distribution(pkginfo.Wheel(str(path)))
except ValueError:
return PackageInfo()
@classmethod
def from_bdist(cls, path): # type: (Path) -> PackageInfo
"""
Gather package information from a bdist (wheel etc.).
:param path: Path to bdist.
"""
if isinstance(path, (pkginfo.BDist, pkginfo.Wheel)):
cls._from_distribution(dist=path)
if path.suffix == ".whl":
return cls.from_wheel(path=path)
try:
return cls._from_distribution(pkginfo.BDist(str(path)))
except ValueError:
raise PackageInfoError(path)
@classmethod
def from_path(cls, path): # type: (Path) -> PackageInfo
"""
Gather package information from a given path (bdist, sdist, directory).
:param path: Path to inspect.
"""
try:
return cls.from_bdist(path=path)
except PackageInfoError:
return cls.from_sdist(path=path)
poetry/puzzle/provider.py
import glob
import logging import logging
import os import os
import re import re
...@@ -10,8 +9,6 @@ from typing import Any ...@@ -10,8 +9,6 @@ from typing import Any
from typing import List from typing import List
from typing import Optional from typing import Optional
import pkginfo
from clikit.ui.components import ProgressIndicator from clikit.ui.components import ProgressIndicator
from poetry.core.packages import Dependency from poetry.core.packages import Dependency
...@@ -20,34 +17,25 @@ from poetry.core.packages import FileDependency ...@@ -20,34 +17,25 @@ from poetry.core.packages import FileDependency
from poetry.core.packages import Package from poetry.core.packages import Package
from poetry.core.packages import URLDependency from poetry.core.packages import URLDependency
from poetry.core.packages import VCSDependency from poetry.core.packages import VCSDependency
from poetry.core.packages import dependency_from_pep_508
from poetry.core.packages.utils.utils import get_python_constraint_from_marker from poetry.core.packages.utils.utils import get_python_constraint_from_marker
from poetry.core.utils.helpers import parse_requires
from poetry.core.vcs.git import Git from poetry.core.vcs.git import Git
from poetry.core.version.markers import MarkerUnion from poetry.core.version.markers import MarkerUnion
from poetry.factory import Factory from poetry.inspection.info import PackageInfo
from poetry.inspection.info import PackageInfoError
from poetry.mixology.incompatibility import Incompatibility from poetry.mixology.incompatibility import Incompatibility
from poetry.mixology.incompatibility_cause import DependencyCause from poetry.mixology.incompatibility_cause import DependencyCause
from poetry.mixology.incompatibility_cause import PythonCause from poetry.mixology.incompatibility_cause import PythonCause
from poetry.mixology.term import Term from poetry.mixology.term import Term
from poetry.packages import DependencyPackage from poetry.packages import DependencyPackage
from poetry.packages.package_collection import PackageCollection from poetry.packages.package_collection import PackageCollection
from poetry.puzzle.exceptions import OverrideNeeded
from poetry.repositories import Pool from poetry.repositories import Pool
from poetry.utils._compat import PY35
from poetry.utils._compat import OrderedDict from poetry.utils._compat import OrderedDict
from poetry.utils._compat import Path from poetry.utils._compat import Path
from poetry.utils._compat import urlparse from poetry.utils._compat import urlparse
from poetry.utils.env import EnvCommandError
from poetry.utils.env import EnvManager
from poetry.utils.env import VirtualEnv
from poetry.utils.helpers import download_file from poetry.utils.helpers import download_file
from poetry.utils.helpers import safe_rmtree from poetry.utils.helpers import safe_rmtree
from poetry.utils.helpers import temporary_directory from poetry.utils.helpers import temporary_directory
from poetry.utils.inspector import Inspector
from poetry.utils.setup_reader import SetupReader
from poetry.utils.toml_file import TomlFile
from .exceptions import OverrideNeeded
logger = logging.getLogger(__name__) logger = logging.getLogger(__name__)
...@@ -68,7 +56,6 @@ class Provider: ...@@ -68,7 +56,6 @@ class Provider:
self._package = package self._package = package
self._pool = pool self._pool = pool
self._io = io self._io = io
self._inspector = Inspector()
self._python_constraint = package.python_constraint self._python_constraint = package.python_constraint
self._search_for = {} self._search_for = {}
self._is_debugging = self._io.is_debug() or self._io.is_very_verbose() self._is_debugging = self._io.is_debug() or self._io.is_very_verbose()
...@@ -245,31 +232,18 @@ class Provider: ...@@ -245,31 +232,18 @@ class Provider:
@classmethod @classmethod
def get_package_from_file(cls, file_path): # type: (Path) -> Package def get_package_from_file(cls, file_path): # type: (Path) -> Package
info = Inspector().inspect(file_path) try:
if not info["name"]: package = PackageInfo.from_path(path=file_path).to_package(
root_dir=file_path
)
except PackageInfoError:
raise RuntimeError( raise RuntimeError(
"Unable to determine the package name of {}".format(file_path) "Unable to determine package info from path: {}".format(file_path)
) )
package = Package(info["name"], info["version"])
package.source_type = "file" package.source_type = "file"
package.source_url = file_path.as_posix() package.source_url = file_path.as_posix()
package.description = info["summary"]
for req in info["requires_dist"]:
dep = dependency_from_pep_508(req)
for extra in dep.in_extras:
if extra not in package.extras:
package.extras[extra] = []
package.extras[extra].append(dep)
if not dep.is_optional():
package.requires.append(dep)
if info["requires_python"]:
package.python_versions = info["requires_python"]
return package return package
def search_for_directory( def search_for_directory(
...@@ -298,136 +272,9 @@ class Provider: ...@@ -298,136 +272,9 @@ class Provider:
def get_package_from_directory( def get_package_from_directory(
cls, directory, name=None cls, directory, name=None
): # type: (Path, Optional[str]) -> Package ): # type: (Path, Optional[str]) -> Package
supports_poetry = False package = PackageInfo.from_directory(
pyproject = directory.joinpath("pyproject.toml") path=directory, allow_build=True
if pyproject.exists(): ).to_package(root_dir=directory)
pyproject = TomlFile(pyproject)
pyproject_content = pyproject.read()
supports_poetry = (
"tool" in pyproject_content and "poetry" in pyproject_content["tool"]
)
if supports_poetry:
poetry = Factory().create_poetry(directory)
pkg = poetry.package
package = Package(pkg.name, pkg.version)
for dep in pkg.requires:
if not dep.is_optional():
package.requires.append(dep)
for extra, deps in pkg.extras.items():
if extra not in package.extras:
package.extras[extra] = []
for dep in deps:
package.extras[extra].append(dep)
package.python_versions = pkg.python_versions
else:
# Execute egg_info
current_dir = os.getcwd()
os.chdir(str(directory))
try:
with temporary_directory() as tmp_dir:
EnvManager.build_venv(tmp_dir)
venv = VirtualEnv(Path(tmp_dir), Path(tmp_dir))
venv.run("python", "setup.py", "egg_info")
except EnvCommandError:
result = SetupReader.read_from_directory(directory)
if not result["name"]:
# The name could not be determined
# We use the dependency name
result["name"] = name
if not result["version"]:
# The version could not be determined
# so we raise an error since it is mandatory
raise RuntimeError(
"Unable to retrieve the package version for {}".format(
directory
)
)
package_name = result["name"]
package_version = result["version"]
python_requires = result["python_requires"]
if python_requires is None:
python_requires = "*"
package_summary = ""
requires = ""
for dep in result["install_requires"]:
requires += dep + "\n"
if result["extras_require"]:
requires += "\n"
for extra_name, deps in result["extras_require"].items():
requires += "[{}]\n".format(extra_name)
for dep in deps:
requires += dep + "\n"
requires += "\n"
reqs = parse_requires(requires)
else:
os.chdir(current_dir)
# Sometimes pathlib will fail on recursive
# symbolic links, so we need to workaround it
# and use the glob module instead.
# Note that this does not happen with pathlib2
# so it's safe to use it for Python < 3.4.
if PY35:
egg_info = next(
Path(p)
for p in glob.glob(
os.path.join(str(directory), "**", "*.egg-info"),
recursive=True,
)
)
else:
egg_info = next(directory.glob("**/*.egg-info"))
meta = pkginfo.UnpackedSDist(str(egg_info))
package_name = meta.name
package_version = meta.version
package_summary = meta.summary
python_requires = meta.requires_python
if meta.requires_dist:
reqs = list(meta.requires_dist)
else:
reqs = []
requires = egg_info / "requires.txt"
if requires.exists():
with requires.open(encoding="utf-8") as f:
reqs = parse_requires(f.read())
finally:
os.chdir(current_dir)
package = Package(package_name, package_version)
package.description = package_summary
for req in reqs:
dep = dependency_from_pep_508(req)
if dep.in_extras:
for extra in dep.in_extras:
if extra not in package.extras:
package.extras[extra] = []
package.extras[extra].append(dep)
if not dep.is_optional():
package.requires.append(dep)
if python_requires:
package.python_versions = python_requires
if name and name != package.name: if name and name != package.name:
# For now, the dependency's name must match the actual package's name # For now, the dependency's name must match the actual package's name
raise RuntimeError( raise RuntimeError(
......
poetry/repositories/legacy_repository.py
...@@ -14,19 +14,17 @@ from cachecontrol.caches.file_cache import FileCache ...@@ -14,19 +14,17 @@ from cachecontrol.caches.file_cache import FileCache
from cachy import CacheManager from cachy import CacheManager
from poetry.core.packages import Package from poetry.core.packages import Package
from poetry.core.packages import dependency_from_pep_508
from poetry.core.packages.utils.link import Link from poetry.core.packages.utils.link import Link
from poetry.core.semver import Version from poetry.core.semver import Version
from poetry.core.semver import VersionConstraint from poetry.core.semver import VersionConstraint
from poetry.core.semver import VersionRange from poetry.core.semver import VersionRange
from poetry.core.semver import parse_constraint from poetry.core.semver import parse_constraint
from poetry.core.version.markers import InvalidMarker
from poetry.locations import REPOSITORY_CACHE_DIR from poetry.locations import REPOSITORY_CACHE_DIR
from poetry.utils._compat import Path from poetry.utils._compat import Path
from poetry.utils.helpers import canonicalize_name from poetry.utils.helpers import canonicalize_name
from poetry.utils.inspector import Inspector
from poetry.utils.patterns import wheel_file_re from poetry.utils.patterns import wheel_file_re
from ..inspection.info import PackageInfo
from .auth import Auth from .auth import Auth
from .exceptions import PackageNotFound from .exceptions import PackageNotFound
from .pypi_repository import PyPiRepository from .pypi_repository import PyPiRepository
...@@ -171,7 +169,6 @@ class LegacyRepository(PyPiRepository): ...@@ -171,7 +169,6 @@ class LegacyRepository(PyPiRepository):
self._auth = auth self._auth = auth
self._client_cert = client_cert self._client_cert = client_cert
self._cert = cert self._cert = cert
self._inspector = Inspector()
self._cache_dir = REPOSITORY_CACHE_DIR / name self._cache_dir = REPOSITORY_CACHE_DIR / name
self._cache = CacheManager( self._cache = CacheManager(
{ {
...@@ -298,63 +295,8 @@ class LegacyRepository(PyPiRepository): ...@@ -298,63 +295,8 @@ class LegacyRepository(PyPiRepository):
return self._packages[index] return self._packages[index]
except ValueError: except ValueError:
if extras is None: package = super(LegacyRepository, self).package(name, version, extras)
extras = []
release_info = self.get_release_info(name, version)
package = Package(name, version, version)
if release_info["requires_python"]:
package.python_versions = release_info["requires_python"]
package.source_url = self._url package.source_url = self._url
package.source_reference = self.name
requires_dist = release_info["requires_dist"] or []
for req in requires_dist:
try:
dependency = dependency_from_pep_508(req)
except InvalidMarker:
# Invalid marker
# We strip the markers hoping for the best
req = req.split(";")[0]
dependency = dependency_from_pep_508(req)
except ValueError:
# Likely unable to parse constraint so we skip it
self._log(
"Invalid constraint ({}) found in {}-{} dependencies, "
"skipping".format(req, package.name, package.version),
level="debug",
)
continue
if dependency.in_extras:
for extra in dependency.in_extras:
if extra not in package.extras:
package.extras[extra] = []
package.extras[extra].append(dependency)
if not dependency.is_optional():
package.requires.append(dependency)
# Adding description
package.description = release_info.get("summary", "")
# Adding hashes information
package.files = release_info["files"]
# Activate extra dependencies
for extra in extras:
if extra in package.extras:
for dep in package.extras[extra]:
dep.activate()
package.requires += package.extras[extra]
self._packages.append(package)
return package return package
def _get_release_info(self, name, version): # type: (str, str) -> dict def _get_release_info(self, name, version): # type: (str, str) -> dict
...@@ -362,15 +304,16 @@ class LegacyRepository(PyPiRepository): ...@@ -362,15 +304,16 @@ class LegacyRepository(PyPiRepository):
if page is None: if page is None:
raise PackageNotFound('No package named "{}"'.format(name)) raise PackageNotFound('No package named "{}"'.format(name))
data = { data = PackageInfo(
"name": name, name=name,
"version": version, version=version,
"summary": "", summary="",
"requires_dist": [], platform=None,
"requires_python": None, requires_dist=[],
"files": [], requires_python=None,
"_cache_version": str(self.CACHE_VERSION), files=[],
} cache_version=str(self.CACHE_VERSION),
)
links = list(page.links_for_version(Version.parse(version))) links = list(page.links_for_version(Version.parse(version)))
if not links: if not links:
...@@ -394,15 +337,15 @@ class LegacyRepository(PyPiRepository): ...@@ -394,15 +337,15 @@ class LegacyRepository(PyPiRepository):
h = link.hash_name + ":" + link.hash h = link.hash_name + ":" + link.hash
files.append({"file": link.filename, "hash": h}) files.append({"file": link.filename, "hash": h})
data["files"] = files data.files = files
info = self._get_info_from_urls(urls) info = self._get_info_from_urls(urls)
data["summary"] = info["summary"] data.summary = info.summary
data["requires_dist"] = info["requires_dist"] data.requires_dist = info.requires_dist
data["requires_python"] = info["requires_python"] data.requires_python = info.requires_python
return data return data.asdict()
def _get(self, endpoint): # type: (str) -> Union[Page, None] def _get(self, endpoint): # type: (str) -> Union[Page, None]
url = self._url + endpoint url = self._url + endpoint
......
poetry/repositories/pypi_repository.py
...@@ -21,16 +21,15 @@ from poetry.core.semver import VersionConstraint ...@@ -21,16 +21,15 @@ from poetry.core.semver import VersionConstraint
from poetry.core.semver import VersionRange from poetry.core.semver import VersionRange
from poetry.core.semver import parse_constraint from poetry.core.semver import parse_constraint
from poetry.core.semver.exceptions import ParseVersionError from poetry.core.semver.exceptions import ParseVersionError
from poetry.core.version.markers import InvalidMarker
from poetry.core.version.markers import parse_marker from poetry.core.version.markers import parse_marker
from poetry.locations import REPOSITORY_CACHE_DIR from poetry.locations import REPOSITORY_CACHE_DIR
from poetry.utils._compat import Path from poetry.utils._compat import Path
from poetry.utils._compat import to_str from poetry.utils._compat import to_str
from poetry.utils.helpers import download_file from poetry.utils.helpers import download_file
from poetry.utils.helpers import temporary_directory from poetry.utils.helpers import temporary_directory
from poetry.utils.inspector import Inspector
from poetry.utils.patterns import wheel_file_re from poetry.utils.patterns import wheel_file_re
from ..inspection.info import PackageInfo
from .exceptions import PackageNotFound from .exceptions import PackageNotFound
from .remote_repository import RemoteRepository from .remote_repository import RemoteRepository
...@@ -73,7 +72,6 @@ class PyPiRepository(RemoteRepository): ...@@ -73,7 +72,6 @@ class PyPiRepository(RemoteRepository):
self._session = CacheControl( self._session = CacheControl(
requests.session(), cache=self._cache_control_cache requests.session(), cache=self._cache_control_cache
) )
self._inspector = Inspector()
self._name = "PyPI" self._name = "PyPI"
...@@ -160,62 +158,8 @@ class PyPiRepository(RemoteRepository): ...@@ -160,62 +158,8 @@ class PyPiRepository(RemoteRepository):
name, # type: str name, # type: str
version, # type: str version, # type: str
extras=None, # type: (Union[list, None]) extras=None, # type: (Union[list, None])
): # type: (...) -> Union[Package, None] ): # type: (...) -> Package
if extras is None: return self.get_release_info(name, version).to_package(name=name, extras=extras)
extras = []
release_info = self.get_release_info(name, version)
package = Package(name, version, version)
requires_dist = release_info["requires_dist"] or []
for req in requires_dist:
try:
dependency = dependency_from_pep_508(req)
except InvalidMarker:
# Invalid marker
# We strip the markers hoping for the best
req = req.split(";")[0]
dependency = dependency_from_pep_508(req)
except ValueError:
# Likely unable to parse constraint so we skip it
self._log(
"Invalid constraint ({}) found in {}-{} dependencies, "
"skipping".format(req, package.name, package.version),
level="debug",
)
continue
if dependency.in_extras:
for extra in dependency.in_extras:
if extra not in package.extras:
package.extras[extra] = []
package.extras[extra].append(dependency)
if not dependency.is_optional():
package.requires.append(dependency)
# Adding description
package.description = release_info.get("summary", "")
if release_info["requires_python"]:
package.python_versions = release_info["requires_python"]
if release_info["platform"]:
package.platform = release_info["platform"]
# Adding hashes information
package.files = release_info["files"]
# Activate extra dependencies
for extra in extras:
if extra in package.extras:
for dep in package.extras[extra]:
dep.activate()
package.requires += package.extras[extra]
return package
def search(self, query): def search(self, query):
results = [] results = []
...@@ -270,7 +214,7 @@ class PyPiRepository(RemoteRepository): ...@@ -270,7 +214,7 @@ class PyPiRepository(RemoteRepository):
return data return data
def get_release_info(self, name, version): # type: (str, str) -> dict def get_release_info(self, name, version): # type: (str, str) -> PackageInfo
""" """
Return the release information given a package name and a version. Return the release information given a package name and a version.
...@@ -278,7 +222,7 @@ class PyPiRepository(RemoteRepository): ...@@ -278,7 +222,7 @@ class PyPiRepository(RemoteRepository):
or retrieved from the remote server. or retrieved from the remote server.
""" """
if self._disable_cache: if self._disable_cache:
return self._get_release_info(name, version) return PackageInfo.load(self._get_release_info(name, version))
cached = self._cache.remember_forever( cached = self._cache.remember_forever(
"{}:{}".format(name, version), lambda: self._get_release_info(name, version) "{}:{}".format(name, version), lambda: self._get_release_info(name, version)
...@@ -295,7 +239,7 @@ class PyPiRepository(RemoteRepository): ...@@ -295,7 +239,7 @@ class PyPiRepository(RemoteRepository):
self._cache.forever("{}:{}".format(name, version), cached) self._cache.forever("{}:{}".format(name, version), cached)
return cached return PackageInfo.load(cached)
def _get_release_info(self, name, version): # type: (str, str) -> dict def _get_release_info(self, name, version): # type: (str, str) -> dict
self._log("Getting info for {} ({}) from PyPI".format(name, version), "debug") self._log("Getting info for {} ({}) from PyPI".format(name, version), "debug")
...@@ -305,16 +249,17 @@ class PyPiRepository(RemoteRepository): ...@@ -305,16 +249,17 @@ class PyPiRepository(RemoteRepository):
raise PackageNotFound("Package [{}] not found.".format(name)) raise PackageNotFound("Package [{}] not found.".format(name))
info = json_data["info"] info = json_data["info"]
data = {
"name": info["name"], data = PackageInfo(
"version": info["version"], name=info["name"],
"summary": info["summary"], version=info["version"],
"platform": info["platform"], summary=info["summary"],
"requires_dist": info["requires_dist"], platform=info["platform"],
"requires_python": info["requires_python"], requires_dist=info["requires_dist"],
"files": [], requires_python=info["requires_python"],
"_cache_version": str(self.CACHE_VERSION), files=info.get("files", []),
} cache_version=str(self.CACHE_VERSION),
)
try: try:
version_info = json_data["releases"][version] version_info = json_data["releases"][version]
...@@ -322,14 +267,14 @@ class PyPiRepository(RemoteRepository): ...@@ -322,14 +267,14 @@ class PyPiRepository(RemoteRepository):
version_info = [] version_info = []
for file_info in version_info: for file_info in version_info:
data["files"].append( data.files.append(
{ {
"file": file_info["filename"], "file": file_info["filename"],
"hash": "sha256:" + file_info["digests"]["sha256"], "hash": "sha256:" + file_info["digests"]["sha256"],
} }
) )
if self._fallback and data["requires_dist"] is None: if self._fallback and data.requires_dist is None:
self._log("No dependencies found, downloading archives", level="debug") self._log("No dependencies found, downloading archives", level="debug")
# No dependencies set (along with other information) # No dependencies set (along with other information)
# This might be due to actually no dependencies # This might be due to actually no dependencies
...@@ -346,16 +291,16 @@ class PyPiRepository(RemoteRepository): ...@@ -346,16 +291,16 @@ class PyPiRepository(RemoteRepository):
urls[dist_type].append(url["url"]) urls[dist_type].append(url["url"])
if not urls: if not urls:
return data return data.asdict()
info = self._get_info_from_urls(urls) info = self._get_info_from_urls(urls)
data["requires_dist"] = info["requires_dist"] data.requires_dist = info.requires_dist
if not data["requires_python"]: if not data.requires_python:
data["requires_python"] = info["requires_python"] data.requires_python = info.requires_python
return data return data.asdict()
def _get(self, endpoint): # type: (str) -> Union[dict, None] def _get(self, endpoint): # type: (str) -> Union[dict, None]
try: try:
...@@ -373,9 +318,7 @@ class PyPiRepository(RemoteRepository): ...@@ -373,9 +318,7 @@ class PyPiRepository(RemoteRepository):
return json_data return json_data
def _get_info_from_urls( def _get_info_from_urls(self, urls): # type: (Dict[str, List[str]]) -> PackageInfo
self, urls
): # type: (Dict[str, List[str]]) -> Dict[str, Union[str, List, None]]
# Checking wheels first as they are more likely to hold # Checking wheels first as they are more likely to hold
# the necessary information # the necessary information
if "bdist_wheel" in urls: if "bdist_wheel" in urls:
...@@ -410,24 +353,24 @@ class PyPiRepository(RemoteRepository): ...@@ -410,24 +353,24 @@ class PyPiRepository(RemoteRepository):
if universal_wheel is not None: if universal_wheel is not None:
return self._get_info_from_wheel(universal_wheel) return self._get_info_from_wheel(universal_wheel)
info = {} info = None
if universal_python2_wheel and universal_python3_wheel: if universal_python2_wheel and universal_python3_wheel:
info = self._get_info_from_wheel(universal_python2_wheel) info = self._get_info_from_wheel(universal_python2_wheel)
py3_info = self._get_info_from_wheel(universal_python3_wheel) py3_info = self._get_info_from_wheel(universal_python3_wheel)
if py3_info["requires_dist"]: if py3_info.requires_dist:
if not info["requires_dist"]: if not info.requires_dist:
info["requires_dist"] = py3_info["requires_dist"] info.requires_dist = py3_info.requires_dist
return info return info
py2_requires_dist = set( py2_requires_dist = set(
dependency_from_pep_508(r).to_pep_508() dependency_from_pep_508(r).to_pep_508()
for r in info["requires_dist"] for r in info.requires_dist
) )
py3_requires_dist = set( py3_requires_dist = set(
dependency_from_pep_508(r).to_pep_508() dependency_from_pep_508(r).to_pep_508()
for r in py3_info["requires_dist"] for r in py3_info.requires_dist
) )
base_requires_dist = py2_requires_dist & py3_requires_dist base_requires_dist = py2_requires_dist & py3_requires_dist
py2_only_requires_dist = py2_requires_dist - py3_requires_dist py2_only_requires_dist = py2_requires_dist - py3_requires_dist
...@@ -449,7 +392,7 @@ class PyPiRepository(RemoteRepository): ...@@ -449,7 +392,7 @@ class PyPiRepository(RemoteRepository):
) )
requires_dist.append(dep.to_pep_508()) requires_dist.append(dep.to_pep_508())
info["requires_dist"] = sorted(list(set(requires_dist))) info.requires_dist = sorted(list(set(requires_dist)))
if info: if info:
return info return info
...@@ -467,9 +410,7 @@ class PyPiRepository(RemoteRepository): ...@@ -467,9 +410,7 @@ class PyPiRepository(RemoteRepository):
return self._get_info_from_sdist(urls["sdist"][0]) return self._get_info_from_sdist(urls["sdist"][0])
def _get_info_from_wheel( def _get_info_from_wheel(self, url): # type: (str) -> PackageInfo
self, url
): # type: (str) -> Dict[str, Union[str, List, None]]
self._log( self._log(
"Downloading wheel: {}".format(urlparse.urlparse(url).path.rsplit("/")[-1]), "Downloading wheel: {}".format(urlparse.urlparse(url).path.rsplit("/")[-1]),
level="debug", level="debug",
...@@ -481,11 +422,9 @@ class PyPiRepository(RemoteRepository): ...@@ -481,11 +422,9 @@ class PyPiRepository(RemoteRepository):
filepath = Path(temp_dir) / filename filepath = Path(temp_dir) / filename
self._download(url, str(filepath)) self._download(url, str(filepath))
return self._inspector.inspect_wheel(filepath) return PackageInfo.from_wheel(filepath)
def _get_info_from_sdist( def _get_info_from_sdist(self, url): # type: (str) -> PackageInfo
self, url
): # type: (str) -> Dict[str, Union[str, List, None]]
self._log( self._log(
"Downloading sdist: {}".format(urlparse.urlparse(url).path.rsplit("/")[-1]), "Downloading sdist: {}".format(urlparse.urlparse(url).path.rsplit("/")[-1]),
level="debug", level="debug",
...@@ -497,7 +436,7 @@ class PyPiRepository(RemoteRepository): ...@@ -497,7 +436,7 @@ class PyPiRepository(RemoteRepository):
filepath = Path(temp_dir) / filename filepath = Path(temp_dir) / filename
self._download(url, str(filepath)) self._download(url, str(filepath))
return self._inspector.inspect_sdist(filepath) return PackageInfo.from_sdist(filepath)
def _download(self, url, dest): # type: (str, str) -> None def _download(self, url, dest): # type: (str, str) -> None
return download_file(url, dest, session=self.session) return download_file(url, dest, session=self.session)
......
poetry/utils/inspector.py deleted 100644 → 0
import logging
import os
import tarfile
import zipfile
from typing import Dict
from typing import List
from typing import Union
import pkginfo
from requests import get
from ._compat import Path
from .helpers import parse_requires
from .setup_reader import SetupReader
from .toml_file import TomlFile
logger = logging.getLogger(__name__)
class Inspector:
"""
A class to download and inspect remote packages.
"""
@classmethod
def download(cls, url, dest): # type: (str, Path) -> None
r = get(url, stream=True)
r.raise_for_status()
with open(str(dest), "wb") as f:
for chunk in r.iter_content(chunk_size=1024):
if chunk:
f.write(chunk)
def inspect(self, file_path): # type: (Path) -> Dict[str, Union[str, List[str]]]
if file_path.suffix == ".whl":
return self.inspect_wheel(file_path)
return self.inspect_sdist(file_path)
def inspect_wheel(
self, file_path
): # type: (Path) -> Dict[str, Union[str, List[str]]]
info = {
"name": "",
"version": "",
"summary": "",
"requires_python": None,
"requires_dist": [],
}
try:
meta = pkginfo.Wheel(str(file_path))
except ValueError:
# Unable to determine dependencies
# Assume none
return info
if meta.name:
info["name"] = meta.name
if meta.version:
info["version"] = meta.version
if meta.summary:
info["summary"] = meta.summary or ""
info["requires_python"] = meta.requires_python
if meta.requires_dist:
info["requires_dist"] = meta.requires_dist
return info
def inspect_sdist(
self, file_path
): # type: (Path) -> Dict[str, Union[str, List[str]]]
info = {
"name": "",
"version": "",
"summary": "",
"requires_python": None,
"requires_dist": None,
}
try:
meta = pkginfo.SDist(str(file_path))
if meta.name:
info["name"] = meta.name
if meta.version:
info["version"] = meta.version
if meta.summary:
info["summary"] = meta.summary
if meta.requires_python:
info["requires_python"] = meta.requires_python
if meta.requires_dist:
info["requires_dist"] = list(meta.requires_dist)
return info
except ValueError:
# Unable to determine dependencies
# We pass and go deeper
pass
# Still not dependencies found
# So, we unpack and introspect
suffix = file_path.suffix
if suffix == ".zip":
tar = zipfile.ZipFile(str(file_path))
else:
if suffix == ".bz2":
suffixes = file_path.suffixes
if len(suffixes) > 1 and suffixes[-2] == ".tar":
suffix = ".tar.bz2"
else:
suffix = ".tar.gz"
tar = tarfile.open(str(file_path))
try:
tar.extractall(os.path.join(str(file_path.parent), "unpacked"))
finally:
tar.close()
unpacked = file_path.parent / "unpacked"
elements = list(unpacked.glob("*"))
if len(elements) == 1 and elements[0].is_dir():
sdist_dir = elements[0]
else:
sdist_dir = unpacked / file_path.name.rstrip(suffix)
pyproject = TomlFile(sdist_dir / "pyproject.toml")
if pyproject.exists():
from poetry.factory import Factory
pyproject_content = pyproject.read()
if "tool" in pyproject_content and "poetry" in pyproject_content["tool"]:
package = Factory().create_poetry(sdist_dir).package
return {
"name": package.name,
"version": package.version.text,
"summary": package.description,
"requires_dist": [dep.to_pep_508() for dep in package.requires],
"requires_python": package.python_versions,
}
# Checking for .egg-info at root
eggs = list(sdist_dir.glob("*.egg-info"))
if eggs:
egg_info = eggs[0]
requires = egg_info / "requires.txt"
if requires.exists():
with requires.open(encoding="utf-8") as f:
info["requires_dist"] = parse_requires(f.read())
return info
# Searching for .egg-info in sub directories
eggs = list(sdist_dir.glob("**/*.egg-info"))
if eggs:
egg_info = eggs[0]
requires = egg_info / "requires.txt"
if requires.exists():
with requires.open(encoding="utf-8") as f:
info["requires_dist"] = parse_requires(f.read())
return info
# Still nothing, try reading (without executing it)
# the setup.py file.
try:
setup_info = self._inspect_sdist_with_setup(sdist_dir)
for key, value in info.items():
if value:
continue
info[key] = setup_info[key]
return info
except Exception as e:
logger.warning(
"An error occurred when reading setup.py or setup.cfg: {}".format(
str(e)
)
)
return info
def _inspect_sdist_with_setup(
self, sdist_dir
): # type: (Path) -> Dict[str, Union[str, List[str]]]
info = {
"name": None,
"version": None,
"summary": "",
"requires_python": None,
"requires_dist": None,
}
result = SetupReader.read_from_directory(sdist_dir)
requires = ""
for dep in result["install_requires"]:
requires += dep + "\n"
if result["extras_require"]:
requires += "\n"
for extra_name, deps in result["extras_require"].items():
requires += "[{}]\n".format(extra_name)
for dep in deps:
requires += dep + "\n"
requires += "\n"
info["name"] = result["name"]
info["version"] = result["version"]
info["requires_dist"] = parse_requires(requires)
info["requires_python"] = result["python_requires"]
return info
tests/conftest.py
...@@ -71,7 +71,6 @@ def config(config_source, auth_config_source, mocker): ...@@ -71,7 +71,6 @@ def config(config_source, auth_config_source, mocker):
@pytest.fixture(autouse=True) @pytest.fixture(autouse=True)
def download_mock(mocker): def download_mock(mocker):
# Patch download to not download anything but to just copy from fixtures # Patch download to not download anything but to just copy from fixtures
mocker.patch("poetry.utils.inspector.Inspector.download", new=mock_download)
mocker.patch("poetry.utils.helpers.download_file", new=mock_download) mocker.patch("poetry.utils.helpers.download_file", new=mock_download)
mocker.patch("poetry.puzzle.provider.download_file", new=mock_download) mocker.patch("poetry.puzzle.provider.download_file", new=mock_download)
mocker.patch("poetry.repositories.pypi_repository.download_file", new=mock_download) mocker.patch("poetry.repositories.pypi_repository.download_file", new=mock_download)
......
tests/console/conftest.py
...@@ -16,7 +16,6 @@ from poetry.utils._compat import Path ...@@ -16,7 +16,6 @@ from poetry.utils._compat import Path
from poetry.utils.env import MockEnv from poetry.utils.env import MockEnv
from poetry.utils.toml_file import TomlFile from poetry.utils.toml_file import TomlFile
from tests.helpers import mock_clone from tests.helpers import mock_clone
from tests.helpers import mock_download
@pytest.fixture() @pytest.fixture()
...@@ -54,9 +53,6 @@ def setup(mocker, installer, installed, config, env): ...@@ -54,9 +53,6 @@ def setup(mocker, installer, installed, config, env):
p = mocker.patch("poetry.core.vcs.git.Git.rev_parse") p = mocker.patch("poetry.core.vcs.git.Git.rev_parse")
p.return_value = "9cf87a285a2d3fbb0b9fa621997b3acc3631ed24" p.return_value = "9cf87a285a2d3fbb0b9fa621997b3acc3631ed24"
# Patch download to not download anything but to just copy from fixtures
mocker.patch("poetry.utils.inspector.Inspector.download", new=mock_download)
# Patch the virtual environment creation do actually do nothing # Patch the virtual environment creation do actually do nothing
mocker.patch("poetry.utils.env.EnvManager.create_venv", return_value=env) mocker.patch("poetry.utils.env.EnvManager.create_venv", return_value=env)
......
tests/installation/fixtures/with-directory-dependency-poetry-transitive.test
[[package]] [[package]]
category = "main" category = "main"
description = "" description = "This is a description"
develop = true develop = true
name = "project-with-extras" name = "project-with-extras"
optional = false optional = false
...@@ -18,7 +18,7 @@ url = "tests/fixtures/directory/project_with_transitive_directory_dependencies/. ...@@ -18,7 +18,7 @@ url = "tests/fixtures/directory/project_with_transitive_directory_dependencies/.
[[package]] [[package]]
category = "main" category = "main"
description = "" description = "This is a description"
develop = true develop = true
name = "project-with-transitive-directory-dependencies" name = "project-with-transitive-directory-dependencies"
optional = false optional = false
......
tests/installation/fixtures/with-directory-dependency-poetry.test
...@@ -8,7 +8,7 @@ version = "1.4.4" ...@@ -8,7 +8,7 @@ version = "1.4.4"
[[package]] [[package]]
category = "main" category = "main"
description = "" description = "This is a description"
develop = true develop = true
name = "project-with-extras" name = "project-with-extras"
optional = false optional = false
......
tests/installation/fixtures/with-file-dependency-transitive.test
...@@ -28,7 +28,7 @@ version = "1.4.4" ...@@ -28,7 +28,7 @@ version = "1.4.4"
[[package]] [[package]]
category = "main" category = "main"
description = "" description = "This is a description"
develop = true develop = true
name = "project-with-transitive-file-dependencies" name = "project-with-transitive-file-dependencies"
optional = false optional = false
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment