Skip to content

P
python-poetry
Commit 10b8eab5 by Arun Babu Neelicattu
Options

unify package inspection logic 

This change brings together logic repeated in various places to
discover package information.
parent f20043af
Showing with 568 additions and 575 deletions
poetry/inspection/info.py 0 → 100644
import glob
import logging
import os
import tarfile
import zipfile
from typing import Dict
from typing import Iterator
from typing import List
from typing import Optional
from typing import Union
import pkginfo
from poetry.core.factory import Factory
from poetry.core.packages import Package
from poetry.core.packages import ProjectPackage
from poetry.core.packages import dependency_from_pep_508
from poetry.core.utils._compat import PY35
from poetry.core.utils._compat import Path
from poetry.core.utils.helpers import parse_requires
from poetry.core.utils.helpers import temporary_directory
from poetry.core.version.markers import InvalidMarker
from poetry.utils.env import EnvCommandError
from poetry.utils.env import EnvManager
from poetry.utils.env import VirtualEnv
from poetry.utils.setup_reader import SetupReader
from poetry.utils.toml_file import TomlFile
logger = logging.getLogger(__name__)
class PackageInfoError(ValueError):
def __init__(self, path): # type: (Union[Path, str]) -> None
super(PackageInfoError, self).__init__(
"Unable to determine package info for path: {}".format(str(path))
)
class PackageInfo:
def __init__(
self,
name=None, # type: Optional[str]
version=None, # type: Optional[str]
summary=None, # type: Optional[str]
platform=None, # type: Optional[str]
requires_dist=None, # type: Optional[List[str]]
requires_python=None, # type: Optional[str]
files=None, # type: Optional[List[str]]
cache_version=None, # type: Optional[str]
):
self.name = name
self.version = version
self.summary = summary
self.platform = platform
self.requires_dist = requires_dist
self.requires_python = requires_python
self.files = files or []
self._cache_version = cache_version
@property
def cache_version(self): # type: () -> Optional[str]
return self._cache_version
def update(self, other): # type: (PackageInfo) -> PackageInfo
self.name = other.name or self.name
self.version = other.version or self.version
self.summary = other.summary or self.summary
self.platform = other.platform or self.platform
self.requires_dist = other.requires_dist or self.requires_dist
self.requires_python = other.requires_python or self.requires_python
self.files = other.files or self.files
self._cache_version = other.cache_version or self._cache_version
return self
def asdict(self): # type: () -> Dict[str, Optional[Union[str, List[str]]]]
"""
Helper method to convert package info into a dictionary used for caching.
"""
return {
"name": self.name,
"version": self.version,
"summary": self.summary,
"platform": self.platform,
"requires_dist": self.requires_dist,
"requires_python": self.requires_python,
"files": self.files,
"_cache_version": self._cache_version,
}
@classmethod
def load(
cls, data
): # type: (Dict[str, Optional[Union[str, List[str]]]]) -> PackageInfo
"""
Helper method to load data from a dictionary produced by `PackageInfo.asdict()`.
:param data: Data to load. This is expected to be a `dict` object output by `asdict()`.
"""
cache_version = data.pop("_cache_version", None)
return cls(cache_version=cache_version, **data)
@classmethod
def _log(cls, msg, level="info"):
"""Internal helper method to log information."""
getattr(logger, level)("<debug>{}:</debug> {}".format(cls.__name__, msg))
def to_package(
self, name=None, extras=None, root_dir=None
): # type: (Optional[str], Optional[List[str]], Optional[Path]) -> Package
"""
Create a new `poetry.core.packages.package.Package` instance using metadata from this instance.
:param name: Name to use for the package, if not specified name from this instance is used.
:param extras: Extras to activate for this package.
:param root_dir: Optional root directory to use for the package. If set, dependency strings
will be parsed relative to this directory.
"""
name = name or self.name
if not self.version:
# The version could not be determined, so we raise an error since it is mandatory.
raise RuntimeError(
"Unable to retrieve the package version for {}".format(name)
)
package = Package(name=name, version=self.version)
package.description = self.summary
package.root_dir = root_dir
package.python_versions = self.requires_python or "*"
package.files = self.files
for req in self.requires_dist or []:
try:
# Attempt to parse the PEP-508 requirement string
dependency = dependency_from_pep_508(req, relative_to=root_dir)
except InvalidMarker:
# Invalid marker, We strip the markers hoping for the best
req = req.split(";")[0]
dependency = dependency_from_pep_508(req, relative_to=root_dir)
except ValueError:
# Likely unable to parse constraint so we skip it
self._log(
"Invalid constraint ({}) found in {}-{} dependencies, "
"skipping".format(req, package.name, package.version),
level="debug",
)
continue
if dependency.in_extras:
# this dependency is required by an extra package
for extra in dependency.in_extras:
if extra not in package.extras:
# this is the first time we encounter this extra for this package
package.extras[extra] = []
# Activate extra dependencies if specified
if extras and extra in extras:
dependency.activate()
package.extras[extra].append(dependency)
if not dependency.is_optional() or dependency.is_activated():
# we skip add only if the dependency is option and was not activated as part of an extra
package.requires.append(dependency)
return package
@classmethod
def _from_distribution(
cls, dist
): # type: (Union[pkginfo.BDist, pkginfo.SDist, pkginfo.Wheel]) -> PackageInfo
"""
Helper method to parse package information from a `pkginfo.Distribution` instance.
:param dist: The distribution instance to parse information from.
"""
if dist.requires_dist:
requirements = list(dist.requires_dist)
else:
requirements = []
requires = Path(dist.filename) / "requires.txt"
if requires.exists():
with requires.open(encoding="utf-8") as f:
requirements = parse_requires(f.read())
return cls(
name=dist.name,
version=dist.version,
summary=dist.summary,
platform=dist.supported_platforms,
requires_dist=requirements or None,
requires_python=dist.requires_python,
)
@classmethod
def _from_sdist_file(cls, path): # type: (Path) -> PackageInfo
"""
Helper method to parse package information from an sdist file. We attempt to first inspect the
file using `pkginfo.SDist`. If this does not provide us with package requirements, we extract the
source and handle it as a directory.
:param path: The sdist file to parse information from.
"""
info = None
try:
info = cls._from_distribution(pkginfo.SDist(str(path)))
except ValueError:
# Unable to determine dependencies
# We pass and go deeper
pass
else:
if info.requires_dist is not None:
# we successfully retrieved dependencies from sdist metadata
return info
# Still not dependencies found
# So, we unpack and introspect
suffix = path.suffix
if suffix == ".zip":
context = zipfile.ZipFile
else:
if suffix == ".bz2":
suffixes = path.suffixes
if len(suffixes) > 1 and suffixes[-2] == ".tar":
suffix = ".tar.bz2"
else:
suffix = ".tar.gz"
context = tarfile.open
with temporary_directory() as tmp:
tmp = Path(tmp)
with context(str(path)) as archive:
archive.extractall(str(tmp))
# a little bit of guess work to determine the directory we care about
elements = list(tmp.glob("*"))
if len(elements) == 1 and elements[0].is_dir():
sdist_dir = elements[0]
else:
sdist_dir = tmp / path.name.rstrip(suffix)
# now this is an unpacked directory we know how to deal with
new_info = cls.from_directory(path=sdist_dir)
if not info:
return new_info
return info.update(new_info)
@classmethod
def from_setup_py(cls, path): # type: (Union[str, Path]) -> PackageInfo
"""
Mechanism to parse package information from a `setup.py` file. This uses the implentation
at `poetry.utils.setup_reader.SetupReader` in order to parse the file. This is not reliable for
complex setup files and should only attempted as a fallback.
:param path: Path to `setup.py` file
:return:
"""
result = SetupReader.read_from_directory(Path(path))
python_requires = result["python_requires"]
if python_requires is None:
python_requires = "*"
requires = ""
for dep in result["install_requires"]:
requires += dep + "\n"
if result["extras_require"]:
requires += "\n"
for extra_name, deps in result["extras_require"].items():
requires += "[{}]\n".format(extra_name)
for dep in deps:
requires += dep + "\n"
requires += "\n"
requirements = parse_requires(requires)
return cls(
name=result.get("name"),
version=result.get("version"),
summary=result.get("description", ""),
requires_dist=requirements or None,
requires_python=python_requires,
)
@staticmethod
def _find_dist_info(path): # type: (Path) -> Iterator[Path]
"""
Discover all `*.*-info` directories in a given path.
:param path: Path to search.
"""
pattern = "**/*.*-info"
if PY35:
# Sometimes pathlib will fail on recursive symbolic links, so we need to workaround it
# and use the glob module instead. Note that this does not happen with pathlib2
# so it's safe to use it for Python < 3.4.
directories = glob.iglob(Path(path, pattern).as_posix(), recursive=True)
else:
directories = path.glob(pattern)
for d in directories:
yield Path(d)
@classmethod
def from_metadata(cls, path): # type: (Union[str, Path]) -> PackageInfo
"""
Helper method to parse package information from an unpacked metadata directory.
:param path: The metadata directory to parse information from.
"""
path = Path(path)
if path.suffix in {".dist-info", ".egg-info"}:
directories = [path.suffix]
else:
directories = cls._find_dist_info(path=path)
for directory in directories:
try:
if directory.suffix == ".egg-info":
dist = pkginfo.UnpackedSDist(str(directory))
elif directory.suffix == ".dist-info":
dist = pkginfo.Wheel(str(directory))
else:
continue
info = cls._from_distribution(dist=dist)
if info:
return info
except ValueError:
pass
@classmethod
def from_package(cls, package): # type: (Package) -> PackageInfo
"""
Helper method to inspect a `Package` object, in order to generate package info.
:param package: This must be a poetry package instance.
"""
requires = {dependency.to_pep_508() for dependency in package.requires}
for extra_requires in package.extras.values():
for dependency in extra_requires:
requires.add(dependency.to_pep_508())
return cls(
name=package.name,
version=package.version,
summary=package.description,
platform=package.platform,
requires_dist=list(requires),
requires_python=package.python_versions,
files=package.files,
)
@staticmethod
def _get_poetry_package(path): # type: (Path) -> Optional[ProjectPackage]
pyproject = path.joinpath("pyproject.toml")
try:
# Note: we ignore any setup.py file at this step
if pyproject.exists():
# TODO: add support for handling non-poetry PEP-517 builds
pyproject = TomlFile(pyproject)
pyproject_content = pyproject.read()
supports_poetry = (
"tool" in pyproject_content
and "poetry" in pyproject_content["tool"]
)
if supports_poetry:
return Factory().create_poetry(path).package
except RuntimeError:
pass
@classmethod
def from_directory(
cls, path, allow_build=False
): # type: (Union[str, Path], bool) -> PackageInfo
"""
Generate package information from a package source directory. When `allow_build` is enabled and
introspection of all available metadata fails, the package is attempted to be build in an isolated
environment so as to generate required metadata.
:param path: Path to generate package information from.
:param allow_build: If enabled, as a fallback, build the project to gather metadata.
"""
path = Path(path)
setup_py = path.joinpath("setup.py")
project_package = cls._get_poetry_package(path)
if project_package:
return cls.from_package(project_package)
if not setup_py.exists():
# this means we cannot do anything else here
raise PackageInfoError(path)
current_dir = os.getcwd()
info = cls.from_metadata(path)
if info and info.requires_dist is not None:
# return only if requirements are discovered
return info
if not allow_build:
return cls.from_setup_py(path=path)
try:
# TODO: replace with PEP517
# we need to switch to the correct path in order for egg_info command to work
os.chdir(str(path))
# Execute egg_info
with temporary_directory() as tmp_dir:
EnvManager.build_venv(tmp_dir)
venv = VirtualEnv(Path(tmp_dir), Path(tmp_dir))
venv.run("python", "setup.py", "egg_info")
except EnvCommandError:
cls._log(
"Falling back to parsing setup.py file for {}".format(path), "debug"
)
# egg_info could not be generated, we fallback to ast parser
return cls.from_setup_py(path=path)
else:
info = cls.from_metadata(path)
if info:
return info
finally:
os.chdir(current_dir)
# if we reach here, everything has failed and all hope is lost
raise PackageInfoError(path)
@classmethod
def from_sdist(cls, path): # type: (Union[Path, pkginfo.SDist]) -> PackageInfo
"""
Gather package information from an sdist file, packed or unpacked.
:param path: Path to an sdist file or unpacked directory.
"""
if path.is_file():
return cls._from_sdist_file(path=path)
# if we get here then it is neither an sdist instance nor a file
# so, we assume this is an directory
return cls.from_directory(path=path)
@classmethod
def from_wheel(cls, path): # type: (Path) -> PackageInfo
"""
Gather package information from a wheel.
:param path: Path to wheel.
"""
try:
return cls._from_distribution(pkginfo.Wheel(str(path)))
except ValueError:
return PackageInfo()
@classmethod
def from_bdist(cls, path): # type: (Path) -> PackageInfo
"""
Gather package information from a bdist (wheel etc.).
:param path: Path to bdist.
"""
if isinstance(path, (pkginfo.BDist, pkginfo.Wheel)):
cls._from_distribution(dist=path)
if path.suffix == ".whl":
return cls.from_wheel(path=path)
try:
return cls._from_distribution(pkginfo.BDist(str(path)))
except ValueError:
raise PackageInfoError(path)
@classmethod
def from_path(cls, path): # type: (Path) -> PackageInfo
"""
Gather package information from a given path (bdist, sdist, directory).
:param path: Path to inspect.
"""
try:
return cls.from_bdist(path=path)
except PackageInfoError:
return cls.from_sdist(path=path)
poetry/puzzle/provider.py
import glob
import logging
import os
import re
......@@ -10,8 +9,6 @@ from typing import Any
from typing import List
from typing import Optional
import pkginfo
from clikit.ui.components import ProgressIndicator
from poetry.core.packages import Dependency
......@@ -20,34 +17,25 @@ from poetry.core.packages import FileDependency
from poetry.core.packages import Package
from poetry.core.packages import URLDependency
from poetry.core.packages import VCSDependency
from poetry.core.packages import dependency_from_pep_508
from poetry.core.packages.utils.utils import get_python_constraint_from_marker
from poetry.core.utils.helpers import parse_requires
from poetry.core.vcs.git import Git
from poetry.core.version.markers import MarkerUnion
from poetry.factory import Factory
from poetry.inspection.info import PackageInfo
from poetry.inspection.info import PackageInfoError
from poetry.mixology.incompatibility import Incompatibility
from poetry.mixology.incompatibility_cause import DependencyCause
from poetry.mixology.incompatibility_cause import PythonCause
from poetry.mixology.term import Term
from poetry.packages import DependencyPackage
from poetry.packages.package_collection import PackageCollection
from poetry.puzzle.exceptions import OverrideNeeded
from poetry.repositories import Pool
from poetry.utils._compat import PY35
from poetry.utils._compat import OrderedDict
from poetry.utils._compat import Path
from poetry.utils._compat import urlparse
from poetry.utils.env import EnvCommandError
from poetry.utils.env import EnvManager
from poetry.utils.env import VirtualEnv
from poetry.utils.helpers import download_file
from poetry.utils.helpers import safe_rmtree
from poetry.utils.helpers import temporary_directory
from poetry.utils.inspector import Inspector
from poetry.utils.setup_reader import SetupReader
from poetry.utils.toml_file import TomlFile
from .exceptions import OverrideNeeded
logger = logging.getLogger(__name__)
......@@ -68,7 +56,6 @@ class Provider:
self._package = package
self._pool = pool
self._io = io
self._inspector = Inspector()
self._python_constraint = package.python_constraint
self._search_for = {}
self._is_debugging = self._io.is_debug() or self._io.is_very_verbose()
......@@ -245,31 +232,18 @@ class Provider:
@classmethod
def get_package_from_file(cls, file_path): # type: (Path) -> Package
info = Inspector().inspect(file_path)
if not info["name"]:
try:
package = PackageInfo.from_path(path=file_path).to_package(
root_dir=file_path
)
except PackageInfoError:
raise RuntimeError(
"Unable to determine the package name of {}".format(file_path)
"Unable to determine package info from path: {}".format(file_path)
)
package = Package(info["name"], info["version"])
package.source_type = "file"
package.source_url = file_path.as_posix()
package.description = info["summary"]
for req in info["requires_dist"]:
dep = dependency_from_pep_508(req)
for extra in dep.in_extras:
if extra not in package.extras:
package.extras[extra] = []
package.extras[extra].append(dep)
if not dep.is_optional():
package.requires.append(dep)
if info["requires_python"]:
package.python_versions = info["requires_python"]
return package
def search_for_directory(
......@@ -298,136 +272,9 @@ class Provider:
def get_package_from_directory(
cls, directory, name=None
): # type: (Path, Optional[str]) -> Package
supports_poetry = False
pyproject = directory.joinpath("pyproject.toml")
if pyproject.exists():
pyproject = TomlFile(pyproject)
pyproject_content = pyproject.read()
supports_poetry = (
"tool" in pyproject_content and "poetry" in pyproject_content["tool"]
)
if supports_poetry:
poetry = Factory().create_poetry(directory)
pkg = poetry.package
package = Package(pkg.name, pkg.version)
for dep in pkg.requires:
if not dep.is_optional():
package.requires.append(dep)
for extra, deps in pkg.extras.items():
if extra not in package.extras:
package.extras[extra] = []
for dep in deps:
package.extras[extra].append(dep)
package.python_versions = pkg.python_versions
else:
# Execute egg_info
current_dir = os.getcwd()
os.chdir(str(directory))
try:
with temporary_directory() as tmp_dir:
EnvManager.build_venv(tmp_dir)
venv = VirtualEnv(Path(tmp_dir), Path(tmp_dir))
venv.run("python", "setup.py", "egg_info")
except EnvCommandError:
result = SetupReader.read_from_directory(directory)
if not result["name"]:
# The name could not be determined
# We use the dependency name
result["name"] = name
if not result["version"]:
# The version could not be determined
# so we raise an error since it is mandatory
raise RuntimeError(
"Unable to retrieve the package version for {}".format(
directory
)
)
package_name = result["name"]
package_version = result["version"]
python_requires = result["python_requires"]
if python_requires is None:
python_requires = "*"
package_summary = ""
requires = ""
for dep in result["install_requires"]:
requires += dep + "\n"
if result["extras_require"]:
requires += "\n"
for extra_name, deps in result["extras_require"].items():
requires += "[{}]\n".format(extra_name)
for dep in deps:
requires += dep + "\n"
requires += "\n"
reqs = parse_requires(requires)
else:
os.chdir(current_dir)
# Sometimes pathlib will fail on recursive
# symbolic links, so we need to workaround it
# and use the glob module instead.
# Note that this does not happen with pathlib2
# so it's safe to use it for Python < 3.4.
if PY35:
egg_info = next(
Path(p)
for p in glob.glob(
os.path.join(str(directory), "**", "*.egg-info"),
recursive=True,
)
)
else:
egg_info = next(directory.glob("**/*.egg-info"))
meta = pkginfo.UnpackedSDist(str(egg_info))
package_name = meta.name
package_version = meta.version
package_summary = meta.summary
python_requires = meta.requires_python
if meta.requires_dist:
reqs = list(meta.requires_dist)
else:
reqs = []
requires = egg_info / "requires.txt"
if requires.exists():
with requires.open(encoding="utf-8") as f:
reqs = parse_requires(f.read())
finally:
os.chdir(current_dir)
package = Package(package_name, package_version)
package.description = package_summary
for req in reqs:
dep = dependency_from_pep_508(req)
if dep.in_extras:
for extra in dep.in_extras:
if extra not in package.extras:
package.extras[extra] = []
package.extras[extra].append(dep)
if not dep.is_optional():
package.requires.append(dep)
if python_requires:
package.python_versions = python_requires
package = PackageInfo.from_directory(
path=directory, allow_build=True
).to_package(root_dir=directory)
if name and name != package.name:
# For now, the dependency's name must match the actual package's name
raise RuntimeError(
......
poetry/repositories/legacy_repository.py
......@@ -14,19 +14,17 @@ from cachecontrol.caches.file_cache import FileCache
from cachy import CacheManager
from poetry.core.packages import Package
from poetry.core.packages import dependency_from_pep_508
from poetry.core.packages.utils.link import Link
from poetry.core.semver import Version
from poetry.core.semver import VersionConstraint
from poetry.core.semver import VersionRange
from poetry.core.semver import parse_constraint
from poetry.core.version.markers import InvalidMarker
from poetry.locations import REPOSITORY_CACHE_DIR
from poetry.utils._compat import Path
from poetry.utils.helpers import canonicalize_name
from poetry.utils.inspector import Inspector
from poetry.utils.patterns import wheel_file_re
from ..inspection.info import PackageInfo
from .auth import Auth
from .exceptions import PackageNotFound
from .pypi_repository import PyPiRepository
......@@ -171,7 +169,6 @@ class LegacyRepository(PyPiRepository):
self._auth = auth
self._client_cert = client_cert
self._cert = cert
self._inspector = Inspector()
self._cache_dir = REPOSITORY_CACHE_DIR / name
self._cache = CacheManager(
{
......@@ -298,63 +295,8 @@ class LegacyRepository(PyPiRepository):
return self._packages[index]
except ValueError:
if extras is None:
extras = []
release_info = self.get_release_info(name, version)
package = Package(name, version, version)
if release_info["requires_python"]:
package.python_versions = release_info["requires_python"]
package = super(LegacyRepository, self).package(name, version, extras)
package.source_url = self._url
package.source_reference = self.name
requires_dist = release_info["requires_dist"] or []
for req in requires_dist:
try:
dependency = dependency_from_pep_508(req)
except InvalidMarker:
# Invalid marker
# We strip the markers hoping for the best
req = req.split(";")[0]
dependency = dependency_from_pep_508(req)
except ValueError:
# Likely unable to parse constraint so we skip it
self._log(
"Invalid constraint ({}) found in {}-{} dependencies, "
"skipping".format(req, package.name, package.version),
level="debug",
)
continue
if dependency.in_extras:
for extra in dependency.in_extras:
if extra not in package.extras:
package.extras[extra] = []
package.extras[extra].append(dependency)
if not dependency.is_optional():
package.requires.append(dependency)
# Adding description
package.description = release_info.get("summary", "")
# Adding hashes information
package.files = release_info["files"]
# Activate extra dependencies
for extra in extras:
if extra in package.extras:
for dep in package.extras[extra]:
dep.activate()
package.requires += package.extras[extra]
self._packages.append(package)
return package
def _get_release_info(self, name, version): # type: (str, str) -> dict
......@@ -362,15 +304,16 @@ class LegacyRepository(PyPiRepository):
if page is None:
raise PackageNotFound('No package named "{}"'.format(name))
data = {
"name": name,
"version": version,
"summary": "",
"requires_dist": [],
"requires_python": None,
"files": [],
"_cache_version": str(self.CACHE_VERSION),
}
data = PackageInfo(
name=name,
version=version,
summary="",
platform=None,
requires_dist=[],
requires_python=None,
files=[],
cache_version=str(self.CACHE_VERSION),
)
links = list(page.links_for_version(Version.parse(version)))
if not links:
......@@ -394,15 +337,15 @@ class LegacyRepository(PyPiRepository):
h = link.hash_name + ":" + link.hash
files.append({"file": link.filename, "hash": h})
data["files"] = files
data.files = files
info = self._get_info_from_urls(urls)
data["summary"] = info["summary"]
data["requires_dist"] = info["requires_dist"]
data["requires_python"] = info["requires_python"]
data.summary = info.summary
data.requires_dist = info.requires_dist
data.requires_python = info.requires_python
return data
return data.asdict()
def _get(self, endpoint): # type: (str) -> Union[Page, None]
url = self._url + endpoint
......
poetry/repositories/pypi_repository.py
......@@ -21,16 +21,15 @@ from poetry.core.semver import VersionConstraint
from poetry.core.semver import VersionRange
from poetry.core.semver import parse_constraint
from poetry.core.semver.exceptions import ParseVersionError
from poetry.core.version.markers import InvalidMarker
from poetry.core.version.markers import parse_marker
from poetry.locations import REPOSITORY_CACHE_DIR
from poetry.utils._compat import Path
from poetry.utils._compat import to_str
from poetry.utils.helpers import download_file
from poetry.utils.helpers import temporary_directory
from poetry.utils.inspector import Inspector
from poetry.utils.patterns import wheel_file_re
from ..inspection.info import PackageInfo
from .exceptions import PackageNotFound
from .remote_repository import RemoteRepository
......@@ -73,7 +72,6 @@ class PyPiRepository(RemoteRepository):
self._session = CacheControl(
requests.session(), cache=self._cache_control_cache
)
self._inspector = Inspector()
self._name = "PyPI"
......@@ -160,62 +158,8 @@ class PyPiRepository(RemoteRepository):
name, # type: str
version, # type: str
extras=None, # type: (Union[list, None])
): # type: (...) -> Union[Package, None]
if extras is None:
extras = []
release_info = self.get_release_info(name, version)
package = Package(name, version, version)
requires_dist = release_info["requires_dist"] or []
for req in requires_dist:
try:
dependency = dependency_from_pep_508(req)
except InvalidMarker:
# Invalid marker
# We strip the markers hoping for the best
req = req.split(";")[0]
dependency = dependency_from_pep_508(req)
except ValueError:
# Likely unable to parse constraint so we skip it
self._log(
"Invalid constraint ({}) found in {}-{} dependencies, "
"skipping".format(req, package.name, package.version),
level="debug",
)
continue
if dependency.in_extras:
for extra in dependency.in_extras:
if extra not in package.extras:
package.extras[extra] = []
package.extras[extra].append(dependency)
if not dependency.is_optional():
package.requires.append(dependency)
# Adding description
package.description = release_info.get("summary", "")
if release_info["requires_python"]:
package.python_versions = release_info["requires_python"]
if release_info["platform"]:
package.platform = release_info["platform"]
# Adding hashes information
package.files = release_info["files"]
# Activate extra dependencies
for extra in extras:
if extra in package.extras:
for dep in package.extras[extra]:
dep.activate()
package.requires += package.extras[extra]
return package
): # type: (...) -> Package
return self.get_release_info(name, version).to_package(name=name, extras=extras)
def search(self, query):
results = []
......@@ -270,7 +214,7 @@ class PyPiRepository(RemoteRepository):
return data
def get_release_info(self, name, version): # type: (str, str) -> dict
def get_release_info(self, name, version): # type: (str, str) -> PackageInfo
"""
Return the release information given a package name and a version.
......@@ -278,7 +222,7 @@ class PyPiRepository(RemoteRepository):
or retrieved from the remote server.
"""
if self._disable_cache:
return self._get_release_info(name, version)
return PackageInfo.load(self._get_release_info(name, version))
cached = self._cache.remember_forever(
"{}:{}".format(name, version), lambda: self._get_release_info(name, version)
......@@ -295,7 +239,7 @@ class PyPiRepository(RemoteRepository):
self._cache.forever("{}:{}".format(name, version), cached)
return cached
return PackageInfo.load(cached)
def _get_release_info(self, name, version): # type: (str, str) -> dict
self._log("Getting info for {} ({}) from PyPI".format(name, version), "debug")
......@@ -305,16 +249,17 @@ class PyPiRepository(RemoteRepository):
raise PackageNotFound("Package [{}] not found.".format(name))
info = json_data["info"]
data = {
"name": info["name"],
"version": info["version"],
"summary": info["summary"],
"platform": info["platform"],
"requires_dist": info["requires_dist"],
"requires_python": info["requires_python"],
"files": [],
"_cache_version": str(self.CACHE_VERSION),
}
data = PackageInfo(
name=info["name"],
version=info["version"],
summary=info["summary"],
platform=info["platform"],
requires_dist=info["requires_dist"],
requires_python=info["requires_python"],
files=info.get("files", []),
cache_version=str(self.CACHE_VERSION),
)
try:
version_info = json_data["releases"][version]
......@@ -322,14 +267,14 @@ class PyPiRepository(RemoteRepository):
version_info = []
for file_info in version_info:
data["files"].append(
data.files.append(
{
"file": file_info["filename"],
"hash": "sha256:" + file_info["digests"]["sha256"],
}
)
if self._fallback and data["requires_dist"] is None:
if self._fallback and data.requires_dist is None:
self._log("No dependencies found, downloading archives", level="debug")
# No dependencies set (along with other information)
# This might be due to actually no dependencies
......@@ -346,16 +291,16 @@ class PyPiRepository(RemoteRepository):
urls[dist_type].append(url["url"])
if not urls:
return data
return data.asdict()
info = self._get_info_from_urls(urls)
data["requires_dist"] = info["requires_dist"]
data.requires_dist = info.requires_dist
if not data["requires_python"]:
data["requires_python"] = info["requires_python"]
if not data.requires_python:
data.requires_python = info.requires_python
return data
return data.asdict()
def _get(self, endpoint): # type: (str) -> Union[dict, None]
try:
......@@ -373,9 +318,7 @@ class PyPiRepository(RemoteRepository):
return json_data
def _get_info_from_urls(
self, urls
): # type: (Dict[str, List[str]]) -> Dict[str, Union[str, List, None]]
def _get_info_from_urls(self, urls): # type: (Dict[str, List[str]]) -> PackageInfo
# Checking wheels first as they are more likely to hold
# the necessary information
if "bdist_wheel" in urls:
......@@ -410,24 +353,24 @@ class PyPiRepository(RemoteRepository):
if universal_wheel is not None:
return self._get_info_from_wheel(universal_wheel)
info = {}
info = None
if universal_python2_wheel and universal_python3_wheel:
info = self._get_info_from_wheel(universal_python2_wheel)
py3_info = self._get_info_from_wheel(universal_python3_wheel)
if py3_info["requires_dist"]:
if not info["requires_dist"]:
info["requires_dist"] = py3_info["requires_dist"]
if py3_info.requires_dist:
if not info.requires_dist:
info.requires_dist = py3_info.requires_dist
return info
py2_requires_dist = set(
dependency_from_pep_508(r).to_pep_508()
for r in info["requires_dist"]
for r in info.requires_dist
)
py3_requires_dist = set(
dependency_from_pep_508(r).to_pep_508()
for r in py3_info["requires_dist"]
for r in py3_info.requires_dist
)
base_requires_dist = py2_requires_dist & py3_requires_dist
py2_only_requires_dist = py2_requires_dist - py3_requires_dist
......@@ -449,7 +392,7 @@ class PyPiRepository(RemoteRepository):
)
requires_dist.append(dep.to_pep_508())
info["requires_dist"] = sorted(list(set(requires_dist)))
info.requires_dist = sorted(list(set(requires_dist)))
if info:
return info
......@@ -467,9 +410,7 @@ class PyPiRepository(RemoteRepository):
return self._get_info_from_sdist(urls["sdist"][0])
def _get_info_from_wheel(
self, url
): # type: (str) -> Dict[str, Union[str, List, None]]
def _get_info_from_wheel(self, url): # type: (str) -> PackageInfo
self._log(
"Downloading wheel: {}".format(urlparse.urlparse(url).path.rsplit("/")[-1]),
level="debug",
......@@ -481,11 +422,9 @@ class PyPiRepository(RemoteRepository):
filepath = Path(temp_dir) / filename
self._download(url, str(filepath))
return self._inspector.inspect_wheel(filepath)
return PackageInfo.from_wheel(filepath)
def _get_info_from_sdist(
self, url
): # type: (str) -> Dict[str, Union[str, List, None]]
def _get_info_from_sdist(self, url): # type: (str) -> PackageInfo
self._log(
"Downloading sdist: {}".format(urlparse.urlparse(url).path.rsplit("/")[-1]),
level="debug",
......@@ -497,7 +436,7 @@ class PyPiRepository(RemoteRepository):
filepath = Path(temp_dir) / filename
self._download(url, str(filepath))
return self._inspector.inspect_sdist(filepath)
return PackageInfo.from_sdist(filepath)
def _download(self, url, dest): # type: (str, str) -> None
return download_file(url, dest, session=self.session)
......
poetry/utils/inspector.py deleted 100644 → 0
import logging
import os
import tarfile
import zipfile
from typing import Dict
from typing import List
from typing import Union
import pkginfo
from requests import get
from ._compat import Path
from .helpers import parse_requires
from .setup_reader import SetupReader
from .toml_file import TomlFile
logger = logging.getLogger(__name__)
class Inspector:
"""
A class to download and inspect remote packages.
"""
@classmethod
def download(cls, url, dest): # type: (str, Path) -> None
r = get(url, stream=True)
r.raise_for_status()
with open(str(dest), "wb") as f:
for chunk in r.iter_content(chunk_size=1024):
if chunk:
f.write(chunk)
def inspect(self, file_path): # type: (Path) -> Dict[str, Union[str, List[str]]]
if file_path.suffix == ".whl":
return self.inspect_wheel(file_path)
return self.inspect_sdist(file_path)
def inspect_wheel(
self, file_path
): # type: (Path) -> Dict[str, Union[str, List[str]]]
info = {
"name": "",
"version": "",
"summary": "",
"requires_python": None,
"requires_dist": [],
}
try:
meta = pkginfo.Wheel(str(file_path))
except ValueError:
# Unable to determine dependencies
# Assume none
return info
if meta.name:
info["name"] = meta.name
if meta.version:
info["version"] = meta.version
if meta.summary:
info["summary"] = meta.summary or ""
info["requires_python"] = meta.requires_python
if meta.requires_dist:
info["requires_dist"] = meta.requires_dist
return info
def inspect_sdist(
self, file_path
): # type: (Path) -> Dict[str, Union[str, List[str]]]
info = {
"name": "",
"version": "",
"summary": "",
"requires_python": None,
"requires_dist": None,
}
try:
meta = pkginfo.SDist(str(file_path))
if meta.name:
info["name"] = meta.name
if meta.version:
info["version"] = meta.version
if meta.summary:
info["summary"] = meta.summary
if meta.requires_python:
info["requires_python"] = meta.requires_python
if meta.requires_dist:
info["requires_dist"] = list(meta.requires_dist)
return info
except ValueError:
# Unable to determine dependencies
# We pass and go deeper
pass
# Still not dependencies found
# So, we unpack and introspect
suffix = file_path.suffix
if suffix == ".zip":
tar = zipfile.ZipFile(str(file_path))
else:
if suffix == ".bz2":
suffixes = file_path.suffixes
if len(suffixes) > 1 and suffixes[-2] == ".tar":
suffix = ".tar.bz2"
else:
suffix = ".tar.gz"
tar = tarfile.open(str(file_path))
try:
tar.extractall(os.path.join(str(file_path.parent), "unpacked"))
finally:
tar.close()
unpacked = file_path.parent / "unpacked"
elements = list(unpacked.glob("*"))
if len(elements) == 1 and elements[0].is_dir():
sdist_dir = elements[0]
else:
sdist_dir = unpacked / file_path.name.rstrip(suffix)
pyproject = TomlFile(sdist_dir / "pyproject.toml")
if pyproject.exists():
from poetry.factory import Factory
pyproject_content = pyproject.read()
if "tool" in pyproject_content and "poetry" in pyproject_content["tool"]:
package = Factory().create_poetry(sdist_dir).package
return {
"name": package.name,
"version": package.version.text,
"summary": package.description,
"requires_dist": [dep.to_pep_508() for dep in package.requires],
"requires_python": package.python_versions,
}
# Checking for .egg-info at root
eggs = list(sdist_dir.glob("*.egg-info"))
if eggs:
egg_info = eggs[0]
requires = egg_info / "requires.txt"
if requires.exists():
with requires.open(encoding="utf-8") as f:
info["requires_dist"] = parse_requires(f.read())
return info
# Searching for .egg-info in sub directories
eggs = list(sdist_dir.glob("**/*.egg-info"))
if eggs:
egg_info = eggs[0]
requires = egg_info / "requires.txt"
if requires.exists():
with requires.open(encoding="utf-8") as f:
info["requires_dist"] = parse_requires(f.read())
return info
# Still nothing, try reading (without executing it)
# the setup.py file.
try:
setup_info = self._inspect_sdist_with_setup(sdist_dir)
for key, value in info.items():
if value:
continue
info[key] = setup_info[key]
return info
except Exception as e:
logger.warning(
"An error occurred when reading setup.py or setup.cfg: {}".format(
str(e)
)
)
return info
def _inspect_sdist_with_setup(
self, sdist_dir
): # type: (Path) -> Dict[str, Union[str, List[str]]]
info = {
"name": None,
"version": None,
"summary": "",
"requires_python": None,
"requires_dist": None,
}
result = SetupReader.read_from_directory(sdist_dir)
requires = ""
for dep in result["install_requires"]:
requires += dep + "\n"
if result["extras_require"]:
requires += "\n"
for extra_name, deps in result["extras_require"].items():
requires += "[{}]\n".format(extra_name)
for dep in deps:
requires += dep + "\n"
requires += "\n"
info["name"] = result["name"]
info["version"] = result["version"]
info["requires_dist"] = parse_requires(requires)
info["requires_python"] = result["python_requires"]
return info
tests/conftest.py
......@@ -71,7 +71,6 @@ def config(config_source, auth_config_source, mocker):
@pytest.fixture(autouse=True)
def download_mock(mocker):
# Patch download to not download anything but to just copy from fixtures
mocker.patch("poetry.utils.inspector.Inspector.download", new=mock_download)
mocker.patch("poetry.utils.helpers.download_file", new=mock_download)
mocker.patch("poetry.puzzle.provider.download_file", new=mock_download)
mocker.patch("poetry.repositories.pypi_repository.download_file", new=mock_download)
......
tests/console/conftest.py
......@@ -16,7 +16,6 @@ from poetry.utils._compat import Path
from poetry.utils.env import MockEnv
from poetry.utils.toml_file import TomlFile
from tests.helpers import mock_clone
from tests.helpers import mock_download
@pytest.fixture()
......@@ -54,9 +53,6 @@ def setup(mocker, installer, installed, config, env):
p = mocker.patch("poetry.core.vcs.git.Git.rev_parse")
p.return_value = "9cf87a285a2d3fbb0b9fa621997b3acc3631ed24"
# Patch download to not download anything but to just copy from fixtures
mocker.patch("poetry.utils.inspector.Inspector.download", new=mock_download)
# Patch the virtual environment creation do actually do nothing
mocker.patch("poetry.utils.env.EnvManager.create_venv", return_value=env)
......
tests/installation/fixtures/with-directory-dependency-poetry-transitive.test
[[package]]
category = "main"
description = ""
description = "This is a description"
develop = true
name = "project-with-extras"
optional = false
......@@ -18,7 +18,7 @@ url = "tests/fixtures/directory/project_with_transitive_directory_dependencies/.
[[package]]
category = "main"
description = ""
description = "This is a description"
develop = true
name = "project-with-transitive-directory-dependencies"
optional = false
......
tests/installation/fixtures/with-directory-dependency-poetry.test
......@@ -8,7 +8,7 @@ version = "1.4.4"
[[package]]
category = "main"
description = ""
description = "This is a description"
develop = true
name = "project-with-extras"
optional = false
......
tests/installation/fixtures/with-file-dependency-transitive.test
......@@ -28,7 +28,7 @@ version = "1.4.4"
[[package]]
category = "main"
description = ""
description = "This is a description"
develop = true
name = "project-with-transitive-file-dependencies"
optional = false
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment