Skip to content

P
python-poetry
Commit ed104cd0 by Sébastien Eustace
Options

Add a fallback mechanism for missing dependencies

parent 81c6fdfb
Showing with 111 additions and 39 deletions
CHANGELOG.md
......@@ -5,6 +5,7 @@
### Added
- Added support for Python 2.7.
- Added a fallback mechanism (opt-in) for missing dependencies.
### Changes
......
docs/docs/basic-usage.md
......@@ -61,6 +61,27 @@ $ poetry add pendulum
It will automatically find a suitable version constraint.
!!!warning
`poetry` uses the PyPI JSON API to retrieve package information.
However, some packages (like `boto3` for example) have missing dependency
information due to bad packaging/publishing which means that `poetry` won't
be able to properly resolve dependencies.
To workaround it you can set the missing dependencies yourself in your `pyproject.toml`
or you can tell `poetry` to use a fallback mechanism by setting the
`settings.pypi.fallback` setting to `true`.
```bash
poetry config settings.pypi.fallback true
```
Note that this is temporary and should be avoided as much as possible since
it increases the dependency resolution time drastically (up to 30 minutes in some cases).
Any case of missing dependencies should be reported to https://github.com/sdispater/poetry/issues
### Version constraints
In our example, we are requesting the `pendulum` package with the version constraint `^1.4`.
......
poetry/config.py
......@@ -33,7 +33,7 @@ class Config:
def content(self):
return self._content
def setting(self, setting_name): # type: (str) -> Any
def setting(self, setting_name, default=None): # type: (str) -> Any
"""
Retrieve a setting value.
"""
......@@ -42,7 +42,7 @@ class Config:
config = self._raw_content
for key in keys:
if key not in config:
return None
return default
config = config[key]
......
poetry/console/commands/check.py
......@@ -10,6 +10,6 @@ class CheckCommand(Command):
def handle(self):
# Load poetry and display errors, if any
self.poetry.check(self.poetry.config, strict=True)
self.poetry.check(self.poetry.local_config, strict=True)
self.info('All set!')
poetry/console/commands/config.py
......@@ -49,11 +49,13 @@ To remove a repository (repo is a short alias for repositories):
# Create config file if it does not exist
if not self._config.file.exists():
self._config.file.parent.mkdir(parents=True, exist_ok=True)
self._config.file.write_text(TEMPLATE)
with self._config.file.open() as f:
f.write(TEMPLATE)
if not self._auth_config.file.exists():
self._auth_config.file.parent.mkdir(parents=True, exist_ok=True)
self._auth_config.file.write_text(AUTH_TEMPLATE)
with self._auth_config.file.open() as f:
f.write(AUTH_TEMPLATE)
def handle(self):
if self.option('list'):
......@@ -98,7 +100,8 @@ To remove a repository (repo is a short alias for repositories):
boolean_normalizer = lambda val: True if val in ['true', '1'] else False
unique_config_values = {
'settings.virtualenvs.create': (boolean_validator, boolean_normalizer)
'settings.virtualenvs.create': (boolean_validator, boolean_normalizer),
'settings.pypi.fallback': (boolean_validator, boolean_normalizer),
}
if setting_key in unique_config_values:
......@@ -216,7 +219,7 @@ To remove a repository (repo is a short alias for repositories):
orig_k = k
for key, value in contents.items():
if k is None and key not in ['config', 'repositories']:
if k is None and key not in ['config', 'repositories', 'settings']:
continue
if isinstance(value, dict) or key == 'repositories' and k is None:
......
poetry/console/commands/script.py
......@@ -16,7 +16,7 @@ class ScriptCommand(VenvCommand):
script = self.argument('script-name')
argv = [script] + self.argument('args')
scripts = self.poetry.config.get('scripts')
scripts = self.poetry.local_config.get('scripts')
if not scripts:
raise RuntimeError('No scripts defined in pyproject.toml')
......
poetry/masonry/builders/builder.py
......@@ -102,8 +102,8 @@ class Builder(object):
# If a README is specificed we need to include it
# to avoid errors
if 'readme' in self._poetry.config:
readme = self._path / self._poetry.config['readme']
if 'readme' in self._poetry.local_config:
readme = self._path / self._poetry.local_config['readme']
if readme.exists():
self._io.writeln(
' - Adding: <comment>{}</comment>'.format(
......@@ -124,11 +124,12 @@ class Builder(object):
result = defaultdict(list)
# Scripts -> Entry points
for name, ep in self._poetry.config.get('scripts', {}).items():
for name, ep in self._poetry.local_config.get('scripts', {}).items():
result['console_scripts'].append('{} = {}'.format(name, ep))
# Plugins -> entry points
for groupname, group in self._poetry.config.get('plugins', {}).items():
plugins = self._poetry.local_config.get('plugins', {})
for groupname, group in plugins.items():
for name, ep in sorted(group.items()):
result[groupname].append('{} = {}'.format(name, ep))
......
poetry/masonry/builders/wheel.py
......@@ -146,7 +146,10 @@ class WheelBuilder(Builder):
self._add_file(str(self._module.path), self._module.path.name)
def write_metadata(self):
if 'scripts' in self._poetry.config or 'plugins' in self._poetry.config:
if (
'scripts' in self._poetry.local_config
or 'plugins' in self._poetry.local_config
):
with self._write_to_zip(self.dist_info + '/entry_points.txt') as f:
self._write_entry_points(f)
......
poetry/poetry.py
......@@ -6,6 +6,8 @@ import json
import jsonschema
from .__version__ import __version__
from .config import Config
from .console.commands.config import TEMPLATE
from .exceptions import InvalidProjectFile
from .packages import Dependency
from .packages import Locker
......@@ -22,23 +24,38 @@ class Poetry:
VERSION = __version__
def __init__(self,
file, # type: Path
config, # type: dict
package, # type: Package
locker # type: Locker
file, # type: Path
local_config, # type: dict
package, # type: Package
locker # type: Locker
):
self._file = TomlFile(file)
self._package = package
self._config = config
self._local_config = local_config
self._locker = locker
self._config = Config.create('config.toml')
# Configure sources
self._pool = Pool()
for source in self._config.get('source', []):
for source in self._local_config.get('source', []):
self._pool.configure(source)
# Always put PyPI last to prefere private repositories
self._pool.add_repository(PyPiRepository())
self._pool.add_repository(
PyPiRepository(
fallback=self._config.setting(
'settings.pypi.fallback',
False
)
)
)
# Adding a fallback for PyPI for when dependencies
# are not retrievable via the JSON API
self._pool.configure({
'name': 'pypi-fallback',
'url': 'https://pypi.org/simple'
})
@property
def file(self):
......@@ -49,8 +66,8 @@ class Poetry:
return self._package
@property
def config(self): # type: () -> dict
return self._config
def local_config(self): # type: () -> dict
return self._local_config
@property
def locker(self): # type: () -> Locker
......
poetry/puzzle/provider.py
......@@ -223,6 +223,5 @@ class Provider(SpecificationProvider):
0 if activated.vertex_named(d.name).payload else 1,
0 if activated.vertex_named(d.name).root else 1,
0 if d.allows_prereleases() else 1,
0 if d.name in conflicts else 1,
0 if activated.vertex_named(d.name).payload else len(self.search_for(d))
0 if d.name in conflicts else 1
])
poetry/repositories/legacy_repository.py
from pathlib import Path
from pip._vendor.pkg_resources import RequirementParseError
from piptools.cache import DependencyCache
from piptools.repositories import PyPIRepository
......@@ -10,10 +9,12 @@ from cachy import CacheManager
import poetry.packages
from poetry.locations import CACHE_DIR
from poetry.packages import Package
from poetry.packages import dependency_from_pep_508
from poetry.semver.constraints import Constraint
from poetry.semver.constraints.base_constraint import BaseConstraint
from poetry.semver.version_parser import VersionParser
from poetry.utils._compat import Path
from poetry.version.markers import InvalidMarker
from .pypi_repository import PyPiRepository
......@@ -61,7 +62,7 @@ class LegacyRepository(PyPiRepository):
key = name
if constraint:
key = f'{key}:{str(constraint)}'
key = '{}:{}'.format(key, str(constraint))
if self._cache.store('matches').has(key):
versions = self._cache.store('matches').get(key)
......@@ -82,7 +83,7 @@ class LegacyRepository(PyPiRepository):
self._cache.store('matches').put(key, versions, 5)
for version in versions:
packages.append(self.package(name, version, extras=extras))
packages.append(Package(name, version, extras=extras))
return packages
......@@ -157,7 +158,7 @@ class LegacyRepository(PyPiRepository):
or retrieved from the remote server.
"""
return self._cache.store('releases').remember_forever(
f'{name}:{version}',
'{}:{}'.format(name, version),
lambda: self._get_release_info(name, version)
)
......@@ -165,7 +166,7 @@ class LegacyRepository(PyPiRepository):
from pip.req import InstallRequirement
from pip.exceptions import InstallationError
ireq = InstallRequirement.from_line(f'{name}=={version}')
ireq = InstallRequirement.from_line('{}=={}'.format(name, version))
resolver = Resolver(
[ireq], self._repository,
cache=DependencyCache(self._cache_dir.as_posix())
......@@ -180,13 +181,18 @@ class LegacyRepository(PyPiRepository):
requires = []
for dep in requirements:
constraint = str(dep.req.specifier)
require = f'{dep.name}'
require = dep.name
if constraint:
require += f' ({constraint})'
require += ' ({})'.format(constraint)
requires.append(require)
hashes = resolver.resolve_hashes([ireq])[ireq]
try:
hashes = resolver.resolve_hashes([ireq])[ireq]
except IndexError:
# Sometimes pip-tools fails when getting indices
hashes = []
hashes = [h.split(':')[1] for h in hashes]
data = {
......
poetry/repositories/pypi_repository.py
......@@ -20,9 +20,13 @@ from .repository import Repository
class PyPiRepository(Repository):
def __init__(self, url='https://pypi.org/', disable_cache=False):
def __init__(self,
url='https://pypi.org/',
disable_cache=False,
fallback=False):
self._url = url
self._disable_cache = disable_cache
self._fallback = fallback
release_cache_dir = Path(CACHE_DIR) / 'cache' / 'repositories' / 'pypi'
self._cache = CacheManager({
......@@ -41,7 +45,7 @@ class PyPiRepository(Repository):
self._session = CacheControl(
session(),
cache=FileCache(str(release_cache_dir / '_packages'))
cache=FileCache(str(release_cache_dir / '_http'))
)
super(PyPiRepository, self).__init__()
......@@ -72,9 +76,7 @@ class PyPiRepository(Repository):
versions.append(version)
for version in versions:
packages.append(
self.package(name, version, extras=extras)
)
packages.append(Package(name, version, version))
return packages
......@@ -82,7 +84,7 @@ class PyPiRepository(Repository):
name, # type: str
version, # type: str
extras=None # type: (Union[list, None])
): # type: (...) -> Package
): # type: (...) -> Union[Package, None]
try:
index = self._packages.index(Package(name, version, version))
......@@ -92,6 +94,19 @@ class PyPiRepository(Repository):
extras = []
release_info = self.get_release_info(name, version)
if (
self._fallback
and release_info['requires_dist'] is None
and not release_info['requires_python']
and not release_info['platform']
):
# No dependencies set (along with other information)
# This might be due to actually no dependencies
# or badly set metadata when uploading
# So, we return None so that the fallback repository
# can pick up more accurate info
return
package = Package(name, version, version)
requires_dist = release_info['requires_dist'] or []
for req in requires_dist:
......@@ -209,7 +224,13 @@ class PyPiRepository(Repository):
'requires_python': info['requires_python'],
'digests': []
}
for file_info in json_data['releases'][version]:
try:
version_info = json_data['releases'][version]
except KeyError:
version_info = []
for file_info in version_info:
data['digests'].append(file_info['digests']['sha256'])
return data
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment