Add a fallback mechanism for missing dependencies

CHANGELOG.md

@@ -5,6 +5,7 @@
 ### Added
 ### Added
 
 
 - Added support for Python 2.7.
 - Added support for Python 2.7.
+- Added a fallback mechanism (opt-in) for missing dependencies.
 
 
 ### Changes
 ### Changes
 
 

docs/docs/basic-usage.md

@@ -61,6 +61,27 @@ $ poetry add pendulum
 
 
 It will automatically find a suitable version constraint.
 It will automatically find a suitable version constraint.
 
 
+!!!warning
+
+    `poetry` uses the PyPI JSON API to retrieve package information.
+    
+    However, some packages (like `boto3` for example) have missing dependency
+    information due to bad packaging/publishing which means that `poetry` won't
+    be able to properly resolve dependencies.
+    
+    To workaround it you can set the missing dependencies yourself in your `pyproject.toml`
+    or you can tell `poetry` to use a fallback mechanism by setting the
+    `settings.pypi.fallback` setting to `true`.
+    
+    ```bash
+    poetry config settings.pypi.fallback true
+    ```
+    
+    Note that this is temporary and should be avoided as much as possible since
+    it increases the dependency resolution time drastically (up to 30 minutes in some cases).
+    
+    Any case of missing dependencies should be reported to https://github.com/sdispater/poetry/issues
+
 ### Version constraints
 ### Version constraints
 
 
 In our example, we are requesting the `pendulum` package with the version constraint `^1.4`.
 In our example, we are requesting the `pendulum` package with the version constraint `^1.4`.

poetry/config.py

@@ -33,7 +33,7 @@ class Config:
     def content(self):
     def content(self):
         return self._content
         return self._content
 
 
-    def setting(self, setting_name):  # type: (str) -> Any
+    def setting(self, setting_name, default=None):  # type: (str) -> Any
         """
         """
         Retrieve a setting value.
         Retrieve a setting value.
         """
         """
@@ -42,7 +42,7 @@ class Config:
         config = self._raw_content
         config = self._raw_content
         for key in keys:
         for key in keys:
             if key not in config:
             if key not in config:
-                return None
+                return default
 
 
             config = config[key]
             config = config[key]
 
 

poetry/console/commands/check.py

@@ -10,6 +10,6 @@ class CheckCommand(Command):
 
 
     def handle(self):
     def handle(self):
         # Load poetry and display errors, if any
         # Load poetry and display errors, if any
-        self.poetry.check(self.poetry.config, strict=True)
+        self.poetry.check(self.poetry.local_config, strict=True)
 
 
         self.info('All set!')
         self.info('All set!')

poetry/console/commands/config.py

@@ -49,11 +49,13 @@ To remove a repository (repo is a short alias for repositories):
         # Create config file if it does not exist
         # Create config file if it does not exist
         if not self._config.file.exists():
         if not self._config.file.exists():
             self._config.file.parent.mkdir(parents=True, exist_ok=True)
             self._config.file.parent.mkdir(parents=True, exist_ok=True)
-            self._config.file.write_text(TEMPLATE)
+            with self._config.file.open() as f:
+                f.write(TEMPLATE)
 
 
         if not self._auth_config.file.exists():
         if not self._auth_config.file.exists():
             self._auth_config.file.parent.mkdir(parents=True, exist_ok=True)
             self._auth_config.file.parent.mkdir(parents=True, exist_ok=True)
-            self._auth_config.file.write_text(AUTH_TEMPLATE)
+            with self._auth_config.file.open() as f:
+                f.write(AUTH_TEMPLATE)
 
 
     def handle(self):
     def handle(self):
         if self.option('list'):
         if self.option('list'):
@@ -98,7 +100,8 @@ To remove a repository (repo is a short alias for repositories):
         boolean_normalizer = lambda val: True if val in ['true', '1'] else False
         boolean_normalizer = lambda val: True if val in ['true', '1'] else False
 
 
         unique_config_values = {
         unique_config_values = {
-            'settings.virtualenvs.create': (boolean_validator, boolean_normalizer)
+            'settings.virtualenvs.create': (boolean_validator, boolean_normalizer),
+            'settings.pypi.fallback': (boolean_validator, boolean_normalizer),
         }
         }
 
 
         if setting_key in unique_config_values:
         if setting_key in unique_config_values:
@@ -216,7 +219,7 @@ To remove a repository (repo is a short alias for repositories):
         orig_k = k
         orig_k = k
 
 
         for key, value in contents.items():
         for key, value in contents.items():
-            if k is None and key not in ['config', 'repositories']:
+            if k is None and key not in ['config', 'repositories', 'settings']:
                 continue
                 continue
 
 
             if isinstance(value, dict) or key == 'repositories' and k is None:
             if isinstance(value, dict) or key == 'repositories' and k is None:

poetry/console/commands/script.py

@@ -16,7 +16,7 @@ class ScriptCommand(VenvCommand):
         script = self.argument('script-name')
         script = self.argument('script-name')
         argv = [script] + self.argument('args')
         argv = [script] + self.argument('args')
 
 
-        scripts = self.poetry.config.get('scripts')
+        scripts = self.poetry.local_config.get('scripts')
         if not scripts:
         if not scripts:
             raise RuntimeError('No scripts defined in pyproject.toml')
             raise RuntimeError('No scripts defined in pyproject.toml')
 
 

poetry/masonry/builders/builder.py

@@ -102,8 +102,8 @@ class Builder(object):
 
 
         # If a README is specificed we need to include it
         # If a README is specificed we need to include it
         # to avoid errors
         # to avoid errors
-        if 'readme' in self._poetry.config:
-            readme = self._path / self._poetry.config['readme']
+        if 'readme' in self._poetry.local_config:
+            readme = self._path / self._poetry.local_config['readme']
             if readme.exists():
             if readme.exists():
                 self._io.writeln(
                 self._io.writeln(
                     ' - Adding: <comment>{}</comment>'.format(
                     ' - Adding: <comment>{}</comment>'.format(
@@ -124,11 +124,12 @@ class Builder(object):
         result = defaultdict(list)
         result = defaultdict(list)
 
 
         # Scripts -> Entry points
         # Scripts -> Entry points
-        for name, ep in self._poetry.config.get('scripts', {}).items():
+        for name, ep in self._poetry.local_config.get('scripts', {}).items():
             result['console_scripts'].append('{} = {}'.format(name, ep))
             result['console_scripts'].append('{} = {}'.format(name, ep))
 
 
         # Plugins -> entry points
         # Plugins -> entry points
-        for groupname, group in self._poetry.config.get('plugins', {}).items():
+        plugins = self._poetry.local_config.get('plugins', {})
+        for groupname, group in plugins.items():
             for name, ep in sorted(group.items()):
             for name, ep in sorted(group.items()):
                 result[groupname].append('{} = {}'.format(name, ep))
                 result[groupname].append('{} = {}'.format(name, ep))
 
 

poetry/masonry/builders/wheel.py

@@ -146,7 +146,10 @@ class WheelBuilder(Builder):
             self._add_file(str(self._module.path), self._module.path.name)
             self._add_file(str(self._module.path), self._module.path.name)
 
 
     def write_metadata(self):
     def write_metadata(self):
-        if 'scripts' in self._poetry.config or 'plugins' in self._poetry.config:
+        if (
+            'scripts' in self._poetry.local_config
+            or 'plugins' in self._poetry.local_config
+        ):
             with self._write_to_zip(self.dist_info + '/entry_points.txt') as f:
             with self._write_to_zip(self.dist_info + '/entry_points.txt') as f:
                 self._write_entry_points(f)
                 self._write_entry_points(f)
 
 

poetry/poetry.py

@@ -6,6 +6,8 @@ import json
 import jsonschema
 import jsonschema
 
 
 from .__version__ import __version__
 from .__version__ import __version__
+from .config import Config
+from .console.commands.config import TEMPLATE
 from .exceptions import InvalidProjectFile
 from .exceptions import InvalidProjectFile
 from .packages import Dependency
 from .packages import Dependency
 from .packages import Locker
 from .packages import Locker
@@ -23,22 +25,37 @@ class Poetry:
 
 
     def __init__(self,
     def __init__(self,
                  file,          # type: Path
                  file,          # type: Path
-                 config,   # type: dict
+                 local_config,  # type: dict
                  package,       # type: Package
                  package,       # type: Package
                  locker         # type: Locker
                  locker         # type: Locker
                  ):
                  ):
         self._file = TomlFile(file)
         self._file = TomlFile(file)
         self._package = package
         self._package = package
-        self._config = config
+        self._local_config = local_config
         self._locker = locker
         self._locker = locker
+        self._config = Config.create('config.toml')
 
 
         # Configure sources
         # Configure sources
         self._pool = Pool()
         self._pool = Pool()
-        for source in self._config.get('source', []):
+        for source in self._local_config.get('source', []):
             self._pool.configure(source)
             self._pool.configure(source)
 
 
         # Always put PyPI last to prefere private repositories
         # Always put PyPI last to prefere private repositories
-        self._pool.add_repository(PyPiRepository())
+        self._pool.add_repository(
+            PyPiRepository(
+                fallback=self._config.setting(
+                    'settings.pypi.fallback',
+                    False
+                )
+            )
+        )
+
+        # Adding a fallback for PyPI for when dependencies
+        # are not retrievable via the JSON API
+        self._pool.configure({
+            'name': 'pypi-fallback',
+            'url': 'https://pypi.org/simple'
+        })
         
         
     @property
     @property
     def file(self):
     def file(self):
@@ -49,8 +66,8 @@ class Poetry:
         return self._package
         return self._package
 
 
     @property
     @property
-    def config(self):  # type: () -> dict
-        return self._config
+    def local_config(self):  # type: () -> dict
+        return self._local_config
 
 
     @property
     @property
     def locker(self):  # type: () -> Locker
     def locker(self):  # type: () -> Locker

poetry/puzzle/provider.py

@@ -223,6 +223,5 @@ class Provider(SpecificationProvider):
             0 if activated.vertex_named(d.name).payload else 1,
             0 if activated.vertex_named(d.name).payload else 1,
             0 if activated.vertex_named(d.name).root else 1,
             0 if activated.vertex_named(d.name).root else 1,
             0 if d.allows_prereleases() else 1,
             0 if d.allows_prereleases() else 1,
-            0 if d.name in conflicts else 1,
-            0 if activated.vertex_named(d.name).payload else len(self.search_for(d))
+            0 if d.name in conflicts else 1
         ])
         ])

poetry/repositories/legacy_repository.py

-from pathlib import Path
 from pip._vendor.pkg_resources import RequirementParseError
 from pip._vendor.pkg_resources import RequirementParseError
 from piptools.cache import DependencyCache
 from piptools.cache import DependencyCache
 from piptools.repositories import PyPIRepository
 from piptools.repositories import PyPIRepository
@@ -10,10 +9,12 @@ from cachy import CacheManager
 import poetry.packages
 import poetry.packages
 
 
 from poetry.locations import CACHE_DIR
 from poetry.locations import CACHE_DIR
+from poetry.packages import Package
 from poetry.packages import dependency_from_pep_508
 from poetry.packages import dependency_from_pep_508
 from poetry.semver.constraints import Constraint
 from poetry.semver.constraints import Constraint
 from poetry.semver.constraints.base_constraint import BaseConstraint
 from poetry.semver.constraints.base_constraint import BaseConstraint
 from poetry.semver.version_parser import VersionParser
 from poetry.semver.version_parser import VersionParser
+from poetry.utils._compat import Path
 from poetry.version.markers import InvalidMarker
 from poetry.version.markers import InvalidMarker
 
 
 from .pypi_repository import PyPiRepository
 from .pypi_repository import PyPiRepository
@@ -61,7 +62,7 @@ class LegacyRepository(PyPiRepository):
 
 
         key = name
         key = name
         if constraint:
         if constraint:
-            key = f'{key}:{str(constraint)}'
+            key = '{}:{}'.format(key, str(constraint))
 
 
         if self._cache.store('matches').has(key):
         if self._cache.store('matches').has(key):
             versions = self._cache.store('matches').get(key)
             versions = self._cache.store('matches').get(key)
@@ -82,7 +83,7 @@ class LegacyRepository(PyPiRepository):
             self._cache.store('matches').put(key, versions, 5)
             self._cache.store('matches').put(key, versions, 5)
 
 
         for version in versions:
         for version in versions:
-            packages.append(self.package(name, version, extras=extras))
+            packages.append(Package(name, version, extras=extras))
 
 
         return packages
         return packages
 
 
@@ -157,7 +158,7 @@ class LegacyRepository(PyPiRepository):
         or retrieved from the remote server.
         or retrieved from the remote server.
         """
         """
         return self._cache.store('releases').remember_forever(
         return self._cache.store('releases').remember_forever(
-            f'{name}:{version}',
+            '{}:{}'.format(name, version),
             lambda: self._get_release_info(name, version)
             lambda: self._get_release_info(name, version)
         )
         )
 
 
@@ -165,7 +166,7 @@ class LegacyRepository(PyPiRepository):
         from pip.req import InstallRequirement
         from pip.req import InstallRequirement
         from pip.exceptions import InstallationError
         from pip.exceptions import InstallationError
 
 
-        ireq = InstallRequirement.from_line(f'{name}=={version}')
+        ireq = InstallRequirement.from_line('{}=={}'.format(name, version))
         resolver = Resolver(
         resolver = Resolver(
             [ireq], self._repository,
             [ireq], self._repository,
             cache=DependencyCache(self._cache_dir.as_posix())
             cache=DependencyCache(self._cache_dir.as_posix())
@@ -180,13 +181,18 @@ class LegacyRepository(PyPiRepository):
         requires = []
         requires = []
         for dep in requirements:
         for dep in requirements:
             constraint = str(dep.req.specifier)
             constraint = str(dep.req.specifier)
-            require = f'{dep.name}'
+            require = dep.name
             if constraint:
             if constraint:
-                require += f' ({constraint})'
+                require += ' ({})'.format(constraint)
 
 
             requires.append(require)
             requires.append(require)
 
 
+        try:
             hashes = resolver.resolve_hashes([ireq])[ireq]
             hashes = resolver.resolve_hashes([ireq])[ireq]
+        except IndexError:
+            # Sometimes pip-tools fails when getting indices
+            hashes = []
+
         hashes = [h.split(':')[1] for h in hashes]
         hashes = [h.split(':')[1] for h in hashes]
 
 
         data = {
         data = {

poetry/repositories/pypi_repository.py

@@ -20,9 +20,13 @@ from .repository import Repository
 
 
 class PyPiRepository(Repository):
 class PyPiRepository(Repository):
 
 
-    def __init__(self, url='https://pypi.org/', disable_cache=False):
+    def __init__(self,
+                 url='https://pypi.org/',
+                 disable_cache=False,
+                 fallback=False):
         self._url = url
         self._url = url
         self._disable_cache = disable_cache
         self._disable_cache = disable_cache
+        self._fallback = fallback
 
 
         release_cache_dir = Path(CACHE_DIR) / 'cache' / 'repositories' / 'pypi'
         release_cache_dir = Path(CACHE_DIR) / 'cache' / 'repositories' / 'pypi'
         self._cache = CacheManager({
         self._cache = CacheManager({
@@ -41,7 +45,7 @@ class PyPiRepository(Repository):
 
 
         self._session = CacheControl(
         self._session = CacheControl(
             session(),
             session(),
-            cache=FileCache(str(release_cache_dir / '_packages'))
+            cache=FileCache(str(release_cache_dir / '_http'))
         )
         )
         
         
         super(PyPiRepository, self).__init__()
         super(PyPiRepository, self).__init__()
@@ -72,9 +76,7 @@ class PyPiRepository(Repository):
                 versions.append(version)
                 versions.append(version)
 
 
         for version in versions:
         for version in versions:
-            packages.append(
-                self.package(name, version, extras=extras)
-            )
+            packages.append(Package(name, version, version))
 
 
         return packages
         return packages
 
 
@@ -82,7 +84,7 @@ class PyPiRepository(Repository):
                 name,        # type: str
                 name,        # type: str
                 version,     # type: str
                 version,     # type: str
                 extras=None  # type: (Union[list, None])
                 extras=None  # type: (Union[list, None])
-                ):  # type: (...) -> Package
+                ):  # type: (...) -> Union[Package, None]
         try:
         try:
             index = self._packages.index(Package(name, version, version))
             index = self._packages.index(Package(name, version, version))
 
 
@@ -92,6 +94,19 @@ class PyPiRepository(Repository):
                 extras = []
                 extras = []
 
 
             release_info = self.get_release_info(name, version)
             release_info = self.get_release_info(name, version)
+            if (
+                self._fallback
+                and release_info['requires_dist'] is None
+                and not release_info['requires_python']
+                and not release_info['platform']
+            ):
+                # No dependencies set (along with other information)
+                # This might be due to actually no dependencies
+                # or badly set metadata when uploading
+                # So, we return None so that the fallback repository
+                # can pick up more accurate info
+                return
+
             package = Package(name, version, version)
             package = Package(name, version, version)
             requires_dist = release_info['requires_dist'] or []
             requires_dist = release_info['requires_dist'] or []
             for req in requires_dist:
             for req in requires_dist:
@@ -209,7 +224,13 @@ class PyPiRepository(Repository):
             'requires_python': info['requires_python'],
             'requires_python': info['requires_python'],
             'digests': []
             'digests': []
         }
         }
-        for file_info in json_data['releases'][version]:
+
+        try:
+            version_info = json_data['releases'][version]
+        except KeyError:
+            version_info = []
+
+        for file_info in version_info:
             data['digests'].append(file_info['digests']['sha256'])
             data['digests'].append(file_info['digests']['sha256'])
 
 
         return data
         return data