Drastically improve dependency resolution speed

CHANGELOG.md

@@ -10,6 +10,7 @@
 
 ### Changed
 
+- Drastically improved dependency resolution speed.
 - Dependency resolution caches now use sha256 hashes.
 - Changed CLI error style.
 - Improved debugging of dependency resolution.

docs/docs/basic-usage.md

@@ -61,33 +61,6 @@ $ poetry add pendulum
 
 It will automatically find a suitable version constraint.
 
-!!!warning
-
-    `poetry` uses the PyPI JSON API to retrieve package information.
-    
-    However, some packages (like `boto3` for example) have missing dependency
-    information due to bad packaging/publishing which means that `poetry` won't
-    be able to properly resolve dependencies.
-    
-    To workaround it, `poetry` has a fallback mechanism that will download packages
-    distributions to check the dependencies.
-    
-    While, in most cases, it will lead to a more exhaustive dependency resolution
-    it will also considerably slow down the process (up to 30 minutes in some extreme cases
-    like `boto3`).
-    
-    If you do not want the fallback mechanism, you can deactivate it like so.
-    
-    ```bash
-    poetry config settings.pypi.fallback false
-    ```
-    
-    In this case you will need to specify the missing dependencies in you `pyproject.toml`
-    file.
-    
-    Any case of missing dependencies should be reported to
-    the offical [repository](https://github.com/sdispater/poetry/issues)
-    and on the repository of the package with missing dependencies.
 
 ### Version constraints
 

poetry/mixology/resolution.py

@@ -935,10 +935,6 @@ class Resolution:
         current_possibility_set = None
 
         for possibility in reversed(possibilities):
-            self._debug(
-                'Getting dependencies for {}'.format(possibility),
-                depth=self.state.depth if self.state else 0
-            )
             dependencies = self._provider.dependencies_for(possibility)
             if current_possibility_set and current_possibility_set.dependencies == dependencies:
                 current_possibility_set.possibilities.insert(0, possibility)

poetry/puzzle/dependencies.py

+class Dependencies:
+    """
+    Proxy to package dependencies to only require them when needed.
+    """
+
+    def __init__(self, package, provider):
+        self._package = package
+        self._provider = provider
+        self._dependencies = None
+
+    @property
+    def dependencies(self):
+        if self._dependencies is None:
+            self._dependencies = self._get_dependencies()
+
+        return self._dependencies
+
+    def _get_dependencies(self):
+        self._provider.debug(
+            'Getting dependencies for {}'.format(self._package), 0
+        )
+        dependencies = self._provider._dependencies_for(self._package)
+
+        if dependencies is None:
+            dependencies = []
+
+        return dependencies
+
+    def __len__(self):
+        return len(self.dependencies)
+
+    def __iter__(self):
+        return self.dependencies.__iter__()
+
+    def __add__(self, other):
+        return self.dependencies + other
+
+    __radd__ = __add__

poetry/puzzle/provider.py

@@ -6,6 +6,7 @@ from functools import cmp_to_key
 from tempfile import mkdtemp
 from typing import Dict
 from typing import List
+from typing import Union
 
 from poetry.mixology import DependencyGraph
 from poetry.mixology.conflict import Conflict
@@ -29,6 +30,8 @@ from poetry.utils.venv import Venv
 
 from poetry.vcs.git import Git
 
+from .dependencies import Dependencies
+
 
 class Provider(SpecificationProvider, UI):
 
@@ -99,6 +102,7 @@ class Provider(SpecificationProvider, UI):
                 dependency.name,
                 constraint,
                 extras=dependency.extras,
+                allow_prereleases=dependency.allows_prereleases()
             )
 
             packages.sort(
@@ -233,27 +237,35 @@ class Provider(SpecificationProvider, UI):
 
         return [package]
 
-    def dependencies_for(self, package):  # type: (Package) -> List[Dependency]
+    def dependencies_for(self, package
+                         ):  # type: (Package) -> Union[List[Dependency], Dependencies]
         if package.source_type in ['git', 'file']:
             # Information should already be set
-            pass
+            return [
+                r for r in package.requires
+                if not r.is_optional()
+                   and r.name not in self.UNSAFE_PACKAGES
+            ]
         else:
-            complete_package = self._pool.package(
-                package.name, package.version,
-                extras=package.requires_extras
-            )
+            return Dependencies(package, self)
+
+    def _dependencies_for(self, package):  # type: (Package) -> List[Dependency]
+        complete_package = self._pool.package(
+            package.name, package.version,
+            extras=package.requires_extras
+        )
 
-            # Update package with new information
-            package.requires = complete_package.requires
-            package.description = complete_package.description
-            package.python_versions = complete_package.python_versions
-            package.platform = complete_package.platform
-            package.hashes = complete_package.hashes
+        # Update package with new information
+        package.requires = complete_package.requires
+        package.description = complete_package.description
+        package.python_versions = complete_package.python_versions
+        package.platform = complete_package.platform
+        package.hashes = complete_package.hashes
 
         return [
             r for r in package.requires
             if not r.is_optional()
-            and r.name not in self.UNSAFE_PACKAGES
+               and r.name not in self.UNSAFE_PACKAGES
         ]
 
     def is_requirement_satisfied_by(self,

poetry/repositories/base_repository.py

@@ -16,7 +16,8 @@ class BaseRepository(object):
     def package(self, name, version, extras=None):
         raise NotImplementedError()
 
-    def find_packages(self, name, constraint=None, extras=None):
+    def find_packages(self, name, constraint=None,
+                      extras=None, allow_prereleases=False):
         raise NotImplementedError()
 
     def search(self, query, mode=SEARCH_FULLTEXT):

poetry/repositories/legacy_repository.py

@@ -64,7 +64,9 @@ class LegacyRepository(PyPiRepository):
     def name(self):
         return self._name
 
-    def find_packages(self, name, constraint=None, extras=None):
+    def find_packages(self, name, constraint=None,
+                      extras=None,
+                      allow_prereleases=False):
         packages = []
 
         if constraint is not None and not isinstance(constraint,

poetry/repositories/pool.py

@@ -79,9 +79,14 @@ class Pool(BaseRepository):
     def find_packages(self,
                       name,
                       constraint=None,
-                      extras=None):
+                      extras=None,
+                      allow_prereleases=False):
         for repository in self._repositories:
-            packages = repository.find_packages(name, constraint, extras=extras)
+            packages = repository.find_packages(
+                name, constraint,
+                extras=extras,
+                allow_prereleases=allow_prereleases
+            )
             if packages:
                 return packages
 

poetry/repositories/pypi_repository.py

@@ -78,22 +78,21 @@ class PyPiRepository(Repository):
         super(PyPiRepository, self).__init__()
 
     def find_packages(self,
-                      name,             # type: str
-                      constraint=None,  # type: Union[Constraint, str, None]
-                      extras=None       # type: Union[list, None]
+                      name,                    # type: str
+                      constraint=None,         # type: Union[Constraint, str, None]
+                      extras=None,             # type: Union[list, None]
+                      allow_prereleases=False  # type: bool
                       ):  # type: (...) -> List[Package]
         """
         Find packages on the remote server.
         """
-        packages = []
-
         if constraint is not None and not isinstance(constraint, BaseConstraint):
             version_parser = VersionParser()
             constraint = version_parser.parse_constraints(constraint)
 
         info = self.get_package_info(name)
 
-        versions = []
+        packages = []
 
         for version, release in info['releases'].items():
             if not release:
@@ -106,18 +105,19 @@ class PyPiRepository(Repository):
                 )
                 continue
 
+            package = Package(name, version)
+
+            if package.is_prerelease() and not allow_prereleases:
+                continue
+
             if (
                 not constraint
                 or (constraint and constraint.matches(Constraint('=', version)))
             ):
-                versions.append(version)
-
-        for version in versions:
-            package = Package(name, version)
-            if extras is not None:
-                package.requires_extras = extras
+                if extras is not None:
+                    package.requires_extras = extras
 
-            packages.append(package)
+                packages.append(package)
 
         self._log(
             '{} packages found for {} {}'.format(

poetry/repositories/repository.py

@@ -26,7 +26,9 @@ class Repository(BaseRepository):
             if name == package.name and package.version == version:
                 return package
 
-    def find_packages(self, name, constraint=None, extras=None):
+    def find_packages(self, name, constraint=None,
+                      extras=None,
+                      allow_prereleases=False):
         name = name.lower()
         packages = []
         if extras is None: