Improve fallback system

docs/docs/basic-usage.md

@@ -69,18 +69,24 @@ It will automatically find a suitable version constraint.
     information due to bad packaging/publishing which means that `poetry` won't
     information due to bad packaging/publishing which means that `poetry` won't
     be able to properly resolve dependencies.
     be able to properly resolve dependencies.
     
     
-    To workaround it you can set the missing dependencies yourself in your `pyproject.toml`
-    or you can tell `poetry` to use a fallback mechanism by setting the
-    `settings.pypi.fallback` setting to `true`.
+    To workaround it, `poetry` has a fallback mechanism that will download packages
+    distributions to check the dependencies.
+    
+    While, in most cases, it will lead to a more exhaustive dependency resolution
+    it will also considerably slow down the process (up to 30 minutes in some extreme cases
+    like `boto3`).
+    
+    If you do not want the fallback mechanism, you can deactivate it like so.
     
     
     ```bash
     ```bash
-    poetry config settings.pypi.fallback true
+    poetry config settings.pypi.fallback false
     ```
     ```
     
     
-    Note that this is temporary and should be avoided as much as possible since
-    it increases the dependency resolution time drastically (up to 30 minutes in some cases).
+    In this case you will need to specify the missing dependencies in you `pyproject.toml`
+    file.
     
     
     Any case of missing dependencies should be reported to https://github.com/sdispater/poetry/issues
     Any case of missing dependencies should be reported to https://github.com/sdispater/poetry/issues
+    and on the repository of the main package.
 
 
 ### Version constraints
 ### Version constraints
 
 

poetry/poetry.py

@@ -42,19 +42,9 @@ class Poetry:
         # Always put PyPI last to prefere private repositories
         # Always put PyPI last to prefere private repositories
         self._pool.add_repository(
         self._pool.add_repository(
             PyPiRepository(
             PyPiRepository(
-                fallback=self._config.setting(
-                    'settings.pypi.fallback',
-                    False
+                fallback=self._config.setting('settings.pypi.fallback', True)
             )
             )
         )
         )
-        )
-
-        # Adding a fallback for PyPI for when dependencies
-        # are not retrievable via the JSON API
-        self._pool.configure({
-            'name': 'pypi-fallback',
-            'url': 'https://pypi.org/simple'
-        })
         
         
     @property
     @property
     def file(self):
     def file(self):

poetry/repositories/pypi_repository.py

+import os
+import tarfile
+import zipfile
+
+import pkginfo
+
+from bz2 import BZ2File
+from gzip import GzipFile
 from typing import List
 from typing import List
 from typing import Union
 from typing import Union
 
 
 try:
 try:
+    import urllib.parse as urlparse
+except ImportError:
+    import urlparse
+
+try:
     from xmlrpc.client import ServerProxy
     from xmlrpc.client import ServerProxy
 except ImportError:
 except ImportError:
     from xmlrpclib import ServerProxy
     from xmlrpclib import ServerProxy
@@ -9,6 +22,7 @@ except ImportError:
 from cachecontrol import CacheControl
 from cachecontrol import CacheControl
 from cachecontrol.caches.file_cache import FileCache
 from cachecontrol.caches.file_cache import FileCache
 from cachy import CacheManager
 from cachy import CacheManager
+from requests import get
 from requests import session
 from requests import session
 
 
 from poetry.locations import CACHE_DIR
 from poetry.locations import CACHE_DIR
@@ -18,6 +32,7 @@ from poetry.semver.constraints import Constraint
 from poetry.semver.constraints.base_constraint import BaseConstraint
 from poetry.semver.constraints.base_constraint import BaseConstraint
 from poetry.semver.version_parser import VersionParser
 from poetry.semver.version_parser import VersionParser
 from poetry.utils._compat import Path
 from poetry.utils._compat import Path
+from poetry.utils.helpers import temporary_directory
 from poetry.version.markers import InvalidMarker
 from poetry.version.markers import InvalidMarker
 
 
 from .repository import Repository
 from .repository import Repository
@@ -28,7 +43,7 @@ class PyPiRepository(Repository):
     def __init__(self,
     def __init__(self,
                  url='https://pypi.org/',
                  url='https://pypi.org/',
                  disable_cache=False,
                  disable_cache=False,
-                 fallback=False):
+                 fallback=True):
         self._url = url
         self._url = url
         self._disable_cache = disable_cache
         self._disable_cache = disable_cache
         self._fallback = fallback
         self._fallback = fallback
@@ -103,13 +118,11 @@ class PyPiRepository(Repository):
                 self._fallback
                 self._fallback
                 and release_info['requires_dist'] is None
                 and release_info['requires_dist'] is None
                 and not release_info['requires_python']
                 and not release_info['requires_python']
+                and '_fallback' not in release_info
             ):
             ):
-                # No dependencies set (along with other information)
-                # This might be due to actually no dependencies
-                # or badly set metadata when uploading
-                # So, we return None so that the fallback repository
-                # can pick up more accurate info
-                return
+                # Force cache update
+                self._cache.forget('{}:{}'.format(name, version))
+                release_info = self.get_release_info(name, version)
 
 
             package = Package(name, version, version)
             package = Package(name, version, version)
             requires_dist = release_info['requires_dist'] or []
             requires_dist = release_info['requires_dist'] or []
@@ -230,7 +243,8 @@ class PyPiRepository(Repository):
             'platform': info['platform'],
             'platform': info['platform'],
             'requires_dist': info['requires_dist'],
             'requires_dist': info['requires_dist'],
             'requires_python': info['requires_python'],
             'requires_python': info['requires_python'],
-            'digests': []
+            'digests': [],
+            '_fallback': False
         }
         }
 
 
         try:
         try:
@@ -241,6 +255,50 @@ class PyPiRepository(Repository):
         for file_info in version_info:
         for file_info in version_info:
             data['digests'].append(file_info['digests']['sha256'])
             data['digests'].append(file_info['digests']['sha256'])
 
 
+        if (
+                self._fallback
+                and data['requires_dist'] is None
+                and not data['requires_python']
+        ):
+            # No dependencies set (along with other information)
+            # This might be due to actually no dependencies
+            # or badly set metadata when uploading
+            # So, we need to make sure there is actually no
+            # dependencies by introspecting packages
+            data['_fallback'] = True
+
+            urls = {}
+            for url in json_data['urls']:
+                # Only get sdist and universal wheels
+                dist_type = url['packagetype']
+                if dist_type not in ['sdist', 'bdist_wheel']:
+                    continue
+
+                if dist_type == 'sdist' and 'dist' not in urls:
+                    urls[url['packagetype']] = url['url']
+                    continue
+
+                if 'bdist_wheel' in urls:
+                    continue
+
+                # If bdist_wheel, check if it's universal
+                python_version = url['python_version']
+                if python_version not in ['py2.py3', 'py3', 'py2']:
+                    continue
+
+                parts = urlparse.urlparse(url['url'])
+                filename = os.path.basename(parts.path)
+
+                if '-none-any' not in filename:
+                    continue
+
+            if not urls:
+                return data
+
+            requires_dist = self._get_requires_dist_from_urls(urls)
+
+            data['requires_dist'] = requires_dist
+
         return data
         return data
 
 
     def _get(self, endpoint):  # type: (str) -> Union[dict, None]
     def _get(self, endpoint):  # type: (str) -> Union[dict, None]
@@ -251,3 +309,121 @@ class PyPiRepository(Repository):
         json_data = json_response.json()
         json_data = json_response.json()
 
 
         return json_data
         return json_data
+
+    def _get_requires_dist_from_urls(self, urls
+                                     ):  # type: (dict) -> Union[list, None]
+        if 'bdist_wheel' in urls:
+            return self._get_requires_dist_from_wheel(urls['bdist_wheek'])
+
+        return self._get_requires_dist_from_sdist(urls['sdist'])
+
+    def _get_requires_dist_from_wheel(self, url
+                                      ):  # type: (str) -> Union[list, None]
+        filename = os.path.basename(urlparse.urlparse(url).path)
+
+        with temporary_directory() as temp_dir:
+            filepath = os.path.join(temp_dir, filename)
+            self._download(url, filepath)
+
+            meta = pkginfo.Wheel(filepath)
+
+        if meta.requires_dist:
+            return meta.requires_dist
+
+    def _get_requires_dist_from_sdist(self, url
+                                      ):  # type: (str) -> Union[list, None]
+        filename = os.path.basename(urlparse.urlparse(url).path)
+
+        with temporary_directory() as temp_dir:
+            filepath = Path(temp_dir) / filename
+            self._download(url, str(filepath))
+
+            meta = pkginfo.SDist(str(filepath))
+
+            if meta.requires_dist:
+                return meta.requires_dist
+
+            # Still not dependencies found
+            # So, we unpack and introspect
+            suffix = filepath.suffix
+            if suffix == '.zip':
+                tar = zipfile.ZipFile(str(filepath))
+            else:
+                if suffix == '.bz2':
+                    gz = BZ2File(str(filepath))
+                else:
+                    gz = GzipFile(str(filepath))
+
+                tar = tarfile.TarFile(str(filepath), fileobj=gz)
+
+            tar.extractall(os.path.join(temp_dir, 'unpacked'))
+
+            unpacked = Path(temp_dir) / 'unpacked'
+            sdist_dir = unpacked / Path(filename).name.rstrip('.tar.gz')
+
+            # Checking for .egg-info
+            eggs = list(sdist_dir.glob('*.egg-info'))
+            if eggs:
+                egg_info = eggs[0]
+
+                requires = egg_info / 'requires.txt'
+                if requires.exists():
+                    with requires.open() as f:
+                        return self._parse_requires(f.read())
+
+                return
+
+            # Still nothing, assume no dependencies
+            # We could probably get them by executing
+            # python setup.py egg-info but I don't feel
+            # confortable executing a file just for the sake
+            # of getting dependencies.
+            return
+
+    def _download(self, url, dest):  # type: (str, str) -> None
+        r = get(url, stream=True)
+        with open(dest, 'wb') as f:
+            for chunk in r.iter_content(chunk_size=1024):
+                if chunk:
+                    f.write(chunk)
+
+    def _parse_requires(self, requires):  # type: (str) -> Union[list, None]
+        lines = requires.split('\n')
+
+        requires_dist = []
+        in_section = False
+        current_marker = None
+        for line in lines:
+            line = line.strip()
+            if not line:
+                if in_section:
+                    in_section = False
+
+                continue
+
+            if line.startswith('['):
+                # extras or conditional dependencies
+                marker = line.lstrip('[').rstrip(']')
+                if ':' not in marker:
+                    extra, marker = marker, None
+                else:
+                    extra, marker = marker.split(':')
+
+                if extra:
+                    if marker:
+                        marker = '{} and extra == "{}"'.format(marker, extra)
+                    else:
+                        marker = 'extra == "{}"'.format(extra)
+
+                if marker:
+                    current_marker = marker
+
+                continue
+
+            if current_marker:
+                line = '{}; {}'.format(line, current_marker)
+
+            requires_dist.append(line)
+
+        if requires_dist:
+            return requires_dist

poetry/utils/helpers.py

 import re
 import re
+import shutil
+import tempfile
+
+from contextlib import contextmanager
 
 
 _canonicalize_regex = re.compile('[-_.]+')
 _canonicalize_regex = re.compile('[-_.]+')
 
 
@@ -9,3 +13,18 @@ def canonicalize_name(name):  # type: (str) -> str
 
 
 def module_name(name):  # type: (str) -> str
 def module_name(name):  # type: (str) -> str
     return canonicalize_name(name).replace('-', '_')
     return canonicalize_name(name).replace('-', '_')
+
+
+@contextmanager
+def temporary_directory(*args, **kwargs):
+    try:
+        from tempfile import TemporaryDirectory
+
+        with TemporaryDirectory(*args, **kwargs) as name:
+            yield name
+    except ImportError:
+        name = tempfile.mkdtemp(*args, **kwargs)
+
+        yield name
+
+        shutil.rmtree(name)

tests/repositories/test_pypi_repository.py

@@ -11,7 +11,8 @@ class MockRepository(PyPiRepository):
     def __init__(self):
     def __init__(self):
         super(MockRepository, self).__init__(
         super(MockRepository, self).__init__(
             url='http://foo.bar',
             url='http://foo.bar',
-            disable_cache=True
+            disable_cache=True,
+            fallback=False
         )
         )
 
 
     def _get(self, url):
     def _get(self, url):
@@ -63,3 +64,52 @@ def test_package_drops_malformed_dependencies():
     dependency_names = [d.name for d in package.requires]
     dependency_names = [d.name for d in package.requires]
 
 
     assert 'setuptools' not in dependency_names
     assert 'setuptools' not in dependency_names
+
+
+def test_parse_requires():
+    requires = """\
+jsonschema>=2.6.0.0,<3.0.0.0
+lockfile>=0.12.0.0,<0.13.0.0
+pip-tools>=1.11.0.0,<2.0.0.0
+pkginfo>=1.4.0.0,<2.0.0.0
+pyrsistent>=0.14.2.0,<0.15.0.0
+toml>=0.9.0.0,<0.10.0.0
+cleo>=0.6.0.0,<0.7.0.0
+cachy>=0.1.1.0,<0.2.0.0
+cachecontrol>=0.12.4.0,<0.13.0.0
+requests>=2.18.0.0,<3.0.0.0
+msgpack-python>=0.5.0.0,<0.6.0.0
+pyparsing>=2.2.0.0,<3.0.0.0
+requests-toolbelt>=0.8.0.0,<0.9.0.0
+
+[:(python_version >= "2.7.0.0" and python_version < "2.8.0.0") or (python_version >= "3.4.0.0" and python_version < "3.5.0.0")]
+typing>=3.6.0.0,<4.0.0.0
+
+[:python_version >= "2.7.0.0" and python_version < "2.8.0.0"]
+virtualenv>=15.2.0.0,<16.0.0.0
+pathlib2>=2.3.0.0,<3.0.0.0
+
+[:python_version >= "3.4.0.0" and python_version < "3.6.0.0"]
+zipfile36>=0.1.0.0,<0.2.0.0    
+"""
+    result = MockRepository()._parse_requires(requires)
+    expected = [
+        'jsonschema>=2.6.0.0,<3.0.0.0',
+        'lockfile>=0.12.0.0,<0.13.0.0',
+        'pip-tools>=1.11.0.0,<2.0.0.0',
+        'pkginfo>=1.4.0.0,<2.0.0.0',
+        'pyrsistent>=0.14.2.0,<0.15.0.0',
+        'toml>=0.9.0.0,<0.10.0.0',
+        'cleo>=0.6.0.0,<0.7.0.0',
+        'cachy>=0.1.1.0,<0.2.0.0',
+        'cachecontrol>=0.12.4.0,<0.13.0.0',
+        'requests>=2.18.0.0,<3.0.0.0',
+        'msgpack-python>=0.5.0.0,<0.6.0.0',
+        'pyparsing>=2.2.0.0,<3.0.0.0',
+        'requests-toolbelt>=0.8.0.0,<0.9.0.0',
+        'typing>=3.6.0.0,<4.0.0.0; (python_version >= "2.7.0.0" and python_version < "2.8.0.0") or (python_version >= "3.4.0.0" and python_version < "3.5.0.0")',
+        'virtualenv>=15.2.0.0,<16.0.0.0; python_version >= "2.7.0.0" and python_version < "2.8.0.0"',
+        'pathlib2>=2.3.0.0,<3.0.0.0; python_version >= "2.7.0.0" and python_version < "2.8.0.0"',
+        'zipfile36>=0.1.0.0,<0.2.0.0; python_version >= "3.4.0.0" and python_version < "3.6.0.0"'
+    ]
+    assert result == expected