Fix information retrieval from archives

49ed4d32 · Sébastien Eustace · 7d970886 · 49ed4d32 · 49ed4d32
Commit 49ed4d32 authored May 18, 2018 by Sébastien Eustace
Show whitespace changes
Inline Side-by-side

Showing with 82 additions and 33 deletions

poetry/puzzle/provider.py
+0 -1

poetry/repositories/pypi_repository.py
+82 -32

No files found.
--- a/poetry/puzzle/provider.py
+++ b/poetry/puzzle/provider.py
@@ -7,7 +7,6 @@ from cleo import ProgressIndicator
 from contextlib import contextmanager
 from functools import cmp_to_key
 from tempfile import mkdtemp
-from typing import Dict
 from typing import List
 from typing import Union

--- a/poetry/repositories/pypi_repository.py
+++ b/poetry/repositories/pypi_repository.py
@@ -7,6 +7,7 @@ import pkginfo
 from bz2 import BZ2File
 from gzip import GzipFile
+from typing import Dict
 from typing import List
 from typing import Union
@@ -26,6 +27,7 @@ from cachy import CacheManager
 from requests import get
 from requests import session
+from poetry.io import NullIO
 from poetry.locations import CACHE_DIR
 from poetry.packages import dependency_from_pep_508
 from poetry.packages import Package
@@ -35,6 +37,7 @@ from poetry.utils._compat import Path
 from poetry.utils._compat import to_str
 from poetry.utils.helpers import parse_requires
 from poetry.utils.helpers import temporary_directory
+from poetry.utils.venv import Venv
 from poetry.version.markers import InvalidMarker
 from .repository import Repository
@@ -140,20 +143,6 @@ class PyPiRepository(Repository):
                extras = []
            release_info = self.get_release_info(name, version)
-            if (
-                self._fallback
-                and release_info['requires_dist'] is None
-                and not release_info['requires_python']
-                and '_fallback' not in release_info
-            ):
-                # Force cache update
-                self._log(
-                    'No dependencies found, downloading archives',
-                    level='debug'
-                )
-                self._cache.forget('{}:{}'.format(name, version))
-                release_info = self.get_release_info(name, version)
            package = Package(name, version, version)
            requires_dist = release_info['requires_dist'] or []
            for req in requires_dist:
@@ -297,13 +286,15 @@ class PyPiRepository(Repository):
                and data['requires_dist'] is None
                and not data['requires_python']
        ):
+            self._log(
+                'No dependencies found, downloading archives',
+                level='debug'
+            )
            # No dependencies set (along with other information)
            # This might be due to actually no dependencies
            # or badly set metadata when uploading
            # So, we need to make sure there is actually no
            # dependencies by introspecting packages
-            data['_fallback'] = True
            urls = {}
            for url in json_data['urls']:
                # Only get sdist and universal wheels
@@ -332,9 +323,10 @@ class PyPiRepository(Repository):
            if not urls:
                return data
-            requires_dist = self._get_requires_dist_from_urls(urls)
+            info = self._get_info_from_urls(urls)
-            data['requires_dist'] = requires_dist
+            data['requires_dist'] = info['requires_dist']
+            data['requires_python'] = info['requires_python']
        return data
@@ -347,15 +339,20 @@ class PyPiRepository(Repository):
        return json_data
-    def _get_requires_dist_from_urls(self, urls
+    def _get_info_from_urls(self, urls
-                                     ):  # type: (dict) -> Union[list, None]
+                            ):  # type: (Dict[str, str]) -> Dict[str, Union[str, List, None]]
        if 'bdist_wheel' in urls:
-            return self._get_requires_dist_from_wheel(urls['bdist_wheek'])
+            return self._get_info_from_wheel(urls['bdist_wheel'])
+        return self._get_info_from_sdist(urls['sdist'])
-        return self._get_requires_dist_from_sdist(urls['sdist'])
+    def _get_info_from_wheel(self, url
+                             ):  # type: (str) -> Dict[str, Union[str, List, None]]
+        info = {
+            'requires_python': None,
+            'requires_dist': None,
+        }
-    def _get_requires_dist_from_wheel(self, url
-                                      ):  # type: (str) -> Union[list, None]
        filename = os.path.basename(urlparse.urlparse(url).path)
        with temporary_directory() as temp_dir:
@@ -367,13 +364,22 @@ class PyPiRepository(Repository):
            except ValueError:
                # Unable to determine dependencies
                # Assume none
-                return
+                return info
+        info['requires_python'] = meta.requires_python
        if meta.requires_dist:
-            return meta.requires_dist
+            info['requires_dist'] = meta.requires_dist
+        return info
+    def _get_info_from_sdist(self, url
+                             ):  # type: (str) -> Dict[str, Union[str, List, None]]
+        info = {
+            'requires_python': None,
+            'requires_dist': None,
+        }
-    def _get_requires_dist_from_sdist(self, url
-                                      ):  # type: (str) -> Union[list, None]
        filename = os.path.basename(urlparse.urlparse(url).path)
        with temporary_directory() as temp_dir:
@@ -382,9 +388,13 @@ class PyPiRepository(Repository):
            try:
                meta = pkginfo.SDist(str(filepath))
+                if meta.requires_python:
+                    info['requires_python'] = meta.requires_python
                if meta.requires_dist:
-                    return meta.requires_dist
+                    info['requires_dist'] = list(meta.requires_dist)
+                    return info
            except ValueError:
                # Unable to determine dependencies
                # We pass and go deeper
@@ -423,16 +433,56 @@ class PyPiRepository(Repository):
                requires = egg_info / 'requires.txt'
                if requires.exists():
                    with requires.open() as f:
-                        return parse_requires(f.read())
+                        info['requires_dist'] = parse_requires(f.read())
-                return
+                        return info
            # Still nothing, assume no dependencies
            # We could probably get them by executing
            # python setup.py egg-info but I don't feel
            # confortable executing a file just for the sake
            # of getting dependencies.
-            return
+            return info
+    def _inspect_sdist_with_setup(self, sdist_dir):
+        info = {
+            'requires_python': None,
+            'requires_dist': None,
+        }
+        setup = sdist_dir / 'setup.py'
+        if not setup.exists():
+            return info
+        venv = Venv.create(NullIO())
+        current_dir = os.getcwd()
+        os.chdir(sdist_dir.as_posix())
+        try:
+            venv.run(
+                'python', 'setup.py', 'egg_info'
+            )
+            egg_info = list(sdist_dir.glob('**/*.egg-info'))[0]
+            meta = pkginfo.UnpackedSDist(str(egg_info))
+            if meta.requires_python:
+                info['requires_python'] = meta.requires_python
+            if meta.requires_dist:
+                info['requires_dist'] = list(meta.requires_dist)
+            else:
+                requires = egg_info / 'requires.txt'
+                if requires.exists():
+                    with requires.open() as f:
+                        info['requires_dist'] = parse_requires(f.read())
+        except Exception:
+            pass
+        os.chdir(current_dir)
+        return info
    def _download(self, url, dest):  # type: (str, str) -> None
        r = get(url, stream=True)