cleanup: reuse parse_requires and download logic

8491b74d · Arun Babu Neelicattu · f020c987 · 8491b74d · 8491b74d · 8491b74d
Commit 8491b74d authored Apr 13, 2020 by Arun Babu Neelicattu
7 changed files
--- a/poetry/puzzle/provider.py
+++ b/poetry/puzzle/provider.py
@@ -22,6 +22,7 @@ from poetry.core.packages import URLDependency
 from poetry.core.packages import VCSDependency
 from poetry.core.packages import dependency_from_pep_508
 from poetry.core.packages.utils.utils import get_python_constraint_from_marker
+from poetry.core.utils.helpers import parse_requires
 from poetry.core.vcs.git import Git
 from poetry.core.version.markers import MarkerUnion
 from poetry.factory import Factory
@@ -39,7 +40,7 @@ from poetry.utils._compat import urlparse
 from poetry.utils.env import EnvCommandError
 from poetry.utils.env import EnvManager
 from poetry.utils.env import VirtualEnv
-from poetry.utils.helpers import parse_requires
+from poetry.utils.helpers import download_file
 from poetry.utils.helpers import safe_rmtree
 from poetry.utils.helpers import temporary_directory
 from poetry.utils.inspector import Inspector
@@ -465,7 +466,7 @@ class Provider:
        with temporary_directory() as temp_dir:
            temp_dir = Path(temp_dir)
            file_name = os.path.basename(urlparse.urlparse(url).path)
-            Inspector().download(url, temp_dir / file_name)
+            download_file(url, temp_dir / file_name)
            package = cls.get_package_from_file(temp_dir / file_name)

--- a/poetry/repositories/legacy_repository.py
+++ b/poetry/repositories/legacy_repository.py
@@ -404,16 +404,9 @@ class LegacyRepository(PyPiRepository):
        return data
-    def _download(self, url, dest):  # type: (str, str) -> None
-        r = self._session.get(url, stream=True)
-        with open(dest, "wb") as f:
-            for chunk in r.iter_content(chunk_size=1024):
-                if chunk:
-                    f.write(chunk)
    def _get(self, endpoint):  # type: (str) -> Union[Page, None]
        url = self._url + endpoint
-        response = self._session.get(url)
+        response = self.session.get(url)
        if response.status_code == 404:
            return

--- a/poetry/repositories/pypi_repository.py
+++ b/poetry/repositories/pypi_repository.py
@@ -6,14 +6,13 @@ from typing import Dict
 from typing import List
 from typing import Union
+import requests
 from cachecontrol import CacheControl
 from cachecontrol.caches.file_cache import FileCache
 from cachecontrol.controller import logger as cache_control_logger
 from cachy import CacheManager
 from html5lib.html5parser import parse
-from requests import get
-from requests import session
-from requests.exceptions import TooManyRedirects
 from poetry.core.packages import Package
 from poetry.core.packages import dependency_from_pep_508
@@ -27,6 +26,7 @@ from poetry.core.version.markers import parse_marker
 from poetry.locations import REPOSITORY_CACHE_DIR
 from poetry.utils._compat import Path
 from poetry.utils._compat import to_str
+from poetry.utils.helpers import download_file
 from poetry.utils.helpers import temporary_directory
 from poetry.utils.inspector import Inspector
 from poetry.utils.patterns import wheel_file_re
@@ -70,11 +70,17 @@ class PyPiRepository(RemoteRepository):
        )
        self._cache_control_cache = FileCache(str(release_cache_dir / "_http"))
-        self._session = CacheControl(session(), cache=self._cache_control_cache)
+        self._session = CacheControl(
+            requests.session(), cache=self._cache_control_cache
+        )
        self._inspector = Inspector()
        self._name = "PyPI"
+    @property
+    def session(self):
+        return self._session
    def find_packages(
        self,
        name,  # type: str
@@ -216,7 +222,7 @@ class PyPiRepository(RemoteRepository):
        search = {"q": query}
-        response = session().get(self._base_url + "search", params=search)
+        response = requests.session().get(self._base_url + "search", params=search)
        content = parse(response.content, namespaceHTMLElements=False)
        for result in content.findall(".//*[@class='package-snippet']"):
            name = result.find("h3/*[@class='package-snippet__name']").text
@@ -353,12 +359,12 @@ class PyPiRepository(RemoteRepository):
    def _get(self, endpoint):  # type: (str) -> Union[dict, None]
        try:
-            json_response = self._session.get(self._base_url + endpoint)
+            json_response = self.session.get(self._base_url + endpoint)
-        except TooManyRedirects:
+        except requests.exceptions.TooManyRedirects:
            # Cache control redirect loop.
            # We try to remove the cache and try again
            self._cache_control_cache.delete(self._base_url + endpoint)
-            json_response = self._session.get(self._base_url + endpoint)
+            json_response = self.session.get(self._base_url + endpoint)
        if json_response.status_code == 404:
            return None
@@ -494,13 +500,7 @@ class PyPiRepository(RemoteRepository):
            return self._inspector.inspect_sdist(filepath)
    def _download(self, url, dest):  # type: (str, str) -> None
-        r = get(url, stream=True)
+        return download_file(url, dest, session=self.session)
-        r.raise_for_status()
-        with open(dest, "wb") as f:
-            for chunk in r.iter_content(chunk_size=1024):
-                if chunk:
-                    f.write(chunk)
    def _log(self, msg, level="info"):
        getattr(logger, level)("<debug>{}:</debug> {}".format(self._name, msg))
--- a/poetry/utils/helpers.py
+++ b/poetry/utils/helpers.py
@@ -5,9 +5,10 @@ import stat
 import tempfile
 from contextlib import contextmanager
-from typing import List
 from typing import Optional
+import requests
 from poetry.config.config import Config
 from poetry.core.version import Version
 from poetry.utils._compat import Path
@@ -49,47 +50,6 @@ def temporary_directory(*args, **kwargs):
        shutil.rmtree(name)
-def parse_requires(requires):  # type: (str) -> List[str]
-    lines = requires.split("\n")
-    requires_dist = []
-    in_section = False
-    current_marker = None
-    for line in lines:
-        line = line.strip()
-        if not line:
-            if in_section:
-                in_section = False
-            continue
-        if line.startswith("["):
-            # extras or conditional dependencies
-            marker = line.lstrip("[").rstrip("]")
-            if ":" not in marker:
-                extra, marker = marker, None
-            else:
-                extra, marker = marker.split(":")
-            if extra:
-                if marker:
-                    marker = '{} and extra == "{}"'.format(marker, extra)
-                else:
-                    marker = 'extra == "{}"'.format(extra)
-            if marker:
-                current_marker = marker
-            continue
-        if current_marker:
-            line = "{}; {}".format(line, current_marker)
-        requires_dist.append(line)
-    return requires_dist
 def get_cert(config, repository_name):  # type: (Config, str) -> Optional[Path]
    cert = config.get("certificates.{}.cert".format(repository_name))
    if cert:
@@ -127,3 +87,17 @@ def merge_dicts(d1, d2):
            merge_dicts(d1[k], d2[k])
        else:
            d1[k] = d2[k]
+def download_file(
+    url, dest, session=None, chunk_size=1024
+):  # type: (str, str, Optional[requests.Session], int) -> None
+    get = requests.get if not session else session.get
+    with get(url, stream=True) as response:
+        response.raise_for_status()
+        with open(dest, "wb") as f:
+            for chunk in response.iter_content(chunk_size=chunk_size):
+                if chunk:
+                    f.write(chunk)
--- a/tests/conftest.py
+++ b/tests/conftest.py
@@ -72,6 +72,9 @@ def config(config_source, auth_config_source, mocker):
 def download_mock(mocker):
    # Patch download to not download anything but to just copy from fixtures
    mocker.patch("poetry.utils.inspector.Inspector.download", new=mock_download)
+    mocker.patch("poetry.utils.helpers.download_file", new=mock_download)
+    mocker.patch("poetry.puzzle.provider.download_file", new=mock_download)
+    mocker.patch("poetry.repositories.pypi_repository.download_file", new=mock_download)
 @pytest.fixture

--- a/tests/helpers.py
+++ b/tests/helpers.py
@@ -81,10 +81,10 @@ def mock_clone(_, source, dest):
    copy_or_symlink(folder, dest)
-def mock_download(self, url, dest):
+def mock_download(url, dest, **__):
    parts = urlparse.urlparse(url)
    fixtures = Path(__file__).parent / "fixtures"
    fixture = fixtures / parts.path.lstrip("/")
-    copy_or_symlink(fixture, dest)
+    copy_or_symlink(fixture, Path(dest))
--- a/tests/utils/test_helpers.py
+++ b/tests/utils/test_helpers.py
+from poetry.core.utils.helpers import parse_requires
 from poetry.utils._compat import Path
 from poetry.utils.helpers import get_cert
 from poetry.utils.helpers import get_client_cert
-from poetry.utils.helpers import parse_requires
 def test_parse_requires():