cleanup: reuse parse_requires and download logic

poetry/puzzle/provider.py

@@ -22,6 +22,7 @@ from poetry.core.packages import URLDependency
 from poetry.core.packages import VCSDependency
 from poetry.core.packages import dependency_from_pep_508
 from poetry.core.packages.utils.utils import get_python_constraint_from_marker
+from poetry.core.utils.helpers import parse_requires
 from poetry.core.vcs.git import Git
 from poetry.core.version.markers import MarkerUnion
 from poetry.factory import Factory
@@ -39,7 +40,7 @@ from poetry.utils._compat import urlparse
 from poetry.utils.env import EnvCommandError
 from poetry.utils.env import EnvManager
 from poetry.utils.env import VirtualEnv
-from poetry.utils.helpers import parse_requires
+from poetry.utils.helpers import download_file
 from poetry.utils.helpers import safe_rmtree
 from poetry.utils.helpers import temporary_directory
 from poetry.utils.inspector import Inspector
@@ -465,7 +466,7 @@ class Provider:
         with temporary_directory() as temp_dir:
             temp_dir = Path(temp_dir)
             file_name = os.path.basename(urlparse.urlparse(url).path)
-            Inspector().download(url, temp_dir / file_name)
+            download_file(url, temp_dir / file_name)
 
             package = cls.get_package_from_file(temp_dir / file_name)
 

poetry/repositories/legacy_repository.py

@@ -404,16 +404,9 @@ class LegacyRepository(PyPiRepository):
 
         return data
 
-    def _download(self, url, dest):  # type: (str, str) -> None
-        r = self._session.get(url, stream=True)
-        with open(dest, "wb") as f:
-            for chunk in r.iter_content(chunk_size=1024):
-                if chunk:
-                    f.write(chunk)
-
     def _get(self, endpoint):  # type: (str) -> Union[Page, None]
         url = self._url + endpoint
-        response = self._session.get(url)
+        response = self.session.get(url)
         if response.status_code == 404:
             return
 

poetry/repositories/pypi_repository.py

@@ -6,14 +6,13 @@ from typing import Dict
 from typing import List
 from typing import Union
 
+import requests
+
 from cachecontrol import CacheControl
 from cachecontrol.caches.file_cache import FileCache
 from cachecontrol.controller import logger as cache_control_logger
 from cachy import CacheManager
 from html5lib.html5parser import parse
-from requests import get
-from requests import session
-from requests.exceptions import TooManyRedirects
 
 from poetry.core.packages import Package
 from poetry.core.packages import dependency_from_pep_508
@@ -27,6 +26,7 @@ from poetry.core.version.markers import parse_marker
 from poetry.locations import REPOSITORY_CACHE_DIR
 from poetry.utils._compat import Path
 from poetry.utils._compat import to_str
+from poetry.utils.helpers import download_file
 from poetry.utils.helpers import temporary_directory
 from poetry.utils.inspector import Inspector
 from poetry.utils.patterns import wheel_file_re
@@ -70,11 +70,17 @@ class PyPiRepository(RemoteRepository):
         )
 
         self._cache_control_cache = FileCache(str(release_cache_dir / "_http"))
-        self._session = CacheControl(session(), cache=self._cache_control_cache)
+        self._session = CacheControl(
+            requests.session(), cache=self._cache_control_cache
+        )
         self._inspector = Inspector()
 
         self._name = "PyPI"
 
+    @property
+    def session(self):
+        return self._session
+
     def find_packages(
         self,
         name,  # type: str
@@ -216,7 +222,7 @@ class PyPiRepository(RemoteRepository):
 
         search = {"q": query}
 
-        response = session().get(self._base_url + "search", params=search)
+        response = requests.session().get(self._base_url + "search", params=search)
         content = parse(response.content, namespaceHTMLElements=False)
         for result in content.findall(".//*[@class='package-snippet']"):
             name = result.find("h3/*[@class='package-snippet__name']").text
@@ -353,12 +359,12 @@ class PyPiRepository(RemoteRepository):
 
     def _get(self, endpoint):  # type: (str) -> Union[dict, None]
         try:
-            json_response = self._session.get(self._base_url + endpoint)
-        except TooManyRedirects:
+            json_response = self.session.get(self._base_url + endpoint)
+        except requests.exceptions.TooManyRedirects:
             # Cache control redirect loop.
             # We try to remove the cache and try again
             self._cache_control_cache.delete(self._base_url + endpoint)
-            json_response = self._session.get(self._base_url + endpoint)
+            json_response = self.session.get(self._base_url + endpoint)
 
         if json_response.status_code == 404:
             return None
@@ -494,13 +500,7 @@ class PyPiRepository(RemoteRepository):
             return self._inspector.inspect_sdist(filepath)
 
     def _download(self, url, dest):  # type: (str, str) -> None
-        r = get(url, stream=True)
-        r.raise_for_status()
-
-        with open(dest, "wb") as f:
-            for chunk in r.iter_content(chunk_size=1024):
-                if chunk:
-                    f.write(chunk)
+        return download_file(url, dest, session=self.session)
 
     def _log(self, msg, level="info"):
         getattr(logger, level)("<debug>{}:</debug> {}".format(self._name, msg))

poetry/utils/helpers.py

@@ -5,9 +5,10 @@ import stat
 import tempfile
 
 from contextlib import contextmanager
-from typing import List
 from typing import Optional
 
+import requests
+
 from poetry.config.config import Config
 from poetry.core.version import Version
 from poetry.utils._compat import Path
@@ -49,47 +50,6 @@ def temporary_directory(*args, **kwargs):
         shutil.rmtree(name)
 
 
-def parse_requires(requires):  # type: (str) -> List[str]
-    lines = requires.split("\n")
-
-    requires_dist = []
-    in_section = False
-    current_marker = None
-    for line in lines:
-        line = line.strip()
-        if not line:
-            if in_section:
-                in_section = False
-
-            continue
-
-        if line.startswith("["):
-            # extras or conditional dependencies
-            marker = line.lstrip("[").rstrip("]")
-            if ":" not in marker:
-                extra, marker = marker, None
-            else:
-                extra, marker = marker.split(":")
-
-            if extra:
-                if marker:
-                    marker = '{} and extra == "{}"'.format(marker, extra)
-                else:
-                    marker = 'extra == "{}"'.format(extra)
-
-            if marker:
-                current_marker = marker
-
-            continue
-
-        if current_marker:
-            line = "{}; {}".format(line, current_marker)
-
-        requires_dist.append(line)
-
-    return requires_dist
-
-
 def get_cert(config, repository_name):  # type: (Config, str) -> Optional[Path]
     cert = config.get("certificates.{}.cert".format(repository_name))
     if cert:
@@ -127,3 +87,17 @@ def merge_dicts(d1, d2):
             merge_dicts(d1[k], d2[k])
         else:
             d1[k] = d2[k]
+
+
+def download_file(
+    url, dest, session=None, chunk_size=1024
+):  # type: (str, str, Optional[requests.Session], int) -> None
+    get = requests.get if not session else session.get
+
+    with get(url, stream=True) as response:
+        response.raise_for_status()
+
+        with open(dest, "wb") as f:
+            for chunk in response.iter_content(chunk_size=chunk_size):
+                if chunk:
+                    f.write(chunk)

tests/conftest.py

@@ -72,6 +72,9 @@ def config(config_source, auth_config_source, mocker):
 def download_mock(mocker):
     # Patch download to not download anything but to just copy from fixtures
     mocker.patch("poetry.utils.inspector.Inspector.download", new=mock_download)
+    mocker.patch("poetry.utils.helpers.download_file", new=mock_download)
+    mocker.patch("poetry.puzzle.provider.download_file", new=mock_download)
+    mocker.patch("poetry.repositories.pypi_repository.download_file", new=mock_download)
 
 
 @pytest.fixture

tests/helpers.py

@@ -81,10 +81,10 @@ def mock_clone(_, source, dest):
     copy_or_symlink(folder, dest)
 
 
-def mock_download(self, url, dest):
+def mock_download(url, dest, **__):
     parts = urlparse.urlparse(url)
 
     fixtures = Path(__file__).parent / "fixtures"
     fixture = fixtures / parts.path.lstrip("/")
 
-    copy_or_symlink(fixture, dest)
+    copy_or_symlink(fixture, Path(dest))

tests/utils/test_helpers.py

+from poetry.core.utils.helpers import parse_requires
 from poetry.utils._compat import Path
 from poetry.utils.helpers import get_cert
 from poetry.utils.helpers import get_client_cert
-from poetry.utils.helpers import parse_requires
 
 
 def test_parse_requires():