Improve support for private repositories.

No longer use pip-tools.

CHANGELOG.md

@@ -14,6 +14,7 @@
 
 - Improved the `show` command to make it easier to check if packages are properly installed.
 - The `script` command has been deprecated, use `run` instead.
+- Improved support for private repositories.
 - Expanded version constraints now keep the original version's precision.
 
 ### Fixed

poetry/packages/__init__.py

@@ -21,6 +21,14 @@ from .vcs_dependency import VCSDependency
 
 
 def dependency_from_pep_508(name):
+    # Removing comments
+    parts = name.split('#', 1)
+    name = parts[0].strip()
+    if len(parts) > 1:
+        rest = parts[1]
+        if ';' in rest:
+            name += ';' + rest.split(';', 1)[1]
+
     req = Requirement(name)
 
     if req.marker:

poetry/packages/constraints/generic_constraint.py

@@ -58,13 +58,8 @@ class GenericConstraint(BaseConstraint):
         return self._version
 
     def matches(self, provider):
-        if not isinstance(provider, (GenericConstraint, EmptyConstraint)):
-            raise ValueError(
-                'Generic constraints can only be compared with each other'
-            )
-
-        if isinstance(provider, EmptyConstraint):
-            return True
+        if not isinstance(provider, GenericConstraint):
+            return provider.matches(self)
 
         is_equal_op = self.OP_EQ is self._operator
         is_non_equal_op = self.OP_NE is self._operator

poetry/repositories/legacy_repository.py

-from pip._vendor.pkg_resources import RequirementParseError
+import cgi
+import re
 
 try:
-    from pip._internal.exceptions import InstallationError
-    from pip._internal.req import InstallRequirement
+    import urllib.parse as urlparse
 except ImportError:
-    from pip.exceptions import InstallationError
-    from pip.req import InstallRequirement
+    import urlparse
 
-from piptools.cache import DependencyCache
-from piptools.repositories import PyPIRepository
-from piptools.resolver import Resolver
-from piptools.scripts.compile import get_pip_command
+try:
+    from html import unescape
+except ImportError:
+    from html.parser import HTMLParser
+    unescape = HTMLParser().unescape
+
+from typing import Generator
+from typing import Union
 
+import html5lib
+import requests
+
+from cachecontrol import CacheControl
+from cachecontrol.caches.file_cache import FileCache
 from cachy import CacheManager
 
 import poetry.packages
 
 from poetry.locations import CACHE_DIR
+from poetry.masonry.publishing.uploader import wheel_file_re
 from poetry.packages import Package
 from poetry.packages import dependency_from_pep_508
+from poetry.packages.utils.link import Link
 from poetry.semver import parse_constraint
 from poetry.semver import Version
 from poetry.semver import VersionConstraint
 from poetry.utils._compat import Path
+from poetry.utils.helpers import canonicalize_name
 from poetry.version.markers import InvalidMarker
 
 from .pypi_repository import PyPiRepository
 
 
+class Page:
+
+    VERSION_REGEX = re.compile('(?i)([a-z0-9_\-.]+?)-(?=\d)([a-z0-9_.!+-]+)')
+
+    def __init__(self, url, content, headers):
+        self._url = url
+        encoding = None
+        if headers and "Content-Type" in headers:
+            content_type, params = cgi.parse_header(headers["Content-Type"])
+
+            if "charset" in params:
+                encoding = params['charset']
+
+        self._content = content
+        self._parsed = html5lib.parse(
+            content,
+            transport_encoding=encoding,
+            namespaceHTMLElements=False,
+        )
+
+    @property
+    def versions(self):  # type: () -> Generator[Version]
+        seen = set()
+        for link in self.links:
+            version = self.link_version(link)
+
+            if not version:
+                continue
+
+            if version in seen:
+                continue
+
+            seen.add(version)
+
+            yield version
+
+    @property
+    def links(self):  # type: () -> Generator[Link]
+        for anchor in self._parsed.findall(".//a"):
+            if anchor.get("href"):
+                href = anchor.get("href")
+                url = self.clean_link(
+                    urlparse.urljoin(self._url, href)
+                )
+                pyrequire = anchor.get('data-requires-python')
+                pyrequire = unescape(pyrequire) if pyrequire else None
+
+                yield Link(url, self, requires_python=pyrequire)
+
+    def links_for_version(self, version):  # type: (Version) -> Generator[Link]
+        for link in self.links:
+            if self.link_version(link) == version:
+                yield link
+
+    def link_version(self, link):  # type: (Link) -> Union[Version, None]
+        m = wheel_file_re.match(link.filename)
+        if m:
+            version = m.group('ver')
+        else:
+            info, ext = link.splitext()
+            match = self.VERSION_REGEX.match(info)
+            if not match:
+                return
+
+            version = match.group(2)
+
+        try:
+            version = Version.parse(version)
+        except ValueError:
+            return
+
+        return version
+
+    _clean_re = re.compile(r'[^a-z0-9$&+,/:;=?@.#%_\\|-]', re.I)
+
+    def clean_link(self, url):
+        """Makes sure a link is fully encoded.  That is, if a ' ' shows up in
+        the link, it will be rewritten to %20 (while not over-quoting
+        % or other characters)."""
+        return self._clean_re.sub(
+            lambda match: '%%%2x' % ord(match.group(0)), url)
+
+
 class LegacyRepository(PyPiRepository):
 
     def __init__(self, name, url):
@@ -36,11 +130,7 @@ class LegacyRepository(PyPiRepository):
 
         self._packages = []
         self._name = name
-        self._url = url
-        command = get_pip_command()
-        opts, _ = command.parse_args([])
-        self._session = command._build_session(opts)
-        self._repository = PyPIRepository(opts, self._session)
+        self._url = url.rstrip('/')
         self._cache_dir = Path(CACHE_DIR) / 'cache' / 'repositories' / name
 
         self._cache = CacheManager({
@@ -60,6 +150,11 @@ class LegacyRepository(PyPiRepository):
             }
         })
 
+        self._session = CacheControl(
+            requests.session(),
+            cache=FileCache(str(self._cache_dir / '_http'))
+        )
+
     @property
     def name(self):
         return self._name
@@ -80,18 +175,12 @@ class LegacyRepository(PyPiRepository):
         if self._cache.store('matches').has(key):
             versions = self._cache.store('matches').get(key)
         else:
-            candidates = [str(c.version) for c in self._repository.find_all_candidates(name)]
+            page = self._get('/{}'.format(canonicalize_name(name).replace('.', '-')))
+            if page is None:
+                raise ValueError('No package named "{}"'.format(name))
 
             versions = []
-            for version in candidates:
-                if version in versions:
-                    continue
-
-                try:
-                    version = Version.parse(version)
-                except ValueError:
-                    continue
-
+            for version in page.versions:
                 if (
                     not constraint
                     or (constraint and constraint.allows(version))
@@ -101,7 +190,11 @@ class LegacyRepository(PyPiRepository):
             self._cache.store('matches').put(key, versions, 5)
 
         for version in versions:
-            packages.append(Package(name, version, extras=extras))
+            package = Package(name, version)
+            if extras is not None:
+                package.requires_extras = extras
+
+            packages.append(package)
 
         return packages
 
@@ -129,8 +222,10 @@ class LegacyRepository(PyPiRepository):
                 extras = []
 
             release_info = self.get_release_info(name, version)
+
             package = poetry.packages.Package(name, version, version)
-            for req in release_info['requires_dist']:
+            requires_dist = release_info['requires_dist'] or []
+            for req in requires_dist:
                 try:
                     dependency = dependency_from_pep_508(req)
                 except InvalidMarker:
@@ -181,43 +276,63 @@ class LegacyRepository(PyPiRepository):
         )
 
     def _get_release_info(self, name, version):  # type: (str, str) -> dict
-        ireq = InstallRequirement.from_line('{}=={}'.format(name, version))
-        resolver = Resolver(
-            [ireq], self._repository,
-            cache=DependencyCache(self._cache_dir.as_posix())
-        )
-        try:
-            requirements = list(resolver._iter_dependencies(ireq))
-        except (InstallationError, RequirementParseError):
-            # setup.py egg-info error most likely
-            # So we assume no dependencies
-            requirements = []
-
-        requires = []
-        for dep in requirements:
-            constraint = str(dep.req.specifier)
-            require = dep.name
-            if constraint:
-                require += ' ({})'.format(constraint)
-
-            requires.append(require)
-
-        try:
-            hashes = resolver.resolve_hashes([ireq])[ireq]
-        except IndexError:
-            # Sometimes pip-tools fails when getting indices
-            hashes = []
-
-        hashes = [h.split(':')[1] for h in hashes]
+        page = self._get('/{}'.format(canonicalize_name(name).replace('.', '-')))
+        if page is None:
+            raise ValueError('No package named "{}"'.format(name))
 
         data = {
             'name': name,
             'version': version,
             'summary': '',
-            'requires_dist': requires,
-            'digests': hashes
+            'requires_dist': [],
+            'requires_python': [],
+            'digests': []
         }
 
-        resolver.repository.freshen_build_caches()
+        links = list(page.links_for_version(Version.parse(version)))
+        urls = {}
+        hashes = []
+        default_link = links[0]
+        for link in links:
+            if link.is_wheel:
+                urls['bdist_wheel'] = link.url
+            elif link.filename.endswith('.tar.gz'):
+                urls['sdist'] = link.url
+            elif link.filename.endswith(('.zip', '.bz2')) and 'sdist' not in urls:
+                urls['sdist'] = link.url
+
+            hash = link.hash
+            if link.hash_name == 'sha256':
+                hashes.append(hash)
+
+        data['digests'] = hashes
+
+        if not urls:
+            if default_link.is_wheel:
+                m = wheel_file_re.match(default_link.filename)
+                python = m.group('pyver')
+                platform = m.group('plat')
+                if python == 'py2.py3' and platform == 'any':
+                    urls['bdist_wheel'] = default_link.url
+            elif default_link.filename.endswith('.tar.gz'):
+                urls['sdist'] = default_link.url
+            elif default_link.filename.endswith(('.zip', '.bz2')) and 'sdist' not in urls:
+                urls['sdist'] = default_link.url
+            else:
+                return data
+
+        info = self._get_info_from_urls(urls)
+
+        data['summary'] = info['summary']
+        data['requires_dist'] = info['requires_dist']
+        data['requires_python'] = info['requires_python']
 
         return data
+
+    def _get(self, endpoint):  # type: (str) -> Union[Page, None]
+        url = self._url + endpoint
+        response = self._session.get(url)
+        if response.status_code == 404:
+            return
+
+        return Page(url, response.content, response.headers)

poetry/repositories/pypi_repository.py

@@ -286,7 +286,6 @@ class PyPiRepository(Repository):
         if (
                 self._fallback
                 and data['requires_dist'] is None
-                and not data['requires_python']
         ):
             self._log(
                 'No dependencies found, downloading archives',
@@ -328,7 +327,9 @@ class PyPiRepository(Repository):
             info = self._get_info_from_urls(urls)
 
             data['requires_dist'] = info['requires_dist']
-            data['requires_python'] = info['requires_python']
+
+            if not data['requires_python']:
+                data['requires_python'] = info['requires_python']
 
         return data
 
@@ -351,6 +352,7 @@ class PyPiRepository(Repository):
     def _get_info_from_wheel(self, url
                              ):  # type: (str) -> Dict[str, Union[str, List, None]]
         info = {
+            'summary': '',
             'requires_python': None,
             'requires_dist': None,
         }
@@ -368,6 +370,9 @@ class PyPiRepository(Repository):
                 # Assume none
                 return info
 
+        if meta.summary:
+            info['summary'] = meta.summary or ''
+
         info['requires_python'] = meta.requires_python
 
         if meta.requires_dist:
@@ -378,6 +383,7 @@ class PyPiRepository(Repository):
     def _get_info_from_sdist(self, url
                              ):  # type: (str) -> Dict[str, Union[str, List, None]]
         info = {
+            'summary': '',
             'requires_python': None,
             'requires_dist': None,
         }
@@ -390,6 +396,9 @@ class PyPiRepository(Repository):
 
             try:
                 meta = pkginfo.SDist(str(filepath))
+                if meta.summary:
+                    info['summary'] = meta.summary
+
                 if meta.requires_python:
                     info['requires_python'] = meta.requires_python
 
@@ -427,7 +436,7 @@ class PyPiRepository(Repository):
             unpacked = Path(temp_dir) / 'unpacked'
             sdist_dir = unpacked / Path(filename).name.rstrip('.tar.gz')
 
-            # Checking for .egg-info
+            # Checking for .egg-info at root
             eggs = list(sdist_dir.glob('*.egg-info'))
             if eggs:
                 egg_info = eggs[0]
@@ -439,6 +448,18 @@ class PyPiRepository(Repository):
 
                         return info
 
+            # Searching for .egg-info in sub directories
+            eggs = list(sdist_dir.glob('**/*.egg-info'))
+            if eggs:
+                egg_info = eggs[0]
+
+                requires = egg_info / 'requires.txt'
+                if requires.exists():
+                    with requires.open() as f:
+                        info['requires_dist'] = parse_requires(f.read())
+
+                        return info
+
             # Still nothing, assume no dependencies
             # We could probably get them by executing
             # python setup.py egg-info but I don't feel

poetry/semver/version.py

@@ -417,5 +417,5 @@ class Version(VersionRange):
             (self.major,
              self.minor,
              self.patch,
-             '.'.join(self.prerelease),
-             '.'.join(self.build)))
+             '.'.join(str(p) for p in self.prerelease),
+             '.'.join(str(p) for p in self.build)))

pyproject.toml

@@ -27,13 +27,13 @@ cleo = "^0.6.6"
 requests = "^2.18"
 toml = "^0.9"
 cachy = "^0.2"
-pip-tools = "^2.0"
 requests-toolbelt = "^0.8.0"
 jsonschema = "^2.6"
 pyrsistent = "^0.14.2"
 pyparsing = "^2.2"
 cachecontrol = { version = "^0.12.4", extras = ["filecache"] }
 pkginfo = "^1.4"
+html5lib = "^1.0"
 
 # The typing module is not in the stdlib in Python 2.7 and 3.4
 typing = { version = "^3.6", python = "~2.7 || ~3.4" }