Add support for private repositories

poetry.toml

@@ -27,6 +27,7 @@ cleo = "^0.6"
 requests = "^2.18"
 toml = "^0.9"
 cachy = "^0.1.0"
+pip-tools = "^1.11"
 
 [dev-dependencies]
 pytest = "~3.4"

poetry/console/commands/add.py

@@ -92,7 +92,7 @@ If you do not specify a version constraint, poetry will choose a suitable one ba
             self.output,
             self.poetry.package,
             self.poetry.locker,
-            self.poetry.repository
+            self.poetry.pool
         )
 
         installer.update(True)
@@ -135,7 +135,7 @@ If you do not specify a version constraint, poetry will choose a suitable one ba
                                        name,
                                        required_version=None
                                        ) -> Tuple[str, str]:
-        selector = VersionSelector(self.poetry.repository)
+        selector = VersionSelector(self.poetry.pool)
         package = selector.find_best_candidate(name, required_version)
 
         if not package:

poetry/console/commands/install.py

@@ -27,7 +27,7 @@ exist it will look for <comment>poetry.toml</> and do the same.
             self.output,
             self.poetry.package,
             self.poetry.locker,
-            PyPiRepository()
+            self.poetry.pool
         )
 
         installer.dev_mode(not self.option('no-dev'))

poetry/console/commands/lock.py

@@ -23,7 +23,7 @@ the current directory, processes it, and locks the depdencies in the <comment>po
             self.output,
             self.poetry.package,
             self.poetry.locker,
-            PyPiRepository()
+            self.poetry.pool
         )
 
         installer.update(True)

poetry/console/commands/remove.py

@@ -83,7 +83,7 @@ list of installed packages
             self.output,
             self.poetry.package,
             self.poetry.locker,
-            self.poetry.repository
+            self.poetry.pool
         )
 
         installer.dry_run(self.option('dry-run'))

poetry/console/commands/update.py

@@ -22,7 +22,7 @@ class UpdateCommand(Command):
             self.output,
             self.poetry.package,
             self.poetry.locker,
-            self.poetry.repository
+            self.poetry.pool
         )
 
         if packages:

poetry/installation/installer.py

@@ -8,6 +8,7 @@ from poetry.puzzle.operations import Install
 from poetry.puzzle.operations import Uninstall
 from poetry.puzzle.operations import Update
 from poetry.puzzle.operations.operation import Operation
+from poetry.repositories import Pool
 from poetry.repositories import Repository
 from poetry.repositories.installed_repository import InstalledRepository
 
@@ -21,11 +22,11 @@ class Installer:
                  io,
                  package: Package,
                  locker: Locker,
-                 repository: Repository):
+                 pool: Pool):
         self._io = io
         self._package = package
         self._locker = locker
-        self._repository = repository
+        self._pool = pool
 
         self._dry_run = False
         self._update = False
@@ -130,7 +131,7 @@ class Installer:
             request = self._package.requires
             request += self._package.dev_requires
 
-            ops = solver.solve(request, self._repository, fixed=fixed)
+            ops = solver.solve(request, self._pool, fixed=fixed)
         else:
             self._io.writeln('<info>Installing dependencies from lock file</>')
             # If we are installing from lock

poetry/installation/pip_installer.py

@@ -10,9 +10,17 @@ class PipInstaller(BaseInstaller):
         self._io = io
 
     def install(self, package):
-        self.run('install', self.requirement(package), '--no-deps')
+        args = ['install', self.requirement(package), '--no-deps']
+        if package.source_type == 'legacy' and package.source_url:
+            args += ['--index-url', package.source_url]
+
+        self.run(*args)
 
     def update(self, source, target):
+        args = ['install', self.requirement(target), '--no-deps', '-U']
+        if target.source_type == 'legacy' and target.source_url:
+            args += ['--index-url', target.source_url]
+
         self.run('install', self.requirement(target), '--no-deps', '-U')
 
     def remove(self, package):

poetry/packages/locker.py

@@ -18,6 +18,7 @@ class Locker:
         'python_versions',
         'dependencies',
         'dev-dependencies',
+        'source',
     ]
 
     def __init__(self, lock: Path, original: Path):

poetry/poetry.py

@@ -2,6 +2,7 @@ from pathlib import Path
 
 from .packages import Locker
 from .packages import Package
+from .repositories import Pool
 from .repositories.pypi_repository import PyPiRepository
 from .semver.helpers import normalize_version
 from .utils.toml_file import TomlFile
@@ -18,7 +19,14 @@ class Poetry:
         self._package = package
         self._config = config
         self._locker = locker
-        self._repository = PyPiRepository()
+
+        # Configure sources
+        self._pool = Pool()
+        for source in self._config.get('source', []):
+            self._pool.configure(source)
+
+        # Always put PyPI last to prefere private repositories
+        self._pool.add_repository(PyPiRepository())
 
     @property
     def package(self) -> Package:
@@ -33,12 +41,8 @@ class Poetry:
         return self._locker
 
     @property
-    def repository(self) -> PyPiRepository:
-        return self._repository
-
-    @property
-    def installer(self):
-        return self._instal
+    def pool(self) -> Pool:
+        return self._pool
 
     @classmethod
     def create(cls, cwd) -> 'Poetry':

poetry/puzzle/provider.py

@@ -17,7 +17,7 @@ from poetry.packages import Dependency
 from poetry.packages import Package
 from poetry.packages import VCSDependency
 
-from poetry.repositories.repository import Repository
+from poetry.repositories import Pool
 
 from poetry.semver import less_than
 from poetry.semver.constraints import Constraint
@@ -31,12 +31,12 @@ class Provider(SpecificationProvider):
 
     UNSAFE_PACKAGES = {'setuptools', 'distribute', 'pip'}
 
-    def __init__(self, repository: Repository):
-        self._repository = repository
+    def __init__(self, pool: Pool):
+        self._pool = pool
 
     @property
-    def repository(self) -> Repository:
-        return self._repository
+    def pool(self) -> Pool:
+        return self._pool
 
     @property
     def name_for_explicit_dependency_source(self) -> str:
@@ -62,7 +62,7 @@ class Provider(SpecificationProvider):
         if dependency.is_vcs():
             return self.search_for_vcs(dependency)
 
-        packages = self._repository.find_packages(
+        packages = self._pool.find_packages(
             dependency.name,
             dependency.constraint
         )
@@ -150,7 +150,7 @@ class Provider(SpecificationProvider):
             # Information should already be set
             pass
         else:
-            package = self._repository.package(package.name, package.version)
+            package = self._pool.package(package.name, package.version)
 
         return [
             r for r in package.requires

poetry/puzzle/solver.py

@@ -20,8 +20,8 @@ class Solver:
         self._locked = locked
         self._io = io
 
-    def solve(self, requested, repository, fixed=None) -> List[Operation]:
-        resolver = Resolver(Provider(repository), UI(self._io))
+    def solve(self, requested, pool, fixed=None) -> List[Operation]:
+        resolver = Resolver(Provider(pool), UI(self._io))
 
         base = None
         if fixed is not None:

poetry/repositories/__init__.py

+from .pool import Pool
 from .repository import Repository

poetry/repositories/legacy_repository.py

+import re
+
+from pathlib import Path
+from piptools.cache import DependencyCache
+from piptools.repositories import PyPIRepository
+from piptools.resolver import Resolver
+from piptools.scripts.compile import get_pip_command
+from pip.req import InstallRequirement
+from pip.exceptions import InstallationError
+
+from cachy import CacheManager
+
+import poetry.packages
+
+from poetry.locations import CACHE_DIR
+from poetry.semver.constraints import Constraint
+from poetry.semver.constraints.base_constraint import BaseConstraint
+from poetry.semver.version_parser import VersionParser
+
+from .repository import Repository
+
+
+class LegacyRepository(Repository):
+
+    def __init__(self, name, url):
+        if name == 'pypi':
+            raise ValueError('The name [pypi] is reserved for repositories')
+
+        self._name = name
+        self._url = url
+        command = get_pip_command()
+        opts, _ = command.parse_args([])
+        self._session = command._build_session(opts)
+        self._repository = PyPIRepository(opts, self._session)
+        self._cache_dir = Path(CACHE_DIR) / 'cache' / 'repositories' / name
+
+        self._cache = CacheManager({
+            'default': 'releases',
+            'serializer': 'json',
+            'stores': {
+                'releases': {
+                    'driver': 'file',
+                    'path': Path(CACHE_DIR) / 'cache' / 'repositories' / name
+                },
+                'packages': {
+                    'driver': 'dict'
+                },
+                'matches': {
+                    'driver': 'dict'
+                }
+            }
+        })
+
+        super().__init__()
+
+    def find_packages(self, name, constraint=None):
+        packages = []
+
+        if constraint is not None and not isinstance(constraint,
+                                                     BaseConstraint):
+            version_parser = VersionParser()
+            constraint = version_parser.parse_constraints(constraint)
+
+        key = name
+        if constraint:
+            key = f'{key}:{str(constraint)}'
+
+        if self._cache.store('matches').has(key):
+            versions = self._cache.store('matches').get(key)
+        else:
+            candidates = [str(c.version) for c in self._repository.find_all_candidates(name)]
+
+            versions = []
+            for version in candidates:
+                if version in versions:
+                    continue
+
+                if (
+                    not constraint
+                    or (constraint and constraint.matches(Constraint('=', version)))
+                ):
+                    versions.append(version)
+
+            self._cache.store('matches').put(key, versions, 5)
+
+        for version in versions:
+            packages.append(self.package(name, version))
+
+        return packages
+
+    def package(self, name, version) -> 'poetry.packages.Package':
+        """
+        Retrieve the release information.
+
+        This is a heavy task which takes time.
+        We have to download a package to get the dependencies.
+        We also need to download every file matching this release
+        to get the various hashes.
+        
+        Note that, this will be cached so the subsequent operations
+        should be much faster.
+        """
+        try:
+            index = self._packages.index(
+                poetry.packages.Package(name, version, version)
+            )
+
+            return self._packages[index]
+        except ValueError:
+            release_info = self.get_release_info(name, version)
+            package = poetry.packages.Package(name, version, version)
+            for dependency in release_info['requires_dist']:
+                m = re.match(
+                    '^(?P<name>[^ ;]+)'
+                    '(?: \((?P<version>.+)\))?'
+                    '(?:;(?P<extra>(.+)))?$',
+                    dependency
+                )
+                package.requires.append(
+                    poetry.packages.Dependency(
+                        m.group('name'),
+                        m.group('version') or '*',
+                        optional=m.group('extra') is not None
+                    )
+                )
+
+            package.source_type = 'legacy'
+            package.source_url = self._url
+
+            # Adding hashes information
+            package.hashes = release_info['digests']
+
+            self._packages.append(package)
+
+            return package
+
+    def get_release_info(self, name: str, version: str) -> dict:
+        """
+        Return the release information given a package name and a version.
+
+        The information is returned from the cache if it exists
+        or retrieved from the remote server.
+        """
+        return self._cache.store('releases').remember_forever(
+            f'{name}:{version}',
+            lambda: self._get_release_info(name, version)
+        )
+
+    def _get_release_info(self, name: str, version: str) -> dict:
+        ireq = InstallRequirement.from_line(f'{name}=={version}')
+        resolver = Resolver(
+            [ireq], self._repository,
+            cache=DependencyCache(self._cache_dir.as_posix())
+        )
+        try:
+            requirements = list(resolver._iter_dependencies(ireq))
+        except InstallationError as e:
+            # setup.py egg-info error most likely
+            # So we assume no dependencies
+            requirements = []
+
+        requires = []
+        for dep in requirements:
+            constraint = str(dep.req.specifier)
+            require = f'{dep.name}'
+            if constraint:
+                require += f' ({constraint})'
+
+            requires.append(require)
+
+        hashes = resolver.resolve_hashes([ireq])[ireq]
+        hashes = [h.split(':')[1] for h in hashes]
+
+        data = {
+            'name': name,
+            'version': version,
+            'requires_dist': requires,
+            'digests': hashes
+        }
+
+        resolver.repository.freshen_build_caches()
+
+        return data

poetry/repositories/pool.py

+from typing import List
+from typing import Union
+
+import poetry.packages
+
+from .base_repository import BaseRepository
+from .legacy_repository import LegacyRepository
+from .repository import Repository
+
+
+class Pool(BaseRepository):
+
+    def __init__(self, repositories: Union[list, None] = None):
+        if repositories is None:
+            repositories = []
+
+        self._repositories = []
+
+        for repository in repositories:
+            self.add_repository(repository)
+
+        super().__init__()
+            
+    @property
+    def repositories(self) -> List[Repository]:
+        return self._repositories
+
+    def add_repository(self, repository: Repository) -> 'Pool':
+        """
+        Adds a repository to the pool.
+        """
+        self._repositories.append(repository)
+
+        return self
+    
+    def configure(self, source: dict) -> 'Pool':
+        """
+        Configures a repository based on a source
+        specification and add it to the pool.
+        """
+        if 'url' in source:
+            # PyPI-like repository
+            if 'name' not in source:
+                raise RuntimeError('Missing [name] in source.')
+
+            repository = LegacyRepository(source['name'], source['url'])
+        else:
+            raise RuntimeError('Unsupported source specified')
+
+        return self.add_repository(repository)
+
+    def has_package(self, package):
+        raise NotImplementedError()
+
+    def package(self, name, version) -> Union['poetry.packages.Package', None]:
+        package = poetry.packages.Package(name, version, version)
+        if package in self._packages:
+            return self._packages[self._packages.index(package)]
+
+        for repository in self._repositories:
+            package = repository.package(name, version)
+            if package:
+                self._packages.append(package)
+
+                return package
+
+        return None
+
+    def find_packages(self,
+                      name,
+                      constraint=None) -> List['poetry.packages.Package']:
+        for repository in self._repositories:
+            packages = repository.find_packages(name, constraint)
+            if packages:
+                return packages
+
+        return []
+
+    def search(self, query, mode=BaseRepository.SEARCH_FULLTEXT):
+        raise NotImplementedError()

poetry/repositories/pypi_repository.py

@@ -27,7 +27,7 @@ class PyPiRepository(Repository):
             'stores': {
                 'releases': {
                     'driver': 'file',
-                    'path': Path(CACHE_DIR) / 'cache'
+                    'path': Path(CACHE_DIR) / 'cache' / 'repositories' / 'pypi'
                 },
                 'packages': {
                     'driver': 'dict'
@@ -35,7 +35,7 @@ class PyPiRepository(Repository):
             }
         })
         
-        super(PyPiRepository, self).__init__()
+        super().__init__()
 
     def find_packages(self,
                       name: str,

poetry/version/version_selector.py

@@ -2,6 +2,7 @@ import re
 from typing import Union
 
 from poetry.packages import Package
+from poetry.semver.comparison import less_than
 from poetry.semver.helpers import normalize_version
 from poetry.semver.version_parser import VersionParser
 
@@ -34,7 +35,7 @@ class VersionSelector(object):
         package = candidates[0]
         for candidate in candidates:
             # Select highest version of the two
-            if package.version < candidate.version:
+            if less_than(package.version, candidate.version):
                 package = candidate
 
         return package

tests/installation/test_installer.py

@@ -7,6 +7,7 @@ from poetry.installation import Installer as BaseInstaller
 from poetry.installation.noop_installer import NoopInstaller
 from poetry.io import NullIO
 from poetry.packages import Locker as BaseLocker
+from poetry.repositories import Pool
 from poetry.repositories import Repository
 
 from tests.helpers import get_package
@@ -58,6 +59,14 @@ def repo():
 
 
 @pytest.fixture()
+def pool(repo):
+    pool = Pool()
+    pool.add_repository(repo)
+
+    return pool
+
+
+@pytest.fixture()
 def locker():
     return Locker()
 
@@ -69,8 +78,8 @@ def fixture(name):
 
 
 @pytest.fixture()
-def installer(package, repo, locker):
-    return Installer(NullIO(), package, locker, repo)
+def installer(package, pool, locker):
+    return Installer(NullIO(), package, locker, pool)
 
 
 def test_run_no_dependencies(installer, locker):