Add support for specifying dependencies extras

README.md

@@ -78,22 +78,22 @@ keywords = ['packaging', 'poetry']
 [tool.poetry.dependencies]
 python = "~2.7 || ^3.2"  # Compatible python versions must be declared here
 toml = "^0.9"
-requests = "^2.13"
-semantic_version = "^2.6"
-pygments = "^2.2"
-twine = "^1.8"
-wheel = "^0.29"
-pip-tools = "^1.8.2"
+# Dependencies with extras
+requests = { version = "^2.13", extras = [ "security" ] }
+# Python specific dependencies with prereleases allowed
+pathlib2 = { version = "^2.2", python = "~2.7", allows_prereleases = true }
+# Git dependencies
 cleo = { git = "https://github.com/sdispater/cleo.git", branch = "master" }
 
+# Optional dependencies (extras)
+pendulum = { version = "^1.4", optional = true}
+
 [tool.poetry.dev-dependencies]
 pytest = "^3.0"
 pytest-cov = "^2.4"
-coverage = "<4.0"
-httpretty = "^0.8.14"
 
 [tool.poetry.scripts]
-poet = 'poet:app.run'
+my-script = 'my_package:main'
 ```
 
 There are some things we can notice here:
@@ -632,21 +632,17 @@ poetry = 'poetry:console.run'
 
 Here, we will have the `poetry` script installed which will execute `console.run` in the `poetry` package.
 
-### `features`
+### `extras`
 
-Poetry supports features to allow expression of:
+Poetry supports extras to allow expression of:
 
 * optional dependencies, which enhance a package, but are not required; and
 * clusters of optional dependencies.
 
 ```toml
-[package]
+[tool.poetry]
 name = "awesome"
 
-[features]
-mysql = ["mysqlclient"]
-pgsql = ["psycopg2"]
-
 [dependencies]
 # These packages are mandatory and form the core of this package’s distribution.
 mandatory = "^1.0"
@@ -655,13 +651,17 @@ mandatory = "^1.0"
 # above `features`. They can be opted into by apps.
 psycopg2 = { version = "^2.7", optional = true }
 mysqlclient = { version = "^1.3", optional = true }
+
+[tool.poetry.extras]
+mysql = ["mysqlclient"]
+pgsql = ["psycopg2"]
 ```
 
-When installing packages, you can specify features by using the `-f|--features` option:
+When installing packages, you can specify features by using the `-E|--extras` option:
 
 ```bash
-poet install --features "mysql pgsql"
-poet install -f mysql -f pgsql
+poet install --extras "mysql pgsql"
+poet install -E mysql -E pgsql
 ```
 
 ### `plugins`

poetry/installation/installer.py

@@ -230,7 +230,7 @@ class Installer:
         if operation.skipped:
             if self._io.is_verbose() and (self._execute_operations or self.is_dry_run()):
                 self._io.writeln(
-                    f'  - Skipping <info>{operation.package.name}</> '
+                    f'  - Skipping <info>{operation.package.pretty_name}</> '
                     f'(<comment>{operation.package.full_pretty_version}</>) '
                     f'{operation.skip_reason}')
 
@@ -238,7 +238,7 @@ class Installer:
 
         if self._execute_operations or self.is_dry_run():
             self._io.writeln(
-                f'  - Installing <info>{operation.package.name}</> '
+                f'  - Installing <info>{operation.package.pretty_name}</> '
                 f'(<comment>{operation.package.full_pretty_version}</>)'
             )
 
@@ -254,7 +254,7 @@ class Installer:
         if operation.skipped:
             if self._io.is_verbose() and (self._execute_operations or self.is_dry_run()):
                 self._io.writeln(
-                    f'  - Skipping <info>{target.name}</> '
+                    f'  - Skipping <info>{target.pretty_name}</> '
                     f'(<comment>{target.full_pretty_version}</>) '
                     f'{operation.skip_reason}')
 
@@ -262,7 +262,7 @@ class Installer:
 
         if self._execute_operations or self.is_dry_run():
             self._io.writeln(
-                f'  - Updating <info>{target.name}</> '
+                f'  - Updating <info>{target.pretty_name}</> '
                 f'(<comment>{source.pretty_version}</>'
                 f' -> <comment>{target.pretty_version}</>)'
             )
@@ -275,7 +275,7 @@ class Installer:
     def _execute_uninstall(self, operation: Uninstall) -> None:
         if self._execute_operations or self.is_dry_run():
             self._io.writeln(
-                f'  - Removing <info>{operation.package.name}</> '
+                f'  - Removing <info>{operation.package.pretty_name}</> '
                 f'(<comment>{operation.package.full_pretty_version}</>)'
             )
 

poetry/packages/dependency.py

@@ -33,6 +33,8 @@ class Dependency:
         self._platform = '*'
         self._platform_constraint = self._parser.parse_constraints('*')
 
+        self._extras = []
+
     @property
     def name(self):
         return self._name
@@ -73,12 +75,15 @@ class Dependency:
     @platform.setter
     def platform(self, value: str):
         self._platform = value
-        self._platform_constraint = self._parser.parse_constraints(value)
 
     @property
     def platform_constraint(self):
         return self._platform_constraint
 
+    @property
+    def extras(self) -> list:
+        return self._extras
+
     def allows_prereleases(self):
         return self._allows_prereleases
 
@@ -129,6 +134,12 @@ class Dependency:
 
         return requirement
 
+    def activate(self):
+        """
+        Set the dependency as mandatory.
+        """
+        self._optional = False
+
     def __eq__(self, other):
         if not isinstance(other, Dependency):
             return NotImplemented

poetry/packages/package.py

@@ -63,6 +63,7 @@ class Package:
 
         self.requires = []
         self.dev_requires = []
+        self.extras = {}
 
         self._parser = VersionParser()
 
@@ -187,7 +188,7 @@ class Package:
                 python_versions = constraint.get('python')
                 platform = constraint.get('platform')
 
-                optional = optional or python_versions is not None or not platform is not None
+                optional = optional or python_versions is not None or platform is not None
 
                 dependency = Dependency(
                     name, version,
@@ -201,6 +202,10 @@ class Package:
 
                 if platform:
                     dependency.platform = platform
+
+            if 'extras' in constraint:
+                for extra in constraint['extras']:
+                    dependency.extras.append(extra)
         else:
             dependency = Dependency(name, constraint, category=category)
 

poetry/puzzle/provider.py

@@ -64,7 +64,8 @@ class Provider(SpecificationProvider):
 
         packages = self._pool.find_packages(
             dependency.name,
-            dependency.constraint
+            dependency.constraint,
+            extras=dependency.extras,
         )
 
         packages.sort(

poetry/repositories/base_repository.py

@@ -16,7 +16,7 @@ class BaseRepository:
     def package(self, name, version):
         raise NotImplementedError()
 
-    def find_packages(self, name, constraint=None):
+    def find_packages(self, name, constraint=None, extras=None):
         raise NotImplementedError()
 
     def search(self, query, mode=SEARCH_FULLTEXT):

poetry/repositories/legacy_repository.py

@@ -17,10 +17,10 @@ from poetry.semver.constraints import Constraint
 from poetry.semver.constraints.base_constraint import BaseConstraint
 from poetry.semver.version_parser import VersionParser
 
-from .repository import Repository
+from .pypi_repository import PyPiRepository
 
 
-class LegacyRepository(Repository):
+class LegacyRepository(PyPiRepository):
 
     def __init__(self, name, url):
         if name == 'pypi':
@@ -51,9 +51,7 @@ class LegacyRepository(Repository):
             }
         })
 
-        super().__init__()
-
-    def find_packages(self, name, constraint=None):
+    def find_packages(self, name, constraint=None, extras=None):
         packages = []
 
         if constraint is not None and not isinstance(constraint,
@@ -84,11 +82,11 @@ class LegacyRepository(Repository):
             self._cache.store('matches').put(key, versions, 5)
 
         for version in versions:
-            packages.append(self.package(name, version))
+            packages.append(self.package(name, version, extras=extras))
 
         return packages
 
-    def package(self, name, version) -> 'poetry.packages.Package':
+    def package(self, name, version, extras=None) -> 'poetry.packages.Package':
         """
         Retrieve the release information.
 
@@ -107,29 +105,78 @@ class LegacyRepository(Repository):
 
             return self._packages[index]
         except ValueError:
+            if extras is None:
+                extras = []
+
             release_info = self.get_release_info(name, version)
             package = poetry.packages.Package(name, version, version)
-            for dependency in release_info['requires_dist']:
-                m = re.match(
-                    '^(?P<name>[^ ;]+)'
-                    '(?: \((?P<version>.+)\))?'
-                    '(?:;(?P<extra>(.+)))?$',
-                    dependency
+            for req in release_info['requires_dist']:
+                req = InstallRequirement.from_line(req)
+
+                name = req.name
+                version = str(req.req.specifier)
+
+                dependency = Dependency(
+                    name,
+                    version,
+                    optional=req.markers
                 )
-                package.requires.append(
-                    poetry.packages.Dependency(
-                        m.group('name'),
-                        m.group('version') or '*',
-                        optional=m.group('extra') is not None
+
+                is_extra = False
+                if req.markers:
+                    # Setting extra dependencies and requirements
+                    requirements = self._convert_markers(
+                        req.markers._markers
                     )
-                )
 
-            package.source_type = 'legacy'
-            package.source_url = self._url
+                    if 'python_version' in requirements:
+                        ors = []
+                        for or_ in requirements['python_version']:
+                            ands = []
+                            for op, version in or_:
+                                ands.append(f'{op}{version}')
+
+                            ors.append(' '.join(ands))
+
+                        dependency.python_versions = ' || '.join(ors)
+
+                    if 'sys_platform' in requirements:
+                        ors = []
+                        for or_ in requirements['sys_platform']:
+                            ands = []
+                            for op, platform in or_:
+                                ands.append(f'{op}{platform}')
+
+                            ors.append(' '.join(ands))
+
+                        dependency.platform = ' || '.join(ors)
+
+                    if 'extra' in requirements:
+                        is_extra = True
+                        for _extras in requirements['extra']:
+                            for _, extra in _extras:
+                                if extra not in package.extras:
+                                    package.extras[extra] = []
+
+                                package.extras[extra].append(dependency)
+
+                if not is_extra:
+                    package.requires.append(dependency)
+
+            # Adding description
+            package.description = release_info.get('summary', '')
 
             # Adding hashes information
             package.hashes = release_info['digests']
 
+            # Activate extra dependencies
+            for extra in extras:
+                if extra in package.extras:
+                    for dep in package.extras[extra]:
+                        dep.activate()
+
+                    package.requires += package.extras[extra]
+
             self._packages.append(package)
 
             return package

poetry/repositories/pool.py

@@ -4,7 +4,6 @@ from typing import Union
 import poetry.packages
 
 from .base_repository import BaseRepository
-from .legacy_repository import LegacyRepository
 from .repository import Repository
 
 
@@ -38,6 +37,8 @@ class Pool(BaseRepository):
         Configures a repository based on a source
         specification and add it to the pool.
         """
+        from .legacy_repository import LegacyRepository
+
         if 'url' in source:
             # PyPI-like repository
             if 'name' not in source:
@@ -68,9 +69,10 @@ class Pool(BaseRepository):
 
     def find_packages(self,
                       name,
-                      constraint=None) -> List['poetry.packages.Package']:
+                      constraint=None,
+                      extras=None) -> List['poetry.packages.Package']:
         for repository in self._repositories:
-            packages = repository.find_packages(name, constraint)
+            packages = repository.find_packages(name, constraint, extras=extras)
             if packages:
                 return packages
 

poetry/repositories/pypi_repository.py

-import re
-
 from pathlib import Path
+from pip.req import InstallRequirement
 from typing import List
 from typing import Union
 
@@ -19,8 +18,9 @@ from .repository import Repository
 
 class PyPiRepository(Repository):
 
-    def __init__(self, url='https://pypi.org/'):
+    def __init__(self, url='https://pypi.org/', disable_cache=False):
         self._url = url
+        self._disable_cache = disable_cache
         self._cache = CacheManager({
             'default': 'releases',
             'serializer': 'json',
@@ -39,7 +39,8 @@ class PyPiRepository(Repository):
 
     def find_packages(self,
                       name: str,
-                      constraint: Union[Constraint, None] = None
+                      constraint: Union[Constraint, str, None] = None,
+                      extras: Union[list, None] = None
                       ) -> List[Package]:
         """
         Find packages on the remote server.
@@ -62,34 +63,78 @@ class PyPiRepository(Repository):
                 versions.append(version)
 
         for version in versions:
-            packages.append(self.package(name, version))
+            packages.append(
+                self.package(name, version, extras=extras)
+            )
 
         return packages
 
     def package(self,
                 name: str,
-                version: str) -> Package:
+                version: str,
+                extras: Union[list, None] = None) -> Package:
         try:
             index = self._packages.index(Package(name, version, version))
 
             return self._packages[index]
         except ValueError:
+            if extras is None:
+                extras = []
+
             release_info = self.get_release_info(name, version)
             package = Package(name, version, version)
-            for dependency in release_info['requires_dist']:
-                m = re.match(
-                    '^(?P<name>[^ ;]+)'
-                    '(?: \((?P<version>.+)\))?'
-                    '(?:;(?P<extra>(.+)))?$',
-                    dependency
+            for req in release_info['requires_dist']:
+                req = InstallRequirement.from_line(req)
+
+                name = req.name
+                version = str(req.req.specifier)
+
+                dependency = Dependency(
+                    name,
+                    version,
+                    optional=req.markers
                 )
-                package.requires.append(
-                    Dependency(
-                        m.group('name'),
-                        m.group('version') or '*',
-                        optional=m.group('extra') is not None
+
+                is_extra = False
+                if req.markers:
+                    # Setting extra dependencies and requirements
+                    requirements = self._convert_markers(
+                        req.markers._markers
                     )
-                )
+
+                    if 'python_version' in requirements:
+                        ors = []
+                        for or_ in requirements['python_version']:
+                            ands = []
+                            for op, version in or_:
+                                ands.append(f'{op}{version}')
+
+                            ors.append(' '.join(ands))
+
+                        dependency.python_versions = ' || '.join(ors)
+
+                    if 'sys_platform' in requirements:
+                        ors = []
+                        for or_ in requirements['sys_platform']:
+                            ands = []
+                            for op, platform in or_:
+                                ands.append(f'{op}{platform}')
+
+                            ors.append(' '.join(ands))
+
+                        dependency.platform = ' || '.join(ors)
+
+                    if 'extra' in requirements:
+                        is_extra = True
+                        for _extras in requirements['extra']:
+                            for _, extra in _extras:
+                                if extra not in package.extras:
+                                    package.extras[extra] = []
+
+                                package.extras[extra].append(dependency)
+
+                if not is_extra:
+                    package.requires.append(dependency)
 
             # Adding description
             package.description = release_info.get('summary', '')
@@ -97,6 +142,14 @@ class PyPiRepository(Repository):
             # Adding hashes information
             package.hashes = release_info['digests']
 
+            # Activate extra dependencies
+            for extra in extras:
+                if extra in package.extras:
+                    for dep in package.extras[extra]:
+                        dep.activate()
+
+                    package.requires += package.extras[extra]
+
             self._packages.append(package)
 
             return package
@@ -130,18 +183,19 @@ class PyPiRepository(Repository):
         The information is returned from the cache if it exists
         or retrieved from the remote server.
         """
+        if self._disable_cache:
+            return self._get_package_info(name)
+
         return self._cache.store('packages').remember_forever(
             f'{name}',
             lambda: self._get_package_info(name)
         )
 
     def _get_package_info(self, name: str) -> dict:
-        json_response = get(self._url + f'pypi/{name}/json')
-        if json_response.status_code == 404:
+        data = self._get(self._url + f'pypi/{name}/json')
+        if data is None:
             raise ValueError(f'Package [{name}] not found.')
 
-        data = json_response.json()
-
         return data
 
     def get_release_info(self, name: str, version: str) -> dict:
@@ -151,17 +205,19 @@ class PyPiRepository(Repository):
         The information is returned from the cache if it exists
         or retrieved from the remote server.
         """
+        if self._disable_cache:
+            return self._get_release_info(name, version)
+
         return self._cache.remember_forever(
             f'{name}:{version}',
             lambda: self._get_release_info(name, version)
         )
 
     def _get_release_info(self, name: str, version: str) -> dict:
-        json_response = get(self._url + f'pypi/{name}/{version}/json')
-        if json_response.status_code == 404:
+        json_data = self._get(self._url + f'pypi/{name}/{version}/json')
+        if json_data is None:
             raise ValueError(f'Package [{name}] not found.')
 
-        json_data = json_response.json()
         info = json_data['info']
         data = {
             'name': info['name'],
@@ -176,3 +232,56 @@ class PyPiRepository(Repository):
             data['digests'].append(file_info['digests']['sha256'])
 
         return data
+
+    def _get(self, url: str) -> Union[dict, None]:
+        json_response = get(url)
+        if json_response.status_code == 404:
+            return None
+
+        json_data = json_response.json()
+
+        return json_data
+
+    def _group_markers(self, markers):
+        groups = [[]]
+
+        for marker in markers:
+            assert isinstance(marker, (list, tuple, str))
+
+            if isinstance(marker, list):
+                groups[-1].append(self._group_markers(marker))
+            elif isinstance(marker, tuple):
+                lhs, op, rhs = marker
+
+                groups[-1].append((lhs.value, op, rhs.value))
+            else:
+                assert marker in ["and", "or"]
+                if marker == "or":
+                    groups.append([])
+
+        return groups
+
+    def _convert_markers(self, markers):
+        groups = self._group_markers(markers)[0]
+
+        requirements = {}
+
+        def _group(_groups, or_=False):
+            nonlocal requirements
+
+            for group in _groups:
+                if isinstance(group, tuple):
+                    variable, op, value = group
+                    group_name = str(variable)
+                    if group_name not in requirements:
+                        requirements[group_name] = [[]]
+                    elif or_:
+                        requirements[group_name].append([])
+
+                    requirements[group_name][-1].append((str(op), str(value)))
+                else:
+                    _group(group, or_=True)
+
+        _group(groups)
+
+        return requirements

poetry/repositories/repository.py

@@ -27,9 +27,11 @@ class Repository(BaseRepository):
             if name == package.name and package.version == version:
                 return package
 
-    def find_packages(self, name, constraint=None):
+    def find_packages(self, name, constraint=None, extras=None):
         name = name.lower()
         packages = []
+        if extras is None:
+            extras = []
 
         if not isinstance(constraint, BaseConstraint):
             parser = VersionParser()
@@ -40,6 +42,13 @@ class Repository(BaseRepository):
                 pkg_constraint = Constraint('==', package.version)
 
                 if constraint is None or constraint.matches(pkg_constraint):
+                    for extra in extras:
+                        if extra in package.extras:
+                            for dep in package.extras[extra]:
+                                dep.activate()
+
+                            package.requires += package.extras[extra]
+
                     packages.append(package)
 
         return packages

tests/fixtures/sample_project/pyproject.toml

@@ -21,7 +21,7 @@ keywords = ["packaging", "dependency", "poetry"]
 python = "~2.7 || ^3.6"
 cleo = "^0.6"
 pendulum = { git = "https://github.com/sdispater/pendulum.git", branch = "2.0" }
-requests = { version = "^2.18", optional = true }
+requests = { version = "^2.18", optional = true, extras=[ "security" ] }
 pathlib2 = { version = "^2.2", python = "~2.7" }
 
 [tool.poetry.dev-dependencies]

tests/installation/fixtures/with-dependencies-extras.test

+[[package]]
+name = "A"
+version = "1.0"
+description = ""
+category = "main"
+optional = false
+python-versions = "*"
+platform = "*"
+
+[[package]]
+name = "B"
+version = "1.0"
+description = ""
+category = "main"
+optional = false
+python-versions = "*"
+platform = "*"
+
+[package.dependencies]
+C = "^1.0"
+
+[[package]]
+name = "C"
+version = "1.0"
+description = ""
+category = "main"
+optional = false
+python-versions = "*"
+platform = "*"
+
+[metadata]
+python-versions = "*"
+platform = "*"
+content-hash = "123456789"
+
+[metadata.hashes]
+"A" = []
+"B" = []
+"C" = []

tests/installation/test_installer.py

@@ -11,6 +11,7 @@ from poetry.packages import Locker as BaseLocker
 from poetry.repositories import Pool
 from poetry.repositories import Repository
 
+from tests.helpers import get_dependency
 from tests.helpers import get_package
 
 
@@ -287,3 +288,25 @@ def test_run_with_optional_and_python_restricted_dependencies(installer, locker,
     # with B's python constraint
     assert len(installer.installs) == 3
 
+
+def test_run_with_dependencies_extras(installer, locker, repo, package):
+    package_a = get_package('A', '1.0')
+    package_b = get_package('B', '1.0')
+    package_c = get_package('C', '1.0')
+
+    package_b.extras = {
+        'foo': [get_dependency('C', '^1.0')]
+    }
+
+    repo.add_package(package_a)
+    repo.add_package(package_b)
+    repo.add_package(package_c)
+
+    package.add_dependency('A', '^1.0')
+    package.add_dependency('B', {'version': '^1.0', 'extras': ['foo']})
+
+    installer.run()
+    expected = fixture('with-dependencies-extras')
+
+    assert locker.written_data == expected
+

tests/puzzle/test_solver.py

@@ -324,3 +324,61 @@ def test_solver_solves_optional_and_compatible_packages(solver, repo, package):
         {'job': 'install', 'package': package_b},
         {'job': 'install', 'package': package_a},
     ])
+
+
+def test_solver_does_not_return_extras_if_not_requested(solver, repo, package):
+    package_a = get_package('A', '1.0')
+    package_b = get_package('B', '1.0')
+    package_c = get_package('C', '1.0')
+
+    package_b.extras = {
+        'foo': [get_dependency('C', '^1.0')]
+    }
+
+    repo.add_package(package_a)
+    repo.add_package(package_b)
+    repo.add_package(package_c)
+
+    dependency_a = get_dependency('A')
+    dependency_b = get_dependency('B')
+    request = [
+        dependency_a,
+        dependency_b
+    ]
+
+    ops = solver.solve(request)
+
+    check_solver_result(ops, [
+        {'job': 'install', 'package': package_b},
+        {'job': 'install', 'package': package_a},
+    ])
+
+
+def test_solver_returns_extras_if_requested(solver, repo, package):
+    package_a = get_package('A', '1.0')
+    package_b = get_package('B', '1.0')
+    package_c = get_package('C', '1.0')
+
+    package_b.extras = {
+        'foo': [get_dependency('C', '^1.0')]
+    }
+
+    repo.add_package(package_a)
+    repo.add_package(package_b)
+    repo.add_package(package_c)
+
+    dependency_a = get_dependency('A')
+    dependency_b = get_dependency('B')
+    dependency_b.extras.append('foo')
+    request = [
+        dependency_a,
+        dependency_b
+    ]
+
+    ops = solver.solve(request)
+
+    check_solver_result(ops, [
+        {'job': 'install', 'package': package_c},
+        {'job': 'install', 'package': package_b},
+        {'job': 'install', 'package': package_a},
+    ])

tests/repositories/test_pypi_repository.py

+import json
+
+from pathlib import Path
+
+from poetry.repositories.pypi_repository import PyPiRepository
+
+
+class MockRepository(PyPiRepository):
+
+    FIXTURES = Path(__file__).parent / 'fixtures' / 'pypi.org' / 'json'
+
+    def __init__(self):
+        super().__init__(url='http://foo.bar', disable_cache=True)
+
+    def _get(self, url: str) -> dict:
+        fixture = self.FIXTURES / 'requests.json'
+
+        with fixture.open() as f:
+            return json.loads(f.read())
+
+
+def test_find_packages():
+    repo = MockRepository()
+    packages = repo.find_packages('requests', '^2.18')
+
+    assert len(packages) == 5
+
+
+def test_package():
+    repo = MockRepository()
+
+    package = repo.package('requests', '2.18.4')
+
+    assert package.name == 'requests'
+    assert len(package.requires) == 4
+    assert len(package.extras['security']) == 3
+    assert len(package.extras['socks']) == 2
+
+    win_inet = package.extras['socks'][0]
+    assert win_inet.name == 'win-inet-pton'
+    assert win_inet.python_versions == '==2.7 || ==2.6'
+    assert win_inet.platform == '==win32'

tests/test_poetry.py

@@ -44,6 +44,7 @@ def test_poetry():
     assert not requests.is_vcs()
     assert not requests.allows_prereleases()
     assert requests.is_optional()
+    assert requests.extras == ['security']
 
     pathlib2 = dependencies[3]
     assert pathlib2.pretty_constraint == '^2.2'