Add support for extras definition

f643de65 · Sébastien Eustace · 3472463e · f643de65 · f643de65 · f643de65
Commit f643de65 authored Mar 06, 2018 by Sébastien Eustace
10 changed files
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -6,6 +6,8 @@
 - Added packaging support (sdist and pure-python wheel).
 - Added the `build` command.
+- Added support for extras definition.
+- Added support for dependencies extras specification.
 ### Changes

--- a/poetry/console/commands/install.py
+++ b/poetry/console/commands/install.py
@@ -12,6 +12,8 @@ class InstallCommand(Command):
        { --no-dev : Do not install dev dependencies. }
        { --dry-run : Outputs the operations but will not execute anything
                      (implicitly enables --verbose). }
+        { --E|extras=* : Extra sets of dependencies to install
+                         (multiple values allowed). }
    """
    help = """The <info>install</info> command reads the <comment>poetry.lock</> file from
@@ -30,6 +32,7 @@ exist it will look for <comment>poetry.toml</> and do the same.
            self.poetry.pool
        )
+        installer.extras(self.option('extras'))
        installer.dev_mode(not self.option('no-dev'))
        installer.dry_run(self.option('dry-run'))

--- a/poetry/installation/installer.py
+++ b/poetry/installation/installer.py
@@ -41,6 +41,8 @@ class Installer:
        self._whitelist = {}
+        self._extras = []
        self._installer = self._get_installer()
    @property
@@ -104,6 +106,11 @@ class Installer:
        return self
+    def extras(self, extras: list) -> 'Installer':
+        self._extras = extras
+        return self
    def _do_install(self, local_repo):
        locked_repository = Repository()
        # initialize locked repo if we are installing from lock
@@ -111,6 +118,11 @@ class Installer:
            locked_repository = self._locker.locked_repository(True)
        if self._update:
+            # Checking extras
+            for extra in self._extras:
+                if extra not in self._package.extras:
+                    raise ValueError(f'Extra [{extra}] is not specified.')
            self._io.writeln('<info>Updating dependencies</>')
            fixed = []
@@ -157,6 +169,10 @@ class Installer:
                    '</warning>'
                )
+            for extra in self._extras:
+                if extra not in self._locker.lock_data.get('extras', {}):
+                    raise ValueError(f'Extra [{extra}] is not specified.')
            # If we are installing from lock
            # Filter the operations by comparing it with what is
            # currently installed
@@ -165,7 +181,8 @@ class Installer:
        self._populate_local_repo(local_repo, ops, locked_repository)
        # We need to filter operations so that packages
-        # not compatible with the current system are dropped
+        # not compatible with the current system,
+        # or optional and not requested, are dropped
        self._filter_operations(ops)
        self._io.new_line()
@@ -319,6 +336,11 @@ class Installer:
                                  ) -> List[Operation]:
        installed_repo = InstalledRepository.load(self._io.venv)
        ops = []
+        extras = []
+        for extra_name, packages in self._locker.lock_data.get('extras').items():
+            if extra_name in self._extras:
+                for package in packages:
+                    extras.append(package.lower())
        for locked in locked_repository.packages:
            is_installed = False
@@ -327,12 +349,20 @@ class Installer:
                    is_installed = True
                    if locked.category == 'dev' and not self.is_dev_mode():
                        ops.append(Uninstall(locked))
+                    elif locked.is_optional() and locked.name not in extras:
+                        # Installed but optional and not requested in extras
+                        ops.append(Uninstall(locked))
                    elif locked.version != installed.version:
                        ops.append(Update(
                            installed, locked
                        ))
            if not is_installed:
+                # If it's optional and not in required extras
+                # we do not install
+                if locked.is_optional() and locked.name not in extras:
+                    continue
                ops.append(Install(locked))
        return ops
@@ -344,7 +374,7 @@ class Installer:
            else:
                package = op.package
-            if not package.requirements or op.job_type == 'uninstall':
+            if op.job_type == 'uninstall':
                continue
            parser = VersionParser()
@@ -356,6 +386,28 @@ class Installer:
                if not python_constraint.matches(Constraint('=', python)):
                    # Incompatible python versions
                    op.skip('Not needed for the current python version')
+                    continue
+            if self._update:
+                extras = {}
+                for extra, deps in self._package.extras.items():
+                    extras[extra] = [dep.name for dep in deps]
+            else:
+                extras = {}
+                for extra, deps in self._locker.lock_data.get('extras', {}):
+                    extras[extra] = [dep.lower() for dep in deps]
+            # If a package is optional and not requested
+            # in any extra we skip it
+            if package.optional:
+                drop = True
+                for extra in self._extras:
+                    if extra in extras and package.name in extras[extra]:
+                        drop = False
+                        continue
+                if drop:
+                    op.skip('Not required')
    def _get_installer(self) -> BaseInstaller:
        return PipInstaller(self._io.venv, self._io)
--- a/poetry/packages/locker.py
+++ b/poetry/packages/locker.py
@@ -123,6 +123,12 @@ class Locker:
            }
        }
+        if root.extras:
+            lock['extras'] = {
+                extra: [dep.pretty_name for dep in deps]
+                for extra, deps in root.extras.items()
+            }
        if not self.is_locked() or lock != self.lock_data:
            self._write_lock_data(lock)

--- a/poetry/packages/package.py
+++ b/poetry/packages/package.py
@@ -165,8 +165,6 @@ class Package:
                python_versions = constraint.get('python')
                platform = constraint.get('platform')
-                optional = optional or python_versions is not None or platform is not None
                dependency = VCSDependency(
                    name,
                    'git', constraint['git'],
@@ -188,8 +186,6 @@ class Package:
                python_versions = constraint.get('python')
                platform = constraint.get('platform')
-                optional = optional or python_versions is not None or platform is not None
                dependency = Dependency(
                    name, version,
                    optional=optional,

--- a/poetry/puzzle/solver.py
+++ b/poetry/puzzle/solver.py
@@ -4,6 +4,8 @@ from poetry.mixology import Resolver
 from poetry.mixology.dependency_graph import DependencyGraph
 from poetry.mixology.exceptions import ResolverError
+from poetry.semver.version_parser import VersionParser
 from .exceptions import SolverProblemError
 from .operations import Install
 from .operations import Uninstall
@@ -50,7 +52,29 @@ class Solver:
                vertex.payload.optional = False
            else:
                vertex.payload.optional = True
-                vertex.payload.requirements = tags['requirements']
+            # Finding the less restrictive requirements
+            requirements = {}
+            parser = VersionParser()
+            for req_name, reqs in tags['requirements'].items():
+                for req in reqs:
+                    if req_name == 'python':
+                        if 'python' not in requirements:
+                            requirements['python'] = req
+                            continue
+                        previous = parser.parse_constraints(requirements['python'])
+                        current = parser.parse_constraints(req)
+                        if current.matches(previous):
+                            requirements['python'] = req
+                    if 'platform' in req:
+                        if 'platform' not in requirements:
+                            requirements['platform'] = req
+                            continue
+            vertex.payload.requirements = requirements
        operations = []
        for package in packages:
@@ -84,7 +108,10 @@ class Solver:
        tags = {
            'category': [],
            'optional': True,
-            'requirements': {}
+            'requirements': {
+                'python': [],
+                'platform': []
+            }
        }
        if not vertex.incoming_edges:
@@ -95,19 +122,21 @@ class Solver:
                    if not req.is_optional():
                        tags['optional'] = False
-                    if req.is_optional():
+                    if req.python_versions != '*':
-                        # Checking installation requirements
+                        tags['requirements']['python'].append(str(req.python_constraint))
-                        if req.python_versions != '*':
-                            tags['requirements']['python'] = str(req.python_constraint)
-                        if req.platform != '*':
+                    if req.platform != '*':
-                            tags['requirements']['platform'] = str(req.platform_constraint)
+                        tags['requirements']['platform'].append(str(req.platform_constraint))
+                    break
        else:
            for edge in vertex.incoming_edges:
                sub_tags = self._get_tags_for_vertex(edge.origin, requested)
                tags['category'] += sub_tags['category']
                tags['optional'] = tags['optional'] and sub_tags['optional']
-                tags['requirements'].update(sub_tags['requirements'])
+                requirements = sub_tags['requirements']
+                tags['requirements']['python'] += requirements.get('python', [])
+                tags['requirements']['platform'] += requirements.get('platform', [])
        return tags
--- a/tests/installation/fixtures/extras.test
+++ b/tests/installation/fixtures/extras.test
+[[package]]
+name = "A"
+version = "1.0"
+description = ""
+category = "main"
+optional = false
+python-versions = "*"
+platform = "*"
+[[package]]
+name = "B"
+version = "1.0"
+description = ""
+category = "main"
+optional = false
+python-versions = "*"
+platform = "*"
+[[package]]
+name = "C"
+version = "1.0"
+description = ""
+category = "main"
+optional = false
+python-versions = "*"
+platform = "*"
+[[package]]
+name = "D"
+version = "1.1"
+description = ""
+category = "main"
+optional = true
+python-versions = "*"
+platform = "*"
+[extras]
+foo = ["D"]
+[metadata]
+python-versions = "*"
+platform = "*"
+content-hash = "123456789"
+[metadata.hashes]
+A = []
+B = []
+C = []
+D = []
--- a/tests/installation/fixtures/with-optional-dependencies.test
+++ b/tests/installation/fixtures/with-optional-dependencies.test
@@ -12,7 +12,7 @@ name = "B"
 version = "1.1"
 description = ""
 category = "main"
-optional = true
+optional = false
 python-versions = "*"
 platform = "*"
@@ -24,7 +24,7 @@ name = "C"
 version = "1.3"
 description = ""
 category = "main"
-optional = true
+optional = false
 python-versions = "*"
 platform = "*"
@@ -39,7 +39,7 @@ name = "D"
 version = "1.4"
 description = ""
 category = "main"
-optional = true
+optional = false
 python-versions = "*"
 platform = "*"

--- a/tests/installation/test_installer.py
+++ b/tests/installation/test_installer.py
@@ -283,10 +283,12 @@ def test_run_with_optional_and_python_restricted_dependencies(installer, locker,
    assert locker.written_data == expected
    installer = installer.installer
-    # We should only have 3 installs
+    # We should only have 2 installs:
-    # A, C, D since the mocked python version is not compatible
+    # C,D since the mocked python version is not compatible
-    # with B's python constraint
+    # with B's python constraint and A is optional
-    assert len(installer.installs) == 3
+    assert len(installer.installs) == 2
+    assert installer.installs[0].name == 'd'
+    assert installer.installs[1].name == 'c'
 def test_run_with_dependencies_extras(installer, locker, repo, package):
@@ -310,3 +312,64 @@ def test_run_with_dependencies_extras(installer, locker, repo, package):
    assert locker.written_data == expected
+def test_run_does_not_install_extras_if_not_requested(installer, locker, repo, package):
+    package.extras['foo'] = [
+        get_dependency('D')
+    ]
+    package_a = get_package('A', '1.0')
+    package_b = get_package('B', '1.0')
+    package_c = get_package('C', '1.0')
+    package_d = get_package('D', '1.1')
+    repo.add_package(package_a)
+    repo.add_package(package_b)
+    repo.add_package(package_c)
+    repo.add_package(package_d)
+    package.add_dependency('A', '^1.0')
+    package.add_dependency('B', '^1.0')
+    package.add_dependency('C', '^1.0')
+    package.add_dependency('D', {'version': '^1.0', 'optional': True})
+    installer.run()
+    expected = fixture('extras')
+    # Extras are pinned in lock
+    assert locker.written_data == expected
+    # But should not be installed
+    installer = installer.installer
+    assert len(installer.installs) == 3  # A, B, C
+def test_run_installs_extras_if_requested(installer, locker, repo, package):
+    package.extras['foo'] = [
+        get_dependency('D')
+    ]
+    package_a = get_package('A', '1.0')
+    package_b = get_package('B', '1.0')
+    package_c = get_package('C', '1.0')
+    package_d = get_package('D', '1.1')
+    repo.add_package(package_a)
+    repo.add_package(package_b)
+    repo.add_package(package_c)
+    repo.add_package(package_d)
+    package.add_dependency('A', '^1.0')
+    package.add_dependency('B', '^1.0')
+    package.add_dependency('C', '^1.0')
+    package.add_dependency('D', {'version': '^1.0', 'optional': True})
+    installer.extras(['foo'])
+    installer.run()
+    expected = fixture('extras')
+    # Extras are pinned in lock
+    assert locker.written_data == expected
+    # But should not be installed
+    installer = installer.installer
+    assert len(installer.installs) == 4  # A, B, C, D
--- a/tests/test_poetry.py
+++ b/tests/test_poetry.py
@@ -49,4 +49,4 @@ def test_poetry():
    pathlib2 = dependencies[3]
    assert pathlib2.pretty_constraint == '^2.2'
    assert pathlib2.python_versions == '~2.7'
-    assert pathlib2.is_optional()
+    assert not pathlib2.is_optional()