Skip to content

P
python-poetry
Commit 3472463e by Sébastien Eustace
Options

Add support for specifying dependencies extras

parent 2367ac9a
Showing with 426 additions and 79 deletions
README.md
......@@ -78,22 +78,22 @@ keywords = ['packaging', 'poetry']
[tool.poetry.dependencies]
python = "~2.7 || ^3.2" # Compatible python versions must be declared here
toml = "^0.9"
requests = "^2.13"
semantic_version = "^2.6"
pygments = "^2.2"
twine = "^1.8"
wheel = "^0.29"
pip-tools = "^1.8.2"
# Dependencies with extras
requests = { version = "^2.13", extras = [ "security" ] }
# Python specific dependencies with prereleases allowed
pathlib2 = { version = "^2.2", python = "~2.7", allows_prereleases = true }
# Git dependencies
cleo = { git = "https://github.com/sdispater/cleo.git", branch = "master" }
# Optional dependencies (extras)
pendulum = { version = "^1.4", optional = true}
[tool.poetry.dev-dependencies]
pytest = "^3.0"
pytest-cov = "^2.4"
coverage = "<4.0"
httpretty = "^0.8.14"
[tool.poetry.scripts]
poet = 'poet:app.run'
my-script = 'my_package:main'
```
There are some things we can notice here:
......@@ -632,21 +632,17 @@ poetry = 'poetry:console.run'
Here, we will have the `poetry` script installed which will execute `console.run` in the `poetry` package.
### `features`
### `extras`
Poetry supports features to allow expression of:
Poetry supports extras to allow expression of:
* optional dependencies, which enhance a package, but are not required; and
* clusters of optional dependencies.
```toml
[package]
[tool.poetry]
name = "awesome"
[features]
mysql = ["mysqlclient"]
pgsql = ["psycopg2"]
[dependencies]
# These packages are mandatory and form the core of this package’s distribution.
mandatory = "^1.0"
......@@ -655,13 +651,17 @@ mandatory = "^1.0"
# above `features`. They can be opted into by apps.
psycopg2 = { version = "^2.7", optional = true }
mysqlclient = { version = "^1.3", optional = true }
[tool.poetry.extras]
mysql = ["mysqlclient"]
pgsql = ["psycopg2"]
```
When installing packages, you can specify features by using the `-f|--features` option:
When installing packages, you can specify features by using the `-E|--extras` option:
```bash
poet install --features "mysql pgsql"
poet install -f mysql -f pgsql
poet install --extras "mysql pgsql"
poet install -E mysql -E pgsql
```
### `plugins`
......
poetry/installation/installer.py
......@@ -230,7 +230,7 @@ class Installer:
if operation.skipped:
if self._io.is_verbose() and (self._execute_operations or self.is_dry_run()):
self._io.writeln(
f' - Skipping <info>{operation.package.name}</> '
f' - Skipping <info>{operation.package.pretty_name}</> '
f'(<comment>{operation.package.full_pretty_version}</>) '
f'{operation.skip_reason}')
......@@ -238,7 +238,7 @@ class Installer:
if self._execute_operations or self.is_dry_run():
self._io.writeln(
f' - Installing <info>{operation.package.name}</> '
f' - Installing <info>{operation.package.pretty_name}</> '
f'(<comment>{operation.package.full_pretty_version}</>)'
)
......@@ -254,7 +254,7 @@ class Installer:
if operation.skipped:
if self._io.is_verbose() and (self._execute_operations or self.is_dry_run()):
self._io.writeln(
f' - Skipping <info>{target.name}</> '
f' - Skipping <info>{target.pretty_name}</> '
f'(<comment>{target.full_pretty_version}</>) '
f'{operation.skip_reason}')
......@@ -262,7 +262,7 @@ class Installer:
if self._execute_operations or self.is_dry_run():
self._io.writeln(
f' - Updating <info>{target.name}</> '
f' - Updating <info>{target.pretty_name}</> '
f'(<comment>{source.pretty_version}</>'
f' -> <comment>{target.pretty_version}</>)'
)
......@@ -275,7 +275,7 @@ class Installer:
def _execute_uninstall(self, operation: Uninstall) -> None:
if self._execute_operations or self.is_dry_run():
self._io.writeln(
f' - Removing <info>{operation.package.name}</> '
f' - Removing <info>{operation.package.pretty_name}</> '
f'(<comment>{operation.package.full_pretty_version}</>)'
)
......
poetry/packages/dependency.py
......@@ -33,6 +33,8 @@ class Dependency:
self._platform = '*'
self._platform_constraint = self._parser.parse_constraints('*')
self._extras = []
@property
def name(self):
return self._name
......@@ -73,12 +75,15 @@ class Dependency:
@platform.setter
def platform(self, value: str):
self._platform = value
self._platform_constraint = self._parser.parse_constraints(value)
@property
def platform_constraint(self):
return self._platform_constraint
@property
def extras(self) -> list:
return self._extras
def allows_prereleases(self):
return self._allows_prereleases
......@@ -129,6 +134,12 @@ class Dependency:
return requirement
def activate(self):
"""
Set the dependency as mandatory.
"""
self._optional = False
def __eq__(self, other):
if not isinstance(other, Dependency):
return NotImplemented
......
poetry/packages/package.py
......@@ -63,6 +63,7 @@ class Package:
self.requires = []
self.dev_requires = []
self.extras = {}
self._parser = VersionParser()
......@@ -187,7 +188,7 @@ class Package:
python_versions = constraint.get('python')
platform = constraint.get('platform')
optional = optional or python_versions is not None or not platform is not None
optional = optional or python_versions is not None or platform is not None
dependency = Dependency(
name, version,
......@@ -201,6 +202,10 @@ class Package:
if platform:
dependency.platform = platform
if 'extras' in constraint:
for extra in constraint['extras']:
dependency.extras.append(extra)
else:
dependency = Dependency(name, constraint, category=category)
......
poetry/puzzle/provider.py
......@@ -64,7 +64,8 @@ class Provider(SpecificationProvider):
packages = self._pool.find_packages(
dependency.name,
dependency.constraint
dependency.constraint,
extras=dependency.extras,
)
packages.sort(
......
poetry/repositories/base_repository.py
......@@ -16,7 +16,7 @@ class BaseRepository:
def package(self, name, version):
raise NotImplementedError()
def find_packages(self, name, constraint=None):
def find_packages(self, name, constraint=None, extras=None):
raise NotImplementedError()
def search(self, query, mode=SEARCH_FULLTEXT):
......
poetry/repositories/legacy_repository.py
......@@ -17,10 +17,10 @@ from poetry.semver.constraints import Constraint
from poetry.semver.constraints.base_constraint import BaseConstraint
from poetry.semver.version_parser import VersionParser
from .repository import Repository
from .pypi_repository import PyPiRepository
class LegacyRepository(Repository):
class LegacyRepository(PyPiRepository):
def __init__(self, name, url):
if name == 'pypi':
......@@ -51,9 +51,7 @@ class LegacyRepository(Repository):
}
})
super().__init__()
def find_packages(self, name, constraint=None):
def find_packages(self, name, constraint=None, extras=None):
packages = []
if constraint is not None and not isinstance(constraint,
......@@ -84,11 +82,11 @@ class LegacyRepository(Repository):
self._cache.store('matches').put(key, versions, 5)
for version in versions:
packages.append(self.package(name, version))
packages.append(self.package(name, version, extras=extras))
return packages
def package(self, name, version) -> 'poetry.packages.Package':
def package(self, name, version, extras=None) -> 'poetry.packages.Package':
"""
Retrieve the release information.
......@@ -107,29 +105,78 @@ class LegacyRepository(Repository):
return self._packages[index]
except ValueError:
if extras is None:
extras = []
release_info = self.get_release_info(name, version)
package = poetry.packages.Package(name, version, version)
for dependency in release_info['requires_dist']:
m = re.match(
'^(?P<name>[^ ;]+)'
'(?: \((?P<version>.+)\))?'
'(?:;(?P<extra>(.+)))?$',
dependency
)
package.requires.append(
poetry.packages.Dependency(
m.group('name'),
m.group('version') or '*',
optional=m.group('extra') is not None
for req in release_info['requires_dist']:
req = InstallRequirement.from_line(req)
name = req.name
version = str(req.req.specifier)
dependency = Dependency(
name,
version,
optional=req.markers
)
is_extra = False
if req.markers:
# Setting extra dependencies and requirements
requirements = self._convert_markers(
req.markers._markers
)
package.source_type = 'legacy'
package.source_url = self._url
if 'python_version' in requirements:
ors = []
for or_ in requirements['python_version']:
ands = []
for op, version in or_:
ands.append(f'{op}{version}')
ors.append(' '.join(ands))
dependency.python_versions = ' || '.join(ors)
if 'sys_platform' in requirements:
ors = []
for or_ in requirements['sys_platform']:
ands = []
for op, platform in or_:
ands.append(f'{op}{platform}')
ors.append(' '.join(ands))
dependency.platform = ' || '.join(ors)
if 'extra' in requirements:
is_extra = True
for _extras in requirements['extra']:
for _, extra in _extras:
if extra not in package.extras:
package.extras[extra] = []
package.extras[extra].append(dependency)
if not is_extra:
package.requires.append(dependency)
# Adding description
package.description = release_info.get('summary', '')
# Adding hashes information
package.hashes = release_info['digests']
# Activate extra dependencies
for extra in extras:
if extra in package.extras:
for dep in package.extras[extra]:
dep.activate()
package.requires += package.extras[extra]
self._packages.append(package)
return package
......
poetry/repositories/pool.py
......@@ -4,7 +4,6 @@ from typing import Union
import poetry.packages
from .base_repository import BaseRepository
from .legacy_repository import LegacyRepository
from .repository import Repository
......@@ -38,6 +37,8 @@ class Pool(BaseRepository):
Configures a repository based on a source
specification and add it to the pool.
"""
from .legacy_repository import LegacyRepository
if 'url' in source:
# PyPI-like repository
if 'name' not in source:
......@@ -68,9 +69,10 @@ class Pool(BaseRepository):
def find_packages(self,
name,
constraint=None) -> List['poetry.packages.Package']:
constraint=None,
extras=None) -> List['poetry.packages.Package']:
for repository in self._repositories:
packages = repository.find_packages(name, constraint)
packages = repository.find_packages(name, constraint, extras=extras)
if packages:
return packages
......
poetry/repositories/pypi_repository.py
import re
from pathlib import Path
from pip.req import InstallRequirement
from typing import List
from typing import Union
......@@ -19,8 +18,9 @@ from .repository import Repository
class PyPiRepository(Repository):
def __init__(self, url='https://pypi.org/'):
def __init__(self, url='https://pypi.org/', disable_cache=False):
self._url = url
self._disable_cache = disable_cache
self._cache = CacheManager({
'default': 'releases',
'serializer': 'json',
......@@ -39,7 +39,8 @@ class PyPiRepository(Repository):
def find_packages(self,
name: str,
constraint: Union[Constraint, None] = None
constraint: Union[Constraint, str, None] = None,
extras: Union[list, None] = None
) -> List[Package]:
"""
Find packages on the remote server.
......@@ -62,41 +63,93 @@ class PyPiRepository(Repository):
versions.append(version)
for version in versions:
packages.append(self.package(name, version))
packages.append(
self.package(name, version, extras=extras)
)
return packages
def package(self,
name: str,
version: str) -> Package:
version: str,
extras: Union[list, None] = None) -> Package:
try:
index = self._packages.index(Package(name, version, version))
return self._packages[index]
except ValueError:
if extras is None:
extras = []
release_info = self.get_release_info(name, version)
package = Package(name, version, version)
for dependency in release_info['requires_dist']:
m = re.match(
'^(?P<name>[^ ;]+)'
'(?: \((?P<version>.+)\))?'
'(?:;(?P<extra>(.+)))?$',
dependency
)
package.requires.append(
Dependency(
m.group('name'),
m.group('version') or '*',
optional=m.group('extra') is not None
for req in release_info['requires_dist']:
req = InstallRequirement.from_line(req)
name = req.name
version = str(req.req.specifier)
dependency = Dependency(
name,
version,
optional=req.markers
)
is_extra = False
if req.markers:
# Setting extra dependencies and requirements
requirements = self._convert_markers(
req.markers._markers
)
if 'python_version' in requirements:
ors = []
for or_ in requirements['python_version']:
ands = []
for op, version in or_:
ands.append(f'{op}{version}')
ors.append(' '.join(ands))
dependency.python_versions = ' || '.join(ors)
if 'sys_platform' in requirements:
ors = []
for or_ in requirements['sys_platform']:
ands = []
for op, platform in or_:
ands.append(f'{op}{platform}')
ors.append(' '.join(ands))
dependency.platform = ' || '.join(ors)
if 'extra' in requirements:
is_extra = True
for _extras in requirements['extra']:
for _, extra in _extras:
if extra not in package.extras:
package.extras[extra] = []
package.extras[extra].append(dependency)
if not is_extra:
package.requires.append(dependency)
# Adding description
package.description = release_info.get('summary', '')
# Adding hashes information
package.hashes = release_info['digests']
# Activate extra dependencies
for extra in extras:
if extra in package.extras:
for dep in package.extras[extra]:
dep.activate()
package.requires += package.extras[extra]
self._packages.append(package)
return package
......@@ -130,18 +183,19 @@ class PyPiRepository(Repository):
The information is returned from the cache if it exists
or retrieved from the remote server.
"""
if self._disable_cache:
return self._get_package_info(name)
return self._cache.store('packages').remember_forever(
f'{name}',
lambda: self._get_package_info(name)
)
def _get_package_info(self, name: str) -> dict:
json_response = get(self._url + f'pypi/{name}/json')
if json_response.status_code == 404:
data = self._get(self._url + f'pypi/{name}/json')
if data is None:
raise ValueError(f'Package [{name}] not found.')
data = json_response.json()
return data
def get_release_info(self, name: str, version: str) -> dict:
......@@ -151,17 +205,19 @@ class PyPiRepository(Repository):
The information is returned from the cache if it exists
or retrieved from the remote server.
"""
if self._disable_cache:
return self._get_release_info(name, version)
return self._cache.remember_forever(
f'{name}:{version}',
lambda: self._get_release_info(name, version)
)
def _get_release_info(self, name: str, version: str) -> dict:
json_response = get(self._url + f'pypi/{name}/{version}/json')
if json_response.status_code == 404:
json_data = self._get(self._url + f'pypi/{name}/{version}/json')
if json_data is None:
raise ValueError(f'Package [{name}] not found.')
json_data = json_response.json()
info = json_data['info']
data = {
'name': info['name'],
......@@ -176,3 +232,56 @@ class PyPiRepository(Repository):
data['digests'].append(file_info['digests']['sha256'])
return data
def _get(self, url: str) -> Union[dict, None]:
json_response = get(url)
if json_response.status_code == 404:
return None
json_data = json_response.json()
return json_data
def _group_markers(self, markers):
groups = [[]]
for marker in markers:
assert isinstance(marker, (list, tuple, str))
if isinstance(marker, list):
groups[-1].append(self._group_markers(marker))
elif isinstance(marker, tuple):
lhs, op, rhs = marker
groups[-1].append((lhs.value, op, rhs.value))
else:
assert marker in ["and", "or"]
if marker == "or":
groups.append([])
return groups
def _convert_markers(self, markers):
groups = self._group_markers(markers)[0]
requirements = {}
def _group(_groups, or_=False):
nonlocal requirements
for group in _groups:
if isinstance(group, tuple):
variable, op, value = group
group_name = str(variable)
if group_name not in requirements:
requirements[group_name] = [[]]
elif or_:
requirements[group_name].append([])
requirements[group_name][-1].append((str(op), str(value)))
else:
_group(group, or_=True)
_group(groups)
return requirements
poetry/repositories/repository.py
......@@ -27,9 +27,11 @@ class Repository(BaseRepository):
if name == package.name and package.version == version:
return package
def find_packages(self, name, constraint=None):
def find_packages(self, name, constraint=None, extras=None):
name = name.lower()
packages = []
if extras is None:
extras = []
if not isinstance(constraint, BaseConstraint):
parser = VersionParser()
......@@ -40,6 +42,13 @@ class Repository(BaseRepository):
pkg_constraint = Constraint('==', package.version)
if constraint is None or constraint.matches(pkg_constraint):
for extra in extras:
if extra in package.extras:
for dep in package.extras[extra]:
dep.activate()
package.requires += package.extras[extra]
packages.append(package)
return packages
......
tests/fixtures/sample_project/pyproject.toml
......@@ -21,7 +21,7 @@ keywords = ["packaging", "dependency", "poetry"]
python = "~2.7 || ^3.6"
cleo = "^0.6"
pendulum = { git = "https://github.com/sdispater/pendulum.git", branch = "2.0" }
requests = { version = "^2.18", optional = true }
requests = { version = "^2.18", optional = true, extras=[ "security" ] }
pathlib2 = { version = "^2.2", python = "~2.7" }
[tool.poetry.dev-dependencies]
......
tests/installation/fixtures/with-dependencies-extras.test 0 → 100644
[[package]]
name = "A"
version = "1.0"
description = ""
category = "main"
optional = false
python-versions = "*"
platform = "*"
[[package]]
name = "B"
version = "1.0"
description = ""
category = "main"
optional = false
python-versions = "*"
platform = "*"
[package.dependencies]
C = "^1.0"
[[package]]
name = "C"
version = "1.0"
description = ""
category = "main"
optional = false
python-versions = "*"
platform = "*"
[metadata]
python-versions = "*"
platform = "*"
content-hash = "123456789"
[metadata.hashes]
"A" = []
"B" = []
"C" = []
tests/installation/test_installer.py
......@@ -11,6 +11,7 @@ from poetry.packages import Locker as BaseLocker
from poetry.repositories import Pool
from poetry.repositories import Repository
from tests.helpers import get_dependency
from tests.helpers import get_package
......@@ -287,3 +288,25 @@ def test_run_with_optional_and_python_restricted_dependencies(installer, locker,
# with B's python constraint
assert len(installer.installs) == 3
def test_run_with_dependencies_extras(installer, locker, repo, package):
package_a = get_package('A', '1.0')
package_b = get_package('B', '1.0')
package_c = get_package('C', '1.0')
package_b.extras = {
'foo': [get_dependency('C', '^1.0')]
}
repo.add_package(package_a)
repo.add_package(package_b)
repo.add_package(package_c)
package.add_dependency('A', '^1.0')
package.add_dependency('B', {'version': '^1.0', 'extras': ['foo']})
installer.run()
expected = fixture('with-dependencies-extras')
assert locker.written_data == expected
tests/puzzle/test_solver.py
......@@ -324,3 +324,61 @@ def test_solver_solves_optional_and_compatible_packages(solver, repo, package):
{'job': 'install', 'package': package_b},
{'job': 'install', 'package': package_a},
])
def test_solver_does_not_return_extras_if_not_requested(solver, repo, package):
package_a = get_package('A', '1.0')
package_b = get_package('B', '1.0')
package_c = get_package('C', '1.0')
package_b.extras = {
'foo': [get_dependency('C', '^1.0')]
}
repo.add_package(package_a)
repo.add_package(package_b)
repo.add_package(package_c)
dependency_a = get_dependency('A')
dependency_b = get_dependency('B')
request = [
dependency_a,
dependency_b
]
ops = solver.solve(request)
check_solver_result(ops, [
{'job': 'install', 'package': package_b},
{'job': 'install', 'package': package_a},
])
def test_solver_returns_extras_if_requested(solver, repo, package):
package_a = get_package('A', '1.0')
package_b = get_package('B', '1.0')
package_c = get_package('C', '1.0')
package_b.extras = {
'foo': [get_dependency('C', '^1.0')]
}
repo.add_package(package_a)
repo.add_package(package_b)
repo.add_package(package_c)
dependency_a = get_dependency('A')
dependency_b = get_dependency('B')
dependency_b.extras.append('foo')
request = [
dependency_a,
dependency_b
]
ops = solver.solve(request)
check_solver_result(ops, [
{'job': 'install', 'package': package_c},
{'job': 'install', 'package': package_b},
{'job': 'install', 'package': package_a},
])
tests/repositories/fixtures/pypi.org/json/requests.json 0 → 100644
This source diff could not be displayed because it is too large. You can view the blob instead.
tests/repositories/test_pypi_repository.py 0 → 100644
import json
from pathlib import Path
from poetry.repositories.pypi_repository import PyPiRepository
class MockRepository(PyPiRepository):
FIXTURES = Path(__file__).parent / 'fixtures' / 'pypi.org' / 'json'
def __init__(self):
super().__init__(url='http://foo.bar', disable_cache=True)
def _get(self, url: str) -> dict:
fixture = self.FIXTURES / 'requests.json'
with fixture.open() as f:
return json.loads(f.read())
def test_find_packages():
repo = MockRepository()
packages = repo.find_packages('requests', '^2.18')
assert len(packages) == 5
def test_package():
repo = MockRepository()
package = repo.package('requests', '2.18.4')
assert package.name == 'requests'
assert len(package.requires) == 4
assert len(package.extras['security']) == 3
assert len(package.extras['socks']) == 2
win_inet = package.extras['socks'][0]
assert win_inet.name == 'win-inet-pton'
assert win_inet.python_versions == '==2.7 || ==2.6'
assert win_inet.platform == '==win32'
tests/test_poetry.py
......@@ -44,6 +44,7 @@ def test_poetry():
assert not requests.is_vcs()
assert not requests.allows_prereleases()
assert requests.is_optional()
assert requests.extras == ['security']
pathlib2 = dependencies[3]
assert pathlib2.pretty_constraint == '^2.2'
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment