Commit 658590c5 by Sébastien Eustace

Fix handling of duplicate dependencies with different constraints

parent a5186e60
...@@ -78,6 +78,9 @@ class DebugResolveCommand(Command): ...@@ -78,6 +78,9 @@ class DebugResolveCommand(Command):
package.name, package.version package.name, package.version
) )
) )
if package.requirements:
for req_name, req_value in package.requirements.items():
self.line(" - {}: {}".format(req_name, req_value))
def _determine_requirements(self, requires): # type: (List[str]) -> List[str] def _determine_requirements(self, requires): # type: (List[str]) -> List[str]
if not requires: if not requires:
......
...@@ -3,6 +3,7 @@ import sys ...@@ -3,6 +3,7 @@ import sys
from typing import List from typing import List
from typing import Union from typing import Union
from poetry.io import NullIO
from poetry.packages import Dependency from poetry.packages import Dependency
from poetry.packages import Locker from poetry.packages import Locker
from poetry.packages import Package from poetry.packages import Package
...@@ -182,6 +183,23 @@ class Installer: ...@@ -182,6 +183,23 @@ class Installer:
self._populate_local_repo(local_repo, ops, locked_repository) self._populate_local_repo(local_repo, ops, locked_repository)
with self._package.with_python_versions(
".".join([str(i) for i in self._venv.version_info[:3]])
):
# We resolve again by only using the lock file
pool = Pool()
pool.add_repository(local_repo)
solver = Solver(
self._package,
pool,
self._installed_repository,
locked_repository,
NullIO(),
)
ops = solver.solve()
# We need to filter operations so that packages # We need to filter operations so that packages
# not compatible with the current system, # not compatible with the current system,
# or optional and not requested, are dropped # or optional and not requested, are dropped
...@@ -365,13 +383,17 @@ class Installer: ...@@ -365,13 +383,17 @@ class Installer:
local_repo.remove_package(pkg) local_repo.remove_package(pkg)
local_repo.add_package(op.target_package) local_repo.add_package(op.target_package)
elif op.job_type == "uninstall": elif op.job_type == "uninstall":
local_repo.remove_package(op.package) if pkg.version == package.version:
local_repo.remove_package(op.package)
else: else:
# Even though the package already exists # Even though the package already exists
# in the lock file we will prefer the new one # in the lock file we will prefer the new one
# to force updates # to force updates
local_repo.remove_package(pkg) if pkg.version == package.version:
local_repo.add_package(package) local_repo.remove_package(pkg)
local_repo.add_package(package)
else:
local_repo.add_package(package)
acted_on = True acted_on = True
......
...@@ -4,5 +4,4 @@ from .version_solver import VersionSolver ...@@ -4,5 +4,4 @@ from .version_solver import VersionSolver
def resolve_version(root, provider, locked=None, use_latest=None): def resolve_version(root, provider, locked=None, use_latest=None):
solver = VersionSolver(root, provider, locked=locked, use_latest=use_latest) solver = VersionSolver(root, provider, locked=locked, use_latest=use_latest)
with provider.progress(): return solver.solve()
return solver.solve()
...@@ -174,7 +174,25 @@ class Locker: ...@@ -174,7 +174,25 @@ class Locker:
if dependency.is_optional() and not dependency.is_activated(): if dependency.is_optional() and not dependency.is_activated():
continue continue
dependencies[dependency.pretty_name] = str(dependency.pretty_constraint) if dependency.pretty_name not in dependencies:
dependencies[dependency.pretty_name] = []
constraint = {"version": str(dependency.pretty_constraint)}
if not dependency.python_constraint.is_any():
constraint["python"] = str(dependency.python_constraint)
if dependency.platform != "*":
constraint["platform"] = dependency.platform
if len(constraint) == 1:
dependencies[dependency.pretty_name].append(constraint["version"])
else:
dependencies[dependency.pretty_name].append(constraint)
for name, constraints in dependencies.items():
if len(constraints) == 1:
dependencies[name] = constraints[0]
data = { data = {
"name": package.pretty_name, "name": package.pretty_name,
......
...@@ -2,6 +2,7 @@ ...@@ -2,6 +2,7 @@
import copy import copy
import re import re
from contextlib import contextmanager
from typing import Union from typing import Union
from poetry.semver import Version from poetry.semver import Version
...@@ -309,6 +310,29 @@ class Package(object): ...@@ -309,6 +310,29 @@ class Package(object):
def to_dependency(self): def to_dependency(self):
return Dependency(self.name, self._version) return Dependency(self.name, self._version)
@contextmanager
def with_python_versions(self, python_versions):
original_python_versions = self.python_versions
self.python_versions = python_versions
yield
self.python_versions = original_python_versions
def clone(self): # type: () -> Package
clone = Package(self.pretty_name, self.version)
clone.category = self.category
clone.optional = self.optional
clone.python_versions = self.python_versions
clone.platform = self.platform
clone.extras = self.extras
for dep in self.requires:
clone.requires.append(dep)
return clone
def __hash__(self): def __hash__(self):
return hash((self._name, self._version)) return hash((self._name, self._version))
......
class CompatibilityError(Exception):
def __init__(self, *constraints):
self._constraints = list(constraints)
@property
def constraints(self):
return self._constraints
class SolverProblemError(Exception): class SolverProblemError(Exception):
def __init__(self, error): def __init__(self, error):
self._error = error self._error = error
......
import logging
import os import os
import pkginfo import pkginfo
import shutil import shutil
...@@ -33,6 +34,10 @@ from poetry.utils.venv import Venv ...@@ -33,6 +34,10 @@ from poetry.utils.venv import Venv
from poetry.vcs.git import Git from poetry.vcs.git import Git
from .dependencies import Dependencies from .dependencies import Dependencies
from .exceptions import CompatibilityError
logger = logging.getLogger(__name__)
class Indicator(ProgressIndicator): class Indicator(ProgressIndicator):
...@@ -65,6 +70,7 @@ class Provider: ...@@ -65,6 +70,7 @@ class Provider:
self._python_constraint = package.python_constraint self._python_constraint = package.python_constraint
self._search_for = {} self._search_for = {}
self._is_debugging = self._io.is_debug() or self._io.is_very_verbose() self._is_debugging = self._io.is_debug() or self._io.is_very_verbose()
self._in_progress = False
@property @property
def pool(self): # type: () -> Pool def pool(self): # type: () -> Pool
...@@ -321,9 +327,8 @@ class Provider: ...@@ -321,9 +327,8 @@ class Provider:
# will become: # will become:
# - enum34; python_version=="2.7" or python_version=="3.3" # - enum34; python_version=="2.7" or python_version=="3.3"
# #
# TODO: If the duplicate dependencies have different constraints # If the duplicate dependencies have different constraints
# we should notify the resolver in some way to make it split the # we have to split the dependency graph.
# current graph.
# #
# An example of this is: # An example of this is:
# - pypiwin32 (220); sys_platform == "win32" and python_version >= "3.6" # - pypiwin32 (220); sys_platform == "win32" and python_version >= "3.6"
...@@ -342,6 +347,8 @@ class Provider: ...@@ -342,6 +347,8 @@ class Provider:
dependencies.append(deps[0]) dependencies.append(deps[0])
continue continue
self.debug("Duplicate dependencies for {}".format(dep_name))
# Regrouping by constraint # Regrouping by constraint
by_constraint = {} by_constraint = {}
for dep in deps: for dep in deps:
...@@ -380,15 +387,24 @@ class Provider: ...@@ -380,15 +387,24 @@ class Provider:
continue continue
if len(by_constraint) == 1: if len(by_constraint) == 1:
self.debug("Merging requirements for {}".format(str(deps[0])))
dependencies.append(list(by_constraint.values())[0][0]) dependencies.append(list(by_constraint.values())[0][0])
continue continue
# At this point, we have one dependency by constraint # At this point, we raise an exception that will
# So we add them to the dependency set # tell the solver to enter compatibility mode
# which means it will resolve for each minor
# Python version supported
python_constraints = []
for constraint, _deps in by_constraint.items(): for constraint, _deps in by_constraint.items():
_dep = _deps[0] python_constraints.append(_deps[0].python_versions)
dependencies.append(_dep) self.debug(
"<warning>Uncompatible constraints for {}.</warning>".format(
dep_name
)
)
raise CompatibilityError(*python_constraints)
package.requires = dependencies package.requires = dependencies
...@@ -400,19 +416,6 @@ class Provider: ...@@ -400,19 +416,6 @@ class Provider:
def output(self): def output(self):
return self._io return self._io
def before_resolution(self):
self._io.write("<info>Resolving dependencies</>")
if self.is_debugging():
self._io.new_line()
def indicate_progress(self):
if not self.is_debugging():
self._io.write(".")
def after_resolution(self):
self._io.new_line()
def debug(self, message, depth=0): def debug(self, message, depth=0):
if self.is_debugging(): if self.is_debugging():
debug_info = str(message) debug_info = str(message)
...@@ -438,3 +441,5 @@ class Provider: ...@@ -438,3 +441,5 @@ class Provider:
with indicator.auto(): with indicator.auto():
yield yield
self._in_progress = False
...@@ -6,6 +6,7 @@ from poetry.packages.constraints.generic_constraint import GenericConstraint ...@@ -6,6 +6,7 @@ from poetry.packages.constraints.generic_constraint import GenericConstraint
from poetry.semver import parse_constraint from poetry.semver import parse_constraint
from .exceptions import CompatibilityError
from .exceptions import SolverProblemError from .exceptions import SolverProblemError
from .operations import Install from .operations import Install
...@@ -23,41 +24,13 @@ class Solver: ...@@ -23,41 +24,13 @@ class Solver:
self._installed = installed self._installed = installed
self._locked = locked self._locked = locked
self._io = io self._io = io
self._provider = Provider(self._package, self._pool, self._io)
def solve(self, use_latest=None): # type: (...) -> List[Operation] def solve(self, use_latest=None): # type: (...) -> List[Operation]
provider = Provider(self._package, self._pool, self._io) with self._provider.progress():
locked = {} packages = self._solve(use_latest=use_latest)
for package in self._locked.packages:
locked[package.name] = package
try:
result = resolve_version(
self._package, provider, locked=locked, use_latest=use_latest
)
except SolveFailure as e:
raise SolverProblemError(e)
packages = result.packages
requested = self._package.all_requires requested = self._package.all_requires
graph = self._build_graph(self._package, packages)
for package in packages:
category, optional, python, platform = self._get_tags_for_package(
package, graph
)
package.category = category
package.optional = optional
# If requirements are empty, drop them
requirements = {}
if python is not None and python != "*":
requirements["python"] = python
if platform is not None and platform != "*":
requirements["platform"] = platform
package.requirements = requirements
operations = [] operations = []
for package in packages: for package in packages:
...@@ -104,9 +77,74 @@ class Solver: ...@@ -104,9 +77,74 @@ class Solver:
key=lambda o: ( key=lambda o: (
1 if o.package.name in requested_names else 0, 1 if o.package.name in requested_names else 0,
o.package.name, o.package.name,
o.package.version,
), ),
) )
def solve_in_compatibility_mode(self, constraints, use_latest=None):
locked = {}
for package in self._locked.packages:
locked[package.name] = package
packages = []
for constraint in constraints:
constraint = parse_constraint(constraint)
intersection = constraint.intersect(self._package.python_constraint)
with self._package.with_python_versions(str(intersection)):
for package in self._solve(use_latest=use_latest):
if package not in packages:
packages.append(package)
continue
current_package = packages[packages.index(package)]
for dep in package.requires:
if dep not in current_package.requires:
current_package.requires.append(dep)
return list(set(packages))
def _solve(self, use_latest=None):
locked = {}
for package in self._locked.packages:
locked[package.name] = package
try:
result = resolve_version(
self._package, self._provider, locked=locked, use_latest=use_latest
)
packages = result.packages
except CompatibilityError as e:
return self.solve_in_compatibility_mode(
e.constraints, use_latest=use_latest
)
except SolveFailure as e:
raise SolverProblemError(e)
requested = self._package.all_requires
graph = self._build_graph(self._package, packages)
for package in packages:
category, optional, python, platform = self._get_tags_for_package(
package, graph
)
package.category = category
package.optional = optional
# If requirements are empty, drop them
requirements = {}
if python is not None and python != "*":
requirements["python"] = python
if platform is not None and platform != "*":
requirements["platform"] = platform
package.requirements = requirements
return packages
def _build_graph(self, package, packages, previous=None, dep=None): def _build_graph(self, package, packages, previous=None, dep=None):
if not previous: if not previous:
category = "dev" category = "dev"
......
...@@ -62,10 +62,6 @@ class Pool(BaseRepository): ...@@ -62,10 +62,6 @@ class Pool(BaseRepository):
raise NotImplementedError() raise NotImplementedError()
def package(self, name, version, extras=None): def package(self, name, version, extras=None):
package = poetry.packages.Package(name, version, version)
if package in self._packages:
return self._packages[self._packages.index(package)]
for repository in self._repositories: for repository in self._repositories:
package = repository.package(name, version, extras=extras) package = repository.package(name, version, extras=extras)
if package: if package:
......
...@@ -130,68 +130,61 @@ class PyPiRepository(Repository): ...@@ -130,68 +130,61 @@ class PyPiRepository(Repository):
version, # type: str version, # type: str
extras=None, # type: (Union[list, None]) extras=None, # type: (Union[list, None])
): # type: (...) -> Union[Package, None] ): # type: (...) -> Union[Package, None]
try: if extras is None:
index = self._packages.index(Package(name, version, version)) extras = []
return self._packages[index]
except ValueError:
if extras is None:
extras = []
release_info = self.get_release_info(name, version)
package = Package(name, version, version)
requires_dist = release_info["requires_dist"] or []
for req in requires_dist:
try:
dependency = dependency_from_pep_508(req)
except InvalidMarker:
# Invalid marker
# We strip the markers hoping for the best
req = req.split(";")[0]
dependency = dependency_from_pep_508(req)
except ValueError:
# Likely unable to parse constraint so we skip it
self._log(
"Invalid constraint ({}) found in {}-{} dependencies, "
"skipping".format(req, package.name, package.version),
level="debug",
)
continue
if dependency.extras: release_info = self.get_release_info(name, version)
for extra in dependency.extras: package = Package(name, version, version)
if extra not in package.extras: requires_dist = release_info["requires_dist"] or []
package.extras[extra] = [] for req in requires_dist:
try:
dependency = dependency_from_pep_508(req)
except InvalidMarker:
# Invalid marker
# We strip the markers hoping for the best
req = req.split(";")[0]
dependency = dependency_from_pep_508(req)
except ValueError:
# Likely unable to parse constraint so we skip it
self._log(
"Invalid constraint ({}) found in {}-{} dependencies, "
"skipping".format(req, package.name, package.version),
level="debug",
)
continue
package.extras[extra].append(dependency) if dependency.extras:
for extra in dependency.extras:
if extra not in package.extras:
package.extras[extra] = []
if not dependency.is_optional(): package.extras[extra].append(dependency)
package.requires.append(dependency)
# Adding description if not dependency.is_optional():
package.description = release_info.get("summary", "") package.requires.append(dependency)
if release_info["requires_python"]: # Adding description
package.python_versions = release_info["requires_python"] package.description = release_info.get("summary", "")
if release_info["platform"]: if release_info["requires_python"]:
package.platform = release_info["platform"] package.python_versions = release_info["requires_python"]
# Adding hashes information if release_info["platform"]:
package.hashes = release_info["digests"] package.platform = release_info["platform"]
# Activate extra dependencies # Adding hashes information
for extra in extras: package.hashes = release_info["digests"]
if extra in package.extras:
for dep in package.extras[extra]:
dep.activate()
package.requires += package.extras[extra] # Activate extra dependencies
for extra in extras:
if extra in package.extras:
for dep in package.extras[extra]:
dep.activate()
self._packages.append(package) package.requires += package.extras[extra]
return package return package
def search(self, query, mode=0): def search(self, query, mode=0):
results = [] results = []
......
...@@ -30,7 +30,7 @@ class Repository(BaseRepository): ...@@ -30,7 +30,7 @@ class Repository(BaseRepository):
if dep.name == extra_dep.lower(): if dep.name == extra_dep.lower():
dep.activate() dep.activate()
return package return package.clone()
def find_packages( def find_packages(
self, name, constraint=None, extras=None, allow_prereleases=False self, name, constraint=None, extras=None, allow_prereleases=False
......
...@@ -9,7 +9,7 @@ platform = "*" ...@@ -9,7 +9,7 @@ platform = "*"
[package.dependencies] [package.dependencies]
"B" = "^1.0" "B" = "^1.0"
"C" = "^1.0" "C" = {"version" = "^1.0", "python" = ">=2.7,<2.8"}
[[package]] [[package]]
name = "B" name = "B"
......
[[package]]
name = "A"
version = "1.0"
description = ""
category = "main"
optional = false
python-versions = "*"
platform = "*"
[package.dependencies]
B = [
{"version" = "^1.0", "python" = "<4.0"},
{"version" = "^2.0", "python" = ">=4.0"},
]
[[package]]
name = "B"
version = "1.0"
description = ""
category = "main"
optional = false
python-versions = "*"
platform = "*"
[package.dependencies]
C = "1.2"
[package.requirements]
python = "<4.0"
[[package]]
name = "B"
version = "2.0"
description = ""
category = "main"
optional = false
python-versions = "*"
platform = "*"
[package.dependencies]
C = "1.5"
[package.requirements]
python = ">=4.0"
[[package]]
name = "C"
version = "1.2"
description = ""
category = "main"
optional = false
python-versions = "*"
platform = "*"
[[package]]
name = "C"
version = "1.5"
description = ""
category = "main"
optional = false
python-versions = "*"
platform = "*"
[metadata]
python-versions = "*"
platform = "*"
content-hash = "123456789"
[metadata.hashes]
A = []
B = []
C = []
...@@ -77,8 +77,8 @@ setuptools = "*" ...@@ -77,8 +77,8 @@ setuptools = "*"
attrs = ">=17.4.0" attrs = ">=17.4.0"
more-itertools = ">=4.0.0" more-itertools = ">=4.0.0"
pluggy = ">=0.5,<0.7" pluggy = ">=0.5,<0.7"
funcsigs = "*" funcsigs = {"version" = "*", "python" = "<3.0"}
colorama = "*" colorama = {"version" = "*", "platform" = "win32"}
[[package]] [[package]]
name = "setuptools" name = "setuptools"
......
...@@ -22,7 +22,7 @@ version = "1.2" ...@@ -22,7 +22,7 @@ version = "1.2"
description = "" description = ""
category = "main" category = "main"
optional = false optional = false
python-versions = "~2.7 || ^3.6" python-versions = "~2.7 || ^3.3"
platform = "*" platform = "*"
[metadata] [metadata]
......
...@@ -279,7 +279,7 @@ def test_run_with_python_versions(installer, locker, repo, package): ...@@ -279,7 +279,7 @@ def test_run_with_python_versions(installer, locker, repo, package):
package_a = get_package("A", "1.0") package_a = get_package("A", "1.0")
package_b = get_package("B", "1.1") package_b = get_package("B", "1.1")
package_c12 = get_package("C", "1.2") package_c12 = get_package("C", "1.2")
package_c12.python_versions = "~2.7 || ^3.6" package_c12.python_versions = "~2.7 || ^3.3"
package_c13 = get_package("C", "1.3") package_c13 = get_package("C", "1.3")
package_c13.python_versions = "~3.3" package_c13.python_versions = "~3.3"
...@@ -740,3 +740,39 @@ def test_run_update_with_locked_extras(installer, locker, repo, package): ...@@ -740,3 +740,39 @@ def test_run_update_with_locked_extras(installer, locker, repo, package):
expected = fixture("update-with-locked-extras") expected = fixture("update-with-locked-extras")
assert locker.written_data == expected assert locker.written_data == expected
def test_run_install_duplicate_dependencies_different_constraints(
installer, locker, repo, package
):
package.add_dependency("A")
package_a = get_package("A", "1.0")
package_a.add_dependency("B", {"version": "^1.0", "python": "<4.0"})
package_a.add_dependency("B", {"version": "^2.0", "python": ">=4.0"})
package_b10 = get_package("B", "1.0")
package_b20 = get_package("B", "2.0")
package_b10.add_dependency("C", "1.2")
package_b20.add_dependency("C", "1.5")
package_c12 = get_package("C", "1.2")
package_c15 = get_package("C", "1.5")
repo.add_package(package_a)
repo.add_package(package_b10)
repo.add_package(package_b20)
repo.add_package(package_c12)
repo.add_package(package_c15)
installer.run()
expected = fixture("with-duplicate-dependencies")
assert locker.written_data == expected
installs = installer.installer.installs
assert len(installs) == 3
assert installs[0] == package_b10
assert installs[1] == package_c12
assert installs[2] == package_a
...@@ -284,9 +284,9 @@ def test_solver_sets_categories(solver, repo, package): ...@@ -284,9 +284,9 @@ def test_solver_sets_categories(solver, repo, package):
], ],
) )
assert package_c.category == "dev" assert ops[0].package.category == "dev"
assert package_b.category == "dev" assert ops[2].package.category == "dev"
assert package_a.category == "main" assert ops[1].package.category == "main"
def test_solver_respects_root_package_python_versions(solver, repo, package): def test_solver_respects_root_package_python_versions(solver, repo, package):
...@@ -785,3 +785,76 @@ def test_solver_duplicate_dependencies_same_constraint(solver, repo, package): ...@@ -785,3 +785,76 @@ def test_solver_duplicate_dependencies_same_constraint(solver, repo, package):
op = ops[0] op = ops[0]
assert op.package.requirements == {"python": "~2.7 || >=3.4"} assert op.package.requirements == {"python": "~2.7 || >=3.4"}
def test_solver_duplicate_dependencies_different_constraints(solver, repo, package):
package.add_dependency("A")
package_a = get_package("A", "1.0")
package_a.add_dependency("B", {"version": "^1.0", "python": "<3.4"})
package_a.add_dependency("B", {"version": "^2.0", "python": ">=3.4"})
package_b10 = get_package("B", "1.0")
package_b20 = get_package("B", "2.0")
repo.add_package(package_a)
repo.add_package(package_b10)
repo.add_package(package_b20)
ops = solver.solve()
check_solver_result(
ops,
[
{"job": "install", "package": package_b10},
{"job": "install", "package": package_b20},
{"job": "install", "package": package_a},
],
)
op = ops[0]
assert op.package.requirements == {"python": "<3.4"}
op = ops[1]
assert op.package.requirements == {"python": ">=3.4"}
def test_solver_duplicate_dependencies_sub_dependencies(solver, repo, package):
package.add_dependency("A")
package_a = get_package("A", "1.0")
package_a.add_dependency("B", {"version": "^1.0", "python": "<3.4"})
package_a.add_dependency("B", {"version": "^2.0", "python": ">=3.4"})
package_b10 = get_package("B", "1.0")
package_b20 = get_package("B", "2.0")
package_b10.add_dependency("C", "1.2")
package_b20.add_dependency("C", "1.5")
package_c12 = get_package("C", "1.2")
package_c15 = get_package("C", "1.5")
repo.add_package(package_a)
repo.add_package(package_b10)
repo.add_package(package_b20)
repo.add_package(package_c12)
repo.add_package(package_c15)
ops = solver.solve()
check_solver_result(
ops,
[
{"job": "install", "package": package_b10},
{"job": "install", "package": package_b20},
{"job": "install", "package": package_c12},
{"job": "install", "package": package_c15},
{"job": "install", "package": package_a},
],
)
op = ops[0]
assert op.package.requirements == {"python": "<3.4"}
op = ops[1]
assert op.package.requirements == {"python": ">=3.4"}
...@@ -27,7 +27,6 @@ from poetry.semver import VersionRange ...@@ -27,7 +27,6 @@ from poetry.semver import VersionRange
) )
def test_parse_valid(input, version): def test_parse_valid(input, version):
parsed = Version.parse(input) parsed = Version.parse(input)
print(parsed.build)
assert parsed == version assert parsed == version
assert parsed.text == input assert parsed.text == input
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment