Skip to content

P
python-poetry
Commit f317b003 by Sébastien Eustace
Options

Improve dependency resolution debugging

parent 92d0eddf
Showing with 133 additions and 65 deletions
poetry/mixology/partial_solution.py
...@@ -4,6 +4,9 @@ from typing import Any ...@@ -4,6 +4,9 @@ from typing import Any
from typing import Dict from typing import Dict
from typing import List from typing import List
from poetry.packages import Dependency
from poetry.packages import Package
from .assignment import Assignment from .assignment import Assignment
from .incompatibility import Incompatibility from .incompatibility import Incompatibility
from .set_relation import SetRelation from .set_relation import SetRelation
...@@ -25,7 +28,7 @@ class PartialSolution: ...@@ -25,7 +28,7 @@ class PartialSolution:
self._assignments = [] # type: List[Assignment] self._assignments = [] # type: List[Assignment]
# The decisions made for each package. # The decisions made for each package.
self._decisions = OrderedDict() # type: Dict[str, Any] self._decisions = OrderedDict() # type: Dict[str, Package]
# The intersection of all positive Assignments for each package, minus any # The intersection of all positive Assignments for each package, minus any
# negative Assignments that refer to that package. # negative Assignments that refer to that package.
...@@ -33,38 +36,41 @@ class PartialSolution: ...@@ -33,38 +36,41 @@ class PartialSolution:
# This is derived from self._assignments. # This is derived from self._assignments.
self._positive = OrderedDict() # type: Dict[str, Term] self._positive = OrderedDict() # type: Dict[str, Term]
# The union of all negative [Assignment]s for each package. # The union of all negative Assignments for each package.
# #
# If a package has any positive [Assignment]s, it doesn't appear in this # If a package has any positive Assignments, it doesn't appear in this
# map. # map.
# #
# This is derived from self._assignments. # This is derived from self._assignments.
self._negative = OrderedDict() # type: Dict[str, Dict[str, Term]] self._negative = OrderedDict() # type: Dict[str, Dict[str, Term]]
# The number of distinct solutions that have been attempted so far.
self._attempted_solutions = 1 self._attempted_solutions = 1
# Whether the solver is currently backtracking.
self._backtracking = False self._backtracking = False
@property @property
def decisions(self): # type: () -> Any def decisions(self): # type: () -> List[Package]
return list(self._decisions.values()) return list(self._decisions.values())
@property @property
def decision_level(self): def decision_level(self): # type: () -> int
return len(self._decisions) return len(self._decisions)
@property @property
def attempted_solutions(self): def attempted_solutions(self): # type: () -> int
return self._attempted_solutions return self._attempted_solutions
@property @property
def unsatisfied(self): def unsatisfied(self): # type: () -> List[Dependency]
return [ return [
term.dependency term.dependency
for term in self._positive.values() for term in self._positive.values()
if term.dependency.name not in self._decisions if term.dependency.name not in self._decisions
] ]
def decide(self, package): # type: (Any) -> None def decide(self, package): # type: (Package) -> None
""" """
Adds an assignment of package as a decision Adds an assignment of package as a decision
and increments the decision level. and increments the decision level.
...@@ -82,7 +88,7 @@ class PartialSolution: ...@@ -82,7 +88,7 @@ class PartialSolution:
self._assign(Assignment.decision(package, self.decision_level, len(self._assignments))) self._assign(Assignment.decision(package, self.decision_level, len(self._assignments)))
def derive(self, dependency, is_positive, cause def derive(self, dependency, is_positive, cause
): # type: (Any, bool, Incompatibility) -> None ): # type: (Dependency, bool, Incompatibility) -> None
""" """
Adds an assignment of package as a derivation. Adds an assignment of package as a derivation.
""" """
...@@ -101,6 +107,10 @@ class PartialSolution: ...@@ -101,6 +107,10 @@ class PartialSolution:
self._register(assignment) self._register(assignment)
def backtrack(self, decision_level): # type: (int) -> None def backtrack(self, decision_level): # type: (int) -> None
"""
Resets the current decision level to decision_level, and removes all
assignments made after that level.
"""
self._backtracking = True self._backtracking = True
packages = set() packages = set()
......
poetry/mixology/term.py
# -*- coding: utf-8 -*- # -*- coding: utf-8 -*-
from typing import Union
from poetry.packages import Dependency from poetry.packages import Dependency
from .set_relation import SetRelation from .set_relation import SetRelation
...@@ -8,9 +10,14 @@ class Term(object): ...@@ -8,9 +10,14 @@ class Term(object):
""" """
A statement about a package which is true or false for a given selection of A statement about a package which is true or false for a given selection of
package versions. package versions.
See https://github.com/dart-lang/pub/tree/master/doc/solver.md#term.
""" """
def __init__(self, dependency, is_positive): def __init__(self,
dependency, # type: Dependency
is_positive # type: bool
):
self._dependency = dependency self._dependency = dependency
self._positive = is_positive self._positive = is_positive
...@@ -100,7 +107,7 @@ class Term(object): ...@@ -100,7 +107,7 @@ class Term(object):
# not foo ^1.5.0 is a superset of not foo ^1.0.0 # not foo ^1.5.0 is a superset of not foo ^1.0.0
return SetRelation.OVERLAPPING return SetRelation.OVERLAPPING
def intersect(self, other): # type: (Term) -> Term def intersect(self, other): # type: (Term) -> Union[Term, None]
""" """
Returns a Term that represents the packages Returns a Term that represents the packages
allowed by both this term and another allowed by both this term and another
......
poetry/mixology/version_solver.py
...@@ -6,6 +6,9 @@ from typing import List ...@@ -6,6 +6,9 @@ from typing import List
from typing import Union from typing import Union
from poetry.packages import Dependency from poetry.packages import Dependency
from poetry.packages import ProjectPackage
from poetry.packages import Package
from poetry.puzzle.provider import Provider
from poetry.semver import Version from poetry.semver import Version
from poetry.semver import VersionRange from poetry.semver import VersionRange
...@@ -25,8 +28,20 @@ _conflict = object() ...@@ -25,8 +28,20 @@ _conflict = object()
class VersionSolver: class VersionSolver:
"""
def __init__(self, root, provider, locked=None, use_latest=None): The version solver that finds a set of package versions that satisfy the
root package's dependencies.
See https://github.com/dart-lang/pub/tree/master/doc/solver.md for details
on how this solver works.
"""
def __init__(self,
root, # type: ProjectPackage
provider, # type: Provider
locked=None, # type: Dict[str, Package]
use_latest=None # type: List[str]
):
self._root = root self._root = root
self._provider = provider self._provider = provider
self._locked = locked or {} self._locked = locked or {}
...@@ -43,7 +58,7 @@ class VersionSolver: ...@@ -43,7 +58,7 @@ class VersionSolver:
def solution(self): # type: () -> PartialSolution def solution(self): # type: () -> PartialSolution
return self._solution return self._solution
def solve(self): def solve(self): # type: () -> SolverResult
""" """
Finds a set of dependencies that match the root package's constraints, Finds a set of dependencies that match the root package's constraints,
or raises an error if no such set is available. or raises an error if no such set is available.
...@@ -97,9 +112,11 @@ class VersionSolver: ...@@ -97,9 +112,11 @@ class VersionSolver:
result = self._propagate_incompatibility(incompatibility) result = self._propagate_incompatibility(incompatibility)
if result is _conflict: if result is _conflict:
# If [incompatibility] is satisfied by [_solution], we use # If the incompatibility is satisfied by the solution, we use
# [_resolveConflict] to determine the root cause of the conflict as a # _resolve_conflict() to determine the root cause of the conflict as a
# new incompatibility. It also backjumps to a point in [_solution] # new incompatibility.
#
# It also backjumps to a point in the solution
# where that incompatibility will allow us to derive new assignments # where that incompatibility will allow us to derive new assignments
# that avoid the conflict. # that avoid the conflict.
root_cause = self._resolve_conflict(incompatibility) root_cause = self._resolve_conflict(incompatibility)
...@@ -114,46 +131,48 @@ class VersionSolver: ...@@ -114,46 +131,48 @@ class VersionSolver:
changed.add(result) changed.add(result)
def _propagate_incompatibility(self, incompatibility def _propagate_incompatibility(self, incompatibility
): # type: (Incompatibility) -> Union[str, None] ): # type: (Incompatibility) -> Union[str, _conflict, None]
""" """
If [incompatibility] is [almost satisfied][] by [_solution], adds the If incompatibility is almost satisfied by _solution, adds the
negation of the unsatisfied term to [_solution]. negation of the unsatisfied term to _solution.
If incompatibility is satisfied by _solution, returns _conflict. If
incompatibility is almost satisfied by _solution, returns the
unsatisfied term's package name.
If [incompatibility] is satisfied by [_solution], returns `#conflict`. If Otherwise, returns None.
[incompatibility] is almost satisfied by [_solution], returns the
unsatisfied term's package name. Otherwise, returns None.
""" """
# The first entry in `incompatibility.terms` that's not yet satisfied by # The first entry in incompatibility.terms that's not yet satisfied by
# [_solution], if one exists. If we find more than one, [_solution] is # _solution, if one exists. If we find more than one, _solution is
# inconclusive for [incompatibility] and we can't deduce anything. # inconclusive for incompatibility and we can't deduce anything.
unsatisfied = None unsatisfied = None
for term in incompatibility.terms: for term in incompatibility.terms:
relation = self._solution.relation(term) relation = self._solution.relation(term)
if relation == SetRelation.DISJOINT: if relation == SetRelation.DISJOINT:
# If [term] is already contradicted by [_solution], then # If term is already contradicted by _solution, then
# [incompatibility] is contradicted as well and there's nothing new we # incompatibility is contradicted as well and there's nothing new we
# can deduce from it. # can deduce from it.
return return
elif relation == SetRelation.OVERLAPPING: elif relation == SetRelation.OVERLAPPING:
# If more than one term is inconclusive, we can't deduce anything about # If more than one term is inconclusive, we can't deduce anything about
# [incompatibility]. # incompatibility.
if unsatisfied is not None: if unsatisfied is not None:
return return
# If exactly one term in [incompatibility] is inconclusive, then it's # If exactly one term in incompatibility is inconclusive, then it's
# almost satisfied and [term] is the unsatisfied term. We can add the # almost satisfied and [term] is the unsatisfied term. We can add the
# inverse of the term to [_solution]. # inverse of the term to _solution.
unsatisfied = term unsatisfied = term
# If *all* terms in [incompatibility] are satisfied by [_solution], then # If *all* terms in incompatibility are satisfied by _solution, then
# [incompatibility] is satisfied and we have a conflict. # incompatibility is satisfied and we have a conflict.
if unsatisfied is None: if unsatisfied is None:
return _conflict return _conflict
self._log( self._log(
'derived: {}{}'.format( '<fg=blue>derived</>: {}{}'.format(
'not ' if unsatisfied.is_positive() else '', 'not ' if unsatisfied.is_positive() else '',
unsatisfied.dependency unsatisfied.dependency
) )
...@@ -169,28 +188,38 @@ class VersionSolver: ...@@ -169,28 +188,38 @@ class VersionSolver:
def _resolve_conflict(self, incompatibility def _resolve_conflict(self, incompatibility
): # type: (Incompatibility) -> Incompatibility ): # type: (Incompatibility) -> Incompatibility
"""
Given an incompatibility that's satisfied by _solution,
The `conflict resolution`_ constructs a new incompatibility that encapsulates the root
cause of the conflict and backtracks _solution until the new
incompatibility will allow _propagate() to deduce new assignments.
Adds the new incompatibility to _incompatibilities and returns it.
.. _conflict resolution: https://github.com/dart-lang/pub/tree/master/doc/solver.md#conflict-resolution
"""
self._log('<fg=red;options=bold>conflict</>: {}'.format(incompatibility)) self._log('<fg=red;options=bold>conflict</>: {}'.format(incompatibility))
new_incompatibility = False new_incompatibility = False
while not incompatibility.is_failure(): while not incompatibility.is_failure():
# The term in `incompatibility.terms` that was most recently satisfied by # The term in incompatibility.terms that was most recently satisfied by
# [_solution]. # _solution.
most_recent_term = None most_recent_term = None
# The earliest assignment in [_solution] such that [incompatibility] is # The earliest assignment in _solution such that incompatibility is
# satisfied by [_solution] up to and including this assignment. # satisfied by _solution up to and including this assignment.
most_recent_satisfier = None most_recent_satisfier = None
# The difference between [most_recent_satisfier] and [most_recent_term]; # The difference between most_recent_satisfier and most_recent_term;
# that is, the versions that are allowed by [most_recent_satisfier] and not # that is, the versions that are allowed by most_recent_satisfier and not
# by [most_recent_term]. This is `null` if [most_recent_satisfier] totally # by most_recent_term. This is None if most_recent_satisfier totally
# satisfies [most_recent_term]. # satisfies most_recent_term.
difference = None difference = None
# The decision level of the earliest assignment in [_solution] *before* # The decision level of the earliest assignment in _solution *before*
# [most_recent_satisfier] such that [incompatibility] is satisfied by # most_recent_satisfier such that incompatibility is satisfied by
# [_solution] up to and including this assignment plus # _solution up to and including this assignment plus
# [most_recent_satisfier]. # most_recent_satisfier.
# #
# Decision level 1 is the level where the root package was selected. It's # Decision level 1 is the level where the root package was selected. It's
# safe to go back to decision level 0, but stopping at 1 tends to produce # safe to go back to decision level 0, but stopping at 1 tends to produce
...@@ -218,7 +247,7 @@ class VersionSolver: ...@@ -218,7 +247,7 @@ class VersionSolver:
) )
if most_recent_term == term: if most_recent_term == term:
# If [most_recent_satisfier] doesn't satisfy [most_recent_term] on its # If most_recent_satisfier doesn't satisfy most_recent_term on its
# own, then the next-most-recent satisfier may be the one that # own, then the next-most-recent satisfier may be the one that
# satisfies the remainder. # satisfies the remainder.
difference = most_recent_satisfier.difference(most_recent_term) difference = most_recent_satisfier.difference(most_recent_term)
...@@ -228,11 +257,11 @@ class VersionSolver: ...@@ -228,11 +257,11 @@ class VersionSolver:
self._solution.satisfier(difference.inverse).decision_level self._solution.satisfier(difference.inverse).decision_level
) )
# If [mostRecentSatisfier] is the only satisfier left at its decision # If most_recent_identifier is the only satisfier left at its decision
# level, or if it has no cause (indicating that it's a decision rather # level, or if it has no cause (indicating that it's a decision rather
# than a derivation), then [incompatibility] is the root cause. We then # than a derivation), then incompatibility is the root cause. We then
# backjump to [previousSatisfierLevel], where [incompatibility] is # backjump to previous_satisfier_level, where incompatibility is
# guaranteed to allow [_propagate] to produce more assignments. # guaranteed to allow _propagate to produce more assignments.
if ( if (
previous_satisfier_level < most_recent_satisfier.decision_level previous_satisfier_level < most_recent_satisfier.decision_level
or most_recent_satisfier.cause is None or most_recent_satisfier.cause is None
...@@ -243,8 +272,8 @@ class VersionSolver: ...@@ -243,8 +272,8 @@ class VersionSolver:
return incompatibility return incompatibility
# Create a new incompatibility by combining [incompatibility] with the # Create a new incompatibility by combining incompatibility with the
# incompatibility that caused [mostRecentSatisfier] to be assigned. Doing # incompatibility that caused most_recent_satisfier to be assigned. Doing
# this iteratively constructs an incompatibility that's guaranteed to be # this iteratively constructs an incompatibility that's guaranteed to be
# true (that is, we know for sure no solution will satisfy the # true (that is, we know for sure no solution will satisfy the
# incompatibility) while also approximating the intuitive notion of the # incompatibility) while also approximating the intuitive notion of the
...@@ -258,16 +287,18 @@ class VersionSolver: ...@@ -258,16 +287,18 @@ class VersionSolver:
if term.dependency != most_recent_satisfier.dependency: if term.dependency != most_recent_satisfier.dependency:
new_terms.append(term) new_terms.append(term)
# The [mostRecentSatisfier] may not satisfy [mostRecentTerm] on its own # The most_recent_satisfier may not satisfy most_recent_term on its own
# if there are a collection of constraints on [mostRecentTerm] that # if there are a collection of constraints on most_recent_term that
# only satisfy it together. For example, if [mostRecentTerm] is # only satisfy it together. For example, if most_recent_term is
# `foo ^1.0.0` and [_solution] contains `[foo >=1.0.0, # `foo ^1.0.0` and _solution contains `[foo >=1.0.0,
# foo <2.0.0]`, then [mostRecentSatisfier] will be `foo <2.0.0` even # foo <2.0.0]`, then most_recent_satisfier will be `foo <2.0.0` even
# though it doesn't totally satisfy `foo ^1.0.0`. # though it doesn't totally satisfy `foo ^1.0.0`.
# #
# In this case, we add `not (mostRecentSatisfier \ mostRecentTerm)` to # In this case, we add `not (most_recent_satisfier \ most_recent_term)` to
# the incompatibility as well, See [the algorithm documentation][] for # the incompatibility as well, See the `algorithm documentation`_ for
# details. # details.
#
# .. _algorithm documentation: https://github.com/dart-lang/pub/tree/master/doc/solver.md#conflict-resolution
if difference is not None: if difference is not None:
new_terms.append(difference.inverse) new_terms.append(difference.inverse)
...@@ -289,13 +320,21 @@ class VersionSolver: ...@@ -289,13 +320,21 @@ class VersionSolver:
def _choose_package_version(self): # type: () -> Union[str, None] def _choose_package_version(self): # type: () -> Union[str, None]
""" """
Tries to select a version of a required package. Tries to select a version of a required package.
Returns the name of the package whose incompatibilities should be
propagated by _propagate(), or None indicating that version solving is
complete and a solution has been found.
""" """
unsatisfied = self._solution.unsatisfied unsatisfied = self._solution.unsatisfied
if not unsatisfied: if not unsatisfied:
return return
# Prefer packages with as few remaining versions as possible,
# so that if a conflict is necessary it's forced quickly.
def _get_min(dependency): def _get_min(dependency):
if dependency.name in self._use_latest: if dependency.name in self._use_latest:
# If we're forced to use the latest version of a package, it effectively
# only has one version to choose from.
return 1 return 1
if dependency.name in self._locked: if dependency.name in self._locked:
...@@ -326,9 +365,11 @@ class VersionSolver: ...@@ -326,9 +365,11 @@ class VersionSolver:
except IndexError: except IndexError:
version = None version = None
else: else:
version = self._locked[dependency.name] version = locked
if version is None: if version is None:
# If there are no versions that satisfy the constraint,
# add an incompatibility that indicates that.
self._add_incompatibility( self._add_incompatibility(
Incompatibility([Term(dependency, True)], NoVersionsCause()) Incompatibility([Term(dependency, True)], NoVersionsCause())
) )
...@@ -339,6 +380,11 @@ class VersionSolver: ...@@ -339,6 +380,11 @@ class VersionSolver:
for incompatibility in self._provider.incompatibilities_for(version): for incompatibility in self._provider.incompatibilities_for(version):
self._add_incompatibility(incompatibility) self._add_incompatibility(incompatibility)
# If an incompatibility is already satisfied, then selecting version
# would cause a conflict.
#
# We'll continue adding its dependencies, then go back to
# unit propagation which will guide us to choose a better version.
conflict = conflict or all([ conflict = conflict or all([
term.dependency.name == dependency.name or self._solution.satisfies(term) term.dependency.name == dependency.name or self._solution.satisfies(term)
for term in incompatibility.terms for term in incompatibility.terms
...@@ -346,7 +392,7 @@ class VersionSolver: ...@@ -346,7 +392,7 @@ class VersionSolver:
if not conflict: if not conflict:
self._solution.decide(version) self._solution.decide(version)
self._log('selecting {}'.format(version)) self._log('<fg=blue>selecting</> {} ({})'.format(version.name, version.full_pretty_version))
return dependency.name return dependency.name
...@@ -354,6 +400,9 @@ class VersionSolver: ...@@ -354,6 +400,9 @@ class VersionSolver:
return isinstance(VersionRange().difference(constraint), Version) return isinstance(VersionRange().difference(constraint), Version)
def _result(self): # type: () -> SolverResult def _result(self): # type: () -> SolverResult
"""
Creates a #SolverResult from the decisions in _solution
"""
decisions = self._solution.decisions decisions = self._solution.decisions
return SolverResult( return SolverResult(
...@@ -363,7 +412,7 @@ class VersionSolver: ...@@ -363,7 +412,7 @@ class VersionSolver:
) )
def _add_incompatibility(self, incompatibility): # type: (Incompatibility) -> None def _add_incompatibility(self, incompatibility): # type: (Incompatibility) -> None
self._log("fact: {}".format(incompatibility)) self._log("<fg=blue>fact</>: {}".format(incompatibility))
for term in incompatibility.terms: for term in incompatibility.terms:
if term.dependency.name not in self._incompatibilities: if term.dependency.name not in self._incompatibilities:
...@@ -381,4 +430,4 @@ class VersionSolver: ...@@ -381,4 +430,4 @@ class VersionSolver:
return self._locked.get(package_name) return self._locked.get(package_name)
def _log(self, text): def _log(self, text):
self._provider.debug(text) self._provider.debug(text, self._solution.attempted_solutions)
poetry/puzzle/provider.py
...@@ -363,7 +363,7 @@ class Provider: ...@@ -363,7 +363,7 @@ class Provider:
if self.is_debugging(): if self.is_debugging():
debug_info = str(message) debug_info = str(message)
debug_info = '\n'.join([ debug_info = '\n'.join([
'<comment>:{}:</> {}'.format(str(depth).rjust(4), s) '<comment>{}:</> {}'.format(str(depth).rjust(4), s)
for s in debug_info.split('\n') for s in debug_info.split('\n')
]) + '\n' ]) + '\n'
......
poetry/repositories/pypi_repository.py
...@@ -257,6 +257,8 @@ class PyPiRepository(Repository): ...@@ -257,6 +257,8 @@ class PyPiRepository(Repository):
) )
def _get_release_info(self, name, version): # type: (str, str) -> dict def _get_release_info(self, name, version): # type: (str, str) -> dict
self._log('Getting info for {} ({}) from PyPI'.format(name, version), 'debug')
json_data = self._get('pypi/{}/{}/json'.format(name, version)) json_data = self._get('pypi/{}/{}/json'.format(name, version))
if json_data is None: if json_data is None:
raise ValueError('Package [{}] not found.'.format(name)) raise ValueError('Package [{}] not found.'.format(name))
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment