Skip to content

P
python-poetry
Commit f317b003 by Sébastien Eustace
Options

Improve dependency resolution debugging

parent 92d0eddf
Showing with 133 additions and 65 deletions
poetry/mixology/partial_solution.py
......@@ -4,6 +4,9 @@ from typing import Any
from typing import Dict
from typing import List
from poetry.packages import Dependency
from poetry.packages import Package
from .assignment import Assignment
from .incompatibility import Incompatibility
from .set_relation import SetRelation
......@@ -25,7 +28,7 @@ class PartialSolution:
self._assignments = [] # type: List[Assignment]
# The decisions made for each package.
self._decisions = OrderedDict() # type: Dict[str, Any]
self._decisions = OrderedDict() # type: Dict[str, Package]
# The intersection of all positive Assignments for each package, minus any
# negative Assignments that refer to that package.
......@@ -33,38 +36,41 @@ class PartialSolution:
# This is derived from self._assignments.
self._positive = OrderedDict() # type: Dict[str, Term]
# The union of all negative [Assignment]s for each package.
# The union of all negative Assignments for each package.
#
# If a package has any positive [Assignment]s, it doesn't appear in this
# If a package has any positive Assignments, it doesn't appear in this
# map.
#
# This is derived from self._assignments.
self._negative = OrderedDict() # type: Dict[str, Dict[str, Term]]
# The number of distinct solutions that have been attempted so far.
self._attempted_solutions = 1
# Whether the solver is currently backtracking.
self._backtracking = False
@property
def decisions(self): # type: () -> Any
def decisions(self): # type: () -> List[Package]
return list(self._decisions.values())
@property
def decision_level(self):
def decision_level(self): # type: () -> int
return len(self._decisions)
@property
def attempted_solutions(self):
def attempted_solutions(self): # type: () -> int
return self._attempted_solutions
@property
def unsatisfied(self):
def unsatisfied(self): # type: () -> List[Dependency]
return [
term.dependency
for term in self._positive.values()
if term.dependency.name not in self._decisions
]
def decide(self, package): # type: (Any) -> None
def decide(self, package): # type: (Package) -> None
"""
Adds an assignment of package as a decision
and increments the decision level.
......@@ -82,7 +88,7 @@ class PartialSolution:
self._assign(Assignment.decision(package, self.decision_level, len(self._assignments)))
def derive(self, dependency, is_positive, cause
): # type: (Any, bool, Incompatibility) -> None
): # type: (Dependency, bool, Incompatibility) -> None
"""
Adds an assignment of package as a derivation.
"""
......@@ -101,6 +107,10 @@ class PartialSolution:
self._register(assignment)
def backtrack(self, decision_level): # type: (int) -> None
"""
Resets the current decision level to decision_level, and removes all
assignments made after that level.
"""
self._backtracking = True
packages = set()
......
poetry/mixology/term.py
# -*- coding: utf-8 -*-
from typing import Union
from poetry.packages import Dependency
from .set_relation import SetRelation
......@@ -8,9 +10,14 @@ class Term(object):
"""
A statement about a package which is true or false for a given selection of
package versions.
See https://github.com/dart-lang/pub/tree/master/doc/solver.md#term.
"""
def __init__(self, dependency, is_positive):
def __init__(self,
dependency, # type: Dependency
is_positive # type: bool
):
self._dependency = dependency
self._positive = is_positive
......@@ -100,7 +107,7 @@ class Term(object):
# not foo ^1.5.0 is a superset of not foo ^1.0.0
return SetRelation.OVERLAPPING
def intersect(self, other): # type: (Term) -> Term
def intersect(self, other): # type: (Term) -> Union[Term, None]
"""
Returns a Term that represents the packages
allowed by both this term and another
......
poetry/mixology/version_solver.py
......@@ -6,6 +6,9 @@ from typing import List
from typing import Union
from poetry.packages import Dependency
from poetry.packages import ProjectPackage
from poetry.packages import Package
from poetry.puzzle.provider import Provider
from poetry.semver import Version
from poetry.semver import VersionRange
......@@ -25,8 +28,20 @@ _conflict = object()
class VersionSolver:
def __init__(self, root, provider, locked=None, use_latest=None):
"""
The version solver that finds a set of package versions that satisfy the
root package's dependencies.
See https://github.com/dart-lang/pub/tree/master/doc/solver.md for details
on how this solver works.
"""
def __init__(self,
root, # type: ProjectPackage
provider, # type: Provider
locked=None, # type: Dict[str, Package]
use_latest=None # type: List[str]
):
self._root = root
self._provider = provider
self._locked = locked or {}
......@@ -43,7 +58,7 @@ class VersionSolver:
def solution(self): # type: () -> PartialSolution
return self._solution
def solve(self):
def solve(self): # type: () -> SolverResult
"""
Finds a set of dependencies that match the root package's constraints,
or raises an error if no such set is available.
......@@ -97,9 +112,11 @@ class VersionSolver:
result = self._propagate_incompatibility(incompatibility)
if result is _conflict:
# If [incompatibility] is satisfied by [_solution], we use
# [_resolveConflict] to determine the root cause of the conflict as a
# new incompatibility. It also backjumps to a point in [_solution]
# If the incompatibility is satisfied by the solution, we use
# _resolve_conflict() to determine the root cause of the conflict as a
# new incompatibility.
#
# It also backjumps to a point in the solution
# where that incompatibility will allow us to derive new assignments
# that avoid the conflict.
root_cause = self._resolve_conflict(incompatibility)
......@@ -114,46 +131,48 @@ class VersionSolver:
changed.add(result)
def _propagate_incompatibility(self, incompatibility
): # type: (Incompatibility) -> Union[str, None]
): # type: (Incompatibility) -> Union[str, _conflict, None]
"""
If [incompatibility] is [almost satisfied][] by [_solution], adds the
negation of the unsatisfied term to [_solution].
If incompatibility is almost satisfied by _solution, adds the
negation of the unsatisfied term to _solution.
If incompatibility is satisfied by _solution, returns _conflict. If
incompatibility is almost satisfied by _solution, returns the
unsatisfied term's package name.
If [incompatibility] is satisfied by [_solution], returns `#conflict`. If
[incompatibility] is almost satisfied by [_solution], returns the
unsatisfied term's package name. Otherwise, returns None.
Otherwise, returns None.
"""
# The first entry in `incompatibility.terms` that's not yet satisfied by
# [_solution], if one exists. If we find more than one, [_solution] is
# inconclusive for [incompatibility] and we can't deduce anything.
# The first entry in incompatibility.terms that's not yet satisfied by
# _solution, if one exists. If we find more than one, _solution is
# inconclusive for incompatibility and we can't deduce anything.
unsatisfied = None
for term in incompatibility.terms:
relation = self._solution.relation(term)
if relation == SetRelation.DISJOINT:
# If [term] is already contradicted by [_solution], then
# [incompatibility] is contradicted as well and there's nothing new we
# If term is already contradicted by _solution, then
# incompatibility is contradicted as well and there's nothing new we
# can deduce from it.
return
elif relation == SetRelation.OVERLAPPING:
# If more than one term is inconclusive, we can't deduce anything about
# [incompatibility].
# incompatibility.
if unsatisfied is not None:
return
# If exactly one term in [incompatibility] is inconclusive, then it's
# If exactly one term in incompatibility is inconclusive, then it's
# almost satisfied and [term] is the unsatisfied term. We can add the
# inverse of the term to [_solution].
# inverse of the term to _solution.
unsatisfied = term
# If *all* terms in [incompatibility] are satisfied by [_solution], then
# [incompatibility] is satisfied and we have a conflict.
# If *all* terms in incompatibility are satisfied by _solution, then
# incompatibility is satisfied and we have a conflict.
if unsatisfied is None:
return _conflict
self._log(
'derived: {}{}'.format(
'<fg=blue>derived</>: {}{}'.format(
'not ' if unsatisfied.is_positive() else '',
unsatisfied.dependency
)
......@@ -169,28 +188,38 @@ class VersionSolver:
def _resolve_conflict(self, incompatibility
): # type: (Incompatibility) -> Incompatibility
"""
Given an incompatibility that's satisfied by _solution,
The `conflict resolution`_ constructs a new incompatibility that encapsulates the root
cause of the conflict and backtracks _solution until the new
incompatibility will allow _propagate() to deduce new assignments.
Adds the new incompatibility to _incompatibilities and returns it.
.. _conflict resolution: https://github.com/dart-lang/pub/tree/master/doc/solver.md#conflict-resolution
"""
self._log('<fg=red;options=bold>conflict</>: {}'.format(incompatibility))
new_incompatibility = False
while not incompatibility.is_failure():
# The term in `incompatibility.terms` that was most recently satisfied by
# [_solution].
# The term in incompatibility.terms that was most recently satisfied by
# _solution.
most_recent_term = None
# The earliest assignment in [_solution] such that [incompatibility] is
# satisfied by [_solution] up to and including this assignment.
# The earliest assignment in _solution such that incompatibility is
# satisfied by _solution up to and including this assignment.
most_recent_satisfier = None
# The difference between [most_recent_satisfier] and [most_recent_term];
# that is, the versions that are allowed by [most_recent_satisfier] and not
# by [most_recent_term]. This is `null` if [most_recent_satisfier] totally
# satisfies [most_recent_term].
# The difference between most_recent_satisfier and most_recent_term;
# that is, the versions that are allowed by most_recent_satisfier and not
# by most_recent_term. This is None if most_recent_satisfier totally
# satisfies most_recent_term.
difference = None
# The decision level of the earliest assignment in [_solution] *before*
# [most_recent_satisfier] such that [incompatibility] is satisfied by
# [_solution] up to and including this assignment plus
# [most_recent_satisfier].
# The decision level of the earliest assignment in _solution *before*
# most_recent_satisfier such that incompatibility is satisfied by
# _solution up to and including this assignment plus
# most_recent_satisfier.
#
# Decision level 1 is the level where the root package was selected. It's
# safe to go back to decision level 0, but stopping at 1 tends to produce
......@@ -218,7 +247,7 @@ class VersionSolver:
)
if most_recent_term == term:
# If [most_recent_satisfier] doesn't satisfy [most_recent_term] on its
# If most_recent_satisfier doesn't satisfy most_recent_term on its
# own, then the next-most-recent satisfier may be the one that
# satisfies the remainder.
difference = most_recent_satisfier.difference(most_recent_term)
......@@ -228,11 +257,11 @@ class VersionSolver:
self._solution.satisfier(difference.inverse).decision_level
)
# If [mostRecentSatisfier] is the only satisfier left at its decision
# If most_recent_identifier is the only satisfier left at its decision
# level, or if it has no cause (indicating that it's a decision rather
# than a derivation), then [incompatibility] is the root cause. We then
# backjump to [previousSatisfierLevel], where [incompatibility] is
# guaranteed to allow [_propagate] to produce more assignments.
# than a derivation), then incompatibility is the root cause. We then
# backjump to previous_satisfier_level, where incompatibility is
# guaranteed to allow _propagate to produce more assignments.
if (
previous_satisfier_level < most_recent_satisfier.decision_level
or most_recent_satisfier.cause is None
......@@ -243,8 +272,8 @@ class VersionSolver:
return incompatibility
# Create a new incompatibility by combining [incompatibility] with the
# incompatibility that caused [mostRecentSatisfier] to be assigned. Doing
# Create a new incompatibility by combining incompatibility with the
# incompatibility that caused most_recent_satisfier to be assigned. Doing
# this iteratively constructs an incompatibility that's guaranteed to be
# true (that is, we know for sure no solution will satisfy the
# incompatibility) while also approximating the intuitive notion of the
......@@ -258,16 +287,18 @@ class VersionSolver:
if term.dependency != most_recent_satisfier.dependency:
new_terms.append(term)
# The [mostRecentSatisfier] may not satisfy [mostRecentTerm] on its own
# if there are a collection of constraints on [mostRecentTerm] that
# only satisfy it together. For example, if [mostRecentTerm] is
# `foo ^1.0.0` and [_solution] contains `[foo >=1.0.0,
# foo <2.0.0]`, then [mostRecentSatisfier] will be `foo <2.0.0` even
# The most_recent_satisfier may not satisfy most_recent_term on its own
# if there are a collection of constraints on most_recent_term that
# only satisfy it together. For example, if most_recent_term is
# `foo ^1.0.0` and _solution contains `[foo >=1.0.0,
# foo <2.0.0]`, then most_recent_satisfier will be `foo <2.0.0` even
# though it doesn't totally satisfy `foo ^1.0.0`.
#
# In this case, we add `not (mostRecentSatisfier \ mostRecentTerm)` to
# the incompatibility as well, See [the algorithm documentation][] for
# In this case, we add `not (most_recent_satisfier \ most_recent_term)` to
# the incompatibility as well, See the `algorithm documentation`_ for
# details.
#
# .. _algorithm documentation: https://github.com/dart-lang/pub/tree/master/doc/solver.md#conflict-resolution
if difference is not None:
new_terms.append(difference.inverse)
......@@ -289,13 +320,21 @@ class VersionSolver:
def _choose_package_version(self): # type: () -> Union[str, None]
"""
Tries to select a version of a required package.
Returns the name of the package whose incompatibilities should be
propagated by _propagate(), or None indicating that version solving is
complete and a solution has been found.
"""
unsatisfied = self._solution.unsatisfied
if not unsatisfied:
return
# Prefer packages with as few remaining versions as possible,
# so that if a conflict is necessary it's forced quickly.
def _get_min(dependency):
if dependency.name in self._use_latest:
# If we're forced to use the latest version of a package, it effectively
# only has one version to choose from.
return 1
if dependency.name in self._locked:
......@@ -326,9 +365,11 @@ class VersionSolver:
except IndexError:
version = None
else:
version = self._locked[dependency.name]
version = locked
if version is None:
# If there are no versions that satisfy the constraint,
# add an incompatibility that indicates that.
self._add_incompatibility(
Incompatibility([Term(dependency, True)], NoVersionsCause())
)
......@@ -339,6 +380,11 @@ class VersionSolver:
for incompatibility in self._provider.incompatibilities_for(version):
self._add_incompatibility(incompatibility)
# If an incompatibility is already satisfied, then selecting version
# would cause a conflict.
#
# We'll continue adding its dependencies, then go back to
# unit propagation which will guide us to choose a better version.
conflict = conflict or all([
term.dependency.name == dependency.name or self._solution.satisfies(term)
for term in incompatibility.terms
......@@ -346,7 +392,7 @@ class VersionSolver:
if not conflict:
self._solution.decide(version)
self._log('selecting {}'.format(version))
self._log('<fg=blue>selecting</> {} ({})'.format(version.name, version.full_pretty_version))
return dependency.name
......@@ -354,6 +400,9 @@ class VersionSolver:
return isinstance(VersionRange().difference(constraint), Version)
def _result(self): # type: () -> SolverResult
"""
Creates a #SolverResult from the decisions in _solution
"""
decisions = self._solution.decisions
return SolverResult(
......@@ -363,7 +412,7 @@ class VersionSolver:
)
def _add_incompatibility(self, incompatibility): # type: (Incompatibility) -> None
self._log("fact: {}".format(incompatibility))
self._log("<fg=blue>fact</>: {}".format(incompatibility))
for term in incompatibility.terms:
if term.dependency.name not in self._incompatibilities:
......@@ -381,4 +430,4 @@ class VersionSolver:
return self._locked.get(package_name)
def _log(self, text):
self._provider.debug(text)
self._provider.debug(text, self._solution.attempted_solutions)
poetry/puzzle/provider.py
......@@ -363,7 +363,7 @@ class Provider:
if self.is_debugging():
debug_info = str(message)
debug_info = '\n'.join([
'<comment>:{}:</> {}'.format(str(depth).rjust(4), s)
'<comment>{}:</> {}'.format(str(depth).rjust(4), s)
for s in debug_info.split('\n')
]) + '\n'
......
poetry/repositories/pypi_repository.py
......@@ -257,6 +257,8 @@ class PyPiRepository(Repository):
)
def _get_release_info(self, name, version): # type: (str, str) -> dict
self._log('Getting info for {} ({}) from PyPI'.format(name, version), 'debug')
json_data = self._get('pypi/{}/{}/json'.format(name, version))
if json_data is None:
raise ValueError('Package [{}] not found.'.format(name))
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment