Improve dependency resolution debugging

poetry/mixology/partial_solution.py

@@ -4,6 +4,9 @@ from typing import Any
 from typing import Dict
 from typing import Dict
 from typing import List
 from typing import List
 
 
+from poetry.packages import Dependency
+from poetry.packages import Package
+
 from .assignment import Assignment
 from .assignment import Assignment
 from .incompatibility import Incompatibility
 from .incompatibility import Incompatibility
 from .set_relation import SetRelation
 from .set_relation import SetRelation
@@ -25,7 +28,7 @@ class PartialSolution:
         self._assignments = []  # type: List[Assignment]
         self._assignments = []  # type: List[Assignment]
 
 
         # The decisions made for each package.
         # The decisions made for each package.
-        self._decisions = OrderedDict()  # type: Dict[str, Any]
+        self._decisions = OrderedDict()  # type: Dict[str, Package]
 
 
         # The intersection of all positive Assignments for each package, minus any
         # The intersection of all positive Assignments for each package, minus any
         # negative Assignments that refer to that package.
         # negative Assignments that refer to that package.
@@ -33,38 +36,41 @@ class PartialSolution:
         # This is derived from self._assignments.
         # This is derived from self._assignments.
         self._positive = OrderedDict()  # type: Dict[str, Term]
         self._positive = OrderedDict()  # type: Dict[str, Term]
 
 
-        # The union of all negative [Assignment]s for each package.
+        # The union of all negative Assignments for each package.
         #
         #
-        # If a package has any positive [Assignment]s, it doesn't appear in this
+        # If a package has any positive Assignments, it doesn't appear in this
         # map.
         # map.
         #
         #
         # This is derived from self._assignments.
         # This is derived from self._assignments.
         self._negative = OrderedDict()  # type: Dict[str, Dict[str, Term]]
         self._negative = OrderedDict()  # type: Dict[str, Dict[str, Term]]
 
 
+        # The number of distinct solutions that have been attempted so far.
         self._attempted_solutions = 1
         self._attempted_solutions = 1
+
+        # Whether the solver is currently backtracking.
         self._backtracking = False
         self._backtracking = False
 
 
     @property
     @property
-    def decisions(self):  # type: () -> Any
+    def decisions(self):  # type: () -> List[Package]
         return list(self._decisions.values())
         return list(self._decisions.values())
 
 
     @property
     @property
-    def decision_level(self):
+    def decision_level(self):  # type: () -> int
         return len(self._decisions)
         return len(self._decisions)
 
 
     @property
     @property
-    def attempted_solutions(self):
+    def attempted_solutions(self):  # type: () -> int
         return self._attempted_solutions
         return self._attempted_solutions
 
 
     @property
     @property
-    def unsatisfied(self):
+    def unsatisfied(self):  # type: () -> List[Dependency]
         return [
         return [
             term.dependency
             term.dependency
             for term in self._positive.values()
             for term in self._positive.values()
             if term.dependency.name not in self._decisions
             if term.dependency.name not in self._decisions
         ]
         ]
 
 
-    def decide(self, package):  # type: (Any) -> None
+    def decide(self, package):  # type: (Package) -> None
         """
         """
         Adds an assignment of package as a decision
         Adds an assignment of package as a decision
         and increments the decision level.
         and increments the decision level.
@@ -82,7 +88,7 @@ class PartialSolution:
         self._assign(Assignment.decision(package, self.decision_level, len(self._assignments)))
         self._assign(Assignment.decision(package, self.decision_level, len(self._assignments)))
 
 
     def derive(self, dependency, is_positive, cause
     def derive(self, dependency, is_positive, cause
-               ):  # type: (Any, bool, Incompatibility) -> None
+               ):  # type: (Dependency, bool, Incompatibility) -> None
         """
         """
         Adds an assignment of package as a derivation.
         Adds an assignment of package as a derivation.
         """
         """
@@ -101,6 +107,10 @@ class PartialSolution:
         self._register(assignment)
         self._register(assignment)
 
 
     def backtrack(self, decision_level):  # type: (int) -> None
     def backtrack(self, decision_level):  # type: (int) -> None
+        """
+        Resets the current decision level to decision_level, and removes all
+        assignments made after that level.
+        """
         self._backtracking = True
         self._backtracking = True
 
 
         packages = set()
         packages = set()

poetry/mixology/term.py

 # -*- coding: utf-8 -*-
 # -*- coding: utf-8 -*-
+from typing import Union
+
 from poetry.packages import Dependency
 from poetry.packages import Dependency
 
 
 from .set_relation import SetRelation
 from .set_relation import SetRelation
@@ -8,9 +10,14 @@ class Term(object):
     """
     """
     A statement about a package which is true or false for a given selection of
     A statement about a package which is true or false for a given selection of
     package versions.
     package versions.
+
+    See https://github.com/dart-lang/pub/tree/master/doc/solver.md#term.
     """
     """
 
 
-    def __init__(self, dependency, is_positive):
+    def __init__(self,
+                 dependency,  # type: Dependency
+                 is_positive  # type: bool
+                 ):
         self._dependency = dependency
         self._dependency = dependency
         self._positive = is_positive
         self._positive = is_positive
 
 
@@ -100,7 +107,7 @@ class Term(object):
                 # not foo ^1.5.0 is a superset of not foo ^1.0.0
                 # not foo ^1.5.0 is a superset of not foo ^1.0.0
                 return SetRelation.OVERLAPPING
                 return SetRelation.OVERLAPPING
 
 
-    def intersect(self, other):  # type: (Term) -> Term
+    def intersect(self, other):  # type: (Term) -> Union[Term, None]
         """
         """
         Returns a Term that represents the packages
         Returns a Term that represents the packages
         allowed by both this term and another
         allowed by both this term and another

poetry/mixology/version_solver.py

@@ -6,6 +6,9 @@ from typing import List
 from typing import Union
 from typing import Union
 
 
 from poetry.packages import Dependency
 from poetry.packages import Dependency
+from poetry.packages import ProjectPackage
+from poetry.packages import Package
+from poetry.puzzle.provider import Provider
 from poetry.semver import Version
 from poetry.semver import Version
 from poetry.semver import VersionRange
 from poetry.semver import VersionRange
 
 
@@ -25,8 +28,20 @@ _conflict = object()
 
 
 
 
 class VersionSolver:
 class VersionSolver:
+    """
+    The version solver that finds a set of package versions that satisfy the
+    root package's dependencies.
+
+    See https://github.com/dart-lang/pub/tree/master/doc/solver.md for details
+    on how this solver works.
+    """
 
 
-    def __init__(self, root, provider, locked=None, use_latest=None):
+    def __init__(self,
+                 root,            # type: ProjectPackage
+                 provider,        # type: Provider
+                 locked=None,     # type: Dict[str, Package]
+                 use_latest=None  # type: List[str]
+                 ):
         self._root = root
         self._root = root
         self._provider = provider
         self._provider = provider
         self._locked = locked or {}
         self._locked = locked or {}
@@ -43,7 +58,7 @@ class VersionSolver:
     def solution(self):  # type: () -> PartialSolution
     def solution(self):  # type: () -> PartialSolution
         return self._solution
         return self._solution
 
 
-    def solve(self):
+    def solve(self):  # type: () -> SolverResult
         """
         """
         Finds a set of dependencies that match the root package's constraints,
         Finds a set of dependencies that match the root package's constraints,
         or raises an error if no such set is available.
         or raises an error if no such set is available.
@@ -97,9 +112,11 @@ class VersionSolver:
                 result = self._propagate_incompatibility(incompatibility)
                 result = self._propagate_incompatibility(incompatibility)
 
 
                 if result is _conflict:
                 if result is _conflict:
-                    # If [incompatibility] is satisfied by [_solution], we use
-                    # [_resolveConflict] to determine the root cause of the conflict as a
-                    # new incompatibility. It also backjumps to a point in [_solution]
+                    # If the incompatibility is satisfied by the solution, we use
+                    # _resolve_conflict() to determine the root cause of the conflict as a
+                    # new incompatibility.
+                    #
+                    # It also backjumps to a point in the solution
                     # where that incompatibility will allow us to derive new assignments
                     # where that incompatibility will allow us to derive new assignments
                     # that avoid the conflict.
                     # that avoid the conflict.
                     root_cause = self._resolve_conflict(incompatibility)
                     root_cause = self._resolve_conflict(incompatibility)
@@ -114,46 +131,48 @@ class VersionSolver:
                     changed.add(result)
                     changed.add(result)
 
 
     def _propagate_incompatibility(self, incompatibility
     def _propagate_incompatibility(self, incompatibility
-                                   ):  # type: (Incompatibility) -> Union[str, None]
+                                   ):  # type: (Incompatibility) -> Union[str, _conflict, None]
         """
         """
-        If [incompatibility] is [almost satisfied][] by [_solution], adds the
-        negation of the unsatisfied term to [_solution].
+        If incompatibility is almost satisfied by _solution, adds the
+        negation of the unsatisfied term to _solution.
+
+        If incompatibility is satisfied by _solution, returns _conflict. If
+        incompatibility is almost satisfied by _solution, returns the
+        unsatisfied term's package name.
 
 
-        If [incompatibility] is satisfied by [_solution], returns `#conflict`. If
-        [incompatibility] is almost satisfied by [_solution], returns the
-        unsatisfied term's package name. Otherwise, returns None.
+        Otherwise, returns None.
         """
         """
-        # The first entry in `incompatibility.terms` that's not yet satisfied by
-        # [_solution], if one exists. If we find more than one, [_solution] is
-        # inconclusive for [incompatibility] and we can't deduce anything.
+        # The first entry in incompatibility.terms that's not yet satisfied by
+        # _solution, if one exists. If we find more than one, _solution is
+        # inconclusive for incompatibility and we can't deduce anything.
         unsatisfied = None
         unsatisfied = None
 
 
         for term in incompatibility.terms:
         for term in incompatibility.terms:
             relation = self._solution.relation(term)
             relation = self._solution.relation(term)
 
 
             if relation == SetRelation.DISJOINT:
             if relation == SetRelation.DISJOINT:
-                # If [term] is already contradicted by [_solution], then
-                # [incompatibility] is contradicted as well and there's nothing new we
+                # If term is already contradicted by _solution, then
+                # incompatibility is contradicted as well and there's nothing new we
                 # can deduce from it.
                 # can deduce from it.
                 return
                 return
             elif relation == SetRelation.OVERLAPPING:
             elif relation == SetRelation.OVERLAPPING:
                 # If more than one term is inconclusive, we can't deduce anything about
                 # If more than one term is inconclusive, we can't deduce anything about
-                # [incompatibility].
+                # incompatibility.
                 if unsatisfied is not None:
                 if unsatisfied is not None:
                     return
                     return
 
 
-                # If exactly one term in [incompatibility] is inconclusive, then it's
+                # If exactly one term in incompatibility is inconclusive, then it's
                 # almost satisfied and [term] is the unsatisfied term. We can add the
                 # almost satisfied and [term] is the unsatisfied term. We can add the
-                # inverse of the term to [_solution].
+                # inverse of the term to _solution.
                 unsatisfied = term
                 unsatisfied = term
 
 
-        # If *all* terms in [incompatibility] are satisfied by [_solution], then
-        # [incompatibility] is satisfied and we have a conflict.
+        # If *all* terms in incompatibility are satisfied by _solution, then
+        # incompatibility is satisfied and we have a conflict.
         if unsatisfied is None:
         if unsatisfied is None:
             return _conflict
             return _conflict
 
 
         self._log(
         self._log(
-            'derived: {}{}'.format(
+            '<fg=blue>derived</>: {}{}'.format(
                 'not ' if unsatisfied.is_positive() else '',
                 'not ' if unsatisfied.is_positive() else '',
                 unsatisfied.dependency
                 unsatisfied.dependency
             )
             )
@@ -169,28 +188,38 @@ class VersionSolver:
 
 
     def _resolve_conflict(self, incompatibility
     def _resolve_conflict(self, incompatibility
                           ):  # type: (Incompatibility) -> Incompatibility
                           ):  # type: (Incompatibility) -> Incompatibility
+        """
+        Given an incompatibility that's satisfied by _solution,
+        The `conflict resolution`_ constructs a new incompatibility that encapsulates the root
+        cause of the conflict and backtracks _solution until the new
+        incompatibility will allow _propagate() to deduce new assignments.
+
+        Adds the new incompatibility to _incompatibilities and returns it.
+
+        .. _conflict resolution: https://github.com/dart-lang/pub/tree/master/doc/solver.md#conflict-resolution
+        """
         self._log('<fg=red;options=bold>conflict</>: {}'.format(incompatibility))
         self._log('<fg=red;options=bold>conflict</>: {}'.format(incompatibility))
 
 
         new_incompatibility = False
         new_incompatibility = False
         while not incompatibility.is_failure():
         while not incompatibility.is_failure():
-            # The term in `incompatibility.terms` that was most recently satisfied by
-            # [_solution].
+            # The term in incompatibility.terms that was most recently satisfied by
+            # _solution.
             most_recent_term = None
             most_recent_term = None
 
 
-            # The earliest assignment in [_solution] such that [incompatibility] is
-            # satisfied by [_solution] up to and including this assignment.
+            # The earliest assignment in _solution such that incompatibility is
+            # satisfied by _solution up to and including this assignment.
             most_recent_satisfier = None
             most_recent_satisfier = None
 
 
-            # The difference between [most_recent_satisfier] and [most_recent_term];
-            # that is, the versions that are allowed by [most_recent_satisfier] and not
-            # by [most_recent_term]. This is `null` if [most_recent_satisfier] totally
-            # satisfies [most_recent_term].
+            # The difference between most_recent_satisfier and most_recent_term;
+            # that is, the versions that are allowed by most_recent_satisfier and not
+            # by most_recent_term. This is None if most_recent_satisfier totally
+            # satisfies most_recent_term.
             difference = None
             difference = None
 
 
-            # The decision level of the earliest assignment in [_solution] *before*
-            # [most_recent_satisfier] such that [incompatibility] is satisfied by
-            # [_solution] up to and including this assignment plus
-            # [most_recent_satisfier].
+            # The decision level of the earliest assignment in _solution *before*
+            # most_recent_satisfier such that incompatibility is satisfied by
+            # _solution up to and including this assignment plus
+            # most_recent_satisfier.
             #
             #
             # Decision level 1 is the level where the root package was selected. It's
             # Decision level 1 is the level where the root package was selected. It's
             # safe to go back to decision level 0, but stopping at 1 tends to produce
             # safe to go back to decision level 0, but stopping at 1 tends to produce
@@ -218,7 +247,7 @@ class VersionSolver:
                     )
                     )
 
 
                 if most_recent_term == term:
                 if most_recent_term == term:
-                    # If [most_recent_satisfier] doesn't satisfy [most_recent_term] on its
+                    # If most_recent_satisfier doesn't satisfy most_recent_term on its
                     # own, then the next-most-recent satisfier may be the one that
                     # own, then the next-most-recent satisfier may be the one that
                     # satisfies the remainder.
                     # satisfies the remainder.
                     difference = most_recent_satisfier.difference(most_recent_term)
                     difference = most_recent_satisfier.difference(most_recent_term)
@@ -228,11 +257,11 @@ class VersionSolver:
                             self._solution.satisfier(difference.inverse).decision_level
                             self._solution.satisfier(difference.inverse).decision_level
                         )
                         )
 
 
-            # If [mostRecentSatisfier] is the only satisfier left at its decision
+            # If most_recent_identifier is the only satisfier left at its decision
             # level, or if it has no cause (indicating that it's a decision rather
             # level, or if it has no cause (indicating that it's a decision rather
-            # than a derivation), then [incompatibility] is the root cause. We then
-            # backjump to [previousSatisfierLevel], where [incompatibility] is
-            # guaranteed to allow [_propagate] to produce more assignments.
+            # than a derivation), then incompatibility is the root cause. We then
+            # backjump to previous_satisfier_level, where incompatibility is
+            # guaranteed to allow _propagate to produce more assignments.
             if (
             if (
                 previous_satisfier_level < most_recent_satisfier.decision_level
                 previous_satisfier_level < most_recent_satisfier.decision_level
                 or most_recent_satisfier.cause is None
                 or most_recent_satisfier.cause is None
@@ -243,8 +272,8 @@ class VersionSolver:
 
 
                 return incompatibility
                 return incompatibility
 
 
-            # Create a new incompatibility by combining [incompatibility] with the
-            # incompatibility that caused [mostRecentSatisfier] to be assigned. Doing
+            # Create a new incompatibility by combining incompatibility with the
+            # incompatibility that caused most_recent_satisfier to be assigned. Doing
             # this iteratively constructs an incompatibility that's guaranteed to be
             # this iteratively constructs an incompatibility that's guaranteed to be
             # true (that is, we know for sure no solution will satisfy the
             # true (that is, we know for sure no solution will satisfy the
             # incompatibility) while also approximating the intuitive notion of the
             # incompatibility) while also approximating the intuitive notion of the
@@ -258,16 +287,18 @@ class VersionSolver:
                 if term.dependency != most_recent_satisfier.dependency:
                 if term.dependency != most_recent_satisfier.dependency:
                     new_terms.append(term)
                     new_terms.append(term)
 
 
-            # The [mostRecentSatisfier] may not satisfy [mostRecentTerm] on its own
-            # if there are a collection of constraints on [mostRecentTerm] that
-            # only satisfy it together. For example, if [mostRecentTerm] is
-            # `foo ^1.0.0` and [_solution] contains `[foo >=1.0.0,
-            # foo <2.0.0]`, then [mostRecentSatisfier] will be `foo <2.0.0` even
+            # The most_recent_satisfier may not satisfy most_recent_term on its own
+            # if there are a collection of constraints on most_recent_term that
+            # only satisfy it together. For example, if most_recent_term is
+            # `foo ^1.0.0` and _solution contains `[foo >=1.0.0,
+            # foo <2.0.0]`, then most_recent_satisfier will be `foo <2.0.0` even
             # though it doesn't totally satisfy `foo ^1.0.0`.
             # though it doesn't totally satisfy `foo ^1.0.0`.
             #
             #
-            # In this case, we add `not (mostRecentSatisfier \ mostRecentTerm)` to
-            # the incompatibility as well, See [the algorithm documentation][] for
+            # In this case, we add `not (most_recent_satisfier \ most_recent_term)` to
+            # the incompatibility as well, See the `algorithm documentation`_ for
             # details.
             # details.
+            #
+            # .. _algorithm documentation: https://github.com/dart-lang/pub/tree/master/doc/solver.md#conflict-resolution
             if difference is not None:
             if difference is not None:
                 new_terms.append(difference.inverse)
                 new_terms.append(difference.inverse)
 
 
@@ -289,13 +320,21 @@ class VersionSolver:
     def _choose_package_version(self):  # type: () -> Union[str, None]
     def _choose_package_version(self):  # type: () -> Union[str, None]
         """
         """
         Tries to select a version of a required package.
         Tries to select a version of a required package.
+
+        Returns the name of the package whose incompatibilities should be
+        propagated by _propagate(), or None indicating that version solving is
+        complete and a solution has been found.
         """
         """
         unsatisfied = self._solution.unsatisfied
         unsatisfied = self._solution.unsatisfied
         if not unsatisfied:
         if not unsatisfied:
             return
             return
 
 
+        # Prefer packages with as few remaining versions as possible,
+        # so that if a conflict is necessary it's forced quickly.
         def _get_min(dependency):
         def _get_min(dependency):
             if dependency.name in self._use_latest:
             if dependency.name in self._use_latest:
+                # If we're forced to use the latest version of a package, it effectively
+                # only has one version to choose from.
                 return 1
                 return 1
 
 
             if dependency.name in self._locked:
             if dependency.name in self._locked:
@@ -326,9 +365,11 @@ class VersionSolver:
             except IndexError:
             except IndexError:
                 version = None
                 version = None
         else:
         else:
-            version = self._locked[dependency.name]
+            version = locked
 
 
         if version is None:
         if version is None:
+            # If there are no versions that satisfy the constraint,
+            # add an incompatibility that indicates that.
             self._add_incompatibility(
             self._add_incompatibility(
                 Incompatibility([Term(dependency, True)], NoVersionsCause())
                 Incompatibility([Term(dependency, True)], NoVersionsCause())
             )
             )
@@ -339,6 +380,11 @@ class VersionSolver:
         for incompatibility in self._provider.incompatibilities_for(version):
         for incompatibility in self._provider.incompatibilities_for(version):
             self._add_incompatibility(incompatibility)
             self._add_incompatibility(incompatibility)
 
 
+            # If an incompatibility is already satisfied, then selecting version
+            # would cause a conflict.
+            #
+            # We'll continue adding its dependencies, then go back to
+            # unit propagation which will guide us to choose a better version.
             conflict = conflict or all([
             conflict = conflict or all([
                 term.dependency.name == dependency.name or self._solution.satisfies(term)
                 term.dependency.name == dependency.name or self._solution.satisfies(term)
                 for term in incompatibility.terms
                 for term in incompatibility.terms
@@ -346,7 +392,7 @@ class VersionSolver:
 
 
         if not conflict:
         if not conflict:
             self._solution.decide(version)
             self._solution.decide(version)
-            self._log('selecting {}'.format(version))
+            self._log('<fg=blue>selecting</> {} ({})'.format(version.name, version.full_pretty_version))
 
 
         return dependency.name
         return dependency.name
 
 
@@ -354,6 +400,9 @@ class VersionSolver:
         return isinstance(VersionRange().difference(constraint), Version)
         return isinstance(VersionRange().difference(constraint), Version)
 
 
     def _result(self):  # type: () -> SolverResult
     def _result(self):  # type: () -> SolverResult
+        """
+        Creates a #SolverResult from the decisions in _solution
+        """
         decisions = self._solution.decisions
         decisions = self._solution.decisions
 
 
         return SolverResult(
         return SolverResult(
@@ -363,7 +412,7 @@ class VersionSolver:
         )
         )
 
 
     def _add_incompatibility(self, incompatibility):  # type: (Incompatibility) -> None
     def _add_incompatibility(self, incompatibility):  # type: (Incompatibility) -> None
-        self._log("fact: {}".format(incompatibility))
+        self._log("<fg=blue>fact</>: {}".format(incompatibility))
 
 
         for term in incompatibility.terms:
         for term in incompatibility.terms:
             if term.dependency.name not in self._incompatibilities:
             if term.dependency.name not in self._incompatibilities:
@@ -381,4 +430,4 @@ class VersionSolver:
         return self._locked.get(package_name)
         return self._locked.get(package_name)
 
 
     def _log(self, text):
     def _log(self, text):
-        self._provider.debug(text)
+        self._provider.debug(text, self._solution.attempted_solutions)

poetry/puzzle/provider.py

@@ -363,7 +363,7 @@ class Provider:
         if self.is_debugging():
         if self.is_debugging():
             debug_info = str(message)
             debug_info = str(message)
             debug_info = '\n'.join([
             debug_info = '\n'.join([
-                '<comment>:{}:</> {}'.format(str(depth).rjust(4), s)
+                '<comment>{}:</> {}'.format(str(depth).rjust(4), s)
                 for s in debug_info.split('\n')
                 for s in debug_info.split('\n')
             ]) + '\n'
             ]) + '\n'
 
 

poetry/repositories/pypi_repository.py

@@ -257,6 +257,8 @@ class PyPiRepository(Repository):
         )
         )
 
 
     def _get_release_info(self, name, version):  # type: (str, str) -> dict
     def _get_release_info(self, name, version):  # type: (str, str) -> dict
+        self._log('Getting info for {} ({}) from PyPI'.format(name, version), 'debug')
+
         json_data = self._get('pypi/{}/{}/json'.format(name, version))
         json_data = self._get('pypi/{}/{}/json'.format(name, version))
         if json_data is None:
         if json_data is None:
             raise ValueError('Package [{}] not found.'.format(name))
             raise ValueError('Package [{}] not found.'.format(name))