sources: introduce priority "explicit" (#7658)

Explicit sources are considered only for packages that explicitly indicate their source.

Co-authored-by: Randy Döring <30527984+radoering@users.noreply.github.com>

docs/cli.md

@@ -786,7 +786,7 @@ You cannot use the name `pypi` as it is reserved for use by the default PyPI sou
 
 * `--default`: Set this source as the [default]({{< relref "repositories#default-package-source" >}}) (disable PyPI). Deprecated in favor of `--priority`.
 * `--secondary`: Set this source as a [secondary]({{< relref "repositories#secondary-package-sources" >}}) source. Deprecated in favor of `--priority`.
-* `--priority`: Set the priority of this source. Accepted values are: [`default`]({{< relref "repositories#default-package-source" >}}), and [`secondary`]({{< relref "repositories#secondary-package-sources" >}}). Refer to the dedicated sections in [Repositories]({{< relref "repositories" >}}) for more information.
+* `--priority`: Set the priority of this source. Accepted values are: [`default`]({{< relref "repositories#default-package-source" >}}), [`secondary`]({{< relref "repositories#secondary-package-sources" >}}), and [`explicit`]({{< relref "repositories#explicit-package-sources" >}}). Refer to the dedicated sections in [Repositories]({{< relref "repositories" >}}) for more information.
 
 {{% note %}}
 At most one of the options above can be provided. See [package sources]({{< relref "repositories#package-sources" >}}) for more information.

docs/repositories.md

@@ -123,7 +123,7 @@ url = "https://foo.bar/simple/"
 priority = "primary"
 ```
 
-If `priority` is undefined, the source is considered a primary source that takes precedence over PyPI and secondary sources.
+If `priority` is undefined, the source is considered a primary source that takes precedence over PyPI, secondary and explicit sources.
 
 Package sources are considered in the following order:
 1. [default source](#default-package-source),
@@ -131,6 +131,8 @@ Package sources are considered in the following order:
 3. PyPI (unless disabled by another default source),
 4. [secondary sources](#secondary-package-sources),
 
+[Explicit sources](#explicit-package-sources) are considered only for packages that explicitly [indicate their source](#package-source-constraint).
+
 Within each priority class, package sources are considered in order of appearance in `pyproject.toml`.
 
 {{% note %}}
@@ -181,6 +183,20 @@ poetry source add --priority=secondary https://foo.bar/simple/
 
 There can be more than one secondary package source.
 
+#### Explicit Package Sources
+
+*Introduced in 1.5.0*
+
+If package sources are configured as explicit, these sources are only searched when a package configuration [explicitly indicates](#package-source-constraint) that it should be found on this package source.
+
+You can configure a package source as an explicit source with `priority = "explicit` in your package source configuration.
+
+```bash
+poetry source add --priority=explicit foo https://foo.bar/simple/
+```
+
+There can be more than one explicit package source.
+
 #### Package Source Constraint
 
 All package sources (including secondary sources) will be searched during the package lookup
@@ -209,6 +225,7 @@ priority = ...
 {{% note %}}
 
 A repository that is configured to be the only source for retrieving a certain package can itself have any priority.
+In particular, it does not need to have priority `"explicit"`.
 If a repository is configured to be the source of a package, it will be the only source that is considered for that package
 and the repository priority will have no effect on the resolution.
 

src/poetry/json/schemas/poetry.json

@@ -45,7 +45,8 @@
           "enum": [
             "primary",
             "default",
-            "secondary"
+            "secondary",
+            "explicit"
           ],
           "description": "Declare the priority of this repository."
         },

src/poetry/repositories/repository_pool.py

@@ -26,6 +26,7 @@ class Priority(IntEnum):
     DEFAULT = enum.auto()
     PRIMARY = enum.auto()
     SECONDARY = enum.auto()
+    EXPLICIT = enum.auto()
 
 
 @dataclass(frozen=True)
@@ -51,11 +52,30 @@ class RepositoryPool(AbstractRepository):
 
     @property
     def repositories(self) -> list[Repository]:
-        unsorted_repositories = self._repositories.values()
-        sorted_repositories = sorted(
-            unsorted_repositories, key=lambda prio_repo: prio_repo.priority
+        """
+        Returns the repositories in the pool,
+        in the order they will be searched for packages.
+
+        ATTENTION: For backwards compatibility and practical reasons,
+                   repositories with priority EXPLICIT are NOT included,
+                   because they will not be searched.
+        """
+        sorted_repositories = self._sorted_repositories
+        return [
+            prio_repo.repository
+            for prio_repo in sorted_repositories
+            if prio_repo.priority is not Priority.EXPLICIT
+        ]
+
+    @property
+    def all_repositories(self) -> list[Repository]:
+        return [prio_repo.repository for prio_repo in self._sorted_repositories]
+
+    @property
+    def _sorted_repositories(self) -> list[PrioritizedRepository]:
+        return sorted(
+            self._repositories.values(), key=lambda prio_repo: prio_repo.priority
         )
-        return [prio_repo.repository for prio_repo in sorted_repositories]
 
     def has_default(self) -> bool:
         return self._contains_priority(Priority.DEFAULT)

tests/console/commands/source/conftest.py

@@ -51,6 +51,13 @@ def source_secondary() -> Source:
     )
 
 
+@pytest.fixture
+def source_explicit() -> Source:
+    return Source(
+        name="explicit", url="https://explicit.com", priority=Priority.EXPLICIT
+    )
+
+
 _existing_source = Source(name="existing", url="https://existing.com")
 
 
@@ -110,11 +117,13 @@ def add_all_source_types(
     source_primary: Source,
     source_default: Source,
     source_secondary: Source,
+    source_explicit: Source,
 ) -> None:
     add = command_tester_factory("source add", poetry=poetry_with_source)
     for source in [
         source_primary,
         source_default,
         source_secondary,
+        source_explicit,
     ]:
         add.execute(f"{source.name} {source.url} --priority={source.name}")

tests/console/commands/source/test_add.py

@@ -136,6 +136,16 @@ def test_source_add_secondary(
     assert_source_added(tester, poetry_with_source, source_existing, source_secondary)
 
 
+def test_source_add_explicit(
+    tester: CommandTester,
+    source_existing: Source,
+    source_explicit: Source,
+    poetry_with_source: Poetry,
+) -> None:
+    tester.execute(f"--priority=explicit {source_explicit.name} {source_explicit.url}")
+    assert_source_added(tester, poetry_with_source, source_existing, source_explicit)
+
+
 def test_source_add_error_default_and_secondary_legacy(tester: CommandTester) -> None:
     tester.execute("--default --secondary error https://error.com")
     assert (

tests/console/commands/source/test_show.py

@@ -101,6 +101,7 @@ priority  : primary
         "source_primary",
         "source_default",
         "source_secondary",
+        "source_explicit",
     ),
 )
 def test_source_show_given_priority(

tests/fixtures/with_explicit_source/pyproject.toml

+[tool.poetry]
+name = "my-package"
+version = "1.2.3"
+description = "Some description."
+authors = [
+    "Your Name <you@example.com>"
+]
+license = "MIT"
+
+# Requirements
+[tool.poetry.dependencies]
+python = "~2.7 || ^3.6"
+
+[tool.poetry.dev-dependencies]
+
+[[tool.poetry.source]]
+name = "explicit"
+url = "https://explicit.com/simple/"
+priority = "explicit"

tests/json/fixtures/source/complete_valid.toml

@@ -10,7 +10,7 @@ python = "^3.10"
 [[tool.poetry.source]]
 name = "pypi-simple"
 url = "https://pypi.org/simple/"
-priority = "primary"
+priority = "explicit"
 
 [build-system]
 requires = ["poetry-core"]

tests/json/test_schema_sources.py

@@ -36,7 +36,7 @@ def test_pyproject_toml_invalid_priority() -> None:
     assert Factory.validate(content) == {
         "errors": [
             "[source.0.priority] 'arbitrary' is not one of ['primary', 'default',"
-            " 'secondary']"
+            " 'secondary', 'explicit']"
         ],
         "warnings": [],
     }

tests/puzzle/test_solver.py

@@ -2998,6 +2998,71 @@ def test_solver_chooses_from_secondary_if_explicit(
     assert ops[2].package.source_url is None
 
 
+def test_solver_does_not_choose_from_explicit_repository(
+    package: ProjectPackage, io: NullIO
+) -> None:
+    package.python_versions = "^3.7"
+    package.add_dependency(Factory.create_dependency("attrs", {"version": "^17.4.0"}))
+
+    pool = RepositoryPool()
+    pool.add_repository(MockPyPIRepository(), priority=Priority.EXPLICIT)
+    pool.add_repository(MockLegacyRepository())
+
+    solver = Solver(package, pool, [], [], io)
+
+    with pytest.raises(SolverProblemError):
+        solver.solve()
+
+
+def test_solver_chooses_direct_dependency_from_explicit_if_explicit(
+    package: ProjectPackage,
+    io: NullIO,
+) -> None:
+    package.python_versions = "^3.7"
+    package.add_dependency(
+        Factory.create_dependency("pylev", {"version": "^1.2.0", "source": "PyPI"})
+    )
+
+    pool = RepositoryPool()
+    pool.add_repository(MockPyPIRepository(), priority=Priority.EXPLICIT)
+    pool.add_repository(MockLegacyRepository())
+
+    solver = Solver(package, pool, [], [], io)
+
+    transaction = solver.solve()
+
+    ops = check_solver_result(
+        transaction,
+        [
+            {"job": "install", "package": get_package("pylev", "1.3.0")},
+        ],
+    )
+
+    assert ops[0].package.source_type is None
+    assert ops[0].package.source_url is None
+
+
+def test_solver_ignores_explicit_repo_for_transient_dependencies(
+    package: ProjectPackage,
+    io: NullIO,
+) -> None:
+    # clikit depends on pylev, which is in MockPyPIRepository (explicit) but not in
+    # MockLegacyRepository
+    package.python_versions = "^3.7"
+    package.add_dependency(
+        Factory.create_dependency("clikit", {"version": "^0.2.0", "source": "PyPI"})
+    )
+
+    pool = RepositoryPool()
+    pool.add_repository(MockPyPIRepository(), priority=Priority.EXPLICIT)
+    pool.add_repository(MockLegacyRepository())
+
+    solver = Solver(package, pool, [], [], io)
+
+    with pytest.raises(SolverProblemError):
+        solver.solve()
+
+
 def test_solver_discards_packages_with_empty_markers(
     package: ProjectPackage,
     repo: Repository,

tests/repositories/test_repository_pool.py

@@ -81,26 +81,43 @@ def test_repository_from_single_repo_pool_legacy(
     assert pool.get_priority("foo") == expected_priority
 
 
-def test_repository_with_normal_default_and_secondary_repositories() -> None:
+def test_repository_with_normal_default_secondary_and_explicit_repositories():
     secondary = LegacyRepository("secondary", "https://secondary.com")
     default = LegacyRepository("default", "https://default.com")
     repo1 = LegacyRepository("foo", "https://foo.bar")
     repo2 = LegacyRepository("bar", "https://bar.baz")
+    explicit = LegacyRepository("explicit", "https://bar.baz")
 
     pool = RepositoryPool()
     pool.add_repository(repo1)
     pool.add_repository(secondary, priority=Priority.SECONDARY)
     pool.add_repository(repo2)
+    pool.add_repository(explicit, priority=Priority.EXPLICIT)
     pool.add_repository(default, priority=Priority.DEFAULT)
 
     assert pool.repository("secondary") is secondary
     assert pool.repository("default") is default
     assert pool.repository("foo") is repo1
     assert pool.repository("bar") is repo2
+    assert pool.repository("explicit") is explicit
     assert pool.has_default()
     assert pool.has_primary_repositories()
 
 
+def test_repository_explicit_repositories_do_not_show() -> None:
+    explicit = LegacyRepository("explicit", "https://explicit.com")
+    default = LegacyRepository("default", "https://default.com")
+
+    pool = RepositoryPool()
+    pool.add_repository(explicit, priority=Priority.EXPLICIT)
+    pool.add_repository(default, priority=Priority.DEFAULT)
+
+    assert pool.repository("explicit") is explicit
+    assert pool.repository("default") is default
+    assert pool.repositories == [default]
+    assert pool.all_repositories == [default, explicit]
+
+
 def test_remove_non_existing_repository_raises_indexerror() -> None:
     pool = RepositoryPool()
 

tests/test_factory.py

@@ -338,6 +338,19 @@ def test_poetry_with_no_default_source():
     assert {repo.name for repo in poetry.pool.repositories} == {"PyPI"}
 
 
+def test_poetry_with_explicit_source(with_simple_keyring: None) -> None:
+    poetry = Factory().create_poetry(fixtures_dir / "with_explicit_source")
+
+    assert len(poetry.pool.repositories) == 1
+    assert len(poetry.pool.all_repositories) == 2
+    assert poetry.pool.has_repository("PyPI")
+    assert poetry.pool.get_priority("PyPI") is Priority.DEFAULT
+    assert isinstance(poetry.pool.repository("PyPI"), PyPiRepository)
+    assert poetry.pool.has_repository("explicit")
+    assert isinstance(poetry.pool.repository("explicit"), LegacyRepository)
+    assert [repo.name for repo in poetry.pool.repositories] == ["PyPI"]
+
+
 def test_poetry_with_two_default_sources_legacy(with_simple_keyring: None):
     with pytest.raises(ValueError) as e:
         Factory().create_poetry(fixtures_dir / "with_two_default_sources_legacy")

tests/utils/test_source.py

@@ -23,10 +23,10 @@ from poetry.utils.source import source_to_table
             },
         ),
         (
-            Source("bar", "https://example.com/bar", priority=Priority.SECONDARY),
+            Source("bar", "https://example.com/bar", priority=Priority.EXPLICIT),
             {
                 "name": "bar",
-                "priority": "secondary",
+                "priority": "explicit",
                 "url": "https://example.com/bar",
             },
         ),