sources: introduce priority "explicit" (#7658)

Explicit sources are considered only for packages that explicitly indicate their source.

Co-authored-by: Randy Döring <30527984+radoering@users.noreply.github.com>

docs/cli.md

@@ -786,7 +786,7 @@ You cannot use the name `pypi` as it is reserved for use by the default PyPI sou
 
 
 * `--default`: Set this source as the [default]({{< relref "repositories#default-package-source" >}}) (disable PyPI). Deprecated in favor of `--priority`.
 * `--default`: Set this source as the [default]({{< relref "repositories#default-package-source" >}}) (disable PyPI). Deprecated in favor of `--priority`.
 * `--secondary`: Set this source as a [secondary]({{< relref "repositories#secondary-package-sources" >}}) source. Deprecated in favor of `--priority`.
 * `--secondary`: Set this source as a [secondary]({{< relref "repositories#secondary-package-sources" >}}) source. Deprecated in favor of `--priority`.
-* `--priority`: Set the priority of this source. Accepted values are: [`default`]({{< relref "repositories#default-package-source" >}}), and [`secondary`]({{< relref "repositories#secondary-package-sources" >}}). Refer to the dedicated sections in [Repositories]({{< relref "repositories" >}}) for more information.
+* `--priority`: Set the priority of this source. Accepted values are: [`default`]({{< relref "repositories#default-package-source" >}}), [`secondary`]({{< relref "repositories#secondary-package-sources" >}}), and [`explicit`]({{< relref "repositories#explicit-package-sources" >}}). Refer to the dedicated sections in [Repositories]({{< relref "repositories" >}}) for more information.
 
 
 {{% note %}}
 {{% note %}}
 At most one of the options above can be provided. See [package sources]({{< relref "repositories#package-sources" >}}) for more information.
 At most one of the options above can be provided. See [package sources]({{< relref "repositories#package-sources" >}}) for more information.

docs/repositories.md

@@ -123,7 +123,7 @@ url = "https://foo.bar/simple/"
 priority = "primary"
 priority = "primary"
 ```
 ```
 
 
-If `priority` is undefined, the source is considered a primary source that takes precedence over PyPI and secondary sources.
+If `priority` is undefined, the source is considered a primary source that takes precedence over PyPI, secondary and explicit sources.
 
 
 Package sources are considered in the following order:
 Package sources are considered in the following order:
 1. [default source](#default-package-source),
 1. [default source](#default-package-source),
@@ -131,6 +131,8 @@ Package sources are considered in the following order:
 3. PyPI (unless disabled by another default source),
 3. PyPI (unless disabled by another default source),
 4. [secondary sources](#secondary-package-sources),
 4. [secondary sources](#secondary-package-sources),
 
 
+[Explicit sources](#explicit-package-sources) are considered only for packages that explicitly [indicate their source](#package-source-constraint).
+
 Within each priority class, package sources are considered in order of appearance in `pyproject.toml`.
 Within each priority class, package sources are considered in order of appearance in `pyproject.toml`.
 
 
 {{% note %}}
 {{% note %}}
@@ -181,6 +183,20 @@ poetry source add --priority=secondary https://foo.bar/simple/
 
 
 There can be more than one secondary package source.
 There can be more than one secondary package source.
 
 
+#### Explicit Package Sources
+
+*Introduced in 1.5.0*
+
+If package sources are configured as explicit, these sources are only searched when a package configuration [explicitly indicates](#package-source-constraint) that it should be found on this package source.
+
+You can configure a package source as an explicit source with `priority = "explicit` in your package source configuration.
+
+```bash
+poetry source add --priority=explicit foo https://foo.bar/simple/
+```
+
+There can be more than one explicit package source.
+
 #### Package Source Constraint
 #### Package Source Constraint
 
 
 All package sources (including secondary sources) will be searched during the package lookup
 All package sources (including secondary sources) will be searched during the package lookup
@@ -209,6 +225,7 @@ priority = ...
 {{% note %}}
 {{% note %}}
 
 
 A repository that is configured to be the only source for retrieving a certain package can itself have any priority.
 A repository that is configured to be the only source for retrieving a certain package can itself have any priority.
+In particular, it does not need to have priority `"explicit"`.
 If a repository is configured to be the source of a package, it will be the only source that is considered for that package
 If a repository is configured to be the source of a package, it will be the only source that is considered for that package
 and the repository priority will have no effect on the resolution.
 and the repository priority will have no effect on the resolution.
 
 

src/poetry/json/schemas/poetry.json

@@ -45,7 +45,8 @@
           "enum": [
           "enum": [
             "primary",
             "primary",
             "default",
             "default",
-            "secondary"
+            "secondary",
+            "explicit"
           ],
           ],
           "description": "Declare the priority of this repository."
           "description": "Declare the priority of this repository."
         },
         },

src/poetry/repositories/repository_pool.py

@@ -26,6 +26,7 @@ class Priority(IntEnum):
     DEFAULT = enum.auto()
     DEFAULT = enum.auto()
     PRIMARY = enum.auto()
     PRIMARY = enum.auto()
     SECONDARY = enum.auto()
     SECONDARY = enum.auto()
+    EXPLICIT = enum.auto()
 
 
 
 
 @dataclass(frozen=True)
 @dataclass(frozen=True)
@@ -51,11 +52,30 @@ class RepositoryPool(AbstractRepository):
 
 
     @property
     @property
     def repositories(self) -> list[Repository]:
     def repositories(self) -> list[Repository]:
-        unsorted_repositories = self._repositories.values()
-        sorted_repositories = sorted(
-            unsorted_repositories, key=lambda prio_repo: prio_repo.priority
+        """
+        Returns the repositories in the pool,
+        in the order they will be searched for packages.
+
+        ATTENTION: For backwards compatibility and practical reasons,
+                   repositories with priority EXPLICIT are NOT included,
+                   because they will not be searched.
+        """
+        sorted_repositories = self._sorted_repositories
+        return [
+            prio_repo.repository
+            for prio_repo in sorted_repositories
+            if prio_repo.priority is not Priority.EXPLICIT
+        ]
+
+    @property
+    def all_repositories(self) -> list[Repository]:
+        return [prio_repo.repository for prio_repo in self._sorted_repositories]
+
+    @property
+    def _sorted_repositories(self) -> list[PrioritizedRepository]:
+        return sorted(
+            self._repositories.values(), key=lambda prio_repo: prio_repo.priority
         )
         )
-        return [prio_repo.repository for prio_repo in sorted_repositories]
 
 
     def has_default(self) -> bool:
     def has_default(self) -> bool:
         return self._contains_priority(Priority.DEFAULT)
         return self._contains_priority(Priority.DEFAULT)

tests/console/commands/source/conftest.py

@@ -51,6 +51,13 @@ def source_secondary() -> Source:
     )
     )
 
 
 
 
+@pytest.fixture
+def source_explicit() -> Source:
+    return Source(
+        name="explicit", url="https://explicit.com", priority=Priority.EXPLICIT
+    )
+
+
 _existing_source = Source(name="existing", url="https://existing.com")
 _existing_source = Source(name="existing", url="https://existing.com")
 
 
 
 
@@ -110,11 +117,13 @@ def add_all_source_types(
     source_primary: Source,
     source_primary: Source,
     source_default: Source,
     source_default: Source,
     source_secondary: Source,
     source_secondary: Source,
+    source_explicit: Source,
 ) -> None:
 ) -> None:
     add = command_tester_factory("source add", poetry=poetry_with_source)
     add = command_tester_factory("source add", poetry=poetry_with_source)
     for source in [
     for source in [
         source_primary,
         source_primary,
         source_default,
         source_default,
         source_secondary,
         source_secondary,
+        source_explicit,
     ]:
     ]:
         add.execute(f"{source.name} {source.url} --priority={source.name}")
         add.execute(f"{source.name} {source.url} --priority={source.name}")

tests/console/commands/source/test_add.py

@@ -136,6 +136,16 @@ def test_source_add_secondary(
     assert_source_added(tester, poetry_with_source, source_existing, source_secondary)
     assert_source_added(tester, poetry_with_source, source_existing, source_secondary)
 
 
 
 
+def test_source_add_explicit(
+    tester: CommandTester,
+    source_existing: Source,
+    source_explicit: Source,
+    poetry_with_source: Poetry,
+) -> None:
+    tester.execute(f"--priority=explicit {source_explicit.name} {source_explicit.url}")
+    assert_source_added(tester, poetry_with_source, source_existing, source_explicit)
+
+
 def test_source_add_error_default_and_secondary_legacy(tester: CommandTester) -> None:
 def test_source_add_error_default_and_secondary_legacy(tester: CommandTester) -> None:
     tester.execute("--default --secondary error https://error.com")
     tester.execute("--default --secondary error https://error.com")
     assert (
     assert (

tests/console/commands/source/test_show.py

@@ -101,6 +101,7 @@ priority  : primary
         "source_primary",
         "source_primary",
         "source_default",
         "source_default",
         "source_secondary",
         "source_secondary",
+        "source_explicit",
     ),
     ),
 )
 )
 def test_source_show_given_priority(
 def test_source_show_given_priority(

tests/fixtures/with_explicit_source/pyproject.toml

+[tool.poetry]
+name = "my-package"
+version = "1.2.3"
+description = "Some description."
+authors = [
+    "Your Name <you@example.com>"
+]
+license = "MIT"
+
+# Requirements
+[tool.poetry.dependencies]
+python = "~2.7 || ^3.6"
+
+[tool.poetry.dev-dependencies]
+
+[[tool.poetry.source]]
+name = "explicit"
+url = "https://explicit.com/simple/"
+priority = "explicit"

tests/json/fixtures/source/complete_valid.toml

@@ -10,7 +10,7 @@ python = "^3.10"
 [[tool.poetry.source]]
 [[tool.poetry.source]]
 name = "pypi-simple"
 name = "pypi-simple"
 url = "https://pypi.org/simple/"
 url = "https://pypi.org/simple/"
-priority = "primary"
+priority = "explicit"
 
 
 [build-system]
 [build-system]
 requires = ["poetry-core"]
 requires = ["poetry-core"]

tests/json/test_schema_sources.py

@@ -36,7 +36,7 @@ def test_pyproject_toml_invalid_priority() -> None:
     assert Factory.validate(content) == {
     assert Factory.validate(content) == {
         "errors": [
         "errors": [
             "[source.0.priority] 'arbitrary' is not one of ['primary', 'default',"
             "[source.0.priority] 'arbitrary' is not one of ['primary', 'default',"
-            " 'secondary']"
+            " 'secondary', 'explicit']"
         ],
         ],
         "warnings": [],
         "warnings": [],
     }
     }

tests/puzzle/test_solver.py

@@ -2998,6 +2998,71 @@ def test_solver_chooses_from_secondary_if_explicit(
     assert ops[2].package.source_url is None
     assert ops[2].package.source_url is None
 
 
 
 
+def test_solver_does_not_choose_from_explicit_repository(
+    package: ProjectPackage, io: NullIO
+) -> None:
+    package.python_versions = "^3.7"
+    package.add_dependency(Factory.create_dependency("attrs", {"version": "^17.4.0"}))
+
+    pool = RepositoryPool()
+    pool.add_repository(MockPyPIRepository(), priority=Priority.EXPLICIT)
+    pool.add_repository(MockLegacyRepository())
+
+    solver = Solver(package, pool, [], [], io)
+
+    with pytest.raises(SolverProblemError):
+        solver.solve()
+
+
+def test_solver_chooses_direct_dependency_from_explicit_if_explicit(
+    package: ProjectPackage,
+    io: NullIO,
+) -> None:
+    package.python_versions = "^3.7"
+    package.add_dependency(
+        Factory.create_dependency("pylev", {"version": "^1.2.0", "source": "PyPI"})
+    )
+
+    pool = RepositoryPool()
+    pool.add_repository(MockPyPIRepository(), priority=Priority.EXPLICIT)
+    pool.add_repository(MockLegacyRepository())
+
+    solver = Solver(package, pool, [], [], io)
+
+    transaction = solver.solve()
+
+    ops = check_solver_result(
+        transaction,
+        [
+            {"job": "install", "package": get_package("pylev", "1.3.0")},
+        ],
+    )
+
+    assert ops[0].package.source_type is None
+    assert ops[0].package.source_url is None
+
+
+def test_solver_ignores_explicit_repo_for_transient_dependencies(
+    package: ProjectPackage,
+    io: NullIO,
+) -> None:
+    # clikit depends on pylev, which is in MockPyPIRepository (explicit) but not in
+    # MockLegacyRepository
+    package.python_versions = "^3.7"
+    package.add_dependency(
+        Factory.create_dependency("clikit", {"version": "^0.2.0", "source": "PyPI"})
+    )
+
+    pool = RepositoryPool()
+    pool.add_repository(MockPyPIRepository(), priority=Priority.EXPLICIT)
+    pool.add_repository(MockLegacyRepository())
+
+    solver = Solver(package, pool, [], [], io)
+
+    with pytest.raises(SolverProblemError):
+        solver.solve()
+
+
 def test_solver_discards_packages_with_empty_markers(
 def test_solver_discards_packages_with_empty_markers(
     package: ProjectPackage,
     package: ProjectPackage,
     repo: Repository,
     repo: Repository,

tests/repositories/test_repository_pool.py

@@ -81,26 +81,43 @@ def test_repository_from_single_repo_pool_legacy(
     assert pool.get_priority("foo") == expected_priority
     assert pool.get_priority("foo") == expected_priority
 
 
 
 
-def test_repository_with_normal_default_and_secondary_repositories() -> None:
+def test_repository_with_normal_default_secondary_and_explicit_repositories():
     secondary = LegacyRepository("secondary", "https://secondary.com")
     secondary = LegacyRepository("secondary", "https://secondary.com")
     default = LegacyRepository("default", "https://default.com")
     default = LegacyRepository("default", "https://default.com")
     repo1 = LegacyRepository("foo", "https://foo.bar")
     repo1 = LegacyRepository("foo", "https://foo.bar")
     repo2 = LegacyRepository("bar", "https://bar.baz")
     repo2 = LegacyRepository("bar", "https://bar.baz")
+    explicit = LegacyRepository("explicit", "https://bar.baz")
 
 
     pool = RepositoryPool()
     pool = RepositoryPool()
     pool.add_repository(repo1)
     pool.add_repository(repo1)
     pool.add_repository(secondary, priority=Priority.SECONDARY)
     pool.add_repository(secondary, priority=Priority.SECONDARY)
     pool.add_repository(repo2)
     pool.add_repository(repo2)
+    pool.add_repository(explicit, priority=Priority.EXPLICIT)
     pool.add_repository(default, priority=Priority.DEFAULT)
     pool.add_repository(default, priority=Priority.DEFAULT)
 
 
     assert pool.repository("secondary") is secondary
     assert pool.repository("secondary") is secondary
     assert pool.repository("default") is default
     assert pool.repository("default") is default
     assert pool.repository("foo") is repo1
     assert pool.repository("foo") is repo1
     assert pool.repository("bar") is repo2
     assert pool.repository("bar") is repo2
+    assert pool.repository("explicit") is explicit
     assert pool.has_default()
     assert pool.has_default()
     assert pool.has_primary_repositories()
     assert pool.has_primary_repositories()
 
 
 
 
+def test_repository_explicit_repositories_do_not_show() -> None:
+    explicit = LegacyRepository("explicit", "https://explicit.com")
+    default = LegacyRepository("default", "https://default.com")
+
+    pool = RepositoryPool()
+    pool.add_repository(explicit, priority=Priority.EXPLICIT)
+    pool.add_repository(default, priority=Priority.DEFAULT)
+
+    assert pool.repository("explicit") is explicit
+    assert pool.repository("default") is default
+    assert pool.repositories == [default]
+    assert pool.all_repositories == [default, explicit]
+
+
 def test_remove_non_existing_repository_raises_indexerror() -> None:
 def test_remove_non_existing_repository_raises_indexerror() -> None:
     pool = RepositoryPool()
     pool = RepositoryPool()
 
 

tests/test_factory.py

@@ -338,6 +338,19 @@ def test_poetry_with_no_default_source():
     assert {repo.name for repo in poetry.pool.repositories} == {"PyPI"}
     assert {repo.name for repo in poetry.pool.repositories} == {"PyPI"}
 
 
 
 
+def test_poetry_with_explicit_source(with_simple_keyring: None) -> None:
+    poetry = Factory().create_poetry(fixtures_dir / "with_explicit_source")
+
+    assert len(poetry.pool.repositories) == 1
+    assert len(poetry.pool.all_repositories) == 2
+    assert poetry.pool.has_repository("PyPI")
+    assert poetry.pool.get_priority("PyPI") is Priority.DEFAULT
+    assert isinstance(poetry.pool.repository("PyPI"), PyPiRepository)
+    assert poetry.pool.has_repository("explicit")
+    assert isinstance(poetry.pool.repository("explicit"), LegacyRepository)
+    assert [repo.name for repo in poetry.pool.repositories] == ["PyPI"]
+
+
 def test_poetry_with_two_default_sources_legacy(with_simple_keyring: None):
 def test_poetry_with_two_default_sources_legacy(with_simple_keyring: None):
     with pytest.raises(ValueError) as e:
     with pytest.raises(ValueError) as e:
         Factory().create_poetry(fixtures_dir / "with_two_default_sources_legacy")
         Factory().create_poetry(fixtures_dir / "with_two_default_sources_legacy")

tests/utils/test_source.py

@@ -23,10 +23,10 @@ from poetry.utils.source import source_to_table
             },
             },
         ),
         ),
         (
         (
-            Source("bar", "https://example.com/bar", priority=Priority.SECONDARY),
+            Source("bar", "https://example.com/bar", priority=Priority.EXPLICIT),
             {
             {
                 "name": "bar",
                 "name": "bar",
-                "priority": "secondary",
+                "priority": "explicit",
                 "url": "https://example.com/bar",
                 "url": "https://example.com/bar",
             },
             },
         ),
         ),