Introduce supplemental package source (#6879)

docs/cli.md

@@ -790,7 +790,7 @@ poetry source add --priority=explicit pypi
 
 * `--default`: Set this source as the [default]({{< relref "repositories#default-package-source" >}}) (disable PyPI). Deprecated in favor of `--priority`.
 * `--secondary`: Set this source as a [secondary]({{< relref "repositories#secondary-package-sources" >}}) source. Deprecated in favor of `--priority`.
-* `--priority`: Set the priority of this source. Accepted values are: [`default`]({{< relref "repositories#default-package-source" >}}), [`secondary`]({{< relref "repositories#secondary-package-sources" >}}), and [`explicit`]({{< relref "repositories#explicit-package-sources" >}}). Refer to the dedicated sections in [Repositories]({{< relref "repositories" >}}) for more information.
+* `--priority`: Set the priority of this source. Accepted values are: [`default`]({{< relref "repositories#default-package-source" >}}), [`secondary`]({{< relref "repositories#secondary-package-sources" >}}), [`supplemental`]({{< relref "repositories#supplemental-package-sources" >}}), and [`explicit`]({{< relref "repositories#explicit-package-sources" >}}). Refer to the dedicated sections in [Repositories]({{< relref "repositories" >}}) for more information.
 
 {{% note %}}
 At most one of the options above can be provided. See [package sources]({{< relref "repositories#package-sources" >}}) for more information.

docs/dependency-specification.md

@@ -258,7 +258,7 @@ you can use the `source` property:
 [[tool.poetry.source]]
 name = "foo"
 url = "https://foo.bar/simple/"
-priority = "secondary"
+priority = "supplemental"
 
 [tool.poetry.dependencies]
 my-cool-package = { version = "*", source = "foo" }

docs/repositories.md

@@ -29,11 +29,11 @@ By default, Poetry discovers and installs packages from [PyPI](https://pypi.org)
 install a dependency to your project for a [simple API repository](#simple-api-repository)? Let's
 do it.
 
-First, [configure](#project-configuration) the [package source](#package-source) as a [secondary package source](#secondary-package-sources) to your
+First, [configure](#project-configuration) the [package source](#package-source) as a [supplemental](#supplemental-package-sources) (or [explicit](#explicit-package-sources)) package source to your
 project.
 
 ```bash
-poetry source add --priority=secondary foo https://pypi.example.org/simple/
+poetry source add --priority=supplemental foo https://pypi.example.org/simple/
 ```
 
 Then, assuming the repository requires authentication, configure credentials for it.
@@ -123,13 +123,14 @@ url = "https://foo.bar/simple/"
 priority = "primary"
 ```
 
-If `priority` is undefined, the source is considered a primary source that takes precedence over PyPI, secondary and explicit sources.
+If `priority` is undefined, the source is considered a primary source that takes precedence over PyPI, secondary, supplemental and explicit sources.
 
 Package sources are considered in the following order:
 1. [default source](#default-package-source),
 2. primary sources,
 3. implicit PyPI (unless disabled by another [default source](#default-package-source) or configured explicitly),
-4. [secondary sources](#secondary-package-sources),
+4. [secondary sources](#secondary-package-sources) (DEPRECATED),
+5. [supplemental sources](#supplemental-package-sources).
 
 [Explicit sources](#explicit-package-sources) are considered only for packages that explicitly [indicate their source](#package-source-constraint).
 
@@ -182,10 +183,12 @@ as a package source for your project.
 
 {{% /warning %}}
 
-#### Secondary Package Sources
+#### Secondary Package Sources (DEPRECATED)
+
+*Deprecated in 1.5.0*
 
 If package sources are configured as secondary, all it means is that these will be given a lower
-priority when selecting compatible package distribution that also exists in your default and primary package sources.
+priority when selecting compatible package distribution that also exists in your default and primary package sources. If the package source should instead be searched only if higher-priority repositories did not return results, please consider a [supplemental source](#supplemental-package-sources) instead.
 
 You can configure a package source as a secondary source with `priority = "secondary"` in your package
 source configuration.
@@ -196,6 +199,28 @@ poetry source add --priority=secondary https://foo.bar/simple/
 
 There can be more than one secondary package source.
 
+{{% warning %}}
+
+Secondary package sources are deprecated in favor of supplemental package sources.
+
+{{% /warning %}}
+
+#### Supplemental Package Sources
+
+*Introduced in 1.5.0*
+
+Package sources configured as supplemental are only searched if no other (higher-priority) source yields a compatible package distribution. This is particularly convenient if the response time of the source is high and relatively few package distributions are to be fetched from this source.
+
+You can configure a package source as a supplemental source with `priority = "supplemental"` in your package
+source configuration.
+
+```bash
+poetry source add --priority=supplemental https://foo.bar/simple/
+```
+
+There can be more than one supplemental package source.
+
+
 #### Explicit Package Sources
 
 *Introduced in 1.5.0*
@@ -212,7 +237,7 @@ There can be more than one explicit package source.
 
 #### Package Source Constraint
 
-All package sources (including secondary sources) will be searched during the package lookup
+All package sources (including secondary and possibly supplemental sources) will be searched during the package lookup
 process. These network requests will occur for all sources, regardless of if the package is
 found at one or more sources.
 

src/poetry/console/commands/source/add.py

@@ -114,6 +114,14 @@ class SourceAddCommand(Command):
         else:
             priority = Priority[priority_str.upper()]
 
+        if priority is Priority.SECONDARY:
+            allowed_prios = (p for p in Priority if p is not Priority.SECONDARY)
+            self.line_error(
+                "<warning>Warning: Priority 'secondary' is deprecated. Consider"
+                " changing the priority to one of the non-deprecated values:"
+                f" {', '.join(repr(p.name.lower()) for p in allowed_prios)}.</warning>"
+            )
+
         sources = AoT([])
         new_source = Source(name=name, url=url, priority=priority)
         is_new_source = True

src/poetry/factory.py

@@ -155,6 +155,16 @@ class Factory(BaseFactory):
                 elif source.get("secondary"):
                     priority = Priority.SECONDARY
 
+            if priority is Priority.SECONDARY:
+                allowed_prios = (p for p in Priority if p is not Priority.SECONDARY)
+                warning = (
+                    "Found deprecated priority 'secondary' for source"
+                    f" '{source.get('name')}' in pyproject.toml. Consider changing the"
+                    " priority to one of the non-deprecated values:"
+                    f" {', '.join(repr(p.name.lower()) for p in allowed_prios)}."
+                )
+                io.write_error_line(f"<warning>Warning: {warning}</warning>")
+
             if io.is_debug():
                 message = f"Adding repository {repository.name} ({repository.url})"
                 if priority is Priority.DEFAULT:

src/poetry/json/schemas/poetry.json

@@ -45,6 +45,7 @@
             "primary",
             "default",
             "secondary",
+            "supplemental",
             "explicit"
           ],
           "description": "Declare the priority of this repository."

src/poetry/repositories/repository_pool.py

@@ -28,6 +28,7 @@ class Priority(IntEnum):
     DEFAULT = enum.auto()
     PRIMARY = enum.auto()
     SECONDARY = enum.auto()
+    SUPPLEMENTAL = enum.auto()
     EXPLICIT = enum.auto()
 
 
@@ -186,11 +187,13 @@ class RepositoryPool(AbstractRepository):
 
         packages: list[Package] = []
         for repo in self.repositories:
+            if packages and self.get_priority(repo.name) is Priority.SUPPLEMENTAL:
+                break
             packages += repo.find_packages(dependency)
         return packages
 
     def search(self, query: str) -> list[Package]:
         results: list[Package] = []
-        for repository in self.repositories:
-            results += repository.search(query)
+        for repo in self.repositories:
+            results += repo.search(query)
         return results

tests/console/commands/source/conftest.py

@@ -52,6 +52,15 @@ def source_secondary() -> Source:
 
 
 @pytest.fixture
+def source_supplemental() -> Source:
+    return Source(
+        name="supplemental",
+        url="https://supplemental.com",
+        priority=Priority.SUPPLEMENTAL,
+    )
+
+
+@pytest.fixture
 def source_explicit() -> Source:
     return Source(
         name="explicit", url="https://explicit.com", priority=Priority.EXPLICIT
@@ -151,6 +160,7 @@ def add_all_source_types(
     source_primary: Source,
     source_default: Source,
     source_secondary: Source,
+    source_supplemental: Source,
     source_explicit: Source,
 ) -> None:
     add = command_tester_factory("source add", poetry=poetry_with_source)
@@ -158,6 +168,7 @@ def add_all_source_types(
         source_primary,
         source_default,
         source_secondary,
+        source_supplemental,
         source_explicit,
     ]:
         add.execute(f"{source.name} {source.url} --priority={source.name}")

tests/console/commands/source/test_add.py

@@ -28,11 +28,20 @@ def assert_source_added_legacy(
     source_existing: Source,
     source_added: Source,
 ) -> None:
+    secondary_deprecated_str = (
+        ""
+        if source_added.priority is not Priority.SECONDARY
+        else (
+            "\nWarning: Priority 'secondary' is deprecated. Consider changing the"
+            " priority to one of the non-deprecated values: 'default', 'primary',"
+            " 'supplemental', 'explicit'."
+        )
+    )
     assert (
         tester.io.fetch_error().strip()
-        == "Warning: Priority was set through a deprecated flag"
-        " (--default or --secondary). Consider using --priority next"
-        " time."
+        == "Warning: Priority was set through a deprecated flag (--default or"
+        " --secondary). Consider using --priority next time."
+        + secondary_deprecated_str
     )
     assert (
         tester.io.fetch_output().strip()
@@ -136,6 +145,20 @@ def test_source_add_secondary(
     assert_source_added(tester, poetry_with_source, source_existing, source_secondary)
 
 
+def test_source_add_supplemental(
+    tester: CommandTester,
+    source_existing: Source,
+    source_supplemental: Source,
+    poetry_with_source: Poetry,
+) -> None:
+    tester.execute(
+        f"--priority=supplemental {source_supplemental.name} {source_supplemental.url}"
+    )
+    assert_source_added(
+        tester, poetry_with_source, source_existing, source_supplemental
+    )
+
+
 def test_source_add_explicit(
     tester: CommandTester,
     source_existing: Source,

tests/console/commands/source/test_show.py

@@ -121,6 +121,7 @@ priority  : primary
         "source_primary",
         "source_default",
         "source_secondary",
+        "source_supplemental",
         "source_explicit",
     ),
 )

tests/fixtures/with_supplemental_source/pyproject.toml

+[tool.poetry]
+name = "with-explicit-source"
+version = "1.2.3"
+description = "Some description."
+authors = [
+    "Your Name <you@example.com>"
+]
+license = "MIT"
+
+# Requirements
+[tool.poetry.dependencies]
+python = "^3.6"
+
+[tool.poetry.dev-dependencies]
+
+[[tool.poetry.source]]
+name = "supplemental"
+url = "https://supplemental.com/simple/"
+priority = "supplemental"

tests/json/test_schema_sources.py

@@ -30,7 +30,7 @@ def test_pyproject_toml_invalid_priority() -> None:
     assert Factory.validate(content) == {
         "errors": [
             "[source.0.priority] 'arbitrary' is not one of ['primary', 'default',"
-            " 'secondary', 'explicit']"
+            " 'secondary', 'supplemental', 'explicit']"
         ],
         "warnings": [],
     }

tests/repositories/test_repository_pool.py

@@ -81,9 +81,10 @@ def test_repository_from_single_repo_pool_legacy(
     assert pool.get_priority("foo") == expected_priority
 
 
-def test_repository_with_normal_default_secondary_and_explicit_repositories() -> None:
+def test_repository_with_all_prio_repositories() -> None:
     secondary = LegacyRepository("secondary", "https://secondary.com")
     default = LegacyRepository("default", "https://default.com")
+    supplemental = LegacyRepository("supplemental", "https://supplemental.com")
     repo1 = LegacyRepository("foo", "https://foo.bar")
     repo2 = LegacyRepository("bar", "https://bar.baz")
     explicit = LegacyRepository("explicit", "https://bar.baz")
@@ -92,6 +93,7 @@ def test_repository_with_normal_default_secondary_and_explicit_repositories() ->
     pool.add_repository(repo1)
     pool.add_repository(secondary, priority=Priority.SECONDARY)
     pool.add_repository(repo2)
+    pool.add_repository(supplemental, priority=Priority.SUPPLEMENTAL)
     pool.add_repository(explicit, priority=Priority.EXPLICIT)
     pool.add_repository(default, priority=Priority.DEFAULT)
 
@@ -99,11 +101,25 @@ def test_repository_with_normal_default_secondary_and_explicit_repositories() ->
     assert pool.repository("default") is default
     assert pool.repository("foo") is repo1
     assert pool.repository("bar") is repo2
+    assert pool.repository("supplemental") is supplemental
     assert pool.repository("explicit") is explicit
     assert pool.has_default()
     assert pool.has_primary_repositories()
 
 
+def test_repository_secondary_and_supplemental_repositories_do_show() -> None:
+    secondary = LegacyRepository("secondary", "https://secondary.com")
+    supplemental = LegacyRepository("supplemental", "https://supplemental.com")
+
+    pool = RepositoryPool()
+    pool.add_repository(secondary, priority=Priority.SECONDARY)
+    pool.add_repository(supplemental, priority=Priority.SUPPLEMENTAL)
+
+    assert pool.repository("secondary") is secondary
+    assert pool.repository("supplemental") is supplemental
+    assert pool.repositories == [secondary, supplemental]
+
+
 def test_repository_explicit_repositories_do_not_show() -> None:
     explicit = LegacyRepository("explicit", "https://explicit.com")
     default = LegacyRepository("default", "https://default.com")
@@ -174,9 +190,11 @@ def test_repository_ordering() -> None:
     secondary1 = LegacyRepository("secondary1", "https://secondary1.com")
     secondary2 = LegacyRepository("secondary2", "https://secondary2.com")
     secondary3 = LegacyRepository("secondary3", "https://secondary3.com")
+    supplemental = LegacyRepository("supplemental", "https://supplemental.com")
 
     pool = RepositoryPool()
     pool.add_repository(secondary1, priority=Priority.SECONDARY)
+    pool.add_repository(supplemental, priority=Priority.SUPPLEMENTAL)
     pool.add_repository(primary1)
     pool.add_repository(default1, priority=Priority.DEFAULT)
     pool.add_repository(primary2)
@@ -188,7 +206,14 @@ def test_repository_ordering() -> None:
     pool.add_repository(primary3)
     pool.add_repository(secondary3, priority=Priority.SECONDARY)
 
-    assert pool.repositories == [default1, primary1, primary3, secondary1, secondary3]
+    assert pool.repositories == [
+        default1,
+        primary1,
+        primary3,
+        secondary1,
+        secondary3,
+        supplemental,
+    ]
     with pytest.raises(ValueError):
         pool.add_repository(default2, priority=Priority.DEFAULT)
 
@@ -207,11 +232,32 @@ def test_pool_get_package_in_any_repository() -> None:
     assert returned_package2 == package2
 
 
+def test_pool_find_packages_only_considers_supplemental_when_needed() -> None:
+    package1 = get_package("foo", "1.1.1")
+    package2 = get_package("foo", "1.2.3")
+    package3 = get_package("foo", "2.0.0")
+    repo1 = Repository("repo1", [package1, package3])
+    repo2 = Repository("repo2", [package1, package2])
+    pool = RepositoryPool([repo1]).add_repository(repo2, priority=Priority.SUPPLEMENTAL)
+
+    dependency_in_nonsupplemental = get_dependency("foo", "^1.0.0")
+    returned_packages_in_nonsupplemental = pool.find_packages(
+        dependency_in_nonsupplemental
+    )
+    dependency_needs_supplemental = get_dependency("foo", "1.2.3")
+    returned_packages_needs_supplemental = pool.find_packages(
+        dependency_needs_supplemental
+    )
+
+    assert returned_packages_in_nonsupplemental == [package1]
+    assert returned_packages_needs_supplemental == [package2]
+
+
 def test_pool_get_package_in_specified_repository() -> None:
     package = get_package("foo", "1.0.0")
-    repo1 = Repository("repo1")
+    repo1 = Repository("repo1", [package])
     repo2 = Repository("repo2", [package])
-    pool = RepositoryPool([repo1, repo2])
+    pool = RepositoryPool([repo1]).add_repository(repo2, priority=Priority.SUPPLEMENTAL)
 
     returned_package = pool.package(
         "foo", Version.parse("1.0.0"), repository_name="repo2"

tests/test_factory.py

@@ -298,7 +298,7 @@ def test_poetry_with_non_default_secondary_source_legacy(
     assert poetry.pool.get_priority("PyPI") is Priority.DEFAULT
     assert poetry.pool.has_repository("foo")
     assert isinstance(poetry.pool.repository("foo"), LegacyRepository)
-    assert [repo.name for repo in poetry.pool.repositories] == ["PyPI", "foo"]
+    assert {repo.name for repo in poetry.pool.repositories} == {"PyPI", "foo"}
 
 
 def test_poetry_with_non_default_secondary_source(
@@ -311,7 +311,7 @@ def test_poetry_with_non_default_secondary_source(
     assert poetry.pool.get_priority("PyPI") is Priority.DEFAULT
     assert poetry.pool.has_repository("foo")
     assert isinstance(poetry.pool.repository("foo"), LegacyRepository)
-    assert [repo.name for repo in poetry.pool.repositories] == ["PyPI", "foo"]
+    assert {repo.name for repo in poetry.pool.repositories} == {"PyPI", "foo"}
 
 
 def test_poetry_with_non_default_multiple_secondary_sources_legacy(
@@ -400,7 +400,12 @@ def test_poetry_with_non_default_multiple_sources_pypi(
     expected = ["bar", "PyPI", "baz", "foo"]
     assert [repo.name for repo in poetry.pool.repositories] == expected
     error = io.fetch_error()
-    assert error == ""
+    assert (
+        error.strip()
+        == "<warning>Warning: Found deprecated priority 'secondary' for source 'foo' in"
+        " pyproject.toml. Consider changing the priority to one of the"
+        " non-deprecated values: 'default', 'primary', 'supplemental', 'explicit'."
+    )
 
 
 def test_poetry_with_no_default_source(fixture_dir: FixtureDirGetter) -> None:
@@ -412,6 +417,20 @@ def test_poetry_with_no_default_source(fixture_dir: FixtureDirGetter) -> None:
     assert {repo.name for repo in poetry.pool.repositories} == {"PyPI"}
 
 
+def test_poetry_with_supplemental_source(
+    fixture_dir: FixtureDirGetter, with_simple_keyring: None
+) -> None:
+    poetry = Factory().create_poetry(fixture_dir("with_supplemental_source"))
+
+    assert poetry.pool.has_repository("PyPI")
+    assert poetry.pool.get_priority("PyPI") is Priority.DEFAULT
+    assert isinstance(poetry.pool.repository("PyPI"), PyPiRepository)
+    assert poetry.pool.has_repository("supplemental")
+    assert poetry.pool.get_priority("supplemental") is Priority.SUPPLEMENTAL
+    assert isinstance(poetry.pool.repository("supplemental"), LegacyRepository)
+    assert {repo.name for repo in poetry.pool.repositories} == {"PyPI", "supplemental"}
+
+
 def test_poetry_with_explicit_source(
     fixture_dir: FixtureDirGetter, with_simple_keyring: None
 ) -> None:
@@ -424,7 +443,7 @@ def test_poetry_with_explicit_source(
     assert isinstance(poetry.pool.repository("PyPI"), PyPiRepository)
     assert poetry.pool.has_repository("explicit")
     assert isinstance(poetry.pool.repository("explicit"), LegacyRepository)
-    assert [repo.name for repo in poetry.pool.repositories] == ["PyPI"]
+    assert {repo.name for repo in poetry.pool.repositories} == {"PyPI"}
 
 
 def test_poetry_with_explicit_pypi_and_other(