repositories: dedupe logic

This change unifies shared logic across repository implementations and
improves inheritance model.

src/poetry/console/commands/debug/resolve.py

@@ -98,7 +98,7 @@ class DebugResolveCommand(InitCommand):
             show_command.init_styles(self.io)
 
             packages = [op.package for op in ops]
-            repo = Repository(packages)
+            repo = Repository(packages=packages)
 
             requires = package.all_requires
             for pkg in repo.packages:

src/poetry/console/commands/show.py

@@ -14,8 +14,8 @@ if TYPE_CHECKING:
     from poetry.core.packages.package import Package
 
     from poetry.packages.project_package import ProjectPackage
-    from poetry.repositories import Repository
     from poetry.repositories.installed_repository import InstalledRepository
+    from poetry.repositories.repository import Repository
 
 
 class ShowCommand(GroupCommand):

src/poetry/repositories/base_repository.py

-from __future__ import annotations
-
-from typing import TYPE_CHECKING
-
-
-if TYPE_CHECKING:
-    from poetry.core.packages.dependency import Dependency
-    from poetry.core.packages.package import Package
-
-
-class BaseRepository:
-    def __init__(self) -> None:
-        self._packages: list[Package] = []
-
-    @property
-    def packages(self) -> list[Package]:
-        return self._packages
-
-    def has_package(self, package: Package) -> bool:
-        raise NotImplementedError()
-
-    def package(
-        self, name: str, version: str, extras: list[str] | None = None
-    ) -> Package:
-        raise NotImplementedError()
-
-    def find_packages(self, dependency: Dependency) -> list[Package]:
-        raise NotImplementedError()
-
-    def search(self, query: str) -> list[Package]:
-        raise NotImplementedError()

src/poetry/repositories/cached.py

+from __future__ import annotations
+
+from abc import ABC
+from abc import abstractmethod
+from typing import TYPE_CHECKING
+
+from cachecontrol.caches import FileCache
+from cachy import CacheManager
+from poetry.core.semver.helpers import parse_constraint
+
+from poetry.locations import REPOSITORY_CACHE_DIR
+from poetry.repositories.repository import Repository
+
+
+if TYPE_CHECKING:
+    from poetry.core.packages.package import Package
+
+    from poetry.inspection.info import PackageInfo
+
+
+class CachedRepository(Repository, ABC):
+    CACHE_VERSION = parse_constraint("1.0.0")
+
+    def __init__(self, name: str, cache_group: str, disable_cache: bool = False):
+        super().__init__(name)
+        self._disable_cache = disable_cache
+        self._cache_dir = REPOSITORY_CACHE_DIR / name
+        self._cache = CacheManager(
+            {
+                "default": "releases",
+                "serializer": "json",
+                "stores": {
+                    "releases": {"driver": "file", "path": str(self._cache_dir)},
+                    "packages": {"driver": "dict"},
+                    "matches": {"driver": "dict"},
+                },
+            }
+        )
+        self._cache_control_cache = FileCache(str(self._cache_dir / cache_group))
+
+    @abstractmethod
+    def _get_release_info(self, name: str, version: str) -> dict:
+        raise NotImplementedError()
+
+    def get_release_info(self, name: str, version: str) -> PackageInfo:
+        """
+        Return the release information given a package name and a version.
+
+        The information is returned from the cache if it exists
+        or retrieved from the remote server.
+        """
+        from poetry.inspection.info import PackageInfo
+
+        if self._disable_cache:
+            return PackageInfo.load(self._get_release_info(name, version))
+
+        cached = self._cache.remember_forever(
+            f"{name}:{version}", lambda: self._get_release_info(name, version)
+        )
+
+        cache_version = cached.get("_cache_version", "0.0.0")
+        if parse_constraint(cache_version) != self.CACHE_VERSION:
+            # The cache must be updated
+            self._log(
+                f"The cache for {name} {version} is outdated. Refreshing.",
+                level="debug",
+            )
+            cached = self._get_release_info(name, version)
+
+            self._cache.forever(f"{name}:{version}", cached)
+
+        return PackageInfo.load(cached)
+
+    def package(
+        self,
+        name: str,
+        version: str,
+        extras: (list | None) = None,
+    ) -> Package:
+        return self.get_release_info(name, version).to_package(name=name, extras=extras)

src/poetry/repositories/link_sources/base.py

+from __future__ import annotations
+
+import contextlib
+import re
+
+from abc import abstractmethod
+from typing import TYPE_CHECKING
+from typing import Iterator
+
+from poetry.core.packages.package import Package
+from poetry.core.semver.version import Version
+
+from poetry.utils.helpers import canonicalize_name
+from poetry.utils.patterns import sdist_file_re
+from poetry.utils.patterns import wheel_file_re
+
+
+if TYPE_CHECKING:
+    from poetry.core.packages.utils.link import Link
+
+
+class LinkSource:
+    VERSION_REGEX = re.compile(r"(?i)([a-z0-9_\-.]+?)-(?=\d)([a-z0-9_.!+-]+)")
+    CLEAN_REGEX = re.compile(r"[^a-z0-9$&+,/:;=?@.#%_\\|-]", re.I)
+    SUPPORTED_FORMATS = [
+        ".tar.gz",
+        ".whl",
+        ".zip",
+        ".tar.bz2",
+        ".tar.xz",
+        ".tar.Z",
+        ".tar",
+    ]
+
+    def __init__(self, url: str) -> None:
+        self._url = url
+
+    @property
+    def url(self) -> str:
+        return self._url
+
+    def versions(self, name: str) -> Iterator[Version]:
+        name = canonicalize_name(name)
+        seen: set[Version] = set()
+
+        for link in self.links:
+            pkg = self.link_package_data(link)
+
+            if pkg.name == name and pkg.version and pkg.version not in seen:
+                seen.add(pkg.version)
+                yield pkg.version
+
+    @property
+    def packages(self) -> Iterator[Package]:
+        for link in self.links:
+            pkg = self.link_package_data(link)
+
+            if pkg.name and pkg.version:
+                yield pkg
+
+    @property
+    @abstractmethod
+    def links(self) -> Iterator[Link]:
+        raise NotImplementedError()
+
+    def link_package_data(self, link: Link) -> Package:
+        name, version = None, None
+        m = wheel_file_re.match(link.filename) or sdist_file_re.match(link.filename)
+
+        if m:
+            name = canonicalize_name(m.group("name"))
+            version = m.group("ver")
+        else:
+            info, ext = link.splitext()
+            match = self.VERSION_REGEX.match(info)
+            if match:
+                version = match.group(2)
+
+        with contextlib.suppress(ValueError):
+            version = Version.parse(version)
+
+        return Package(name, version, source_url=link.url)
+
+    def links_for_version(self, name: str, version: Version) -> Iterator[Link]:
+        name = canonicalize_name(name)
+
+        for link in self.links:
+            pkg = self.link_package_data(link)
+
+            if pkg.name == name and pkg.version and pkg.version == version:
+                yield link
+
+    def clean_link(self, url: str) -> str:
+        """Makes sure a link is fully encoded.  That is, if a ' ' shows up in
+        the link, it will be rewritten to %20 (while not over-quoting
+        % or other characters)."""
+        return self.CLEAN_REGEX.sub(lambda match: f"%{ord(match.group(0)):02x}", url)

src/poetry/repositories/link_sources/html.py

+from __future__ import annotations
+
+import urllib.parse
+import warnings
+
+from html import unescape
+from typing import Iterator
+
+from poetry.core.packages.utils.link import Link
+
+from poetry.repositories.link_sources.base import LinkSource
+
+
+with warnings.catch_warnings():
+    warnings.simplefilter("ignore")
+    import html5lib
+
+
+class HTMLPage(LinkSource):
+    def __init__(self, url: str, content: str) -> None:
+        super().__init__(url=url)
+
+        self._parsed = html5lib.parse(content, namespaceHTMLElements=False)
+
+    @property
+    def links(self) -> Iterator[Link]:
+        for anchor in self._parsed.findall(".//a"):
+            if anchor.get("href"):
+                href = anchor.get("href")
+                url = self.clean_link(urllib.parse.urljoin(self._url, href))
+                pyrequire = anchor.get("data-requires-python")
+                pyrequire = unescape(pyrequire) if pyrequire else None
+                link = Link(url, self, requires_python=pyrequire)
+
+                if link.ext not in self.SUPPORTED_FORMATS:
+                    continue
+
+                yield link
+
+
+class SimpleRepositoryPage(HTMLPage):
+    def __init__(self, url: str, content: str) -> None:
+        if not url.endswith("/"):
+            url += "/"
+        super().__init__(url=url, content=content)

src/poetry/repositories/pool.py

@@ -3,23 +3,23 @@ from __future__ import annotations
 from contextlib import suppress
 from typing import TYPE_CHECKING
 
-from poetry.repositories.base_repository import BaseRepository
 from poetry.repositories.exceptions import PackageNotFound
+from poetry.repositories.repository import Repository
 
 
 if TYPE_CHECKING:
     from poetry.core.packages.dependency import Dependency
     from poetry.core.packages.package import Package
 
-    from poetry.repositories.repository import Repository
 
-
-class Pool(BaseRepository):
+class Pool(Repository):
     def __init__(
         self,
         repositories: list[Repository] | None = None,
         ignore_repository_names: bool = False,
     ) -> None:
+        super().__init__()
+
         if repositories is None:
             repositories = []
 

src/poetry/repositories/remote_repository.py

-from __future__ import annotations
-
-from poetry.repositories.repository import Repository
-
-
-class RemoteRepository(Repository):
-    def __init__(self, url: str) -> None:
-        self._url = url
-
-        super().__init__()
-
-    @property
-    def url(self) -> str:
-        return self._url
-
-    @property
-    def authenticated_url(self) -> str:
-        return self._url

src/poetry/repositories/repository.py

 from __future__ import annotations
 
+import logging
+
 from typing import TYPE_CHECKING
 
-from poetry.repositories.base_repository import BaseRepository
+from poetry.core.semver.helpers import parse_constraint
+from poetry.core.semver.version_constraint import VersionConstraint
+from poetry.core.semver.version_range import VersionRange
 
 
 if TYPE_CHECKING:
     from poetry.core.packages.dependency import Dependency
     from poetry.core.packages.package import Package
     from poetry.core.packages.utils.link import Link
+    from poetry.core.semver.helpers import VersionTypes
 
 
-class Repository(BaseRepository):
-    def __init__(self, packages: list[Package] = None, name: str = None) -> None:
-        super().__init__()
-
+class Repository:
+    def __init__(self, name: str = None, packages: list[Package] = None) -> None:
         self._name = name
+        self._packages: list[Package] = []
 
-        if packages is None:
-            packages = []
-
-        for package in packages:
+        for package in packages or []:
             self.add_package(package)
 
     @property
     def name(self) -> str | None:
         return self._name
 
-    def package(
-        self, name: str, version: str, extras: list[str] | None = None
-    ) -> Package:
-        name = name.lower()
-
-        for package in self.packages:
-            if name == package.name and package.version.text == version:
-                return package.clone()
+    @property
+    def packages(self) -> list[Package]:
+        return self._packages
 
     def find_packages(self, dependency: Dependency) -> list[Package]:
-        from poetry.core.semver.helpers import parse_constraint
-        from poetry.core.semver.version_constraint import VersionConstraint
-        from poetry.core.semver.version_range import VersionRange
-
-        constraint = dependency.constraint
         packages = []
         ignored_pre_release_packages = []
-
-        if constraint is None:
-            constraint = "*"
-
-        if not isinstance(constraint, VersionConstraint):
-            constraint = parse_constraint(constraint)
-
-        allow_prereleases = dependency.allows_prereleases()
-        if isinstance(constraint, VersionRange) and (
-            constraint.max is not None
-            and constraint.max.is_unstable()
-            or constraint.min is not None
-            and constraint.min.is_unstable()
-        ):
-            allow_prereleases = True
+        constraint, allow_prereleases = self._get_constraints_from_dependency(
+            dependency
+        )
 
         for package in self.packages:
             if dependency.name == package.name:
@@ -103,9 +82,6 @@ class Repository(BaseRepository):
         if index is not None:
             del self._packages[index]
 
-    def find_links_for_package(self, package: Package) -> list[Link]:
-        return []
-
     def search(self, query: str) -> list[Package]:
         results: list[Package] = []
 
@@ -115,5 +91,44 @@ class Repository(BaseRepository):
 
         return results
 
+    @staticmethod
+    def _get_constraints_from_dependency(
+        dependency: Dependency,
+    ) -> tuple[VersionTypes, bool]:
+        constraint = dependency.constraint
+        if constraint is None:
+            constraint = "*"
+
+        if not isinstance(constraint, VersionConstraint):
+            constraint = parse_constraint(constraint)
+
+        allow_prereleases = dependency.allows_prereleases()
+        if isinstance(constraint, VersionRange) and (
+            constraint.max is not None
+            and constraint.max.is_unstable()
+            or constraint.min is not None
+            and constraint.min.is_unstable()
+        ):
+            allow_prereleases = True
+
+        return constraint, allow_prereleases
+
+    def _log(self, msg: str, level: str = "info") -> None:
+        getattr(logging.getLogger(self.__class__.__name__), level)(
+            f"<debug>{self.name}:</debug> {msg}"
+        )
+
     def __len__(self) -> int:
         return len(self._packages)
+
+    def find_links_for_package(self, package: Package) -> list[Link]:
+        return []
+
+    def package(
+        self, name: str, version: str, extras: list[str] | None = None
+    ) -> Package:
+        name = name.lower()
+
+        for package in self.packages:
+            if name == package.name and package.version.text == version:
+                return package.clone()

src/poetry/utils/patterns.py

@@ -12,3 +12,8 @@ wheel_file_re = re.compile(
     r"\.whl|\.dist-info$",
     re.VERBOSE,
 )
+
+sdist_file_re = re.compile(
+    r"^(?P<namever>(?P<name>.+?)-(?P<ver>\d.*?))"
+    r"(\.sdist)?\.(?P<format>(zip|tar(\.(gz|bz2|xz|Z))?))$"
+)

tests/conftest.py

@@ -191,7 +191,7 @@ def download_mock(mocker: MockerFixture) -> None:
     # Patch download to not download anything but to just copy from fixtures
     mocker.patch("poetry.utils.helpers.download_file", new=mock_download)
     mocker.patch("poetry.puzzle.provider.download_file", new=mock_download)
-    mocker.patch("poetry.repositories.pypi_repository.download_file", new=mock_download)
+    mocker.patch("poetry.repositories.http.download_file", new=mock_download)
 
 
 @pytest.fixture(autouse=True)

tests/repositories/test_legacy_repository.py

@@ -14,7 +14,7 @@ from poetry.factory import Factory
 from poetry.repositories.exceptions import PackageNotFound
 from poetry.repositories.exceptions import RepositoryError
 from poetry.repositories.legacy_repository import LegacyRepository
-from poetry.repositories.legacy_repository import Page
+from poetry.repositories.link_sources.html import SimpleRepositoryPage
 
 
 try:
@@ -35,7 +35,7 @@ class MockRepository(LegacyRepository):
     def __init__(self) -> None:
         super().__init__("legacy", url="http://legacy.foo.bar", disable_cache=True)
 
-    def _get_page(self, endpoint: str) -> Page | None:
+    def _get_page(self, endpoint: str) -> SimpleRepositoryPage | None:
         parts = endpoint.split("/")
         name = parts[1]
 
@@ -44,7 +44,7 @@ class MockRepository(LegacyRepository):
             return
 
         with fixture.open(encoding="utf-8") as f:
-            return Page(self._url + endpoint, f.read(), {})
+            return SimpleRepositoryPage(self._url + endpoint, f.read())
 
     def _download(self, url: str, dest: Path) -> None:
         filename = urlparse.urlparse(url).path.rsplit("/")[-1]