add initial support for single page repositories

This change allows users to make use of find-links style single page sources that do not present a PEP 503 simple API repository.

add initial support for single page repositories
This change allows users to make use of find-links style single page sources that do not present a PEP 503 simple API repository.
36eff809 · Arun Babu Neelicattu · bb305300 · 36eff809 · 36eff809 · 36eff809
Commit 36eff809 authored Apr 29, 2022 by Arun Babu Neelicattu
Showing with 111 additions and 1 deletions

src/poetry/factory.py
+8 -1

src/poetry/repositories/single_page_repository.py
+15 -0

tests/repositories/fixtures/single-page/jax_releases.html
+27 -0

tests/repositories/test_single_page_repository.py
+61 -0

No files found.
--- a/src/poetry/factory.py
+++ b/src/poetry/factory.py
@@ -2,6 +2,7 @@ from __future__ import annotations
 import contextlib
 import logging
+import re
 from typing import TYPE_CHECKING
 from typing import Any
@@ -193,6 +194,7 @@ class Factory(BaseFactory):
        cls, source: dict[str, str], auth_config: Config, disable_cache: bool = False
    ) -> LegacyRepository:
        from poetry.repositories.legacy_repository import LegacyRepository
+        from poetry.repositories.single_page_repository import SinglePageRepository
        if "url" not in source:
            raise RuntimeError("Unsupported source specified")
@@ -203,7 +205,12 @@ class Factory(BaseFactory):
        name = source["name"]
        url = source["url"]
-        return LegacyRepository(
+        repository_class = LegacyRepository
+        if re.match(r".*\.(htm|html)$", url):
+            repository_class = SinglePageRepository
+        return repository_class(
            name,
            url,
            config=auth_config,

--- a/src/poetry/repositories/single_page_repository.py
+++ b/src/poetry/repositories/single_page_repository.py
+from __future__ import annotations
+from poetry.repositories.legacy_repository import LegacyRepository
+from poetry.repositories.link_sources.html import SimpleRepositoryPage
+class SinglePageRepository(LegacyRepository):
+    def _get_page(self, endpoint: str = None) -> SimpleRepositoryPage | None:
+        """
+        Single page repositories only have one page irrespective of endpoint.
+        """
+        response = self._get_response("")
+        if not response:
+            return None
+        return SimpleRepositoryPage(response.url, response.text)
--- a/tests/repositories/fixtures/single-page/jax_releases.html
+++ b/tests/repositories/fixtures/single-page/jax_releases.html
+<!DOCTYPE html>
+<html>
+<head><meta http-equiv="Content-Type" content="text/html; charset=utf-8"></head>
+<body>
+<a href="https://storage.googleapis.com/jax-releases/nocuda/jaxlib-0.3.0-cp310-none-manylinux2010_x86_64.whl">nocuda/jaxlib-0.3.0-cp310-none-manylinux2010_x86_64.whl</a><br>
+<a href="https://storage.googleapis.com/jax-releases/nocuda/jaxlib-0.3.0-cp37-none-manylinux2010_x86_64.whl">nocuda/jaxlib-0.3.0-cp37-none-manylinux2010_x86_64.whl</a><br>
+<a href="https://storage.googleapis.com/jax-releases/nocuda/jaxlib-0.3.0-cp38-none-manylinux2010_x86_64.whl">nocuda/jaxlib-0.3.0-cp38-none-manylinux2010_x86_64.whl</a><br>
+<a href="https://storage.googleapis.com/jax-releases/nocuda/jaxlib-0.3.0-cp39-none-manylinux2010_x86_64.whl">nocuda/jaxlib-0.3.0-cp39-none-manylinux2010_x86_64.whl</a><br>
+<a href="https://storage.googleapis.com/jax-releases/nocuda/jaxlib-0.3.2-cp310-none-manylinux2010_x86_64.whl">nocuda/jaxlib-0.3.2-cp310-none-manylinux2010_x86_64.whl</a><br>
+<a href="https://storage.googleapis.com/jax-releases/nocuda/jaxlib-0.3.2-cp37-none-manylinux2010_x86_64.whl">nocuda/jaxlib-0.3.2-cp37-none-manylinux2010_x86_64.whl</a><br>
+<a href="https://storage.googleapis.com/jax-releases/nocuda/jaxlib-0.3.2-cp38-none-manylinux2010_x86_64.whl">nocuda/jaxlib-0.3.2-cp38-none-manylinux2010_x86_64.whl</a><br>
+<a href="https://storage.googleapis.com/jax-releases/nocuda/jaxlib-0.3.2-cp39-none-manylinux2010_x86_64.whl">nocuda/jaxlib-0.3.2-cp39-none-manylinux2010_x86_64.whl</a><br>
+<a href="https://storage.googleapis.com/jax-releases/nocuda/jaxlib-0.3.5-cp310-none-manylinux2010_x86_64.whl">nocuda/jaxlib-0.3.5-cp310-none-manylinux2010_x86_64.whl</a><br>
+<a href="https://storage.googleapis.com/jax-releases/nocuda/jaxlib-0.3.5-cp37-none-manylinux2010_x86_64.whl">nocuda/jaxlib-0.3.5-cp37-none-manylinux2010_x86_64.whl</a><br>
+<a href="https://storage.googleapis.com/jax-releases/nocuda/jaxlib-0.3.5-cp38-none-manylinux2010_x86_64.whl">nocuda/jaxlib-0.3.5-cp38-none-manylinux2010_x86_64.whl</a><br>
+<a href="https://storage.googleapis.com/jax-releases/nocuda/jaxlib-0.3.5-cp39-none-manylinux2010_x86_64.whl">nocuda/jaxlib-0.3.5-cp39-none-manylinux2010_x86_64.whl</a><br>
+<a href="https://storage.googleapis.com/jax-releases/nocuda/jaxlib-0.3.7-cp310-none-manylinux2014_x86_64.whl">nocuda/jaxlib-0.3.7-cp310-none-manylinux2014_x86_64.whl</a><br>
+<a href="https://storage.googleapis.com/jax-releases/nocuda/jaxlib-0.3.7-cp37-none-manylinux2014_x86_64.whl">nocuda/jaxlib-0.3.7-cp37-none-manylinux2014_x86_64.whl</a><br>
+<a href="https://storage.googleapis.com/jax-releases/nocuda/jaxlib-0.3.7-cp38-none-manylinux2014_x86_64.whl">nocuda/jaxlib-0.3.7-cp38-none-manylinux2014_x86_64.whl</a><br>
+<a href="https://storage.googleapis.com/jax-releases/nocuda/jaxlib-0.3.7-cp39-none-manylinux2014_x86_64.whl">nocuda/jaxlib-0.3.7-cp39-none-manylinux2014_x86_64.whl</a><br>
+<a href="https://storage.googleapis.com/jax-releases/jax/jax-0.3.0.tar.gz">jax/jax-0.3.0.tar.gz</a><br>
+<a href="https://storage.googleapis.com/jax-releases/jax/jax-0.3.2.tar.gz">jax/jax-0.3.2.tar.gz</a><br>
+<a href="https://storage.googleapis.com/jax-releases/jax/jax-0.3.5.tar.gz">jax/jax-0.3.5.tar.gz</a><br>
+<a href="https://storage.googleapis.com/jax-releases/jax/jax-0.3.6.tar.gz">jax/jax-0.3.6.tar.gz</a><br>
+<a href="https://storage.googleapis.com/jax-releases/jax/jax-0.3.7.tar.gz">jax/jax-0.3.7.tar.gz</a><br>
+</body>
+</html>
--- a/tests/repositories/test_single_page_repository.py
+++ b/tests/repositories/test_single_page_repository.py
+from __future__ import annotations
+import re
+from pathlib import Path
+from poetry.core.packages.dependency import Dependency
+from poetry.repositories.link_sources.html import SimpleRepositoryPage
+from poetry.repositories.single_page_repository import SinglePageRepository
+class MockSinglePageRepository(SinglePageRepository):
+    FIXTURES = Path(__file__).parent / "fixtures" / "single-page"
+    def __init__(self, page: str) -> None:
+        super().__init__(
+            "single-page",
+            url=f"http://single-page.foo.bar/{page}.html",
+            disable_cache=True,
+        )
+    def _get_page(self, endpoint: str = None) -> SimpleRepositoryPage | None:
+        fixture = self.FIXTURES / self.url.rsplit("/", 1)[-1]
+        if not fixture.exists():
+            return
+        with fixture.open(encoding="utf-8") as f:
+            return SimpleRepositoryPage(self._url, f.read())
+    def _download(self, url: str, dest: Path) -> None:
+        raise RuntimeError("Tests are not configured for downloads")
+def test_single_page_repository_get_page():
+    repo = MockSinglePageRepository("jax_releases")
+    page = repo._get_page("/ignored")
+    links = list(page.links)
+    assert len(links) == 21
+    for link in links:
+        assert re.match(r"^(jax|jaxlib)-0\.3\.\d.*\.(whl|tar\.gz)$", link.filename)
+        assert link.netloc == "storage.googleapis.com"
+        assert link.path.startswith("/jax-releases/")
+def test_single_page_repository_find_packages():
+    repo = MockSinglePageRepository("jax_releases")
+    dep = Dependency("jaxlib", "0.3.7")
+    packages = repo.find_packages(dep)
+    assert len(packages) == 1
+    package = packages[0]
+    assert package.name == dep.name
+    assert package.to_dependency().to_pep_508() == dep.to_pep_508()