Skip to content

P
python-poetry
Commit c1d77a1a by Randy Döring
Options

repositories: support for yanked releases according to PEP 592

parent cce1abc6
Showing with 449 additions and 10 deletions
src/poetry/inspection/info.py
......@@ -69,6 +69,7 @@ class PackageInfoError(ValueError):
class PackageInfo:
def __init__(
self,
*,
name: str | None = None,
version: str | None = None,
summary: str | None = None,
......@@ -76,6 +77,7 @@ class PackageInfo:
requires_dist: list[str] | None = None,
requires_python: str | None = None,
files: list[dict[str, str]] | None = None,
yanked: str | bool = False,
cache_version: str | None = None,
) -> None:
self.name = name
......@@ -85,6 +87,7 @@ class PackageInfo:
self.requires_dist = requires_dist
self.requires_python = requires_python
self.files = files or []
self.yanked = yanked
self._cache_version = cache_version
self._source_type: str | None = None
self._source_url: str | None = None
......@@ -117,6 +120,7 @@ class PackageInfo:
"requires_dist": self.requires_dist,
"requires_python": self.requires_python,
"files": self.files,
"yanked": self.yanked,
"_cache_version": self._cache_version,
}
......@@ -163,6 +167,7 @@ class PackageInfo:
source_type=self._source_type,
source_url=self._source_url,
source_reference=self._source_reference,
yanked=self.yanked,
)
if self.summary is not None:
package.description = self.summary
......@@ -450,6 +455,7 @@ class PackageInfo:
requires_dist=list(requires),
requires_python=package.python_versions,
files=package.files,
yanked=package.yanked_reason if package.yanked else False,
)
@staticmethod
......
src/poetry/repositories/cached.py
......@@ -21,7 +21,7 @@ if TYPE_CHECKING:
class CachedRepository(Repository, ABC):
CACHE_VERSION = parse_constraint("1.0.0")
CACHE_VERSION = parse_constraint("1.1.0")
def __init__(
self, name: str, disable_cache: bool = False, config: Config | None = None
......
src/poetry/repositories/http.py
......@@ -210,6 +210,9 @@ class HTTPRepository(CachedRepository, ABC):
urls = defaultdict(list)
files: list[dict[str, Any]] = []
for link in links:
if link.yanked and not data.yanked:
# drop yanked files unless the entire release is yanked
continue
if link.is_wheel:
urls["bdist_wheel"].append(link.url)
elif link.filename.endswith(
......
src/poetry/repositories/legacy_repository.py
......@@ -72,7 +72,7 @@ class LegacyRepository(HTTPRepository):
"""
Find packages on the remote server.
"""
versions: list[Version]
versions: list[tuple[Version, str | bool]]
key: str = name
if not constraint.is_any():
......@@ -90,7 +90,9 @@ class LegacyRepository(HTTPRepository):
return []
versions = [
version for version in page.versions(name) if constraint.allows(version)
(version, page.yanked(name, version))
for version in page.versions(name)
if constraint.allows(version)
]
self._cache.store("matches").put(key, versions, 5)
......@@ -101,8 +103,9 @@ class LegacyRepository(HTTPRepository):
source_type="legacy",
source_reference=self.name,
source_url=self._url,
yanked=yanked,
)
for version in versions
for version, yanked in versions
]
def _get_release_info(
......@@ -113,6 +116,7 @@ class LegacyRepository(HTTPRepository):
raise PackageNotFound(f'No package named "{name}"')
links = list(page.links_for_version(name, version))
yanked = page.yanked(name, version)
return self._links_to_data(
links,
......@@ -124,6 +128,7 @@ class LegacyRepository(HTTPRepository):
requires_dist=[],
requires_python=None,
files=[],
yanked=yanked,
cache_version=str(self.CACHE_VERSION),
),
)
......
src/poetry/repositories/pypi_repository.py
......@@ -144,7 +144,10 @@ class PyPiRepository(HTTPRepository):
continue
if constraint.allows(version):
packages.append(Package(info["info"]["name"], version))
# PEP 592: PyPI always yanks entire releases, not individual files,
# so we just have to look for the first file
yanked = self._get_yanked(release[0])
packages.append(Package(info["info"]["name"], version, yanked=yanked))
return packages
......@@ -163,7 +166,7 @@ class PyPiRepository(HTTPRepository):
links = []
for url in json_data["urls"]:
h = f"sha256={url['digests']['sha256']}"
links.append(Link(url["url"] + "#" + h))
links.append(Link(url["url"] + "#" + h, yanked=self._get_yanked(url)))
return links
......@@ -188,6 +191,7 @@ class PyPiRepository(HTTPRepository):
requires_dist=info["requires_dist"],
requires_python=info["requires_python"],
files=info.get("files", []),
yanked=self._get_yanked(info),
cache_version=str(self.CACHE_VERSION),
)
......@@ -254,3 +258,9 @@ class PyPiRepository(HTTPRepository):
json: dict[str, Any] = json_response.json()
return json
@staticmethod
def _get_yanked(json_data: dict[str, Any]) -> str | bool:
if json_data.get("yanked", False):
return json_data.get("yanked_reason") or True # noqa: SIM222
return False
src/poetry/repositories/repository.py
......@@ -5,6 +5,7 @@ import logging
from typing import TYPE_CHECKING
from poetry.core.semver.helpers import parse_constraint
from poetry.core.semver.version import Version
from poetry.core.semver.version_constraint import VersionConstraint
from poetry.core.semver.version_range import VersionRange
......@@ -16,7 +17,6 @@ if TYPE_CHECKING:
from poetry.core.packages.dependency import Dependency
from poetry.core.packages.package import Package
from poetry.core.packages.utils.link import Link
from poetry.core.semver.version import Version
class Repository:
......@@ -43,6 +43,11 @@ class Repository:
ignored_pre_release_packages = []
for package in self._find_packages(dependency.name, constraint):
if package.yanked and not isinstance(constraint, Version):
# PEP 592: yanked files are always ignored, unless they are the only
# file that matches a version specifier that "pins" to an exact
# version
continue
if (
package.is_prerelease()
and not allow_prereleases
......
tests/console/commands/test_add.py
......@@ -823,7 +823,9 @@ def test_add_constraint_with_source(
):
repo = LegacyRepository(name="my-index", url="https://my-index.fake")
repo.add_package(get_package("cachy", "0.2.0"))
repo._cache.store("matches").put("cachy:0.2.0", [Version.parse("0.2.0")], 5)
repo._cache.store("matches").put(
"cachy:0.2.0", [(Version.parse("0.2.0"), False)], 5
)
poetry.pool.add_repository(repo)
......@@ -1810,7 +1812,9 @@ def test_add_constraint_with_source_old_installer(
):
repo = LegacyRepository(name="my-index", url="https://my-index.fake")
repo.add_package(get_package("cachy", "0.2.0"))
repo._cache.store("matches").put("cachy:0.2.0", [Version.parse("0.2.0")], 5)
repo._cache.store("matches").put(
"cachy:0.2.0", [(Version.parse("0.2.0"), False)], 5
)
poetry.pool.add_repository(repo)
......
tests/repositories/fixtures/legacy/black.html
......@@ -4,7 +4,8 @@
<title>Links for black</title></head>
<body>
<h1>Links for black</h1>
<a href="https://files.pythonhosted.org/packages/b0/dc/ecd83b973fb7b82c34d828aad621a6e5865764d52375b8ac1d7a45e23c8d/black-19.10b0.tar.gz#sha256=c2edb73a08e9e0e6f65a0e6af18b059b8b1cdd5bef997d7a0b181df93dc81539" data-requires-python=">=3.6">black-19.10b0.tar.gz</a>
<a href="https://files.pythonhosted.org/packages/fd/bb/ad34bbc93d1bea3de086d7c59e528d4a503ac8fe318bd1fa48605584c3d2/black-19.10b0-py36-none-any.whl#sha256=1b30e59be925fafc1ee4565e5e08abef6b03fe455102883820fe5ee2e4734e0b" data-requires-python=">=3.6">black-19.10b0-py36-none-any.whl</a>
<a href="https://files.pythonhosted.org/packages/3d/ad/1cf514e7f9ee4c3d8df7c839d7977f7605ad76557f3fca741ec67f76dba6/black-21.11b0-py3-none-any.whl#sha256=0b1f66cbfadcd332ceeaeecf6373d9991d451868d2e2219ad0ac1213fb701117" data-requires-python=">=3.6.2" data-yanked="Broken regex dependency. Use 21.11b1 instead.">black-21.11b0-py3-none-any.whl</a>
</body>
</html>
<!--SERIAL 6044498-->
tests/repositories/fixtures/legacy/futures_partial_yank.html 0 → 100644
<!DOCTYPE html>
<html>
<head>
<title>Links for futures</title>
</head>
<body>
<h1>Links for futures</h1>
<a href="https://files.pythonhosted.org/packages/2d/99/b2c4e9d5a30f6471e410a146232b4118e697fa3ffc06d6a65efde84debd0/futures-3.2.0-py2-none-any.whl#sha256=ec0a6cb848cc212002b9828c3e34c675e0c9ff6741dc445cab6fdd4e1085d1f1" data-requires-python="&gt;=2.6, &lt;3" data-yanked>futures-3.2.0-py2-none-any.whl</a><br/>
<a href="https://files.pythonhosted.org/packages/1f/9e/7b2ff7e965fc654592269f2906ade1c7d705f1bf25b7d469fa153f7d19eb/futures-3.2.0.tar.gz#sha256=9ec02aa7d674acb8618afb127e27fde7fc68994c0437ad759fa094a574adb265" data-requires-python="&gt;=2.6, &lt;3">futures-3.2.0.tar.gz</a><br/>
</body>
</html>
<!--SERIAL 3865286-->
tests/repositories/fixtures/pypi.org/json/black.json
......@@ -99,6 +99,48 @@
"yanked": false,
"yanked_reason": null
}
],
"21.11b0": [
{
"comment_text": "",
"digests": {
"md5": "945da11b34c11738560fc6698cffa425",
"sha256": "0b1f66cbfadcd332ceeaeecf6373d9991d451868d2e2219ad0ac1213fb701117"
},
"downloads": -1,
"filename": "black-21.11b0-py3-none-any.whl",
"has_sig": false,
"md5_digest": "945da11b34c11738560fc6698cffa425",
"packagetype": "bdist_wheel",
"python_version": "py3",
"requires_python": ">=3.6.2",
"size": 155131,
"upload_time": "2021-11-17T02:32:14",
"upload_time_iso_8601": "2021-11-17T02:32:14.551680Z",
"url": "https://files.pythonhosted.org/packages/3d/ad/1cf514e7f9ee4c3d8df7c839d7977f7605ad76557f3fca741ec67f76dba6/black-21.11b0-py3-none-any.whl",
"yanked": true,
"yanked_reason": "Broken regex dependency. Use 21.11b1 instead."
},
{
"comment_text": "",
"digests": {
"md5": "6040b4e4c6ccc4e7eb81bb2634ef299a",
"sha256": "83f3852301c8dcb229e9c444dd79f573c8d31c7c2dad9bbaaa94c808630e32aa"
},
"downloads": -1,
"filename": "black-21.11b0.tar.gz",
"has_sig": false,
"md5_digest": "6040b4e4c6ccc4e7eb81bb2634ef299a",
"packagetype": "sdist",
"python_version": "source",
"requires_python": ">=3.6.2",
"size": 593164,
"upload_time": "2021-11-17T02:32:16",
"upload_time_iso_8601": "2021-11-17T02:32:16.396821Z",
"url": "https://files.pythonhosted.org/packages/2f/db/03e8cef689ab0ff857576ee2ee288d1ff2110ef7f3a77cac62e61f18acaf/black-21.11b0.tar.gz",
"yanked": true,
"yanked_reason": "Broken regex dependency. Use 21.11b1 instead."
}
]
},
"urls": [
......
tests/repositories/fixtures/pypi.org/json/black/21.11b0.json 0 → 100644
{
"info": {
"author": "Łukasz Langa",
"author_email": "lukasz@langa.pl",
"bugtrack_url": null,
"classifiers": [
"Development Status :: 4 - Beta",
"Environment :: Console",
"Intended Audience :: Developers",
"License :: OSI Approved :: MIT License",
"Operating System :: OS Independent",
"Programming Language :: Python",
"Programming Language :: Python :: 3.6",
"Programming Language :: Python :: 3.7",
"Programming Language :: Python :: 3.8",
"Programming Language :: Python :: 3 :: Only",
"Topic :: Software Development :: Libraries :: Python Modules",
"Topic :: Software Development :: Quality Assurance"
],
"description": "",
"description_content_type": "text/markdown",
"docs_url": null,
"download_url": "",
"downloads": {
"last_day": -1,
"last_month": -1,
"last_week": -1
},
"home_page": "https://github.com/psf/black",
"keywords": "automation formatter yapf autopep8 pyfmt gofmt rustfmt",
"license": "MIT",
"maintainer": "",
"maintainer_email": "",
"name": "black",
"package_url": "https://pypi.org/project/black/",
"platform": "",
"project_url": "https://pypi.org/project/black/",
"project_urls": {
"Changelog": "https://github.com/psf/black/blob/main/CHANGES.md",
"Homepage": "https://github.com/psf/black"
},
"release_url": "https://pypi.org/project/black/21.11b0/",
"requires_dist": [
"click (>=7.1.2)",
"platformdirs (>=2)",
"tomli (<2.0.0,>=0.2.6)",
"regex (>=2020.1.8)",
"pathspec (<1,>=0.9.0)",
"typing-extensions (>=3.10.0.0)",
"mypy-extensions (>=0.4.3)",
"dataclasses (>=0.6) ; python_version < \"3.7\"",
"typed-ast (>=1.4.2) ; python_version < \"3.8\" and implementation_name == \"cpython\"",
"typing-extensions (!=3.10.0.1) ; python_version >= \"3.10\"",
"colorama (>=0.4.3) ; extra == 'colorama'",
"aiohttp (>=3.7.4) ; extra == 'd'",
"ipython (>=7.8.0) ; extra == 'jupyter'",
"tokenize-rt (>=3.2.0) ; extra == 'jupyter'",
"typed-ast (>=1.4.3) ; extra == 'python2'",
"uvloop (>=0.15.2) ; extra == 'uvloop'"
],
"requires_python": ">=3.6.2",
"summary": "The uncompromising code formatter.",
"version": "21.11b0",
"yanked": true,
"yanked_reason": "Broken regex dependency. Use 21.11b1 instead."
},
"last_serial": 13326107,
"releases": {
"21.11b0": [
{
"comment_text": "",
"digests": {
"md5": "945da11b34c11738560fc6698cffa425",
"sha256": "0b1f66cbfadcd332ceeaeecf6373d9991d451868d2e2219ad0ac1213fb701117"
},
"downloads": -1,
"filename": "black-21.11b0-py3-none-any.whl",
"has_sig": false,
"md5_digest": "945da11b34c11738560fc6698cffa425",
"packagetype": "bdist_wheel",
"python_version": "py3",
"requires_python": ">=3.6.2",
"size": 155131,
"upload_time": "2021-11-17T02:32:14",
"upload_time_iso_8601": "2021-11-17T02:32:14.551680Z",
"url": "https://files.pythonhosted.org/packages/3d/ad/1cf514e7f9ee4c3d8df7c839d7977f7605ad76557f3fca741ec67f76dba6/black-21.11b0-py3-none-any.whl",
"yanked": true,
"yanked_reason": "Broken regex dependency. Use 21.11b1 instead."
},
{
"comment_text": "",
"digests": {
"md5": "6040b4e4c6ccc4e7eb81bb2634ef299a",
"sha256": "83f3852301c8dcb229e9c444dd79f573c8d31c7c2dad9bbaaa94c808630e32aa"
},
"downloads": -1,
"filename": "black-21.11b0.tar.gz",
"has_sig": false,
"md5_digest": "6040b4e4c6ccc4e7eb81bb2634ef299a",
"packagetype": "sdist",
"python_version": "source",
"requires_python": ">=3.6.2",
"size": 593164,
"upload_time": "2021-11-17T02:32:16",
"upload_time_iso_8601": "2021-11-17T02:32:16.396821Z",
"url": "https://files.pythonhosted.org/packages/2f/db/03e8cef689ab0ff857576ee2ee288d1ff2110ef7f3a77cac62e61f18acaf/black-21.11b0.tar.gz",
"yanked": true,
"yanked_reason": "Broken regex dependency. Use 21.11b1 instead."
}
]
},
"urls": [
{
"comment_text": "",
"digests": {
"md5": "945da11b34c11738560fc6698cffa425",
"sha256": "0b1f66cbfadcd332ceeaeecf6373d9991d451868d2e2219ad0ac1213fb701117"
},
"downloads": -1,
"filename": "black-21.11b0-py3-none-any.whl",
"has_sig": false,
"md5_digest": "945da11b34c11738560fc6698cffa425",
"packagetype": "bdist_wheel",
"python_version": "py3",
"requires_python": ">=3.6.2",
"size": 155131,
"upload_time": "2021-11-17T02:32:14",
"upload_time_iso_8601": "2021-11-17T02:32:14.551680Z",
"url": "https://files.pythonhosted.org/packages/3d/ad/1cf514e7f9ee4c3d8df7c839d7977f7605ad76557f3fca741ec67f76dba6/black-21.11b0-py3-none-any.whl",
"yanked": true,
"yanked_reason": "Broken regex dependency. Use 21.11b1 instead."
},
{
"comment_text": "",
"digests": {
"md5": "6040b4e4c6ccc4e7eb81bb2634ef299a",
"sha256": "83f3852301c8dcb229e9c444dd79f573c8d31c7c2dad9bbaaa94c808630e32aa"
},
"downloads": -1,
"filename": "black-21.11b0.tar.gz",
"has_sig": false,
"md5_digest": "6040b4e4c6ccc4e7eb81bb2634ef299a",
"packagetype": "sdist",
"python_version": "source",
"requires_python": ">=3.6.2",
"size": 593164,
"upload_time": "2021-11-17T02:32:16",
"upload_time_iso_8601": "2021-11-17T02:32:16.396821Z",
"url": "https://files.pythonhosted.org/packages/2f/db/03e8cef689ab0ff857576ee2ee288d1ff2110ef7f3a77cac62e61f18acaf/black-21.11b0.tar.gz",
"yanked": true,
"yanked_reason": "Broken regex dependency. Use 21.11b1 instead."
}
],
"vulnerabilities": []
}
tests/repositories/test_legacy_repository.py
......@@ -234,6 +234,24 @@ def test_find_packages_only_prereleases_empty_when_not_any() -> None:
assert len(packages) == 0
@pytest.mark.parametrize(
["constraint", "expected"],
[
# yanked 21.11b0 is ignored except for pinned version
("*", ["19.10b0"]),
(">=19.0a0", ["19.10b0"]),
(">=20.0a0", []),
(">=21.11b0", []),
("==21.11b0", ["21.11b0"]),
],
)
def test_find_packages_yanked(constraint: str, expected: list[str]) -> None:
repo = MockRepository()
packages = repo.find_packages(Factory.create_dependency("black", constraint))
assert [str(p.version) for p in packages] == expected
def test_get_package_information_chooses_correct_distribution() -> None:
repo = MockRepository()
......@@ -402,6 +420,62 @@ def test_get_package_retrieves_packages_with_no_hashes() -> None:
] == package.files
@pytest.mark.parametrize(
"package_name, version, yanked, yanked_reason",
[
("black", "19.10b0", False, ""),
("black", "21.11b0", True, "Broken regex dependency. Use 21.11b1 instead."),
],
)
def test_package_yanked(
package_name: str, version: str, yanked: bool, yanked_reason: str
) -> None:
repo = MockRepository()
package = repo.package(canonicalize_name(package_name), Version.parse(version))
assert package.name == package_name
assert str(package.version) == version
assert package.yanked is yanked
assert package.yanked_reason == yanked_reason
def test_package_partial_yank():
class SpecialMockRepository(MockRepository):
def _get_page(self, endpoint: str) -> SimpleRepositoryPage | None:
return super()._get_page(f"/{endpoint.strip('/')}_partial_yank/")
repo = MockRepository()
package = repo.package(canonicalize_name("futures"), Version.parse("3.2.0"))
assert len(package.files) == 2
repo = SpecialMockRepository()
package = repo.package(canonicalize_name("futures"), Version.parse("3.2.0"))
assert len(package.files) == 1
assert package.files[0]["file"].endswith(".tar.gz")
@pytest.mark.parametrize(
"package_name, version, yanked, yanked_reason",
[
("black", "19.10b0", False, ""),
("black", "21.11b0", True, "Broken regex dependency. Use 21.11b1 instead."),
],
)
def test_find_links_for_package_yanked(
package_name: str, version: str, yanked: bool, yanked_reason: str
) -> None:
repo = MockRepository()
package = repo.package(canonicalize_name(package_name), Version.parse(version))
links = repo.find_links_for_package(package)
assert len(links) == 1
for link in links:
assert link.yanked == yanked
assert link.yanked_reason == yanked_reason
class MockHttpRepository(LegacyRepository):
def __init__(
self, endpoint_responses: dict, http: type[httpretty.httpretty]
......
tests/repositories/test_pypi_repository.py
......@@ -96,6 +96,24 @@ def test_find_packages_only_prereleases(constraint: str, count: int) -> None:
assert len(packages) == count
@pytest.mark.parametrize(
["constraint", "expected"],
[
# yanked 21.11b0 is ignored except for pinned version
("*", ["19.10b0"]),
(">=19.0a0", ["19.10b0"]),
(">=20.0a0", []),
(">=21.11b0", []),
("==21.11b0", ["21.11b0"]),
],
)
def test_find_packages_yanked(constraint: str, expected: list[str]) -> None:
repo = MockRepository()
packages = repo.find_packages(Factory.create_dependency("black", constraint))
assert [str(p.version) for p in packages] == expected
def test_package() -> None:
repo = MockRepository()
......@@ -128,6 +146,47 @@ def test_package() -> None:
)
@pytest.mark.parametrize(
"package_name, version, yanked, yanked_reason",
[
("black", "19.10b0", False, ""),
("black", "21.11b0", True, "Broken regex dependency. Use 21.11b1 instead."),
],
)
def test_package_yanked(
package_name: str, version: str, yanked: bool, yanked_reason: str
) -> None:
repo = MockRepository()
package = repo.package(package_name, version)
assert package.name == package_name
assert str(package.version) == version
assert package.yanked is yanked
assert package.yanked_reason == yanked_reason
@pytest.mark.parametrize(
"package_name, version, yanked, yanked_reason",
[
("black", "19.10b0", False, ""),
("black", "21.11b0", True, "Broken regex dependency. Use 21.11b1 instead."),
],
)
def test_find_links_for_package_yanked(
package_name: str, version: str, yanked: bool, yanked_reason: str
) -> None:
repo = MockRepository()
package = repo.package(package_name, version)
links = repo.find_links_for_package(package)
assert len(links) == 2
for link in links:
assert link.yanked == yanked
assert link.yanked_reason == yanked_reason
def test_fallback_on_downloading_packages() -> None:
repo = MockRepository(fallback=True)
......
tests/repositories/test_repository.py 0 → 100644
from __future__ import annotations
import pytest
from packaging.utils import canonicalize_name
from poetry.core.packages.package import Package
from poetry.core.semver.version import Version
from poetry.factory import Factory
from poetry.repositories import Repository
@pytest.fixture(scope="module")
def black_repository() -> Repository:
repo = Repository("repo")
repo.add_package(Package("black", "19.10b0"))
repo.add_package(Package("black", "21.11b0", yanked="reason"))
return repo
@pytest.mark.parametrize(
["constraint", "expected"],
[
# yanked 21.11b0 is ignored except for pinned version
("*", ["19.10b0"]),
(">=19.0a0", ["19.10b0"]),
(">=20.0a0", []),
(">=21.11b0", []),
("==21.11b0", ["21.11b0"]),
],
)
def test_find_packages_yanked(
black_repository: Repository, constraint: str, expected: list[str]
) -> None:
packages = black_repository.find_packages(
Factory.create_dependency("black", constraint)
)
assert [str(p.version) for p in packages] == expected
@pytest.mark.parametrize(
"package_name, version, yanked, yanked_reason",
[
("black", "19.10b0", False, ""),
("black", "21.11b0", True, "reason"),
],
)
def test_package_yanked(
black_repository: Repository,
package_name: str,
version: str,
yanked: bool,
yanked_reason: str,
) -> None:
package = black_repository.package(
canonicalize_name(package_name), Version.parse(version)
)
assert package.name == package_name
assert str(package.version) == version
assert package.yanked is yanked
assert package.yanked_reason == yanked_reason
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment