feat: use ArtifactCache in get_package_from_url (#7693)

Co-authored-by: Randy Döring <30527984+radoering@users.noreply.github.com>

src/poetry/console/commands/debug/resolve.py

@@ -113,7 +113,7 @@ class DebugResolveCommand(InitCommand):
 
 
         if self.option("install"):
         if self.option("install"):
             env = EnvManager(self.poetry).get()
             env = EnvManager(self.poetry).get()
-            pool = RepositoryPool()
+            pool = RepositoryPool(config=self.poetry.config)
             locked_repository = Repository("poetry-locked")
             locked_repository = Repository("poetry-locked")
             for op in ops:
             for op in ops:
                 locked_repository.add_package(op.package)
                 locked_repository.add_package(op.package)

src/poetry/console/commands/init.py

@@ -434,11 +434,15 @@ You can specify a package in the following forms:
 
 
         try:
         try:
             cwd = self.poetry.file.parent
             cwd = self.poetry.file.parent
+            artifact_cache = self.poetry.pool.artifact_cache
         except (PyProjectException, RuntimeError):
         except (PyProjectException, RuntimeError):
             cwd = Path.cwd()
             cwd = Path.cwd()
+            artifact_cache = self._get_pool().artifact_cache
 
 
         parser = RequirementsParser(
         parser = RequirementsParser(
-            self.env if isinstance(self, EnvCommand) else None, cwd
+            artifact_cache=artifact_cache,
+            env=self.env if isinstance(self, EnvCommand) else None,
+            cwd=cwd,
         )
         )
         return [parser.parse(requirement) for requirement in requirements]
         return [parser.parse(requirement) for requirement in requirements]
 
 

src/poetry/console/commands/show.py

@@ -213,7 +213,7 @@ lists all packages available."""
         from poetry.utils.helpers import get_package_version_display_string
         from poetry.utils.helpers import get_package_version_display_string
 
 
         locked_packages = locked_repository.packages
         locked_packages = locked_repository.packages
-        pool = RepositoryPool(ignore_repository_names=True)
+        pool = RepositoryPool(ignore_repository_names=True, config=self.poetry.config)
         pool.add_repository(locked_repository)
         pool.add_repository(locked_repository)
         solver = Solver(
         solver = Solver(
             root,
             root,

src/poetry/factory.py

@@ -119,7 +119,7 @@ class Factory(BaseFactory):
     @classmethod
     @classmethod
     def create_pool(
     def create_pool(
         cls,
         cls,
-        auth_config: Config,
+        config: Config,
         sources: Iterable[dict[str, Any]] = (),
         sources: Iterable[dict[str, Any]] = (),
         io: IO | None = None,
         io: IO | None = None,
         disable_cache: bool = False,
         disable_cache: bool = False,
@@ -133,12 +133,12 @@ class Factory(BaseFactory):
         if disable_cache:
         if disable_cache:
             logger.debug("Disabling source caches")
             logger.debug("Disabling source caches")
 
 
-        pool = RepositoryPool()
+        pool = RepositoryPool(config=config)
 
 
         explicit_pypi = False
         explicit_pypi = False
         for source in sources:
         for source in sources:
             repository = cls.create_package_source(
             repository = cls.create_package_source(
-                source, auth_config, disable_cache=disable_cache
+                source, config, disable_cache=disable_cache
             )
             )
             priority = Priority[source.get("priority", Priority.PRIMARY.name).upper()]
             priority = Priority[source.get("priority", Priority.PRIMARY.name).upper()]
             if "default" in source or "secondary" in source:
             if "default" in source or "secondary" in source:
@@ -207,7 +207,7 @@ class Factory(BaseFactory):
 
 
     @classmethod
     @classmethod
     def create_package_source(
     def create_package_source(
-        cls, source: dict[str, str], auth_config: Config, disable_cache: bool = False
+        cls, source: dict[str, str], config: Config, disable_cache: bool = False
     ) -> HTTPRepository:
     ) -> HTTPRepository:
         from poetry.repositories.exceptions import InvalidSourceError
         from poetry.repositories.exceptions import InvalidSourceError
         from poetry.repositories.legacy_repository import LegacyRepository
         from poetry.repositories.legacy_repository import LegacyRepository
@@ -239,7 +239,7 @@ class Factory(BaseFactory):
         return repository_class(
         return repository_class(
             name,
             name,
             url,
             url,
-            config=auth_config,
+            config=config,
             disable_cache=disable_cache,
             disable_cache=disable_cache,
         )
         )
 
 

src/poetry/installation/installer.py

@@ -47,6 +47,7 @@ class Installer:
         self._package = package
         self._package = package
         self._locker = locker
         self._locker = locker
         self._pool = pool
         self._pool = pool
+        self._config = config
 
 
         self._dry_run = False
         self._dry_run = False
         self._requires_synchronization = False
         self._requires_synchronization = False
@@ -290,7 +291,7 @@ class Installer:
             )
             )
 
 
         # We resolve again by only using the lock file
         # We resolve again by only using the lock file
-        pool = RepositoryPool(ignore_repository_names=True)
+        pool = RepositoryPool(ignore_repository_names=True, config=self._config)
 
 
         # Making a new repo containing the packages
         # Making a new repo containing the packages
         # newly resolved and the ones from the current lock file
         # newly resolved and the ones from the current lock file

src/poetry/packages/direct_origin.py

@@ -2,12 +2,13 @@ from __future__ import annotations
 
 
 import functools
 import functools
 import os
 import os
-import tempfile
 import urllib.parse
 import urllib.parse
 
 
 from pathlib import Path
 from pathlib import Path
 from typing import TYPE_CHECKING
 from typing import TYPE_CHECKING
 
 
+from poetry.core.packages.utils.link import Link
+
 from poetry.inspection.info import PackageInfo
 from poetry.inspection.info import PackageInfo
 from poetry.inspection.info import PackageInfoError
 from poetry.inspection.info import PackageInfoError
 from poetry.utils.helpers import download_file
 from poetry.utils.helpers import download_file
@@ -18,6 +19,8 @@ from poetry.vcs.git import Git
 if TYPE_CHECKING:
 if TYPE_CHECKING:
     from poetry.core.packages.package import Package
     from poetry.core.packages.package import Package
 
 
+    from poetry.utils.cache import ArtifactCache
+
 
 
 @functools.lru_cache(maxsize=None)
 @functools.lru_cache(maxsize=None)
 def _get_package_from_git(
 def _get_package_from_git(
@@ -53,6 +56,9 @@ def _get_package_from_git(
 
 
 
 
 class DirectOrigin:
 class DirectOrigin:
+    def __init__(self, artifact_cache: ArtifactCache) -> None:
+        self._artifact_cache = artifact_cache
+
     @classmethod
     @classmethod
     def get_package_from_file(cls, file_path: Path) -> Package:
     def get_package_from_file(cls, file_path: Path) -> Package:
         try:
         try:
@@ -70,17 +76,22 @@ class DirectOrigin:
     def get_package_from_directory(cls, directory: Path) -> Package:
     def get_package_from_directory(cls, directory: Path) -> Package:
         return PackageInfo.from_directory(path=directory).to_package(root_dir=directory)
         return PackageInfo.from_directory(path=directory).to_package(root_dir=directory)
 
 
-    @classmethod
-    def get_package_from_url(cls, url: str) -> Package:
+    def get_package_from_url(self, url: str) -> Package:
         file_name = os.path.basename(urllib.parse.urlparse(url).path)
         file_name = os.path.basename(urllib.parse.urlparse(url).path)
-        with tempfile.TemporaryDirectory() as temp_dir:
-            dest = Path(temp_dir) / file_name
-            download_file(url, dest)
-            package = cls.get_package_from_file(dest)
-
-            package.files = [
-                {"file": file_name, "hash": "sha256:" + get_file_hash(dest)}
-            ]
+        link = Link(url)
+        artifact = self._artifact_cache.get_cached_archive_for_link(link, strict=True)
+
+        if not artifact:
+            artifact = (
+                self._artifact_cache.get_cache_directory_for_link(link) / file_name
+            )
+            artifact.parent.mkdir(parents=True, exist_ok=True)
+            download_file(url, artifact)
+
+        package = self.get_package_from_file(artifact)
+        package.files = [
+            {"file": file_name, "hash": "sha256:" + get_file_hash(artifact)}
+        ]
 
 
         package._source_type = "url"
         package._source_type = "url"
         package._source_url = url
         package._source_url = url

src/poetry/poetry.py

@@ -49,7 +49,7 @@ class Poetry(BasePoetry):
 
 
         self._locker = locker
         self._locker = locker
         self._config = config
         self._config = config
-        self._pool = RepositoryPool()
+        self._pool = RepositoryPool(config=config)
         self._plugin_manager: PluginManager | None = None
         self._plugin_manager: PluginManager | None = None
         self._disable_cache = disable_cache
         self._disable_cache = disable_cache
 
 

src/poetry/puzzle/provider.py

@@ -107,6 +107,7 @@ class Provider:
     ) -> None:
     ) -> None:
         self._package = package
         self._package = package
         self._pool = pool
         self._pool = pool
+        self._direct_origin = DirectOrigin(self._pool.artifact_cache)
         self._io = io
         self._io = io
         self._env: Env | None = None
         self._env: Env | None = None
         self._python_constraint = package.python_constraint
         self._python_constraint = package.python_constraint
@@ -311,7 +312,7 @@ class Provider:
         Basically, we clone the repository in a temporary directory
         Basically, we clone the repository in a temporary directory
         and get the information we need by checking out the specified reference.
         and get the information we need by checking out the specified reference.
         """
         """
-        package = DirectOrigin.get_package_from_vcs(
+        package = self._direct_origin.get_package_from_vcs(
             dependency.vcs,
             dependency.vcs,
             dependency.source,
             dependency.source,
             branch=dependency.branch,
             branch=dependency.branch,
@@ -330,7 +331,7 @@ class Provider:
 
 
     def _search_for_file(self, dependency: FileDependency) -> Package:
     def _search_for_file(self, dependency: FileDependency) -> Package:
         dependency.validate(raise_error=True)
         dependency.validate(raise_error=True)
-        package = DirectOrigin.get_package_from_file(dependency.full_path)
+        package = self._direct_origin.get_package_from_file(dependency.full_path)
 
 
         self.validate_package_for_dependency(dependency=dependency, package=package)
         self.validate_package_for_dependency(dependency=dependency, package=package)
 
 
@@ -348,7 +349,7 @@ class Provider:
 
 
     def _search_for_directory(self, dependency: DirectoryDependency) -> Package:
     def _search_for_directory(self, dependency: DirectoryDependency) -> Package:
         dependency.validate(raise_error=True)
         dependency.validate(raise_error=True)
-        package = DirectOrigin.get_package_from_directory(dependency.full_path)
+        package = self._direct_origin.get_package_from_directory(dependency.full_path)
 
 
         self.validate_package_for_dependency(dependency=dependency, package=package)
         self.validate_package_for_dependency(dependency=dependency, package=package)
 
 
@@ -360,7 +361,7 @@ class Provider:
         return package
         return package
 
 
     def _search_for_url(self, dependency: URLDependency) -> Package:
     def _search_for_url(self, dependency: URLDependency) -> Package:
-        package = DirectOrigin.get_package_from_url(dependency.url)
+        package = self._direct_origin.get_package_from_url(dependency.url)
 
 
         self.validate_package_for_dependency(dependency=dependency, package=package)
         self.validate_package_for_dependency(dependency=dependency, package=package)
 
 

src/poetry/repositories/repository_pool.py

@@ -8,8 +8,10 @@ from dataclasses import dataclass
 from enum import IntEnum
 from enum import IntEnum
 from typing import TYPE_CHECKING
 from typing import TYPE_CHECKING
 
 
+from poetry.config.config import Config
 from poetry.repositories.abstract_repository import AbstractRepository
 from poetry.repositories.abstract_repository import AbstractRepository
 from poetry.repositories.exceptions import PackageNotFound
 from poetry.repositories.exceptions import PackageNotFound
+from poetry.utils.cache import ArtifactCache
 
 
 
 
 if TYPE_CHECKING:
 if TYPE_CHECKING:
@@ -40,6 +42,8 @@ class RepositoryPool(AbstractRepository):
         self,
         self,
         repositories: list[Repository] | None = None,
         repositories: list[Repository] | None = None,
         ignore_repository_names: bool = False,
         ignore_repository_names: bool = False,
+        *,
+        config: Config | None = None,
     ) -> None:
     ) -> None:
         super().__init__("poetry-repository-pool")
         super().__init__("poetry-repository-pool")
         self._repositories: OrderedDict[str, PrioritizedRepository] = OrderedDict()
         self._repositories: OrderedDict[str, PrioritizedRepository] = OrderedDict()
@@ -50,6 +54,10 @@ class RepositoryPool(AbstractRepository):
         for repository in repositories:
         for repository in repositories:
             self.add_repository(repository)
             self.add_repository(repository)
 
 
+        self._artifact_cache = ArtifactCache(
+            cache_dir=(config or Config.create()).artifacts_cache_directory
+        )
+
     @property
     @property
     def repositories(self) -> list[Repository]:
     def repositories(self) -> list[Repository]:
         """
         """
@@ -77,6 +85,10 @@ class RepositoryPool(AbstractRepository):
             self._repositories.values(), key=lambda prio_repo: prio_repo.priority
             self._repositories.values(), key=lambda prio_repo: prio_repo.priority
         )
         )
 
 
+    @property
+    def artifact_cache(self) -> ArtifactCache:
+        return self._artifact_cache
+
     def has_default(self) -> bool:
     def has_default(self) -> bool:
         return self._contains_priority(Priority.DEFAULT)
         return self._contains_priority(Priority.DEFAULT)
 
 

src/poetry/utils/dependency_specification.py

@@ -22,6 +22,7 @@ from poetry.packages.direct_origin import DirectOrigin
 if TYPE_CHECKING:
 if TYPE_CHECKING:
     from poetry.core.packages.vcs_dependency import VCSDependency
     from poetry.core.packages.vcs_dependency import VCSDependency
 
 
+    from poetry.utils.cache import ArtifactCache
     from poetry.utils.env import Env
     from poetry.utils.env import Env
 
 
 
 
@@ -57,7 +58,14 @@ def dependency_to_specification(
 
 
 
 
 class RequirementsParser:
 class RequirementsParser:
-    def __init__(self, env: Env | None = None, cwd: Path | None = None) -> None:
+    def __init__(
+        self,
+        *,
+        artifact_cache: ArtifactCache,
+        env: Env | None = None,
+        cwd: Path | None = None,
+    ) -> None:
+        self._direct_origin = DirectOrigin(artifact_cache)
         self._env = env
         self._env = env
         self._cwd = cwd or Path.cwd()
         self._cwd = cwd or Path.cwd()
 
 
@@ -120,7 +128,7 @@ class RequirementsParser:
             pair["subdirectory"] = parsed.subdirectory
             pair["subdirectory"] = parsed.subdirectory
 
 
         source_root = self._env.path.joinpath("src") if self._env else None
         source_root = self._env.path.joinpath("src") if self._env else None
-        package = DirectOrigin.get_package_from_vcs(
+        package = self._direct_origin.get_package_from_vcs(
             "git",
             "git",
             url=url.url,
             url=url.url,
             rev=pair.get("rev"),
             rev=pair.get("rev"),
@@ -139,7 +147,7 @@ class RequirementsParser:
             return self._parse_git_url(requirement)
             return self._parse_git_url(requirement)
 
 
         if url_parsed.scheme in ["http", "https"]:
         if url_parsed.scheme in ["http", "https"]:
-            package = DirectOrigin.get_package_from_url(requirement)
+            package = self._direct_origin.get_package_from_url(requirement)
             assert package.source_url is not None
             assert package.source_url is not None
             return {"name": package.name, "url": package.source_url}
             return {"name": package.name, "url": package.source_url}
 
 
@@ -158,9 +166,9 @@ class RequirementsParser:
                 path = self._cwd.joinpath(requirement)
                 path = self._cwd.joinpath(requirement)
 
 
             if path.is_file():
             if path.is_file():
-                package = DirectOrigin.get_package_from_file(path.resolve())
+                package = self._direct_origin.get_package_from_file(path.resolve())
             else:
             else:
-                package = DirectOrigin.get_package_from_directory(path.resolve())
+                package = self._direct_origin.get_package_from_directory(path.resolve())
 
 
             return {
             return {
                 "name": package.name,
                 "name": package.name,

tests/conftest.py

@@ -24,6 +24,7 @@ from poetry.inspection.info import PackageInfoError
 from poetry.layouts import layout
 from poetry.layouts import layout
 from poetry.repositories import Repository
 from poetry.repositories import Repository
 from poetry.repositories import RepositoryPool
 from poetry.repositories import RepositoryPool
+from poetry.utils.cache import ArtifactCache
 from poetry.utils.env import EnvManager
 from poetry.utils.env import EnvManager
 from poetry.utils.env import SystemEnv
 from poetry.utils.env import SystemEnv
 from poetry.utils.env import VirtualEnv
 from poetry.utils.env import VirtualEnv
@@ -222,6 +223,11 @@ def config(
     return c
     return c
 
 
 
 
+@pytest.fixture
+def artifact_cache(config: Config) -> ArtifactCache:
+    return ArtifactCache(cache_dir=config.artifacts_cache_directory)
+
+
 @pytest.fixture()
 @pytest.fixture()
 def config_dir(tmp_path: Path) -> Path:
 def config_dir(tmp_path: Path) -> Path:
     path = tmp_path / "config"
     path = tmp_path / "config"

tests/installation/test_chef.py

@@ -14,7 +14,6 @@ from poetry.core.packages.utils.link import Link
 from poetry.factory import Factory
 from poetry.factory import Factory
 from poetry.installation.chef import Chef
 from poetry.installation.chef import Chef
 from poetry.repositories import RepositoryPool
 from poetry.repositories import RepositoryPool
-from poetry.utils.cache import ArtifactCache
 from poetry.utils.env import EnvManager
 from poetry.utils.env import EnvManager
 from tests.repositories.test_pypi_repository import MockRepository
 from tests.repositories.test_pypi_repository import MockRepository
 
 
@@ -22,6 +21,7 @@ from tests.repositories.test_pypi_repository import MockRepository
 if TYPE_CHECKING:
 if TYPE_CHECKING:
     from pytest_mock import MockerFixture
     from pytest_mock import MockerFixture
 
 
+    from poetry.utils.cache import ArtifactCache
     from tests.conftest import Config
     from tests.conftest import Config
     from tests.types import FixtureDirGetter
     from tests.types import FixtureDirGetter
 
 
@@ -40,11 +40,6 @@ def setup(mocker: MockerFixture, pool: RepositoryPool) -> None:
     mocker.patch.object(Factory, "create_pool", return_value=pool)
     mocker.patch.object(Factory, "create_pool", return_value=pool)
 
 
 
 
-@pytest.fixture
-def artifact_cache(config: Config) -> ArtifactCache:
-    return ArtifactCache(cache_dir=config.artifacts_cache_directory)
-
-
 def test_prepare_sdist(
 def test_prepare_sdist(
     config: Config,
     config: Config,
     config_cache_dir: Path,
     config_cache_dir: Path,

tests/installation/test_executor.py

@@ -136,11 +136,6 @@ def pool() -> RepositoryPool:
 
 
 
 
 @pytest.fixture
 @pytest.fixture
-def artifact_cache(config: Config) -> ArtifactCache:
-    return ArtifactCache(cache_dir=config.artifacts_cache_directory)
-
-
-@pytest.fixture
 def mock_file_downloads(
 def mock_file_downloads(
     http: type[httpretty.httpretty], fixture_dir: FixtureDirGetter
     http: type[httpretty.httpretty], fixture_dir: FixtureDirGetter
 ) -> None:
 ) -> None:

tests/packages/test_direct_origin.py

+from __future__ import annotations
+
+from typing import TYPE_CHECKING
+from unittest.mock import MagicMock
+
+from poetry.core.packages.utils.link import Link
+
+from poetry.packages.direct_origin import DirectOrigin
+from poetry.utils.cache import ArtifactCache
+
+
+if TYPE_CHECKING:
+    from pathlib import Path
+
+    from pytest_mock import MockerFixture
+
+    from tests.types import FixtureDirGetter
+
+
+def test_direct_origin_get_package_from_file(fixture_dir: FixtureDirGetter) -> None:
+    wheel_path = fixture_dir("distributions") / "demo-0.1.2-py2.py3-none-any.whl"
+    package = DirectOrigin.get_package_from_file(wheel_path)
+    assert package.name == "demo"
+
+
+def test_direct_origin_caches_url_dependency(tmp_path: Path) -> None:
+    artifact_cache = ArtifactCache(cache_dir=tmp_path)
+    direct_origin = DirectOrigin(artifact_cache)
+    url = "https://python-poetry.org/distributions/demo-0.1.0-py2.py3-none-any.whl"
+
+    package = direct_origin.get_package_from_url(url)
+
+    assert package.name == "demo"
+    assert artifact_cache.get_cached_archive_for_link(Link(url), strict=True)
+
+
+def test_direct_origin_does_not_download_url_dependency_when_cached(
+    fixture_dir: FixtureDirGetter, mocker: MockerFixture
+) -> None:
+    artifact_cache = MagicMock()
+    artifact_cache.get_cached_archive_for_link = MagicMock(
+        return_value=fixture_dir("distributions") / "demo-0.1.2-py2.py3-none-any.whl"
+    )
+    direct_origin = DirectOrigin(artifact_cache)
+    url = "https://python-poetry.org/distributions/demo-0.1.0-py2.py3-none-any.whl"
+    mocker.patch(
+        "poetry.packages.direct_origin.download_file",
+        side_effect=Exception("download_file should not be called"),
+    )
+
+    package = direct_origin.get_package_from_url(url)
+
+    assert package.name == "demo"
+    artifact_cache.get_cached_archive_for_link.assert_called_once_with(
+        Link(url), strict=True
+    )

tests/test_factory.py

@@ -543,7 +543,7 @@ def test_create_package_source_invalid(
     fixture_dir: FixtureDirGetter,
     fixture_dir: FixtureDirGetter,
 ) -> None:
 ) -> None:
     with pytest.raises(InvalidSourceError) as e:
     with pytest.raises(InvalidSourceError) as e:
-        Factory.create_package_source(source, auth_config=config)
+        Factory.create_package_source(source, config=config)
         Factory().create_poetry(fixture_dir("with_source_pypi_url"))
         Factory().create_poetry(fixture_dir("with_source_pypi_url"))
 
 
     assert str(e.value) == expected
     assert str(e.value) == expected

tests/utils/test_dependency_specification.py

@@ -13,6 +13,7 @@ from poetry.utils.dependency_specification import RequirementsParser
 if TYPE_CHECKING:
 if TYPE_CHECKING:
     from pytest_mock import MockerFixture
     from pytest_mock import MockerFixture
 
 
+    from poetry.utils.cache import ArtifactCache
     from poetry.utils.dependency_specification import DependencySpec
     from poetry.utils.dependency_specification import DependencySpec
 
 
 
 
@@ -104,7 +105,10 @@ if TYPE_CHECKING:
     ],
     ],
 )
 )
 def test_parse_dependency_specification(
 def test_parse_dependency_specification(
-    requirement: str, specification: DependencySpec, mocker: MockerFixture
+    requirement: str,
+    specification: DependencySpec,
+    mocker: MockerFixture,
+    artifact_cache: ArtifactCache,
 ) -> None:
 ) -> None:
     original = Path.exists
     original = Path.exists
 
 
@@ -116,5 +120,7 @@ def test_parse_dependency_specification(
     mocker.patch("pathlib.Path.exists", _mock)
     mocker.patch("pathlib.Path.exists", _mock)
 
 
     assert not DeepDiff(
     assert not DeepDiff(
-        RequirementsParser().parse(requirement), specification, ignore_order=True
+        RequirementsParser(artifact_cache=artifact_cache).parse(requirement),
+        specification,
+        ignore_order=True,
     )
     )