Skip to content

P
python-poetry
Commit ad1b0938 by Arun Babu Neelicattu
Options

replace git command use with dulwich 

This change introduces dulwich as the git backend, instead of system
git executable. Together with an LRU cache when inspecting git package,
this considerable improves performance for dependency solver and reuse
of source when project has git dependencies.

In cases where dulwich fails with an HTTPUnauthorized error, Poetry
falls back to system provided git client as a temporary measure. This
will be replaced in the future once dulwich supports git credentials.
parent bf04e205
Showing with 506 additions and 158 deletions
.github/workflows/main.yml
...@@ -85,6 +85,9 @@ jobs: ...@@ -85,6 +85,9 @@ jobs:
- name: Run pytest - name: Run pytest
run: poetry run python -m pytest -p no:sugar -q tests/ run: poetry run python -m pytest -p no:sugar -q tests/
- name: Run pytest (integration suite)
run: poetry run python -m pytest -p no:sugar -q --integration tests/integration
- name: Get Plugin Version (poetry-plugin-export) - name: Get Plugin Version (poetry-plugin-export)
id: poetry-plugin-export-version id: poetry-plugin-export-version
run: | run: |
......
poetry.lock
...@@ -22,7 +22,7 @@ tests_no_zope = ["coverage[toml] (>=5.0.2)", "hypothesis", "pympler", "pytest (> ...@@ -22,7 +22,7 @@ tests_no_zope = ["coverage[toml] (>=5.0.2)", "hypothesis", "pympler", "pytest (>
[[package]] [[package]]
name = "cachecontrol" name = "cachecontrol"
version = "0.12.10" version = "0.12.11"
description = "httplib2 caching for requests" description = "httplib2 caching for requests"
category = "main" category = "main"
optional = false optional = false
...@@ -169,6 +169,24 @@ optional = false ...@@ -169,6 +169,24 @@ optional = false
python-versions = "*" python-versions = "*"
[[package]] [[package]]
name = "dulwich"
version = "0.20.35"
description = "Python Git Library"
category = "main"
optional = false
python-versions = ">=3.6"
[package.dependencies]
certifi = "*"
urllib3 = ">=1.24.1"
[package.extras]
fastimport = ["fastimport"]
https = ["urllib3[secure] (>=1.24.1)"]
pgp = ["gpg"]
watch = ["pyinotify"]
[[package]]
name = "entrypoints" name = "entrypoints"
version = "0.3" version = "0.3"
description = "Discover and load entry points from installed packages." description = "Discover and load entry points from installed packages."
...@@ -721,7 +739,7 @@ testing = ["pytest (>=6)", "pytest-checkdocs (>=2.4)", "pytest-flake8", "pytest- ...@@ -721,7 +739,7 @@ testing = ["pytest (>=6)", "pytest-checkdocs (>=2.4)", "pytest-flake8", "pytest-
[metadata] [metadata]
lock-version = "1.1" lock-version = "1.1"
python-versions = "^3.7" python-versions = "^3.7"
content-hash = "f74aedfd57d8aa47486cacfd4e2f5a24e952cfe1aee43c7b6a6d801eec5254ea" content-hash = "2bf89b93e12d19fdadc3799785ef9cae5fd5d3d964ac2cfc4861b5e9d7e9554a"
[metadata.files] [metadata.files]
atomicwrites = [ atomicwrites = [
...@@ -733,8 +751,8 @@ attrs = [ ...@@ -733,8 +751,8 @@ attrs = [
{file = "attrs-21.4.0.tar.gz", hash = "sha256:626ba8234211db98e869df76230a137c4c40a12d72445c45d5f5b716f076e2fd"}, {file = "attrs-21.4.0.tar.gz", hash = "sha256:626ba8234211db98e869df76230a137c4c40a12d72445c45d5f5b716f076e2fd"},
] ]
cachecontrol = [ cachecontrol = [
{file = "CacheControl-0.12.10-py2.py3-none-any.whl", hash = "sha256:b0d43d8f71948ef5ebdee5fe236b86c6ffc7799370453dccb0e894c20dfa487c"}, {file = "CacheControl-0.12.11-py2.py3-none-any.whl", hash = "sha256:2c75d6a8938cb1933c75c50184549ad42728a27e9f6b92fd677c3151aa72555b"},
{file = "CacheControl-0.12.10.tar.gz", hash = "sha256:d8aca75b82eec92d84b5d6eb8c8f66ea16f09d2adb09dbca27fe2d5fc8d3732d"}, {file = "CacheControl-0.12.11.tar.gz", hash = "sha256:a5b9fcc986b184db101aa280b42ecdcdfc524892596f606858e0b7a8b4d9e144"},
] ]
cachy = [ cachy = [
{file = "cachy-0.3.0-py2.py3-none-any.whl", hash = "sha256:338ca09c8860e76b275aff52374330efedc4d5a5e45dc1c5b539c1ead0786fe7"}, {file = "cachy-0.3.0-py2.py3-none-any.whl", hash = "sha256:338ca09c8860e76b275aff52374330efedc4d5a5e45dc1c5b539c1ead0786fe7"},
...@@ -889,6 +907,29 @@ distlib = [ ...@@ -889,6 +907,29 @@ distlib = [
{file = "distlib-0.3.4-py2.py3-none-any.whl", hash = "sha256:6564fe0a8f51e734df6333d08b8b94d4ea8ee6b99b5ed50613f731fd4089f34b"}, {file = "distlib-0.3.4-py2.py3-none-any.whl", hash = "sha256:6564fe0a8f51e734df6333d08b8b94d4ea8ee6b99b5ed50613f731fd4089f34b"},
{file = "distlib-0.3.4.zip", hash = "sha256:e4b58818180336dc9c529bfb9a0b58728ffc09ad92027a3f30b7cd91e3458579"}, {file = "distlib-0.3.4.zip", hash = "sha256:e4b58818180336dc9c529bfb9a0b58728ffc09ad92027a3f30b7cd91e3458579"},
] ]
dulwich = [
{file = "dulwich-0.20.35-cp310-cp310-macosx_10_15_x86_64.whl", hash = "sha256:428b5fbb79f8cfba2f5ac6826cc813d1903b44b0780e9ec57e54cbd0f44feb61"},
{file = "dulwich-0.20.35-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:581c6aa825c9267794747c5cc5ec3831960d96ca7fd9eb0158989e9a4099cbb1"},
{file = "dulwich-0.20.35-cp310-cp310-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:e11cc7a30b42dbbe5a0b6ebbfbfbb07138a5ffd6175bab2ddbabc9882a1c0438"},
{file = "dulwich-0.20.35-cp310-cp310-win_amd64.whl", hash = "sha256:22c61a24edb699564b49a9701b723a08fa773f5d3322e8a0cabda897ae86816e"},
{file = "dulwich-0.20.35-cp36-cp36m-macosx_10_14_x86_64.whl", hash = "sha256:9759cf611503681bcdd2950c9d2db04d1c057ecbb62d6fccd095b13771864f1c"},
{file = "dulwich-0.20.35-cp36-cp36m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:8d683b4f30b1dae6b1668336f62f10ff57ebf2a1252c7cc76ad3eeff973879eb"},
{file = "dulwich-0.20.35-cp36-cp36m-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:9d85b6b41c4be6df9ecdc4014d3cbe78a5a44a73c97bccbefac3e5de83bb74be"},
{file = "dulwich-0.20.35-cp36-cp36m-win_amd64.whl", hash = "sha256:6dc9b082f6ace9890de572260a575a09a996d617f5930edd2858c6f8fedfd7fb"},
{file = "dulwich-0.20.35-cp37-cp37m-macosx_10_14_x86_64.whl", hash = "sha256:28ac2374f09487b02a8cb9b2fad083c358fc927bcfe9803d971614bc00e25076"},
{file = "dulwich-0.20.35-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:195b21c7a8f85cb2de8938d54fcc6d589d1ccbceaa63bb117796b531065bb68b"},
{file = "dulwich-0.20.35-cp37-cp37m-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:9bdea3a4e8e5e3b1dbd513d9ab8a692f8a9a6f4760633e25c006446bce56fc5e"},
{file = "dulwich-0.20.35-cp37-cp37m-win_amd64.whl", hash = "sha256:3d3d07b5aa51e6b7d08707c62932da86adbbaaa62552a0129b37d413735c7786"},
{file = "dulwich-0.20.35-cp38-cp38-macosx_10_14_x86_64.whl", hash = "sha256:5d94cd182fb0da4ec2f182be977b27b9cc1d7dbd0ee9bbf991e101a95fdcd3d8"},
{file = "dulwich-0.20.35-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:f563e9f51e83c47a7df2f3cea79919f700e50d1e5556b6b753730b9cd2be1f47"},
{file = "dulwich-0.20.35-cp38-cp38-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:f221c3c2fd10260419905bb673cd00129d491e3ed38c7a8d3ac2c7662682dd9b"},
{file = "dulwich-0.20.35-cp38-cp38-win_amd64.whl", hash = "sha256:c4f4c59445dc5c2341e9cb2fe35e51a890e8a5f42178abec0a96044811c558a9"},
{file = "dulwich-0.20.35-cp39-cp39-macosx_10_15_x86_64.whl", hash = "sha256:3616a949053eb6bdf34581f57d1f6cb7192a4bb635be1a02c37f6f6dda032277"},
{file = "dulwich-0.20.35-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:134a2f586847c2c58569959a784d7a875b551df4226b639267302217799e4234"},
{file = "dulwich-0.20.35-cp39-cp39-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:c008b6b562af76cf011d3b5450a0d30edc96feeee7856b081d7400bc7cf42653"},
{file = "dulwich-0.20.35-cp39-cp39-win_amd64.whl", hash = "sha256:bf228800785754d7a55d52c5f122c26c3ced51f0f3df727fde2c9fefb71d5d76"},
{file = "dulwich-0.20.35.tar.gz", hash = "sha256:953f6301a9df8a091fa88d55eed394a88bf9988cde8be341775354910918c196"},
]
entrypoints = [ entrypoints = [
{file = "entrypoints-0.3-py2.py3-none-any.whl", hash = "sha256:589f874b313739ad35be6e0cd7efde2a4e9b6fea91edcc34e58ecbb8dbe56d19"}, {file = "entrypoints-0.3-py2.py3-none-any.whl", hash = "sha256:589f874b313739ad35be6e0cd7efde2a4e9b6fea91edcc34e58ecbb8dbe56d19"},
{file = "entrypoints-0.3.tar.gz", hash = "sha256:c70dd71abe5a8c85e55e12c19bd91ccfeec11a6e99044204511f9ed547d48451"}, {file = "entrypoints-0.3.tar.gz", hash = "sha256:c70dd71abe5a8c85e55e12c19bd91ccfeec11a6e99044204511f9ed547d48451"},
......
pyproject.toml
...@@ -56,6 +56,7 @@ tomlkit = ">=0.7.0,<1.0.0" ...@@ -56,6 +56,7 @@ tomlkit = ">=0.7.0,<1.0.0"
# exclude 20.4.5 - 20.4.6 due to https://github.com/pypa/pip/issues/9953 # exclude 20.4.5 - 20.4.6 due to https://github.com/pypa/pip/issues/9953
virtualenv = "(>=20.4.3,<20.4.5 || >=20.4.7)" virtualenv = "(>=20.4.3,<20.4.5 || >=20.4.7)"
urllib3 = "^1.26.0" urllib3 = "^1.26.0"
dulwich = "^0.20.35"
[tool.poetry.dev-dependencies] [tool.poetry.dev-dependencies]
tox = "^3.18" tox = "^3.18"
......
src/poetry/console/commands/init.py
...@@ -436,8 +436,13 @@ You can specify a package in the following forms: ...@@ -436,8 +436,13 @@ You can specify a package in the following forms:
if extras: if extras:
pair["extras"] = extras pair["extras"] = extras
source_root = (
self.env.path.joinpath("src")
if isinstance(self, EnvCommand) and self.env
else None
)
package = Provider.get_package_from_vcs( package = Provider.get_package_from_vcs(
"git", url.url, rev=pair.get("rev") "git", url=url.url, rev=pair.get("rev"), source_root=source_root
) )
pair["name"] = package.name pair["name"] = package.name
result.append(pair) result.append(pair)
......
src/poetry/installation/executor.py
...@@ -576,7 +576,7 @@ class Executor: ...@@ -576,7 +576,7 @@ class Executor:
return self.pip_install(req, upgrade=True) return self.pip_install(req, upgrade=True)
def _install_git(self, operation: Install | Update) -> int: def _install_git(self, operation: Install | Update) -> int:
from poetry.core.vcs import Git from poetry.vcs.git import Git
package = operation.package package = operation.package
operation_message = self.get_operation_message(operation) operation_message = self.get_operation_message(operation)
...@@ -586,24 +586,15 @@ class Executor: ...@@ -586,24 +586,15 @@ class Executor:
) )
self._write(operation, message) self._write(operation, message)
src_dir = self._env.path / "src" / package.name source = Git.clone(
if src_dir.exists(): url=package.source_url,
remove_directory(src_dir, force=True) source_root=self._env.path / "src",
revision=package.source_resolved_reference or package.source_reference,
src_dir.parent.mkdir(exist_ok=True) )
git = Git()
git.clone(package.source_url, src_dir)
reference = package.source_resolved_reference
if not reference:
reference = package.source_reference
git.checkout(reference, src_dir)
# Now we just need to install from the source directory # Now we just need to install from the source directory
original_url = package.source_url original_url = package.source_url
package._source_url = str(src_dir) package._source_url = str(source.path)
status_code = self._install_directory(operation) status_code = self._install_directory(operation)
......
src/poetry/installation/installer.py
...@@ -199,7 +199,10 @@ class Installer: ...@@ -199,7 +199,10 @@ class Installer:
self._io, self._io,
) )
ops = solver.solve(use_latest=[]).calculate_operations() with solver.provider.use_source_root(
source_root=self._env.path.joinpath("src")
):
ops = solver.solve(use_latest=[]).calculate_operations()
local_repo = Repository() local_repo = Repository()
self._populate_local_repo(local_repo, ops) self._populate_local_repo(local_repo, ops)
...@@ -236,7 +239,10 @@ class Installer: ...@@ -236,7 +239,10 @@ class Installer:
self._io, self._io,
) )
ops = solver.solve(use_latest=self._whitelist).calculate_operations() with solver.provider.use_source_root(
source_root=self._env.path.joinpath("src")
):
ops = solver.solve(use_latest=self._whitelist).calculate_operations()
else: else:
self._io.write_line("<info>Installing dependencies from lock file</>") self._io.write_line("<info>Installing dependencies from lock file</>")
......
src/poetry/installation/pip_installer.py
...@@ -248,27 +248,19 @@ class PipInstaller(BaseInstaller): ...@@ -248,27 +248,19 @@ class PipInstaller(BaseInstaller):
def install_git(self, package: Package) -> None: def install_git(self, package: Package) -> None:
from poetry.core.packages.package import Package from poetry.core.packages.package import Package
from poetry.core.vcs.git import Git
src_dir = self._env.path / "src" / package.name from poetry.vcs.git import Git
if src_dir.exists():
remove_directory(src_dir, force=True)
src_dir.parent.mkdir(exist_ok=True) source = Git.clone(
url=package.source_url,
git = Git() source_root=self._env.path / "src",
git.clone(package.source_url, src_dir) revision=package.source_resolved_reference or package.source_reference,
)
reference = package.source_resolved_reference
if not reference:
reference = package.source_reference
git.checkout(reference, src_dir)
# Now we just need to install from the source directory # Now we just need to install from the source directory
pkg = Package(package.name, package.version) pkg = Package(package.name, package.version)
pkg._source_type = "directory" pkg._source_type = "directory"
pkg._source_url = str(src_dir) pkg._source_url = str(source.path)
pkg.develop = package.develop pkg.develop = package.develop
self.install_directory(pkg) self.install_directory(pkg)
src/poetry/puzzle/provider.py
from __future__ import annotations from __future__ import annotations
import functools
import logging import logging
import os import os
import re import re
...@@ -10,7 +11,6 @@ import urllib.parse ...@@ -10,7 +11,6 @@ import urllib.parse
from collections import defaultdict from collections import defaultdict
from contextlib import contextmanager from contextlib import contextmanager
from pathlib import Path from pathlib import Path
from tempfile import mkdtemp
from typing import TYPE_CHECKING from typing import TYPE_CHECKING
from typing import Any from typing import Any
from typing import Iterable from typing import Iterable
...@@ -20,7 +20,6 @@ from cleo.ui.progress_indicator import ProgressIndicator ...@@ -20,7 +20,6 @@ from cleo.ui.progress_indicator import ProgressIndicator
from poetry.core.packages.utils.utils import get_python_constraint_from_marker from poetry.core.packages.utils.utils import get_python_constraint_from_marker
from poetry.core.semver.empty_constraint import EmptyConstraint from poetry.core.semver.empty_constraint import EmptyConstraint
from poetry.core.semver.version import Version from poetry.core.semver.version import Version
from poetry.core.vcs.git import Git
from poetry.core.version.markers import AnyMarker from poetry.core.version.markers import AnyMarker
from poetry.core.version.markers import MarkerUnion from poetry.core.version.markers import MarkerUnion
...@@ -34,7 +33,7 @@ from poetry.packages import DependencyPackage ...@@ -34,7 +33,7 @@ from poetry.packages import DependencyPackage
from poetry.packages.package_collection import PackageCollection from poetry.packages.package_collection import PackageCollection
from poetry.puzzle.exceptions import OverrideNeeded from poetry.puzzle.exceptions import OverrideNeeded
from poetry.utils.helpers import download_file from poetry.utils.helpers import download_file
from poetry.utils.helpers import remove_directory from poetry.vcs.git import Git
if TYPE_CHECKING: if TYPE_CHECKING:
...@@ -61,12 +60,43 @@ class Indicator(ProgressIndicator): ...@@ -61,12 +60,43 @@ class Indicator(ProgressIndicator):
return f"{elapsed:.1f}s" return f"{elapsed:.1f}s"
@functools.lru_cache(maxsize=None)
def _get_package_from_git(
url: str,
branch: str | None = None,
tag: str | None = None,
rev: str | None = None,
source_root: Path | None = None,
) -> Package:
source = Git.clone(
url=url,
source_root=source_root,
branch=branch,
tag=tag,
revision=rev,
clean=False,
)
revision = Git.get_revision(source)
package = Provider.get_package_from_directory(Path(source.path))
package._source_type = "git"
package._source_url = url
package._source_reference = rev or tag or branch or "HEAD"
package._source_resolved_reference = revision
return package
class Provider: class Provider:
UNSAFE_PACKAGES: set[str] = set() UNSAFE_PACKAGES: set[str] = set()
def __init__( def __init__(
self, package: Package, pool: Pool, io: Any, env: Env | None = None self,
package: Package,
pool: Pool,
io: Any,
env: Env | None = None,
) -> None: ) -> None:
self._package = package self._package = package
self._pool = pool self._pool = pool
...@@ -78,6 +108,7 @@ class Provider: ...@@ -78,6 +108,7 @@ class Provider:
self._overrides: dict[DependencyPackage, dict[str, Dependency]] = {} self._overrides: dict[DependencyPackage, dict[str, Dependency]] = {}
self._deferred_cache: dict[Dependency, Package] = {} self._deferred_cache: dict[Dependency, Package] = {}
self._load_deferred = True self._load_deferred = True
self._source_root: Path | None = None
@property @property
def pool(self) -> Pool: def pool(self) -> Pool:
...@@ -93,6 +124,15 @@ class Provider: ...@@ -93,6 +124,15 @@ class Provider:
self._load_deferred = load_deferred self._load_deferred = load_deferred
@contextmanager @contextmanager
def use_source_root(self, source_root: Path) -> Iterator[Provider]:
original_source_root = self._source_root
self._source_root = source_root
yield self
self._source_root = original_source_root
@contextmanager
def use_environment(self, env: Env) -> Iterator[Provider]: def use_environment(self, env: Env) -> Iterator[Provider]:
original_env = self._env original_env = self._env
original_python_constraint = self._python_constraint original_python_constraint = self._python_constraint
...@@ -105,6 +145,17 @@ class Provider: ...@@ -105,6 +145,17 @@ class Provider:
self._env = original_env self._env = original_env
self._python_constraint = original_python_constraint self._python_constraint = original_python_constraint
@staticmethod
def validate_package_for_dependency(
dependency: Dependency, package: Package
) -> None:
if dependency.name != package.name:
# For now, the dependency's name must match the actual package's name
raise RuntimeError(
f"The dependency name for {dependency.name} does not match the actual"
f" package's name: {package.name}"
)
def search_for( def search_for(
self, self,
dependency: ( dependency: (
...@@ -161,8 +212,12 @@ class Provider: ...@@ -161,8 +212,12 @@ class Provider:
branch=dependency.branch, branch=dependency.branch,
tag=dependency.tag, tag=dependency.tag,
rev=dependency.rev, rev=dependency.rev,
name=dependency.name, source_root=self._source_root
or (self._env.path.joinpath("src") if self._env else None),
) )
self.validate_package_for_dependency(dependency=dependency, package=package)
package.develop = dependency.develop package.develop = dependency.develop
dependency._constraint = package.version dependency._constraint = package.version
...@@ -176,44 +231,21 @@ class Provider: ...@@ -176,44 +231,21 @@ class Provider:
return [package] return [package]
@classmethod @staticmethod
def get_package_from_vcs( def get_package_from_vcs(
cls,
vcs: str, vcs: str,
url: str, url: str,
branch: str | None = None, branch: str | None = None,
tag: str | None = None, tag: str | None = None,
rev: str | None = None, rev: str | None = None,
name: str | None = None, source_root: Path | None = None,
) -> Package: ) -> Package:
if vcs != "git": if vcs != "git":
raise ValueError(f"Unsupported VCS dependency {vcs}") raise ValueError(f"Unsupported VCS dependency {vcs}")
suffix = url.split("/")[-1].rstrip(".git") return _get_package_from_git(
tmp_dir = Path(mkdtemp(prefix=f"pypoetry-git-{suffix}")) url=url, branch=branch, tag=tag, rev=rev, source_root=source_root
)
try:
git = Git()
git.clone(url, tmp_dir)
reference = branch or tag or rev
if reference is not None:
git.checkout(reference, tmp_dir)
else:
reference = "HEAD"
revision = git.rev_parse(reference, tmp_dir).strip()
package = cls.get_package_from_directory(tmp_dir, name=name)
package._source_type = "git"
package._source_url = url
package._source_reference = reference
package._source_resolved_reference = revision
except Exception:
raise
finally:
remove_directory(tmp_dir, force=True)
return package
def search_for_file(self, dependency: FileDependency) -> list[Package]: def search_for_file(self, dependency: FileDependency) -> list[Package]:
if dependency in self._deferred_cache: if dependency in self._deferred_cache:
...@@ -228,12 +260,7 @@ class Provider: ...@@ -228,12 +260,7 @@ class Provider:
self._deferred_cache[dependency] = (dependency, package) self._deferred_cache[dependency] = (dependency, package)
if dependency.name != package.name: self.validate_package_for_dependency(dependency=dependency, package=package)
# For now, the dependency's name must match the actual package's name
raise RuntimeError(
f"The dependency name for {dependency.name} does not match the actual"
f" package's name: {package.name}"
)
if dependency.base is not None: if dependency.base is not None:
package.root_dir = dependency.base package.root_dir = dependency.base
...@@ -263,15 +290,15 @@ class Provider: ...@@ -263,15 +290,15 @@ class Provider:
package = _package.clone() package = _package.clone()
else: else:
package = self.get_package_from_directory( package = self.get_package_from_directory(dependency.full_path)
dependency.full_path, name=dependency.name
)
dependency._constraint = package.version dependency._constraint = package.version
dependency._pretty_constraint = package.version.text dependency._pretty_constraint = package.version.text
self._deferred_cache[dependency] = (dependency, package) self._deferred_cache[dependency] = (dependency, package)
self.validate_package_for_dependency(dependency=dependency, package=package)
package.develop = dependency.develop package.develop = dependency.develop
if dependency.base is not None: if dependency.base is not None:
...@@ -280,21 +307,8 @@ class Provider: ...@@ -280,21 +307,8 @@ class Provider:
return [package] return [package]
@classmethod @classmethod
def get_package_from_directory( def get_package_from_directory(cls, directory: Path) -> Package:
cls, directory: Path, name: str | None = None return PackageInfo.from_directory(path=directory).to_package(root_dir=directory)
) -> Package:
package = PackageInfo.from_directory(path=directory).to_package(
root_dir=directory
)
if name and name != package.name:
# For now, the dependency's name must match the actual package's name
raise RuntimeError(
f"The dependency name for {name} does not match the actual package's"
f" name: {package.name}"
)
return package
def search_for_url(self, dependency: URLDependency) -> list[Package]: def search_for_url(self, dependency: URLDependency) -> list[Package]:
if dependency in self._deferred_cache: if dependency in self._deferred_cache:
...@@ -302,12 +316,7 @@ class Provider: ...@@ -302,12 +316,7 @@ class Provider:
package = self.get_package_from_url(dependency.url) package = self.get_package_from_url(dependency.url)
if dependency.name != package.name: self.validate_package_for_dependency(dependency=dependency, package=package)
# For now, the dependency's name must match the actual package's name
raise RuntimeError(
f"The dependency name for {dependency.name} does not match the actual"
f" package's name: {package.name}"
)
for extra in dependency.extras: for extra in dependency.extras:
if extra in package.extras: if extra in package.extras:
......
src/poetry/repositories/installed_repository.py
...@@ -77,13 +77,10 @@ class InstalledRepository(Repository): ...@@ -77,13 +77,10 @@ class InstalledRepository(Repository):
@classmethod @classmethod
def get_package_vcs_properties_from_path(cls, src: Path) -> tuple[str, str, str]: def get_package_vcs_properties_from_path(cls, src: Path) -> tuple[str, str, str]:
from poetry.core.vcs.git import Git from poetry.vcs.git import Git
git = Git() info = Git.info(repo=src)
revision = git.rev_parse("HEAD", src).strip() return "git", info.origin, info.revision
url = git.remote_url(src)
return "git", url, revision
@classmethod @classmethod
def is_vcs_package(cls, package: Path | Package, env: Env) -> bool: def is_vcs_package(cls, package: Path | Package, env: Env) -> bool:
......
src/poetry/vcs/git/__init__.py 0 → 100644
from __future__ import annotations
from poetry.vcs.git.backend import Git
__all__ = [Git.__name__]
src/poetry/vcs/git/system.py 0 → 100644
from __future__ import annotations
import subprocess
from typing import TYPE_CHECKING
from dulwich.client import find_git_command
if TYPE_CHECKING:
from pathlib import Path
from typing import Any
class SystemGit:
@classmethod
def clone(cls, repository: str, dest: Path) -> str:
cls._check_parameter(repository)
return cls.run("clone", "--recurse-submodules", "--", repository, str(dest))
@classmethod
def checkout(cls, rev: str, target: Path | None = None) -> str:
args = []
if target:
args += [
"--git-dir",
(target / ".git").as_posix(),
"--work-tree",
target.as_posix(),
]
cls._check_parameter(rev)
args += ["checkout", rev]
return cls.run(*args)
@staticmethod
def run(*args: Any, **kwargs: Any) -> str:
folder = kwargs.pop("folder", None)
if folder:
args = (
"--git-dir",
(folder / ".git").as_posix(),
"--work-tree",
folder.as_posix(),
) + args
return (
subprocess.check_output(
find_git_command() + list(args), stderr=subprocess.STDOUT
)
.decode()
.strip()
)
@staticmethod
def _check_parameter(parameter: str) -> None:
"""
Checks a git parameter to avoid unwanted code execution.
"""
if parameter.strip().startswith("-"):
raise RuntimeError(f"Invalid Git parameter: {parameter}")
tests/conftest.py
...@@ -38,6 +38,8 @@ from tests.helpers import mock_download ...@@ -38,6 +38,8 @@ from tests.helpers import mock_download
if TYPE_CHECKING: if TYPE_CHECKING:
from _pytest.config import Config as PyTestConfig
from _pytest.config.argparsing import Parser
from pytest_mock import MockerFixture from pytest_mock import MockerFixture
from poetry.poetry import Poetry from poetry.poetry import Poetry
...@@ -45,6 +47,23 @@ if TYPE_CHECKING: ...@@ -45,6 +47,23 @@ if TYPE_CHECKING:
from tests.types import ProjectFactory from tests.types import ProjectFactory
def pytest_addoption(parser: Parser) -> None:
parser.addoption(
"--integration",
action="store_true",
dest="integration",
default=False,
help="enable integration tests",
)
def pytest_configure(config: PyTestConfig) -> None:
config.addinivalue_line("markers", "integration: mark integration tests")
if not config.option.integration:
config.option.markexpr = "not integration"
class Config(BaseConfig): class Config(BaseConfig):
def get(self, setting_name: str, default: Any = None) -> Any: def get(self, setting_name: str, default: Any = None) -> Any:
self.merge(self._config_source.config) self.merge(self._config_source.config)
...@@ -252,9 +271,8 @@ def isolate_environ() -> Iterator[None]: ...@@ -252,9 +271,8 @@ def isolate_environ() -> Iterator[None]:
@pytest.fixture(autouse=True) @pytest.fixture(autouse=True)
def git_mock(mocker: MockerFixture) -> None: def git_mock(mocker: MockerFixture) -> None:
# Patch git module to not actually clone projects # Patch git module to not actually clone projects
mocker.patch("poetry.core.vcs.git.Git.clone", new=mock_clone) mocker.patch("poetry.vcs.git.Git.clone", new=mock_clone)
mocker.patch("poetry.core.vcs.git.Git.checkout", new=lambda *_: None) p = mocker.patch("poetry.vcs.git.Git.get_revision")
p = mocker.patch("poetry.core.vcs.git.Git.rev_parse")
p.return_value = "9cf87a285a2d3fbb0b9fa621997b3acc3631ed24" p.return_value = "9cf87a285a2d3fbb0b9fa621997b3acc3631ed24"
......
tests/console/conftest.py
...@@ -71,9 +71,8 @@ def setup( ...@@ -71,9 +71,8 @@ def setup(
p.return_value = installed p.return_value = installed
# Patch git module to not actually clone projects # Patch git module to not actually clone projects
mocker.patch("poetry.core.vcs.git.Git.clone", new=mock_clone) mocker.patch("poetry.vcs.git.Git.clone", new=mock_clone)
mocker.patch("poetry.core.vcs.git.Git.checkout", new=lambda *_: None) p = mocker.patch("poetry.vcs.git.Git.get_revision")
p = mocker.patch("poetry.core.vcs.git.Git.rev_parse")
p.return_value = "9cf87a285a2d3fbb0b9fa621997b3acc3631ed24" p.return_value = "9cf87a285a2d3fbb0b9fa621997b3acc3631ed24"
# Patch the virtual environment creation do actually do nothing # Patch the virtual environment creation do actually do nothing
...@@ -99,6 +98,7 @@ def project_directory() -> str: ...@@ -99,6 +98,7 @@ def project_directory() -> str:
@pytest.fixture @pytest.fixture
def poetry(repo: TestRepository, project_directory: str, config: Config) -> Poetry: def poetry(repo: TestRepository, project_directory: str, config: Config) -> Poetry:
p = Factory().create_poetry( p = Factory().create_poetry(
Path(__file__).parent.parent / "fixtures" / project_directory Path(__file__).parent.parent / "fixtures" / project_directory
) )
......
tests/helpers.py
from __future__ import annotations from __future__ import annotations
import os import os
import re
import shutil import shutil
import urllib.parse import urllib.parse
...@@ -90,19 +91,34 @@ def copy_or_symlink(source: Path, dest: Path) -> None: ...@@ -90,19 +91,34 @@ def copy_or_symlink(source: Path, dest: Path) -> None:
os.symlink(str(source), str(dest)) os.symlink(str(source), str(dest))
def mock_clone(_: Any, source: str, dest: Path) -> None: class MockDulwichRepo:
def __init__(self, root: Path | str, **__: Any) -> None:
self.path = str(root)
def head(self) -> bytes:
return b"9cf87a285a2d3fbb0b9fa621997b3acc3631ed24"
def mock_clone(
url: str,
*_: Any,
source_root: Path | None = None,
**__: Any,
) -> MockDulwichRepo:
# Checking source to determine which folder we need to copy # Checking source to determine which folder we need to copy
parsed = ParsedUrl.parse(source) parsed = ParsedUrl.parse(url)
path = re.sub(r"(.git)?$", "", parsed.pathname.lstrip("/"))
folder = Path(__file__).parent / "fixtures" / "git" / parsed.resource / path
if not source_root:
source_root = Path(Factory.create_config().get("cache-dir")) / "src"
folder = ( dest = source_root / path
Path(__file__).parent dest.parent.mkdir(parents=True, exist_ok=True)
/ "fixtures"
/ "git"
/ parsed.resource
/ parsed.pathname.lstrip("/").rstrip(".git")
)
copy_or_symlink(folder, dest) copy_or_symlink(folder, dest)
return MockDulwichRepo(dest)
def mock_download(url: str, dest: str, **__: Any) -> None: def mock_download(url: str, dest: str, **__: Any) -> None:
......
tests/integration/test_utils_vcs_git.py 0 → 100644
from __future__ import annotations
import uuid
from copy import deepcopy
from hashlib import sha1
from pathlib import Path
from typing import TYPE_CHECKING
import pytest
from dulwich.client import HTTPUnauthorized
from dulwich.client import get_transport_and_path
from dulwich.repo import Repo
from poetry.core.pyproject.toml import PyProjectTOML
from poetry.console.exceptions import PoetrySimpleConsoleException
from poetry.vcs.git import Git
from poetry.vcs.git.backend import GitRefSpec
if TYPE_CHECKING:
from _pytest.tmpdir import TempdirFactory
from dulwich.client import FetchPackResult
from dulwich.client import GitClient
from pytest_mock import MockerFixture
from tests.conftest import Config
# these tests are integration as they rely on an external repository
# see `source_url` fixture
pytestmark = pytest.mark.integration
@pytest.fixture(autouse=True)
def git_mock() -> None:
pass
@pytest.fixture(autouse=True)
def setup(config: Config) -> None:
pass
REVISION_TO_VERSION_MAP = {
"b6204750a763268e941cec1f05f8986b6c66913e": "0.1.0", # Annotated Tag
"18d3ff247d288da701fc7f9ce2ec718388fca266": "0.1.1-alpha.0",
"dd07e8d4efb82690e7975b289917a7782fbef29b": "0.2.0-alpha.0",
"7263819922b4cd008afbb447f425a562432dad7d": "0.2.0-alpha.1",
}
BRANCH_TO_REVISION_MAP = {"0.1": "18d3ff247d288da701fc7f9ce2ec718388fca266"}
TAG_TO_REVISION_MAP = {"v0.1.0": "b6204750a763268e941cec1f05f8986b6c66913e"}
REF_TO_REVISION_MAP = {
"branch": BRANCH_TO_REVISION_MAP,
"tag": TAG_TO_REVISION_MAP,
}
@pytest.fixture(scope="module")
def source_url() -> str:
return "https://github.com/python-poetry/test-fixture-vcs-repository.git"
@pytest.fixture(scope="module")
def source_directory_name(source_url: str) -> str:
return Git.get_name_from_source_url(url=source_url)
@pytest.fixture(scope="module")
def local_repo(tmpdir_factory: TempdirFactory, source_directory_name: str) -> Repo:
with Repo.init(
tmpdir_factory.mktemp("src") / source_directory_name, mkdir=True
) as repo:
yield repo
@pytest.fixture(scope="module")
def _remote_refs(source_url: str, local_repo: Repo) -> FetchPackResult:
client: GitClient
path: str
client, path = get_transport_and_path(source_url)
return client.fetch(
path, local_repo, determine_wants=local_repo.object_store.determine_wants_all
)
@pytest.fixture
def remote_refs(_remote_refs: FetchPackResult) -> FetchPackResult:
return deepcopy(_remote_refs)
@pytest.fixture(scope="module")
def remote_default_ref(_remote_refs: FetchPackResult) -> bytes:
return _remote_refs.symrefs[b"HEAD"]
@pytest.fixture(scope="module")
def remote_default_branch(remote_default_ref: bytes) -> str:
return remote_default_ref.decode("utf-8").replace("refs/heads/", "")
def test_git_clone_default_branch_head(
source_url: str, remote_refs: FetchPackResult, remote_default_ref: bytes
):
with Git.clone(url=source_url) as repo:
assert remote_refs.refs[remote_default_ref] == repo.head()
def test_git_clone_fails_for_non_existent_branch(source_url: str):
branch = uuid.uuid4().hex
with pytest.raises(PoetrySimpleConsoleException) as e:
Git.clone(url=source_url, branch=branch)
assert f"Failed to clone {source_url} at '{branch}'" in str(e.value)
def test_git_clone_fails_for_non_existent_revision(source_url: str):
revision = sha1(uuid.uuid4().bytes).hexdigest()
with pytest.raises(PoetrySimpleConsoleException) as e:
Git.clone(url=source_url, revision=revision)
assert f"Failed to clone {source_url} at '{revision}'" in str(e.value)
def assert_version(repo: Repo, expected_revision: str) -> None:
version = PyProjectTOML(
path=Path(repo.path).joinpath("pyproject.toml")
).poetry_config["version"]
revision = Git.get_revision(repo=repo)
assert revision == expected_revision
assert revision in REVISION_TO_VERSION_MAP
assert version == REVISION_TO_VERSION_MAP[revision]
def test_git_clone_when_branch_is_ref(source_url: str) -> None:
with Git.clone(url=source_url, branch="refs/heads/0.1") as repo:
assert_version(repo, BRANCH_TO_REVISION_MAP["0.1"])
@pytest.mark.parametrize("branch", [*BRANCH_TO_REVISION_MAP.keys()])
def test_git_clone_branch(
source_url: str, remote_refs: FetchPackResult, branch: str
) -> None:
with Git.clone(url=source_url, branch=branch) as repo:
assert_version(repo, BRANCH_TO_REVISION_MAP[branch])
@pytest.mark.parametrize("tag", [*TAG_TO_REVISION_MAP.keys()])
def test_git_clone_tag(source_url: str, remote_refs: FetchPackResult, tag: str) -> None:
with Git.clone(url=source_url, tag=tag) as repo:
assert_version(repo, TAG_TO_REVISION_MAP[tag])
def test_git_clone_multiple_times(
source_url: str, remote_refs: FetchPackResult
) -> None:
for revision in REVISION_TO_VERSION_MAP:
with Git.clone(url=source_url, revision=revision) as repo:
assert_version(repo, revision)
def test_git_clone_revision_is_branch(
source_url: str, remote_refs: FetchPackResult
) -> None:
with Git.clone(url=source_url, revision="0.1") as repo:
assert_version(repo, BRANCH_TO_REVISION_MAP["0.1"])
def test_git_clone_revision_is_ref(
source_url: str, remote_refs: FetchPackResult
) -> None:
with Git.clone(url=source_url, revision="refs/heads/0.1") as repo:
assert_version(repo, BRANCH_TO_REVISION_MAP["0.1"])
@pytest.mark.parametrize(
("revision", "expected_revision"),
[
("0.1", BRANCH_TO_REVISION_MAP["0.1"]),
("v0.1.0", TAG_TO_REVISION_MAP["v0.1.0"]),
*zip(REVISION_TO_VERSION_MAP, REVISION_TO_VERSION_MAP),
],
)
def test_git_clone_revision_is_tag(
source_url: str, remote_refs: FetchPackResult, revision: str, expected_revision: str
) -> None:
with Git.clone(url=source_url, revision=revision) as repo:
assert_version(repo, expected_revision)
def test_git_clone_clones_submodules(source_url: str) -> None:
with Git.clone(url=source_url) as repo:
submodule_package_directory = (
Path(repo.path) / "submodules" / "sample-namespace-packages"
)
assert submodule_package_directory.exists()
assert submodule_package_directory.joinpath("README.md").exists()
assert len(list(submodule_package_directory.glob("*"))) > 1
def test_system_git_fallback_on_http_401(
mocker: MockerFixture, source_url: str
) -> None:
spy = mocker.spy(Git, "_clone_legacy")
mocker.patch.object(Git, "_clone", side_effect=HTTPUnauthorized(None, None))
with Git.clone(url=source_url, branch="0.1") as repo:
path = Path(repo.path)
assert_version(repo, BRANCH_TO_REVISION_MAP["0.1"])
spy.assert_called_with(
url="https://github.com/python-poetry/test-fixture-vcs-repository.git",
target=path,
refspec=GitRefSpec(branch="0.1", revision=None, tag=None, ref=b"HEAD"),
)
spy.assert_called_once()
tests/puzzle/conftest.py
from __future__ import annotations from __future__ import annotations
import shutil
from pathlib import Path
from typing import TYPE_CHECKING from typing import TYPE_CHECKING
import pytest import pytest
from tests.helpers import mock_clone
try:
import urllib.parse as urlparse
except ImportError:
import urlparse
if TYPE_CHECKING: if TYPE_CHECKING:
from poetry.core.vcs import Git
from pytest_mock import MockerFixture from pytest_mock import MockerFixture
def mock_clone(self: Git, source: str, dest: Path) -> None:
# Checking source to determine which folder we need to copy
parts = urlparse.urlparse(source)
folder = (
Path(__file__).parent.parent
/ "fixtures"
/ "git"
/ parts.netloc
/ parts.path.lstrip("/").rstrip(".git")
)
shutil.rmtree(str(dest))
shutil.copytree(str(folder), str(dest))
@pytest.fixture(autouse=True) @pytest.fixture(autouse=True)
def setup(mocker: MockerFixture) -> None: def setup(mocker: MockerFixture) -> None:
# Patch git module to not actually clone projects # Patch git module to not actually clone projects
mocker.patch("poetry.core.vcs.git.Git.clone", new=mock_clone) mocker.patch("poetry.vcs.git.Git.clone", new=mock_clone)
mocker.patch("poetry.core.vcs.git.Git.checkout", new=lambda *_: None) p = mocker.patch("poetry.vcs.git.Git.get_revision")
p = mocker.patch("poetry.core.vcs.git.Git.rev_parse")
p.return_value = "9cf87a285a2d3fbb0b9fa621997b3acc3631ed24" p.return_value = "9cf87a285a2d3fbb0b9fa621997b3acc3631ed24"
tests/repositories/test_installed_repository.py
from __future__ import annotations from __future__ import annotations
from collections import namedtuple
from pathlib import Path from pathlib import Path
from typing import TYPE_CHECKING from typing import TYPE_CHECKING
...@@ -71,15 +72,11 @@ def repository(mocker: MockerFixture, env: MockEnv) -> InstalledRepository: ...@@ -71,15 +72,11 @@ def repository(mocker: MockerFixture, env: MockEnv) -> InstalledRepository:
return_value=INSTALLED_RESULTS, return_value=INSTALLED_RESULTS,
) )
mocker.patch( mocker.patch(
"poetry.core.vcs.git.Git.rev_parse", "poetry.vcs.git.Git.info",
return_value="bb058f6b78b2d28ef5d9a5e759cfa179a1a713d6", return_value=namedtuple("GitRepoLocalInfo", "origin revision")(
) origin="https://github.com/sdispater/pendulum.git",
mocker.patch( revision="bb058f6b78b2d28ef5d9a5e759cfa179a1a713d6",
"poetry.core.vcs.git.Git.remote_urls", ),
side_effect=[
{"remote.origin.url": "https://github.com/sdispater/pendulum.git"},
{"remote.origin.url": "git@github.com:sdispater/pendulum.git"},
],
) )
mocker.patch("poetry.repositories.installed_repository._VENDORS", str(VENDOR_DIR)) mocker.patch("poetry.repositories.installed_repository._VENDORS", str(VENDOR_DIR))
return InstalledRepository.load(env) return InstalledRepository.load(env)
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment