Skip to content

P
python-poetry
Commit e4e8e3cb by Sébastien Eustace Committed by GitHub
Options

Fix resolution of path, url and VCS dependencies (#2398) 

* Fix resolution of path, url and VCS dependencies

* Fix editable installation of Poetry packages
parent 7ee66c0a
Showing with 353 additions and 41 deletions
poetry/installation/pip_installer.py
import os import os
import tempfile import tempfile
from io import open
from subprocess import CalledProcessError from subprocess import CalledProcessError
from clikit.api.io import IO from clikit.api.io import IO
...@@ -181,9 +180,7 @@ class PipInstaller(BaseInstaller): ...@@ -181,9 +180,7 @@ class PipInstaller(BaseInstaller):
return name return name
def install_directory(self, package): def install_directory(self, package):
from poetry.masonry.builder import SdistBuilder
from poetry.factory import Factory from poetry.factory import Factory
from poetry.utils._compat import decode
from poetry.utils.env import NullEnv from poetry.utils.env import NullEnv
from poetry.utils.toml_file import TomlFile from poetry.utils.toml_file import TomlFile
...@@ -210,17 +207,20 @@ class PipInstaller(BaseInstaller): ...@@ -210,17 +207,20 @@ class PipInstaller(BaseInstaller):
setup = os.path.join(req, "setup.py") setup = os.path.join(req, "setup.py")
has_setup = os.path.exists(setup) has_setup = os.path.exists(setup)
if not has_setup and has_poetry and (package.develop or not has_build_system): if has_poetry and (package.develop or not has_build_system):
# We actually need to rely on creating a temporary setup.py # We actually need to rely on creating a temporary setup.py
# file since pip, as of this comment, does not support # file since pip, as of this comment, does not support
# build-system for editable packages # build-system for editable packages
# We also need it for non-PEP-517 packages # We also need it for non-PEP-517 packages
builder = SdistBuilder( from poetry.masonry.builders.editable import EditableBuilder
builder = EditableBuilder(
Factory().create_poetry(pyproject.parent), NullEnv(), NullIO() Factory().create_poetry(pyproject.parent), NullEnv(), NullIO()
) )
with open(setup, "w", encoding="utf-8") as f: builder.build()
f.write(decode(builder.build_setup()))
return
if package.develop: if package.develop:
args.append("-e") args.append("-e")
......
poetry/mixology/version_solver.py
...@@ -339,6 +339,16 @@ class VersionSolver: ...@@ -339,6 +339,16 @@ class VersionSolver:
if dependency.name in self._locked: if dependency.name in self._locked:
return 1 return 1
# VCS, URL, File or Directory dependencies
# represent a single version
if (
dependency.is_vcs()
or dependency.is_url()
or dependency.is_file()
or dependency.is_directory()
):
return 1
try: try:
return len(self._provider.search_for(dependency)) return len(self._provider.search_for(dependency))
except ValueError: except ValueError:
......
poetry/packages/directory_dependency.py
...@@ -79,3 +79,14 @@ class DirectoryDependency(Dependency): ...@@ -79,3 +79,14 @@ class DirectoryDependency(Dependency):
def is_directory(self): def is_directory(self):
return True return True
def __str__(self):
if self.is_root:
return self._pretty_name
return "{} ({} {})".format(
self._pretty_name, self._pretty_constraint, self._path
)
def __hash__(self):
return hash((self._name, self._full_path))
poetry/packages/file_dependency.py
...@@ -42,6 +42,10 @@ class FileDependency(Dependency): ...@@ -42,6 +42,10 @@ class FileDependency(Dependency):
) )
@property @property
def base(self):
return self._base
@property
def path(self): def path(self):
return self._path return self._path
...@@ -59,3 +63,14 @@ class FileDependency(Dependency): ...@@ -59,3 +63,14 @@ class FileDependency(Dependency):
h.update(content) h.update(content)
return h.hexdigest() return h.hexdigest()
def __str__(self):
if self.is_root:
return self._pretty_name
return "{} ({} {})".format(
self._pretty_name, self._pretty_constraint, self._path
)
def __hash__(self):
return hash((self._name, self._full_path))
poetry/packages/package.py
...@@ -409,6 +409,7 @@ class Package(object): ...@@ -409,6 +409,7 @@ class Package(object):
def clone(self): # type: () -> Package def clone(self): # type: () -> Package
clone = self.__class__(self.pretty_name, self.version) clone = self.__class__(self.pretty_name, self.version)
clone.description = self.description
clone.category = self.category clone.category = self.category
clone.optional = self.optional clone.optional = self.optional
clone.python_versions = self.python_versions clone.python_versions = self.python_versions
......
poetry/packages/url_dependency.py
...@@ -38,3 +38,9 @@ class URLDependency(Dependency): ...@@ -38,3 +38,9 @@ class URLDependency(Dependency):
def is_url(self): # type: () -> bool def is_url(self): # type: () -> bool
return True return True
def __str__(self):
return "{} ({} url)".format(self._pretty_name, self._pretty_constraint)
def __hash__(self):
return hash((self._name, self._url))
poetry/packages/vcs_dependency.py
...@@ -94,3 +94,11 @@ class VCSDependency(Dependency): ...@@ -94,3 +94,11 @@ class VCSDependency(Dependency):
def accepts_prereleases(self): # type: () -> bool def accepts_prereleases(self): # type: () -> bool
return True return True
def __str__(self):
return "{} ({} {})".format(
self._pretty_name, self._pretty_constraint, self._vcs
)
def __hash__(self):
return hash((self._name, self._vcs, self._branch, self._tag, self._rev))
poetry/puzzle/provider.py
...@@ -72,6 +72,7 @@ class Provider: ...@@ -72,6 +72,7 @@ class Provider:
self._search_for = {} self._search_for = {}
self._is_debugging = self._io.is_debug() or self._io.is_very_verbose() self._is_debugging = self._io.is_debug() or self._io.is_very_verbose()
self._in_progress = False self._in_progress = False
self._deferred_cache = {}
@property @property
def pool(self): # type: () -> Pool def pool(self): # type: () -> Pool
...@@ -164,6 +165,9 @@ class Provider: ...@@ -164,6 +165,9 @@ class Provider:
Basically, we clone the repository in a temporary directory Basically, we clone the repository in a temporary directory
and get the information we need by checking out the specified reference. and get the information we need by checking out the specified reference.
""" """
if dependency in self._deferred_cache:
return [self._deferred_cache[dependency]]
package = self.get_package_from_vcs( package = self.get_package_from_vcs(
dependency.vcs, dependency.vcs,
dependency.source, dependency.source,
...@@ -178,6 +182,11 @@ class Provider: ...@@ -178,6 +182,11 @@ class Provider:
package.requires += package.extras[extra] package.requires += package.extras[extra]
dependency._constraint = package.version
dependency._pretty_constraint = package.version.text
self._deferred_cache[dependency] = package
return [package] return [package]
@classmethod @classmethod
...@@ -214,7 +223,17 @@ class Provider: ...@@ -214,7 +223,17 @@ class Provider:
return package return package
def search_for_file(self, dependency): # type: (FileDependency) -> List[Package] def search_for_file(self, dependency): # type: (FileDependency) -> List[Package]
package = self.get_package_from_file(dependency.full_path) if dependency in self._deferred_cache:
dependency, _package = self._deferred_cache[dependency]
package = _package.clone()
else:
package = self.get_package_from_file(dependency.full_path)
dependency._constraint = package.version
dependency._pretty_constraint = package.version.text
self._deferred_cache[dependency] = (dependency, package)
if dependency.name != package.name: if dependency.name != package.name:
# For now, the dependency's name must match the actual package's name # For now, the dependency's name must match the actual package's name
...@@ -224,6 +243,9 @@ class Provider: ...@@ -224,6 +243,9 @@ class Provider:
) )
) )
if dependency.base is not None:
package.root_dir = dependency.base
package.source_url = dependency.path.as_posix() package.source_url = dependency.path.as_posix()
package.files = [ package.files = [
{"file": dependency.path.name, "hash": "sha256:" + dependency.hash()} {"file": dependency.path.name, "hash": "sha256:" + dependency.hash()}
...@@ -270,15 +292,25 @@ class Provider: ...@@ -270,15 +292,25 @@ class Provider:
def search_for_directory( def search_for_directory(
self, dependency self, dependency
): # type: (DirectoryDependency) -> List[Package] ): # type: (DirectoryDependency) -> List[Package]
package = self.get_package_from_directory( if dependency in self._deferred_cache:
dependency.full_path, name=dependency.name dependency, _package = self._deferred_cache[dependency]
)
package = _package.clone()
else:
package = self.get_package_from_directory(
dependency.full_path, name=dependency.name
)
dependency._constraint = package.version
dependency._pretty_constraint = package.version.text
self._deferred_cache[dependency] = (dependency, package)
package.source_url = dependency.path.as_posix() package.source_url = dependency.path.as_posix()
package.develop = dependency.develop package.develop = dependency.develop
if dependency.base is not None: if dependency.base is not None:
package.root_dir = dependency.base.as_posix() package.root_dir = dependency.base
for extra in dependency.extras: for extra in dependency.extras:
if extra in package.extras: if extra in package.extras:
...@@ -434,6 +466,9 @@ class Provider: ...@@ -434,6 +466,9 @@ class Provider:
return package return package
def search_for_url(self, dependency): # type: (URLDependency) -> List[Package] def search_for_url(self, dependency): # type: (URLDependency) -> List[Package]
if dependency in self._deferred_cache:
return [self._deferred_cache[dependency]]
package = self.get_package_from_url(dependency.url) package = self.get_package_from_url(dependency.url)
if dependency.name != package.name: if dependency.name != package.name:
...@@ -451,6 +486,11 @@ class Provider: ...@@ -451,6 +486,11 @@ class Provider:
package.requires += package.extras[extra] package.requires += package.extras[extra]
dependency._constraint = package.version
dependency._pretty_constraint = package.version.text
self._deferred_cache[dependency] = package
return [package] return [package]
@classmethod @classmethod
...@@ -551,6 +591,17 @@ class Provider: ...@@ -551,6 +591,17 @@ class Provider:
else: else:
requires = package.requires requires = package.requires
# Retrieving constraints for deferred dependencies
for r in requires:
if r.is_directory():
self.search_for_directory(r)
elif r.is_file():
self.search_for_file(r)
elif r.is_vcs():
self.search_for_vcs(r)
elif r.is_url():
self.search_for_url(r)
dependencies = [ dependencies = [
r r
for r in requires for r in requires
...@@ -696,15 +747,15 @@ class Provider: ...@@ -696,15 +747,15 @@ class Provider:
if (package.dependency.is_directory() or package.dependency.is_file()) and ( if (package.dependency.is_directory() or package.dependency.is_file()) and (
dep.is_directory() or dep.is_file() dep.is_directory() or dep.is_file()
): ):
if dep.path.as_posix().startswith(package.source_url): relative_path = Path(
relative = (Path(package.source_url) / dep.path).relative_to( os.path.relpath(
package.source_url dep.full_path.as_posix(), package.root_dir.as_posix()
) )
else: )
relative = Path(package.source_url) / dep.path
# TODO: Improve the way we set the correct relative path for dependencies # TODO: Improve the way we set the correct relative path for dependencies
dep._path = relative dep._path = relative_path
clean_dependencies.append(dep) clean_dependencies.append(dep)
package.requires = clean_dependencies package.requires = clean_dependencies
......
tests/fixtures/directory/project_with_transitive_directory_dependencies/pyproject.toml
...@@ -8,5 +8,6 @@ license = "MIT" ...@@ -8,5 +8,6 @@ license = "MIT"
[tool.poetry.dependencies] [tool.poetry.dependencies]
python = "*" python = "*"
project-with-extras = {path = "../../project_with_extras/"} project-with-extras = {path = "../../project_with_extras/"}
project-with-transitive-file-dependencies = {path = "../project_with_transitive_file_dependencies/"}
[tool.poetry.dev-dependencies] [tool.poetry.dev-dependencies]
tests/fixtures/directory/project_with_transitive_file_dependencies/inner-directory-project/pyproject.toml 0 → 100644
[tool.poetry]
name = "inner-directory-project"
version = "1.2.4"
description = "This is a description"
authors = ["Your Name <you@example.com>"]
license = "MIT"
[tool.poetry.dependencies]
python = "*"
[tool.poetry.dev-dependencies]
tests/fixtures/directory/project_with_transitive_file_dependencies/pyproject.toml
...@@ -8,5 +8,6 @@ license = "MIT" ...@@ -8,5 +8,6 @@ license = "MIT"
[tool.poetry.dependencies] [tool.poetry.dependencies]
python = "*" python = "*"
demo = {path = "../../distributions/demo-0.1.0-py2.py3-none-any.whl"} demo = {path = "../../distributions/demo-0.1.0-py2.py3-none-any.whl"}
inner-directory-project = {path = "./inner-directory-project"}
[tool.poetry.dev-dependencies] [tool.poetry.dev-dependencies]
tests/installation/fixtures/with-directory-dependency-poetry-transitive.test
[[package]] [[package]]
category = "main" category = "main"
description = "" description = ""
name = "demo"
optional = false
python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*"
version = "0.1.0"
[package.dependencies]
pendulum = ">=1.4.4"
[package.extras]
bar = ["tomlkit"]
foo = ["cleo"]
[package.source]
reference = ""
type = "file"
url = "../../distributions/demo-0.1.0-py2.py3-none-any.whl"
[[package]]
category = "main"
description = ""
develop = true
name = "inner-directory-project"
optional = false
python-versions = "*"
version = "1.2.4"
[package.source]
reference = ""
type = "directory"
url = "../project_with_transitive_file_dependencies/inner-directory-project"
[[package]]
category = "main"
description = ""
name = "pendulum"
optional = false
python-versions = "*"
version = "1.4.4"
[[package]]
category = "main"
description = ""
develop = true develop = true
name = "project-with-extras" name = "project-with-extras"
optional = false optional = false
...@@ -14,7 +56,7 @@ extras_b = ["cachy (>=0.2.0)"] ...@@ -14,7 +56,7 @@ extras_b = ["cachy (>=0.2.0)"]
[package.source] [package.source]
reference = "" reference = ""
type = "directory" type = "directory"
url = "tests/fixtures/directory/project_with_transitive_directory_dependencies/../../project_with_extras" url = "../project_with_extras"
[[package]] [[package]]
category = "main" category = "main"
...@@ -26,17 +68,42 @@ python-versions = "*" ...@@ -26,17 +68,42 @@ python-versions = "*"
version = "1.2.3" version = "1.2.3"
[package.dependencies] [package.dependencies]
project-with-extras = "*" project-with-extras = "1.2.3"
project-with-transitive-file-dependencies = "1.2.3"
[package.source]
reference = ""
type = "directory"
url = "project_with_transitive_directory_dependencies"
[[package]]
category = "main"
description = ""
develop = true
name = "project-with-transitive-file-dependencies"
optional = false
python-versions = "*"
version = "1.2.3"
[package.dependencies]
demo = "0.1.0"
inner-directory-project = "1.2.4"
[package.source] [package.source]
reference = "" reference = ""
type = "directory" type = "directory"
url = "tests/fixtures/directory/project_with_transitive_directory_dependencies" url = "project_with_transitive_file_dependencies"
[metadata] [metadata]
content-hash = "123456789" content-hash = "123456789"
python-versions = "*" python-versions = "*"
[metadata.files] [metadata.files]
demo = [
{file = "demo-0.1.0-py2.py3-none-any.whl", hash = "sha256:70e704135718fffbcbf61ed1fc45933cfd86951a744b681000eaaa75da31f17a"},
]
inner-directory-project = []
pendulum = []
project-with-extras = [] project-with-extras = []
project-with-transitive-directory-dependencies = [] project-with-transitive-directory-dependencies = []
project-with-transitive-file-dependencies = []
tests/installation/fixtures/with-file-dependency-transitive.test
...@@ -16,7 +16,21 @@ foo = ["cleo"] ...@@ -16,7 +16,21 @@ foo = ["cleo"]
[package.source] [package.source]
reference = "" reference = ""
type = "file" type = "file"
url = "tests/fixtures/directory/project_with_transitive_file_dependencies/../../distributions/demo-0.1.0-py2.py3-none-any.whl" url = "../distributions/demo-0.1.0-py2.py3-none-any.whl"
[[package]]
category = "main"
description = ""
develop = true
name = "inner-directory-project"
optional = false
python-versions = "*"
version = "1.2.4"
[package.source]
reference = ""
type = "directory"
url = "project_with_transitive_file_dependencies/inner-directory-project"
[[package]] [[package]]
category = "main" category = "main"
...@@ -36,12 +50,13 @@ python-versions = "*" ...@@ -36,12 +50,13 @@ python-versions = "*"
version = "1.2.3" version = "1.2.3"
[package.dependencies] [package.dependencies]
demo = "*" demo = "0.1.0"
inner-directory-project = "1.2.4"
[package.source] [package.source]
reference = "" reference = ""
type = "directory" type = "directory"
url = "tests/fixtures/directory/project_with_transitive_file_dependencies" url = "project_with_transitive_file_dependencies"
[metadata] [metadata]
content-hash = "123456789" content-hash = "123456789"
...@@ -51,5 +66,6 @@ python-versions = "*" ...@@ -51,5 +66,6 @@ python-versions = "*"
demo = [ demo = [
{file = "demo-0.1.0-py2.py3-none-any.whl", hash = "sha256:70e704135718fffbcbf61ed1fc45933cfd86951a744b681000eaaa75da31f17a"}, {file = "demo-0.1.0-py2.py3-none-any.whl", hash = "sha256:70e704135718fffbcbf61ed1fc45933cfd86951a744b681000eaaa75da31f17a"},
] ]
inner-directory-project = []
pendulum = [] pendulum = []
project-with-transitive-file-dependencies = [] project-with-transitive-file-dependencies = []
tests/installation/test_installer.py
...@@ -716,11 +716,13 @@ def test_run_installs_with_local_poetry_directory_and_extras( ...@@ -716,11 +716,13 @@ def test_run_installs_with_local_poetry_directory_and_extras(
def test_run_installs_with_local_poetry_directory_transitive( def test_run_installs_with_local_poetry_directory_transitive(
installer, locker, repo, package, tmpdir installer, locker, repo, package, tmpdir
): ):
file_path = ( package.root_dir = fixtures_dir.joinpath("directory")
fixtures_dir / "directory/project_with_transitive_directory_dependencies/" directory = fixtures_dir.joinpath("directory").joinpath(
"project_with_transitive_directory_dependencies"
) )
package.add_dependency( package.add_dependency(
"project-with-transitive-directory-dependencies", {"path": str(file_path)} "project-with-transitive-directory-dependencies",
{"path": str(directory.relative_to(fixtures_dir.joinpath("directory")))},
) )
repo.add_package(get_package("pendulum", "1.4.4")) repo.add_package(get_package("pendulum", "1.4.4"))
...@@ -732,15 +734,19 @@ def test_run_installs_with_local_poetry_directory_transitive( ...@@ -732,15 +734,19 @@ def test_run_installs_with_local_poetry_directory_transitive(
assert locker.written_data == expected assert locker.written_data == expected
assert len(installer.installer.installs) == 2 assert len(installer.installer.installs) == 6
def test_run_installs_with_local_poetry_file_transitive( def test_run_installs_with_local_poetry_file_transitive(
installer, locker, repo, package, tmpdir installer, locker, repo, package, tmpdir
): ):
file_path = fixtures_dir / "directory/project_with_transitive_file_dependencies/" package.root_dir = fixtures_dir.joinpath("directory")
directory = fixtures_dir.joinpath("directory").joinpath(
"project_with_transitive_file_dependencies"
)
package.add_dependency( package.add_dependency(
"project-with-transitive-file-dependencies", {"path": str(file_path)} "project-with-transitive-file-dependencies",
{"path": str(directory.relative_to(fixtures_dir.joinpath("directory")))},
) )
repo.add_package(get_package("pendulum", "1.4.4")) repo.add_package(get_package("pendulum", "1.4.4"))
...@@ -752,7 +758,7 @@ def test_run_installs_with_local_poetry_file_transitive( ...@@ -752,7 +758,7 @@ def test_run_installs_with_local_poetry_file_transitive(
assert locker.written_data == expected assert locker.written_data == expected
assert len(installer.installer.installs) == 3 assert len(installer.installer.installs) == 4
def test_run_installs_with_local_setuptools_directory( def test_run_installs_with_local_setuptools_directory(
......
tests/puzzle/test_provider.py
...@@ -204,16 +204,13 @@ def test_search_for_directory_setup_with_base(provider, directory): ...@@ -204,16 +204,13 @@ def test_search_for_directory_setup_with_base(provider, directory):
"foo": [get_dependency("cleo")], "foo": [get_dependency("cleo")],
"bar": [get_dependency("tomlkit")], "bar": [get_dependency("tomlkit")],
} }
assert ( assert package.root_dir == (
package.root_dir Path(__file__).parent.parent
== ( / "fixtures"
Path(__file__).parent.parent / "git"
/ "fixtures" / "github.com"
/ "git" / "demo"
/ "github.com" / directory
/ "demo"
/ directory
).as_posix()
) )
......
tests/puzzle/test_solver.py
...@@ -1928,3 +1928,114 @@ def test_solver_properly_propagates_markers(solver, repo, package): ...@@ -1928,3 +1928,114 @@ def test_solver_properly_propagates_markers(solver, repo, package):
str(ops[0].package.marker) str(ops[0].package.marker)
== 'python_version >= "3.6" and implementation_name != "pypy"' == 'python_version >= "3.6" and implementation_name != "pypy"'
) )
def test_solver_cannot_choose_another_version_for_directory_dependencies(
solver, repo, package
):
pendulum = get_package("pendulum", "2.0.3")
demo = get_package("demo", "0.1.0")
foo = get_package("foo", "1.2.3")
foo.add_dependency("demo", "<0.1.2")
repo.add_package(foo)
repo.add_package(demo)
repo.add_package(pendulum)
path = (
Path(__file__).parent.parent
/ "fixtures"
/ "git"
/ "github.com"
/ "demo"
/ "demo"
).as_posix()
package.add_dependency("demo", {"path": path})
package.add_dependency("foo", "^1.2.3")
# This is not solvable since the demo version is pinned
# via the directory dependency
with pytest.raises(SolverProblemError):
solver.solve()
def test_solver_cannot_choose_another_version_for_file_dependencies(
solver, repo, package
):
pendulum = get_package("pendulum", "2.0.3")
demo = get_package("demo", "0.0.8")
foo = get_package("foo", "1.2.3")
foo.add_dependency("demo", "<0.1.0")
repo.add_package(foo)
repo.add_package(demo)
repo.add_package(pendulum)
path = (
Path(__file__).parent.parent
/ "fixtures"
/ "distributions"
/ "demo-0.1.0-py2.py3-none-any.whl"
).as_posix()
package.add_dependency("demo", {"path": path})
package.add_dependency("foo", "^1.2.3")
# This is not solvable since the demo version is pinned
# via the file dependency
with pytest.raises(SolverProblemError):
solver.solve()
def test_solver_cannot_choose_another_version_for_git_dependencies(
solver, repo, package
):
pendulum = get_package("pendulum", "2.0.3")
demo = get_package("demo", "0.0.8")
foo = get_package("foo", "1.2.3")
foo.add_dependency("demo", "<0.1.0")
repo.add_package(foo)
repo.add_package(demo)
repo.add_package(pendulum)
package.add_dependency("demo", {"git": "https://github.com/demo/demo.git"})
package.add_dependency("foo", "^1.2.3")
# This is not solvable since the demo version is pinned
# via the file dependency
with pytest.raises(SolverProblemError):
solver.solve()
def test_solver_cannot_choose_another_version_for_url_dependencies(
solver, repo, package, http
):
path = (
Path(__file__).parent.parent
/ "fixtures"
/ "distributions"
/ "demo-0.1.0-py2.py3-none-any.whl"
)
http.register_uri(
"GET",
"https://foo.bar/demo-0.1.0-py2.py3-none-any.whl",
body=path.read_bytes(),
streaming=True,
)
pendulum = get_package("pendulum", "2.0.3")
demo = get_package("demo", "0.0.8")
foo = get_package("foo", "1.2.3")
foo.add_dependency("demo", "<0.1.0")
repo.add_package(foo)
repo.add_package(demo)
repo.add_package(pendulum)
package.add_dependency(
"demo", {"url": "https://foo.bar/distributions/demo-0.1.0-py2.py3-none-any.whl"}
)
package.add_dependency("foo", "^1.2.3")
# This is not solvable since the demo version is pinned
# via the git dependency
with pytest.raises(SolverProblemError):
solver.solve()
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment