Skip to content

P
python-poetry
Commit c7055be9 by Sébastien Eustace Committed by GitHub
Options

New and faster installer implementation (#2595) 

* Improve the way packages are installed

* Add support for parallelized operations

* Improve the Chooser class

* Make the new installer the default

* Add an Authenticator class to be able to download from protected urls

* Adapt code and tests to latest changes

* Update lock file and some dependencies

* Improve installations information and caching

* Make the final adjustments and tests for the new installer

* Use the preview version of Poetry for the CI

* Rename Executor.run() to Executor.run_pip()

* Gracefully handle errors when executing operations
parent d6289470
Showing with 1439 additions and 178 deletions
.cirrus.yml
...@@ -16,7 +16,8 @@ test_task: ...@@ -16,7 +16,8 @@ test_task:
- pkg install -y git-lite $PYPACKAGE $SQLPACKAGE - pkg install -y git-lite $PYPACKAGE $SQLPACKAGE
pip_script: pip_script:
- $PYTHON -m ensurepip - $PYTHON -m ensurepip
- $PYTHON -m pip install -U pip tox poetry - $PYTHON -m pip install -U pip tox
- $PYTHON -m pip install -U --pre poetry
- poetry config virtualenvs.in-project true - poetry config virtualenvs.in-project true
tox_script: $PYTHON -m tox -e py -- -q --junitxml=junit.xml tests tox_script: $PYTHON -m tox -e py -- -q --junitxml=junit.xml tests
on_failure: on_failure:
......
.flake8
...@@ -16,6 +16,7 @@ exclude = ...@@ -16,6 +16,7 @@ exclude =
.vscode .vscode
.github .github
poetry/utils/_compat.py poetry/utils/_compat.py
poetry/utils/env_scripts/tags.py
tests/fixtures/ tests/fixtures/
tests/repositories/fixtures/ tests/repositories/fixtures/
tests/utils/fixtures/ tests/utils/fixtures/
.github/workflows/main.yml
...@@ -49,7 +49,7 @@ jobs: ...@@ -49,7 +49,7 @@ jobs:
shell: bash shell: bash
run: | run: |
curl -fsS -o get-poetry.py https://raw.githubusercontent.com/python-poetry/poetry/master/get-poetry.py curl -fsS -o get-poetry.py https://raw.githubusercontent.com/python-poetry/poetry/master/get-poetry.py
python get-poetry.py -y python get-poetry.py -y --preview
echo "::set-env name=PATH::$HOME/.poetry/bin:$PATH" echo "::set-env name=PATH::$HOME/.poetry/bin:$PATH"
- name: Configure poetry - name: Configure poetry
......
poetry/config/config.py
...@@ -37,6 +37,7 @@ class Config(object): ...@@ -37,6 +37,7 @@ class Config(object):
"in-project": False, "in-project": False,
"path": os.path.join("{cache-dir}", "virtualenvs"), "path": os.path.join("{cache-dir}", "virtualenvs"),
}, },
"experimental": {"new-installer": True},
} }
def __init__( def __init__(
......
poetry/console/commands/add.py
from cleo import argument from cleo import argument
from cleo import option from cleo import option
from .env_command import EnvCommand
from .init import InitCommand from .init import InitCommand
from .installer_command import InstallerCommand
class AddCommand(EnvCommand, InitCommand): class AddCommand(InstallerCommand, InitCommand):
name = "add" name = "add"
description = "Adds a new dependency to <comment>pyproject.toml</>." description = "Adds a new dependency to <comment>pyproject.toml</>."
...@@ -56,7 +56,6 @@ If you do not specify a version constraint, poetry will choose a suitable one ba ...@@ -56,7 +56,6 @@ If you do not specify a version constraint, poetry will choose a suitable one ba
loggers = ["poetry.repositories.pypi_repository"] loggers = ["poetry.repositories.pypi_repository"]
def handle(self): def handle(self):
from poetry.installation.installer import Installer
from poetry.core.semver import parse_constraint from poetry.core.semver import parse_constraint
from tomlkit import inline_table from tomlkit import inline_table
...@@ -149,18 +148,17 @@ If you do not specify a version constraint, poetry will choose a suitable one ba ...@@ -149,18 +148,17 @@ If you do not specify a version constraint, poetry will choose a suitable one ba
# Update packages # Update packages
self.reset_poetry() self.reset_poetry()
installer = Installer( self._installer.set_package(self.poetry.package)
self.io, self.env, self.poetry.package, self.poetry.locker, self.poetry.pool self._installer.dry_run(self.option("dry-run"))
) self._installer.verbose(self._io.is_verbose())
self._installer.update(True)
installer.dry_run(self.option("dry-run"))
installer.update(True)
if self.option("lock"): if self.option("lock"):
installer.lock() self._installer.lock()
installer.whitelist([r["name"] for r in requirements])
self._installer.whitelist([r["name"] for r in requirements])
try: try:
status = installer.run() status = self._installer.run()
except Exception: except Exception:
self.poetry.file.write(original_content) self.poetry.file.write(original_content)
...@@ -169,10 +167,10 @@ If you do not specify a version constraint, poetry will choose a suitable one ba ...@@ -169,10 +167,10 @@ If you do not specify a version constraint, poetry will choose a suitable one ba
if status != 0 or self.option("dry-run"): if status != 0 or self.option("dry-run"):
# Revert changes # Revert changes
if not self.option("dry-run"): if not self.option("dry-run"):
self.error( self.line_error(
"\n" "\n"
"Addition failed, reverting pyproject.toml " "<error>Failed to add packages, reverting the pyproject.toml file "
"to its original content." "to its original content.</error>"
) )
self.poetry.file.write(original_content) self.poetry.file.write(original_content)
......
poetry/console/commands/config.py
...@@ -57,6 +57,11 @@ To remove a repository (repo is a short alias for repositories): ...@@ -57,6 +57,11 @@ To remove a repository (repo is a short alias for repositories):
lambda val: str(Path(val)), lambda val: str(Path(val)),
str(Path(CACHE_DIR) / "virtualenvs"), str(Path(CACHE_DIR) / "virtualenvs"),
), ),
"experimental.new-installer": (
boolean_validator,
boolean_normalizer,
True,
),
} }
return unique_config_values return unique_config_values
......
poetry/console/commands/install.py
from cleo import option from cleo import option
from .env_command import EnvCommand from .installer_command import InstallerCommand
class InstallCommand(EnvCommand): class InstallCommand(InstallerCommand):
name = "install" name = "install"
description = "Installs the project dependencies." description = "Installs the project dependencies."
...@@ -48,12 +48,11 @@ dependencies and not including the current project, run the command with the ...@@ -48,12 +48,11 @@ dependencies and not including the current project, run the command with the
_loggers = ["poetry.repositories.pypi_repository"] _loggers = ["poetry.repositories.pypi_repository"]
def handle(self): def handle(self):
from poetry.installation.installer import Installer
from poetry.masonry.builders import EditableBuilder from poetry.masonry.builders import EditableBuilder
from poetry.core.masonry.utils.module import ModuleOrPackageNotFound from poetry.core.masonry.utils.module import ModuleOrPackageNotFound
installer = Installer( self._installer.use_executor(
self.io, self.env, self.poetry.package, self.poetry.locker, self.poetry.pool self.poetry.config.get("experimental.new-installer", False)
) )
extras = [] extras = []
...@@ -63,13 +62,13 @@ dependencies and not including the current project, run the command with the ...@@ -63,13 +62,13 @@ dependencies and not including the current project, run the command with the
else: else:
extras.append(extra) extras.append(extra)
installer.extras(extras) self._installer.extras(extras)
installer.dev_mode(not self.option("no-dev")) self._installer.dev_mode(not self.option("no-dev"))
installer.dry_run(self.option("dry-run")) self._installer.dry_run(self.option("dry-run"))
installer.remove_untracked(self.option("remove-untracked")) self._installer.remove_untracked(self.option("remove-untracked"))
installer.verbose(self.option("verbose")) self._installer.verbose(self._io.is_verbose())
return_code = installer.run() return_code = self._installer.run()
if return_code != 0: if return_code != 0:
return return_code return return_code
...@@ -85,15 +84,32 @@ dependencies and not including the current project, run the command with the ...@@ -85,15 +84,32 @@ dependencies and not including the current project, run the command with the
# If this is a true error it will be picked up later by build anyway. # If this is a true error it will be picked up later by build anyway.
return 0 return 0
self.line("")
if not self._io.supports_ansi() or self.io.is_debug():
self.line( self.line(
" - Installing <c1>{}</c1> (<c2>{}</c2>)".format( "<b>Installing</> the current project: <c1>{}</c1> (<c2>{}</c2>)".format(
self.poetry.package.pretty_name, self.poetry.package.pretty_version
)
)
else:
self.write(
"<b>Installing</> the current project: <c1>{}</c1> (<c2>{}</c2>)".format(
self.poetry.package.pretty_name, self.poetry.package.pretty_version self.poetry.package.pretty_name, self.poetry.package.pretty_version
) )
) )
if self.option("dry-run"): if self.option("dry-run"):
self.line("")
return 0 return 0
builder.build() builder.build()
if self._io.supports_ansi() and not self.io.is_debug():
self.overwrite(
"<b>Installing</> the current project: <c1>{}</c1> (<success>{}</success>)".format(
self.poetry.package.pretty_name, self.poetry.package.pretty_version
)
)
self.line("")
return 0 return 0
poetry/console/commands/installer_command.py 0 → 100644
from typing import TYPE_CHECKING
from .env_command import EnvCommand
if TYPE_CHECKING:
from poetry.installation.installer import Installer
class InstallerCommand(EnvCommand):
def __init__(self):
self._installer = None
super(InstallerCommand, self).__init__()
@property
def installer(self): # type: () -> Installer
return self._installer
def set_installer(self, installer): # type: (Installer) -> None
self._installer = installer
poetry/console/commands/lock.py
from .env_command import EnvCommand from .installer_command import InstallerCommand
class LockCommand(EnvCommand): class LockCommand(InstallerCommand):
name = "lock" name = "lock"
description = "Locks the project dependencies." description = "Locks the project dependencies."
...@@ -17,12 +17,10 @@ file. ...@@ -17,12 +17,10 @@ file.
loggers = ["poetry.repositories.pypi_repository"] loggers = ["poetry.repositories.pypi_repository"]
def handle(self): def handle(self):
from poetry.installation.installer import Installer self._installer.use_executor(
self.poetry.config.get("experimental.new-installer", False)
installer = Installer(
self.io, self.env, self.poetry.package, self.poetry.locker, self.poetry.pool
) )
installer.lock() self._installer.lock()
return installer.run() return self._installer.run()
poetry/console/commands/remove.py
from cleo import argument from cleo import argument
from cleo import option from cleo import option
from .env_command import EnvCommand from .installer_command import InstallerCommand
class RemoveCommand(EnvCommand): class RemoveCommand(InstallerCommand):
name = "remove" name = "remove"
description = "Removes a package from the project dependencies." description = "Removes a package from the project dependencies."
...@@ -28,8 +28,6 @@ list of installed packages ...@@ -28,8 +28,6 @@ list of installed packages
loggers = ["poetry.repositories.pypi_repository"] loggers = ["poetry.repositories.pypi_repository"]
def handle(self): def handle(self):
from poetry.installation.installer import Installer
packages = self.argument("packages") packages = self.argument("packages")
is_dev = self.option("dev") is_dev = self.option("dev")
...@@ -62,16 +60,18 @@ list of installed packages ...@@ -62,16 +60,18 @@ list of installed packages
# Update packages # Update packages
self.reset_poetry() self.reset_poetry()
installer = Installer( self._installer.set_package(self.poetry.package)
self.io, self.env, self.poetry.package, self.poetry.locker, self.poetry.pool self._installer.use_executor(
self.poetry.config.get("experimental.new-installer", False)
) )
installer.dry_run(self.option("dry-run")) self._installer.dry_run(self.option("dry-run"))
installer.update(True) self._installer.verbose(self._io.is_verbose())
installer.whitelist(requirements) self._installer.update(True)
self._installer.whitelist(requirements)
try: try:
status = installer.run() status = self._installer.run()
except Exception: except Exception:
self.poetry.file.write(original_content) self.poetry.file.write(original_content)
...@@ -80,7 +80,7 @@ list of installed packages ...@@ -80,7 +80,7 @@ list of installed packages
if status != 0 or self.option("dry-run"): if status != 0 or self.option("dry-run"):
# Revert changes # Revert changes
if not self.option("dry-run"): if not self.option("dry-run"):
self.error( self.line_error(
"\n" "\n"
"Removal failed, reverting pyproject.toml " "Removal failed, reverting pyproject.toml "
"to its original content." "to its original content."
......
poetry/console/commands/update.py
from cleo import argument from cleo import argument
from cleo import option from cleo import option
from .env_command import EnvCommand from .installer_command import InstallerCommand
class UpdateCommand(EnvCommand): class UpdateCommand(InstallerCommand):
name = "update" name = "update"
description = ( description = (
...@@ -28,22 +28,20 @@ class UpdateCommand(EnvCommand): ...@@ -28,22 +28,20 @@ class UpdateCommand(EnvCommand):
loggers = ["poetry.repositories.pypi_repository"] loggers = ["poetry.repositories.pypi_repository"]
def handle(self): def handle(self):
from poetry.installation.installer import Installer
packages = self.argument("packages") packages = self.argument("packages")
installer = Installer( self._installer.use_executor(
self.io, self.env, self.poetry.package, self.poetry.locker, self.poetry.pool self.poetry.config.get("experimental.new-installer", False)
) )
if packages: if packages:
installer.whitelist({name: "*" for name in packages}) self._installer.whitelist({name: "*" for name in packages})
installer.dev_mode(not self.option("no-dev")) self._installer.dev_mode(not self.option("no-dev"))
installer.dry_run(self.option("dry-run")) self._installer.dry_run(self.option("dry-run"))
installer.execute_operations(not self.option("lock")) self._installer.execute_operations(not self.option("lock"))
# Force update # Force update
installer.update(True) self._installer.update(True)
return installer.run() return self._installer.run()
poetry/console/config/application_config.py
...@@ -27,6 +27,7 @@ from clikit.io.output_stream import StandardOutputStream ...@@ -27,6 +27,7 @@ from clikit.io.output_stream import StandardOutputStream
from poetry.console.commands.command import Command from poetry.console.commands.command import Command
from poetry.console.commands.env_command import EnvCommand from poetry.console.commands.env_command import EnvCommand
from poetry.console.commands.installer_command import InstallerCommand
from poetry.console.logging.io_formatter import IOFormatter from poetry.console.logging.io_formatter import IOFormatter
from poetry.console.logging.io_handler import IOHandler from poetry.console.logging.io_handler import IOHandler
from poetry.utils._compat import PY36 from poetry.utils._compat import PY36
...@@ -37,15 +38,22 @@ class ApplicationConfig(BaseApplicationConfig): ...@@ -37,15 +38,22 @@ class ApplicationConfig(BaseApplicationConfig):
super(ApplicationConfig, self).configure() super(ApplicationConfig, self).configure()
self.add_style(Style("c1").fg("cyan")) self.add_style(Style("c1").fg("cyan"))
self.add_style(Style("c2").fg("green")) self.add_style(Style("c2").fg("default").bold())
self.add_style(Style("info").fg("blue")) self.add_style(Style("info").fg("blue"))
self.add_style(Style("comment").fg("green")) self.add_style(Style("comment").fg("green"))
self.add_style(Style("error").fg("red").bold()) self.add_style(Style("error").fg("red").bold())
self.add_style(Style("warning").fg("yellow").bold()) self.add_style(Style("warning").fg("yellow").bold())
self.add_style(Style("debug").fg("default").dark()) self.add_style(Style("debug").fg("default").dark())
self.add_style(Style("success").fg("green"))
# Dark variants
self.add_style(Style("c1_dark").fg("cyan").dark())
self.add_style(Style("c2_dark").fg("default").bold().dark())
self.add_style(Style("success_dark").fg("green").dark())
self.add_event_listener(PRE_HANDLE, self.register_command_loggers) self.add_event_listener(PRE_HANDLE, self.register_command_loggers)
self.add_event_listener(PRE_HANDLE, self.set_env) self.add_event_listener(PRE_HANDLE, self.set_env)
self.add_event_listener(PRE_HANDLE, self.set_installer)
if PY36: if PY36:
from poetry.mixology.solutions.providers import ( from poetry.mixology.solutions.providers import (
...@@ -93,6 +101,9 @@ class ApplicationConfig(BaseApplicationConfig): ...@@ -93,6 +101,9 @@ class ApplicationConfig(BaseApplicationConfig):
if not isinstance(command, EnvCommand): if not isinstance(command, EnvCommand):
return return
if command.env is not None:
return
io = event.io io = event.io
poetry = command.poetry poetry = command.poetry
...@@ -104,6 +115,32 @@ class ApplicationConfig(BaseApplicationConfig): ...@@ -104,6 +115,32 @@ class ApplicationConfig(BaseApplicationConfig):
command.set_env(env) command.set_env(env)
def set_installer(
self, event, event_name, _
): # type: (PreHandleEvent, str, Any) -> None
command = event.command.config.handler # type: InstallerCommand
if not isinstance(command, InstallerCommand):
return
# If the command already has an installer
# we skip this step
if command.installer is not None:
return
from poetry.installation.installer import Installer
poetry = command.poetry
installer = Installer(
event.io,
command.env,
poetry.package,
poetry.locker,
poetry.pool,
poetry.config,
)
installer.use_executor(poetry.config.get("experimental.new-installer", False))
command.set_installer(installer)
def resolve_help_command( def resolve_help_command(
self, event, event_name, dispatcher self, event, event_name, dispatcher
): # type: (PreResolveEvent, str, EventDispatcher) -> None ): # type: (PreResolveEvent, str, EventDispatcher) -> None
......
poetry/installation/authenticator.py 0 → 100644
from typing import TYPE_CHECKING
from poetry.utils._compat import urlparse
from poetry.utils.password_manager import PasswordManager
if TYPE_CHECKING:
from typing import Any
from typing import Optional
from typing import Tuple
from clikit.api.io import IO
from requests import Request # noqa
from requests import Response # noqa
from requests import Session # noqa
from poetry.config.config import Config
class Authenticator(object):
def __init__(self, config, io): # type: (Config, IO) -> None
self._config = config
self._io = io
self._session = None
self._credentials = {}
self._password_manager = PasswordManager(self._config)
@property
def session(self): # type: () -> Session
from requests import Session # noqa
if self._session is None:
self._session = Session()
return self._session
def request(self, method, url, **kwargs): # type: (str, str, Any) -> Response
from requests import Request # noqa
from requests.auth import HTTPBasicAuth
request = Request(method, url)
username, password = self._get_credentials_for_url(url)
if username is not None and password is not None:
request = HTTPBasicAuth(username, password)(request)
session = self.session
prepared_request = session.prepare_request(request)
proxies = kwargs.get("proxies", {})
stream = kwargs.get("stream")
verify = kwargs.get("verify")
cert = kwargs.get("cert")
settings = session.merge_environment_settings(
prepared_request.url, proxies, stream, verify, cert
)
# Send the request.
send_kwargs = {
"timeout": kwargs.get("timeout"),
"allow_redirects": kwargs.get("allow_redirects", True),
}
send_kwargs.update(settings)
resp = session.send(prepared_request, **send_kwargs)
resp.raise_for_status()
return resp
def _get_credentials_for_url(
self, url
): # type: (str) -> Tuple[Optional[str], Optional[str]]
parsed_url = urlparse.urlsplit(url)
netloc = parsed_url.netloc
credentials = self._credentials.get(netloc, (None, None))
if credentials == (None, None):
if "@" not in netloc:
credentials = self._get_credentials_for_netloc_from_config(netloc)
else:
# Split from the right because that's how urllib.parse.urlsplit()
# behaves if more than one @ is present (which can be checked using
# the password attribute of urlsplit()'s return value).
auth, netloc = netloc.rsplit("@", 1)
if ":" in auth:
# Split from the left because that's how urllib.parse.urlsplit()
# behaves if more than one : is present (which again can be checked
# using the password attribute of the return value)
credentials = auth.split(":", 1)
else:
credentials = auth, None
credentials = tuple(
None if x is None else urlparse.unquote(x) for x in credentials
)
if credentials[0] is not None or credentials[1] is not None:
credentials = (credentials[0] or "", credentials[1] or "")
self._credentials[netloc] = credentials
return credentials[0], credentials[1]
def _get_credentials_for_netloc_from_config(
self, netloc
): # type: (str) -> Tuple[Optional[str], Optional[str]]
credentials = (None, None)
for repository_name in self._config.get("http-basic", {}):
repository_config = self._config.get(
"repositories.{}".format(repository_name)
)
if not repository_config:
continue
url = repository_config.get("url")
if not url:
continue
parsed_url = urlparse.urlsplit(url)
if netloc == parsed_url.netloc:
auth = self._password_manager.get_http_auth(repository_name)
if auth is None:
continue
return auth["username"], auth["password"]
return credentials
poetry/installation/chef.py 0 → 100644
import hashlib
import json
from typing import TYPE_CHECKING
from poetry.core.packages.utils.link import Link
from poetry.utils._compat import Path
from .chooser import InvalidWheelName
from .chooser import Wheel
if TYPE_CHECKING:
from typing import List
from typing import Optional
from poetry.config.config import Config
from poetry.utils.env import Env
class Chef:
def __init__(self, config, env): # type: (Config, Env) -> None
self._config = config
self._env = env
self._cache_dir = (
Path(config.get("cache-dir")).expanduser().joinpath("artifacts")
)
def prepare(self, archive): # type: (Path) -> Path
return archive
def prepare_sdist(self, archive): # type: (Path) -> Path
return archive
def prepare_wheel(self, archive): # type: (Path) -> Path
return archive
def should_prepare(self, archive): # type: (Path) -> bool
return not self.is_wheel(archive)
def is_wheel(self, archive): # type: (Path) -> bool
return archive.suffix == ".whl"
def get_cached_archive_for_link(self, link): # type: (Link) -> Optional[Link]
# If the archive is already a wheel, there is no need to cache it.
if link.is_wheel:
pass
archives = self.get_cached_archives_for_link(link)
if not archives:
return link
candidates = []
for archive in archives:
if not archive.is_wheel:
candidates.append((float("inf"), archive))
continue
try:
wheel = Wheel(archive.filename)
except InvalidWheelName:
continue
if not wheel.is_supported_by_environment(self._env):
continue
candidates.append(
(wheel.get_minimum_supported_index(self._env.supported_tags), archive),
)
if not candidates:
return link
return min(candidates)[1]
def get_cached_archives_for_link(self, link): # type: (Link) -> List[Link]
cache_dir = self.get_cache_directory_for_link(link)
archive_types = ["whl", "tar.gz", "tar.bz2", "bz2", "zip"]
links = []
for archive_type in archive_types:
for archive in cache_dir.glob("*.{}".format(archive_type)):
links.append(Link("file://{}".format(str(archive))))
return links
def get_cache_directory_for_link(self, link): # type: (Link) -> Path
key_parts = {"url": link.url_without_fragment}
if link.hash_name is not None and link.hash is not None:
key_parts[link.hash_name] = link.hash
if link.subdirectory_fragment:
key_parts["subdirectory"] = link.subdirectory_fragment
key_parts["interpreter_name"] = self._env.marker_env["interpreter_name"]
key_parts["interpreter_version"] = "".join(
self._env.marker_env["interpreter_version"].split(".")[:2]
)
key = hashlib.sha256(
json.dumps(
key_parts, sort_keys=True, separators=(",", ":"), ensure_ascii=True
).encode("ascii")
).hexdigest()
split_key = [key[:2], key[2:4], key[4:6], key[6:]]
return self._cache_dir.joinpath(*split_key)
poetry/installation/chooser.py 0 → 100644
import re
from typing import List
from typing import Tuple
from packaging.tags import Tag
from poetry.core.packages.package import Package
from poetry.core.packages.utils.link import Link
from poetry.repositories.pool import Pool
from poetry.utils.env import Env
from poetry.utils.patterns import wheel_file_re
class InvalidWheelName(Exception):
pass
class Wheel(object):
def __init__(self, filename): # type: (str) -> None
wheel_info = wheel_file_re.match(filename)
if not wheel_info:
raise InvalidWheelName("{} is not a valid wheel filename.".format(filename))
self.filename = filename
self.name = wheel_info.group("name").replace("_", "-")
self.version = wheel_info.group("ver").replace("_", "-")
self.build_tag = wheel_info.group("build")
self.pyversions = wheel_info.group("pyver").split(".")
self.abis = wheel_info.group("abi").split(".")
self.plats = wheel_info.group("plat").split(".")
self.tags = {
Tag(x, y, z) for x in self.pyversions for y in self.abis for z in self.plats
}
def get_minimum_supported_index(self, tags):
indexes = [tags.index(t) for t in self.tags if t in tags]
return min(indexes) if indexes else None
def is_supported_by_environment(self, env):
return bool(set(env.supported_tags).intersection(self.tags))
class Chooser:
"""
A Chooser chooses an appropriate release archive for packages.
"""
def __init__(self, pool, env): # type: (Pool, Env) -> None
self._pool = pool
self._env = env
def choose_for(self, package): # type: (Package) -> Link
"""
Return the url of the selected archive for a given package.
"""
links = []
for link in self._get_links(package):
if link.is_wheel and not Wheel(link.filename).is_supported_by_environment(
self._env
):
continue
if link.ext == ".egg":
continue
links.append(link)
if not links:
raise RuntimeError(
"Unable to find installation candidates for {}".format(package)
)
# Get the best link
chosen = max(links, key=lambda link: self._sort_key(package, link))
if not chosen:
raise RuntimeError(
"Unable to find installation candidates for {}".format(package)
)
return chosen
def _get_links(self, package): # type: (Package) -> List[Link]
if not package.source_type:
if not self._pool.has_repository("pypi"):
repository = self._pool.repositories[0]
else:
repository = self._pool.repository("pypi")
else:
repository = self._pool.repository(package.source_reference)
links = repository.find_links_for_package(package)
hashes = [f["hash"] for f in package.files]
if not hashes:
return links
selected_links = []
for link in links:
if not link.hash:
selected_links.append(link)
continue
h = link.hash_name + ":" + link.hash
if h not in hashes:
continue
selected_links.append(link)
return selected_links
def _sort_key(self, package, link): # type: (Package, Link) -> Tuple
"""
Function to pass as the `key` argument to a call to sorted() to sort
InstallationCandidates by preference.
Returns a tuple such that tuples sorting as greater using Python's
default comparison operator are more preferred.
The preference is as follows:
First and foremost, candidates with allowed (matching) hashes are
always preferred over candidates without matching hashes. This is
because e.g. if the only candidate with an allowed hash is yanked,
we still want to use that candidate.
Second, excepting hash considerations, candidates that have been
yanked (in the sense of PEP 592) are always less preferred than
candidates that haven't been yanked. Then:
If not finding wheels, they are sorted by version only.
If finding wheels, then the sort order is by version, then:
1. existing installs
2. wheels ordered via Wheel.support_index_min(self._supported_tags)
3. source archives
If prefer_binary was set, then all wheels are sorted above sources.
Note: it was considered to embed this logic into the Link
comparison operators, but then different sdist links
with the same version, would have to be considered equal
"""
support_num = len(self._env.supported_tags)
build_tag = ()
binary_preference = 0
if link.is_wheel:
wheel = Wheel(link.filename)
if not wheel.is_supported_by_environment(self._env):
raise RuntimeError(
"{} is not a supported wheel for this platform. It "
"can't be sorted.".format(wheel.filename)
)
# TODO: Binary preference
pri = -(wheel.get_minimum_supported_index(self._env.supported_tags))
if wheel.build_tag is not None:
match = re.match(r"^(\d+)(.*)$", wheel.build_tag)
build_tag_groups = match.groups()
build_tag = (int(build_tag_groups[0]), build_tag_groups[1])
else: # sdist
pri = -support_num
has_allowed_hash = int(self._is_link_hash_allowed_for_package(link, package))
# TODO: Proper yank value
yank_value = 0
return (
has_allowed_hash,
yank_value,
binary_preference,
package.version,
build_tag,
pri,
)
def _is_link_hash_allowed_for_package(
self, link, package
): # type: (Link, Package) -> bool
if not link.hash:
return True
h = link.hash_name + ":" + link.hash
return h in {f["hash"] for f in package.files}
poetry/installation/installer.py
from typing import List from typing import List
from typing import Optional
from typing import Union from typing import Union
from clikit.api.io import IO from clikit.api.io import IO
from poetry.config.config import Config
from poetry.core.packages.project_package import ProjectPackage from poetry.core.packages.project_package import ProjectPackage
from poetry.io.null_io import NullIO from poetry.io.null_io import NullIO
from poetry.packages import Locker from poetry.packages import Locker
from poetry.puzzle import Solver
from poetry.puzzle.operations import Install
from poetry.puzzle.operations import Uninstall
from poetry.puzzle.operations import Update
from poetry.puzzle.operations.operation import Operation
from poetry.repositories import Pool from poetry.repositories import Pool
from poetry.repositories import Repository from poetry.repositories import Repository
from poetry.repositories.installed_repository import InstalledRepository from poetry.repositories.installed_repository import InstalledRepository
...@@ -18,6 +15,11 @@ from poetry.utils.extras import get_extra_package_names ...@@ -18,6 +15,11 @@ from poetry.utils.extras import get_extra_package_names
from poetry.utils.helpers import canonicalize_name from poetry.utils.helpers import canonicalize_name
from .base_installer import BaseInstaller from .base_installer import BaseInstaller
from .executor import Executor
from .operations import Install
from .operations import Uninstall
from .operations import Update
from .operations.operation import Operation
from .pip_installer import PipInstaller from .pip_installer import PipInstaller
...@@ -29,7 +31,9 @@ class Installer: ...@@ -29,7 +31,9 @@ class Installer:
package, # type: ProjectPackage package, # type: ProjectPackage
locker, # type: Locker locker, # type: Locker
pool, # type: Pool pool, # type: Pool
installed=None, # type: (Union[InstalledRepository, None]) config, # type: Config
installed=None, # type: Union[InstalledRepository, None]
executor=None, # type: Optional[Executor]
): ):
self._io = io self._io = io
self._env = env self._env = env
...@@ -50,6 +54,12 @@ class Installer: ...@@ -50,6 +54,12 @@ class Installer:
self._extras = [] self._extras = []
if executor is None:
executor = Executor(self._env, self._pool, config, self._io)
self._executor = executor
self._use_executor = False
self._installer = self._get_installer() self._installer = self._get_installer()
if installed is None: if installed is None:
installed = self._get_installed() installed = self._get_installed()
...@@ -57,9 +67,18 @@ class Installer: ...@@ -57,9 +67,18 @@ class Installer:
self._installed_repository = installed self._installed_repository = installed
@property @property
def executor(self):
return self._executor
@property
def installer(self): def installer(self):
return self._installer return self._installer
def set_package(self, package): # type: (ProjectPackage) -> Installer
self._package = package
return self
def run(self): def run(self):
# Force update if there is no lock file present # Force update if there is no lock file present
if not self._update and not self._locker.is_locked(): if not self._update and not self._locker.is_locked():
...@@ -71,12 +90,12 @@ class Installer: ...@@ -71,12 +90,12 @@ class Installer:
self._execute_operations = False self._execute_operations = False
local_repo = Repository() local_repo = Repository()
self._do_install(local_repo)
return 0 return self._do_install(local_repo)
def dry_run(self, dry_run=True): # type: (bool) -> Installer def dry_run(self, dry_run=True): # type: (bool) -> Installer
self._dry_run = dry_run self._dry_run = dry_run
self._executor.dry_run(dry_run)
return self return self
...@@ -93,6 +112,7 @@ class Installer: ...@@ -93,6 +112,7 @@ class Installer:
def verbose(self, verbose=True): # type: (bool) -> Installer def verbose(self, verbose=True): # type: (bool) -> Installer
self._verbose = verbose self._verbose = verbose
self._executor.verbose(verbose)
return self return self
...@@ -128,6 +148,9 @@ class Installer: ...@@ -128,6 +148,9 @@ class Installer:
def execute_operations(self, execute=True): # type: (bool) -> Installer def execute_operations(self, execute=True): # type: (bool) -> Installer
self._execute_operations = execute self._execute_operations = execute
if not execute:
self._executor.disable()
return self return self
def whitelist(self, packages): # type: (dict) -> Installer def whitelist(self, packages): # type: (dict) -> Installer
...@@ -140,7 +163,14 @@ class Installer: ...@@ -140,7 +163,14 @@ class Installer:
return self return self
def use_executor(self, use_executor=True): # type: (bool) -> Installer
self._use_executor = use_executor
return self
def _do_install(self, local_repo): def _do_install(self, local_repo):
from poetry.puzzle import Solver
locked_repository = Repository() locked_repository = Repository()
if self._update: if self._update:
if self._locker.is_locked() and not self._lock: if self._locker.is_locked() and not self._lock:
...@@ -247,19 +277,30 @@ class Installer: ...@@ -247,19 +277,30 @@ class Installer:
# or optional and not requested, are dropped # or optional and not requested, are dropped
self._filter_operations(ops, local_repo) self._filter_operations(ops, local_repo)
# Execute operations
return self._execute(ops)
def _write_lock_file(self, repo): # type: (Repository) -> None
if self._update and self._write_lock:
updated_lock = self._locker.set_lock_data(self._package, repo.packages)
if updated_lock:
self._io.write_line("") self._io.write_line("")
self._io.write_line("<info>Writing lock file</>")
# Execute operations def _execute(self, operations):
actual_ops = [op for op in ops if not op.skipped] if self._use_executor:
if not actual_ops and (self._execute_operations or self._dry_run): return self._executor.execute(operations)
if not operations and (self._execute_operations or self._dry_run):
self._io.write_line("No dependencies to install or update") self._io.write_line("No dependencies to install or update")
if actual_ops and (self._execute_operations or self._dry_run): if operations and (self._execute_operations or self._dry_run):
installs = 0 installs = 0
updates = 0 updates = 0
uninstalls = 0 uninstalls = 0
skipped = 0 skipped = 0
for op in ops: for op in operations:
if op.skipped: if op.skipped:
skipped += 1 skipped += 1
elif op.job_type == "install": elif op.job_type == "install":
...@@ -289,18 +330,13 @@ class Installer: ...@@ -289,18 +330,13 @@ class Installer:
) )
self._io.write_line("") self._io.write_line("")
for op in ops:
self._execute(op)
def _write_lock_file(self, repo): # type: (Repository) -> None for op in operations:
if self._update and self._write_lock: self._execute_operation(op)
updated_lock = self._locker.set_lock_data(self._package, repo.packages)
if updated_lock: return 0
self._io.write_line("")
self._io.write_line("<info>Writing lock file</>")
def _execute(self, operation): # type: (Operation) -> None def _execute_operation(self, operation): # type: (Operation) -> None
""" """
Execute a given operation. Execute a given operation.
""" """
......
poetry/puzzle/operations/install.py → poetry/installation/operations/install.py
...@@ -2,8 +2,8 @@ from .operation import Operation ...@@ -2,8 +2,8 @@ from .operation import Operation
class Install(Operation): class Install(Operation):
def __init__(self, package, reason=None): def __init__(self, package, reason=None, priority=0):
super(Install, self).__init__(reason) super(Install, self).__init__(reason, priority=priority)
self._package = package self._package = package
......
poetry/puzzle/operations/operation.py → poetry/installation/operations/operation.py
...@@ -4,11 +4,14 @@ from typing import Union ...@@ -4,11 +4,14 @@ from typing import Union
class Operation(object): class Operation(object):
def __init__(self, reason=None): # type: (Union[str, None]) -> None def __init__(
self, reason=None, priority=0
): # type: (Union[str, None], int) -> None
self._reason = reason self._reason = reason
self._skipped = False self._skipped = False
self._skip_reason = None self._skip_reason = None
self._priority = priority
@property @property
def job_type(self): # type: () -> str def job_type(self): # type: () -> str
...@@ -27,6 +30,10 @@ class Operation(object): ...@@ -27,6 +30,10 @@ class Operation(object):
return self._skip_reason return self._skip_reason
@property @property
def priority(self): # type: () -> int
return self._priority
@property
def package(self): def package(self):
raise NotImplementedError() raise NotImplementedError()
......
poetry/puzzle/operations/uninstall.py → poetry/installation/operations/uninstall.py
...@@ -2,8 +2,8 @@ from .operation import Operation ...@@ -2,8 +2,8 @@ from .operation import Operation
class Uninstall(Operation): class Uninstall(Operation):
def __init__(self, package, reason=None): def __init__(self, package, reason=None, priority=float("inf")):
super(Uninstall, self).__init__(reason) super(Uninstall, self).__init__(reason, priority=priority)
self._package = package self._package = package
......
poetry/puzzle/operations/update.py → poetry/installation/operations/update.py
...@@ -2,11 +2,11 @@ from .operation import Operation ...@@ -2,11 +2,11 @@ from .operation import Operation
class Update(Operation): class Update(Operation):
def __init__(self, initial, target, reason=None): def __init__(self, initial, target, reason=None, priority=0):
self._initial_package = initial self._initial_package = initial
self._target_package = target self._target_package = target
super(Update, self).__init__(reason) super(Update, self).__init__(reason, priority=priority)
@property @property
def initial_package(self): def initial_package(self):
......
poetry/puzzle/solver.py
...@@ -10,6 +10,10 @@ from clikit.io import ConsoleIO ...@@ -10,6 +10,10 @@ from clikit.io import ConsoleIO
from poetry.core.packages import Package from poetry.core.packages import Package
from poetry.core.packages.project_package import ProjectPackage from poetry.core.packages.project_package import ProjectPackage
from poetry.installation.operations import Install
from poetry.installation.operations import Uninstall
from poetry.installation.operations import Update
from poetry.installation.operations.operation import Operation
from poetry.mixology import resolve_version from poetry.mixology import resolve_version
from poetry.mixology.failure import SolveFailure from poetry.mixology.failure import SolveFailure
from poetry.packages import DependencyPackage from poetry.packages import DependencyPackage
...@@ -19,10 +23,6 @@ from poetry.utils.env import Env ...@@ -19,10 +23,6 @@ from poetry.utils.env import Env
from .exceptions import OverrideNeeded from .exceptions import OverrideNeeded
from .exceptions import SolverProblemError from .exceptions import SolverProblemError
from .operations import Install
from .operations import Uninstall
from .operations import Update
from .operations.operation import Operation
from .provider import Provider from .provider import Provider
...@@ -78,7 +78,7 @@ class Solver: ...@@ -78,7 +78,7 @@ class Solver:
) )
operations = [] operations = []
for package in packages: for i, package in enumerate(packages):
installed = False installed = False
for pkg in self._installed.packages: for pkg in self._installed.packages:
if package.name == pkg.name: if package.name == pkg.name:
...@@ -113,23 +113,27 @@ class Solver: ...@@ -113,23 +113,27 @@ class Solver:
package.source_reference package.source_reference
) )
): ):
operations.append(Update(pkg, package)) operations.append(Update(pkg, package, priority=depths[i]))
else: else:
operations.append( operations.append(
Install(package).skip("Already installed") Install(package).skip("Already installed")
) )
elif package.version != pkg.version: elif package.version != pkg.version:
# Checking version # Checking version
operations.append(Update(pkg, package)) operations.append(Update(pkg, package, priority=depths[i]))
elif pkg.source_type and package.source_type != pkg.source_type: elif pkg.source_type and package.source_type != pkg.source_type:
operations.append(Update(pkg, package)) operations.append(Update(pkg, package, priority=depths[i]))
else: else:
operations.append(Install(package).skip("Already installed")) operations.append(
Install(package, priority=depths[i]).skip(
"Already installed"
)
)
break break
if not installed: if not installed:
operations.append(Install(package)) operations.append(Install(package, priority=depths[i]))
# Checking for removals # Checking for removals
for pkg in self._locked.packages: for pkg in self._locked.packages:
...@@ -165,15 +169,7 @@ class Solver: ...@@ -165,15 +169,7 @@ class Solver:
operations.append(Uninstall(installed)) operations.append(Uninstall(installed))
return sorted( return sorted(
operations, operations, key=lambda o: (-o.priority, o.package.name, o.package.version,),
key=lambda o: (
o.job_type == "uninstall",
# Packages to be uninstalled have no depth so we default to 0
# since it actually doesn't matter since removals are always on top.
-depths[packages.index(o.package)] if o.job_type != "uninstall" else 0,
o.package.name,
o.package.version,
),
) )
def solve_in_compatibility_mode(self, overrides, use_latest=None): def solve_in_compatibility_mode(self, overrides, use_latest=None):
......
poetry/repositories/installed_repository.py
...@@ -90,6 +90,7 @@ class InstalledRepository(Repository): ...@@ -90,6 +90,7 @@ class InstalledRepository(Repository):
# TODO: handle multiple source directories? # TODO: handle multiple source directories?
package.source_type = "directory" package.source_type = "directory"
package.source_url = paths.pop().as_posix() package.source_url = paths.pop().as_posix()
continue continue
src_path = env.path / "src" src_path = env.path / "src"
......
poetry/repositories/legacy_repository.py
...@@ -304,6 +304,13 @@ class LegacyRepository(PyPiRepository): ...@@ -304,6 +304,13 @@ class LegacyRepository(PyPiRepository):
return package return package
def find_links_for_package(self, package):
page = self._get("/{}/".format(package.name.replace(".", "-")))
if page is None:
return []
return list(page.links_for_version(package.version))
def _get_release_info(self, name, version): # type: (str, str) -> dict def _get_release_info(self, name, version): # type: (str, str) -> dict
page = self._get("/{}/".format(canonicalize_name(name).replace(".", "-"))) page = self._get("/{}/".format(canonicalize_name(name).replace(".", "-")))
if page is None: if page is None:
......
poetry/repositories/pool.py
...@@ -33,7 +33,15 @@ class Pool(BaseRepository): ...@@ -33,7 +33,15 @@ class Pool(BaseRepository):
def has_default(self): # type: () -> bool def has_default(self): # type: () -> bool
return self._default return self._default
def has_repository(self, name): # type: (str) -> bool
name = name.lower() if name is not None else None
return name in self._lookup
def repository(self, name): # type: (str) -> Repository def repository(self, name): # type: (str) -> Repository
if name is not None:
name = name.lower()
if name in self._lookup: if name in self._lookup:
return self._repositories[self._lookup[name]] return self._repositories[self._lookup[name]]
...@@ -45,6 +53,9 @@ class Pool(BaseRepository): ...@@ -45,6 +53,9 @@ class Pool(BaseRepository):
""" """
Adds a repository to the pool. Adds a repository to the pool.
""" """
repository_name = (
repository.name.lower() if repository.name is not None else None
)
if default: if default:
if self.has_default(): if self.has_default():
raise ValueError("Only one repository can be the default") raise ValueError("Only one repository can be the default")
...@@ -57,17 +68,17 @@ class Pool(BaseRepository): ...@@ -57,17 +68,17 @@ class Pool(BaseRepository):
if self._secondary_start_idx is not None: if self._secondary_start_idx is not None:
self._secondary_start_idx += 1 self._secondary_start_idx += 1
self._lookup[repository.name] = 0 self._lookup[repository_name] = 0
elif secondary: elif secondary:
if self._secondary_start_idx is None: if self._secondary_start_idx is None:
self._secondary_start_idx = len(self._repositories) self._secondary_start_idx = len(self._repositories)
self._repositories.append(repository) self._repositories.append(repository)
self._lookup[repository.name] = len(self._repositories) - 1 self._lookup[repository_name] = len(self._repositories) - 1
else: else:
if self._secondary_start_idx is None: if self._secondary_start_idx is None:
self._repositories.append(repository) self._repositories.append(repository)
self._lookup[repository.name] = len(self._repositories) - 1 self._lookup[repository_name] = len(self._repositories) - 1
else: else:
self._repositories.insert(self._secondary_start_idx, repository) self._repositories.insert(self._secondary_start_idx, repository)
...@@ -77,12 +88,15 @@ class Pool(BaseRepository): ...@@ -77,12 +88,15 @@ class Pool(BaseRepository):
self._lookup[name] += 1 self._lookup[name] += 1
self._lookup[repository.name] = self._secondary_start_idx self._lookup[repository_name] = self._secondary_start_idx
self._secondary_start_idx += 1 self._secondary_start_idx += 1
return self return self
def remove_repository(self, repository_name): # type: (str) -> Pool def remove_repository(self, repository_name): # type: (str) -> Pool
if repository_name is not None:
repository_name = repository_name.lower()
idx = self._lookup.get(repository_name) idx = self._lookup.get(repository_name)
if idx is not None: if idx is not None:
del self._repositories[idx] del self._repositories[idx]
...@@ -95,6 +109,9 @@ class Pool(BaseRepository): ...@@ -95,6 +109,9 @@ class Pool(BaseRepository):
def package( def package(
self, name, version, extras=None, repository=None self, name, version, extras=None, repository=None
): # type: (str, str, List[str], str) -> "Package" ): # type: (str, str, List[str], str) -> "Package"
if repository is not None:
repository = repository.lower()
if ( if (
repository is not None repository is not None
and repository not in self._lookup and repository not in self._lookup
...@@ -104,9 +121,7 @@ class Pool(BaseRepository): ...@@ -104,9 +121,7 @@ class Pool(BaseRepository):
if repository is not None and not self._ignore_repository_names: if repository is not None and not self._ignore_repository_names:
try: try:
return self._repositories[self._lookup[repository]].package( return self.repository(repository).package(name, version, extras=extras)
name, version, extras=extras
)
except PackageNotFound: except PackageNotFound:
pass pass
else: else:
...@@ -131,6 +146,9 @@ class Pool(BaseRepository): ...@@ -131,6 +146,9 @@ class Pool(BaseRepository):
allow_prereleases=False, allow_prereleases=False,
repository=None, repository=None,
): ):
if repository is not None:
repository = repository.lower()
if ( if (
repository is not None repository is not None
and repository not in self._lookup and repository not in self._lookup
...@@ -139,7 +157,7 @@ class Pool(BaseRepository): ...@@ -139,7 +157,7 @@ class Pool(BaseRepository):
raise ValueError('Repository "{}" does not exist.'.format(repository)) raise ValueError('Repository "{}" does not exist.'.format(repository))
if repository is not None and not self._ignore_repository_names: if repository is not None and not self._ignore_repository_names:
return self._repositories[self._lookup[repository]].find_packages( return self.repository(repository).find_packages(
name, constraint, extras=extras, allow_prereleases=allow_prereleases name, constraint, extras=extras, allow_prereleases=allow_prereleases
) )
......
poetry/repositories/pypi_repository.py
...@@ -241,6 +241,18 @@ class PyPiRepository(RemoteRepository): ...@@ -241,6 +241,18 @@ class PyPiRepository(RemoteRepository):
return PackageInfo.load(cached) return PackageInfo.load(cached)
def find_links_for_package(self, package):
json_data = self._get("pypi/{}/{}/json".format(package.name, package.version))
if json_data is None:
return []
links = []
for url in json_data["urls"]:
h = "sha256={}".format(url["digests"]["sha256"])
links.append(Link(url["url"] + "#" + h))
return links
def _get_release_info(self, name, version): # type: (str, str) -> dict def _get_release_info(self, name, version): # type: (str, str) -> dict
self._log("Getting info for {} ({}) from PyPI".format(name, version), "debug") self._log("Getting info for {} ({}) from PyPI".format(name, version), "debug")
......
poetry/repositories/repository.py
...@@ -6,10 +6,10 @@ from .base_repository import BaseRepository ...@@ -6,10 +6,10 @@ from .base_repository import BaseRepository
class Repository(BaseRepository): class Repository(BaseRepository):
def __init__(self, packages=None): def __init__(self, packages=None, name=None):
super(Repository, self).__init__() super(Repository, self).__init__()
self._name = None self._name = name
if packages is None: if packages is None:
packages = [] packages = []
...@@ -115,6 +115,9 @@ class Repository(BaseRepository): ...@@ -115,6 +115,9 @@ class Repository(BaseRepository):
if index is not None: if index is not None:
del self._packages[index] del self._packages[index]
def find_links_for_package(self, package):
return []
def search(self, query): def search(self, query):
results = [] results = []
......
poetry/utils/_compat.py
...@@ -23,6 +23,11 @@ try: ...@@ -23,6 +23,11 @@ try:
except ImportError: except ImportError:
import urlparse import urlparse
try:
from os import cpu_count
except ImportError: # Python 2
from multiprocessing import cpu_count
try: # Python 2 try: # Python 2
long = long long = long
unicode = unicode unicode = unicode
...@@ -50,6 +55,14 @@ else: ...@@ -50,6 +55,14 @@ else:
shell_quote = shlex.quote shell_quote = shlex.quote
if PY34:
from importlib.machinery import EXTENSION_SUFFIXES
else:
from imp import get_suffixes
EXTENSION_SUFFIXES = [suffix[0] for suffix in get_suffixes()]
if PY35: if PY35:
from pathlib import Path from pathlib import Path
else: else:
......
poetry/utils/env.py
...@@ -7,7 +7,7 @@ import re ...@@ -7,7 +7,7 @@ import re
import shutil import shutil
import sys import sys
import sysconfig import sysconfig
import warnings import textwrap
from contextlib import contextmanager from contextlib import contextmanager
from typing import Any from typing import Any
...@@ -20,6 +20,12 @@ import tomlkit ...@@ -20,6 +20,12 @@ import tomlkit
from clikit.api.io import IO from clikit.api.io import IO
import packaging.tags
from packaging.tags import Tag
from packaging.tags import interpreter_name
from packaging.tags import interpreter_version
from packaging.tags import sys_tags
from poetry.core.semver import parse_constraint from poetry.core.semver import parse_constraint
from poetry.core.semver.version import Version from poetry.core.semver.version import Version
from poetry.core.version.markers import BaseMarker from poetry.core.version.markers import BaseMarker
...@@ -39,6 +45,36 @@ import json ...@@ -39,6 +45,36 @@ import json
import os import os
import platform import platform
import sys import sys
import sysconfig
INTERPRETER_SHORT_NAMES = {
"python": "py",
"cpython": "cp",
"pypy": "pp",
"ironpython": "ip",
"jython": "jy",
}
def interpreter_version():
version = sysconfig.get_config_var("interpreter_version")
if version:
version = str(version)
else:
version = _version_nodot(sys.version_info[:2])
return version
def _version_nodot(version):
# type: (PythonVersion) -> str
if any(v >= 10 for v in version):
sep = "_"
else:
sep = ""
return sep.join(map(str, version))
if hasattr(sys, "implementation"): if hasattr(sys, "implementation"):
info = sys.implementation.version info = sys.implementation.version
...@@ -50,7 +86,7 @@ if hasattr(sys, "implementation"): ...@@ -50,7 +86,7 @@ if hasattr(sys, "implementation"):
implementation_name = sys.implementation.name implementation_name = sys.implementation.name
else: else:
iver = "0" iver = "0"
implementation_name = "" implementation_name = platform.python_implementation().lower()
env = { env = {
"implementation_name": implementation_name, "implementation_name": implementation_name,
...@@ -65,6 +101,9 @@ env = { ...@@ -65,6 +101,9 @@ env = {
"python_version": platform.python_version()[:3], "python_version": platform.python_version()[:3],
"sys_platform": sys.platform, "sys_platform": sys.platform,
"version_info": tuple(sys.version_info), "version_info": tuple(sys.version_info),
# Extra information
"interpreter_name": INTERPRETER_SHORT_NAMES.get(implementation_name, implementation_name),
"interpreter_version": interpreter_version(),
} }
print(json.dumps(env)) print(json.dumps(env))
...@@ -82,12 +121,6 @@ else: ...@@ -82,12 +121,6 @@ else:
print(sys.prefix) print(sys.prefix)
""" """
GET_CONFIG_VAR = """\
import sysconfig
print(sysconfig.get_config_var("{config_var}")),
"""
GET_PYTHON_VERSION = """\ GET_PYTHON_VERSION = """\
import sys import sys
...@@ -742,6 +775,7 @@ class Env(object): ...@@ -742,6 +775,7 @@ class Env(object):
self._pip_version = None self._pip_version = None
self._site_packages = None self._site_packages = None
self._paths = None self._paths = None
self._supported_tags = None
@property @property
def path(self): # type: () -> Path def path(self): # type: () -> Path
...@@ -813,6 +847,13 @@ class Env(object): ...@@ -813,6 +847,13 @@ class Env(object):
return self._paths return self._paths
@property
def supported_tags(self): # type: () -> List[Tag]
if self._supported_tags is None:
self._supported_tags = self.get_supported_tags()
return self._supported_tags
@classmethod @classmethod
def get_base_prefix(cls): # type: () -> Path def get_base_prefix(cls): # type: () -> Path
if hasattr(sys, "real_prefix"): if hasattr(sys, "real_prefix"):
...@@ -835,7 +876,7 @@ class Env(object): ...@@ -835,7 +876,7 @@ class Env(object):
def get_pip_command(self): # type: () -> List[str] def get_pip_command(self): # type: () -> List[str]
raise NotImplementedError() raise NotImplementedError()
def config_var(self, var): # type: (str) -> Any def get_supported_tags(self): # type: () -> List[Tag]
raise NotImplementedError() raise NotImplementedError()
def get_pip_version(self): # type: () -> Version def get_pip_version(self): # type: () -> Version
...@@ -987,6 +1028,9 @@ class SystemEnv(Env): ...@@ -987,6 +1028,9 @@ class SystemEnv(Env):
return paths return paths
def get_supported_tags(self): # type: () -> List[Tag]
return list(sys_tags())
def get_marker_env(self): # type: () -> Dict[str, Any] def get_marker_env(self): # type: () -> Dict[str, Any]
if hasattr(sys, "implementation"): if hasattr(sys, "implementation"):
info = sys.implementation.version info = sys.implementation.version
...@@ -1015,16 +1059,11 @@ class SystemEnv(Env): ...@@ -1015,16 +1059,11 @@ class SystemEnv(Env):
), ),
"sys_platform": sys.platform, "sys_platform": sys.platform,
"version_info": sys.version_info, "version_info": sys.version_info,
# Extra information
"interpreter_name": interpreter_name(),
"interpreter_version": interpreter_version(),
} }
def config_var(self, var): # type: (str) -> Any
try:
return sysconfig.get_config_var(var)
except IOError as e:
warnings.warn("{0}".format(e), RuntimeWarning)
return
def get_pip_version(self): # type: () -> Version def get_pip_version(self): # type: () -> Version
from pip import __version__ from pip import __version__
...@@ -1068,28 +1107,40 @@ class VirtualEnv(Env): ...@@ -1068,28 +1107,40 @@ class VirtualEnv(Env):
# so assume that we have a functional pip # so assume that we have a functional pip
return [self._bin("pip")] return [self._bin("pip")]
def get_marker_env(self): # type: () -> Dict[str, Any] def get_supported_tags(self): # type: () -> List[Tag]
output = self.run("python", "-", input_=GET_ENVIRONMENT_INFO) file_path = Path(packaging.tags.__file__)
if file_path.suffix == ".pyc":
# Python 2
file_path = file_path.with_suffix(".py")
return json.loads(output) with file_path.open(encoding="utf-8") as f:
script = decode(f.read())
def config_var(self, var): # type: (str) -> Any script = script.replace(
try: "from ._typing import TYPE_CHECKING, cast",
value = self.run( "TYPE_CHECKING = False\ncast = lambda type_, value: value",
"python", "-", input_=GET_CONFIG_VAR.format(config_var=var) )
).strip() script = script.replace(
except EnvCommandError as e: "from ._typing import MYPY_CHECK_RUNNING, cast",
warnings.warn("{0}".format(e), RuntimeWarning) "MYPY_CHECK_RUNNING = False\ncast = lambda type_, value: value",
return None )
script += textwrap.dedent(
"""
import json
print(json.dumps([(t.interpreter, t.abi, t.platform) for t in sys_tags()]))
"""
)
output = self.run("python", "-", input_=script)
return [Tag(*t) for t in json.loads(output)]
if value == "None": def get_marker_env(self): # type: () -> Dict[str, Any]
value = None output = self.run("python", "-", input_=GET_ENVIRONMENT_INFO)
elif value == "1":
value = 1
elif value == "0":
value = 0
return value return json.loads(output)
def get_pip_version(self): # type: () -> Version def get_pip_version(self): # type: () -> Version
output = self.run_pip("--version").strip() output = self.run_pip("--version").strip()
...@@ -1188,7 +1239,7 @@ class MockEnv(NullEnv): ...@@ -1188,7 +1239,7 @@ class MockEnv(NullEnv):
pip_version="19.1", pip_version="19.1",
sys_path=None, sys_path=None,
marker_env=None, marker_env=None,
config_vars=None, supported_tags=None,
**kwargs **kwargs
): ):
super(MockEnv, self).__init__(**kwargs) super(MockEnv, self).__init__(**kwargs)
...@@ -1201,15 +1252,7 @@ class MockEnv(NullEnv): ...@@ -1201,15 +1252,7 @@ class MockEnv(NullEnv):
self._pip_version = Version.parse(pip_version) self._pip_version = Version.parse(pip_version)
self._sys_path = sys_path self._sys_path = sys_path
self._mock_marker_env = marker_env self._mock_marker_env = marker_env
self._config_vars = config_vars self._supported_tags = supported_tags
@property
def version_info(self): # type: () -> Tuple[int]
return self._version_info
@property
def python_implementation(self): # type: () -> str
return self._python_implementation
@property @property
def platform(self): # type: () -> str def platform(self): # type: () -> str
...@@ -1240,17 +1283,12 @@ class MockEnv(NullEnv): ...@@ -1240,17 +1283,12 @@ class MockEnv(NullEnv):
marker_env["python_version"] = ".".join(str(v) for v in self._version_info[:2]) marker_env["python_version"] = ".".join(str(v) for v in self._version_info[:2])
marker_env["python_full_version"] = ".".join(str(v) for v in self._version_info) marker_env["python_full_version"] = ".".join(str(v) for v in self._version_info)
marker_env["sys_platform"] = self._platform marker_env["sys_platform"] = self._platform
marker_env["interpreter_name"] = self._python_implementation.lower()
marker_env["interpreter_version"] = "cp" + "".join(
str(v) for v in self._version_info[:2]
)
return marker_env return marker_env
def is_venv(self): # type: () -> bool def is_venv(self): # type: () -> bool
return self._is_venv return self._is_venv
def config_var(self, var): # type: (str) -> Any
if self._config_vars is None:
return super().config_var(var)
else:
try:
return self._config_vars[var]
except KeyError:
return None
pyproject.toml
...@@ -37,12 +37,15 @@ html5lib = "^1.0" ...@@ -37,12 +37,15 @@ html5lib = "^1.0"
shellingham = "^1.1" shellingham = "^1.1"
tomlkit = "^0.5.11" tomlkit = "^0.5.11"
pexpect = "^4.7.0" pexpect = "^4.7.0"
packaging = "^20.4"
# The typing module is not in the stdlib in Python 2.7 # The typing module is not in the stdlib in Python 2.7
typing = { version = "^3.6", python = "~2.7" } typing = { version = "^3.6", python = "~2.7" }
# Use pathlib2 for Python 2.7 # Use pathlib2 for Python 2.7
pathlib2 = { version = "^2.3", python = "~2.7" } pathlib2 = { version = "^2.3", python = "~2.7" }
# Use futures on Python 2.7
futures = { version = "^3.3.0", python = "~2.7" }
# Use glob2 for Python 2.7 and 3.4 # Use glob2 for Python 2.7 and 3.4
glob2 = { version = "^0.6", python = "~2.7" } glob2 = { version = "^0.6", python = "~2.7" }
# Use virtualenv for Python 2.7 since venv does not exist # Use virtualenv for Python 2.7 since venv does not exist
......
tests/conftest.py
...@@ -104,11 +104,13 @@ def git_mock(mocker): ...@@ -104,11 +104,13 @@ def git_mock(mocker):
@pytest.fixture @pytest.fixture
def http(): def http():
httpretty.enable() httpretty.reset()
httpretty.enable(allow_net_connect=False)
yield httpretty yield httpretty
httpretty.disable() httpretty.activate()
httpretty.reset()
@pytest.fixture @pytest.fixture
......
tests/console/commands/test_config.py
...@@ -13,6 +13,7 @@ def test_list_displays_default_value_if_not_set(app, config): ...@@ -13,6 +13,7 @@ def test_list_displays_default_value_if_not_set(app, config):
tester.execute("--list") tester.execute("--list")
expected = """cache-dir = "/foo" expected = """cache-dir = "/foo"
experimental.new-installer = true
virtualenvs.create = true virtualenvs.create = true
virtualenvs.in-project = false virtualenvs.in-project = false
virtualenvs.path = {path} # /foo{sep}virtualenvs virtualenvs.path = {path} # /foo{sep}virtualenvs
...@@ -32,6 +33,7 @@ def test_list_displays_set_get_setting(app, config): ...@@ -32,6 +33,7 @@ def test_list_displays_set_get_setting(app, config):
tester.execute("--list") tester.execute("--list")
expected = """cache-dir = "/foo" expected = """cache-dir = "/foo"
experimental.new-installer = true
virtualenvs.create = false virtualenvs.create = false
virtualenvs.in-project = false virtualenvs.in-project = false
virtualenvs.path = {path} # /foo{sep}virtualenvs virtualenvs.path = {path} # /foo{sep}virtualenvs
...@@ -79,6 +81,7 @@ def test_list_displays_set_get_local_setting(app, config): ...@@ -79,6 +81,7 @@ def test_list_displays_set_get_local_setting(app, config):
tester.execute("--list") tester.execute("--list")
expected = """cache-dir = "/foo" expected = """cache-dir = "/foo"
experimental.new-installer = true
virtualenvs.create = false virtualenvs.create = false
virtualenvs.in-project = false virtualenvs.in-project = false
virtualenvs.path = {path} # /foo{sep}virtualenvs virtualenvs.path = {path} # /foo{sep}virtualenvs
......
tests/console/conftest.py
import os import os
import re
import pytest import pytest
from cleo import ApplicationTester from cleo import ApplicationTester
from poetry.console import Application as BaseApplication from poetry.console import Application as BaseApplication
from poetry.core.masonry.utils.helpers import escape_name
from poetry.core.masonry.utils.helpers import escape_version
from poetry.core.packages.utils.link import Link
from poetry.factory import Factory from poetry.factory import Factory
from poetry.installation.executor import Executor as BaseExecutor
from poetry.installation.noop_installer import NoopInstaller from poetry.installation.noop_installer import NoopInstaller
from poetry.io.null_io import NullIO
from poetry.packages import Locker as BaseLocker from poetry.packages import Locker as BaseLocker
from poetry.poetry import Poetry as BasePoetry from poetry.poetry import Poetry as BasePoetry
from poetry.repositories import Pool from poetry.repositories import Pool
...@@ -18,6 +24,42 @@ from poetry.utils.toml_file import TomlFile ...@@ -18,6 +24,42 @@ from poetry.utils.toml_file import TomlFile
from tests.helpers import mock_clone from tests.helpers import mock_clone
class Executor(BaseExecutor):
def __init__(self, *args, **kwargs):
super(Executor, self).__init__(*args, **kwargs)
self._installs = []
self._updates = []
self._uninstalls = []
@property
def installations(self):
return self._installs
@property
def updates(self):
return self._updates
@property
def removals(self):
return self._uninstalls
def _do_execute_operation(self, operation):
super(Executor, self)._do_execute_operation(operation)
if not operation.skipped:
getattr(self, "_{}s".format(operation.job_type)).append(operation.package)
def _execute_install(self, operation):
return 0
def _execute_update(self, operation):
return 0
def _execute_remove(self, operation):
return 0
@pytest.fixture() @pytest.fixture()
def installer(): def installer():
return NoopInstaller() return NoopInstaller()
...@@ -39,6 +81,9 @@ def setup(mocker, installer, installed, config, env): ...@@ -39,6 +81,9 @@ def setup(mocker, installer, installed, config, env):
p = mocker.patch("poetry.installation.installer.Installer._get_installer") p = mocker.patch("poetry.installation.installer.Installer._get_installer")
p.return_value = installer p.return_value = installer
# Do not run pip commands of the executor
mocker.patch("poetry.installation.executor.Executor.run_pip")
p = mocker.patch("poetry.installation.installer.Installer._get_installed") p = mocker.patch("poetry.installation.installer.Installer._get_installed")
p.return_value = installed p.return_value = installed
...@@ -144,10 +189,22 @@ class Repository(BaseRepository): ...@@ -144,10 +189,22 @@ class Repository(BaseRepository):
raise PackageNotFound("Package [{}] not found.".format(name)) raise PackageNotFound("Package [{}] not found.".format(name))
return packages return packages
def find_links_for_package(self, package):
return [
Link(
"https://foo.bar/files/{}-{}-py2.py3-none-any.whl".format(
escape_name(package.name), escape_version(package.version.text)
)
)
]
@pytest.fixture @pytest.fixture
def repo(): def repo(http):
return Repository() http.register_uri(
http.GET, re.compile("^https?://foo.bar/(.+?)$"),
)
return Repository(name="foo")
@pytest.fixture @pytest.fixture
...@@ -188,3 +245,13 @@ def app(poetry): ...@@ -188,3 +245,13 @@ def app(poetry):
@pytest.fixture @pytest.fixture
def app_tester(app): def app_tester(app):
return ApplicationTester(app) return ApplicationTester(app)
@pytest.fixture
def new_installer_disabled(config):
config.merge({"experimental": {"new-installer": False}})
@pytest.fixture()
def executor(poetry, config, env):
return Executor(env, poetry.pool, config, NullIO())
tests/installation/test_authenticator.py 0 → 100644
import re
import pytest
from poetry.installation.authenticator import Authenticator
from poetry.io.null_io import NullIO
@pytest.fixture()
def mock_remote(http):
http.register_uri(
http.GET, re.compile("^https?://foo.bar/(.+?)$"),
)
def test_authenticator_uses_url_provided_credentials(config, mock_remote, http):
config.merge(
{
"repositories": {"foo": {"url": "https://foo.bar/simple/"}},
"http-basic": {"foo": {"username": "bar", "password": "baz"}},
}
)
authenticator = Authenticator(config, NullIO())
authenticator.request("get", "https://foo001:bar002@foo.bar/files/foo-0.1.0.tar.gz")
request = http.last_request()
assert "Basic Zm9vMDAxOmJhcjAwMg==" == request.headers["Authorization"]
def test_authenticator_uses_credentials_from_config_if_not_provided(
config, mock_remote, http
):
config.merge(
{
"repositories": {"foo": {"url": "https://foo.bar/simple/"}},
"http-basic": {"foo": {"username": "bar", "password": "baz"}},
}
)
authenticator = Authenticator(config, NullIO())
authenticator.request("get", "https://foo.bar/files/foo-0.1.0.tar.gz")
request = http.last_request()
assert "Basic YmFyOmJheg==" == request.headers["Authorization"]
def test_authenticator_uses_username_only_credentials(config, mock_remote, http):
config.merge(
{
"repositories": {"foo": {"url": "https://foo.bar/simple/"}},
"http-basic": {"foo": {"username": "bar", "password": "baz"}},
}
)
authenticator = Authenticator(config, NullIO())
authenticator.request("get", "https://foo001@foo.bar/files/foo-0.1.0.tar.gz")
request = http.last_request()
assert "Basic Zm9vMDAxOg==" == request.headers["Authorization"]
def test_authenticator_uses_password_only_credentials(config, mock_remote, http):
config.merge(
{
"repositories": {"foo": {"url": "https://foo.bar/simple/"}},
"http-basic": {"foo": {"username": "bar", "password": "baz"}},
}
)
authenticator = Authenticator(config, NullIO())
authenticator.request("get", "https://:bar002@foo.bar/files/foo-0.1.0.tar.gz")
request = http.last_request()
assert "Basic OmJhcjAwMg==" == request.headers["Authorization"]
def test_authenticator_uses_empty_strings_as_default_password(
config, mock_remote, http
):
config.merge(
{
"repositories": {"foo": {"url": "https://foo.bar/simple/"}},
"http-basic": {"foo": {"username": "bar"}},
}
)
authenticator = Authenticator(config, NullIO())
authenticator.request("get", "https://foo.bar/files/foo-0.1.0.tar.gz")
request = http.last_request()
assert "Basic YmFyOg==" == request.headers["Authorization"]
def test_authenticator_uses_empty_strings_as_default_username(
config, mock_remote, http
):
config.merge(
{
"repositories": {"foo": {"url": "https://foo.bar/simple/"}},
"http-basic": {"foo": {"username": None, "password": "bar"}},
}
)
authenticator = Authenticator(config, NullIO())
authenticator.request("get", "https://foo.bar/files/foo-0.1.0.tar.gz")
request = http.last_request()
assert "Basic OmJhcg==" == request.headers["Authorization"]
tests/installation/test_chef.py 0 → 100644
from packaging.tags import Tag
from poetry.core.packages.utils.link import Link
from poetry.installation.chef import Chef
from poetry.utils._compat import Path
from poetry.utils.env import MockEnv
def test_get_cached_archive_for_link(config, mocker):
chef = Chef(
config,
MockEnv(
version_info=(3, 8, 3),
marker_env={"interpreter_name": "cpython", "interpreter_version": "3.8.3"},
supported_tags=[
Tag("cp38", "cp38", "macosx_10_15_x86_64"),
Tag("py3", "none", "any"),
],
),
)
mocker.patch.object(
chef,
"get_cached_archives_for_link",
return_value=[
Link("file:///foo/demo-0.1.0-py2.py3-none-any"),
Link("file:///foo/demo-0.1.0.tar.gz"),
Link("file:///foo/demo-0.1.0-cp38-cp38-macosx_10_15_x86_64.whl"),
Link("file:///foo/demo-0.1.0-cp37-cp37-macosx_10_15_x86_64.whl"),
],
)
archive = chef.get_cached_archive_for_link(
Link("https://files.python-poetry.org/demo-0.1.0.tar.gz")
)
assert Link("file:///foo/demo-0.1.0-cp38-cp38-macosx_10_15_x86_64.whl") == archive
def test_get_cached_archives_for_link(config, mocker):
chef = Chef(
config,
MockEnv(
marker_env={"interpreter_name": "cpython", "interpreter_version": "3.8.3"}
),
)
mocker.patch.object(
chef,
"get_cache_directory_for_link",
return_value=Path(__file__).parent.parent.joinpath("fixtures/distributions"),
)
archives = chef.get_cached_archives_for_link(
Link("https://files.python-poetry.org/demo-0.1.0.tar.gz")
)
assert 2 == len(archives)
def test_get_cache_directory_for_link(config):
chef = Chef(
config,
MockEnv(
marker_env={"interpreter_name": "cpython", "interpreter_version": "3.8.3"}
),
)
directory = chef.get_cache_directory_for_link(
Link("https://files.python-poetry.org/poetry-1.1.0.tar.gz")
)
expected = Path(
"/foo/artifacts/ba/63/13/283a3b3b7f95f05e9e6f84182d276f7bb0951d5b0cc24422b33f7a4648"
)
assert expected == directory
tests/installation/test_chooser.py 0 → 100644
import re
import pytest
from packaging.tags import Tag
from poetry.core.packages.package import Package
from poetry.installation.chooser import Chooser
from poetry.repositories.legacy_repository import LegacyRepository
from poetry.repositories.pool import Pool
from poetry.repositories.pypi_repository import PyPiRepository
from poetry.utils._compat import Path
from poetry.utils.env import MockEnv
JSON_FIXTURES = (
Path(__file__).parent.parent / "repositories" / "fixtures" / "pypi.org" / "json"
)
LEGACY_FIXTURES = Path(__file__).parent.parent / "repositories" / "fixtures" / "legacy"
@pytest.fixture()
def env():
return MockEnv(
supported_tags=[
Tag("cp37", "cp37", "macosx_10_15_x86_64"),
Tag("py3", "none", "any"),
]
)
@pytest.fixture()
def mock_pypi(http):
def callback(request, uri, headers):
parts = uri.rsplit("/")
name = parts[-3]
version = parts[-2]
fixture = JSON_FIXTURES / name / (version + ".json")
if not fixture.exists():
fixture = JSON_FIXTURES / (name + ".json")
if not fixture.exists():
return
with fixture.open(encoding="utf-8") as f:
return [200, headers, f.read()]
http.register_uri(
http.GET, re.compile("^https://pypi.org/(.+?)/(.+?)/json$"), body=callback,
)
@pytest.fixture()
def mock_legacy(http):
def callback(request, uri, headers):
parts = uri.rsplit("/")
name = parts[-2]
fixture = LEGACY_FIXTURES / (name + ".html")
with fixture.open(encoding="utf-8") as f:
return [200, headers, f.read()]
http.register_uri(
http.GET, re.compile("^https://foo.bar/simple/(.+?)$"), body=callback,
)
@pytest.fixture()
def pool():
pool = Pool()
pool.add_repository(PyPiRepository(disable_cache=True))
pool.add_repository(
LegacyRepository("foo", "https://foo.bar/simple/", disable_cache=True)
)
return pool
@pytest.mark.parametrize("source_type", ["", "legacy"])
def test_chooser_chooses_universal_wheel_link_if_available(
env, mock_pypi, mock_legacy, source_type, pool
):
chooser = Chooser(pool, env)
package = Package("pytest", "3.5.0")
if source_type == "legacy":
package.source_type = "legacy"
package.source_reference = "foo"
package.source_url = "https://foo.bar/simple/"
link = chooser.choose_for(package)
assert "pytest-3.5.0-py2.py3-none-any.whl" == link.filename
@pytest.mark.parametrize("source_type", ["", "legacy"])
def test_chooser_chooses_specific_python_universal_wheel_link_if_available(
env, mock_pypi, mock_legacy, source_type, pool
):
chooser = Chooser(pool, env)
package = Package("isort", "4.3.4")
if source_type == "legacy":
package.source_type = "legacy"
package.source_reference = "foo"
package.source_url = "https://foo.bar/simple/"
link = chooser.choose_for(package)
assert "isort-4.3.4-py3-none-any.whl" == link.filename
@pytest.mark.parametrize("source_type", ["", "legacy"])
def test_chooser_chooses_system_specific_wheel_link_if_available(
mock_pypi, mock_legacy, source_type, pool
):
env = MockEnv(
supported_tags=[Tag("cp37", "cp37m", "win32"), Tag("py3", "none", "any")]
)
chooser = Chooser(pool, env)
package = Package("pyyaml", "3.13.0")
if source_type == "legacy":
package.source_type = "legacy"
package.source_reference = "foo"
package.source_url = "https://foo.bar/simple/"
link = chooser.choose_for(package)
assert "PyYAML-3.13-cp37-cp37m-win32.whl" == link.filename
@pytest.mark.parametrize("source_type", ["", "legacy"])
def test_chooser_chooses_sdist_if_no_compatible_wheel_link_is_available(
env, mock_pypi, mock_legacy, source_type, pool,
):
chooser = Chooser(pool, env)
package = Package("pyyaml", "3.13.0")
if source_type == "legacy":
package.source_type = "legacy"
package.source_reference = "foo"
package.source_url = "https://foo.bar/simple/"
link = chooser.choose_for(package)
assert "PyYAML-3.13.tar.gz" == link.filename
@pytest.mark.parametrize("source_type", ["", "legacy"])
def test_chooser_chooses_distributions_that_match_the_package_hashes(
env, mock_pypi, mock_legacy, source_type, pool,
):
chooser = Chooser(pool, env)
package = Package("isort", "4.3.4")
package.files = [
{
"hash": "sha256:b9c40e9750f3d77e6e4d441d8b0266cf555e7cdabdcff33c4fd06366ca761ef8",
"filename": "isort-4.3.4.tar.gz",
}
]
if source_type == "legacy":
package.source_type = "legacy"
package.source_reference = "foo"
package.source_url = "https://foo.bar/simple/"
link = chooser.choose_for(package)
assert "isort-4.3.4.tar.gz" == link.filename
tests/installation/test_executor.py 0 → 100644
# -*- coding: utf-8 -*-
from __future__ import unicode_literals
import re
import pytest
from clikit.api.formatter.style import Style
from clikit.io.buffered_io import BufferedIO
from poetry.config.config import Config
from poetry.core.packages.package import Package
from poetry.installation.executor import Executor
from poetry.installation.operations import Install
from poetry.installation.operations import Uninstall
from poetry.installation.operations import Update
from poetry.repositories.pool import Pool
from poetry.utils._compat import PY36
from poetry.utils._compat import Path
from poetry.utils.env import MockEnv
from tests.repositories.test_pypi_repository import MockRepository
@pytest.fixture()
def io():
io = BufferedIO()
io.formatter.add_style(Style("c1_dark").fg("cyan").dark())
io.formatter.add_style(Style("c2_dark").fg("default").bold().dark())
io.formatter.add_style(Style("success_dark").fg("green").dark())
io.formatter.add_style(Style("warning").fg("yellow"))
return io
@pytest.fixture()
def pool():
pool = Pool()
pool.add_repository(MockRepository())
return pool
@pytest.fixture()
def mock_file_downloads(http):
def callback(request, uri, headers):
fixture = Path(__file__).parent.parent.joinpath(
"fixtures/distributions/demo-0.1.0-py2.py3-none-any.whl"
)
with fixture.open("rb") as f:
return [200, headers, f.read()]
http.register_uri(
http.GET, re.compile("^https://files.pythonhosted.org/.*$"), body=callback,
)
def test_execute_executes_a_batch_of_operations(
config, pool, io, tmp_dir, mock_file_downloads
):
config = Config()
config.merge({"cache-dir": tmp_dir})
env = MockEnv(path=Path(tmp_dir))
executor = Executor(env, pool, config, io)
file_package = Package("demo", "0.1.0")
file_package.source_type = "file"
file_package.source_url = str(
Path(__file__)
.parent.parent.joinpath(
"fixtures/distributions/demo-0.1.0-py2.py3-none-any.whl"
)
.resolve()
)
directory_package = Package("simple-project", "1.2.3")
directory_package.source_type = "directory"
directory_package.source_url = str(
Path(__file__).parent.parent.joinpath("fixtures/simple_project").resolve()
)
git_package = Package("demo", "0.1.0")
git_package.source_type = "git"
git_package.source_reference = "master"
git_package.source_url = "https://github.com/demo/demo.git"
assert 0 == executor.execute(
[
Install(Package("pytest", "3.5.2")),
Uninstall(Package("attrs", "17.4.0")),
Update(Package("requests", "2.18.3"), Package("requests", "2.18.4")),
Uninstall(Package("clikit", "0.2.3")).skip("Not currently installed"),
Install(file_package),
Install(directory_package),
Install(git_package),
]
)
expected = """
Package operations: 4 installs, 1 update, 1 removal
• Installing pytest (3.5.2)
• Removing attrs (17.4.0)
• Updating requests (2.18.3 -> 2.18.4)
• Installing demo (0.1.0 {})
• Installing simple-project (1.2.3 {})
• Installing demo (0.1.0 master)
""".format(
file_package.source_url, directory_package.source_url
)
assert expected == io.fetch_output()
assert 5 == len(env.executed)
def test_execute_shows_skipped_operations_if_verbose(config, pool, io):
config = Config()
config.merge({"cache-dir": "/foo"})
env = MockEnv()
executor = Executor(env, pool, config, io)
executor.verbose()
assert 0 == executor.execute(
[Uninstall(Package("clikit", "0.2.3")).skip("Not currently installed")]
)
expected = """
Package operations: 0 installs, 0 updates, 0 removals, 1 skipped
• Removing clikit (0.2.3): Skipped for the following reason: Not currently installed
"""
assert expected == io.fetch_output()
assert 0 == len(env.executed)
@pytest.mark.skipif(
not PY36, reason="Improved error rendering is only available on Python >=3.6"
)
def test_execute_should_show_errors(config, mocker, io):
env = MockEnv()
executor = Executor(env, pool, config, io)
executor.verbose()
mocker.patch.object(executor, "_install", side_effect=Exception("It failed!"))
assert 1 == executor.execute([Install(Package("clikit", "0.2.3"))])
expected = """
Package operations: 1 install, 0 updates, 0 removals
• Installing clikit (0.2.3)
Exception
It failed!
"""
assert expected in io.fetch_output()
def test_execute_should_show_operation_as_cancelled_on_subprocess_keyboard_interrupt(
config, mocker, io
):
env = MockEnv()
executor = Executor(env, pool, config, io)
executor.verbose()
# A return code of -2 means KeyboardInterrupt in the pip subprocess
mocker.patch.object(executor, "_install", return_value=-2)
assert 1 == executor.execute([Install(Package("clikit", "0.2.3"))])
expected = """
Package operations: 1 install, 0 updates, 0 removals
• Installing clikit (0.2.3)
• Installing clikit (0.2.3): Cancelled
"""
assert expected == io.fetch_output()
tests/repositories/fixtures/legacy/pytest.html 0 → 100644
<!DOCTYPE html>
<html>
<head>
<title>Links for pytest</title>
</head>
<body>
<h1>Links for pytest</h1><a href="https://files.pythonhosted.org/packages/ed/96/271c93f75212c06e2a7ec3e2fa8a9c90acee0a4838dc05bf379ea09aae31/pytest-3.5.0-py2.py3-none-any.whl#sha256=6266f87ab64692112e5477eba395cfedda53b1933ccd29478e671e73b420c19c" data-requires-python="&gt;=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*">pytest-3.5.0-py2.py3-none-any.whl</a><br/>
<a href="https://files.pythonhosted.org/packages/2d/56/6019153cdd743300c5688ab3b07702355283e53c83fbf922242c053ffb7b/pytest-3.5.0.tar.gz#sha256=fae491d1874f199537fd5872b5e1f0e74a009b979df9d53d1553fd03da1703e1" data-requires-python="&gt;=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*">pytest-3.5.0.tar.gz</a><br/>
</body>
</html>
<!--SERIAL 7198641-->
tests/repositories/fixtures/legacy/pyyaml.html
...@@ -5,6 +5,7 @@ ...@@ -5,6 +5,7 @@
</head> </head>
<body> <body>
<h1>Links for python-language-server</h1> <h1>Links for python-language-server</h1>
<a href="https://files.pythonhosted.org/packages/5c/ed/d6557f70daaaab6ee5cd2f8ccf7bedd63081e522e38679c03840e1acc114/PyYAML-3.13-cp37-cp37m-win32.whl#sha256=e170a9e6fcfd19021dd29845af83bb79236068bf5fd4df3327c1be18182b2531">PyYAML-3.13-cp37-cp37m-win32.whl</a><br/>
<a href="https://files.pythonhosted.org/packages/9e/a3/1d13970c3f36777c583f136c136f804d70f500168edc1edea6daa7200769/PyYAML-3.13.tar.gz#sha256=3ef3092145e9b70e3ddd2c7ad59bdd0252a94dfe3949721633e41344de00a6bf">PyYAML-3.13.tar.gz</a><br/> <a href="https://files.pythonhosted.org/packages/9e/a3/1d13970c3f36777c583f136c136f804d70f500168edc1edea6daa7200769/PyYAML-3.13.tar.gz#sha256=3ef3092145e9b70e3ddd2c7ad59bdd0252a94dfe3949721633e41344de00a6bf">PyYAML-3.13.tar.gz</a><br/>
<a href="https://files.pythonhosted.org/packages/0f/9d/f98ed0a460dc540f720bbe5c6e076f025595cdfa3e318fad27165db13cf9/PyYAML-4.2b2.tar.gz#sha256=406b717f739e2d00c49873068b71f5454c2420157db51b082d4d2beb17ffffb6">PyYAML-4.2b2.tar.gz</a><br/> <a href="https://files.pythonhosted.org/packages/0f/9d/f98ed0a460dc540f720bbe5c6e076f025595cdfa3e318fad27165db13cf9/PyYAML-4.2b2.tar.gz#sha256=406b717f739e2d00c49873068b71f5454c2420157db51b082d4d2beb17ffffb6">PyYAML-4.2b2.tar.gz</a><br/>
</body> </body>
......
tox.ini
...@@ -7,5 +7,5 @@ envlist = py27, py35, py36, py37, py38 ...@@ -7,5 +7,5 @@ envlist = py27, py35, py36, py37, py38
whitelist_externals = poetry whitelist_externals = poetry
skip_install = true skip_install = true
commands = commands =
poetry install -vvv poetry install -vv
poetry run pytest {posargs} tests/ poetry run pytest {posargs} tests/
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment