Skip to content

P
python-poetry
Commit c7055be9 by Sébastien Eustace Committed by GitHub
Options

New and faster installer implementation (#2595) 

* Improve the way packages are installed

* Add support for parallelized operations

* Improve the Chooser class

* Make the new installer the default

* Add an Authenticator class to be able to download from protected urls

* Adapt code and tests to latest changes

* Update lock file and some dependencies

* Improve installations information and caching

* Make the final adjustments and tests for the new installer

* Use the preview version of Poetry for the CI

* Rename Executor.run() to Executor.run_pip()

* Gracefully handle errors when executing operations
parent d6289470
Showing with 1443 additions and 182 deletions
.cirrus.yml
......@@ -16,7 +16,8 @@ test_task:
- pkg install -y git-lite $PYPACKAGE $SQLPACKAGE
pip_script:
- $PYTHON -m ensurepip
- $PYTHON -m pip install -U pip tox poetry
- $PYTHON -m pip install -U pip tox
- $PYTHON -m pip install -U --pre poetry
- poetry config virtualenvs.in-project true
tox_script: $PYTHON -m tox -e py -- -q --junitxml=junit.xml tests
on_failure:
......
.flake8
......@@ -16,6 +16,7 @@ exclude =
.vscode
.github
poetry/utils/_compat.py
poetry/utils/env_scripts/tags.py
tests/fixtures/
tests/repositories/fixtures/
tests/utils/fixtures/
.github/workflows/main.yml
......@@ -49,7 +49,7 @@ jobs:
shell: bash
run: |
curl -fsS -o get-poetry.py https://raw.githubusercontent.com/python-poetry/poetry/master/get-poetry.py
python get-poetry.py -y
python get-poetry.py -y --preview
echo "::set-env name=PATH::$HOME/.poetry/bin:$PATH"
- name: Configure poetry
......
poetry/config/config.py
......@@ -37,6 +37,7 @@ class Config(object):
"in-project": False,
"path": os.path.join("{cache-dir}", "virtualenvs"),
},
"experimental": {"new-installer": True},
}
def __init__(
......
poetry/console/commands/add.py
from cleo import argument
from cleo import option
from .env_command import EnvCommand
from .init import InitCommand
from .installer_command import InstallerCommand
class AddCommand(EnvCommand, InitCommand):
class AddCommand(InstallerCommand, InitCommand):
name = "add"
description = "Adds a new dependency to <comment>pyproject.toml</>."
......@@ -56,7 +56,6 @@ If you do not specify a version constraint, poetry will choose a suitable one ba
loggers = ["poetry.repositories.pypi_repository"]
def handle(self):
from poetry.installation.installer import Installer
from poetry.core.semver import parse_constraint
from tomlkit import inline_table
......@@ -149,18 +148,17 @@ If you do not specify a version constraint, poetry will choose a suitable one ba
# Update packages
self.reset_poetry()
installer = Installer(
self.io, self.env, self.poetry.package, self.poetry.locker, self.poetry.pool
)
installer.dry_run(self.option("dry-run"))
installer.update(True)
self._installer.set_package(self.poetry.package)
self._installer.dry_run(self.option("dry-run"))
self._installer.verbose(self._io.is_verbose())
self._installer.update(True)
if self.option("lock"):
installer.lock()
installer.whitelist([r["name"] for r in requirements])
self._installer.lock()
self._installer.whitelist([r["name"] for r in requirements])
try:
status = installer.run()
status = self._installer.run()
except Exception:
self.poetry.file.write(original_content)
......@@ -169,10 +167,10 @@ If you do not specify a version constraint, poetry will choose a suitable one ba
if status != 0 or self.option("dry-run"):
# Revert changes
if not self.option("dry-run"):
self.error(
self.line_error(
"\n"
"Addition failed, reverting pyproject.toml "
"to its original content."
"<error>Failed to add packages, reverting the pyproject.toml file "
"to its original content.</error>"
)
self.poetry.file.write(original_content)
......
poetry/console/commands/config.py
......@@ -57,6 +57,11 @@ To remove a repository (repo is a short alias for repositories):
lambda val: str(Path(val)),
str(Path(CACHE_DIR) / "virtualenvs"),
),
"experimental.new-installer": (
boolean_validator,
boolean_normalizer,
True,
),
}
return unique_config_values
......
poetry/console/commands/install.py
from cleo import option
from .env_command import EnvCommand
from .installer_command import InstallerCommand
class InstallCommand(EnvCommand):
class InstallCommand(InstallerCommand):
name = "install"
description = "Installs the project dependencies."
......@@ -48,12 +48,11 @@ dependencies and not including the current project, run the command with the
_loggers = ["poetry.repositories.pypi_repository"]
def handle(self):
from poetry.installation.installer import Installer
from poetry.masonry.builders import EditableBuilder
from poetry.core.masonry.utils.module import ModuleOrPackageNotFound
installer = Installer(
self.io, self.env, self.poetry.package, self.poetry.locker, self.poetry.pool
self._installer.use_executor(
self.poetry.config.get("experimental.new-installer", False)
)
extras = []
......@@ -63,13 +62,13 @@ dependencies and not including the current project, run the command with the
else:
extras.append(extra)
installer.extras(extras)
installer.dev_mode(not self.option("no-dev"))
installer.dry_run(self.option("dry-run"))
installer.remove_untracked(self.option("remove-untracked"))
installer.verbose(self.option("verbose"))
self._installer.extras(extras)
self._installer.dev_mode(not self.option("no-dev"))
self._installer.dry_run(self.option("dry-run"))
self._installer.remove_untracked(self.option("remove-untracked"))
self._installer.verbose(self._io.is_verbose())
return_code = installer.run()
return_code = self._installer.run()
if return_code != 0:
return return_code
......@@ -85,15 +84,32 @@ dependencies and not including the current project, run the command with the
# If this is a true error it will be picked up later by build anyway.
return 0
self.line(
" - Installing <c1>{}</c1> (<c2>{}</c2>)".format(
self.poetry.package.pretty_name, self.poetry.package.pretty_version
self.line("")
if not self._io.supports_ansi() or self.io.is_debug():
self.line(
"<b>Installing</> the current project: <c1>{}</c1> (<c2>{}</c2>)".format(
self.poetry.package.pretty_name, self.poetry.package.pretty_version
)
)
else:
self.write(
"<b>Installing</> the current project: <c1>{}</c1> (<c2>{}</c2>)".format(
self.poetry.package.pretty_name, self.poetry.package.pretty_version
)
)
)
if self.option("dry-run"):
self.line("")
return 0
builder.build()
if self._io.supports_ansi() and not self.io.is_debug():
self.overwrite(
"<b>Installing</> the current project: <c1>{}</c1> (<success>{}</success>)".format(
self.poetry.package.pretty_name, self.poetry.package.pretty_version
)
)
self.line("")
return 0
poetry/console/commands/installer_command.py 0 → 100644
from typing import TYPE_CHECKING
from .env_command import EnvCommand
if TYPE_CHECKING:
from poetry.installation.installer import Installer
class InstallerCommand(EnvCommand):
def __init__(self):
self._installer = None
super(InstallerCommand, self).__init__()
@property
def installer(self): # type: () -> Installer
return self._installer
def set_installer(self, installer): # type: (Installer) -> None
self._installer = installer
poetry/console/commands/lock.py
from .env_command import EnvCommand
from .installer_command import InstallerCommand
class LockCommand(EnvCommand):
class LockCommand(InstallerCommand):
name = "lock"
description = "Locks the project dependencies."
......@@ -17,12 +17,10 @@ file.
loggers = ["poetry.repositories.pypi_repository"]
def handle(self):
from poetry.installation.installer import Installer
installer = Installer(
self.io, self.env, self.poetry.package, self.poetry.locker, self.poetry.pool
self._installer.use_executor(
self.poetry.config.get("experimental.new-installer", False)
)
installer.lock()
self._installer.lock()
return installer.run()
return self._installer.run()
poetry/console/commands/remove.py
from cleo import argument
from cleo import option
from .env_command import EnvCommand
from .installer_command import InstallerCommand
class RemoveCommand(EnvCommand):
class RemoveCommand(InstallerCommand):
name = "remove"
description = "Removes a package from the project dependencies."
......@@ -28,8 +28,6 @@ list of installed packages
loggers = ["poetry.repositories.pypi_repository"]
def handle(self):
from poetry.installation.installer import Installer
packages = self.argument("packages")
is_dev = self.option("dev")
......@@ -62,16 +60,18 @@ list of installed packages
# Update packages
self.reset_poetry()
installer = Installer(
self.io, self.env, self.poetry.package, self.poetry.locker, self.poetry.pool
self._installer.set_package(self.poetry.package)
self._installer.use_executor(
self.poetry.config.get("experimental.new-installer", False)
)
installer.dry_run(self.option("dry-run"))
installer.update(True)
installer.whitelist(requirements)
self._installer.dry_run(self.option("dry-run"))
self._installer.verbose(self._io.is_verbose())
self._installer.update(True)
self._installer.whitelist(requirements)
try:
status = installer.run()
status = self._installer.run()
except Exception:
self.poetry.file.write(original_content)
......@@ -80,7 +80,7 @@ list of installed packages
if status != 0 or self.option("dry-run"):
# Revert changes
if not self.option("dry-run"):
self.error(
self.line_error(
"\n"
"Removal failed, reverting pyproject.toml "
"to its original content."
......
poetry/console/commands/update.py
from cleo import argument
from cleo import option
from .env_command import EnvCommand
from .installer_command import InstallerCommand
class UpdateCommand(EnvCommand):
class UpdateCommand(InstallerCommand):
name = "update"
description = (
......@@ -28,22 +28,20 @@ class UpdateCommand(EnvCommand):
loggers = ["poetry.repositories.pypi_repository"]
def handle(self):
from poetry.installation.installer import Installer
packages = self.argument("packages")
installer = Installer(
self.io, self.env, self.poetry.package, self.poetry.locker, self.poetry.pool
self._installer.use_executor(
self.poetry.config.get("experimental.new-installer", False)
)
if packages:
installer.whitelist({name: "*" for name in packages})
self._installer.whitelist({name: "*" for name in packages})
installer.dev_mode(not self.option("no-dev"))
installer.dry_run(self.option("dry-run"))
installer.execute_operations(not self.option("lock"))
self._installer.dev_mode(not self.option("no-dev"))
self._installer.dry_run(self.option("dry-run"))
self._installer.execute_operations(not self.option("lock"))
# Force update
installer.update(True)
self._installer.update(True)
return installer.run()
return self._installer.run()
poetry/console/config/application_config.py
......@@ -27,6 +27,7 @@ from clikit.io.output_stream import StandardOutputStream
from poetry.console.commands.command import Command
from poetry.console.commands.env_command import EnvCommand
from poetry.console.commands.installer_command import InstallerCommand
from poetry.console.logging.io_formatter import IOFormatter
from poetry.console.logging.io_handler import IOHandler
from poetry.utils._compat import PY36
......@@ -37,15 +38,22 @@ class ApplicationConfig(BaseApplicationConfig):
super(ApplicationConfig, self).configure()
self.add_style(Style("c1").fg("cyan"))
self.add_style(Style("c2").fg("green"))
self.add_style(Style("c2").fg("default").bold())
self.add_style(Style("info").fg("blue"))
self.add_style(Style("comment").fg("green"))
self.add_style(Style("error").fg("red").bold())
self.add_style(Style("warning").fg("yellow").bold())
self.add_style(Style("debug").fg("default").dark())
self.add_style(Style("success").fg("green"))
# Dark variants
self.add_style(Style("c1_dark").fg("cyan").dark())
self.add_style(Style("c2_dark").fg("default").bold().dark())
self.add_style(Style("success_dark").fg("green").dark())
self.add_event_listener(PRE_HANDLE, self.register_command_loggers)
self.add_event_listener(PRE_HANDLE, self.set_env)
self.add_event_listener(PRE_HANDLE, self.set_installer)
if PY36:
from poetry.mixology.solutions.providers import (
......@@ -93,6 +101,9 @@ class ApplicationConfig(BaseApplicationConfig):
if not isinstance(command, EnvCommand):
return
if command.env is not None:
return
io = event.io
poetry = command.poetry
......@@ -104,6 +115,32 @@ class ApplicationConfig(BaseApplicationConfig):
command.set_env(env)
def set_installer(
self, event, event_name, _
): # type: (PreHandleEvent, str, Any) -> None
command = event.command.config.handler # type: InstallerCommand
if not isinstance(command, InstallerCommand):
return
# If the command already has an installer
# we skip this step
if command.installer is not None:
return
from poetry.installation.installer import Installer
poetry = command.poetry
installer = Installer(
event.io,
command.env,
poetry.package,
poetry.locker,
poetry.pool,
poetry.config,
)
installer.use_executor(poetry.config.get("experimental.new-installer", False))
command.set_installer(installer)
def resolve_help_command(
self, event, event_name, dispatcher
): # type: (PreResolveEvent, str, EventDispatcher) -> None
......
poetry/installation/authenticator.py 0 → 100644
from typing import TYPE_CHECKING
from poetry.utils._compat import urlparse
from poetry.utils.password_manager import PasswordManager
if TYPE_CHECKING:
from typing import Any
from typing import Optional
from typing import Tuple
from clikit.api.io import IO
from requests import Request # noqa
from requests import Response # noqa
from requests import Session # noqa
from poetry.config.config import Config
class Authenticator(object):
def __init__(self, config, io): # type: (Config, IO) -> None
self._config = config
self._io = io
self._session = None
self._credentials = {}
self._password_manager = PasswordManager(self._config)
@property
def session(self): # type: () -> Session
from requests import Session # noqa
if self._session is None:
self._session = Session()
return self._session
def request(self, method, url, **kwargs): # type: (str, str, Any) -> Response
from requests import Request # noqa
from requests.auth import HTTPBasicAuth
request = Request(method, url)
username, password = self._get_credentials_for_url(url)
if username is not None and password is not None:
request = HTTPBasicAuth(username, password)(request)
session = self.session
prepared_request = session.prepare_request(request)
proxies = kwargs.get("proxies", {})
stream = kwargs.get("stream")
verify = kwargs.get("verify")
cert = kwargs.get("cert")
settings = session.merge_environment_settings(
prepared_request.url, proxies, stream, verify, cert
)
# Send the request.
send_kwargs = {
"timeout": kwargs.get("timeout"),
"allow_redirects": kwargs.get("allow_redirects", True),
}
send_kwargs.update(settings)
resp = session.send(prepared_request, **send_kwargs)
resp.raise_for_status()
return resp
def _get_credentials_for_url(
self, url
): # type: (str) -> Tuple[Optional[str], Optional[str]]
parsed_url = urlparse.urlsplit(url)
netloc = parsed_url.netloc
credentials = self._credentials.get(netloc, (None, None))
if credentials == (None, None):
if "@" not in netloc:
credentials = self._get_credentials_for_netloc_from_config(netloc)
else:
# Split from the right because that's how urllib.parse.urlsplit()
# behaves if more than one @ is present (which can be checked using
# the password attribute of urlsplit()'s return value).
auth, netloc = netloc.rsplit("@", 1)
if ":" in auth:
# Split from the left because that's how urllib.parse.urlsplit()
# behaves if more than one : is present (which again can be checked
# using the password attribute of the return value)
credentials = auth.split(":", 1)
else:
credentials = auth, None
credentials = tuple(
None if x is None else urlparse.unquote(x) for x in credentials
)
if credentials[0] is not None or credentials[1] is not None:
credentials = (credentials[0] or "", credentials[1] or "")
self._credentials[netloc] = credentials
return credentials[0], credentials[1]
def _get_credentials_for_netloc_from_config(
self, netloc
): # type: (str) -> Tuple[Optional[str], Optional[str]]
credentials = (None, None)
for repository_name in self._config.get("http-basic", {}):
repository_config = self._config.get(
"repositories.{}".format(repository_name)
)
if not repository_config:
continue
url = repository_config.get("url")
if not url:
continue
parsed_url = urlparse.urlsplit(url)
if netloc == parsed_url.netloc:
auth = self._password_manager.get_http_auth(repository_name)
if auth is None:
continue
return auth["username"], auth["password"]
return credentials
poetry/installation/chef.py 0 → 100644
import hashlib
import json
from typing import TYPE_CHECKING
from poetry.core.packages.utils.link import Link
from poetry.utils._compat import Path
from .chooser import InvalidWheelName
from .chooser import Wheel
if TYPE_CHECKING:
from typing import List
from typing import Optional
from poetry.config.config import Config
from poetry.utils.env import Env
class Chef:
def __init__(self, config, env): # type: (Config, Env) -> None
self._config = config
self._env = env
self._cache_dir = (
Path(config.get("cache-dir")).expanduser().joinpath("artifacts")
)
def prepare(self, archive): # type: (Path) -> Path
return archive
def prepare_sdist(self, archive): # type: (Path) -> Path
return archive
def prepare_wheel(self, archive): # type: (Path) -> Path
return archive
def should_prepare(self, archive): # type: (Path) -> bool
return not self.is_wheel(archive)
def is_wheel(self, archive): # type: (Path) -> bool
return archive.suffix == ".whl"
def get_cached_archive_for_link(self, link): # type: (Link) -> Optional[Link]
# If the archive is already a wheel, there is no need to cache it.
if link.is_wheel:
pass
archives = self.get_cached_archives_for_link(link)
if not archives:
return link
candidates = []
for archive in archives:
if not archive.is_wheel:
candidates.append((float("inf"), archive))
continue
try:
wheel = Wheel(archive.filename)
except InvalidWheelName:
continue
if not wheel.is_supported_by_environment(self._env):
continue
candidates.append(
(wheel.get_minimum_supported_index(self._env.supported_tags), archive),
)
if not candidates:
return link
return min(candidates)[1]
def get_cached_archives_for_link(self, link): # type: (Link) -> List[Link]
cache_dir = self.get_cache_directory_for_link(link)
archive_types = ["whl", "tar.gz", "tar.bz2", "bz2", "zip"]
links = []
for archive_type in archive_types:
for archive in cache_dir.glob("*.{}".format(archive_type)):
links.append(Link("file://{}".format(str(archive))))
return links
def get_cache_directory_for_link(self, link): # type: (Link) -> Path
key_parts = {"url": link.url_without_fragment}
if link.hash_name is not None and link.hash is not None:
key_parts[link.hash_name] = link.hash
if link.subdirectory_fragment:
key_parts["subdirectory"] = link.subdirectory_fragment
key_parts["interpreter_name"] = self._env.marker_env["interpreter_name"]
key_parts["interpreter_version"] = "".join(
self._env.marker_env["interpreter_version"].split(".")[:2]
)
key = hashlib.sha256(
json.dumps(
key_parts, sort_keys=True, separators=(",", ":"), ensure_ascii=True
).encode("ascii")
).hexdigest()
split_key = [key[:2], key[2:4], key[4:6], key[6:]]
return self._cache_dir.joinpath(*split_key)
poetry/installation/chooser.py 0 → 100644
import re
from typing import List
from typing import Tuple
from packaging.tags import Tag
from poetry.core.packages.package import Package
from poetry.core.packages.utils.link import Link
from poetry.repositories.pool import Pool
from poetry.utils.env import Env
from poetry.utils.patterns import wheel_file_re
class InvalidWheelName(Exception):
pass
class Wheel(object):
def __init__(self, filename): # type: (str) -> None
wheel_info = wheel_file_re.match(filename)
if not wheel_info:
raise InvalidWheelName("{} is not a valid wheel filename.".format(filename))
self.filename = filename
self.name = wheel_info.group("name").replace("_", "-")
self.version = wheel_info.group("ver").replace("_", "-")
self.build_tag = wheel_info.group("build")
self.pyversions = wheel_info.group("pyver").split(".")
self.abis = wheel_info.group("abi").split(".")
self.plats = wheel_info.group("plat").split(".")
self.tags = {
Tag(x, y, z) for x in self.pyversions for y in self.abis for z in self.plats
}
def get_minimum_supported_index(self, tags):
indexes = [tags.index(t) for t in self.tags if t in tags]
return min(indexes) if indexes else None
def is_supported_by_environment(self, env):
return bool(set(env.supported_tags).intersection(self.tags))
class Chooser:
"""
A Chooser chooses an appropriate release archive for packages.
"""
def __init__(self, pool, env): # type: (Pool, Env) -> None
self._pool = pool
self._env = env
def choose_for(self, package): # type: (Package) -> Link
"""
Return the url of the selected archive for a given package.
"""
links = []
for link in self._get_links(package):
if link.is_wheel and not Wheel(link.filename).is_supported_by_environment(
self._env
):
continue
if link.ext == ".egg":
continue
links.append(link)
if not links:
raise RuntimeError(
"Unable to find installation candidates for {}".format(package)
)
# Get the best link
chosen = max(links, key=lambda link: self._sort_key(package, link))
if not chosen:
raise RuntimeError(
"Unable to find installation candidates for {}".format(package)
)
return chosen
def _get_links(self, package): # type: (Package) -> List[Link]
if not package.source_type:
if not self._pool.has_repository("pypi"):
repository = self._pool.repositories[0]
else:
repository = self._pool.repository("pypi")
else:
repository = self._pool.repository(package.source_reference)
links = repository.find_links_for_package(package)
hashes = [f["hash"] for f in package.files]
if not hashes:
return links
selected_links = []
for link in links:
if not link.hash:
selected_links.append(link)
continue
h = link.hash_name + ":" + link.hash
if h not in hashes:
continue
selected_links.append(link)
return selected_links
def _sort_key(self, package, link): # type: (Package, Link) -> Tuple
"""
Function to pass as the `key` argument to a call to sorted() to sort
InstallationCandidates by preference.
Returns a tuple such that tuples sorting as greater using Python's
default comparison operator are more preferred.
The preference is as follows:
First and foremost, candidates with allowed (matching) hashes are
always preferred over candidates without matching hashes. This is
because e.g. if the only candidate with an allowed hash is yanked,
we still want to use that candidate.
Second, excepting hash considerations, candidates that have been
yanked (in the sense of PEP 592) are always less preferred than
candidates that haven't been yanked. Then:
If not finding wheels, they are sorted by version only.
If finding wheels, then the sort order is by version, then:
1. existing installs
2. wheels ordered via Wheel.support_index_min(self._supported_tags)
3. source archives
If prefer_binary was set, then all wheels are sorted above sources.
Note: it was considered to embed this logic into the Link
comparison operators, but then different sdist links
with the same version, would have to be considered equal
"""
support_num = len(self._env.supported_tags)
build_tag = ()
binary_preference = 0
if link.is_wheel:
wheel = Wheel(link.filename)
if not wheel.is_supported_by_environment(self._env):
raise RuntimeError(
"{} is not a supported wheel for this platform. It "
"can't be sorted.".format(wheel.filename)
)
# TODO: Binary preference
pri = -(wheel.get_minimum_supported_index(self._env.supported_tags))
if wheel.build_tag is not None:
match = re.match(r"^(\d+)(.*)$", wheel.build_tag)
build_tag_groups = match.groups()
build_tag = (int(build_tag_groups[0]), build_tag_groups[1])
else: # sdist
pri = -support_num
has_allowed_hash = int(self._is_link_hash_allowed_for_package(link, package))
# TODO: Proper yank value
yank_value = 0
return (
has_allowed_hash,
yank_value,
binary_preference,
package.version,
build_tag,
pri,
)
def _is_link_hash_allowed_for_package(
self, link, package
): # type: (Link, Package) -> bool
if not link.hash:
return True
h = link.hash_name + ":" + link.hash
return h in {f["hash"] for f in package.files}
poetry/installation/installer.py
from typing import List
from typing import Optional
from typing import Union
from clikit.api.io import IO
from poetry.config.config import Config
from poetry.core.packages.project_package import ProjectPackage
from poetry.io.null_io import NullIO
from poetry.packages import Locker
from poetry.puzzle import Solver
from poetry.puzzle.operations import Install
from poetry.puzzle.operations import Uninstall
from poetry.puzzle.operations import Update
from poetry.puzzle.operations.operation import Operation
from poetry.repositories import Pool
from poetry.repositories import Repository
from poetry.repositories.installed_repository import InstalledRepository
......@@ -18,6 +15,11 @@ from poetry.utils.extras import get_extra_package_names
from poetry.utils.helpers import canonicalize_name
from .base_installer import BaseInstaller
from .executor import Executor
from .operations import Install
from .operations import Uninstall
from .operations import Update
from .operations.operation import Operation
from .pip_installer import PipInstaller
......@@ -29,7 +31,9 @@ class Installer:
package, # type: ProjectPackage
locker, # type: Locker
pool, # type: Pool
installed=None, # type: (Union[InstalledRepository, None])
config, # type: Config
installed=None, # type: Union[InstalledRepository, None]
executor=None, # type: Optional[Executor]
):
self._io = io
self._env = env
......@@ -50,6 +54,12 @@ class Installer:
self._extras = []
if executor is None:
executor = Executor(self._env, self._pool, config, self._io)
self._executor = executor
self._use_executor = False
self._installer = self._get_installer()
if installed is None:
installed = self._get_installed()
......@@ -57,9 +67,18 @@ class Installer:
self._installed_repository = installed
@property
def executor(self):
return self._executor
@property
def installer(self):
return self._installer
def set_package(self, package): # type: (ProjectPackage) -> Installer
self._package = package
return self
def run(self):
# Force update if there is no lock file present
if not self._update and not self._locker.is_locked():
......@@ -71,12 +90,12 @@ class Installer:
self._execute_operations = False
local_repo = Repository()
self._do_install(local_repo)
return 0
return self._do_install(local_repo)
def dry_run(self, dry_run=True): # type: (bool) -> Installer
self._dry_run = dry_run
self._executor.dry_run(dry_run)
return self
......@@ -93,6 +112,7 @@ class Installer:
def verbose(self, verbose=True): # type: (bool) -> Installer
self._verbose = verbose
self._executor.verbose(verbose)
return self
......@@ -128,6 +148,9 @@ class Installer:
def execute_operations(self, execute=True): # type: (bool) -> Installer
self._execute_operations = execute
if not execute:
self._executor.disable()
return self
def whitelist(self, packages): # type: (dict) -> Installer
......@@ -140,7 +163,14 @@ class Installer:
return self
def use_executor(self, use_executor=True): # type: (bool) -> Installer
self._use_executor = use_executor
return self
def _do_install(self, local_repo):
from poetry.puzzle import Solver
locked_repository = Repository()
if self._update:
if self._locker.is_locked() and not self._lock:
......@@ -247,19 +277,30 @@ class Installer:
# or optional and not requested, are dropped
self._filter_operations(ops, local_repo)
self._io.write_line("")
# Execute operations
actual_ops = [op for op in ops if not op.skipped]
if not actual_ops and (self._execute_operations or self._dry_run):
return self._execute(ops)
def _write_lock_file(self, repo): # type: (Repository) -> None
if self._update and self._write_lock:
updated_lock = self._locker.set_lock_data(self._package, repo.packages)
if updated_lock:
self._io.write_line("")
self._io.write_line("<info>Writing lock file</>")
def _execute(self, operations):
if self._use_executor:
return self._executor.execute(operations)
if not operations and (self._execute_operations or self._dry_run):
self._io.write_line("No dependencies to install or update")
if actual_ops and (self._execute_operations or self._dry_run):
if operations and (self._execute_operations or self._dry_run):
installs = 0
updates = 0
uninstalls = 0
skipped = 0
for op in ops:
for op in operations:
if op.skipped:
skipped += 1
elif op.job_type == "install":
......@@ -289,18 +330,13 @@ class Installer:
)
self._io.write_line("")
for op in ops:
self._execute(op)
def _write_lock_file(self, repo): # type: (Repository) -> None
if self._update and self._write_lock:
updated_lock = self._locker.set_lock_data(self._package, repo.packages)
for op in operations:
self._execute_operation(op)
if updated_lock:
self._io.write_line("")
self._io.write_line("<info>Writing lock file</>")
return 0
def _execute(self, operation): # type: (Operation) -> None
def _execute_operation(self, operation): # type: (Operation) -> None
"""
Execute a given operation.
"""
......
poetry/puzzle/operations/install.py → poetry/installation/operations/install.py
......@@ -2,8 +2,8 @@ from .operation import Operation
class Install(Operation):
def __init__(self, package, reason=None):
super(Install, self).__init__(reason)
def __init__(self, package, reason=None, priority=0):
super(Install, self).__init__(reason, priority=priority)
self._package = package
......
poetry/puzzle/operations/operation.py → poetry/installation/operations/operation.py
......@@ -4,11 +4,14 @@ from typing import Union
class Operation(object):
def __init__(self, reason=None): # type: (Union[str, None]) -> None
def __init__(
self, reason=None, priority=0
): # type: (Union[str, None], int) -> None
self._reason = reason
self._skipped = False
self._skip_reason = None
self._priority = priority
@property
def job_type(self): # type: () -> str
......@@ -27,6 +30,10 @@ class Operation(object):
return self._skip_reason
@property
def priority(self): # type: () -> int
return self._priority
@property
def package(self):
raise NotImplementedError()
......
poetry/puzzle/operations/uninstall.py → poetry/installation/operations/uninstall.py
......@@ -2,8 +2,8 @@ from .operation import Operation
class Uninstall(Operation):
def __init__(self, package, reason=None):
super(Uninstall, self).__init__(reason)
def __init__(self, package, reason=None, priority=float("inf")):
super(Uninstall, self).__init__(reason, priority=priority)
self._package = package
......
poetry/puzzle/operations/update.py → poetry/installation/operations/update.py
......@@ -2,11 +2,11 @@ from .operation import Operation
class Update(Operation):
def __init__(self, initial, target, reason=None):
def __init__(self, initial, target, reason=None, priority=0):
self._initial_package = initial
self._target_package = target
super(Update, self).__init__(reason)
super(Update, self).__init__(reason, priority=priority)
@property
def initial_package(self):
......
poetry/puzzle/solver.py
......@@ -10,6 +10,10 @@ from clikit.io import ConsoleIO
from poetry.core.packages import Package
from poetry.core.packages.project_package import ProjectPackage
from poetry.installation.operations import Install
from poetry.installation.operations import Uninstall
from poetry.installation.operations import Update
from poetry.installation.operations.operation import Operation
from poetry.mixology import resolve_version
from poetry.mixology.failure import SolveFailure
from poetry.packages import DependencyPackage
......@@ -19,10 +23,6 @@ from poetry.utils.env import Env
from .exceptions import OverrideNeeded
from .exceptions import SolverProblemError
from .operations import Install
from .operations import Uninstall
from .operations import Update
from .operations.operation import Operation
from .provider import Provider
......@@ -78,7 +78,7 @@ class Solver:
)
operations = []
for package in packages:
for i, package in enumerate(packages):
installed = False
for pkg in self._installed.packages:
if package.name == pkg.name:
......@@ -113,23 +113,27 @@ class Solver:
package.source_reference
)
):
operations.append(Update(pkg, package))
operations.append(Update(pkg, package, priority=depths[i]))
else:
operations.append(
Install(package).skip("Already installed")
)
elif package.version != pkg.version:
# Checking version
operations.append(Update(pkg, package))
operations.append(Update(pkg, package, priority=depths[i]))
elif pkg.source_type and package.source_type != pkg.source_type:
operations.append(Update(pkg, package))
operations.append(Update(pkg, package, priority=depths[i]))
else:
operations.append(Install(package).skip("Already installed"))
operations.append(
Install(package, priority=depths[i]).skip(
"Already installed"
)
)
break
if not installed:
operations.append(Install(package))
operations.append(Install(package, priority=depths[i]))
# Checking for removals
for pkg in self._locked.packages:
......@@ -165,15 +169,7 @@ class Solver:
operations.append(Uninstall(installed))
return sorted(
operations,
key=lambda o: (
o.job_type == "uninstall",
# Packages to be uninstalled have no depth so we default to 0
# since it actually doesn't matter since removals are always on top.
-depths[packages.index(o.package)] if o.job_type != "uninstall" else 0,
o.package.name,
o.package.version,
),
operations, key=lambda o: (-o.priority, o.package.name, o.package.version,),
)
def solve_in_compatibility_mode(self, overrides, use_latest=None):
......
poetry/repositories/installed_repository.py
......@@ -90,6 +90,7 @@ class InstalledRepository(Repository):
# TODO: handle multiple source directories?
package.source_type = "directory"
package.source_url = paths.pop().as_posix()
continue
src_path = env.path / "src"
......
poetry/repositories/legacy_repository.py
......@@ -304,6 +304,13 @@ class LegacyRepository(PyPiRepository):
return package
def find_links_for_package(self, package):
page = self._get("/{}/".format(package.name.replace(".", "-")))
if page is None:
return []
return list(page.links_for_version(package.version))
def _get_release_info(self, name, version): # type: (str, str) -> dict
page = self._get("/{}/".format(canonicalize_name(name).replace(".", "-")))
if page is None:
......
poetry/repositories/pool.py
......@@ -33,7 +33,15 @@ class Pool(BaseRepository):
def has_default(self): # type: () -> bool
return self._default
def has_repository(self, name): # type: (str) -> bool
name = name.lower() if name is not None else None
return name in self._lookup
def repository(self, name): # type: (str) -> Repository
if name is not None:
name = name.lower()
if name in self._lookup:
return self._repositories[self._lookup[name]]
......@@ -45,6 +53,9 @@ class Pool(BaseRepository):
"""
Adds a repository to the pool.
"""
repository_name = (
repository.name.lower() if repository.name is not None else None
)
if default:
if self.has_default():
raise ValueError("Only one repository can be the default")
......@@ -57,17 +68,17 @@ class Pool(BaseRepository):
if self._secondary_start_idx is not None:
self._secondary_start_idx += 1
self._lookup[repository.name] = 0
self._lookup[repository_name] = 0
elif secondary:
if self._secondary_start_idx is None:
self._secondary_start_idx = len(self._repositories)
self._repositories.append(repository)
self._lookup[repository.name] = len(self._repositories) - 1
self._lookup[repository_name] = len(self._repositories) - 1
else:
if self._secondary_start_idx is None:
self._repositories.append(repository)
self._lookup[repository.name] = len(self._repositories) - 1
self._lookup[repository_name] = len(self._repositories) - 1
else:
self._repositories.insert(self._secondary_start_idx, repository)
......@@ -77,12 +88,15 @@ class Pool(BaseRepository):
self._lookup[name] += 1
self._lookup[repository.name] = self._secondary_start_idx
self._lookup[repository_name] = self._secondary_start_idx
self._secondary_start_idx += 1
return self
def remove_repository(self, repository_name): # type: (str) -> Pool
if repository_name is not None:
repository_name = repository_name.lower()
idx = self._lookup.get(repository_name)
if idx is not None:
del self._repositories[idx]
......@@ -95,6 +109,9 @@ class Pool(BaseRepository):
def package(
self, name, version, extras=None, repository=None
): # type: (str, str, List[str], str) -> "Package"
if repository is not None:
repository = repository.lower()
if (
repository is not None
and repository not in self._lookup
......@@ -104,9 +121,7 @@ class Pool(BaseRepository):
if repository is not None and not self._ignore_repository_names:
try:
return self._repositories[self._lookup[repository]].package(
name, version, extras=extras
)
return self.repository(repository).package(name, version, extras=extras)
except PackageNotFound:
pass
else:
......@@ -131,6 +146,9 @@ class Pool(BaseRepository):
allow_prereleases=False,
repository=None,
):
if repository is not None:
repository = repository.lower()
if (
repository is not None
and repository not in self._lookup
......@@ -139,7 +157,7 @@ class Pool(BaseRepository):
raise ValueError('Repository "{}" does not exist.'.format(repository))
if repository is not None and not self._ignore_repository_names:
return self._repositories[self._lookup[repository]].find_packages(
return self.repository(repository).find_packages(
name, constraint, extras=extras, allow_prereleases=allow_prereleases
)
......
poetry/repositories/pypi_repository.py
......@@ -241,6 +241,18 @@ class PyPiRepository(RemoteRepository):
return PackageInfo.load(cached)
def find_links_for_package(self, package):
json_data = self._get("pypi/{}/{}/json".format(package.name, package.version))
if json_data is None:
return []
links = []
for url in json_data["urls"]:
h = "sha256={}".format(url["digests"]["sha256"])
links.append(Link(url["url"] + "#" + h))
return links
def _get_release_info(self, name, version): # type: (str, str) -> dict
self._log("Getting info for {} ({}) from PyPI".format(name, version), "debug")
......
poetry/repositories/repository.py
......@@ -6,10 +6,10 @@ from .base_repository import BaseRepository
class Repository(BaseRepository):
def __init__(self, packages=None):
def __init__(self, packages=None, name=None):
super(Repository, self).__init__()
self._name = None
self._name = name
if packages is None:
packages = []
......@@ -115,6 +115,9 @@ class Repository(BaseRepository):
if index is not None:
del self._packages[index]
def find_links_for_package(self, package):
return []
def search(self, query):
results = []
......
poetry/utils/_compat.py
......@@ -23,6 +23,11 @@ try:
except ImportError:
import urlparse
try:
from os import cpu_count
except ImportError: # Python 2
from multiprocessing import cpu_count
try: # Python 2
long = long
unicode = unicode
......@@ -50,6 +55,14 @@ else:
shell_quote = shlex.quote
if PY34:
from importlib.machinery import EXTENSION_SUFFIXES
else:
from imp import get_suffixes
EXTENSION_SUFFIXES = [suffix[0] for suffix in get_suffixes()]
if PY35:
from pathlib import Path
else:
......
poetry/utils/env.py
......@@ -7,7 +7,7 @@ import re
import shutil
import sys
import sysconfig
import warnings
import textwrap
from contextlib import contextmanager
from typing import Any
......@@ -20,6 +20,12 @@ import tomlkit
from clikit.api.io import IO
import packaging.tags
from packaging.tags import Tag
from packaging.tags import interpreter_name
from packaging.tags import interpreter_version
from packaging.tags import sys_tags
from poetry.core.semver import parse_constraint
from poetry.core.semver.version import Version
from poetry.core.version.markers import BaseMarker
......@@ -39,6 +45,36 @@ import json
import os
import platform
import sys
import sysconfig
INTERPRETER_SHORT_NAMES = {
"python": "py",
"cpython": "cp",
"pypy": "pp",
"ironpython": "ip",
"jython": "jy",
}
def interpreter_version():
version = sysconfig.get_config_var("interpreter_version")
if version:
version = str(version)
else:
version = _version_nodot(sys.version_info[:2])
return version
def _version_nodot(version):
# type: (PythonVersion) -> str
if any(v >= 10 for v in version):
sep = "_"
else:
sep = ""
return sep.join(map(str, version))
if hasattr(sys, "implementation"):
info = sys.implementation.version
......@@ -50,7 +86,7 @@ if hasattr(sys, "implementation"):
implementation_name = sys.implementation.name
else:
iver = "0"
implementation_name = ""
implementation_name = platform.python_implementation().lower()
env = {
"implementation_name": implementation_name,
......@@ -65,6 +101,9 @@ env = {
"python_version": platform.python_version()[:3],
"sys_platform": sys.platform,
"version_info": tuple(sys.version_info),
# Extra information
"interpreter_name": INTERPRETER_SHORT_NAMES.get(implementation_name, implementation_name),
"interpreter_version": interpreter_version(),
}
print(json.dumps(env))
......@@ -82,12 +121,6 @@ else:
print(sys.prefix)
"""
GET_CONFIG_VAR = """\
import sysconfig
print(sysconfig.get_config_var("{config_var}")),
"""
GET_PYTHON_VERSION = """\
import sys
......@@ -742,6 +775,7 @@ class Env(object):
self._pip_version = None
self._site_packages = None
self._paths = None
self._supported_tags = None
@property
def path(self): # type: () -> Path
......@@ -813,6 +847,13 @@ class Env(object):
return self._paths
@property
def supported_tags(self): # type: () -> List[Tag]
if self._supported_tags is None:
self._supported_tags = self.get_supported_tags()
return self._supported_tags
@classmethod
def get_base_prefix(cls): # type: () -> Path
if hasattr(sys, "real_prefix"):
......@@ -835,7 +876,7 @@ class Env(object):
def get_pip_command(self): # type: () -> List[str]
raise NotImplementedError()
def config_var(self, var): # type: (str) -> Any
def get_supported_tags(self): # type: () -> List[Tag]
raise NotImplementedError()
def get_pip_version(self): # type: () -> Version
......@@ -987,6 +1028,9 @@ class SystemEnv(Env):
return paths
def get_supported_tags(self): # type: () -> List[Tag]
return list(sys_tags())
def get_marker_env(self): # type: () -> Dict[str, Any]
if hasattr(sys, "implementation"):
info = sys.implementation.version
......@@ -1015,16 +1059,11 @@ class SystemEnv(Env):
),
"sys_platform": sys.platform,
"version_info": sys.version_info,
# Extra information
"interpreter_name": interpreter_name(),
"interpreter_version": interpreter_version(),
}
def config_var(self, var): # type: (str) -> Any
try:
return sysconfig.get_config_var(var)
except IOError as e:
warnings.warn("{0}".format(e), RuntimeWarning)
return
def get_pip_version(self): # type: () -> Version
from pip import __version__
......@@ -1068,29 +1107,41 @@ class VirtualEnv(Env):
# so assume that we have a functional pip
return [self._bin("pip")]
def get_supported_tags(self): # type: () -> List[Tag]
file_path = Path(packaging.tags.__file__)
if file_path.suffix == ".pyc":
# Python 2
file_path = file_path.with_suffix(".py")
with file_path.open(encoding="utf-8") as f:
script = decode(f.read())
script = script.replace(
"from ._typing import TYPE_CHECKING, cast",
"TYPE_CHECKING = False\ncast = lambda type_, value: value",
)
script = script.replace(
"from ._typing import MYPY_CHECK_RUNNING, cast",
"MYPY_CHECK_RUNNING = False\ncast = lambda type_, value: value",
)
script += textwrap.dedent(
"""
import json
print(json.dumps([(t.interpreter, t.abi, t.platform) for t in sys_tags()]))
"""
)
output = self.run("python", "-", input_=script)
return [Tag(*t) for t in json.loads(output)]
def get_marker_env(self): # type: () -> Dict[str, Any]
output = self.run("python", "-", input_=GET_ENVIRONMENT_INFO)
return json.loads(output)
def config_var(self, var): # type: (str) -> Any
try:
value = self.run(
"python", "-", input_=GET_CONFIG_VAR.format(config_var=var)
).strip()
except EnvCommandError as e:
warnings.warn("{0}".format(e), RuntimeWarning)
return None
if value == "None":
value = None
elif value == "1":
value = 1
elif value == "0":
value = 0
return value
def get_pip_version(self): # type: () -> Version
output = self.run_pip("--version").strip()
m = re.match("pip (.+?)(?: from .+)?$", output)
......@@ -1188,7 +1239,7 @@ class MockEnv(NullEnv):
pip_version="19.1",
sys_path=None,
marker_env=None,
config_vars=None,
supported_tags=None,
**kwargs
):
super(MockEnv, self).__init__(**kwargs)
......@@ -1201,15 +1252,7 @@ class MockEnv(NullEnv):
self._pip_version = Version.parse(pip_version)
self._sys_path = sys_path
self._mock_marker_env = marker_env
self._config_vars = config_vars
@property
def version_info(self): # type: () -> Tuple[int]
return self._version_info
@property
def python_implementation(self): # type: () -> str
return self._python_implementation
self._supported_tags = supported_tags
@property
def platform(self): # type: () -> str
......@@ -1240,17 +1283,12 @@ class MockEnv(NullEnv):
marker_env["python_version"] = ".".join(str(v) for v in self._version_info[:2])
marker_env["python_full_version"] = ".".join(str(v) for v in self._version_info)
marker_env["sys_platform"] = self._platform
marker_env["interpreter_name"] = self._python_implementation.lower()
marker_env["interpreter_version"] = "cp" + "".join(
str(v) for v in self._version_info[:2]
)
return marker_env
def is_venv(self): # type: () -> bool
return self._is_venv
def config_var(self, var): # type: (str) -> Any
if self._config_vars is None:
return super().config_var(var)
else:
try:
return self._config_vars[var]
except KeyError:
return None
pyproject.toml
......@@ -37,12 +37,15 @@ html5lib = "^1.0"
shellingham = "^1.1"
tomlkit = "^0.5.11"
pexpect = "^4.7.0"
packaging = "^20.4"
# The typing module is not in the stdlib in Python 2.7
typing = { version = "^3.6", python = "~2.7" }
# Use pathlib2 for Python 2.7
pathlib2 = { version = "^2.3", python = "~2.7" }
# Use futures on Python 2.7
futures = { version = "^3.3.0", python = "~2.7" }
# Use glob2 for Python 2.7 and 3.4
glob2 = { version = "^0.6", python = "~2.7" }
# Use virtualenv for Python 2.7 since venv does not exist
......
tests/conftest.py
......@@ -104,11 +104,13 @@ def git_mock(mocker):
@pytest.fixture
def http():
httpretty.enable()
httpretty.reset()
httpretty.enable(allow_net_connect=False)
yield httpretty
httpretty.disable()
httpretty.activate()
httpretty.reset()
@pytest.fixture
......
tests/console/commands/test_config.py
......@@ -13,6 +13,7 @@ def test_list_displays_default_value_if_not_set(app, config):
tester.execute("--list")
expected = """cache-dir = "/foo"
experimental.new-installer = true
virtualenvs.create = true
virtualenvs.in-project = false
virtualenvs.path = {path} # /foo{sep}virtualenvs
......@@ -32,6 +33,7 @@ def test_list_displays_set_get_setting(app, config):
tester.execute("--list")
expected = """cache-dir = "/foo"
experimental.new-installer = true
virtualenvs.create = false
virtualenvs.in-project = false
virtualenvs.path = {path} # /foo{sep}virtualenvs
......@@ -79,6 +81,7 @@ def test_list_displays_set_get_local_setting(app, config):
tester.execute("--list")
expected = """cache-dir = "/foo"
experimental.new-installer = true
virtualenvs.create = false
virtualenvs.in-project = false
virtualenvs.path = {path} # /foo{sep}virtualenvs
......
tests/console/conftest.py
import os
import re
import pytest
from cleo import ApplicationTester
from poetry.console import Application as BaseApplication
from poetry.core.masonry.utils.helpers import escape_name
from poetry.core.masonry.utils.helpers import escape_version
from poetry.core.packages.utils.link import Link
from poetry.factory import Factory
from poetry.installation.executor import Executor as BaseExecutor
from poetry.installation.noop_installer import NoopInstaller
from poetry.io.null_io import NullIO
from poetry.packages import Locker as BaseLocker
from poetry.poetry import Poetry as BasePoetry
from poetry.repositories import Pool
......@@ -18,6 +24,42 @@ from poetry.utils.toml_file import TomlFile
from tests.helpers import mock_clone
class Executor(BaseExecutor):
def __init__(self, *args, **kwargs):
super(Executor, self).__init__(*args, **kwargs)
self._installs = []
self._updates = []
self._uninstalls = []
@property
def installations(self):
return self._installs
@property
def updates(self):
return self._updates
@property
def removals(self):
return self._uninstalls
def _do_execute_operation(self, operation):
super(Executor, self)._do_execute_operation(operation)
if not operation.skipped:
getattr(self, "_{}s".format(operation.job_type)).append(operation.package)
def _execute_install(self, operation):
return 0
def _execute_update(self, operation):
return 0
def _execute_remove(self, operation):
return 0
@pytest.fixture()
def installer():
return NoopInstaller()
......@@ -39,6 +81,9 @@ def setup(mocker, installer, installed, config, env):
p = mocker.patch("poetry.installation.installer.Installer._get_installer")
p.return_value = installer
# Do not run pip commands of the executor
mocker.patch("poetry.installation.executor.Executor.run_pip")
p = mocker.patch("poetry.installation.installer.Installer._get_installed")
p.return_value = installed
......@@ -144,10 +189,22 @@ class Repository(BaseRepository):
raise PackageNotFound("Package [{}] not found.".format(name))
return packages
def find_links_for_package(self, package):
return [
Link(
"https://foo.bar/files/{}-{}-py2.py3-none-any.whl".format(
escape_name(package.name), escape_version(package.version.text)
)
)
]
@pytest.fixture
def repo():
return Repository()
def repo(http):
http.register_uri(
http.GET, re.compile("^https?://foo.bar/(.+?)$"),
)
return Repository(name="foo")
@pytest.fixture
......@@ -188,3 +245,13 @@ def app(poetry):
@pytest.fixture
def app_tester(app):
return ApplicationTester(app)
@pytest.fixture
def new_installer_disabled(config):
config.merge({"experimental": {"new-installer": False}})
@pytest.fixture()
def executor(poetry, config, env):
return Executor(env, poetry.pool, config, NullIO())
tests/installation/test_authenticator.py 0 → 100644
import re
import pytest
from poetry.installation.authenticator import Authenticator
from poetry.io.null_io import NullIO
@pytest.fixture()
def mock_remote(http):
http.register_uri(
http.GET, re.compile("^https?://foo.bar/(.+?)$"),
)
def test_authenticator_uses_url_provided_credentials(config, mock_remote, http):
config.merge(
{
"repositories": {"foo": {"url": "https://foo.bar/simple/"}},
"http-basic": {"foo": {"username": "bar", "password": "baz"}},
}
)
authenticator = Authenticator(config, NullIO())
authenticator.request("get", "https://foo001:bar002@foo.bar/files/foo-0.1.0.tar.gz")
request = http.last_request()
assert "Basic Zm9vMDAxOmJhcjAwMg==" == request.headers["Authorization"]
def test_authenticator_uses_credentials_from_config_if_not_provided(
config, mock_remote, http
):
config.merge(
{
"repositories": {"foo": {"url": "https://foo.bar/simple/"}},
"http-basic": {"foo": {"username": "bar", "password": "baz"}},
}
)
authenticator = Authenticator(config, NullIO())
authenticator.request("get", "https://foo.bar/files/foo-0.1.0.tar.gz")
request = http.last_request()
assert "Basic YmFyOmJheg==" == request.headers["Authorization"]
def test_authenticator_uses_username_only_credentials(config, mock_remote, http):
config.merge(
{
"repositories": {"foo": {"url": "https://foo.bar/simple/"}},
"http-basic": {"foo": {"username": "bar", "password": "baz"}},
}
)
authenticator = Authenticator(config, NullIO())
authenticator.request("get", "https://foo001@foo.bar/files/foo-0.1.0.tar.gz")
request = http.last_request()
assert "Basic Zm9vMDAxOg==" == request.headers["Authorization"]
def test_authenticator_uses_password_only_credentials(config, mock_remote, http):
config.merge(
{
"repositories": {"foo": {"url": "https://foo.bar/simple/"}},
"http-basic": {"foo": {"username": "bar", "password": "baz"}},
}
)
authenticator = Authenticator(config, NullIO())
authenticator.request("get", "https://:bar002@foo.bar/files/foo-0.1.0.tar.gz")
request = http.last_request()
assert "Basic OmJhcjAwMg==" == request.headers["Authorization"]
def test_authenticator_uses_empty_strings_as_default_password(
config, mock_remote, http
):
config.merge(
{
"repositories": {"foo": {"url": "https://foo.bar/simple/"}},
"http-basic": {"foo": {"username": "bar"}},
}
)
authenticator = Authenticator(config, NullIO())
authenticator.request("get", "https://foo.bar/files/foo-0.1.0.tar.gz")
request = http.last_request()
assert "Basic YmFyOg==" == request.headers["Authorization"]
def test_authenticator_uses_empty_strings_as_default_username(
config, mock_remote, http
):
config.merge(
{
"repositories": {"foo": {"url": "https://foo.bar/simple/"}},
"http-basic": {"foo": {"username": None, "password": "bar"}},
}
)
authenticator = Authenticator(config, NullIO())
authenticator.request("get", "https://foo.bar/files/foo-0.1.0.tar.gz")
request = http.last_request()
assert "Basic OmJhcg==" == request.headers["Authorization"]
tests/installation/test_chef.py 0 → 100644
from packaging.tags import Tag
from poetry.core.packages.utils.link import Link
from poetry.installation.chef import Chef
from poetry.utils._compat import Path
from poetry.utils.env import MockEnv
def test_get_cached_archive_for_link(config, mocker):
chef = Chef(
config,
MockEnv(
version_info=(3, 8, 3),
marker_env={"interpreter_name": "cpython", "interpreter_version": "3.8.3"},
supported_tags=[
Tag("cp38", "cp38", "macosx_10_15_x86_64"),
Tag("py3", "none", "any"),
],
),
)
mocker.patch.object(
chef,
"get_cached_archives_for_link",
return_value=[
Link("file:///foo/demo-0.1.0-py2.py3-none-any"),
Link("file:///foo/demo-0.1.0.tar.gz"),
Link("file:///foo/demo-0.1.0-cp38-cp38-macosx_10_15_x86_64.whl"),
Link("file:///foo/demo-0.1.0-cp37-cp37-macosx_10_15_x86_64.whl"),
],
)
archive = chef.get_cached_archive_for_link(
Link("https://files.python-poetry.org/demo-0.1.0.tar.gz")
)
assert Link("file:///foo/demo-0.1.0-cp38-cp38-macosx_10_15_x86_64.whl") == archive
def test_get_cached_archives_for_link(config, mocker):
chef = Chef(
config,
MockEnv(
marker_env={"interpreter_name": "cpython", "interpreter_version": "3.8.3"}
),
)
mocker.patch.object(
chef,
"get_cache_directory_for_link",
return_value=Path(__file__).parent.parent.joinpath("fixtures/distributions"),
)
archives = chef.get_cached_archives_for_link(
Link("https://files.python-poetry.org/demo-0.1.0.tar.gz")
)
assert 2 == len(archives)
def test_get_cache_directory_for_link(config):
chef = Chef(
config,
MockEnv(
marker_env={"interpreter_name": "cpython", "interpreter_version": "3.8.3"}
),
)
directory = chef.get_cache_directory_for_link(
Link("https://files.python-poetry.org/poetry-1.1.0.tar.gz")
)
expected = Path(
"/foo/artifacts/ba/63/13/283a3b3b7f95f05e9e6f84182d276f7bb0951d5b0cc24422b33f7a4648"
)
assert expected == directory
tests/installation/test_chooser.py 0 → 100644
import re
import pytest
from packaging.tags import Tag
from poetry.core.packages.package import Package
from poetry.installation.chooser import Chooser
from poetry.repositories.legacy_repository import LegacyRepository
from poetry.repositories.pool import Pool
from poetry.repositories.pypi_repository import PyPiRepository
from poetry.utils._compat import Path
from poetry.utils.env import MockEnv
JSON_FIXTURES = (
Path(__file__).parent.parent / "repositories" / "fixtures" / "pypi.org" / "json"
)
LEGACY_FIXTURES = Path(__file__).parent.parent / "repositories" / "fixtures" / "legacy"
@pytest.fixture()
def env():
return MockEnv(
supported_tags=[
Tag("cp37", "cp37", "macosx_10_15_x86_64"),
Tag("py3", "none", "any"),
]
)
@pytest.fixture()
def mock_pypi(http):
def callback(request, uri, headers):
parts = uri.rsplit("/")
name = parts[-3]
version = parts[-2]
fixture = JSON_FIXTURES / name / (version + ".json")
if not fixture.exists():
fixture = JSON_FIXTURES / (name + ".json")
if not fixture.exists():
return
with fixture.open(encoding="utf-8") as f:
return [200, headers, f.read()]
http.register_uri(
http.GET, re.compile("^https://pypi.org/(.+?)/(.+?)/json$"), body=callback,
)
@pytest.fixture()
def mock_legacy(http):
def callback(request, uri, headers):
parts = uri.rsplit("/")
name = parts[-2]
fixture = LEGACY_FIXTURES / (name + ".html")
with fixture.open(encoding="utf-8") as f:
return [200, headers, f.read()]
http.register_uri(
http.GET, re.compile("^https://foo.bar/simple/(.+?)$"), body=callback,
)
@pytest.fixture()
def pool():
pool = Pool()
pool.add_repository(PyPiRepository(disable_cache=True))
pool.add_repository(
LegacyRepository("foo", "https://foo.bar/simple/", disable_cache=True)
)
return pool
@pytest.mark.parametrize("source_type", ["", "legacy"])
def test_chooser_chooses_universal_wheel_link_if_available(
env, mock_pypi, mock_legacy, source_type, pool
):
chooser = Chooser(pool, env)
package = Package("pytest", "3.5.0")
if source_type == "legacy":
package.source_type = "legacy"
package.source_reference = "foo"
package.source_url = "https://foo.bar/simple/"
link = chooser.choose_for(package)
assert "pytest-3.5.0-py2.py3-none-any.whl" == link.filename
@pytest.mark.parametrize("source_type", ["", "legacy"])
def test_chooser_chooses_specific_python_universal_wheel_link_if_available(
env, mock_pypi, mock_legacy, source_type, pool
):
chooser = Chooser(pool, env)
package = Package("isort", "4.3.4")
if source_type == "legacy":
package.source_type = "legacy"
package.source_reference = "foo"
package.source_url = "https://foo.bar/simple/"
link = chooser.choose_for(package)
assert "isort-4.3.4-py3-none-any.whl" == link.filename
@pytest.mark.parametrize("source_type", ["", "legacy"])
def test_chooser_chooses_system_specific_wheel_link_if_available(
mock_pypi, mock_legacy, source_type, pool
):
env = MockEnv(
supported_tags=[Tag("cp37", "cp37m", "win32"), Tag("py3", "none", "any")]
)
chooser = Chooser(pool, env)
package = Package("pyyaml", "3.13.0")
if source_type == "legacy":
package.source_type = "legacy"
package.source_reference = "foo"
package.source_url = "https://foo.bar/simple/"
link = chooser.choose_for(package)
assert "PyYAML-3.13-cp37-cp37m-win32.whl" == link.filename
@pytest.mark.parametrize("source_type", ["", "legacy"])
def test_chooser_chooses_sdist_if_no_compatible_wheel_link_is_available(
env, mock_pypi, mock_legacy, source_type, pool,
):
chooser = Chooser(pool, env)
package = Package("pyyaml", "3.13.0")
if source_type == "legacy":
package.source_type = "legacy"
package.source_reference = "foo"
package.source_url = "https://foo.bar/simple/"
link = chooser.choose_for(package)
assert "PyYAML-3.13.tar.gz" == link.filename
@pytest.mark.parametrize("source_type", ["", "legacy"])
def test_chooser_chooses_distributions_that_match_the_package_hashes(
env, mock_pypi, mock_legacy, source_type, pool,
):
chooser = Chooser(pool, env)
package = Package("isort", "4.3.4")
package.files = [
{
"hash": "sha256:b9c40e9750f3d77e6e4d441d8b0266cf555e7cdabdcff33c4fd06366ca761ef8",
"filename": "isort-4.3.4.tar.gz",
}
]
if source_type == "legacy":
package.source_type = "legacy"
package.source_reference = "foo"
package.source_url = "https://foo.bar/simple/"
link = chooser.choose_for(package)
assert "isort-4.3.4.tar.gz" == link.filename
tests/installation/test_executor.py 0 → 100644
# -*- coding: utf-8 -*-
from __future__ import unicode_literals
import re
import pytest
from clikit.api.formatter.style import Style
from clikit.io.buffered_io import BufferedIO
from poetry.config.config import Config
from poetry.core.packages.package import Package
from poetry.installation.executor import Executor
from poetry.installation.operations import Install
from poetry.installation.operations import Uninstall
from poetry.installation.operations import Update
from poetry.repositories.pool import Pool
from poetry.utils._compat import PY36
from poetry.utils._compat import Path
from poetry.utils.env import MockEnv
from tests.repositories.test_pypi_repository import MockRepository
@pytest.fixture()
def io():
io = BufferedIO()
io.formatter.add_style(Style("c1_dark").fg("cyan").dark())
io.formatter.add_style(Style("c2_dark").fg("default").bold().dark())
io.formatter.add_style(Style("success_dark").fg("green").dark())
io.formatter.add_style(Style("warning").fg("yellow"))
return io
@pytest.fixture()
def pool():
pool = Pool()
pool.add_repository(MockRepository())
return pool
@pytest.fixture()
def mock_file_downloads(http):
def callback(request, uri, headers):
fixture = Path(__file__).parent.parent.joinpath(
"fixtures/distributions/demo-0.1.0-py2.py3-none-any.whl"
)
with fixture.open("rb") as f:
return [200, headers, f.read()]
http.register_uri(
http.GET, re.compile("^https://files.pythonhosted.org/.*$"), body=callback,
)
def test_execute_executes_a_batch_of_operations(
config, pool, io, tmp_dir, mock_file_downloads
):
config = Config()
config.merge({"cache-dir": tmp_dir})
env = MockEnv(path=Path(tmp_dir))
executor = Executor(env, pool, config, io)
file_package = Package("demo", "0.1.0")
file_package.source_type = "file"
file_package.source_url = str(
Path(__file__)
.parent.parent.joinpath(
"fixtures/distributions/demo-0.1.0-py2.py3-none-any.whl"
)
.resolve()
)
directory_package = Package("simple-project", "1.2.3")
directory_package.source_type = "directory"
directory_package.source_url = str(
Path(__file__).parent.parent.joinpath("fixtures/simple_project").resolve()
)
git_package = Package("demo", "0.1.0")
git_package.source_type = "git"
git_package.source_reference = "master"
git_package.source_url = "https://github.com/demo/demo.git"
assert 0 == executor.execute(
[
Install(Package("pytest", "3.5.2")),
Uninstall(Package("attrs", "17.4.0")),
Update(Package("requests", "2.18.3"), Package("requests", "2.18.4")),
Uninstall(Package("clikit", "0.2.3")).skip("Not currently installed"),
Install(file_package),
Install(directory_package),
Install(git_package),
]
)
expected = """
Package operations: 4 installs, 1 update, 1 removal
• Installing pytest (3.5.2)
• Removing attrs (17.4.0)
• Updating requests (2.18.3 -> 2.18.4)
• Installing demo (0.1.0 {})
• Installing simple-project (1.2.3 {})
• Installing demo (0.1.0 master)
""".format(
file_package.source_url, directory_package.source_url
)
assert expected == io.fetch_output()
assert 5 == len(env.executed)
def test_execute_shows_skipped_operations_if_verbose(config, pool, io):
config = Config()
config.merge({"cache-dir": "/foo"})
env = MockEnv()
executor = Executor(env, pool, config, io)
executor.verbose()
assert 0 == executor.execute(
[Uninstall(Package("clikit", "0.2.3")).skip("Not currently installed")]
)
expected = """
Package operations: 0 installs, 0 updates, 0 removals, 1 skipped
• Removing clikit (0.2.3): Skipped for the following reason: Not currently installed
"""
assert expected == io.fetch_output()
assert 0 == len(env.executed)
@pytest.mark.skipif(
not PY36, reason="Improved error rendering is only available on Python >=3.6"
)
def test_execute_should_show_errors(config, mocker, io):
env = MockEnv()
executor = Executor(env, pool, config, io)
executor.verbose()
mocker.patch.object(executor, "_install", side_effect=Exception("It failed!"))
assert 1 == executor.execute([Install(Package("clikit", "0.2.3"))])
expected = """
Package operations: 1 install, 0 updates, 0 removals
• Installing clikit (0.2.3)
Exception
It failed!
"""
assert expected in io.fetch_output()
def test_execute_should_show_operation_as_cancelled_on_subprocess_keyboard_interrupt(
config, mocker, io
):
env = MockEnv()
executor = Executor(env, pool, config, io)
executor.verbose()
# A return code of -2 means KeyboardInterrupt in the pip subprocess
mocker.patch.object(executor, "_install", return_value=-2)
assert 1 == executor.execute([Install(Package("clikit", "0.2.3"))])
expected = """
Package operations: 1 install, 0 updates, 0 removals
• Installing clikit (0.2.3)
• Installing clikit (0.2.3): Cancelled
"""
assert expected == io.fetch_output()
tests/repositories/fixtures/legacy/pytest.html 0 → 100644
<!DOCTYPE html>
<html>
<head>
<title>Links for pytest</title>
</head>
<body>
<h1>Links for pytest</h1><a href="https://files.pythonhosted.org/packages/ed/96/271c93f75212c06e2a7ec3e2fa8a9c90acee0a4838dc05bf379ea09aae31/pytest-3.5.0-py2.py3-none-any.whl#sha256=6266f87ab64692112e5477eba395cfedda53b1933ccd29478e671e73b420c19c" data-requires-python="&gt;=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*">pytest-3.5.0-py2.py3-none-any.whl</a><br/>
<a href="https://files.pythonhosted.org/packages/2d/56/6019153cdd743300c5688ab3b07702355283e53c83fbf922242c053ffb7b/pytest-3.5.0.tar.gz#sha256=fae491d1874f199537fd5872b5e1f0e74a009b979df9d53d1553fd03da1703e1" data-requires-python="&gt;=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*">pytest-3.5.0.tar.gz</a><br/>
</body>
</html>
<!--SERIAL 7198641-->
tests/repositories/fixtures/legacy/pyyaml.html
......@@ -5,6 +5,7 @@
</head>
<body>
<h1>Links for python-language-server</h1>
<a href="https://files.pythonhosted.org/packages/5c/ed/d6557f70daaaab6ee5cd2f8ccf7bedd63081e522e38679c03840e1acc114/PyYAML-3.13-cp37-cp37m-win32.whl#sha256=e170a9e6fcfd19021dd29845af83bb79236068bf5fd4df3327c1be18182b2531">PyYAML-3.13-cp37-cp37m-win32.whl</a><br/>
<a href="https://files.pythonhosted.org/packages/9e/a3/1d13970c3f36777c583f136c136f804d70f500168edc1edea6daa7200769/PyYAML-3.13.tar.gz#sha256=3ef3092145e9b70e3ddd2c7ad59bdd0252a94dfe3949721633e41344de00a6bf">PyYAML-3.13.tar.gz</a><br/>
<a href="https://files.pythonhosted.org/packages/0f/9d/f98ed0a460dc540f720bbe5c6e076f025595cdfa3e318fad27165db13cf9/PyYAML-4.2b2.tar.gz#sha256=406b717f739e2d00c49873068b71f5454c2420157db51b082d4d2beb17ffffb6">PyYAML-4.2b2.tar.gz</a><br/>
</body>
......
tox.ini
......@@ -7,5 +7,5 @@ envlist = py27, py35, py36, py37, py38
whitelist_externals = poetry
skip_install = true
commands =
poetry install -vvv
poetry install -vv
poetry run pytest {posargs} tests/
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment