Commit 2a104075 by Sébastien Eustace

Add basic lock and install mechanism

parent e853ed1e
from poetry.console import Application
from .poetry import Poetry
__version__ = '0.1.0'
__version__ = Poetry.VERSION
console = Application('Poetry', __version__)
console = Application()
from .application import Application
import os
from cleo import Application as BaseApplication
from poetry.poetry import Poetry
from poetry.utils.venv import Venv
from .commands import AboutCommand
from .commands import InstallCommand
class Application(BaseApplication):
def __init__(self):
super().__init__('Poetry', Poetry.VERSION)
self._poetry = None
self._venv = Venv.create()
@property
def poetry(self) -> Poetry:
if self._poetry is not None:
return self._poetry
self._poetry = Poetry.create(os.getcwd())
return self._poetry
@property
def venv(self) -> Venv:
return self._venv
def get_default_commands(self) -> list:
commands = super(Application, self).get_default_commands()
return commands + [
AboutCommand(),
InstallCommand()
]
def do_run(self, i, o) -> int:
if self._venv.is_venv() and o.is_debug():
o.writeln(f'Using virtualenv: <comment>{self._venv.venv}</>')
return super().do_run(i, o)
from .about import AboutCommand
from .install import InstallCommand
from .command import Command
class AboutCommand(Command):
"""
Short information about Poetry.
about
"""
def handle(self):
self.line("""<info>Poetry - Package Management for Python</info>
<comment>Poetry is a dependency manager tracking local dependencies of your projects and libraries.
See <fg=blue>https://github.com/sdispater/poetry</> for more information.</comment>
""")
from cleo import Command as BaseCommand
from poetry.poetry import Poetry
from ..styles.poetry import PoetryStyle
class Command(BaseCommand):
@property
def poetry(self) -> Poetry:
return self.get_application().poetry
def run(self, i, o) -> int:
"""
Initialize command.
"""
self.input = i
self.output = PoetryStyle(i, o, self.get_application().venv)
return super(BaseCommand, self).run(i, o)
from poetry.installation import Installer
from poetry.repositories.pypi_repository import PyPiRepository
from .command import Command
class InstallCommand(Command):
"""
Installs the project dependencies.
install
{ --no-dev : Do not install dev dependencies. }
{ --dry-run : Outputs the operations but will not execute anything
(implicitly enables --verbose). }
"""
help = """The <info>install</info> command reads the <comment>poetry.lock</> file from
the current directory, processes it, and downloads and installs all the
libraries and dependencies outlined in that file. If the file does not
exist it will look for <comment>poetry.json</> and do the same.
<info>poetry install</info>
"""
def handle(self):
installer = Installer(
self.output,
self.poetry.package,
self.poetry.locker,
PyPiRepository()
)
installer.dev_mode(not self.option('no-dev'))
return installer.run()
from cleo.styles import CleoStyle
class PoetryStyle(CleoStyle):
def __init__(self, i, o, venv):
self._venv = venv
super().__init__(i, o)
@property
def venv(self):
return self._venv
from .installer import Installer
from typing import List
from poetry.packages import Locker
from poetry.packages import Package
from poetry.puzzle import Solver
from poetry.puzzle.operations import Install
from poetry.puzzle.operations import Uninstall
from poetry.puzzle.operations import Update
from poetry.puzzle.operations.operation import Operation
from poetry.repositories import Repository
from poetry.repositories.installed_repository import InstalledRepository
class Installer:
def __init__(self,
io,
package: Package,
locker: Locker,
repository: Repository):
self._io = io
self._package = package
self._locker = locker
self._repository = repository
self._dry_run = False
self._update = False
self._verbose = False
self._write_lock = True
self._dev_mode = True
self._execute_operations = True
def run(self):
# Force update if there is no lock file present
if not self._update and not self._locker.is_locked():
self._update = True
if self.is_dry_run():
self.verbose(True)
self._write_lock = False
self._execute_operations = False
local_repo = Repository()
self._do_install(local_repo)
if self._update and self._write_lock:
updated_lock = self._locker.set_lock_data(
self._package,
local_repo.packages
)
if updated_lock:
self._io.writeln('<info>Writing lock file</>')
return 0
def dry_run(self, dry_run=True) -> 'Installer':
self._dry_run = dry_run
return self
def is_dry_run(self) -> bool:
return self._dry_run
def verbose(self, verbose=True) -> 'Installer':
self._verbose = verbose
return self
def is_verbose(self) -> bool:
return self._verbose
def dev_mode(self, dev_mode=True) -> 'Installer':
self._dev_mode = dev_mode
return self
def is_dev_mode(self) -> bool:
return self._dev_mode
def _do_install(self, local_repo):
locked_repository = Repository()
# initialize locked repo if we are installing from lock
if not self._update:
locked_repository = self._locker.locked_repository(self._dev_mode)
solver = Solver(locked_repository, self._io)
request = self._package.requires
if self.is_dev_mode():
request += self._package.dev_requires
if self._update:
self._io.writeln('<info>Updating dependencies</>')
ops = solver.solve(request, self._repository)
else:
self._io.writeln('<info>Installing dependencies from lock file</>')
# If we are installing from lock
# Filter the operations by comparing it with what is
# currently installed
ops = self._get_operations_from_lock(locked_repository)
self._io.new_line()
# Execute operations
if not ops:
self._io.writeln('Nothing to install or update')
# extract dev packages and mark them to be skipped
# if it's a --no-dev install or update
# we also force them to be uninstalled
# if they are present in the local repo
# TODO
if ops:
installs = []
updates = []
uninstalls = []
for op in ops:
if op.job_type == 'install':
installs.append(
f'{op.package.pretty_name}'
f':{op.package.full_pretty_version}'
)
elif op.job_type == 'update':
updates.append(
f'{op.target_package.pretty_name}'
f':{op.target_package.full_pretty_version}'
)
elif op.job_type == 'uninstall':
uninstalls.append(
f'{op.package.pretty_name}'
)
self._io.new_line()
self._io.writeln(
'Package operations: '
f'<info>{len(installs)}</> install{"" if len(installs) == 1 else "s"}, '
f'<info>{len(updates)}</> update{"" if len(updates) == 1 else "s"}, '
f'<info>{len(uninstalls)}</> removal{"" if len(uninstalls) == 1 else "s"}'
f''
)
self._io.new_line()
for op in ops:
if op.job_type == 'install':
local_repo.add_package(op.package)
elif op.job_type == 'update':
local_repo.add_package(op.target_package)
self._execute(op)
def _execute(self, operation: Operation) -> None:
"""
Execute a given operation.
"""
method = operation.job_type
getattr(self, f'_execute_{method}')(operation)
def _execute_install(self, operation: Install) -> None:
self._io.writeln(
f' - Installing <info>{operation.package.name}</> '
f'(<comment>{operation.package.full_pretty_version}</>)'
)
def _execute_update(self, operation: Update) -> None:
name = operation.target_package.name
original = operation.initial_package.pretty_version
target = operation.target_package.pretty_version
self._io.writeln(
f' - Updating <info>{name}</> '
f'(<comment>{original}</>'
f' -> <comment>{target}</>)'
)
def _execute_uninstall(self, operation: Uninstall) -> None:
self._io.writeln(
f' - Removing <info>{operation.package.name}</> '
f'(<comment>{operation.package.full_pretty_version}</>)'
)
def _get_operations_from_lock(self,
locked_repository: Repository
) -> List[Operation]:
installed_repo = InstalledRepository.load(self._io.venv)
ops = []
for locked in locked_repository.packages:
is_installed = False
for installed in installed_repo.packages:
if locked.name == installed.name:
is_installed = True
if locked.version != installed.version:
ops.append(Update(
installed, locked
))
if not is_installed:
ops.append(Install(locked))
return ops
from .dependency import Dependency
from .locker import Locker
from .package import Package
import json
import poetry.packages
from hashlib import sha256
from pathlib import Path
from typing import List
from poetry.repositories import Repository
from poetry.utils.toml_file import TomlFile
class Locker:
_relevant_keys = [
'name',
'version',
'python_versions',
'dependencies',
'dev-dependencies',
]
def __init__(self, lock: Path, original: Path):
self._lock = TomlFile(lock)
self._original = TomlFile(original)
self._lock_data = None
self._content_hash = self._get_content_hash()
@property
def lock_data(self):
if self._lock_data is None:
self._lock_data = self._get_lock_data()
return self._lock_data
def is_locked(self) -> bool:
"""
Checks whether the locker has been locked (lockfile found).
"""
if not self._lock.exists():
return False
return 'packages' in self.lock_data
def is_fresh(self) -> bool:
"""
Checks whether the lock file is still up to date with the current hash.
"""
lock = self._lock.read()
if 'content-hash' in lock:
return self._content_hash == lock['content-hash']
return False
def locked_repository(self, with_dev_reqs: bool = False) -> Repository:
"""
Searches and returns a repository of locked packages.
"""
if not self.is_locked():
return Repository()
lock_data = self.lock_data
packages = Repository()
if with_dev_reqs:
locked_packages = lock_data['packages']
else:
locked_packages = [
p for p in lock_data['packages'] if p['category'] == 'main'
]
if not locked_packages:
return packages
for info in locked_packages:
packages.add_package(
poetry.packages.Package(
info['name'], info['version'], info['version']
)
)
return packages
def set_lock_data(self,
root, packages,
python_versions=None, platform=None) -> bool:
lock = {
'root': {
'name': root.name,
'version': root.version,
'python_versions': python_versions or '*'
},
'packages': None,
'metadata': {
'content-hash': self._content_hash
}
}
if platform is not None:
lock['root']['platform'] = platform
lock['packages'] = self._lock_packages(packages)
if not self.is_locked() or lock != self.lock_data:
self._lock.write(lock)
self._lock_data = None
return True
return False
def _get_content_hash(self) -> str:
"""
Returns the sha256 hash of the sorted content of the composer file.
"""
content = self._original.read()
relevant_content = {}
package = content['package']
for key in ['name', 'version', 'python-versions', 'platform']:
relevant_content[key] = package.get(key, '')
for key in ['dependencies', 'dev-dependencies']:
relevant_content[key] = content[key]
content_hash = sha256(
json.dumps(relevant_content, sort_keys=True).encode()
).hexdigest()
return content_hash
def _get_lock_data(self) -> dict:
if not self._lock.exists():
raise RuntimeError(
'No lockfile found. Unable to read locked packages'
)
return self._lock.read()
def _lock_packages(self, packages: List['poetry.packages.Package']) -> list:
locked = []
for package in sorted(packages, key=lambda x: x.name):
spec = self._dump_package(package)
locked.append(spec)
return locked
def _dump_package(self, package: 'poetry.packages.Package') -> dict:
data = {
'name': package.pretty_name,
'version': package.pretty_version,
'category': package.category,
'optional': package.optional,
'python-versions': package.python_versions,
'checksum': package.hashes
}
return data
from poetry.semver.helpers import parse_stability
from .dependency import Dependency
class Package:
......@@ -43,7 +45,6 @@ class Package:
"""
self._pretty_name = name
self._name = name.lower()
self._id = -1
self._version = version
self._pretty_version = pretty_version
......@@ -58,6 +59,12 @@ class Package:
self.requires = []
self.dev_requires = []
self.category = 'main'
self.hashes = []
self.optional = False
self.python_versions = '*'
self.platform = None
@property
def name(self):
return self._name
......@@ -104,6 +111,28 @@ class Package:
def is_prerelease(self):
return self._stability != 'stable'
def add_dependency(self, name, constraint=None, dev=False):
if constraint is None:
constraint = '*'
if isinstance(constraint, dict):
if 'git' in constraint:
# VCS dependency
pass
else:
version = constraint['version']
optional = constraint.get('optional', False)
dependency = Dependency(name, version, optional=optional)
else:
dependency = Dependency(name, constraint)
if dev:
self.dev_requires.append(dependency)
else:
self.requires.append(dependency)
return dependency
def __hash__(self):
return hash((self._name, self._version))
......
from pathlib import Path
from .packages import Locker
from .packages import Package
from .semver.helpers import normalize_version
from .utils.toml_file import TomlFile
class Poetry:
VERSION = '0.1.0'
def __init__(self,
config: dict,
package: Package,
locker: Locker):
self._package = package
self._config = config
self._locker = locker
@property
def package(self):
return self._package
@property
def config(self):
return self._config
@property
def locker(self):
return self._locker
@classmethod
def create(cls, cwd) -> 'Poetry':
poetry_file = Path(cwd) / 'poetry.toml'
if not poetry_file.exists():
raise RuntimeError(
f'Poetry could not find a poetry.json file in {cwd}'
)
# TODO: validate file content
local_config = TomlFile(poetry_file.as_posix()).read()
# Load package
package_config = local_config['package']
name = package_config['name']
pretty_version = package_config['version']
version = normalize_version(pretty_version)
package = Package(name, version, pretty_version)
if 'python_versions' in package_config:
package.python_versions = package_config['python_versions']
if 'platform' in package_config:
package.platform = package_config['platform']
if 'dependencies' in local_config:
for name, constraint in local_config['dependencies'].items():
package.add_dependency(name, constraint)
if 'dev-dependencies' in local_config:
for name, constraint in local_config['dev-dependencies'].items():
package.add_dependency(name, constraint, dev=True)
locker = Locker(poetry_file.with_suffix('.lock'), poetry_file)
return cls(local_config, package, locker)
......@@ -7,6 +7,10 @@ class Operation:
self._reason = reason
@property
def job_type(self):
raise NotImplementedError
@property
def reason(self) -> str:
return self._reason
......
......@@ -17,6 +17,8 @@ from poetry.semver.constraints import Constraint
class Provider(SpecificationProvider):
UNSAFE_PACKAGES = {'setuptools', 'distribute', 'pip'}
def __init__(self, repository: Repository):
self._repository = repository
......@@ -63,7 +65,11 @@ class Provider(SpecificationProvider):
def dependencies_for(self, package: Package):
package = self._repository.package(package.name, package.version)
return [r for r in package.requires if not r.is_optional()]
return [
r for r in package.requires
if not r.is_optional()
and r.name not in self.UNSAFE_PACKAGES
]
def is_requirement_satisfied_by(self,
requirement: Dependency,
......
from poetry.semver.constraints import Constraint
class BaseRepository:
SEARCH_FULLTEXT = 0
......@@ -24,112 +21,3 @@ class BaseRepository:
def search(self, query, mode=SEARCH_FULLTEXT):
raise NotImplementedError()
def get_dependents(self, needle,
constraint=None, invert=False,
recurse=True, packages_found=None):
results = {}
needles = needle
if not isinstance(needles, list):
needles = [needles]
# initialize the list with the needles before any recursion occurs
if packages_found is None:
packages_found = needles
# locate root package for use below
root_package = None
for package in self.packages:
if isinstance(package, RootPackage):
root_package = package
break
# Loop over all currently installed packages.
for package in self.packages:
links = package.requires
# each loop needs its own "tree"
# as we want to show the complete dependent set of every needle
# without warning all the time about finding circular deps
packages_in_tree = packages_found
# Require-dev is only relevant for the root package
if isinstance(package, RootPackage):
links += package.dev_requires
# Cross-reference all discovered links to the needles
for link in links:
for needle in needles:
if link.target == needle:
if (
constraint is None
or link.constraint.matches(constraint) is not invert
):
# already displayed this node's dependencies,
# cutting short
if link.source in packages_in_tree:
results[link.source] = (package, link, False)
continue
packages_in_tree.append(link.source)
if recurse:
dependents = self.get_dependents(
link.source, None, False, True, packages_in_tree
)
else:
dependents = {}
results[link.source] = (package, link, dependents)
# When inverting, we need to check for conflicts
# of the needles against installed packages
if invert and package.name in needles:
for link in package.conflicts:
for pkg in self.find_packages(link.target):
version = Constraint('=', pkg.version)
if link.constraint.matches(version) is invert:
results[len(results) - 1] = (package, link, False)
# When inverting, we need to check for conflicts of the needles'
# requirements against installed packages
if (
invert
and constraint
and package.name in needles
and constraint.matches(Constraint('=', package.version))
):
for link in package.requires:
for pkg in self._packages:
if link.target not in pkg.names:
continue
version = Constraint('=', pkg.version)
if not link.constraint.matches(version):
# if we have a root package
# we show the root requires as well
# to perhaps allow to find an issue there
if root_package:
for root_req in root_package.requires + root_package.dev_requires:
if root_req.target in pkg.names and not root_req.constraint.matches(link.constraint):
results[len(results) - 1] = (package, link, False)
results[len(results) - 1] = (root_package, root_req, False)
continue
results[len(results) - 1] = (package, link, False)
lnk = Link(
root_package.name,
link.target,
None,
'does not require',
'but {} is installed'.format(
pkg.pretty_version
)
)
results[len(results) - 1] = (package, lnk, False)
else:
results[len(results) - 1] = (package, link, False)
continue
return results
from pip.commands.freeze import freeze
from poetry.packages import Package
from poetry.utils.venv import Venv
from .repository import Repository
class InstalledRepository(Repository):
def __init__(self, packages=None):
super(InstalledRepository, self).__init__(packages)
@classmethod
def load(cls, venv: Venv) -> 'InstalledRepository':
"""
Load installed packages.
For now, it uses the pip "freeze" command.
"""
repo = cls()
freeze_output = venv.run('pip', 'freeze')
for line in freeze_output.split('\n'):
if '==' in line:
name, version = line.split('==')
repo.add_package(Package(name, version, version))
return repo
......@@ -62,7 +62,7 @@ class PyPiRepository(Repository):
versions.append(version)
for version in versions:
packages.append(Package(name, version, version))
packages.append(self.package(name, version))
return packages
......@@ -90,6 +90,10 @@ class PyPiRepository(Repository):
optional=m.group('extra') is not None
)
)
# Adding hashes information
package.hashes = release_info['digests']
self._packages.append(package)
return package
......
from pathlib import Path
from toml import dumps
from toml import loads
class TomlFile:
def __init__(self, path):
self._path = Path(path)
def read(self) -> dict:
return loads(self._path.read_text())
def write(self, data) -> None:
self._path.write_text(dumps(data))
def exists(self) -> bool:
return self._path.exists()
import glob
import os
import subprocess
import sys
class Venv:
def __init__(self, venv=None):
self._venv = venv
@classmethod
def create(cls) -> 'Venv':
if 'VIRTUAL_ENV' not in os.environ:
# Not in a virtualenv
return cls()
# venv detection:
# stdlib venv may symlink sys.executable, so we can't use realpath.
# but others can symlink *to* the venv Python,
# so we can't just use sys.executable.
# So we just check every item in the symlink tree (generally <= 3)
p = os.path.normcase(sys.executable)
paths = [p]
while os.path.islink(p):
p = os.path.normcase(
os.path.join(os.path.dirname(p), os.readlink(p)))
paths.append(p)
p_venv = os.path.normcase(os.environ['VIRTUAL_ENV'])
if any(p.startswith(p_venv) for p in paths):
# Running properly in the virtualenv, don't need to do anything
return cls()
if sys.platform == "win32":
venv = os.path.join(
os.environ['VIRTUAL_ENV'], 'Lib', 'site-packages'
)
else:
lib = os.path.join(
os.environ['VIRTUAL_ENV'], 'lib'
)
python = glob.glob(
os.path.join(lib, 'python*')
)[0].replace(
lib + '/', ''
)
venv = os.path.join(
lib,
python,
'site-packages'
)
return cls(venv)
@property
def venv(self):
return self._venv
@property
def python(self) -> str:
"""
Path to current python executable
"""
return self._bin('python')
@property
def pip(self) -> str:
"""
Path to current pip executable
"""
return self._bin('pip')
def run(self, bin: str, *args) -> str:
"""
Run a command inside the virtual env.
"""
cmd = [self._bin(bin)] + list(args)
output = subprocess.check_output(cmd, stderr=subprocess.STDOUT)
return output.decode()
def _bin(self, bin) -> str:
"""
Return path to the given executable.
"""
if not self.is_venv():
return bin
return os.path.realpath(
os.path.join(self._venv, '..', '..', '..', 'bin', bin)
)
def is_venv(self) -> bool:
return self._venv is not None
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment