Commit 89208696 by Sébastien Eustace

Add experimental support for packages with C extensions

parent c467b34c
......@@ -3,6 +3,10 @@
## [Unreleased]
### Added
- Added experimental support for package with C extensions.
### Changed
- Added hashes check when installing packages.
......
......@@ -8,7 +8,7 @@ class BuildCommand(Command):
Builds a package, as a tarball and a wheel by default.
build
{ --f|format=* : Limit the format to either wheel or sdist}
{ --f|format= : Limit the format to either wheel or sdist. }
"""
def handle(self):
......
......@@ -4,22 +4,26 @@ from poetry.utils.venv import Venv
class NullVenv(Venv):
def __init__(self):
def __init__(self, execute=False):
super().__init__()
self.executed = []
self._execute = execute
def run(self, bin: str, *args):
self.executed.append([bin] + list(args))
if self._execute:
return super().run(bin, *args)
def _bin(self, bin):
return bin
class NullIO(PoetryStyle):
def __init__(self):
self._venv = NullVenv()
def __init__(self, execute=False):
self._venv = NullVenv(execute=execute)
@property
def venv(self) -> NullVenv:
......
......@@ -29,7 +29,9 @@ class Builder:
self._io = io
self._package = poetry.package
self._path = poetry.file.parent
self._module = Module(self._package.name, self._path.as_posix())
self._module = Module(
self._package.name, self._path.as_posix()
)
def build(self):
raise NotImplementedError()
......@@ -53,7 +55,7 @@ class Builder:
return result
def find_files_to_add(self) -> list:
def find_files_to_add(self, exclude_build=True) -> list:
"""
Finds all files to add to the tarball
......@@ -102,6 +104,11 @@ class Builder:
)
to_add.append(readme.relative_to(self._path))
# If a build script is specified and explicitely required
# we add it to the list of files
if self._package.build and not exclude_build:
to_add.append(Path(self._package.build))
return sorted(to_add)
def convert_entry_points(self) -> dict:
......
......@@ -26,7 +26,8 @@ class CompleteBuilder(Builder):
dist_dir = self._path / 'dist'
with self.unpacked_tarball(sdist_file) as tmpdir:
wheel_info = WheelBuilder.make_in(
poetry.Poetry.create(tmpdir), self._io, dist_dir
poetry.Poetry.create(tmpdir), self._io, dist_dir,
original=self._poetry
)
return SimpleNamespace(wheel=wheel_info, sdist=sdist_info)
......
......@@ -22,15 +22,19 @@ SETUP = """\
from distutils.core import setup
{before}
setup(
name={name!r},
version={version!r},
description={description!r},
author={author!r},
author_email={author_email!r},
url={url!r},
setup_kwargs = {{
'name': {name!r},
'version': {version!r},
'description': {description!r},
'long_description': {long_description!r},
'author': {author!r},
'author_email': {author_email!r},
'url': {url!r},
{extra}
)
}}
{after}
setup(**setup_kwargs)
"""
......@@ -67,7 +71,7 @@ class SdistBuilder(Builder):
try:
tar_dir = f'{self._package.pretty_name}-{self._package.version}'
files_to_add = self.find_files_to_add()
files_to_add = self.find_files_to_add(exclude_build=False)
for relpath in files_to_add:
path = self._path / relpath
......@@ -105,12 +109,19 @@ class SdistBuilder(Builder):
tar.close()
gz.close()
self._io.writeln(f' - Built <comment>{target.name}</>')
self._io.writeln(f' - Built <fg=cyan>{target.name}</>')
return target
def build_setup(self) -> bytes:
before, extra = [], []
before, extra, after = [], [], []
# If we have a build script, use it
if self._package.build:
after += [
f'from {self._package.build.split(".")[0]} import *',
'build(setup_kwargs)'
]
if self._module.is_package():
packages, package_data = self.find_packages(
......@@ -118,24 +129,24 @@ class SdistBuilder(Builder):
)
before.append("packages = \\\n{}\n".format(pformat(sorted(packages))))
before.append("package_data = \\\n{}\n".format(pformat(package_data)))
extra.append("packages=packages,")
extra.append("package_data=package_data,")
extra.append("'packages': packages,")
extra.append("'package_data': package_data,")
else:
extra.append('py_modules={!r},'.format(self._module.name))
extra.append("'py_modules': {!r},".format(self._module.name))
dependencies, extras = self.convert_dependencies(self._package.requires)
if dependencies:
before.append("install_requires = \\\n{}\n".format(pformat(dependencies)))
extra.append("install_requires=install_requires,")
extra.append("'install_requires': install_requires,")
if extras:
before.append("extras_require = \\\n{}\n".format(pformat(extras)))
extra.append("extras_require=extras_require,")
extra.append("'extras_require': extras_require,")
entry_points = self.convert_entry_points()
if entry_points:
before.append("entry_points = \\\n{}\n".format(pformat(entry_points)))
extra.append("entry_points=entry_points,")
extra.append("'entry_points': entry_points,")
if self._package.python_versions != '*':
constraint = self._package.python_constraint
......@@ -146,7 +157,7 @@ class SdistBuilder(Builder):
else:
python_requires = str(constraint).replace(' ', '')
extra.append('python_requires={!r},'.format(python_requires))
extra.append("'python_requires': {!r},".format(python_requires))
author = self.convert_author(self._package.authors[0])
......@@ -155,10 +166,12 @@ class SdistBuilder(Builder):
name=self._package.name,
version=self._package.version,
description=self._package.description,
long_description=self._package.readme,
author=author['name'],
author_email=author['email'],
url=self._package.homepage or self._package.repository_url,
extra='\n '.join(extra),
after='\n'.join(after)
).encode('utf-8')
@classmethod
......
......@@ -3,6 +3,7 @@ import hashlib
import os
import re
import tempfile
import shutil
import stat
import zipfile
......@@ -14,8 +15,13 @@ from types import SimpleNamespace
from poetry.__version__ import __version__
from poetry.semver.constraints import Constraint
from poetry.semver.constraints import MultiConstraint
from poetry.vcs import get_vcs
from ..utils.helpers import normalize_file_permissions
from ..utils.tags import get_abbr_impl
from ..utils.tags import get_abi_tag
from ..utils.tags import get_impl_ver
from ..utils.tags import get_platform
from .builder import Builder
......@@ -28,24 +34,27 @@ Root-Is-Purelib: true
class WheelBuilder(Builder):
def __init__(self, poetry, io, target_fp):
def __init__(self, poetry, io, target_fp, original=None):
super().__init__(poetry, io)
self._records = []
self._original_path = self._path
if original:
self._original_path = original.file.parent
# Open the zip file ready to write
self._wheel_zip = zipfile.ZipFile(target_fp, 'w',
compression=zipfile.ZIP_DEFLATED)
@classmethod
def make_in(cls, poetry, io, directory) -> SimpleNamespace:
def make_in(cls, poetry, io, directory, original=None) -> SimpleNamespace:
# We don't know the final filename until metadata is loaded, so write to
# a temporary_file, and rename it afterwards.
(fd, temp_path) = tempfile.mkstemp(suffix='.whl',
dir=str(directory))
try:
with open(fd, 'w+b') as fp:
wb = WheelBuilder(poetry, io, fp)
wb = WheelBuilder(poetry, io, fp, original=original)
wb.build()
wheel_path = directory / wb.wheel_filename
......@@ -71,13 +80,38 @@ class WheelBuilder(Builder):
def build(self) -> None:
self._io.writeln(' - Building <info>wheel</info>')
try:
self._build()
self.copy_module()
self.write_metadata()
self.write_record()
finally:
self._wheel_zip.close()
self._io.writeln(f' - Built <comment>{self.wheel_filename}</>')
self._io.writeln(f' - Built <fg=cyan>{self.wheel_filename}</>')
def _build(self) -> None:
if self._package.build:
setup = self._path / 'setup.py'
# We need to place ourselves in the temporary
# directory in order to build the package
current_path = os.getcwd()
try:
os.chdir(str(self._path))
self._io.venv.run(
'python',
str(setup),
'build',
'-b', str(self._path / 'build')
)
finally:
os.chdir(current_path)
build_dir = self._path / 'build'
lib = list(build_dir.glob('lib.*'))[0]
for pkg in lib.glob('*'):
shutil.rmtree(str(self._path / pkg.name))
shutil.copytree(str(pkg), str(self._path / pkg.name))
def copy_module(self) -> None:
if self._module.is_package():
......@@ -119,17 +153,54 @@ class WheelBuilder(Builder):
# RECORD itself is recorded with no hash or size
f.write(self.dist_info + '/RECORD,,\n')
def find_excluded_files(self) -> list:
# Checking VCS
vcs = get_vcs(self._original_path)
if not vcs:
return []
ignored = vcs.get_ignored_files()
result = []
for file in ignored:
try:
file = Path(file).absolute().relative_to(self._original_path)
except ValueError:
# Should only happen in tests
continue
result.append(file)
return result
@property
def dist_info(self) -> str:
return self.dist_info_name(self._package.name, self._package.version)
@property
def wheel_filename(self) -> str:
tag = ('py2.' if self.supports_python2() else '') + 'py3-none-any'
if self._package.build:
platform = get_platform().replace('.', '_').replace('-', '_')
impl_name = get_abbr_impl()
impl_ver = get_impl_ver()
impl = impl_name + impl_ver
abi_tag = str(get_abi_tag()).lower()
tag = (impl, abi_tag, platform)
else:
platform = 'any'
if self.supports_python2():
impl = 'py2.py3'
else:
impl = 'py3'
tag = (impl, 'none', platform)
tag = '-'.join(tag)
return '{}-{}-{}.whl'.format(
re.sub("[^\w\d.]+", "_", self._package.pretty_name, flags=re.UNICODE),
re.sub("[^\w\d.]+", "_", self._package.version, flags=re.UNICODE),
tag)
tag
)
def supports_python2(self):
return self._package.python_constraint.matches(
......
"""Generate and work with PEP 425 Compatibility Tags."""
import distutils.util
import platform
import sys
import sysconfig
import warnings
def get_config_var(var):
try:
return sysconfig.get_config_var(var)
except IOError as e:
warnings.warn("{0}".format(e), RuntimeWarning)
return None
def get_abbr_impl():
"""Return abbreviated implementation name."""
impl = platform.python_implementation()
if impl == 'PyPy':
return 'pp'
elif impl == 'Jython':
return 'jy'
elif impl == 'IronPython':
return 'ip'
elif impl == 'CPython':
return 'cp'
raise LookupError('Unknown Python implementation: ' + impl)
def get_impl_ver():
"""Return implementation version."""
impl_ver = get_config_var("py_version_nodot")
if not impl_ver or get_abbr_impl() == 'pp':
impl_ver = ''.join(map(str, get_impl_version_info()))
return impl_ver
def get_impl_version_info():
"""Return sys.version_info-like tuple for use in decrementing the minor
version."""
if get_abbr_impl() == 'pp':
# as per https://github.com/pypa/pip/issues/2882
return (sys.version_info[0], sys.pypy_version_info.major,
sys.pypy_version_info.minor)
else:
return sys.version_info[0], sys.version_info[1]
def get_flag(var, fallback, expected=True, warn=True):
"""Use a fallback method for determining SOABI flags if the needed config
var is unset or unavailable."""
val = get_config_var(var)
if val is None:
if warn:
warnings.warn("Config variable '{0}' is unset, Python ABI tag may "
"be incorrect".format(var), RuntimeWarning, 2)
return fallback()
return val == expected
def get_abi_tag():
"""Return the ABI tag based on SOABI (if available) or emulate SOABI
(CPython 2, PyPy)."""
soabi = get_config_var('SOABI')
impl = get_abbr_impl()
if not soabi and impl in ('cp', 'pp') and hasattr(sys, 'maxunicode'):
d = ''
m = ''
u = ''
if get_flag('Py_DEBUG',
lambda: hasattr(sys, 'gettotalrefcount'),
warn=(impl == 'cp')):
d = 'd'
if get_flag('WITH_PYMALLOC',
lambda: impl == 'cp',
warn=(impl == 'cp')):
m = 'm'
if get_flag('Py_UNICODE_SIZE',
lambda: sys.maxunicode == 0x10ffff,
expected=4,
warn=(impl == 'cp' and
sys.version_info < (3, 3))) \
and sys.version_info < (3, 3):
u = 'u'
abi = '%s%s%s%s%s' % (impl, get_impl_ver(), d, m, u)
elif soabi and soabi.startswith('cpython-'):
abi = 'cp' + soabi.split('-')[1]
elif soabi:
abi = soabi.replace('.', '_').replace('-', '_')
else:
abi = None
return abi
def get_platform():
"""Return our platform name 'win32', 'linux_x86_64'"""
# XXX remove distutils dependency
result = distutils.util.get_platform().replace('.', '_').replace('-', '_')
if result == "linux_x86_64" and sys.maxsize == 2147483647:
# pip pull request #3497
result = "linux_i686"
return result
def get_supported(versions=None, supplied_platform=None):
"""Return a list of supported tags for each version specified in
`versions`.
:param versions: a list of string versions, of the form ["33", "32"],
or None. The first version will be assumed to support our ABI.
"""
supported = []
# Versions must be given with respect to the preference
if versions is None:
versions = []
version_info = get_impl_version_info()
major = version_info[:-1]
# Support all previous minor Python versions.
for minor in range(version_info[-1], -1, -1):
versions.append(''.join(map(str, major + (minor,))))
impl = get_abbr_impl()
abis = []
abi = get_abi_tag()
if abi:
abis[0:0] = [abi]
abi3s = set()
import imp
for suffix in imp.get_suffixes():
if suffix[0].startswith('.abi'):
abi3s.add(suffix[0].split('.', 2)[1])
abis.extend(sorted(list(abi3s)))
abis.append('none')
platforms = []
if supplied_platform:
platforms.append(supplied_platform)
platforms.append(get_platform())
# Current version, current API (built specifically for our Python):
for abi in abis:
for arch in platforms:
supported.append(('%s%s' % (impl, versions[0]), abi, arch))
# abi3 modules compatible with older version of Python
for version in versions[1:]:
# abi3 was introduced in Python 3.2
if version in ('31', '30'):
break
for abi in abi3s: # empty set if not Python 3
for arch in platforms:
supported.append(("%s%s" % (impl, version), abi, arch))
# No abi / arch, but requires our implementation:
for i, version in enumerate(versions):
supported.append(('%s%s' % (impl, version), 'none', 'any'))
if i == 0:
# Tagged specifically as being cross-version compatible
# (with just the major version specified)
supported.append(('%s%s' % (impl, versions[0][0]), 'none', 'any'))
# Major Python version + platform; e.g. binaries not using the Python API
supported.append(('py%s' % (versions[0][0]), 'none', arch))
# No abi / arch, generic Python
for i, version in enumerate(versions):
supported.append(('py%s' % (version,), 'none', 'any'))
if i == 0:
supported.append(('py%s' % (version[0]), 'none', 'any'))
return supported
......@@ -86,6 +86,10 @@ class Package:
# Requirements for making it mandatory
self.requirements = {}
self.build = None
self.include = []
self.exclude = []
self._python_versions = '*'
self._python_constraint = self._parser.parse_constraints('*')
self._platform = '*'
......
......@@ -107,6 +107,15 @@ class Poetry:
Dependency(req, '*') for req in requirements
]
if 'build' in local_config:
package.build = local_config['build']
if 'include' in local_config:
package.include = local_config['include']
if 'exclude' in local_config:
package.exclude = local_config['exclude']
locker = Locker(poetry_file.with_suffix('.lock'), local_config)
return cls(poetry_file, local_config, package, locker)
......@@ -3,6 +3,22 @@ import os
import subprocess
import sys
from subprocess import CalledProcessError
class VenvError(Exception):
pass
class VenvCommandError(VenvError):
def __init__(self, e: CalledProcessError):
message = f'Command {e.cmd} errored with the following output: \n' \
f'{e.output.decode()}'
super().__init__(message)
class Venv:
......@@ -79,10 +95,13 @@ class Venv:
"""
cmd = [self._bin(bin)] + list(args)
output = subprocess.check_output(
cmd, stderr=subprocess.STDOUT,
**kwargs
)
try:
output = subprocess.check_output(
cmd, stderr=subprocess.STDOUT,
**kwargs
)
except CalledProcessError as e:
raise VenvCommandError(e)
return output.decode()
......
......@@ -8,4 +8,4 @@ def get_vcs(directory: Path):
for p in [directory] + list(directory.parents):
if (p / '.git').is_dir():
return Git()
return Git(p)
......@@ -26,8 +26,9 @@ class GitConfig:
class Git:
def __init__(self):
def __init__(self, work_dir=None):
self._config = GitConfig()
self._work_dir = work_dir
@property
def config(self) -> GitConfig:
......@@ -36,24 +37,55 @@ class Git:
def clone(self, repository, dest) -> str:
return self.run('clone', repository, dest)
def checkout(self, rev, folder) -> str:
return self.run(
'--git-dir', (folder / '.git').as_posix(),
'--work-tree', folder.as_posix(),
def checkout(self, rev, folder=None) -> str:
args = []
if folder is None and self._work_dir:
folder = self._work_dir
if folder:
args += [
'--git-dir', (folder / '.git').as_posix(),
'--work-tree', folder.as_posix()
]
args += [
'checkout', rev
)
]
return self.run(*args)
def rev_parse(self, rev, folder=None) -> str:
args = []
if folder is None and self._work_dir:
folder = self._work_dir
def rev_parse(self, rev, folder) -> str:
return self.run(
'--git-dir', (folder / '.git').as_posix(),
'--work-tree', folder.as_posix(),
if folder:
args += [
'--git-dir', (folder / '.git').as_posix(),
'--work-tree', folder.as_posix()
]
args += [
'rev-parse', rev
)
]
return self.run(*args)
def get_ignored_files(self, folder=None) -> list:
args = []
if folder is None and self._work_dir:
folder = self._work_dir
if folder:
args += [
'--git-dir', (folder / '.git').as_posix(),
'--work-tree', folder.as_posix()
]
def get_ignored_files(self) -> list:
output = self.run(
args += [
'ls-files', '--others', '-i', '--exclude-standard'
)
]
output = self.run(*args)
return output.split('\n')
......
from distutils.core import Extension
extensions = [
Extension('extended.extended', ['extended/extended.c']),
]
def build(setup_kwargs):
setup_kwargs.update({
'ext_modules': extensions
})
#include <Python.h>
static PyObject *hello(PyObject *self) {
return PyUnicode_FromString("Hello");
}
static PyMethodDef module_methods[] = {
{
"hello",
(PyCFunction) hello,
NULL,
PyDoc_STR("Say hello.")
},
{NULL}
};
static struct PyModuleDef moduledef = {
PyModuleDef_HEAD_INIT,
"extended",
NULL,
-1,
module_methods,
NULL,
NULL,
NULL,
NULL,
};
PyMODINIT_FUNC
PyInit_extended(void)
{
PyObject *module;
module = PyModule_Create(&moduledef);
if (module == NULL)
return NULL;
return module;
}
[tool.poetry]
name = "extended"
version = "0.1"
description = "Some description."
authors = [
"Sébastien Eustace <sebastien@eustace.io>"
]
license = "MIT"
readme = "README.rst"
homepage = "https://poetry.eustace.io/"
build = "build.py"
include = ["extended.c", "README.rst", "build.py"]
import pytest
import shutil
import tarfile
import zipfile
from pathlib import Path
from poetry import Poetry
from poetry.io import NullIO
from poetry.masonry.builders import CompleteBuilder
fixtures_dir = Path(__file__).parent / 'fixtures'
@pytest.fixture(autouse=True)
def setup():
clear_samples_dist()
yield
clear_samples_dist()
def clear_samples_dist():
for dist in fixtures_dir.glob('**/dist'):
if dist.is_dir():
shutil.rmtree(str(dist))
def test_wheel_c_extension():
module_path = fixtures_dir / 'extended'
builder = CompleteBuilder(Poetry.create(module_path), NullIO(True))
builder.build()
sdist = fixtures_dir / 'extended' / 'dist' / 'extended-0.1.tar.gz'
assert sdist.exists()
tar = tarfile.open(str(sdist), 'r')
assert 'extended-0.1/build.py' in tar.getnames()
assert 'extended-0.1/extended/extended.c' in tar.getnames()
whl = list((module_path / 'dist').glob('extended-0.1-cp3*-cp3*m-*.whl'))[0]
assert whl.exists()
zip = zipfile.ZipFile(whl)
has_compiled_extension = False
for name in zip.namelist():
if name.startswith('extended/extended') and name.endswith('.so'):
has_compiled_extension = True
assert has_compiled_extension
import ast
import pytest
import shutil
import tarfile
from pathlib import Path
......@@ -127,3 +128,20 @@ def test_prelease():
sdist = fixtures_dir / 'prerelease' / 'dist' / 'prerelease-0.1b1.tar.gz'
assert sdist.exists()
def test_with_c_extensions():
poetry = Poetry.create(project('extended'))
builder = SdistBuilder(poetry, NullIO())
builder.build()
sdist = fixtures_dir / 'extended' / 'dist' / 'extended-0.1.tar.gz'
assert sdist.exists()
tar = tarfile.open(str(sdist), 'r')
assert 'extended-0.1/build.py' in tar.getnames()
assert 'extended-0.1/extended/extended.c' in tar.getnames()
......@@ -87,7 +87,6 @@ cwd = "./handlers"
REDIS_PASSWORD = "MYPASSWORD"
#REDIS_PASSWORD = ""
"""
print(f.dumps())
assert expected == f.dumps()
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment