Add a wheel builder

poetry/masonry/builders/builder.py

+import os
+import re
+
+from collections import defaultdict
+from pathlib import Path
+
+from poetry.semver.constraints import Constraint
+from poetry.semver.version_parser import VersionParser
+from poetry.vcs import get_vcs
+
+from ..utils.module import Module
+
+
+AUTHOR_REGEX = re.compile('(?u)^(?P<name>[- .,\w\d\'’"()]+) <(?P<email>.+?)>$')
+
+
+class Builder:
+
+    AVAILABLE_PYTHONS = {
+        '2',
+        '2.7',
+        '3',
+        '3.4', '3.5', '3.6', '3.7'
+    }
+
+    def __init__(self, poetry):
+        self._poetry = poetry
+        self._package = poetry.package
+        self._path = poetry.file.parent
+        self._module = Module(self._package.name, self._path.as_posix())
+
+    def build(self):
+        raise NotImplementedError()
+
+    def find_excluded_files(self) -> list:
+        # Checking VCS
+        vcs = get_vcs(self._path)
+        if not vcs:
+            return []
+
+        ignored = vcs.get_ignored_files()
+        result = []
+        for file in ignored:
+            try:
+                file = Path(file).absolute().relative_to(self._path)
+            except ValueError:
+                # Should only happen in tests
+                continue
+
+            result.append(file)
+
+        return result
+
+    def find_files_to_add(self) -> list:
+        """
+        Finds all files to add to the tarball
+
+        TODO: Support explicit include/exclude
+        """
+        excluded = self.find_excluded_files()
+        src = self._module.path
+        to_add = []
+
+        for root, dirs, files in os.walk(src.as_posix()):
+            root = Path(root)
+            if root.name == '__pycache__':
+                continue
+
+            for file in files:
+                file = root / file
+                file = file.relative_to(self._path)
+
+                if file in excluded:
+                    continue
+
+                if file.suffix == '.pyc':
+                    continue
+
+                to_add.append(file)
+
+        # Include project files
+        to_add.append(Path('pyproject.toml'))
+
+        # If a README is specificed we need to include it
+        # to avoid errors
+        if 'readme' in self._poetry.config:
+            readme = self._path / self._poetry.config['readme']
+            if readme.exists():
+                to_add.append(readme.relative_to(self._path))
+
+        return sorted(to_add)
+
+    def convert_entry_points(self) -> dict:
+        result = defaultdict(list)
+
+        # Scripts -> Entry points
+        for name, ep in self._poetry.config.get('scripts', {}).items():
+            result['console_scripts'].append(f'{name} = {ep}')
+
+        # Plugins -> entry points
+        for groupname, group in self._poetry.config.get('plugins', {}).items():
+            for name, ep in sorted(group.items()):
+                result[groupname].append(f'{name} = {ep}')
+
+        return dict(result)
+
+    @classmethod
+    def convert_author(cls, author) -> dict:
+        m = AUTHOR_REGEX.match(author)
+
+        name = m.group('name')
+        email = m.group('email')
+
+        return {
+            'name': name,
+            'email': email
+        }
+
+    def get_classifers(self):
+        classifiers = []
+
+        # Automatically set python classifiers
+        parser = VersionParser()
+        if self._package.python_versions == '*':
+            python_constraint = parser.parse_constraints('~2.7 || ^3.4')
+        else:
+            python_constraint = self._package.python_constraint
+
+        for version in sorted(self.AVAILABLE_PYTHONS):
+            if python_constraint.matches(Constraint('=', version)):
+                classifiers.append(f'Programming Language :: Python :: {version}')
+
+        return classifiers

poetry/masonry/builders/complete.py

-from .sdist import SdistBuilder
+import os
+import tarfile
+
+import poetry
 
 
+from contextlib import contextmanager
+from tempfile import TemporaryDirectory
+from types import SimpleNamespace
 
 
-class CompleteBuilder:
+from .builder import Builder
+from .sdist import SdistBuilder
+from .wheel import WheelBuilder
 
 
-    def __init__(self, poetry):
-        self._poetry = poetry
+
+class CompleteBuilder(Builder):
 
 
     def build(self):
     def build(self):
         # We start by building the tarball
         # We start by building the tarball
         # We will use it to build the wheel
         # We will use it to build the wheel
         sdist_builder = SdistBuilder(self._poetry)
         sdist_builder = SdistBuilder(self._poetry)
         sdist_file = sdist_builder.build()
         sdist_file = sdist_builder.build()
+        sdist_info = SimpleNamespace(builder=sdist_builder, file=sdist_file)
+
+        dist_dir = self._path / 'dist'
+        with self.unpacked_tarball(sdist_file) as tmpdir:
+            wheel_info = WheelBuilder.make_in(poetry.Poetry.create(tmpdir), dist_dir)
+
+        return SimpleNamespace(wheel=wheel_info, sdist=sdist_info)
+
+    @classmethod
+    @contextmanager
+    def unpacked_tarball(cls, path):
+        tf = tarfile.open(str(path))
+
+        with TemporaryDirectory() as tmpdir:
+            tf.extractall(tmpdir)
+            files = os.listdir(tmpdir)
+
+            assert len(files) == 1, files
+
+            yield os.path.join(tmpdir, files[0])

poetry/masonry/builders/sdist.py

 import os
 import os
-import re
 import tarfile
 import tarfile
 
 
 from collections import defaultdict
 from collections import defaultdict
@@ -13,10 +12,10 @@ from typing import List
 
 
 from poetry.packages import Dependency
 from poetry.packages import Dependency
 from poetry.semver.constraints import MultiConstraint
 from poetry.semver.constraints import MultiConstraint
-from poetry.vcs import get_vcs
 
 
 from ..utils.helpers import normalize_file_permissions
 from ..utils.helpers import normalize_file_permissions
-from ..utils.module import Module
+
+from .builder import Builder
 
 
 
 
 SETUP = """\
 SETUP = """\
@@ -45,16 +44,10 @@ Author-email: {author_email}
 """
 """
 
 
 
 
-AUTHOR_REGEX = re.compile('(?u)^(?P<name>[- .,\w\d\'’"()]+) <(?P<email>.+?)>$')
-
-
-class SdistBuilder:
+class SdistBuilder(Builder):
 
 
     def __init__(self, poetry):
     def __init__(self, poetry):
-        self._poetry = poetry
-        self._package = self._poetry.package
-        self._path = poetry.file.parent
-        self._module = Module(self._package.name, self._path.as_posix())
+        super().__init__(poetry)
 
 
     def build(self, target_dir: Path = None) -> Path:
     def build(self, target_dir: Path = None) -> Path:
         if target_dir is None:
         if target_dir is None:
@@ -112,54 +105,6 @@ class SdistBuilder:
 
 
         return target
         return target
 
 
-    def find_excluded_files(self) -> list:
-        # Checking VCS
-        vcs = get_vcs(self._path)
-        if not vcs:
-            return []
-
-        ignored = vcs.get_ignored_files()
-        result = []
-        for file in ignored:
-            try:
-                file = Path(file).absolute().relative_to(self._path)
-            except ValueError:
-                # Should only happen in tests
-                continue
-
-            result.append(file)
-
-        return result
-
-    def find_files_to_add(self) -> list:
-        """
-        Finds all files to add to the tarball
-
-        TODO: Support explicit include/exclude
-        """
-        excluded = self.find_excluded_files()
-        src = self._module.path
-        to_add = []
-
-        for root, dirs, files in os.walk(src.as_posix()):
-            root = Path(root)
-            if root.name == '__pycache__':
-                continue
-
-            for file in files:
-                file = root / file
-                file = file.relative_to(self._path)
-
-                if file in excluded:
-                    continue
-
-                if file.suffix == '.pyc':
-                    continue
-
-                to_add.append(file)
-
-        return to_add
-
     def build_setup(self) -> bytes:
     def build_setup(self) -> bytes:
         before, extra = [], []
         before, extra = [], []
 
 
@@ -268,59 +213,15 @@ class SdistBuilder:
         extras = []
         extras = []
 
 
         for dependency in dependencies:
         for dependency in dependencies:
-            is_extra = False
-            requirement = dependency.pretty_name
-            constraint = dependency.constraint
-            if isinstance(constraint, MultiConstraint):
-                requirement += ','.join(
-                    [str(c).replace(' ', '') for c in constraint.constraints]
-                )
-            else:
-                requirement += str(constraint).replace(' ', '')
-            if str(dependency.python_constraint) != '*':
-                is_extra = True
-                python_constraint = dependency.python_constraint
-                requirement += '; python_version'
-                if isinstance(python_constraint, MultiConstraint):
-                    requirement += ','.join(
-                        [str(c).replace(' ', '') for c in python_constraint.constraints]
-                    )
-                else:
-                    requirement += str(python_constraint).replace(' ', '')
+            requirement = dependency.to_pep_508()
 
 
-            if is_extra:
+            if ';' in requirement:
                 extras.append(requirement)
                 extras.append(requirement)
             else:
             else:
                 main.append(requirement)
                 main.append(requirement)
 
 
         return main, extras
         return main, extras
 
 
-    def convert_entry_points(self) -> dict:
-        result = defaultdict(list)
-
-        # Scripts -> Entry points
-        for name, ep in self._poetry.config.get('scripts', {}).items():
-            result['console_scripts'].append(f'{name} = {ep}')
-
-        # Plugins -> entry points
-        for groupname, group in self._poetry.config.get('plugins', {}).items():
-            for name, ep in sorted(group.items()):
-                result[groupname].append(f'{name} = {ep}')
-
-        return dict(result)
-
-    @classmethod
-    def convert_author(cls, author) -> dict:
-        m = AUTHOR_REGEX.match(author)
-
-        name = m.group('name')
-        email = m.group('email')
-
-        return {
-            'name': name,
-            'email': email
-        }
-
     @classmethod
     @classmethod
     def clean_tarinfo(cls, tar_info):
     def clean_tarinfo(cls, tar_info):
         """
         """

poetry/masonry/builders/wheel.py

-class WheelBuilder:
+import contextlib
+import hashlib
+import os
+import re
+import tempfile
+import stat
+import zipfile
 
 
-    def __init__(self):
-        pass
+from base64 import urlsafe_b64encode
+from io import StringIO
+from pathlib import Path
+from types import SimpleNamespace
+
+from poetry.__version__ import __version__
+from poetry.semver.constraints import Constraint
+from poetry.semver.constraints import MultiConstraint
+
+from ..utils.helpers import normalize_file_permissions
+from .builder import Builder
+
+
+wheel_file_template = """\
+Wheel-Version: 1.0
+Generator: poetry {version}
+Root-Is-Purelib: true
+""".format(version=__version__)
+
+
+class WheelBuilder(Builder):
+
+    def __init__(self, poetry, target_fp):
+        super().__init__(poetry)
+
+        self._records = []
+
+        # Open the zip file ready to write
+        self._wheel_zip = zipfile.ZipFile(target_fp, 'w',
+                                          compression=zipfile.ZIP_DEFLATED)
+
+    @classmethod
+    def make_in(cls, poetry, directory) -> SimpleNamespace:
+        # We don't know the final filename until metadata is loaded, so write to
+        # a temporary_file, and rename it afterwards.
+        (fd, temp_path) = tempfile.mkstemp(suffix='.whl',
+                                           dir=str(directory))
+        try:
+            with open(fd, 'w+b') as fp:
+                wb = WheelBuilder(poetry, fp)
+                wb.build()
+
+            wheel_path = directory / wb.wheel_filename
+            os.replace(temp_path, str(wheel_path))
+        except:
+            os.unlink(temp_path)
+            raise
+
+        return SimpleNamespace(builder=wb, file=wheel_path)
+
+    @classmethod
+    def make(cls, poetry) -> SimpleNamespace:
+        """Build a wheel in the dist/ directory, and optionally upload it.
+            """
+        dist_dir = poetry.file.parent / 'dist'
+        try:
+            dist_dir.mkdir()
+        except FileExistsError:
+            pass
+
+        return cls.make_in(poetry, dist_dir)
+
+    def build(self) -> None:
+        try:
+            self.copy_module()
+            self.write_metadata()
+            self.write_record()
+        finally:
+            self._wheel_zip.close()
+
+    def copy_module(self) -> None:
+        if self._module.is_package():
+            files = self.find_files_to_add()
+
+            # Walk the files and compress them,
+            # sorting everything so the order is stable.
+            for file in sorted(files):
+                full_path = self._path / file
+
+                # Do not include topmost files
+                if full_path.relative_to(self._path) == Path(file.name):
+                    continue
+
+                self._add_file(full_path, file)
+        else:
+            self._add_file(str(self._module.path), self._module.path.name)
+
+    def write_metadata(self):
+        if 'scripts' in self._poetry.config or 'plugins' in self._poetry.config:
+            with self._write_to_zip(self.dist_info + '/entry_points.txt') as f:
+                self._write_entry_points(f)
+
+        for base in ('COPYING', 'LICENSE'):
+            for path in sorted(self._path.glob(base + '*')):
+                self._add_file(path, '%s/%s' % (self.dist_info, path.name))
+
+        with self._write_to_zip(self.dist_info + '/WHEEL') as f:
+            self._write_wheel_file(f)
+
+        with self._write_to_zip(self.dist_info + '/METADATA') as f:
+            self._write_metadata_file(f)
+
+    def write_record(self):
+        # Write a record of the files in the wheel
+        with self._write_to_zip(self.dist_info + '/RECORD') as f:
+            for path, hash, size in self._records:
+                f.write('{},sha256={},{}\n'.format(path, hash, size))
+            # RECORD itself is recorded with no hash or size
+            f.write(self.dist_info + '/RECORD,,\n')
+
+    @property
+    def dist_info(self) -> str:
+        return self.dist_info_name(self._package.name, self._package.version)
+
+    @property
+    def wheel_filename(self) -> str:
+        tag = ('py2.' if self.supports_python2() else '') + 'py3-none-any'
+        return '{}-{}-{}.whl'.format(
+            re.sub("[^\w\d.]+", "_", self._package.pretty_name, flags=re.UNICODE),
+            re.sub("[^\w\d.]+", "_", self._package.pretty_version, flags=re.UNICODE),
+            tag)
+
+    def supports_python2(self):
+        return self._package.python_constraint.matches(
+            MultiConstraint([
+                Constraint('>=', '2.0.0'),
+                Constraint('<', '3.0.0')
+            ])
+        )
+
+    def dist_info_name(self, distribution, version) -> str:
+        escaped_name = re.sub("[^\w\d.]+", "_", distribution, flags=re.UNICODE)
+        escaped_version = re.sub("[^\w\d.]+", "_", version, flags=re.UNICODE)
+
+        return '{}-{}.dist-info'.format(escaped_name, escaped_version)
+
+    def _add_file(self, full_path, rel_path):
+        full_path, rel_path = str(full_path), str(rel_path)
+        if os.sep != '/':
+            # We always want to have /-separated paths in the zip file and in
+            # RECORD
+            rel_path = rel_path.replace(os.sep, '/')
+
+        zinfo = zipfile.ZipInfo.from_file(full_path, rel_path)
+
+        # Normalize permission bits to either 755 (executable) or 644
+        st_mode = os.stat(full_path).st_mode
+        new_mode = normalize_file_permissions(st_mode)
+        zinfo.external_attr = (new_mode & 0xFFFF) << 16  # Unix attributes
+
+        if stat.S_ISDIR(st_mode):
+            zinfo.external_attr |= 0x10  # MS-DOS directory flag
+
+        hashsum = hashlib.sha256()
+        with open(full_path, 'rb') as src, self._wheel_zip.open(zinfo,
+                                                                'w') as dst:
+            while True:
+                buf = src.read(1024 * 8)
+                if not buf:
+                    break
+                hashsum.update(buf)
+                dst.write(buf)
+
+        size = os.stat(full_path).st_size
+        hash_digest = urlsafe_b64encode(hashsum.digest()).decode(
+            'ascii').rstrip('=')
+
+        self._records.append((rel_path, hash_digest, size))
+
+    @contextlib.contextmanager
+    def _write_to_zip(self, rel_path):
+        sio = StringIO()
+        yield sio
+
+        # The default is a fixed timestamp rather than the current time, so
+        # that building a wheel twice on the same computer can automatically
+        # give you the exact same result.
+        date_time = (2016, 1, 1, 0, 0, 0)
+        zi = zipfile.ZipInfo(rel_path, date_time)
+        b = sio.getvalue().encode('utf-8')
+        hashsum = hashlib.sha256(b)
+        hash_digest = urlsafe_b64encode(
+            hashsum.digest()
+        ).decode('ascii').rstrip('=')
+
+        self._wheel_zip.writestr(zi, b, compress_type=zipfile.ZIP_DEFLATED)
+        self._records.append((rel_path, hash_digest, len(b)))
+
+    def _write_entry_points(self, fp):
+        """
+        Write entry_points.txt.
+        """
+        entry_points = self.convert_entry_points()
+
+        for group_name in sorted(entry_points):
+            fp.write('[{}]\n'.format(group_name))
+            for ep in sorted(entry_points[group_name]):
+                fp.write(ep.replace(' ', ''))
+
+            fp.write('\n')
+
+    def _write_wheel_file(self, fp):
+        fp.write(wheel_file_template)
+
+        if self.supports_python2():
+            fp.write("Tag: py2-none-any\n")
+
+        fp.write("Tag: py3-none-any\n")
+
+    def _write_metadata_file(self, fp):
+        """
+        Write out metadata in the 1.x format (email like)
+        """
+        fp.write('Metadata-Version: 1.2\n')
+        fp.write(f'Name: {self._package.name}\n')
+        fp.write(f'Version: {self._package.version}\n')
+        fp.write(f'Summary: {self._package.description}\n')
+        fp.write(f'Home-page: {self._package.homepage or self._package.repository_url or "UNKNOWN"}\n')
+        fp.write(f'License: {self._package.license or "UNKOWN"}\n')
+
+        # Optional fields
+        if self._package.keywords:
+            fp.write(f"Keywords: {','.join(self._package.keywords)}\n")
+
+        if self._package.authors:
+            author = self.convert_author(self._package.authors[0])
+
+            fp.write(f'Author: {author["name"]}\n')
+            fp.write(f'Author-email: {author["email"]}\n')
+
+        if self._package.python_versions != '*':
+            constraint = self._package.python_constraint
+            if isinstance(constraint, MultiConstraint):
+                python_requires = ','.join(
+                    [str(c).replace(' ', '') for c in constraint.constraints]
+                )
+            else:
+                python_requires = str(constraint).replace(' ', '')
+
+            fp.write(f'Requires-Python: {python_requires}\n')
+
+        classifiers = self.get_classifers()
+        for classifier in classifiers:
+            fp.write(f'Classifier: {classifier}\n')
+
+        for dep in self._package.requires:
+            fp.write('Requires-Dist: {}\n'.format(dep.to_pep_508()))
+
+        if self._package.readme is not None:
+            fp.write('\n' + self._package.readme + '\n')

poetry/packages/dependency.py

 import poetry.packages
 import poetry.packages
 
 
 from poetry.semver.constraints import Constraint
 from poetry.semver.constraints import Constraint
+from poetry.semver.constraints import MultiConstraint
 from poetry.semver.constraints.base_constraint import BaseConstraint
 from poetry.semver.constraints.base_constraint import BaseConstraint
 from poetry.semver.version_parser import VersionParser
 from poetry.semver.version_parser import VersionParser
 
 
@@ -97,6 +98,37 @@ class Dependency:
             and (not package.is_prerelease() or self.allows_prereleases())
             and (not package.is_prerelease() or self.allows_prereleases())
         )
         )
 
 
+    def to_pep_508(self) -> str:
+        requirement = f'{self.pretty_name}'
+
+        if isinstance(self.constraint, MultiConstraint):
+            requirement += ','.join(
+                [str(c).replace(' ', '') for c in self.constraint.constraints]
+            )
+        else:
+            requirement += str(self.constraint).replace(' ', '')
+
+        # Markers
+        markers = []
+
+        # Python marker
+        if self.python_versions != '*':
+            python_constraint = self.python_constraint
+            marker = 'python_version'
+            if isinstance(python_constraint, MultiConstraint):
+                marker += ','.join(
+                    [str(c).replace(' ', '') for c in python_constraint.constraints]
+                )
+            else:
+                marker += str(python_constraint).replace(' ', '')
+
+            markers.append(marker)
+
+        if markers:
+            requirement += f'; {" and ".join(markers)}'
+
+        return requirement
+
     def __eq__(self, other):
     def __eq__(self, other):
         if not isinstance(other, Dependency):
         if not isinstance(other, Dependency):
             return NotImplemented
             return NotImplemented

poetry/packages/package.py

@@ -53,6 +53,8 @@ class Package:
         self.homepage = None
         self.homepage = None
         self.repository_url = None
         self.repository_url = None
         self.keywords = []
         self.keywords = []
+        self.license = None
+        self.readme = ''
 
 
         self.source_type = ''
         self.source_type = ''
         self.source_reference = ''
         self.source_reference = ''

poetry/poetry.py

@@ -80,6 +80,12 @@ class Poetry:
         package.description = local_config.get('description', '')
         package.description = local_config.get('description', '')
         package.homepage = local_config.get('homepage')
         package.homepage = local_config.get('homepage')
         package.repository_url = local_config.get('repository')
         package.repository_url = local_config.get('repository')
+        package.license = local_config.get('license')
+        package.keywords = local_config.get('keywords', [])
+
+        if 'readme' in local_config:
+            with open(poetry_file.parent / local_config['readme']) as f:
+                package.readme = f.read()
 
 
         if 'platform' in local_config:
         if 'platform' in local_config:
             package.platform = local_config['platform']
             package.platform = local_config['platform']

pyproject.toml

@@ -26,3 +26,4 @@ pip-tools = "^1.11"
 
 
 [tool.poetry.dev-dependencies]
 [tool.poetry.dev-dependencies]
 pytest = "~3.4"
 pytest = "~3.4"
+pytest-cov = "^2.5"

tests/masonry/builders/fixtures/module1/README.rst

+Module 1
+========

tests/masonry/builders/fixtures/module1/module1.py

+"""Example module"""
+
+__version__ = '0.1'

tests/masonry/builders/fixtures/module1/pyproject.toml

+[tool.poetry]
+name = "module1"
+version = "0.1"
+description = "Some description."
+authors = [
+    "Sébastien Eustace <sebastien@eustace.io>"
+]
+license = "MIT"
+
+readme = "README.rst"
+
+homepage = "https://poetry.eustace.io/"

tests/masonry/builders/test_sdist.py

@@ -76,9 +76,11 @@ def test_find_files_to_add():
     result = builder.find_files_to_add()
     result = builder.find_files_to_add()
 
 
     assert result == [
     assert result == [
+        Path('README.rst'),
         Path('my_package/__init__.py'),
         Path('my_package/__init__.py'),
-        Path('my_package/sub_pkg1/__init__.py'),
         Path('my_package/data1/test.json'),
         Path('my_package/data1/test.json'),
+        Path('my_package/sub_pkg1/__init__.py'),
         Path('my_package/sub_pkg2/__init__.py'),
         Path('my_package/sub_pkg2/__init__.py'),
         Path('my_package/sub_pkg2/data2/data.json'),
         Path('my_package/sub_pkg2/data2/data.json'),
+        Path('pyproject.toml'),
     ]
     ]

tests/masonry/builders/test_wheel.py

+import pytest
+import shutil
+
+from pathlib import Path
+
+from poetry import Poetry
+from poetry.masonry.builders import WheelBuilder
+
+
+fixtures_dir = Path(__file__).parent / 'fixtures'
+
+
+@pytest.fixture(autouse=True)
+def setup():
+    clear_samples_dist()
+
+    yield
+
+    clear_samples_dist()
+
+
+def clear_samples_dist():
+    for dist in fixtures_dir.glob('**/dist'):
+        if dist.is_dir():
+            shutil.rmtree(str(dist))
+
+
+def test_wheel_module():
+    module_path = fixtures_dir / 'module1'
+    WheelBuilder.make(Poetry.create(str(module_path)))
+
+    whl = module_path / 'dist' / 'module1-0.1-py2.py3-none-any.whl'
+
+    assert whl.exists()
+
+
+def test_wheel_package():
+    module_path = fixtures_dir / 'complete'
+    WheelBuilder.make(Poetry.create(str(module_path)))
+
+    whl = module_path / 'dist' / 'my_package-1.2.3-py3-none-any.whl'
+
+    assert whl.exists()