Add the build command with a tarball builder (sdist)

poetry/__version__.py

-__version__ = '0.3.0'
+__version__ = '0.4.0-beta'

poetry/console/application.py

@@ -7,6 +7,7 @@ from poetry.utils.venv import Venv
 
 from .commands import AboutCommand
 from .commands import AddCommand
+from .commands import BuildCommand
 from .commands import InstallCommand
 from .commands import LockCommand
 from .commands import NewCommand
@@ -45,6 +46,7 @@ class Application(BaseApplication):
         return commands + [
             AboutCommand(),
             AddCommand(),
+            BuildCommand(),
             InstallCommand(),
             LockCommand(),
             NewCommand(),

poetry/console/commands/__init__.py

 from .about import AboutCommand
 from .add import AddCommand
+from .build import BuildCommand
 from .install import InstallCommand
 from .lock import LockCommand
 from .new import NewCommand

poetry/console/commands/build.py

+from .command import Command
+
+from poetry.masonry import Builder
+
+
+class BuildCommand(Command):
+    """
+    Builds a package, as a tarball and a wheel by default.
+
+    build
+        { --f|format=* : Limit the format to either wheel or sdist}
+    """
+
+    def handle(self):
+        fmt = 'all'
+        if self.option('format'):
+            fmt = self.option('format')
+
+        builder = Builder(self.poetry)
+        builder.build(fmt)

poetry/layouts/layout.py

-# -*- coding: utf-8 -*-
-
-import re
-
 import toml
 
+from poetry.utils.helpers import module_name
 from poetry.vcs.git import Git
 
-_canonicalize_regex = re.compile(r"[-_.]+")
-
 
 TESTS_DEFAULT = """from {package_name} import __version__
 
@@ -21,7 +16,7 @@ class Layout(object):
 
     def __init__(self, project, version='0.1.0', readme_format='md', author=None):
         self._project = project
-        self._package_name = self._canonicalize_name(project).replace('-', '_')
+        self._package_name = module_name(project)
         self._version = version
         self._readme_format = readme_format
         self._dependencies = {}
@@ -118,6 +113,3 @@ class Layout(object):
 
         with poetry.open('w') as f:
             f.write(content)
-
-    def _canonicalize_name(self, name: str) -> str:
-        return _canonicalize_regex.sub("-", name).lower()

poetry/masonry/__init__.py

+"""
+This module handles the packaging and publishing
+of python projects.
+
+A lot of the code used here has been taken from
+`flit <https://github.com/takluyver/flit>`__ and adapted
+to work with the poetry codebase, so kudos to them for showing the way.
+"""
+
+from .builder import Builder

poetry/masonry/builder.py

+from .builders import CompleteBuilder
+from .builders import SdistBuilder
+from .builders import WheelBuilder
+
+
+class Builder:
+
+    _FORMATS = {
+        'sdist': SdistBuilder,
+        'wheel': WheelBuilder,
+        'all': CompleteBuilder
+    }
+
+    def __init__(self, poetry):
+        self._poetry = poetry
+
+    def build(self, fmt: str):
+        if fmt not in self._FORMATS:
+            raise ValueError(f'Invalid format: {fmt}')
+
+        builder = self._FORMATS[fmt](self._poetry)
+
+        return builder.build()

poetry/masonry/builders/__init__.py

+from .complete import CompleteBuilder
+from .sdist import SdistBuilder
+from .wheel import WheelBuilder

poetry/masonry/builders/complete.py

+from .sdist import SdistBuilder
+
+
+class CompleteBuilder:
+
+    def __init__(self, poetry):
+        self._poetry = poetry
+
+    def build(self):
+        # We start by building the tarball
+        # We will use it to build the wheel
+        sdist_builder = SdistBuilder(self._poetry)
+        sdist_file = sdist_builder.build()

poetry/masonry/builders/sdist.py

+import os
+import re
+import tarfile
+
+from collections import defaultdict
+from copy import copy
+from gzip import GzipFile
+from io import BytesIO
+from pathlib import Path
+from posixpath import join as pjoin
+from pprint import pformat
+from typing import List
+
+from poetry.packages import Dependency
+from poetry.semver.constraints import MultiConstraint
+from poetry.vcs import get_vcs
+
+from ..utils.helpers import normalize_file_permissions
+from ..utils.module import Module
+
+
+SETUP = """\
+from setuptools import setup
+
+{before}
+setup(
+    name={name!r},
+    description={description!r},
+    author={author!r},
+    author_email={author_email!r},
+    url={url!r},
+    {extra}
+)
+"""
+
+
+PKG_INFO = """\
+Metadata-Version: 1.1
+Name: {name}
+Version: {version}
+Summary: {summary}
+Home-page: {home_page}
+Author: {author}
+Author-email: {author_email}
+"""
+
+
+AUTHOR_REGEX = re.compile('(?u)^(?P<name>[- .,\w\d\'’"()]+) <(?P<email>.+?)>$')
+
+
+class SdistBuilder:
+
+    def __init__(self, poetry):
+        self._poetry = poetry
+        self._package = self._poetry.package
+        self._path = poetry.file.parent
+        self._module = Module(self._package.name, self._path.as_posix())
+
+    def build(self, target_dir: Path = None) -> Path:
+        if target_dir is None:
+            target_dir = self._path / 'dist'
+
+        if not target_dir.exists():
+            target_dir.mkdir(parents=True)
+
+        target = target_dir / f'{self._package.name}' \
+                              f'-{self._package.version}.tar.gz'
+        gz = GzipFile(target.as_posix(), mode='wb')
+        tar = tarfile.TarFile(target.as_posix(), mode='w', fileobj=gz,
+                              format=tarfile.PAX_FORMAT)
+
+        try:
+            tar_dir = f'{self._package.name}-{self._package.version}'
+
+            files_to_add = self.find_files_to_add()
+
+            for relpath in files_to_add:
+                path = self._path / relpath
+                tar_info = tar.gettarinfo(
+                    str(path),
+                    arcname=pjoin(tar_dir, relpath)
+                )
+                tar_info = self.clean_tarinfo(tar_info)
+
+                if tar_info.isreg():
+                    with path.open('rb') as f:
+                        tar.addfile(tar_info, f)
+                else:
+                    tar.addfile(tar_info)  # Symlinks & ?
+
+            setup = self.build_setup()
+            tar_info = tarfile.TarInfo(pjoin(tar_dir, 'setup.py'))
+            tar_info.size = len(setup)
+            tar.addfile(tar_info, BytesIO(setup))
+
+            author = self.convert_author(self._package.authors[0])
+            pkg_info = PKG_INFO.format(
+                name=self._package.name,
+                version=self._package.version,
+                summary=self._package.description,
+                home_page=self._package.homepage or self._package.repository_url,
+                author=author['name'],
+                author_email=author['email'],
+            ).encode('utf-8')
+
+            tar_info = tarfile.TarInfo(pjoin(tar_dir, 'PKG-INFO'))
+            tar_info.size = len(pkg_info)
+            tar.addfile(tar_info, BytesIO(pkg_info))
+        finally:
+            tar.close()
+            gz.close()
+
+        return target
+
+    def find_excluded_files(self) -> list:
+        # Checking VCS
+        vcs = get_vcs(self._path)
+        if not vcs:
+            return []
+
+        ignored = vcs.get_ignored_files()
+        result = []
+        for file in ignored:
+            try:
+                file = Path(file).absolute().relative_to(self._path)
+            except ValueError:
+                # Should only happen in tests
+                continue
+
+            result.append(file)
+
+        return result
+
+    def find_files_to_add(self) -> list:
+        """
+        Finds all files to add to the tarball
+
+        TODO: Support explicit include/exclude
+        """
+        excluded = self.find_excluded_files()
+        src = self._module.path
+        to_add = []
+
+        for root, dirs, files in os.walk(src.as_posix()):
+            root = Path(root)
+            if root.name == '__pycache__':
+                continue
+
+            for file in files:
+                file = root / file
+                file = file.relative_to(self._path)
+
+                if file in excluded:
+                    continue
+
+                if file.suffix == '.pyc':
+                    continue
+
+                to_add.append(file)
+
+        return to_add
+
+    def build_setup(self) -> bytes:
+        before, extra = [], []
+
+        if self._module.is_package():
+            packages, package_data = self.find_packages(
+                self._module.path.as_posix()
+            )
+            before.append("packages = \\\n{}\n".format(pformat(sorted(packages))))
+            before.append("package_data = \\\n{}\n".format(pformat(package_data)))
+            extra.append("packages=packages,")
+            extra.append("package_data=package_data,")
+        else:
+            extra.append('py_modules={!r},'.format(self._module.name))
+
+        dependencies, extras = self.convert_dependencies(self._package.requires)
+        if dependencies:
+            before.append("install_requires = \\\n{}\n".format(pformat(dependencies)))
+            extra.append("install_requires=install_requires,")
+
+        if extras:
+            before.append("extras_require = \\\n{}\n".format(pformat(extras)))
+            extra.append("extras_require=extras_require,")
+
+        entry_points = self.convert_entry_points()
+        if entry_points:
+            before.append("entry_points = \\\n{}\n".format(pformat(entry_points)))
+            extra.append("entry_points=entry_points,")
+
+        if self._package.python_versions != '*':
+            constraint = self._package.python_constraint
+            if isinstance(constraint, MultiConstraint):
+                python_requires = ','.join(
+                    [str(c).replace(' ', '') for c in constraint.constraints]
+                )
+            else:
+                python_requires = str(constraint).replace(' ', '')
+
+            extra.append('python_requires={!r},'.format(python_requires))
+
+        author = self.convert_author(self._package.authors[0])
+
+        return SETUP.format(
+            before='\n'.join(before),
+            name=self._package.name,
+            version=self._package.version,
+            description=self._package.description,
+            author=author['name'],
+            author_email=author['email'],
+            url=self._package.homepage or self._package.repository_url,
+            extra='\n    '.join(extra),
+        ).encode('utf-8')
+
+    @classmethod
+    def find_packages(cls, path: str):
+        """
+        Discover subpackages and data.
+
+        It also retrieve necessary files
+        """
+        pkgdir = os.path.normpath(path)
+        pkg_name = os.path.basename(pkgdir)
+        pkg_data = defaultdict(list)
+        # Undocumented distutils feature:
+        # the empty string matches all package names
+        pkg_data[''].append('*')
+        packages = [pkg_name]
+        subpkg_paths = set()
+
+        def find_nearest_pkg(rel_path):
+            parts = rel_path.split(os.sep)
+            for i in reversed(range(1, len(parts))):
+                ancestor = '/'.join(parts[:i])
+                if ancestor in subpkg_paths:
+                    pkg = '.'.join([pkg_name] + parts[:i])
+                    return pkg, '/'.join(parts[i:])
+
+            # Relative to the top-level package
+            return pkg_name, rel_path
+
+        for path, dirnames, filenames in os.walk(pkgdir, topdown=True):
+            if os.path.basename(path) == '__pycache__':
+                continue
+
+            from_top_level = os.path.relpath(path, pkgdir)
+            if from_top_level == '.':
+                continue
+
+            is_subpkg = '__init__.py' in filenames
+            if is_subpkg:
+                subpkg_paths.add(from_top_level)
+                parts = from_top_level.split(os.sep)
+                packages.append('.'.join([pkg_name] + parts))
+            else:
+                pkg, from_nearest_pkg = find_nearest_pkg(from_top_level)
+                pkg_data[pkg].append(pjoin(from_nearest_pkg, '*'))
+
+        # Sort values in pkg_data
+        pkg_data = {k: sorted(v) for (k, v) in pkg_data.items()}
+
+        return sorted(packages), pkg_data
+
+    @classmethod
+    def convert_dependencies(cls,
+                             dependencies: List[Dependency]):
+        main = []
+        extras = []
+
+        for dependency in dependencies:
+            is_extra = False
+            requirement = dependency.pretty_name
+            constraint = dependency.constraint
+            if isinstance(constraint, MultiConstraint):
+                requirement += ','.join(
+                    [str(c).replace(' ', '') for c in constraint.constraints]
+                )
+            else:
+                requirement += str(constraint).replace(' ', '')
+            if str(dependency.python_constraint) != '*':
+                is_extra = True
+                python_constraint = dependency.python_constraint
+                requirement += '; python_version'
+                if isinstance(python_constraint, MultiConstraint):
+                    requirement += ','.join(
+                        [str(c).replace(' ', '') for c in python_constraint.constraints]
+                    )
+                else:
+                    requirement += str(python_constraint).replace(' ', '')
+
+            if is_extra:
+                extras.append(requirement)
+            else:
+                main.append(requirement)
+
+        return main, extras
+
+    def convert_entry_points(self) -> dict:
+        result = defaultdict(list)
+
+        # Scripts -> Entry points
+        for name, ep in self._poetry.config.get('scripts', {}).items():
+            result['console_scripts'].append(f'{name} = {ep}')
+
+        # Plugins -> entry points
+        for groupname, group in self._poetry.config.get('plugins', {}).items():
+            for name, ep in sorted(group.items()):
+                result[groupname].append(f'{name} = {ep}')
+
+        return dict(result)
+
+    @classmethod
+    def convert_author(cls, author) -> dict:
+        m = AUTHOR_REGEX.match(author)
+
+        name = m.group('name')
+        email = m.group('email')
+
+        return {
+            'name': name,
+            'email': email
+        }
+
+    @classmethod
+    def clean_tarinfo(cls, tar_info):
+        """
+        Clean metadata from a TarInfo object to make it more reproducible.
+
+            - Set uid & gid to 0
+            - Set uname and gname to ""
+            - Normalise permissions to 644 or 755
+            - Set mtime if not None
+        """
+        ti = copy(tar_info)
+        ti.uid = 0
+        ti.gid = 0
+        ti.uname = ''
+        ti.gname = ''
+        ti.mode = normalize_file_permissions(ti.mode)
+        
+        return ti

poetry/masonry/builders/wheel.py

+class WheelBuilder:
+
+    def __init__(self):
+        pass

poetry/masonry/utils/helpers.py

+def normalize_file_permissions(st_mode):
+    """
+    Normalizes the permission bits in the st_mode field from stat to 644/755
+
+    Popular VCSs only track whether a file is executable or not. The exact
+    permissions can vary on systems with different umasks. Normalising
+    to 644 (non executable) or 755 (executable) makes builds more reproducible.
+    """
+    # Set 644 permissions, leaving higher bits of st_mode unchanged
+    new_mode = (st_mode | 0o644) & ~0o133
+    if st_mode & 0o100:
+        new_mode |= 0o111  # Executable: 644 -> 755
+
+    return new_mode

poetry/masonry/utils/module.py

+from pathlib import Path
+
+from poetry.utils.helpers import module_name
+
+
+class Module:
+
+    def __init__(self, name, directory='.'):
+        self._name = module_name(name)
+
+        # It must exist either as a .py file or a directory, but not both
+        pkg_dir = Path(directory, self._name)
+        py_file = Path(directory, self._name + '.py')
+        if pkg_dir.is_dir() and py_file.is_file():
+            raise ValueError("Both {} and {} exist".format(pkg_dir, py_file))
+        elif pkg_dir.is_dir():
+            self._path = pkg_dir
+            self._is_package = True
+        elif py_file.is_file():
+            self._path = py_file
+            self._is_package = False
+        else:
+            raise ValueError("No file/folder found for package {}".format(name))
+
+    @property
+    def name(self) -> str:
+        return self._name
+
+    @property
+    def path(self) -> Path:
+        return self._path
+
+    @property
+    def file(self) -> Path:
+        if self._is_package:
+            return self._path / '__init__.py'
+        else:
+            return self._path
+
+    def is_package(self) -> bool:
+        return self._is_package

poetry/packages/package.py

@@ -36,15 +36,6 @@ class Package:
     def __init__(self, name, version, pretty_version=None):
         """
         Creates a new in memory package.
-
-        :param name: The package's name
-        :type name: str
-
-        :param version: The package's version
-        :type version: str
-
-        :param pretty_version: The package's non-normalized version
-        :type pretty_version: str
         """
         self._pretty_name = name
         self._name = name.lower()
@@ -57,6 +48,12 @@ class Package:
         self._stability = parse_stability(version)
         self._dev = self._stability == 'dev'
 
+        self._authors = []
+
+        self.homepage = None
+        self.repository_url = None
+        self.keywords = []
+
         self.source_type = ''
         self.source_reference = ''
         self.source_url = ''
@@ -112,6 +109,10 @@ class Package:
         return '{} {}'.format(self._pretty_version, self.source_reference)
 
     @property
+    def authors(self) -> list:
+        return self._authors
+
+    @property
     def python_versions(self):
         return self._python_versions
 

poetry/poetry.py

@@ -74,6 +74,13 @@ class Poetry:
         version = normalize_version(pretty_version)
         package = Package(name, version, pretty_version)
 
+        for author in local_config['authors']:
+            package.authors.append(author)
+
+        package.description = local_config.get('description', '')
+        package.homepage = local_config.get('homepage')
+        package.repository_url = local_config.get('repository')
+
         if 'platform' in local_config:
             package.platform = local_config['platform']
 

poetry/utils/helpers.py

+import re
+
+_canonicalize_regex = re.compile('[-_.]+')
+
+
+def canonicalize_name(name: str) -> str:
+    return _canonicalize_regex.sub('-', name).lower()
+
+
+def module_name(name: str) -> str:
+    return canonicalize_name(name).replace('-', '_')

poetry/utils/toml_file.py

@@ -30,5 +30,5 @@ class TomlFile:
 
         self._path.write_text(data)
 
-    def exists(self) -> bool:
-        return self._path.exists()
+    def __getattr__(self, item):
+        return getattr(self._path, item)

poetry/vcs/__init__.py

+from pathlib import Path
+
+from .git import Git
+
+
+def get_vcs(directory: Path):
+    directory = directory.resolve()
+
+    for p in [directory] + list(directory.parents):
+        if (p / '.git').is_dir():
+            return Git()

poetry/vcs/git.py

@@ -50,6 +50,13 @@ class Git:
             'rev-parse', rev
         )
 
+    def get_ignored_files(self) -> list:
+        output = self.run(
+            'ls-files', '--others', '-i', '--exclude-standard'
+        )
+
+        return output.split('\n')
+
     def run(self, *args) -> str:
         return subprocess.check_output(
             ['git'] + list(args),

pyproject.toml

 [tool.poetry]
 name = "poetry"
-version = "0.3.0"
+version = "0.4.0-beta"
 description = "Python dependency management and packaging made easy."
 authors = [
     "Sébastien Eustace <sebastien@eustace.io>"

tests/masonry/builders/fixtures/complete/README.rst

+My Package
+==========

tests/masonry/builders/fixtures/complete/my_package/__init__.py

+__version__ = '1.2.3'

tests/masonry/builders/fixtures/complete/pyproject.toml

+[tool.poetry]
+name = "my-package"
+version = "1.2.3"
+description = "Some description."
+authors = [
+    "Sébastien Eustace <sebastien@eustace.io>"
+]
+license = "MIT"
+
+readme = "README.rst"
+
+homepage = "https://poetry.eustace.io/"
+repository = "https://github.com/sdispater/poetry"
+documentation = "https://poetry.eustace.io/docs"
+
+keywords = ["packaging", "dependency", "poetry"]
+
+
+# Requirements
+[tool.poetry.dependencies]
+python = "^3.6"
+cleo = "^0.6"
+
+[tool.poetry.dev-dependencies]
+pytest = "~3.4"
+
+
+[tool.poetry.scripts]
+my-script = "my_package:main"

tests/masonry/builders/test_sdist.py

+import ast
+
+from pathlib import Path
+
+from poetry import Poetry
+from poetry.masonry.builders.sdist import SdistBuilder
+
+from tests.helpers import get_dependency
+
+
+def project(name):
+    return Path(__file__).parent / 'fixtures' / name
+
+
+def test_convert_dependencies():
+    result = SdistBuilder.convert_dependencies([
+        get_dependency('A', '^1.0'),
+        get_dependency('B', '~1.0'),
+        get_dependency('C', '1.2.3'),
+    ])
+    main = [
+        'A>=1.0.0.0,<2.0.0.0',
+        'B>=1.0.0.0,<1.1.0.0',
+        'C==1.2.3.0',
+    ]
+    extras = []
+
+    assert result == (main, extras)
+
+    dependency_with_python = get_dependency('A', '^1.0')
+    dependency_with_python.python_versions = '^3.4'
+
+    result = SdistBuilder.convert_dependencies([
+        dependency_with_python,
+        get_dependency('B', '~1.0'),
+        get_dependency('C', '1.2.3'),
+    ])
+    main = [
+        'B>=1.0.0.0,<1.1.0.0',
+        'C==1.2.3.0',
+    ]
+    extras = [
+        'A>=1.0.0.0,<2.0.0.0; python_version>=3.4.0.0,<4.0.0.0',
+    ]
+
+    assert result == (main, extras)
+
+
+def test_make_setup():
+    poetry = Poetry.create(project('complete'))
+
+    builder = SdistBuilder(poetry)
+    setup = builder.build_setup()
+    setup_ast = ast.parse(setup)
+
+    setup_ast.body = [n for n in setup_ast.body if isinstance(n, ast.Assign)]
+    ns = {}
+    exec(compile(setup_ast, filename="setup.py", mode="exec"), ns)
+    assert ns['packages'] == [
+        'my_package',
+        'my_package.sub_pkg1',
+        'my_package.sub_pkg2'
+    ]
+    assert ns['install_requires'] == [
+        'cleo>=0.6.0.0,<0.7.0.0'
+    ]
+    assert ns['entry_points'] == {
+        'console_scripts': ['my-script = my_package:main']
+    }
+
+
+def test_find_files_to_add():
+    poetry = Poetry.create(project('complete'))
+
+    builder = SdistBuilder(poetry)
+    result = builder.find_files_to_add()
+
+    assert result == [
+        Path('my_package/__init__.py'),
+        Path('my_package/sub_pkg1/__init__.py'),
+        Path('my_package/data1/test.json'),
+        Path('my_package/sub_pkg2/__init__.py'),
+        Path('my_package/sub_pkg2/data2/data.json'),
+    ]