Commit 41fce882 by Sébastien Eustace

Add support for packages, include and exclude

parent b963b8fb
......@@ -2,6 +2,10 @@
## [Unreleased]
### Added
- Added support for `packages`, `include` and `exclude` properties.
### Fixed
- Fixed handling of duplicate dependencies with different constraints.
......
......@@ -68,6 +68,53 @@ An URL to the documentation of the project. **Optional**
A list of keywords (max: 5) that the package is related to. **Optional**
## packages
A list of packages and modules to include in the final distribution.
If your project structure differs from the standard one supported by `poetry`,
you can specify the packages you want to include in the final distribution.
```toml
[tool.poetry]
# ...
packages = [
{ include = "mypackage" },
{ include = "extra_package/**/*.py" },
]
```
If your package is stored inside a "source" directory, you must specify it:
```toml
[tool.poetry]
# ...
packages = [
{ include = "mypackage", from = "lib" },
]
```
!!!note
Using `packages` disables the package auto-detection feature meaning you have to
**explicitly** specify the "default" package.
For instance, if you have a package named `my_package` and you want to also include
another package named `extra_package`, you will need to specify `my_package` explicitely:
```toml
packages = [
{ include = "mypackage" },
{ include = "extra_package" },
]
```
!!!note
Poetry is clever enough to detect Python subpackages.
So, if you only have to specify the directory where you root package resides.
## include and exclude
A list of patterns that will be included in the final package.
......@@ -78,13 +125,13 @@ The globs specified in the exclude field identify a set of files that are not in
If a VCS is being used for a package, the exclude field will be seeded with the VCS’ ignore settings (`.gitignore` for git for example).
```toml
[package]
[tool.poetry]
# ...
include = ["package/**/*.py", "package/**/.c"]
include = ["CHANGELOG.md"]
```
```toml
exclude = ["package/excluded.py"]
exclude = ["my_package/excluded.py"]
```
## `dependencies` and `dev-dependencies`
......@@ -109,7 +156,7 @@ url = 'http://example.com/simple'
Be aware that declaring the python version for which your package
is compatible is mandatory:
```toml
[tool.poetry.dependencies]
python = "^3.6"
......
......@@ -3,7 +3,11 @@
"name": "Package",
"type": "object",
"additionalProperties": false,
"required": [ "name", "version", "description" ],
"required": [
"name",
"version",
"description"
],
"properties": {
"name": {
"type": "string",
......@@ -50,14 +54,46 @@
"type": "string",
"description": "The path to the README file"
},
"classifiers": {
"classifiers": {
"type": "array",
"description": "A list of trove classifers."
},
"packages": {
"type": "array",
"description": "A list of packages to include in the final distribution.",
"items": {
"type": "object",
"description": "Information about where the package resides.",
"additionalProperties": false,
"required": [
"include"
],
"properties": {
"include": {
"type": "string",
"description": "What to include in the package."
},
"from": {
"type": "string",
"description": "Where the source directory of the package resides."
}
}
}
},
"include": {
"type": "array",
"description": "A list of files and folders to include."
},
"exclude": {
"type": "array",
"description": "A list of files and folders to exclude."
},
"dependencies": {
"type": "object",
"description": "This is a hash of package name (keys) and version constraints (values) that are required to run this package.",
"required": ["python"],
"required": [
"python"
],
"properties": {
"python": {
"type": "string",
......@@ -171,7 +207,9 @@
},
"long-dependency": {
"type": "object",
"required": ["version"],
"required": [
"version"
],
"additionalProperties": false,
"properties": {
"version": {
......@@ -205,7 +243,9 @@
},
"git-dependency": {
"type": "object",
"required": ["git"],
"required": [
"git"
],
"additionalProperties": false,
"properties": {
"git": {
......@@ -252,7 +292,9 @@
},
"file-dependency": {
"type": "object",
"required": ["file"],
"required": [
"file"
],
"additionalProperties": false,
"properties": {
"file": {
......@@ -282,7 +324,9 @@
},
"path-dependency": {
"type": "object",
"required": ["path"],
"required": [
"path"
],
"additionalProperties": false,
"properties": {
"path": {
......
......@@ -27,7 +27,12 @@ class Builder(object):
self._io = io
self._package = poetry.package
self._path = poetry.file.parent
self._module = Module(self._package.name, self._path.as_posix())
self._module = Module(
self._package.name,
self._path.as_posix(),
packages=self._package.packages,
includes=self._package.include,
)
self._meta = Metadata.from_package(self._package)
def build(self):
......@@ -39,7 +44,12 @@ class Builder(object):
if not vcs:
return []
ignored = vcs.get_ignored_files()
explicitely_excluded = []
for excluded_glob in self._package.exclude:
for excluded in self._path.glob(excluded_glob):
explicitely_excluded.append(excluded)
ignored = vcs.get_ignored_files() + explicitely_excluded
result = []
for file in ignored:
try:
......@@ -55,39 +65,32 @@ class Builder(object):
def find_files_to_add(self, exclude_build=True): # type: () -> list
"""
Finds all files to add to the tarball
TODO: Support explicit include/exclude
"""
excluded = self.find_excluded_files()
src = self._module.path
to_add = []
if not self._module.is_package():
if self._module.is_in_src():
to_add.append(src.relative_to(src.parent.parent))
else:
to_add.append(src.relative_to(src.parent))
else:
for root, dirs, files in os.walk(src.as_posix()):
root = Path(root)
if root.name == "__pycache__":
for include in self._module.includes:
for file in include.elements:
if "__pycache__" in str(file):
continue
if file.is_dir():
continue
for file in files:
file = root / file
file = file.relative_to(self._path)
file = file.relative_to(self._path)
if file in excluded:
continue
if file in excluded:
continue
if file.suffix == ".pyc":
continue
if file.suffix == ".pyc":
continue
self._io.writeln(
" - Adding: <comment>{}</comment>".format(str(file)),
verbosity=self._io.VERBOSITY_VERY_VERBOSE,
)
to_add.append(file)
self._io.writeln(
" - Adding: <comment>{}</comment>".format(str(file)),
verbosity=self._io.VERBOSITY_VERY_VERBOSE,
)
to_add.append(file)
# Include project files
self._io.writeln(
......
......@@ -17,6 +17,7 @@ from poetry.utils._compat import encode
from poetry.utils._compat import to_str
from ..utils.helpers import normalize_file_permissions
from ..utils.package_include import PackageInclude
from .builder import Builder
......@@ -108,6 +109,7 @@ class SdistBuilder(Builder):
def build_setup(self): # type: () -> bytes
before, extra, after = [], [], []
package_dir = {}
# If we have a build script, use it
if self._package.build:
......@@ -116,18 +118,42 @@ class SdistBuilder(Builder):
"build(setup_kwargs)",
]
if self._module.is_in_src():
before.append("package_dir = \\\n{}\n".format(pformat({"": "src"})))
modules = []
packages = []
package_data = {}
for include in self._module.includes:
if isinstance(include, PackageInclude):
if include.is_package():
pkg_dir, _packages, _package_data = self.find_packages(include)
if pkg_dir is not None:
package_dir[""] = str(pkg_dir.relative_to(self._path))
packages += _packages
package_data.update(_package_data)
else:
if include.source is not None:
package_dir[""] = str(include.base.relative_to(self._path))
modules.append(include.elements[0].relative_to(include.base).stem)
else:
pass
if package_dir:
before.append("package_dir = \\\n{}\n".format(pformat(package_dir)))
extra.append("'package_dir': package_dir,")
if self._module.is_package():
packages, package_data = self.find_packages(self._module.path.as_posix())
if packages:
before.append("packages = \\\n{}\n".format(pformat(sorted(packages))))
before.append("package_data = \\\n{}\n".format(pformat(package_data)))
extra.append("'packages': packages,")
if package_data:
before.append("package_data = \\\n{}\n".format(pformat(package_data)))
extra.append("'package_data': package_data,")
else:
extra.append("'py_modules': {!r},".format(to_str(self._module.name)))
if modules:
before.append("modules = \\\n{}".format(pformat(modules)))
extra.append("'py_modules': modules,".format())
dependencies, extras = self.convert_dependencies(
self._package, self._package.requires
......@@ -195,14 +221,19 @@ class SdistBuilder(Builder):
return encode(pkg_info)
@classmethod
def find_packages(cls, path):
def find_packages(cls, include):
"""
Discover subpackages and data.
It also retrieve necessary files
It also retrieves necessary files.
"""
pkgdir = os.path.normpath(path)
pkg_name = os.path.basename(pkgdir)
pkgdir = None
if include.source is not None:
pkgdir = include.base
base = include.elements[0].parent
pkg_name = include.package
pkg_data = defaultdict(list)
# Undocumented distutils feature:
# the empty string matches all package names
......@@ -221,11 +252,11 @@ class SdistBuilder(Builder):
# Relative to the top-level package
return pkg_name, rel_path
for path, dirnames, filenames in os.walk(pkgdir, topdown=True):
for path, dirnames, filenames in os.walk(str(base), topdown=True):
if os.path.basename(path) == "__pycache__":
continue
from_top_level = os.path.relpath(path, pkgdir)
from_top_level = os.path.relpath(path, base)
if from_top_level == ".":
continue
......@@ -241,7 +272,7 @@ class SdistBuilder(Builder):
# Sort values in pkg_data
pkg_data = {k: sorted(v) for (k, v) in pkg_data.items()}
return sorted(packages), pkg_data
return pkgdir, sorted(packages), pkg_data
@classmethod
def convert_dependencies(
......
......@@ -17,6 +17,7 @@ from poetry.semver import parse_constraint
from poetry.utils._compat import Path
from ..utils.helpers import normalize_file_permissions
from ..utils.package_include import PackageInclude
from ..utils.tags import get_abbr_impl
from ..utils.tags import get_abi_tag
from ..utils.tags import get_impl_ver
......@@ -120,29 +121,41 @@ class WheelBuilder(Builder):
shutil.copytree(str(pkg), str(self._path / pkg.name))
def copy_module(self):
if self._module.is_package():
files = self.find_files_to_add()
# Walk the files and compress them,
# sorting everything so the order is stable.
for file in sorted(files):
full_path = self._path / file
if self._module.is_in_src():
try:
file = file.relative_to(
self._module.path.parent.relative_to(self._path)
)
except ValueError:
pass
# Do not include topmost files
if full_path.relative_to(self._path) == Path(file.name):
excluded = self.find_excluded_files()
src = self._module.path
to_add = []
for include in self._module.includes:
include.refresh()
for file in include.elements:
if "__pycache__" in str(file):
continue
self._add_file(full_path, file)
else:
self._add_file(str(self._module.path), self._module.path.name)
if file.is_dir():
continue
if isinstance(include, PackageInclude) and include.source:
rel_file = file.relative_to(include.base)
else:
rel_file = file.relative_to(self._path)
if file in excluded:
continue
if file.suffix == ".pyc":
continue
self._io.writeln(
" - Adding: <comment>{}</comment>".format(str(file)),
verbosity=self._io.VERBOSITY_VERY_VERBOSE,
)
to_add.append((file, rel_file))
# Walk the files and compress them,
# sorting everything so the order is stable.
for full_path, rel_path in sorted(to_add, key=lambda x: x[1]):
self._add_file(full_path, rel_path)
def write_metadata(self):
if (
......
from typing import List
from poetry.utils._compat import Path
class Include(object):
"""
Represents an "include" entry.
It can be a glob string, a single file or a directory.
This class will then detect the type of this include:
- a package
- a module
- a file
- a directory
"""
def __init__(self, base, include): # type: (Path, str) -> None
self._base = base
self._include = include
self._elements = sorted(list(self._base.glob(self._include)))
@property
def base(self): # type: () -> Path
return self._base
@property
def elements(self): # type: () -> List[Path]
return self._elements
def is_empty(self): # type: () -> bool
return len(self._elements) == 0
def refresh(self): # type: () -> Include
self._elements = sorted(list(self._base.glob(self._include)))
return self
from poetry.utils._compat import Path
from poetry.utils.helpers import module_name
from .include import Include
from .package_include import PackageInclude
class Module:
def __init__(self, name, directory="."):
def __init__(self, name, directory=".", packages=None, includes=None):
self._name = module_name(name)
self._in_src = False
self._is_package = False
self._path = Path(directory)
self._includes = []
packages = packages or []
includes = includes or []
# It must exist either as a .py file or a directory, but not both
pkg_dir = Path(directory, self._name)
py_file = Path(directory, self._name + ".py")
if pkg_dir.is_dir() and py_file.is_file():
raise ValueError("Both {} and {} exist".format(pkg_dir, py_file))
elif pkg_dir.is_dir():
self._path = pkg_dir
self._is_package = True
elif py_file.is_file():
self._path = py_file
self._is_package = False
else:
# Searching for a src module
src_pkg_dir = Path(directory, "src", self._name)
src_py_file = Path(directory, "src", self._name + ".py")
if src_pkg_dir.is_dir() and src_py_file.is_file():
if not packages:
# It must exist either as a .py file or a directory, but not both
pkg_dir = Path(directory, self._name)
py_file = Path(directory, self._name + ".py")
if pkg_dir.is_dir() and py_file.is_file():
raise ValueError("Both {} and {} exist".format(pkg_dir, py_file))
elif src_pkg_dir.is_dir():
self._in_src = True
self._path = src_pkg_dir
self._is_package = True
elif src_py_file.is_file():
self._in_src = True
self._path = src_py_file
self._is_package = False
elif pkg_dir.is_dir():
packages = [{"include": str(pkg_dir.relative_to(self._path))}]
elif py_file.is_file():
packages = [{"include": str(py_file.relative_to(self._path))}]
else:
raise ValueError("No file/folder found for package {}".format(name))
# Searching for a src module
src = Path(directory, "src")
src_pkg_dir = src / self._name
src_py_file = src / (self._name + ".py")
if src_pkg_dir.is_dir() and src_py_file.is_file():
raise ValueError("Both {} and {} exist".format(pkg_dir, py_file))
elif src_pkg_dir.is_dir():
packages = [
{
"include": str(src_pkg_dir.relative_to(src)),
"from": str(src.relative_to(self._path)),
}
]
elif src_py_file.is_file():
packages = [
{
"include": str(src_py_file.relative_to(src)),
"from": str(src.relative_to(self._path)),
}
]
else:
raise ValueError("No file/folder found for package {}".format(name))
for package in packages:
self._includes.append(
PackageInclude(self._path, package["include"], package.get("from"))
)
for include in includes:
self._includes.append(Include(self._path, include))
@property
def name(self): # type: () -> str
......@@ -51,6 +73,10 @@ class Module:
else:
return self._path
@property
def includes(self): # type: () -> List
return self._includes
def is_package(self): # type: () -> bool
return self._is_package
......
from .include import Include
class PackageInclude(Include):
def __init__(self, base, include, source=None):
self._package = None
self._is_package = False
self._is_module = False
self._source = source
if source is not None:
base = base / source
super(PackageInclude, self).__init__(base, include)
self.check_elements()
@property
def package(self): # type: () -> str
return self._package
@property
def source(self): # type: () -> str
return self._source
def is_package(self): # type: () -> bool
return self._is_package
def is_module(self): # type: ()
return self._is_module
def refresh(self): # type: () -> PackageInclude
super(PackageInclude, self).refresh()
return self.check_elements()
def check_elements(self): # type: () -> PackageInclude
if not self._elements:
raise ValueError("{} does not contain any element".format(base / include))
if len(self._elements) > 1:
# Probably glob
self._is_package = True
# The __init__.py file should be first
root = self._elements[0]
if root.name != "__init__.py":
raise ValueError("{} is not a package.".format(root))
self._package = root.parent.name
else:
if self._elements[0].is_dir():
# If it's a directory, we include everything inside it
self._package = self._elements[0].name
self._elements = sorted(list(self._elements[0].glob("**/*")))
self._is_package = True
else:
self._package = self._elements[0].stem
self._is_module = True
return self
......@@ -26,11 +26,6 @@ class Package(object):
AVAILABLE_PYTHONS = {"2", "2.7", "3", "3.4", "3.5", "3.6", "3.7"}
supported_link_types = {
"require": {"description": "requires", "method": "requires"},
"provide": {"description": "provides", "method": "provides"},
}
def __init__(self, name, version, pretty_version=None):
"""
Creates a new in memory package.
......@@ -71,10 +66,6 @@ class Package(object):
# Requirements for making it mandatory
self.requirements = {}
self.build = None
self.include = []
self.exclude = []
self.classifiers = []
self._python_versions = "*"
......
......@@ -2,6 +2,14 @@ from .package import Package
class ProjectPackage(Package):
def __init__(self, name, version, pretty_version=None):
super(ProjectPackage, self).__init__(name, version, pretty_version)
self.build = None
self.packages = []
self.include = []
self.exclude = []
def is_root(self):
return True
......
......@@ -154,6 +154,9 @@ class Poetry:
if "exclude" in local_config:
package.exclude = local_config["exclude"]
if "packages" in local_config:
package.packages = local_config["packages"]
locker = Locker(poetry_file.with_suffix(".lock"), local_config)
return cls(poetry_file, local_config, package, locker)
......
Copyright (c) 2018 Sébastien Eustace
Permission is hereby granted, free of charge, to any person obtaining
a copy of this software and associated documentation files (the
"Software"), to deal in the Software without restriction, including
without limitation the rights to use, copy, modify, merge, publish,
distribute, sublicense, and/or sell copies of the Software, and to
permit persons to whom the Software is furnished to do so, subject to
the following conditions:
The above copyright notice and this permission notice shall be
included in all copies or substantial portions of the Software.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
[tool.poetry]
name = "with-include"
version = "1.2.3"
description = "Some description."
authors = [
"Sébastien Eustace <sebastien@eustace.io>"
]
license = "MIT"
readme = "README.rst"
homepage = "https://poetry.eustace.io/"
repository = "https://github.com/sdispater/poetry"
documentation = "https://poetry.eustace.io/docs"
keywords = ["packaging", "dependency", "poetry"]
classifiers = [
"Topic :: Software Development :: Build Tools",
"Topic :: Software Development :: Libraries :: Python Modules"
]
packages = [
{ include = "extra_dir/**/*.py" },
{ include = "my_module.py" },
{ include = "package_with_include" },
]
include = [
"notes.txt"
]
# Requirements
[tool.poetry.dependencies]
python = "^3.6"
cleo = "^0.6"
cachy = { version = "^0.2.0", extras = ["msgpack"] }
pendulum = { version = "^1.4", optional = true }
[tool.poetry.dev-dependencies]
pytest = "~3.4"
[tool.poetry.extras]
time = ["pendulum"]
[tool.poetry.scripts]
my-script = "my_package:main"
my-2nd-script = "my_package:main2"
......@@ -6,6 +6,7 @@ import tarfile
from poetry.io import NullIO
from poetry.masonry.builders.sdist import SdistBuilder
from poetry.masonry.utils.package_include import PackageInclude
from poetry.packages import Package
from poetry.poetry import Poetry
from poetry.utils._compat import Path
......@@ -142,6 +143,38 @@ def test_find_files_to_add():
]
def test_find_packages():
poetry = Poetry.create(project("complete"))
builder = SdistBuilder(poetry, NullVenv(), NullIO())
base = project("complete")
include = PackageInclude(base, "my_package")
pkg_dir, packages, pkg_data = builder.find_packages(include)
assert pkg_dir is None
assert packages == ["my_package", "my_package.sub_pkg1", "my_package.sub_pkg2"]
assert pkg_data == {
"": ["*"],
"my_package": ["data1/*"],
"my_package.sub_pkg2": ["data2/*"],
}
poetry = Poetry.create(project("source_package"))
builder = SdistBuilder(poetry, NullVenv(), NullIO())
base = project("source_package")
include = PackageInclude(base, "package_src", "src")
pkg_dir, packages, pkg_data = builder.find_packages(include)
assert pkg_dir == base / "src"
assert packages == ["package_src"]
assert pkg_data == {"": ["*"]}
def test_package():
poetry = Poetry.create(project("complete"))
......@@ -212,7 +245,7 @@ def test_with_src_module_file():
ns = {}
exec(compile(setup_ast, filename="setup.py", mode="exec"), ns)
assert ns["package_dir"] == {"": "src"}
assert re.search("'py_modules': 'module_src'", to_str(setup)) is not None
assert ns["modules"] == ["module_src"]
builder.build()
......@@ -250,3 +283,40 @@ def test_with_src_module_dir():
assert "package-src-0.1/src/package_src/__init__.py" in tar.getnames()
assert "package-src-0.1/src/package_src/module.py" in tar.getnames()
def test_package_with_include():
poetry = Poetry.create(project("with-include"))
builder = SdistBuilder(poetry, NullVenv(), NullIO())
# Check setup.py
setup = builder.build_setup()
setup_ast = ast.parse(setup)
setup_ast.body = [n for n in setup_ast.body if isinstance(n, ast.Assign)]
ns = {}
exec(compile(setup_ast, filename="setup.py", mode="exec"), ns)
assert "package_dir" not in ns
assert ns["packages"] == ["extra_dir", "extra_dir.sub_pkg", "package_with_include"]
assert ns["modules"] == ["my_module"]
builder.build()
sdist = fixtures_dir / "with-include" / "dist" / "with-include-1.2.3.tar.gz"
assert sdist.exists()
tar = tarfile.open(str(sdist), "r")
names = tar.getnames()
assert "with-include-1.2.3/LICENSE" in names
assert "with-include-1.2.3/README.rst" in names
assert "with-include-1.2.3/extra_dir/__init__.py" in names
assert "with-include-1.2.3/extra_dir/sub_pkg/__init__.py" in names
assert "with-include-1.2.3/my_module.py" in names
assert "with-include-1.2.3/notes.txt" in names
assert "with-include-1.2.3/package_with_include/__init__.py" in names
assert "with-include-1.2.3/pyproject.toml" in names
assert "with-include-1.2.3/setup.py" in names
assert "with-include-1.2.3/PKG-INFO" in names
......@@ -102,6 +102,22 @@ def test_poetry():
]
def test_poetry_with_packages_and_includes():
poetry = Poetry.create(
str(fixtures_dir.parent / "masonry" / "builders" / "fixtures" / "with-include")
)
package = poetry.package
assert package.packages == [
{"include": "extra_dir/**/*.py"},
{"include": "my_module.py"},
{"include": "package_with_include"},
]
assert package.include == ["notes.txt"]
def test_check():
complete = fixtures_dir / "complete.toml"
with complete.open() as f:
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment