Exclude nested items (#784) (#1464)

* This PR impliments the feature request #784.

When a folder is explicit defined in `pyproject.toml` as excluded, all nested data, including subfolder, are excluded. It is no longer neccessary to use the glob `folder/**/*`

* use `Path` instead of `os.path.join` to create string for globbing

* try to fix linting error

* create glob pattern string by concatenating and not using Path

* using `os.path.isdir()`` for checking of explicit excluded name is a folder, because pathlib's `is_dir()` raises in exception under windows of name contains globing characters

* Remove nested data when wildcards where used.

Steps to do this are:
1. expand any wildcard used
2. if expanded path is a folder append  **/* and expand again

* fix linting

* only glob a second time if path is dir

* implement @sdispater 's suggestion for better readability

* fix glob for windows?

* On Windows, testing if a path with a glob is a directory will raise an OSError

* pathlibs  glob function doesn't return the correct case (https://bugs.python.org/issue26655). So switching back to  glob.glob()

* removing obsolete imports

poetry/masonry/builders/builder.py

 # -*- coding: utf-8 -*-
 # -*- coding: utf-8 -*-
-import os
 import re
 import re
 import shutil
 import shutil
 import tempfile
 import tempfile
-
 from collections import defaultdict
 from collections import defaultdict
 from contextlib import contextmanager
 from contextlib import contextmanager
 from typing import Set
 from typing import Set
@@ -17,12 +15,10 @@ from poetry.utils._compat import glob
 from poetry.utils._compat import lru_cache
 from poetry.utils._compat import lru_cache
 from poetry.utils._compat import to_str
 from poetry.utils._compat import to_str
 from poetry.vcs import get_vcs
 from poetry.vcs import get_vcs
-
 from ..metadata import Metadata
 from ..metadata import Metadata
 from ..utils.module import Module
 from ..utils.module import Module
 from ..utils.package_include import PackageInclude
 from ..utils.package_include import PackageInclude
 
 
-
 AUTHOR_REGEX = re.compile(r"(?u)^(?P<name>[- .,\w\d'’\"()]+) <(?P<email>.+?)>$")
 AUTHOR_REGEX = re.compile(r"(?u)^(?P<name>[- .,\w\d'’\"()]+) <(?P<email>.+?)>$")
 
 
 METADATA_BASE = """\
 METADATA_BASE = """\
@@ -34,7 +30,6 @@ Summary: {summary}
 
 
 
 
 class Builder(object):
 class Builder(object):
-
     AVAILABLE_PYTHONS = {"2", "2.7", "3", "3.4", "3.5", "3.6", "3.7"}
     AVAILABLE_PYTHONS = {"2", "2.7", "3", "3.4", "3.5", "3.6", "3.7"}
 
 
     format = None
     format = None
@@ -86,8 +81,18 @@ class Builder(object):
 
 
         explicitely_excluded = set()
         explicitely_excluded = set()
         for excluded_glob in self._package.exclude:
         for excluded_glob in self._package.exclude:
+            excluded_path = Path(self._path, excluded_glob)
+
+            try:
+                is_dir = excluded_path.is_dir()
+            except OSError:
+                # On Windows, testing if a path with a glob is a directory will raise an OSError
+                is_dir = False
+            if is_dir:
+                excluded_glob = Path(excluded_glob, "**/*")
+
             for excluded in glob(
             for excluded in glob(
-                os.path.join(self._path.as_posix(), str(excluded_glob)), recursive=True
+                Path(self._path, excluded_glob).as_posix(), recursive=True
             ):
             ):
                 explicitely_excluded.add(
                 explicitely_excluded.add(
                     Path(excluded).relative_to(self._path).as_posix()
                     Path(excluded).relative_to(self._path).as_posix()

tests/masonry/builders/fixtures/exclude_nested_data_toml/LICENSE

+Copyright (c) 2018 Sébastien Eustace
+
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+"Software"), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

tests/masonry/builders/fixtures/exclude_nested_data_toml/README.rst

+My Package
+==========

tests/masonry/builders/fixtures/exclude_nested_data_toml/pyproject.toml

+[tool.poetry]
+name = "my-package"
+version = "1.2.3"
+description = "Some description."
+authors = [
+    "Sébastien Eustace <sebastien@eustace.io>"
+]
+license = "MIT"
+
+readme = "README.rst"
+
+exclude = ["my_package/data/", "**/*/item*"]
+
+homepage = "https://poetry.eustace.io/"
+repository = "https://github.com/sdispater/poetry"
+documentation = "https://poetry.eustace.io/docs"
+
+keywords = ["packaging", "dependency", "poetry"]
+
+classifiers = [
+    "Topic :: Software Development :: Build Tools",
+    "Topic :: Software Development :: Libraries :: Python Modules"
+]
+
+# Requirements
+[tool.poetry.dependencies]
+python = "^3.6"
+cleo = "^0.6"
+cachy = { version = "^0.2.0", extras = ["msgpack"] }
+
+pendulum = { version = "^1.4", optional = true }
+
+[tool.poetry.dev-dependencies]
+pytest = "~3.4"
+
+[tool.poetry.extras]
+time = ["pendulum"]
+
+[tool.poetry.scripts]
+my-script = "my_package:main"
+my-2nd-script = "my_package:main2"

tests/masonry/builders/test_sdist.py

@@ -410,6 +410,38 @@ def test_default_with_excluded_data(mocker):
         assert "my-package-1.2.3/PKG-INFO" in names
         assert "my-package-1.2.3/PKG-INFO" in names
 
 
 
 
+def test_src_excluded_nested_data():
+    module_path = fixtures_dir / "exclude_nested_data_toml"
+    poetry = Factory().create_poetry(module_path)
+
+    builder = SdistBuilder(poetry, NullEnv(), NullIO())
+    builder.build()
+
+    sdist = module_path / "dist" / "my-package-1.2.3.tar.gz"
+
+    assert sdist.exists()
+
+    with tarfile.open(str(sdist), "r") as tar:
+        names = tar.getnames()
+        assert len(names) == len(set(names))
+        assert "my-package-1.2.3/LICENSE" in names
+        assert "my-package-1.2.3/README.rst" in names
+        assert "my-package-1.2.3/pyproject.toml" in names
+        assert "my-package-1.2.3/setup.py" in names
+        assert "my-package-1.2.3/PKG-INFO" in names
+        assert "my-package-1.2.3/my_package/__init__.py" in names
+        assert "my-package-1.2.3/my_package/data/sub_data/data2.txt" not in names
+        assert "my-package-1.2.3/my_package/data/sub_data/data3.txt" not in names
+        assert "my-package-1.2.3/my_package/data/data1.txt" not in names
+        assert "my-package-1.2.3/my_package/puplic/publicdata.txt" in names
+        assert "my-package-1.2.3/my_package/public/item1/itemdata1.txt" not in names
+        assert (
+            "my-package-1.2.3/my_package/public/item1/subitem/subitemdata.txt"
+            not in names
+        )
+        assert "my-package-1.2.3/my_package/public/item2/itemdata2.txt" not in names
+
+
 def test_proper_python_requires_if_two_digits_precision_version_specified():
 def test_proper_python_requires_if_two_digits_precision_version_specified():
     poetry = Factory().create_poetry(project("simple_version"))
     poetry = Factory().create_poetry(project("simple_version"))
 
 

tests/masonry/builders/test_wheel.py

@@ -77,6 +77,26 @@ def test_wheel_excluded_data():
         assert "my_package/data/data1.txt" not in z.namelist()
         assert "my_package/data/data1.txt" not in z.namelist()
 
 
 
 
+def test_wheel_excluded_nested_data():
+    module_path = fixtures_dir / "exclude_nested_data_toml"
+    poetry = Factory().create_poetry(module_path)
+    WheelBuilder.make(poetry, NullEnv(), NullIO())
+
+    whl = module_path / "dist" / "my_package-1.2.3-py3-none-any.whl"
+
+    assert whl.exists()
+
+    with zipfile.ZipFile(str(whl)) as z:
+        assert "my_package/__init__.py" in z.namelist()
+        assert "my_package/data/sub_data/data2.txt" not in z.namelist()
+        assert "my_package/data/sub_data/data3.txt" not in z.namelist()
+        assert "my_package/data/data1.txt" not in z.namelist()
+        assert "my_package/puplic/publicdata.txt" in z.namelist()
+        assert "my_package/public/item1/itemdata1.txt" not in z.namelist()
+        assert "my_package/public/item1/subitem/subitemdata.txt" not in z.namelist()
+        assert "my_package/public/item2/itemdata2.txt" not in z.namelist()
+
+
 def test_wheel_localversionlabel():
 def test_wheel_localversionlabel():
     module_path = fixtures_dir / "localversionlabel"
     module_path = fixtures_dir / "localversionlabel"
     WheelBuilder.make(Factory().create_poetry(module_path), NullEnv(), NullIO())
     WheelBuilder.make(Factory().create_poetry(module_path), NullEnv(), NullIO())