Fix performance issues when packaging if a lot of files were excluded

CHANGELOG.md

@@ -8,6 +8,7 @@
 - Fixed a `KeyError` when getting information for packages which require reading setup files.
 - Fixed a `KeyError` when getting information for packages which require reading setup files.
 - Fixed the building of wheels with C extensions and an `src` layout.
 - Fixed the building of wheels with C extensions and an `src` layout.
 - Fixed extras being selected when resolving dependencies even when not required.
 - Fixed extras being selected when resolving dependencies even when not required.
+- Fixed performance issues when packaging projects if a lot of files were excluded.
 
 
 
 
 ## [0.12.6] - 2018-11-05
 ## [0.12.6] - 2018-11-05

poetry.lock

@@ -737,7 +737,7 @@ python-versions = "*"
 version = "0.5.1"
 version = "0.5.1"
 
 
 [metadata]
 [metadata]
-content-hash = "7d4198bfede7f32d4992ec73c05701c841eea21306a8d3acfdde6a1db49a34f2"
+content-hash = "bba4def1ba8240833a7d211dc6a7998d44faf3d592a9b7084490f240f7fd05b6"
 python-versions = "~2.7 || ^3.4"
 python-versions = "~2.7 || ^3.4"
 
 
 [metadata.hashes]
 [metadata.hashes]

poetry/masonry/builders/builder.py

 # -*- coding: utf-8 -*-
 # -*- coding: utf-8 -*-
-import os
 import re
 import re
 import shutil
 import shutil
 import tempfile
 import tempfile
 
 
 from collections import defaultdict
 from collections import defaultdict
 from contextlib import contextmanager
 from contextlib import contextmanager
+from typing import Set
+from typing import Union
 
 
 from poetry.utils._compat import Path
 from poetry.utils._compat import Path
+from poetry.utils._compat import basestring
+from poetry.utils._compat import lru_cache
 from poetry.vcs import get_vcs
 from poetry.vcs import get_vcs
 
 
 from ..metadata import Metadata
 from ..metadata import Metadata
@@ -39,40 +42,40 @@ class Builder(object):
     def build(self):
     def build(self):
         raise NotImplementedError()
         raise NotImplementedError()
 
 
-    def find_excluded_files(self):  # type: () -> list
+    @lru_cache(maxsize=None)
+    def find_excluded_files(self):  # type: () -> Set[str]
         # Checking VCS
         # Checking VCS
         vcs = get_vcs(self._path)
         vcs = get_vcs(self._path)
         if not vcs:
         if not vcs:
-            vcs_ignored_files = []
+            vcs_ignored_files = set()
         else:
         else:
-            vcs_ignored_files = vcs.get_ignored_files()
+            vcs_ignored_files = set(vcs.get_ignored_files())
 
 
-        explicitely_excluded = []
+        explicitely_excluded = set()
         for excluded_glob in self._package.exclude:
         for excluded_glob in self._package.exclude:
             for excluded in self._path.glob(excluded_glob):
             for excluded in self._path.glob(excluded_glob):
-                explicitely_excluded.append(excluded)
+                explicitely_excluded.add(str(excluded.relative_to(self._path)))
 
 
-        ignored = vcs_ignored_files + explicitely_excluded
-        result = []
+        ignored = vcs_ignored_files | explicitely_excluded
+        result = set()
         for file in ignored:
         for file in ignored:
-            try:
-                file = Path(file).absolute().relative_to(self._path)
-            except ValueError:
-                # Should only happen in tests
-                continue
-
-            result.append(file)
+            result.add(file)
 
 
         # The list of excluded files might be big and we will do a lot
         # The list of excluded files might be big and we will do a lot
         # containment check (x in excluded).
         # containment check (x in excluded).
         # Returning a set make those tests much much faster.
         # Returning a set make those tests much much faster.
-        return set(result)
+        return result
+
+    def is_excluded(self, filepath):  # type: (Union[str, Path]) -> bool
+        if not isinstance(filepath, basestring):
+            filepath = str(filepath)
+
+        return filepath in self.find_excluded_files()
 
 
-    def find_files_to_add(self, exclude_build=True):  # type: () -> list
+    def find_files_to_add(self, exclude_build=True):  # type: (bool) -> list
         """
         """
         Finds all files to add to the tarball
         Finds all files to add to the tarball
         """
         """
-        excluded = self.find_excluded_files()
         to_add = []
         to_add = []
 
 
         for include in self._module.includes:
         for include in self._module.includes:
@@ -85,7 +88,7 @@ class Builder(object):
 
 
                 file = file.relative_to(self._path)
                 file = file.relative_to(self._path)
 
 
-                if file in excluded and isinstance(include, PackageInclude):
+                if self.is_excluded(file) and isinstance(include, PackageInclude):
                     continue
                     continue
 
 
                 if file.suffix == ".pyc":
                 if file.suffix == ".pyc":

poetry/masonry/builders/sdist.py

@@ -278,7 +278,7 @@ class SdistBuilder(Builder):
                     if not f.is_dir()
                     if not f.is_dir()
                 ]
                 ]
 
 
-                data = [e for e in data_elements if e not in excluded_files]
+                data = [e for e in data_elements if not self.is_excluded(e)]
                 if not data:
                 if not data:
                     continue
                     continue
 
 

poetry/masonry/builders/wheel.py

@@ -11,6 +11,7 @@ import zipfile
 
 
 from base64 import urlsafe_b64encode
 from base64 import urlsafe_b64encode
 from io import StringIO
 from io import StringIO
+from typing import Set
 
 
 from poetry.__version__ import __version__
 from poetry.__version__ import __version__
 from poetry.semver import parse_constraint
 from poetry.semver import parse_constraint
@@ -185,9 +186,9 @@ class WheelBuilder(Builder):
             # RECORD itself is recorded with no hash or size
             # RECORD itself is recorded with no hash or size
             f.write(self.dist_info + "/RECORD,,\n")
             f.write(self.dist_info + "/RECORD,,\n")
 
 
-    def find_excluded_files(self):  # type: () -> list
+    def find_excluded_files(self):  # type: () -> Set
         # Checking VCS
         # Checking VCS
-        return []
+        return set()
 
 
     @property
     @property
     def dist_info(self):  # type: () -> str
     def dist_info(self):  # type: () -> str

poetry/utils/_compat.py

@@ -6,6 +6,11 @@ try:
 except ImportError:
 except ImportError:
     from pathlib import Path
     from pathlib import Path
 
 
+try:
+    from functools32 import lru_cache
+except ImportError:
+    from functools import lru_cache
+
 try:  # Python 2
 try:  # Python 2
     long = long
     long = long
     unicode = unicode
     unicode = unicode

pyproject.toml

@@ -43,6 +43,8 @@ typing = { version = "^3.6", python = "~2.7 || ~3.4" }
 pathlib2 = { version = "^2.3", python = "~2.7 || ~3.4" }
 pathlib2 = { version = "^2.3", python = "~2.7 || ~3.4" }
 # Use virtualenv for Python 2.7 since venv does not exist
 # Use virtualenv for Python 2.7 since venv does not exist
 virtualenv = { version = "^16.0", python = "~2.7" }
 virtualenv = { version = "^16.0", python = "~2.7" }
+# functools32 is needed for Python 2.7
+functools32 = { version = "^3.2.3", python = "~2.7" }
 
 
 [tool.poetry.dev-dependencies]
 [tool.poetry.dev-dependencies]
 pytest = "^3.4"
 pytest = "^3.4"

tests/masonry/builders/test_sdist.py

@@ -439,6 +439,7 @@ def test_default_with_excluded_data(mocker):
     p = mocker.patch("poetry.vcs.git.Git.get_ignored_files")
     p = mocker.patch("poetry.vcs.git.Git.get_ignored_files")
     p.return_value = [
     p.return_value = [
         str(
         str(
+            (
                 Path(__file__).parent
                 Path(__file__).parent
                 / "fixtures"
                 / "fixtures"
                 / "default_with_excluded_data"
                 / "default_with_excluded_data"
@@ -446,6 +447,7 @@ def test_default_with_excluded_data(mocker):
                 / "data"
                 / "data"
                 / "sub_data"
                 / "sub_data"
                 / "data2.txt"
                 / "data2.txt"
+            ).relative_to(project("default_with_excluded_data"))
         )
         )
     ]
     ]
     poetry = Poetry.create(project("default_with_excluded_data"))
     poetry = Poetry.create(project("default_with_excluded_data"))