Add maximum of workers to config (#3516)

cdbacd64 · Maximilian · GitHub · 28591019 · cdbacd64 · cdbacd64
Commit cdbacd64 authored Nov 21, 2021 by Maximilian Committed by GitHub Nov 21, 2021
10 changed files
--- a/.gitignore
+++ b/.gitignore
@@ -38,3 +38,5 @@ MANIFEST.in
 /releases/*
 pip-wheel-metadata
 /poetry.toml
+poetry/core/*
--- a/docs/configuration.md
+++ b/docs/configuration.md
@@ -123,6 +123,21 @@ Defaults to one of the following directories:
 Use parallel execution when using the new (`>=1.1.0`) installer.
 Defaults to `true`.
+### `installer.max-workers`
+**Type**: int
+Set the maximum number of workers while using the parallel installer. Defaults to `number_of_cores + 4`.
+The `number_of_cores` is determined by `os.cpu_count()`.
+If this raises a `NotImplentedError` exception `number_of_cores` is assumed to be 1.
+If this configuration parameter is set to a value greater than `number_of_cores + 4`,
+the number of maximum workers is still limited at `number_of_cores + 4`.
+{{% note %}}
+This configuration will be ignored when `installer.parallel` is set to false.
+{{% /note %}}
 ### `virtualenvs.create`
 **Type**: boolean

--- a/poetry.lock
+++ b/poetry.lock
@@ -212,6 +212,14 @@ docs = ["furo (>=2021.8.17b43)", "sphinx (>=4.1)", "sphinx-autodoc-typehints (>=
 testing = ["covdefaults (>=1.2.0)", "coverage (>=4)", "pytest (>=4)", "pytest-cov", "pytest-timeout (>=1.4.2)"]
 [[package]]
+name = "flatdict"
+version = "4.0.1"
+description = "Python module for interacting with nested dicts as a single level dict with delimited keys."
+category = "dev"
+optional = false
+python-versions = "*"
+[[package]]
 name = "html5lib"
 version = "1.1"
 description = "HTML parser based on the WHATWG HTML specification"
@@ -739,7 +747,7 @@ testing = ["pytest (>=4.6)", "pytest-checkdocs (>=2.4)", "pytest-flake8", "pytes
 [metadata]
 lock-version = "1.1"
 python-versions = "^3.6"
-content-hash = "9d2e32899df46f2c63018e9a3f5e95dbbeb1ec41291c31289cff40f6f2d935a4"
+content-hash = "d427df125a868ada92bbb6d3a8cc90def6034ad684c1546afb519729048ab150"
 [metadata.files]
 atomicwrites = [
@@ -929,6 +937,9 @@ filelock = [
    {file = "filelock-3.4.0-py3-none-any.whl", hash = "sha256:2e139a228bcf56dd8b2274a65174d005c4a6b68540ee0bdbb92c76f43f29f7e8"},
    {file = "filelock-3.4.0.tar.gz", hash = "sha256:93d512b32a23baf4cac44ffd72ccf70732aeff7b8050fcaf6d3ec406d954baf4"},
 ]
+flatdict = [
+    {file = "flatdict-4.0.1.tar.gz", hash = "sha256:cd32f08fd31ed21eb09ebc76f06b6bd12046a24f77beb1fd0281917e47f26742"},
+]
 html5lib = [
    {file = "html5lib-1.1-py2.py3-none-any.whl", hash = "sha256:0d78f8fde1c230e99fe37986a60526d7049ed4bf8a9fadbad5f00e22e58e041d"},
    {file = "html5lib-1.1.tar.gz", hash = "sha256:b2e5b40261e20f354d198eae92afc10d750afb487ed5e50f9c4eaf07c184146f"},

--- a/pyproject.toml
+++ b/pyproject.toml
@@ -65,6 +65,7 @@ deepdiff = "^5.0"
 httpretty = "^1.0"
 typing-extensions = { version = "^4.0.0", python = "<3.8" }
 zipp = { version = "^3.4", python = "<3.8" }
+flatdict = "^4.0.1"
 [tool.poetry.scripts]
 poetry = "poetry.console.application:main"

--- a/src/poetry/config/config.py
+++ b/src/poetry/config/config.py
@@ -25,6 +25,10 @@ def boolean_normalizer(val: str) -> bool:
    return val in ["true", "1"]
+def int_normalizer(val: str) -> int:
+    return int(val)
 class Config:
    default_config: Dict[str, Any] = {
@@ -36,7 +40,7 @@ class Config:
            "options": {"always-copy": False, "system-site-packages": False},
        },
        "experimental": {"new-installer": True},
-        "installer": {"parallel": True},
+        "installer": {"parallel": True, "max-workers": None},
    }
    def __init__(
@@ -129,7 +133,8 @@ class Config:
        return re.sub(r"{(.+?)}", lambda m: self.get(m.group(1)), value)
-    def _get_normalizer(self, name: str) -> Callable:
+    @staticmethod
+    def _get_normalizer(name: str) -> Callable:
        if name in {
            "virtualenvs.create",
            "virtualenvs.in-project",
@@ -143,4 +148,7 @@ class Config:
        if name == "virtualenvs.path":
            return lambda val: str(Path(val))
+        if name == "installer.max-workers":
+            return int_normalizer
        return lambda val: val
--- a/src/poetry/console/commands/config.py
+++ b/src/poetry/console/commands/config.py
@@ -52,6 +52,7 @@ To remove a repository (repo is a short alias for repositories):
        from poetry.config.config import boolean_normalizer
        from poetry.config.config import boolean_validator
+        from poetry.config.config import int_normalizer
        from poetry.locations import CACHE_DIR
        unique_config_values = {
@@ -87,6 +88,11 @@ To remove a repository (repo is a short alias for repositories):
                boolean_normalizer,
                True,
            ),
+            "installer.max-workers": (
+                lambda val: int(val) > 0,
+                int_normalizer,
+                None,
+            ),
        }
        return unique_config_values

--- a/src/poetry/installation/executor.py
+++ b/src/poetry/installation/executor.py
@@ -12,6 +12,7 @@ from typing import TYPE_CHECKING
 from typing import Any
 from typing import Dict
 from typing import List
+from typing import Optional
 from typing import Union
 from cleo.io.null_io import NullIO
@@ -66,14 +67,9 @@ class Executor:
            parallel = config.get("installer.parallel", True)
        if parallel:
-            # This should be directly handled by ThreadPoolExecutor
+            self._max_workers = self._get_max_workers(
-            # however, on some systems the number of CPUs cannot be determined
+                desired_max_workers=config.get("installer.max-workers")
-            # (it raises a NotImplementedError), so, in this case, we assume
+            )
-            # that the system only has one CPU.
-            try:
-                self._max_workers = os.cpu_count() + 4
-            except NotImplementedError:
-                self._max_workers = 5
        else:
            self._max_workers = 1
@@ -190,6 +186,21 @@ class Executor:
        return 1 if self._shutdown else 0
+    @staticmethod
+    def _get_max_workers(desired_max_workers: Optional[int] = None):
+        # This should be directly handled by ThreadPoolExecutor
+        # however, on some systems the number of CPUs cannot be determined
+        # (it raises a NotImplementedError), so, in this case, we assume
+        # that the system only has one CPU.
+        try:
+            default_max_workers = os.cpu_count() + 4
+        except NotImplementedError:
+            default_max_workers = 5
+        if desired_max_workers is None:
+            return default_max_workers
+        return min(default_max_workers, desired_max_workers)
    def _write(self, operation: "OperationTypes", line: str) -> None:
        if not self.supports_fancy_output() or not self._should_write_operation(
            operation

--- a/tests/config/test_config.py
+++ b/tests/config/test_config.py
@@ -2,31 +2,29 @@ import os
 import re
 from typing import TYPE_CHECKING
-from typing import Any
+from typing import Callable
-from typing import Dict
 from typing import Iterator
-from typing import Optional
 from typing import Tuple
 import pytest
+from flatdict import FlatDict
 from poetry.config.config import Config
+from poetry.config.config import boolean_normalizer
+from poetry.config.config import int_normalizer
 if TYPE_CHECKING:
    from pathlib import Path
-def get_boolean_options(config: Optional[Dict[str, Any]] = None) -> str:
+def get_options_based_on_normalizer(normalizer: Callable) -> str:
-    if config is None:
+    flattened_config = FlatDict(Config.default_config, delimiter=".")
-        config = Config.default_config
-    for k, v in config.items():
+    for k in flattened_config:
-        if isinstance(v, bool) or v is None:
+        if Config._get_normalizer(k) == normalizer:
            yield k
-        if isinstance(v, dict):
-            for suboption in get_boolean_options(v):
-                yield f"{k}.{suboption}"
 @pytest.mark.parametrize(
@@ -43,10 +41,14 @@ def test_config_get_processes_depended_on_values(
 def generate_environment_variable_tests() -> Iterator[Tuple[str, str, str, bool]]:
-    for env_value, value in [("true", True), ("false", False)]:
+    for normalizer, values in [
-        for name in get_boolean_options():
+        (boolean_normalizer, [("true", True), ("false", False)]),
-            env_var = "POETRY_{}".format(re.sub("[.-]+", "_", name).upper())
+        (int_normalizer, [("4", 4), ("2", 2)]),
-            yield name, env_var, env_value, value
+    ]:
+        for env_value, value in values:
+            for name in get_options_based_on_normalizer(normalizer=normalizer):
+                env_var = "POETRY_" + re.sub("[.-]+", "_", name).upper()
+                yield name, env_var, env_value, value
 @pytest.mark.parametrize(

--- a/tests/console/commands/test_config.py
+++ b/tests/console/commands/test_config.py
@@ -46,6 +46,7 @@ def test_list_displays_default_value_if_not_set(
    expected = """cache-dir = {cache}
 experimental.new-installer = true
+installer.max-workers = null
 installer.parallel = true
 virtualenvs.create = true
 virtualenvs.in-project = null
@@ -70,6 +71,7 @@ def test_list_displays_set_get_setting(
    expected = """cache-dir = {cache}
 experimental.new-installer = true
+installer.max-workers = null
 installer.parallel = true
 virtualenvs.create = false
 virtualenvs.in-project = null
@@ -118,6 +120,7 @@ def test_list_displays_set_get_local_setting(
    expected = """cache-dir = {cache}
 experimental.new-installer = true
+installer.max-workers = null
 installer.parallel = true
 virtualenvs.create = false
 virtualenvs.in-project = null

--- a/tests/installation/test_executor.py
+++ b/tests/installation/test_executor.py
@@ -559,3 +559,35 @@ def test_executor_should_use_cached_link_and_hash(
        Link("https://example.com/demo-0.1.0-py2.py3-none-any.whl"),
    )
    assert archive == link_cached
+@pytest.mark.parametrize(
+    ("max_workers", "cpu_count", "side_effect", "expected_workers"),
+    [
+        (None, 3, None, 7),
+        (3, 4, None, 3),
+        (8, 3, None, 7),
+        (None, 8, NotImplementedError(), 5),
+        (2, 8, NotImplementedError(), 2),
+        (8, 8, NotImplementedError(), 5),
+    ],
+)
+def test_executor_should_be_initialized_with_correct_workers(
+    tmp_venv,
+    pool,
+    config,
+    io,
+    mocker,
+    max_workers,
+    cpu_count,
+    side_effect,
+    expected_workers,
+):
+    config = Config()
+    config.merge({"installer": {"max-workers": max_workers}})
+    mocker.patch("os.cpu_count", return_value=cpu_count, side_effect=side_effect)
+    executor = Executor(tmp_venv, pool, config, io)
+    assert executor._max_workers == expected_workers