tests: enable parallel test execution (#5745)

also support symlinked temp directories

tests: enable parallel test execution (#5745)
also support symlinked temp directories
89e3f7ba · Arun Babu Neelicattu · GitHub · 2cc07cfe · 89e3f7ba · 89e3f7ba
Commit 89e3f7ba authored Jun 03, 2022 by Arun Babu Neelicattu Committed by GitHub Jun 02, 2022
10 changed files
--- a/.cirrus.yml
+++ b/.cirrus.yml
@@ -27,7 +27,7 @@ test_task:
    - poetry config virtualenvs.in-project true
  test_script:
    - poetry install
-    - poetry run pytest -q --junitxml=junit.xml tests
+    - poetry run pytest -n auto -q --junitxml=junit.xml tests
  on_failure:
    annotate_failure_artifacts:
      path: junit.xml

--- a/.github/workflows/main.yml
+++ b/.github/workflows/main.yml
@@ -88,13 +88,13 @@ jobs:
        run: poetry run pip install pytest-github-actions-annotate-failures
      - name: Run pytest
-        run: poetry run python -m pytest -p no:sugar -q tests/
+        run: poetry run python -m pytest -n auto -p no:sugar -q tests/
      - name: Run pytest (integration suite)
        env:
          POETRY_TEST_INTEGRATION_GIT_USERNAME: ${GITHUB_ACTOR}
          POETRY_TEST_INTEGRATION_GIT_PASSWORD: ${{ secrets.GITHUB_TOKEN }}
-        run: poetry run python -m pytest -p no:sugar -q --integration tests/integration
+        run: poetry run python -m pytest -n auto -p no:sugar -q --integration tests/integration
      - name: Get Plugin Version (poetry-plugin-export)
        id: poetry-plugin-export-version

--- a/poetry.lock
+++ b/poetry.lock
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -75,6 +75,7 @@ pytest = "^7.1"
 pytest-cov = "^3.0"
 pytest-mock = "^3.5"
 pytest-sugar = "^0.9"
+pytest-xdist = { version = "^2.5", extras = ["psutil"] }
 pre-commit = "^2.6"
 deepdiff = "^5.0"
 httpretty = "^1.0"

--- a/tests/conftest.py
+++ b/tests/conftest.py
@@ -46,6 +46,7 @@ if TYPE_CHECKING:
    from pytest_mock import MockerFixture
    from poetry.poetry import Poetry
+    from tests.types import FixtureCopier
    from tests.types import FixtureDirGetter
    from tests.types import ProjectFactory
@@ -298,7 +299,7 @@ def fixture_dir(fixture_base: Path) -> FixtureDirGetter:
 def tmp_dir() -> Iterator[str]:
    dir_ = tempfile.mkdtemp(prefix="poetry_")
-    yield dir_
+    yield Path(dir_).resolve().as_posix()
    remove_directory(dir_, force=True)
@@ -366,6 +367,7 @@ def project_factory(
    repo: TestRepository,
    installed: Repository,
    default_python: str,
+    load_required_fixtures: None,
 ) -> ProjectFactory:
    workspace = Path(tmp_dir)
@@ -376,17 +378,25 @@ def project_factory(
        pyproject_content: str | None = None,
        poetry_lock_content: str | None = None,
        install_deps: bool = True,
+        source: Path | None = None,
+        locker_config: dict[str, Any] | None = None,
    ) -> Poetry:
        project_dir = workspace / f"poetry-fixture-{name}"
        dependencies = dependencies or {}
        dev_dependencies = dev_dependencies or {}
-        if pyproject_content:
+        if pyproject_content or source:
-            project_dir.mkdir(parents=True, exist_ok=True)
+            if source:
-            with project_dir.joinpath("pyproject.toml").open(
+                project_dir.parent.mkdir(parents=True, exist_ok=True)
-                "w", encoding="utf-8"
+                shutil.copytree(source, project_dir)
-            ) as f:
+            else:
-                f.write(pyproject_content)
+                project_dir.mkdir(parents=True, exist_ok=True)
+            if pyproject_content:
+                with project_dir.joinpath("pyproject.toml").open(
+                    "w", encoding="utf-8"
+                ) as f:
+                    f.write(pyproject_content)
        else:
            layout("src")(
                name,
@@ -404,7 +414,9 @@ def project_factory(
        poetry = Factory().create_poetry(project_dir)
-        locker = TestLocker(poetry.locker.lock.path, poetry.locker._local_config)
+        locker = TestLocker(
+            poetry.locker.lock.path, locker_config or poetry.locker._local_config
+        )
        locker.write()
        poetry.set_locker(locker)
@@ -441,3 +453,36 @@ def set_simple_log_formatter() -> None:
        for handler in logging.getLogger(name).handlers:
            # replace formatter with simple formatter for testing
            handler.setFormatter(logging.Formatter(fmt="%(message)s"))
+@pytest.fixture
+def fixture_copier(fixture_base: Path, tmp_dir: str) -> FixtureCopier:
+    def _copy(relative_path: str, target: Path | None = None) -> Path:
+        path = fixture_base.joinpath(relative_path)
+        target = target or Path(tmp_dir, relative_path)
+        target.parent.mkdir(parents=True, exist_ok=True)
+        if target.exists():
+            return target
+        if path.is_dir():
+            shutil.copytree(path, target)
+        else:
+            shutil.copyfile(path, target)
+        return target
+    return _copy
+@pytest.fixture
+def required_fixtures() -> list[str]:
+    return []
+@pytest.fixture(autouse=True)
+def load_required_fixtures(
+    required_fixtures: list[str], fixture_copier: FixtureCopier
+) -> None:
+    for fixture in required_fixtures:
+        fixture_copier(fixture)
--- a/tests/console/commands/test_add.py
+++ b/tests/console/commands/test_add.py
@@ -414,17 +414,17 @@ Package operations: 2 installs, 0 updates, 0 removals
    assert content["dependencies"]["demo"] == expected
+@pytest.mark.parametrize(
+    "required_fixtures",
+    [["git/github.com/demo/demo"]],
+)
 @pytest.mark.parametrize("editable", [False, True])
 def test_add_directory_constraint(
    editable: bool,
    app: PoetryTestApplication,
    repo: TestRepository,
    tester: CommandTester,
-    mocker: MockerFixture,
 ):
-    p = mocker.patch("pathlib.Path.cwd")
-    p.return_value = Path(__file__).parent
    repo.add_package(get_package("pendulum", "1.4.4"))
    repo.add_package(get_package("cleo", "0.6.5"))
@@ -451,22 +451,22 @@ Package operations: 2 installs, 0 updates, 0 removals
    assert "demo" in content["dependencies"]
-    expected = {"path": "../git/github.com/demo/demo"}
+    expected = {"path": path}
    if editable:
        expected["develop"] = True
    assert content["dependencies"]["demo"] == expected
+@pytest.mark.parametrize(
+    "required_fixtures",
+    [["git/github.com/demo/pyproject-demo"]],
+)
 def test_add_directory_with_poetry(
    app: PoetryTestApplication,
    repo: TestRepository,
    tester: CommandTester,
-    mocker: MockerFixture,
 ):
-    p = mocker.patch("pathlib.Path.cwd")
-    p.return_value = Path(__file__) / ".."
    repo.add_package(get_package("pendulum", "1.4.4"))
    path = "../git/github.com/demo/pyproject-demo"
@@ -489,16 +489,16 @@ Package operations: 2 installs, 0 updates, 0 removals
    assert tester.command.installer.executor.installations_count == 2
+@pytest.mark.parametrize(
+    "required_fixtures",
+    [["distributions/demo-0.1.0-py2.py3-none-any.whl"]],
+)
 def test_add_file_constraint_wheel(
    app: PoetryTestApplication,
    repo: TestRepository,
    tester: CommandTester,
-    mocker: MockerFixture,
    poetry: Poetry,
 ):
-    p = mocker.patch("pathlib.Path.cwd")
-    p.return_value = poetry.file.parent
    repo.add_package(get_package("pendulum", "1.4.4"))
    path = "../distributions/demo-0.1.0-py2.py3-none-any.whl"
@@ -523,20 +523,18 @@ Package operations: 2 installs, 0 updates, 0 removals
    content = app.poetry.file.read()["tool"]["poetry"]
    assert "demo" in content["dependencies"]
-    assert content["dependencies"]["demo"] == {
+    assert content["dependencies"]["demo"] == {"path": path}
-        "path": "../distributions/demo-0.1.0-py2.py3-none-any.whl"
-    }
+@pytest.mark.parametrize(
+    "required_fixtures",
+    [["distributions/demo-0.1.0.tar.gz"]],
+)
 def test_add_file_constraint_sdist(
    app: PoetryTestApplication,
    repo: TestRepository,
    tester: CommandTester,
-    mocker: MockerFixture,
 ):
-    p = mocker.patch("pathlib.Path.cwd")
-    p.return_value = Path(__file__) / ".."
    repo.add_package(get_package("pendulum", "1.4.4"))
    path = "../distributions/demo-0.1.0.tar.gz"
@@ -561,9 +559,7 @@ Package operations: 2 installs, 0 updates, 0 removals
    content = app.poetry.file.read()["tool"]["poetry"]
    assert "demo" in content["dependencies"]
-    assert content["dependencies"]["demo"] == {
+    assert content["dependencies"]["demo"] == {"path": path}
-        "path": "../distributions/demo-0.1.0.tar.gz"
-    }
 def test_add_constraint_with_extras_option(
@@ -1379,16 +1375,16 @@ Package operations: 2 installs, 0 updates, 0 removals
    }
+@pytest.mark.parametrize(
+    "required_fixtures",
+    [["git/github.com/demo/demo"]],
+)
 def test_add_directory_constraint_old_installer(
    app: PoetryTestApplication,
    repo: TestRepository,
    installer: NoopInstaller,
-    mocker: MockerFixture,
    old_tester: CommandTester,
 ):
-    p = mocker.patch("pathlib.Path.cwd")
-    p.return_value = Path(__file__) / ".."
    repo.add_package(get_package("pendulum", "1.4.4"))
    repo.add_package(get_package("cleo", "0.6.5"))
@@ -1415,19 +1411,19 @@ Package operations: 2 installs, 0 updates, 0 removals
    content = app.poetry.file.read()["tool"]["poetry"]
    assert "demo" in content["dependencies"]
-    assert content["dependencies"]["demo"] == {"path": "../git/github.com/demo/demo"}
+    assert content["dependencies"]["demo"] == {"path": path}
+@pytest.mark.parametrize(
+    "required_fixtures",
+    [["git/github.com/demo/pyproject-demo"]],
+)
 def test_add_directory_with_poetry_old_installer(
    app: PoetryTestApplication,
    repo: TestRepository,
    installer: NoopInstaller,
-    mocker: MockerFixture,
    old_tester: CommandTester,
 ):
-    p = mocker.patch("pathlib.Path.cwd")
-    p.return_value = Path(__file__) / ".."
    repo.add_package(get_package("pendulum", "1.4.4"))
    path = "../git/github.com/demo/pyproject-demo"
@@ -1451,16 +1447,16 @@ Package operations: 2 installs, 0 updates, 0 removals
    assert len(installer.installs) == 2
+@pytest.mark.parametrize(
+    "required_fixtures",
+    [["distributions/demo-0.1.0-py2.py3-none-any.whl"]],
+)
 def test_add_file_constraint_wheel_old_installer(
    app: PoetryTestApplication,
    repo: TestRepository,
    installer: NoopInstaller,
-    mocker: MockerFixture,
    old_tester: CommandTester,
 ):
-    p = mocker.patch("pathlib.Path.cwd")
-    p.return_value = Path(__file__) / ".."
    repo.add_package(get_package("pendulum", "1.4.4"))
    path = "../distributions/demo-0.1.0-py2.py3-none-any.whl"
@@ -1486,21 +1482,19 @@ Package operations: 2 installs, 0 updates, 0 removals
    content = app.poetry.file.read()["tool"]["poetry"]
    assert "demo" in content["dependencies"]
-    assert content["dependencies"]["demo"] == {
+    assert content["dependencies"]["demo"] == {"path": path}
-        "path": "../distributions/demo-0.1.0-py2.py3-none-any.whl"
-    }
+@pytest.mark.parametrize(
+    "required_fixtures",
+    [["distributions/demo-0.1.0.tar.gz"]],
+)
 def test_add_file_constraint_sdist_old_installer(
    app: PoetryTestApplication,
    repo: TestRepository,
    installer: NoopInstaller,
-    mocker: MockerFixture,
    old_tester: CommandTester,
 ):
-    p = mocker.patch("pathlib.Path.cwd")
-    p.return_value = Path(__file__) / ".."
    repo.add_package(get_package("pendulum", "1.4.4"))
    path = "../distributions/demo-0.1.0.tar.gz"
@@ -1526,9 +1520,7 @@ Package operations: 2 installs, 0 updates, 0 removals
    content = app.poetry.file.read()["tool"]["poetry"]
    assert "demo" in content["dependencies"]
-    assert content["dependencies"]["demo"] == {
+    assert content["dependencies"]["demo"] == {"path": path}
-        "path": "../distributions/demo-0.1.0.tar.gz"
-    }
 def test_add_constraint_with_extras_option_old_installer(

--- a/tests/console/commands/test_show.py
+++ b/tests/console/commands/test_show.py
@@ -932,7 +932,15 @@ pendulum 2.0.0 2.0.1 Pendulum package
    assert tester.io.fetch_output() == expected
-@pytest.mark.parametrize("project_directory", ["project_with_local_dependencies"])
+@pytest.mark.parametrize(
+    ("project_directory", "required_fixtures"),
+    [
+        (
+            "project_with_local_dependencies",
+            ["distributions/demo-0.1.0-py2.py3-none-any.whl", "project_with_setup"],
+        ),
+    ],
+)
 def test_show_outdated_local_dependencies(
    tester: CommandTester,
    poetry: Poetry,
@@ -952,13 +960,13 @@ def test_show_outdated_local_dependencies(
    demo_010 = get_package("demo", "0.1.0")
    demo_010.description = ""
-    my_package_012 = get_package("project-with-setup", "0.1.2")
+    my_package_011 = get_package("project-with-setup", "0.1.1")
-    my_package_012.description = "Demo project."
+    my_package_011.description = "Demo project."
    installed.add_package(cachy_020)
    installed.add_package(pendulum_200)
    installed.add_package(demo_010)
-    installed.add_package(my_package_012)
+    installed.add_package(my_package_011)
    repo.add_package(cachy_020)
    repo.add_package(cachy_030)

--- a/tests/console/conftest.py
+++ b/tests/console/conftest.py
@@ -11,14 +11,11 @@ from cleo.io.null_io import NullIO
 from cleo.testers.application_tester import ApplicationTester
 from cleo.testers.command_tester import CommandTester
-from poetry.factory import Factory
 from poetry.installation import Installer
 from poetry.installation.noop_installer import NoopInstaller
-from poetry.repositories import Pool
 from poetry.utils.env import MockEnv
 from tests.helpers import PoetryTestApplication
 from tests.helpers import TestExecutor
-from tests.helpers import TestLocker
 from tests.helpers import mock_clone
@@ -32,8 +29,8 @@ if TYPE_CHECKING:
    from poetry.repositories import Repository
    from poetry.utils.env import Env
    from tests.conftest import Config
-    from tests.helpers import TestRepository
    from tests.types import CommandTesterFactory
+    from tests.types import ProjectFactory
 @pytest.fixture()
@@ -98,26 +95,11 @@ def project_directory() -> str:
 @pytest.fixture
-def poetry(repo: TestRepository, project_directory: str, config: Config) -> Poetry:
+def poetry(project_directory: str, project_factory: ProjectFactory) -> Poetry:
+    return project_factory(
-    p = Factory().create_poetry(
+        name="simple",
-        Path(__file__).parent.parent / "fixtures" / project_directory
+        source=Path(__file__).parent.parent / "fixtures" / project_directory,
    )
-    p.set_locker(TestLocker(p.locker.lock.path, p.locker._local_config))
-    with p.file.path.open(encoding="utf-8") as f:
-        content = f.read()
-    p.set_config(config)
-    pool = Pool()
-    pool.add_repository(repo)
-    p.set_pool(pool)
-    yield p
-    with p.file.path.open("w", encoding="utf-8") as f:
-        f.write(content)
 @pytest.fixture

--- a/tests/types.py
+++ b/tests/types.py
@@ -45,6 +45,7 @@ class ProjectFactory(Protocol):
        pyproject_content: str | None = None,
        poetry_lock_content: str | None = None,
        install_deps: bool = True,
+        source: Path | None = None,
    ) -> Poetry:
        ...
@@ -52,3 +53,8 @@ class ProjectFactory(Protocol):
 class FixtureDirGetter(Protocol):
    def __call__(self, name: str) -> Path:
        ...
+class FixtureCopier(Protocol):
+    def __call__(self, relative_path: str, target: Path | None = None) -> Path:
+        ...
--- a/tests/utils/test_env.py
+++ b/tests/utils/test_env.py
@@ -39,6 +39,7 @@ if TYPE_CHECKING:
    from poetry.poetry import Poetry
    from tests.conftest import Config
+    from tests.types import ProjectFactory
 MINIMAL_SCRIPT = """\
@@ -73,13 +74,9 @@ class MockVirtualEnv(VirtualEnv):
 @pytest.fixture()
-def poetry(config: Config) -> Poetry:
+def poetry(project_factory: ProjectFactory) -> Poetry:
-    poetry = Factory().create_poetry(
+    fixture = Path(__file__).parent.parent / "fixtures" / "simple_project"
-        Path(__file__).parent.parent / "fixtures" / "simple_project"
+    return project_factory("simple", source=fixture)
-    )
-    poetry.set_config(config)
-    return poetry
 @pytest.fixture()