Use tomli for reading the lock file (#6562)

Co-authored-by: Randy Döring <30527984+radoering@users.noreply.github.com>

Use tomli for reading the lock file (#6562)
Co-authored-by: Randy Döring <30527984+radoering@users.noreply.github.com>
b28339d8 · David Hotham · GitHub · 3a961dfb · b28339d8 · b28339d8
Commit b28339d8 authored Nov 05, 2022 by David Hotham Committed by GitHub Nov 05, 2022
9 changed files
--- a/poetry.lock
+++ b/poetry.lock
@@ -10,7 +10,7 @@ python-versions = ">=3.5"
 dev = ["cloudpickle", "coverage[toml] (>=5.0.2)", "furo", "hypothesis", "mypy (>=0.900,!=0.940)", "pre-commit", "pympler", "pytest (>=4.3.0)", "pytest-mypy-plugins", "sphinx", "sphinx-notfound-page", "zope.interface"]
 docs = ["furo", "sphinx", "sphinx-notfound-page", "zope.interface"]
 tests = ["cloudpickle", "coverage[toml] (>=5.0.2)", "hypothesis", "mypy (>=0.900,!=0.940)", "pympler", "pytest (>=4.3.0)", "pytest-mypy-plugins", "zope.interface"]
-tests_no_zope = ["cloudpickle", "coverage[toml] (>=5.0.2)", "hypothesis", "mypy (>=0.900,!=0.940)", "pympler", "pytest (>=4.3.0)", "pytest-mypy-plugins"]
+tests-no-zope = ["cloudpickle", "coverage[toml] (>=5.0.2)", "hypothesis", "mypy (>=0.900,!=0.940)", "pympler", "pytest (>=4.3.0)", "pytest-mypy-plugins"]
 [[package]]
 name = "backports.cached-property"
@@ -86,7 +86,7 @@ optional = false
 python-versions = ">=3.6.0"
 [package.extras]
-unicode_backport = ["unicodedata2"]
+unicode-backport = ["unicodedata2"]
 [[package]]
 name = "cleo"
@@ -524,15 +524,15 @@ importlib-metadata = {version = ">=1.7.0", markers = "python_version < \"3.8\""}
 [[package]]
 name = "poetry-plugin-export"
-version = "1.1.2"
+version = "1.2.0"
 description = "Poetry plugin to export the dependencies to various formats"
 category = "main"
 optional = false
 python-versions = ">=3.7,<4.0"
 [package.dependencies]
-poetry = ">=1.2.0,<2.0.0"
+poetry = ">=1.2.2,<2.0.0"
-poetry-core = ">=1.1.0,<2.0.0"
+poetry-core = ">=1.3.0,<2.0.0"
 [[package]]
 name = "pre-commit"
@@ -749,7 +749,7 @@ urllib3 = ">=1.21.1,<1.27"
 [package.extras]
 socks = ["PySocks (>=1.5.6,!=1.5.7)"]
-use_chardet_on_py3 = ["chardet (>=3.0.2,<6)"]
+use-chardet-on-py3 = ["chardet (>=3.0.2,<6)"]
 [[package]]
 name = "requests-toolbelt"
@@ -815,7 +815,7 @@ python-versions = ">=2.6, !=3.0.*, !=3.1.*, !=3.2.*"
 name = "tomli"
 version = "2.0.1"
 description = "A lil' TOML parser"
-category = "dev"
+category = "main"
 optional = false
 python-versions = ">=3.7"
@@ -951,7 +951,7 @@ testing = ["flake8 (<5)", "func-timeout", "jaraco.functools", "jaraco.itertools"
 [metadata]
 lock-version = "1.1"
 python-versions = "^3.7"
-content-hash = "7db62de4ee4f2831829fc08b6a3d190f177c62d5af7676e20fb228bb58274850"
+content-hash = "703e97d65aae512130b6a43ef7c82a27c914b9f4023f5267824b54e901d4bde3"
 [metadata.files]
 attrs = [
@@ -1364,8 +1364,8 @@ poetry-core = [
    {file = "poetry_core-1.3.2-py3-none-any.whl", hash = "sha256:ea0f5a90b339cde132b4e43cff78a1b440cd928db864bb67cfc97fdfcefe7218"},
 ]
 poetry-plugin-export = [
-    {file = "poetry-plugin-export-1.1.2.tar.gz", hash = "sha256:5e92525dd63f38ce74a51ed68ea91d753523f21ce5f9ef8d3b793e2a4b2222ef"},
+    {file = "poetry_plugin_export-1.2.0-py3-none-any.whl", hash = "sha256:109fb43ebfd0e79d8be2e7f9d43ba2ae357c4975a18dfc0cfdd9597dd086790e"},
-    {file = "poetry_plugin_export-1.1.2-py3-none-any.whl", hash = "sha256:946e3313b3d00c18fb9a50522e9d5e6a7e111beaba8d6ae33297662fc2070ac1"},
+    {file = "poetry_plugin_export-1.2.0.tar.gz", hash = "sha256:9a1dd42765408931d7831738749022651d43a2968b67c988db1b7a567dfe41ef"},
 ]
 pre-commit = [
    {file = "pre_commit-2.20.0-py2.py3-none-any.whl", hash = "sha256:51a5ba7c480ae8072ecdb6933df22d2f812dc897d5fe848778116129a681aac7"},

--- a/pyproject.toml
+++ b/pyproject.toml
@@ -48,7 +48,7 @@ generate-setup-file = false
 python = "^3.7"
 poetry-core = "^1.3.2"
-poetry-plugin-export = "^1.1.2"
+poetry-plugin-export = "^1.2.0"
 "backports.cached-property" = { version = "^1.0.2", python = "<3.8" }
 cachecontrol = { version = "^0.12.9", extras = ["filecache"] }
 cleo = "^1.0.0a5"
@@ -67,6 +67,7 @@ platformdirs = "^2.5.2"
 requests = "^2.18"
 requests-toolbelt = ">=0.9.1,<0.11.0"
 shellingham = "^1.5"
+tomli = { version = "^2.0.1", python = "<3.11" }
 # exclude 0.11.2 and 0.11.3 due to https://github.com/sdispater/tomlkit/issues/225
 tomlkit = ">=0.11.1,<1.0.0,!=0.11.2,!=0.11.3"
 trove-classifiers = "^2022.5.19"

--- a/src/poetry/console/commands/add.py
+++ b/src/poetry/console/commands/add.py
@@ -241,7 +241,7 @@ The add command adds required packages to your <comment>pyproject.toml</> and in
        # Refresh the locker
        self.poetry.set_locker(
-            self.poetry.locker.__class__(self.poetry.locker.lock.path, poetry_content)
+            self.poetry.locker.__class__(self.poetry.locker.lock, poetry_content)
        )
        self.installer.set_locker(self.poetry.locker)

--- a/src/poetry/console/commands/remove.py
+++ b/src/poetry/console/commands/remove.py
@@ -103,7 +103,7 @@ list of installed packages
        # Refresh the locker
        self.poetry.set_locker(
-            self.poetry.locker.__class__(self.poetry.locker.lock.path, poetry_content)
+            self.poetry.locker.__class__(self.poetry.locker.lock, poetry_content)
        )
        self.installer.set_locker(self.poetry.locker)

--- a/src/poetry/packages/locker.py
+++ b/src/poetry/packages/locker.py
@@ -24,7 +24,8 @@ from tomlkit import comment
 from tomlkit import document
 from tomlkit import inline_table
 from tomlkit import table
-from tomlkit.exceptions import TOMLKitError
+from poetry.utils._compat import tomllib
 if TYPE_CHECKING:
@@ -32,7 +33,6 @@ if TYPE_CHECKING:
    from poetry.core.packages.file_dependency import FileDependency
    from poetry.core.packages.url_dependency import URLDependency
    from poetry.core.packages.vcs_dependency import VCSDependency
-    from tomlkit.items import Table
    from tomlkit.toml_document import TOMLDocument
    from poetry.repositories.lockfile_repository import LockfileRepository
@@ -53,17 +53,17 @@ class Locker:
    _relevant_keys = [*_legacy_keys, "group"]
    def __init__(self, lock: str | Path, local_config: dict[str, Any]) -> None:
-        self._lock = TOMLFile(lock)
+        self._lock = lock if isinstance(lock, Path) else Path(lock)
        self._local_config = local_config
-        self._lock_data: TOMLDocument | None = None
+        self._lock_data: dict[str, Any] | None = None
        self._content_hash = self._get_content_hash()
    @property
-    def lock(self) -> TOMLFile:
+    def lock(self) -> Path:
        return self._lock
    @property
-    def lock_data(self) -> TOMLDocument:
+    def lock_data(self) -> dict[str, Any]:
        if self._lock_data is None:
            self._lock_data = self._get_lock_data()
@@ -79,7 +79,8 @@ class Locker:
        """
        Checks whether the lock file is still up to date with the current hash.
        """
-        lock = self._lock.read()
+        with self.lock.open("rb") as f:
+            lock = tomllib.load(f)
        metadata = lock.get("metadata", {})
        if "content-hash" in metadata:
@@ -111,7 +112,7 @@ class Locker:
            source_type = source.get("type")
            url = source.get("url")
            if source_type in ["directory", "file"]:
-                url = self._lock.path.parent.joinpath(url).resolve().as_posix()
+                url = self.lock.parent.joinpath(url).resolve().as_posix()
            name = info["name"]
            package = Package(
@@ -196,7 +197,7 @@ class Locker:
                        package.marker = parse_marker(split_dep[1].strip())
            for dep_name, constraint in info.get("dependencies", {}).items():
-                root_dir = self._lock.path.parent
+                root_dir = self.lock.parent
                if package.source_type == "directory":
                    # root dir should be the source of the package relative to the lock
                    # path
@@ -267,11 +268,8 @@ class Locker:
        return do_write
    def _write_lock_data(self, data: TOMLDocument) -> None:
-        self.lock.write(data)
+        lockfile = TOMLFile(self.lock)
+        lockfile.write(data)
-        # Checking lock file data consistency
-        if data != self.lock.read():
-            raise RuntimeError("Inconsistent lock file data.")
        self._lock_data = None
@@ -292,16 +290,17 @@ class Locker:
        return sha256(json.dumps(relevant_content, sort_keys=True).encode()).hexdigest()
-    def _get_lock_data(self) -> TOMLDocument:
+    def _get_lock_data(self) -> dict[str, Any]:
-        if not self._lock.exists():
+        if not self.lock.exists():
            raise RuntimeError("No lockfile found. Unable to read locked packages")
-        try:
+        with self.lock.open("rb") as f:
-            lock_data: TOMLDocument = self._lock.read()
+            try:
-        except TOMLKitError as e:
+                lock_data = tomllib.load(f)
-            raise RuntimeError(f"Unable to read the lock file ({e}).")
+            except tomllib.TOMLDecodeError as e:
+                raise RuntimeError(f"Unable to read the lock file ({e}).")
-        metadata = cast("Table", lock_data["metadata"])
+        metadata = lock_data["metadata"]
        lock_version = Version.parse(metadata.get("lock-version", "1.0"))
        current_version = Version.parse(self._VERSION)
        accepted_versions = parse_constraint(self._READ_VERSION_RANGE)
@@ -441,7 +440,7 @@ class Locker:
                url = Path(
                    os.path.relpath(
                        Path(url).resolve(),
-                        Path(self._lock.path.parent).resolve(),
+                        Path(self.lock.parent).resolve(),
                    )
                ).as_posix()

--- a/src/poetry/utils/_compat.py
+++ b/src/poetry/utils/_compat.py
@@ -7,6 +7,14 @@ from contextlib import suppress
 # TODO: use try/except ImportError when
 # https://github.com/python/mypy/issues/1393 is fixed
+if sys.version_info < (3, 11):
+    # compatibility for python <3.11
+    import tomli as tomllib
+else:
+    import tomllib  # nopycln: import
 if sys.version_info < (3, 10):
    # compatibility for python <3.10
    import importlib_metadata as metadata
@@ -67,4 +75,5 @@ __all__ = [
    "list_to_shell_command",
    "metadata",
    "to_str",
+    "tomllib",
 ]
--- a/tests/conftest.py
+++ b/tests/conftest.py
@@ -412,7 +412,7 @@ def project_factory(
        poetry = Factory().create_poetry(project_dir)
        locker = TestLocker(
-            poetry.locker.lock.path, locker_config or poetry.locker._local_config
+            poetry.locker.lock, locker_config or poetry.locker._local_config
        )
        locker.write()

--- a/tests/helpers.py
+++ b/tests/helpers.py
@@ -181,7 +181,7 @@ class PoetryTestApplication(Application):
        self._poetry.set_pool(poetry.pool)
        self._poetry.set_config(poetry.config)
        self._poetry.set_locker(
-            TestLocker(poetry.locker.lock.path, self._poetry.local_config)
+            TestLocker(poetry.locker.lock, self._poetry.local_config)
        )

--- a/tests/packages/test_locker.py
+++ b/tests/packages/test_locker.py
@@ -231,7 +231,9 @@ python-versions = "~2.7 || ^3.4"
 content-hash = "c3d07fca33fba542ef2b2a4d75bf5b48d892d21a830e2ad9c952ba5123a52f77"
 """  # noqa: E800
-    locker.lock.write(tomlkit.parse(content))
+    data = tomlkit.parse(content)
+    with open(locker.lock, "w", encoding="utf-8") as f:
+        f.write(data.as_string())
    packages = locker.locked_repository().packages
@@ -294,7 +296,9 @@ lock-version = "2.0"
 content-hash = "123456789"
 """  # noqa: E800
-    locker.lock.write(tomlkit.parse(content))
+    data = tomlkit.parse(content)
+    with open(locker.lock, "w", encoding="utf-8") as f:
+        f.write(data.as_string())
    repository = locker.locked_repository()
    assert len(repository.packages) == 3
@@ -359,7 +363,9 @@ lock-version = "2.0"
 content-hash = "123456789"
 """  # noqa: E800
-    locker.lock.write(tomlkit.parse(content))
+    data = tomlkit.parse(content)
+    with open(locker.lock, "w", encoding="utf-8") as f:
+        f.write(data.as_string())
    repository = locker.locked_repository()
    assert len(repository.packages) == 2
@@ -399,7 +405,9 @@ lock-version = "2.0"
 python-versions = "*"
 content-hash = "115cf985d932e9bf5f540555bbdd75decbb62cac81e399375fc19f6277f8c1d8"
 """
-    locker.lock.write(tomlkit.parse(content))
+    data = tomlkit.parse(content)
+    with open(locker.lock, "w", encoding="utf-8") as f:
+        f.write(data.as_string())
    repository = locker.locked_repository()
    assert len(repository.packages) == 1
@@ -495,7 +503,9 @@ demo = [
    {file = "demo-1.0-py3-none-any.whl", hash = "sha256"},
 ]
 """
-    locker.lock.write(tomlkit.parse(content))
+    data = tomlkit.parse(content)
+    with open(locker.lock, "w", encoding="utf-8") as f:
+        f.write(data.as_string())
    repository = locker.locked_repository()
    assert len(repository.packages) == 5
@@ -687,7 +697,9 @@ content-hash = "c3d07fca33fba542ef2b2a4d75bf5b48d892d21a830e2ad9c952ba5123a52f77
 """
    caplog.set_level(logging.WARNING, logger="poetry.packages.locker")
-    locker.lock.write(tomlkit.parse(content))
+    data = tomlkit.parse(content)
+    with open(locker.lock, "w", encoding="utf-8") as f:
+        f.write(data.as_string())
    _ = locker.lock_data
@@ -717,7 +729,9 @@ content-hash = "c3d07fca33fba542ef2b2a4d75bf5b48d892d21a830e2ad9c952ba5123a52f77
 """  # noqa: E800
    caplog.set_level(logging.WARNING, logger="poetry.packages.locker")
-    locker.lock.write(tomlkit.parse(content))
+    data = tomlkit.parse(content)
+    with open(locker.lock, "w", encoding="utf-8") as f:
+        f.write(data.as_string())
    with pytest.raises(RuntimeError, match="^The lock file is not compatible"):
        _ = locker.lock_data
@@ -775,7 +789,9 @@ content-hash = "c3d07fca33fba542ef2b2a4d75bf5b48d892d21a830e2ad9c952ba5123a52f77
 """
    caplog.set_level(logging.WARNING, logger="poetry.packages.locker")
-    locker.lock.write(tomlkit.parse(content))
+    data = tomlkit.parse(content)
+    with open(locker.lock, "w", encoding="utf-8") as f:
+        f.write(data.as_string())
    _ = locker.lock_data
@@ -970,7 +986,10 @@ python-versions = "*"
 content-hash = "115cf985d932e9bf5f540555bbdd75decbb62cac81e399375fc19f6277f8c1d8"
 """  # noqa: E800
-    locker.lock.write(tomlkit.parse(content))
+    data = tomlkit.parse(content)
+    with open(locker.lock, "w", encoding="utf-8") as f:
+        f.write(data.as_string())
    create_dependency_patch = mocker.patch(
        "poetry.factory.Factory.create_dependency", autospec=True
    )
@@ -983,7 +1002,7 @@ content-hash = "115cf985d932e9bf5f540555bbdd75decbb62cac81e399375fc19f6277f8c1d8
    root_dir = call_kwargs["root_dir"]
    assert root_dir.match("*/lib/libA")
    # relative_to raises an exception if not relative - is_relative_to comes in py3.9
-    assert root_dir.relative_to(locker.lock.path.parent.resolve()) is not None
+    assert root_dir.relative_to(locker.lock.parent.resolve()) is not None
 @pytest.mark.parametrize(
@@ -1017,7 +1036,7 @@ def test_content_hash_with_legacy_is_compatible(
        relevant_content[key] = local_config.get(key)
    locker = locker.__class__(
-        lock=locker.lock.path,
+        lock=locker.lock,
        local_config=local_config,
    )