Review of (explicit) encoding for files being created in text mode (#1088)

* Fix encoding for files created for building package (issue #1027)

This is minimal set of modifications, which fixed the problem.

There are still more places, where files are open in text mode
without explicit encoding, thus possibly failing on systems not using
UTF-8 console encoding.

* test_api.py use explicit encoding for files.

* get-poetry.py: fix explicit (utf-8) encoding for files open in text mode

* tests: fix explicit (utf-8) encoding for files open in text mode

* poetry init: use explicit utf-8 for created pyproject.toml

* installed: use explicit utf-8 for text files

* layouts: use explicit utf-8 for open text files

* spdx: explicit utf-8 for open text files

* pypi_repository: explicit utf-8 for open text files

* explicit utf-8 for json

* explicit utf-8 for metadata

* explicit utf-8 for requires.txt

* open files with explicit encoding in py27

* fix creation of poetry.bat on Windows

* Make get-poetry.py compatible to py2/3

* Blacked two files to pass linter test

* remove extranous encoding related comments

* rewert broken move of import related to ansible

* removed extraneous comments about opening files in UTF-8

* rewert ansible.release import (keep order of vars unchanged)

* Revert to original content (as it comes from external project)

* all vendored setup.py files in fixtures reverted back to org

get-poetry.py

@@ -33,7 +33,7 @@ from contextlib import closing
 from contextlib import contextmanager
 from functools import cmp_to_key
 from gzip import GzipFile
-from io import UnsupportedOperation
+from io import UnsupportedOperation, open
 
 try:
     from urllib.error import HTTPError
@@ -58,6 +58,10 @@ try:
 except ImportError:
     winreg = None
 
+try:
+    u = unicode
+except NameError:
+    u = str
 
 WINDOWS = sys.platform.startswith("win") or (sys.platform == "cli" and os.name == "nt")
 
@@ -191,6 +195,7 @@ POETRY_LIB_BACKUP = os.path.join(POETRY_HOME, "lib-backup")
 
 
 BIN = """#!/usr/bin/env python
+# -*- coding: utf-8 -*-
 import glob
 import sys
 import os
@@ -205,7 +210,7 @@ if __name__ == "__main__":
     main()
 """
 
-BAT = '@echo off\r\npython "{poetry_bin}" %*\r\n'
+BAT = u('@echo off\r\npython "{poetry_bin}" %*\r\n')
 
 
 PRE_MESSAGE = """# Welcome to {poetry}!
@@ -387,7 +392,9 @@ class Installer:
 
         current_version = None
         if os.path.exists(POETRY_LIB):
-            with open(os.path.join(POETRY_LIB, "poetry", "__version__.py")) as f:
+            with open(
+                os.path.join(POETRY_LIB, "poetry", "__version__.py"), encoding="utf-8"
+            ) as f:
                 version_content = f.read()
 
             current_version_re = re.match(
@@ -563,15 +570,17 @@ class Installer:
         if WINDOWS:
             with open(os.path.join(POETRY_BIN, "poetry.bat"), "w") as f:
                 f.write(
-                    BAT.format(
-                        poetry_bin=os.path.join(POETRY_BIN, "poetry").replace(
-                            os.environ["USERPROFILE"], "%USERPROFILE%"
+                    u(
+                        BAT.format(
+                            poetry_bin=os.path.join(POETRY_BIN, "poetry").replace(
+                                os.environ["USERPROFILE"], "%USERPROFILE%"
+                            )
                         )
                     )
                 )
 
-        with open(os.path.join(POETRY_BIN, "poetry"), "w") as f:
-            f.write(BIN)
+        with open(os.path.join(POETRY_BIN, "poetry"), "w", encoding="utf-8") as f:
+            f.write(u(BIN))
 
         if not WINDOWS:
             # Making the file executable
@@ -583,7 +592,7 @@ class Installer:
             return
 
         with open(os.path.join(POETRY_HOME, "env"), "w") as f:
-            f.write(self.get_export_string())
+            f.write(u(self.get_export_string()))
 
     def update_path(self):
         """
@@ -608,7 +617,7 @@ class Installer:
 
             if addition not in content:
                 with open(profile, "a") as f:
-                    f.write(addition)
+                    f.write(u(addition))
 
                 updated.append(os.path.relpath(profile, HOME))
 

poetry/console/commands/init.py

@@ -158,7 +158,7 @@ The <info>init</info> command creates a basic <comment>pyproject.toml</> file in
 
             return 1
 
-        with (Path.cwd() / "pyproject.toml").open("w") as f:
+        with (Path.cwd() / "pyproject.toml").open("w", encoding="utf-8") as f:
             f.write(content)
 
     def _determine_requirements(

poetry/installation/pip_installer.py

@@ -193,7 +193,7 @@ class PipInstaller(BaseInstaller):
             # We also need it for non-PEP-517 packages
             builder = SdistBuilder(Poetry.create(pyproject.parent), NullEnv(), NullIO())
 
-            with open(setup, "w") as f:
+            with open(setup, "w", encoding="utf-8") as f:
                 f.write(decode(builder.build_setup()))
 
         if package.develop:

poetry/json/__init__.py

@@ -3,6 +3,7 @@ import os
 
 import jsonschema
 
+from io import open
 from typing import List
 
 SCHEMA_DIR = os.path.join(os.path.dirname(__file__), "schemas")
@@ -19,7 +20,7 @@ def validate_object(obj, schema_name):  # type: (dict, str) -> List[str]
     if not os.path.exists(schema):
         raise ValueError("Schema {} does not exist.".format(schema_name))
 
-    with open(schema) as f:
+    with open(schema, encoding="utf-8") as f:
         schema = json.loads(f.read())
 
     validator = jsonschema.Draft7Validator(schema)

poetry/layouts/layout.py

@@ -140,7 +140,7 @@ class Layout(object):
         tests.mkdir()
         tests_init.touch(exist_ok=False)
 
-        with tests_default.open("w") as f:
+        with tests_default.open("w", encoding="utf-8") as f:
             f.write(
                 TESTS_DEFAULT.format(
                     package_name=self._package_name, version=self._version
@@ -152,5 +152,5 @@ class Layout(object):
 
         poetry = path / "pyproject.toml"
 
-        with poetry.open("w") as f:
+        with poetry.open("w", encoding="utf-8") as f:
             f.write(content)

poetry/layouts/src.py

@@ -14,5 +14,5 @@ class SrcLayout(Layout):
 
         package_path.mkdir(parents=True)
 
-        with package_init.open("w") as f:
+        with package_init.open("w", encoding="utf-8") as f:
             f.write(DEFAULT.format(version=self._version))

poetry/layouts/standard.py

@@ -14,5 +14,5 @@ class StandardLayout(Layout):
 
         package_path.mkdir()
 
-        with package_init.open("w") as f:
+        with package_init.open("w", encoding="utf-8") as f:
             f.write(DEFAULT.format(version=self._version))

poetry/masonry/api.py

@@ -39,13 +39,13 @@ def prepare_metadata_for_build_wheel(metadata_directory, config_settings=None):
     dist_info.mkdir()
 
     if "scripts" in poetry.local_config or "plugins" in poetry.local_config:
-        with (dist_info / "entry_points.txt").open("w") as f:
+        with (dist_info / "entry_points.txt").open("w", encoding="utf-8") as f:
             builder._write_entry_points(f)
 
-    with (dist_info / "WHEEL").open("w") as f:
+    with (dist_info / "WHEEL").open("w", encoding="utf-8") as f:
         builder._write_wheel_file(f)
 
-    with (dist_info / "METADATA").open("w") as f:
+    with (dist_info / "METADATA").open("w", encoding="utf-8") as f:
         builder._write_metadata_file(f)
 
     return dist_info.name

poetry/masonry/metadata.py

@@ -46,7 +46,7 @@ class Metadata:
         meta.version = normalize_version(package.version.text)
         meta.summary = package.description
         if package.readme:
-            with package.readme.open() as f:
+            with package.readme.open(encoding="utf-8") as f:
                 meta.description = f.read()
 
         meta.keywords = ",".join(package.keywords)

poetry/puzzle/provider.py

@@ -346,7 +346,7 @@ class Provider:
                     reqs = []
                     requires = egg_info / "requires.txt"
                     if requires.exists():
-                        with requires.open() as f:
+                        with requires.open(encoding="utf-8") as f:
                             reqs = parse_requires(f.read())
             finally:
                 os.chdir(current_dir)

poetry/repositories/pypi_repository.py

@@ -533,7 +533,7 @@ class PyPiRepository(Repository):
 
                 requires = egg_info / "requires.txt"
                 if requires.exists():
-                    with requires.open() as f:
+                    with requires.open(encoding="utf-8") as f:
                         info["requires_dist"] = parse_requires(f.read())
 
                         return info
@@ -545,7 +545,7 @@ class PyPiRepository(Repository):
 
                 requires = egg_info / "requires.txt"
                 if requires.exists():
-                    with requires.open() as f:
+                    with requires.open(encoding="utf-8") as f:
                         info["requires_dist"] = parse_requires(f.read())
 
                         return info

poetry/spdx/__init__.py

 import json
 import os
 
+from io import open
+
 from .license import License
 from .updater import Updater
 
@@ -26,7 +28,7 @@ def load_licenses():
 
     licenses_file = os.path.join(os.path.dirname(__file__), "data", "licenses.json")
 
-    with open(licenses_file) as f:
+    with open(licenses_file, encoding="utf-8") as f:
         data = json.loads(f.read())
 
     for name, license in data.items():

poetry/spdx/updater.py

 import json
 import os
 
+from io import open
+
 try:
     from urllib.request import urlopen
 except ImportError:
@@ -20,7 +22,7 @@ class Updater:
 
         licenses_url = self._base_url + "licenses.json"
 
-        with open(file, "w") as f:
+        with open(file, "w", encoding="utf-8") as f:
             f.write(
                 json.dumps(self.get_licenses(licenses_url), indent=2, sort_keys=True)
             )

tests/console/conftest.py

@@ -176,7 +176,7 @@ def repo():
 def poetry(repo):
     p = Poetry.create(Path(__file__).parent.parent / "fixtures" / "simple_project")
 
-    with p.file.path.open() as f:
+    with p.file.path.open(encoding="utf-8") as f:
         content = f.read()
 
     p.pool.remove_repository("pypi")
@@ -184,7 +184,7 @@ def poetry(repo):
 
     yield p
 
-    with p.file.path.open("w") as f:
+    with p.file.path.open("w", encoding="utf-8") as f:
         f.write(content)
 
 

tests/masonry/test_api.py

@@ -117,11 +117,11 @@ My Package
         assert (dist_info / "WHEEL").exists()
         assert (dist_info / "METADATA").exists()
 
-        with (dist_info / "entry_points.txt").open() as f:
+        with (dist_info / "entry_points.txt").open(encoding="utf-8") as f:
             assert entry_points == decode(f.read())
 
-        with (dist_info / "WHEEL").open() as f:
+        with (dist_info / "WHEEL").open(encoding="utf-8") as f:
             assert wheel_data == decode(f.read())
 
-        with (dist_info / "METADATA").open() as f:
+        with (dist_info / "METADATA").open(encoding="utf-8") as f:
             assert metadata == decode(f.read())

tests/repositories/test_legacy_repository.py

@@ -29,7 +29,7 @@ class MockRepository(LegacyRepository):
 
         fixture = self.FIXTURES / (name + ".html")
 
-        with fixture.open() as f:
+        with fixture.open(encoding="utf-8") as f:
             return Page(self._url + endpoint, f.read(), {})
 
     def _download(self, url, dest):

tests/repositories/test_pypi_repository.py

@@ -33,7 +33,7 @@ class MockRepository(PyPiRepository):
             if not fixture.exists():
                 fixture = self.JSON_FIXTURES / (name + ".json")
 
-        with fixture.open() as f:
+        with fixture.open(encoding="utf-8") as f:
             return json.loads(f.read())
 
     def _download(self, url, dest):