Review of (explicit) encoding for files being created in text mode (#1088)

* Fix encoding for files created for building package (issue #1027)

This is minimal set of modifications, which fixed the problem.

There are still more places, where files are open in text mode
without explicit encoding, thus possibly failing on systems not using
UTF-8 console encoding.

* test_api.py use explicit encoding for files.

* get-poetry.py: fix explicit (utf-8) encoding for files open in text mode

* tests: fix explicit (utf-8) encoding for files open in text mode

* poetry init: use explicit utf-8 for created pyproject.toml

* installed: use explicit utf-8 for text files

* layouts: use explicit utf-8 for open text files

* spdx: explicit utf-8 for open text files

* pypi_repository: explicit utf-8 for open text files

* explicit utf-8 for json

* explicit utf-8 for metadata

* explicit utf-8 for requires.txt

* open files with explicit encoding in py27

* fix creation of poetry.bat on Windows

* Make get-poetry.py compatible to py2/3

* Blacked two files to pass linter test

* remove extranous encoding related comments

* rewert broken move of import related to ansible

* removed extraneous comments about opening files in UTF-8

* rewert ansible.release import (keep order of vars unchanged)

* Revert to original content (as it comes from external project)

* all vendored setup.py files in fixtures reverted back to org

get-poetry.py

@@ -33,7 +33,7 @@ from contextlib import closing
 from contextlib import contextmanager
 from contextlib import contextmanager
 from functools import cmp_to_key
 from functools import cmp_to_key
 from gzip import GzipFile
 from gzip import GzipFile
-from io import UnsupportedOperation
+from io import UnsupportedOperation, open
 
 
 try:
 try:
     from urllib.error import HTTPError
     from urllib.error import HTTPError
@@ -58,6 +58,10 @@ try:
 except ImportError:
 except ImportError:
     winreg = None
     winreg = None
 
 
+try:
+    u = unicode
+except NameError:
+    u = str
 
 
 WINDOWS = sys.platform.startswith("win") or (sys.platform == "cli" and os.name == "nt")
 WINDOWS = sys.platform.startswith("win") or (sys.platform == "cli" and os.name == "nt")
 
 
@@ -191,6 +195,7 @@ POETRY_LIB_BACKUP = os.path.join(POETRY_HOME, "lib-backup")
 
 
 
 
 BIN = """#!/usr/bin/env python
 BIN = """#!/usr/bin/env python
+# -*- coding: utf-8 -*-
 import glob
 import glob
 import sys
 import sys
 import os
 import os
@@ -205,7 +210,7 @@ if __name__ == "__main__":
     main()
     main()
 """
 """
 
 
-BAT = '@echo off\r\npython "{poetry_bin}" %*\r\n'
+BAT = u('@echo off\r\npython "{poetry_bin}" %*\r\n')
 
 
 
 
 PRE_MESSAGE = """# Welcome to {poetry}!
 PRE_MESSAGE = """# Welcome to {poetry}!
@@ -387,7 +392,9 @@ class Installer:
 
 
         current_version = None
         current_version = None
         if os.path.exists(POETRY_LIB):
         if os.path.exists(POETRY_LIB):
-            with open(os.path.join(POETRY_LIB, "poetry", "__version__.py")) as f:
+            with open(
+                os.path.join(POETRY_LIB, "poetry", "__version__.py"), encoding="utf-8"
+            ) as f:
                 version_content = f.read()
                 version_content = f.read()
 
 
             current_version_re = re.match(
             current_version_re = re.match(
@@ -563,15 +570,17 @@ class Installer:
         if WINDOWS:
         if WINDOWS:
             with open(os.path.join(POETRY_BIN, "poetry.bat"), "w") as f:
             with open(os.path.join(POETRY_BIN, "poetry.bat"), "w") as f:
                 f.write(
                 f.write(
+                    u(
                         BAT.format(
                         BAT.format(
                             poetry_bin=os.path.join(POETRY_BIN, "poetry").replace(
                             poetry_bin=os.path.join(POETRY_BIN, "poetry").replace(
                                 os.environ["USERPROFILE"], "%USERPROFILE%"
                                 os.environ["USERPROFILE"], "%USERPROFILE%"
                             )
                             )
                         )
                         )
                     )
                     )
+                )
 
 
-        with open(os.path.join(POETRY_BIN, "poetry"), "w") as f:
-            f.write(BIN)
+        with open(os.path.join(POETRY_BIN, "poetry"), "w", encoding="utf-8") as f:
+            f.write(u(BIN))
 
 
         if not WINDOWS:
         if not WINDOWS:
             # Making the file executable
             # Making the file executable
@@ -583,7 +592,7 @@ class Installer:
             return
             return
 
 
         with open(os.path.join(POETRY_HOME, "env"), "w") as f:
         with open(os.path.join(POETRY_HOME, "env"), "w") as f:
-            f.write(self.get_export_string())
+            f.write(u(self.get_export_string()))
 
 
     def update_path(self):
     def update_path(self):
         """
         """
@@ -608,7 +617,7 @@ class Installer:
 
 
             if addition not in content:
             if addition not in content:
                 with open(profile, "a") as f:
                 with open(profile, "a") as f:
-                    f.write(addition)
+                    f.write(u(addition))
 
 
                 updated.append(os.path.relpath(profile, HOME))
                 updated.append(os.path.relpath(profile, HOME))
 
 

poetry/console/commands/init.py

@@ -158,7 +158,7 @@ The <info>init</info> command creates a basic <comment>pyproject.toml</> file in
 
 
             return 1
             return 1
 
 
-        with (Path.cwd() / "pyproject.toml").open("w") as f:
+        with (Path.cwd() / "pyproject.toml").open("w", encoding="utf-8") as f:
             f.write(content)
             f.write(content)
 
 
     def _determine_requirements(
     def _determine_requirements(

poetry/installation/pip_installer.py

@@ -193,7 +193,7 @@ class PipInstaller(BaseInstaller):
             # We also need it for non-PEP-517 packages
             # We also need it for non-PEP-517 packages
             builder = SdistBuilder(Poetry.create(pyproject.parent), NullEnv(), NullIO())
             builder = SdistBuilder(Poetry.create(pyproject.parent), NullEnv(), NullIO())
 
 
-            with open(setup, "w") as f:
+            with open(setup, "w", encoding="utf-8") as f:
                 f.write(decode(builder.build_setup()))
                 f.write(decode(builder.build_setup()))
 
 
         if package.develop:
         if package.develop:

poetry/json/__init__.py

@@ -3,6 +3,7 @@ import os
 
 
 import jsonschema
 import jsonschema
 
 
+from io import open
 from typing import List
 from typing import List
 
 
 SCHEMA_DIR = os.path.join(os.path.dirname(__file__), "schemas")
 SCHEMA_DIR = os.path.join(os.path.dirname(__file__), "schemas")
@@ -19,7 +20,7 @@ def validate_object(obj, schema_name):  # type: (dict, str) -> List[str]
     if not os.path.exists(schema):
     if not os.path.exists(schema):
         raise ValueError("Schema {} does not exist.".format(schema_name))
         raise ValueError("Schema {} does not exist.".format(schema_name))
 
 
-    with open(schema) as f:
+    with open(schema, encoding="utf-8") as f:
         schema = json.loads(f.read())
         schema = json.loads(f.read())
 
 
     validator = jsonschema.Draft7Validator(schema)
     validator = jsonschema.Draft7Validator(schema)

poetry/layouts/layout.py

@@ -140,7 +140,7 @@ class Layout(object):
         tests.mkdir()
         tests.mkdir()
         tests_init.touch(exist_ok=False)
         tests_init.touch(exist_ok=False)
 
 
-        with tests_default.open("w") as f:
+        with tests_default.open("w", encoding="utf-8") as f:
             f.write(
             f.write(
                 TESTS_DEFAULT.format(
                 TESTS_DEFAULT.format(
                     package_name=self._package_name, version=self._version
                     package_name=self._package_name, version=self._version
@@ -152,5 +152,5 @@ class Layout(object):
 
 
         poetry = path / "pyproject.toml"
         poetry = path / "pyproject.toml"
 
 
-        with poetry.open("w") as f:
+        with poetry.open("w", encoding="utf-8") as f:
             f.write(content)
             f.write(content)

poetry/layouts/src.py

@@ -14,5 +14,5 @@ class SrcLayout(Layout):
 
 
         package_path.mkdir(parents=True)
         package_path.mkdir(parents=True)
 
 
-        with package_init.open("w") as f:
+        with package_init.open("w", encoding="utf-8") as f:
             f.write(DEFAULT.format(version=self._version))
             f.write(DEFAULT.format(version=self._version))

poetry/layouts/standard.py

@@ -14,5 +14,5 @@ class StandardLayout(Layout):
 
 
         package_path.mkdir()
         package_path.mkdir()
 
 
-        with package_init.open("w") as f:
+        with package_init.open("w", encoding="utf-8") as f:
             f.write(DEFAULT.format(version=self._version))
             f.write(DEFAULT.format(version=self._version))

poetry/masonry/api.py

@@ -39,13 +39,13 @@ def prepare_metadata_for_build_wheel(metadata_directory, config_settings=None):
     dist_info.mkdir()
     dist_info.mkdir()
 
 
     if "scripts" in poetry.local_config or "plugins" in poetry.local_config:
     if "scripts" in poetry.local_config or "plugins" in poetry.local_config:
-        with (dist_info / "entry_points.txt").open("w") as f:
+        with (dist_info / "entry_points.txt").open("w", encoding="utf-8") as f:
             builder._write_entry_points(f)
             builder._write_entry_points(f)
 
 
-    with (dist_info / "WHEEL").open("w") as f:
+    with (dist_info / "WHEEL").open("w", encoding="utf-8") as f:
         builder._write_wheel_file(f)
         builder._write_wheel_file(f)
 
 
-    with (dist_info / "METADATA").open("w") as f:
+    with (dist_info / "METADATA").open("w", encoding="utf-8") as f:
         builder._write_metadata_file(f)
         builder._write_metadata_file(f)
 
 
     return dist_info.name
     return dist_info.name

poetry/masonry/metadata.py

@@ -46,7 +46,7 @@ class Metadata:
         meta.version = normalize_version(package.version.text)
         meta.version = normalize_version(package.version.text)
         meta.summary = package.description
         meta.summary = package.description
         if package.readme:
         if package.readme:
-            with package.readme.open() as f:
+            with package.readme.open(encoding="utf-8") as f:
                 meta.description = f.read()
                 meta.description = f.read()
 
 
         meta.keywords = ",".join(package.keywords)
         meta.keywords = ",".join(package.keywords)

poetry/puzzle/provider.py

@@ -346,7 +346,7 @@ class Provider:
                     reqs = []
                     reqs = []
                     requires = egg_info / "requires.txt"
                     requires = egg_info / "requires.txt"
                     if requires.exists():
                     if requires.exists():
-                        with requires.open() as f:
+                        with requires.open(encoding="utf-8") as f:
                             reqs = parse_requires(f.read())
                             reqs = parse_requires(f.read())
             finally:
             finally:
                 os.chdir(current_dir)
                 os.chdir(current_dir)

poetry/repositories/pypi_repository.py

@@ -533,7 +533,7 @@ class PyPiRepository(Repository):
 
 
                 requires = egg_info / "requires.txt"
                 requires = egg_info / "requires.txt"
                 if requires.exists():
                 if requires.exists():
-                    with requires.open() as f:
+                    with requires.open(encoding="utf-8") as f:
                         info["requires_dist"] = parse_requires(f.read())
                         info["requires_dist"] = parse_requires(f.read())
 
 
                         return info
                         return info
@@ -545,7 +545,7 @@ class PyPiRepository(Repository):
 
 
                 requires = egg_info / "requires.txt"
                 requires = egg_info / "requires.txt"
                 if requires.exists():
                 if requires.exists():
-                    with requires.open() as f:
+                    with requires.open(encoding="utf-8") as f:
                         info["requires_dist"] = parse_requires(f.read())
                         info["requires_dist"] = parse_requires(f.read())
 
 
                         return info
                         return info

poetry/spdx/__init__.py

 import json
 import json
 import os
 import os
 
 
+from io import open
+
 from .license import License
 from .license import License
 from .updater import Updater
 from .updater import Updater
 
 
@@ -26,7 +28,7 @@ def load_licenses():
 
 
     licenses_file = os.path.join(os.path.dirname(__file__), "data", "licenses.json")
     licenses_file = os.path.join(os.path.dirname(__file__), "data", "licenses.json")
 
 
-    with open(licenses_file) as f:
+    with open(licenses_file, encoding="utf-8") as f:
         data = json.loads(f.read())
         data = json.loads(f.read())
 
 
     for name, license in data.items():
     for name, license in data.items():

poetry/spdx/updater.py

 import json
 import json
 import os
 import os
 
 
+from io import open
+
 try:
 try:
     from urllib.request import urlopen
     from urllib.request import urlopen
 except ImportError:
 except ImportError:
@@ -20,7 +22,7 @@ class Updater:
 
 
         licenses_url = self._base_url + "licenses.json"
         licenses_url = self._base_url + "licenses.json"
 
 
-        with open(file, "w") as f:
+        with open(file, "w", encoding="utf-8") as f:
             f.write(
             f.write(
                 json.dumps(self.get_licenses(licenses_url), indent=2, sort_keys=True)
                 json.dumps(self.get_licenses(licenses_url), indent=2, sort_keys=True)
             )
             )

tests/console/conftest.py

@@ -176,7 +176,7 @@ def repo():
 def poetry(repo):
 def poetry(repo):
     p = Poetry.create(Path(__file__).parent.parent / "fixtures" / "simple_project")
     p = Poetry.create(Path(__file__).parent.parent / "fixtures" / "simple_project")
 
 
-    with p.file.path.open() as f:
+    with p.file.path.open(encoding="utf-8") as f:
         content = f.read()
         content = f.read()
 
 
     p.pool.remove_repository("pypi")
     p.pool.remove_repository("pypi")
@@ -184,7 +184,7 @@ def poetry(repo):
 
 
     yield p
     yield p
 
 
-    with p.file.path.open("w") as f:
+    with p.file.path.open("w", encoding="utf-8") as f:
         f.write(content)
         f.write(content)
 
 
 
 

tests/masonry/test_api.py

@@ -117,11 +117,11 @@ My Package
         assert (dist_info / "WHEEL").exists()
         assert (dist_info / "WHEEL").exists()
         assert (dist_info / "METADATA").exists()
         assert (dist_info / "METADATA").exists()
 
 
-        with (dist_info / "entry_points.txt").open() as f:
+        with (dist_info / "entry_points.txt").open(encoding="utf-8") as f:
             assert entry_points == decode(f.read())
             assert entry_points == decode(f.read())
 
 
-        with (dist_info / "WHEEL").open() as f:
+        with (dist_info / "WHEEL").open(encoding="utf-8") as f:
             assert wheel_data == decode(f.read())
             assert wheel_data == decode(f.read())
 
 
-        with (dist_info / "METADATA").open() as f:
+        with (dist_info / "METADATA").open(encoding="utf-8") as f:
             assert metadata == decode(f.read())
             assert metadata == decode(f.read())

tests/repositories/test_legacy_repository.py

@@ -29,7 +29,7 @@ class MockRepository(LegacyRepository):
 
 
         fixture = self.FIXTURES / (name + ".html")
         fixture = self.FIXTURES / (name + ".html")
 
 
-        with fixture.open() as f:
+        with fixture.open(encoding="utf-8") as f:
             return Page(self._url + endpoint, f.read(), {})
             return Page(self._url + endpoint, f.read(), {})
 
 
     def _download(self, url, dest):
     def _download(self, url, dest):

tests/repositories/test_pypi_repository.py

@@ -33,7 +33,7 @@ class MockRepository(PyPiRepository):
             if not fixture.exists():
             if not fixture.exists():
                 fixture = self.JSON_FIXTURES / (name + ".json")
                 fixture = self.JSON_FIXTURES / (name + ".json")
 
 
-        with fixture.open() as f:
+        with fixture.open(encoding="utf-8") as f:
             return json.loads(f.read())
             return json.loads(f.read())
 
 
     def _download(self, url, dest):
     def _download(self, url, dest):