Commit 7ae77fd7 by Sébastien Eustace

Fix unicode characters in toml

parent 887427bb
...@@ -4,7 +4,6 @@ A converter of python values to TOML Token instances. ...@@ -4,7 +4,6 @@ A converter of python values to TOML Token instances.
""" """
import codecs import codecs
import datetime import datetime
import six
import re import re
from .. import tokens from .. import tokens
...@@ -47,8 +46,8 @@ def create_primitive_token(value, multiline_strings_allowed=True): ...@@ -47,8 +46,8 @@ def create_primitive_token(value, multiline_strings_allowed=True):
elif isinstance(value, float): elif isinstance(value, float):
return tokens.Token(tokens.TYPE_FLOAT, u'{}'.format(value)) return tokens.Token(tokens.TYPE_FLOAT, u'{}'.format(value))
elif isinstance(value, (datetime.datetime, datetime.date, datetime.time)): elif isinstance(value, (datetime.datetime, datetime.date, datetime.time)):
return tokens.Token(tokens.TYPE_DATE, strict_rfc3339.timestamp_to_rfc3339_utcoffset(ts)) return tokens.Token(tokens.TYPE_DATE, value.isoformat())
elif isinstance(value, six.string_types): elif isinstance(value, str):
return create_string_token(value, multiline_strings_allowed=multiline_strings_allowed) return create_string_token(value, multiline_strings_allowed=multiline_strings_allowed)
raise NotPrimitiveError("{} of type {}".format(value, type(value))) raise NotPrimitiveError("{} of type {}".format(value, type(value)))
...@@ -63,7 +62,7 @@ def create_string_token(text, bare_string_allowed=False, multiline_strings_allow ...@@ -63,7 +62,7 @@ def create_string_token(text, bare_string_allowed=False, multiline_strings_allow
Raises ValueError on non-string input. Raises ValueError on non-string input.
""" """
if not isinstance(text, six.string_types): if not isinstance(text, str):
raise ValueError('Given value must be a string') raise ValueError('Given value must be a string')
if text == '': if text == '':
...@@ -78,10 +77,7 @@ def create_string_token(text, bare_string_allowed=False, multiline_strings_allow ...@@ -78,10 +77,7 @@ def create_string_token(text, bare_string_allowed=False, multiline_strings_allow
def _escape_single_line_quoted_string(text): def _escape_single_line_quoted_string(text):
if six.PY2: return text.replace('"', '\\"')
return text.encode('unicode-escape').encode('string-escape').replace('"', '\\"').replace("\\'", "'")
else:
return codecs.encode(text, 'unicode-escape').decode().replace('"', '\\"')
def _create_multiline_string_token(text): def _create_multiline_string_token(text):
...@@ -125,6 +121,6 @@ def create_whitespace(source_substring): ...@@ -125,6 +121,6 @@ def create_whitespace(source_substring):
def create_multiline_string(text, maximum_line_length=120): def create_multiline_string(text, maximum_line_length=120):
def escape(t): def escape(t):
return t.replace(u'"""', six.u(r'\"\"\"')) return t.replace('"""', '\"\"\"')
source_substring = u'"""\n{}"""'.format(u'\\\n'.join(chunkate_string(escape(text), maximum_line_length))) source_substring = '"""\n{}"""'.format('\\\n'.join(chunkate_string(escape(text), maximum_line_length)))
return Token(tokens.TYPE_MULTILINE_STRING, source_substring) return Token(tokens.TYPE_MULTILINE_STRING, source_substring)
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment