Force utf-8 encoding when opening files

The default `open(..., 'r')` may lead to errors when utf-8 encoded
files are read on a platform with non-utf-8 default locale /
preferred encoding. The `io` package allows to specify `encoding`
for `open` in a Python 2 & 3-compatible way.

diff --git a/chevron/main.py b/chevron/main.py
index 649e776b217a21e44b410c3bf146458fa350b3e7..35934ee6ff4f6128e49b167b417fc318d378878c 100755 (executable)
--- a/chevron/main.py
+++ b/chevron/main.py
@@ -1,5 +1,6 @@
 #!/usr/bin/python
 
+import io
 import sys
 
 try:
@@ -16,9 +17,9 @@ except (ValueError, SystemError):  # python 2
 
 
 def main(template, data={}, **kwargs):
-    with open(template, 'r') as template_file:
+    with io.open(template, 'r', encoding='utf-8') as template_file:
         if data != {}:
-            data_file = open(data, 'r')
+            data_file = io.open(data, 'r', encoding='utf-8')
             data = json.load(data_file)
             data_file.close()
 

diff --git a/chevron/renderer.py b/chevron/renderer.py
index 3944f9971c9843916311d186f627d684bba765df..e6460949c9b5ea3997df2dc2d2678e3544ef9f84 100644 (file)
--- a/chevron/renderer.py
+++ b/chevron/renderer.py
@@ -1,5 +1,7 @@
 # -*- coding: utf-8 -*-
 
+import io
+
 try:
     from .tokenizer import tokenize
 except (ValueError, SystemError):  # python 2
@@ -91,7 +93,7 @@ def _get_partial(name, partials_dict, partials_path, partials_ext):
         try:
             # Maybe it's in the file system
             path = partials_path + '/' + name + '.' + partials_ext
-            with open(path, 'r') as partial:
+            with io.open(path, 'r', encoding='utf-8') as partial:
                 return partial.read()
 
         except IOError:

diff --git a/test_spec.py b/test_spec.py
index 67d5ff5d7bc0621f11d0d1b31e1503154cae4923..627e9112460be3b2c91eadfe90559855208d9811 100755 (executable)
--- a/test_spec.py
+++ b/test_spec.py
@@ -4,9 +4,17 @@
 import unittest
 import os
 import json
+import io
 
 import chevron
 
+import sys
+if sys.version_info[0] == 3:
+    python3 = True
+else:  # python 2
+    python3 = False
+
+
 SPECS_PATH = os.path.join('spec', 'specs')
 if os.path.exists(SPECS_PATH):
     SPECS = [path for path in os.listdir(SPECS_PATH) if path.endswith('.json')]
@@ -35,7 +43,7 @@ def _test_case_from_path(json_path):
                                                                  obj['desc'])
             return test_case
 
-        with open(json_path, 'r') as f:
+        with io.open(json_path, 'r', encoding='utf-8') as f:
             yaml = json.load(f)
 
         # Generates a unit test for each test object
@@ -124,8 +132,10 @@ class ExpandedCoverage(unittest.TestCase):
         result = chevron.main('tests/test.mustache', 'tests/data.json',
                               partials_path='tests')
 
-        with open('tests/test.rendered', 'r') as f:
+        with io.open('tests/test.rendered', 'r', encoding='utf-8') as f:
             expected = f.read()
+            if not python3:
+                expected = expected.encode('utf-8')
 
         self.assertEqual(result, expected)