renderer: support unicode in render_from_file.

If a file passed to render_from_file had non-ascii text then
jinja in python2 would decode as ascii, which would cause
UnicodeDecodeError.  This issue can be re-created in python2
with just:
 'can\xe2\x80\x99t'.decode()

The solution here is to explicitly pass in unicode supporting
type (py3 str, py2 unicode).  Those are six.text_type.
Then jinja does not try to decode.

The reason we hit this is that load_file calls decode_binary.
decode_binary believes it has no work to do if it got a six.string_types.
  isinstance('can\xe2\x80\x99t', six.string_types) == True
So it returns the original string which will blow up for jinja.

Our fix here then is to load the file in binary mode and explicitly
decode it to utf-8.  Then in python2 we'll have a unicode type
and in python3 we'll have a string type.

2 files changed, 42 insertions, 3 deletions

diff --git a/tests/unittests/test_handler/test_handler_ntp.py b/tests/unittests/test_handler/test_handler_ntp.py
index 1b3ca570..02676aa6 100644
--- a/tests/unittests/test_handler/test_handler_ntp.py
+++ b/tests/unittests/test_handler/test_handler_ntp.py
@@ -272,12 +272,12 @@ class TestNtp(FilesystemMockingTestCase):
                     expected_servers = '\n'.join([
                         'server {0} iburst'.format(srv) for srv in servers])
                     print('distro=%s client=%s' % (distro, client))
-                    self.assertIn(expected_servers, content.decode(),
+                    self.assertIn(expected_servers, content.decode('utf-8'),
                                   ('failed to render {0} conf'
                                    ' for distro:{1}'.format(client, distro)))
                     expected_pools = '\n'.join([
                         'pool {0} iburst'.format(pool) for pool in pools])
-                    self.assertIn(expected_pools, content.decode(),
+                    self.assertIn(expected_pools, content.decode('utf-8'),
                                   ('failed to render {0} conf'
                                    ' for distro:{1}'.format(client, distro)))
                 elif client == 'systemd-timesyncd':
diff --git a/tests/unittests/test_templating.py b/tests/unittests/test_templating.py
index 53154d33..1080e135 100644
--- a/tests/unittests/test_templating.py
+++ b/tests/unittests/test_templating.py
@@ -10,6 +10,7 @@ from cloudinit.tests import helpers as test_helpers
 import textwrap
 
 from cloudinit import templater
+from cloudinit.util import load_file, write_file
 
 try:
     import Cheetah
@@ -19,7 +20,17 @@ except ImportError:
     HAS_CHEETAH = False
 
 
-class TestTemplates(test_helpers.TestCase):
+class TestTemplates(test_helpers.CiTestCase):
+    jinja_utf8 = b'It\xe2\x80\x99s not ascii, {{name}}\n'
+    jinja_utf8_rbob = b'It\xe2\x80\x99s not ascii, bob\n'.decode('utf-8')
+
+    @staticmethod
+    def add_header(renderer, data):
+        """Return text (py2 unicode/py3 str) with template header."""
+        if isinstance(data, bytes):
+            data = data.decode('utf-8')
+        return "## template: %s\n" % renderer + data
+
     def test_render_basic(self):
         in_data = textwrap.dedent("""
             ${b}
@@ -106,4 +117,32 @@ $a,$b'''
                                            'codename': codename})
         self.assertEqual(ex_data, out_data)
 
+    def test_jinja_nonascii_render_to_string(self):
+        """Test jinja render_to_string with non-ascii content."""
+        self.assertEqual(
+            templater.render_string(
+                self.add_header("jinja", self.jinja_utf8), {"name": "bob"}),
+            self.jinja_utf8_rbob)
+
+    def test_jinja_nonascii_render_to_file(self):
+        """Test jinja render_to_file of a filename with non-ascii content."""
+        tmpl_fn = self.tmp_path("j-render-to-file.template")
+        out_fn = self.tmp_path("j-render-to-file.out")
+        write_file(filename=tmpl_fn, omode="wb",
+                   content=self.add_header(
+                       "jinja", self.jinja_utf8).encode('utf-8'))
+        templater.render_to_file(tmpl_fn, out_fn, {"name": "bob"})
+        result = load_file(out_fn, decode=False).decode('utf-8')
+        self.assertEqual(result, self.jinja_utf8_rbob)
+
+    def test_jinja_nonascii_render_from_file(self):
+        """Test jinja render_from_file with non-ascii content."""
+        tmpl_fn = self.tmp_path("j-render-from-file.template")
+        write_file(tmpl_fn, omode="wb",
+                   content=self.add_header(
+                       "jinja", self.jinja_utf8).encode('utf-8'))
+        result = templater.render_from_file(tmpl_fn, {"name": "bob"})
+        self.assertEqual(result, self.jinja_utf8_rbob)
+
+
 # vi: ts=4 expandtab