renderer: support unicode in render_from_file.

If a file passed to render_from_file had non-ascii text then
jinja in python2 would decode as ascii, which would cause
UnicodeDecodeError.  This issue can be re-created in python2
with just:
 'can\xe2\x80\x99t'.decode()

The solution here is to explicitly pass in unicode supporting
type (py3 str, py2 unicode).  Those are six.text_type.
Then jinja does not try to decode.

The reason we hit this is that load_file calls decode_binary.
decode_binary believes it has no work to do if it got a six.string_types.
  isinstance('can\xe2\x80\x99t', six.string_types) == True
So it returns the original string which will blow up for jinja.

Our fix here then is to load the file in binary mode and explicitly
decode it to utf-8.  Then in python2 we'll have a unicode type
and in python3 we'll have a string type.

5 files changed, 53 insertions, 6 deletions

diff --git a/cloudinit/templater.py b/cloudinit/templater.py
index b3ea64e4..9a087e1c 100644
--- a/cloudinit/templater.py
+++ b/cloudinit/templater.py
@@ -121,7 +121,11 @@ def detect_template(text):
 def render_from_file(fn, params):
     if not params:
         params = {}
-    template_type, renderer, content = detect_template(util.load_file(fn))
+    # jinja in python2 uses unicode internally.  All py2 str will be decoded.
+    # If it is given a str that has non-ascii then it will raise a
+    # UnicodeDecodeError.  So we explicitly convert to unicode type here.
+    template_type, renderer, content = detect_template(
+        util.load_file(fn, decode=False).decode('utf-8'))
     LOG.debug("Rendering content of '%s' using renderer %s", fn, template_type)
     return renderer(content, params)
 
@@ -132,11 +136,15 @@ def render_to_file(fn, outfn, params, mode=0o644):
 
 
 def render_string_to_file(content, outfn, params, mode=0o644):
+    """Render string (or py2 unicode) to file.
+    Warning: py2 str with non-ascii chars will cause UnicodeDecodeError."""
     contents = render_string(content, params)
     util.write_file(outfn, contents, mode=mode)
 
 
 def render_string(content, params):
+    """Render string (or py2 unicode).
+    Warning: py2 str with non-ascii chars will cause UnicodeDecodeError."""
     if not params:
         params = {}
     template_type, renderer, content = detect_template(content)
diff --git a/templates/chrony.conf.debian.tmpl b/templates/chrony.conf.debian.tmpl
index e72bfee1..661bf04e 100644
--- a/templates/chrony.conf.debian.tmpl
+++ b/templates/chrony.conf.debian.tmpl
@@ -30,7 +30,7 @@ logdir /var/log/chrony
 maxupdateskew 100.0
 
 # This directive enables kernel synchronisation (every 11 minutes) of the
-# real-time clock. Note that it can't be used along with the 'rtcfile' directive.
+# real-time clock. Note that it can’t be used along with the 'rtcfile' directive.
 rtcsync
 
 # Step the system clock instead of slewing it if the adjustment is larger than
diff --git a/templates/chrony.conf.ubuntu.tmpl b/templates/chrony.conf.ubuntu.tmpl
index da7f16a4..50a6f518 100644
--- a/templates/chrony.conf.ubuntu.tmpl
+++ b/templates/chrony.conf.ubuntu.tmpl
@@ -34,7 +34,7 @@ logdir /var/log/chrony
 maxupdateskew 100.0
 
 # This directive enables kernel synchronisation (every 11 minutes) of the
-# real-time clock. Note that it can't be used along with the 'rtcfile' directive.
+# real-time clock. Note that it can’t be used along with the 'rtcfile' directive.
 rtcsync
 
 # Step the system clock instead of slewing it if the adjustment is larger than
diff --git a/tests/unittests/test_handler/test_handler_ntp.py b/tests/unittests/test_handler/test_handler_ntp.py
index 1b3ca570..02676aa6 100644
--- a/tests/unittests/test_handler/test_handler_ntp.py
+++ b/tests/unittests/test_handler/test_handler_ntp.py
@@ -272,12 +272,12 @@ class TestNtp(FilesystemMockingTestCase):
                     expected_servers = '\n'.join([
                         'server {0} iburst'.format(srv) for srv in servers])
                     print('distro=%s client=%s' % (distro, client))
-                    self.assertIn(expected_servers, content.decode(),
+                    self.assertIn(expected_servers, content.decode('utf-8'),
                                   ('failed to render {0} conf'
                                    ' for distro:{1}'.format(client, distro)))
                     expected_pools = '\n'.join([
                         'pool {0} iburst'.format(pool) for pool in pools])
-                    self.assertIn(expected_pools, content.decode(),
+                    self.assertIn(expected_pools, content.decode('utf-8'),
                                   ('failed to render {0} conf'
                                    ' for distro:{1}'.format(client, distro)))
                 elif client == 'systemd-timesyncd':
diff --git a/tests/unittests/test_templating.py b/tests/unittests/test_templating.py
index 53154d33..1080e135 100644
--- a/tests/unittests/test_templating.py
+++ b/tests/unittests/test_templating.py
@@ -10,6 +10,7 @@ from cloudinit.tests import helpers as test_helpers
 import textwrap
 
 from cloudinit import templater
+from cloudinit.util import load_file, write_file
 
 try:
     import Cheetah
@@ -19,7 +20,17 @@ except ImportError:
     HAS_CHEETAH = False
 
 
-class TestTemplates(test_helpers.TestCase):
+class TestTemplates(test_helpers.CiTestCase):
+    jinja_utf8 = b'It\xe2\x80\x99s not ascii, {{name}}\n'
+    jinja_utf8_rbob = b'It\xe2\x80\x99s not ascii, bob\n'.decode('utf-8')
+
+    @staticmethod
+    def add_header(renderer, data):
+        """Return text (py2 unicode/py3 str) with template header."""
+        if isinstance(data, bytes):
+            data = data.decode('utf-8')
+        return "## template: %s\n" % renderer + data
+
     def test_render_basic(self):
         in_data = textwrap.dedent("""
             ${b}
@@ -106,4 +117,32 @@ $a,$b'''
                                            'codename': codename})
         self.assertEqual(ex_data, out_data)
 
+    def test_jinja_nonascii_render_to_string(self):
+        """Test jinja render_to_string with non-ascii content."""
+        self.assertEqual(
+            templater.render_string(
+                self.add_header("jinja", self.jinja_utf8), {"name": "bob"}),
+            self.jinja_utf8_rbob)
+
+    def test_jinja_nonascii_render_to_file(self):
+        """Test jinja render_to_file of a filename with non-ascii content."""
+        tmpl_fn = self.tmp_path("j-render-to-file.template")
+        out_fn = self.tmp_path("j-render-to-file.out")
+        write_file(filename=tmpl_fn, omode="wb",
+                   content=self.add_header(
+                       "jinja", self.jinja_utf8).encode('utf-8'))
+        templater.render_to_file(tmpl_fn, out_fn, {"name": "bob"})
+        result = load_file(out_fn, decode=False).decode('utf-8')
+        self.assertEqual(result, self.jinja_utf8_rbob)
+
+    def test_jinja_nonascii_render_from_file(self):
+        """Test jinja render_from_file with non-ascii content."""
+        tmpl_fn = self.tmp_path("j-render-from-file.template")
+        write_file(tmpl_fn, omode="wb",
+                   content=self.add_header(
+                       "jinja", self.jinja_utf8).encode('utf-8'))
+        result = templater.render_from_file(tmpl_fn, {"name": "bob"})
+        self.assertEqual(result, self.jinja_utf8_rbob)
+
+
 # vi: ts=4 expandtab