fix cloud-config-archive handling

handling of cloud-config-archive input would fail in fully_decoded_payload.
part.get_charset() would return a Charset object, but
get_charset.input_codec is a string suitable for passing to decode.

This handles that correctly, and is more careful about binary data inside
input.

The test added verifies that cloud-config inside a cloud-config-archive
is handled correctly and also that binary data there is ignored without
exceptions raised.

LP: #1445143

4 files changed, 44 insertions, 5 deletions

diff --git a/cloudinit/handlers/__init__.py b/cloudinit/handlers/__init__.py
index 52defe66..53d5604a 100644
--- a/cloudinit/handlers/__init__.py
+++ b/cloudinit/handlers/__init__.py
@@ -263,7 +263,10 @@ def fixup_handler(mod, def_freq=PER_INSTANCE):
 
 
 def type_from_starts_with(payload, default=None):
-    payload_lc = payload.lower()
+    try:
+        payload_lc = util.decode_binary(payload).lower()
+    except UnicodeDecodeError:
+        return default
     payload_lc = payload_lc.lstrip()
     for text in INCLUSION_SRCH:
         if payload_lc.startswith(text):
diff --git a/cloudinit/user_data.py b/cloudinit/user_data.py
index eb3c7336..f7c5787c 100644
--- a/cloudinit/user_data.py
+++ b/cloudinit/user_data.py
@@ -49,6 +49,7 @@ INCLUDE_TYPES = ['text/x-include-url', 'text/x-include-once-url']
 ARCHIVE_TYPES = ["text/cloud-config-archive"]
 UNDEF_TYPE = "text/plain"
 ARCHIVE_UNDEF_TYPE = "text/cloud-config"
+ARCHIVE_UNDEF_BINARY_TYPE = "application/octet-stream"
 
 # This seems to hit most of the gzip possible content types.
 DECOMP_TYPES = [
@@ -265,11 +266,15 @@ class UserDataProcessor(object):
             content = ent.get('content', '')
             mtype = ent.get('type')
             if not mtype:
-                mtype = handlers.type_from_starts_with(content,
-                                                       ARCHIVE_UNDEF_TYPE)
+                default = ARCHIVE_UNDEF_TYPE
+                if isinstance(content, six.binary_type):
+                    default = ARCHIVE_UNDEF_BINARY_TYPE
+                mtype = handlers.type_from_starts_with(content, default)
 
             maintype, subtype = mtype.split('/', 1)
             if maintype == "text":
+                if isinstance(content, six.binary_type):
+                    content = content.decode()
                 msg = MIMEText(content, _subtype=subtype)
             else:
                 msg = MIMEBase(maintype, subtype)
diff --git a/cloudinit/util.py b/cloudinit/util.py
index 971c1c2d..cae57770 100644
--- a/cloudinit/util.py
+++ b/cloudinit/util.py
@@ -121,8 +121,12 @@ def fully_decoded_payload(part):
     if (six.PY3 and
             part.get_content_maintype() == 'text' and
             isinstance(cte_payload, bytes)):
-        charset = part.get_charset() or 'utf-8'
-        return cte_payload.decode(charset, errors='surrogateescape')
+        charset = part.get_charset()
+        if charset and charset.input_codec:
+            encoding = charset.input_codec
+        else:
+            encoding = 'utf-8'
+        return cte_payload.decode(encoding, errors='surrogateescape')
     return cte_payload
 
 
diff --git a/tests/unittests/test_data.py b/tests/unittests/test_data.py
index b950c9a5..1b15dafa 100644
--- a/tests/unittests/test_data.py
+++ b/tests/unittests/test_data.py
@@ -512,6 +512,33 @@ c: 4
             ci.paths.get_ipath("cloud_config"), "", 0o600)
 
 
+    def test_cloud_config_archive(self):
+        non_decodable = b'\x11\xc9\xb4gTH\xee\x12'
+        data = [{'content': '#cloud-config\npassword: gocubs\n'},
+                {'content': '#cloud-config\nlocale: chicago\n'},
+                {'content': non_decodable}]
+        message = b'#cloud-config-archive\n' + util.yaml_dumps(data).encode()
+
+        ci = stages.Init()
+        ci.datasource = FakeDataSource(message)
+
+        fs = {}
+
+        def fsstore(filename, content, mode=0o0644, omode="wb"):
+            fs[filename] = content
+
+        # consuming the user-data provided should write 'cloud_config' file
+        # which will have our yaml in it.
+        with mock.patch('cloudinit.util.write_file') as mockobj:
+            mockobj.side_effect = fsstore
+            ci.fetch()
+            ci.consume_data()
+
+        cfg = util.load_yaml(fs[ci.paths.get_ipath("cloud_config")])
+        self.assertEqual(cfg.get('password'), 'gocubs')
+        self.assertEqual(cfg.get('locale'), 'chicago')
+
+
 class TestUDProcess(helpers.ResourceUsingTestCase):
 
     def test_bytes_in_userdata(self):