userdata-handlers: python3-related fixes on do-not-process-this-part path

Cloud-init crashed when received multipart userdata object with
'application/octet-stream' part or some other 'application/*' part
except archived ones (x-gzip and friends). These parts are not
processed by cloud-init and result only in a message in the log.
We used some non-python3-friendly techniques while generating
this log message which was a reason for the crash.

2 files changed, 36 insertions, 6 deletions

diff --git a/cloudinit/handlers/__init__.py b/cloudinit/handlers/__init__.py
index 6b7abbcd..d62fcd19 100644
--- a/cloudinit/handlers/__init__.py
+++ b/cloudinit/handlers/__init__.py
@@ -163,12 +163,19 @@ def walker_handle_handler(pdata, _ctype, _filename, payload):
 
 
 def _extract_first_or_bytes(blob, size):
-    # Extract the first line upto X bytes or X bytes from more than the
-    # first line if the first line does not contain enough bytes
-    first_line = blob.split("\n", 1)[0]
-    if len(first_line) >= size:
-        start = first_line[:size]
-    else:
+    # Extract the first line or upto X symbols for text objects
+    # Extract first X bytes for binary objects
+    try:
+        if isinstance(blob, six.string_types):
+            start = blob.split("\n", 1)[0]
+        else:
+            # We want to avoid decoding the whole blob (it might be huge)
+            # By taking 4*size bytes we have a guarantee to decode size utf8 chars
+            start = blob[:4*size].decode(errors='ignore').split("\n", 1)[0]
+        if len(start) >= size:
+            start = start[:size]
+    except UnicodeDecodeError:
+        # Bytes array doesn't contain a text object -- return chunk of raw bytes
         start = blob[0:size]
     return start
 
@@ -183,6 +190,11 @@ def _escape_string(text):
         except TypeError:
             # Give up...
             pass
+    except AttributeError:
+        # We're in Python3 and received blob as text
+        # No escaping is needed because bytes are printed
+        # as 'b\xAA\xBB' automatically in Python3
+        pass
     return text
 
 
diff --git a/tests/unittests/test_data.py b/tests/unittests/test_data.py
index 8fc280e4..4f24e2dd 100644
--- a/tests/unittests/test_data.py
+++ b/tests/unittests/test_data.py
@@ -13,6 +13,7 @@ except ImportError:
 
 from six import BytesIO, StringIO
 
+from email import encoders
 from email.mime.application import MIMEApplication
 from email.mime.base import MIMEBase
 from email.mime.multipart import MIMEMultipart
@@ -492,6 +493,23 @@ c: 4
             mock.call(ci.paths.get_ipath("cloud_config"), "", 0o600),
             ])
 
+    def test_mime_application_octet_stream(self):
+        """Mime message of type application/octet-stream is ignored but shows warning."""
+        ci = stages.Init()
+        message = MIMEBase("application", "octet-stream")
+        message.set_payload(b'\xbf\xe6\xb2\xc3\xd3\xba\x13\xa4\xd8\xa1\xcc\xbf')
+        encoders.encode_base64(message)
+        ci.datasource = FakeDataSource(message.as_string().encode())
+
+        with mock.patch('cloudinit.util.write_file') as mockobj:
+            log_file = self.capture_log(logging.WARNING)
+            ci.fetch()
+            ci.consume_data()
+            self.assertIn(
+                "Unhandled unknown content-type (application/octet-stream)",
+                log_file.getvalue())
+        mockobj.assert_called_once_with(
+            ci.paths.get_ipath("cloud_config"), "", 0o600)
 
 class TestUDProcess(helpers.ResourceUsingTestCase):