From 30e730f7ca111487d243ba9f40c66df6d7a49953 Mon Sep 17 00:00:00 2001 From: Scott Moser Date: Thu, 17 May 2018 14:59:54 -0600 Subject: read_file_or_url: move to url_helper, fix bug in its FileResponse. The result of a read_file_or_url on a file and on a url would differ in behavior. str(UrlResponse) would return UrlResponse.contents.decode('utf-8') while str(FileResponse) would return str(FileResponse.contents) The difference being "b'foo'" versus "foo". As part of the general goal of cleaning util, move read_file_or_url into url_helper. --- cloudinit/user_data.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) (limited to 'cloudinit/user_data.py') diff --git a/cloudinit/user_data.py b/cloudinit/user_data.py index cc55daf8..8f6aba1e 100644 --- a/cloudinit/user_data.py +++ b/cloudinit/user_data.py @@ -19,7 +19,7 @@ import six from cloudinit import handlers from cloudinit import log as logging -from cloudinit.url_helper import UrlError +from cloudinit.url_helper import read_file_or_url, UrlError from cloudinit import util LOG = logging.getLogger(__name__) @@ -224,8 +224,8 @@ class UserDataProcessor(object): content = util.load_file(include_once_fn) else: try: - resp = util.read_file_or_url(include_url, - ssl_details=self.ssl_details) + resp = read_file_or_url(include_url, + ssl_details=self.ssl_details) if include_once_on and resp.ok(): util.write_file(include_once_fn, resp.contents, mode=0o600) -- cgit v1.2.3 From faa6f07e9de4058083a5f69ed508b6e24bd53b23 Mon Sep 17 00:00:00 2001 From: Scott Moser Date: Wed, 13 Jun 2018 12:54:43 -0400 Subject: Be more safe on string/bytes when writing multipart user-data to disk. When creating the multipart mime message that is written as user-data.txt.i, cloud-init losing data on conversion to some things as a string. LP: #1768600 Author: Scott Moser Co-Authored-By: Chad Smith --- cloudinit/user_data.py | 22 +++++++++++++--------- tests/unittests/test_data.py | 11 ++++++++++- 2 files changed, 23 insertions(+), 10 deletions(-) (limited to 'cloudinit/user_data.py') diff --git a/cloudinit/user_data.py b/cloudinit/user_data.py index 8f6aba1e..ed83d2d8 100644 --- a/cloudinit/user_data.py +++ b/cloudinit/user_data.py @@ -337,8 +337,10 @@ def is_skippable(part): # Coverts a raw string into a mime message def convert_string(raw_data, content_type=NOT_MULTIPART_TYPE): + """convert a string (more likely bytes) or a message into + a mime message.""" if not raw_data: - raw_data = '' + raw_data = b'' def create_binmsg(data, content_type): maintype, subtype = content_type.split("/", 1) @@ -346,15 +348,17 @@ def convert_string(raw_data, content_type=NOT_MULTIPART_TYPE): msg.set_payload(data) return msg - try: - data = util.decode_binary(util.decomp_gzip(raw_data)) - if "mime-version:" in data[0:4096].lower(): - msg = util.message_from_string(data) - else: - msg = create_binmsg(data, content_type) - except UnicodeDecodeError: - msg = create_binmsg(raw_data, content_type) + if isinstance(raw_data, six.text_type): + bdata = raw_data.encode('utf-8') + else: + bdata = raw_data + bdata = util.decomp_gzip(bdata, decode=False) + if b"mime-version:" in bdata[0:4096].lower(): + msg = util.message_from_string(bdata.decode('utf-8')) + else: + msg = create_binmsg(bdata, content_type) return msg + # vi: ts=4 expandtab diff --git a/tests/unittests/test_data.py b/tests/unittests/test_data.py index 91d35cb8..3efe7adf 100644 --- a/tests/unittests/test_data.py +++ b/tests/unittests/test_data.py @@ -606,8 +606,10 @@ class TestUDProcess(helpers.ResourceUsingTestCase): class TestConvertString(helpers.TestCase): + def test_handles_binary_non_utf8_decodable(self): - blob = b'\x32\x99' + """Printable unicode (not utf8-decodable) is safely converted.""" + blob = b'#!/bin/bash\necho \xc3\x84\n' msg = ud.convert_string(blob) self.assertEqual(blob, msg.get_payload(decode=True)) @@ -621,6 +623,13 @@ class TestConvertString(helpers.TestCase): msg = ud.convert_string(text) self.assertEqual(text, msg.get_payload(decode=False)) + def test_handle_mime_parts(self): + """Mime parts are properly returned as a mime message.""" + message = MIMEBase("text", "plain") + message.set_payload("Just text") + msg = ud.convert_string(str(message)) + self.assertEqual("Just text", msg.get_payload(decode=False)) + class TestFetchBaseConfig(helpers.TestCase): def test_only_builtin_gets_builtin(self): -- cgit v1.2.3