From f895cb12141281702b34da18f2384deb64c881e7 Mon Sep 17 00:00:00 2001 From: Barry Warsaw Date: Wed, 21 Jan 2015 17:56:53 -0500 Subject: Largely merge lp:~harlowja/cloud-init/py2-3 albeit manually because it seemed to be behind trunk. `tox -e py27` passes full test suite. Now to work on replacing mocker. --- cloudinit/user_data.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) (limited to 'cloudinit/user_data.py') diff --git a/cloudinit/user_data.py b/cloudinit/user_data.py index de6487d8..9111bd39 100644 --- a/cloudinit/user_data.py +++ b/cloudinit/user_data.py @@ -29,6 +29,8 @@ from email.mime.multipart import MIMEMultipart from email.mime.nonmultipart import MIMENonMultipart from email.mime.text import MIMEText +import six + from cloudinit import handlers from cloudinit import log as logging from cloudinit import util @@ -235,7 +237,7 @@ class UserDataProcessor(object): resp = util.read_file_or_url(include_url, ssl_details=self.ssl_details) if include_once_on and resp.ok(): - util.write_file(include_once_fn, str(resp), mode=0600) + util.write_file(include_once_fn, str(resp), mode=0o600) if resp.ok(): content = str(resp) else: @@ -256,7 +258,7 @@ class UserDataProcessor(object): # filename and type not be present # or # scalar(payload) - if isinstance(ent, (str, basestring)): + if isinstance(ent, six.string_types): ent = {'content': ent} if not isinstance(ent, (dict)): # TODO(harlowja) raise? @@ -337,7 +339,7 @@ def convert_string(raw_data, headers=None): data = util.decomp_gzip(raw_data) if "mime-version:" in data[0:4096].lower(): msg = email.message_from_string(data) - for (key, val) in headers.iteritems(): + for (key, val) in headers.items(): _replace_header(msg, key, val) else: mtype = headers.get(CONTENT_TYPE, NOT_MULTIPART_TYPE) -- cgit v1.2.3 From 841db73600e3f203243c773109d71ab88d3334bc Mon Sep 17 00:00:00 2001 From: Barry Warsaw Date: Mon, 26 Jan 2015 11:14:06 -0500 Subject: More test repairs. --- cloudinit/distros/__init__.py | 2 +- cloudinit/user_data.py | 9 +++++++ tests/unittests/helpers.py | 12 ++++++--- tests/unittests/test_builtin_handlers.py | 1 - tests/unittests/test_datasource/test_azure.py | 31 +++++++++++++--------- tests/unittests/test_datasource/test_gce.py | 2 +- tests/unittests/test_datasource/test_opennebula.py | 10 +++++-- tests/unittests/test_datasource/test_smartos.py | 12 ++++++--- tests/unittests/test_filters/test_launch_index.py | 8 +++--- tests/unittests/test_handler/test_handler_chef.py | 3 ++- .../test_handler/test_handler_seed_random.py | 11 ++++++-- tests/unittests/test_util.py | 6 ++--- 12 files changed, 73 insertions(+), 34 deletions(-) (limited to 'cloudinit/user_data.py') diff --git a/cloudinit/distros/__init__.py b/cloudinit/distros/__init__.py index 00fb95fb..ab874b45 100644 --- a/cloudinit/distros/__init__.py +++ b/cloudinit/distros/__init__.py @@ -857,7 +857,7 @@ def extract_default(users, default_name=None, default_config=None): if not tmp_users: return (default_name, default_config) else: - name = tmp_users.keys()[0] + name = list(tmp_users)[0] config = tmp_users[name] config.pop('default', None) return (name, config) diff --git a/cloudinit/user_data.py b/cloudinit/user_data.py index 9111bd39..ff21259c 100644 --- a/cloudinit/user_data.py +++ b/cloudinit/user_data.py @@ -109,6 +109,15 @@ class UserDataProcessor(object): ctype = None ctype_orig = part.get_content_type() payload = part.get_payload(decode=True) + # In Python 3, decoding the payload will ironically hand us a + # bytes object. 'decode' means to decode according to + # Content-Transfer-Encoding, not according to any charset in the + # Content-Type. So, if we end up with bytes, first try to decode + # to str via CT charset, and failing that, try utf-8 using + # surrogate escapes. + if six.PY3 and isinstance(payload, bytes): + charset = part.get_charset() or 'utf-8' + payload = payload.decode(charset, errors='surrogateescape') was_compressed = False # When the message states it is of a gzipped content type ensure diff --git a/tests/unittests/helpers.py b/tests/unittests/helpers.py index 70b8116f..4b8dcc5c 100644 --- a/tests/unittests/helpers.py +++ b/tests/unittests/helpers.py @@ -1,8 +1,11 @@ import os import sys +import shutil import tempfile import unittest +import six + try: from unittest import mock except ImportError: @@ -15,8 +18,6 @@ except ImportError: from cloudinit import helpers as ch from cloudinit import util -import shutil - # Used for detecting different python versions PY2 = False PY26 = False @@ -115,7 +116,12 @@ def retarget_many_wrapper(new_base, am, old_func): nam = len(n_args) for i in range(0, nam): path = args[i] - n_args[i] = rebase_path(path, new_base) + # patchOS() wraps various os and os.path functions, however in + # Python 3 some of these now accept file-descriptors (integers). + # That breaks rebase_path() so in lieu of a better solution, just + # don't rebase if we get a fd. + if isinstance(path, six.string_types): + n_args[i] = rebase_path(path, new_base) return old_func(*n_args, **kwds) return wrapper diff --git a/tests/unittests/test_builtin_handlers.py b/tests/unittests/test_builtin_handlers.py index 47ff6318..ad32d0b2 100644 --- a/tests/unittests/test_builtin_handlers.py +++ b/tests/unittests/test_builtin_handlers.py @@ -21,7 +21,6 @@ from cloudinit.settings import (PER_ALWAYS, PER_INSTANCE) class TestBuiltins(test_helpers.FilesystemMockingTestCase): - def test_upstart_frequency_no_out(self): c_root = tempfile.mkdtemp() self.addCleanup(shutil.rmtree, c_root) diff --git a/tests/unittests/test_datasource/test_azure.py b/tests/unittests/test_datasource/test_azure.py index 2dbcd389..1f0330b3 100644 --- a/tests/unittests/test_datasource/test_azure.py +++ b/tests/unittests/test_datasource/test_azure.py @@ -22,6 +22,13 @@ import tempfile import unittest +def b64(source): + # In Python 3, b64encode only accepts bytes and returns bytes. + if not isinstance(source, bytes): + source = source.encode('utf-8') + return base64.b64encode(source).decode('us-ascii') + + def construct_valid_ovf_env(data=None, pubkeys=None, userdata=None): if data is None: data = {'HostName': 'FOOHOST'} @@ -51,7 +58,7 @@ def construct_valid_ovf_env(data=None, pubkeys=None, userdata=None): content += "<%s%s>%s\n" % (key, attrs, val, key) if userdata: - content += "%s\n" % (base64.b64encode(userdata)) + content += "%s\n" % (b64(userdata)) if pubkeys: content += "\n" @@ -181,7 +188,7 @@ class TestAzureDataSource(unittest.TestCase): # set dscfg in via base64 encoded yaml cfg = {'agent_command': "my_command"} odata = {'HostName': "myhost", 'UserName': "myuser", - 'dscfg': {'text': base64.b64encode(yaml.dump(cfg)), + 'dscfg': {'text': b64(yaml.dump(cfg)), 'encoding': 'base64'}} data = {'ovfcontent': construct_valid_ovf_env(data=odata)} @@ -233,13 +240,13 @@ class TestAzureDataSource(unittest.TestCase): def test_userdata_found(self): mydata = "FOOBAR" - odata = {'UserData': base64.b64encode(mydata)} + odata = {'UserData': b64(mydata)} data = {'ovfcontent': construct_valid_ovf_env(data=odata)} dsrc = self._get_ds(data) ret = dsrc.get_data() self.assertTrue(ret) - self.assertEqual(dsrc.userdata_raw, mydata) + self.assertEqual(dsrc.userdata_raw, mydata.encode('utf-8')) def test_no_datasource_expected(self): # no source should be found if no seed_dir and no devs @@ -281,7 +288,7 @@ class TestAzureDataSource(unittest.TestCase): 'command': 'my-bounce-command', 'hostname_command': 'my-hostname-command'}} odata = {'HostName': "xhost", - 'dscfg': {'text': base64.b64encode(yaml.dump(cfg)), + 'dscfg': {'text': b64(yaml.dump(cfg)), 'encoding': 'base64'}} data = {'ovfcontent': construct_valid_ovf_env(data=odata)} self._get_ds(data).get_data() @@ -296,7 +303,7 @@ class TestAzureDataSource(unittest.TestCase): # config specifying set_hostname off should not bounce cfg = {'set_hostname': False} odata = {'HostName': "xhost", - 'dscfg': {'text': base64.b64encode(yaml.dump(cfg)), + 'dscfg': {'text': b64(yaml.dump(cfg)), 'encoding': 'base64'}} data = {'ovfcontent': construct_valid_ovf_env(data=odata)} self._get_ds(data).get_data() @@ -325,7 +332,7 @@ class TestAzureDataSource(unittest.TestCase): # Make sure that user can affect disk aliases dscfg = {'disk_aliases': {'ephemeral0': '/dev/sdc'}} odata = {'HostName': "myhost", 'UserName': "myuser", - 'dscfg': {'text': base64.b64encode(yaml.dump(dscfg)), + 'dscfg': {'text': b64(yaml.dump(dscfg)), 'encoding': 'base64'}} usercfg = {'disk_setup': {'/dev/sdc': {'something': '...'}, 'ephemeral0': False}} @@ -347,7 +354,7 @@ class TestAzureDataSource(unittest.TestCase): dsrc = self._get_ds(data) dsrc.get_data() - self.assertEqual(userdata, dsrc.userdata_raw) + self.assertEqual(userdata.encode('us-ascii'), dsrc.userdata_raw) def test_ovf_env_arrives_in_waagent_dir(self): xml = construct_valid_ovf_env(data={}, userdata="FOODATA") @@ -362,7 +369,7 @@ class TestAzureDataSource(unittest.TestCase): def test_existing_ovf_same(self): # waagent/SharedConfig left alone if found ovf-env.xml same as cached - odata = {'UserData': base64.b64encode("SOMEUSERDATA")} + odata = {'UserData': b64("SOMEUSERDATA")} data = {'ovfcontent': construct_valid_ovf_env(data=odata)} populate_dir(self.waagent_d, @@ -386,9 +393,9 @@ class TestAzureDataSource(unittest.TestCase): # 'get_data' should remove SharedConfig.xml in /var/lib/waagent # if ovf-env.xml differs. cached_ovfenv = construct_valid_ovf_env( - {'userdata': base64.b64encode("FOO_USERDATA")}) + {'userdata': b64("FOO_USERDATA")}) new_ovfenv = construct_valid_ovf_env( - {'userdata': base64.b64encode("NEW_USERDATA")}) + {'userdata': b64("NEW_USERDATA")}) populate_dir(self.waagent_d, {'ovf-env.xml': cached_ovfenv, @@ -398,7 +405,7 @@ class TestAzureDataSource(unittest.TestCase): dsrc = self._get_ds({'ovfcontent': new_ovfenv}) ret = dsrc.get_data() self.assertTrue(ret) - self.assertEqual(dsrc.userdata_raw, "NEW_USERDATA") + self.assertEqual(dsrc.userdata_raw, b"NEW_USERDATA") self.assertTrue(os.path.exists( os.path.join(self.waagent_d, 'otherfile'))) self.assertFalse( diff --git a/tests/unittests/test_datasource/test_gce.py b/tests/unittests/test_datasource/test_gce.py index aa60eb33..6dd4b5ed 100644 --- a/tests/unittests/test_datasource/test_gce.py +++ b/tests/unittests/test_datasource/test_gce.py @@ -45,7 +45,7 @@ GCE_META_ENCODING = { 'instance/id': '12345', 'instance/hostname': 'server.project-baz.local', 'instance/zone': 'baz/bang', - 'instance/attributes/user-data': b64encode('/bin/echo baz\n'), + 'instance/attributes/user-data': b64encode(b'/bin/echo baz\n'), 'instance/attributes/user-data-encoding': 'base64', } diff --git a/tests/unittests/test_datasource/test_opennebula.py b/tests/unittests/test_datasource/test_opennebula.py index b79237f0..1a8d2122 100644 --- a/tests/unittests/test_datasource/test_opennebula.py +++ b/tests/unittests/test_datasource/test_opennebula.py @@ -10,6 +10,12 @@ import shutil import tempfile import unittest +def b64(source): + # In Python 3, b64encode only accepts bytes and returns bytes. + if not isinstance(source, bytes): + source = source.encode('utf-8') + return b64encode(source).decode('us-ascii') + TEST_VARS = { 'VAR1': 'single', @@ -180,7 +186,7 @@ class TestOpenNebulaDataSource(unittest.TestCase): self.assertEqual(USER_DATA, results['userdata']) def test_user_data_encoding_required_for_decode(self): - b64userdata = b64encode(USER_DATA) + b64userdata = b64(USER_DATA) for k in ('USER_DATA', 'USERDATA'): my_d = os.path.join(self.tmp, k) populate_context_dir(my_d, {k: b64userdata}) @@ -192,7 +198,7 @@ class TestOpenNebulaDataSource(unittest.TestCase): def test_user_data_base64_encoding(self): for k in ('USER_DATA', 'USERDATA'): my_d = os.path.join(self.tmp, k) - populate_context_dir(my_d, {k: b64encode(USER_DATA), + populate_context_dir(my_d, {k: b64(USER_DATA), 'USERDATA_ENCODING': 'base64'}) results = ds.read_context_disk_dir(my_d) diff --git a/tests/unittests/test_datasource/test_smartos.py b/tests/unittests/test_datasource/test_smartos.py index 01b9b73e..2fb9e1b6 100644 --- a/tests/unittests/test_datasource/test_smartos.py +++ b/tests/unittests/test_datasource/test_smartos.py @@ -36,6 +36,12 @@ import tempfile import stat import uuid +def b64(source): + # In Python 3, b64encode only accepts bytes and returns bytes. + if not isinstance(source, bytes): + source = source.encode('utf-8') + return base64.b64encode(source).decode('us-ascii') + MOCK_RETURNS = { 'hostname': 'test-host', @@ -233,7 +239,7 @@ class TestSmartOSDataSource(helpers.FilesystemMockingTestCase): my_returns = MOCK_RETURNS.copy() my_returns['base64_all'] = "true" for k in ('hostname', 'cloud-init:user-data'): - my_returns[k] = base64.b64encode(my_returns[k]) + my_returns[k] = b64(my_returns[k]) dsrc = self._get_ds(mockdata=my_returns) ret = dsrc.get_data() @@ -254,7 +260,7 @@ class TestSmartOSDataSource(helpers.FilesystemMockingTestCase): my_returns['b64-cloud-init:user-data'] = "true" my_returns['b64-hostname'] = "true" for k in ('hostname', 'cloud-init:user-data'): - my_returns[k] = base64.b64encode(my_returns[k]) + my_returns[k] = b64(my_returns[k]) dsrc = self._get_ds(mockdata=my_returns) ret = dsrc.get_data() @@ -270,7 +276,7 @@ class TestSmartOSDataSource(helpers.FilesystemMockingTestCase): my_returns = MOCK_RETURNS.copy() my_returns['base64_keys'] = 'hostname,ignored' for k in ('hostname',): - my_returns[k] = base64.b64encode(my_returns[k]) + my_returns[k] = b64(my_returns[k]) dsrc = self._get_ds(mockdata=my_returns) ret = dsrc.get_data() diff --git a/tests/unittests/test_filters/test_launch_index.py b/tests/unittests/test_filters/test_launch_index.py index 2f4c2fda..95d24b9b 100644 --- a/tests/unittests/test_filters/test_launch_index.py +++ b/tests/unittests/test_filters/test_launch_index.py @@ -2,7 +2,7 @@ import copy from .. import helpers -import itertools +from six.moves import filterfalse from cloudinit.filters import launch_index from cloudinit import user_data as ud @@ -36,11 +36,9 @@ class TestLaunchFilter(helpers.ResourceUsingTestCase): return False # Do some basic payload checking msg1_msgs = [m for m in msg1.walk()] - msg1_msgs = [m for m in - itertools.ifilterfalse(ud.is_skippable, msg1_msgs)] + msg1_msgs = [m for m in filterfalse(ud.is_skippable, msg1_msgs)] msg2_msgs = [m for m in msg2.walk()] - msg2_msgs = [m for m in - itertools.ifilterfalse(ud.is_skippable, msg2_msgs)] + msg2_msgs = [m for m in filterfalse(ud.is_skippable, msg2_msgs)] for i in range(0, len(msg2_msgs)): m1_msg = msg1_msgs[i] m2_msg = msg2_msgs[i] diff --git a/tests/unittests/test_handler/test_handler_chef.py b/tests/unittests/test_handler/test_handler_chef.py index b06a160c..8ab27911 100644 --- a/tests/unittests/test_handler/test_handler_chef.py +++ b/tests/unittests/test_handler/test_handler_chef.py @@ -11,6 +11,7 @@ from cloudinit.sources import DataSourceNone from .. import helpers as t_help +import six import logging import shutil import tempfile @@ -77,7 +78,7 @@ class TestChef(t_help.FilesystemMockingTestCase): for k, v in cfg['chef'].items(): self.assertIn(v, c) for k, v in cc_chef.CHEF_RB_TPL_DEFAULTS.items(): - if isinstance(v, basestring): + if isinstance(v, six.string_types): self.assertIn(v, c) c = util.load_file(cc_chef.CHEF_FB_PATH) self.assertEqual({}, json.loads(c)) diff --git a/tests/unittests/test_handler/test_handler_seed_random.py b/tests/unittests/test_handler/test_handler_seed_random.py index 579377fb..c2da5ced 100644 --- a/tests/unittests/test_handler/test_handler_seed_random.py +++ b/tests/unittests/test_handler/test_handler_seed_random.py @@ -38,6 +38,13 @@ import logging LOG = logging.getLogger(__name__) +def b64(source): + # In Python 3, b64encode only accepts bytes and returns bytes. + if not isinstance(source, bytes): + source = source.encode('utf-8') + return base64.b64encode(source).decode('us-ascii') + + class TestRandomSeed(t_help.TestCase): def setUp(self): super(TestRandomSeed, self).setUp() @@ -134,7 +141,7 @@ class TestRandomSeed(t_help.TestCase): self.assertEquals("big-toe", contents) def test_append_random_base64(self): - data = base64.b64encode('bubbles') + data = b64('bubbles') cfg = { 'random_seed': { 'file': self._seed_file, @@ -147,7 +154,7 @@ class TestRandomSeed(t_help.TestCase): self.assertEquals("bubbles", contents) def test_append_random_b64(self): - data = base64.b64encode('kit-kat') + data = b64('kit-kat') cfg = { 'random_seed': { 'file': self._seed_file, diff --git a/tests/unittests/test_util.py b/tests/unittests/test_util.py index b1f5d62c..b0207ace 100644 --- a/tests/unittests/test_util.py +++ b/tests/unittests/test_util.py @@ -119,7 +119,7 @@ class TestWriteFile(unittest.TestCase): # Create file first with basic content with open(path, "wb") as f: - f.write("LINE1\n") + f.write(b"LINE1\n") util.write_file(path, contents, omode="a") self.assertTrue(os.path.exists(path)) @@ -194,7 +194,7 @@ class TestDeleteDirContents(unittest.TestCase): os.mkdir(os.path.join(self.tmp, "new_dir")) f_name = os.path.join(self.tmp, "new_dir", "new_file.txt") with open(f_name, "wb") as f: - f.write("DELETE ME") + f.write(b"DELETE ME") util.delete_dir_contents(self.tmp) @@ -205,7 +205,7 @@ class TestDeleteDirContents(unittest.TestCase): file_name = os.path.join(self.tmp, "new_file.txt") link_name = os.path.join(self.tmp, "new_file_link.txt") with open(file_name, "wb") as f: - f.write("DELETE ME") + f.write(b"DELETE ME") os.symlink(file_name, link_name) util.delete_dir_contents(self.tmp) -- cgit v1.2.3 From 2329d28a316e0ea6874b9457a1c04b37895adfd2 Mon Sep 17 00:00:00 2001 From: Barry Warsaw Date: Mon, 26 Jan 2015 16:18:57 -0500 Subject: Python 3 tests pass, except for skips. --- cloudinit/user_data.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) (limited to 'cloudinit/user_data.py') diff --git a/cloudinit/user_data.py b/cloudinit/user_data.py index ff21259c..3f860f3b 100644 --- a/cloudinit/user_data.py +++ b/cloudinit/user_data.py @@ -108,6 +108,7 @@ class UserDataProcessor(object): ctype = None ctype_orig = part.get_content_type() + ctype_main = part.get_content_maintype() payload = part.get_payload(decode=True) # In Python 3, decoding the payload will ironically hand us a # bytes object. 'decode' means to decode according to @@ -115,7 +116,7 @@ class UserDataProcessor(object): # Content-Type. So, if we end up with bytes, first try to decode # to str via CT charset, and failing that, try utf-8 using # surrogate escapes. - if six.PY3 and isinstance(payload, bytes): + if six.PY3 and ctype_main == 'text' and isinstance(payload, bytes): charset = part.get_charset() or 'utf-8' payload = payload.decode(charset, errors='surrogateescape') was_compressed = False @@ -131,6 +132,7 @@ class UserDataProcessor(object): ctype_orig = None was_compressed = True except util.DecompressionError as e: + import pdb; pdb.set_trace() LOG.warn("Failed decompressing payload from %s of length" " %s due to: %s", ctype_orig, len(payload), e) continue -- cgit v1.2.3 From 1e76dad45e3bce4dac5a638dda970fc02a044dbb Mon Sep 17 00:00:00 2001 From: Barry Warsaw Date: Tue, 27 Jan 2015 14:24:22 -0500 Subject: Respond to review: - Remove str() wrappers to second argument to write_files() where it is no longer necessary. Also: Fixed a couple of other octal literals which clearly weren't being tested. --- cloudinit/config/cc_chef.py | 2 +- cloudinit/config/cc_puppet.py | 2 +- cloudinit/config/cc_rightscale_userdata.py | 2 +- cloudinit/config/cc_runcmd.py | 2 +- cloudinit/config/cc_salt_minion.py | 2 +- cloudinit/distros/arch.py | 2 +- cloudinit/distros/gentoo.py | 2 +- cloudinit/distros/rhel_util.py | 2 +- cloudinit/user_data.py | 2 +- 9 files changed, 9 insertions(+), 9 deletions(-) (limited to 'cloudinit/user_data.py') diff --git a/cloudinit/config/cc_chef.py b/cloudinit/config/cc_chef.py index 584199e5..e18c5405 100644 --- a/cloudinit/config/cc_chef.py +++ b/cloudinit/config/cc_chef.py @@ -302,7 +302,7 @@ def install_chef(cloud, chef_cfg, log): with util.tempdir() as tmpd: # Use tmpdir over tmpfile to avoid 'text file busy' on execute tmpf = "%s/chef-omnibus-install" % tmpd - util.write_file(tmpf, str(content), mode=0o700) + util.write_file(tmpf, content, mode=0o700) util.subp([tmpf], capture=False) else: log.warn("Unknown chef install type '%s'", install_type) diff --git a/cloudinit/config/cc_puppet.py b/cloudinit/config/cc_puppet.py index 6f1b3c57..4501598e 100644 --- a/cloudinit/config/cc_puppet.py +++ b/cloudinit/config/cc_puppet.py @@ -91,7 +91,7 @@ def handle(name, cfg, cloud, log, _args): util.chownbyname(PUPPET_SSL_DIR, 'puppet', 'root') util.ensure_dir(PUPPET_SSL_CERT_DIR) util.chownbyname(PUPPET_SSL_CERT_DIR, 'puppet', 'root') - util.write_file(PUPPET_SSL_CERT_PATH, str(cfg)) + util.write_file(PUPPET_SSL_CERT_PATH, cfg) util.chownbyname(PUPPET_SSL_CERT_PATH, 'puppet', 'root') else: # Iterate throug the config items, we'll use ConfigParser.set diff --git a/cloudinit/config/cc_rightscale_userdata.py b/cloudinit/config/cc_rightscale_userdata.py index 7d2ec10a..1f769c0a 100644 --- a/cloudinit/config/cc_rightscale_userdata.py +++ b/cloudinit/config/cc_rightscale_userdata.py @@ -82,7 +82,7 @@ def handle(name, _cfg, cloud, log, _args): resp = uhelp.readurl(url) # Ensure its a valid http response (and something gotten) if resp.ok() and resp.contents: - util.write_file(fname, str(resp), mode=0700) + util.write_file(fname, resp, mode=0o700) wrote_fns.append(fname) except Exception as e: captured_excps.append(e) diff --git a/cloudinit/config/cc_runcmd.py b/cloudinit/config/cc_runcmd.py index 598c3a3e..66dc3363 100644 --- a/cloudinit/config/cc_runcmd.py +++ b/cloudinit/config/cc_runcmd.py @@ -33,6 +33,6 @@ def handle(name, cfg, cloud, log, _args): cmd = cfg["runcmd"] try: content = util.shellify(cmd) - util.write_file(out_fn, content, 0700) + util.write_file(out_fn, content, 0o700) except: util.logexc(log, "Failed to shellify %s into file %s", cmd, out_fn) diff --git a/cloudinit/config/cc_salt_minion.py b/cloudinit/config/cc_salt_minion.py index 53013dcb..f5786a31 100644 --- a/cloudinit/config/cc_salt_minion.py +++ b/cloudinit/config/cc_salt_minion.py @@ -47,7 +47,7 @@ def handle(name, cfg, cloud, log, _args): # ... copy the key pair if specified if 'public_key' in salt_cfg and 'private_key' in salt_cfg: pki_dir = salt_cfg.get('pki_dir', '/etc/salt/pki') - with util.umask(077): + with util.umask(0o77): util.ensure_dir(pki_dir) pub_name = os.path.join(pki_dir, 'minion.pub') pem_name = os.path.join(pki_dir, 'minion.pem') diff --git a/cloudinit/distros/arch.py b/cloudinit/distros/arch.py index e540e0bc..45fcf26f 100644 --- a/cloudinit/distros/arch.py +++ b/cloudinit/distros/arch.py @@ -129,7 +129,7 @@ class Distro(distros.Distro): if not conf: conf = HostnameConf('') conf.set_hostname(your_hostname) - util.write_file(out_fn, str(conf), 0644) + util.write_file(out_fn, conf, 0o644) def _read_system_hostname(self): sys_hostname = self._read_hostname(self.hostname_conf_fn) diff --git a/cloudinit/distros/gentoo.py b/cloudinit/distros/gentoo.py index 09dd0d73..9e80583c 100644 --- a/cloudinit/distros/gentoo.py +++ b/cloudinit/distros/gentoo.py @@ -108,7 +108,7 @@ class Distro(distros.Distro): if not conf: conf = HostnameConf('') conf.set_hostname(your_hostname) - util.write_file(out_fn, str(conf), 0644) + util.write_file(out_fn, conf, 0o644) def _read_system_hostname(self): sys_hostname = self._read_hostname(self.hostname_conf_fn) diff --git a/cloudinit/distros/rhel_util.py b/cloudinit/distros/rhel_util.py index 903d7793..84aad623 100644 --- a/cloudinit/distros/rhel_util.py +++ b/cloudinit/distros/rhel_util.py @@ -86,4 +86,4 @@ def update_resolve_conf_file(fn, dns_servers, search_servers): r_conf.add_search_domain(s) except ValueError: util.logexc(LOG, "Failed at adding search domain %s", s) - util.write_file(fn, str(r_conf), 0o644) + util.write_file(fn, r_conf, 0o644) diff --git a/cloudinit/user_data.py b/cloudinit/user_data.py index 3f860f3b..bf5642a5 100644 --- a/cloudinit/user_data.py +++ b/cloudinit/user_data.py @@ -248,7 +248,7 @@ class UserDataProcessor(object): resp = util.read_file_or_url(include_url, ssl_details=self.ssl_details) if include_once_on and resp.ok(): - util.write_file(include_once_fn, str(resp), mode=0o600) + util.write_file(include_once_fn, resp, mode=0o600) if resp.ok(): content = str(resp) else: -- cgit v1.2.3 From 96d130e7732f1242d71c65a32412ae56cb229abf Mon Sep 17 00:00:00 2001 From: Barry Warsaw Date: Tue, 27 Jan 2015 15:11:53 -0500 Subject: Respond to review: - Refactor "fully" decoding the payload of a text/* part. In Python 3, decode=True only means to decode according to Content-Transfer-Encoding, not according to any charset in the Content-Type header. So do that. --- cloudinit/handlers/__init__.py | 11 +---------- cloudinit/user_data.py | 12 +----------- cloudinit/util.py | 15 +++++++++++++++ 3 files changed, 17 insertions(+), 21 deletions(-) (limited to 'cloudinit/user_data.py') diff --git a/cloudinit/handlers/__init__.py b/cloudinit/handlers/__init__.py index cdccf122..6b7abbcd 100644 --- a/cloudinit/handlers/__init__.py +++ b/cloudinit/handlers/__init__.py @@ -233,16 +233,7 @@ def walk(msg, callback, data): headers = dict(part) LOG.debug(headers) headers['Content-Type'] = ctype - payload = part.get_payload(decode=True) - # In Python 3, decoding the payload will ironically hand us a bytes - # object. 'decode' means to decode according to - # Content-Transfer-Encoding, not according to any charset in the - # Content-Type. So, if we end up with bytes, first try to decode to - # str via CT charset, and failing that, try utf-8 using surrogate - # escapes. - if six.PY3 and isinstance(payload, bytes): - charset = part.get_charset() or 'utf-8' - payload = payload.decode(charset, errors='surrogateescape') + payload = util.fully_decoded_payload(part) callback(data, filename, payload, headers) partnum = partnum + 1 diff --git a/cloudinit/user_data.py b/cloudinit/user_data.py index bf5642a5..5fdc46f2 100644 --- a/cloudinit/user_data.py +++ b/cloudinit/user_data.py @@ -108,17 +108,7 @@ class UserDataProcessor(object): ctype = None ctype_orig = part.get_content_type() - ctype_main = part.get_content_maintype() - payload = part.get_payload(decode=True) - # In Python 3, decoding the payload will ironically hand us a - # bytes object. 'decode' means to decode according to - # Content-Transfer-Encoding, not according to any charset in the - # Content-Type. So, if we end up with bytes, first try to decode - # to str via CT charset, and failing that, try utf-8 using - # surrogate escapes. - if six.PY3 and ctype_main == 'text' and isinstance(payload, bytes): - charset = part.get_charset() or 'utf-8' - payload = payload.decode(charset, errors='surrogateescape') + payload = util.fully_decoded_payload(part) was_compressed = False # When the message states it is of a gzipped content type ensure diff --git a/cloudinit/util.py b/cloudinit/util.py index 8916cc11..3a921afe 100644 --- a/cloudinit/util.py +++ b/cloudinit/util.py @@ -110,6 +110,21 @@ def b64e(source): return b64encode(source).decode('utf-8') +def fully_decoded_payload(part): + # In Python 3, decoding the payload will ironically hand us a bytes object. + # 'decode' means to decode according to Content-Transfer-Encoding, not + # according to any charset in the Content-Type. So, if we end up with + # bytes, first try to decode to str via CT charset, and failing that, try + # utf-8 using surrogate escapes. + cte_payload = part.get_payload(decode=True) + if ( six.PY3 and + part.get_content_maintype() == 'text' and + isinstance(cte_payload, bytes)): + charset = part.get_charset() or 'utf-8' + return cte_payload.decode(charset, errors='surrogateescape') + return cte_payload + + # Path for DMI Data DMI_SYS_PATH = "/sys/class/dmi/id" -- cgit v1.2.3 From f24d43d9df2b972720a93120bd60bc8ce86dc1f6 Mon Sep 17 00:00:00 2001 From: Barry Warsaw Date: Tue, 27 Jan 2015 15:13:24 -0500 Subject: Remove debugging turd. --- cloudinit/user_data.py | 1 - 1 file changed, 1 deletion(-) (limited to 'cloudinit/user_data.py') diff --git a/cloudinit/user_data.py b/cloudinit/user_data.py index 5fdc46f2..fe343d0c 100644 --- a/cloudinit/user_data.py +++ b/cloudinit/user_data.py @@ -122,7 +122,6 @@ class UserDataProcessor(object): ctype_orig = None was_compressed = True except util.DecompressionError as e: - import pdb; pdb.set_trace() LOG.warn("Failed decompressing payload from %s of length" " %s due to: %s", ctype_orig, len(payload), e) continue -- cgit v1.2.3