From 508670bdaf9545b6bcc8e2009e8bd3f08d6f8796 Mon Sep 17 00:00:00 2001
From: Barry Warsaw <barry@python.org>
Date: Thu, 22 Jan 2015 18:38:30 -0500
Subject: More test ports from mocker to mock.

---
 tests/unittests/test_datasource/test_smartos.py | 10 +++++++---
 1 file changed, 7 insertions(+), 3 deletions(-)

(limited to 'tests/unittests/test_datasource/test_smartos.py')

diff --git a/tests/unittests/test_datasource/test_smartos.py b/tests/unittests/test_datasource/test_smartos.py
index 65675106..35d7ef5e 100644
--- a/tests/unittests/test_datasource/test_smartos.py
+++ b/tests/unittests/test_datasource/test_smartos.py
@@ -29,9 +29,12 @@ from .. import helpers
 import os
 import os.path
 import re
+import shutil
+import tempfile
 import stat
 import uuid
 
+
 MOCK_RETURNS = {
     'hostname': 'test-host',
     'root_authorized_keys': 'ssh-rsa AAAAB3Nz...aC1yc2E= keyname',
@@ -109,9 +112,10 @@ class TestSmartOSDataSource(helpers.FilesystemMockingTestCase):
     def setUp(self):
         helpers.FilesystemMockingTestCase.setUp(self)
 
-        # makeDir comes from MockerTestCase
-        self.tmp = self.makeDir()
-        self.legacy_user_d = self.makeDir()
+        self.tmp = tempfile.mkdtemp()
+        self.addCleanup(shutil.rmtree, self.tmp)
+        self.legacy_user_d = tempfile.mkdtemp()
+        self.addCleanup(shutil.rmtree, self.legacy_user_d)
 
         # If you should want to watch the logs...
         self._log = None
-- 
cgit v1.2.3


From 3b798b5d5c3caa5d0e8e534855e29010ca932aaa Mon Sep 17 00:00:00 2001
From: Barry Warsaw <barry@python.org>
Date: Thu, 22 Jan 2015 21:21:04 -0500
Subject: Low hanging Python 3 fruit.

---
 cloudinit/config/cc_ca_certs.py                          |  4 ++--
 cloudinit/config/cc_chef.py                              |  6 ++++--
 cloudinit/distros/__init__.py                            | 12 ++++++++++--
 cloudinit/distros/debian.py                              |  2 +-
 cloudinit/distros/rhel_util.py                           |  4 ++--
 cloudinit/distros/sles.py                                |  2 +-
 cloudinit/sources/DataSourceAltCloud.py                  | 12 ++++++------
 cloudinit/sources/DataSourceAzure.py                     |  4 ++--
 cloudinit/sources/DataSourceMAAS.py                      | 10 ++++++----
 cloudinit/sources/DataSourceOpenNebula.py                |  2 +-
 cloudinit/templater.py                                   |  2 +-
 cloudinit/util.py                                        |  7 +++++--
 templates/resolv.conf.tmpl                               |  2 +-
 tests/unittests/helpers.py                               |  4 ++--
 tests/unittests/test_datasource/test_configdrive.py      |  2 +-
 tests/unittests/test_datasource/test_digitalocean.py     |  7 +++----
 tests/unittests/test_datasource/test_gce.py              |  2 +-
 tests/unittests/test_datasource/test_opennebula.py       |  2 +-
 tests/unittests/test_datasource/test_smartos.py          |  4 +++-
 .../unittests/test_handler/test_handler_apt_configure.py |  2 +-
 tests/unittests/test_merging.py                          | 16 +++++++++-------
 tools/ccfg-merge-debug                                   |  4 ++--
 22 files changed, 65 insertions(+), 47 deletions(-)

(limited to 'tests/unittests/test_datasource/test_smartos.py')

diff --git a/cloudinit/config/cc_ca_certs.py b/cloudinit/config/cc_ca_certs.py
index 4f2a46a1..8248b020 100644
--- a/cloudinit/config/cc_ca_certs.py
+++ b/cloudinit/config/cc_ca_certs.py
@@ -44,7 +44,7 @@ def add_ca_certs(certs):
     if certs:
         # First ensure they are strings...
         cert_file_contents = "\n".join([str(c) for c in certs])
-        util.write_file(CA_CERT_FULL_PATH, cert_file_contents, mode=0644)
+        util.write_file(CA_CERT_FULL_PATH, cert_file_contents, mode=0o644)
 
         # Append cert filename to CA_CERT_CONFIG file.
         # We have to strip the content because blank lines in the file
@@ -63,7 +63,7 @@ def remove_default_ca_certs():
     """
     util.delete_dir_contents(CA_CERT_PATH)
     util.delete_dir_contents(CA_CERT_SYSTEM_PATH)
-    util.write_file(CA_CERT_CONFIG, "", mode=0644)
+    util.write_file(CA_CERT_CONFIG, "", mode=0o644)
     debconf_sel = "ca-certificates ca-certificates/trust_new_crts select no"
     util.subp(('debconf-set-selections', '-'), debconf_sel)
 
diff --git a/cloudinit/config/cc_chef.py b/cloudinit/config/cc_chef.py
index fc837363..584199e5 100644
--- a/cloudinit/config/cc_chef.py
+++ b/cloudinit/config/cc_chef.py
@@ -76,6 +76,8 @@ from cloudinit import templater
 from cloudinit import url_helper
 from cloudinit import util
 
+import six
+
 RUBY_VERSION_DEFAULT = "1.8"
 
 CHEF_DIRS = tuple([
@@ -261,7 +263,7 @@ def run_chef(chef_cfg, log):
         cmd_args = chef_cfg['exec_arguments']
         if isinstance(cmd_args, (list, tuple)):
             cmd.extend(cmd_args)
-        elif isinstance(cmd_args, (str, basestring)):
+        elif isinstance(cmd_args, six.string_types):
             cmd.append(cmd_args)
         else:
             log.warn("Unknown type %s provided for chef"
@@ -300,7 +302,7 @@ def install_chef(cloud, chef_cfg, log):
         with util.tempdir() as tmpd:
             # Use tmpdir over tmpfile to avoid 'text file busy' on execute
             tmpf = "%s/chef-omnibus-install" % tmpd
-            util.write_file(tmpf, str(content), mode=0700)
+            util.write_file(tmpf, str(content), mode=0o700)
             util.subp([tmpf], capture=False)
     else:
         log.warn("Unknown chef install type '%s'", install_type)
diff --git a/cloudinit/distros/__init__.py b/cloudinit/distros/__init__.py
index 4ebccdda..6b96d58c 100644
--- a/cloudinit/distros/__init__.py
+++ b/cloudinit/distros/__init__.py
@@ -25,7 +25,6 @@ import six
 from six import StringIO
 
 import abc
-import itertools
 import os
 import re
 
@@ -37,6 +36,15 @@ from cloudinit import util
 
 from cloudinit.distros.parsers import hosts
 
+try:
+    # Python 3
+    from six import filter
+except ImportError:
+    # Python 2
+    from itertools import ifilter as filter
+
+
+
 OSFAMILIES = {
     'debian': ['debian', 'ubuntu'],
     'redhat': ['fedora', 'rhel'],
@@ -853,7 +861,7 @@ def extract_default(users, default_name=None, default_config=None):
             return config['default']
 
     tmp_users = users.items()
-    tmp_users = dict(itertools.ifilter(safe_find, tmp_users))
+    tmp_users = dict(filter(safe_find, tmp_users))
     if not tmp_users:
         return (default_name, default_config)
     else:
diff --git a/cloudinit/distros/debian.py b/cloudinit/distros/debian.py
index b09eb094..6d3a82bf 100644
--- a/cloudinit/distros/debian.py
+++ b/cloudinit/distros/debian.py
@@ -97,7 +97,7 @@ class Distro(distros.Distro):
         if not conf:
             conf = HostnameConf('')
         conf.set_hostname(your_hostname)
-        util.write_file(out_fn, str(conf), 0644)
+        util.write_file(out_fn, str(conf), 0o644)
 
     def _read_system_hostname(self):
         sys_hostname = self._read_hostname(self.hostname_conf_fn)
diff --git a/cloudinit/distros/rhel_util.py b/cloudinit/distros/rhel_util.py
index 063d536e..903d7793 100644
--- a/cloudinit/distros/rhel_util.py
+++ b/cloudinit/distros/rhel_util.py
@@ -50,7 +50,7 @@ def update_sysconfig_file(fn, adjustments, allow_empty=False):
         ]
         if not exists:
             lines.insert(0, util.make_header())
-        util.write_file(fn, "\n".join(lines) + "\n", 0644)
+        util.write_file(fn, "\n".join(lines) + "\n", 0o644)
 
 
 # Helper function to read a RHEL/SUSE /etc/sysconfig/* file
@@ -86,4 +86,4 @@ def update_resolve_conf_file(fn, dns_servers, search_servers):
                 r_conf.add_search_domain(s)
             except ValueError:
                 util.logexc(LOG, "Failed at adding search domain %s", s)
-    util.write_file(fn, str(r_conf), 0644)
+    util.write_file(fn, str(r_conf), 0o644)
diff --git a/cloudinit/distros/sles.py b/cloudinit/distros/sles.py
index 0c6d1203..620c974c 100644
--- a/cloudinit/distros/sles.py
+++ b/cloudinit/distros/sles.py
@@ -113,7 +113,7 @@ class Distro(distros.Distro):
         if not conf:
             conf = HostnameConf('')
         conf.set_hostname(hostname)
-        util.write_file(out_fn, str(conf), 0644)
+        util.write_file(out_fn, str(conf), 0o644)
 
     def _read_system_hostname(self):
         host_fn = self.hostname_conf_fn
diff --git a/cloudinit/sources/DataSourceAltCloud.py b/cloudinit/sources/DataSourceAltCloud.py
index 1e913a6e..69053d0b 100644
--- a/cloudinit/sources/DataSourceAltCloud.py
+++ b/cloudinit/sources/DataSourceAltCloud.py
@@ -124,11 +124,11 @@ class DataSourceAltCloud(sources.DataSource):
         cmd = CMD_DMI_SYSTEM
         try:
             (cmd_out, _err) = util.subp(cmd)
-        except ProcessExecutionError, _err:
+        except ProcessExecutionError as _err:
             LOG.debug(('Failed command: %s\n%s') % \
                 (' '.join(cmd), _err.message))
             return 'UNKNOWN'
-        except OSError, _err:
+        except OSError as _err:
             LOG.debug(('Failed command: %s\n%s') % \
                 (' '.join(cmd), _err.message))
             return 'UNKNOWN'
@@ -211,11 +211,11 @@ class DataSourceAltCloud(sources.DataSource):
             cmd = CMD_PROBE_FLOPPY
             (cmd_out, _err) = util.subp(cmd)
             LOG.debug(('Command: %s\nOutput%s') % (' '.join(cmd), cmd_out))
-        except ProcessExecutionError, _err:
+        except ProcessExecutionError as _err:
             util.logexc(LOG, 'Failed command: %s\n%s', ' '.join(cmd),
                         _err.message)
             return False
-        except OSError, _err:
+        except OSError as _err:
             util.logexc(LOG, 'Failed command: %s\n%s', ' '.join(cmd),
                         _err.message)
             return False
@@ -228,11 +228,11 @@ class DataSourceAltCloud(sources.DataSource):
             cmd.append('--exit-if-exists=' + floppy_dev)
             (cmd_out, _err) = util.subp(cmd)
             LOG.debug(('Command: %s\nOutput%s') % (' '.join(cmd), cmd_out))
-        except ProcessExecutionError, _err:
+        except ProcessExecutionError as _err:
             util.logexc(LOG, 'Failed command: %s\n%s', ' '.join(cmd),
                         _err.message)
             return False
-        except OSError, _err:
+        except OSError as _err:
             util.logexc(LOG, 'Failed command: %s\n%s', ' '.join(cmd),
                         _err.message)
             return False
diff --git a/cloudinit/sources/DataSourceAzure.py b/cloudinit/sources/DataSourceAzure.py
index 09bc196d..29ae2c22 100644
--- a/cloudinit/sources/DataSourceAzure.py
+++ b/cloudinit/sources/DataSourceAzure.py
@@ -151,7 +151,7 @@ class DataSourceAzureNet(sources.DataSource):
 
         # walinux agent writes files world readable, but expects
         # the directory to be protected.
-        write_files(ddir, files, dirmode=0700)
+        write_files(ddir, files, dirmode=0o700)
 
         # handle the hostname 'publishing'
         try:
@@ -390,7 +390,7 @@ def write_files(datadir, files, dirmode=None):
     util.ensure_dir(datadir, dirmode)
     for (name, content) in files.items():
         util.write_file(filename=os.path.join(datadir, name),
-                        content=content, mode=0600)
+                        content=content, mode=0o600)
 
 
 def invoke_agent(cmd):
diff --git a/cloudinit/sources/DataSourceMAAS.py b/cloudinit/sources/DataSourceMAAS.py
index 9a3e30c5..8f9c81de 100644
--- a/cloudinit/sources/DataSourceMAAS.py
+++ b/cloudinit/sources/DataSourceMAAS.py
@@ -18,6 +18,8 @@
 #    You should have received a copy of the GNU General Public License
 #    along with this program.  If not, see <http://www.gnu.org/licenses/>.
 
+from __future__ import print_function
+
 from email.utils import parsedate
 import errno
 import oauth.oauth as oauth
@@ -361,7 +363,7 @@ if __name__ == "__main__":
             return (urllib2.urlopen(req).read())
 
         def printurl(url, headers_cb):
-            print "== %s ==\n%s\n" % (url, geturl(url, headers_cb))
+            print("== %s ==\n%s\n" % (url, geturl(url, headers_cb)))
 
         def crawl(url, headers_cb=None):
             if url.endswith("/"):
@@ -386,9 +388,9 @@ if __name__ == "__main__":
                                                           version=args.apiver)
             else:
                 (userdata, metadata) = read_maas_seed_url(args.url)
-            print "=== userdata ==="
-            print userdata
-            print "=== metadata ==="
+            print("=== userdata ===")
+            print(userdata)
+            print("=== metadata ===")
             pprint.pprint(metadata)
 
         elif args.subcmd == "get":
diff --git a/cloudinit/sources/DataSourceOpenNebula.py b/cloudinit/sources/DataSourceOpenNebula.py
index e2469f6e..f9dac29e 100644
--- a/cloudinit/sources/DataSourceOpenNebula.py
+++ b/cloudinit/sources/DataSourceOpenNebula.py
@@ -280,7 +280,7 @@ def parse_shell_config(content, keylist=None, bash=None, asuser=None,
 
     # allvars expands to all existing variables by using '${!x*}' notation
     # where x is lower or upper case letters or '_'
-    allvars = ["${!%s*}" % x for x in string.letters + "_"]
+    allvars = ["${!%s*}" % x for x in string.ascii_letters + "_"]
 
     keylist_in = keylist
     if keylist is None:
diff --git a/cloudinit/templater.py b/cloudinit/templater.py
index 4cd3f13d..a9231482 100644
--- a/cloudinit/templater.py
+++ b/cloudinit/templater.py
@@ -137,7 +137,7 @@ def render_from_file(fn, params):
     return renderer(content, params)
 
 
-def render_to_file(fn, outfn, params, mode=0644):
+def render_to_file(fn, outfn, params, mode=0o644):
     contents = render_from_file(fn, params)
     util.write_file(outfn, contents, mode=mode)
 
diff --git a/cloudinit/util.py b/cloudinit/util.py
index 434ba7fb..94fd5c70 100644
--- a/cloudinit/util.py
+++ b/cloudinit/util.py
@@ -142,6 +142,9 @@ class ProcessExecutionError(IOError):
             'reason': self.reason,
         }
         IOError.__init__(self, message)
+        # For backward compatibility with Python 2.
+        if not hasattr(self, 'message'):
+            self.message = message
 
 
 class SeLinuxGuard(object):
@@ -260,7 +263,7 @@ def translate_bool(val, addons=None):
 
 def rand_str(strlen=32, select_from=None):
     if not select_from:
-        select_from = string.letters + string.digits
+        select_from = string.ascii_letters + string.digits
     return "".join([random.choice(select_from) for _x in range(0, strlen)])
 
 
@@ -1127,7 +1130,7 @@ def pipe_in_out(in_fh, out_fh, chunk_size=1024, chunk_cb=None):
     bytes_piped = 0
     while True:
         data = in_fh.read(chunk_size)
-        if data == '':
+        if len(data) == 0:
             break
         else:
             out_fh.write(data)
diff --git a/templates/resolv.conf.tmpl b/templates/resolv.conf.tmpl
index 1300156c..bfae80db 100644
--- a/templates/resolv.conf.tmpl
+++ b/templates/resolv.conf.tmpl
@@ -24,7 +24,7 @@ sortlist {% for sort in sortlist %}{{sort}} {% endfor %}
 {% if options or flags %}
 
 options {% for flag in flags %}{{flag}} {% endfor %}
-{% for key, value in options.iteritems() -%}
+{% for key, value in options.items() -%}
  {{key}}:{{value}}
 {% endfor %}
 {% endif %}
diff --git a/tests/unittests/helpers.py b/tests/unittests/helpers.py
index 38a2176d..70b8116f 100644
--- a/tests/unittests/helpers.py
+++ b/tests/unittests/helpers.py
@@ -65,7 +65,7 @@ if PY26:
         def assertDictContainsSubset(self, expected, actual, msg=None):
             missing = []
             mismatched = []
-            for k, v in expected.iteritems():
+            for k, v in expected.items():
                 if k not in actual:
                     missing.append(k)
                 elif actual[k] != v:
@@ -243,7 +243,7 @@ class HttprettyTestCase(TestCase):
 def populate_dir(path, files):
     if not os.path.exists(path):
         os.makedirs(path)
-    for (name, content) in files.iteritems():
+    for (name, content) in files.items():
         with open(os.path.join(path, name), "w") as fp:
             fp.write(content)
             fp.close()
diff --git a/tests/unittests/test_datasource/test_configdrive.py b/tests/unittests/test_datasource/test_configdrive.py
index 800c5fd8..258c68e2 100644
--- a/tests/unittests/test_datasource/test_configdrive.py
+++ b/tests/unittests/test_datasource/test_configdrive.py
@@ -338,7 +338,7 @@ def populate_ds_from_read_config(cfg_ds, source, results):
 
 
 def populate_dir(seed_dir, files):
-    for (name, content) in files.iteritems():
+    for (name, content) in files.items():
         path = os.path.join(seed_dir, name)
         dirname = os.path.dirname(path)
         if not os.path.isdir(dirname):
diff --git a/tests/unittests/test_datasource/test_digitalocean.py b/tests/unittests/test_datasource/test_digitalocean.py
index d1270fc2..98f9cfac 100644
--- a/tests/unittests/test_datasource/test_digitalocean.py
+++ b/tests/unittests/test_datasource/test_digitalocean.py
@@ -18,8 +18,7 @@
 import httpretty
 import re
 
-from types import ListType
-from urlparse import urlparse
+from six.moves.urllib_parse import urlparse
 
 from cloudinit import settings
 from cloudinit import helpers
@@ -110,7 +109,7 @@ class TestDataSourceDigitalOcean(test_helpers.HttprettyTestCase):
         self.assertEqual([DO_META.get('public-keys')],
                          self.ds.get_public_ssh_keys())
 
-        self.assertIs(type(self.ds.get_public_ssh_keys()), ListType)
+        self.assertIsInstance(self.ds.get_public_ssh_keys(), list)
 
     @httpretty.activate
     def test_multiple_ssh_keys(self):
@@ -124,4 +123,4 @@ class TestDataSourceDigitalOcean(test_helpers.HttprettyTestCase):
         self.assertEqual(DO_META.get('public-keys').splitlines(),
                          self.ds.get_public_ssh_keys())
 
-        self.assertIs(type(self.ds.get_public_ssh_keys()), ListType)
+        self.assertIsInstance(self.ds.get_public_ssh_keys(), list)
diff --git a/tests/unittests/test_datasource/test_gce.py b/tests/unittests/test_datasource/test_gce.py
index 06050bb1..aa60eb33 100644
--- a/tests/unittests/test_datasource/test_gce.py
+++ b/tests/unittests/test_datasource/test_gce.py
@@ -19,7 +19,7 @@ import httpretty
 import re
 
 from base64 import b64encode, b64decode
-from urlparse import urlparse
+from six.moves.urllib_parse import urlparse
 
 from cloudinit import settings
 from cloudinit import helpers
diff --git a/tests/unittests/test_datasource/test_opennebula.py b/tests/unittests/test_datasource/test_opennebula.py
index ddf77265..b79237f0 100644
--- a/tests/unittests/test_datasource/test_opennebula.py
+++ b/tests/unittests/test_datasource/test_opennebula.py
@@ -294,7 +294,7 @@ class TestParseShellConfig(unittest.TestCase):
 
 def populate_context_dir(path, variables):
     data = "# Context variables generated by OpenNebula\n"
-    for (k, v) in variables.iteritems():
+    for (k, v) in variables.items():
         data += ("%s='%s'\n" % (k.upper(), v.replace(r"'", r"'\''")))
     populate_dir(path, {'context.sh': data})
 
diff --git a/tests/unittests/test_datasource/test_smartos.py b/tests/unittests/test_datasource/test_smartos.py
index 35d7ef5e..01b9b73e 100644
--- a/tests/unittests/test_datasource/test_smartos.py
+++ b/tests/unittests/test_datasource/test_smartos.py
@@ -22,6 +22,8 @@
 #   return responses.
 #
 
+from __future__ import print_function
+
 import base64
 from cloudinit import helpers as c_helpers
 from cloudinit.sources import DataSourceSmartOS
@@ -369,7 +371,7 @@ class TestSmartOSDataSource(helpers.FilesystemMockingTestCase):
                 permissions = oct(os.stat(name_f)[stat.ST_MODE])[-3:]
                 if re.match(r'.*\/mdata-user-data$', name_f):
                     found_new = True
-                    print name_f
+                    print(name_f)
                     self.assertEquals(permissions, '400')
 
         self.assertFalse(found_new)
diff --git a/tests/unittests/test_handler/test_handler_apt_configure.py b/tests/unittests/test_handler/test_handler_apt_configure.py
index 2c3dad72..d72fa8c7 100644
--- a/tests/unittests/test_handler/test_handler_apt_configure.py
+++ b/tests/unittests/test_handler/test_handler_apt_configure.py
@@ -62,7 +62,7 @@ class TestAptProxyConfig(unittest.TestCase):
 
         contents = str(util.read_file_or_url(self.pfile))
 
-        for ptype, pval in values.iteritems():
+        for ptype, pval in values.items():
             self.assertTrue(self._search_apt_config(contents, ptype, pval))
 
     def test_proxy_deleted(self):
diff --git a/tests/unittests/test_merging.py b/tests/unittests/test_merging.py
index 07b610f7..976d8283 100644
--- a/tests/unittests/test_merging.py
+++ b/tests/unittests/test_merging.py
@@ -11,11 +11,13 @@ import glob
 import os
 import random
 import re
+import six
 import string
 
 SOURCE_PAT = "source*.*yaml"
 EXPECTED_PAT = "expected%s.yaml"
-TYPES = [long, int, dict, str, list, tuple, None]
+TYPES = [dict, str, list, tuple, None]
+TYPES.extend(six.integer_types)
 
 
 def _old_mergedict(src, cand):
@@ -25,7 +27,7 @@ def _old_mergedict(src, cand):
     Nested dictionaries are merged recursively.
     """
     if isinstance(src, dict) and isinstance(cand, dict):
-        for (k, v) in cand.iteritems():
+        for (k, v) in cand.items():
             if k not in src:
                 src[k] = v
             else:
@@ -42,8 +44,8 @@ def _old_mergemanydict(*args):
 
 def _random_str(rand):
     base = ''
-    for _i in xrange(rand.randint(1, 2 ** 8)):
-        base += rand.choice(string.letters + string.digits)
+    for _i in range(rand.randint(1, 2 ** 8)):
+        base += rand.choice(string.ascii_letters + string.digits)
     return base
 
 
@@ -64,7 +66,7 @@ def _make_dict(current_depth, max_depth, rand):
     if t in [dict, list, tuple]:
         if t in [dict]:
             amount = rand.randint(0, 5)
-            keys = [_random_str(rand) for _i in xrange(0, amount)]
+            keys = [_random_str(rand) for _i in range(0, amount)]
             base = {}
             for k in keys:
                 try:
@@ -74,14 +76,14 @@ def _make_dict(current_depth, max_depth, rand):
         elif t in [list, tuple]:
             base = []
             amount = rand.randint(0, 5)
-            for _i in xrange(0, amount):
+            for _i in range(0, amount):
                 try:
                     base.append(_make_dict(current_depth + 1, max_depth, rand))
                 except _NoMoreException:
                     pass
             if t in [tuple]:
                 base = tuple(base)
-    elif t in [long, int]:
+    elif t in six.integer_types:
         base = rand.randint(0, 2 ** 8)
     elif t in [str]:
         base = _random_str(rand)
diff --git a/tools/ccfg-merge-debug b/tools/ccfg-merge-debug
index 85227da7..1f08e0cb 100755
--- a/tools/ccfg-merge-debug
+++ b/tools/ccfg-merge-debug
@@ -51,7 +51,7 @@ def main():
     c_handlers.register(ccph)
 
     called = []
-    for (_ctype, mod) in c_handlers.iteritems():
+    for (_ctype, mod) in c_handlers.items():
         if mod in called:
             continue
         handlers.call_begin(mod, data, frequency)
@@ -76,7 +76,7 @@ def main():
 
     # Give callbacks opportunity to finalize
     called = []
-    for (_ctype, mod) in c_handlers.iteritems():
+    for (_ctype, mod) in c_handlers.items():
         if mod in called:
             continue
         handlers.call_end(mod, data, frequency)
-- 
cgit v1.2.3


From 841db73600e3f203243c773109d71ab88d3334bc Mon Sep 17 00:00:00 2001
From: Barry Warsaw <barry@python.org>
Date: Mon, 26 Jan 2015 11:14:06 -0500
Subject: More test repairs.

---
 cloudinit/distros/__init__.py                      |  2 +-
 cloudinit/user_data.py                             |  9 +++++++
 tests/unittests/helpers.py                         | 12 ++++++---
 tests/unittests/test_builtin_handlers.py           |  1 -
 tests/unittests/test_datasource/test_azure.py      | 31 +++++++++++++---------
 tests/unittests/test_datasource/test_gce.py        |  2 +-
 tests/unittests/test_datasource/test_opennebula.py | 10 +++++--
 tests/unittests/test_datasource/test_smartos.py    | 12 ++++++---
 tests/unittests/test_filters/test_launch_index.py  |  8 +++---
 tests/unittests/test_handler/test_handler_chef.py  |  3 ++-
 .../test_handler/test_handler_seed_random.py       | 11 ++++++--
 tests/unittests/test_util.py                       |  6 ++---
 12 files changed, 73 insertions(+), 34 deletions(-)

(limited to 'tests/unittests/test_datasource/test_smartos.py')

diff --git a/cloudinit/distros/__init__.py b/cloudinit/distros/__init__.py
index 00fb95fb..ab874b45 100644
--- a/cloudinit/distros/__init__.py
+++ b/cloudinit/distros/__init__.py
@@ -857,7 +857,7 @@ def extract_default(users, default_name=None, default_config=None):
     if not tmp_users:
         return (default_name, default_config)
     else:
-        name = tmp_users.keys()[0]
+        name = list(tmp_users)[0]
         config = tmp_users[name]
         config.pop('default', None)
         return (name, config)
diff --git a/cloudinit/user_data.py b/cloudinit/user_data.py
index 9111bd39..ff21259c 100644
--- a/cloudinit/user_data.py
+++ b/cloudinit/user_data.py
@@ -109,6 +109,15 @@ class UserDataProcessor(object):
             ctype = None
             ctype_orig = part.get_content_type()
             payload = part.get_payload(decode=True)
+            # In Python 3, decoding the payload will ironically hand us a
+            # bytes object.  'decode' means to decode according to
+            # Content-Transfer-Encoding, not according to any charset in the
+            # Content-Type.  So, if we end up with bytes, first try to decode
+            # to str via CT charset, and failing that, try utf-8 using
+            # surrogate escapes.
+            if six.PY3 and isinstance(payload, bytes):
+                charset = part.get_charset() or 'utf-8'
+                payload = payload.decode(charset, errors='surrogateescape')
             was_compressed = False
 
             # When the message states it is of a gzipped content type ensure
diff --git a/tests/unittests/helpers.py b/tests/unittests/helpers.py
index 70b8116f..4b8dcc5c 100644
--- a/tests/unittests/helpers.py
+++ b/tests/unittests/helpers.py
@@ -1,8 +1,11 @@
 import os
 import sys
+import shutil
 import tempfile
 import unittest
 
+import six
+
 try:
     from unittest import mock
 except ImportError:
@@ -15,8 +18,6 @@ except ImportError:
 from cloudinit import helpers as ch
 from cloudinit import util
 
-import shutil
-
 # Used for detecting different python versions
 PY2 = False
 PY26 = False
@@ -115,7 +116,12 @@ def retarget_many_wrapper(new_base, am, old_func):
             nam = len(n_args)
         for i in range(0, nam):
             path = args[i]
-            n_args[i] = rebase_path(path, new_base)
+            # patchOS() wraps various os and os.path functions, however in
+            # Python 3 some of these now accept file-descriptors (integers).
+            # That breaks rebase_path() so in lieu of a better solution, just
+            # don't rebase if we get a fd.
+            if isinstance(path, six.string_types):
+                n_args[i] = rebase_path(path, new_base)
         return old_func(*n_args, **kwds)
     return wrapper
 
diff --git a/tests/unittests/test_builtin_handlers.py b/tests/unittests/test_builtin_handlers.py
index 47ff6318..ad32d0b2 100644
--- a/tests/unittests/test_builtin_handlers.py
+++ b/tests/unittests/test_builtin_handlers.py
@@ -21,7 +21,6 @@ from cloudinit.settings import (PER_ALWAYS, PER_INSTANCE)
 
 
 class TestBuiltins(test_helpers.FilesystemMockingTestCase):
-
     def test_upstart_frequency_no_out(self):
         c_root = tempfile.mkdtemp()
         self.addCleanup(shutil.rmtree, c_root)
diff --git a/tests/unittests/test_datasource/test_azure.py b/tests/unittests/test_datasource/test_azure.py
index 2dbcd389..1f0330b3 100644
--- a/tests/unittests/test_datasource/test_azure.py
+++ b/tests/unittests/test_datasource/test_azure.py
@@ -22,6 +22,13 @@ import tempfile
 import unittest
 
 
+def b64(source):
+    # In Python 3, b64encode only accepts bytes and returns bytes.
+    if not isinstance(source, bytes):
+        source = source.encode('utf-8')
+    return base64.b64encode(source).decode('us-ascii')
+
+
 def construct_valid_ovf_env(data=None, pubkeys=None, userdata=None):
     if data is None:
         data = {'HostName': 'FOOHOST'}
@@ -51,7 +58,7 @@ def construct_valid_ovf_env(data=None, pubkeys=None, userdata=None):
         content += "<%s%s>%s</%s>\n" % (key, attrs, val, key)
 
     if userdata:
-        content += "<UserData>%s</UserData>\n" % (base64.b64encode(userdata))
+        content += "<UserData>%s</UserData>\n" % (b64(userdata))
 
     if pubkeys:
         content += "<SSH><PublicKeys>\n"
@@ -181,7 +188,7 @@ class TestAzureDataSource(unittest.TestCase):
         # set dscfg in via base64 encoded yaml
         cfg = {'agent_command': "my_command"}
         odata = {'HostName': "myhost", 'UserName': "myuser",
-                'dscfg': {'text': base64.b64encode(yaml.dump(cfg)),
+                'dscfg': {'text': b64(yaml.dump(cfg)),
                           'encoding': 'base64'}}
         data = {'ovfcontent': construct_valid_ovf_env(data=odata)}
 
@@ -233,13 +240,13 @@ class TestAzureDataSource(unittest.TestCase):
 
     def test_userdata_found(self):
         mydata = "FOOBAR"
-        odata = {'UserData': base64.b64encode(mydata)}
+        odata = {'UserData': b64(mydata)}
         data = {'ovfcontent': construct_valid_ovf_env(data=odata)}
 
         dsrc = self._get_ds(data)
         ret = dsrc.get_data()
         self.assertTrue(ret)
-        self.assertEqual(dsrc.userdata_raw, mydata)
+        self.assertEqual(dsrc.userdata_raw, mydata.encode('utf-8'))
 
     def test_no_datasource_expected(self):
         # no source should be found if no seed_dir and no devs
@@ -281,7 +288,7 @@ class TestAzureDataSource(unittest.TestCase):
                                    'command': 'my-bounce-command',
                                    'hostname_command': 'my-hostname-command'}}
         odata = {'HostName': "xhost",
-                'dscfg': {'text': base64.b64encode(yaml.dump(cfg)),
+                'dscfg': {'text': b64(yaml.dump(cfg)),
                           'encoding': 'base64'}}
         data = {'ovfcontent': construct_valid_ovf_env(data=odata)}
         self._get_ds(data).get_data()
@@ -296,7 +303,7 @@ class TestAzureDataSource(unittest.TestCase):
         # config specifying set_hostname off should not bounce
         cfg = {'set_hostname': False}
         odata = {'HostName': "xhost",
-                'dscfg': {'text': base64.b64encode(yaml.dump(cfg)),
+                'dscfg': {'text': b64(yaml.dump(cfg)),
                           'encoding': 'base64'}}
         data = {'ovfcontent': construct_valid_ovf_env(data=odata)}
         self._get_ds(data).get_data()
@@ -325,7 +332,7 @@ class TestAzureDataSource(unittest.TestCase):
         # Make sure that user can affect disk aliases
         dscfg = {'disk_aliases': {'ephemeral0': '/dev/sdc'}}
         odata = {'HostName': "myhost", 'UserName': "myuser",
-                'dscfg': {'text': base64.b64encode(yaml.dump(dscfg)),
+                'dscfg': {'text': b64(yaml.dump(dscfg)),
                           'encoding': 'base64'}}
         usercfg = {'disk_setup': {'/dev/sdc': {'something': '...'},
                                   'ephemeral0': False}}
@@ -347,7 +354,7 @@ class TestAzureDataSource(unittest.TestCase):
         dsrc = self._get_ds(data)
         dsrc.get_data()
 
-        self.assertEqual(userdata, dsrc.userdata_raw)
+        self.assertEqual(userdata.encode('us-ascii'), dsrc.userdata_raw)
 
     def test_ovf_env_arrives_in_waagent_dir(self):
         xml = construct_valid_ovf_env(data={}, userdata="FOODATA")
@@ -362,7 +369,7 @@ class TestAzureDataSource(unittest.TestCase):
 
     def test_existing_ovf_same(self):
         # waagent/SharedConfig left alone if found ovf-env.xml same as cached
-        odata = {'UserData': base64.b64encode("SOMEUSERDATA")}
+        odata = {'UserData': b64("SOMEUSERDATA")}
         data = {'ovfcontent': construct_valid_ovf_env(data=odata)}
 
         populate_dir(self.waagent_d,
@@ -386,9 +393,9 @@ class TestAzureDataSource(unittest.TestCase):
         # 'get_data' should remove SharedConfig.xml in /var/lib/waagent
         # if ovf-env.xml differs.
         cached_ovfenv = construct_valid_ovf_env(
-            {'userdata': base64.b64encode("FOO_USERDATA")})
+            {'userdata': b64("FOO_USERDATA")})
         new_ovfenv = construct_valid_ovf_env(
-            {'userdata': base64.b64encode("NEW_USERDATA")})
+            {'userdata': b64("NEW_USERDATA")})
 
         populate_dir(self.waagent_d,
             {'ovf-env.xml': cached_ovfenv,
@@ -398,7 +405,7 @@ class TestAzureDataSource(unittest.TestCase):
         dsrc = self._get_ds({'ovfcontent': new_ovfenv})
         ret = dsrc.get_data()
         self.assertTrue(ret)
-        self.assertEqual(dsrc.userdata_raw, "NEW_USERDATA")
+        self.assertEqual(dsrc.userdata_raw, b"NEW_USERDATA")
         self.assertTrue(os.path.exists(
             os.path.join(self.waagent_d, 'otherfile')))
         self.assertFalse(
diff --git a/tests/unittests/test_datasource/test_gce.py b/tests/unittests/test_datasource/test_gce.py
index aa60eb33..6dd4b5ed 100644
--- a/tests/unittests/test_datasource/test_gce.py
+++ b/tests/unittests/test_datasource/test_gce.py
@@ -45,7 +45,7 @@ GCE_META_ENCODING = {
     'instance/id': '12345',
     'instance/hostname': 'server.project-baz.local',
     'instance/zone': 'baz/bang',
-    'instance/attributes/user-data': b64encode('/bin/echo baz\n'),
+    'instance/attributes/user-data': b64encode(b'/bin/echo baz\n'),
     'instance/attributes/user-data-encoding': 'base64',
 }
 
diff --git a/tests/unittests/test_datasource/test_opennebula.py b/tests/unittests/test_datasource/test_opennebula.py
index b79237f0..1a8d2122 100644
--- a/tests/unittests/test_datasource/test_opennebula.py
+++ b/tests/unittests/test_datasource/test_opennebula.py
@@ -10,6 +10,12 @@ import shutil
 import tempfile
 import unittest
 
+def b64(source):
+    # In Python 3, b64encode only accepts bytes and returns bytes.
+    if not isinstance(source, bytes):
+        source = source.encode('utf-8')
+    return b64encode(source).decode('us-ascii')
+
 
 TEST_VARS = {
     'VAR1': 'single',
@@ -180,7 +186,7 @@ class TestOpenNebulaDataSource(unittest.TestCase):
             self.assertEqual(USER_DATA, results['userdata'])
 
     def test_user_data_encoding_required_for_decode(self):
-        b64userdata = b64encode(USER_DATA)
+        b64userdata = b64(USER_DATA)
         for k in ('USER_DATA', 'USERDATA'):
             my_d = os.path.join(self.tmp, k)
             populate_context_dir(my_d, {k: b64userdata})
@@ -192,7 +198,7 @@ class TestOpenNebulaDataSource(unittest.TestCase):
     def test_user_data_base64_encoding(self):
         for k in ('USER_DATA', 'USERDATA'):
             my_d = os.path.join(self.tmp, k)
-            populate_context_dir(my_d, {k: b64encode(USER_DATA),
+            populate_context_dir(my_d, {k: b64(USER_DATA),
                                         'USERDATA_ENCODING': 'base64'})
             results = ds.read_context_disk_dir(my_d)
 
diff --git a/tests/unittests/test_datasource/test_smartos.py b/tests/unittests/test_datasource/test_smartos.py
index 01b9b73e..2fb9e1b6 100644
--- a/tests/unittests/test_datasource/test_smartos.py
+++ b/tests/unittests/test_datasource/test_smartos.py
@@ -36,6 +36,12 @@ import tempfile
 import stat
 import uuid
 
+def b64(source):
+    # In Python 3, b64encode only accepts bytes and returns bytes.
+    if not isinstance(source, bytes):
+        source = source.encode('utf-8')
+    return base64.b64encode(source).decode('us-ascii')
+
 
 MOCK_RETURNS = {
     'hostname': 'test-host',
@@ -233,7 +239,7 @@ class TestSmartOSDataSource(helpers.FilesystemMockingTestCase):
         my_returns = MOCK_RETURNS.copy()
         my_returns['base64_all'] = "true"
         for k in ('hostname', 'cloud-init:user-data'):
-            my_returns[k] = base64.b64encode(my_returns[k])
+            my_returns[k] = b64(my_returns[k])
 
         dsrc = self._get_ds(mockdata=my_returns)
         ret = dsrc.get_data()
@@ -254,7 +260,7 @@ class TestSmartOSDataSource(helpers.FilesystemMockingTestCase):
         my_returns['b64-cloud-init:user-data'] = "true"
         my_returns['b64-hostname'] = "true"
         for k in ('hostname', 'cloud-init:user-data'):
-            my_returns[k] = base64.b64encode(my_returns[k])
+            my_returns[k] = b64(my_returns[k])
 
         dsrc = self._get_ds(mockdata=my_returns)
         ret = dsrc.get_data()
@@ -270,7 +276,7 @@ class TestSmartOSDataSource(helpers.FilesystemMockingTestCase):
         my_returns = MOCK_RETURNS.copy()
         my_returns['base64_keys'] = 'hostname,ignored'
         for k in ('hostname',):
-            my_returns[k] = base64.b64encode(my_returns[k])
+            my_returns[k] = b64(my_returns[k])
 
         dsrc = self._get_ds(mockdata=my_returns)
         ret = dsrc.get_data()
diff --git a/tests/unittests/test_filters/test_launch_index.py b/tests/unittests/test_filters/test_launch_index.py
index 2f4c2fda..95d24b9b 100644
--- a/tests/unittests/test_filters/test_launch_index.py
+++ b/tests/unittests/test_filters/test_launch_index.py
@@ -2,7 +2,7 @@ import copy
 
 from .. import helpers
 
-import itertools
+from six.moves import filterfalse
 
 from cloudinit.filters import launch_index
 from cloudinit import user_data as ud
@@ -36,11 +36,9 @@ class TestLaunchFilter(helpers.ResourceUsingTestCase):
             return False
         # Do some basic payload checking
         msg1_msgs = [m for m in msg1.walk()]
-        msg1_msgs = [m for m in
-                     itertools.ifilterfalse(ud.is_skippable, msg1_msgs)]
+        msg1_msgs = [m for m in filterfalse(ud.is_skippable, msg1_msgs)]
         msg2_msgs = [m for m in msg2.walk()]
-        msg2_msgs = [m for m in
-                     itertools.ifilterfalse(ud.is_skippable, msg2_msgs)]
+        msg2_msgs = [m for m in filterfalse(ud.is_skippable, msg2_msgs)]
         for i in range(0, len(msg2_msgs)):
             m1_msg = msg1_msgs[i]
             m2_msg = msg2_msgs[i]
diff --git a/tests/unittests/test_handler/test_handler_chef.py b/tests/unittests/test_handler/test_handler_chef.py
index b06a160c..8ab27911 100644
--- a/tests/unittests/test_handler/test_handler_chef.py
+++ b/tests/unittests/test_handler/test_handler_chef.py
@@ -11,6 +11,7 @@ from cloudinit.sources import DataSourceNone
 
 from .. import helpers as t_help
 
+import six
 import logging
 import shutil
 import tempfile
@@ -77,7 +78,7 @@ class TestChef(t_help.FilesystemMockingTestCase):
         for k, v in cfg['chef'].items():
             self.assertIn(v, c)
         for k, v in cc_chef.CHEF_RB_TPL_DEFAULTS.items():
-            if isinstance(v, basestring):
+            if isinstance(v, six.string_types):
                 self.assertIn(v, c)
         c = util.load_file(cc_chef.CHEF_FB_PATH)
         self.assertEqual({}, json.loads(c))
diff --git a/tests/unittests/test_handler/test_handler_seed_random.py b/tests/unittests/test_handler/test_handler_seed_random.py
index 579377fb..c2da5ced 100644
--- a/tests/unittests/test_handler/test_handler_seed_random.py
+++ b/tests/unittests/test_handler/test_handler_seed_random.py
@@ -38,6 +38,13 @@ import logging
 LOG = logging.getLogger(__name__)
 
 
+def b64(source):
+    # In Python 3, b64encode only accepts bytes and returns bytes.
+    if not isinstance(source, bytes):
+        source = source.encode('utf-8')
+    return base64.b64encode(source).decode('us-ascii')
+
+
 class TestRandomSeed(t_help.TestCase):
     def setUp(self):
         super(TestRandomSeed, self).setUp()
@@ -134,7 +141,7 @@ class TestRandomSeed(t_help.TestCase):
         self.assertEquals("big-toe", contents)
 
     def test_append_random_base64(self):
-        data = base64.b64encode('bubbles')
+        data = b64('bubbles')
         cfg = {
             'random_seed': {
                 'file': self._seed_file,
@@ -147,7 +154,7 @@ class TestRandomSeed(t_help.TestCase):
         self.assertEquals("bubbles", contents)
 
     def test_append_random_b64(self):
-        data = base64.b64encode('kit-kat')
+        data = b64('kit-kat')
         cfg = {
             'random_seed': {
                 'file': self._seed_file,
diff --git a/tests/unittests/test_util.py b/tests/unittests/test_util.py
index b1f5d62c..b0207ace 100644
--- a/tests/unittests/test_util.py
+++ b/tests/unittests/test_util.py
@@ -119,7 +119,7 @@ class TestWriteFile(unittest.TestCase):
 
         # Create file first with basic content
         with open(path, "wb") as f:
-            f.write("LINE1\n")
+            f.write(b"LINE1\n")
         util.write_file(path, contents, omode="a")
 
         self.assertTrue(os.path.exists(path))
@@ -194,7 +194,7 @@ class TestDeleteDirContents(unittest.TestCase):
         os.mkdir(os.path.join(self.tmp, "new_dir"))
         f_name = os.path.join(self.tmp, "new_dir", "new_file.txt")
         with open(f_name, "wb") as f:
-            f.write("DELETE ME")
+            f.write(b"DELETE ME")
 
         util.delete_dir_contents(self.tmp)
 
@@ -205,7 +205,7 @@ class TestDeleteDirContents(unittest.TestCase):
         file_name = os.path.join(self.tmp, "new_file.txt")
         link_name = os.path.join(self.tmp, "new_file_link.txt")
         with open(file_name, "wb") as f:
-            f.write("DELETE ME")
+            f.write(b"DELETE ME")
         os.symlink(file_name, link_name)
 
         util.delete_dir_contents(self.tmp)
-- 
cgit v1.2.3


From e2cfd12dc79a6350d9f1f3111a15bdb6666ccad7 Mon Sep 17 00:00:00 2001
From: Barry Warsaw <barry@python.org>
Date: Mon, 26 Jan 2015 20:05:48 -0500
Subject: super() works in all of Python 2.6, 2.7, and 3.4.

---
 tests/unittests/helpers.py                      | 6 +++---
 tests/unittests/test_data.py                    | 2 +-
 tests/unittests/test_datasource/test_smartos.py | 2 +-
 tests/unittests/test_pathprefix2dict.py         | 2 +-
 4 files changed, 6 insertions(+), 6 deletions(-)

(limited to 'tests/unittests/test_datasource/test_smartos.py')

diff --git a/tests/unittests/helpers.py b/tests/unittests/helpers.py
index 424d0626..f92e7ac2 100644
--- a/tests/unittests/helpers.py
+++ b/tests/unittests/helpers.py
@@ -43,7 +43,7 @@ if PY26:
     # all this once Python 2.6 is dropped as a minimum requirement.
     class TestCase(unittest.TestCase):
         def setUp(self):
-            unittest.TestCase.setUp(self)
+            super(TestCase, self).setUp()
             self.__all_cleanups = ExitStack()
 
         def tearDown(self):
@@ -147,7 +147,7 @@ def retarget_many_wrapper(new_base, am, old_func):
 
 class ResourceUsingTestCase(TestCase):
     def setUp(self):
-        TestCase.setUp(self)
+        super(ResourceUsingTestCase, self).setUp()
         self.resource_path = None
 
     def resourceLocation(self, subname=None):
@@ -186,7 +186,7 @@ class ResourceUsingTestCase(TestCase):
 
 class FilesystemMockingTestCase(ResourceUsingTestCase):
     def setUp(self):
-        ResourceUsingTestCase.setUp(self)
+        super(FilesystemMockingTestCase, self).setUp()
         self.patched_funcs = ExitStack()
 
     def tearDown(self):
diff --git a/tests/unittests/test_data.py b/tests/unittests/test_data.py
index 7598837e..e900faa8 100644
--- a/tests/unittests/test_data.py
+++ b/tests/unittests/test_data.py
@@ -44,7 +44,7 @@ class FakeDataSource(sources.DataSource):
 class TestConsumeUserData(helpers.FilesystemMockingTestCase):
 
     def setUp(self):
-        helpers.FilesystemMockingTestCase.setUp(self)
+        super(TestConsumeUserData, self).setUp()
         self._log = None
         self._log_file = None
         self._log_handler = None
diff --git a/tests/unittests/test_datasource/test_smartos.py b/tests/unittests/test_datasource/test_smartos.py
index 2fb9e1b6..b5ebf94d 100644
--- a/tests/unittests/test_datasource/test_smartos.py
+++ b/tests/unittests/test_datasource/test_smartos.py
@@ -118,7 +118,7 @@ class MockSerial(object):
 
 class TestSmartOSDataSource(helpers.FilesystemMockingTestCase):
     def setUp(self):
-        helpers.FilesystemMockingTestCase.setUp(self)
+        super(TestSmartOSDataSource, self).setUp()
 
         self.tmp = tempfile.mkdtemp()
         self.addCleanup(shutil.rmtree, self.tmp)
diff --git a/tests/unittests/test_pathprefix2dict.py b/tests/unittests/test_pathprefix2dict.py
index d38260e6..7089bde6 100644
--- a/tests/unittests/test_pathprefix2dict.py
+++ b/tests/unittests/test_pathprefix2dict.py
@@ -9,7 +9,7 @@ import tempfile
 class TestPathPrefix2Dict(TestCase):
 
     def setUp(self):
-        TestCase.setUp(self)
+        super(TestPathPrefix2Dict, self).setUp()
         self.tmp = tempfile.mkdtemp()
         self.addCleanup(shutil.rmtree, self.tmp)
 
-- 
cgit v1.2.3


From 6e742d20e9ed56498925c7c850cd5da65d063b4b Mon Sep 17 00:00:00 2001
From: Barry Warsaw <barry@python.org>
Date: Tue, 27 Jan 2015 15:03:52 -0500
Subject: Respond to review:

- Refactor both the base64 encoding and decoding into utility functions.

Also:

- Mechanically fix some other broken untested code.
---
 cloudinit/config/cc_seed_random.py                 |  8 +------
 cloudinit/config/cc_ssh_authkey_fingerprints.py    |  2 +-
 cloudinit/sources/DataSourceOpenNebula.py          |  7 +-----
 cloudinit/sources/DataSourceSmartOS.py             | 11 +--------
 cloudinit/util.py                                  | 20 ++++++++++++++++
 tests/unittests/test_datasource/test_azure.py      | 28 ++++++++--------------
 tests/unittests/test_datasource/test_opennebula.py | 11 ++-------
 tests/unittests/test_datasource/test_smartos.py    | 14 ++++-------
 .../test_handler/test_handler_seed_random.py       | 12 ++--------
 9 files changed, 42 insertions(+), 71 deletions(-)

(limited to 'tests/unittests/test_datasource/test_smartos.py')

diff --git a/cloudinit/config/cc_seed_random.py b/cloudinit/config/cc_seed_random.py
index 981e1b08..bb64b0f5 100644
--- a/cloudinit/config/cc_seed_random.py
+++ b/cloudinit/config/cc_seed_random.py
@@ -38,13 +38,7 @@ def _decode(data, encoding=None):
     if not encoding or encoding.lower() in ['raw']:
         return data
     elif encoding.lower() in ['base64', 'b64']:
-        # Try to give us a native string in both Python 2 and 3, and remember
-        # that b64decode() returns bytes in Python 3.
-        decoded = base64.b64decode(data)
-        try:
-            return decoded.decode('utf-8')
-        except UnicodeDecodeError:
-            return decoded
+        return util.b64d(data)
     elif encoding.lower() in ['gzip', 'gz']:
         return util.decomp_gzip(data, quiet=False)
     else:
diff --git a/cloudinit/config/cc_ssh_authkey_fingerprints.py b/cloudinit/config/cc_ssh_authkey_fingerprints.py
index 51580633..6ce831bc 100644
--- a/cloudinit/config/cc_ssh_authkey_fingerprints.py
+++ b/cloudinit/config/cc_ssh_authkey_fingerprints.py
@@ -32,7 +32,7 @@ from cloudinit import util
 
 def _split_hash(bin_hash):
     split_up = []
-    for i in xrange(0, len(bin_hash), 2):
+    for i in range(0, len(bin_hash), 2):
         split_up.append(bin_hash[i:i + 2])
     return split_up
 
diff --git a/cloudinit/sources/DataSourceOpenNebula.py b/cloudinit/sources/DataSourceOpenNebula.py
index a0275cda..61709c1b 100644
--- a/cloudinit/sources/DataSourceOpenNebula.py
+++ b/cloudinit/sources/DataSourceOpenNebula.py
@@ -426,12 +426,7 @@ def read_context_disk_dir(source_dir, asuser=None):
                                context.get('USER_DATA_ENCODING'))
         if encoding == "base64":
             try:
-                userdata = base64.b64decode(results['userdata'])
-                # In Python 3 we still expect a str, but b64decode will return
-                # bytes.  Convert to str.
-                if isinstance(userdata, bytes):
-                    userdata = userdata.decode('utf-8')
-                results['userdata'] = userdata
+                results['userdata'] = util.b64d(results['userdata'])
             except TypeError:
                 LOG.warn("Failed base64 decoding of userdata")
 
diff --git a/cloudinit/sources/DataSourceSmartOS.py b/cloudinit/sources/DataSourceSmartOS.py
index f59ad3d6..9d48beab 100644
--- a/cloudinit/sources/DataSourceSmartOS.py
+++ b/cloudinit/sources/DataSourceSmartOS.py
@@ -351,16 +351,7 @@ def query_data(noun, seed_device, seed_timeout, strip=False, default=None,
 
     if b64:
         try:
-            # Generally, we want native strings in the values.  Python 3's
-            # b64decode will return bytes though, so decode them to utf-8 if
-            # possible.  If that fails, return the bytes.
-            decoded = base64.b64decode(resp)
-            try:
-                if isinstance(decoded, bytes):
-                    return decoded.decode('utf-8')
-            except UnicodeDecodeError:
-                pass
-            return decoded
+            return util.b64d(resp)
         # Bogus input produces different errors in Python 2 and 3; catch both.
         except (TypeError, binascii.Error):
             LOG.warn("Failed base64 decoding key '%s'", noun)
diff --git a/cloudinit/util.py b/cloudinit/util.py
index 766f8e32..8916cc11 100644
--- a/cloudinit/util.py
+++ b/cloudinit/util.py
@@ -44,6 +44,7 @@ import sys
 import tempfile
 import time
 
+from base64 import b64decode, b64encode
 from six.moves.urllib import parse as urlparse
 
 import six
@@ -90,6 +91,25 @@ def encode_text(text, encoding='utf-8'):
         return text
     return text.encode(encoding)
 
+
+def b64d(source):
+    # Base64 decode some data, accepting bytes or unicode/str, and returning
+    # str/unicode if the result is utf-8 compatible, otherwise returning bytes.
+    decoded = b64decode(source)
+    if isinstance(decoded, bytes):
+        try:
+            return decoded.decode('utf-8')
+        except UnicodeDecodeError:
+            return decoded
+
+def b64e(source):
+    # Base64 encode some data, accepting bytes or unicode/str, and returning
+    # str/unicode if the result is utf-8 compatible, otherwise returning bytes.
+    if not isinstance(source, bytes):
+        source = source.encode('utf-8')
+    return b64encode(source).decode('utf-8')
+
+
 # Path for DMI Data
 DMI_SYS_PATH = "/sys/class/dmi/id"
 
diff --git a/tests/unittests/test_datasource/test_azure.py b/tests/unittests/test_datasource/test_azure.py
index 97a53bee..965bce4b 100644
--- a/tests/unittests/test_datasource/test_azure.py
+++ b/tests/unittests/test_datasource/test_azure.py
@@ -1,5 +1,5 @@
 from cloudinit import helpers
-from cloudinit.util import load_file
+from cloudinit.util import b64e, load_file
 from cloudinit.sources import DataSourceAzure
 from ..helpers import TestCase, populate_dir
 
@@ -12,7 +12,6 @@ try:
 except ImportError:
     from contextlib2 import ExitStack
 
-import base64
 import crypt
 import os
 import stat
@@ -22,13 +21,6 @@ import tempfile
 import unittest
 
 
-def b64(source):
-    # In Python 3, b64encode only accepts bytes and returns bytes.
-    if not isinstance(source, bytes):
-        source = source.encode('utf-8')
-    return base64.b64encode(source).decode('us-ascii')
-
-
 def construct_valid_ovf_env(data=None, pubkeys=None, userdata=None):
     if data is None:
         data = {'HostName': 'FOOHOST'}
@@ -58,7 +50,7 @@ def construct_valid_ovf_env(data=None, pubkeys=None, userdata=None):
         content += "<%s%s>%s</%s>\n" % (key, attrs, val, key)
 
     if userdata:
-        content += "<UserData>%s</UserData>\n" % (b64(userdata))
+        content += "<UserData>%s</UserData>\n" % (b64e(userdata))
 
     if pubkeys:
         content += "<SSH><PublicKeys>\n"
@@ -189,7 +181,7 @@ class TestAzureDataSource(TestCase):
         # set dscfg in via base64 encoded yaml
         cfg = {'agent_command': "my_command"}
         odata = {'HostName': "myhost", 'UserName': "myuser",
-                'dscfg': {'text': b64(yaml.dump(cfg)),
+                'dscfg': {'text': b64e(yaml.dump(cfg)),
                           'encoding': 'base64'}}
         data = {'ovfcontent': construct_valid_ovf_env(data=odata)}
 
@@ -241,7 +233,7 @@ class TestAzureDataSource(TestCase):
 
     def test_userdata_found(self):
         mydata = "FOOBAR"
-        odata = {'UserData': b64(mydata)}
+        odata = {'UserData': b64e(mydata)}
         data = {'ovfcontent': construct_valid_ovf_env(data=odata)}
 
         dsrc = self._get_ds(data)
@@ -289,7 +281,7 @@ class TestAzureDataSource(TestCase):
                                    'command': 'my-bounce-command',
                                    'hostname_command': 'my-hostname-command'}}
         odata = {'HostName': "xhost",
-                'dscfg': {'text': b64(yaml.dump(cfg)),
+                'dscfg': {'text': b64e(yaml.dump(cfg)),
                           'encoding': 'base64'}}
         data = {'ovfcontent': construct_valid_ovf_env(data=odata)}
         self._get_ds(data).get_data()
@@ -304,7 +296,7 @@ class TestAzureDataSource(TestCase):
         # config specifying set_hostname off should not bounce
         cfg = {'set_hostname': False}
         odata = {'HostName': "xhost",
-                'dscfg': {'text': b64(yaml.dump(cfg)),
+                'dscfg': {'text': b64e(yaml.dump(cfg)),
                           'encoding': 'base64'}}
         data = {'ovfcontent': construct_valid_ovf_env(data=odata)}
         self._get_ds(data).get_data()
@@ -333,7 +325,7 @@ class TestAzureDataSource(TestCase):
         # Make sure that user can affect disk aliases
         dscfg = {'disk_aliases': {'ephemeral0': '/dev/sdc'}}
         odata = {'HostName': "myhost", 'UserName': "myuser",
-                'dscfg': {'text': b64(yaml.dump(dscfg)),
+                'dscfg': {'text': b64e(yaml.dump(dscfg)),
                           'encoding': 'base64'}}
         usercfg = {'disk_setup': {'/dev/sdc': {'something': '...'},
                                   'ephemeral0': False}}
@@ -370,7 +362,7 @@ class TestAzureDataSource(TestCase):
 
     def test_existing_ovf_same(self):
         # waagent/SharedConfig left alone if found ovf-env.xml same as cached
-        odata = {'UserData': b64("SOMEUSERDATA")}
+        odata = {'UserData': b64e("SOMEUSERDATA")}
         data = {'ovfcontent': construct_valid_ovf_env(data=odata)}
 
         populate_dir(self.waagent_d,
@@ -394,9 +386,9 @@ class TestAzureDataSource(TestCase):
         # 'get_data' should remove SharedConfig.xml in /var/lib/waagent
         # if ovf-env.xml differs.
         cached_ovfenv = construct_valid_ovf_env(
-            {'userdata': b64("FOO_USERDATA")})
+            {'userdata': b64e("FOO_USERDATA")})
         new_ovfenv = construct_valid_ovf_env(
-            {'userdata': b64("NEW_USERDATA")})
+            {'userdata': b64e("NEW_USERDATA")})
 
         populate_dir(self.waagent_d,
             {'ovf-env.xml': cached_ovfenv,
diff --git a/tests/unittests/test_datasource/test_opennebula.py b/tests/unittests/test_datasource/test_opennebula.py
index e5a4bd18..27adf21b 100644
--- a/tests/unittests/test_datasource/test_opennebula.py
+++ b/tests/unittests/test_datasource/test_opennebula.py
@@ -3,19 +3,12 @@ from cloudinit.sources import DataSourceOpenNebula as ds
 from cloudinit import util
 from ..helpers import TestCase, populate_dir
 
-from base64 import b64encode
 import os
 import pwd
 import shutil
 import tempfile
 import unittest
 
-def b64(source):
-    # In Python 3, b64encode only accepts bytes and returns bytes.
-    if not isinstance(source, bytes):
-        source = source.encode('utf-8')
-    return b64encode(source).decode('us-ascii')
-
 
 TEST_VARS = {
     'VAR1': 'single',
@@ -186,7 +179,7 @@ class TestOpenNebulaDataSource(TestCase):
             self.assertEqual(USER_DATA, results['userdata'])
 
     def test_user_data_encoding_required_for_decode(self):
-        b64userdata = b64(USER_DATA)
+        b64userdata = util.b64e(USER_DATA)
         for k in ('USER_DATA', 'USERDATA'):
             my_d = os.path.join(self.tmp, k)
             populate_context_dir(my_d, {k: b64userdata})
@@ -198,7 +191,7 @@ class TestOpenNebulaDataSource(TestCase):
     def test_user_data_base64_encoding(self):
         for k in ('USER_DATA', 'USERDATA'):
             my_d = os.path.join(self.tmp, k)
-            populate_context_dir(my_d, {k: b64(USER_DATA),
+            populate_context_dir(my_d, {k: util.b64e(USER_DATA),
                                         'USERDATA_ENCODING': 'base64'})
             results = ds.read_context_disk_dir(my_d)
 
diff --git a/tests/unittests/test_datasource/test_smartos.py b/tests/unittests/test_datasource/test_smartos.py
index b5ebf94d..8b62b1b1 100644
--- a/tests/unittests/test_datasource/test_smartos.py
+++ b/tests/unittests/test_datasource/test_smartos.py
@@ -24,9 +24,9 @@
 
 from __future__ import print_function
 
-import base64
 from cloudinit import helpers as c_helpers
 from cloudinit.sources import DataSourceSmartOS
+from cloudinit.util import b64e
 from .. import helpers
 import os
 import os.path
@@ -36,12 +36,6 @@ import tempfile
 import stat
 import uuid
 
-def b64(source):
-    # In Python 3, b64encode only accepts bytes and returns bytes.
-    if not isinstance(source, bytes):
-        source = source.encode('utf-8')
-    return base64.b64encode(source).decode('us-ascii')
-
 
 MOCK_RETURNS = {
     'hostname': 'test-host',
@@ -239,7 +233,7 @@ class TestSmartOSDataSource(helpers.FilesystemMockingTestCase):
         my_returns = MOCK_RETURNS.copy()
         my_returns['base64_all'] = "true"
         for k in ('hostname', 'cloud-init:user-data'):
-            my_returns[k] = b64(my_returns[k])
+            my_returns[k] = b64e(my_returns[k])
 
         dsrc = self._get_ds(mockdata=my_returns)
         ret = dsrc.get_data()
@@ -260,7 +254,7 @@ class TestSmartOSDataSource(helpers.FilesystemMockingTestCase):
         my_returns['b64-cloud-init:user-data'] = "true"
         my_returns['b64-hostname'] = "true"
         for k in ('hostname', 'cloud-init:user-data'):
-            my_returns[k] = b64(my_returns[k])
+            my_returns[k] = b64e(my_returns[k])
 
         dsrc = self._get_ds(mockdata=my_returns)
         ret = dsrc.get_data()
@@ -276,7 +270,7 @@ class TestSmartOSDataSource(helpers.FilesystemMockingTestCase):
         my_returns = MOCK_RETURNS.copy()
         my_returns['base64_keys'] = 'hostname,ignored'
         for k in ('hostname',):
-            my_returns[k] = b64(my_returns[k])
+            my_returns[k] = b64e(my_returns[k])
 
         dsrc = self._get_ds(mockdata=my_returns)
         ret = dsrc.get_data()
diff --git a/tests/unittests/test_handler/test_handler_seed_random.py b/tests/unittests/test_handler/test_handler_seed_random.py
index d3f18fa0..0bcdcb31 100644
--- a/tests/unittests/test_handler/test_handler_seed_random.py
+++ b/tests/unittests/test_handler/test_handler_seed_random.py
@@ -18,7 +18,6 @@
 
 from cloudinit.config import cc_seed_random
 
-import base64
 import gzip
 import tempfile
 
@@ -38,13 +37,6 @@ import logging
 LOG = logging.getLogger(__name__)
 
 
-def b64(source):
-    # In Python 3, b64encode only accepts bytes and returns bytes.
-    if not isinstance(source, bytes):
-        source = source.encode('utf-8')
-    return base64.b64encode(source).decode('us-ascii')
-
-
 class TestRandomSeed(t_help.TestCase):
     def setUp(self):
         super(TestRandomSeed, self).setUp()
@@ -141,7 +133,7 @@ class TestRandomSeed(t_help.TestCase):
         self.assertEquals("big-toe", contents)
 
     def test_append_random_base64(self):
-        data = b64('bubbles')
+        data = util.b64e('bubbles')
         cfg = {
             'random_seed': {
                 'file': self._seed_file,
@@ -154,7 +146,7 @@ class TestRandomSeed(t_help.TestCase):
         self.assertEquals("bubbles", contents)
 
     def test_append_random_b64(self):
-        data = b64('kit-kat')
+        data = util.b64e('kit-kat')
         cfg = {
             'random_seed': {
                 'file': self._seed_file,
-- 
cgit v1.2.3