From f895cb12141281702b34da18f2384deb64c881e7 Mon Sep 17 00:00:00 2001
From: Barry Warsaw <barry@python.org>
Date: Wed, 21 Jan 2015 17:56:53 -0500
Subject: Largely merge lp:~harlowja/cloud-init/py2-3 albeit manually because
 it seemed to be behind trunk.

`tox -e py27` passes full test suite.  Now to work on replacing mocker.
---
 cloudinit/sources/DataSourceConfigDrive.py  |  4 ++--
 cloudinit/sources/DataSourceDigitalOcean.py |  9 +++++----
 cloudinit/sources/DataSourceEc2.py          |  4 ++--
 cloudinit/sources/DataSourceMAAS.py         |  2 +-
 cloudinit/sources/DataSourceOVF.py          |  6 +++---
 cloudinit/sources/DataSourceSmartOS.py      | 15 ++++++++-------
 cloudinit/sources/__init__.py               | 10 ++++++----
 cloudinit/sources/helpers/openstack.py      | 10 ++++++----
 8 files changed, 33 insertions(+), 27 deletions(-)

(limited to 'cloudinit/sources')

diff --git a/cloudinit/sources/DataSourceConfigDrive.py b/cloudinit/sources/DataSourceConfigDrive.py
index 15244a0d..eb474079 100644
--- a/cloudinit/sources/DataSourceConfigDrive.py
+++ b/cloudinit/sources/DataSourceConfigDrive.py
@@ -216,11 +216,11 @@ def on_first_boot(data, distro=None):
     files = data.get('files', {})
     if files:
         LOG.debug("Writing %s injected files", len(files))
-        for (filename, content) in files.iteritems():
+        for (filename, content) in files.items():
             if not filename.startswith(os.sep):
                 filename = os.sep + filename
             try:
-                util.write_file(filename, content, mode=0660)
+                util.write_file(filename, content, mode=0o660)
             except IOError:
                 util.logexc(LOG, "Failed writing file: %s", filename)
 
diff --git a/cloudinit/sources/DataSourceDigitalOcean.py b/cloudinit/sources/DataSourceDigitalOcean.py
index 8f27ee89..b20ce2a1 100644
--- a/cloudinit/sources/DataSourceDigitalOcean.py
+++ b/cloudinit/sources/DataSourceDigitalOcean.py
@@ -18,7 +18,7 @@ from cloudinit import log as logging
 from cloudinit import util
 from cloudinit import sources
 from cloudinit import ec2_utils
-from types import StringType
+
 import functools
 
 
@@ -72,10 +72,11 @@ class DataSourceDigitalOcean(sources.DataSource):
         return "\n".join(self.metadata['vendor-data'])
 
     def get_public_ssh_keys(self):
-        if type(self.metadata['public-keys']) is StringType:
-            return [self.metadata['public-keys']]
+        public_keys = self.metadata['public-keys']
+        if isinstance(public_keys, list):
+            return public_keys
         else:
-            return self.metadata['public-keys']
+            return [public_keys]
 
     @property
     def availability_zone(self):
diff --git a/cloudinit/sources/DataSourceEc2.py b/cloudinit/sources/DataSourceEc2.py
index 1b20ecf3..798869b7 100644
--- a/cloudinit/sources/DataSourceEc2.py
+++ b/cloudinit/sources/DataSourceEc2.py
@@ -156,8 +156,8 @@ class DataSourceEc2(sources.DataSource):
         # 'ephemeral0': '/dev/sdb',
         # 'root': '/dev/sda1'}
         found = None
-        bdm_items = self.metadata['block-device-mapping'].iteritems()
-        for (entname, device) in bdm_items:
+        bdm = self.metadata['block-device-mapping']
+        for (entname, device) in bdm.items():
             if entname == name:
                 found = device
                 break
diff --git a/cloudinit/sources/DataSourceMAAS.py b/cloudinit/sources/DataSourceMAAS.py
index dfe90bc6..9a3e30c5 100644
--- a/cloudinit/sources/DataSourceMAAS.py
+++ b/cloudinit/sources/DataSourceMAAS.py
@@ -262,7 +262,7 @@ def check_seed_contents(content, seed):
 
     userdata = content.get('user-data', "")
     md = {}
-    for (key, val) in content.iteritems():
+    for (key, val) in content.items():
         if key == 'user-data':
             continue
         md[key] = val
diff --git a/cloudinit/sources/DataSourceOVF.py b/cloudinit/sources/DataSourceOVF.py
index 7ba60735..58a4b2a2 100644
--- a/cloudinit/sources/DataSourceOVF.py
+++ b/cloudinit/sources/DataSourceOVF.py
@@ -66,7 +66,7 @@ class DataSourceOVF(sources.DataSource):
             np = {'iso': transport_iso9660,
                   'vmware-guestd': transport_vmware_guestd, }
             name = None
-            for (name, transfunc) in np.iteritems():
+            for (name, transfunc) in np.items():
                 (contents, _dev, _fname) = transfunc()
                 if contents:
                     break
@@ -138,7 +138,7 @@ def read_ovf_environment(contents):
     ud = ""
     cfg_props = ['password']
     md_props = ['seedfrom', 'local-hostname', 'public-keys', 'instance-id']
-    for (prop, val) in props.iteritems():
+    for (prop, val) in props.items():
         if prop == 'hostname':
             prop = "local-hostname"
         if prop in md_props:
@@ -183,7 +183,7 @@ def transport_iso9660(require_iso=True):
 
     # Go through mounts to see if it was already mounted
     mounts = util.mounts()
-    for (dev, info) in mounts.iteritems():
+    for (dev, info) in mounts.items():
         fstype = info['fstype']
         if fstype != "iso9660" and require_iso:
             continue
diff --git a/cloudinit/sources/DataSourceSmartOS.py b/cloudinit/sources/DataSourceSmartOS.py
index 2733a2f6..7a975d78 100644
--- a/cloudinit/sources/DataSourceSmartOS.py
+++ b/cloudinit/sources/DataSourceSmartOS.py
@@ -30,12 +30,12 @@
 #       Comments with "@datadictionary" are snippets of the definition
 
 import base64
+import os
+import serial
+
 from cloudinit import log as logging
 from cloudinit import sources
 from cloudinit import util
-import os
-import os.path
-import serial
 
 
 LOG = logging.getLogger(__name__)
@@ -201,7 +201,7 @@ class DataSourceSmartOS(sources.DataSource):
         if b64_all is not None:
             self.b64_all = util.is_true(b64_all)
 
-        for ci_noun, attribute in SMARTOS_ATTRIB_MAP.iteritems():
+        for ci_noun, attribute in SMARTOS_ATTRIB_MAP.items():
             smartos_noun, strip = attribute
             md[ci_noun] = self.query(smartos_noun, strip=strip)
 
@@ -218,11 +218,12 @@ class DataSourceSmartOS(sources.DataSource):
         user_script = os.path.join(data_d, 'user-script')
         u_script_l = "%s/user-script" % LEGACY_USER_D
         write_boot_content(md.get('user-script'), content_f=user_script,
-                           link=u_script_l, shebang=True, mode=0700)
+                           link=u_script_l, shebang=True, mode=0o700)
 
         operator_script = os.path.join(data_d, 'operator-script')
         write_boot_content(md.get('operator-script'),
-                           content_f=operator_script, shebang=False, mode=0700)
+                           content_f=operator_script, shebang=False,
+                           mode=0o700)
 
         # @datadictionary:  This key has no defined format, but its value
         # is written to the file /var/db/mdata-user-data on each boot prior
@@ -381,7 +382,7 @@ def dmi_data():
 
 
 def write_boot_content(content, content_f, link=None, shebang=False,
-                       mode=0400):
+                       mode=0o400):
     """
     Write the content to content_f. Under the following rules:
         1. If no content, remove the file
diff --git a/cloudinit/sources/__init__.py b/cloudinit/sources/__init__.py
index 7c7ef9ab..39eab51b 100644
--- a/cloudinit/sources/__init__.py
+++ b/cloudinit/sources/__init__.py
@@ -23,6 +23,8 @@
 import abc
 import os
 
+import six
+
 from cloudinit import importer
 from cloudinit import log as logging
 from cloudinit import type_utils
@@ -130,7 +132,7 @@ class DataSource(object):
         # we want to return the correct value for what will actually
         # exist in this instance
         mappings = {"sd": ("vd", "xvd", "vtb")}
-        for (nfrom, tlist) in mappings.iteritems():
+        for (nfrom, tlist) in mappings.items():
             if not short_name.startswith(nfrom):
                 continue
             for nto in tlist:
@@ -218,18 +220,18 @@ def normalize_pubkey_data(pubkey_data):
     if not pubkey_data:
         return keys
 
-    if isinstance(pubkey_data, (basestring, str)):
+    if isinstance(pubkey_data, six.string_types):
         return str(pubkey_data).splitlines()
 
     if isinstance(pubkey_data, (list, set)):
         return list(pubkey_data)
 
     if isinstance(pubkey_data, (dict)):
-        for (_keyname, klist) in pubkey_data.iteritems():
+        for (_keyname, klist) in pubkey_data.items():
             # lp:506332 uec metadata service responds with
             # data that makes boto populate a string for 'klist' rather
             # than a list.
-            if isinstance(klist, (str, basestring)):
+            if isinstance(klist, six.string_types):
                 klist = [klist]
             if isinstance(klist, (list, set)):
                 for pkey in klist:
diff --git a/cloudinit/sources/helpers/openstack.py b/cloudinit/sources/helpers/openstack.py
index b7e19314..88c7a198 100644
--- a/cloudinit/sources/helpers/openstack.py
+++ b/cloudinit/sources/helpers/openstack.py
@@ -24,6 +24,8 @@ import copy
 import functools
 import os
 
+import six
+
 from cloudinit import ec2_utils
 from cloudinit import log as logging
 from cloudinit import sources
@@ -205,7 +207,7 @@ class BaseReader(object):
         """
 
         load_json_anytype = functools.partial(
-            util.load_json, root_types=(dict, basestring, list))
+            util.load_json, root_types=(dict, list) + six.string_types)
 
         def datafiles(version):
             files = {}
@@ -234,7 +236,7 @@ class BaseReader(object):
             'version': 2,
         }
         data = datafiles(self._find_working_version())
-        for (name, (path, required, translator)) in data.iteritems():
+        for (name, (path, required, translator)) in data.items():
             path = self._path_join(self.base_path, path)
             data = None
             found = False
@@ -364,7 +366,7 @@ class ConfigDriveReader(BaseReader):
             raise NonReadable("%s: no files found" % (self.base_path))
 
         md = {}
-        for (name, (key, translator, default)) in FILES_V1.iteritems():
+        for (name, (key, translator, default)) in FILES_V1.items():
             if name in found:
                 path = found[name]
                 try:
@@ -478,7 +480,7 @@ def convert_vendordata_json(data, recurse=True):
     """
     if not data:
         return None
-    if isinstance(data, (str, unicode, basestring)):
+    if isinstance(data, six.string_types):
         return data
     if isinstance(data, list):
         return copy.deepcopy(data)
-- 
cgit v1.2.3


From 3b798b5d5c3caa5d0e8e534855e29010ca932aaa Mon Sep 17 00:00:00 2001
From: Barry Warsaw <barry@python.org>
Date: Thu, 22 Jan 2015 21:21:04 -0500
Subject: Low hanging Python 3 fruit.

---
 cloudinit/config/cc_ca_certs.py                          |  4 ++--
 cloudinit/config/cc_chef.py                              |  6 ++++--
 cloudinit/distros/__init__.py                            | 12 ++++++++++--
 cloudinit/distros/debian.py                              |  2 +-
 cloudinit/distros/rhel_util.py                           |  4 ++--
 cloudinit/distros/sles.py                                |  2 +-
 cloudinit/sources/DataSourceAltCloud.py                  | 12 ++++++------
 cloudinit/sources/DataSourceAzure.py                     |  4 ++--
 cloudinit/sources/DataSourceMAAS.py                      | 10 ++++++----
 cloudinit/sources/DataSourceOpenNebula.py                |  2 +-
 cloudinit/templater.py                                   |  2 +-
 cloudinit/util.py                                        |  7 +++++--
 templates/resolv.conf.tmpl                               |  2 +-
 tests/unittests/helpers.py                               |  4 ++--
 tests/unittests/test_datasource/test_configdrive.py      |  2 +-
 tests/unittests/test_datasource/test_digitalocean.py     |  7 +++----
 tests/unittests/test_datasource/test_gce.py              |  2 +-
 tests/unittests/test_datasource/test_opennebula.py       |  2 +-
 tests/unittests/test_datasource/test_smartos.py          |  4 +++-
 .../unittests/test_handler/test_handler_apt_configure.py |  2 +-
 tests/unittests/test_merging.py                          | 16 +++++++++-------
 tools/ccfg-merge-debug                                   |  4 ++--
 22 files changed, 65 insertions(+), 47 deletions(-)

(limited to 'cloudinit/sources')

diff --git a/cloudinit/config/cc_ca_certs.py b/cloudinit/config/cc_ca_certs.py
index 4f2a46a1..8248b020 100644
--- a/cloudinit/config/cc_ca_certs.py
+++ b/cloudinit/config/cc_ca_certs.py
@@ -44,7 +44,7 @@ def add_ca_certs(certs):
     if certs:
         # First ensure they are strings...
         cert_file_contents = "\n".join([str(c) for c in certs])
-        util.write_file(CA_CERT_FULL_PATH, cert_file_contents, mode=0644)
+        util.write_file(CA_CERT_FULL_PATH, cert_file_contents, mode=0o644)
 
         # Append cert filename to CA_CERT_CONFIG file.
         # We have to strip the content because blank lines in the file
@@ -63,7 +63,7 @@ def remove_default_ca_certs():
     """
     util.delete_dir_contents(CA_CERT_PATH)
     util.delete_dir_contents(CA_CERT_SYSTEM_PATH)
-    util.write_file(CA_CERT_CONFIG, "", mode=0644)
+    util.write_file(CA_CERT_CONFIG, "", mode=0o644)
     debconf_sel = "ca-certificates ca-certificates/trust_new_crts select no"
     util.subp(('debconf-set-selections', '-'), debconf_sel)
 
diff --git a/cloudinit/config/cc_chef.py b/cloudinit/config/cc_chef.py
index fc837363..584199e5 100644
--- a/cloudinit/config/cc_chef.py
+++ b/cloudinit/config/cc_chef.py
@@ -76,6 +76,8 @@ from cloudinit import templater
 from cloudinit import url_helper
 from cloudinit import util
 
+import six
+
 RUBY_VERSION_DEFAULT = "1.8"
 
 CHEF_DIRS = tuple([
@@ -261,7 +263,7 @@ def run_chef(chef_cfg, log):
         cmd_args = chef_cfg['exec_arguments']
         if isinstance(cmd_args, (list, tuple)):
             cmd.extend(cmd_args)
-        elif isinstance(cmd_args, (str, basestring)):
+        elif isinstance(cmd_args, six.string_types):
             cmd.append(cmd_args)
         else:
             log.warn("Unknown type %s provided for chef"
@@ -300,7 +302,7 @@ def install_chef(cloud, chef_cfg, log):
         with util.tempdir() as tmpd:
             # Use tmpdir over tmpfile to avoid 'text file busy' on execute
             tmpf = "%s/chef-omnibus-install" % tmpd
-            util.write_file(tmpf, str(content), mode=0700)
+            util.write_file(tmpf, str(content), mode=0o700)
             util.subp([tmpf], capture=False)
     else:
         log.warn("Unknown chef install type '%s'", install_type)
diff --git a/cloudinit/distros/__init__.py b/cloudinit/distros/__init__.py
index 4ebccdda..6b96d58c 100644
--- a/cloudinit/distros/__init__.py
+++ b/cloudinit/distros/__init__.py
@@ -25,7 +25,6 @@ import six
 from six import StringIO
 
 import abc
-import itertools
 import os
 import re
 
@@ -37,6 +36,15 @@ from cloudinit import util
 
 from cloudinit.distros.parsers import hosts
 
+try:
+    # Python 3
+    from six import filter
+except ImportError:
+    # Python 2
+    from itertools import ifilter as filter
+
+
+
 OSFAMILIES = {
     'debian': ['debian', 'ubuntu'],
     'redhat': ['fedora', 'rhel'],
@@ -853,7 +861,7 @@ def extract_default(users, default_name=None, default_config=None):
             return config['default']
 
     tmp_users = users.items()
-    tmp_users = dict(itertools.ifilter(safe_find, tmp_users))
+    tmp_users = dict(filter(safe_find, tmp_users))
     if not tmp_users:
         return (default_name, default_config)
     else:
diff --git a/cloudinit/distros/debian.py b/cloudinit/distros/debian.py
index b09eb094..6d3a82bf 100644
--- a/cloudinit/distros/debian.py
+++ b/cloudinit/distros/debian.py
@@ -97,7 +97,7 @@ class Distro(distros.Distro):
         if not conf:
             conf = HostnameConf('')
         conf.set_hostname(your_hostname)
-        util.write_file(out_fn, str(conf), 0644)
+        util.write_file(out_fn, str(conf), 0o644)
 
     def _read_system_hostname(self):
         sys_hostname = self._read_hostname(self.hostname_conf_fn)
diff --git a/cloudinit/distros/rhel_util.py b/cloudinit/distros/rhel_util.py
index 063d536e..903d7793 100644
--- a/cloudinit/distros/rhel_util.py
+++ b/cloudinit/distros/rhel_util.py
@@ -50,7 +50,7 @@ def update_sysconfig_file(fn, adjustments, allow_empty=False):
         ]
         if not exists:
             lines.insert(0, util.make_header())
-        util.write_file(fn, "\n".join(lines) + "\n", 0644)
+        util.write_file(fn, "\n".join(lines) + "\n", 0o644)
 
 
 # Helper function to read a RHEL/SUSE /etc/sysconfig/* file
@@ -86,4 +86,4 @@ def update_resolve_conf_file(fn, dns_servers, search_servers):
                 r_conf.add_search_domain(s)
             except ValueError:
                 util.logexc(LOG, "Failed at adding search domain %s", s)
-    util.write_file(fn, str(r_conf), 0644)
+    util.write_file(fn, str(r_conf), 0o644)
diff --git a/cloudinit/distros/sles.py b/cloudinit/distros/sles.py
index 0c6d1203..620c974c 100644
--- a/cloudinit/distros/sles.py
+++ b/cloudinit/distros/sles.py
@@ -113,7 +113,7 @@ class Distro(distros.Distro):
         if not conf:
             conf = HostnameConf('')
         conf.set_hostname(hostname)
-        util.write_file(out_fn, str(conf), 0644)
+        util.write_file(out_fn, str(conf), 0o644)
 
     def _read_system_hostname(self):
         host_fn = self.hostname_conf_fn
diff --git a/cloudinit/sources/DataSourceAltCloud.py b/cloudinit/sources/DataSourceAltCloud.py
index 1e913a6e..69053d0b 100644
--- a/cloudinit/sources/DataSourceAltCloud.py
+++ b/cloudinit/sources/DataSourceAltCloud.py
@@ -124,11 +124,11 @@ class DataSourceAltCloud(sources.DataSource):
         cmd = CMD_DMI_SYSTEM
         try:
             (cmd_out, _err) = util.subp(cmd)
-        except ProcessExecutionError, _err:
+        except ProcessExecutionError as _err:
             LOG.debug(('Failed command: %s\n%s') % \
                 (' '.join(cmd), _err.message))
             return 'UNKNOWN'
-        except OSError, _err:
+        except OSError as _err:
             LOG.debug(('Failed command: %s\n%s') % \
                 (' '.join(cmd), _err.message))
             return 'UNKNOWN'
@@ -211,11 +211,11 @@ class DataSourceAltCloud(sources.DataSource):
             cmd = CMD_PROBE_FLOPPY
             (cmd_out, _err) = util.subp(cmd)
             LOG.debug(('Command: %s\nOutput%s') % (' '.join(cmd), cmd_out))
-        except ProcessExecutionError, _err:
+        except ProcessExecutionError as _err:
             util.logexc(LOG, 'Failed command: %s\n%s', ' '.join(cmd),
                         _err.message)
             return False
-        except OSError, _err:
+        except OSError as _err:
             util.logexc(LOG, 'Failed command: %s\n%s', ' '.join(cmd),
                         _err.message)
             return False
@@ -228,11 +228,11 @@ class DataSourceAltCloud(sources.DataSource):
             cmd.append('--exit-if-exists=' + floppy_dev)
             (cmd_out, _err) = util.subp(cmd)
             LOG.debug(('Command: %s\nOutput%s') % (' '.join(cmd), cmd_out))
-        except ProcessExecutionError, _err:
+        except ProcessExecutionError as _err:
             util.logexc(LOG, 'Failed command: %s\n%s', ' '.join(cmd),
                         _err.message)
             return False
-        except OSError, _err:
+        except OSError as _err:
             util.logexc(LOG, 'Failed command: %s\n%s', ' '.join(cmd),
                         _err.message)
             return False
diff --git a/cloudinit/sources/DataSourceAzure.py b/cloudinit/sources/DataSourceAzure.py
index 09bc196d..29ae2c22 100644
--- a/cloudinit/sources/DataSourceAzure.py
+++ b/cloudinit/sources/DataSourceAzure.py
@@ -151,7 +151,7 @@ class DataSourceAzureNet(sources.DataSource):
 
         # walinux agent writes files world readable, but expects
         # the directory to be protected.
-        write_files(ddir, files, dirmode=0700)
+        write_files(ddir, files, dirmode=0o700)
 
         # handle the hostname 'publishing'
         try:
@@ -390,7 +390,7 @@ def write_files(datadir, files, dirmode=None):
     util.ensure_dir(datadir, dirmode)
     for (name, content) in files.items():
         util.write_file(filename=os.path.join(datadir, name),
-                        content=content, mode=0600)
+                        content=content, mode=0o600)
 
 
 def invoke_agent(cmd):
diff --git a/cloudinit/sources/DataSourceMAAS.py b/cloudinit/sources/DataSourceMAAS.py
index 9a3e30c5..8f9c81de 100644
--- a/cloudinit/sources/DataSourceMAAS.py
+++ b/cloudinit/sources/DataSourceMAAS.py
@@ -18,6 +18,8 @@
 #    You should have received a copy of the GNU General Public License
 #    along with this program.  If not, see <http://www.gnu.org/licenses/>.
 
+from __future__ import print_function
+
 from email.utils import parsedate
 import errno
 import oauth.oauth as oauth
@@ -361,7 +363,7 @@ if __name__ == "__main__":
             return (urllib2.urlopen(req).read())
 
         def printurl(url, headers_cb):
-            print "== %s ==\n%s\n" % (url, geturl(url, headers_cb))
+            print("== %s ==\n%s\n" % (url, geturl(url, headers_cb)))
 
         def crawl(url, headers_cb=None):
             if url.endswith("/"):
@@ -386,9 +388,9 @@ if __name__ == "__main__":
                                                           version=args.apiver)
             else:
                 (userdata, metadata) = read_maas_seed_url(args.url)
-            print "=== userdata ==="
-            print userdata
-            print "=== metadata ==="
+            print("=== userdata ===")
+            print(userdata)
+            print("=== metadata ===")
             pprint.pprint(metadata)
 
         elif args.subcmd == "get":
diff --git a/cloudinit/sources/DataSourceOpenNebula.py b/cloudinit/sources/DataSourceOpenNebula.py
index e2469f6e..f9dac29e 100644
--- a/cloudinit/sources/DataSourceOpenNebula.py
+++ b/cloudinit/sources/DataSourceOpenNebula.py
@@ -280,7 +280,7 @@ def parse_shell_config(content, keylist=None, bash=None, asuser=None,
 
     # allvars expands to all existing variables by using '${!x*}' notation
     # where x is lower or upper case letters or '_'
-    allvars = ["${!%s*}" % x for x in string.letters + "_"]
+    allvars = ["${!%s*}" % x for x in string.ascii_letters + "_"]
 
     keylist_in = keylist
     if keylist is None:
diff --git a/cloudinit/templater.py b/cloudinit/templater.py
index 4cd3f13d..a9231482 100644
--- a/cloudinit/templater.py
+++ b/cloudinit/templater.py
@@ -137,7 +137,7 @@ def render_from_file(fn, params):
     return renderer(content, params)
 
 
-def render_to_file(fn, outfn, params, mode=0644):
+def render_to_file(fn, outfn, params, mode=0o644):
     contents = render_from_file(fn, params)
     util.write_file(outfn, contents, mode=mode)
 
diff --git a/cloudinit/util.py b/cloudinit/util.py
index 434ba7fb..94fd5c70 100644
--- a/cloudinit/util.py
+++ b/cloudinit/util.py
@@ -142,6 +142,9 @@ class ProcessExecutionError(IOError):
             'reason': self.reason,
         }
         IOError.__init__(self, message)
+        # For backward compatibility with Python 2.
+        if not hasattr(self, 'message'):
+            self.message = message
 
 
 class SeLinuxGuard(object):
@@ -260,7 +263,7 @@ def translate_bool(val, addons=None):
 
 def rand_str(strlen=32, select_from=None):
     if not select_from:
-        select_from = string.letters + string.digits
+        select_from = string.ascii_letters + string.digits
     return "".join([random.choice(select_from) for _x in range(0, strlen)])
 
 
@@ -1127,7 +1130,7 @@ def pipe_in_out(in_fh, out_fh, chunk_size=1024, chunk_cb=None):
     bytes_piped = 0
     while True:
         data = in_fh.read(chunk_size)
-        if data == '':
+        if len(data) == 0:
             break
         else:
             out_fh.write(data)
diff --git a/templates/resolv.conf.tmpl b/templates/resolv.conf.tmpl
index 1300156c..bfae80db 100644
--- a/templates/resolv.conf.tmpl
+++ b/templates/resolv.conf.tmpl
@@ -24,7 +24,7 @@ sortlist {% for sort in sortlist %}{{sort}} {% endfor %}
 {% if options or flags %}
 
 options {% for flag in flags %}{{flag}} {% endfor %}
-{% for key, value in options.iteritems() -%}
+{% for key, value in options.items() -%}
  {{key}}:{{value}}
 {% endfor %}
 {% endif %}
diff --git a/tests/unittests/helpers.py b/tests/unittests/helpers.py
index 38a2176d..70b8116f 100644
--- a/tests/unittests/helpers.py
+++ b/tests/unittests/helpers.py
@@ -65,7 +65,7 @@ if PY26:
         def assertDictContainsSubset(self, expected, actual, msg=None):
             missing = []
             mismatched = []
-            for k, v in expected.iteritems():
+            for k, v in expected.items():
                 if k not in actual:
                     missing.append(k)
                 elif actual[k] != v:
@@ -243,7 +243,7 @@ class HttprettyTestCase(TestCase):
 def populate_dir(path, files):
     if not os.path.exists(path):
         os.makedirs(path)
-    for (name, content) in files.iteritems():
+    for (name, content) in files.items():
         with open(os.path.join(path, name), "w") as fp:
             fp.write(content)
             fp.close()
diff --git a/tests/unittests/test_datasource/test_configdrive.py b/tests/unittests/test_datasource/test_configdrive.py
index 800c5fd8..258c68e2 100644
--- a/tests/unittests/test_datasource/test_configdrive.py
+++ b/tests/unittests/test_datasource/test_configdrive.py
@@ -338,7 +338,7 @@ def populate_ds_from_read_config(cfg_ds, source, results):
 
 
 def populate_dir(seed_dir, files):
-    for (name, content) in files.iteritems():
+    for (name, content) in files.items():
         path = os.path.join(seed_dir, name)
         dirname = os.path.dirname(path)
         if not os.path.isdir(dirname):
diff --git a/tests/unittests/test_datasource/test_digitalocean.py b/tests/unittests/test_datasource/test_digitalocean.py
index d1270fc2..98f9cfac 100644
--- a/tests/unittests/test_datasource/test_digitalocean.py
+++ b/tests/unittests/test_datasource/test_digitalocean.py
@@ -18,8 +18,7 @@
 import httpretty
 import re
 
-from types import ListType
-from urlparse import urlparse
+from six.moves.urllib_parse import urlparse
 
 from cloudinit import settings
 from cloudinit import helpers
@@ -110,7 +109,7 @@ class TestDataSourceDigitalOcean(test_helpers.HttprettyTestCase):
         self.assertEqual([DO_META.get('public-keys')],
                          self.ds.get_public_ssh_keys())
 
-        self.assertIs(type(self.ds.get_public_ssh_keys()), ListType)
+        self.assertIsInstance(self.ds.get_public_ssh_keys(), list)
 
     @httpretty.activate
     def test_multiple_ssh_keys(self):
@@ -124,4 +123,4 @@ class TestDataSourceDigitalOcean(test_helpers.HttprettyTestCase):
         self.assertEqual(DO_META.get('public-keys').splitlines(),
                          self.ds.get_public_ssh_keys())
 
-        self.assertIs(type(self.ds.get_public_ssh_keys()), ListType)
+        self.assertIsInstance(self.ds.get_public_ssh_keys(), list)
diff --git a/tests/unittests/test_datasource/test_gce.py b/tests/unittests/test_datasource/test_gce.py
index 06050bb1..aa60eb33 100644
--- a/tests/unittests/test_datasource/test_gce.py
+++ b/tests/unittests/test_datasource/test_gce.py
@@ -19,7 +19,7 @@ import httpretty
 import re
 
 from base64 import b64encode, b64decode
-from urlparse import urlparse
+from six.moves.urllib_parse import urlparse
 
 from cloudinit import settings
 from cloudinit import helpers
diff --git a/tests/unittests/test_datasource/test_opennebula.py b/tests/unittests/test_datasource/test_opennebula.py
index ddf77265..b79237f0 100644
--- a/tests/unittests/test_datasource/test_opennebula.py
+++ b/tests/unittests/test_datasource/test_opennebula.py
@@ -294,7 +294,7 @@ class TestParseShellConfig(unittest.TestCase):
 
 def populate_context_dir(path, variables):
     data = "# Context variables generated by OpenNebula\n"
-    for (k, v) in variables.iteritems():
+    for (k, v) in variables.items():
         data += ("%s='%s'\n" % (k.upper(), v.replace(r"'", r"'\''")))
     populate_dir(path, {'context.sh': data})
 
diff --git a/tests/unittests/test_datasource/test_smartos.py b/tests/unittests/test_datasource/test_smartos.py
index 35d7ef5e..01b9b73e 100644
--- a/tests/unittests/test_datasource/test_smartos.py
+++ b/tests/unittests/test_datasource/test_smartos.py
@@ -22,6 +22,8 @@
 #   return responses.
 #
 
+from __future__ import print_function
+
 import base64
 from cloudinit import helpers as c_helpers
 from cloudinit.sources import DataSourceSmartOS
@@ -369,7 +371,7 @@ class TestSmartOSDataSource(helpers.FilesystemMockingTestCase):
                 permissions = oct(os.stat(name_f)[stat.ST_MODE])[-3:]
                 if re.match(r'.*\/mdata-user-data$', name_f):
                     found_new = True
-                    print name_f
+                    print(name_f)
                     self.assertEquals(permissions, '400')
 
         self.assertFalse(found_new)
diff --git a/tests/unittests/test_handler/test_handler_apt_configure.py b/tests/unittests/test_handler/test_handler_apt_configure.py
index 2c3dad72..d72fa8c7 100644
--- a/tests/unittests/test_handler/test_handler_apt_configure.py
+++ b/tests/unittests/test_handler/test_handler_apt_configure.py
@@ -62,7 +62,7 @@ class TestAptProxyConfig(unittest.TestCase):
 
         contents = str(util.read_file_or_url(self.pfile))
 
-        for ptype, pval in values.iteritems():
+        for ptype, pval in values.items():
             self.assertTrue(self._search_apt_config(contents, ptype, pval))
 
     def test_proxy_deleted(self):
diff --git a/tests/unittests/test_merging.py b/tests/unittests/test_merging.py
index 07b610f7..976d8283 100644
--- a/tests/unittests/test_merging.py
+++ b/tests/unittests/test_merging.py
@@ -11,11 +11,13 @@ import glob
 import os
 import random
 import re
+import six
 import string
 
 SOURCE_PAT = "source*.*yaml"
 EXPECTED_PAT = "expected%s.yaml"
-TYPES = [long, int, dict, str, list, tuple, None]
+TYPES = [dict, str, list, tuple, None]
+TYPES.extend(six.integer_types)
 
 
 def _old_mergedict(src, cand):
@@ -25,7 +27,7 @@ def _old_mergedict(src, cand):
     Nested dictionaries are merged recursively.
     """
     if isinstance(src, dict) and isinstance(cand, dict):
-        for (k, v) in cand.iteritems():
+        for (k, v) in cand.items():
             if k not in src:
                 src[k] = v
             else:
@@ -42,8 +44,8 @@ def _old_mergemanydict(*args):
 
 def _random_str(rand):
     base = ''
-    for _i in xrange(rand.randint(1, 2 ** 8)):
-        base += rand.choice(string.letters + string.digits)
+    for _i in range(rand.randint(1, 2 ** 8)):
+        base += rand.choice(string.ascii_letters + string.digits)
     return base
 
 
@@ -64,7 +66,7 @@ def _make_dict(current_depth, max_depth, rand):
     if t in [dict, list, tuple]:
         if t in [dict]:
             amount = rand.randint(0, 5)
-            keys = [_random_str(rand) for _i in xrange(0, amount)]
+            keys = [_random_str(rand) for _i in range(0, amount)]
             base = {}
             for k in keys:
                 try:
@@ -74,14 +76,14 @@ def _make_dict(current_depth, max_depth, rand):
         elif t in [list, tuple]:
             base = []
             amount = rand.randint(0, 5)
-            for _i in xrange(0, amount):
+            for _i in range(0, amount):
                 try:
                     base.append(_make_dict(current_depth + 1, max_depth, rand))
                 except _NoMoreException:
                     pass
             if t in [tuple]:
                 base = tuple(base)
-    elif t in [long, int]:
+    elif t in six.integer_types:
         base = rand.randint(0, 2 ** 8)
     elif t in [str]:
         base = _random_str(rand)
diff --git a/tools/ccfg-merge-debug b/tools/ccfg-merge-debug
index 85227da7..1f08e0cb 100755
--- a/tools/ccfg-merge-debug
+++ b/tools/ccfg-merge-debug
@@ -51,7 +51,7 @@ def main():
     c_handlers.register(ccph)
 
     called = []
-    for (_ctype, mod) in c_handlers.iteritems():
+    for (_ctype, mod) in c_handlers.items():
         if mod in called:
             continue
         handlers.call_begin(mod, data, frequency)
@@ -76,7 +76,7 @@ def main():
 
     # Give callbacks opportunity to finalize
     called = []
-    for (_ctype, mod) in c_handlers.iteritems():
+    for (_ctype, mod) in c_handlers.items():
         if mod in called:
             continue
         handlers.call_end(mod, data, frequency)
-- 
cgit v1.2.3


From de5974fe93dd717e0c7ba6de17db3192cc258cff Mon Sep 17 00:00:00 2001
From: Barry Warsaw <barry@python.org>
Date: Mon, 26 Jan 2015 14:31:09 -0500
Subject: * More str/bytes fixes. * Temporarily skip the MAAS tests in py3
 since they need to be ported to oauthlib.

---
 cloudinit/sources/DataSourceOpenNebula.py          | 12 +++++++++---
 cloudinit/sources/DataSourceSmartOS.py             | 15 +++++++++++++--
 tests/unittests/test_datasource/test_maas.py       |  7 ++++++-
 tests/unittests/test_datasource/test_opennebula.py |  4 ++--
 4 files changed, 30 insertions(+), 8 deletions(-)

(limited to 'cloudinit/sources')

diff --git a/cloudinit/sources/DataSourceOpenNebula.py b/cloudinit/sources/DataSourceOpenNebula.py
index f9dac29e..691b39f8 100644
--- a/cloudinit/sources/DataSourceOpenNebula.py
+++ b/cloudinit/sources/DataSourceOpenNebula.py
@@ -379,7 +379,8 @@ def read_context_disk_dir(source_dir, asuser=None):
                 raise BrokenContextDiskDir("configured user '%s' "
                                            "does not exist", asuser)
         try:
-            with open(os.path.join(source_dir, 'context.sh'), 'r') as f:
+            path = os.path.join(source_dir, 'context.sh')
+            with open(path, 'r', encoding='utf-8') as f:
                 content = f.read().strip()
 
             context = parse_shell_config(content, asuser=asuser)
@@ -426,14 +427,19 @@ def read_context_disk_dir(source_dir, asuser=None):
                                context.get('USER_DATA_ENCODING'))
         if encoding == "base64":
             try:
-                results['userdata'] = base64.b64decode(results['userdata'])
+                userdata = base64.b64decode(results['userdata'])
+                # In Python 3 we still expect a str, but b64decode will return
+                # bytes.  Convert to str.
+                if isinstance(userdata, bytes):
+                    userdata = userdata.decode('utf-8')
+                results['userdata'] = userdata
             except TypeError:
                 LOG.warn("Failed base64 decoding of userdata")
 
     # generate static /etc/network/interfaces
     # only if there are any required context variables
     # http://opennebula.org/documentation:rel3.8:cong#network_configuration
-    for k in context.keys():
+    for k in context:
         if re.match(r'^ETH\d+_IP$', k):
             (out, _) = util.subp(['/sbin/ip', 'link'])
             net = OpenNebulaNetwork(out, context)
diff --git a/cloudinit/sources/DataSourceSmartOS.py b/cloudinit/sources/DataSourceSmartOS.py
index 7a975d78..d3ed40c5 100644
--- a/cloudinit/sources/DataSourceSmartOS.py
+++ b/cloudinit/sources/DataSourceSmartOS.py
@@ -30,6 +30,7 @@
 #       Comments with "@datadictionary" are snippets of the definition
 
 import base64
+import binascii
 import os
 import serial
 
@@ -350,8 +351,18 @@ def query_data(noun, seed_device, seed_timeout, strip=False, default=None,
 
     if b64:
         try:
-            return base64.b64decode(resp)
-        except TypeError:
+            # Generally, we want native strings in the values.  Python 3's
+            # b64decode will return bytes though, so decode them to utf-8 if
+            # possible.  If that fails, return the bytes.
+            decoded = base64.b64decode(resp)
+            try:
+                if isinstance(decoded, bytes):
+                    return decoded.decode('utf-8')
+            except UnicodeDecodeError:
+                pass
+            return decoded
+        # Bogus input produces different errors in Python 2 and 3; catch both.
+        except (TypeError, binascii.Error):
             LOG.warn("Failed base64 decoding key '%s'", noun)
             return resp
 
diff --git a/tests/unittests/test_datasource/test_maas.py b/tests/unittests/test_datasource/test_maas.py
index 6af0cd82..66fe22ae 100644
--- a/tests/unittests/test_datasource/test_maas.py
+++ b/tests/unittests/test_datasource/test_maas.py
@@ -4,7 +4,11 @@ import shutil
 import tempfile
 import unittest
 
-from cloudinit.sources import DataSourceMAAS
+# XXX DataSourceMAAS must be ported to oauthlib for Python 3
+import six
+if not six.PY3:
+    from cloudinit.sources import DataSourceMAAS
+
 from cloudinit import url_helper
 from ..helpers import populate_dir
 
@@ -14,6 +18,7 @@ except ImportError:
     import mock
 
 
+@unittest.skipIf(six.PY3, 'DataSourceMAAS must be ported to oauthlib')
 class TestMAASDataSource(unittest.TestCase):
 
     def setUp(self):
diff --git a/tests/unittests/test_datasource/test_opennebula.py b/tests/unittests/test_datasource/test_opennebula.py
index 31c6232f..ef534bab 100644
--- a/tests/unittests/test_datasource/test_opennebula.py
+++ b/tests/unittests/test_datasource/test_opennebula.py
@@ -33,7 +33,7 @@ TEST_VARS = {
 }
 
 INVALID_CONTEXT = ';'
-USER_DATA = b'#cloud-config\napt_upgrade: true'
+USER_DATA = '#cloud-config\napt_upgrade: true'
 SSH_KEY = 'ssh-rsa AAAAB3NzaC1....sIkJhq8wdX+4I3A4cYbYP ubuntu@server-460-%i'
 HOSTNAME = 'foo.example.com'
 PUBLIC_IP = '10.0.0.3'
@@ -300,7 +300,7 @@ class TestParseShellConfig(unittest.TestCase):
 
 def populate_context_dir(path, variables):
     data = "# Context variables generated by OpenNebula\n"
-    for (k, v) in variables.items():
+    for k, v in variables.items():
         data += ("%s='%s'\n" % (k.upper(), v.replace(r"'", r"'\''")))
     populate_dir(path, {'context.sh': data})
 
-- 
cgit v1.2.3


From 18b35de06432869a9d859e2978e7e9567eba66a2 Mon Sep 17 00:00:00 2001
From: Barry Warsaw <barry@python.org>
Date: Mon, 26 Jan 2015 14:48:23 -0500
Subject: Another handling of b64decode.

Also, restore Python 2 compatibility.
---
 cloudinit/config/cc_seed_random.py        |  8 +++++++-
 cloudinit/sources/DataSourceOpenNebula.py | 11 ++++++++++-
 2 files changed, 17 insertions(+), 2 deletions(-)

(limited to 'cloudinit/sources')

diff --git a/cloudinit/config/cc_seed_random.py b/cloudinit/config/cc_seed_random.py
index 3b7235bf..981e1b08 100644
--- a/cloudinit/config/cc_seed_random.py
+++ b/cloudinit/config/cc_seed_random.py
@@ -38,7 +38,13 @@ def _decode(data, encoding=None):
     if not encoding or encoding.lower() in ['raw']:
         return data
     elif encoding.lower() in ['base64', 'b64']:
-        return base64.b64decode(data)
+        # Try to give us a native string in both Python 2 and 3, and remember
+        # that b64decode() returns bytes in Python 3.
+        decoded = base64.b64decode(data)
+        try:
+            return decoded.decode('utf-8')
+        except UnicodeDecodeError:
+            return decoded
     elif encoding.lower() in ['gzip', 'gz']:
         return util.decomp_gzip(data, quiet=False)
     else:
diff --git a/cloudinit/sources/DataSourceOpenNebula.py b/cloudinit/sources/DataSourceOpenNebula.py
index 691b39f8..6da569ec 100644
--- a/cloudinit/sources/DataSourceOpenNebula.py
+++ b/cloudinit/sources/DataSourceOpenNebula.py
@@ -25,6 +25,7 @@
 #    along with this program.  If not, see <http://www.gnu.org/licenses/>.
 
 import base64
+import codecs
 import os
 import pwd
 import re
@@ -34,6 +35,8 @@ from cloudinit import log as logging
 from cloudinit import sources
 from cloudinit import util
 
+import six
+
 LOG = logging.getLogger(__name__)
 
 DEFAULT_IID = "iid-dsopennebula"
@@ -43,6 +46,12 @@ CONTEXT_DISK_FILES = ["context.sh"]
 VALID_DSMODES = ("local", "net", "disabled")
 
 
+def utf8_open(path):
+    if six.PY3:
+        return open(path, 'r', encoding='utf-8')
+    return codecs.open(path, 'r', encoding='utf-8')
+
+
 class DataSourceOpenNebula(sources.DataSource):
     def __init__(self, sys_cfg, distro, paths):
         sources.DataSource.__init__(self, sys_cfg, distro, paths)
@@ -380,7 +389,7 @@ def read_context_disk_dir(source_dir, asuser=None):
                                            "does not exist", asuser)
         try:
             path = os.path.join(source_dir, 'context.sh')
-            with open(path, 'r', encoding='utf-8') as f:
+            with utf8_open(path) as f:
                 content = f.read().strip()
 
             context = parse_shell_config(content, asuser=asuser)
-- 
cgit v1.2.3


From fabff4aec884467729fc372bb67f240752c15511 Mon Sep 17 00:00:00 2001
From: Barry Warsaw <barry@python.org>
Date: Mon, 26 Jan 2015 16:37:29 -0500
Subject: Port the MAAS code to oauthlib.

---
 cloudinit/sources/DataSourceMAAS.py          | 56 ++++++++++++++++------------
 requirements.txt                             |  2 +-
 tests/unittests/test_datasource/test_maas.py |  7 +---
 3 files changed, 35 insertions(+), 30 deletions(-)

(limited to 'cloudinit/sources')

diff --git a/cloudinit/sources/DataSourceMAAS.py b/cloudinit/sources/DataSourceMAAS.py
index 8f9c81de..39296f08 100644
--- a/cloudinit/sources/DataSourceMAAS.py
+++ b/cloudinit/sources/DataSourceMAAS.py
@@ -22,10 +22,11 @@ from __future__ import print_function
 
 from email.utils import parsedate
 import errno
-import oauth.oauth as oauth
+import oauthlib
 import os
 import time
-import urllib2
+
+from six.moves.urllib_request import Request, urlopen
 
 from cloudinit import log as logging
 from cloudinit import sources
@@ -274,25 +275,34 @@ def check_seed_contents(content, seed):
 
 def oauth_headers(url, consumer_key, token_key, token_secret, consumer_secret,
                   timestamp=None):
-    consumer = oauth.OAuthConsumer(consumer_key, consumer_secret)
-    token = oauth.OAuthToken(token_key, token_secret)
-
-    if timestamp is None:
-        ts = int(time.time())
-    else:
-        ts = timestamp
-
-    params = {
-        'oauth_version': "1.0",
-        'oauth_nonce': oauth.generate_nonce(),
-        'oauth_timestamp': ts,
-        'oauth_token': token.key,
-        'oauth_consumer_key': consumer.key,
-    }
-    req = oauth.OAuthRequest(http_url=url, parameters=params)
-    req.sign_request(oauth.OAuthSignatureMethod_PLAINTEXT(),
-                     consumer, token)
-    return req.to_header()
+    client = oauthlib.oauth1.Client(
+        consumer_key,
+        client_secret=consumer_secret,
+        resource_owner_key=token_key,
+        resource_owner_secret=token_secret,
+        signature_method=oauthlib.SIGNATURE_PLAINTEXT)
+    uri, signed_headers, body = client.sign(url)
+    return signed_headers
+
+    ## consumer = oauth.OAuthConsumer(consumer_key, consumer_secret)
+    ## token = oauth.OAuthToken(token_key, token_secret)
+
+    ## if timestamp is None:
+    ##     ts = int(time.time())
+    ## else:
+    ##     ts = timestamp
+
+    ## params = {
+    ##     'oauth_version': "1.0",
+    ##     'oauth_nonce': oauth.generate_nonce(),
+    ##     'oauth_timestamp': ts,
+    ##     'oauth_token': token.key,
+    ##     'oauth_consumer_key': consumer.key,
+    ## }
+    ## req = oauth.OAuthRequest(http_url=url, parameters=params)
+    ## req.sign_request(oauth.OAuthSignatureMethod_PLAINTEXT(),
+    ##                  consumer, token)
+    ## return req.to_header()
 
 
 class MAASSeedDirNone(Exception):
@@ -359,8 +369,8 @@ if __name__ == "__main__":
                     creds[key] = cfg[key]
 
         def geturl(url, headers_cb):
-            req = urllib2.Request(url, data=None, headers=headers_cb(url))
-            return (urllib2.urlopen(req).read())
+            req = Request(url, data=None, headers=headers_cb(url))
+            return urlopen(req).read()
 
         def printurl(url, headers_cb):
             print("== %s ==\n%s\n" % (url, geturl(url, headers_cb)))
diff --git a/requirements.txt b/requirements.txt
index 2a12ca3e..19c88857 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -8,7 +8,7 @@ PrettyTable
 
 # This one is currently only used by the MAAS datasource. If that
 # datasource is removed, this is no longer needed
-oauth
+oauthlib
 
 # This one is currently used only by the CloudSigma and SmartOS datasources.
 # If these datasources are removed, this is no longer needed
diff --git a/tests/unittests/test_datasource/test_maas.py b/tests/unittests/test_datasource/test_maas.py
index 66fe22ae..6af0cd82 100644
--- a/tests/unittests/test_datasource/test_maas.py
+++ b/tests/unittests/test_datasource/test_maas.py
@@ -4,11 +4,7 @@ import shutil
 import tempfile
 import unittest
 
-# XXX DataSourceMAAS must be ported to oauthlib for Python 3
-import six
-if not six.PY3:
-    from cloudinit.sources import DataSourceMAAS
-
+from cloudinit.sources import DataSourceMAAS
 from cloudinit import url_helper
 from ..helpers import populate_dir
 
@@ -18,7 +14,6 @@ except ImportError:
     import mock
 
 
-@unittest.skipIf(six.PY3, 'DataSourceMAAS must be ported to oauthlib')
 class TestMAASDataSource(unittest.TestCase):
 
     def setUp(self):
-- 
cgit v1.2.3


From c3ced2d4bdbbbdcb2466202e1571d4ea7bfc7c72 Mon Sep 17 00:00:00 2001
From: Barry Warsaw <barry@python.org>
Date: Tue, 27 Jan 2015 14:36:10 -0500
Subject: Remove a comment turd.

---
 cloudinit/sources/DataSourceMAAS.py | 20 --------------------
 1 file changed, 20 deletions(-)

(limited to 'cloudinit/sources')

diff --git a/cloudinit/sources/DataSourceMAAS.py b/cloudinit/sources/DataSourceMAAS.py
index 39296f08..082cc58f 100644
--- a/cloudinit/sources/DataSourceMAAS.py
+++ b/cloudinit/sources/DataSourceMAAS.py
@@ -284,26 +284,6 @@ def oauth_headers(url, consumer_key, token_key, token_secret, consumer_secret,
     uri, signed_headers, body = client.sign(url)
     return signed_headers
 
-    ## consumer = oauth.OAuthConsumer(consumer_key, consumer_secret)
-    ## token = oauth.OAuthToken(token_key, token_secret)
-
-    ## if timestamp is None:
-    ##     ts = int(time.time())
-    ## else:
-    ##     ts = timestamp
-
-    ## params = {
-    ##     'oauth_version': "1.0",
-    ##     'oauth_nonce': oauth.generate_nonce(),
-    ##     'oauth_timestamp': ts,
-    ##     'oauth_token': token.key,
-    ##     'oauth_consumer_key': consumer.key,
-    ## }
-    ## req = oauth.OAuthRequest(http_url=url, parameters=params)
-    ## req.sign_request(oauth.OAuthSignatureMethod_PLAINTEXT(),
-    ##                  consumer, token)
-    ## return req.to_header()
-
 
 class MAASSeedDirNone(Exception):
     pass
-- 
cgit v1.2.3


From 69c64029997599b3f1764ef48fe571094e2ee5f2 Mon Sep 17 00:00:00 2001
From: Barry Warsaw <barry@python.org>
Date: Tue, 27 Jan 2015 14:40:05 -0500
Subject: Respond to review:

- Just use util.load_file() instead of yet another way to open and read the
  file.
---
 cloudinit/sources/DataSourceOpenNebula.py | 12 +-----------
 1 file changed, 1 insertion(+), 11 deletions(-)

(limited to 'cloudinit/sources')

diff --git a/cloudinit/sources/DataSourceOpenNebula.py b/cloudinit/sources/DataSourceOpenNebula.py
index 6da569ec..a0275cda 100644
--- a/cloudinit/sources/DataSourceOpenNebula.py
+++ b/cloudinit/sources/DataSourceOpenNebula.py
@@ -25,7 +25,6 @@
 #    along with this program.  If not, see <http://www.gnu.org/licenses/>.
 
 import base64
-import codecs
 import os
 import pwd
 import re
@@ -35,7 +34,6 @@ from cloudinit import log as logging
 from cloudinit import sources
 from cloudinit import util
 
-import six
 
 LOG = logging.getLogger(__name__)
 
@@ -46,12 +44,6 @@ CONTEXT_DISK_FILES = ["context.sh"]
 VALID_DSMODES = ("local", "net", "disabled")
 
 
-def utf8_open(path):
-    if six.PY3:
-        return open(path, 'r', encoding='utf-8')
-    return codecs.open(path, 'r', encoding='utf-8')
-
-
 class DataSourceOpenNebula(sources.DataSource):
     def __init__(self, sys_cfg, distro, paths):
         sources.DataSource.__init__(self, sys_cfg, distro, paths)
@@ -389,9 +381,7 @@ def read_context_disk_dir(source_dir, asuser=None):
                                            "does not exist", asuser)
         try:
             path = os.path.join(source_dir, 'context.sh')
-            with utf8_open(path) as f:
-                content = f.read().strip()
-
+            content = util.load_file(path)
             context = parse_shell_config(content, asuser=asuser)
         except util.ProcessExecutionError as e:
             raise BrokenContextDiskDir("Error processing context.sh: %s" % (e))
-- 
cgit v1.2.3


From 6e742d20e9ed56498925c7c850cd5da65d063b4b Mon Sep 17 00:00:00 2001
From: Barry Warsaw <barry@python.org>
Date: Tue, 27 Jan 2015 15:03:52 -0500
Subject: Respond to review:

- Refactor both the base64 encoding and decoding into utility functions.

Also:

- Mechanically fix some other broken untested code.
---
 cloudinit/config/cc_seed_random.py                 |  8 +------
 cloudinit/config/cc_ssh_authkey_fingerprints.py    |  2 +-
 cloudinit/sources/DataSourceOpenNebula.py          |  7 +-----
 cloudinit/sources/DataSourceSmartOS.py             | 11 +--------
 cloudinit/util.py                                  | 20 ++++++++++++++++
 tests/unittests/test_datasource/test_azure.py      | 28 ++++++++--------------
 tests/unittests/test_datasource/test_opennebula.py | 11 ++-------
 tests/unittests/test_datasource/test_smartos.py    | 14 ++++-------
 .../test_handler/test_handler_seed_random.py       | 12 ++--------
 9 files changed, 42 insertions(+), 71 deletions(-)

(limited to 'cloudinit/sources')

diff --git a/cloudinit/config/cc_seed_random.py b/cloudinit/config/cc_seed_random.py
index 981e1b08..bb64b0f5 100644
--- a/cloudinit/config/cc_seed_random.py
+++ b/cloudinit/config/cc_seed_random.py
@@ -38,13 +38,7 @@ def _decode(data, encoding=None):
     if not encoding or encoding.lower() in ['raw']:
         return data
     elif encoding.lower() in ['base64', 'b64']:
-        # Try to give us a native string in both Python 2 and 3, and remember
-        # that b64decode() returns bytes in Python 3.
-        decoded = base64.b64decode(data)
-        try:
-            return decoded.decode('utf-8')
-        except UnicodeDecodeError:
-            return decoded
+        return util.b64d(data)
     elif encoding.lower() in ['gzip', 'gz']:
         return util.decomp_gzip(data, quiet=False)
     else:
diff --git a/cloudinit/config/cc_ssh_authkey_fingerprints.py b/cloudinit/config/cc_ssh_authkey_fingerprints.py
index 51580633..6ce831bc 100644
--- a/cloudinit/config/cc_ssh_authkey_fingerprints.py
+++ b/cloudinit/config/cc_ssh_authkey_fingerprints.py
@@ -32,7 +32,7 @@ from cloudinit import util
 
 def _split_hash(bin_hash):
     split_up = []
-    for i in xrange(0, len(bin_hash), 2):
+    for i in range(0, len(bin_hash), 2):
         split_up.append(bin_hash[i:i + 2])
     return split_up
 
diff --git a/cloudinit/sources/DataSourceOpenNebula.py b/cloudinit/sources/DataSourceOpenNebula.py
index a0275cda..61709c1b 100644
--- a/cloudinit/sources/DataSourceOpenNebula.py
+++ b/cloudinit/sources/DataSourceOpenNebula.py
@@ -426,12 +426,7 @@ def read_context_disk_dir(source_dir, asuser=None):
                                context.get('USER_DATA_ENCODING'))
         if encoding == "base64":
             try:
-                userdata = base64.b64decode(results['userdata'])
-                # In Python 3 we still expect a str, but b64decode will return
-                # bytes.  Convert to str.
-                if isinstance(userdata, bytes):
-                    userdata = userdata.decode('utf-8')
-                results['userdata'] = userdata
+                results['userdata'] = util.b64d(results['userdata'])
             except TypeError:
                 LOG.warn("Failed base64 decoding of userdata")
 
diff --git a/cloudinit/sources/DataSourceSmartOS.py b/cloudinit/sources/DataSourceSmartOS.py
index f59ad3d6..9d48beab 100644
--- a/cloudinit/sources/DataSourceSmartOS.py
+++ b/cloudinit/sources/DataSourceSmartOS.py
@@ -351,16 +351,7 @@ def query_data(noun, seed_device, seed_timeout, strip=False, default=None,
 
     if b64:
         try:
-            # Generally, we want native strings in the values.  Python 3's
-            # b64decode will return bytes though, so decode them to utf-8 if
-            # possible.  If that fails, return the bytes.
-            decoded = base64.b64decode(resp)
-            try:
-                if isinstance(decoded, bytes):
-                    return decoded.decode('utf-8')
-            except UnicodeDecodeError:
-                pass
-            return decoded
+            return util.b64d(resp)
         # Bogus input produces different errors in Python 2 and 3; catch both.
         except (TypeError, binascii.Error):
             LOG.warn("Failed base64 decoding key '%s'", noun)
diff --git a/cloudinit/util.py b/cloudinit/util.py
index 766f8e32..8916cc11 100644
--- a/cloudinit/util.py
+++ b/cloudinit/util.py
@@ -44,6 +44,7 @@ import sys
 import tempfile
 import time
 
+from base64 import b64decode, b64encode
 from six.moves.urllib import parse as urlparse
 
 import six
@@ -90,6 +91,25 @@ def encode_text(text, encoding='utf-8'):
         return text
     return text.encode(encoding)
 
+
+def b64d(source):
+    # Base64 decode some data, accepting bytes or unicode/str, and returning
+    # str/unicode if the result is utf-8 compatible, otherwise returning bytes.
+    decoded = b64decode(source)
+    if isinstance(decoded, bytes):
+        try:
+            return decoded.decode('utf-8')
+        except UnicodeDecodeError:
+            return decoded
+
+def b64e(source):
+    # Base64 encode some data, accepting bytes or unicode/str, and returning
+    # str/unicode if the result is utf-8 compatible, otherwise returning bytes.
+    if not isinstance(source, bytes):
+        source = source.encode('utf-8')
+    return b64encode(source).decode('utf-8')
+
+
 # Path for DMI Data
 DMI_SYS_PATH = "/sys/class/dmi/id"
 
diff --git a/tests/unittests/test_datasource/test_azure.py b/tests/unittests/test_datasource/test_azure.py
index 97a53bee..965bce4b 100644
--- a/tests/unittests/test_datasource/test_azure.py
+++ b/tests/unittests/test_datasource/test_azure.py
@@ -1,5 +1,5 @@
 from cloudinit import helpers
-from cloudinit.util import load_file
+from cloudinit.util import b64e, load_file
 from cloudinit.sources import DataSourceAzure
 from ..helpers import TestCase, populate_dir
 
@@ -12,7 +12,6 @@ try:
 except ImportError:
     from contextlib2 import ExitStack
 
-import base64
 import crypt
 import os
 import stat
@@ -22,13 +21,6 @@ import tempfile
 import unittest
 
 
-def b64(source):
-    # In Python 3, b64encode only accepts bytes and returns bytes.
-    if not isinstance(source, bytes):
-        source = source.encode('utf-8')
-    return base64.b64encode(source).decode('us-ascii')
-
-
 def construct_valid_ovf_env(data=None, pubkeys=None, userdata=None):
     if data is None:
         data = {'HostName': 'FOOHOST'}
@@ -58,7 +50,7 @@ def construct_valid_ovf_env(data=None, pubkeys=None, userdata=None):
         content += "<%s%s>%s</%s>\n" % (key, attrs, val, key)
 
     if userdata:
-        content += "<UserData>%s</UserData>\n" % (b64(userdata))
+        content += "<UserData>%s</UserData>\n" % (b64e(userdata))
 
     if pubkeys:
         content += "<SSH><PublicKeys>\n"
@@ -189,7 +181,7 @@ class TestAzureDataSource(TestCase):
         # set dscfg in via base64 encoded yaml
         cfg = {'agent_command': "my_command"}
         odata = {'HostName': "myhost", 'UserName': "myuser",
-                'dscfg': {'text': b64(yaml.dump(cfg)),
+                'dscfg': {'text': b64e(yaml.dump(cfg)),
                           'encoding': 'base64'}}
         data = {'ovfcontent': construct_valid_ovf_env(data=odata)}
 
@@ -241,7 +233,7 @@ class TestAzureDataSource(TestCase):
 
     def test_userdata_found(self):
         mydata = "FOOBAR"
-        odata = {'UserData': b64(mydata)}
+        odata = {'UserData': b64e(mydata)}
         data = {'ovfcontent': construct_valid_ovf_env(data=odata)}
 
         dsrc = self._get_ds(data)
@@ -289,7 +281,7 @@ class TestAzureDataSource(TestCase):
                                    'command': 'my-bounce-command',
                                    'hostname_command': 'my-hostname-command'}}
         odata = {'HostName': "xhost",
-                'dscfg': {'text': b64(yaml.dump(cfg)),
+                'dscfg': {'text': b64e(yaml.dump(cfg)),
                           'encoding': 'base64'}}
         data = {'ovfcontent': construct_valid_ovf_env(data=odata)}
         self._get_ds(data).get_data()
@@ -304,7 +296,7 @@ class TestAzureDataSource(TestCase):
         # config specifying set_hostname off should not bounce
         cfg = {'set_hostname': False}
         odata = {'HostName': "xhost",
-                'dscfg': {'text': b64(yaml.dump(cfg)),
+                'dscfg': {'text': b64e(yaml.dump(cfg)),
                           'encoding': 'base64'}}
         data = {'ovfcontent': construct_valid_ovf_env(data=odata)}
         self._get_ds(data).get_data()
@@ -333,7 +325,7 @@ class TestAzureDataSource(TestCase):
         # Make sure that user can affect disk aliases
         dscfg = {'disk_aliases': {'ephemeral0': '/dev/sdc'}}
         odata = {'HostName': "myhost", 'UserName': "myuser",
-                'dscfg': {'text': b64(yaml.dump(dscfg)),
+                'dscfg': {'text': b64e(yaml.dump(dscfg)),
                           'encoding': 'base64'}}
         usercfg = {'disk_setup': {'/dev/sdc': {'something': '...'},
                                   'ephemeral0': False}}
@@ -370,7 +362,7 @@ class TestAzureDataSource(TestCase):
 
     def test_existing_ovf_same(self):
         # waagent/SharedConfig left alone if found ovf-env.xml same as cached
-        odata = {'UserData': b64("SOMEUSERDATA")}
+        odata = {'UserData': b64e("SOMEUSERDATA")}
         data = {'ovfcontent': construct_valid_ovf_env(data=odata)}
 
         populate_dir(self.waagent_d,
@@ -394,9 +386,9 @@ class TestAzureDataSource(TestCase):
         # 'get_data' should remove SharedConfig.xml in /var/lib/waagent
         # if ovf-env.xml differs.
         cached_ovfenv = construct_valid_ovf_env(
-            {'userdata': b64("FOO_USERDATA")})
+            {'userdata': b64e("FOO_USERDATA")})
         new_ovfenv = construct_valid_ovf_env(
-            {'userdata': b64("NEW_USERDATA")})
+            {'userdata': b64e("NEW_USERDATA")})
 
         populate_dir(self.waagent_d,
             {'ovf-env.xml': cached_ovfenv,
diff --git a/tests/unittests/test_datasource/test_opennebula.py b/tests/unittests/test_datasource/test_opennebula.py
index e5a4bd18..27adf21b 100644
--- a/tests/unittests/test_datasource/test_opennebula.py
+++ b/tests/unittests/test_datasource/test_opennebula.py
@@ -3,19 +3,12 @@ from cloudinit.sources import DataSourceOpenNebula as ds
 from cloudinit import util
 from ..helpers import TestCase, populate_dir
 
-from base64 import b64encode
 import os
 import pwd
 import shutil
 import tempfile
 import unittest
 
-def b64(source):
-    # In Python 3, b64encode only accepts bytes and returns bytes.
-    if not isinstance(source, bytes):
-        source = source.encode('utf-8')
-    return b64encode(source).decode('us-ascii')
-
 
 TEST_VARS = {
     'VAR1': 'single',
@@ -186,7 +179,7 @@ class TestOpenNebulaDataSource(TestCase):
             self.assertEqual(USER_DATA, results['userdata'])
 
     def test_user_data_encoding_required_for_decode(self):
-        b64userdata = b64(USER_DATA)
+        b64userdata = util.b64e(USER_DATA)
         for k in ('USER_DATA', 'USERDATA'):
             my_d = os.path.join(self.tmp, k)
             populate_context_dir(my_d, {k: b64userdata})
@@ -198,7 +191,7 @@ class TestOpenNebulaDataSource(TestCase):
     def test_user_data_base64_encoding(self):
         for k in ('USER_DATA', 'USERDATA'):
             my_d = os.path.join(self.tmp, k)
-            populate_context_dir(my_d, {k: b64(USER_DATA),
+            populate_context_dir(my_d, {k: util.b64e(USER_DATA),
                                         'USERDATA_ENCODING': 'base64'})
             results = ds.read_context_disk_dir(my_d)
 
diff --git a/tests/unittests/test_datasource/test_smartos.py b/tests/unittests/test_datasource/test_smartos.py
index b5ebf94d..8b62b1b1 100644
--- a/tests/unittests/test_datasource/test_smartos.py
+++ b/tests/unittests/test_datasource/test_smartos.py
@@ -24,9 +24,9 @@
 
 from __future__ import print_function
 
-import base64
 from cloudinit import helpers as c_helpers
 from cloudinit.sources import DataSourceSmartOS
+from cloudinit.util import b64e
 from .. import helpers
 import os
 import os.path
@@ -36,12 +36,6 @@ import tempfile
 import stat
 import uuid
 
-def b64(source):
-    # In Python 3, b64encode only accepts bytes and returns bytes.
-    if not isinstance(source, bytes):
-        source = source.encode('utf-8')
-    return base64.b64encode(source).decode('us-ascii')
-
 
 MOCK_RETURNS = {
     'hostname': 'test-host',
@@ -239,7 +233,7 @@ class TestSmartOSDataSource(helpers.FilesystemMockingTestCase):
         my_returns = MOCK_RETURNS.copy()
         my_returns['base64_all'] = "true"
         for k in ('hostname', 'cloud-init:user-data'):
-            my_returns[k] = b64(my_returns[k])
+            my_returns[k] = b64e(my_returns[k])
 
         dsrc = self._get_ds(mockdata=my_returns)
         ret = dsrc.get_data()
@@ -260,7 +254,7 @@ class TestSmartOSDataSource(helpers.FilesystemMockingTestCase):
         my_returns['b64-cloud-init:user-data'] = "true"
         my_returns['b64-hostname'] = "true"
         for k in ('hostname', 'cloud-init:user-data'):
-            my_returns[k] = b64(my_returns[k])
+            my_returns[k] = b64e(my_returns[k])
 
         dsrc = self._get_ds(mockdata=my_returns)
         ret = dsrc.get_data()
@@ -276,7 +270,7 @@ class TestSmartOSDataSource(helpers.FilesystemMockingTestCase):
         my_returns = MOCK_RETURNS.copy()
         my_returns['base64_keys'] = 'hostname,ignored'
         for k in ('hostname',):
-            my_returns[k] = b64(my_returns[k])
+            my_returns[k] = b64e(my_returns[k])
 
         dsrc = self._get_ds(mockdata=my_returns)
         ret = dsrc.get_data()
diff --git a/tests/unittests/test_handler/test_handler_seed_random.py b/tests/unittests/test_handler/test_handler_seed_random.py
index d3f18fa0..0bcdcb31 100644
--- a/tests/unittests/test_handler/test_handler_seed_random.py
+++ b/tests/unittests/test_handler/test_handler_seed_random.py
@@ -18,7 +18,6 @@
 
 from cloudinit.config import cc_seed_random
 
-import base64
 import gzip
 import tempfile
 
@@ -38,13 +37,6 @@ import logging
 LOG = logging.getLogger(__name__)
 
 
-def b64(source):
-    # In Python 3, b64encode only accepts bytes and returns bytes.
-    if not isinstance(source, bytes):
-        source = source.encode('utf-8')
-    return base64.b64encode(source).decode('us-ascii')
-
-
 class TestRandomSeed(t_help.TestCase):
     def setUp(self):
         super(TestRandomSeed, self).setUp()
@@ -141,7 +133,7 @@ class TestRandomSeed(t_help.TestCase):
         self.assertEquals("big-toe", contents)
 
     def test_append_random_base64(self):
-        data = b64('bubbles')
+        data = util.b64e('bubbles')
         cfg = {
             'random_seed': {
                 'file': self._seed_file,
@@ -154,7 +146,7 @@ class TestRandomSeed(t_help.TestCase):
         self.assertEquals("bubbles", contents)
 
     def test_append_random_b64(self):
-        data = b64('kit-kat')
+        data = util.b64e('kit-kat')
         cfg = {
             'random_seed': {
                 'file': self._seed_file,
-- 
cgit v1.2.3


From f62b86bd45c8df78ada32ab4040a639c9d096202 Mon Sep 17 00:00:00 2001
From: Scott Moser <smoser@ubuntu.com>
Date: Wed, 11 Feb 2015 01:09:34 +0000
Subject: fix random_seed module

---
 cloudinit/config/cc_seed_random.py   | 16 ++++++++--------
 cloudinit/sources/DataSourceAzure.py |  3 ++-
 cloudinit/util.py                    | 16 +++++++++-------
 3 files changed, 19 insertions(+), 16 deletions(-)

(limited to 'cloudinit/sources')

diff --git a/cloudinit/config/cc_seed_random.py b/cloudinit/config/cc_seed_random.py
index bb64b0f5..3288a853 100644
--- a/cloudinit/config/cc_seed_random.py
+++ b/cloudinit/config/cc_seed_random.py
@@ -22,7 +22,7 @@
 import base64
 import os
 
-from six import StringIO
+from six import BytesIO
 
 from cloudinit.settings import PER_INSTANCE
 from cloudinit import log as logging
@@ -34,13 +34,13 @@ LOG = logging.getLogger(__name__)
 
 def _decode(data, encoding=None):
     if not data:
-        return ''
+        return b''
     if not encoding or encoding.lower() in ['raw']:
-        return data
+        return util.encode_text(data)
     elif encoding.lower() in ['base64', 'b64']:
-        return util.b64d(data)
+        return base64.b64decode(data)
     elif encoding.lower() in ['gzip', 'gz']:
-        return util.decomp_gzip(data, quiet=False)
+        return util.decomp_gzip(data, quiet=False, decode=None)
     else:
         raise IOError("Unknown random_seed encoding: %s" % (encoding))
 
@@ -65,9 +65,9 @@ def handle_random_seed_command(command, required, env=None):
 def handle(name, cfg, cloud, log, _args):
     mycfg = cfg.get('random_seed', {})
     seed_path = mycfg.get('file', '/dev/urandom')
-    seed_data = mycfg.get('data', '')
+    seed_data = mycfg.get('data', b'')
 
-    seed_buf = StringIO()
+    seed_buf = BytesIO()
     if seed_data:
         seed_buf.write(_decode(seed_data, encoding=mycfg.get('encoding')))
 
@@ -75,7 +75,7 @@ def handle(name, cfg, cloud, log, _args):
     # openstack meta_data.json
     metadata = cloud.datasource.metadata
     if metadata and 'random_seed' in metadata:
-        seed_buf.write(metadata['random_seed'])
+        seed_buf.write(util.encode_text(metadata['random_seed']))
 
     seed_data = seed_buf.getvalue()
     if len(seed_data):
diff --git a/cloudinit/sources/DataSourceAzure.py b/cloudinit/sources/DataSourceAzure.py
index 29ae2c22..c599d50f 100644
--- a/cloudinit/sources/DataSourceAzure.py
+++ b/cloudinit/sources/DataSourceAzure.py
@@ -124,7 +124,8 @@ class DataSourceAzureNet(sources.DataSource):
             LOG.debug("using files cached in %s", ddir)
 
         # azure / hyper-v provides random data here
-        seed = util.load_file("/sys/firmware/acpi/tables/OEM0", quiet=True)
+        seed = util.load_file("/sys/firmware/acpi/tables/OEM0",
+                              quiet=True, decode=False)
         if seed:
             self.metadata['random_seed'] = seed
 
diff --git a/cloudinit/util.py b/cloudinit/util.py
index 3a921afe..c998154a 100644
--- a/cloudinit/util.py
+++ b/cloudinit/util.py
@@ -96,11 +96,10 @@ def b64d(source):
     # Base64 decode some data, accepting bytes or unicode/str, and returning
     # str/unicode if the result is utf-8 compatible, otherwise returning bytes.
     decoded = b64decode(source)
-    if isinstance(decoded, bytes):
-        try:
-            return decoded.decode('utf-8')
-        except UnicodeDecodeError:
-            return decoded
+    try:
+        return decoded.decode('utf-8')
+    except UnicodeDecodeError:
+        return decoded
 
 def b64e(source):
     # Base64 encode some data, accepting bytes or unicode/str, and returning
@@ -354,11 +353,14 @@ def clean_filename(fn):
     return fn
 
 
-def decomp_gzip(data, quiet=True):
+def decomp_gzip(data, quiet=True, decode=True):
     try:
         buf = six.BytesIO(encode_text(data))
         with contextlib.closing(gzip.GzipFile(None, "rb", 1, buf)) as gh:
-            return decode_binary(gh.read())
+            if decode:
+                return decode_binary(gh.read())
+            else:
+                return gh.read()
     except Exception as e:
         if quiet:
             return data
-- 
cgit v1.2.3