* More str/bytes fixes.

* Temporarily skip the MAAS tests in py3 since they need to be ported to oauthlib.

4 files changed, 30 insertions, 8 deletions

diff --git a/cloudinit/sources/DataSourceOpenNebula.py b/cloudinit/sources/DataSourceOpenNebula.py
index f9dac29e..691b39f8 100644
--- a/cloudinit/sources/DataSourceOpenNebula.py
+++ b/cloudinit/sources/DataSourceOpenNebula.py
@@ -379,7 +379,8 @@ def read_context_disk_dir(source_dir, asuser=None):
                 raise BrokenContextDiskDir("configured user '%s' "
                                            "does not exist", asuser)
         try:
-            with open(os.path.join(source_dir, 'context.sh'), 'r') as f:
+            path = os.path.join(source_dir, 'context.sh')
+            with open(path, 'r', encoding='utf-8') as f:
                 content = f.read().strip()
 
             context = parse_shell_config(content, asuser=asuser)
@@ -426,14 +427,19 @@ def read_context_disk_dir(source_dir, asuser=None):
                                context.get('USER_DATA_ENCODING'))
         if encoding == "base64":
             try:
-                results['userdata'] = base64.b64decode(results['userdata'])
+                userdata = base64.b64decode(results['userdata'])
+                # In Python 3 we still expect a str, but b64decode will return
+                # bytes.  Convert to str.
+                if isinstance(userdata, bytes):
+                    userdata = userdata.decode('utf-8')
+                results['userdata'] = userdata
             except TypeError:
                 LOG.warn("Failed base64 decoding of userdata")
 
     # generate static /etc/network/interfaces
     # only if there are any required context variables
     # http://opennebula.org/documentation:rel3.8:cong#network_configuration
-    for k in context.keys():
+    for k in context:
         if re.match(r'^ETH\d+_IP$', k):
             (out, _) = util.subp(['/sbin/ip', 'link'])
             net = OpenNebulaNetwork(out, context)
diff --git a/cloudinit/sources/DataSourceSmartOS.py b/cloudinit/sources/DataSourceSmartOS.py
index 7a975d78..d3ed40c5 100644
--- a/cloudinit/sources/DataSourceSmartOS.py
+++ b/cloudinit/sources/DataSourceSmartOS.py
@@ -30,6 +30,7 @@
 #       Comments with "@datadictionary" are snippets of the definition
 
 import base64
+import binascii
 import os
 import serial
 
@@ -350,8 +351,18 @@ def query_data(noun, seed_device, seed_timeout, strip=False, default=None,
 
     if b64:
         try:
-            return base64.b64decode(resp)
-        except TypeError:
+            # Generally, we want native strings in the values.  Python 3's
+            # b64decode will return bytes though, so decode them to utf-8 if
+            # possible.  If that fails, return the bytes.
+            decoded = base64.b64decode(resp)
+            try:
+                if isinstance(decoded, bytes):
+                    return decoded.decode('utf-8')
+            except UnicodeDecodeError:
+                pass
+            return decoded
+        # Bogus input produces different errors in Python 2 and 3; catch both.
+        except (TypeError, binascii.Error):
             LOG.warn("Failed base64 decoding key '%s'", noun)
             return resp
 
diff --git a/tests/unittests/test_datasource/test_maas.py b/tests/unittests/test_datasource/test_maas.py
index 6af0cd82..66fe22ae 100644
--- a/tests/unittests/test_datasource/test_maas.py
+++ b/tests/unittests/test_datasource/test_maas.py
@@ -4,7 +4,11 @@ import shutil
 import tempfile
 import unittest
 
-from cloudinit.sources import DataSourceMAAS
+# XXX DataSourceMAAS must be ported to oauthlib for Python 3
+import six
+if not six.PY3:
+    from cloudinit.sources import DataSourceMAAS
+
 from cloudinit import url_helper
 from ..helpers import populate_dir
 
@@ -14,6 +18,7 @@ except ImportError:
     import mock
 
 
+@unittest.skipIf(six.PY3, 'DataSourceMAAS must be ported to oauthlib')
 class TestMAASDataSource(unittest.TestCase):
 
     def setUp(self):
diff --git a/tests/unittests/test_datasource/test_opennebula.py b/tests/unittests/test_datasource/test_opennebula.py
index 31c6232f..ef534bab 100644
--- a/tests/unittests/test_datasource/test_opennebula.py
+++ b/tests/unittests/test_datasource/test_opennebula.py
@@ -33,7 +33,7 @@ TEST_VARS = {
 }
 
 INVALID_CONTEXT = ';'
-USER_DATA = b'#cloud-config\napt_upgrade: true'
+USER_DATA = '#cloud-config\napt_upgrade: true'
 SSH_KEY = 'ssh-rsa AAAAB3NzaC1....sIkJhq8wdX+4I3A4cYbYP ubuntu@server-460-%i'
 HOSTNAME = 'foo.example.com'
 PUBLIC_IP = '10.0.0.3'
@@ -300,7 +300,7 @@ class TestParseShellConfig(unittest.TestCase):
 
 def populate_context_dir(path, variables):
     data = "# Context variables generated by OpenNebula\n"
-    for (k, v) in variables.items():
+    for k, v in variables.items():
         data += ("%s='%s'\n" % (k.upper(), v.replace(r"'", r"'\''")))
     populate_dir(path, {'context.sh': data})