readurl, read_file_or_url returns bytes, user must convert as necessary

 * explicitly test compressed user-data.
 * userdata_raw is now bytes
 * add load_tfile_or_url for loading text file or url
 * ec2_utils: all meta-data is text, remove non-obvious string translations
 * DigitalOcean: adjust for ec2_utils
 * DataSourceGCE, DataSourceMAAS: user-data is binary other fields are text.
 * openstack.py: read paths without decoding to text.  This is ok as paths
                 other than user-data are json, and load_json will handle
 * load_file still returns text, and that is what most things use.

LP: #1424900

1 files changed, 11 insertions, 3 deletions

diff --git a/cloudinit/sources/DataSourceMAAS.py b/cloudinit/sources/DataSourceMAAS.py
index 082cc58f..35c5b5e1 100644
--- a/cloudinit/sources/DataSourceMAAS.py
+++ b/cloudinit/sources/DataSourceMAAS.py
@@ -36,6 +36,8 @@ from cloudinit import util
 LOG = logging.getLogger(__name__)
 MD_VERSION = "2012-03-01"
 
+BINARY_FIELDS = ('user-data',)
+
 
 class DataSourceMAAS(sources.DataSource):
     """
@@ -185,7 +187,9 @@ def read_maas_seed_dir(seed_d):
     md = {}
     for fname in files:
         try:
-            md[fname] = util.load_file(os.path.join(seed_d, fname))
+            print("fname: %s / %s" % (fname, fname not in BINARY_FIELDS))
+            md[fname] = util.load_file(os.path.join(seed_d, fname),
+                                       decode=fname not in BINARY_FIELDS)
         except IOError as e:
             if e.errno != errno.ENOENT:
                 raise
@@ -218,6 +222,7 @@ def read_maas_seed_url(seed_url, header_cb=None, timeout=None,
         'public-keys': "%s/%s" % (base_url, 'meta-data/public-keys'),
         'user-data': "%s/%s" % (base_url, 'user-data'),
     }
+
     md = {}
     for name in file_order:
         url = files.get(name)
@@ -238,7 +243,10 @@ def read_maas_seed_url(seed_url, header_cb=None, timeout=None,
                                          timeout=timeout,
                                          ssl_details=ssl_details)
             if resp.ok():
-                md[name] = str(resp)
+                if name in BINARY_FIELDS:
+                    md[name] = resp.contents
+                else:
+                    md[name] = util.decode_binary(resp.contents)
             else:
                 LOG.warn(("Fetching from %s resulted in"
                           " an invalid http code %s"), url, resp.code)
@@ -263,7 +271,7 @@ def check_seed_contents(content, seed):
     if len(missing):
         raise MAASSeedDirMalformed("%s: missing files %s" % (seed, missing))
 
-    userdata = content.get('user-data', "")
+    userdata = content.get('user-data', b"")
     md = {}
     for (key, val) in content.items():
         if key == 'user-data':