move towards user-data being binary

UrlResponse: biggest change... make readurl return bytes, making user
             know what to do with it.
util: add load_tfile_or_url for loading text file or url
      as  read_file_or_url now returns bytes

ec2_utils: all meta-data is text, remove non-obvious string translations
DigitalOcean: adjust for ec2_utils

DataSourceGCE, DataSourceMAAS: user-data is binary other fields are text.
openstack.py: read paths without decoding to text.  This is ok as paths
              other than user-data are json, and load_json will handle

load_file still returns text, and that is what most things use.

7 files changed, 50 insertions, 22 deletions

diff --git a/cloudinit/ec2_utils.py b/cloudinit/ec2_utils.py
index e1ed4091..7cf99186 100644
--- a/cloudinit/ec2_utils.py
+++ b/cloudinit/ec2_utils.py
@@ -41,6 +41,10 @@ class MetadataLeafDecoder(object):
     def __call__(self, field, blob):
         if not blob:
             return blob
+        try:
+            blob = util.decode_binary(blob)
+        except UnicodeDecodeError:
+            return blob
         if self._maybe_json_object(blob):
             try:
                 # Assume it's json, unless it fails parsing...
@@ -69,6 +73,8 @@ class MetadataMaterializer(object):
     def _parse(self, blob):
         leaves = {}
         children = []
+        blob = util.decode_binary(blob)
+
         if not blob:
             return (leaves, children)
 
@@ -117,12 +123,12 @@ class MetadataMaterializer(object):
             child_url = url_helper.combine_url(base_url, c)
             if not child_url.endswith("/"):
                 child_url += "/"
-            child_blob = str(self._caller(child_url))
+            child_blob = self._caller(child_url)
             child_contents[c] = self._materialize(child_blob, child_url)
         leaf_contents = {}
         for (field, resource) in leaves.items():
             leaf_url = url_helper.combine_url(base_url, resource)
-            leaf_blob = self._caller(leaf_url).contents
+            leaf_blob = self._caller(leaf_url)
             leaf_contents[field] = self._leaf_decoder(field, leaf_blob)
         joined = {}
         joined.update(child_contents)
@@ -179,11 +185,13 @@ def get_instance_metadata(api_version='latest',
     caller = functools.partial(util.read_file_or_url,
                                ssl_details=ssl_details, timeout=timeout,
                                retries=retries)
+    def mcaller(url):
+        return caller(url).contents
 
     try:
         response = caller(md_url)
         materializer = MetadataMaterializer(response.contents,
-                                            md_url, caller,
+                                            md_url, mcaller,
                                             leaf_decoder=leaf_decoder)
         md = materializer.materialize()
         if not isinstance(md, (dict)):
diff --git a/cloudinit/sources/DataSourceDigitalOcean.py b/cloudinit/sources/DataSourceDigitalOcean.py
index 76ddaa9d..5d47564d 100644
--- a/cloudinit/sources/DataSourceDigitalOcean.py
+++ b/cloudinit/sources/DataSourceDigitalOcean.py
@@ -54,9 +54,13 @@ class DataSourceDigitalOcean(sources.DataSource):
     def get_data(self):
         caller = functools.partial(util.read_file_or_url,
                                    timeout=self.timeout, retries=self.retries)
-        md = ec2_utils.MetadataMaterializer(str(caller(self.metadata_address)),
+
+        def mcaller(url):
+            return caller(url).contents
+
+        md = ec2_utils.MetadataMaterializer(mcaller(self.metadata_address),
                                             base_url=self.metadata_address,
-                                            caller=caller)
+                                            caller=mcaller)
 
         self.metadata = md.materialize()
 
diff --git a/cloudinit/sources/DataSourceGCE.py b/cloudinit/sources/DataSourceGCE.py
index 6936c74e..608c07f1 100644
--- a/cloudinit/sources/DataSourceGCE.py
+++ b/cloudinit/sources/DataSourceGCE.py
@@ -53,15 +53,15 @@ class DataSourceGCE(sources.DataSource):
         # GCE metadata server requires a custom header since v1
         headers = {'X-Google-Metadata-Request': True}
 
-        # url_map: (our-key, path, required)
+        # url_map: (our-key, path, required, is_text)
         url_map = [
-            ('instance-id', 'instance/id', True),
-            ('availability-zone', 'instance/zone', True),
-            ('local-hostname', 'instance/hostname', True),
-            ('public-keys', 'project/attributes/sshKeys', False),
-            ('user-data', 'instance/attributes/user-data', False),
+            ('instance-id', 'instance/id', True, True),
+            ('availability-zone', 'instance/zone', True, True),
+            ('local-hostname', 'instance/hostname', True, True),
+            ('public-keys', 'project/attributes/sshKeys', False, True),
+            ('user-data', 'instance/attributes/user-data', False, False),
             ('user-data-encoding', 'instance/attributes/user-data-encoding',
-             False),
+             False, True),
         ]
 
         # if we cannot resolve the metadata server, then no point in trying
@@ -71,13 +71,16 @@ class DataSourceGCE(sources.DataSource):
 
         # iterate over url_map keys to get metadata items
         found = False
-        for (mkey, path, required) in url_map:
+        for (mkey, path, required, is_text) in url_map:
             try:
                 resp = url_helper.readurl(url=self.metadata_address + path,
                                           headers=headers)
                 if resp.code == 200:
                     found = True
-                    self.metadata[mkey] = resp.contents
+                    if is_text:
+                        self.metadata[mkey] = util.decode_binary(resp.contents)
+                    else:
+                        self.metadata[mkey] = resp.contents
                 else:
                     if required:
                         msg = "required url %s returned code %s. not GCE"
diff --git a/cloudinit/sources/DataSourceMAAS.py b/cloudinit/sources/DataSourceMAAS.py
index 082cc58f..35c5b5e1 100644
--- a/cloudinit/sources/DataSourceMAAS.py
+++ b/cloudinit/sources/DataSourceMAAS.py
@@ -36,6 +36,8 @@ from cloudinit import util
 LOG = logging.getLogger(__name__)
 MD_VERSION = "2012-03-01"
 
+BINARY_FIELDS = ('user-data',)
+
 
 class DataSourceMAAS(sources.DataSource):
     """
@@ -185,7 +187,9 @@ def read_maas_seed_dir(seed_d):
     md = {}
     for fname in files:
         try:
-            md[fname] = util.load_file(os.path.join(seed_d, fname))
+            print("fname: %s / %s" % (fname, fname not in BINARY_FIELDS))
+            md[fname] = util.load_file(os.path.join(seed_d, fname),
+                                       decode=fname not in BINARY_FIELDS)
         except IOError as e:
             if e.errno != errno.ENOENT:
                 raise
@@ -218,6 +222,7 @@ def read_maas_seed_url(seed_url, header_cb=None, timeout=None,
         'public-keys': "%s/%s" % (base_url, 'meta-data/public-keys'),
         'user-data': "%s/%s" % (base_url, 'user-data'),
     }
+
     md = {}
     for name in file_order:
         url = files.get(name)
@@ -238,7 +243,10 @@ def read_maas_seed_url(seed_url, header_cb=None, timeout=None,
                                          timeout=timeout,
                                          ssl_details=ssl_details)
             if resp.ok():
-                md[name] = str(resp)
+                if name in BINARY_FIELDS:
+                    md[name] = resp.contents
+                else:
+                    md[name] = util.decode_binary(resp.contents)
             else:
                 LOG.warn(("Fetching from %s resulted in"
                           " an invalid http code %s"), url, resp.code)
@@ -263,7 +271,7 @@ def check_seed_contents(content, seed):
     if len(missing):
         raise MAASSeedDirMalformed("%s: missing files %s" % (seed, missing))
 
-    userdata = content.get('user-data', "")
+    userdata = content.get('user-data', b"")
     md = {}
     for (key, val) in content.items():
         if key == 'user-data':
diff --git a/cloudinit/sources/helpers/openstack.py b/cloudinit/sources/helpers/openstack.py
index 88c7a198..bd93d22f 100644
--- a/cloudinit/sources/helpers/openstack.py
+++ b/cloudinit/sources/helpers/openstack.py
@@ -327,7 +327,7 @@ class ConfigDriveReader(BaseReader):
         return os.path.join(*components)
 
     def _path_read(self, path):
-        return util.load_file(path)
+        return util.load_file(path, decode=False)
 
     def _fetch_available_versions(self):
         if self._versions is None:
diff --git a/cloudinit/url_helper.py b/cloudinit/url_helper.py
index 62001dff..2d81a062 100644
--- a/cloudinit/url_helper.py
+++ b/cloudinit/url_helper.py
@@ -119,7 +119,7 @@ class UrlResponse(object):
 
     @property
     def contents(self):
-        return self._response.text
+        return self._response.content
 
     @property
     def url(self):
diff --git a/cloudinit/util.py b/cloudinit/util.py
index 4fbdf0a9..efbc3c8d 100644
--- a/cloudinit/util.py
+++ b/cloudinit/util.py
@@ -739,6 +739,10 @@ def fetch_ssl_details(paths=None):
     return ssl_details
 
 
+def load_tfile_or_url(*args, **kwargs):
+    return(decode_binary(read_file_or_url(*args, **kwargs).contents))
+
+
 def read_file_or_url(url, timeout=5, retries=10,
                      headers=None, data=None, sec_between=1, ssl_details=None,
                      headers_cb=None, exception_cb=None):
@@ -750,7 +754,7 @@ def read_file_or_url(url, timeout=5, retries=10,
             LOG.warn("Unable to post data to file resource %s", url)
         file_path = url[len("file://"):]
         try:
-            contents = load_file(file_path)
+            contents = load_file(file_path, decode=False)
         except IOError as e:
             code = e.errno
             if e.errno == errno.ENOENT:
@@ -806,7 +810,7 @@ def read_seeded(base="", ext="", timeout=5, retries=10, file_retries=0):
         ud_url = "%s%s%s" % (base, "user-data", ext)
         md_url = "%s%s%s" % (base, "meta-data", ext)
 
-    md_resp = read_file_or_url(md_url, timeout, retries, file_retries)
+    md_resp = load_tfile_or_url(md_url, timeout, retries, file_retries)
     md = None
     if md_resp.ok():
         md = load_yaml(md_resp.contents, default={})
@@ -815,6 +819,7 @@ def read_seeded(base="", ext="", timeout=5, retries=10, file_retries=0):
     ud = None
     if ud_resp.ok():
         ud = ud_resp.contents
+    print("returning %s (%s)" % (ud_resp.contents.__class__, ud_resp.contents))
 
     return (md, ud)
 
@@ -2030,7 +2035,7 @@ def pathprefix2dict(base, required=None, optional=None, delim=os.path.sep):
     ret = {}
     for f in required + optional:
         try:
-            ret[f] = load_file(base + delim + f, quiet=False)
+            ret[f] = load_file(base + delim + f, quiet=False, decode=False)
         except IOError as e:
             if e.errno != errno.ENOENT:
                 raise