readurl, read_file_or_url returns bytes, user must convert as necessary

 * explicitly test compressed user-data.
 * userdata_raw is now bytes
 * add load_tfile_or_url for loading text file or url
 * ec2_utils: all meta-data is text, remove non-obvious string translations
 * DigitalOcean: adjust for ec2_utils
 * DataSourceGCE, DataSourceMAAS: user-data is binary other fields are text.
 * openstack.py: read paths without decoding to text.  This is ok as paths
                 other than user-data are json, and load_json will handle
 * load_file still returns text, and that is what most things use.

LP: #1424900

1 files changed, 11 insertions, 3 deletions

diff --git a/cloudinit/ec2_utils.py b/cloudinit/ec2_utils.py
index e1ed4091..7cf99186 100644
--- a/cloudinit/ec2_utils.py
+++ b/cloudinit/ec2_utils.py
@@ -41,6 +41,10 @@ class MetadataLeafDecoder(object):
     def __call__(self, field, blob):
         if not blob:
             return blob
+        try:
+            blob = util.decode_binary(blob)
+        except UnicodeDecodeError:
+            return blob
         if self._maybe_json_object(blob):
             try:
                 # Assume it's json, unless it fails parsing...
@@ -69,6 +73,8 @@ class MetadataMaterializer(object):
     def _parse(self, blob):
         leaves = {}
         children = []
+        blob = util.decode_binary(blob)
+
         if not blob:
             return (leaves, children)
 
@@ -117,12 +123,12 @@ class MetadataMaterializer(object):
             child_url = url_helper.combine_url(base_url, c)
             if not child_url.endswith("/"):
                 child_url += "/"
-            child_blob = str(self._caller(child_url))
+            child_blob = self._caller(child_url)
             child_contents[c] = self._materialize(child_blob, child_url)
         leaf_contents = {}
         for (field, resource) in leaves.items():
             leaf_url = url_helper.combine_url(base_url, resource)
-            leaf_blob = self._caller(leaf_url).contents
+            leaf_blob = self._caller(leaf_url)
             leaf_contents[field] = self._leaf_decoder(field, leaf_blob)
         joined = {}
         joined.update(child_contents)
@@ -179,11 +185,13 @@ def get_instance_metadata(api_version='latest',
     caller = functools.partial(util.read_file_or_url,
                                ssl_details=ssl_details, timeout=timeout,
                                retries=retries)
+    def mcaller(url):
+        return caller(url).contents
 
     try:
         response = caller(md_url)
         materializer = MetadataMaterializer(response.contents,
-                                            md_url, caller,
+                                            md_url, mcaller,
                                             leaf_decoder=leaf_decoder)
         md = materializer.materialize()
         if not isinstance(md, (dict)):