1. Add a url response class that urlreading now returns (instead of a tuple).

   a. This allows for more properties to be added as needed in the future, instead of being very restrictive.
2. Fix up all uses of the url reading to now use this new response object.
3. Also fixup user data including, such that if no response actual occurs the url content is not further processed.

5 files changed, 70 insertions, 32 deletions

diff --git a/cloudinit/sources/DataSourceMAAS.py b/cloudinit/sources/DataSourceMAAS.py
index 40ace947..bb8fbac1 100644
--- a/cloudinit/sources/DataSourceMAAS.py
+++ b/cloudinit/sources/DataSourceMAAS.py
@@ -185,9 +185,12 @@ def read_maas_seed_url(seed_url, header_cb=None, timeout=None,
         else:
             headers = {}
         try:
-            (resp, sc) = uhelp.readurl(url, headers=headers, timeout=timeout)
-            if uhelp.ok_http_code(sc):
-                md[name] = resp
+            resp = uhelp.readurl(url, headers=headers, timeout=timeout)
+            if resp.ok():
+                md[name] = str(resp)
+            else:
+                LOG.warn(("Fetching from %s resulted in"
+                          " an invalid http code %s"), url, resp.code)
         except urllib2.HTTPError as e:
             if e.code != 404:
                 raise
diff --git a/cloudinit/transforms/rightscale_userdata.py b/cloudinit/transforms/rightscale_userdata.py
index 40d76c89..8dfd845f 100644
--- a/cloudinit/transforms/rightscale_userdata.py
+++ b/cloudinit/transforms/rightscale_userdata.py
@@ -79,10 +79,10 @@ def handle(name, _cfg, cloud, log, _args):
     for (i, url) in enumerate(urls):
         fname = os.path.join(scripts_d, "rightscale-%02i" % (i))
         try:
-            (content, st) = uhelp.readurl(url)
+            resp = uhelp.readurl(url)
             # Ensure its a valid http response (and something gotten)
-            if uhelp.ok_http_code(st) and content:
-                util.write_file(fname, content, mode=0700)
+            if resp.ok() and resp.contents:
+                util.write_file(fname, str(resp), mode=0700)
                 wrote_fns.append(fname)
         except Exception as e:
             captured_excps.append(e)
diff --git a/cloudinit/url_helper.py b/cloudinit/url_helper.py
index 95de9c7a..56649c1b 100644
--- a/cloudinit/url_helper.py
+++ b/cloudinit/url_helper.py
@@ -34,11 +34,35 @@ from cloudinit import version
 LOG = logging.getLogger(__name__)
 
 
-def ok_http_code(st, redirects_ok=False):
-    if redirects_ok:
-        return st in xrange(200, 400)
-    else:
-        return st in xrange(200, 300)
+class UrlResponse(object):
+    def __init__(self, status_code, contents=None, headers=None):
+        self._status_code = status_code
+        self._contents = contents
+        self._headers = headers
+
+    @property
+    def code(self):
+        return self._status_code
+
+    @property
+    def contents(self):
+        return self._contents
+    
+    @property
+    def headers(self):
+        return self._headers
+    
+    def __str__(self):
+        if not self.contents:
+            return ''
+        else:
+            return str(self.contents)
+
+    def ok(self, redirects_ok=False):
+        if redirects_ok:
+            return self.code in xrange(200, 400)
+        else:
+            return self.code in xrange(200, 300)
 
 
 def readurl(url, data=None, timeout=None,
@@ -74,9 +98,12 @@ def readurl(url, data=None, timeout=None,
                 if status is None:
                     # This seems to happen when files are read...
                     status = 200
+                headers = {}
+                if rh.headers:
+                    headers = dict(rh.headers)
                 LOG.info("Read from %s (%s, %sb) after %s attempts",
                          url, status, len(content), (i + 1))
-                return (content, status)
+                return UrlResponse(status, content, headers)
         except urllib2.HTTPError as e:
             excepts.append(e)
         except urllib2.URLError as e:
@@ -162,11 +189,11 @@ def wait_for_url(urls, max_wait=None, timeout=None,
                 else:
                     headers = {}
 
-                (resp, sc) = readurl(url, headers=headers, timeout=timeout)
-                if not resp:
-                    reason = "empty response [%s]" % sc
-                elif not ok_http_code(sc):
-                    reason = "bad status code [%s]" % sc
+                resp = readurl(url, headers=headers, timeout=timeout)
+                if not resp.contents:
+                    reason = "empty response [%s]" % (resp.code)
+                elif not resp.ok():
+                    reason = "bad status code [%s]" % (resp.code)
                 else:
                     return url
             except urllib2.HTTPError as e:
diff --git a/cloudinit/user_data.py b/cloudinit/user_data.py
index 663f7cda..bf34943d 100644
--- a/cloudinit/user_data.py
+++ b/cloudinit/user_data.py
@@ -121,19 +121,25 @@ class UserDataProcessor(object):
                 continue
 
             include_once_fn = None
+            content = None
             if include_once_on:
                 include_once_fn = self._get_include_once_filename(include_url)
             if include_once_on and os.path.isfile(include_once_fn):
                 content = util.load_file(include_once_fn)
             else:
-                (content, st) = url_helper.readurl(include_url)
-                if include_once_on and url_helper.ok_http_code(st):
-                    util.write_file(include_once_fn, content, mode=0600)
-                if not url_helper.ok_http_code(st):
-                    content = ''
-
-            new_msg = convert_string(content)
-            self._process_msg(new_msg, append_msg)
+                resp = url_helper.readurl(include_url)
+                if include_once_on and resp.ok():
+                    util.write_file(include_once_fn, str(resp), mode=0600)
+                if resp.ok():
+                    content = str(resp)
+                else:
+                    LOG.warn(("Fetching from %s resulted in"
+                              " a invalid http code of %s"),
+                             include_url, resp.code)
+
+            if content is not None:
+                new_msg = convert_string(content)
+                self._process_msg(new_msg, append_msg)
 
     def _explode_archive(self, archive, append_msg):
         entries = util.load_yaml(archive, default=[], allowed=[list, set])
diff --git a/cloudinit/util.py b/cloudinit/util.py
index 11986447..b6fa959b 100644
--- a/cloudinit/util.py
+++ b/cloudinit/util.py
@@ -595,14 +595,16 @@ def read_seeded(base="", ext="", timeout=5, retries=10, file_retries=0):
         ud_url = "%s%s%s" % (base, "user-data", ext)
         md_url = "%s%s%s" % (base, "meta-data", ext)
 
-    (md_str, msc) = read_file_or_url(md_url, timeout, retries, file_retries)
+    md_resp = read_file_or_url(md_url, timeout, retries, file_retries)
     md = None
-    if md_str and uhelp.ok_http_code(msc):
+    if md_resp.ok():
+        md_str = str(md_resp)
         md = load_yaml(md_str, default={})
 
-    (ud_str, usc) = read_file_or_url(ud_url, timeout, retries, file_retries)
+    ud_resp = read_file_or_url(ud_url, timeout, retries, file_retries)
     ud = None
-    if ud_str and uhelp.ok_http_code(usc):
+    if ud_resp.ok():
+        ud_str = str(ud_resp)
         ud = ud_str
 
     return (md, ud)
@@ -769,9 +771,9 @@ def get_cmdline_url(names=None, starts=None, cmdline=None):
     if not url:
         return (None, None, None)
 
-    (contents, sc) = uhelp.readurl(url)
-    if contents.startswith(starts) and uhelp.ok_http_code(sc):
-        return (key, url, contents)
+    resp = uhelp.readurl(url)
+    if resp.contents.startswith(starts) and resp.ok():
+        return (key, url, str(resp))
 
     return (key, url, None)