Get tests working and further adjustments.

6 files changed, 60 insertions, 33 deletions

diff --git a/cloudinit/sources/DataSourceMAAS.py b/cloudinit/sources/DataSourceMAAS.py
index 6e1133b2..0c526305 100644
--- a/cloudinit/sources/DataSourceMAAS.py
+++ b/cloudinit/sources/DataSourceMAAS.py
@@ -223,9 +223,12 @@ def read_maas_seed_url(seed_url, header_cb=None, timeout=None,
         else:
             headers = {}
         try:
-            resp = util.read_file_or_url(url, headers=headers, timeout=timeout,
-                                         ssl_details=util.fetch_ssl_details(paths))
-            if resp.ok:
+            ssl_details = util.fetch_ssl_details(paths)
+            resp = util.read_file_or_url(url,
+                                         headers=headers,
+                                         timeout=timeout,
+                                         ssl_details=ssl_details)
+            if resp.ok():
                 md[name] = str(resp)
             else:
                 LOG.warn(("Fetching from %s resulted in"
diff --git a/cloudinit/url_helper.py b/cloudinit/url_helper.py
index 0839e63b..300e70c2 100644
--- a/cloudinit/url_helper.py
+++ b/cloudinit/url_helper.py
@@ -70,9 +70,14 @@ class UrlResponse(object):
     def url(self):
         return self._response.url
 
-    @property
-    def ok(self):
-        return self._response.ok
+    def ok(self, redirects_ok=False):
+        upper = 300
+        if redirects_ok:
+            upper = 400
+        if self.code >= 200 and self.code < upper:
+            return True
+        else:
+            return False
 
     @property
     def headers(self):
@@ -158,11 +163,8 @@ def readurl(url, data=None, timeout=None, retries=0, sec_between=1,
             r = requests.request(**req_args)
             if check_status:
                 r.raise_for_status()
-            contents = r.content
-            status = r.status_code
-            headers = r.headers
             LOG.debug("Read from %s (%s, %sb) after %s attempts", url,
-                      status, len(contents), (i + 1))
+                      r.status_code, len(r.content), (i + 1))
             # Doesn't seem like we can make it use a different
             # subclass for responses, so add our own backward-compat
             # attrs
@@ -256,8 +258,9 @@ def wait_for_url(urls, max_wait=None, timeout=None,
 
             time_taken = int(time.time() - start_time)
             status_msg = "Calling '%s' failed [%s/%ss]: %s" % (url,
-                                                             time_taken,
-                                                             max_wait, reason)
+                                                               time_taken,
+                                                               max_wait,
+                                                               reason)
             status_cb(status_msg)
             if exception_cb:
                 exception_cb(msg=status_msg, exception=e)
diff --git a/cloudinit/user_data.py b/cloudinit/user_data.py
index eaf448a7..df069ff8 100644
--- a/cloudinit/user_data.py
+++ b/cloudinit/user_data.py
@@ -59,6 +59,7 @@ EXAMINE_FOR_LAUNCH_INDEX = ["text/cloud-config"]
 class UserDataProcessor(object):
     def __init__(self, paths):
         self.paths = paths
+        self.ssl_details = util.fetch_ssl_details(paths)
 
     def process(self, blob):
         accumulating_msg = MIMEMultipart()
@@ -172,10 +173,11 @@ class UserDataProcessor(object):
             if include_once_on and os.path.isfile(include_once_fn):
                 content = util.load_file(include_once_fn)
             else:
-                resp = util.read_file_or_url(include_url)
-                if include_once_on and resp.ok:
+                resp = util.read_file_or_url(include_url,
+                                             ssl_details=self.ssl_details)
+                if include_once_on and resp.ok():
                     util.write_file(include_once_fn, str(resp), mode=0600)
-                if resp.ok:
+                if resp.ok():
                     content = str(resp)
                 else:
                     LOG.warn(("Fetching from %s resulted in"
diff --git a/cloudinit/util.py b/cloudinit/util.py
index 42b3ab01..dc3c5639 100644
--- a/cloudinit/util.py
+++ b/cloudinit/util.py
@@ -70,18 +70,31 @@ FN_ALLOWED = ('_-.()' + string.digits + string.ascii_letters)
 CONTAINER_TESTS = ['running-in-container', 'lxc-is-container']
 
 
-class FileResponse(object):
-    def __init__(self, path, contents):
-        self.code = 200
+# Made to have same accessors as UrlResponse so that the
+# read_file_or_url can return this or that object and the
+# 'user' of those objects will not need to know the difference.
+class StringResponse(object):
+    def __init__(self, contents, code=200):
+        self.code = code
         self.headers = {}
         self.contents = contents
-        self.ok = True
-        self.url = path
+        self.url = None
+
+    def ok(self, *args, **kwargs):
+        if self.code != 200:
+            return False
+        return True
 
     def __str__(self):
         return self.contents
 
 
+class FileResponse(StringResponse):
+    def __init__(self, path, contents, code=200):
+        StringResponse.__init__(self, contents, code=code)
+        self.url = path
+
+
 class ProcessExecutionError(IOError):
 
     MESSAGE_TMPL = ('%(description)s\n'
@@ -630,7 +643,7 @@ def read_optional_seed(fill, base="", ext="", timeout=5):
         fill['user-data'] = ud
         fill['meta-data'] = md
         return True
-    except OSError as e:
+    except IOError as e:
         if e.errno == errno.ENOENT:
             return False
         raise
@@ -670,9 +683,12 @@ def fetch_ssl_details(paths=None):
 
 def read_file_or_url(url, timeout=5, retries=10,
                      headers=None, data=None, sec_between=1, ssl_details=None):
+    url = url.lstrip()
     if url.startswith("/"):
         url = "file://%s" % url
     if url.lower().startswith("file://"):
+        if data:
+            LOG.warn("Unable to post data to file resource %s", url)
         file_path = url[len("file://"):]
         return FileResponse(file_path, contents=load_file(file_path))
     else:
@@ -724,13 +740,13 @@ def read_seeded(base="", ext="", timeout=5, retries=10, file_retries=0):
 
     md_resp = read_file_or_url(md_url, timeout, retries, file_retries)
     md = None
-    if md_resp.ok:
+    if md_resp.ok():
         md_str = str(md_resp)
         md = load_yaml(md_str, default={})
 
     ud_resp = read_file_or_url(ud_url, timeout, retries, file_retries)
     ud = None
-    if ud_resp.ok:
+    if ud_resp.ok():
         ud_str = str(ud_resp)
         ud = ud_str
 
@@ -900,7 +916,7 @@ def get_cmdline_url(names=('cloud-config-url', 'url'),
         return (None, None, None)
 
     resp = read_file_or_url(url)
-    if resp.contents.startswith(starts) and resp.ok:
+    if resp.contents.startswith(starts) and resp.ok():
         return (key, url, str(resp))
 
     return (key, url, None)
diff --git a/tests/unittests/test__init__.py b/tests/unittests/test__init__.py
index ac082076..d707afa9 100644
--- a/tests/unittests/test__init__.py
+++ b/tests/unittests/test__init__.py
@@ -191,8 +191,8 @@ class TestCmdlineUrl(MockerTestCase):
 
         mock_readurl = self.mocker.replace(url_helper.readurl,
                                            passthrough=False)
-        mock_readurl(url)
-        self.mocker.result(url_helper.UrlResponse(200, payload))
+        mock_readurl(url, ARGS, KWARGS)
+        self.mocker.result(util.StringResponse(payload))
         self.mocker.replay()
 
         self.assertEqual((key, url, None),
@@ -207,8 +207,8 @@ class TestCmdlineUrl(MockerTestCase):
 
         mock_readurl = self.mocker.replace(url_helper.readurl,
                                            passthrough=False)
-        mock_readurl(url)
-        self.mocker.result(url_helper.UrlResponse(200, payload))
+        mock_readurl(url, ARGS, KWARGS)
+        self.mocker.result(util.StringResponse(payload))
         self.mocker.replay()
 
         self.assertEqual((key, url, payload),
@@ -221,7 +221,7 @@ class TestCmdlineUrl(MockerTestCase):
         cmdline = "ro %s=%s bar=1" % (key, url)
 
         self.mocker.replace(url_helper.readurl, passthrough=False)
-        self.mocker.result(url_helper.UrlResponse(400))
+        self.mocker.result(util.StringResponse(""))
         self.mocker.replay()
 
         self.assertEqual((None, None, None),
diff --git a/tests/unittests/test_datasource/test_maas.py b/tests/unittests/test_datasource/test_maas.py
index b56fea82..47f8caa4 100644
--- a/tests/unittests/test_datasource/test_maas.py
+++ b/tests/unittests/test_datasource/test_maas.py
@@ -3,12 +3,13 @@ import os
 
 from cloudinit.sources import DataSourceMAAS
 from cloudinit import url_helper
+from cloudinit import util
 from tests.unittests.helpers import populate_dir
 
-from mocker import MockerTestCase
+import mocker
 
 
-class TestMAASDataSource(MockerTestCase):
+class TestMAASDataSource(mocker.MockerTestCase):
 
     def setUp(self):
         super(TestMAASDataSource, self).setUp()
@@ -115,9 +116,11 @@ class TestMAASDataSource(MockerTestCase):
 
         for key in valid_order:
             url = "%s/%s/%s" % (my_seed, my_ver, key)
-            mock_request(url, headers=my_headers, timeout=None)
+            mock_request(url, headers=my_headers, timeout=mocker.ANY,
+                         data=mocker.ANY, sec_between=mocker.ANY,
+                         ssl_details=mocker.ANY, retries=mocker.ANY)
             resp = valid.get(key)
-            self.mocker.result(url_helper.UrlResponse(200, resp))
+            self.mocker.result(util.StringResponse(resp))
         self.mocker.replay()
 
         (userdata, metadata) = DataSourceMAAS.read_maas_seed_url(my_seed,