Fix the maas callback mechanism now that requests is used.

2 files changed, 36 insertions, 27 deletions

diff --git a/cloudinit/sources/DataSourceMAAS.py b/cloudinit/sources/DataSourceMAAS.py
index 0c526305..f3759b4b 100644
--- a/cloudinit/sources/DataSourceMAAS.py
+++ b/cloudinit/sources/DataSourceMAAS.py
@@ -25,8 +25,6 @@ import os
 import time
 import urllib2
 
-import requests
-
 from cloudinit import log as logging
 from cloudinit import sources
 from cloudinit import url_helper
@@ -81,7 +79,7 @@ class DataSourceMAAS(sources.DataSource):
             self.base_url = url
 
             (userdata, metadata) = read_maas_seed_url(self.base_url,
-                                                      self.md_headers,
+                                                      self._md_headers,
                                                       paths=self.paths)
             self.userdata_raw = userdata
             self.metadata = metadata
@@ -90,7 +88,7 @@ class DataSourceMAAS(sources.DataSource):
             util.logexc(LOG, "Failed fetching metadata from url %s", url)
             return False
 
-    def md_headers(self, url):
+    def _md_headers(self, url):
         mcfg = self.ds_cfg
 
         # If we are missing token_key, token_secret or consumer_key
@@ -134,9 +132,10 @@ class DataSourceMAAS(sources.DataSource):
         starttime = time.time()
         check_url = "%s/%s/meta-data/instance-id" % (url, MD_VERSION)
         urls = [check_url]
-        url = uhelp.wait_for_url(urls=urls, max_wait=max_wait,
-                                 timeout=timeout, exception_cb=self._except_cb,
-                                 headers_cb=self.md_headers)
+        url = url_helper.wait_for_url(urls=urls, max_wait=max_wait,
+                                      timeout=timeout,
+                                      exception_cb=self._except_cb,
+                                      headers_cb=self._md_headers)
 
         if url:
             LOG.debug("Using metadata source: '%s'", url)
@@ -147,23 +146,23 @@ class DataSourceMAAS(sources.DataSource):
         return bool(url)
 
     def _except_cb(self, msg, exception):
-        if not (isinstance(exception, urllib2.HTTPError) and
+        if not (isinstance(exception, url_helper.UrlError) and
                 (exception.code == 403 or exception.code == 401)):
             return
+
         if 'date' not in exception.headers:
-            LOG.warn("date field not in %d headers" % exception.code)
+            LOG.warn("Missing header 'date' in %s response", exception.code)
             return
 
         date = exception.headers['date']
-
         try:
             ret_time = time.mktime(parsedate(date))
-        except:
-            LOG.warn("failed to convert datetime '%s'")
+        except Exception as e:
+            LOG.warn("Failed to convert datetime '%s': %s", date, e)
             return
 
         self.oauth_clockskew = int(ret_time - time.time())
-        LOG.warn("set oauth clockskew to %d" % self.oauth_clockskew)
+        LOG.warn("Setting oauth clockskew to %d", self.oauth_clockskew)
         return
 
 
diff --git a/cloudinit/url_helper.py b/cloudinit/url_helper.py
index 300e70c2..6f06761a 100644
--- a/cloudinit/url_helper.py
+++ b/cloudinit/url_helper.py
@@ -92,13 +92,13 @@ class UrlResponse(object):
 
 
 class UrlError(IOError):
-    def __init__(self, cause):
+    def __init__(self, cause, code=None, headers=None):
         IOError.__init__(self, str(cause))
         self.cause = cause
-        if isinstance(cause, exceptions.HTTPError) and cause.response:
-            self.code = cause.response.status_code
-        else:
-            self.code = None
+        self.code = code
+        self.headers = headers
+        if self.headers is None:
+            self.headers = {}
 
 
 def readurl(url, data=None, timeout=None, retries=0, sec_between=1,
@@ -170,7 +170,11 @@ def readurl(url, data=None, timeout=None, retries=0, sec_between=1,
             # attrs
             return UrlResponse(r)
         except exceptions.RequestException as e:
-            excps.append(UrlError(e))
+            if isinstance(e, (exceptions.HTTPError)) and e.response:
+                excps.append(UrlError(e, code=e.response.status_code,
+                                      headers=e.response.headers))
+            else:
+                excps.append(UrlError(e))
             if i + 1 < manual_tries and sec_between > 0:
                 LOG.debug("Please wait %s seconds while we wait to try again",
                           sec_between)
@@ -235,20 +239,23 @@ def wait_for_url(urls, max_wait=None, timeout=None,
                     timeout = int((start_time + max_wait) - now)
 
             reason = ""
+            e = None
             try:
                 if headers_cb is not None:
                     headers = headers_cb(url)
                 else:
                     headers = {}
 
-                resp = readurl(url, headers=headers, timeout=timeout,
-                               check_status=False)
-                if not resp.contents:
-                    reason = "empty response [%s]" % (resp.code)
-                    e = ValueError(reason)
-                elif not resp.ok():
-                    reason = "bad status code [%s]" % (resp.code)
-                    e = ValueError(reason)
+                response = readurl(url, headers=headers, timeout=timeout,
+                                   check_status=False)
+                if not response.contents:
+                    reason = "empty response [%s]" % (response.code)
+                    e = UrlError(ValueError(reason),
+                                 code=response.code, headers=response.headers)
+                elif not response.ok():
+                    reason = "bad status code [%s]" % (response.code)
+                    e = UrlError(ValueError(reason),
+                                 code=response.code, headers=response.headers)
                 else:
                     return url
             except UrlError as e:
@@ -263,6 +270,9 @@ def wait_for_url(urls, max_wait=None, timeout=None,
                                                                reason)
             status_cb(status_msg)
             if exception_cb:
+                # This can be used to alter the headers that will be sent
+                # in the future, for example this is what the MAAS datasource
+                # does.
                 exception_cb(msg=status_msg, exception=e)
 
         if timeup(max_wait, start_time):