move wait_for_metadata_service for util, rename to wait_for_url

Also, add in the headers_cb which will be required for oauth.

3 files changed, 90 insertions, 170 deletions

diff --git a/cloudinit/DataSourceEc2.py b/cloudinit/DataSourceEc2.py
index 06635746..4e06803d 100644
--- a/cloudinit/DataSourceEc2.py
+++ b/cloudinit/DataSourceEc2.py
@@ -134,8 +134,8 @@ class DataSourceEc2(DataSource.DataSource):
             url2base[cur] = url
 
         starttime = time.time()
-        url = wait_for_metadata_service(urls=urls, max_wait=max_wait,
-                  timeout=timeout, status_cb=log.warn)
+        url = util.wait_for_url(urls=urls, max_wait=max_wait,
+                                timeout=timeout, status_cb=log.warn)
 
         if url:
             log.debug("Using metadata source: '%s'" % url2base[url])
@@ -208,87 +208,6 @@ class DataSourceEc2(DataSource.DataSource):
         return False
 
 
-def wait_for_metadata_service(urls, max_wait=None, timeout=None,
-                              status_cb=None):
-    """
-    urls:      a list of urls to try
-    max_wait:  roughly the maximum time to wait before giving up
-               The max time is *actually* len(urls)*timeout as each url will
-               be tried once and given the timeout provided.
-    timeout:   the timeout provided to urllib2.urlopen
-    status_cb: call method with string message when a url is not available
-
-    the idea of this routine is to wait for the EC2 metdata service to
-    come up.  On both Eucalyptus and EC2 we have seen the case where
-    the instance hit the MD before the MD service was up.  EC2 seems
-    to have permenantely fixed this, though.
-
-    In openstack, the metadata service might be painfully slow, and
-    unable to avoid hitting a timeout of even up to 10 seconds or more
-    (LP: #894279) for a simple GET.
-
-    Offset those needs with the need to not hang forever (and block boot)
-    on a system where cloud-init is configured to look for EC2 Metadata
-    service but is not going to find one.  It is possible that the instance
-    data host (169.254.169.254) may be firewalled off Entirely for a sytem,
-    meaning that the connection will block forever unless a timeout is set.
-    """
-    starttime = time.time()
-
-    sleeptime = 1
-
-    def nullstatus_cb(msg):
-        return
-
-    if status_cb == None:
-        status_cb = nullstatus_cb
-
-    def timeup(max_wait, starttime):
-        return((max_wait <= 0 or max_wait == None) or
-               (time.time() - starttime > max_wait))
-
-    loop_n = 0
-    while True:
-        sleeptime = int(loop_n / 5) + 1
-        for url in urls:
-            now = time.time()
-            if loop_n != 0:
-                if timeup(max_wait, starttime):
-                    break
-                if timeout and (now + timeout > (starttime + max_wait)):
-                    # shorten timeout to not run way over max_time
-                    timeout = int((starttime + max_wait) - now)
-
-            reason = ""
-            try:
-                req = urllib2.Request(url)
-                resp = urllib2.urlopen(req, timeout=timeout)
-                if resp.read() != "":
-                    return url
-                reason = "empty data [%s]" % resp.getcode()
-            except urllib2.HTTPError as e:
-                reason = "http error [%s]" % e.code
-            except urllib2.URLError as e:
-                reason = "url error [%s]" % e.reason
-            except socket.timeout as e:
-                reason = "socket timeout [%s]" % e
-            except Exception as e:
-                reason = "unexpected error [%s]" % e
-
-            if log:
-                status_cb("'%s' failed [%s/%ss]: %s" %
-                          (url, int(time.time() - starttime), max_wait,
-                           reason))
-
-        if timeup(max_wait, starttime):
-            break
-
-        loop_n = loop_n + 1
-        time.sleep(sleeptime)
-
-    return False
-
-
 datasources = [
   (DataSourceEc2, (DataSource.DEP_FILESYSTEM, DataSource.DEP_NETWORK)),
 ]
diff --git a/cloudinit/DataSourceMaaS.py b/cloudinit/DataSourceMaaS.py
index 2bc1f71f..d902ccb4 100644
--- a/cloudinit/DataSourceMaaS.py
+++ b/cloudinit/DataSourceMaaS.py
@@ -116,7 +116,7 @@ class DataSourceMaaS(DataSource.DataSource):
 
         starttime = time.time()
         check_url = "%s/instance-id" % url
-        url = wait_for_metadata_service(urls=[check_url], max_wait=max_wait,
+        url = util.wait_for_url(urls=[check_url], max_wait=max_wait,
             timeout=timeout, status_cb=log.warn,
             headers_cb=self.md_headers)
 
@@ -129,92 +129,6 @@ class DataSourceMaaS(DataSource.DataSource):
         return (bool(url))
 
 
-def wait_for_metadata_service(urls, max_wait=None, timeout=None,
-                              status_cb=None, headers_cb=None):
-    """
-    urls:      a list of urls to try
-    max_wait:  roughly the maximum time to wait before giving up
-               The max time is *actually* len(urls)*timeout as each url will
-               be tried once and given the timeout provided.
-    timeout:   the timeout provided to urllib2.urlopen
-    status_cb: call method with string message when a url is not available
-
-    the idea of this routine is to wait for the EC2 metdata service to
-    come up.  On both Eucalyptus and EC2 we have seen the case where
-    the instance hit the MD before the MD service was up.  EC2 seems
-    to have permenantely fixed this, though.
-
-    In openstack, the metadata service might be painfully slow, and
-    unable to avoid hitting a timeout of even up to 10 seconds or more
-    (LP: #894279) for a simple GET.
-
-    Offset those needs with the need to not hang forever (and block boot)
-    on a system where cloud-init is configured to look for EC2 Metadata
-    service but is not going to find one.  It is possible that the instance
-    data host (169.254.169.254) may be firewalled off Entirely for a sytem,
-    meaning that the connection will block forever unless a timeout is set.
-    """
-    starttime = time.time()
-
-    sleeptime = 1
-
-    def nullstatus_cb(msg):
-        return
-
-    if status_cb == None:
-        status_cb = nullstatus_cb
-
-    def timeup(max_wait, starttime):
-        return((max_wait <= 0 or max_wait == None) or
-               (time.time() - starttime > max_wait))
-
-    loop_n = 0
-    while True:
-        sleeptime = int(loop_n / 5) + 1
-        for url in urls:
-            now = time.time()
-            if loop_n != 0:
-                if timeup(max_wait, starttime):
-                    break
-                if timeout and (now + timeout > (starttime + max_wait)):
-                    # shorten timeout to not run way over max_time
-                    timeout = int((starttime + max_wait) - now)
-
-            reason = ""
-            try:
-                if headers_cb != None:
-                    headers = headers_cb(url)
-                else:
-                    headers = {}
-
-                req = urllib2.Request(url, data=None, headers=headers)
-                resp = urllib2.urlopen(req, timeout=timeout)
-                if resp.read() != "":
-                    return url
-                reason = "empty data [%s]" % resp.getcode()
-            except urllib2.HTTPError as e:
-                reason = "http error [%s]" % e.code
-            except urllib2.URLError as e:
-                reason = "url error [%s]" % e.reason
-            except socket.timeout as e:
-                reason = "socket timeout [%s]" % e
-            except Exception as e:
-                reason = "unexpected error [%s]" % e
-
-            if log:
-                status_cb("'%s' failed [%s/%ss]: %s" %
-                          (url, int(time.time() - starttime), max_wait,
-                           reason))
-
-        if timeup(max_wait, starttime):
-            break
-
-        loop_n = loop_n + 1
-        time.sleep(sleeptime)
-
-    return False
-
-
 def read_maas_seed_dir(seed_d):
     """
     Return user-data and metadata for a maas seed dir in seed_d.
diff --git a/cloudinit/util.py b/cloudinit/util.py
index c37f0316..882fd9fb 100644
--- a/cloudinit/util.py
+++ b/cloudinit/util.py
@@ -751,3 +751,90 @@ def mount_callback_umount(device, callback, data=None):
     _cleanup(umount, tmpd)
 
     return(ret)
+
+def wait_for_url(urls, max_wait=None, timeout=None,
+                 status_cb=None, headers_cb=None):
+    """
+    urls:      a list of urls to try
+    max_wait:  roughly the maximum time to wait before giving up
+               The max time is *actually* len(urls)*timeout as each url will
+               be tried once and given the timeout provided.
+    timeout:   the timeout provided to urllib2.urlopen
+    status_cb: call method with string message when a url is not available
+
+    the idea of this routine is to wait for the EC2 metdata service to
+    come up.  On both Eucalyptus and EC2 we have seen the case where
+    the instance hit the MD before the MD service was up.  EC2 seems
+    to have permenantely fixed this, though.
+
+    In openstack, the metadata service might be painfully slow, and
+    unable to avoid hitting a timeout of even up to 10 seconds or more
+    (LP: #894279) for a simple GET.
+
+    Offset those needs with the need to not hang forever (and block boot)
+    on a system where cloud-init is configured to look for EC2 Metadata
+    service but is not going to find one.  It is possible that the instance
+    data host (169.254.169.254) may be firewalled off Entirely for a sytem,
+    meaning that the connection will block forever unless a timeout is set.
+    """
+    starttime = time.time()
+
+    sleeptime = 1
+
+    def nullstatus_cb(msg):
+        return
+
+    if status_cb == None:
+        status_cb = nullstatus_cb
+
+    def timeup(max_wait, starttime):
+        return((max_wait <= 0 or max_wait == None) or
+               (time.time() - starttime > max_wait))
+
+    loop_n = 0
+    while True:
+        sleeptime = int(loop_n / 5) + 1
+        for url in urls:
+            now = time.time()
+            if loop_n != 0:
+                if timeup(max_wait, starttime):
+                    break
+                if timeout and (now + timeout > (starttime + max_wait)):
+                    # shorten timeout to not run way over max_time
+                    timeout = int((starttime + max_wait) - now)
+
+            reason = ""
+            try:
+                if headers_cb != None:
+                    headers = headers_cb(url)
+                else:
+                    headers = {}
+
+                req = urllib2.Request(url, data=None, headers=headers)
+                resp = urllib2.urlopen(req, timeout=timeout)
+                if resp.read() != "":
+                    return url
+                reason = "empty data [%s]" % resp.getcode()
+            except urllib2.HTTPError as e:
+                reason = "http error [%s]" % e.code
+            except urllib2.URLError as e:
+                reason = "url error [%s]" % e.reason
+            except socket.timeout as e:
+                reason = "socket timeout [%s]" % e
+            except Exception as e:
+                reason = "unexpected error [%s]" % e
+
+            if log:
+                status_cb("'%s' failed [%s/%ss]: %s" %
+                          (url, int(time.time() - starttime), max_wait,
+                           reason))
+
+        if timeup(max_wait, starttime):
+            break
+
+        loop_n = loop_n + 1
+        time.sleep(sleeptime)
+
+    return False
+
+