Add a openstack specific datasource

Openstack has a unique derivative datasource that is gaining usage.
Previously the config drive datasource provided part of this functionality
as well as the ec2 datasource, but since new functionality is being added
to openstack's special datasource it seems beneficial to combine the used
parts into a new datasource just made for handling openstack deployments
that use the openstack metadata service (possibly in combination with the
ec2 metadata service).

This patch factors out the common logic shared between the config drive
and the openstack metadata datasource and places that in a shared helper
file and then creates a new openstack datasource that readers from the
openstack metadata service and refactors the config drive datasource to
use this common logic.

1 files changed, 40 insertions, 15 deletions

diff --git a/cloudinit/url_helper.py b/cloudinit/url_helper.py
index 97ed75ad..4a83169a 100644
--- a/cloudinit/url_helper.py
+++ b/cloudinit/url_helper.py
@@ -22,6 +22,7 @@
 
 import httplib
 import time
+import urllib
 
 import requests
 from requests import exceptions
@@ -38,6 +39,7 @@ NOT_FOUND = httplib.NOT_FOUND
 # Check if requests has ssl support (added in requests >= 0.8.8)
 SSL_ENABLED = False
 CONFIG_ENABLED = False  # This was added in 0.7 (but taken out in >=1.0)
+_REQ_VER = None
 try:
     from distutils.version import LooseVersion
     import pkg_resources
@@ -61,6 +63,23 @@ def _cleanurl(url):
     return urlunparse(parsed_url)
 
 
+def combine_url(base, *add_ons):
+
+    def combine_single(url, add_on):
+        url_parsed = list(urlparse(url))
+        path = url_parsed[2]
+        if path and not path.endswith("/"):
+            path += "/"
+        path += urllib.quote(str(add_on), safe="/:")
+        url_parsed[2] = path
+        return urlunparse(url_parsed)
+
+    url = base
+    for add_on in add_ons:
+        url = combine_single(url, add_on)
+    return url
+
+
 # Made to have same accessors as UrlResponse so that the
 # read_file_or_url can return this or that object and the
 # 'user' of those objects will not need to know the difference.
@@ -129,28 +148,34 @@ class UrlError(IOError):
             self.headers = {}
 
 
-def readurl(url, data=None, timeout=None, retries=0, sec_between=1,
-            headers=None, headers_cb=None, ssl_details=None,
-            check_status=True, allow_redirects=True, exception_cb=None):
-    url = _cleanurl(url)
-    req_args = {
-        'url': url,
-    }
+def _get_ssl_args(url, ssl_details):
+    ssl_args = {}
     scheme = urlparse(url).scheme  # pylint: disable=E1101
     if scheme == 'https' and ssl_details:
         if not SSL_ENABLED:
-            LOG.warn("SSL is not enabled, cert. verification can not occur!")
+            LOG.warn("SSL is not supported in requests v%s, "
+                     "cert. verification can not occur!", _REQ_VER)
         else:
             if 'ca_certs' in ssl_details and ssl_details['ca_certs']:
-                req_args['verify'] = ssl_details['ca_certs']
+                ssl_args['verify'] = ssl_details['ca_certs']
             else:
-                req_args['verify'] = True
+                ssl_args['verify'] = True
             if 'cert_file' in ssl_details and 'key_file' in ssl_details:
-                req_args['cert'] = [ssl_details['cert_file'],
+                ssl_args['cert'] = [ssl_details['cert_file'],
                                     ssl_details['key_file']]
             elif 'cert_file' in ssl_details:
-                req_args['cert'] = str(ssl_details['cert_file'])
+                ssl_args['cert'] = str(ssl_details['cert_file'])
+    return ssl_args
 
+
+def readurl(url, data=None, timeout=None, retries=0, sec_between=1,
+            headers=None, headers_cb=None, ssl_details=None,
+            check_status=True, allow_redirects=True, exception_cb=None):
+    url = _cleanurl(url)
+    req_args = {
+        'url': url,
+    }
+    req_args.update(_get_ssl_args(url, ssl_details))
     req_args['allow_redirects'] = allow_redirects
     req_args['method'] = 'GET'
     if timeout is not None:
@@ -181,12 +206,11 @@ def readurl(url, data=None, timeout=None, retries=0, sec_between=1,
         def _cb(url):
             return headers
         headers_cb = _cb
-
     if data:
-        # Do this after the log (it might be large)
         req_args['data'] = data
     if sec_between is None:
         sec_between = -1
+
     excps = []
     # Handle retrying ourselves since the built-in support
     # doesn't handle sleeping between tries...
@@ -223,7 +247,7 @@ def readurl(url, data=None, timeout=None, retries=0, sec_between=1,
                     # ssl exceptions are not going to get fixed by waiting a
                     # few seconds
                     break
-            if exception_cb and not exception_cb(filtered_req_args, excps[-1]):
+            if exception_cb and not exception_cb(req_args.copy(), excps[-1]):
                 break
             if i + 1 < manual_tries and sec_between > 0:
                 LOG.debug("Please wait %s seconds while we wait to try again",
@@ -242,6 +266,7 @@ def wait_for_url(urls, max_wait=None, timeout=None,
     max_wait:  roughly the maximum time to wait before giving up
                The max time is *actually* len(urls)*timeout as each url will
                be tried once and given the timeout provided.
+               a number <= 0 will always result in only one try
     timeout:   the timeout provided to urlopen
     status_cb: call method with string message when a url is not available
     headers_cb: call method with single argument of url to get headers