Continue working on integrating requests.

7 files changed, 129 insertions, 45 deletions

diff --git a/cloudinit/config/cc_phone_home.py b/cloudinit/config/cc_phone_home.py
index ae1349eb..90834080 100644
--- a/cloudinit/config/cc_phone_home.py
+++ b/cloudinit/config/cc_phone_home.py
@@ -19,7 +19,6 @@
 #    along with this program.  If not, see <http://www.gnu.org/licenses/>.
 
 from cloudinit import templater
-from cloudinit import url_helper as uhelp
 from cloudinit import util
 
 from cloudinit.settings import PER_INSTANCE
@@ -112,7 +111,8 @@ def handle(name, cfg, cloud, log, args):
     }
     url = templater.render_string(url, url_params)
     try:
-        uhelp.readurl(url, data=real_submit_keys, retries=tries, sec_between=3)
+        util.read_file_or_url(url, data=real_submit_keys,
+                              retries=tries, sec_between=3)
     except:
         util.logexc(log, ("Failed to post phone home data to"
                           " %s in %s tries"), url, tries)
diff --git a/cloudinit/ec2_utils.py b/cloudinit/ec2_utils.py
index b9d7a2f7..c422eea9 100644
--- a/cloudinit/ec2_utils.py
+++ b/cloudinit/ec2_utils.py
@@ -22,7 +22,6 @@ import json
 import urllib
 
 from cloudinit import log as logging
-from cloudinit import url_helper as uh
 from cloudinit import util
 
 LOG = logging.getLogger(__name__)
@@ -40,11 +39,10 @@ def combine_url(base, add_on):
 
 # See: http://bit.ly/TyoUQs
 class MetadataMaterializer(object):
-    def __init__(self, blob, base_url, **fetch_settings):
+    def __init__(self, blob, base_url):
         self._blob = blob
         self._md = None
         self._base_url = base_url
-        self._fetch_settings = fetch_settings
 
     def _parse(self, blob):
         leaves = {}
@@ -90,8 +88,8 @@ class MetadataMaterializer(object):
         self._md = self._materialize(self._blob, self._base_url)
         return self._md
 
-    def _fetch_url(self, url, **opts):
-        response = uh.readurl(url, **opts)
+    def _fetch_url(self, url):
+        response = util.read_file_or_url(url)
         return str(response)
 
     def _decode_leaf_blob(self, blob):
@@ -115,12 +113,12 @@ class MetadataMaterializer(object):
             child_url = combine_url(base_url, c)
             if not child_url.endswith("/"):
                 child_url += "/"
-            child_blob = self._fetch_url(child_url, **self._fetch_settings)
+            child_blob = self._fetch_url(child_url)
             child_contents[c] = self._materialize(child_blob, child_url)
         leaf_contents = {}
         for (field, resource) in leaves.items():
             leaf_url = combine_url(base_url, resource)
-            leaf_blob = self._fetch_url(leaf_url, **self._fetch_settings)
+            leaf_blob = self._fetch_url(leaf_url)
             leaf_contents[field] = self._decode_leaf_blob(leaf_blob)
         joined = {}
         joined.update(child_contents)
@@ -136,23 +134,19 @@ def get_instance_userdata(url, version='latest', ssl_details=None):
     ud_url = combine_url(url, version)
     ud_url = combine_url(ud_url, 'user-data')
     try:
-        response = uh.readurl(ud_url, timeout=5,
-                              retries=10, ssl_details=ssl_details)
+        response = util.read_file_or_url(ud_url)
         return str(response)
     except Exception:
         util.logexc(LOG, "Failed fetching userdata from url %s", ud_url)
         return None
 
 
-def get_instance_metadata(url, version='latest', ssl_details=None):
+def get_instance_metadata(url, version='latest'):
     md_url = combine_url(url, version)
     md_url = combine_url(md_url, 'meta-data')
     try:
-        response = uh.readurl(md_url, timeout=5,
-                              retries=10, ssl_details=ssl_details)
-        materializer = MetadataMaterializer(str(response), md_url,
-                                            timeout=5, retries=10,
-                                            ssl_details=ssl_details)
+        response = util.read_file_or_url(md_url)
+        materializer = MetadataMaterializer(str(response), md_url)
         return materializer.materialize()
     except Exception:
         util.logexc(LOG, "Failed fetching metadata from url %s", md_url)
diff --git a/cloudinit/sources/DataSourceEc2.py b/cloudinit/sources/DataSourceEc2.py
index 204963e7..47f677d4 100644
--- a/cloudinit/sources/DataSourceEc2.py
+++ b/cloudinit/sources/DataSourceEc2.py
@@ -137,7 +137,7 @@ class DataSourceEc2(sources.DataSource):
 
         start_time = time.time()
         url = uhelp.wait_for_url(urls=urls, max_wait=max_wait,
-                                timeout=timeout, status_cb=LOG.warn)
+                                 timeout=timeout, status_cb=LOG.warn)
 
         if url:
             LOG.debug("Using metadata source: '%s'", url2base[url])
diff --git a/cloudinit/sources/DataSourceMAAS.py b/cloudinit/sources/DataSourceMAAS.py
index e187aec9..2de31305 100644
--- a/cloudinit/sources/DataSourceMAAS.py
+++ b/cloudinit/sources/DataSourceMAAS.py
@@ -25,9 +25,11 @@ import os
 import time
 import urllib2
 
+import requests
+
 from cloudinit import log as logging
 from cloudinit import sources
-from cloudinit import url_helper as uhelp
+from cloudinit import url_helper
 from cloudinit import util
 
 LOG = logging.getLogger(__name__)
@@ -191,8 +193,8 @@ def read_maas_seed_url(seed_url, header_cb=None, timeout=None,
     version=MD_VERSION):
     """
     Read the maas datasource at seed_url.
-    header_cb is a method that should return a headers dictionary that will
-    be given to urllib2.Request()
+      - header_cb is a method that should return a headers dictionary for
+        a given url
 
     Expected format of seed_url is are the following files:
       * <seed_url>/<version>/meta-data/instance-id
@@ -220,13 +222,13 @@ def read_maas_seed_url(seed_url, header_cb=None, timeout=None,
         else:
             headers = {}
         try:
-            resp = uhelp.readurl(url, headers=headers, timeout=timeout)
-            if resp.ok():
+            resp = util.read_file_or_url(url, headers=headers, timeout=timeout)
+            if resp.ok:
                 md[name] = str(resp)
             else:
                 LOG.warn(("Fetching from %s resulted in"
-                          " an invalid http code %s"), url, resp.code)
-        except urllib2.HTTPError as e:
+                          " an invalid http code %s"), url, resp.status_code)
+        except url_helper.UrlError as e:
             if e.code != 404:
                 raise
     return check_seed_contents(md, seed_url)
diff --git a/cloudinit/url_helper.py b/cloudinit/url_helper.py
index e8330e24..0839e63b 100644
--- a/cloudinit/url_helper.py
+++ b/cloudinit/url_helper.py
@@ -58,6 +58,44 @@ def _cleanurl(url):
     return urlunparse(parsed_url)
 
 
+class UrlResponse(object):
+    def __init__(self, response):
+        self._response = response
+
+    @property
+    def contents(self):
+        return self._response.content
+
+    @property
+    def url(self):
+        return self._response.url
+
+    @property
+    def ok(self):
+        return self._response.ok
+
+    @property
+    def headers(self):
+        return self._response.headers
+
+    @property
+    def code(self):
+        return self._response.status_code
+
+    def __str__(self):
+        return self.contents
+
+
+class UrlError(IOError):
+    def __init__(self, cause):
+        IOError.__init__(self, str(cause))
+        self.cause = cause
+        if isinstance(cause, exceptions.HTTPError) and cause.response:
+            self.code = cause.response.status_code
+        else:
+            self.code = None
+
+
 def readurl(url, data=None, timeout=None, retries=0, sec_between=1,
             headers=None, ssl_details=None, check_status=True,
             allow_redirects=False):
@@ -76,6 +114,8 @@ def readurl(url, data=None, timeout=None, retries=0, sec_between=1,
             if 'cert_file' in ssl_details and 'key_file' in ssl_details:
                 req_args['cert'] = [ssl_details['cert_file'],
                                     ssl_details['key_file']]
+            elif 'cert_file' in ssl_details:
+                req_args['cert'] = str(ssl_details['cert_file'])
                                     
     req_args['allow_redirects'] = allow_redirects
     req_args['method'] = 'GET'
@@ -126,13 +166,9 @@ def readurl(url, data=None, timeout=None, retries=0, sec_between=1,
             # Doesn't seem like we can make it use a different
             # subclass for responses, so add our own backward-compat
             # attrs
-            if not hasattr(r, 'code'):
-                setattr(r, 'code', status)
-            if not hasattr(r, 'contents'):
-                setattr(r, 'contents', contents)
-            return r
+            return UrlResponse(r)
         except exceptions.RequestException as e:
-            excps.append(e)
+            excps.append(UrlError(e))
             if i + 1 < manual_tries and sec_between > 0:
                 LOG.debug("Please wait %s seconds while we wait to try again",
                           sec_between)
@@ -213,7 +249,7 @@ def wait_for_url(urls, max_wait=None, timeout=None,
                     e = ValueError(reason)
                 else:
                     return url
-            except exceptions.RequestException as e:
+            except UrlError as e:
                 reason = "request error [%s]" % e
             except Exception as e:
                 reason = "unexpected error [%s]" % e
diff --git a/cloudinit/user_data.py b/cloudinit/user_data.py
index 803ffc3a..4a640f1e 100644
--- a/cloudinit/user_data.py
+++ b/cloudinit/user_data.py
@@ -29,7 +29,6 @@ from email.mime.text import MIMEText
 
 from cloudinit import handlers
 from cloudinit import log as logging
-from cloudinit import url_helper
 from cloudinit import util
 
 LOG = logging.getLogger(__name__)
@@ -173,10 +172,10 @@ class UserDataProcessor(object):
             if include_once_on and os.path.isfile(include_once_fn):
                 content = util.load_file(include_once_fn)
             else:
-                resp = url_helper.readurl(include_url)
-                if include_once_on and resp.ok():
+                resp = util.read_file_or_url(include_url)
+                if include_once_on and resp.ok:
                     util.write_file(include_once_fn, str(resp), mode=0600)
-                if resp.ok():
+                if resp.ok:
                     content = str(resp)
                 else:
                     LOG.warn(("Fetching from %s resulted in"
diff --git a/cloudinit/util.py b/cloudinit/util.py
index f5a7ac12..da2cdeda 100644
--- a/cloudinit/util.py
+++ b/cloudinit/util.py
@@ -51,7 +51,7 @@ import yaml
 from cloudinit import importer
 from cloudinit import log as logging
 from cloudinit import safeyaml
-from cloudinit import url_helper as uhelp
+from cloudinit import url_helper
 
 from cloudinit.settings import (CFG_BUILTIN)
 
@@ -69,6 +69,18 @@ FN_ALLOWED = ('_-.()' + string.digits + string.ascii_letters)
 CONTAINER_TESTS = ['running-in-container', 'lxc-is-container']
 
 
+class FileResponse(object):
+    def __init__(self, path, contents):
+        self.code = 200
+        self.headers = {}
+        self.contents = contents
+        self.ok = True
+        self.url = path
+
+    def __str__(self):
+        return self.contents
+
+
 class ProcessExecutionError(IOError):
 
     MESSAGE_TMPL = ('%(description)s\n'
@@ -628,12 +640,53 @@ def read_optional_seed(fill, base="", ext="", timeout=5):
         raise
 
 
-def read_file_or_url(url, timeout=5, retries=10, file_retries=0):
+def fetch_ssl_details(paths=None):
+    ssl_details = {}
+    # Lookup in these locations for ssl key/cert files
+    ssl_cert_paths = [
+        '/var/lib/cloud/data/ssl',
+        '/var/lib/cloud/instance/data/ssl',
+    ]
+    if paths:
+        ssl_cert_paths.extend([
+            os.path.join(paths.get_ipath_cur('data'), 'ssl'),
+            os.path.join(paths.get_cpath('data'), 'ssl'),
+        ])
+    ssl_cert_paths = uniq_merge(ssl_cert_paths)
+    ssl_cert_paths = [d for d in ssl_cert_paths if d and os.path.isdir(d)]
+    cert_file = None
+    for d in ssl_cert_paths:
+        if os.path.isfile(os.path.join(d, 'cert.pem')):
+            cert_file = os.path.join(d, 'cert.pem')
+            break
+    key_file = None
+    for d in ssl_cert_paths:
+        if os.path.isfile(os.path.join(d, 'key.pem')):
+            key_file = os.path.join(d, 'key.pem')
+            break
+    if cert_file and key_file:
+        ssl_details['cert_file'] = cert_file
+        ssl_details['key_file'] = key_file
+    elif cert_file:
+        ssl_details['cert_file'] = cert_file
+    return ssl_details
+
+
+def read_file_or_url(url, timeout=5, retries=10,
+                     headers=None, data=None, sec_between=1, paths=None):
     if url.startswith("/"):
         url = "file://%s" % url
-    if url.startswith("file://"):
-        retries = file_retries
-    return uhelp.readurl(url, timeout=timeout, retries=retries)
+    if url.lower().startswith("file://"):
+        file_path = url[len("file://"):]
+        return FileResponse(file_path, contents=load_file(file_path))
+    else:
+        return url_helper.readurl(url,
+                                  timeout=timeout,
+                                  retries=retries,
+                                  headers=headers,
+                                  data=data,
+                                  sec_between=sec_between,
+                                  ssl_details=fetch_ssl_details(paths))
 
 
 def load_yaml(blob, default=None, allowed=(dict,)):
@@ -675,13 +728,13 @@ def read_seeded(base="", ext="", timeout=5, retries=10, file_retries=0):
 
     md_resp = read_file_or_url(md_url, timeout, retries, file_retries)
     md = None
-    if md_resp.ok():
+    if md_resp.ok:
         md_str = str(md_resp)
         md = load_yaml(md_str, default={})
 
     ud_resp = read_file_or_url(ud_url, timeout, retries, file_retries)
     ud = None
-    if ud_resp.ok():
+    if ud_resp.ok:
         ud_str = str(ud_resp)
         ud = ud_str
 
@@ -850,8 +903,8 @@ def get_cmdline_url(names=('cloud-config-url', 'url'),
     if not url:
         return (None, None, None)
 
-    resp = uhelp.readurl(url)
-    if resp.contents.startswith(starts) and resp.ok():
+    resp = read_file_or_url(url)
+    if resp.contents.startswith(starts) and resp.ok:
         return (key, url, str(resp))
 
     return (key, url, None)