summaryrefslogtreecommitdiff
diff options
context:
space:
mode:
authorharlowja <harlowja@virtualbox.rhel>2013-02-19 22:51:49 -0800
committerharlowja <harlowja@virtualbox.rhel>2013-02-19 22:51:49 -0800
commit6d40d5b2e3da9577d4a2686444d47125e62817fe (patch)
tree3dc28b3e6157513aedb4a21fcabf35aaefaf5e1f
parent6a6a2f7c337e18d84d7031ba60d5adf6a93256fc (diff)
downloadvyos-cloud-init-6d40d5b2e3da9577d4a2686444d47125e62817fe.tar.gz
vyos-cloud-init-6d40d5b2e3da9577d4a2686444d47125e62817fe.zip
Continue working on integrating requests.
-rw-r--r--cloudinit/config/cc_phone_home.py4
-rw-r--r--cloudinit/ec2_utils.py24
-rw-r--r--cloudinit/sources/DataSourceEc2.py2
-rw-r--r--cloudinit/sources/DataSourceMAAS.py16
-rw-r--r--cloudinit/url_helper.py50
-rw-r--r--cloudinit/user_data.py7
-rw-r--r--cloudinit/util.py71
7 files changed, 129 insertions, 45 deletions
diff --git a/cloudinit/config/cc_phone_home.py b/cloudinit/config/cc_phone_home.py
index ae1349eb..90834080 100644
--- a/cloudinit/config/cc_phone_home.py
+++ b/cloudinit/config/cc_phone_home.py
@@ -19,7 +19,6 @@
# along with this program. If not, see <http://www.gnu.org/licenses/>.
from cloudinit import templater
-from cloudinit import url_helper as uhelp
from cloudinit import util
from cloudinit.settings import PER_INSTANCE
@@ -112,7 +111,8 @@ def handle(name, cfg, cloud, log, args):
}
url = templater.render_string(url, url_params)
try:
- uhelp.readurl(url, data=real_submit_keys, retries=tries, sec_between=3)
+ util.read_file_or_url(url, data=real_submit_keys,
+ retries=tries, sec_between=3)
except:
util.logexc(log, ("Failed to post phone home data to"
" %s in %s tries"), url, tries)
diff --git a/cloudinit/ec2_utils.py b/cloudinit/ec2_utils.py
index b9d7a2f7..c422eea9 100644
--- a/cloudinit/ec2_utils.py
+++ b/cloudinit/ec2_utils.py
@@ -22,7 +22,6 @@ import json
import urllib
from cloudinit import log as logging
-from cloudinit import url_helper as uh
from cloudinit import util
LOG = logging.getLogger(__name__)
@@ -40,11 +39,10 @@ def combine_url(base, add_on):
# See: http://bit.ly/TyoUQs
class MetadataMaterializer(object):
- def __init__(self, blob, base_url, **fetch_settings):
+ def __init__(self, blob, base_url):
self._blob = blob
self._md = None
self._base_url = base_url
- self._fetch_settings = fetch_settings
def _parse(self, blob):
leaves = {}
@@ -90,8 +88,8 @@ class MetadataMaterializer(object):
self._md = self._materialize(self._blob, self._base_url)
return self._md
- def _fetch_url(self, url, **opts):
- response = uh.readurl(url, **opts)
+ def _fetch_url(self, url):
+ response = util.read_file_or_url(url)
return str(response)
def _decode_leaf_blob(self, blob):
@@ -115,12 +113,12 @@ class MetadataMaterializer(object):
child_url = combine_url(base_url, c)
if not child_url.endswith("/"):
child_url += "/"
- child_blob = self._fetch_url(child_url, **self._fetch_settings)
+ child_blob = self._fetch_url(child_url)
child_contents[c] = self._materialize(child_blob, child_url)
leaf_contents = {}
for (field, resource) in leaves.items():
leaf_url = combine_url(base_url, resource)
- leaf_blob = self._fetch_url(leaf_url, **self._fetch_settings)
+ leaf_blob = self._fetch_url(leaf_url)
leaf_contents[field] = self._decode_leaf_blob(leaf_blob)
joined = {}
joined.update(child_contents)
@@ -136,23 +134,19 @@ def get_instance_userdata(url, version='latest', ssl_details=None):
ud_url = combine_url(url, version)
ud_url = combine_url(ud_url, 'user-data')
try:
- response = uh.readurl(ud_url, timeout=5,
- retries=10, ssl_details=ssl_details)
+ response = util.read_file_or_url(ud_url)
return str(response)
except Exception:
util.logexc(LOG, "Failed fetching userdata from url %s", ud_url)
return None
-def get_instance_metadata(url, version='latest', ssl_details=None):
+def get_instance_metadata(url, version='latest'):
md_url = combine_url(url, version)
md_url = combine_url(md_url, 'meta-data')
try:
- response = uh.readurl(md_url, timeout=5,
- retries=10, ssl_details=ssl_details)
- materializer = MetadataMaterializer(str(response), md_url,
- timeout=5, retries=10,
- ssl_details=ssl_details)
+ response = util.read_file_or_url(md_url)
+ materializer = MetadataMaterializer(str(response), md_url)
return materializer.materialize()
except Exception:
util.logexc(LOG, "Failed fetching metadata from url %s", md_url)
diff --git a/cloudinit/sources/DataSourceEc2.py b/cloudinit/sources/DataSourceEc2.py
index 204963e7..47f677d4 100644
--- a/cloudinit/sources/DataSourceEc2.py
+++ b/cloudinit/sources/DataSourceEc2.py
@@ -137,7 +137,7 @@ class DataSourceEc2(sources.DataSource):
start_time = time.time()
url = uhelp.wait_for_url(urls=urls, max_wait=max_wait,
- timeout=timeout, status_cb=LOG.warn)
+ timeout=timeout, status_cb=LOG.warn)
if url:
LOG.debug("Using metadata source: '%s'", url2base[url])
diff --git a/cloudinit/sources/DataSourceMAAS.py b/cloudinit/sources/DataSourceMAAS.py
index e187aec9..2de31305 100644
--- a/cloudinit/sources/DataSourceMAAS.py
+++ b/cloudinit/sources/DataSourceMAAS.py
@@ -25,9 +25,11 @@ import os
import time
import urllib2
+import requests
+
from cloudinit import log as logging
from cloudinit import sources
-from cloudinit import url_helper as uhelp
+from cloudinit import url_helper
from cloudinit import util
LOG = logging.getLogger(__name__)
@@ -191,8 +193,8 @@ def read_maas_seed_url(seed_url, header_cb=None, timeout=None,
version=MD_VERSION):
"""
Read the maas datasource at seed_url.
- header_cb is a method that should return a headers dictionary that will
- be given to urllib2.Request()
+ - header_cb is a method that should return a headers dictionary for
+ a given url
Expected format of seed_url is are the following files:
* <seed_url>/<version>/meta-data/instance-id
@@ -220,13 +222,13 @@ def read_maas_seed_url(seed_url, header_cb=None, timeout=None,
else:
headers = {}
try:
- resp = uhelp.readurl(url, headers=headers, timeout=timeout)
- if resp.ok():
+ resp = util.read_file_or_url(url, headers=headers, timeout=timeout)
+ if resp.ok:
md[name] = str(resp)
else:
LOG.warn(("Fetching from %s resulted in"
- " an invalid http code %s"), url, resp.code)
- except urllib2.HTTPError as e:
+ " an invalid http code %s"), url, resp.status_code)
+ except url_helper.UrlError as e:
if e.code != 404:
raise
return check_seed_contents(md, seed_url)
diff --git a/cloudinit/url_helper.py b/cloudinit/url_helper.py
index e8330e24..0839e63b 100644
--- a/cloudinit/url_helper.py
+++ b/cloudinit/url_helper.py
@@ -58,6 +58,44 @@ def _cleanurl(url):
return urlunparse(parsed_url)
+class UrlResponse(object):
+ def __init__(self, response):
+ self._response = response
+
+ @property
+ def contents(self):
+ return self._response.content
+
+ @property
+ def url(self):
+ return self._response.url
+
+ @property
+ def ok(self):
+ return self._response.ok
+
+ @property
+ def headers(self):
+ return self._response.headers
+
+ @property
+ def code(self):
+ return self._response.status_code
+
+ def __str__(self):
+ return self.contents
+
+
+class UrlError(IOError):
+ def __init__(self, cause):
+ IOError.__init__(self, str(cause))
+ self.cause = cause
+ if isinstance(cause, exceptions.HTTPError) and cause.response:
+ self.code = cause.response.status_code
+ else:
+ self.code = None
+
+
def readurl(url, data=None, timeout=None, retries=0, sec_between=1,
headers=None, ssl_details=None, check_status=True,
allow_redirects=False):
@@ -76,6 +114,8 @@ def readurl(url, data=None, timeout=None, retries=0, sec_between=1,
if 'cert_file' in ssl_details and 'key_file' in ssl_details:
req_args['cert'] = [ssl_details['cert_file'],
ssl_details['key_file']]
+ elif 'cert_file' in ssl_details:
+ req_args['cert'] = str(ssl_details['cert_file'])
req_args['allow_redirects'] = allow_redirects
req_args['method'] = 'GET'
@@ -126,13 +166,9 @@ def readurl(url, data=None, timeout=None, retries=0, sec_between=1,
# Doesn't seem like we can make it use a different
# subclass for responses, so add our own backward-compat
# attrs
- if not hasattr(r, 'code'):
- setattr(r, 'code', status)
- if not hasattr(r, 'contents'):
- setattr(r, 'contents', contents)
- return r
+ return UrlResponse(r)
except exceptions.RequestException as e:
- excps.append(e)
+ excps.append(UrlError(e))
if i + 1 < manual_tries and sec_between > 0:
LOG.debug("Please wait %s seconds while we wait to try again",
sec_between)
@@ -213,7 +249,7 @@ def wait_for_url(urls, max_wait=None, timeout=None,
e = ValueError(reason)
else:
return url
- except exceptions.RequestException as e:
+ except UrlError as e:
reason = "request error [%s]" % e
except Exception as e:
reason = "unexpected error [%s]" % e
diff --git a/cloudinit/user_data.py b/cloudinit/user_data.py
index 803ffc3a..4a640f1e 100644
--- a/cloudinit/user_data.py
+++ b/cloudinit/user_data.py
@@ -29,7 +29,6 @@ from email.mime.text import MIMEText
from cloudinit import handlers
from cloudinit import log as logging
-from cloudinit import url_helper
from cloudinit import util
LOG = logging.getLogger(__name__)
@@ -173,10 +172,10 @@ class UserDataProcessor(object):
if include_once_on and os.path.isfile(include_once_fn):
content = util.load_file(include_once_fn)
else:
- resp = url_helper.readurl(include_url)
- if include_once_on and resp.ok():
+ resp = util.read_file_or_url(include_url)
+ if include_once_on and resp.ok:
util.write_file(include_once_fn, str(resp), mode=0600)
- if resp.ok():
+ if resp.ok:
content = str(resp)
else:
LOG.warn(("Fetching from %s resulted in"
diff --git a/cloudinit/util.py b/cloudinit/util.py
index f5a7ac12..da2cdeda 100644
--- a/cloudinit/util.py
+++ b/cloudinit/util.py
@@ -51,7 +51,7 @@ import yaml
from cloudinit import importer
from cloudinit import log as logging
from cloudinit import safeyaml
-from cloudinit import url_helper as uhelp
+from cloudinit import url_helper
from cloudinit.settings import (CFG_BUILTIN)
@@ -69,6 +69,18 @@ FN_ALLOWED = ('_-.()' + string.digits + string.ascii_letters)
CONTAINER_TESTS = ['running-in-container', 'lxc-is-container']
+class FileResponse(object):
+ def __init__(self, path, contents):
+ self.code = 200
+ self.headers = {}
+ self.contents = contents
+ self.ok = True
+ self.url = path
+
+ def __str__(self):
+ return self.contents
+
+
class ProcessExecutionError(IOError):
MESSAGE_TMPL = ('%(description)s\n'
@@ -628,12 +640,53 @@ def read_optional_seed(fill, base="", ext="", timeout=5):
raise
-def read_file_or_url(url, timeout=5, retries=10, file_retries=0):
+def fetch_ssl_details(paths=None):
+ ssl_details = {}
+ # Lookup in these locations for ssl key/cert files
+ ssl_cert_paths = [
+ '/var/lib/cloud/data/ssl',
+ '/var/lib/cloud/instance/data/ssl',
+ ]
+ if paths:
+ ssl_cert_paths.extend([
+ os.path.join(paths.get_ipath_cur('data'), 'ssl'),
+ os.path.join(paths.get_cpath('data'), 'ssl'),
+ ])
+ ssl_cert_paths = uniq_merge(ssl_cert_paths)
+ ssl_cert_paths = [d for d in ssl_cert_paths if d and os.path.isdir(d)]
+ cert_file = None
+ for d in ssl_cert_paths:
+ if os.path.isfile(os.path.join(d, 'cert.pem')):
+ cert_file = os.path.join(d, 'cert.pem')
+ break
+ key_file = None
+ for d in ssl_cert_paths:
+ if os.path.isfile(os.path.join(d, 'key.pem')):
+ key_file = os.path.join(d, 'key.pem')
+ break
+ if cert_file and key_file:
+ ssl_details['cert_file'] = cert_file
+ ssl_details['key_file'] = key_file
+ elif cert_file:
+ ssl_details['cert_file'] = cert_file
+ return ssl_details
+
+
+def read_file_or_url(url, timeout=5, retries=10,
+ headers=None, data=None, sec_between=1, paths=None):
if url.startswith("/"):
url = "file://%s" % url
- if url.startswith("file://"):
- retries = file_retries
- return uhelp.readurl(url, timeout=timeout, retries=retries)
+ if url.lower().startswith("file://"):
+ file_path = url[len("file://"):]
+ return FileResponse(file_path, contents=load_file(file_path))
+ else:
+ return url_helper.readurl(url,
+ timeout=timeout,
+ retries=retries,
+ headers=headers,
+ data=data,
+ sec_between=sec_between,
+ ssl_details=fetch_ssl_details(paths))
def load_yaml(blob, default=None, allowed=(dict,)):
@@ -675,13 +728,13 @@ def read_seeded(base="", ext="", timeout=5, retries=10, file_retries=0):
md_resp = read_file_or_url(md_url, timeout, retries, file_retries)
md = None
- if md_resp.ok():
+ if md_resp.ok:
md_str = str(md_resp)
md = load_yaml(md_str, default={})
ud_resp = read_file_or_url(ud_url, timeout, retries, file_retries)
ud = None
- if ud_resp.ok():
+ if ud_resp.ok:
ud_str = str(ud_resp)
ud = ud_str
@@ -850,8 +903,8 @@ def get_cmdline_url(names=('cloud-config-url', 'url'),
if not url:
return (None, None, None)
- resp = uhelp.readurl(url)
- if resp.contents.startswith(starts) and resp.ok():
+ resp = read_file_or_url(url)
+ if resp.contents.startswith(starts) and resp.ok:
return (key, url, str(resp))
return (key, url, None)