From 7c9bbbc9b49425e3ba8e0517908477c58ea51d4b Mon Sep 17 00:00:00 2001 From: Joshua Harlow Date: Fri, 19 Oct 2012 14:06:21 -0700 Subject: Remove the need for boto just for fetching the userdata and metadata. Add in this crawling functionality to the ec2_utils module that will fully crawl (not lazily) the ec2 metadata and parse it in the same manner as boto. 1. Make the ec2 datasource + cloudstack now call into these. 2. Fix phone_home due to urllib3 change (TBD) --- cloudinit/sources/DataSourceCloudStack.py | 9 +++------ cloudinit/sources/DataSourceEc2.py | 9 +++------ 2 files changed, 6 insertions(+), 12 deletions(-) (limited to 'cloudinit/sources') diff --git a/cloudinit/sources/DataSourceCloudStack.py b/cloudinit/sources/DataSourceCloudStack.py index f7ffa7cb..2654df53 100644 --- a/cloudinit/sources/DataSourceCloudStack.py +++ b/cloudinit/sources/DataSourceCloudStack.py @@ -26,8 +26,7 @@ from struct import pack import os import time -import boto.utils as boto_utils - +from cloudinit import ec2_utils from cloudinit import log as logging from cloudinit import sources from cloudinit import url_helper as uhelp @@ -116,10 +115,8 @@ class DataSourceCloudStack(sources.DataSource): if not self.wait_for_metadata_service(): return False start_time = time.time() - self.userdata_raw = boto_utils.get_instance_userdata(self.api_ver, - None, self.metadata_address) - self.metadata = boto_utils.get_instance_metadata(self.api_ver, - self.metadata_address) + self.userdata_raw = ec2_utils.get_instance_userdata(self.metadata_address, self.api_ver) + self.metadata = ec2_utils.get_instance_metadata(self.metadata_address, self.api_ver) LOG.debug("Crawl of metadata service took %s seconds", int(time.time() - start_time)) return True diff --git a/cloudinit/sources/DataSourceEc2.py b/cloudinit/sources/DataSourceEc2.py index 3686fa10..204963e7 100644 --- a/cloudinit/sources/DataSourceEc2.py +++ b/cloudinit/sources/DataSourceEc2.py @@ -23,8 +23,7 @@ import os import time -import boto.utils as boto_utils - +from cloudinit import ec2_utils from cloudinit import log as logging from cloudinit import sources from cloudinit import url_helper as uhelp @@ -65,10 +64,8 @@ class DataSourceEc2(sources.DataSource): if not self.wait_for_metadata_service(): return False start_time = time.time() - self.userdata_raw = boto_utils.get_instance_userdata(self.api_ver, - None, self.metadata_address) - self.metadata = boto_utils.get_instance_metadata(self.api_ver, - self.metadata_address) + self.userdata_raw = ec2_utils.get_instance_userdata(self.metadata_address, self.api_ver) + self.metadata = ec2_utils.get_instance_metadata(self.metadata_address, self.api_ver) LOG.debug("Crawl of metadata service took %s seconds", int(time.time() - start_time)) return True -- cgit v1.2.3 From 6d40d5b2e3da9577d4a2686444d47125e62817fe Mon Sep 17 00:00:00 2001 From: harlowja Date: Tue, 19 Feb 2013 22:51:49 -0800 Subject: Continue working on integrating requests. --- cloudinit/config/cc_phone_home.py | 4 +-- cloudinit/ec2_utils.py | 24 +++++-------- cloudinit/sources/DataSourceEc2.py | 2 +- cloudinit/sources/DataSourceMAAS.py | 16 +++++---- cloudinit/url_helper.py | 50 ++++++++++++++++++++++---- cloudinit/user_data.py | 7 ++-- cloudinit/util.py | 71 ++++++++++++++++++++++++++++++++----- 7 files changed, 129 insertions(+), 45 deletions(-) (limited to 'cloudinit/sources') diff --git a/cloudinit/config/cc_phone_home.py b/cloudinit/config/cc_phone_home.py index ae1349eb..90834080 100644 --- a/cloudinit/config/cc_phone_home.py +++ b/cloudinit/config/cc_phone_home.py @@ -19,7 +19,6 @@ # along with this program. If not, see . from cloudinit import templater -from cloudinit import url_helper as uhelp from cloudinit import util from cloudinit.settings import PER_INSTANCE @@ -112,7 +111,8 @@ def handle(name, cfg, cloud, log, args): } url = templater.render_string(url, url_params) try: - uhelp.readurl(url, data=real_submit_keys, retries=tries, sec_between=3) + util.read_file_or_url(url, data=real_submit_keys, + retries=tries, sec_between=3) except: util.logexc(log, ("Failed to post phone home data to" " %s in %s tries"), url, tries) diff --git a/cloudinit/ec2_utils.py b/cloudinit/ec2_utils.py index b9d7a2f7..c422eea9 100644 --- a/cloudinit/ec2_utils.py +++ b/cloudinit/ec2_utils.py @@ -22,7 +22,6 @@ import json import urllib from cloudinit import log as logging -from cloudinit import url_helper as uh from cloudinit import util LOG = logging.getLogger(__name__) @@ -40,11 +39,10 @@ def combine_url(base, add_on): # See: http://bit.ly/TyoUQs class MetadataMaterializer(object): - def __init__(self, blob, base_url, **fetch_settings): + def __init__(self, blob, base_url): self._blob = blob self._md = None self._base_url = base_url - self._fetch_settings = fetch_settings def _parse(self, blob): leaves = {} @@ -90,8 +88,8 @@ class MetadataMaterializer(object): self._md = self._materialize(self._blob, self._base_url) return self._md - def _fetch_url(self, url, **opts): - response = uh.readurl(url, **opts) + def _fetch_url(self, url): + response = util.read_file_or_url(url) return str(response) def _decode_leaf_blob(self, blob): @@ -115,12 +113,12 @@ class MetadataMaterializer(object): child_url = combine_url(base_url, c) if not child_url.endswith("/"): child_url += "/" - child_blob = self._fetch_url(child_url, **self._fetch_settings) + child_blob = self._fetch_url(child_url) child_contents[c] = self._materialize(child_blob, child_url) leaf_contents = {} for (field, resource) in leaves.items(): leaf_url = combine_url(base_url, resource) - leaf_blob = self._fetch_url(leaf_url, **self._fetch_settings) + leaf_blob = self._fetch_url(leaf_url) leaf_contents[field] = self._decode_leaf_blob(leaf_blob) joined = {} joined.update(child_contents) @@ -136,23 +134,19 @@ def get_instance_userdata(url, version='latest', ssl_details=None): ud_url = combine_url(url, version) ud_url = combine_url(ud_url, 'user-data') try: - response = uh.readurl(ud_url, timeout=5, - retries=10, ssl_details=ssl_details) + response = util.read_file_or_url(ud_url) return str(response) except Exception: util.logexc(LOG, "Failed fetching userdata from url %s", ud_url) return None -def get_instance_metadata(url, version='latest', ssl_details=None): +def get_instance_metadata(url, version='latest'): md_url = combine_url(url, version) md_url = combine_url(md_url, 'meta-data') try: - response = uh.readurl(md_url, timeout=5, - retries=10, ssl_details=ssl_details) - materializer = MetadataMaterializer(str(response), md_url, - timeout=5, retries=10, - ssl_details=ssl_details) + response = util.read_file_or_url(md_url) + materializer = MetadataMaterializer(str(response), md_url) return materializer.materialize() except Exception: util.logexc(LOG, "Failed fetching metadata from url %s", md_url) diff --git a/cloudinit/sources/DataSourceEc2.py b/cloudinit/sources/DataSourceEc2.py index 204963e7..47f677d4 100644 --- a/cloudinit/sources/DataSourceEc2.py +++ b/cloudinit/sources/DataSourceEc2.py @@ -137,7 +137,7 @@ class DataSourceEc2(sources.DataSource): start_time = time.time() url = uhelp.wait_for_url(urls=urls, max_wait=max_wait, - timeout=timeout, status_cb=LOG.warn) + timeout=timeout, status_cb=LOG.warn) if url: LOG.debug("Using metadata source: '%s'", url2base[url]) diff --git a/cloudinit/sources/DataSourceMAAS.py b/cloudinit/sources/DataSourceMAAS.py index e187aec9..2de31305 100644 --- a/cloudinit/sources/DataSourceMAAS.py +++ b/cloudinit/sources/DataSourceMAAS.py @@ -25,9 +25,11 @@ import os import time import urllib2 +import requests + from cloudinit import log as logging from cloudinit import sources -from cloudinit import url_helper as uhelp +from cloudinit import url_helper from cloudinit import util LOG = logging.getLogger(__name__) @@ -191,8 +193,8 @@ def read_maas_seed_url(seed_url, header_cb=None, timeout=None, version=MD_VERSION): """ Read the maas datasource at seed_url. - header_cb is a method that should return a headers dictionary that will - be given to urllib2.Request() + - header_cb is a method that should return a headers dictionary for + a given url Expected format of seed_url is are the following files: * //meta-data/instance-id @@ -220,13 +222,13 @@ def read_maas_seed_url(seed_url, header_cb=None, timeout=None, else: headers = {} try: - resp = uhelp.readurl(url, headers=headers, timeout=timeout) - if resp.ok(): + resp = util.read_file_or_url(url, headers=headers, timeout=timeout) + if resp.ok: md[name] = str(resp) else: LOG.warn(("Fetching from %s resulted in" - " an invalid http code %s"), url, resp.code) - except urllib2.HTTPError as e: + " an invalid http code %s"), url, resp.status_code) + except url_helper.UrlError as e: if e.code != 404: raise return check_seed_contents(md, seed_url) diff --git a/cloudinit/url_helper.py b/cloudinit/url_helper.py index e8330e24..0839e63b 100644 --- a/cloudinit/url_helper.py +++ b/cloudinit/url_helper.py @@ -58,6 +58,44 @@ def _cleanurl(url): return urlunparse(parsed_url) +class UrlResponse(object): + def __init__(self, response): + self._response = response + + @property + def contents(self): + return self._response.content + + @property + def url(self): + return self._response.url + + @property + def ok(self): + return self._response.ok + + @property + def headers(self): + return self._response.headers + + @property + def code(self): + return self._response.status_code + + def __str__(self): + return self.contents + + +class UrlError(IOError): + def __init__(self, cause): + IOError.__init__(self, str(cause)) + self.cause = cause + if isinstance(cause, exceptions.HTTPError) and cause.response: + self.code = cause.response.status_code + else: + self.code = None + + def readurl(url, data=None, timeout=None, retries=0, sec_between=1, headers=None, ssl_details=None, check_status=True, allow_redirects=False): @@ -76,6 +114,8 @@ def readurl(url, data=None, timeout=None, retries=0, sec_between=1, if 'cert_file' in ssl_details and 'key_file' in ssl_details: req_args['cert'] = [ssl_details['cert_file'], ssl_details['key_file']] + elif 'cert_file' in ssl_details: + req_args['cert'] = str(ssl_details['cert_file']) req_args['allow_redirects'] = allow_redirects req_args['method'] = 'GET' @@ -126,13 +166,9 @@ def readurl(url, data=None, timeout=None, retries=0, sec_between=1, # Doesn't seem like we can make it use a different # subclass for responses, so add our own backward-compat # attrs - if not hasattr(r, 'code'): - setattr(r, 'code', status) - if not hasattr(r, 'contents'): - setattr(r, 'contents', contents) - return r + return UrlResponse(r) except exceptions.RequestException as e: - excps.append(e) + excps.append(UrlError(e)) if i + 1 < manual_tries and sec_between > 0: LOG.debug("Please wait %s seconds while we wait to try again", sec_between) @@ -213,7 +249,7 @@ def wait_for_url(urls, max_wait=None, timeout=None, e = ValueError(reason) else: return url - except exceptions.RequestException as e: + except UrlError as e: reason = "request error [%s]" % e except Exception as e: reason = "unexpected error [%s]" % e diff --git a/cloudinit/user_data.py b/cloudinit/user_data.py index 803ffc3a..4a640f1e 100644 --- a/cloudinit/user_data.py +++ b/cloudinit/user_data.py @@ -29,7 +29,6 @@ from email.mime.text import MIMEText from cloudinit import handlers from cloudinit import log as logging -from cloudinit import url_helper from cloudinit import util LOG = logging.getLogger(__name__) @@ -173,10 +172,10 @@ class UserDataProcessor(object): if include_once_on and os.path.isfile(include_once_fn): content = util.load_file(include_once_fn) else: - resp = url_helper.readurl(include_url) - if include_once_on and resp.ok(): + resp = util.read_file_or_url(include_url) + if include_once_on and resp.ok: util.write_file(include_once_fn, str(resp), mode=0600) - if resp.ok(): + if resp.ok: content = str(resp) else: LOG.warn(("Fetching from %s resulted in" diff --git a/cloudinit/util.py b/cloudinit/util.py index f5a7ac12..da2cdeda 100644 --- a/cloudinit/util.py +++ b/cloudinit/util.py @@ -51,7 +51,7 @@ import yaml from cloudinit import importer from cloudinit import log as logging from cloudinit import safeyaml -from cloudinit import url_helper as uhelp +from cloudinit import url_helper from cloudinit.settings import (CFG_BUILTIN) @@ -69,6 +69,18 @@ FN_ALLOWED = ('_-.()' + string.digits + string.ascii_letters) CONTAINER_TESTS = ['running-in-container', 'lxc-is-container'] +class FileResponse(object): + def __init__(self, path, contents): + self.code = 200 + self.headers = {} + self.contents = contents + self.ok = True + self.url = path + + def __str__(self): + return self.contents + + class ProcessExecutionError(IOError): MESSAGE_TMPL = ('%(description)s\n' @@ -628,12 +640,53 @@ def read_optional_seed(fill, base="", ext="", timeout=5): raise -def read_file_or_url(url, timeout=5, retries=10, file_retries=0): +def fetch_ssl_details(paths=None): + ssl_details = {} + # Lookup in these locations for ssl key/cert files + ssl_cert_paths = [ + '/var/lib/cloud/data/ssl', + '/var/lib/cloud/instance/data/ssl', + ] + if paths: + ssl_cert_paths.extend([ + os.path.join(paths.get_ipath_cur('data'), 'ssl'), + os.path.join(paths.get_cpath('data'), 'ssl'), + ]) + ssl_cert_paths = uniq_merge(ssl_cert_paths) + ssl_cert_paths = [d for d in ssl_cert_paths if d and os.path.isdir(d)] + cert_file = None + for d in ssl_cert_paths: + if os.path.isfile(os.path.join(d, 'cert.pem')): + cert_file = os.path.join(d, 'cert.pem') + break + key_file = None + for d in ssl_cert_paths: + if os.path.isfile(os.path.join(d, 'key.pem')): + key_file = os.path.join(d, 'key.pem') + break + if cert_file and key_file: + ssl_details['cert_file'] = cert_file + ssl_details['key_file'] = key_file + elif cert_file: + ssl_details['cert_file'] = cert_file + return ssl_details + + +def read_file_or_url(url, timeout=5, retries=10, + headers=None, data=None, sec_between=1, paths=None): if url.startswith("/"): url = "file://%s" % url - if url.startswith("file://"): - retries = file_retries - return uhelp.readurl(url, timeout=timeout, retries=retries) + if url.lower().startswith("file://"): + file_path = url[len("file://"):] + return FileResponse(file_path, contents=load_file(file_path)) + else: + return url_helper.readurl(url, + timeout=timeout, + retries=retries, + headers=headers, + data=data, + sec_between=sec_between, + ssl_details=fetch_ssl_details(paths)) def load_yaml(blob, default=None, allowed=(dict,)): @@ -675,13 +728,13 @@ def read_seeded(base="", ext="", timeout=5, retries=10, file_retries=0): md_resp = read_file_or_url(md_url, timeout, retries, file_retries) md = None - if md_resp.ok(): + if md_resp.ok: md_str = str(md_resp) md = load_yaml(md_str, default={}) ud_resp = read_file_or_url(ud_url, timeout, retries, file_retries) ud = None - if ud_resp.ok(): + if ud_resp.ok: ud_str = str(ud_resp) ud = ud_str @@ -850,8 +903,8 @@ def get_cmdline_url(names=('cloud-config-url', 'url'), if not url: return (None, None, None) - resp = uhelp.readurl(url) - if resp.contents.startswith(starts) and resp.ok(): + resp = read_file_or_url(url) + if resp.contents.startswith(starts) and resp.ok: return (key, url, str(resp)) return (key, url, None) -- cgit v1.2.3 From 9dfb60d3144860334ab1ad1d72920d962139461f Mon Sep 17 00:00:00 2001 From: harlowja Date: Thu, 21 Feb 2013 22:39:30 -0800 Subject: More work on requests integration. --- cloudinit/config/cc_phone_home.py | 3 ++- cloudinit/ec2_utils.py | 17 +++++++++++------ cloudinit/sources/DataSourceEc2.py | 6 ++++-- cloudinit/sources/DataSourceMAAS.py | 15 +++++++++------ cloudinit/util.py | 4 ++-- 5 files changed, 28 insertions(+), 17 deletions(-) (limited to 'cloudinit/sources') diff --git a/cloudinit/config/cc_phone_home.py b/cloudinit/config/cc_phone_home.py index 90834080..5a4332ef 100644 --- a/cloudinit/config/cc_phone_home.py +++ b/cloudinit/config/cc_phone_home.py @@ -112,7 +112,8 @@ def handle(name, cfg, cloud, log, args): url = templater.render_string(url, url_params) try: util.read_file_or_url(url, data=real_submit_keys, - retries=tries, sec_between=3) + retries=tries, sec_between=3, + ssl_details=util.fetch_ssl_details(cloud.paths)) except: util.logexc(log, ("Failed to post phone home data to" " %s in %s tries"), url, tries) diff --git a/cloudinit/ec2_utils.py b/cloudinit/ec2_utils.py index c422eea9..026ee178 100644 --- a/cloudinit/ec2_utils.py +++ b/cloudinit/ec2_utils.py @@ -38,11 +38,16 @@ def combine_url(base, add_on): # See: http://bit.ly/TyoUQs +# +# Since boto metadata reader uses the old urllib which does not +# support ssl, we need to ahead and create our own reader which +# works the same as the boto one (for now). class MetadataMaterializer(object): - def __init__(self, blob, base_url): + def __init__(self, blob, base_url, ssl_details): self._blob = blob self._md = None self._base_url = base_url + self._ssl_details = ssl_details def _parse(self, blob): leaves = {} @@ -89,7 +94,7 @@ class MetadataMaterializer(object): return self._md def _fetch_url(self, url): - response = util.read_file_or_url(url) + response = util.read_file_or_url(url, ssl_details=self._ssl_details) return str(response) def _decode_leaf_blob(self, blob): @@ -134,19 +139,19 @@ def get_instance_userdata(url, version='latest', ssl_details=None): ud_url = combine_url(url, version) ud_url = combine_url(ud_url, 'user-data') try: - response = util.read_file_or_url(ud_url) + response = util.read_file_or_url(ud_url, ssl_details=ssl_details) return str(response) except Exception: util.logexc(LOG, "Failed fetching userdata from url %s", ud_url) return None -def get_instance_metadata(url, version='latest'): +def get_instance_metadata(url, version='latest', ssl_details=None): md_url = combine_url(url, version) md_url = combine_url(md_url, 'meta-data') try: - response = util.read_file_or_url(md_url) - materializer = MetadataMaterializer(str(response), md_url) + response = util.read_file_or_url(md_url, ssl_details=ssl_details) + materializer = MetadataMaterializer(str(response), md_url, ssl_details) return materializer.materialize() except Exception: util.logexc(LOG, "Failed fetching metadata from url %s", md_url) diff --git a/cloudinit/sources/DataSourceEc2.py b/cloudinit/sources/DataSourceEc2.py index 47f677d4..eee4e6bc 100644 --- a/cloudinit/sources/DataSourceEc2.py +++ b/cloudinit/sources/DataSourceEc2.py @@ -64,8 +64,10 @@ class DataSourceEc2(sources.DataSource): if not self.wait_for_metadata_service(): return False start_time = time.time() - self.userdata_raw = ec2_utils.get_instance_userdata(self.metadata_address, self.api_ver) - self.metadata = ec2_utils.get_instance_metadata(self.metadata_address, self.api_ver) + self.userdata_raw = ec2_utils.get_instance_userdata(self.metadata_address, self.api_ver, + ssl_details=util.fetch_ssl_details(self.paths)) + self.metadata = ec2_utils.get_instance_metadata(self.metadata_address, self.api_ver, + ssl_details=util.fetch_ssl_details(self.paths)) LOG.debug("Crawl of metadata service took %s seconds", int(time.time() - start_time)) return True diff --git a/cloudinit/sources/DataSourceMAAS.py b/cloudinit/sources/DataSourceMAAS.py index 2de31305..dc048943 100644 --- a/cloudinit/sources/DataSourceMAAS.py +++ b/cloudinit/sources/DataSourceMAAS.py @@ -81,7 +81,8 @@ class DataSourceMAAS(sources.DataSource): self.base_url = url (userdata, metadata) = read_maas_seed_url(self.base_url, - self.md_headers) + self.md_headers, + paths=self.paths) self.userdata_raw = userdata self.metadata = metadata return True @@ -141,7 +142,7 @@ class DataSourceMAAS(sources.DataSource): LOG.debug("Using metadata source: '%s'", url) else: LOG.critical("Giving up on md from %s after %i seconds", - urls, int(time.time() - starttime)) + urls, int(time.time() - starttime)) return bool(url) @@ -190,7 +191,7 @@ def read_maas_seed_dir(seed_d): def read_maas_seed_url(seed_url, header_cb=None, timeout=None, - version=MD_VERSION): + version=MD_VERSION, paths=None): """ Read the maas datasource at seed_url. - header_cb is a method that should return a headers dictionary for @@ -222,12 +223,13 @@ def read_maas_seed_url(seed_url, header_cb=None, timeout=None, else: headers = {} try: - resp = util.read_file_or_url(url, headers=headers, timeout=timeout) + resp = util.read_file_or_url(url, headers=headers, timeout=timeout, + ssl_details=util.fetch_ssl_details(paths)) if resp.ok: md[name] = str(resp) else: LOG.warn(("Fetching from %s resulted in" - " an invalid http code %s"), url, resp.status_code) + " an invalid http code %s"), url, resp.code) except url_helper.UrlError as e: if e.code != 404: raise @@ -372,7 +374,8 @@ if __name__ == "__main__": if args.subcmd == "check-seed": if args.url.startswith("http"): (userdata, metadata) = read_maas_seed_url(args.url, - header_cb=my_headers, version=args.apiver) + header_cb=my_headers, + version=args.apiver) else: (userdata, metadata) = read_maas_seed_url(args.url) print "=== userdata ===" diff --git a/cloudinit/util.py b/cloudinit/util.py index da2cdeda..307ed613 100644 --- a/cloudinit/util.py +++ b/cloudinit/util.py @@ -673,7 +673,7 @@ def fetch_ssl_details(paths=None): def read_file_or_url(url, timeout=5, retries=10, - headers=None, data=None, sec_between=1, paths=None): + headers=None, data=None, sec_between=1, ssl_details=None): if url.startswith("/"): url = "file://%s" % url if url.lower().startswith("file://"): @@ -686,7 +686,7 @@ def read_file_or_url(url, timeout=5, retries=10, headers=headers, data=data, sec_between=sec_between, - ssl_details=fetch_ssl_details(paths)) + ssl_details=ssl_details) def load_yaml(blob, default=None, allowed=(dict,)): -- cgit v1.2.3 From eacfc7ffbec3e6a0348ed484da895e2d2fc5ba10 Mon Sep 17 00:00:00 2001 From: harlowja Date: Sat, 23 Feb 2013 21:23:24 -0800 Subject: Get tests working and further adjustments. --- cloudinit/sources/DataSourceMAAS.py | 9 +++++--- cloudinit/url_helper.py | 21 +++++++++-------- cloudinit/user_data.py | 8 ++++--- cloudinit/util.py | 34 ++++++++++++++++++++-------- tests/unittests/test__init__.py | 10 ++++---- tests/unittests/test_datasource/test_maas.py | 11 +++++---- 6 files changed, 60 insertions(+), 33 deletions(-) (limited to 'cloudinit/sources') diff --git a/cloudinit/sources/DataSourceMAAS.py b/cloudinit/sources/DataSourceMAAS.py index 6e1133b2..0c526305 100644 --- a/cloudinit/sources/DataSourceMAAS.py +++ b/cloudinit/sources/DataSourceMAAS.py @@ -223,9 +223,12 @@ def read_maas_seed_url(seed_url, header_cb=None, timeout=None, else: headers = {} try: - resp = util.read_file_or_url(url, headers=headers, timeout=timeout, - ssl_details=util.fetch_ssl_details(paths)) - if resp.ok: + ssl_details = util.fetch_ssl_details(paths) + resp = util.read_file_or_url(url, + headers=headers, + timeout=timeout, + ssl_details=ssl_details) + if resp.ok(): md[name] = str(resp) else: LOG.warn(("Fetching from %s resulted in" diff --git a/cloudinit/url_helper.py b/cloudinit/url_helper.py index 0839e63b..300e70c2 100644 --- a/cloudinit/url_helper.py +++ b/cloudinit/url_helper.py @@ -70,9 +70,14 @@ class UrlResponse(object): def url(self): return self._response.url - @property - def ok(self): - return self._response.ok + def ok(self, redirects_ok=False): + upper = 300 + if redirects_ok: + upper = 400 + if self.code >= 200 and self.code < upper: + return True + else: + return False @property def headers(self): @@ -158,11 +163,8 @@ def readurl(url, data=None, timeout=None, retries=0, sec_between=1, r = requests.request(**req_args) if check_status: r.raise_for_status() - contents = r.content - status = r.status_code - headers = r.headers LOG.debug("Read from %s (%s, %sb) after %s attempts", url, - status, len(contents), (i + 1)) + r.status_code, len(r.content), (i + 1)) # Doesn't seem like we can make it use a different # subclass for responses, so add our own backward-compat # attrs @@ -256,8 +258,9 @@ def wait_for_url(urls, max_wait=None, timeout=None, time_taken = int(time.time() - start_time) status_msg = "Calling '%s' failed [%s/%ss]: %s" % (url, - time_taken, - max_wait, reason) + time_taken, + max_wait, + reason) status_cb(status_msg) if exception_cb: exception_cb(msg=status_msg, exception=e) diff --git a/cloudinit/user_data.py b/cloudinit/user_data.py index eaf448a7..df069ff8 100644 --- a/cloudinit/user_data.py +++ b/cloudinit/user_data.py @@ -59,6 +59,7 @@ EXAMINE_FOR_LAUNCH_INDEX = ["text/cloud-config"] class UserDataProcessor(object): def __init__(self, paths): self.paths = paths + self.ssl_details = util.fetch_ssl_details(paths) def process(self, blob): accumulating_msg = MIMEMultipart() @@ -172,10 +173,11 @@ class UserDataProcessor(object): if include_once_on and os.path.isfile(include_once_fn): content = util.load_file(include_once_fn) else: - resp = util.read_file_or_url(include_url) - if include_once_on and resp.ok: + resp = util.read_file_or_url(include_url, + ssl_details=self.ssl_details) + if include_once_on and resp.ok(): util.write_file(include_once_fn, str(resp), mode=0600) - if resp.ok: + if resp.ok(): content = str(resp) else: LOG.warn(("Fetching from %s resulted in" diff --git a/cloudinit/util.py b/cloudinit/util.py index 42b3ab01..dc3c5639 100644 --- a/cloudinit/util.py +++ b/cloudinit/util.py @@ -70,18 +70,31 @@ FN_ALLOWED = ('_-.()' + string.digits + string.ascii_letters) CONTAINER_TESTS = ['running-in-container', 'lxc-is-container'] -class FileResponse(object): - def __init__(self, path, contents): - self.code = 200 +# Made to have same accessors as UrlResponse so that the +# read_file_or_url can return this or that object and the +# 'user' of those objects will not need to know the difference. +class StringResponse(object): + def __init__(self, contents, code=200): + self.code = code self.headers = {} self.contents = contents - self.ok = True - self.url = path + self.url = None + + def ok(self, *args, **kwargs): + if self.code != 200: + return False + return True def __str__(self): return self.contents +class FileResponse(StringResponse): + def __init__(self, path, contents, code=200): + StringResponse.__init__(self, contents, code=code) + self.url = path + + class ProcessExecutionError(IOError): MESSAGE_TMPL = ('%(description)s\n' @@ -630,7 +643,7 @@ def read_optional_seed(fill, base="", ext="", timeout=5): fill['user-data'] = ud fill['meta-data'] = md return True - except OSError as e: + except IOError as e: if e.errno == errno.ENOENT: return False raise @@ -670,9 +683,12 @@ def fetch_ssl_details(paths=None): def read_file_or_url(url, timeout=5, retries=10, headers=None, data=None, sec_between=1, ssl_details=None): + url = url.lstrip() if url.startswith("/"): url = "file://%s" % url if url.lower().startswith("file://"): + if data: + LOG.warn("Unable to post data to file resource %s", url) file_path = url[len("file://"):] return FileResponse(file_path, contents=load_file(file_path)) else: @@ -724,13 +740,13 @@ def read_seeded(base="", ext="", timeout=5, retries=10, file_retries=0): md_resp = read_file_or_url(md_url, timeout, retries, file_retries) md = None - if md_resp.ok: + if md_resp.ok(): md_str = str(md_resp) md = load_yaml(md_str, default={}) ud_resp = read_file_or_url(ud_url, timeout, retries, file_retries) ud = None - if ud_resp.ok: + if ud_resp.ok(): ud_str = str(ud_resp) ud = ud_str @@ -900,7 +916,7 @@ def get_cmdline_url(names=('cloud-config-url', 'url'), return (None, None, None) resp = read_file_or_url(url) - if resp.contents.startswith(starts) and resp.ok: + if resp.contents.startswith(starts) and resp.ok(): return (key, url, str(resp)) return (key, url, None) diff --git a/tests/unittests/test__init__.py b/tests/unittests/test__init__.py index ac082076..d707afa9 100644 --- a/tests/unittests/test__init__.py +++ b/tests/unittests/test__init__.py @@ -191,8 +191,8 @@ class TestCmdlineUrl(MockerTestCase): mock_readurl = self.mocker.replace(url_helper.readurl, passthrough=False) - mock_readurl(url) - self.mocker.result(url_helper.UrlResponse(200, payload)) + mock_readurl(url, ARGS, KWARGS) + self.mocker.result(util.StringResponse(payload)) self.mocker.replay() self.assertEqual((key, url, None), @@ -207,8 +207,8 @@ class TestCmdlineUrl(MockerTestCase): mock_readurl = self.mocker.replace(url_helper.readurl, passthrough=False) - mock_readurl(url) - self.mocker.result(url_helper.UrlResponse(200, payload)) + mock_readurl(url, ARGS, KWARGS) + self.mocker.result(util.StringResponse(payload)) self.mocker.replay() self.assertEqual((key, url, payload), @@ -221,7 +221,7 @@ class TestCmdlineUrl(MockerTestCase): cmdline = "ro %s=%s bar=1" % (key, url) self.mocker.replace(url_helper.readurl, passthrough=False) - self.mocker.result(url_helper.UrlResponse(400)) + self.mocker.result(util.StringResponse("")) self.mocker.replay() self.assertEqual((None, None, None), diff --git a/tests/unittests/test_datasource/test_maas.py b/tests/unittests/test_datasource/test_maas.py index b56fea82..47f8caa4 100644 --- a/tests/unittests/test_datasource/test_maas.py +++ b/tests/unittests/test_datasource/test_maas.py @@ -3,12 +3,13 @@ import os from cloudinit.sources import DataSourceMAAS from cloudinit import url_helper +from cloudinit import util from tests.unittests.helpers import populate_dir -from mocker import MockerTestCase +import mocker -class TestMAASDataSource(MockerTestCase): +class TestMAASDataSource(mocker.MockerTestCase): def setUp(self): super(TestMAASDataSource, self).setUp() @@ -115,9 +116,11 @@ class TestMAASDataSource(MockerTestCase): for key in valid_order: url = "%s/%s/%s" % (my_seed, my_ver, key) - mock_request(url, headers=my_headers, timeout=None) + mock_request(url, headers=my_headers, timeout=mocker.ANY, + data=mocker.ANY, sec_between=mocker.ANY, + ssl_details=mocker.ANY, retries=mocker.ANY) resp = valid.get(key) - self.mocker.result(url_helper.UrlResponse(200, resp)) + self.mocker.result(util.StringResponse(resp)) self.mocker.replay() (userdata, metadata) = DataSourceMAAS.read_maas_seed_url(my_seed, -- cgit v1.2.3 From f22388a04076247490c02f605b176c3c45ecd425 Mon Sep 17 00:00:00 2001 From: harlowja Date: Sat, 23 Feb 2013 23:22:57 -0800 Subject: Fix the maas callback mechanism now that requests is used. --- cloudinit/sources/DataSourceMAAS.py | 25 ++++++++++++------------ cloudinit/url_helper.py | 38 +++++++++++++++++++++++-------------- 2 files changed, 36 insertions(+), 27 deletions(-) (limited to 'cloudinit/sources') diff --git a/cloudinit/sources/DataSourceMAAS.py b/cloudinit/sources/DataSourceMAAS.py index 0c526305..f3759b4b 100644 --- a/cloudinit/sources/DataSourceMAAS.py +++ b/cloudinit/sources/DataSourceMAAS.py @@ -25,8 +25,6 @@ import os import time import urllib2 -import requests - from cloudinit import log as logging from cloudinit import sources from cloudinit import url_helper @@ -81,7 +79,7 @@ class DataSourceMAAS(sources.DataSource): self.base_url = url (userdata, metadata) = read_maas_seed_url(self.base_url, - self.md_headers, + self._md_headers, paths=self.paths) self.userdata_raw = userdata self.metadata = metadata @@ -90,7 +88,7 @@ class DataSourceMAAS(sources.DataSource): util.logexc(LOG, "Failed fetching metadata from url %s", url) return False - def md_headers(self, url): + def _md_headers(self, url): mcfg = self.ds_cfg # If we are missing token_key, token_secret or consumer_key @@ -134,9 +132,10 @@ class DataSourceMAAS(sources.DataSource): starttime = time.time() check_url = "%s/%s/meta-data/instance-id" % (url, MD_VERSION) urls = [check_url] - url = uhelp.wait_for_url(urls=urls, max_wait=max_wait, - timeout=timeout, exception_cb=self._except_cb, - headers_cb=self.md_headers) + url = url_helper.wait_for_url(urls=urls, max_wait=max_wait, + timeout=timeout, + exception_cb=self._except_cb, + headers_cb=self._md_headers) if url: LOG.debug("Using metadata source: '%s'", url) @@ -147,23 +146,23 @@ class DataSourceMAAS(sources.DataSource): return bool(url) def _except_cb(self, msg, exception): - if not (isinstance(exception, urllib2.HTTPError) and + if not (isinstance(exception, url_helper.UrlError) and (exception.code == 403 or exception.code == 401)): return + if 'date' not in exception.headers: - LOG.warn("date field not in %d headers" % exception.code) + LOG.warn("Missing header 'date' in %s response", exception.code) return date = exception.headers['date'] - try: ret_time = time.mktime(parsedate(date)) - except: - LOG.warn("failed to convert datetime '%s'") + except Exception as e: + LOG.warn("Failed to convert datetime '%s': %s", date, e) return self.oauth_clockskew = int(ret_time - time.time()) - LOG.warn("set oauth clockskew to %d" % self.oauth_clockskew) + LOG.warn("Setting oauth clockskew to %d", self.oauth_clockskew) return diff --git a/cloudinit/url_helper.py b/cloudinit/url_helper.py index 300e70c2..6f06761a 100644 --- a/cloudinit/url_helper.py +++ b/cloudinit/url_helper.py @@ -92,13 +92,13 @@ class UrlResponse(object): class UrlError(IOError): - def __init__(self, cause): + def __init__(self, cause, code=None, headers=None): IOError.__init__(self, str(cause)) self.cause = cause - if isinstance(cause, exceptions.HTTPError) and cause.response: - self.code = cause.response.status_code - else: - self.code = None + self.code = code + self.headers = headers + if self.headers is None: + self.headers = {} def readurl(url, data=None, timeout=None, retries=0, sec_between=1, @@ -170,7 +170,11 @@ def readurl(url, data=None, timeout=None, retries=0, sec_between=1, # attrs return UrlResponse(r) except exceptions.RequestException as e: - excps.append(UrlError(e)) + if isinstance(e, (exceptions.HTTPError)) and e.response: + excps.append(UrlError(e, code=e.response.status_code, + headers=e.response.headers)) + else: + excps.append(UrlError(e)) if i + 1 < manual_tries and sec_between > 0: LOG.debug("Please wait %s seconds while we wait to try again", sec_between) @@ -235,20 +239,23 @@ def wait_for_url(urls, max_wait=None, timeout=None, timeout = int((start_time + max_wait) - now) reason = "" + e = None try: if headers_cb is not None: headers = headers_cb(url) else: headers = {} - resp = readurl(url, headers=headers, timeout=timeout, - check_status=False) - if not resp.contents: - reason = "empty response [%s]" % (resp.code) - e = ValueError(reason) - elif not resp.ok(): - reason = "bad status code [%s]" % (resp.code) - e = ValueError(reason) + response = readurl(url, headers=headers, timeout=timeout, + check_status=False) + if not response.contents: + reason = "empty response [%s]" % (response.code) + e = UrlError(ValueError(reason), + code=response.code, headers=response.headers) + elif not response.ok(): + reason = "bad status code [%s]" % (response.code) + e = UrlError(ValueError(reason), + code=response.code, headers=response.headers) else: return url except UrlError as e: @@ -263,6 +270,9 @@ def wait_for_url(urls, max_wait=None, timeout=None, reason) status_cb(status_msg) if exception_cb: + # This can be used to alter the headers that will be sent + # in the future, for example this is what the MAAS datasource + # does. exception_cb(msg=status_msg, exception=e) if timeup(max_wait, start_time): -- cgit v1.2.3 From 20a2d9961697fbd6ef0e74cd3770b6601b141bcd Mon Sep 17 00:00:00 2001 From: Joshua Harlow Date: Tue, 19 Mar 2013 15:53:16 -0700 Subject: Move back to using boto for now. --- Requires | 3 + cloudinit/ec2_utils.py | 183 +++++++----------------------- cloudinit/sources/DataSourceCloudStack.py | 7 +- cloudinit/sources/DataSourceEc2.py | 7 +- 4 files changed, 48 insertions(+), 152 deletions(-) (limited to 'cloudinit/sources') diff --git a/Requires b/Requires index 0313d569..de51a4e4 100644 --- a/Requires +++ b/Requires @@ -24,3 +24,6 @@ argparse # Requests handles ssl correctly! requests + +# Boto for ec2 +boto diff --git a/cloudinit/ec2_utils.py b/cloudinit/ec2_utils.py index 71c84206..29393ce1 100644 --- a/cloudinit/ec2_utils.py +++ b/cloudinit/ec2_utils.py @@ -16,146 +16,45 @@ # You should have received a copy of the GNU General Public License # along with this program. If not, see . -from urlparse import (urlparse, urlunparse) +import boto.utils as boto_utils + +# Versions of boto >= 2.6.0 (and possibly 2.5.2) +# try to lazily load the metadata backing, which +# doesn't work so well in cloud-init especially +# since the metadata is serialized and actions are +# performed where the metadata server may be blocked +# (thus the datasource will start failing) resulting +# in url exceptions when fields that do exist (or +# would have existed) do not exist due to the blocking +# that occurred. + + +def _unlazy_dict(mp): + if not isinstance(mp, (dict)): + return mp + # Walk over the keys/values which + # forces boto to unlazy itself and + # has no effect on dictionaries that + # already have there items. + for (_k, v) in mp.items(): + _unlazy_dict(v) + return mp + + +def get_instance_userdata(api_version, metadata_address): + # Note: boto.utils.get_instance_metadata returns '' for empty string + # so the change from non-true to '' is not specifically necessary, but + # this way cloud-init will get consistent behavior even if boto changed + # in the future to return a None on "no user-data provided". + ud = boto_utils.get_instance_userdata(api_version, None, metadata_address) + if not ud: + ud = '' + return ud + + +def get_instance_metadata(api_version, metadata_address): + metadata = boto_utils.get_instance_metadata(api_version, metadata_address) + if not isinstance(metadata, (dict)): + metadata = {} + return _unlazy_dict(metadata) -import json -import urllib - -from cloudinit import log as logging -from cloudinit import util - -LOG = logging.getLogger(__name__) - - -def combine_url(base, add_on): - base_parsed = list(urlparse(base)) - path = base_parsed[2] - if path and not path.endswith("/"): - path += "/" - path += urllib.quote(str(add_on), safe="/:") - base_parsed[2] = path - return urlunparse(base_parsed) - - -# See: http://bit.ly/TyoUQs -# -# Since boto metadata reader uses the old urllib which does not -# support ssl, we need to ahead and create our own reader which -# works the same as the boto one (for now). -class MetadataMaterializer(object): - def __init__(self, blob, base_url, ssl_details): - self._blob = blob - self._md = None - self._base_url = base_url - self._ssl_details = ssl_details - - def _parse(self, blob): - leaves = {} - children = [] - if not blob: - return (leaves, children) - - def has_children(item): - if item.endswith("/"): - return True - else: - return False - - def get_name(item): - if item.endswith("/"): - return item.rstrip("/") - return item - - for field in blob.splitlines(): - field = field.strip() - field_name = get_name(field) - if not field or not field_name: - continue - if has_children(field): - if field_name not in children: - children.append(field_name) - else: - contents = field.split("=", 1) - resource = field_name - if len(contents) > 1: - # What a PITA... - (ident, sub_contents) = contents - checked_ident = util.safe_int(ident) - if checked_ident is not None: - resource = "%s/openssh-key" % (checked_ident) - field_name = sub_contents - leaves[field_name] = resource - return (leaves, children) - - def materialize(self): - if self._md is not None: - return self._md - self._md = self._materialize(self._blob, self._base_url) - return self._md - - def _fetch_url(self, url): - response = util.read_file_or_url(url, ssl_details=self._ssl_details) - return str(response) - - def _decode_leaf_blob(self, blob): - if not blob: - return blob - stripped_blob = blob.strip() - if stripped_blob.startswith("{") and stripped_blob.endswith("}"): - # Assume and try with json - try: - return json.loads(blob) - except (ValueError, TypeError): - pass - if blob.find("\n") != -1: - return blob.splitlines() - return blob - - def _materialize(self, blob, base_url): - (leaves, children) = self._parse(blob) - child_contents = {} - for c in children: - child_url = combine_url(base_url, c) - if not child_url.endswith("/"): - child_url += "/" - child_blob = self._fetch_url(child_url) - child_contents[c] = self._materialize(child_blob, child_url) - leaf_contents = {} - for (field, resource) in leaves.items(): - leaf_url = combine_url(base_url, resource) - leaf_blob = self._fetch_url(leaf_url) - leaf_contents[field] = self._decode_leaf_blob(leaf_blob) - joined = {} - joined.update(child_contents) - for field in leaf_contents.keys(): - if field in joined: - LOG.warn("Duplicate key found in results from %s", base_url) - else: - joined[field] = leaf_contents[field] - return joined - - -def get_instance_userdata(version='latest', url='http://169.254.169.254', - ssl_details=None): - ud_url = combine_url(url, version) - ud_url = combine_url(ud_url, 'user-data') - try: - response = util.read_file_or_url(ud_url, ssl_details=ssl_details) - return str(response) - except Exception: - util.logexc(LOG, "Failed fetching userdata from url %s", ud_url) - return None - - -def get_instance_metadata(version='latest', url='http://169.254.169.254', - ssl_details=None, timeout=5, num_retries=5): - md_url = combine_url(metadata_address, version) - md_url = combine_url(md_url, 'meta-data') - try: - response = util.read_file_or_url(md_url, ssl_details=ssl_details, - timeout=timeout, retries=num_retries) - materializer = MetadataMaterializer(str(response), md_url, ssl_details) - return materializer.materialize() - except Exception: - util.logexc(LOG, "Failed fetching metadata from url %s", md_url) - return None diff --git a/cloudinit/sources/DataSourceCloudStack.py b/cloudinit/sources/DataSourceCloudStack.py index 7fd38982..b4ca6d93 100644 --- a/cloudinit/sources/DataSourceCloudStack.py +++ b/cloudinit/sources/DataSourceCloudStack.py @@ -102,13 +102,10 @@ class DataSourceCloudStack(sources.DataSource): return False start_time = time.time() md_addr = self.metadata_address - ssl_details = util.fetch_ssl_details(self.paths) self.userdata_raw = ec2_utils.get_instance_userdata(self.api_ver, - md_addr, - ssl_details) + md_addr) self.metadata = ec2_utils.get_instance_metadata(self.api_ver, - md_addr, - ssl_details) + md_addr) LOG.debug("Crawl of metadata service took %s seconds", int(time.time() - start_time)) return True diff --git a/cloudinit/sources/DataSourceEc2.py b/cloudinit/sources/DataSourceEc2.py index 0d62cf01..bd35c8b0 100644 --- a/cloudinit/sources/DataSourceEc2.py +++ b/cloudinit/sources/DataSourceEc2.py @@ -62,13 +62,10 @@ class DataSourceEc2(sources.DataSource): return False start_time = time.time() md_addr = self.metadata_address - ssl_details = util.fetch_ssl_details(self.paths) self.userdata_raw = ec2_utils.get_instance_userdata(self.api_ver, - md_addr, - ssl_details) + md_addr) self.metadata = ec2_utils.get_instance_metadata(self.api_ver, - md_addr, - ssl_details) + md_addr) LOG.debug("Crawl of metadata service took %s seconds", int(time.time() - start_time)) return True -- cgit v1.2.3 From c49b92e2e1f32cbb32c856ba246ef97026318dbf Mon Sep 17 00:00:00 2001 From: Scott Moser Date: Wed, 20 Mar 2013 08:30:43 -0400 Subject: remove some churn cloudinit/sources/DataSourceCloudStack.py and cloudinit/sources/DataSourceEc2.py are reverted entirely back to trunk versions now, rather than the non-behavior change that was left in place. Also, remove inadvertantly added trailing newline from cloudinit/ec2_utils.py Overall, this just makes the diff when merged to trunk carry more focused changes. --- cloudinit/ec2_utils.py | 1 - cloudinit/sources/DataSourceCloudStack.py | 11 +++++------ cloudinit/sources/DataSourceEc2.py | 13 ++++++------- 3 files changed, 11 insertions(+), 14 deletions(-) (limited to 'cloudinit/sources') diff --git a/cloudinit/ec2_utils.py b/cloudinit/ec2_utils.py index 175f96aa..5fa8c5ad 100644 --- a/cloudinit/ec2_utils.py +++ b/cloudinit/ec2_utils.py @@ -61,4 +61,3 @@ def get_instance_metadata(api_version, metadata_address): if not isinstance(metadata, (dict)): metadata = {} return _unlazy_dict(metadata) - diff --git a/cloudinit/sources/DataSourceCloudStack.py b/cloudinit/sources/DataSourceCloudStack.py index b4ca6d93..81c8cda9 100644 --- a/cloudinit/sources/DataSourceCloudStack.py +++ b/cloudinit/sources/DataSourceCloudStack.py @@ -25,7 +25,7 @@ import os import time -from cloudinit import ec2_utils +from cloudinit import ec2_utils as ec2 from cloudinit import log as logging from cloudinit import sources from cloudinit import url_helper as uhelp @@ -101,11 +101,10 @@ class DataSourceCloudStack(sources.DataSource): if not self.wait_for_metadata_service(): return False start_time = time.time() - md_addr = self.metadata_address - self.userdata_raw = ec2_utils.get_instance_userdata(self.api_ver, - md_addr) - self.metadata = ec2_utils.get_instance_metadata(self.api_ver, - md_addr) + self.userdata_raw = ec2.get_instance_userdata(self.api_ver, + self.metadata_address) + self.metadata = ec2.get_instance_metadata(self.api_ver, + self.metadata_address) LOG.debug("Crawl of metadata service took %s seconds", int(time.time() - start_time)) return True diff --git a/cloudinit/sources/DataSourceEc2.py b/cloudinit/sources/DataSourceEc2.py index bd35c8b0..f010e640 100644 --- a/cloudinit/sources/DataSourceEc2.py +++ b/cloudinit/sources/DataSourceEc2.py @@ -23,7 +23,7 @@ import os import time -from cloudinit import ec2_utils +from cloudinit import ec2_utils as ec2 from cloudinit import log as logging from cloudinit import sources from cloudinit import url_helper as uhelp @@ -61,11 +61,10 @@ class DataSourceEc2(sources.DataSource): if not self.wait_for_metadata_service(): return False start_time = time.time() - md_addr = self.metadata_address - self.userdata_raw = ec2_utils.get_instance_userdata(self.api_ver, - md_addr) - self.metadata = ec2_utils.get_instance_metadata(self.api_ver, - md_addr) + self.userdata_raw = ec2.get_instance_userdata(self.api_ver, + self.metadata_address) + self.metadata = ec2.get_instance_metadata(self.api_ver, + self.metadata_address) LOG.debug("Crawl of metadata service took %s seconds", int(time.time() - start_time)) return True @@ -134,7 +133,7 @@ class DataSourceEc2(sources.DataSource): start_time = time.time() url = uhelp.wait_for_url(urls=urls, max_wait=max_wait, - timeout=timeout, status_cb=LOG.warn) + timeout=timeout, status_cb=LOG.warn) if url: LOG.debug("Using metadata source: '%s'", url2base[url]) -- cgit v1.2.3