From c104d6dfa464a8906c16b4f09b4b76ab5bf2e4e1 Mon Sep 17 00:00:00 2001 From: Joshua Harlow Date: Sat, 1 Feb 2014 22:48:55 -0800 Subject: Add a openstack specific datasource Openstack has a unique derivative datasource that is gaining usage. Previously the config drive datasource provided part of this functionality as well as the ec2 datasource, but since new functionality is being added to openstack is seems benefical to combine the used parts into one datasource just made for handling openstack deployments. This patch factors out the common logic shared between the config drive and the openstack metadata datasource and places that in a shared helper file and then creates a new openstack datasource that readers from the openstack metadata service and refactors the config drive datasource to use this common logic. --- cloudinit/url_helper.py | 64 +++++++++++++++++++++++++++++++++++++++---------- 1 file changed, 52 insertions(+), 12 deletions(-) (limited to 'cloudinit/url_helper.py') diff --git a/cloudinit/url_helper.py b/cloudinit/url_helper.py index 19a30409..5c33d1e4 100644 --- a/cloudinit/url_helper.py +++ b/cloudinit/url_helper.py @@ -21,6 +21,7 @@ # along with this program. If not, see . import time +import urllib import requests from requests import exceptions @@ -58,6 +59,23 @@ def _cleanurl(url): return urlunparse(parsed_url) +def combine_url(base, *add_ons): + + def combine_single(url, add_on): + url_parsed = list(urlparse(url)) + path = url_parsed[2] + if path and not path.endswith("/"): + path += "/" + path += urllib.quote(str(add_on), safe="/:") + url_parsed[2] = path + return urlunparse(url_parsed) + + url = base + for add_on in add_ons: + url = combine_single(url, add_on) + return url + + class UrlResponse(object): def __init__(self, response): self._response = response @@ -101,30 +119,52 @@ class UrlError(IOError): self.headers = {} -def readurl(url, data=None, timeout=None, retries=0, sec_between=1, - headers=None, headers_cb=None, ssl_details=None, - check_status=True, allow_redirects=True): - url = _cleanurl(url) - req_args = { - 'url': url, - } +def _get_ssl_args(url, ssl_details): + ssl_args = {} scheme = urlparse(url).scheme # pylint: disable=E1101 if scheme == 'https' and ssl_details: if not SSL_ENABLED: LOG.warn("SSL is not enabled, cert. verification can not occur!") else: if 'ca_certs' in ssl_details and ssl_details['ca_certs']: - req_args['verify'] = ssl_details['ca_certs'] + ssl_args['verify'] = ssl_details['ca_certs'] else: - req_args['verify'] = True + ssl_args['verify'] = True if 'cert_file' in ssl_details and 'key_file' in ssl_details: - req_args['cert'] = [ssl_details['cert_file'], + ssl_args['cert'] = [ssl_details['cert_file'], ssl_details['key_file']] elif 'cert_file' in ssl_details: - req_args['cert'] = str(ssl_details['cert_file']) + ssl_args['cert'] = str(ssl_details['cert_file']) + return ssl_args + +def existsurl(url, ssl_details=None, timeout=None): + r = _readurl(url, ssl_details=ssl_details, timeout=timeout, + method='HEAD', check_status=False) + return r.ok() + + +def readurl(url, data=None, timeout=None, retries=0, sec_between=1, + headers=None, headers_cb=None, ssl_details=None, + check_status=True, allow_redirects=True): + return _readurl(url, data=data, timeout=timeout, retries=retries, + sec_between=sec_between, headers=headers, + headers_cb=headers_cb, ssl_details=ssl_details, + check_status=check_status, + allow_redirects=allow_redirects) + + +def _readurl(url, data=None, timeout=None, retries=0, sec_between=1, + headers=None, headers_cb=None, ssl_details=None, + check_status=True, allow_redirects=True, method='GET'): + url = _cleanurl(url) + req_args = { + 'url': url, + } + req_args.update(_get_ssl_args(url, ssl_details)) + scheme = urlparse(url).scheme # pylint: disable=E1101 req_args['allow_redirects'] = allow_redirects - req_args['method'] = 'GET' + req_args['method'] = method if timeout is not None: req_args['timeout'] = max(float(timeout), 0) if data: -- cgit v1.2.3 From 098a74e6207f5d91f515fac63e970375d52795c0 Mon Sep 17 00:00:00 2001 From: Joshua Harlow Date: Sat, 8 Feb 2014 12:20:33 -0800 Subject: Remove HEAD usage and other small adjustments --- cloudinit/ec2_utils.py | 4 +-- cloudinit/sources/DataSourceOpenStack.py | 1 + cloudinit/sources/helpers/openstack.py | 41 ++++++++++++++--------- cloudinit/url_helper.py | 23 ++----------- tests/unittests/test_datasource/test_openstack.py | 11 ------ 5 files changed, 30 insertions(+), 50 deletions(-) (limited to 'cloudinit/url_helper.py') diff --git a/cloudinit/ec2_utils.py b/cloudinit/ec2_utils.py index 91cba20f..a7c9c9ab 100644 --- a/cloudinit/ec2_utils.py +++ b/cloudinit/ec2_utils.py @@ -16,10 +16,8 @@ # You should have received a copy of the GNU General Public License # along with this program. If not, see . -import httplib -from urlparse import (urlparse, urlunparse) - import functools +import httplib import json from cloudinit import log as logging diff --git a/cloudinit/sources/DataSourceOpenStack.py b/cloudinit/sources/DataSourceOpenStack.py index 621572de..2c50ed84 100644 --- a/cloudinit/sources/DataSourceOpenStack.py +++ b/cloudinit/sources/DataSourceOpenStack.py @@ -44,6 +44,7 @@ class DataSourceOpenStack(openstack.SourceMixin, sources.DataSource): self.ssl_details = util.fetch_ssl_details(self.paths) self.version = None self.files = {} + self.ec2_metadata = None def __str__(self): root = sources.DataSource.__str__(self) diff --git a/cloudinit/sources/helpers/openstack.py b/cloudinit/sources/helpers/openstack.py index 9dbef677..09fb4ad8 100644 --- a/cloudinit/sources/helpers/openstack.py +++ b/cloudinit/sources/helpers/openstack.py @@ -21,7 +21,6 @@ import abc import base64 import copy -import functools import os from cloudinit import ec2_utils @@ -395,26 +394,38 @@ class ConfigDriveReader(BaseReader): class MetadataReader(BaseReader): def __init__(self, base_url, ssl_details=None, timeout=5, retries=5): super(MetadataReader, self).__init__(base_url) - self._url_reader = functools.partial(url_helper.readurl, - retries=retries, - ssl_details=ssl_details, - timeout=timeout) - self._url_checker = functools.partial(url_helper.existsurl, - ssl_details=ssl_details, - timeout=timeout) - self._ec2_reader = functools.partial(ec2_utils.get_instance_metadata, - ssl_details=ssl_details, - timeout=timeout, - retries=retries) + self.ssl_details = ssl_details + self.timeout = float(timeout) + self.retries = int(retries) def _path_read(self, path): - return str(self._url_reader(path)) + response = url_helper.readurl(path, + retries=self.retries, + ssl_details=self.ssl_details, + timeout=self.timeout) + return response.contents def _path_exists(self, path): - return self._url_checker(path) + + def should_retry_cb(request, cause): + if cause.code >= 400: + return False + return True + + try: + response = url_helper.readurl(path, + retries=self.retries, + ssl_details=self.ssl_details, + timeout=self.timeout, + exception_cb=should_retry_cb) + return response.ok() + except IOError: + return False def _path_join(self, base, *add_ons): return url_helper.combine_url(base, *add_ons) def _read_ec2_metadata(self): - return self._ec2_reader() + return ec2_utils.get_instance_metadata(ssl_details=self.ssl_details, + timeout=self.timeout, + retries=self.retries) diff --git a/cloudinit/url_helper.py b/cloudinit/url_helper.py index 76a8e29b..a477b185 100644 --- a/cloudinit/url_helper.py +++ b/cloudinit/url_helper.py @@ -166,35 +166,16 @@ def _get_ssl_args(url, ssl_details): return ssl_args -def existsurl(url, ssl_details=None, timeout=None): - r = _readurl(url, ssl_details=ssl_details, timeout=timeout, - method='HEAD', check_status=False) - return r.ok() - - def readurl(url, data=None, timeout=None, retries=0, sec_between=1, - headers=None, headers_cb=None, ssl_details=None, - check_status=True, allow_redirects=True, exception_cb=None): - return _readurl(url, data=data, timeout=timeout, retries=retries, - sec_between=sec_between, headers=headers, - headers_cb=headers_cb, ssl_details=ssl_details, - check_status=check_status, - allow_redirects=allow_redirects, - exception_cb=exception_cb) - - -def _readurl(url, data=None, timeout=None, retries=0, sec_between=1, headers=None, headers_cb=None, ssl_details=None, - check_status=True, allow_redirects=True, exception_cb=None, - method='GET'): + check_status=True, allow_redirects=True, exception_cb=None): url = _cleanurl(url) req_args = { 'url': url, } req_args.update(_get_ssl_args(url, ssl_details)) - scheme = urlparse(url).scheme # pylint: disable=E1101 req_args['allow_redirects'] = allow_redirects - req_args['method'] = method + req_args['method'] = 'GET' if timeout is not None: req_args['timeout'] = max(float(timeout), 0) if data: diff --git a/tests/unittests/test_datasource/test_openstack.py b/tests/unittests/test_datasource/test_openstack.py index 3fcf8bc9..3a64430a 100644 --- a/tests/unittests/test_datasource/test_openstack.py +++ b/tests/unittests/test_datasource/test_openstack.py @@ -117,20 +117,9 @@ def _register_uris(version, ec2_files, ec2_meta, os_files): return (200, headers, os_files.get(path)) return match_ec2_url(uri, headers) - def head_request_callback(method, uri, headers): - uri = urlparse(uri) - path = uri.path.lstrip("/") - for key in os_files.keys(): - if key.startswith(path): - return (200, headers, '') - return (404, headers, '') - hp.register_uri(hp.GET, re.compile(r'http://169.254.169.254/.*'), body=get_request_callback) - hp.register_uri(hp.HEAD, re.compile(r'http://169.254.169.254/.*'), - body=head_request_callback) - class TestOpenStackDataSource(test_helpers.TestCase): VERSION = 'latest' -- cgit v1.2.3 From 1edc8d3697d05f66195b9a425771f60cf6f9c27e Mon Sep 17 00:00:00 2001 From: Joshua Harlow Date: Sat, 8 Feb 2014 13:28:07 -0800 Subject: Update requests ssl not supported message --- cloudinit/url_helper.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) (limited to 'cloudinit/url_helper.py') diff --git a/cloudinit/url_helper.py b/cloudinit/url_helper.py index a477b185..43e879d2 100644 --- a/cloudinit/url_helper.py +++ b/cloudinit/url_helper.py @@ -39,6 +39,7 @@ NOT_FOUND = httplib.NOT_FOUND # Check if requests has ssl support (added in requests >= 0.8.8) SSL_ENABLED = False CONFIG_ENABLED = False # This was added in 0.7 (but taken out in >=1.0) +_REQ_VER = None try: from distutils.version import LooseVersion import pkg_resources @@ -152,7 +153,8 @@ def _get_ssl_args(url, ssl_details): scheme = urlparse(url).scheme # pylint: disable=E1101 if scheme == 'https' and ssl_details: if not SSL_ENABLED: - LOG.warn("SSL is not enabled, cert. verification can not occur!") + LOG.warn("SSL is not supported in requests v%s, " + "cert. verification can not occur!", _REQ_VER) else: if 'ca_certs' in ssl_details and ssl_details['ca_certs']: ssl_args['verify'] = ssl_details['ca_certs'] -- cgit v1.2.3 From 1597aca9b0606d02c045549afce4395370502231 Mon Sep 17 00:00:00 2001 From: Joshua Harlow Date: Sat, 8 Feb 2014 13:39:27 -0800 Subject: Spacing and comment cleanup --- cloudinit/url_helper.py | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) (limited to 'cloudinit/url_helper.py') diff --git a/cloudinit/url_helper.py b/cloudinit/url_helper.py index 43e879d2..c116a484 100644 --- a/cloudinit/url_helper.py +++ b/cloudinit/url_helper.py @@ -169,8 +169,8 @@ def _get_ssl_args(url, ssl_details): def readurl(url, data=None, timeout=None, retries=0, sec_between=1, - headers=None, headers_cb=None, ssl_details=None, - check_status=True, allow_redirects=True, exception_cb=None): + headers=None, headers_cb=None, ssl_details=None, + check_status=True, allow_redirects=True, exception_cb=None): url = _cleanurl(url) req_args = { 'url': url, @@ -206,12 +206,11 @@ def readurl(url, data=None, timeout=None, retries=0, sec_between=1, def _cb(url): return headers headers_cb = _cb - if data: - # Do this after the log (it might be large) req_args['data'] = data if sec_between is None: sec_between = -1 + excps = [] # Handle retrying ourselves since the built-in support # doesn't handle sleeping between tries... -- cgit v1.2.3 From 87d0fa867f27f101e93006ba8dc8a395098e8df1 Mon Sep 17 00:00:00 2001 From: Scott Moser Date: Thu, 13 Feb 2014 12:13:42 -0500 Subject: wait less for the metadata service (by default) Waiting around for a metadata service in a given datasource means that if its not there all the subsequent datasources have to wait, and boot is slowed down. As it is right now, EC2 is the only one that has the right to wait. In the past, we had to wait around for the EC2 metadata service. I really do not want to extend that courtesy to other cloud platforms. A network based metadata service should be up as soon as networking is up. --- cloudinit/sources/DataSourceOpenStack.py | 28 +++++++++++++--------------- cloudinit/url_helper.py | 1 + 2 files changed, 14 insertions(+), 15 deletions(-) (limited to 'cloudinit/url_helper.py') diff --git a/cloudinit/sources/DataSourceOpenStack.py b/cloudinit/sources/DataSourceOpenStack.py index 69807798..7fafa3f7 100644 --- a/cloudinit/sources/DataSourceOpenStack.py +++ b/cloudinit/sources/DataSourceOpenStack.py @@ -45,6 +45,8 @@ class DataSourceOpenStack(openstack.SourceMixin, sources.DataSource): self.version = None self.files = {} self.ec2_metadata = None + if not self.ds_cfg: + self.ds_cfg = {} def __str__(self): root = sources.DataSource.__str__(self) @@ -54,27 +56,25 @@ class DataSourceOpenStack(openstack.SourceMixin, sources.DataSource): def _get_url_settings(self): # TODO(harlowja): this is shared with ec2 datasource, we should just # move it to a shared location instead... - ds_cfg = self.ds_cfg - if not ds_cfg: - ds_cfg = {} - max_wait = 120 + # Note: the defaults here are different though. + + # max_wait < 0 indicates do not wait + max_wait = -1 + timeout = 10 + try: - max_wait = int(ds_cfg.get("max_wait", max_wait)) + max_wait = int(self.ds_cfg.get("max_wait", max_wait)) except Exception: util.logexc(LOG, "Failed to get max wait. using %s", max_wait) - timeout = 50 try: - timeout = max(0, int(ds_cfg.get("timeout", timeout))) + timeout = max(0, int(self.ds_cfg.get("timeout", timeout))) except Exception: util.logexc(LOG, "Failed to get timeout, using %s", timeout) return (max_wait, timeout) def wait_for_metadata_service(self): - ds_cfg = self.ds_cfg - if not ds_cfg: - ds_cfg = {} - urls = ds_cfg.get("metadata_urls", [DEF_MD_URL]) + urls = self.ds_cfg.get("metadata_urls", [DEF_MD_URL]) filtered = [x for x in urls if util.is_resolvable_url(x)] if set(filtered) != set(urls): LOG.debug("Removed the following from metadata urls: %s", @@ -95,8 +95,6 @@ class DataSourceOpenStack(openstack.SourceMixin, sources.DataSource): url2base[md_url] = url (max_wait, timeout) = self._get_url_settings() - if max_wait <= 0: - return False start_time = time.time() avail_url = url_helper.wait_for_url(urls=md_urls, max_wait=max_wait, timeout=timeout, @@ -104,8 +102,8 @@ class DataSourceOpenStack(openstack.SourceMixin, sources.DataSource): if avail_url: LOG.debug("Using metadata source: '%s'", url2base[avail_url]) else: - LOG.critical("Giving up on md from %s after %s seconds", - md_urls, int(time.time() - start_time)) + LOG.debug("Giving up on OpenStack md from %s after %s seconds", + md_urls, int(time.time() - start_time)) self.metadata_address = url2base.get(avail_url) return bool(avail_url) diff --git a/cloudinit/url_helper.py b/cloudinit/url_helper.py index c116a484..4a83169a 100644 --- a/cloudinit/url_helper.py +++ b/cloudinit/url_helper.py @@ -266,6 +266,7 @@ def wait_for_url(urls, max_wait=None, timeout=None, max_wait: roughly the maximum time to wait before giving up The max time is *actually* len(urls)*timeout as each url will be tried once and given the timeout provided. + a number <= 0 will always result in only one try timeout: the timeout provided to urlopen status_cb: call method with string message when a url is not available headers_cb: call method with single argument of url to get headers -- cgit v1.2.3