From 7c9bbbc9b49425e3ba8e0517908477c58ea51d4b Mon Sep 17 00:00:00 2001 From: Joshua Harlow Date: Fri, 19 Oct 2012 14:06:21 -0700 Subject: Remove the need for boto just for fetching the userdata and metadata. Add in this crawling functionality to the ec2_utils module that will fully crawl (not lazily) the ec2 metadata and parse it in the same manner as boto. 1. Make the ec2 datasource + cloudstack now call into these. 2. Fix phone_home due to urllib3 change (TBD) --- cloudinit/sources/DataSourceCloudStack.py | 9 +++------ cloudinit/sources/DataSourceEc2.py | 9 +++------ 2 files changed, 6 insertions(+), 12 deletions(-) (limited to 'cloudinit/sources') diff --git a/cloudinit/sources/DataSourceCloudStack.py b/cloudinit/sources/DataSourceCloudStack.py index f7ffa7cb..2654df53 100644 --- a/cloudinit/sources/DataSourceCloudStack.py +++ b/cloudinit/sources/DataSourceCloudStack.py @@ -26,8 +26,7 @@ from struct import pack import os import time -import boto.utils as boto_utils - +from cloudinit import ec2_utils from cloudinit import log as logging from cloudinit import sources from cloudinit import url_helper as uhelp @@ -116,10 +115,8 @@ class DataSourceCloudStack(sources.DataSource): if not self.wait_for_metadata_service(): return False start_time = time.time() - self.userdata_raw = boto_utils.get_instance_userdata(self.api_ver, - None, self.metadata_address) - self.metadata = boto_utils.get_instance_metadata(self.api_ver, - self.metadata_address) + self.userdata_raw = ec2_utils.get_instance_userdata(self.metadata_address, self.api_ver) + self.metadata = ec2_utils.get_instance_metadata(self.metadata_address, self.api_ver) LOG.debug("Crawl of metadata service took %s seconds", int(time.time() - start_time)) return True diff --git a/cloudinit/sources/DataSourceEc2.py b/cloudinit/sources/DataSourceEc2.py index 3686fa10..204963e7 100644 --- a/cloudinit/sources/DataSourceEc2.py +++ b/cloudinit/sources/DataSourceEc2.py @@ -23,8 +23,7 @@ import os import time -import boto.utils as boto_utils - +from cloudinit import ec2_utils from cloudinit import log as logging from cloudinit import sources from cloudinit import url_helper as uhelp @@ -65,10 +64,8 @@ class DataSourceEc2(sources.DataSource): if not self.wait_for_metadata_service(): return False start_time = time.time() - self.userdata_raw = boto_utils.get_instance_userdata(self.api_ver, - None, self.metadata_address) - self.metadata = boto_utils.get_instance_metadata(self.api_ver, - self.metadata_address) + self.userdata_raw = ec2_utils.get_instance_userdata(self.metadata_address, self.api_ver) + self.metadata = ec2_utils.get_instance_metadata(self.metadata_address, self.api_ver) LOG.debug("Crawl of metadata service took %s seconds", int(time.time() - start_time)) return True -- cgit v1.2.3 From 6d40d5b2e3da9577d4a2686444d47125e62817fe Mon Sep 17 00:00:00 2001 From: harlowja Date: Tue, 19 Feb 2013 22:51:49 -0800 Subject: Continue working on integrating requests. --- cloudinit/config/cc_phone_home.py | 4 +-- cloudinit/ec2_utils.py | 24 +++++-------- cloudinit/sources/DataSourceEc2.py | 2 +- cloudinit/sources/DataSourceMAAS.py | 16 +++++---- cloudinit/url_helper.py | 50 ++++++++++++++++++++++---- cloudinit/user_data.py | 7 ++-- cloudinit/util.py | 71 ++++++++++++++++++++++++++++++++----- 7 files changed, 129 insertions(+), 45 deletions(-) (limited to 'cloudinit/sources') diff --git a/cloudinit/config/cc_phone_home.py b/cloudinit/config/cc_phone_home.py index ae1349eb..90834080 100644 --- a/cloudinit/config/cc_phone_home.py +++ b/cloudinit/config/cc_phone_home.py @@ -19,7 +19,6 @@ # along with this program. If not, see . from cloudinit import templater -from cloudinit import url_helper as uhelp from cloudinit import util from cloudinit.settings import PER_INSTANCE @@ -112,7 +111,8 @@ def handle(name, cfg, cloud, log, args): } url = templater.render_string(url, url_params) try: - uhelp.readurl(url, data=real_submit_keys, retries=tries, sec_between=3) + util.read_file_or_url(url, data=real_submit_keys, + retries=tries, sec_between=3) except: util.logexc(log, ("Failed to post phone home data to" " %s in %s tries"), url, tries) diff --git a/cloudinit/ec2_utils.py b/cloudinit/ec2_utils.py index b9d7a2f7..c422eea9 100644 --- a/cloudinit/ec2_utils.py +++ b/cloudinit/ec2_utils.py @@ -22,7 +22,6 @@ import json import urllib from cloudinit import log as logging -from cloudinit import url_helper as uh from cloudinit import util LOG = logging.getLogger(__name__) @@ -40,11 +39,10 @@ def combine_url(base, add_on): # See: http://bit.ly/TyoUQs class MetadataMaterializer(object): - def __init__(self, blob, base_url, **fetch_settings): + def __init__(self, blob, base_url): self._blob = blob self._md = None self._base_url = base_url - self._fetch_settings = fetch_settings def _parse(self, blob): leaves = {} @@ -90,8 +88,8 @@ class MetadataMaterializer(object): self._md = self._materialize(self._blob, self._base_url) return self._md - def _fetch_url(self, url, **opts): - response = uh.readurl(url, **opts) + def _fetch_url(self, url): + response = util.read_file_or_url(url) return str(response) def _decode_leaf_blob(self, blob): @@ -115,12 +113,12 @@ class MetadataMaterializer(object): child_url = combine_url(base_url, c) if not child_url.endswith("/"): child_url += "/" - child_blob = self._fetch_url(child_url, **self._fetch_settings) + child_blob = self._fetch_url(child_url) child_contents[c] = self._materialize(child_blob, child_url) leaf_contents = {} for (field, resource) in leaves.items(): leaf_url = combine_url(base_url, resource) - leaf_blob = self._fetch_url(leaf_url, **self._fetch_settings) + leaf_blob = self._fetch_url(leaf_url) leaf_contents[field] = self._decode_leaf_blob(leaf_blob) joined = {} joined.update(child_contents) @@ -136,23 +134,19 @@ def get_instance_userdata(url, version='latest', ssl_details=None): ud_url = combine_url(url, version) ud_url = combine_url(ud_url, 'user-data') try: - response = uh.readurl(ud_url, timeout=5, - retries=10, ssl_details=ssl_details) + response = util.read_file_or_url(ud_url) return str(response) except Exception: util.logexc(LOG, "Failed fetching userdata from url %s", ud_url) return None -def get_instance_metadata(url, version='latest', ssl_details=None): +def get_instance_metadata(url, version='latest'): md_url = combine_url(url, version) md_url = combine_url(md_url, 'meta-data') try: - response = uh.readurl(md_url, timeout=5, - retries=10, ssl_details=ssl_details) - materializer = MetadataMaterializer(str(response), md_url, - timeout=5, retries=10, - ssl_details=ssl_details) + response = util.read_file_or_url(md_url) + materializer = MetadataMaterializer(str(response), md_url) return materializer.materialize() except Exception: util.logexc(LOG, "Failed fetching metadata from url %s", md_url) diff --git a/cloudinit/sources/DataSourceEc2.py b/cloudinit/sources/DataSourceEc2.py index 204963e7..47f677d4 100644 --- a/cloudinit/sources/DataSourceEc2.py +++ b/cloudinit/sources/DataSourceEc2.py @@ -137,7 +137,7 @@ class DataSourceEc2(sources.DataSource): start_time = time.time() url = uhelp.wait_for_url(urls=urls, max_wait=max_wait, - timeout=timeout, status_cb=LOG.warn) + timeout=timeout, status_cb=LOG.warn) if url: LOG.debug("Using metadata source: '%s'", url2base[url]) diff --git a/cloudinit/sources/DataSourceMAAS.py b/cloudinit/sources/DataSourceMAAS.py index e187aec9..2de31305 100644 --- a/cloudinit/sources/DataSourceMAAS.py +++ b/cloudinit/sources/DataSourceMAAS.py @@ -25,9 +25,11 @@ import os import time import urllib2 +import requests + from cloudinit import log as logging from cloudinit import sources -from cloudinit import url_helper as uhelp +from cloudinit import url_helper from cloudinit import util LOG = logging.getLogger(__name__) @@ -191,8 +193,8 @@ def read_maas_seed_url(seed_url, header_cb=None, timeout=None, version=MD_VERSION): """ Read the maas datasource at seed_url. - header_cb is a method that should return a headers dictionary that will - be given to urllib2.Request() + - header_cb is a method that should return a headers dictionary for + a given url Expected format of seed_url is are the following files: * //meta-data/instance-id @@ -220,13 +222,13 @@ def read_maas_seed_url(seed_url, header_cb=None, timeout=None, else: headers = {} try: - resp = uhelp.readurl(url, headers=headers, timeout=timeout) - if resp.ok(): + resp = util.read_file_or_url(url, headers=headers, timeout=timeout) + if resp.ok: md[name] = str(resp) else: LOG.warn(("Fetching from %s resulted in" - " an invalid http code %s"), url, resp.code) - except urllib2.HTTPError as e: + " an invalid http code %s"), url, resp.status_code) + except url_helper.UrlError as e: if e.code != 404: raise return check_seed_contents(md, seed_url) diff --git a/cloudinit/url_helper.py b/cloudinit/url_helper.py index e8330e24..0839e63b 100644 --- a/cloudinit/url_helper.py +++ b/cloudinit/url_helper.py @@ -58,6 +58,44 @@ def _cleanurl(url): return urlunparse(parsed_url) +class UrlResponse(object): + def __init__(self, response): + self._response = response + + @property + def contents(self): + return self._response.content + + @property + def url(self): + return self._response.url + + @property + def ok(self): + return self._response.ok + + @property + def headers(self): + return self._response.headers + + @property + def code(self): + return self._response.status_code + + def __str__(self): + return self.contents + + +class UrlError(IOError): + def __init__(self, cause): + IOError.__init__(self, str(cause)) + self.cause = cause + if isinstance(cause, exceptions.HTTPError) and cause.response: + self.code = cause.response.status_code + else: + self.code = None + + def readurl(url, data=None, timeout=None, retries=0, sec_between=1, headers=None, ssl_details=None, check_status=True, allow_redirects=False): @@ -76,6 +114,8 @@ def readurl(url, data=None, timeout=None, retries=0, sec_between=1, if 'cert_file' in ssl_details and 'key_file' in ssl_details: req_args['cert'] = [ssl_details['cert_file'], ssl_details['key_file']] + elif 'cert_file' in ssl_details: + req_args['cert'] = str(ssl_details['cert_file']) req_args['allow_redirects'] = allow_redirects req_args['method'] = 'GET' @@ -126,13 +166,9 @@ def readurl(url, data=None, timeout=None, retries=0, sec_between=1, # Doesn't seem like we can make it use a different # subclass for responses, so add our own backward-compat # attrs - if not hasattr(r, 'code'): - setattr(r, 'code', status) - if not hasattr(r, 'contents'): - setattr(r, 'contents', contents) - return r + return UrlResponse(r) except exceptions.RequestException as e: - excps.append(e) + excps.append(UrlError(e)) if i + 1 < manual_tries and sec_between > 0: LOG.debug("Please wait %s seconds while we wait to try again", sec_between) @@ -213,7 +249,7 @@ def wait_for_url(urls, max_wait=None, timeout=None, e = ValueError(reason) else: return url - except exceptions.RequestException as e: + except UrlError as e: reason = "request error [%s]" % e except Exception as e: reason = "unexpected error [%s]" % e diff --git a/cloudinit/user_data.py b/cloudinit/user_data.py index 803ffc3a..4a640f1e 100644 --- a/cloudinit/user_data.py +++ b/cloudinit/user_data.py @@ -29,7 +29,6 @@ from email.mime.text import MIMEText from cloudinit import handlers from cloudinit import log as logging -from cloudinit import url_helper from cloudinit import util LOG = logging.getLogger(__name__) @@ -173,10 +172,10 @@ class UserDataProcessor(object): if include_once_on and os.path.isfile(include_once_fn): content = util.load_file(include_once_fn) else: - resp = url_helper.readurl(include_url) - if include_once_on and resp.ok(): + resp = util.read_file_or_url(include_url) + if include_once_on and resp.ok: util.write_file(include_once_fn, str(resp), mode=0600) - if resp.ok(): + if resp.ok: content = str(resp) else: LOG.warn(("Fetching from %s resulted in" diff --git a/cloudinit/util.py b/cloudinit/util.py index f5a7ac12..da2cdeda 100644 --- a/cloudinit/util.py +++ b/cloudinit/util.py @@ -51,7 +51,7 @@ import yaml from cloudinit import importer from cloudinit import log as logging from cloudinit import safeyaml -from cloudinit import url_helper as uhelp +from cloudinit import url_helper from cloudinit.settings import (CFG_BUILTIN) @@ -69,6 +69,18 @@ FN_ALLOWED = ('_-.()' + string.digits + string.ascii_letters) CONTAINER_TESTS = ['running-in-container', 'lxc-is-container'] +class FileResponse(object): + def __init__(self, path, contents): + self.code = 200 + self.headers = {} + self.contents = contents + self.ok = True + self.url = path + + def __str__(self): + return self.contents + + class ProcessExecutionError(IOError): MESSAGE_TMPL = ('%(description)s\n' @@ -628,12 +640,53 @@ def read_optional_seed(fill, base="", ext="", timeout=5): raise -def read_file_or_url(url, timeout=5, retries=10, file_retries=0): +def fetch_ssl_details(paths=None): + ssl_details = {} + # Lookup in these locations for ssl key/cert files + ssl_cert_paths = [ + '/var/lib/cloud/data/ssl', + '/var/lib/cloud/instance/data/ssl', + ] + if paths: + ssl_cert_paths.extend([ + os.path.join(paths.get_ipath_cur('data'), 'ssl'), + os.path.join(paths.get_cpath('data'), 'ssl'), + ]) + ssl_cert_paths = uniq_merge(ssl_cert_paths) + ssl_cert_paths = [d for d in ssl_cert_paths if d and os.path.isdir(d)] + cert_file = None + for d in ssl_cert_paths: + if os.path.isfile(os.path.join(d, 'cert.pem')): + cert_file = os.path.join(d, 'cert.pem') + break + key_file = None + for d in ssl_cert_paths: + if os.path.isfile(os.path.join(d, 'key.pem')): + key_file = os.path.join(d, 'key.pem') + break + if cert_file and key_file: + ssl_details['cert_file'] = cert_file + ssl_details['key_file'] = key_file + elif cert_file: + ssl_details['cert_file'] = cert_file + return ssl_details + + +def read_file_or_url(url, timeout=5, retries=10, + headers=None, data=None, sec_between=1, paths=None): if url.startswith("/"): url = "file://%s" % url - if url.startswith("file://"): - retries = file_retries - return uhelp.readurl(url, timeout=timeout, retries=retries) + if url.lower().startswith("file://"): + file_path = url[len("file://"):] + return FileResponse(file_path, contents=load_file(file_path)) + else: + return url_helper.readurl(url, + timeout=timeout, + retries=retries, + headers=headers, + data=data, + sec_between=sec_between, + ssl_details=fetch_ssl_details(paths)) def load_yaml(blob, default=None, allowed=(dict,)): @@ -675,13 +728,13 @@ def read_seeded(base="", ext="", timeout=5, retries=10, file_retries=0): md_resp = read_file_or_url(md_url, timeout, retries, file_retries) md = None - if md_resp.ok(): + if md_resp.ok: md_str = str(md_resp) md = load_yaml(md_str, default={}) ud_resp = read_file_or_url(ud_url, timeout, retries, file_retries) ud = None - if ud_resp.ok(): + if ud_resp.ok: ud_str = str(ud_resp) ud = ud_str @@ -850,8 +903,8 @@ def get_cmdline_url(names=('cloud-config-url', 'url'), if not url: return (None, None, None) - resp = uhelp.readurl(url) - if resp.contents.startswith(starts) and resp.ok(): + resp = read_file_or_url(url) + if resp.contents.startswith(starts) and resp.ok: return (key, url, str(resp)) return (key, url, None) -- cgit v1.2.3 From 9dfb60d3144860334ab1ad1d72920d962139461f Mon Sep 17 00:00:00 2001 From: harlowja Date: Thu, 21 Feb 2013 22:39:30 -0800 Subject: More work on requests integration. --- cloudinit/config/cc_phone_home.py | 3 ++- cloudinit/ec2_utils.py | 17 +++++++++++------ cloudinit/sources/DataSourceEc2.py | 6 ++++-- cloudinit/sources/DataSourceMAAS.py | 15 +++++++++------ cloudinit/util.py | 4 ++-- 5 files changed, 28 insertions(+), 17 deletions(-) (limited to 'cloudinit/sources') diff --git a/cloudinit/config/cc_phone_home.py b/cloudinit/config/cc_phone_home.py index 90834080..5a4332ef 100644 --- a/cloudinit/config/cc_phone_home.py +++ b/cloudinit/config/cc_phone_home.py @@ -112,7 +112,8 @@ def handle(name, cfg, cloud, log, args): url = templater.render_string(url, url_params) try: util.read_file_or_url(url, data=real_submit_keys, - retries=tries, sec_between=3) + retries=tries, sec_between=3, + ssl_details=util.fetch_ssl_details(cloud.paths)) except: util.logexc(log, ("Failed to post phone home data to" " %s in %s tries"), url, tries) diff --git a/cloudinit/ec2_utils.py b/cloudinit/ec2_utils.py index c422eea9..026ee178 100644 --- a/cloudinit/ec2_utils.py +++ b/cloudinit/ec2_utils.py @@ -38,11 +38,16 @@ def combine_url(base, add_on): # See: http://bit.ly/TyoUQs +# +# Since boto metadata reader uses the old urllib which does not +# support ssl, we need to ahead and create our own reader which +# works the same as the boto one (for now). class MetadataMaterializer(object): - def __init__(self, blob, base_url): + def __init__(self, blob, base_url, ssl_details): self._blob = blob self._md = None self._base_url = base_url + self._ssl_details = ssl_details def _parse(self, blob): leaves = {} @@ -89,7 +94,7 @@ class MetadataMaterializer(object): return self._md def _fetch_url(self, url): - response = util.read_file_or_url(url) + response = util.read_file_or_url(url, ssl_details=self._ssl_details) return str(response) def _decode_leaf_blob(self, blob): @@ -134,19 +139,19 @@ def get_instance_userdata(url, version='latest', ssl_details=None): ud_url = combine_url(url, version) ud_url = combine_url(ud_url, 'user-data') try: - response = util.read_file_or_url(ud_url) + response = util.read_file_or_url(ud_url, ssl_details=ssl_details) return str(response) except Exception: util.logexc(LOG, "Failed fetching userdata from url %s", ud_url) return None -def get_instance_metadata(url, version='latest'): +def get_instance_metadata(url, version='latest', ssl_details=None): md_url = combine_url(url, version) md_url = combine_url(md_url, 'meta-data') try: - response = util.read_file_or_url(md_url) - materializer = MetadataMaterializer(str(response), md_url) + response = util.read_file_or_url(md_url, ssl_details=ssl_details) + materializer = MetadataMaterializer(str(response), md_url, ssl_details) return materializer.materialize() except Exception: util.logexc(LOG, "Failed fetching metadata from url %s", md_url) diff --git a/cloudinit/sources/DataSourceEc2.py b/cloudinit/sources/DataSourceEc2.py index 47f677d4..eee4e6bc 100644 --- a/cloudinit/sources/DataSourceEc2.py +++ b/cloudinit/sources/DataSourceEc2.py @@ -64,8 +64,10 @@ class DataSourceEc2(sources.DataSource): if not self.wait_for_metadata_service(): return False start_time = time.time() - self.userdata_raw = ec2_utils.get_instance_userdata(self.metadata_address, self.api_ver) - self.metadata = ec2_utils.get_instance_metadata(self.metadata_address, self.api_ver) + self.userdata_raw = ec2_utils.get_instance_userdata(self.metadata_address, self.api_ver, + ssl_details=util.fetch_ssl_details(self.paths)) + self.metadata = ec2_utils.get_instance_metadata(self.metadata_address, self.api_ver, + ssl_details=util.fetch_ssl_details(self.paths)) LOG.debug("Crawl of metadata service took %s seconds", int(time.time() - start_time)) return True diff --git a/cloudinit/sources/DataSourceMAAS.py b/cloudinit/sources/DataSourceMAAS.py index 2de31305..dc048943 100644 --- a/cloudinit/sources/DataSourceMAAS.py +++ b/cloudinit/sources/DataSourceMAAS.py @@ -81,7 +81,8 @@ class DataSourceMAAS(sources.DataSource): self.base_url = url (userdata, metadata) = read_maas_seed_url(self.base_url, - self.md_headers) + self.md_headers, + paths=self.paths) self.userdata_raw = userdata self.metadata = metadata return True @@ -141,7 +142,7 @@ class DataSourceMAAS(sources.DataSource): LOG.debug("Using metadata source: '%s'", url) else: LOG.critical("Giving up on md from %s after %i seconds", - urls, int(time.time() - starttime)) + urls, int(time.time() - starttime)) return bool(url) @@ -190,7 +191,7 @@ def read_maas_seed_dir(seed_d): def read_maas_seed_url(seed_url, header_cb=None, timeout=None, - version=MD_VERSION): + version=MD_VERSION, paths=None): """ Read the maas datasource at seed_url. - header_cb is a method that should return a headers dictionary for @@ -222,12 +223,13 @@ def read_maas_seed_url(seed_url, header_cb=None, timeout=None, else: headers = {} try: - resp = util.read_file_or_url(url, headers=headers, timeout=timeout) + resp = util.read_file_or_url(url, headers=headers, timeout=timeout, + ssl_details=util.fetch_ssl_details(paths)) if resp.ok: md[name] = str(resp) else: LOG.warn(("Fetching from %s resulted in" - " an invalid http code %s"), url, resp.status_code) + " an invalid http code %s"), url, resp.code) except url_helper.UrlError as e: if e.code != 404: raise @@ -372,7 +374,8 @@ if __name__ == "__main__": if args.subcmd == "check-seed": if args.url.startswith("http"): (userdata, metadata) = read_maas_seed_url(args.url, - header_cb=my_headers, version=args.apiver) + header_cb=my_headers, + version=args.apiver) else: (userdata, metadata) = read_maas_seed_url(args.url) print "=== userdata ===" diff --git a/cloudinit/util.py b/cloudinit/util.py index da2cdeda..307ed613 100644 --- a/cloudinit/util.py +++ b/cloudinit/util.py @@ -673,7 +673,7 @@ def fetch_ssl_details(paths=None): def read_file_or_url(url, timeout=5, retries=10, - headers=None, data=None, sec_between=1, paths=None): + headers=None, data=None, sec_between=1, ssl_details=None): if url.startswith("/"): url = "file://%s" % url if url.lower().startswith("file://"): @@ -686,7 +686,7 @@ def read_file_or_url(url, timeout=5, retries=10, headers=headers, data=data, sec_between=sec_between, - ssl_details=fetch_ssl_details(paths)) + ssl_details=ssl_details) def load_yaml(blob, default=None, allowed=(dict,)): -- cgit v1.2.3 From eacfc7ffbec3e6a0348ed484da895e2d2fc5ba10 Mon Sep 17 00:00:00 2001 From: harlowja Date: Sat, 23 Feb 2013 21:23:24 -0800 Subject: Get tests working and further adjustments. --- cloudinit/sources/DataSourceMAAS.py | 9 +++++--- cloudinit/url_helper.py | 21 +++++++++-------- cloudinit/user_data.py | 8 ++++--- cloudinit/util.py | 34 ++++++++++++++++++++-------- tests/unittests/test__init__.py | 10 ++++---- tests/unittests/test_datasource/test_maas.py | 11 +++++---- 6 files changed, 60 insertions(+), 33 deletions(-) (limited to 'cloudinit/sources') diff --git a/cloudinit/sources/DataSourceMAAS.py b/cloudinit/sources/DataSourceMAAS.py index 6e1133b2..0c526305 100644 --- a/cloudinit/sources/DataSourceMAAS.py +++ b/cloudinit/sources/DataSourceMAAS.py @@ -223,9 +223,12 @@ def read_maas_seed_url(seed_url, header_cb=None, timeout=None, else: headers = {} try: - resp = util.read_file_or_url(url, headers=headers, timeout=timeout, - ssl_details=util.fetch_ssl_details(paths)) - if resp.ok: + ssl_details = util.fetch_ssl_details(paths) + resp = util.read_file_or_url(url, + headers=headers, + timeout=timeout, + ssl_details=ssl_details) + if resp.ok(): md[name] = str(resp) else: LOG.warn(("Fetching from %s resulted in" diff --git a/cloudinit/url_helper.py b/cloudinit/url_helper.py index 0839e63b..300e70c2 100644 --- a/cloudinit/url_helper.py +++ b/cloudinit/url_helper.py @@ -70,9 +70,14 @@ class UrlResponse(object): def url(self): return self._response.url - @property - def ok(self): - return self._response.ok + def ok(self, redirects_ok=False): + upper = 300 + if redirects_ok: + upper = 400 + if self.code >= 200 and self.code < upper: + return True + else: + return False @property def headers(self): @@ -158,11 +163,8 @@ def readurl(url, data=None, timeout=None, retries=0, sec_between=1, r = requests.request(**req_args) if check_status: r.raise_for_status() - contents = r.content - status = r.status_code - headers = r.headers LOG.debug("Read from %s (%s, %sb) after %s attempts", url, - status, len(contents), (i + 1)) + r.status_code, len(r.content), (i + 1)) # Doesn't seem like we can make it use a different # subclass for responses, so add our own backward-compat # attrs @@ -256,8 +258,9 @@ def wait_for_url(urls, max_wait=None, timeout=None, time_taken = int(time.time() - start_time) status_msg = "Calling '%s' failed [%s/%ss]: %s" % (url, - time_taken, - max_wait, reason) + time_taken, + max_wait, + reason) status_cb(status_msg) if exception_cb: exception_cb(msg=status_msg, exception=e) diff --git a/cloudinit/user_data.py b/cloudinit/user_data.py index eaf448a7..df069ff8 100644 --- a/cloudinit/user_data.py +++ b/cloudinit/user_data.py @@ -59,6 +59,7 @@ EXAMINE_FOR_LAUNCH_INDEX = ["text/cloud-config"] class UserDataProcessor(object): def __init__(self, paths): self.paths = paths + self.ssl_details = util.fetch_ssl_details(paths) def process(self, blob): accumulating_msg = MIMEMultipart() @@ -172,10 +173,11 @@ class UserDataProcessor(object): if include_once_on and os.path.isfile(include_once_fn): content = util.load_file(include_once_fn) else: - resp = util.read_file_or_url(include_url) - if include_once_on and resp.ok: + resp = util.read_file_or_url(include_url, + ssl_details=self.ssl_details) + if include_once_on and resp.ok(): util.write_file(include_once_fn, str(resp), mode=0600) - if resp.ok: + if resp.ok(): content = str(resp) else: LOG.warn(("Fetching from %s resulted in" diff --git a/cloudinit/util.py b/cloudinit/util.py index 42b3ab01..dc3c5639 100644 --- a/cloudinit/util.py +++ b/cloudinit/util.py @@ -70,18 +70,31 @@ FN_ALLOWED = ('_-.()' + string.digits + string.ascii_letters) CONTAINER_TESTS = ['running-in-container', 'lxc-is-container'] -class FileResponse(object): - def __init__(self, path, contents): - self.code = 200 +# Made to have same accessors as UrlResponse so that the +# read_file_or_url can return this or that object and the +# 'user' of those objects will not need to know the difference. +class StringResponse(object): + def __init__(self, contents, code=200): + self.code = code self.headers = {} self.contents = contents - self.ok = True - self.url = path + self.url = None + + def ok(self, *args, **kwargs): + if self.code != 200: + return False + return True def __str__(self): return self.contents +class FileResponse(StringResponse): + def __init__(self, path, contents, code=200): + StringResponse.__init__(self, contents, code=code) + self.url = path + + class ProcessExecutionError(IOError): MESSAGE_TMPL = ('%(description)s\n' @@ -630,7 +643,7 @@ def read_optional_seed(fill, base="", ext="", timeout=5): fill['user-data'] = ud fill['meta-data'] = md return True - except OSError as e: + except IOError as e: if e.errno == errno.ENOENT: return False raise @@ -670,9 +683,12 @@ def fetch_ssl_details(paths=None): def read_file_or_url(url, timeout=5, retries=10, headers=None, data=None, sec_between=1, ssl_details=None): + url = url.lstrip() if url.startswith("/"): url = "file://%s" % url if url.lower().startswith("file://"): + if data: + LOG.warn("Unable to post data to file resource %s", url) file_path = url[len("file://"):] return FileResponse(file_path, contents=load_file(file_path)) else: @@ -724,13 +740,13 @@ def read_seeded(base="", ext="", timeout=5, retries=10, file_retries=0): md_resp = read_file_or_url(md_url, timeout, retries, file_retries) md = None - if md_resp.ok: + if md_resp.ok(): md_str = str(md_resp) md = load_yaml(md_str, default={}) ud_resp = read_file_or_url(ud_url, timeout, retries, file_retries) ud = None - if ud_resp.ok: + if ud_resp.ok(): ud_str = str(ud_resp) ud = ud_str @@ -900,7 +916,7 @@ def get_cmdline_url(names=('cloud-config-url', 'url'), return (None, None, None) resp = read_file_or_url(url) - if resp.contents.startswith(starts) and resp.ok: + if resp.contents.startswith(starts) and resp.ok(): return (key, url, str(resp)) return (key, url, None) diff --git a/tests/unittests/test__init__.py b/tests/unittests/test__init__.py index ac082076..d707afa9 100644 --- a/tests/unittests/test__init__.py +++ b/tests/unittests/test__init__.py @@ -191,8 +191,8 @@ class TestCmdlineUrl(MockerTestCase): mock_readurl = self.mocker.replace(url_helper.readurl, passthrough=False) - mock_readurl(url) - self.mocker.result(url_helper.UrlResponse(200, payload)) + mock_readurl(url, ARGS, KWARGS) + self.mocker.result(util.StringResponse(payload)) self.mocker.replay() self.assertEqual((key, url, None), @@ -207,8 +207,8 @@ class TestCmdlineUrl(MockerTestCase): mock_readurl = self.mocker.replace(url_helper.readurl, passthrough=False) - mock_readurl(url) - self.mocker.result(url_helper.UrlResponse(200, payload)) + mock_readurl(url, ARGS, KWARGS) + self.mocker.result(util.StringResponse(payload)) self.mocker.replay() self.assertEqual((key, url, payload), @@ -221,7 +221,7 @@ class TestCmdlineUrl(MockerTestCase): cmdline = "ro %s=%s bar=1" % (key, url) self.mocker.replace(url_helper.readurl, passthrough=False) - self.mocker.result(url_helper.UrlResponse(400)) + self.mocker.result(util.StringResponse("")) self.mocker.replay() self.assertEqual((None, None, None), diff --git a/tests/unittests/test_datasource/test_maas.py b/tests/unittests/test_datasource/test_maas.py index b56fea82..47f8caa4 100644 --- a/tests/unittests/test_datasource/test_maas.py +++ b/tests/unittests/test_datasource/test_maas.py @@ -3,12 +3,13 @@ import os from cloudinit.sources import DataSourceMAAS from cloudinit import url_helper +from cloudinit import util from tests.unittests.helpers import populate_dir -from mocker import MockerTestCase +import mocker -class TestMAASDataSource(MockerTestCase): +class TestMAASDataSource(mocker.MockerTestCase): def setUp(self): super(TestMAASDataSource, self).setUp() @@ -115,9 +116,11 @@ class TestMAASDataSource(MockerTestCase): for key in valid_order: url = "%s/%s/%s" % (my_seed, my_ver, key) - mock_request(url, headers=my_headers, timeout=None) + mock_request(url, headers=my_headers, timeout=mocker.ANY, + data=mocker.ANY, sec_between=mocker.ANY, + ssl_details=mocker.ANY, retries=mocker.ANY) resp = valid.get(key) - self.mocker.result(url_helper.UrlResponse(200, resp)) + self.mocker.result(util.StringResponse(resp)) self.mocker.replay() (userdata, metadata) = DataSourceMAAS.read_maas_seed_url(my_seed, -- cgit v1.2.3 From f22388a04076247490c02f605b176c3c45ecd425 Mon Sep 17 00:00:00 2001 From: harlowja Date: Sat, 23 Feb 2013 23:22:57 -0800 Subject: Fix the maas callback mechanism now that requests is used. --- cloudinit/sources/DataSourceMAAS.py | 25 ++++++++++++------------ cloudinit/url_helper.py | 38 +++++++++++++++++++++++-------------- 2 files changed, 36 insertions(+), 27 deletions(-) (limited to 'cloudinit/sources') diff --git a/cloudinit/sources/DataSourceMAAS.py b/cloudinit/sources/DataSourceMAAS.py index 0c526305..f3759b4b 100644 --- a/cloudinit/sources/DataSourceMAAS.py +++ b/cloudinit/sources/DataSourceMAAS.py @@ -25,8 +25,6 @@ import os import time import urllib2 -import requests - from cloudinit import log as logging from cloudinit import sources from cloudinit import url_helper @@ -81,7 +79,7 @@ class DataSourceMAAS(sources.DataSource): self.base_url = url (userdata, metadata) = read_maas_seed_url(self.base_url, - self.md_headers, + self._md_headers, paths=self.paths) self.userdata_raw = userdata self.metadata = metadata @@ -90,7 +88,7 @@ class DataSourceMAAS(sources.DataSource): util.logexc(LOG, "Failed fetching metadata from url %s", url) return False - def md_headers(self, url): + def _md_headers(self, url): mcfg = self.ds_cfg # If we are missing token_key, token_secret or consumer_key @@ -134,9 +132,10 @@ class DataSourceMAAS(sources.DataSource): starttime = time.time() check_url = "%s/%s/meta-data/instance-id" % (url, MD_VERSION) urls = [check_url] - url = uhelp.wait_for_url(urls=urls, max_wait=max_wait, - timeout=timeout, exception_cb=self._except_cb, - headers_cb=self.md_headers) + url = url_helper.wait_for_url(urls=urls, max_wait=max_wait, + timeout=timeout, + exception_cb=self._except_cb, + headers_cb=self._md_headers) if url: LOG.debug("Using metadata source: '%s'", url) @@ -147,23 +146,23 @@ class DataSourceMAAS(sources.DataSource): return bool(url) def _except_cb(self, msg, exception): - if not (isinstance(exception, urllib2.HTTPError) and + if not (isinstance(exception, url_helper.UrlError) and (exception.code == 403 or exception.code == 401)): return + if 'date' not in exception.headers: - LOG.warn("date field not in %d headers" % exception.code) + LOG.warn("Missing header 'date' in %s response", exception.code) return date = exception.headers['date'] - try: ret_time = time.mktime(parsedate(date)) - except: - LOG.warn("failed to convert datetime '%s'") + except Exception as e: + LOG.warn("Failed to convert datetime '%s': %s", date, e) return self.oauth_clockskew = int(ret_time - time.time()) - LOG.warn("set oauth clockskew to %d" % self.oauth_clockskew) + LOG.warn("Setting oauth clockskew to %d", self.oauth_clockskew) return diff --git a/cloudinit/url_helper.py b/cloudinit/url_helper.py index 300e70c2..6f06761a 100644 --- a/cloudinit/url_helper.py +++ b/cloudinit/url_helper.py @@ -92,13 +92,13 @@ class UrlResponse(object): class UrlError(IOError): - def __init__(self, cause): + def __init__(self, cause, code=None, headers=None): IOError.__init__(self, str(cause)) self.cause = cause - if isinstance(cause, exceptions.HTTPError) and cause.response: - self.code = cause.response.status_code - else: - self.code = None + self.code = code + self.headers = headers + if self.headers is None: + self.headers = {} def readurl(url, data=None, timeout=None, retries=0, sec_between=1, @@ -170,7 +170,11 @@ def readurl(url, data=None, timeout=None, retries=0, sec_between=1, # attrs return UrlResponse(r) except exceptions.RequestException as e: - excps.append(UrlError(e)) + if isinstance(e, (exceptions.HTTPError)) and e.response: + excps.append(UrlError(e, code=e.response.status_code, + headers=e.response.headers)) + else: + excps.append(UrlError(e)) if i + 1 < manual_tries and sec_between > 0: LOG.debug("Please wait %s seconds while we wait to try again", sec_between) @@ -235,20 +239,23 @@ def wait_for_url(urls, max_wait=None, timeout=None, timeout = int((start_time + max_wait) - now) reason = "" + e = None try: if headers_cb is not None: headers = headers_cb(url) else: headers = {} - resp = readurl(url, headers=headers, timeout=timeout, - check_status=False) - if not resp.contents: - reason = "empty response [%s]" % (resp.code) - e = ValueError(reason) - elif not resp.ok(): - reason = "bad status code [%s]" % (resp.code) - e = ValueError(reason) + response = readurl(url, headers=headers, timeout=timeout, + check_status=False) + if not response.contents: + reason = "empty response [%s]" % (response.code) + e = UrlError(ValueError(reason), + code=response.code, headers=response.headers) + elif not response.ok(): + reason = "bad status code [%s]" % (response.code) + e = UrlError(ValueError(reason), + code=response.code, headers=response.headers) else: return url except UrlError as e: @@ -263,6 +270,9 @@ def wait_for_url(urls, max_wait=None, timeout=None, reason) status_cb(status_msg) if exception_cb: + # This can be used to alter the headers that will be sent + # in the future, for example this is what the MAAS datasource + # does. exception_cb(msg=status_msg, exception=e) if timeup(max_wait, start_time): -- cgit v1.2.3 From ac83536339d2622f4a896f50681497a388e7e26f Mon Sep 17 00:00:00 2001 From: Scott Moser Date: Fri, 1 Mar 2013 16:03:02 -0500 Subject: fix regression on expected label of filesystem for DataSourceNone Last addition to DataSourceNoCloud left it looking for a filesystem named 'None'. --- cloudinit/sources/DataSourceNoCloud.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'cloudinit/sources') diff --git a/cloudinit/sources/DataSourceNoCloud.py b/cloudinit/sources/DataSourceNoCloud.py index 097bbc52..603f0155 100644 --- a/cloudinit/sources/DataSourceNoCloud.py +++ b/cloudinit/sources/DataSourceNoCloud.py @@ -86,11 +86,11 @@ class DataSourceNoCloud(sources.DataSource): if 'ds_config' not in found: found.append("ds_config") - if self.ds_cfg.get('fs_label', "cidata"): + label = self.ds_cfg.get('fs_label', "cidata") + if label is not None: fslist = util.find_devs_with("TYPE=vfat") fslist.extend(util.find_devs_with("TYPE=iso9660")) - label = self.ds_cfg.get('fs_label') label_list = util.find_devs_with("LABEL=%s" % label) devlist = list(set(fslist) & set(label_list)) devlist.sort(reverse=True) -- cgit v1.2.3 From fc6aa5aa54ee35ff0a3eff823bae0d3cf9b34bc1 Mon Sep 17 00:00:00 2001 From: Joshua Harlow Date: Wed, 6 Mar 2013 19:24:05 -0800 Subject: Continue working on merging code. --- cloudinit/config/cc_landscape.py | 3 ++- cloudinit/config/cc_mounts.py | 3 ++- cloudinit/distros/__init__.py | 15 +++++++------ cloudinit/handlers/__init__.py | 3 ++- cloudinit/handlers/cloud_config.py | 15 ++++++------- cloudinit/helpers.py | 3 ++- cloudinit/mergers/__init__.py | 13 +++++++++--- cloudinit/sources/DataSourceAltCloud.py | 5 +++-- cloudinit/sources/DataSourceCloudStack.py | 3 --- cloudinit/sources/DataSourceConfigDrive.py | 4 +++- cloudinit/sources/DataSourceEc2.py | 3 --- cloudinit/sources/DataSourceMAAS.py | 3 ++- cloudinit/sources/DataSourceNoCloud.py | 5 ++--- cloudinit/sources/DataSourceNone.py | 3 --- cloudinit/sources/DataSourceOVF.py | 3 ++- cloudinit/sources/__init__.py | 10 ++++++--- cloudinit/stages.py | 9 ++++---- cloudinit/type_utils.py | 34 ++++++++++++++++++++++++++++++ cloudinit/util.py | 33 ++++++++++------------------- tests/unittests/test_userdata.py | 4 +++- 20 files changed, 104 insertions(+), 70 deletions(-) create mode 100644 cloudinit/type_utils.py (limited to 'cloudinit/sources') diff --git a/cloudinit/config/cc_landscape.py b/cloudinit/config/cc_landscape.py index 02610dd0..6734efee 100644 --- a/cloudinit/config/cc_landscape.py +++ b/cloudinit/config/cc_landscape.py @@ -24,6 +24,7 @@ from StringIO import StringIO from configobj import ConfigObj +from cloudinit import type_utils from cloudinit import util from cloudinit.settings import PER_INSTANCE @@ -58,7 +59,7 @@ def handle(_name, cfg, cloud, log, _args): if not isinstance(ls_cloudcfg, (dict)): raise RuntimeError(("'landscape' key existed in config," " but not a dictionary type," - " is a %s instead"), util.obj_name(ls_cloudcfg)) + " is a %s instead"), type_utils.obj_name(ls_cloudcfg)) if not ls_cloudcfg: return diff --git a/cloudinit/config/cc_mounts.py b/cloudinit/config/cc_mounts.py index cb772c86..6ebe563d 100644 --- a/cloudinit/config/cc_mounts.py +++ b/cloudinit/config/cc_mounts.py @@ -22,6 +22,7 @@ from string import whitespace # pylint: disable=W0402 import re +from cloudinit import type_utils from cloudinit import util # Shortname matches 'sda', 'sda1', 'xvda', 'hda', 'sdb', xvdb, vda, vdd1 @@ -60,7 +61,7 @@ def handle(_name, cfg, cloud, log, _args): # skip something that wasn't a list if not isinstance(cfgmnt[i], list): log.warn("Mount option %s not a list, got a %s instead", - (i + 1), util.obj_name(cfgmnt[i])) + (i + 1), type_utils.obj_name(cfgmnt[i])) continue startname = str(cfgmnt[i][0]) diff --git a/cloudinit/distros/__init__.py b/cloudinit/distros/__init__.py index 6a684b89..eeea6af1 100644 --- a/cloudinit/distros/__init__.py +++ b/cloudinit/distros/__init__.py @@ -31,6 +31,7 @@ import re from cloudinit import importer from cloudinit import log as logging from cloudinit import ssh_util +from cloudinit import type_utils from cloudinit import util from cloudinit.distros.parsers import hosts @@ -427,7 +428,7 @@ class Distro(object): lines.append("%s %s" % (user, rules)) else: msg = "Can not create sudoers rule addition with type %r" - raise TypeError(msg % (util.obj_name(rules))) + raise TypeError(msg % (type_utils.obj_name(rules))) content = "\n".join(lines) content += "\n" # trailing newline @@ -550,7 +551,7 @@ def _normalize_groups(grp_cfg): c_grp_cfg[k] = [v] else: raise TypeError("Bad group member type %s" % - util.obj_name(v)) + type_utils.obj_name(v)) else: if isinstance(v, (list)): c_grp_cfg[k].extend(v) @@ -558,13 +559,13 @@ def _normalize_groups(grp_cfg): c_grp_cfg[k].append(v) else: raise TypeError("Bad group member type %s" % - util.obj_name(v)) + type_utils.obj_name(v)) elif isinstance(i, (str, basestring)): if i not in c_grp_cfg: c_grp_cfg[i] = [] else: raise TypeError("Unknown group name type %s" % - util.obj_name(i)) + type_utils.obj_name(i)) grp_cfg = c_grp_cfg groups = {} if isinstance(grp_cfg, (dict)): @@ -573,7 +574,7 @@ def _normalize_groups(grp_cfg): else: raise TypeError(("Group config must be list, dict " " or string types only and not %s") % - util.obj_name(grp_cfg)) + type_utils.obj_name(grp_cfg)) return groups @@ -604,7 +605,7 @@ def _normalize_users(u_cfg, def_user_cfg=None): ad_ucfg.append(v) else: raise TypeError(("Unmappable user value type %s" - " for key %s") % (util.obj_name(v), k)) + " for key %s") % (type_utils.obj_name(v), k)) u_cfg = ad_ucfg elif isinstance(u_cfg, (str, basestring)): u_cfg = util.uniq_merge_sorted(u_cfg) @@ -629,7 +630,7 @@ def _normalize_users(u_cfg, def_user_cfg=None): else: raise TypeError(("User config must be dictionary/list " " or string types only and not %s") % - util.obj_name(user_config)) + type_utils.obj_name(user_config)) # Ensure user options are in the right python friendly format if users: diff --git a/cloudinit/handlers/__init__.py b/cloudinit/handlers/__init__.py index 63fdb948..924463ce 100644 --- a/cloudinit/handlers/__init__.py +++ b/cloudinit/handlers/__init__.py @@ -27,6 +27,7 @@ from cloudinit.settings import (PER_ALWAYS, PER_INSTANCE, FREQUENCIES) from cloudinit import importer from cloudinit import log as logging +from cloudinit import type_utils from cloudinit import util LOG = logging.getLogger(__name__) @@ -76,7 +77,7 @@ class Handler(object): self.frequency = frequency def __repr__(self): - return "%s: [%s]" % (util.obj_name(self), self.list_types()) + return "%s: [%s]" % (type_utils.obj_name(self), self.list_types()) @abc.abstractmethod def list_types(self): diff --git a/cloudinit/handlers/cloud_config.py b/cloudinit/handlers/cloud_config.py index d458dee2..5f519f78 100644 --- a/cloudinit/handlers/cloud_config.py +++ b/cloudinit/handlers/cloud_config.py @@ -29,8 +29,8 @@ from cloudinit.settings import (PER_ALWAYS) LOG = logging.getLogger(__name__) -DEF_MERGE_TYPE = "list(extend)+dict()+str(append)" MERGE_HEADER = 'Merge-Type' +DEF_MERGERS = mergers.default_mergers() class CloudConfigPartHandler(handlers.Handler): @@ -39,9 +39,7 @@ class CloudConfigPartHandler(handlers.Handler): self.cloud_buf = None self.cloud_fn = paths.get_ipath("cloud_config") self.file_names = [] - self.mergers = [ - mergers.string_extract_mergers(DEF_MERGE_TYPE), - ] + self.mergers = [DEF_MERGERS] def list_types(self): return [ @@ -59,6 +57,7 @@ class CloudConfigPartHandler(handlers.Handler): file_lines.append("# %s" % (fn)) file_lines.append("") if self.cloud_buf is not None: + # Something was actually gathered.... lines = [ "#cloud-config", '', @@ -86,7 +85,7 @@ class CloudConfigPartHandler(handlers.Handler): all_mergers.extend(mergers_yaml) all_mergers.extend(mergers_header) if not all_mergers: - all_mergers = mergers.string_extract_mergers(DEF_MERGE_TYPE) + all_mergers = DEF_MERGERS return all_mergers def _merge_part(self, payload, headers): @@ -94,7 +93,7 @@ class CloudConfigPartHandler(handlers.Handler): # Use the merger list from the last call, since it is the one # that will be defining how to merge with the next payload. curr_mergers = list(self.mergers[-1]) - LOG.debug("Merging with %s", curr_mergers) + LOG.debug("Merging by applying %s", curr_mergers) self.mergers.append(next_mergers) merger = mergers.construct(curr_mergers) if self.cloud_buf is None: @@ -106,9 +105,7 @@ class CloudConfigPartHandler(handlers.Handler): def _reset(self): self.file_names = [] self.cloud_buf = None - self.mergers = [ - mergers.string_extract_mergers(DEF_MERGE_TYPE), - ] + self.mergers = [DEF_MERGERS] def handle_part(self, _data, ctype, filename, payload, _freq, headers): if ctype == handlers.CONTENT_START: diff --git a/cloudinit/helpers.py b/cloudinit/helpers.py index 2077401c..a4e6fb03 100644 --- a/cloudinit/helpers.py +++ b/cloudinit/helpers.py @@ -32,6 +32,7 @@ from cloudinit.settings import (PER_INSTANCE, PER_ALWAYS, PER_ONCE, CFG_ENV_NAME) from cloudinit import log as logging +from cloudinit import type_utils from cloudinit import util LOG = logging.getLogger(__name__) @@ -68,7 +69,7 @@ class FileLock(object): self.fn = fn def __str__(self): - return "<%s using file %r>" % (util.obj_name(self), self.fn) + return "<%s using file %r>" % (type_utils.obj_name(self), self.fn) def canon_sem_name(name): diff --git a/cloudinit/mergers/__init__.py b/cloudinit/mergers/__init__.py index 4a112165..453426af 100644 --- a/cloudinit/mergers/__init__.py +++ b/cloudinit/mergers/__init__.py @@ -20,11 +20,12 @@ import re from cloudinit import importer from cloudinit import log as logging -from cloudinit import util +from cloudinit import type_utils NAME_MTCH = re.compile(r"(^[a-zA-Z_][A-Za-z0-9_]*)\((.*?)\)$") LOG = logging.getLogger(__name__) +DEF_MERGE_TYPE = "list(extend)+dict()+str(append)" class UnknownMerger(object): @@ -42,7 +43,7 @@ class UnknownMerger(object): # If not found the merge will be given to a '_handle_unknown' # function which can decide what to do wit the 2 values. def merge(self, source, merge_with): - type_name = util.obj_name(source) + type_name = type_utils.obj_name(source) type_name = type_name.lower() method_name = "_on_%s" % (type_name) meth = None @@ -127,6 +128,10 @@ def string_extract_mergers(merge_how): return parsed_mergers +def default_mergers(): + return tuple(string_extract_mergers(DEF_MERGE_TYPE)) + + def construct(parsed_mergers): mergers_to_be = [] for (m_name, m_ops) in parsed_mergers: @@ -145,4 +150,6 @@ def construct(parsed_mergers): root = LookupMerger(mergers) for (attr, opts) in mergers_to_be: mergers.append(attr(root, opts)) - return root \ No newline at end of file + return root + + diff --git a/cloudinit/sources/DataSourceAltCloud.py b/cloudinit/sources/DataSourceAltCloud.py index 9812bdcb..64548d43 100644 --- a/cloudinit/sources/DataSourceAltCloud.py +++ b/cloudinit/sources/DataSourceAltCloud.py @@ -30,6 +30,7 @@ import os.path from cloudinit import log as logging from cloudinit import sources from cloudinit import util + from cloudinit.util import ProcessExecutionError LOG = logging.getLogger(__name__) @@ -91,8 +92,8 @@ class DataSourceAltCloud(sources.DataSource): self.supported_seed_starts = ("/", "file://") def __str__(self): - mstr = "%s [seed=%s]" % (util.obj_name(self), self.seed) - return mstr + root = sources.DataSource.__str__(self) + return "%s [seed=%s]" % (root, self.seed) def get_cloud_type(self): ''' diff --git a/cloudinit/sources/DataSourceCloudStack.py b/cloudinit/sources/DataSourceCloudStack.py index 076dba5a..c0e1a23c 100644 --- a/cloudinit/sources/DataSourceCloudStack.py +++ b/cloudinit/sources/DataSourceCloudStack.py @@ -59,9 +59,6 @@ class DataSourceCloudStack(sources.DataSource): return gw return None - def __str__(self): - return util.obj_name(self) - def _get_url_settings(self): mcfg = self.ds_cfg if not mcfg: diff --git a/cloudinit/sources/DataSourceConfigDrive.py b/cloudinit/sources/DataSourceConfigDrive.py index c7826851..46abd772 100644 --- a/cloudinit/sources/DataSourceConfigDrive.py +++ b/cloudinit/sources/DataSourceConfigDrive.py @@ -51,7 +51,9 @@ class DataSourceConfigDrive(sources.DataSource): self.ec2_metadata = None def __str__(self): - mstr = "%s [%s,ver=%s]" % (util.obj_name(self), self.dsmode, + root = sources.DataSource.__str__(self) + mstr = "%s [%s,ver=%s]" % (root, + self.dsmode, self.version) mstr += "[source=%s]" % (self.source) return mstr diff --git a/cloudinit/sources/DataSourceEc2.py b/cloudinit/sources/DataSourceEc2.py index 2db53446..f010e640 100644 --- a/cloudinit/sources/DataSourceEc2.py +++ b/cloudinit/sources/DataSourceEc2.py @@ -49,9 +49,6 @@ class DataSourceEc2(sources.DataSource): self.seed_dir = os.path.join(paths.seed_dir, "ec2") self.api_ver = DEF_MD_VERSION - def __str__(self): - return util.obj_name(self) - def get_data(self): seed_ret = {} if util.read_optional_seed(seed_ret, base=(self.seed_dir + "/")): diff --git a/cloudinit/sources/DataSourceMAAS.py b/cloudinit/sources/DataSourceMAAS.py index b55d8a21..612d8ffa 100644 --- a/cloudinit/sources/DataSourceMAAS.py +++ b/cloudinit/sources/DataSourceMAAS.py @@ -50,7 +50,8 @@ class DataSourceMAAS(sources.DataSource): self.oauth_clockskew = None def __str__(self): - return "%s [%s]" % (util.obj_name(self), self.base_url) + root = sources.DataSource.__str__(self) + return "%s [%s]" % (root, self.base_url) def get_data(self): mcfg = self.ds_cfg diff --git a/cloudinit/sources/DataSourceNoCloud.py b/cloudinit/sources/DataSourceNoCloud.py index bed500a2..9a770d38 100644 --- a/cloudinit/sources/DataSourceNoCloud.py +++ b/cloudinit/sources/DataSourceNoCloud.py @@ -40,9 +40,8 @@ class DataSourceNoCloud(sources.DataSource): self.supported_seed_starts = ("/", "file://") def __str__(self): - mstr = "%s [seed=%s][dsmode=%s]" % (util.obj_name(self), - self.seed, self.dsmode) - return mstr + root = sources.DataSource.__str__(self) + return "%s [seed=%s][dsmode=%s]" % (root, self.seed, self.dsmode) def get_data(self): defaults = { diff --git a/cloudinit/sources/DataSourceNone.py b/cloudinit/sources/DataSourceNone.py index c2125bee..e2175e1f 100644 --- a/cloudinit/sources/DataSourceNone.py +++ b/cloudinit/sources/DataSourceNone.py @@ -41,9 +41,6 @@ class DataSourceNone(sources.DataSource): def get_instance_id(self): return 'iid-datasource-none' - def __str__(self): - return util.obj_name(self) - @property def is_disconnected(self): return True diff --git a/cloudinit/sources/DataSourceOVF.py b/cloudinit/sources/DataSourceOVF.py index e90150c6..ae139074 100644 --- a/cloudinit/sources/DataSourceOVF.py +++ b/cloudinit/sources/DataSourceOVF.py @@ -43,7 +43,8 @@ class DataSourceOVF(sources.DataSource): self.supported_seed_starts = ("/", "file://") def __str__(self): - return "%s [seed=%s]" % (util.obj_name(self), self.seed) + root = sources.DataSource.__str__(self) + return "%s [seed=%s]" % (root, self.seed) def get_data(self): found = [] diff --git a/cloudinit/sources/__init__.py b/cloudinit/sources/__init__.py index 96baff90..d8fbacdd 100644 --- a/cloudinit/sources/__init__.py +++ b/cloudinit/sources/__init__.py @@ -25,6 +25,7 @@ import os from cloudinit import importer from cloudinit import log as logging +from cloudinit import type_utils from cloudinit import user_data as ud from cloudinit import util @@ -52,7 +53,7 @@ class DataSource(object): self.userdata = None self.metadata = None self.userdata_raw = None - name = util.obj_name(self) + name = type_utils.obj_name(self) if name.startswith(DS_PREFIX): name = name[len(DS_PREFIX):] self.ds_cfg = util.get_cfg_by_path(self.sys_cfg, @@ -62,6 +63,9 @@ class DataSource(object): else: self.ud_proc = ud_proc + def __str__(self): + return type_utils.obj_name(self) + def get_userdata(self, apply_filter=False): if self.userdata is None: self.userdata = self.ud_proc.process(self.get_userdata_raw()) @@ -214,7 +218,7 @@ def normalize_pubkey_data(pubkey_data): def find_source(sys_cfg, distro, paths, ds_deps, cfg_list, pkg_list): ds_list = list_sources(cfg_list, ds_deps, pkg_list) - ds_names = [util.obj_name(f) for f in ds_list] + ds_names = [type_utils.obj_name(f) for f in ds_list] LOG.debug("Searching for data source in: %s", ds_names) for cls in ds_list: @@ -222,7 +226,7 @@ def find_source(sys_cfg, distro, paths, ds_deps, cfg_list, pkg_list): LOG.debug("Seeing if we can get any data from %s", cls) s = cls(sys_cfg, distro, paths) if s.get_data(): - return (s, util.obj_name(cls)) + return (s, type_utils.obj_name(cls)) except Exception: util.logexc(LOG, "Getting data from %s failed", cls) diff --git a/cloudinit/stages.py b/cloudinit/stages.py index 94a267df..531e7997 100644 --- a/cloudinit/stages.py +++ b/cloudinit/stages.py @@ -43,6 +43,7 @@ from cloudinit import helpers from cloudinit import importer from cloudinit import log as logging from cloudinit import sources +from cloudinit import type_utils from cloudinit import util LOG = logging.getLogger(__name__) @@ -220,7 +221,7 @@ class Init(object): # Any config provided??? pkg_list = self.cfg.get('datasource_pkg_list') or [] # Add the defaults at the end - for n in ['', util.obj_name(sources)]: + for n in ['', type_utils.obj_name(sources)]: if n not in pkg_list: pkg_list.append(n) cfg_list = self.cfg.get('datasource_list') or [] @@ -280,7 +281,7 @@ class Init(object): dp = self.paths.get_cpath('data') # Write what the datasource was and is.. - ds = "%s: %s" % (util.obj_name(self.datasource), self.datasource) + ds = "%s: %s" % (type_utils.obj_name(self.datasource), self.datasource) previous_ds = None ds_fn = os.path.join(idir, 'datasource') try: @@ -497,7 +498,7 @@ class Modules(object): else: raise TypeError(("Failed to read '%s' item in config," " unknown type %s") % - (item, util.obj_name(item))) + (item, type_utils.obj_name(item))) return module_list def _fixup_modules(self, raw_mods): @@ -515,7 +516,7 @@ class Modules(object): # Reset it so when ran it will get set to a known value freq = None mod_locs = importer.find_module(mod_name, - ['', util.obj_name(config)], + ['', type_utils.obj_name(config)], ['handle']) if not mod_locs: LOG.warn("Could not find module named %s", mod_name) diff --git a/cloudinit/type_utils.py b/cloudinit/type_utils.py new file mode 100644 index 00000000..2decbfc5 --- /dev/null +++ b/cloudinit/type_utils.py @@ -0,0 +1,34 @@ +# vi: ts=4 expandtab +# +# Copyright (C) 2012 Canonical Ltd. +# Copyright (C) 2012 Hewlett-Packard Development Company, L.P. +# Copyright (C) 2012 Yahoo! Inc. +# +# Author: Scott Moser +# Author: Juerg Haefliger +# Author: Joshua Harlow +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 3, as +# published by the Free Software Foundation. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +# +# pylint: disable=C0302 + +import types + + +def obj_name(obj): + if isinstance(obj, (types.TypeType, + types.ModuleType, + types.FunctionType, + types.LambdaType)): + return str(obj.__name__) + return obj_name(obj.__class__) diff --git a/cloudinit/util.py b/cloudinit/util.py index ab918433..73bf6304 100644 --- a/cloudinit/util.py +++ b/cloudinit/util.py @@ -43,14 +43,15 @@ import subprocess import sys import tempfile import time -import types import urlparse import yaml from cloudinit import importer from cloudinit import log as logging +from cloudinit import mergers from cloudinit import safeyaml +from cloudinit import type_utils from cloudinit import url_helper as uhelp from cloudinit import version @@ -194,11 +195,12 @@ def fork_cb(child_cb, *args): os._exit(0) # pylint: disable=W0212 except: logexc(LOG, ("Failed forking and" - " calling callback %s"), obj_name(child_cb)) + " calling callback %s"), + type_utils.obj_name(child_cb)) os._exit(1) # pylint: disable=W0212 else: LOG.debug("Forked child %s who will run callback %s", - fid, obj_name(child_cb)) + fid, type_utils.obj_name(child_cb)) def is_true(val, addons=None): @@ -513,15 +515,6 @@ def make_url(scheme, host, port=None, return urlparse.urlunparse(pieces) -def obj_name(obj): - if isinstance(obj, (types.TypeType, - types.ModuleType, - types.FunctionType, - types.LambdaType)): - return str(obj.__name__) - return obj_name(obj.__class__) - - def mergemanydict(srcs, reverse=False): if reverse: srcs = reversed(srcs) @@ -538,13 +531,9 @@ def mergedict(src, cand): If C{src} has a key C{cand} will not override. Nested dictionaries are merged recursively. """ - if isinstance(src, dict) and isinstance(cand, dict): - for (k, v) in cand.iteritems(): - if k not in src: - src[k] = v - else: - src[k] = mergedict(src[k], v) - return src + raw_mergers = mergers.default_mergers() + merger = mergers.construct(raw_mergers) + return merger.merge(src, cand) @contextlib.contextmanager @@ -645,7 +634,7 @@ def load_yaml(blob, default=None, allowed=(dict,)): # Yes this will just be caught, but thats ok for now... raise TypeError(("Yaml load allows %s root types," " but got %s instead") % - (allowed, obj_name(converted))) + (allowed, type_utils.obj_name(converted))) loaded = converted except (yaml.YAMLError, TypeError, ValueError): if len(blob) == 0: @@ -714,7 +703,7 @@ def read_conf_with_confd(cfgfile): if not isinstance(confd, (str, basestring)): raise TypeError(("Config file %s contains 'conf_d' " "with non-string type %s") % - (cfgfile, obj_name(confd))) + (cfgfile, type_utils.obj_name(confd))) else: confd = str(confd).strip() elif os.path.isdir("%s.d" % cfgfile): @@ -1472,7 +1461,7 @@ def shellify(cmdlist, add_header=True): else: raise RuntimeError(("Unable to shellify type %s" " which is not a list or string") - % (obj_name(args))) + % (type_utils.obj_name(args))) LOG.debug("Shellified %s commands.", cmds_made) return content diff --git a/tests/unittests/test_userdata.py b/tests/unittests/test_userdata.py index 9e1fed7e..ef0dd7b8 100644 --- a/tests/unittests/test_userdata.py +++ b/tests/unittests/test_userdata.py @@ -74,7 +74,7 @@ run: - morestuff ''' message2 = MIMEBase("text", "cloud-config") - message2['Merge-Type'] = 'dict()+list(extend)+str()' + message2['X-Merge-Type'] = 'dict()+list(extend)+str()' message2.set_payload(blob2) blob3 = ''' @@ -83,6 +83,7 @@ e: - 1 - 2 - 3 +p: 1 ''' message3 = MIMEBase("text", "cloud-config") message3['Merge-Type'] = 'dict()+list()+str()' @@ -109,6 +110,7 @@ e: self.assertEquals(contents['run'], ['b', 'c', 'stuff', 'morestuff']) self.assertEquals(contents['a'], 'be') self.assertEquals(contents['e'], 'fg') + self.assertEquals(contents['p'], 1) def test_unhandled_type_warning(self): """Raw text without magic is ignored but shows warning.""" -- cgit v1.2.3 From 1e4f41e900a9c942354428b0f312428af00031ce Mon Sep 17 00:00:00 2001 From: Joshua Harlow Date: Wed, 6 Mar 2013 19:36:31 -0800 Subject: Make conf.d and the default merging use the new merging algos. --- cloudinit/sources/DataSourceConfigDrive.py | 2 +- cloudinit/sources/DataSourceNoCloud.py | 8 ++++---- cloudinit/sources/DataSourceOVF.py | 4 ++-- cloudinit/util.py | 25 ++++++++++++------------- 4 files changed, 19 insertions(+), 20 deletions(-) (limited to 'cloudinit/sources') diff --git a/cloudinit/sources/DataSourceConfigDrive.py b/cloudinit/sources/DataSourceConfigDrive.py index 46abd772..0216ed07 100644 --- a/cloudinit/sources/DataSourceConfigDrive.py +++ b/cloudinit/sources/DataSourceConfigDrive.py @@ -154,7 +154,7 @@ class DataSourceConfigDrive(sources.DataSource): return False md = results['metadata'] - md = util.mergedict(md, DEFAULT_METADATA) + md = util.mergemanydict([md, DEFAULT_METADATA]) # Perform some metadata 'fixups' # diff --git a/cloudinit/sources/DataSourceNoCloud.py b/cloudinit/sources/DataSourceNoCloud.py index 9a770d38..7800812b 100644 --- a/cloudinit/sources/DataSourceNoCloud.py +++ b/cloudinit/sources/DataSourceNoCloud.py @@ -64,7 +64,7 @@ class DataSourceNoCloud(sources.DataSource): # Check to see if the seed dir has data. seedret = {} if util.read_optional_seed(seedret, base=self.seed_dir + "/"): - md = util.mergedict(md, seedret['meta-data']) + md = util.mergemanydict([md, seedret['meta-data']]) ud = seedret['user-data'] found.append(self.seed_dir) LOG.debug("Using seeded cache data from %s", self.seed_dir) @@ -88,7 +88,7 @@ class DataSourceNoCloud(sources.DataSource): LOG.debug("Attempting to use data from %s", dev) (newmd, newud) = util.mount_cb(dev, util.read_seeded) - md = util.mergedict(newmd, md) + md = util.mergemanydict([newmd, md]) ud = newud # For seed from a device, the default mode is 'net'. @@ -139,11 +139,11 @@ class DataSourceNoCloud(sources.DataSource): LOG.debug("Using seeded cache data from %s", seedfrom) # Values in the command line override those from the seed - md = util.mergedict(md, md_seed) + md = util.mergemanydict([md, md_seed]) found.append(seedfrom) # Now that we have exhausted any other places merge in the defaults - md = util.mergedict(md, defaults) + md = util.mergemanydict([md, defaults]) # Update the network-interfaces if metadata had 'network-interfaces' # entry and this is the local datasource, or 'seedfrom' was used diff --git a/cloudinit/sources/DataSourceOVF.py b/cloudinit/sources/DataSourceOVF.py index ae139074..0530c4b7 100644 --- a/cloudinit/sources/DataSourceOVF.py +++ b/cloudinit/sources/DataSourceOVF.py @@ -94,11 +94,11 @@ class DataSourceOVF(sources.DataSource): (md_seed, ud) = util.read_seeded(seedfrom, timeout=None) LOG.debug("Using seeded cache data from %s", seedfrom) - md = util.mergedict(md, md_seed) + md = util.mergemanydict([md, md_seed]) found.append(seedfrom) # Now that we have exhausted any other places merge in the defaults - md = util.mergedict(md, defaults) + md = util.mergemanydict([md, defaults]) self.seed = ",".join(found) self.metadata = md diff --git a/cloudinit/util.py b/cloudinit/util.py index 73bf6304..e5c6f4ea 100644 --- a/cloudinit/util.py +++ b/cloudinit/util.py @@ -519,23 +519,22 @@ def mergemanydict(srcs, reverse=False): if reverse: srcs = reversed(srcs) m_cfg = {} + merge_how = [mergers.default_mergers()] for a_cfg in srcs: if a_cfg: - m_cfg = mergedict(m_cfg, a_cfg) + # Take the last merger as the one that + # will define how to merge next... + mergers_to_apply = list(merge_how[-1]) + merger = mergers.construct(mergers_to_apply) + m_cfg = merger.merge(m_cfg, a_cfg) + # If the config has now has new merger set, + # extract them to be used next time... + new_mergers = mergers.dict_extract_mergers(m_cfg) + if new_mergers: + merge_how.append(new_mergers) return m_cfg -def mergedict(src, cand): - """ - Merge values from C{cand} into C{src}. - If C{src} has a key C{cand} will not override. - Nested dictionaries are merged recursively. - """ - raw_mergers = mergers.default_mergers() - merger = mergers.construct(raw_mergers) - return merger.merge(src, cand) - - @contextlib.contextmanager def chdir(ndir): curr = os.getcwd() @@ -714,7 +713,7 @@ def read_conf_with_confd(cfgfile): # Conf.d settings override input configuration confd_cfg = read_conf_d(confd) - return mergedict(confd_cfg, cfg) + return mergemanydict([confd_cfg, cfg]) def read_cc_from_cmdline(cmdline=None): -- cgit v1.2.3 From aae7fe638f61aaf02c6579d5b691a8641455c875 Mon Sep 17 00:00:00 2001 From: Scott Moser Date: Thu, 7 Mar 2013 16:47:54 -0500 Subject: fix pep8 and pylint --- cloudinit/config/cc_landscape.py | 3 ++- cloudinit/distros/__init__.py | 4 ++-- cloudinit/mergers/__init__.py | 5 +++-- cloudinit/mergers/str.py | 2 +- cloudinit/sources/DataSourceNone.py | 1 - tests/unittests/test__init__.py | 2 -- tests/unittests/test_merging.py | 8 ++++---- tests/unittests/test_userdata.py | 39 +++++++++++++++++++++---------------- 8 files changed, 34 insertions(+), 30 deletions(-) (limited to 'cloudinit/sources') diff --git a/cloudinit/config/cc_landscape.py b/cloudinit/config/cc_landscape.py index 47c10a97..8a709677 100644 --- a/cloudinit/config/cc_landscape.py +++ b/cloudinit/config/cc_landscape.py @@ -59,7 +59,8 @@ def handle(_name, cfg, cloud, log, _args): if not isinstance(ls_cloudcfg, (dict)): raise RuntimeError(("'landscape' key existed in config," " but not a dictionary type," - " is a %s instead"), type_utils.obj_name(ls_cloudcfg)) + " is a %s instead"), + type_utils.obj_name(ls_cloudcfg)) if not ls_cloudcfg: return diff --git a/cloudinit/distros/__init__.py b/cloudinit/distros/__init__.py index 7b6276c5..50d52594 100644 --- a/cloudinit/distros/__init__.py +++ b/cloudinit/distros/__init__.py @@ -741,7 +741,7 @@ def normalize_users_groups(cfg, distro): } if not isinstance(old_user, (dict)): LOG.warn(("Format for 'user' key must be a string or " - "dictionary and not %s"), util.obj_name(old_user)) + "dictionary and not %s"), type_utils.obj_name(old_user)) old_user = {} # If no old user format, then assume the distro @@ -767,7 +767,7 @@ def normalize_users_groups(cfg, distro): if not isinstance(base_users, (list, dict, str, basestring)): LOG.warn(("Format for 'users' key must be a comma separated string" " or a dictionary or a list and not %s"), - util.obj_name(base_users)) + type_utils.obj_name(base_users)) base_users = [] if old_user: diff --git a/cloudinit/mergers/__init__.py b/cloudinit/mergers/__init__.py index 3b56686f..ac16f143 100644 --- a/cloudinit/mergers/__init__.py +++ b/cloudinit/mergers/__init__.py @@ -32,7 +32,7 @@ class UnknownMerger(object): # Named differently so auto-method finding # doesn't pick this up if there is ever a type # named "unknown" - def _handle_unknown(self, meth_wanted, value, merge_with): + def _handle_unknown(self, _meth_wanted, value, _merge_with): return value # This merging will attempt to look for a '_on_X' method @@ -119,7 +119,8 @@ def string_extract_mergers(merge_how): continue match = NAME_MTCH.match(m_name) if not match: - msg = "Matcher identifer '%s' is not in the right format" % (m_name) + msg = ("Matcher identifer '%s' is not in the right format" % + (m_name)) raise ValueError(msg) (m_name, m_ops) = match.groups() m_ops = m_ops.strip().split(",") diff --git a/cloudinit/mergers/str.py b/cloudinit/mergers/str.py index f1534c5b..291c91c2 100644 --- a/cloudinit/mergers/str.py +++ b/cloudinit/mergers/str.py @@ -18,7 +18,7 @@ class Merger(object): - def __init__(self, merger, opts): + def __init__(self, _merger, opts): self._append = 'append' in opts # On encountering a unicode object to merge value with diff --git a/cloudinit/sources/DataSourceNone.py b/cloudinit/sources/DataSourceNone.py index e2175e1f..12a8a992 100644 --- a/cloudinit/sources/DataSourceNone.py +++ b/cloudinit/sources/DataSourceNone.py @@ -18,7 +18,6 @@ from cloudinit import log as logging from cloudinit import sources -from cloudinit import util LOG = logging.getLogger(__name__) diff --git a/tests/unittests/test__init__.py b/tests/unittests/test__init__.py index 7924755a..56ccbcfb 100644 --- a/tests/unittests/test__init__.py +++ b/tests/unittests/test__init__.py @@ -24,8 +24,6 @@ class FakeModule(handlers.Handler): def handle_part(self, data, ctype, filename, payload, frequency): pass - - class TestWalkerHandleHandler(MockerTestCase): diff --git a/tests/unittests/test_merging.py b/tests/unittests/test_merging.py index fa7ee8e4..591a99c8 100644 --- a/tests/unittests/test_merging.py +++ b/tests/unittests/test_merging.py @@ -1,5 +1,3 @@ -import os - from tests.unittests import helpers from cloudinit import mergers @@ -107,8 +105,10 @@ class TestSimpleRun(helpers.MockerTestCase): self.assertEquals(merged['a'], [1, 'b', 2, 'e', 'f', 'g']) self.assertEquals(merged['b'], 'blahblahmore') self.assertEquals(merged['c']['f'], 'bigblobofstuff') - self.assertEquals(merged['run'], ['runme', 'runme2', 'morecmd', 'moremoremore']) - self.assertEquals(merged['runmereally'], ['e', ['a'], 'd', 'blah', ['b'], 'e']) + self.assertEquals(merged['run'], ['runme', 'runme2', 'morecmd', + 'moremoremore']) + self.assertEquals(merged['runmereally'], ['e', ['a'], 'd', 'blah', + ['b'], 'e']) def test_dict_overwrite_layered(self): source = { diff --git a/tests/unittests/test_userdata.py b/tests/unittests/test_userdata.py index ef0dd7b8..48ad9c5f 100644 --- a/tests/unittests/test_userdata.py +++ b/tests/unittests/test_userdata.py @@ -7,8 +7,6 @@ import os from email.mime.base import MIMEBase -from mocker import MockerTestCase - from cloudinit import handlers from cloudinit import helpers as c_helpers from cloudinit import log @@ -97,14 +95,16 @@ p: 1 new_root = self.makeDir() self.patchUtils(new_root) self.patchOS(new_root) - cloud_cfg.handle_part(None, handlers.CONTENT_START, None, None, None, None) + cloud_cfg.handle_part(None, handlers.CONTENT_START, None, None, None, + None) for i, m in enumerate(messages): headers = dict(m) fn = "part-%s" % (i + 1) payload = m.get_payload(decode=True) cloud_cfg.handle_part(None, headers['Content-Type'], fn, payload, None, headers) - cloud_cfg.handle_part(None, handlers.CONTENT_END, None, None, None, None) + cloud_cfg.handle_part(None, handlers.CONTENT_END, None, None, None, + None) contents = util.load_file(paths.get_ipath('cloud_config')) contents = util.load_yaml(contents) self.assertEquals(contents['run'], ['b', 'c', 'stuff', 'morestuff']) @@ -118,8 +118,9 @@ p: 1 data = "arbitrary text\n" ci.datasource = FakeDataSource(data) - self.mock_write = self.mocker.replace("cloudinit.util.write_file", passthrough=False) - self.mock_write(ci.paths.get_ipath("cloud_config"), "", 0600) + mock_write = self.mocker.replace("cloudinit.util.write_file", + passthrough=False) + mock_write(ci.paths.get_ipath("cloud_config"), "", 0600) self.mocker.replay() log_file = self.capture_log(logging.WARNING) @@ -136,8 +137,9 @@ p: 1 message.set_payload("Just text") ci.datasource = FakeDataSource(message.as_string()) - self.mock_write = self.mocker.replace("cloudinit.util.write_file", passthrough=False) - self.mock_write(ci.paths.get_ipath("cloud_config"), "", 0600) + mock_write = self.mocker.replace("cloudinit.util.write_file", + passthrough=False) + mock_write(ci.paths.get_ipath("cloud_config"), "", 0600) self.mocker.replay() log_file = self.capture_log(logging.WARNING) @@ -154,9 +156,10 @@ p: 1 ci.datasource = FakeDataSource(script) outpath = os.path.join(ci.paths.get_ipath_cur("scripts"), "part-001") - self.mock_write = self.mocker.replace("cloudinit.util.write_file", passthrough=False) - self.mock_write(ci.paths.get_ipath("cloud_config"), "", 0600) - self.mock_write(outpath, script, 0700) + mock_write = self.mocker.replace("cloudinit.util.write_file", + passthrough=False) + mock_write(ci.paths.get_ipath("cloud_config"), "", 0600) + mock_write(outpath, script, 0700) self.mocker.replay() log_file = self.capture_log(logging.WARNING) @@ -173,9 +176,10 @@ p: 1 ci.datasource = FakeDataSource(message.as_string()) outpath = os.path.join(ci.paths.get_ipath_cur("scripts"), "part-001") - self.mock_write = self.mocker.replace("cloudinit.util.write_file", passthrough=False) - self.mock_write(ci.paths.get_ipath("cloud_config"), "", 0600) - self.mock_write(outpath, script, 0700) + mock_write = self.mocker.replace("cloudinit.util.write_file", + passthrough=False) + mock_write(ci.paths.get_ipath("cloud_config"), "", 0600) + mock_write(outpath, script, 0700) self.mocker.replay() log_file = self.capture_log(logging.WARNING) @@ -192,9 +196,10 @@ p: 1 ci.datasource = FakeDataSource(message.as_string()) outpath = os.path.join(ci.paths.get_ipath_cur("scripts"), "part-001") - self.mock_write = self.mocker.replace("cloudinit.util.write_file", passthrough=False) - self.mock_write(outpath, script, 0700) - self.mock_write(ci.paths.get_ipath("cloud_config"), "", 0600) + mock_write = self.mocker.replace("cloudinit.util.write_file", + passthrough=False) + mock_write(outpath, script, 0700) + mock_write(ci.paths.get_ipath("cloud_config"), "", 0600) self.mocker.replay() log_file = self.capture_log(logging.WARNING) -- cgit v1.2.3 From 20a2d9961697fbd6ef0e74cd3770b6601b141bcd Mon Sep 17 00:00:00 2001 From: Joshua Harlow Date: Tue, 19 Mar 2013 15:53:16 -0700 Subject: Move back to using boto for now. --- Requires | 3 + cloudinit/ec2_utils.py | 183 +++++++----------------------- cloudinit/sources/DataSourceCloudStack.py | 7 +- cloudinit/sources/DataSourceEc2.py | 7 +- 4 files changed, 48 insertions(+), 152 deletions(-) (limited to 'cloudinit/sources') diff --git a/Requires b/Requires index 0313d569..de51a4e4 100644 --- a/Requires +++ b/Requires @@ -24,3 +24,6 @@ argparse # Requests handles ssl correctly! requests + +# Boto for ec2 +boto diff --git a/cloudinit/ec2_utils.py b/cloudinit/ec2_utils.py index 71c84206..29393ce1 100644 --- a/cloudinit/ec2_utils.py +++ b/cloudinit/ec2_utils.py @@ -16,146 +16,45 @@ # You should have received a copy of the GNU General Public License # along with this program. If not, see . -from urlparse import (urlparse, urlunparse) +import boto.utils as boto_utils + +# Versions of boto >= 2.6.0 (and possibly 2.5.2) +# try to lazily load the metadata backing, which +# doesn't work so well in cloud-init especially +# since the metadata is serialized and actions are +# performed where the metadata server may be blocked +# (thus the datasource will start failing) resulting +# in url exceptions when fields that do exist (or +# would have existed) do not exist due to the blocking +# that occurred. + + +def _unlazy_dict(mp): + if not isinstance(mp, (dict)): + return mp + # Walk over the keys/values which + # forces boto to unlazy itself and + # has no effect on dictionaries that + # already have there items. + for (_k, v) in mp.items(): + _unlazy_dict(v) + return mp + + +def get_instance_userdata(api_version, metadata_address): + # Note: boto.utils.get_instance_metadata returns '' for empty string + # so the change from non-true to '' is not specifically necessary, but + # this way cloud-init will get consistent behavior even if boto changed + # in the future to return a None on "no user-data provided". + ud = boto_utils.get_instance_userdata(api_version, None, metadata_address) + if not ud: + ud = '' + return ud + + +def get_instance_metadata(api_version, metadata_address): + metadata = boto_utils.get_instance_metadata(api_version, metadata_address) + if not isinstance(metadata, (dict)): + metadata = {} + return _unlazy_dict(metadata) -import json -import urllib - -from cloudinit import log as logging -from cloudinit import util - -LOG = logging.getLogger(__name__) - - -def combine_url(base, add_on): - base_parsed = list(urlparse(base)) - path = base_parsed[2] - if path and not path.endswith("/"): - path += "/" - path += urllib.quote(str(add_on), safe="/:") - base_parsed[2] = path - return urlunparse(base_parsed) - - -# See: http://bit.ly/TyoUQs -# -# Since boto metadata reader uses the old urllib which does not -# support ssl, we need to ahead and create our own reader which -# works the same as the boto one (for now). -class MetadataMaterializer(object): - def __init__(self, blob, base_url, ssl_details): - self._blob = blob - self._md = None - self._base_url = base_url - self._ssl_details = ssl_details - - def _parse(self, blob): - leaves = {} - children = [] - if not blob: - return (leaves, children) - - def has_children(item): - if item.endswith("/"): - return True - else: - return False - - def get_name(item): - if item.endswith("/"): - return item.rstrip("/") - return item - - for field in blob.splitlines(): - field = field.strip() - field_name = get_name(field) - if not field or not field_name: - continue - if has_children(field): - if field_name not in children: - children.append(field_name) - else: - contents = field.split("=", 1) - resource = field_name - if len(contents) > 1: - # What a PITA... - (ident, sub_contents) = contents - checked_ident = util.safe_int(ident) - if checked_ident is not None: - resource = "%s/openssh-key" % (checked_ident) - field_name = sub_contents - leaves[field_name] = resource - return (leaves, children) - - def materialize(self): - if self._md is not None: - return self._md - self._md = self._materialize(self._blob, self._base_url) - return self._md - - def _fetch_url(self, url): - response = util.read_file_or_url(url, ssl_details=self._ssl_details) - return str(response) - - def _decode_leaf_blob(self, blob): - if not blob: - return blob - stripped_blob = blob.strip() - if stripped_blob.startswith("{") and stripped_blob.endswith("}"): - # Assume and try with json - try: - return json.loads(blob) - except (ValueError, TypeError): - pass - if blob.find("\n") != -1: - return blob.splitlines() - return blob - - def _materialize(self, blob, base_url): - (leaves, children) = self._parse(blob) - child_contents = {} - for c in children: - child_url = combine_url(base_url, c) - if not child_url.endswith("/"): - child_url += "/" - child_blob = self._fetch_url(child_url) - child_contents[c] = self._materialize(child_blob, child_url) - leaf_contents = {} - for (field, resource) in leaves.items(): - leaf_url = combine_url(base_url, resource) - leaf_blob = self._fetch_url(leaf_url) - leaf_contents[field] = self._decode_leaf_blob(leaf_blob) - joined = {} - joined.update(child_contents) - for field in leaf_contents.keys(): - if field in joined: - LOG.warn("Duplicate key found in results from %s", base_url) - else: - joined[field] = leaf_contents[field] - return joined - - -def get_instance_userdata(version='latest', url='http://169.254.169.254', - ssl_details=None): - ud_url = combine_url(url, version) - ud_url = combine_url(ud_url, 'user-data') - try: - response = util.read_file_or_url(ud_url, ssl_details=ssl_details) - return str(response) - except Exception: - util.logexc(LOG, "Failed fetching userdata from url %s", ud_url) - return None - - -def get_instance_metadata(version='latest', url='http://169.254.169.254', - ssl_details=None, timeout=5, num_retries=5): - md_url = combine_url(metadata_address, version) - md_url = combine_url(md_url, 'meta-data') - try: - response = util.read_file_or_url(md_url, ssl_details=ssl_details, - timeout=timeout, retries=num_retries) - materializer = MetadataMaterializer(str(response), md_url, ssl_details) - return materializer.materialize() - except Exception: - util.logexc(LOG, "Failed fetching metadata from url %s", md_url) - return None diff --git a/cloudinit/sources/DataSourceCloudStack.py b/cloudinit/sources/DataSourceCloudStack.py index 7fd38982..b4ca6d93 100644 --- a/cloudinit/sources/DataSourceCloudStack.py +++ b/cloudinit/sources/DataSourceCloudStack.py @@ -102,13 +102,10 @@ class DataSourceCloudStack(sources.DataSource): return False start_time = time.time() md_addr = self.metadata_address - ssl_details = util.fetch_ssl_details(self.paths) self.userdata_raw = ec2_utils.get_instance_userdata(self.api_ver, - md_addr, - ssl_details) + md_addr) self.metadata = ec2_utils.get_instance_metadata(self.api_ver, - md_addr, - ssl_details) + md_addr) LOG.debug("Crawl of metadata service took %s seconds", int(time.time() - start_time)) return True diff --git a/cloudinit/sources/DataSourceEc2.py b/cloudinit/sources/DataSourceEc2.py index 0d62cf01..bd35c8b0 100644 --- a/cloudinit/sources/DataSourceEc2.py +++ b/cloudinit/sources/DataSourceEc2.py @@ -62,13 +62,10 @@ class DataSourceEc2(sources.DataSource): return False start_time = time.time() md_addr = self.metadata_address - ssl_details = util.fetch_ssl_details(self.paths) self.userdata_raw = ec2_utils.get_instance_userdata(self.api_ver, - md_addr, - ssl_details) + md_addr) self.metadata = ec2_utils.get_instance_metadata(self.api_ver, - md_addr, - ssl_details) + md_addr) LOG.debug("Crawl of metadata service took %s seconds", int(time.time() - start_time)) return True -- cgit v1.2.3 From c49b92e2e1f32cbb32c856ba246ef97026318dbf Mon Sep 17 00:00:00 2001 From: Scott Moser Date: Wed, 20 Mar 2013 08:30:43 -0400 Subject: remove some churn cloudinit/sources/DataSourceCloudStack.py and cloudinit/sources/DataSourceEc2.py are reverted entirely back to trunk versions now, rather than the non-behavior change that was left in place. Also, remove inadvertantly added trailing newline from cloudinit/ec2_utils.py Overall, this just makes the diff when merged to trunk carry more focused changes. --- cloudinit/ec2_utils.py | 1 - cloudinit/sources/DataSourceCloudStack.py | 11 +++++------ cloudinit/sources/DataSourceEc2.py | 13 ++++++------- 3 files changed, 11 insertions(+), 14 deletions(-) (limited to 'cloudinit/sources') diff --git a/cloudinit/ec2_utils.py b/cloudinit/ec2_utils.py index 175f96aa..5fa8c5ad 100644 --- a/cloudinit/ec2_utils.py +++ b/cloudinit/ec2_utils.py @@ -61,4 +61,3 @@ def get_instance_metadata(api_version, metadata_address): if not isinstance(metadata, (dict)): metadata = {} return _unlazy_dict(metadata) - diff --git a/cloudinit/sources/DataSourceCloudStack.py b/cloudinit/sources/DataSourceCloudStack.py index b4ca6d93..81c8cda9 100644 --- a/cloudinit/sources/DataSourceCloudStack.py +++ b/cloudinit/sources/DataSourceCloudStack.py @@ -25,7 +25,7 @@ import os import time -from cloudinit import ec2_utils +from cloudinit import ec2_utils as ec2 from cloudinit import log as logging from cloudinit import sources from cloudinit import url_helper as uhelp @@ -101,11 +101,10 @@ class DataSourceCloudStack(sources.DataSource): if not self.wait_for_metadata_service(): return False start_time = time.time() - md_addr = self.metadata_address - self.userdata_raw = ec2_utils.get_instance_userdata(self.api_ver, - md_addr) - self.metadata = ec2_utils.get_instance_metadata(self.api_ver, - md_addr) + self.userdata_raw = ec2.get_instance_userdata(self.api_ver, + self.metadata_address) + self.metadata = ec2.get_instance_metadata(self.api_ver, + self.metadata_address) LOG.debug("Crawl of metadata service took %s seconds", int(time.time() - start_time)) return True diff --git a/cloudinit/sources/DataSourceEc2.py b/cloudinit/sources/DataSourceEc2.py index bd35c8b0..f010e640 100644 --- a/cloudinit/sources/DataSourceEc2.py +++ b/cloudinit/sources/DataSourceEc2.py @@ -23,7 +23,7 @@ import os import time -from cloudinit import ec2_utils +from cloudinit import ec2_utils as ec2 from cloudinit import log as logging from cloudinit import sources from cloudinit import url_helper as uhelp @@ -61,11 +61,10 @@ class DataSourceEc2(sources.DataSource): if not self.wait_for_metadata_service(): return False start_time = time.time() - md_addr = self.metadata_address - self.userdata_raw = ec2_utils.get_instance_userdata(self.api_ver, - md_addr) - self.metadata = ec2_utils.get_instance_metadata(self.api_ver, - md_addr) + self.userdata_raw = ec2.get_instance_userdata(self.api_ver, + self.metadata_address) + self.metadata = ec2.get_instance_metadata(self.api_ver, + self.metadata_address) LOG.debug("Crawl of metadata service took %s seconds", int(time.time() - start_time)) return True @@ -134,7 +133,7 @@ class DataSourceEc2(sources.DataSource): start_time = time.time() url = uhelp.wait_for_url(urls=urls, max_wait=max_wait, - timeout=timeout, status_cb=LOG.warn) + timeout=timeout, status_cb=LOG.warn) if url: LOG.debug("Using metadata source: '%s'", url2base[url]) -- cgit v1.2.3 From 984c72e522c585c6d3f6b3d3aec39fb21dd84028 Mon Sep 17 00:00:00 2001 From: Greg Padgett Date: Tue, 26 Mar 2013 17:48:40 -0400 Subject: compatibility fixes for Fedora and RHEL This patch fixes issues in Fedora 18 (and upcoming RHEL 7) which are present due to their use of systemd: - store locale configuration in /etc/locale.conf - store hostname in /etc/hostname - use a symlink for /etc/localtime (prior code would set the timezone but corrupt data in /usr/share/zoneinfo due to presence of symlink) It also contains fixes for issues unrelated to systemd adoption: - explicitly scan /dev/sr0 with blkid in order to get the optical drive in the blkid cache. This prevents an issue on systems running 2.6 kernels (such as RHEL 6) in which config disks on some devices won't be detected unless the device has previously been queried. (For reference, see https://patchwork.kernel.org/patch/1770241/) - append a newline when rewriting sysconfig files, as this is customary text configuration file formatting and is expected by some parsers (such as the ifcfg-rh plugin for NetworkManager) --- cloudinit/distros/rhel.py | 73 ++++++++++++++++------ cloudinit/sources/DataSourceConfigDrive.py | 3 + cloudinit/sources/DataSourceNoCloud.py | 3 + cloudinit/util.py | 1 + .../unittests/test_datasource/test_configdrive.py | 5 +- 5 files changed, 63 insertions(+), 22 deletions(-) (limited to 'cloudinit/sources') diff --git a/cloudinit/distros/rhel.py b/cloudinit/distros/rhel.py index 9fee5fd1..174da3ab 100644 --- a/cloudinit/distros/rhel.py +++ b/cloudinit/distros/rhel.py @@ -47,8 +47,10 @@ class Distro(distros.Distro): # See: http://tiny.cc/6r99fw clock_conf_fn = "/etc/sysconfig/clock" locale_conf_fn = '/etc/sysconfig/i18n' + systemd_locale_conf_fn = '/etc/locale.conf' network_conf_fn = "/etc/sysconfig/network" hostname_conf_fn = "/etc/sysconfig/network" + systemd_hostname_conf_fn = "/etc/hostname" network_script_tpl = '/etc/sysconfig/network-scripts/ifcfg-%s' resolve_conf_fn = "/etc/resolv.conf" tz_local_fn = "/etc/localtime" @@ -143,21 +145,36 @@ class Distro(distros.Distro): ] if not exists: lines.insert(0, util.make_header()) - util.write_file(fn, "\n".join(lines), 0644) + util.write_file(fn, "\n".join(lines) + "\n", 0644) + + def _dist_uses_systemd(self): + # Fedora 18 and RHEL 7 were the first adopters in their series + (dist, vers) = util.system_info()['dist'][:2] + major = (int)(vers.split('.')[0]) + return ((dist.startswith('Red Hat Enterprise Linux') and major >= 7) + or (dist.startswith('Fedora') and major >= 18)) def apply_locale(self, locale, out_fn=None): - if not out_fn: - out_fn = self.locale_conf_fn + if self._dist_uses_systemd(): + if not out_fn: + out_fn = self.systemd_locale_conf_fn + out_fn = self.systemd_locale_conf_fn + else: + if not out_fn: + out_fn = self.locale_conf_fn locale_cfg = { 'LANG': locale, } self._update_sysconfig_file(out_fn, locale_cfg) def _write_hostname(self, hostname, out_fn): - host_cfg = { - 'HOSTNAME': hostname, - } - self._update_sysconfig_file(out_fn, host_cfg) + if self._dist_uses_systemd(): + util.subp(['hostnamectl', 'set-hostname', str(hostname)]) + else: + host_cfg = { + 'HOSTNAME': hostname, + } + self._update_sysconfig_file(out_fn, host_cfg) def _select_hostname(self, hostname, fqdn): # See: http://bit.ly/TwitgL @@ -167,15 +184,25 @@ class Distro(distros.Distro): return hostname def _read_system_hostname(self): - return (self.network_conf_fn, - self._read_hostname(self.network_conf_fn)) + if self._dist_uses_systemd(): + host_fn = self.systemd_hostname_conf_fn + else: + host_fn = self.hostname_conf_fn + return (host_fn, self._read_hostname(host_fn)) def _read_hostname(self, filename, default=None): - (_exists, contents) = self._read_conf(filename) - if 'HOSTNAME' in contents: - return contents['HOSTNAME'] + if self._dist_uses_systemd(): + (out, _err) = util.subp(['hostname']) + if len(out): + return out + else: + return default else: - return default + (_exists, contents) = self._read_conf(filename) + if 'HOSTNAME' in contents: + return contents['HOSTNAME'] + else: + return default def _read_conf(self, fn): exists = False @@ -200,13 +227,19 @@ class Distro(distros.Distro): if not os.path.isfile(tz_file): raise RuntimeError(("Invalid timezone %s," " no file found at %s") % (tz, tz_file)) - # Adjust the sysconfig clock zone setting - clock_cfg = { - 'ZONE': str(tz), - } - self._update_sysconfig_file(self.clock_conf_fn, clock_cfg) - # This ensures that the correct tz will be used for the system - util.copy(tz_file, self.tz_local_fn) + if self._dist_uses_systemd(): + # Currently, timedatectl complains if invoked during startup + # so for compatibility, create the link manually. + util.del_file(self.tz_local_fn) + util.sym_link(tz_file, self.tz_local_fn) + else: + # Adjust the sysconfig clock zone setting + clock_cfg = { + 'ZONE': str(tz), + } + self._update_sysconfig_file(self.clock_conf_fn, clock_cfg) + # This ensures that the correct tz will be used for the system + util.copy(tz_file, self.tz_local_fn) def package_command(self, command, args=None, pkgs=None): if pkgs is None: diff --git a/cloudinit/sources/DataSourceConfigDrive.py b/cloudinit/sources/DataSourceConfigDrive.py index 5f152299..d3443c2b 100644 --- a/cloudinit/sources/DataSourceConfigDrive.py +++ b/cloudinit/sources/DataSourceConfigDrive.py @@ -258,6 +258,9 @@ def find_candidate_devs(): * labeled with 'config-2' """ + # Query optical drive to get it in blkid cache for 2.6 kernels + util.find_devs_with(path="/dev/sr0") + by_fstype = (util.find_devs_with("TYPE=vfat") + util.find_devs_with("TYPE=iso9660")) by_label = util.find_devs_with("LABEL=config-2") diff --git a/cloudinit/sources/DataSourceNoCloud.py b/cloudinit/sources/DataSourceNoCloud.py index 08a853cc..01c99028 100644 --- a/cloudinit/sources/DataSourceNoCloud.py +++ b/cloudinit/sources/DataSourceNoCloud.py @@ -87,6 +87,9 @@ class DataSourceNoCloud(sources.DataSource): label = self.ds_cfg.get('fs_label', "cidata") if label is not None: + # Query optical drive to get it in blkid cache for 2.6 kernels + util.find_devs_with(path="/dev/sr0") + fslist = util.find_devs_with("TYPE=vfat") fslist.extend(util.find_devs_with("TYPE=iso9660")) diff --git a/cloudinit/util.py b/cloudinit/util.py index 36e9b83b..50de55fe 100644 --- a/cloudinit/util.py +++ b/cloudinit/util.py @@ -408,6 +408,7 @@ def system_info(): 'release': platform.release(), 'python': platform.python_version(), 'uname': platform.uname(), + 'dist': platform.linux_distribution(), } diff --git a/tests/unittests/test_datasource/test_configdrive.py b/tests/unittests/test_datasource/test_configdrive.py index 930086db..d5935294 100644 --- a/tests/unittests/test_datasource/test_configdrive.py +++ b/tests/unittests/test_datasource/test_configdrive.py @@ -259,8 +259,9 @@ class TestConfigDriveDataSource(MockerTestCase): def test_find_candidates(self): devs_with_answers = {} - def my_devs_with(criteria): - return devs_with_answers[criteria] + def my_devs_with(*args, **kwargs): + criteria = args[0] if len(args) else kwargs.pop('criteria', None) + return devs_with_answers.get(criteria, []) def my_is_partition(dev): return dev[-1] in "0123456789" and not dev.startswith("sr") -- cgit v1.2.3 From 1b7e36a966ce1a0964e93eefa98c9efcbc4c323d Mon Sep 17 00:00:00 2001 From: Scott Moser Date: Thu, 25 Apr 2013 11:58:38 -0400 Subject: re-work maas datasource and headers call backs Couple of things here: * do not re-try on user-data (404 means 'not here') * re-generate headers on retry requests LP: #1172742 --- cloudinit/sources/DataSourceMAAS.py | 16 +++++++++++----- cloudinit/url_helper.py | 18 +++++++++++++----- cloudinit/util.py | 4 +++- 3 files changed, 27 insertions(+), 11 deletions(-) (limited to 'cloudinit/sources') diff --git a/cloudinit/sources/DataSourceMAAS.py b/cloudinit/sources/DataSourceMAAS.py index de822924..dfe90bc6 100644 --- a/cloudinit/sources/DataSourceMAAS.py +++ b/cloudinit/sources/DataSourceMAAS.py @@ -218,14 +218,20 @@ def read_maas_seed_url(seed_url, header_cb=None, timeout=None, md = {} for name in file_order: url = files.get(name) - if header_cb: - headers = header_cb(url) + if not header_cb: + def _cb(url): + return {} + header_cb = _cb + + if name == 'user-data': + retries = 0 else: - headers = {} + retries = None + try: ssl_details = util.fetch_ssl_details(paths) - resp = util.read_file_or_url(url, - headers=headers, + resp = util.read_file_or_url(url, retries=retries, + headers_cb=header_cb, timeout=timeout, ssl_details=ssl_details) if resp.ok(): diff --git a/cloudinit/url_helper.py b/cloudinit/url_helper.py index 6b4516e0..24ce6871 100644 --- a/cloudinit/url_helper.py +++ b/cloudinit/url_helper.py @@ -102,8 +102,8 @@ class UrlError(IOError): def readurl(url, data=None, timeout=None, retries=0, sec_between=1, - headers=None, ssl_details=None, check_status=True, - allow_redirects=True): + headers=None, headers_cb=None, ssl_details=None, + check_status=True, allow_redirects=True): url = _cleanurl(url) req_args = { 'url': url, @@ -149,8 +149,11 @@ def readurl(url, data=None, timeout=None, retries=0, sec_between=1, headers = { 'User-Agent': 'Cloud-Init/%s' % (version.version_string()), } - req_args['headers'] = headers - LOG.debug("Attempting to open '%s' with %s configuration", url, req_args) + if not headers_cb: + def _cb(url): + return headers + headers_cb = _cb + if data: # Do this after the log (it might be large) req_args['data'] = data @@ -161,6 +164,11 @@ def readurl(url, data=None, timeout=None, retries=0, sec_between=1, # doesn't handle sleeping between tries... for i in range(0, manual_tries): try: + req_args['headers'] = headers_cb(url) + LOG.debug("[%s/%s] open '%s' with %s configuration", i, + manual_tries, url, + {k: req_args[k] for k in req_args if k != 'data'}) + r = requests.request(**req_args) if check_status: r.raise_for_status() # pylint: disable=E1103 @@ -174,7 +182,7 @@ def readurl(url, data=None, timeout=None, retries=0, sec_between=1, except exceptions.RequestException as e: if (isinstance(e, (exceptions.HTTPError)) and hasattr(e, 'response') # This appeared in v 0.10.8 - and e.response): + and hasattr(e.response, 'status_code')): excps.append(UrlError(e, code=e.response.status_code, headers=e.response.headers)) else: diff --git a/cloudinit/util.py b/cloudinit/util.py index 50de55fe..053fa95d 100644 --- a/cloudinit/util.py +++ b/cloudinit/util.py @@ -671,7 +671,8 @@ def fetch_ssl_details(paths=None): def read_file_or_url(url, timeout=5, retries=10, - headers=None, data=None, sec_between=1, ssl_details=None): + headers=None, data=None, sec_between=1, ssl_details=None, + headers_cb=None): url = url.lstrip() if url.startswith("/"): url = "file://%s" % url @@ -685,6 +686,7 @@ def read_file_or_url(url, timeout=5, retries=10, timeout=timeout, retries=retries, headers=headers, + headers_cb=headers_cb, data=data, sec_between=sec_between, ssl_details=ssl_details) -- cgit v1.2.3 From 48a3b4ca0ba55699825d6eabe75d843286c23545 Mon Sep 17 00:00:00 2001 From: Greg Padgett Date: Tue, 4 Jun 2013 20:42:55 -0400 Subject: support optical drives with dev node /dev/sr1 Extend a prior fix which helped discovery of media on systems using 2.6 kernels. /dev/sr0 covers only some of the use cases, /dev/sr1 is also common. --- cloudinit/sources/DataSourceConfigDrive.py | 1 + cloudinit/sources/DataSourceNoCloud.py | 1 + 2 files changed, 2 insertions(+) (limited to 'cloudinit/sources') diff --git a/cloudinit/sources/DataSourceConfigDrive.py b/cloudinit/sources/DataSourceConfigDrive.py index d3443c2b..835f2a9a 100644 --- a/cloudinit/sources/DataSourceConfigDrive.py +++ b/cloudinit/sources/DataSourceConfigDrive.py @@ -260,6 +260,7 @@ def find_candidate_devs(): # Query optical drive to get it in blkid cache for 2.6 kernels util.find_devs_with(path="/dev/sr0") + util.find_devs_with(path="/dev/sr1") by_fstype = (util.find_devs_with("TYPE=vfat") + util.find_devs_with("TYPE=iso9660")) diff --git a/cloudinit/sources/DataSourceNoCloud.py b/cloudinit/sources/DataSourceNoCloud.py index 01c99028..084abca7 100644 --- a/cloudinit/sources/DataSourceNoCloud.py +++ b/cloudinit/sources/DataSourceNoCloud.py @@ -89,6 +89,7 @@ class DataSourceNoCloud(sources.DataSource): if label is not None: # Query optical drive to get it in blkid cache for 2.6 kernels util.find_devs_with(path="/dev/sr0") + util.find_devs_with(path="/dev/sr1") fslist = util.find_devs_with("TYPE=vfat") fslist.extend(util.find_devs_with("TYPE=iso9660")) -- cgit v1.2.3 From 233761a7fe4a995fbe0c8f65f512172a2b64090c Mon Sep 17 00:00:00 2001 From: Scott Moser Date: Fri, 7 Jun 2013 13:30:03 -0400 Subject: DataSourceOVF: small fix for iso9660 transport This is not really a problem, because nothing would call transport_iso9660 with 'require_iso' as False, but if it did, then we would have still required iso9660 filesystem on the mount. --- ChangeLog | 1 + cloudinit/sources/DataSourceOVF.py | 7 ++++++- 2 files changed, 7 insertions(+), 1 deletion(-) (limited to 'cloudinit/sources') diff --git a/ChangeLog b/ChangeLog index adc8b954..6495a19b 100644 --- a/ChangeLog +++ b/ChangeLog @@ -1,5 +1,6 @@ 0.7.3: - fix omnibus chef installer (LP: #1182265) [Chris Wing] + - small fix for OVF datasource for iso transport on non-iso9660 filesystem 0.7.2: - add a debian watch file - add 'sudo' entry to ubuntu's default user (LP: #1080717) diff --git a/cloudinit/sources/DataSourceOVF.py b/cloudinit/sources/DataSourceOVF.py index 0530c4b7..77b43e17 100644 --- a/cloudinit/sources/DataSourceOVF.py +++ b/cloudinit/sources/DataSourceOVF.py @@ -194,6 +194,11 @@ def transport_iso9660(require_iso=True): if contents is not False: return (contents, dev, fname) + if require_iso: + mtype = "iso9660" + else: + mtype = None + devs = os.listdir("/dev/") devs.sort() for dev in devs: @@ -211,7 +216,7 @@ def transport_iso9660(require_iso=True): try: (fname, contents) = util.mount_cb(fullp, - get_ovf_env, mtype="iso9660") + get_ovf_env, mtype=mtype) except util.MountFailedError: LOG.debug("%s not mountable as iso9660" % fullp) continue -- cgit v1.2.3 From 944623f4ad3e4c7319758c64053d06a3b05555a2 Mon Sep 17 00:00:00 2001 From: Juerg Haefliger Date: Wed, 19 Jun 2013 08:44:00 +0200 Subject: fix and cleanup usage of util.logexc --- cloudinit/config/cc_bootcmd.py | 5 ++--- cloudinit/config/cc_growpart.py | 8 +++++--- cloudinit/config/cc_phone_home.py | 14 +++++++------- cloudinit/config/cc_rightscale_userdata.py | 10 +++++----- cloudinit/config/cc_set_hostname.py | 6 +++--- cloudinit/config/cc_set_passwords.py | 6 +++--- cloudinit/config/cc_ssh.py | 10 +++++----- cloudinit/config/cc_ssh_import_id.py | 6 +++--- cloudinit/config/cc_update_hostname.py | 6 +++--- cloudinit/distros/__init__.py | 26 ++++++++++++------------- cloudinit/distros/rhel.py | 7 +++---- cloudinit/handlers/__init__.py | 13 ++++++------- cloudinit/handlers/boot_hook.py | 4 ++-- cloudinit/helpers.py | 18 ++++++++--------- cloudinit/sources/DataSourceAltCloud.py | 31 +++++++++++++++--------------- cloudinit/sources/DataSourceCloudStack.py | 6 ++++-- cloudinit/sources/DataSourceNoCloud.py | 6 +++--- cloudinit/ssh_util.py | 8 +++----- cloudinit/stages.py | 7 +++---- cloudinit/util.py | 5 ++--- 20 files changed, 100 insertions(+), 102 deletions(-) (limited to 'cloudinit/sources') diff --git a/cloudinit/config/cc_bootcmd.py b/cloudinit/config/cc_bootcmd.py index 896cb4d0..3ac22967 100644 --- a/cloudinit/config/cc_bootcmd.py +++ b/cloudinit/config/cc_bootcmd.py @@ -1,7 +1,7 @@ # vi: ts=4 expandtab # # Copyright (C) 2009-2011 Canonical Ltd. -# Copyright (C) 2012 Hewlett-Packard Development Company, L.P. +# Copyright (C) 2012, 2013 Hewlett-Packard Development Company, L.P. # # Author: Scott Moser # Author: Juerg Haefliger @@ -50,6 +50,5 @@ def handle(name, cfg, cloud, log, _args): cmd = ['/bin/sh', tmpf.name] util.subp(cmd, env=env, capture=False) except: - util.logexc(log, - ("Failed to run bootcmd module %s"), name) + util.logexc(log, "Failed to run bootcmd module %s", name) raise diff --git a/cloudinit/config/cc_growpart.py b/cloudinit/config/cc_growpart.py index b6e1fd37..4f8c8f80 100644 --- a/cloudinit/config/cc_growpart.py +++ b/cloudinit/config/cc_growpart.py @@ -1,8 +1,10 @@ # vi: ts=4 expandtab # # Copyright (C) 2011 Canonical Ltd. +# Copyright (C) 2013 Hewlett-Packard Development Company, L.P. # # Author: Scott Moser +# Author: Juerg Haefliger # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License version 3, as @@ -121,15 +123,15 @@ class ResizeGrowPart(object): util.subp(["growpart", '--dry-run', diskdev, partnum]) except util.ProcessExecutionError as e: if e.exit_code != 1: - util.logexc(LOG, ("Failed growpart --dry-run for (%s, %s)" % - (diskdev, partnum))) + util.logexc(LOG, "Failed growpart --dry-run for (%s, %s)", + diskdev, partnum) raise ResizeFailedException(e) return (before, before) try: util.subp(["growpart", diskdev, partnum]) except util.ProcessExecutionError as e: - util.logexc(LOG, "Failed: growpart %s %s" % (diskdev, partnum)) + util.logexc(LOG, "Failed: growpart %s %s", diskdev, partnum) raise ResizeFailedException(e) return (before, get_size(partdev)) diff --git a/cloudinit/config/cc_phone_home.py b/cloudinit/config/cc_phone_home.py index c873c8a8..2e058ccd 100644 --- a/cloudinit/config/cc_phone_home.py +++ b/cloudinit/config/cc_phone_home.py @@ -1,7 +1,7 @@ # vi: ts=4 expandtab # # Copyright (C) 2011 Canonical Ltd. -# Copyright (C) 2012 Hewlett-Packard Development Company, L.P. +# Copyright (C) 2012, 2013 Hewlett-Packard Development Company, L.P. # # Author: Scott Moser # Author: Juerg Haefliger @@ -65,8 +65,8 @@ def handle(name, cfg, cloud, log, args): tries = int(tries) except: tries = 10 - util.logexc(log, ("Configuration entry 'tries'" - " is not an integer, using %s instead"), tries) + util.logexc(log, "Configuration entry 'tries' is not an integer, " + "using %s instead", tries) if post_list == "all": post_list = POST_LIST_ALL @@ -85,8 +85,8 @@ def handle(name, cfg, cloud, log, args): try: all_keys[n] = util.load_file(path) except: - util.logexc(log, ("%s: failed to open, can not" - " phone home that data!"), path) + util.logexc(log, "%s: failed to open, can not phone home that " + "data!", path) submit_keys = {} for k in post_list: @@ -115,5 +115,5 @@ def handle(name, cfg, cloud, log, args): retries=tries, sec_between=3, ssl_details=util.fetch_ssl_details(cloud.paths)) except: - util.logexc(log, ("Failed to post phone home data to" - " %s in %s tries"), url, tries) + util.logexc(log, "Failed to post phone home data to %s in %s tries", + url, tries) diff --git a/cloudinit/config/cc_rightscale_userdata.py b/cloudinit/config/cc_rightscale_userdata.py index 4bf18516..c771728d 100644 --- a/cloudinit/config/cc_rightscale_userdata.py +++ b/cloudinit/config/cc_rightscale_userdata.py @@ -1,7 +1,7 @@ # vi: ts=4 expandtab # # Copyright (C) 2011 Canonical Ltd. -# Copyright (C) 2012 Hewlett-Packard Development Company, L.P. +# Copyright (C) 2012, 2013 Hewlett-Packard Development Company, L.P. # # Author: Scott Moser # Author: Juerg Haefliger @@ -64,8 +64,8 @@ def handle(name, _cfg, cloud, log, _args): " raw userdata"), name, MY_HOOKNAME) return except: - util.logexc(log, ("Failed to parse query string %s" - " into a dictionary"), ud) + util.logexc(log, "Failed to parse query string %s into a dictionary", + ud) raise wrote_fns = [] @@ -86,8 +86,8 @@ def handle(name, _cfg, cloud, log, _args): wrote_fns.append(fname) except Exception as e: captured_excps.append(e) - util.logexc(log, "%s failed to read %s and write %s", - MY_NAME, url, fname) + util.logexc(log, "%s failed to read %s and write %s", MY_NAME, url, + fname) if wrote_fns: log.debug("Wrote out rightscale userdata to %s files", len(wrote_fns)) diff --git a/cloudinit/config/cc_set_hostname.py b/cloudinit/config/cc_set_hostname.py index 2b32fc94..5d7f4331 100644 --- a/cloudinit/config/cc_set_hostname.py +++ b/cloudinit/config/cc_set_hostname.py @@ -1,7 +1,7 @@ # vi: ts=4 expandtab # # Copyright (C) 2011 Canonical Ltd. -# Copyright (C) 2012 Hewlett-Packard Development Company, L.P. +# Copyright (C) 2012, 2013 Hewlett-Packard Development Company, L.P. # # Author: Scott Moser # Author: Juerg Haefliger @@ -32,6 +32,6 @@ def handle(name, cfg, cloud, log, _args): log.debug("Setting the hostname to %s (%s)", fqdn, hostname) cloud.distro.set_hostname(hostname, fqdn) except Exception: - util.logexc(log, "Failed to set the hostname to %s (%s)", - fqdn, hostname) + util.logexc(log, "Failed to set the hostname to %s (%s)", fqdn, + hostname) raise diff --git a/cloudinit/config/cc_set_passwords.py b/cloudinit/config/cc_set_passwords.py index c6bf62fd..e93c8c6f 100644 --- a/cloudinit/config/cc_set_passwords.py +++ b/cloudinit/config/cc_set_passwords.py @@ -1,7 +1,7 @@ # vi: ts=4 expandtab # # Copyright (C) 2009-2010 Canonical Ltd. -# Copyright (C) 2012 Hewlett-Packard Development Company, L.P. +# Copyright (C) 2012, 2013 Hewlett-Packard Development Company, L.P. # # Author: Scott Moser # Author: Juerg Haefliger @@ -81,8 +81,8 @@ def handle(_name, cfg, cloud, log, args): util.subp(['chpasswd'], ch_in) except Exception as e: errors.append(e) - util.logexc(log, - "Failed to set passwords with chpasswd for %s", users) + util.logexc(log, "Failed to set passwords with chpasswd for %s", + users) if len(randlist): blurb = ("Set the following 'random' passwords\n", diff --git a/cloudinit/config/cc_ssh.py b/cloudinit/config/cc_ssh.py index 7ef20d9f..64a5e3cb 100644 --- a/cloudinit/config/cc_ssh.py +++ b/cloudinit/config/cc_ssh.py @@ -1,7 +1,7 @@ # vi: ts=4 expandtab # # Copyright (C) 2009-2010 Canonical Ltd. -# Copyright (C) 2012 Hewlett-Packard Development Company, L.P. +# Copyright (C) 2012, 2013 Hewlett-Packard Development Company, L.P. # # Author: Scott Moser # Author: Juerg Haefliger @@ -85,8 +85,8 @@ def handle(_name, cfg, cloud, log, _args): util.subp(cmd, capture=False) log.debug("Generated a key for %s from %s", pair[0], pair[1]) except: - util.logexc(log, ("Failed generated a key" - " for %s from %s"), pair[0], pair[1]) + util.logexc(log, "Failed generated a key for %s from %s", + pair[0], pair[1]) else: # if not, generate them genkeys = util.get_cfg_option_list(cfg, @@ -102,8 +102,8 @@ def handle(_name, cfg, cloud, log, _args): with util.SeLinuxGuard("/etc/ssh", recursive=True): util.subp(cmd, capture=False) except: - util.logexc(log, ("Failed generating key type" - " %s to file %s"), keytype, keyfile) + util.logexc(log, "Failed generating key type %s to " + "file %s", keytype, keyfile) try: (users, _groups) = ds.normalize_users_groups(cfg, cloud.distro) diff --git a/cloudinit/config/cc_ssh_import_id.py b/cloudinit/config/cc_ssh_import_id.py index 83af36e9..50d96e15 100644 --- a/cloudinit/config/cc_ssh_import_id.py +++ b/cloudinit/config/cc_ssh_import_id.py @@ -1,7 +1,7 @@ # vi: ts=4 expandtab # # Copyright (C) 2009-2010 Canonical Ltd. -# Copyright (C) 2012 Hewlett-Packard Development Company, L.P. +# Copyright (C) 2012, 2013 Hewlett-Packard Development Company, L.P. # # Author: Scott Moser # Author: Juerg Haefliger @@ -71,8 +71,8 @@ def handle(_name, cfg, cloud, log, args): try: import_ssh_ids(import_ids, user, log) except Exception as exc: - util.logexc(log, "ssh-import-id failed for: %s %s" % - (user, import_ids), exc) + util.logexc(log, "ssh-import-id failed for: %s %s", user, + import_ids) elist.append(exc) if len(elist): diff --git a/cloudinit/config/cc_update_hostname.py b/cloudinit/config/cc_update_hostname.py index 52225cd8..e396ba13 100644 --- a/cloudinit/config/cc_update_hostname.py +++ b/cloudinit/config/cc_update_hostname.py @@ -1,7 +1,7 @@ # vi: ts=4 expandtab # # Copyright (C) 2011 Canonical Ltd. -# Copyright (C) 2012 Hewlett-Packard Development Company, L.P. +# Copyright (C) 2012, 2013 Hewlett-Packard Development Company, L.P. # # Author: Scott Moser # Author: Juerg Haefliger @@ -38,6 +38,6 @@ def handle(name, cfg, cloud, log, _args): log.debug("Updating hostname to %s (%s)", fqdn, hostname) cloud.distro.update_hostname(hostname, fqdn, prev_fn) except Exception: - util.logexc(log, "Failed to update the hostname to %s (%s)", - fqdn, hostname) + util.logexc(log, "Failed to update the hostname to %s (%s)", fqdn, + hostname) raise diff --git a/cloudinit/distros/__init__.py b/cloudinit/distros/__init__.py index 50d52594..e99cb16f 100644 --- a/cloudinit/distros/__init__.py +++ b/cloudinit/distros/__init__.py @@ -1,7 +1,7 @@ # vi: ts=4 expandtab # # Copyright (C) 2012 Canonical Ltd. -# Copyright (C) 2012 Hewlett-Packard Development Company, L.P. +# Copyright (C) 2012, 2013 Hewlett-Packard Development Company, L.P. # Copyright (C) 2012 Yahoo! Inc. # # Author: Scott Moser @@ -142,8 +142,8 @@ class Distro(object): try: util.subp(['hostname', hostname]) except util.ProcessExecutionError: - util.logexc(LOG, ("Failed to non-persistently adjust" - " the system hostname to %s"), hostname) + util.logexc(LOG, "Failed to non-persistently adjust the system " + "hostname to %s", hostname) @abc.abstractmethod def _select_hostname(self, hostname, fqdn): @@ -200,8 +200,8 @@ class Distro(object): try: self._write_hostname(hostname, fn) except IOError: - util.logexc(LOG, "Failed to write hostname %s to %s", - hostname, fn) + util.logexc(LOG, "Failed to write hostname %s to %s", hostname, + fn) if (sys_hostname and prev_hostname and sys_hostname != prev_hostname): @@ -347,7 +347,7 @@ class Distro(object): try: util.subp(adduser_cmd, logstring=x_adduser_cmd) except Exception as e: - util.logexc(LOG, "Failed to create user %s due to error.", e) + util.logexc(LOG, "Failed to create user %s", name) raise e # Set password if plain-text password provided @@ -360,8 +360,8 @@ class Distro(object): try: util.subp(['passwd', '--lock', name]) except Exception as e: - util.logexc(LOG, ("Failed to disable password logins for" - "user %s" % name), e) + util.logexc(LOG, "Failed to disable password logins for " + "user %s", name) raise e # Configure sudo access @@ -385,7 +385,7 @@ class Distro(object): try: util.subp(cmd, pass_string, logstring="chpasswd for %s" % user) except Exception as e: - util.logexc(LOG, "Failed to set password for %s" % user) + util.logexc(LOG, "Failed to set password for %s", user) raise e return True @@ -427,7 +427,7 @@ class Distro(object): util.append_file(sudo_base, sudoers_contents) LOG.debug("Added '#includedir %s' to %s" % (path, sudo_base)) except IOError as e: - util.logexc(LOG, "Failed to write %s" % sudo_base, e) + util.logexc(LOG, "Failed to write %s", sudo_base) raise e util.ensure_dir(path, 0750) @@ -478,15 +478,15 @@ class Distro(object): try: util.subp(group_add_cmd) LOG.info("Created new group %s" % name) - except Exception as e: - util.logexc("Failed to create group %s" % name, e) + except Exception: + util.logexc("Failed to create group %s", name) # Add members to the group, if so defined if len(members) > 0: for member in members: if not util.is_user(member): LOG.warn("Unable to add group member '%s' to group '%s'" - "; user does not exist." % (member, name)) + "; user does not exist.", member, name) continue util.subp(['usermod', '-a', '-G', name, member]) diff --git a/cloudinit/distros/rhel.py b/cloudinit/distros/rhel.py index 174da3ab..0727ecd1 100644 --- a/cloudinit/distros/rhel.py +++ b/cloudinit/distros/rhel.py @@ -1,7 +1,7 @@ # vi: ts=4 expandtab # # Copyright (C) 2012 Canonical Ltd. -# Copyright (C) 2012 Hewlett-Packard Development Company, L.P. +# Copyright (C) 2012, 2013 Hewlett-Packard Development Company, L.P. # Copyright (C) 2012 Yahoo! Inc. # # Author: Scott Moser @@ -72,9 +72,8 @@ class Distro(distros.Distro): r_conf = ResolvConf(util.load_file(self.resolve_conf_fn)) r_conf.parse() except IOError: - util.logexc(LOG, - "Failed at parsing %s reverting to an empty instance", - self.resolve_conf_fn) + util.logexc(LOG, "Failed at parsing %s reverting to an empty " + "instance", self.resolve_conf_fn) r_conf = ResolvConf('') r_conf.parse() if dns_servers: diff --git a/cloudinit/handlers/__init__.py b/cloudinit/handlers/__init__.py index 924463ce..497d68c5 100644 --- a/cloudinit/handlers/__init__.py +++ b/cloudinit/handlers/__init__.py @@ -1,7 +1,7 @@ # vi: ts=4 expandtab # # Copyright (C) 2012 Canonical Ltd. -# Copyright (C) 2012 Hewlett-Packard Development Company, L.P. +# Copyright (C) 2012, 2013 Hewlett-Packard Development Company, L.P. # Copyright (C) 2012 Yahoo! Inc. # # Author: Scott Moser @@ -117,10 +117,9 @@ def run_part(mod, data, filename, payload, frequency, headers): else: raise ValueError("Unknown module version %s" % (mod_ver)) except: - util.logexc(LOG, ("Failed calling handler %s (%s, %s, %s)" - " with frequency %s"), - mod, content_type, filename, - mod_ver, frequency) + util.logexc(LOG, "Failed calling handler %s (%s, %s, %s) with " + "frequency %s", mod, content_type, filename, mod_ver, + frequency) def call_begin(mod, data, frequency): @@ -158,8 +157,8 @@ def walker_handle_handler(pdata, _ctype, _filename, payload): handlers.register(mod) pdata['handlercount'] = curcount + 1 except: - util.logexc(LOG, ("Failed at registering python file: %s" - " (part handler %s)"), modfname, curcount) + util.logexc(LOG, "Failed at registering python file: %s (part " + "handler %s)", modfname, curcount) def _extract_first_or_bytes(blob, size): diff --git a/cloudinit/handlers/boot_hook.py b/cloudinit/handlers/boot_hook.py index bf2899ab..11ac4fe5 100644 --- a/cloudinit/handlers/boot_hook.py +++ b/cloudinit/handlers/boot_hook.py @@ -70,5 +70,5 @@ class BootHookPartHandler(handlers.Handler): except util.ProcessExecutionError: util.logexc(LOG, "Boothooks script %s execution error", filepath) except Exception: - util.logexc(LOG, ("Boothooks unknown " - "error when running %s"), filepath) + util.logexc(LOG, "Boothooks unknown error when running %s", + filepath) diff --git a/cloudinit/helpers.py b/cloudinit/helpers.py index a4e6fb03..b91c1290 100644 --- a/cloudinit/helpers.py +++ b/cloudinit/helpers.py @@ -1,7 +1,7 @@ # vi: ts=4 expandtab # # Copyright (C) 2012 Canonical Ltd. -# Copyright (C) 2012 Hewlett-Packard Development Company, L.P. +# Copyright (C) 2012, 2013 Hewlett-Packard Development Company, L.P. # Copyright (C) 2012 Yahoo! Inc. # # Author: Scott Moser @@ -216,8 +216,8 @@ class ConfigMerger(object): if ds_cfg and isinstance(ds_cfg, (dict)): d_cfgs.append(ds_cfg) except: - util.logexc(LOG, ("Failed loading of datasource" - " config object from %s"), self._ds) + util.logexc(LOG, "Failed loading of datasource config object " + "from %s", self._ds) return d_cfgs def _get_env_configs(self): @@ -227,8 +227,8 @@ class ConfigMerger(object): try: e_cfgs.append(util.read_conf(e_fn)) except: - util.logexc(LOG, ('Failed loading of env. config' - ' from %s'), e_fn) + util.logexc(LOG, 'Failed loading of env. config from %s', + e_fn) return e_cfgs def _get_instance_configs(self): @@ -242,8 +242,8 @@ class ConfigMerger(object): try: i_cfgs.append(util.read_conf(cc_fn)) except: - util.logexc(LOG, ('Failed loading of cloud-config' - ' from %s'), cc_fn) + util.logexc(LOG, 'Failed loading of cloud-config from %s', + cc_fn) return i_cfgs def _read_cfg(self): @@ -259,8 +259,8 @@ class ConfigMerger(object): try: cfgs.append(util.read_conf(c_fn)) except: - util.logexc(LOG, ("Failed loading of configuration" - " from %s"), c_fn) + util.logexc(LOG, "Failed loading of configuration from %s", + c_fn) cfgs.extend(self._get_env_configs()) cfgs.extend(self._get_instance_configs()) diff --git a/cloudinit/sources/DataSourceAltCloud.py b/cloudinit/sources/DataSourceAltCloud.py index 64548d43..a834f8eb 100644 --- a/cloudinit/sources/DataSourceAltCloud.py +++ b/cloudinit/sources/DataSourceAltCloud.py @@ -1,10 +1,11 @@ # vi: ts=4 expandtab # # Copyright (C) 2009-2010 Canonical Ltd. -# Copyright (C) 2012 Hewlett-Packard Development Company, L.P. +# Copyright (C) 2012, 2013 Hewlett-Packard Development Company, L.P. # Copyright (C) 2012 Yahoo! Inc. # # Author: Joe VLcek +# Author: Juerg Haefliger # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License version 3, as @@ -79,7 +80,7 @@ def read_user_data_callback(mount_dir): try: user_data = util.load_file(user_data_file).strip() except IOError: - util.logexc(LOG, ('Failed accessing user data file.')) + util.logexc(LOG, 'Failed accessing user data file.') return None return user_data @@ -178,7 +179,7 @@ class DataSourceAltCloud(sources.DataSource): return False # No user data found - util.logexc(LOG, ('Failed accessing user data.')) + util.logexc(LOG, 'Failed accessing user data.') return False def user_data_rhevm(self): @@ -205,12 +206,12 @@ class DataSourceAltCloud(sources.DataSource): (cmd_out, _err) = util.subp(cmd) LOG.debug(('Command: %s\nOutput%s') % (' '.join(cmd), cmd_out)) except ProcessExecutionError, _err: - util.logexc(LOG, (('Failed command: %s\n%s') % \ - (' '.join(cmd), _err.message))) + util.logexc(LOG, 'Failed command: %s\n%s', ' '.join(cmd), + _err.message) return False except OSError, _err: - util.logexc(LOG, (('Failed command: %s\n%s') % \ - (' '.join(cmd), _err.message))) + util.logexc(LOG, 'Failed command: %s\n%s', ' '.join(cmd), + _err.message) return False floppy_dev = '/dev/fd0' @@ -222,12 +223,12 @@ class DataSourceAltCloud(sources.DataSource): (cmd_out, _err) = util.subp(cmd) LOG.debug(('Command: %s\nOutput%s') % (' '.join(cmd), cmd_out)) except ProcessExecutionError, _err: - util.logexc(LOG, (('Failed command: %s\n%s') % \ - (' '.join(cmd), _err.message))) + util.logexc(LOG, 'Failed command: %s\n%s', ' '.join(cmd), + _err.message) return False except OSError, _err: - util.logexc(LOG, (('Failed command: %s\n%s') % \ - (' '.join(cmd), _err.message))) + util.logexc(LOG, 'Failed command: %s\n%s', ' '.join(cmd), + _err.message) return False try: @@ -236,8 +237,8 @@ class DataSourceAltCloud(sources.DataSource): if err.errno != errno.ENOENT: raise except util.MountFailedError: - util.logexc(LOG, ("Failed to mount %s" - " when looking for user data"), floppy_dev) + util.logexc(LOG, "Failed to mount %s when looking for user data", + floppy_dev) self.userdata_raw = return_str self.metadata = META_DATA_NOT_SUPPORTED @@ -272,8 +273,8 @@ class DataSourceAltCloud(sources.DataSource): if err.errno != errno.ENOENT: raise except util.MountFailedError: - util.logexc(LOG, ("Failed to mount %s" - " when looking for user data"), cdrom_dev) + util.logexc(LOG, "Failed to mount %s when looking for user " + "data", cdrom_dev) self.userdata_raw = return_str self.metadata = META_DATA_NOT_SUPPORTED diff --git a/cloudinit/sources/DataSourceCloudStack.py b/cloudinit/sources/DataSourceCloudStack.py index 81c8cda9..08f661e4 100644 --- a/cloudinit/sources/DataSourceCloudStack.py +++ b/cloudinit/sources/DataSourceCloudStack.py @@ -4,11 +4,13 @@ # Copyright (C) 2012 Cosmin Luta # Copyright (C) 2012 Yahoo! Inc. # Copyright (C) 2012 Gerard Dethier +# Copyright (C) 2013 Hewlett-Packard Development Company, L.P. # # Author: Cosmin Luta # Author: Scott Moser # Author: Joshua Harlow # Author: Gerard Dethier +# Author: Juerg Haefliger # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License version 3, as @@ -109,8 +111,8 @@ class DataSourceCloudStack(sources.DataSource): int(time.time() - start_time)) return True except Exception: - util.logexc(LOG, ('Failed fetching from metadata ' - 'service %s'), self.metadata_address) + util.logexc(LOG, 'Failed fetching from metadata service %s', + self.metadata_address) return False def get_instance_id(self): diff --git a/cloudinit/sources/DataSourceNoCloud.py b/cloudinit/sources/DataSourceNoCloud.py index 084abca7..4ef92a56 100644 --- a/cloudinit/sources/DataSourceNoCloud.py +++ b/cloudinit/sources/DataSourceNoCloud.py @@ -1,7 +1,7 @@ # vi: ts=4 expandtab # # Copyright (C) 2009-2010 Canonical Ltd. -# Copyright (C) 2012 Hewlett-Packard Development Company, L.P. +# Copyright (C) 2012, 2013 Hewlett-Packard Development Company, L.P. # Copyright (C) 2012 Yahoo! Inc. # # Author: Scott Moser @@ -119,8 +119,8 @@ class DataSourceNoCloud(sources.DataSource): if e.errno != errno.ENOENT: raise except util.MountFailedError: - util.logexc(LOG, ("Failed to mount %s" - " when looking for data"), dev) + util.logexc(LOG, "Failed to mount %s when looking for " + "data", dev) # There was no indication on kernel cmdline or data # in the seeddir suggesting this handler should be used. diff --git a/cloudinit/ssh_util.py b/cloudinit/ssh_util.py index 95133236..70a577bc 100644 --- a/cloudinit/ssh_util.py +++ b/cloudinit/ssh_util.py @@ -229,11 +229,9 @@ def extract_authorized_keys(username): except (IOError, OSError): # Give up and use a default key filename auth_key_fn = os.path.join(ssh_dir, 'authorized_keys') - util.logexc(LOG, ("Failed extracting 'AuthorizedKeysFile'" - " in ssh config" - " from %r, using 'AuthorizedKeysFile' file" - " %r instead"), - DEF_SSHD_CFG, auth_key_fn) + util.logexc(LOG, "Failed extracting 'AuthorizedKeysFile' in ssh " + "config from %r, using 'AuthorizedKeysFile' file " + "%r instead", DEF_SSHD_CFG, auth_key_fn) return (auth_key_fn, parse_authorized_keys(auth_key_fn)) diff --git a/cloudinit/stages.py b/cloudinit/stages.py index 543d247f..df49cabb 100644 --- a/cloudinit/stages.py +++ b/cloudinit/stages.py @@ -1,7 +1,7 @@ # vi: ts=4 expandtab # # Copyright (C) 2012 Canonical Ltd. -# Copyright (C) 2012 Hewlett-Packard Development Company, L.P. +# Copyright (C) 2012, 2013 Hewlett-Packard Development Company, L.P. # Copyright (C) 2012 Yahoo! Inc. # # Author: Scott Moser @@ -154,9 +154,8 @@ class Init(object): try: util.chownbyname(log_file, u, g) except OSError: - util.logexc(LOG, ("Unable to change the ownership" - " of %s to user %s, group %s"), - log_file, u, g) + util.logexc(LOG, "Unable to change the ownership of %s to " + "user %s, group %s", log_file, u, g) def read_cfg(self, extra_fns=None): # None check so that we don't keep on re-loading if empty diff --git a/cloudinit/util.py b/cloudinit/util.py index b27b3567..c45aae06 100644 --- a/cloudinit/util.py +++ b/cloudinit/util.py @@ -1,7 +1,7 @@ # vi: ts=4 expandtab # # Copyright (C) 2012 Canonical Ltd. -# Copyright (C) 2012 Hewlett-Packard Development Company, L.P. +# Copyright (C) 2012, 2013 Hewlett-Packard Development Company, L.P. # Copyright (C) 2012 Yahoo! Inc. # # Author: Scott Moser @@ -219,8 +219,7 @@ def fork_cb(child_cb, *args): child_cb(*args) os._exit(0) # pylint: disable=W0212 except: - logexc(LOG, ("Failed forking and" - " calling callback %s"), + logexc(LOG, "Failed forking and calling callback %s", type_utils.obj_name(child_cb)) os._exit(1) # pylint: disable=W0212 else: -- cgit v1.2.3 From 691fe6d4ef3dad5d77e1b250d05bb0858234afee Mon Sep 17 00:00:00 2001 From: Scott Moser Date: Mon, 8 Jul 2013 17:04:53 -0400 Subject: commit dev state --- cloudinit/sources/DataSourceAzure.py | 205 ++++++++++++++++++++++++++++++ doc/examples/cloud-config-datasources.txt | 4 + 2 files changed, 209 insertions(+) create mode 100644 cloudinit/sources/DataSourceAzure.py (limited to 'cloudinit/sources') diff --git a/cloudinit/sources/DataSourceAzure.py b/cloudinit/sources/DataSourceAzure.py new file mode 100644 index 00000000..83c4603c --- /dev/null +++ b/cloudinit/sources/DataSourceAzure.py @@ -0,0 +1,205 @@ +# vi: ts=4 expandtab +# +# Copyright (C) 2013 Canonical Ltd. +# +# Author: Scott Moser +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 3, as +# published by the Free Software Foundation. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +import base64 +import os +from xml.dom import minidom + +from cloudinit import log as logging +from cloudinit import sources +from cloudinit import util + +LOG = logging.getLogger(__name__) + +DS_NAME = 'Azure' +DEFAULT_METADATA = {"instance-id": "iid-AZURE-NODE"} +AGENT_START = ['service', 'walinuxagent', 'start'] +DEFAULT_DS_CONFIG = {'datasource': {DS_NAME: {'agent_command': AGENT_START}}} + + +class DataSourceAzureNet(sources.DataSource): + def __init__(self, sys_cfg, distro, paths): + sources.DataSource.__init__(self, sys_cfg, distro, paths) + self.seed_dir = os.path.join(paths.seed_dir, 'azure') + self.cfg = {} + self.seed = None + + def __str__(self): + root = sources.DataSource.__str__(self) + return "%s [seed=%s]" % (root, self.seed) + + def get_data(self): + candidates = [self.seed_dir] + candidates.extend(list_possible_azure_ds_devs()) + found = None + + for cdev in candidates: + try: + if cdev.startswith("/dev/"): + ret = util.mount_cb(cdev, load_azure_ds_dir) + else: + ret = load_azure_ds_dir(cdev) + + except NonAzureDataSource: + pass + except BrokenAzureDataSource as exc: + raise exc + except util.MountFailedError: + LOG.warn("%s was not mountable" % cdev) + + (md, self.userdata_raw, cfg) = ret + self.seed = cdev + self.metadata = util.mergemanydict([md, DEFAULT_METADATA]) + self.cfg = util.mergemanydict([cfg, DEFAULT_DS_CONFIG]) + found = cdev + + LOG.debug("found datasource in %s", cdev) + break + + if not found: + return False + + path = ['datasource', DS_NAME, 'agent_command'] + cmd = None + for cfg in (self.cfg, self.sys_cfg): + cmd = util.get_cfg_by_path(cfg, keyp=path) + if cmd is not None: + break + invoke_agent(cmd) + + def get_config_obj(self): + return self.cfg + + +def invoke_agent(cmd): + if cmd: + LOG.debug("invoking agent: %s" % cmd) + util.subp(cmd, shell=(not isinstance(cmd, list))) + else: + LOG.debug("not invoking agent") + + +def find_child(node, filter_func): + ret = [] + if not node.hasChildNodes(): + return ret + for child in node.childNodes: + if filter_func(child): + ret.append(child) + return ret + + +def read_azure_ovf(contents): + dom = minidom.parseString(contents) + results = find_child(dom.documentElement, + lambda n: n.localName == "ProvisioningSection") + + if len(results) == 0: + raise NonAzureDataSource("No ProvisioningSection") + if len(results) > 1: + raise BrokenAzureDataSource("found '%d' ProvisioningSection items" % + len(results)) + provSection = results[0] + + lpcs_nodes = find_child(provSection, + lambda n: n.localName == "LinuxProvisioningConfigurationSet") + + if len(results) == 0: + raise NonAzureDataSource("No LinuxProvisioningConfigurationSet") + if len(results) > 1: + raise BrokenAzureDataSource("found '%d' %ss" % + ("LinuxProvisioningConfigurationSet", + len(results))) + lpcs = lpcs_nodes[0] + + if not lpcs.hasChildNodes(): + raise BrokenAzureDataSource("no child nodes of configuration set") + + md_props = 'seedfrom' + md = {'azure_data': {}} + cfg = {} + ud = "" + + for child in lpcs.childNodes: + if child.nodeType == dom.TEXT_NODE or not child.localName: + continue + + name = child.localName.lower() + + simple = False + if (len(child.childNodes) == 1 and + child.childNodes[0].nodeType == dom.TEXT_NODE): + simple = True + value = child.childNodes[0].wholeText + + if name == "userdata": + ud = base64.b64decode(''.join(value.split())) + elif name == "username": + cfg['system_info'] = {'default_user': {'name': value}} + elif name == "hostname": + md['local-hostname'] = value + elif name == "dscfg": + cfg['datasource'] = {DS_NAME: util.load_yaml(value, default={})} + elif simple: + if name in md_props: + md[name] = value + else: + md['azure_data'][name] = value + + return (md, ud, cfg) + + +def list_possible_azure_ds_devs(): + # return a sorted list of devices that might have a azure datasource + devlist = [] + for fstype in ("iso9660", "udf"): + devlist.extend(util.find_devs_with("TYPE=%s" % fstype)) + + devlist.sort(reverse=True) + return devlist + + +def load_azure_ds_dir(source_dir): + ovf_file = os.path.join(source_dir, "ovf-env.xml") + + if not os.path.isfile(ovf_file): + raise NonAzureDataSource("No ovf-env file found") + + with open(ovf_file, "r") as fp: + contents = fp.read() + + return read_azure_ovf(contents) + + +class BrokenAzureDataSource(Exception): + pass + + +class NonAzureDataSource(Exception): + pass + + +# Used to match classes to dependencies +datasources = [ + (DataSourceAzureNet, (sources.DEP_FILESYSTEM, sources.DEP_NETWORK)), +] + + +# Return a list of data sources that match this set of dependencies +def get_datasource_list(depends): + return sources.list_from_depends(depends, datasources) diff --git a/doc/examples/cloud-config-datasources.txt b/doc/examples/cloud-config-datasources.txt index fc8c22d4..fbabcad9 100644 --- a/doc/examples/cloud-config-datasources.txt +++ b/doc/examples/cloud-config-datasources.txt @@ -42,3 +42,7 @@ datasource: meta-data: instance-id: i-87018aed local-hostname: myhost.internal + + Azure: + agent_command: [service, walinuxagent, start] + -- cgit v1.2.3 From 21ea6154ab2aafbe51c7b23fd56e43bd1cc26b00 Mon Sep 17 00:00:00 2001 From: Scott Moser Date: Tue, 9 Jul 2013 01:35:49 -0400 Subject: add some unit tests, fix things found by doing so --- cloudinit/sources/DataSourceAzure.py | 26 +++- tests/unittests/test_datasource/test_azure.py | 168 ++++++++++++++++++++++++++ 2 files changed, 188 insertions(+), 6 deletions(-) create mode 100644 tests/unittests/test_datasource/test_azure.py (limited to 'cloudinit/sources') diff --git a/cloudinit/sources/DataSourceAzure.py b/cloudinit/sources/DataSourceAzure.py index 83c4603c..6a04b333 100644 --- a/cloudinit/sources/DataSourceAzure.py +++ b/cloudinit/sources/DataSourceAzure.py @@ -18,6 +18,7 @@ import base64 import os +import os.path from xml.dom import minidom from cloudinit import log as logging @@ -29,7 +30,7 @@ LOG = logging.getLogger(__name__) DS_NAME = 'Azure' DEFAULT_METADATA = {"instance-id": "iid-AZURE-NODE"} AGENT_START = ['service', 'walinuxagent', 'start'] -DEFAULT_DS_CONFIG = {'datasource': {DS_NAME: {'agent_command': AGENT_START}}} +BUILTIN_DS_CONFIG = {'datasource': {DS_NAME: {'agent_command': AGENT_START}}} class DataSourceAzureNet(sources.DataSource): @@ -56,16 +57,17 @@ class DataSourceAzureNet(sources.DataSource): ret = load_azure_ds_dir(cdev) except NonAzureDataSource: - pass + continue except BrokenAzureDataSource as exc: raise exc except util.MountFailedError: LOG.warn("%s was not mountable" % cdev) + continue (md, self.userdata_raw, cfg) = ret self.seed = cdev self.metadata = util.mergemanydict([md, DEFAULT_METADATA]) - self.cfg = util.mergemanydict([cfg, DEFAULT_DS_CONFIG]) + self.cfg = cfg found = cdev LOG.debug("found datasource in %s", cdev) @@ -76,17 +78,25 @@ class DataSourceAzureNet(sources.DataSource): path = ['datasource', DS_NAME, 'agent_command'] cmd = None - for cfg in (self.cfg, self.sys_cfg): + for cfg in (self.cfg, self.sys_cfg, BUILTIN_DS_CONFIG): cmd = util.get_cfg_by_path(cfg, keyp=path) if cmd is not None: break - invoke_agent(cmd) + + try: + invoke_agent(cmd) + except util.ProcessExecutionError: + # claim the datasource even if the command failed + util.logexc(LOG, "agent command '%s' failed.", cmd) + + return True def get_config_obj(self): return self.cfg def invoke_agent(cmd): + # this is a function itself to simplify patching it for test if cmd: LOG.debug("invoking agent: %s" % cmd) util.subp(cmd, shell=(not isinstance(cmd, list))) @@ -105,7 +115,11 @@ def find_child(node, filter_func): def read_azure_ovf(contents): - dom = minidom.parseString(contents) + try: + dom = minidom.parseString(contents) + except Exception as e: + raise NonAzureDataSource("invalid xml: %s" % e) + results = find_child(dom.documentElement, lambda n: n.localName == "ProvisioningSection") diff --git a/tests/unittests/test_datasource/test_azure.py b/tests/unittests/test_datasource/test_azure.py new file mode 100644 index 00000000..179fb50a --- /dev/null +++ b/tests/unittests/test_datasource/test_azure.py @@ -0,0 +1,168 @@ +from cloudinit import helpers +from cloudinit.sources import DataSourceAzure +from tests.unittests.helpers import populate_dir + +import base64 +from mocker import MockerTestCase +import os +import yaml + + +def construct_valid_ovf_env(data=None, pubkeys=None, userdata=None): + if data is None: + data = {'HostName': 'FOOHOST'} + if pubkeys is None: + pubkeys = {} + + content = """ + + + 1.0 + + LinuxProvisioningConfiguration + """ + for key, val in data.items(): + content += "<%s>%s\n" % (key, val, key) + + if userdata: + content += "%s\n" % (base64.b64encode(userdata)) + + if pubkeys: + content += "\n" + for fp, path in pubkeys.items(): + content += " " + content += ("%s%s" % + (fp, path)) + content += " " + content += """ + + + 1.0 + + kms.core.windows.net + false + + + + """ + + return content + + +class TestAzureDataSource(MockerTestCase): + + def setUp(self): + # makeDir comes from MockerTestCase + self.tmp = self.makeDir() + + # patch cloud_dir, so our 'seed_dir' is guaranteed empty + self.paths = helpers.Paths({'cloud_dir': self.tmp}) + + self.unapply = [] + super(TestAzureDataSource, self).setUp() + + def tearDown(self): + apply_patches([i for i in reversed(self.unapply)]) + super(TestAzureDataSource, self).tearDown() + + def apply_patches(self, patches): + ret = apply_patches(patches) + self.unapply += ret + + def _get_ds(self, data): + + def dsdevs(): + return data.get('dsdevs', []) + + def invoker(cmd): + data['agent_invoked'] = cmd + + if data.get('ovfcontent') is not None: + populate_dir(os.path.join(self.paths.seed_dir, "azure"), + {'ovf-env.xml': data['ovfcontent']}) + + mod = DataSourceAzure + + if data.get('dsdevs'): + self.apply_patches([(mod, 'list_possible_azure_ds_devs', dsdevs)]) + + self.apply_patches([(mod, 'invoke_agent', invoker)]) + + dsrc = mod.DataSourceAzureNet( + data.get('sys_cfg', {}), distro=None, paths=self.paths) + + return dsrc + + def test_basic_seed_dir(self): + odata = {'HostName': "myhost", 'UserName': "myuser"} + data = {'ovfcontent': construct_valid_ovf_env(data=odata), + 'sys_cfg': {}} + + dsrc = self._get_ds(data) + ret = dsrc.get_data() + self.assertTrue(ret) + self.assertEqual(dsrc.userdata_raw, "") + self.assertEqual(dsrc.metadata['local-hostname'], odata['HostName']) + + def test_user_cfg_set_agent_command(self): + cfg = {'agent_command': "my_command"} + odata = {'HostName': "myhost", 'UserName': "myuser", + 'dscfg': yaml.dump(cfg)} + data = {'ovfcontent': construct_valid_ovf_env(data=odata)} + + dsrc = self._get_ds(data) + ret = dsrc.get_data() + self.assertTrue(ret) + self.assertEqual(data['agent_invoked'], cfg['agent_command']) + + def test_sys_cfg_set_agent_command(self): + sys_cfg = {'datasource': {'Azure': {'agent_command': '_COMMAND'}}} + data = {'ovfcontent': construct_valid_ovf_env(data={}), + 'sys_cfg': sys_cfg} + + dsrc = self._get_ds(data) + ret = dsrc.get_data() + self.assertTrue(ret) + self.assertEqual(data['agent_invoked'], '_COMMAND') + + def test_userdata_found(self): + mydata = "FOOBAR" + odata = {'UserData': base64.b64encode(mydata)} + data = {'ovfcontent': construct_valid_ovf_env(data=odata)} + + dsrc = self._get_ds(data) + ret = dsrc.get_data() + self.assertTrue(ret) + self.assertEqual(dsrc.userdata_raw, mydata) + + def test_no_datasource_expected(self): + #no source should be found if no seed_dir and no devs + data = {} + dsrc = self._get_ds({}) + ret = dsrc.get_data() + self.assertFalse(ret) + self.assertFalse('agent_invoked' in data) + + +class TestReadAzureOvf(MockerTestCase): + def test_invalid_xml_raises_non_azure_ds(self): + invalid_xml = "" + construct_valid_ovf_env(data={}) + self.assertRaises(DataSourceAzure.NonAzureDataSource, + DataSourceAzure.read_azure_ovf, invalid_xml) + + +def apply_patches(patches): + ret = [] + for (ref, name, replace) in patches: + if replace is None: + continue + orig = getattr(ref, name) + setattr(ref, name, replace) + ret.append((ref, name, orig)) + return ret -- cgit v1.2.3 From d1d96451a420672a01d9097cc0b14ee13e8a9256 Mon Sep 17 00:00:00 2001 From: Scott Moser Date: Tue, 9 Jul 2013 14:13:24 -0400 Subject: add stub for ssh keys The idea is that in the future, the DataSources get_data could check to see if there were ssh keys provided. if there were, it could poll and wait (or inotify) until the .crt files appeared in the /var/lib/walinux directory. Once they did, it'd populate the metadata's public keys. --- cloudinit/sources/DataSourceAzure.py | 14 ++++++++++++++ 1 file changed, 14 insertions(+) (limited to 'cloudinit/sources') diff --git a/cloudinit/sources/DataSourceAzure.py b/cloudinit/sources/DataSourceAzure.py index 6a04b333..143b7e4a 100644 --- a/cloudinit/sources/DataSourceAzure.py +++ b/cloudinit/sources/DataSourceAzure.py @@ -114,6 +114,18 @@ def find_child(node, filter_func): return ret +def load_azure_ovf_pubkeys(sshnode): + # in the future this would return a list of dicts like: + # [{'fp': '6BE7A7C3C8A8F4B123CCA5D0C2F1BE4CA7B63ED7', + # 'path': 'where/to/go'}] + # + # + # ABC/ABC + # ... + # + return [] + + def read_azure_ovf(contents): try: dom = minidom.parseString(contents) @@ -169,6 +181,8 @@ def read_azure_ovf(contents): md['local-hostname'] = value elif name == "dscfg": cfg['datasource'] = {DS_NAME: util.load_yaml(value, default={})} + elif name == "ssh": + cfg['_pubkeys'] = loadAzurePubkeys(child) elif simple: if name in md_props: md[name] = value -- cgit v1.2.3 From 6bea1cb867c13e05e3548c648d5f051d2c49f07b Mon Sep 17 00:00:00 2001 From: Scott Moser Date: Tue, 9 Jul 2013 14:41:55 -0400 Subject: better handling for user/password --- cloudinit/sources/DataSourceAzure.py | 21 ++++++++++++++++++++- tests/unittests/test_datasource/test_azure.py | 27 +++++++++++++++++++++++++++ 2 files changed, 47 insertions(+), 1 deletion(-) (limited to 'cloudinit/sources') diff --git a/cloudinit/sources/DataSourceAzure.py b/cloudinit/sources/DataSourceAzure.py index 143b7e4a..5037c1a3 100644 --- a/cloudinit/sources/DataSourceAzure.py +++ b/cloudinit/sources/DataSourceAzure.py @@ -160,6 +160,8 @@ def read_azure_ovf(contents): md = {'azure_data': {}} cfg = {} ud = "" + password = None + username = None for child in lpcs.childNodes: if child.nodeType == dom.TEXT_NODE or not child.localName: @@ -176,19 +178,36 @@ def read_azure_ovf(contents): if name == "userdata": ud = base64.b64decode(''.join(value.split())) elif name == "username": - cfg['system_info'] = {'default_user': {'name': value}} + username = value + elif name == "userpassword": + password = value elif name == "hostname": md['local-hostname'] = value elif name == "dscfg": cfg['datasource'] = {DS_NAME: util.load_yaml(value, default={})} elif name == "ssh": cfg['_pubkeys'] = loadAzurePubkeys(child) + elif name == "disablesshpasswordauthentication": + cfg['ssh_pwauth'] = util.is_true(value) elif simple: if name in md_props: md[name] = value else: md['azure_data'][name] = value + defuser = {} + if username: + defuser['name'] = username + if password: + defuser['password'] = password + defuser['lock_passwd'] = False + + if defuser: + cfg['system_info'] = {'default_user': defuser} + + if 'ssh_pwauth' not in cfg and password: + cfg['ssh_pwauth'] = True + return (md, ud, cfg) diff --git a/tests/unittests/test_datasource/test_azure.py b/tests/unittests/test_datasource/test_azure.py index 179fb50a..a2347f1b 100644 --- a/tests/unittests/test_datasource/test_azure.py +++ b/tests/unittests/test_datasource/test_azure.py @@ -131,6 +131,33 @@ class TestAzureDataSource(MockerTestCase): self.assertTrue(ret) self.assertEqual(data['agent_invoked'], '_COMMAND') + def test_username_used(self): + odata = {'HostName': "myhost", 'UserName': "myuser"} + data = {'ovfcontent': construct_valid_ovf_env(data=odata)} + + dsrc = self._get_ds(data) + ret = dsrc.get_data() + self.assertTrue(ret) + self.assertEqual(dsrc.cfg['system_info']['default_user']['name'], + "myuser") + + def test_password_given(self): + odata = {'HostName': "myhost", 'UserName': "myuser", + 'UserPassword': "mypass"} + data = {'ovfcontent': construct_valid_ovf_env(data=odata)} + + dsrc = self._get_ds(data) + ret = dsrc.get_data() + self.assertTrue(ret) + self.assertTrue('default_user' in dsrc.cfg['system_info']) + defuser = dsrc.cfg['system_info']['default_user'] + + # default user shoudl be updated for password and username + # and should not be locked. + self.assertEqual(defuser['name'], odata['UserName']) + self.assertEqual(defuser['password'], odata['UserPassword']) + self.assertFalse(defuser['lock_passwd']) + def test_userdata_found(self): mydata = "FOOBAR" odata = {'UserData': base64.b64encode(mydata)} -- cgit v1.2.3 From cf1b10900626dfa6194c77b6720291e7edbaf9f6 Mon Sep 17 00:00:00 2001 From: Scott Moser Date: Tue, 9 Jul 2013 15:07:38 -0400 Subject: populate /var/lib/waagent with ovf-env.xml this will copy the ovf-env.xml file that was found to the configured directory (default /var/lib/waagent) --- cloudinit/sources/DataSourceAzure.py | 43 +++++++++++++++++++-------- tests/unittests/test_datasource/test_azure.py | 8 +++++ 2 files changed, 39 insertions(+), 12 deletions(-) (limited to 'cloudinit/sources') diff --git a/cloudinit/sources/DataSourceAzure.py b/cloudinit/sources/DataSourceAzure.py index 5037c1a3..f1c7c771 100644 --- a/cloudinit/sources/DataSourceAzure.py +++ b/cloudinit/sources/DataSourceAzure.py @@ -30,7 +30,9 @@ LOG = logging.getLogger(__name__) DS_NAME = 'Azure' DEFAULT_METADATA = {"instance-id": "iid-AZURE-NODE"} AGENT_START = ['service', 'walinuxagent', 'start'] -BUILTIN_DS_CONFIG = {'datasource': {DS_NAME: {'agent_command': AGENT_START}}} +BUILTIN_DS_CONFIG = {'datasource': {DS_NAME: { + 'agent_command': AGENT_START, + 'data_dir': "/var/lib/waagent"}}} class DataSourceAzureNet(sources.DataSource): @@ -64,7 +66,7 @@ class DataSourceAzureNet(sources.DataSource): LOG.warn("%s was not mountable" % cdev) continue - (md, self.userdata_raw, cfg) = ret + (md, self.userdata_raw, cfg, files) = ret self.seed = cdev self.metadata = util.mergemanydict([md, DEFAULT_METADATA]) self.cfg = cfg @@ -76,18 +78,24 @@ class DataSourceAzureNet(sources.DataSource): if not found: return False - path = ['datasource', DS_NAME, 'agent_command'] - cmd = None + fields = [('cmd', ['datasource', DS_NAME, 'agent_command']), + ('datadir', ['datasource', DS_NAME, 'data_dir'])] + mycfg = {} for cfg in (self.cfg, self.sys_cfg, BUILTIN_DS_CONFIG): - cmd = util.get_cfg_by_path(cfg, keyp=path) - if cmd is not None: - break + for name, path in fields: + if name in mycfg: + continue + value = util.get_cfg_by_path(cfg, keyp=path) + if value is not None: + mycfg[name] = value + + write_files(mycfg['datadir'], files) try: - invoke_agent(cmd) + invoke_agent(mycfg['cmd']) except util.ProcessExecutionError: # claim the datasource even if the command failed - util.logexc(LOG, "agent command '%s' failed.", cmd) + util.logexc(LOG, "agent command '%s' failed.", mycfg['cmd']) return True @@ -95,6 +103,16 @@ class DataSourceAzureNet(sources.DataSource): return self.cfg +def write_files(datadir, files): + if not datadir: + return + if not files: + files = {} + for (name, content) in files.items(): + util.write_file(filename=os.path.join(datadir, name), + content=content, mode=0600) + + def invoke_agent(cmd): # this is a function itself to simplify patching it for test if cmd: @@ -114,7 +132,7 @@ def find_child(node, filter_func): return ret -def load_azure_ovf_pubkeys(sshnode): +def load_azure_ovf_pubkeys(_sshnode): # in the future this would return a list of dicts like: # [{'fp': '6BE7A7C3C8A8F4B123CCA5D0C2F1BE4CA7B63ED7', # 'path': 'where/to/go'}] @@ -186,7 +204,7 @@ def read_azure_ovf(contents): elif name == "dscfg": cfg['datasource'] = {DS_NAME: util.load_yaml(value, default={})} elif name == "ssh": - cfg['_pubkeys'] = loadAzurePubkeys(child) + cfg['_pubkeys'] = load_azure_ovf_pubkeys(child) elif name == "disablesshpasswordauthentication": cfg['ssh_pwauth'] = util.is_true(value) elif simple: @@ -230,7 +248,8 @@ def load_azure_ds_dir(source_dir): with open(ovf_file, "r") as fp: contents = fp.read() - return read_azure_ovf(contents) + md, ud, cfg = read_azure_ovf(contents) + return (md, ud, cfg, {'ovf-env.xml': contents}) class BrokenAzureDataSource(Exception): diff --git a/tests/unittests/test_datasource/test_azure.py b/tests/unittests/test_datasource/test_azure.py index a2347f1b..68f4bcca 100644 --- a/tests/unittests/test_datasource/test_azure.py +++ b/tests/unittests/test_datasource/test_azure.py @@ -83,6 +83,12 @@ class TestAzureDataSource(MockerTestCase): def invoker(cmd): data['agent_invoked'] = cmd + def file_writer(datadir, files): + data['files'] = {} + data['datadir'] = datadir + for (fname, content) in files.items(): + data['files'][fname] = content + if data.get('ovfcontent') is not None: populate_dir(os.path.join(self.paths.seed_dir, "azure"), {'ovf-env.xml': data['ovfcontent']}) @@ -93,6 +99,7 @@ class TestAzureDataSource(MockerTestCase): self.apply_patches([(mod, 'list_possible_azure_ds_devs', dsdevs)]) self.apply_patches([(mod, 'invoke_agent', invoker)]) + self.apply_patches([(mod, 'write_files', file_writer)]) dsrc = mod.DataSourceAzureNet( data.get('sys_cfg', {}), distro=None, paths=self.paths) @@ -109,6 +116,7 @@ class TestAzureDataSource(MockerTestCase): self.assertTrue(ret) self.assertEqual(dsrc.userdata_raw, "") self.assertEqual(dsrc.metadata['local-hostname'], odata['HostName']) + self.assertTrue('ovf-env.xml' in data['files']) def test_user_cfg_set_agent_command(self): cfg = {'agent_command': "my_command"} -- cgit v1.2.3 From c12845193066ac2eb14b9bbef75657d579b696b5 Mon Sep 17 00:00:00 2001 From: Scott Moser Date: Tue, 9 Jul 2013 15:49:55 -0400 Subject: search in the default storage directory for cached ovf-env.xml since azure ejects the disk on reboot, we need to look there to find this datasource. --- cloudinit/sources/DataSourceAzure.py | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) (limited to 'cloudinit/sources') diff --git a/cloudinit/sources/DataSourceAzure.py b/cloudinit/sources/DataSourceAzure.py index f1c7c771..92b6172b 100644 --- a/cloudinit/sources/DataSourceAzure.py +++ b/cloudinit/sources/DataSourceAzure.py @@ -47,8 +47,19 @@ class DataSourceAzureNet(sources.DataSource): return "%s [seed=%s]" % (root, self.seed) def get_data(self): + ddir_cfgpath = ['datasource', DS_NAME, 'data_dir'] + # azure removes/ejects the cdrom containing the ovf-env.xml + # file on reboot. So, in order to successfully reboot we + # need to look in the datadir and consider that valid + ddir = util.get_cfg_by_path(self.sys_cfg, ddir_cfgpath) + if ddir is None: + ddir = util.get_cfg_by_path(BUILTIN_DS_CONFIG, ddir_cfgpath) + candidates = [self.seed_dir] candidates.extend(list_possible_azure_ds_devs()) + if ddir: + candidates.append(ddir) + found = None for cdev in candidates: @@ -79,7 +90,7 @@ class DataSourceAzureNet(sources.DataSource): return False fields = [('cmd', ['datasource', DS_NAME, 'agent_command']), - ('datadir', ['datasource', DS_NAME, 'data_dir'])] + ('datadir', ddir_cfgpath)] mycfg = {} for cfg in (self.cfg, self.sys_cfg, BUILTIN_DS_CONFIG): for name, path in fields: -- cgit v1.2.3 From 4368b264be42472c53bc3333587c7029373ad56a Mon Sep 17 00:00:00 2001 From: Scott Moser Date: Tue, 9 Jul 2013 16:03:22 -0400 Subject: mention using cached --- cloudinit/sources/DataSourceAzure.py | 3 +++ 1 file changed, 3 insertions(+) (limited to 'cloudinit/sources') diff --git a/cloudinit/sources/DataSourceAzure.py b/cloudinit/sources/DataSourceAzure.py index 92b6172b..d8e39392 100644 --- a/cloudinit/sources/DataSourceAzure.py +++ b/cloudinit/sources/DataSourceAzure.py @@ -89,6 +89,9 @@ class DataSourceAzureNet(sources.DataSource): if not found: return False + if found == ddir: + LOG.debug("using cached datasource in %s", ddir) + fields = [('cmd', ['datasource', DS_NAME, 'agent_command']), ('datadir', ddir_cfgpath)] mycfg = {} -- cgit v1.2.3 From 950762bb008d25f529c71aae4c0b04f6b0134abb Mon Sep 17 00:00:00 2001 From: Scott Moser Date: Tue, 9 Jul 2013 20:20:55 -0400 Subject: fill out load_azure_ovf_pubkeys now if there are pubkeys, the cfg['_pubkeys'] entry will have a list of dicts where each dict has 'fingerprint' and 'path' entries. The next thing to do is to block waiting for the .crt files to appear in /var/lib/waagent. --- cloudinit/sources/DataSourceAzure.py | 40 +++++++++++++++++++++++++-- tests/unittests/test_datasource/test_azure.py | 13 +++++++-- 2 files changed, 49 insertions(+), 4 deletions(-) (limited to 'cloudinit/sources') diff --git a/cloudinit/sources/DataSourceAzure.py b/cloudinit/sources/DataSourceAzure.py index d8e39392..43a963ad 100644 --- a/cloudinit/sources/DataSourceAzure.py +++ b/cloudinit/sources/DataSourceAzure.py @@ -146,7 +146,7 @@ def find_child(node, filter_func): return ret -def load_azure_ovf_pubkeys(_sshnode): +def load_azure_ovf_pubkeys(sshnode): # in the future this would return a list of dicts like: # [{'fp': '6BE7A7C3C8A8F4B123CCA5D0C2F1BE4CA7B63ED7', # 'path': 'where/to/go'}] @@ -155,7 +155,43 @@ def load_azure_ovf_pubkeys(_sshnode): # ABC/ABC # ... # - return [] + results = find_child(sshnode, lambda n: n.localName == "PublicKeys") + if len(results) == 0: + return [] + if len(results) > 1: + raise BrokenAzureDataSource("Multiple 'PublicKeys'(%s) in SSH node" % + len(results)) + + pubkeys_node = results[0] + pubkeys = find_child(pubkeys_node, lambda n: n.localName == "PublicKey") + + if len(pubkeys) == 0: + return [] + + found = [] + text_node = minidom.Document.TEXT_NODE + + for pk_node in pubkeys: + if not pk_node.hasChildNodes(): + continue + cur = {'fingerprint': "", 'path': ""} + for child in pk_node.childNodes: + if (child.nodeType == text_node or not child.localName): + continue + + name = child.localName.lower() + + if name not in cur.keys(): + continue + + if (len(child.childNodes) != 1 or + child.childNodes[0].nodeType != text_node): + continue + + cur[name] = child.childNodes[0].wholeText.strip() + found.append(cur) + + return found def read_azure_ovf(contents): diff --git a/tests/unittests/test_datasource/test_azure.py b/tests/unittests/test_datasource/test_azure.py index 68f4bcca..be6fab70 100644 --- a/tests/unittests/test_datasource/test_azure.py +++ b/tests/unittests/test_datasource/test_azure.py @@ -34,11 +34,12 @@ def construct_valid_ovf_env(data=None, pubkeys=None, userdata=None): if pubkeys: content += "\n" - for fp, path in pubkeys.items(): + for fp, path in pubkeys: content += " " content += ("%s%s" % (fp, path)) - content += " " + content += "\n" + content += "" content += """ @@ -191,6 +192,14 @@ class TestReadAzureOvf(MockerTestCase): self.assertRaises(DataSourceAzure.NonAzureDataSource, DataSourceAzure.read_azure_ovf, invalid_xml) + def test_load_with_pubkeys(self): + mypklist = [{'fingerprint': 'fp1', 'path': 'path1'}] + pubkeys = [(x['fingerprint'], x['path']) for x in mypklist] + content = construct_valid_ovf_env(pubkeys=pubkeys) + (md, ud, cfg) = DataSourceAzure.read_azure_ovf(content) + for mypk in mypklist: + self.assertIn(mypk, cfg['_pubkeys']) + def apply_patches(patches): ret = [] -- cgit v1.2.3 From ec22feeefe309187107e0fb5471136f1c8a646c9 Mon Sep 17 00:00:00 2001 From: Scott Moser Date: Tue, 9 Jul 2013 20:36:28 -0400 Subject: build up the 'wait_for' list including fingerprint.crt files --- cloudinit/sources/DataSourceAzure.py | 6 ++++++ tests/unittests/test_datasource/test_azure.py | 13 +++++++++++++ 2 files changed, 19 insertions(+) (limited to 'cloudinit/sources') diff --git a/cloudinit/sources/DataSourceAzure.py b/cloudinit/sources/DataSourceAzure.py index 43a963ad..ab570344 100644 --- a/cloudinit/sources/DataSourceAzure.py +++ b/cloudinit/sources/DataSourceAzure.py @@ -111,6 +111,12 @@ class DataSourceAzureNet(sources.DataSource): # claim the datasource even if the command failed util.logexc(LOG, "agent command '%s' failed.", mycfg['cmd']) + wait_for = [os.path.join(mycfg['datadir'], "SharedConfig.xml")] + + for pk in self.cfg.get('_pubkeys', []): + bname = pk['fingerprint'] + ".crt" + wait_for += [os.path.join(mycfg['datadir'], bname)] + return True def get_config_obj(self): diff --git a/tests/unittests/test_datasource/test_azure.py b/tests/unittests/test_datasource/test_azure.py index be6fab70..a7094ec6 100644 --- a/tests/unittests/test_datasource/test_azure.py +++ b/tests/unittests/test_datasource/test_azure.py @@ -185,6 +185,19 @@ class TestAzureDataSource(MockerTestCase): self.assertFalse(ret) self.assertFalse('agent_invoked' in data) + def test_cfg_has_pubkeys(self): + odata = {'HostName': "myhost", 'UserName': "myuser"} + mypklist = [{'fingerprint': 'fp1', 'path': 'path1'}] + pubkeys = [(x['fingerprint'], x['path']) for x in mypklist] + data = {'ovfcontent': construct_valid_ovf_env(data=odata, + pubkeys=pubkeys)} + + dsrc = self._get_ds(data) + ret = dsrc.get_data() + self.assertTrue(ret) + for mypk in mypklist: + self.assertIn(mypk, dsrc.cfg['_pubkeys']) + class TestReadAzureOvf(MockerTestCase): def test_invalid_xml_raises_non_azure_ds(self): -- cgit v1.2.3 From ce949d5b4c94caf9c1df6393abe86de2872e05ae Mon Sep 17 00:00:00 2001 From: Scott Moser Date: Wed, 10 Jul 2013 13:08:23 -0400 Subject: add waiting for files and reading of crt keys --- cloudinit/sources/DataSourceAzure.py | 46 ++++++++++++++++++++++++++- packages/debian/changelog.in | 2 +- tests/unittests/test_datasource/test_azure.py | 21 +++++++++--- 3 files changed, 62 insertions(+), 7 deletions(-) (limited to 'cloudinit/sources') diff --git a/cloudinit/sources/DataSourceAzure.py b/cloudinit/sources/DataSourceAzure.py index ab570344..200bede5 100644 --- a/cloudinit/sources/DataSourceAzure.py +++ b/cloudinit/sources/DataSourceAzure.py @@ -19,6 +19,7 @@ import base64 import os import os.path +import time from xml.dom import minidom from cloudinit import log as logging @@ -113,9 +114,18 @@ class DataSourceAzureNet(sources.DataSource): wait_for = [os.path.join(mycfg['datadir'], "SharedConfig.xml")] + fp_files = [] for pk in self.cfg.get('_pubkeys', []): bname = pk['fingerprint'] + ".crt" - wait_for += [os.path.join(mycfg['datadir'], bname)] + fp_files += [os.path.join(mycfg['datadir'], bname)] + + missing = wait_for_files(wait_for + fp_files) + if len(missing): + LOG.warn("Did not find files, but going on: %s" % missing) + + pubkeys = pubkeys_from_crt_files(fp_files) + + self.metadata['public-keys'] = pubkeys return True @@ -123,6 +133,40 @@ class DataSourceAzureNet(sources.DataSource): return self.cfg +def crtfile_to_pubkey(fname): + pipeline = ('openssl x509 -noout -pubkey < "$0" |' + 'ssh-keygen -i -m PKCS8 -f /dev/stdin') + (out, _err) = util.subp(['sh', '-c', pipeline, fname], capture=True) + return out.rstrip() + + +def pubkeys_from_crt_files(flist): + pubkeys = [] + errors = [] + for fname in flist: + try: + pubkeys.append(crtfile_to_pubkey(fname)) + except util.ProcessExecutionError: + errors.extend(fname) + + if errors: + LOG.warn("failed to convert the crt files to pubkey: %s" % errors) + + return pubkeys + + +def wait_for_files(flist, maxwait=60, naplen=.5): + need = set(flist) + waited = 0 + while waited < maxwait: + need -= set([f for f in need if os.path.exists(f)]) + if len(need) == 0: + return [] + time.sleep(naplen) + waited += naplen + return need + + def write_files(datadir, files): if not datadir: return diff --git a/packages/debian/changelog.in b/packages/debian/changelog.in index e3e94f54..4944230b 100644 --- a/packages/debian/changelog.in +++ b/packages/debian/changelog.in @@ -1,5 +1,5 @@ ## This is a cheetah template -cloud-init (${version}~bzr${revision}-1) UNRELEASED; urgency=low +cloud-init (${version}~bzr${revision}-1) raring; urgency=low * build diff --git a/tests/unittests/test_datasource/test_azure.py b/tests/unittests/test_datasource/test_azure.py index a7094ec6..74ed7197 100644 --- a/tests/unittests/test_datasource/test_azure.py +++ b/tests/unittests/test_datasource/test_azure.py @@ -81,15 +81,23 @@ class TestAzureDataSource(MockerTestCase): def dsdevs(): return data.get('dsdevs', []) - def invoker(cmd): + def _invoke_agent(cmd): data['agent_invoked'] = cmd - def file_writer(datadir, files): + def _write_files(datadir, files): data['files'] = {} data['datadir'] = datadir for (fname, content) in files.items(): data['files'][fname] = content + def _wait_for_files(flist, _maxwait=None, _naplen=None): + data['waited'] = flist + return [] + + def _pubkeys_from_crt_files(flist): + data['pubkey_files'] = flist + return ["pubkey_from: %s" % f for f in flist] + if data.get('ovfcontent') is not None: populate_dir(os.path.join(self.paths.seed_dir, "azure"), {'ovf-env.xml': data['ovfcontent']}) @@ -99,8 +107,11 @@ class TestAzureDataSource(MockerTestCase): if data.get('dsdevs'): self.apply_patches([(mod, 'list_possible_azure_ds_devs', dsdevs)]) - self.apply_patches([(mod, 'invoke_agent', invoker)]) - self.apply_patches([(mod, 'write_files', file_writer)]) + self.apply_patches([(mod, 'invoke_agent', _invoke_agent), + (mod, 'write_files', _write_files), + (mod, 'wait_for_files', _wait_for_files), + (mod, 'pubkeys_from_crt_files', + _pubkeys_from_crt_files)]) dsrc = mod.DataSourceAzureNet( data.get('sys_cfg', {}), distro=None, paths=self.paths) @@ -209,7 +220,7 @@ class TestReadAzureOvf(MockerTestCase): mypklist = [{'fingerprint': 'fp1', 'path': 'path1'}] pubkeys = [(x['fingerprint'], x['path']) for x in mypklist] content = construct_valid_ovf_env(pubkeys=pubkeys) - (md, ud, cfg) = DataSourceAzure.read_azure_ovf(content) + (_md, _ud, cfg) = DataSourceAzure.read_azure_ovf(content) for mypk in mypklist: self.assertIn(mypk, cfg['_pubkeys']) -- cgit v1.2.3 From 8c15320283012d218c2165f5dfb38a29d2cf7b53 Mon Sep 17 00:00:00 2001 From: Scott Moser Date: Thu, 11 Jul 2013 10:16:18 -0400 Subject: DataSourceAzure.py: allow 'customdata' to be the name for userdata Also, fix a comment and write a log message on how long we waited for the files to appear. --- cloudinit/sources/DataSourceAzure.py | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) (limited to 'cloudinit/sources') diff --git a/cloudinit/sources/DataSourceAzure.py b/cloudinit/sources/DataSourceAzure.py index 200bede5..2818408c 100644 --- a/cloudinit/sources/DataSourceAzure.py +++ b/cloudinit/sources/DataSourceAzure.py @@ -119,9 +119,13 @@ class DataSourceAzureNet(sources.DataSource): bname = pk['fingerprint'] + ".crt" fp_files += [os.path.join(mycfg['datadir'], bname)] + start = time.time() missing = wait_for_files(wait_for + fp_files) if len(missing): - LOG.warn("Did not find files, but going on: %s" % missing) + LOG.warn("Did not find files, but going on: %s", missing) + else: + LOG.debug("waited %.3f seconds for %d files to appear", + time.time() - start, len(wait_for)) pubkeys = pubkeys_from_crt_files(fp_files) @@ -197,7 +201,8 @@ def find_child(node, filter_func): def load_azure_ovf_pubkeys(sshnode): - # in the future this would return a list of dicts like: + # This parses a 'SSH' node formatted like below, and returns + # an array of dicts. # [{'fp': '6BE7A7C3C8A8F4B123CCA5D0C2F1BE4CA7B63ED7', # 'path': 'where/to/go'}] # @@ -293,7 +298,9 @@ def read_azure_ovf(contents): simple = True value = child.childNodes[0].wholeText - if name == "userdata": + # we accept either UserData or CustomData. If both are present + # then behavior is undefined. + if (name == "userdata" or name == "customdata"): ud = base64.b64decode(''.join(value.split())) elif name == "username": username = value -- cgit v1.2.3 From e2e5becb83827e741bfaaeac5edc1c7937718faa Mon Sep 17 00:00:00 2001 From: Ben Howard Date: Tue, 16 Jul 2013 15:21:26 -0600 Subject: Use the inverse of DisableSshPasswordAuthentication for ssh_pwauth. (LP: 1201969) --- cloudinit/sources/DataSourceAzure.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) (limited to 'cloudinit/sources') diff --git a/cloudinit/sources/DataSourceAzure.py b/cloudinit/sources/DataSourceAzure.py index 2818408c..5071ee67 100644 --- a/cloudinit/sources/DataSourceAzure.py +++ b/cloudinit/sources/DataSourceAzure.py @@ -313,7 +313,10 @@ def read_azure_ovf(contents): elif name == "ssh": cfg['_pubkeys'] = load_azure_ovf_pubkeys(child) elif name == "disablesshpasswordauthentication": - cfg['ssh_pwauth'] = util.is_true(value) + # The verb 'disablessh..." asks whether to disable password auth. + # Since it is disabled by default, "false" indicates that + # password authentication should be enabled. + cfg['ssh_pwauth'] = util.is_false(value) elif simple: if name in md_props: md[name] = value -- cgit v1.2.3 From 8f70bb7e7144f2225b4e9a589d16ae6d15992a3d Mon Sep 17 00:00:00 2001 From: Scott Moser Date: Wed, 17 Jul 2013 13:36:32 -0400 Subject: Azure: make /var/lib/waagent with 0700 perms The walinux agent expects that the files it writes with 0644 (default umask) permissions are not globally readable. Since we were creating the directory for it, and using default umaks (0755), the files inside were readable to non-priviledged users. --- cloudinit/sources/DataSourceAzure.py | 7 +++++-- tests/unittests/test_datasource/test_azure.py | 4 +++- 2 files changed, 8 insertions(+), 3 deletions(-) (limited to 'cloudinit/sources') diff --git a/cloudinit/sources/DataSourceAzure.py b/cloudinit/sources/DataSourceAzure.py index f1419296..c90d7b07 100644 --- a/cloudinit/sources/DataSourceAzure.py +++ b/cloudinit/sources/DataSourceAzure.py @@ -104,7 +104,9 @@ class DataSourceAzureNet(sources.DataSource): if value is not None: mycfg[name] = value - write_files(mycfg['datadir'], files) + # walinux agent writes files world readable, but expects + # the directory to be protected. + write_files(mycfg['datadir'], files, dirmode=0700) try: invoke_agent(mycfg['cmd']) @@ -171,11 +173,12 @@ def wait_for_files(flist, maxwait=60, naplen=.5): return need -def write_files(datadir, files): +def write_files(datadir, files, dirmode=None): if not datadir: return if not files: files = {} + util.ensure_dir(datadir, dirmode) for (name, content) in files.items(): util.write_file(filename=os.path.join(datadir, name), content=content, mode=0600) diff --git a/tests/unittests/test_datasource/test_azure.py b/tests/unittests/test_datasource/test_azure.py index 74ed7197..c79c25d8 100644 --- a/tests/unittests/test_datasource/test_azure.py +++ b/tests/unittests/test_datasource/test_azure.py @@ -84,9 +84,10 @@ class TestAzureDataSource(MockerTestCase): def _invoke_agent(cmd): data['agent_invoked'] = cmd - def _write_files(datadir, files): + def _write_files(datadir, files, dirmode): data['files'] = {} data['datadir'] = datadir + data['datadir_mode'] = dirmode for (fname, content) in files.items(): data['files'][fname] = content @@ -129,6 +130,7 @@ class TestAzureDataSource(MockerTestCase): self.assertEqual(dsrc.userdata_raw, "") self.assertEqual(dsrc.metadata['local-hostname'], odata['HostName']) self.assertTrue('ovf-env.xml' in data['files']) + self.assertEqual(0700, data['datadir_mode']) def test_user_cfg_set_agent_command(self): cfg = {'agent_command': "my_command"} -- cgit v1.2.3 From 6b7e65e4f57902c25363c78a7e47aa2caa579b7b Mon Sep 17 00:00:00 2001 From: Ben Howard Date: Thu, 18 Jul 2013 15:37:18 -0600 Subject: Added SmartOS datasource and unit tests. --- cloudinit/settings.py | 1 + cloudinit/sources/DataSourceSmartOS.py | 172 +++++++++++++++++++++ cloudinit/util.py | 18 +++ tests/unittests/test_datasource/test_smartos.py | 191 ++++++++++++++++++++++++ 4 files changed, 382 insertions(+) create mode 100644 cloudinit/sources/DataSourceSmartOS.py create mode 100644 tests/unittests/test_datasource/test_smartos.py (limited to 'cloudinit/sources') diff --git a/cloudinit/settings.py b/cloudinit/settings.py index dc371cd2..9f6badae 100644 --- a/cloudinit/settings.py +++ b/cloudinit/settings.py @@ -37,6 +37,7 @@ CFG_BUILTIN = { 'MAAS', 'Ec2', 'CloudStack', + 'SmartOS', # At the end to act as a 'catch' when none of the above work... 'None', ], diff --git a/cloudinit/sources/DataSourceSmartOS.py b/cloudinit/sources/DataSourceSmartOS.py new file mode 100644 index 00000000..f9b724eb --- /dev/null +++ b/cloudinit/sources/DataSourceSmartOS.py @@ -0,0 +1,172 @@ +# vi: ts=4 expandtab +# +# Copyright (C) 2013 Canonical Ltd. +# +# Author: Ben Howard +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 3, as +# published by the Free Software Foundation. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +# +# +# Datasource for provisioning on SmartOS. This works on Joyent +# and public/private Clouds using SmartOS. +# +# SmartOS hosts use a serial console (/dev/ttyS1) on Linux Guests. +# The meta-data is transmitted via key/value pairs made by +# requests on the console. For example, to get the hostname, you +# would send "GET hostname" on /dev/ttyS1. +# + + +import os +import os.path +import serial +from cloudinit import log as logging +from cloudinit import sources +from cloudinit import util + + +TTY_LOC = '/dev/ttyS1' +LOG = logging.getLogger(__name__) + + +class DataSourceSmartOS(sources.DataSource): + def __init__(self, sys_cfg, distro, paths): + sources.DataSource.__init__(self, sys_cfg, distro, paths) + self.seed_dir = os.path.join(paths.seed_dir, 'sdc') + self.seed = None + self.is_smartdc = None + + def __str__(self): + root = sources.DataSource.__str__(self) + return "%s [seed=%s]" % (root, self.seed) + + def get_data(self): + md = {} + ud = "" + + if not os.path.exists(TTY_LOC): + LOG.debug("Host does not appear to be on SmartOS") + return False + self.seed = TTY_LOC + + system_uuid, system_type = dmi_data() + if 'smartdc' not in system_type.lower(): + LOG.debug("Host is not on SmartOS") + return False + self.is_smartdc = True + + hostname = query_data("hostname", strip=True) + if not hostname: + hostname = system_uuid + + md['local-hostname'] = hostname + md['instance-id'] = system_uuid + md['public-keys'] = query_data("root_authorized_keys", strip=True) + ud = query_data("user-script") + md['iptables_disable'] = query_data("disable_iptables_flag", + strip=True) + md['motd_sys_info'] = query_data("enable_motd_sys_info", strip=True) + + self.metadata = md + self.userdata_raw = ud + return True + + def get_instance_id(self): + return self.metadata['instance-id'] + + +def get_serial(): + """This is replaced in unit testing, allowing us to replace + serial.Serial with a mocked class""" + return serial.Serial() + + +def query_data(noun, strip=False): + """Makes a request to via the serial console via "GET " + + In the response, the first line is the status, while subsequent lines + are is the value. A blank line with a "." is used to indicate end of + response. + + The timeout value of 60 seconds should never be hit. The value + is taken from SmartOS own provisioning tools. Since we are reading + each line individually up until the single ".", the transfer is + usually very fast (i.e. microseconds) to get the response. + """ + if not noun: + return False + + ser = get_serial() + ser.port = '/dev/ttyS1' + ser.open() + if not ser.isOpen(): + LOG.debug("Serial console is not open") + return False + + ser.write("GET %s\n" % noun.rstrip()) + status = str(ser.readline()).rstrip() + response = [] + eom_found = False + + if 'SUCCESS' not in status: + ser.close() + return None + + while not eom_found: + m = ser.readline() + if m.rstrip() == ".": + eom_found = True + else: + response.append(m) + + ser.close() + if not strip: + return "".join(response) + else: + return "".join(response).rstrip() + + return None + + +def dmi_data(): + sys_uuid, sys_type = None, None + dmidecode_path = util.which('dmidecode') + if not dmidecode_path: + return False + + sys_uuid_cmd = [dmidecode_path, "-s", "system-uuid"] + try: + LOG.debug("Getting hostname from dmidecode") + (sys_uuid, _err) = util.subp(sys_uuid_cmd) + except Exception as e: + util.logexc(LOG, "Failed to get system UUID", e) + + sys_type_cmd = [dmidecode_path, "-s", "system-product-name"] + try: + LOG.debug("Determining hypervisor product name via dmidecode") + (sys_type, _err) = util.subp(sys_type_cmd) + except Exception as e: + util.logexc(LOG, "Failed to get system UUID", e) + + return sys_uuid.lower(), sys_type + + +# Used to match classes to dependencies +datasources = [ + (DataSourceSmartOS, (sources.DEP_FILESYSTEM, sources.DEP_NETWORK)), +] + + +# Return a list of data sources that match this set of dependencies +def get_datasource_list(depends): + return sources.list_from_depends(depends, datasources) diff --git a/cloudinit/util.py b/cloudinit/util.py index c45aae06..7163225f 100644 --- a/cloudinit/util.py +++ b/cloudinit/util.py @@ -1743,3 +1743,21 @@ def get_mount_info(path, log=LOG): mountinfo_path = '/proc/%s/mountinfo' % os.getpid() lines = load_file(mountinfo_path).splitlines() return parse_mount_info(path, lines, log) + +def which(program): + # Return path of program for execution if found in path + def is_exe(fpath): + return os.path.isfile(fpath) and os.access(fpath, os.X_OK) + + fpath, fname = os.path.split(program) + if fpath: + if is_exe(program): + return program + else: + for path in os.environ["PATH"].split(os.pathsep): + path = path.strip('"') + exe_file = os.path.join(path, program) + if is_exe(exe_file): + return exe_file + + return None diff --git a/tests/unittests/test_datasource/test_smartos.py b/tests/unittests/test_datasource/test_smartos.py new file mode 100644 index 00000000..494f9828 --- /dev/null +++ b/tests/unittests/test_datasource/test_smartos.py @@ -0,0 +1,191 @@ +# vi: ts=4 expandtab +# +# Copyright (C) 2013 Canonical Ltd. +# +# Author: Ben Howard +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 3, as +# published by the Free Software Foundation. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +# +# +# This is a testcase for the SmartOS datasource. It replicates a serial +# console and acts like the SmartOS console does in order to validate +# return responses. +# + +from cloudinit import helpers +from cloudinit.sources import DataSourceSmartOS + +from mocker import MockerTestCase +import uuid + +mock_returns = { + 'hostname': 'test-host', + 'root_authorized_keys': 'ssh-rsa AAAAB3Nz...aC1yc2E= keyname', + 'disable_iptables_flag': False, + 'enable_motd_sys_info': False, + 'system_uuid': str(uuid.uuid4()), + 'smartdc': 'smartdc', + 'userdata': """ +#!/bin/sh +/bin/true +""", +} + + +class MockSerial(object): + """Fake a serial terminal for testing the code that + interfaces with the serial""" + + port = None + + def __init__(self): + self.last = None + self.last = None + self.new = True + self.count = 0 + self.mocked_out = [] + + def open(self): + return True + + def close(self): + return True + + def isOpen(self): + return True + + def write(self, line): + line = line.replace('GET ', '') + self.last = line.rstrip() + + def readline(self): + if self.new: + self.new = False + if self.last in mock_returns: + return 'SUCCESS\n' + else: + return 'NOTFOUND %s\n' % self.last + + if self.last in mock_returns: + if not self.mocked_out: + self.mocked_out = [x for x in self._format_out()] + print self.mocked_out + + if len(self.mocked_out) > self.count: + self.count += 1 + return self.mocked_out[self.count - 1] + + def _format_out(self): + if self.last in mock_returns: + try: + for l in mock_returns[self.last].splitlines(): + yield "%s\n" % l + except: + yield "%s\n" % mock_returns[self.last] + + yield '\n' + yield '.' + + +class TestSmartOSDataSource(MockerTestCase): + def setUp(self): + # makeDir comes from MockerTestCase + self.tmp = self.makeDir() + + # patch cloud_dir, so our 'seed_dir' is guaranteed empty + self.paths = helpers.Paths({'cloud_dir': self.tmp}) + + self.unapply = [] + super(TestSmartOSDataSource, self).setUp() + + def tearDown(self): + apply_patches([i for i in reversed(self.unapply)]) + super(TestSmartOSDataSource, self).tearDown() + + def apply_patches(self, patches): + ret = apply_patches(patches) + self.unapply += ret + + def _get_ds(self): + + def _get_serial(): + return MockSerial() + + def _dmi_data(): + return mock_returns['system_uuid'], 'smartdc' + + data = {'sys_cfg': {}} + mod = DataSourceSmartOS + self.apply_patches([(mod, 'get_serial', _get_serial)]) + self.apply_patches([(mod, 'dmi_data', _dmi_data)]) + dsrc = mod.DataSourceSmartOS( + data.get('sys_cfg', {}), distro=None, paths=self.paths) + return dsrc + + def test_seed(self): + dsrc = self._get_ds() + ret = dsrc.get_data() + self.assertTrue(ret) + self.assertEquals('/dev/ttyS1', dsrc.seed) + + def test_issmartdc(self): + dsrc = self._get_ds() + ret = dsrc.get_data() + self.assertTrue(ret) + self.assertTrue(dsrc.is_smartdc) + + def test_uuid(self): + dsrc = self._get_ds() + ret = dsrc.get_data() + self.assertTrue(ret) + self.assertEquals(mock_returns['system_uuid'], + dsrc.metadata['instance-id']) + + def test_root_keys(self): + dsrc = self._get_ds() + ret = dsrc.get_data() + self.assertTrue(ret) + self.assertEquals(mock_returns['root_authorized_keys'], + dsrc.metadata['public-keys']) + + def test_hostname(self): + dsrc = self._get_ds() + ret = dsrc.get_data() + self.assertTrue(ret) + self.assertEquals(mock_returns['hostname'], + dsrc.metadata['local-hostname']) + + def test_disable_iptables_flag(self): + dsrc = self._get_ds() + ret = dsrc.get_data() + self.assertTrue(ret) + self.assertEquals(str(mock_returns['disable_iptables_flag']), + dsrc.metadata['iptables_disable']) + + def test_motd_sys_info(self): + dsrc = self._get_ds() + ret = dsrc.get_data() + self.assertTrue(ret) + self.assertEquals(str(mock_returns['enable_motd_sys_info']), + dsrc.metadata['motd_sys_info']) + + +def apply_patches(patches): + ret = [] + for (ref, name, replace) in patches: + if replace is None: + continue + orig = getattr(ref, name) + setattr(ref, name, replace) + ret.append((ref, name, orig)) + return ret -- cgit v1.2.3 From ccbdf8c360e4272055208afb013a17a218c9f097 Mon Sep 17 00:00:00 2001 From: Scott Moser Date: Tue, 23 Jul 2013 13:10:33 -0400 Subject: alias 'availability_zone' to availability-zone in metadata service. The place this was noticed was in trying to use the 'nova.clouds.archive.ubuntu.com' mirror selection. Because the config-drive-v2 has a metadata entry of 'availability_zone', it didn't get found by the availabilty_zone property in cloudinit/sources/__init__.py LP: #1190431 --- cloudinit/sources/__init__.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) (limited to 'cloudinit/sources') diff --git a/cloudinit/sources/__init__.py b/cloudinit/sources/__init__.py index d8fbacdd..974c0407 100644 --- a/cloudinit/sources/__init__.py +++ b/cloudinit/sources/__init__.py @@ -135,7 +135,8 @@ class DataSource(object): @property def availability_zone(self): - return self.metadata.get('availability-zone') + return self.metadata.get('availability-zone', + self.metadata.get('availability_zone')) def get_instance_id(self): if not self.metadata or 'instance-id' not in self.metadata: -- cgit v1.2.3 From c67cc904bfddc9ed462f54d85d31236b39b6285f Mon Sep 17 00:00:00 2001 From: Ben Howard Date: Tue, 23 Jul 2013 12:36:15 -0600 Subject: Move more functionality into get_serial() --- cloudinit/sources/DataSourceSmartOS.py | 42 ++++++++++++++++++++++------------ 1 file changed, 27 insertions(+), 15 deletions(-) (limited to 'cloudinit/sources') diff --git a/cloudinit/sources/DataSourceSmartOS.py b/cloudinit/sources/DataSourceSmartOS.py index f9b724eb..129020ec 100644 --- a/cloudinit/sources/DataSourceSmartOS.py +++ b/cloudinit/sources/DataSourceSmartOS.py @@ -35,7 +35,8 @@ from cloudinit import sources from cloudinit import util -TTY_LOC = '/dev/ttyS1' +DEF_TTY_LOC = '/dev/ttyS1' +TTY_LOC = None LOG = logging.getLogger(__name__) @@ -54,6 +55,7 @@ class DataSourceSmartOS(sources.DataSource): md = {} ud = "" + TTY_LOC = self.sys_cfg.get("serial_device", DEF_TTY_LOC) if not os.path.exists(TTY_LOC): LOG.debug("Host does not appear to be on SmartOS") return False @@ -72,11 +74,17 @@ class DataSourceSmartOS(sources.DataSource): md['local-hostname'] = hostname md['instance-id'] = system_uuid md['public-keys'] = query_data("root_authorized_keys", strip=True) - ud = query_data("user-script") + md['user-script'] = query_data("user-script") + md['user-data'] = query_data("user-script") md['iptables_disable'] = query_data("disable_iptables_flag", strip=True) md['motd_sys_info'] = query_data("enable_motd_sys_info", strip=True) + if md['user-data']: + ud = md['user-data'] + else: + ud = md['user-script'] + self.metadata = md self.userdata_raw = ud return True @@ -87,8 +95,22 @@ class DataSourceSmartOS(sources.DataSource): def get_serial(): """This is replaced in unit testing, allowing us to replace - serial.Serial with a mocked class""" - return serial.Serial() + serial.Serial with a mocked class + + The timeout value of 60 seconds should never be hit. The value + is taken from SmartOS own provisioning tools. Since we are reading + each line individually up until the single ".", the transfer is + usually very fast (i.e. microseconds) to get the response. + """ + if not TTY_LOC: + raise AttributeError("TTY_LOC value is not set") + + _ret = serial.Serial(TTY_LOC, timeout=60) + if not _ret.isOpen(): + raise SystemError("Unable to open %s" % TTY_LOC) + + return _ret + def query_data(noun, strip=False): @@ -97,22 +119,12 @@ def query_data(noun, strip=False): In the response, the first line is the status, while subsequent lines are is the value. A blank line with a "." is used to indicate end of response. + """ - The timeout value of 60 seconds should never be hit. The value - is taken from SmartOS own provisioning tools. Since we are reading - each line individually up until the single ".", the transfer is - usually very fast (i.e. microseconds) to get the response. - """ if not noun: return False ser = get_serial() - ser.port = '/dev/ttyS1' - ser.open() - if not ser.isOpen(): - LOG.debug("Serial console is not open") - return False - ser.write("GET %s\n" % noun.rstrip()) status = str(ser.readline()).rstrip() response = [] -- cgit v1.2.3 From a4310ee3db0b394dcebd4f6b49d3b25bba37fedf Mon Sep 17 00:00:00 2001 From: Scott Moser Date: Tue, 23 Jul 2013 15:17:50 -0400 Subject: on azure datasource, grab use Deployment as the instance-id LP: #1204190 --- cloudinit/sources/DataSourceAzure.py | 44 ++++++++++++++++++++++++++- tests/unittests/test_datasource/test_azure.py | 23 +++++++++++++- 2 files changed, 65 insertions(+), 2 deletions(-) (limited to 'cloudinit/sources') diff --git a/cloudinit/sources/DataSourceAzure.py b/cloudinit/sources/DataSourceAzure.py index c90d7b07..0a5caebe 100644 --- a/cloudinit/sources/DataSourceAzure.py +++ b/cloudinit/sources/DataSourceAzure.py @@ -114,7 +114,8 @@ class DataSourceAzureNet(sources.DataSource): # claim the datasource even if the command failed util.logexc(LOG, "agent command '%s' failed.", mycfg['cmd']) - wait_for = [os.path.join(mycfg['datadir'], "SharedConfig.xml")] + shcfgxml = os.path.join(mycfg['datadir'], "SharedConfig.xml") + wait_for = [shcfgxml] fp_files = [] for pk in self.cfg.get('_pubkeys', []): @@ -129,6 +130,14 @@ class DataSourceAzureNet(sources.DataSource): LOG.debug("waited %.3f seconds for %d files to appear", time.time() - start, len(wait_for)) + if shcfgxml in missing: + LOG.warn("SharedConfig.xml missing, using static instance-id") + else: + try: + self.metadata['instance-id'] = iid_from_shared_config(shcfgxml) + except ValueError as e: + LOG.warn("failed to get instance id in %s: %s" % (shcfgxml, e)) + pubkeys = pubkeys_from_crt_files(fp_files) self.metadata['public-keys'] = pubkeys @@ -252,6 +261,20 @@ def load_azure_ovf_pubkeys(sshnode): return found +def single_node_at_path(node, pathlist): + curnode = node + for tok in pathlist: + results = find_child(curnode, lambda n: n.localName == tok) + if len(results) == 0: + raise ValueError("missing %s token in %s" % (tok, str(pathlist))) + if len(results) > 1: + raise ValueError("found %s nodes of type %s looking for %s" % + (len(results), tok, str(pathlist))) + curnode = results[0] + + return curnode + + def read_azure_ovf(contents): try: dom = minidom.parseString(contents) @@ -362,6 +385,25 @@ def load_azure_ds_dir(source_dir): return (md, ud, cfg, {'ovf-env.xml': contents}) +def iid_from_shared_config(path): + with open(path, "rb") as fp: + content = fp.read() + return iid_from_shared_config_content(content) + + +def iid_from_shared_config_content(content): + """ + find INSTANCE_ID in: + + + + + """ + dom = minidom.parseString(content) + depnode = single_node_at_path(dom, ["SharedConfig", "Deployment"]) + return depnode.attributes.get('name').value + + class BrokenAzureDataSource(Exception): pass diff --git a/tests/unittests/test_datasource/test_azure.py b/tests/unittests/test_datasource/test_azure.py index c79c25d8..2e8583f9 100644 --- a/tests/unittests/test_datasource/test_azure.py +++ b/tests/unittests/test_datasource/test_azure.py @@ -99,6 +99,10 @@ class TestAzureDataSource(MockerTestCase): data['pubkey_files'] = flist return ["pubkey_from: %s" % f for f in flist] + def _iid_from_shared_config(path): + data['iid_from_shared_cfg'] = path + return 'i-my-azure-id' + if data.get('ovfcontent') is not None: populate_dir(os.path.join(self.paths.seed_dir, "azure"), {'ovf-env.xml': data['ovfcontent']}) @@ -112,7 +116,9 @@ class TestAzureDataSource(MockerTestCase): (mod, 'write_files', _write_files), (mod, 'wait_for_files', _wait_for_files), (mod, 'pubkeys_from_crt_files', - _pubkeys_from_crt_files)]) + _pubkeys_from_crt_files), + (mod, 'iid_from_shared_config', + _iid_from_shared_config), ]) dsrc = mod.DataSourceAzureNet( data.get('sys_cfg', {}), distro=None, paths=self.paths) @@ -131,6 +137,7 @@ class TestAzureDataSource(MockerTestCase): self.assertEqual(dsrc.metadata['local-hostname'], odata['HostName']) self.assertTrue('ovf-env.xml' in data['files']) self.assertEqual(0700, data['datadir_mode']) + self.assertEqual(dsrc.metadata['instance-id'], 'i-my-azure-id') def test_user_cfg_set_agent_command(self): cfg = {'agent_command': "my_command"} @@ -227,6 +234,20 @@ class TestReadAzureOvf(MockerTestCase): self.assertIn(mypk, cfg['_pubkeys']) +class TestReadAzureSharedConfig(MockerTestCase): + def test_valid_content(self): + xml = """ + + + + + + + """ + ret = DataSourceAzure.iid_from_shared_config_content(xml) + self.assertEqual("MY_INSTANCE_ID", ret) + + def apply_patches(patches): ret = [] for (ref, name, replace) in patches: -- cgit v1.2.3 From 4b41f7dc3d37d5bf7397bbc34d8a5e0c56798ac7 Mon Sep 17 00:00:00 2001 From: Ben Howard Date: Tue, 23 Jul 2013 16:33:46 -0600 Subject: Changed get_serial to be fully parameterized and return the serial initialized. Added a mapping of attributes between cloud-init and smartos. --- cloudinit/sources/DataSourceSmartOS.py | 64 ++++++++++++++----------- cloudinit/util.py | 5 +- tests/unittests/test_datasource/test_smartos.py | 10 ++-- 3 files changed, 43 insertions(+), 36 deletions(-) (limited to 'cloudinit/sources') diff --git a/cloudinit/sources/DataSourceSmartOS.py b/cloudinit/sources/DataSourceSmartOS.py index 129020ec..d6589f57 100644 --- a/cloudinit/sources/DataSourceSmartOS.py +++ b/cloudinit/sources/DataSourceSmartOS.py @@ -27,25 +27,37 @@ # -import os -import os.path -import serial from cloudinit import log as logging from cloudinit import sources from cloudinit import util +import os +import os.path +import serial DEF_TTY_LOC = '/dev/ttyS1' -TTY_LOC = None +DEF_TTY_TIMEOUT = 60 LOG = logging.getLogger(__name__) +SMARTOS_ATTRIB_MAP = { + #Cloud-init Key : (SmartOS Key, Strip line endings) + 'local-hostname': ('hostname', True), + 'public-keys': ('root_authorized_keys', True), + 'user-script': ('user-script', False), + 'user-data': ('user-data', False), + 'iptables_disable': ('iptables_disable', True), + 'motd_sys_info': ('motd_sys_info', True), +} + class DataSourceSmartOS(sources.DataSource): def __init__(self, sys_cfg, distro, paths): sources.DataSource.__init__(self, sys_cfg, distro, paths) self.seed_dir = os.path.join(paths.seed_dir, 'sdc') - self.seed = None self.is_smartdc = None + self.seed = self.sys_cfg.get("serial_device", DEF_TTY_LOC) + self.seed_timeout = self.sys_cfg.get("serial_timeout", + DEF_TTY_TIMEOUT) def __str__(self): root = sources.DataSource.__str__(self) @@ -55,30 +67,25 @@ class DataSourceSmartOS(sources.DataSource): md = {} ud = "" - TTY_LOC = self.sys_cfg.get("serial_device", DEF_TTY_LOC) - if not os.path.exists(TTY_LOC): + if not os.path.exists(self.seed): LOG.debug("Host does not appear to be on SmartOS") return False - self.seed = TTY_LOC + self.seed = self.seed system_uuid, system_type = dmi_data() if 'smartdc' not in system_type.lower(): LOG.debug("Host is not on SmartOS") return False self.is_smartdc = True + md['instance-id'] = system_uuid - hostname = query_data("hostname", strip=True) - if not hostname: - hostname = system_uuid + for ci_noun, attribute in SMARTOS_ATTRIB_MAP.iteritems(): + smartos_noun, strip = attribute + md[ci_noun] = query_data(smartos_noun, self.seed, + self.seed_timeout, strip=strip) - md['local-hostname'] = hostname - md['instance-id'] = system_uuid - md['public-keys'] = query_data("root_authorized_keys", strip=True) - md['user-script'] = query_data("user-script") - md['user-data'] = query_data("user-script") - md['iptables_disable'] = query_data("disable_iptables_flag", - strip=True) - md['motd_sys_info'] = query_data("enable_motd_sys_info", strip=True) + if not md['local-hostname']: + md['local-hostname'] = system_uuid if md['user-data']: ud = md['user-data'] @@ -93,7 +100,7 @@ class DataSourceSmartOS(sources.DataSource): return self.metadata['instance-id'] -def get_serial(): +def get_serial(seed_device, seed_timeout): """This is replaced in unit testing, allowing us to replace serial.Serial with a mocked class @@ -102,18 +109,17 @@ def get_serial(): each line individually up until the single ".", the transfer is usually very fast (i.e. microseconds) to get the response. """ - if not TTY_LOC: - raise AttributeError("TTY_LOC value is not set") - - _ret = serial.Serial(TTY_LOC, timeout=60) - if not _ret.isOpen(): - raise SystemError("Unable to open %s" % TTY_LOC) + if not seed_device: + raise AttributeError("seed_device value is not set") - return _ret + ser = serial.Serial(seed_device, timeout=seed_timeout) + if not ser.isOpen(): + raise SystemError("Unable to open %s" % seed_device) + return ser -def query_data(noun, strip=False): +def query_data(noun, seed_device, seed_timeout, strip=False): """Makes a request to via the serial console via "GET " In the response, the first line is the status, while subsequent lines @@ -124,7 +130,7 @@ def query_data(noun, strip=False): if not noun: return False - ser = get_serial() + ser = get_serial(seed_device, seed_timeout) ser.write("GET %s\n" % noun.rstrip()) status = str(ser.readline()).rstrip() response = [] diff --git a/cloudinit/util.py b/cloudinit/util.py index 7163225f..a2fbc004 100644 --- a/cloudinit/util.py +++ b/cloudinit/util.py @@ -1744,13 +1744,14 @@ def get_mount_info(path, log=LOG): lines = load_file(mountinfo_path).splitlines() return parse_mount_info(path, lines, log) + def which(program): # Return path of program for execution if found in path def is_exe(fpath): return os.path.isfile(fpath) and os.access(fpath, os.X_OK) - fpath, fname = os.path.split(program) - if fpath: + _fpath, _ = os.path.split(program) + if _fpath: if is_exe(program): return program else: diff --git a/tests/unittests/test_datasource/test_smartos.py b/tests/unittests/test_datasource/test_smartos.py index 494f9828..6c12f1e2 100644 --- a/tests/unittests/test_datasource/test_smartos.py +++ b/tests/unittests/test_datasource/test_smartos.py @@ -31,8 +31,8 @@ import uuid mock_returns = { 'hostname': 'test-host', 'root_authorized_keys': 'ssh-rsa AAAAB3Nz...aC1yc2E= keyname', - 'disable_iptables_flag': False, - 'enable_motd_sys_info': False, + 'disable_iptables_flag': None, + 'enable_motd_sys_info': None, 'system_uuid': str(uuid.uuid4()), 'smartdc': 'smartdc', 'userdata': """ @@ -118,7 +118,7 @@ class TestSmartOSDataSource(MockerTestCase): def _get_ds(self): - def _get_serial(): + def _get_serial(*_): return MockSerial() def _dmi_data(): @@ -169,14 +169,14 @@ class TestSmartOSDataSource(MockerTestCase): dsrc = self._get_ds() ret = dsrc.get_data() self.assertTrue(ret) - self.assertEquals(str(mock_returns['disable_iptables_flag']), + self.assertEquals(mock_returns['disable_iptables_flag'], dsrc.metadata['iptables_disable']) def test_motd_sys_info(self): dsrc = self._get_ds() ret = dsrc.get_data() self.assertTrue(ret) - self.assertEquals(str(mock_returns['enable_motd_sys_info']), + self.assertEquals(mock_returns['enable_motd_sys_info'], dsrc.metadata['motd_sys_info']) -- cgit v1.2.3 From 0891f6611d1c264220a6f71306802db1e70651fc Mon Sep 17 00:00:00 2001 From: Scott Moser Date: Wed, 24 Jul 2013 10:44:21 -0400 Subject: DataSourceSmartOS: fix issue if dmidecode is not present --- cloudinit/sources/DataSourceSmartOS.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) (limited to 'cloudinit/sources') diff --git a/cloudinit/sources/DataSourceSmartOS.py b/cloudinit/sources/DataSourceSmartOS.py index d6589f57..1ce20c10 100644 --- a/cloudinit/sources/DataSourceSmartOS.py +++ b/cloudinit/sources/DataSourceSmartOS.py @@ -72,7 +72,12 @@ class DataSourceSmartOS(sources.DataSource): return False self.seed = self.seed - system_uuid, system_type = dmi_data() + dmi_info = dmi_data() + if dmi_info is False: + LOG.debug("No dmidata utility found") + return False + + system_uuid, system_type = dmi_info if 'smartdc' not in system_type.lower(): LOG.debug("Host is not on SmartOS") return False -- cgit v1.2.3 From 3d10b8d080a874be022f9e25063ba77f0293c5e8 Mon Sep 17 00:00:00 2001 From: Scott Moser Date: Thu, 25 Jul 2013 14:37:10 -0400 Subject: azure: support bouncing interfaces to publish new hostname See the added doc/sources/azure/README.rst for why this is necessary. Essentially, we now are doing the following in the get_data() method of azure datasource to publish this NewHostname: hostname NewHostName ifdown eth0; ifup eth0 LP: #1202758 --- cloudinit/sources/DataSourceAzure.py | 114 +++++++++++++++++----- doc/examples/cloud-config-datasources.txt | 5 + doc/sources/azure/README.rst | 134 ++++++++++++++++++++++++++ tests/unittests/test_datasource/test_azure.py | 77 ++++++++++++++- 4 files changed, 300 insertions(+), 30 deletions(-) create mode 100644 doc/sources/azure/README.rst (limited to 'cloudinit/sources') diff --git a/cloudinit/sources/DataSourceAzure.py b/cloudinit/sources/DataSourceAzure.py index 0a5caebe..30b06fef 100644 --- a/cloudinit/sources/DataSourceAzure.py +++ b/cloudinit/sources/DataSourceAzure.py @@ -31,9 +31,20 @@ LOG = logging.getLogger(__name__) DS_NAME = 'Azure' DEFAULT_METADATA = {"instance-id": "iid-AZURE-NODE"} AGENT_START = ['service', 'walinuxagent', 'start'] -BUILTIN_DS_CONFIG = {'datasource': {DS_NAME: { - 'agent_command': AGENT_START, - 'data_dir': "/var/lib/waagent"}}} +BOUNCE_COMMAND = ("i=$interface; x=0; ifdown $i || x=$?; " + "ifup $i || x=$?; exit $x") +BUILTIN_DS_CONFIG = { + 'agent_command': AGENT_START, + 'data_dir': "/var/lib/waagent", + 'set_hostname': True, + 'hostname_bounce': { + 'interface': 'eth0', + 'policy': True, + 'command': BOUNCE_COMMAND, + 'hostname_command': 'hostname', + } +} +DS_CFG_PATH = ['datasource', DS_NAME] class DataSourceAzureNet(sources.DataSource): @@ -42,19 +53,19 @@ class DataSourceAzureNet(sources.DataSource): self.seed_dir = os.path.join(paths.seed_dir, 'azure') self.cfg = {} self.seed = None + self.ds_cfg = util.mergemanydict([ + util.get_cfg_by_path(sys_cfg, DS_CFG_PATH), + BUILTIN_DS_CONFIG]) def __str__(self): root = sources.DataSource.__str__(self) return "%s [seed=%s]" % (root, self.seed) def get_data(self): - ddir_cfgpath = ['datasource', DS_NAME, 'data_dir'] # azure removes/ejects the cdrom containing the ovf-env.xml # file on reboot. So, in order to successfully reboot we # need to look in the datadir and consider that valid - ddir = util.get_cfg_by_path(self.sys_cfg, ddir_cfgpath) - if ddir is None: - ddir = util.get_cfg_by_path(BUILTIN_DS_CONFIG, ddir_cfgpath) + ddir = self.ds_cfg['data_dir'] candidates = [self.seed_dir] candidates.extend(list_possible_azure_ds_devs()) @@ -91,36 +102,40 @@ class DataSourceAzureNet(sources.DataSource): return False if found == ddir: - LOG.debug("using cached datasource in %s", ddir) - - fields = [('cmd', ['datasource', DS_NAME, 'agent_command']), - ('datadir', ddir_cfgpath)] - mycfg = {} - for cfg in (self.cfg, self.sys_cfg, BUILTIN_DS_CONFIG): - for name, path in fields: - if name in mycfg: - continue - value = util.get_cfg_by_path(cfg, keyp=path) - if value is not None: - mycfg[name] = value + LOG.debug("using files cached in %s", ddir) + + # now update ds_cfg to reflect contents pass in config + usercfg = util.get_cfg_by_path(self.cfg, DS_CFG_PATH, {}) + self.ds_cfg = util.mergemanydict([usercfg, self.ds_cfg]) + mycfg = self.ds_cfg # walinux agent writes files world readable, but expects # the directory to be protected. - write_files(mycfg['datadir'], files, dirmode=0700) + write_files(mycfg['data_dir'], files, dirmode=0700) + + # handle the hostname 'publishing' + try: + handle_set_hostname(mycfg.get('set_hostname'), + self.metadata.get('local-hostname'), + mycfg['hostname_bounce']) + except Exception as e: + LOG.warn("Failed publishing hostname: %s" % e) + util.logexc(LOG, "handling set_hostname failed") try: - invoke_agent(mycfg['cmd']) + invoke_agent(mycfg['agent_command']) except util.ProcessExecutionError: # claim the datasource even if the command failed - util.logexc(LOG, "agent command '%s' failed.", mycfg['cmd']) + util.logexc(LOG, "agent command '%s' failed.", + mycfg['agent_command']) - shcfgxml = os.path.join(mycfg['datadir'], "SharedConfig.xml") + shcfgxml = os.path.join(mycfg['data_dir'], "SharedConfig.xml") wait_for = [shcfgxml] fp_files = [] for pk in self.cfg.get('_pubkeys', []): bname = pk['fingerprint'] + ".crt" - fp_files += [os.path.join(mycfg['datadir'], bname)] + fp_files += [os.path.join(mycfg['data_dir'], bname)] start = time.time() missing = wait_for_files(wait_for + fp_files) @@ -148,6 +163,43 @@ class DataSourceAzureNet(sources.DataSource): return self.cfg +def handle_set_hostname(enabled, hostname, cfg): + if not util.is_true(enabled): + return + + if not hostname: + LOG.warn("set_hostname was true but no local-hostname") + return + + apply_hostname_bounce(hostname=hostname, policy=cfg['policy'], + interface=cfg['interface'], + command=cfg['command'], + hostname_command=cfg['hostname_command']) + + +def apply_hostname_bounce(hostname, policy, interface, command, + hostname_command="hostname"): + # set the hostname to 'hostname' if it is not already set to that. + # then, if policy is not off, bounce the interface using command + prev_hostname = util.subp(hostname_command, capture=True)[0].strip() + + util.subp([hostname_command, hostname]) + + if util.is_false(policy): + return + + if prev_hostname == hostname and policy != "force": + return + + env = os.environ.copy() + env['interface'] = interface + + if command == "builtin": + command = BOUNCE_COMMAND + + util.subp(command, shell=(not isinstance(command, list)), capture=True) + + def crtfile_to_pubkey(fname): pipeline = ('openssl x509 -noout -pubkey < "$0" |' 'ssh-keygen -i -m PKCS8 -f /dev/stdin') @@ -319,15 +371,21 @@ def read_azure_ovf(contents): name = child.localName.lower() simple = False + value = "" if (len(child.childNodes) == 1 and child.childNodes[0].nodeType == dom.TEXT_NODE): simple = True value = child.childNodes[0].wholeText + attrs = {k: v for k, v in child.attributes.items()} + # we accept either UserData or CustomData. If both are present # then behavior is undefined. if (name == "userdata" or name == "customdata"): - ud = base64.b64decode(''.join(value.split())) + if attrs.get('encoding') in (None, "base64"): + ud = base64.b64decode(''.join(value.split())) + else: + ud = value elif name == "username": username = value elif name == "userpassword": @@ -335,7 +393,11 @@ def read_azure_ovf(contents): elif name == "hostname": md['local-hostname'] = value elif name == "dscfg": - cfg['datasource'] = {DS_NAME: util.load_yaml(value, default={})} + if attrs.get('encoding') in (None, "base64"): + dscfg = base64.b64decode(''.join(value.split())) + else: + dscfg = value + cfg['datasource'] = {DS_NAME: util.load_yaml(dscfg, default={})} elif name == "ssh": cfg['_pubkeys'] = load_azure_ovf_pubkeys(child) elif name == "disablesshpasswordauthentication": diff --git a/doc/examples/cloud-config-datasources.txt b/doc/examples/cloud-config-datasources.txt index a19353fc..6544448e 100644 --- a/doc/examples/cloud-config-datasources.txt +++ b/doc/examples/cloud-config-datasources.txt @@ -45,6 +45,11 @@ datasource: Azure: agent_command: [service, walinuxagent, start] + set_hostname: True + hostname_bounce: + interface: eth0 + policy: on # [can be 'on', 'off' or 'force'] + } SmartOS: # Smart OS datasource works over a serial console interacting with diff --git a/doc/sources/azure/README.rst b/doc/sources/azure/README.rst new file mode 100644 index 00000000..8239d1fa --- /dev/null +++ b/doc/sources/azure/README.rst @@ -0,0 +1,134 @@ +================ +Azure Datasource +================ + +This datasource finds metadata and user-data from the Azure cloud platform. + +Azure Platform +-------------- +The azure cloud-platform provides initial data to an instance via an attached +CD formated in UDF. That CD contains a 'ovf-env.xml' file that provides some +information. Additional information is obtained via interaction with the +"endpoint". The ip address of the endpoint is advertised to the instance +inside of dhcp option 245. On ubuntu, that can be seen in +/var/lib/dhcp/dhclient.eth0.leases as a colon delimited hex value (example: +``option unknown-245 64:41:60:82;`` is 100.65.96.130) + +walinuxagent +------------ +In order to operate correctly, cloud-init needs walinuxagent to provide much +of the interaction with azure. In addition to "provisioning" code, walinux +does the following on the agent is a long running daemon that handles the +following things: +- generate a x509 certificate and send that to the endpoint + +waagent.conf config +~~~~~~~~~~~~~~~~~~~ +in order to use waagent.conf with cloud-init, the following settings are recommended. Other values can be changed or set to the defaults. + + :: + + # disabling provisioning turns off all 'Provisioning.*' function + Provisioning.Enabled=n + # this is currently not handled by cloud-init, so let walinuxagent do it. + ResourceDisk.Format=y + ResourceDisk.MountPoint=/mnt + + +Userdata +-------- +Userdata is provided to cloud-init inside the ovf-env.xml file. Cloud-init +expects that user-data will be provided as base64 encoded value inside the +text child of a element named ``UserData`` or ``CustomData`` which is a direct +child of the ``LinuxProvisioningConfigurationSet`` (a sibling to ``UserName``) +If both ``UserData`` and ``CustomData`` are provided behavior is undefined on +which will be selected. + +In the example below, user-data provided is 'this is my userdata', and the +datasource config provided is ``{"agent_command": ["start", "walinuxagent"]}``. +That agent command will take affect as if it were specified in system config. + +Example: + +.. code:: + + + 1.0 + + LinuxProvisioningConfiguration + myHost + myuser + + dGhpcyBpcyBteSB1c2VyZGF0YQ=== + eyJhZ2VudF9jb21tYW5kIjogWyJzdGFydCIsICJ3YWxpbnV4YWdlbnQiXX0= + true + + + + 6BE7A7C3C8A8F4B123CCA5D0C2F1BE4CA7B63ED7 + this-value-unused + + + + + + +Configuration +------------- +Configuration for the datasource can be read from the system config's or set +via the `dscfg` entry in the `LinuxProvisioningConfigurationSet`. Content in +dscfg node is expected to be base64 encoded yaml content, and it will be +merged into the 'datasource: Azure' entry. + +The '``hostname_bounce: command``' entry can be either the literal string +'builtin' or a command to execute. The command will be invoked after the +hostname is set, and will have the 'interface' in its environment. If +``set_hostname`` is not true, then ``hostname_bounce`` will be ignored. + +An example might be: + command: ["sh", "-c", "killall dhclient; dhclient $interface"] + +.. code:: + + datasource: + agent_command + Azure: + agent_command: [service, walinuxagent, start] + set_hostname: True + hostname_bounce: + # the name of the interface to bounce + interface: eth0 + # policy can be 'on', 'off' or 'force' + policy: on + # the method 'bounce' command. + command: "builtin" + hostname_command: "hostname" + } + +hostname +-------- +When the user launches an instance, they provide a hostname for that instance. +The hostname is provided to the instance in the ovf-env.xml file as +``HostName``. + +Whatever value the instance provides in its dhcp request will resolve in the +domain returned in the 'search' request. + +The interesting issue is that a generic image will already have a hostname +configured. The ubuntu cloud images have 'ubuntu' as the hostname of the +system, and the initial dhcp request on eth0 is not guaranteed to occur after +the datasource code has been run. So, on first boot, that initial value will +be sent in the dhcp request and *that* value will resolve. + +In order to make the ``HostName`` provided in the ovf-env.xml resolve, a +dhcp request must be made with the new value. Walinuxagent (in its current +version) handles this by polling the state of hostname and bouncing ('``ifdown +eth0; ifup eth0``' the network interface if it sees that a change has been +made. + +cloud-init handles this by setting the hostname in the DataSource's 'get_data' +method via '``hostname $HostName``', and then bouncing the interface. This +behavior can be configured or disabled in the datasource config. See +'Configuration' above. diff --git a/tests/unittests/test_datasource/test_azure.py b/tests/unittests/test_datasource/test_azure.py index 2e8583f9..c944cb13 100644 --- a/tests/unittests/test_datasource/test_azure.py +++ b/tests/unittests/test_datasource/test_azure.py @@ -26,8 +26,15 @@ def construct_valid_ovf_env(data=None, pubkeys=None, userdata=None): xmlns:i="http://www.w3.org/2001/XMLSchema-instance"> LinuxProvisioningConfiguration """ - for key, val in data.items(): - content += "<%s>%s\n" % (key, val, key) + for key, dval in data.items(): + if isinstance(dval, dict): + val = dval.get('text') + attrs = ' ' + ' '.join(["%s='%s'" % (k, v) for k, v in dval.items() + if k != 'text']) + else: + val = dval + attrs = "" + content += "<%s%s>%s\n" % (key, attrs, val, key) if userdata: content += "%s\n" % (base64.b64encode(userdata)) @@ -103,6 +110,9 @@ class TestAzureDataSource(MockerTestCase): data['iid_from_shared_cfg'] = path return 'i-my-azure-id' + def _apply_hostname_bounce(**kwargs): + data['apply_hostname_bounce'] = kwargs + if data.get('ovfcontent') is not None: populate_dir(os.path.join(self.paths.seed_dir, "azure"), {'ovf-env.xml': data['ovfcontent']}) @@ -118,7 +128,9 @@ class TestAzureDataSource(MockerTestCase): (mod, 'pubkeys_from_crt_files', _pubkeys_from_crt_files), (mod, 'iid_from_shared_config', - _iid_from_shared_config), ]) + _iid_from_shared_config), + (mod, 'apply_hostname_bounce', + _apply_hostname_bounce), ]) dsrc = mod.DataSourceAzureNet( data.get('sys_cfg', {}), distro=None, paths=self.paths) @@ -139,11 +151,26 @@ class TestAzureDataSource(MockerTestCase): self.assertEqual(0700, data['datadir_mode']) self.assertEqual(dsrc.metadata['instance-id'], 'i-my-azure-id') + def test_user_cfg_set_agent_command_plain(self): + # set dscfg in via plaintext + cfg = {'agent_command': "my_command"} + odata = {'HostName': "myhost", 'UserName': "myuser", + 'dscfg': {'text': yaml.dump(cfg), 'encoding': 'plain'}} + data = {'ovfcontent': construct_valid_ovf_env(data=odata)} + + dsrc = self._get_ds(data) + ret = dsrc.get_data() + self.assertTrue(ret) + self.assertEqual(data['agent_invoked'], cfg['agent_command']) + def test_user_cfg_set_agent_command(self): + # set dscfg in via base64 encoded yaml cfg = {'agent_command': "my_command"} odata = {'HostName': "myhost", 'UserName': "myuser", - 'dscfg': yaml.dump(cfg)} + 'dscfg': {'text': base64.b64encode(yaml.dump(cfg)), + 'encoding': 'base64'}} data = {'ovfcontent': construct_valid_ovf_env(data=odata)} + print data dsrc = self._get_ds(data) ret = dsrc.get_data() @@ -218,6 +245,48 @@ class TestAzureDataSource(MockerTestCase): for mypk in mypklist: self.assertIn(mypk, dsrc.cfg['_pubkeys']) + def test_disabled_bounce(self): + pass + + def test_apply_bounce_call_1(self): + # hostname needs to get through to apply_hostname_bounce + mydata = "FOOBAR" + odata = {'HostName': 'my-random-hostname'} + data = {'ovfcontent': construct_valid_ovf_env(data=odata)} + + self._get_ds(data).get_data() + self.assertIn('hostname', data['apply_hostname_bounce']) + self.assertEqual(data['apply_hostname_bounce']['hostname'], + odata['HostName']) + + def test_apply_bounce_call_configurable(self): + # hostname_bounce should be configurable in datasource cfg + cfg = {'hostname_bounce': {'interface': 'eth1', 'policy': 'off', + 'command': 'my-bounce-command', + 'hostname_command': 'my-hostname-command'}} + odata = {'HostName': "xhost", + 'dscfg': {'text': base64.b64encode(yaml.dump(cfg)), + 'encoding': 'base64'}} + data = {'ovfcontent': construct_valid_ovf_env(data=odata)} + self._get_ds(data).get_data() + + for k in cfg['hostname_bounce']: + self.assertIn(k, data['apply_hostname_bounce']) + + for k, v in cfg['hostname_bounce'].items(): + self.assertEqual(data['apply_hostname_bounce'][k], v) + + def test_set_hostname_disabled(self): + # config specifying set_hostname off should not bounce + cfg = {'set_hostname': False} + odata = {'HostName': "xhost", + 'dscfg': {'text': base64.b64encode(yaml.dump(cfg)), + 'encoding': 'base64'}} + data = {'ovfcontent': construct_valid_ovf_env(data=odata)} + self._get_ds(data).get_data() + + self.assertEqual(data.get('apply_hostname_bounce', "N/A"), "N/A") + class TestReadAzureOvf(MockerTestCase): def test_invalid_xml_raises_non_azure_ds(self): -- cgit v1.2.3 From 4d9668dac5fed8f713f3b4300fdb574f399c14ee Mon Sep 17 00:00:00 2001 From: Scott Moser Date: Thu, 25 Jul 2013 15:27:44 -0400 Subject: minor azure cleanup --- cloudinit/sources/DataSourceAzure.py | 2 +- tests/unittests/test_datasource/test_azure.py | 1 - 2 files changed, 1 insertion(+), 2 deletions(-) (limited to 'cloudinit/sources') diff --git a/cloudinit/sources/DataSourceAzure.py b/cloudinit/sources/DataSourceAzure.py index 30b06fef..d4863429 100644 --- a/cloudinit/sources/DataSourceAzure.py +++ b/cloudinit/sources/DataSourceAzure.py @@ -54,7 +54,7 @@ class DataSourceAzureNet(sources.DataSource): self.cfg = {} self.seed = None self.ds_cfg = util.mergemanydict([ - util.get_cfg_by_path(sys_cfg, DS_CFG_PATH), + util.get_cfg_by_path(sys_cfg, DS_CFG_PATH, {}), BUILTIN_DS_CONFIG]) def __str__(self): diff --git a/tests/unittests/test_datasource/test_azure.py b/tests/unittests/test_datasource/test_azure.py index c944cb13..4cd3f213 100644 --- a/tests/unittests/test_datasource/test_azure.py +++ b/tests/unittests/test_datasource/test_azure.py @@ -170,7 +170,6 @@ class TestAzureDataSource(MockerTestCase): 'dscfg': {'text': base64.b64encode(yaml.dump(cfg)), 'encoding': 'base64'}} data = {'ovfcontent': construct_valid_ovf_env(data=odata)} - print data dsrc = self._get_ds(data) ret = dsrc.get_data() -- cgit v1.2.3 From 9da084bda820d0ab8500ea6ee13a014f46d3bfab Mon Sep 17 00:00:00 2001 From: Scott Moser Date: Fri, 26 Jul 2013 14:05:52 -0400 Subject: azure: fix bouncing of interface the environment that was set up to include 'interface' was not actually being passed on to 'subp', so when the command ran it wasn't available. --- cloudinit/sources/DataSourceAzure.py | 17 ++++++++++++++--- 1 file changed, 14 insertions(+), 3 deletions(-) (limited to 'cloudinit/sources') diff --git a/cloudinit/sources/DataSourceAzure.py b/cloudinit/sources/DataSourceAzure.py index d4863429..9503b045 100644 --- a/cloudinit/sources/DataSourceAzure.py +++ b/cloudinit/sources/DataSourceAzure.py @@ -31,8 +31,9 @@ LOG = logging.getLogger(__name__) DS_NAME = 'Azure' DEFAULT_METADATA = {"instance-id": "iid-AZURE-NODE"} AGENT_START = ['service', 'walinuxagent', 'start'] -BOUNCE_COMMAND = ("i=$interface; x=0; ifdown $i || x=$?; " - "ifup $i || x=$?; exit $x") +BOUNCE_COMMAND = ['sh', '-xc', + "i=$interface; x=0; ifdown $i || x=$?; ifup $i || x=$?; exit $x"] + BUILTIN_DS_CONFIG = { 'agent_command': AGENT_START, 'data_dir': "/var/lib/waagent", @@ -185,19 +186,29 @@ def apply_hostname_bounce(hostname, policy, interface, command, util.subp([hostname_command, hostname]) + msg = ("phostname=%s hostname=%s policy=%s interface=%s" % + (prev_hostname, hostname, policy, interface)) + if util.is_false(policy): + LOG.debug("pubhname: policy false, skipping [%s]", msg) return if prev_hostname == hostname and policy != "force": + LOG.debug("pubhname: no change, policy != force. skipping. [%s]", msg) return env = os.environ.copy() env['interface'] = interface + env['hostname'] = hostname + env['old_hostname'] = prev_hostname if command == "builtin": command = BOUNCE_COMMAND - util.subp(command, shell=(not isinstance(command, list)), capture=True) + LOG.debug("pubhname: publishing hostname [%s]", msg) + shell = not isinstance(command, (list, tuple)) + (output, err) = util.subp(command, shell=shell, capture=True, env=env) + LOG.debug("output: %s. err: %s", output, err) def crtfile_to_pubkey(fname): -- cgit v1.2.3 From e9f1190993fc5e66cd7bcc47016a5d7335fe8763 Mon Sep 17 00:00:00 2001 From: Scott Moser Date: Mon, 29 Jul 2013 12:02:03 -0400 Subject: DataSourceAzure: do not capture output of bounce command As shown in comments of bug 1202758 and filing of ntp bug 1206164, waiting for the output of this command causes us to wait for ntpdate to fully finish. Ideally I think we'd disable ntpdate running on this run, but that is not trivially possible. --- cloudinit/sources/DataSourceAzure.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) (limited to 'cloudinit/sources') diff --git a/cloudinit/sources/DataSourceAzure.py b/cloudinit/sources/DataSourceAzure.py index 9503b045..2f28702e 100644 --- a/cloudinit/sources/DataSourceAzure.py +++ b/cloudinit/sources/DataSourceAzure.py @@ -206,9 +206,11 @@ def apply_hostname_bounce(hostname, policy, interface, command, command = BOUNCE_COMMAND LOG.debug("pubhname: publishing hostname [%s]", msg) + start = time.time() shell = not isinstance(command, (list, tuple)) - (output, err) = util.subp(command, shell=shell, capture=True, env=env) - LOG.debug("output: %s. err: %s", output, err) + # capture=False, see comments in bug 1202758 and bug 1206164. + (output, err) = util.subp(command, shell=shell, capture=False, env=env) + LOG.debug("publishing hostname took %.3f seconds", time.time() - start) def crtfile_to_pubkey(fname): -- cgit v1.2.3 From 66490ebb92af59d148f79aae42a2eddc1ecedb7e Mon Sep 17 00:00:00 2001 From: Scott Moser Date: Tue, 30 Jul 2013 14:23:10 -0400 Subject: add util.log_time helper The reason for this is that more and more things I was wanting to be able to see how long they took. This puts that time logic into a single place. It also supports (by default) reading from /proc/uptime as the timing mechanism. While that is almost certainly slower than time.time(), it does give millisecond granularity and is not affected by 'ntpdate' having run in between the two events. --- ChangeLog | 3 +++ bin/cloud-init | 5 ++++- cloudinit/config/cc_growpart.py | 3 ++- cloudinit/config/cc_resizefs.py | 11 +++++------ cloudinit/sources/DataSourceAzure.py | 14 ++++++-------- cloudinit/util.py | 35 +++++++++++++++++++++++++++++++++++ 6 files changed, 55 insertions(+), 16 deletions(-) (limited to 'cloudinit/sources') diff --git a/ChangeLog b/ChangeLog index a255d24a..68d03376 100644 --- a/ChangeLog +++ b/ChangeLog @@ -11,6 +11,9 @@ - always finalize handlers even if processing failed (LP: #1203368) - support merging into cloud-config via jsonp. (LP: #1200476) - add datasource 'SmartOS' for Joyent Cloud. Adds a dependency on serial. + - add 'log_time' helper to util for timing how long things take + which also reads from uptime. uptime is useful as clock may change during + boot due to ntp. 0.7.2: - add a debian watch file - add 'sudo' entry to ubuntu's default user (LP: #1080717) diff --git a/bin/cloud-init b/bin/cloud-init index c5a5b949..bd9ddc04 100755 --- a/bin/cloud-init +++ b/bin/cloud-init @@ -502,7 +502,10 @@ def main(): signal_handler.attach_handlers() (name, functor) = args.action - return functor(name, args) + + return util.log_time(logfunc=LOG.debug, + msg="cloud-init mode '%s'" % name, uptime=True, + func=functor, args=(name, args)) if __name__ == '__main__': diff --git a/cloudinit/config/cc_growpart.py b/cloudinit/config/cc_growpart.py index 4f8c8f80..ba6c58af 100644 --- a/cloudinit/config/cc_growpart.py +++ b/cloudinit/config/cc_growpart.py @@ -264,7 +264,8 @@ def handle(_name, cfg, _cloud, log, _args): raise e return - resized = resize_devices(resizer, devices) + resized = util.log_time(logfunc=log.debug, msg="resize_devices", + func=resize_devices, args=(resizer, devices)) for (entry, action, msg) in resized: if action == RESIZE.CHANGED: log.info("'%s' resized: %s" % (entry, msg)) diff --git a/cloudinit/config/cc_resizefs.py b/cloudinit/config/cc_resizefs.py index b4ee16b2..56040fdd 100644 --- a/cloudinit/config/cc_resizefs.py +++ b/cloudinit/config/cc_resizefs.py @@ -21,7 +21,6 @@ import errno import os import stat -import time from cloudinit.settings import PER_ALWAYS from cloudinit import util @@ -120,9 +119,12 @@ def handle(name, cfg, _cloud, log, args): if resize_root == NOBLOCK: # Fork to a child that will run # the resize command - util.fork_cb(do_resize, resize_cmd, log) + util.fork_cb( + util.log_time(logfunc=log.debug, msg="backgrounded Resizing", + func=do_resize, args=(resize_cmd, log))) else: - do_resize(resize_cmd, log) + util.log_time(logfunc=log.debug, msg="Resizing", + func=do_resize, args=(resize_cmd, log)) action = 'Resized' if resize_root == NOBLOCK: @@ -132,13 +134,10 @@ def handle(name, cfg, _cloud, log, args): def do_resize(resize_cmd, log): - start = time.time() try: util.subp(resize_cmd) except util.ProcessExecutionError: util.logexc(log, "Failed to resize filesystem (cmd=%s)", resize_cmd) raise - tot_time = time.time() - start - log.debug("Resizing took %.3f seconds", tot_time) # TODO(harlowja): Should we add a fsck check after this to make # sure we didn't corrupt anything? diff --git a/cloudinit/sources/DataSourceAzure.py b/cloudinit/sources/DataSourceAzure.py index 2f28702e..f906b8fa 100644 --- a/cloudinit/sources/DataSourceAzure.py +++ b/cloudinit/sources/DataSourceAzure.py @@ -138,13 +138,11 @@ class DataSourceAzureNet(sources.DataSource): bname = pk['fingerprint'] + ".crt" fp_files += [os.path.join(mycfg['data_dir'], bname)] - start = time.time() - missing = wait_for_files(wait_for + fp_files) + missing = util.log_time(logfunc=LOG.debug, msg="waiting for files", + func=wait_for_files, + args=(wait_for + fp_files,)) if len(missing): LOG.warn("Did not find files, but going on: %s", missing) - else: - LOG.debug("waited %.3f seconds for %d files to appear", - time.time() - start, len(wait_for)) if shcfgxml in missing: LOG.warn("SharedConfig.xml missing, using static instance-id") @@ -206,11 +204,11 @@ def apply_hostname_bounce(hostname, policy, interface, command, command = BOUNCE_COMMAND LOG.debug("pubhname: publishing hostname [%s]", msg) - start = time.time() shell = not isinstance(command, (list, tuple)) # capture=False, see comments in bug 1202758 and bug 1206164. - (output, err) = util.subp(command, shell=shell, capture=False, env=env) - LOG.debug("publishing hostname took %.3f seconds", time.time() - start) + util.log_time(logfunc=LOG.debug, msg="publishing hostname", + func=util.subp, kwargs={'command': command, 'shell': shell, + 'capture': False, 'env': env}) def crtfile_to_pubkey(fname): diff --git a/cloudinit/util.py b/cloudinit/util.py index 8542fe27..b0eb56e6 100644 --- a/cloudinit/util.py +++ b/cloudinit/util.py @@ -1770,3 +1770,38 @@ def which(program): return exe_file return None + + +def log_time(logfunc, msg, func, args=None, kwargs=None, get_uptime=True): + if args is None: + args = [] + if kwargs is None: + kwargs = {} + + start = time.time() + + ustart = None + if get_uptime: + try: + ustart = float(uptime()) + except ValueError: + pass + + try: + ret = func(*args, **kwargs) + finally: + delta = time.time() - start + if ustart is not None: + try: + udelta = float(uptime()) - ustart + except ValueError: + udelta = "N/A" + + tmsg = " took %0.3f seconds" % delta + if get_uptime: + tmsg += "(%0.2f)" % udelta + try: + logfunc(msg + tmsg) + except: + pass + return ret -- cgit v1.2.3 From 0ca150b08433fbc57e10d599a46e300142c955c5 Mon Sep 17 00:00:00 2001 From: Scott Moser Date: Tue, 30 Jul 2013 14:28:09 -0400 Subject: set reading /proc/uptime to false by default. reading /proc/uptime is going to be slower, and no reason to do it on most things. Better to only do it when you suspect maybe a need for it. --- bin/cloud-init | 5 ++--- cloudinit/sources/DataSourceAzure.py | 5 +++-- cloudinit/util.py | 2 +- 3 files changed, 6 insertions(+), 6 deletions(-) (limited to 'cloudinit/sources') diff --git a/bin/cloud-init b/bin/cloud-init index bd9ddc04..b4f9fd07 100755 --- a/bin/cloud-init +++ b/bin/cloud-init @@ -503,9 +503,8 @@ def main(): (name, functor) = args.action - return util.log_time(logfunc=LOG.debug, - msg="cloud-init mode '%s'" % name, uptime=True, - func=functor, args=(name, args)) + return util.log_time(logfunc=LOG.debug, msg="cloud-init mode '%s'" % name, + get_uptime=True, func=functor, args=(name, args)) if __name__ == '__main__': diff --git a/cloudinit/sources/DataSourceAzure.py b/cloudinit/sources/DataSourceAzure.py index f906b8fa..1a74de21 100644 --- a/cloudinit/sources/DataSourceAzure.py +++ b/cloudinit/sources/DataSourceAzure.py @@ -207,8 +207,9 @@ def apply_hostname_bounce(hostname, policy, interface, command, shell = not isinstance(command, (list, tuple)) # capture=False, see comments in bug 1202758 and bug 1206164. util.log_time(logfunc=LOG.debug, msg="publishing hostname", - func=util.subp, kwargs={'command': command, 'shell': shell, - 'capture': False, 'env': env}) + get_uptime=True, func=util.subp, + kwargs={'command': command, 'shell': shell, 'capture': False, + 'env': env}) def crtfile_to_pubkey(fname): diff --git a/cloudinit/util.py b/cloudinit/util.py index b0eb56e6..4a74ba57 100644 --- a/cloudinit/util.py +++ b/cloudinit/util.py @@ -1772,7 +1772,7 @@ def which(program): return None -def log_time(logfunc, msg, func, args=None, kwargs=None, get_uptime=True): +def log_time(logfunc, msg, func, args=None, kwargs=None, get_uptime=False): if args is None: args = [] if kwargs is None: -- cgit v1.2.3 From 8c4d88a630025b6fe9f90957343a94105768533f Mon Sep 17 00:00:00 2001 From: Ben Howard Date: Tue, 30 Jul 2013 17:00:33 -0600 Subject: Added base64 support to SmartOS datasource. Added documentation on SmartOS datasource. --- cloudinit/sources/DataSourceSmartOS.py | 60 +++++++++++++++++++--- doc/sources/smartos/README.rst | 66 ++++++++++++++++++++++++ tests/unittests/test_datasource/test_smartos.py | 67 +++++++++++++++++++++---- 3 files changed, 176 insertions(+), 17 deletions(-) create mode 100644 doc/sources/smartos/README.rst (limited to 'cloudinit/sources') diff --git a/cloudinit/sources/DataSourceSmartOS.py b/cloudinit/sources/DataSourceSmartOS.py index 1ce20c10..e0bb871c 100644 --- a/cloudinit/sources/DataSourceSmartOS.py +++ b/cloudinit/sources/DataSourceSmartOS.py @@ -27,6 +27,7 @@ # +import base64 from cloudinit import log as logging from cloudinit import sources from cloudinit import util @@ -49,6 +50,10 @@ SMARTOS_ATTRIB_MAP = { 'motd_sys_info': ('motd_sys_info', True), } +# These are values which will never be base64 encoded. +SMARTOS_NO_BASE64 = ['root_authorized_keys', 'motd_sys_info', + 'iptables_disable'] + class DataSourceSmartOS(sources.DataSource): def __init__(self, sys_cfg, distro, paths): @@ -56,6 +61,10 @@ class DataSourceSmartOS(sources.DataSource): self.seed_dir = os.path.join(paths.seed_dir, 'sdc') self.is_smartdc = None self.seed = self.sys_cfg.get("serial_device", DEF_TTY_LOC) + self.all_base64 = self.sys_cfg.get("decode_base64", False) + self.base_64_encoded = [] + self.smartos_no_base64 = self.sys_cfg.get("no_base64_decode", + SMARTOS_NO_BASE64) self.seed_timeout = self.sys_cfg.get("serial_timeout", DEF_TTY_TIMEOUT) @@ -84,17 +93,41 @@ class DataSourceSmartOS(sources.DataSource): self.is_smartdc = True md['instance-id'] = system_uuid + self.base_64_encoded = query_data('base_64_enocded', + self.seed, + self.seed_timeout, + strip=True) + if self.base_64_encoded: + self.base_64_encoded = str(self.base_64_encoded).split(',') + else: + self.base_64_encoded = [] + + if not self.all_base64: + self.all_base64 = util.is_true(query_data('meta_encoded_base64', + self.seed, + self.seed_timeout, + strip=True)) + for ci_noun, attribute in SMARTOS_ATTRIB_MAP.iteritems(): smartos_noun, strip = attribute + + b64encoded = False + if self.all_base64 and \ + (smartos_noun not in self.smartos_no_base64 and \ + ci_noun not in self.smartos_no_base64): + b64encoded = True + md[ci_noun] = query_data(smartos_noun, self.seed, - self.seed_timeout, strip=strip) + self.seed_timeout, strip=strip, + b64encoded=b64encoded) if not md['local-hostname']: md['local-hostname'] = system_uuid + ud = None if md['user-data']: ud = md['user-data'] - else: + elif md['user-script']: ud = md['user-script'] self.metadata = md @@ -124,12 +157,17 @@ def get_serial(seed_device, seed_timeout): return ser -def query_data(noun, seed_device, seed_timeout, strip=False): +def query_data(noun, seed_device, seed_timeout, strip=False, b64encoded=False): """Makes a request to via the serial console via "GET " In the response, the first line is the status, while subsequent lines are is the value. A blank line with a "." is used to indicate end of response. + + If the response is expected to be base64 encoded, then set b64encoded + to true. Unfortantely, there is no way to know if something is 100% + encoded, so this method relies on being told if the data is base64 or + not. """ if not noun: @@ -153,12 +191,22 @@ def query_data(noun, seed_device, seed_timeout, strip=False): response.append(m) ser.close() + + resp = None if not strip: - return "".join(response) + resp = "".join(response) + elif b64encoded: + resp = "".join(response).rstrip() else: - return "".join(response).rstrip() + resp = "".join(response).rstrip() + + if b64encoded: + try: + return base64.b64decode(resp) + except TypeError: + return resp - return None + return resp def dmi_data(): diff --git a/doc/sources/smartos/README.rst b/doc/sources/smartos/README.rst new file mode 100644 index 00000000..96310857 --- /dev/null +++ b/doc/sources/smartos/README.rst @@ -0,0 +1,66 @@ +================== +SmartOS Datasource +================== + +This datasource finds metadata and user-data from the SmartOS virtualization +platform (i.e. Joyent). + +SmartOS Platform +---------------- +The SmartOS virtualization platform meta-data to the instance via the second +serial console. On Linux, this is /dev/ttyS1. The data is a provided via a +simple protocol, where something queries for the userdata, where the console +responds with the status and if "SUCCESS" returns until a single ".\n". + +The format is lossy. As such, new versions of the SmartOS tooling will include +support for base64 encoded data. + +Userdata +-------- + +In SmartOS parlance, user-data is a actually meta-data. This userdata can be +provided a key-value pairs. + +Cloud-init supports reading the traditional meta-data fields supported by the +SmartOS tools. These are: + * root_authorized_keys + * hostname + * enable_motd_sys_info + * iptables_disable + +Note: At this time iptables_disable and enable_motd_sys_info are read but + are not actioned. + +user-script +----------- + +SmartOS traditionally supports sending over a user-script for execution at the +rc.local level. Cloud-init supports running user-scripts as if they were +cloud-init user-data. In this sense, anything with a shell interpetter +directive will run + +user-data and user-script +------------------------- + +In the event that a user defines the meta-data key of "user-data" it will +always supercede any user-script data. This is for consistency. + +base64 +------ + +In order to provide a lossy format, all data except for: + * root_authorized_keys + * enable_motd_sys_info + * iptables_disable + +This means that user-script and user-data as well as other values can be +base64 encoded to provide a lossy format. Since Cloud-init can only guess +as to whether or not something is truly base64 encoded, the following +meta-data keys are hints as to whether or not to base64 decode something: + * decode_base64: Except for excluded keys, attempt to base64 decode + the values. If the value fails to decode properly, it will be + returned in its text + * base_64_encoded: A comma deliminated list of which values are base64 + encoded. + * no_base64_decode: This is a configuration setting (i.e. /etc/cloud/cloud.cfg.d) + that sets which values should not be base64 decoded. diff --git a/tests/unittests/test_datasource/test_smartos.py b/tests/unittests/test_datasource/test_smartos.py index 6c12f1e2..ae621433 100644 --- a/tests/unittests/test_datasource/test_smartos.py +++ b/tests/unittests/test_datasource/test_smartos.py @@ -22,6 +22,7 @@ # return responses. # +import base64 from cloudinit import helpers from cloudinit.sources import DataSourceSmartOS @@ -35,7 +36,7 @@ mock_returns = { 'enable_motd_sys_info': None, 'system_uuid': str(uuid.uuid4()), 'smartdc': 'smartdc', - 'userdata': """ + 'user-data': """ #!/bin/sh /bin/true """, @@ -48,12 +49,14 @@ class MockSerial(object): port = None - def __init__(self): + def __init__(self, b64encode=False): self.last = None self.last = None self.new = True self.count = 0 self.mocked_out = [] + self.b64encode = b64encode + self.b64excluded = DataSourceSmartOS.SMARTOS_NO_BASE64 def open(self): return True @@ -87,11 +90,17 @@ class MockSerial(object): def _format_out(self): if self.last in mock_returns: - try: - for l in mock_returns[self.last].splitlines(): - yield "%s\n" % l - except: - yield "%s\n" % mock_returns[self.last] + _mret = mock_returns[self.last] + if self.b64encode and \ + self.last not in self.b64excluded: + yield base64.b64encode(_mret) + + else: + try: + for l in _mret.splitlines(): + yield "%s\n" % l.rstrip() + except: + yield "%s\n" % _mret.rstrip() yield '\n' yield '.' @@ -116,16 +125,19 @@ class TestSmartOSDataSource(MockerTestCase): ret = apply_patches(patches) self.unapply += ret - def _get_ds(self): + def _get_ds(self, b64encode=False, sys_cfg=None): + mod = DataSourceSmartOS def _get_serial(*_): - return MockSerial() + return MockSerial(b64encode=b64encode) def _dmi_data(): return mock_returns['system_uuid'], 'smartdc' - data = {'sys_cfg': {}} - mod = DataSourceSmartOS + if not sys_cfg: + sys_cfg = {} + + data = {'sys_cfg': sys_cfg} self.apply_patches([(mod, 'get_serial', _get_serial)]) self.apply_patches([(mod, 'dmi_data', _dmi_data)]) dsrc = mod.DataSourceSmartOS( @@ -158,6 +170,13 @@ class TestSmartOSDataSource(MockerTestCase): self.assertEquals(mock_returns['root_authorized_keys'], dsrc.metadata['public-keys']) + def test_hostname_b64(self): + dsrc = self._get_ds(b64encode=True) + ret = dsrc.get_data() + self.assertTrue(ret) + self.assertEquals(base64.b64encode(mock_returns['hostname']), + dsrc.metadata['local-hostname']) + def test_hostname(self): dsrc = self._get_ds() ret = dsrc.get_data() @@ -165,6 +184,32 @@ class TestSmartOSDataSource(MockerTestCase): self.assertEquals(mock_returns['hostname'], dsrc.metadata['local-hostname']) + def test_base64(self): + """This tests to make sure that SmartOS system key/value pairs + are not interpetted as being base64 encoded, while making + sure that the others are when 'decode_base64' is set""" + dsrc = self._get_ds(sys_cfg={'decode_base64': True}, + b64encode=True) + ret = dsrc.get_data() + self.assertTrue(ret) + self.assertEquals(mock_returns['hostname'], + dsrc.metadata['local-hostname']) + self.assertEquals("%s" % mock_returns['user-data'], + dsrc.userdata_raw) + self.assertEquals(mock_returns['root_authorized_keys'], + dsrc.metadata['public-keys']) + self.assertEquals(mock_returns['disable_iptables_flag'], + dsrc.metadata['iptables_disable']) + self.assertEquals(mock_returns['enable_motd_sys_info'], + dsrc.metadata['motd_sys_info']) + + def test_userdata(self): + dsrc = self._get_ds() + ret = dsrc.get_data() + self.assertTrue(ret) + self.assertEquals("%s\n" % mock_returns['user-data'], + dsrc.userdata_raw) + def test_disable_iptables_flag(self): dsrc = self._get_ds() ret = dsrc.get_data() -- cgit v1.2.3 From e668da729a0f9cd5d93d909a9b44d74cf6925dd5 Mon Sep 17 00:00:00 2001 From: Scott Moser Date: Thu, 15 Aug 2013 13:16:01 -0400 Subject: do not set 'password', but set 'passwd' to crypt'd value 'password' was the wrong key. It should have been setting the default user's "plain_text_password". Instead of doing that, though, we're encrypting the value and putting it in 'passwd', which will then be passed on to useradd. The key value in doing this is that the plain text password will not be stored in obj.pkl. (admittedly it is still in plain text in the ovf-env.xml file). --- cloudinit/sources/DataSourceAzure.py | 7 ++++++- tests/unittests/test_datasource/test_azure.py | 11 ++++++++--- 2 files changed, 14 insertions(+), 4 deletions(-) (limited to 'cloudinit/sources') diff --git a/cloudinit/sources/DataSourceAzure.py b/cloudinit/sources/DataSourceAzure.py index 1a74de21..7ec622bf 100644 --- a/cloudinit/sources/DataSourceAzure.py +++ b/cloudinit/sources/DataSourceAzure.py @@ -17,6 +17,7 @@ # along with this program. If not, see . import base64 +import crypt import os import os.path import time @@ -424,7 +425,7 @@ def read_azure_ovf(contents): if username: defuser['name'] = username if password: - defuser['password'] = password + defuser['passwd'] = encrypt_pass(password) defuser['lock_passwd'] = False if defuser: @@ -436,6 +437,10 @@ def read_azure_ovf(contents): return (md, ud, cfg) +def encrypt_pass(password, salt_id="$6$"): + return crypt.crypt(password, salt_id + util.rand_str(strlen=16)) + + def list_possible_azure_ds_devs(): # return a sorted list of devices that might have a azure datasource devlist = [] diff --git a/tests/unittests/test_datasource/test_azure.py b/tests/unittests/test_datasource/test_azure.py index 06f8a5d2..1ca6a79d 100644 --- a/tests/unittests/test_datasource/test_azure.py +++ b/tests/unittests/test_datasource/test_azure.py @@ -2,6 +2,7 @@ from cloudinit import helpers from cloudinit.sources import DataSourceAzure from tests.unittests.helpers import populate_dir +import crypt import base64 from mocker import MockerTestCase import os @@ -207,11 +208,15 @@ class TestAzureDataSource(MockerTestCase): self.assertTrue('default_user' in dsrc.cfg['system_info']) defuser = dsrc.cfg['system_info']['default_user'] - # default user shoudl be updated for password and username - # and should not be locked. + # default user should be updated username and should not be locked. self.assertEqual(defuser['name'], odata['UserName']) - self.assertEqual(defuser['password'], odata['UserPassword']) self.assertFalse(defuser['lock_passwd']) + # passwd is crypt formated string $id$salt$encrypted + # encrypting plaintext with salt value of everything up to final '$' + # should equal that after the '$' + pos = defuser['passwd'].rfind("$") + 1 + self.assertEqual(defuser['passwd'], + crypt.crypt(odata['UserPassword'], defuser['passwd'][0:pos])) def test_userdata_found(self): mydata = "FOOBAR" -- cgit v1.2.3 From e683ab2baa3e67614edcd409122bd1aec99737e0 Mon Sep 17 00:00:00 2001 From: Ben Howard Date: Tue, 20 Aug 2013 09:56:25 -0600 Subject: Fixed no_base64_decode settings --- cloudinit/sources/DataSourceSmartOS.py | 26 ++++++++++++++++++++----- doc/examples/cloud-config-datasources.txt | 2 +- doc/sources/smartos/README.rst | 12 ++++++------ tests/unittests/test_datasource/test_smartos.py | 8 ++++++++ 4 files changed, 36 insertions(+), 12 deletions(-) (limited to 'cloudinit/sources') diff --git a/cloudinit/sources/DataSourceSmartOS.py b/cloudinit/sources/DataSourceSmartOS.py index e0bb871c..1cf9e4f0 100644 --- a/cloudinit/sources/DataSourceSmartOS.py +++ b/cloudinit/sources/DataSourceSmartOS.py @@ -35,7 +35,7 @@ import os import os.path import serial - +DS_NAME = 'SmartOS' DEF_TTY_LOC = '/dev/ttyS1' DEF_TTY_TIMEOUT = 60 LOG = logging.getLogger(__name__) @@ -60,13 +60,14 @@ class DataSourceSmartOS(sources.DataSource): sources.DataSource.__init__(self, sys_cfg, distro, paths) self.seed_dir = os.path.join(paths.seed_dir, 'sdc') self.is_smartdc = None + self.base_64_encoded = [] self.seed = self.sys_cfg.get("serial_device", DEF_TTY_LOC) self.all_base64 = self.sys_cfg.get("decode_base64", False) - self.base_64_encoded = [] - self.smartos_no_base64 = self.sys_cfg.get("no_base64_decode", - SMARTOS_NO_BASE64) self.seed_timeout = self.sys_cfg.get("serial_timeout", DEF_TTY_TIMEOUT) + self.smartos_no_base64 = SMARTOS_NO_BASE64 + if 'no_base64_decode' in self.ds_cfg: + self.smartos_no_base64 = self.ds_cfg['no_base64_decode'] def __str__(self): root = sources.DataSource.__str__(self) @@ -137,10 +138,25 @@ class DataSourceSmartOS(sources.DataSource): def get_instance_id(self): return self.metadata['instance-id'] + def not_b64_var(self, var): + """Return true if value is read as b64.""" + if var in self.smartos_no_base64 or \ + not self.all_base64: + return True + return False + + def is_b64_var(self, var): + """Return true if value is read as b64.""" + if self.all_base64 or ( + var not in self.smartos_no_base64 and + var in self.base_64_encoded): + return True + return False + def get_serial(seed_device, seed_timeout): """This is replaced in unit testing, allowing us to replace - serial.Serial with a mocked class + serial.Serial with a mocked class. The timeout value of 60 seconds should never be hit. The value is taken from SmartOS own provisioning tools. Since we are reading diff --git a/doc/examples/cloud-config-datasources.txt b/doc/examples/cloud-config-datasources.txt index 6544448e..6ec0d57e 100644 --- a/doc/examples/cloud-config-datasources.txt +++ b/doc/examples/cloud-config-datasources.txt @@ -55,5 +55,5 @@ datasource: # Smart OS datasource works over a serial console interacting with # a server on the other end. By default, the second serial console is the # device. SmartOS also uses a serial timeout of 60 seconds. - serial device: /dev/ttyS1 + serial_device: /dev/ttyS1 serial timeout: 60 diff --git a/doc/sources/smartos/README.rst b/doc/sources/smartos/README.rst index 96310857..ba90e7af 100644 --- a/doc/sources/smartos/README.rst +++ b/doc/sources/smartos/README.rst @@ -12,8 +12,7 @@ serial console. On Linux, this is /dev/ttyS1. The data is a provided via a simple protocol, where something queries for the userdata, where the console responds with the status and if "SUCCESS" returns until a single ".\n". -The format is lossy. As such, new versions of the SmartOS tooling will include -support for base64 encoded data. +New versions of the SmartOS tooling will include support for base64 encoded data. Userdata -------- @@ -48,15 +47,16 @@ always supercede any user-script data. This is for consistency. base64 ------ -In order to provide a lossy format, all data except for: +The following are excempt from base64 encoding, owing to the fact that they +are provided by SmartOS: * root_authorized_keys * enable_motd_sys_info * iptables_disable This means that user-script and user-data as well as other values can be -base64 encoded to provide a lossy format. Since Cloud-init can only guess -as to whether or not something is truly base64 encoded, the following -meta-data keys are hints as to whether or not to base64 decode something: +base64 encoded. Since Cloud-init can only guess as to whether or not something +is truly base64 encoded, the following meta-data keys are hints as to whether +or not to base64 decode something: * decode_base64: Except for excluded keys, attempt to base64 decode the values. If the value fails to decode properly, it will be returned in its text diff --git a/tests/unittests/test_datasource/test_smartos.py b/tests/unittests/test_datasource/test_smartos.py index ae621433..b9b3a479 100644 --- a/tests/unittests/test_datasource/test_smartos.py +++ b/tests/unittests/test_datasource/test_smartos.py @@ -36,6 +36,7 @@ mock_returns = { 'enable_motd_sys_info': None, 'system_uuid': str(uuid.uuid4()), 'smartdc': 'smartdc', + 'test-var1': 'some data', 'user-data': """ #!/bin/sh /bin/true @@ -156,6 +157,13 @@ class TestSmartOSDataSource(MockerTestCase): self.assertTrue(ret) self.assertTrue(dsrc.is_smartdc) + def test_no_base64(self): + sys_cfg = {'no_base64_decode': ['test_var1'], 'all_base': True} + dsrc = self._get_ds(sys_cfg=sys_cfg) + ret = dsrc.get_data() + self.assertTrue(ret) + self.assertTrue(dsrc.not_b64_var('test-var')) + def test_uuid(self): dsrc = self._get_ds() ret = dsrc.get_data() -- cgit v1.2.3 From 7af11ba50c8311ceb545b830716c78929079a0cd Mon Sep 17 00:00:00 2001 From: Scott Moser Date: Tue, 20 Aug 2013 16:00:09 -0400 Subject: fix bad arguments to subp inside of apply_hostname_bounce This simply correctly invokes subp through util.log_time. The arguments to subp is named 'args' not 'command'. LP: #1214541 --- cloudinit/sources/DataSourceAzure.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'cloudinit/sources') diff --git a/cloudinit/sources/DataSourceAzure.py b/cloudinit/sources/DataSourceAzure.py index 7ec622bf..66d7728b 100644 --- a/cloudinit/sources/DataSourceAzure.py +++ b/cloudinit/sources/DataSourceAzure.py @@ -209,7 +209,7 @@ def apply_hostname_bounce(hostname, policy, interface, command, # capture=False, see comments in bug 1202758 and bug 1206164. util.log_time(logfunc=LOG.debug, msg="publishing hostname", get_uptime=True, func=util.subp, - kwargs={'command': command, 'shell': shell, 'capture': False, + kwargs={'args': command, 'shell': shell, 'capture': False, 'env': env}) -- cgit v1.2.3 From 0a667454289a788a6f406e66d78a34c7ec914daa Mon Sep 17 00:00:00 2001 From: Ben Howard Date: Fri, 23 Aug 2013 09:10:30 -0600 Subject: Fixed some typos. Change decode_base64 from sys_cfg to ds_cfg --- cloudinit/sources/DataSourceSmartOS.py | 5 ++++- doc/sources/smartos/README.rst | 6 +++--- 2 files changed, 7 insertions(+), 4 deletions(-) (limited to 'cloudinit/sources') diff --git a/cloudinit/sources/DataSourceSmartOS.py b/cloudinit/sources/DataSourceSmartOS.py index 1cf9e4f0..45f03a7e 100644 --- a/cloudinit/sources/DataSourceSmartOS.py +++ b/cloudinit/sources/DataSourceSmartOS.py @@ -62,9 +62,12 @@ class DataSourceSmartOS(sources.DataSource): self.is_smartdc = None self.base_64_encoded = [] self.seed = self.sys_cfg.get("serial_device", DEF_TTY_LOC) - self.all_base64 = self.sys_cfg.get("decode_base64", False) self.seed_timeout = self.sys_cfg.get("serial_timeout", DEF_TTY_TIMEOUT) + self.all_base64 = False + if 'decode_base64' in self.ds_cfg: + self.all_base64 = self.ds_cfg['decode_base64'] + self.smartos_no_base64 = SMARTOS_NO_BASE64 if 'no_base64_decode' in self.ds_cfg: self.smartos_no_base64 = self.ds_cfg['no_base64_decode'] diff --git a/doc/sources/smartos/README.rst b/doc/sources/smartos/README.rst index ba90e7af..8f72fa0f 100644 --- a/doc/sources/smartos/README.rst +++ b/doc/sources/smartos/README.rst @@ -35,7 +35,7 @@ user-script SmartOS traditionally supports sending over a user-script for execution at the rc.local level. Cloud-init supports running user-scripts as if they were -cloud-init user-data. In this sense, anything with a shell interpetter +cloud-init user-data. In this sense, anything with a shell interpreter directive will run user-data and user-script @@ -47,7 +47,7 @@ always supercede any user-script data. This is for consistency. base64 ------ -The following are excempt from base64 encoding, owing to the fact that they +The following are exempt from base64 encoding, owing to the fact that they are provided by SmartOS: * root_authorized_keys * enable_motd_sys_info @@ -63,4 +63,4 @@ or not to base64 decode something: * base_64_encoded: A comma deliminated list of which values are base64 encoded. * no_base64_decode: This is a configuration setting (i.e. /etc/cloud/cloud.cfg.d) - that sets which values should not be base64 decoded. + that sets which values should not be base64 decoded. -- cgit v1.2.3 From 10c8ec1e5c1b16572a38afd08ee794d28c450054 Mon Sep 17 00:00:00 2001 From: Scott Moser Date: Sat, 24 Aug 2013 00:07:35 -0400 Subject: changes to behavior on specifying keys. The most likely end user operation (or at least a valid one) for base64 encoding would be to encode the user-data, but leave all other values as plaintext. In order to facilitate that, the user can simply add: b64-user-data=true to indicate that user-data is base64 encoded. Other changes here are to change the cloud-config and metadata keynames that are used. base64_all = boolean(True) base64_keys = [list, of, keys] Fixed up tests to accomodate. --- cloudinit/sources/DataSourceSmartOS.py | 94 ++++++--------- doc/examples/cloud-config-datasources.txt | 10 +- doc/sources/smartos/README.rst | 16 ++- tests/unittests/test_datasource/test_smartos.py | 153 ++++++++++++++---------- 4 files changed, 149 insertions(+), 124 deletions(-) (limited to 'cloudinit/sources') diff --git a/cloudinit/sources/DataSourceSmartOS.py b/cloudinit/sources/DataSourceSmartOS.py index 45f03a7e..d348d20b 100644 --- a/cloudinit/sources/DataSourceSmartOS.py +++ b/cloudinit/sources/DataSourceSmartOS.py @@ -35,7 +35,6 @@ import os import os.path import serial -DS_NAME = 'SmartOS' DEF_TTY_LOC = '/dev/ttyS1' DEF_TTY_TIMEOUT = 60 LOG = logging.getLogger(__name__) @@ -51,6 +50,7 @@ SMARTOS_ATTRIB_MAP = { } # These are values which will never be base64 encoded. +# They come from the cloud platform, not user SMARTOS_NO_BASE64 = ['root_authorized_keys', 'motd_sys_info', 'iptables_disable'] @@ -60,17 +60,13 @@ class DataSourceSmartOS(sources.DataSource): sources.DataSource.__init__(self, sys_cfg, distro, paths) self.seed_dir = os.path.join(paths.seed_dir, 'sdc') self.is_smartdc = None - self.base_64_encoded = [] - self.seed = self.sys_cfg.get("serial_device", DEF_TTY_LOC) - self.seed_timeout = self.sys_cfg.get("serial_timeout", - DEF_TTY_TIMEOUT) - self.all_base64 = False - if 'decode_base64' in self.ds_cfg: - self.all_base64 = self.ds_cfg['decode_base64'] - - self.smartos_no_base64 = SMARTOS_NO_BASE64 - if 'no_base64_decode' in self.ds_cfg: - self.smartos_no_base64 = self.ds_cfg['no_base64_decode'] + + self.seed = self.ds_cfg.get("serial_device", DEF_TTY_LOC) + self.seed_timeout = self.ds_cfg.get("serial_timeout", DEF_TTY_TIMEOUT) + self.smartos_no_base64 = self.ds_cfg.get('no_base64_decode', + SMARTOS_NO_BASE64) + self.b64_keys = self.ds_cfg.get('base64_keys', []) + self.b64_all = self.ds_cfg.get('base64_all', False) def __str__(self): root = sources.DataSource.__str__(self) @@ -92,38 +88,22 @@ class DataSourceSmartOS(sources.DataSource): system_uuid, system_type = dmi_info if 'smartdc' not in system_type.lower(): - LOG.debug("Host is not on SmartOS") + LOG.debug("Host is not on SmartOS. system_type=%s", system_type) return False self.is_smartdc = True md['instance-id'] = system_uuid - self.base_64_encoded = query_data('base_64_enocded', - self.seed, - self.seed_timeout, - strip=True) - if self.base_64_encoded: - self.base_64_encoded = str(self.base_64_encoded).split(',') - else: - self.base_64_encoded = [] + b64_keys = self.query('base64_keys', strip=True, b64=False) + if b64_keys is not None: + self.b64_keys = [k.strip() for k in str(b64_keys).split(',')] - if not self.all_base64: - self.all_base64 = util.is_true(query_data('meta_encoded_base64', - self.seed, - self.seed_timeout, - strip=True)) + b64_all = self.query('base64_all', strip=True, b64=False) + if b64_all is not None: + self.b64_all = util.is_true(b64_all) for ci_noun, attribute in SMARTOS_ATTRIB_MAP.iteritems(): smartos_noun, strip = attribute - - b64encoded = False - if self.all_base64 and \ - (smartos_noun not in self.smartos_no_base64 and \ - ci_noun not in self.smartos_no_base64): - b64encoded = True - - md[ci_noun] = query_data(smartos_noun, self.seed, - self.seed_timeout, strip=strip, - b64encoded=b64encoded) + md[ci_noun] = self.query(smartos_noun, strip=strip) if not md['local-hostname']: md['local-hostname'] = system_uuid @@ -141,20 +121,16 @@ class DataSourceSmartOS(sources.DataSource): def get_instance_id(self): return self.metadata['instance-id'] - def not_b64_var(self, var): - """Return true if value is read as b64.""" - if var in self.smartos_no_base64 or \ - not self.all_base64: - return True - return False + def query(self, noun, strip=False, default=None, b64=None): + if b64 is None: + if noun in self.smartos_no_base64: + b64 = False + elif self.b64_all or noun in self.b64_keys: + b64 = True - def is_b64_var(self, var): - """Return true if value is read as b64.""" - if self.all_base64 or ( - var not in self.smartos_no_base64 and - var in self.base_64_encoded): - return True - return False + return query_data(noun=noun, strip=strip, seed_device=self.seed, + seed_timeout=self.seed_timeout, default=default, + b64=b64) def get_serial(seed_device, seed_timeout): @@ -176,7 +152,8 @@ def get_serial(seed_device, seed_timeout): return ser -def query_data(noun, seed_device, seed_timeout, strip=False, b64encoded=False): +def query_data(noun, seed_device, seed_timeout, strip=False, default=None, + b64=None): """Makes a request to via the serial console via "GET " In the response, the first line is the status, while subsequent lines @@ -200,7 +177,7 @@ def query_data(noun, seed_device, seed_timeout, strip=False, b64encoded=False): if 'SUCCESS' not in status: ser.close() - return None + return default while not eom_found: m = ser.readline() @@ -211,18 +188,23 @@ def query_data(noun, seed_device, seed_timeout, strip=False, b64encoded=False): ser.close() + if b64 is None: + b64 = query_data('b64-%s' % noun, seed_device=seed_device, + seed_timeout=seed_timeout, b64=False, + default=False, strip=True) + b64 = util.is_true(b64) + resp = None - if not strip: - resp = "".join(response) - elif b64encoded: + if b64 or strip: resp = "".join(response).rstrip() else: - resp = "".join(response).rstrip() + resp = "".join(response) - if b64encoded: + if b64: try: return base64.b64decode(resp) except TypeError: + LOG.warn("Failed base64 decoding key '%s'", noun) return resp return resp diff --git a/doc/examples/cloud-config-datasources.txt b/doc/examples/cloud-config-datasources.txt index 6ec0d57e..65a3cdf5 100644 --- a/doc/examples/cloud-config-datasources.txt +++ b/doc/examples/cloud-config-datasources.txt @@ -56,4 +56,12 @@ datasource: # a server on the other end. By default, the second serial console is the # device. SmartOS also uses a serial timeout of 60 seconds. serial_device: /dev/ttyS1 - serial timeout: 60 + serial_timeout: 60 + + # a list of keys that will not be base64 decoded even if base64_all + no_base64_decode: ['root_authorized_keys', 'motd_sys_info', + 'iptables_disable'] + # a plaintext, comma delimited list of keys whose values are b64 encoded + base64_keys: [] + # a boolean indicating that all keys not in 'no_base64_decode' are encoded + base64_all: False diff --git a/doc/sources/smartos/README.rst b/doc/sources/smartos/README.rst index 8f72fa0f..fd4e496d 100644 --- a/doc/sources/smartos/README.rst +++ b/doc/sources/smartos/README.rst @@ -53,14 +53,20 @@ are provided by SmartOS: * enable_motd_sys_info * iptables_disable +This list can be changed through system config of variable 'no_base64_decode'. + This means that user-script and user-data as well as other values can be base64 encoded. Since Cloud-init can only guess as to whether or not something is truly base64 encoded, the following meta-data keys are hints as to whether or not to base64 decode something: - * decode_base64: Except for excluded keys, attempt to base64 decode + * base64_all: Except for excluded keys, attempt to base64 decode the values. If the value fails to decode properly, it will be returned in its text - * base_64_encoded: A comma deliminated list of which values are base64 - encoded. - * no_base64_decode: This is a configuration setting (i.e. /etc/cloud/cloud.cfg.d) - that sets which values should not be base64 decoded. + * base64_keys: A comma deliminated list of which keys are base64 encoded. + * b64-: + for any key, if there exists an entry in the metadata for 'b64-' + Then 'b64-' is expected to be a plaintext boolean indicating whether + or not its value is encoded. + * no_base64_decode: This is a configuration setting + (i.e. /etc/cloud/cloud.cfg.d) that sets which values should not be + base64 decoded. diff --git a/tests/unittests/test_datasource/test_smartos.py b/tests/unittests/test_datasource/test_smartos.py index b9b3a479..f53715b0 100644 --- a/tests/unittests/test_datasource/test_smartos.py +++ b/tests/unittests/test_datasource/test_smartos.py @@ -29,20 +29,17 @@ from cloudinit.sources import DataSourceSmartOS from mocker import MockerTestCase import uuid -mock_returns = { +MOCK_RETURNS = { 'hostname': 'test-host', 'root_authorized_keys': 'ssh-rsa AAAAB3Nz...aC1yc2E= keyname', 'disable_iptables_flag': None, 'enable_motd_sys_info': None, - 'system_uuid': str(uuid.uuid4()), - 'smartdc': 'smartdc', 'test-var1': 'some data', - 'user-data': """ -#!/bin/sh -/bin/true -""", + 'user-data': '\n'.join(['#!/bin/sh', '/bin/true', '']), } +DMI_DATA_RETURN = (str(uuid.uuid4()), 'smartdc') + class MockSerial(object): """Fake a serial terminal for testing the code that @@ -50,14 +47,13 @@ class MockSerial(object): port = None - def __init__(self, b64encode=False): + def __init__(self, mockdata): self.last = None self.last = None self.new = True self.count = 0 self.mocked_out = [] - self.b64encode = b64encode - self.b64excluded = DataSourceSmartOS.SMARTOS_NO_BASE64 + self.mockdata = mockdata def open(self): return True @@ -75,12 +71,12 @@ class MockSerial(object): def readline(self): if self.new: self.new = False - if self.last in mock_returns: + if self.last in self.mockdata: return 'SUCCESS\n' else: return 'NOTFOUND %s\n' % self.last - if self.last in mock_returns: + if self.last in self.mockdata: if not self.mocked_out: self.mocked_out = [x for x in self._format_out()] print self.mocked_out @@ -90,21 +86,16 @@ class MockSerial(object): return self.mocked_out[self.count - 1] def _format_out(self): - if self.last in mock_returns: - _mret = mock_returns[self.last] - if self.b64encode and \ - self.last not in self.b64excluded: - yield base64.b64encode(_mret) - - else: - try: - for l in _mret.splitlines(): - yield "%s\n" % l.rstrip() - except: - yield "%s\n" % _mret.rstrip() + if self.last in self.mockdata: + _mret = self.mockdata[self.last] + try: + for l in _mret.splitlines(): + yield "%s\n" % l.rstrip() + except: + yield "%s\n" % _mret.rstrip() - yield '\n' yield '.' + yield '\n' class TestSmartOSDataSource(MockerTestCase): @@ -126,26 +117,36 @@ class TestSmartOSDataSource(MockerTestCase): ret = apply_patches(patches) self.unapply += ret - def _get_ds(self, b64encode=False, sys_cfg=None): + def _get_ds(self, sys_cfg=None, ds_cfg=None, mockdata=None, dmi_data=None): mod = DataSourceSmartOS + if mockdata is None: + mockdata = MOCK_RETURNS + + if dmi_data is None: + dmi_data = DMI_DATA_RETURN + def _get_serial(*_): - return MockSerial(b64encode=b64encode) + return MockSerial(mockdata) def _dmi_data(): - return mock_returns['system_uuid'], 'smartdc' + return dmi_data - if not sys_cfg: + if sys_cfg is None: sys_cfg = {} - data = {'sys_cfg': sys_cfg} + if ds_cfg is not None: + sys_cfg['datasource'] = sys_cfg.get('datasource', {}) + sys_cfg['datasource']['SmartOS'] = ds_cfg + self.apply_patches([(mod, 'get_serial', _get_serial)]) self.apply_patches([(mod, 'dmi_data', _dmi_data)]) - dsrc = mod.DataSourceSmartOS( - data.get('sys_cfg', {}), distro=None, paths=self.paths) + dsrc = mod.DataSourceSmartOS(sys_cfg, distro=None, + paths=self.paths) return dsrc def test_seed(self): + # default seed should be /dev/ttyS1 dsrc = self._get_ds() ret = dsrc.get_data() self.assertTrue(ret) @@ -158,78 +159,106 @@ class TestSmartOSDataSource(MockerTestCase): self.assertTrue(dsrc.is_smartdc) def test_no_base64(self): - sys_cfg = {'no_base64_decode': ['test_var1'], 'all_base': True} - dsrc = self._get_ds(sys_cfg=sys_cfg) + ds_cfg = {'no_base64_decode': ['test_var1'], 'all_base': True} + dsrc = self._get_ds(ds_cfg=ds_cfg) ret = dsrc.get_data() self.assertTrue(ret) - self.assertTrue(dsrc.not_b64_var('test-var')) def test_uuid(self): - dsrc = self._get_ds() + dsrc = self._get_ds(mockdata=MOCK_RETURNS) ret = dsrc.get_data() self.assertTrue(ret) - self.assertEquals(mock_returns['system_uuid'], - dsrc.metadata['instance-id']) + self.assertEquals(DMI_DATA_RETURN[0], dsrc.metadata['instance-id']) def test_root_keys(self): - dsrc = self._get_ds() + dsrc = self._get_ds(mockdata=MOCK_RETURNS) ret = dsrc.get_data() self.assertTrue(ret) - self.assertEquals(mock_returns['root_authorized_keys'], + self.assertEquals(MOCK_RETURNS['root_authorized_keys'], dsrc.metadata['public-keys']) def test_hostname_b64(self): - dsrc = self._get_ds(b64encode=True) + dsrc = self._get_ds(mockdata=MOCK_RETURNS) ret = dsrc.get_data() self.assertTrue(ret) - self.assertEquals(base64.b64encode(mock_returns['hostname']), + self.assertEquals(MOCK_RETURNS['hostname'], dsrc.metadata['local-hostname']) def test_hostname(self): - dsrc = self._get_ds() + dsrc = self._get_ds(mockdata=MOCK_RETURNS) ret = dsrc.get_data() self.assertTrue(ret) - self.assertEquals(mock_returns['hostname'], + self.assertEquals(MOCK_RETURNS['hostname'], dsrc.metadata['local-hostname']) - def test_base64(self): - """This tests to make sure that SmartOS system key/value pairs - are not interpetted as being base64 encoded, while making - sure that the others are when 'decode_base64' is set""" - dsrc = self._get_ds(sys_cfg={'decode_base64': True}, - b64encode=True) + def test_base64_all(self): + # metadata provided base64_all of true + my_returns = MOCK_RETURNS.copy() + my_returns['base64_all'] = "true" + for k in ('hostname', 'user-data'): + my_returns[k] = base64.b64encode(my_returns[k]) + + dsrc = self._get_ds(mockdata=my_returns) ret = dsrc.get_data() self.assertTrue(ret) - self.assertEquals(mock_returns['hostname'], + self.assertEquals(MOCK_RETURNS['hostname'], dsrc.metadata['local-hostname']) - self.assertEquals("%s" % mock_returns['user-data'], + self.assertEquals(MOCK_RETURNS['user-data'], dsrc.userdata_raw) - self.assertEquals(mock_returns['root_authorized_keys'], + self.assertEquals(MOCK_RETURNS['root_authorized_keys'], dsrc.metadata['public-keys']) - self.assertEquals(mock_returns['disable_iptables_flag'], + self.assertEquals(MOCK_RETURNS['disable_iptables_flag'], dsrc.metadata['iptables_disable']) - self.assertEquals(mock_returns['enable_motd_sys_info'], + self.assertEquals(MOCK_RETURNS['enable_motd_sys_info'], dsrc.metadata['motd_sys_info']) + def test_b64_userdata(self): + my_returns = MOCK_RETURNS.copy() + my_returns['b64-user-data'] = "true" + my_returns['b64-hostname'] = "true" + for k in ('hostname', 'user-data'): + my_returns[k] = base64.b64encode(my_returns[k]) + + dsrc = self._get_ds(mockdata=my_returns) + ret = dsrc.get_data() + self.assertTrue(ret) + self.assertEquals(MOCK_RETURNS['hostname'], + dsrc.metadata['local-hostname']) + self.assertEquals(MOCK_RETURNS['user-data'], dsrc.userdata_raw) + self.assertEquals(MOCK_RETURNS['root_authorized_keys'], + dsrc.metadata['public-keys']) + + def test_b64_keys(self): + my_returns = MOCK_RETURNS.copy() + my_returns['base64_keys'] = 'hostname,ignored' + for k in ('hostname',): + my_returns[k] = base64.b64encode(my_returns[k]) + + dsrc = self._get_ds(mockdata=my_returns) + ret = dsrc.get_data() + self.assertTrue(ret) + self.assertEquals(MOCK_RETURNS['hostname'], + dsrc.metadata['local-hostname']) + self.assertEquals(MOCK_RETURNS['user-data'], dsrc.userdata_raw) + def test_userdata(self): - dsrc = self._get_ds() + dsrc = self._get_ds(mockdata=MOCK_RETURNS) ret = dsrc.get_data() self.assertTrue(ret) - self.assertEquals("%s\n" % mock_returns['user-data'], - dsrc.userdata_raw) + self.assertEquals(MOCK_RETURNS['user-data'], dsrc.userdata_raw) def test_disable_iptables_flag(self): - dsrc = self._get_ds() + dsrc = self._get_ds(mockdata=MOCK_RETURNS) ret = dsrc.get_data() self.assertTrue(ret) - self.assertEquals(mock_returns['disable_iptables_flag'], + self.assertEquals(MOCK_RETURNS['disable_iptables_flag'], dsrc.metadata['iptables_disable']) def test_motd_sys_info(self): - dsrc = self._get_ds() + dsrc = self._get_ds(mockdata=MOCK_RETURNS) ret = dsrc.get_data() self.assertTrue(ret) - self.assertEquals(mock_returns['enable_motd_sys_info'], + self.assertEquals(MOCK_RETURNS['enable_motd_sys_info'], dsrc.metadata['motd_sys_info']) -- cgit v1.2.3