diff options
Diffstat (limited to 'cloudinit/handlers')
| -rw-r--r-- | cloudinit/handlers/__init__.py | 94 | ||||
| -rw-r--r-- | cloudinit/handlers/boot_hook.py | 20 | ||||
| -rw-r--r-- | cloudinit/handlers/cloud_config.py | 136 | ||||
| -rw-r--r-- | cloudinit/handlers/shell_script.py | 6 | ||||
| -rw-r--r-- | cloudinit/handlers/upstart_job.py | 60 |
5 files changed, 241 insertions, 75 deletions
diff --git a/cloudinit/handlers/__init__.py b/cloudinit/handlers/__init__.py index 8d6dcd4d..2ddc75f4 100644 --- a/cloudinit/handlers/__init__.py +++ b/cloudinit/handlers/__init__.py @@ -1,7 +1,7 @@ # vi: ts=4 expandtab # # Copyright (C) 2012 Canonical Ltd. -# Copyright (C) 2012 Hewlett-Packard Development Company, L.P. +# Copyright (C) 2012, 2013 Hewlett-Packard Development Company, L.P. # Copyright (C) 2012 Yahoo! Inc. # # Author: Scott Moser <scott.moser@canonical.com> @@ -27,6 +27,7 @@ from cloudinit.settings import (PER_ALWAYS, PER_INSTANCE, FREQUENCIES) from cloudinit import importer from cloudinit import log as logging +from cloudinit import type_utils from cloudinit import util LOG = logging.getLogger(__name__) @@ -61,6 +62,7 @@ INCLUSION_TYPES_MAP = { '#part-handler': 'text/part-handler', '#cloud-boothook': 'text/cloud-boothook', '#cloud-config-archive': 'text/cloud-config-archive', + '#cloud-config-jsonp': 'text/cloud-config-jsonp', } # Sorted longest first @@ -69,7 +71,6 @@ INCLUSION_SRCH = sorted(list(INCLUSION_TYPES_MAP.keys()), class Handler(object): - __metaclass__ = abc.ABCMeta def __init__(self, frequency, version=2): @@ -77,53 +78,65 @@ class Handler(object): self.frequency = frequency def __repr__(self): - return "%s: [%s]" % (util.obj_name(self), self.list_types()) + return "%s: [%s]" % (type_utils.obj_name(self), self.list_types()) @abc.abstractmethod def list_types(self): raise NotImplementedError() - def handle_part(self, data, ctype, filename, payload, frequency): - return self._handle_part(data, ctype, filename, payload, frequency) - @abc.abstractmethod - def _handle_part(self, data, ctype, filename, payload, frequency): + def handle_part(self, *args, **kwargs): raise NotImplementedError() -def run_part(mod, data, ctype, filename, payload, frequency): +def run_part(mod, data, filename, payload, frequency, headers): mod_freq = mod.frequency if not (mod_freq == PER_ALWAYS or (frequency == PER_INSTANCE and mod_freq == PER_INSTANCE)): return - mod_ver = mod.handler_version # Sanity checks on version (should be an int convertable) try: + mod_ver = mod.handler_version mod_ver = int(mod_ver) - except: + except (TypeError, ValueError, AttributeError): mod_ver = 1 + content_type = headers['Content-Type'] try: LOG.debug("Calling handler %s (%s, %s, %s) with frequency %s", - mod, ctype, filename, mod_ver, frequency) - if mod_ver >= 2: + mod, content_type, filename, mod_ver, frequency) + if mod_ver == 3: + # Treat as v. 3 which does get a frequency + headers + mod.handle_part(data, content_type, filename, + payload, frequency, headers) + elif mod_ver == 2: # Treat as v. 2 which does get a frequency - mod.handle_part(data, ctype, filename, payload, frequency) - else: + mod.handle_part(data, content_type, filename, + payload, frequency) + elif mod_ver == 1: # Treat as v. 1 which gets no frequency - mod.handle_part(data, ctype, filename, payload) + mod.handle_part(data, content_type, filename, payload) + else: + raise ValueError("Unknown module version %s" % (mod_ver)) except: - util.logexc(LOG, ("Failed calling handler %s (%s, %s, %s)" - " with frequency %s"), - mod, ctype, filename, - mod_ver, frequency) + util.logexc(LOG, "Failed calling handler %s (%s, %s, %s) with " + "frequency %s", mod, content_type, filename, mod_ver, + frequency) def call_begin(mod, data, frequency): - run_part(mod, data, CONTENT_START, None, None, frequency) + # Create a fake header set + headers = { + 'Content-Type': CONTENT_START, + } + run_part(mod, data, None, None, frequency, headers) def call_end(mod, data, frequency): - run_part(mod, data, CONTENT_END, None, None, frequency) + # Create a fake header set + headers = { + 'Content-Type': CONTENT_END, + } + run_part(mod, data, None, None, frequency, headers) def walker_handle_handler(pdata, _ctype, _filename, payload): @@ -139,14 +152,13 @@ def walker_handle_handler(pdata, _ctype, _filename, payload): try: mod = fixup_handler(importer.import_module(modname)) call_begin(mod, pdata['data'], frequency) - # Only register and increment - # after the above have worked (so we don't if it - # fails) - handlers.register(mod) + # Only register and increment after the above have worked, so we don't + # register if it fails starting. + handlers.register(mod, initialized=True) pdata['handlercount'] = curcount + 1 except: - util.logexc(LOG, ("Failed at registering python file: %s" - " (part handler %s)"), modfname, curcount) + util.logexc(LOG, "Failed at registering python file: %s (part " + "handler %s)", modfname, curcount) def _extract_first_or_bytes(blob, size): @@ -173,26 +185,27 @@ def _escape_string(text): return text -def walker_callback(pdata, ctype, filename, payload): - if ctype in PART_CONTENT_TYPES: - walker_handle_handler(pdata, ctype, filename, payload) +def walker_callback(data, filename, payload, headers): + content_type = headers['Content-Type'] + if content_type in PART_CONTENT_TYPES: + walker_handle_handler(data, content_type, filename, payload) return - handlers = pdata['handlers'] - if ctype in pdata['handlers']: - run_part(handlers[ctype], pdata['data'], ctype, filename, - payload, pdata['frequency']) + handlers = data['handlers'] + if content_type in handlers: + run_part(handlers[content_type], data['data'], filename, + payload, data['frequency'], headers) elif payload: # Extract the first line or 24 bytes for displaying in the log start = _extract_first_or_bytes(payload, 24) details = "'%s...'" % (_escape_string(start)) - if ctype == NOT_MULTIPART_TYPE: + if content_type == NOT_MULTIPART_TYPE: LOG.warning("Unhandled non-multipart (%s) userdata: %s", - ctype, details) + content_type, details) else: LOG.warning("Unhandled unknown content-type (%s) userdata: %s", - ctype, details) + content_type, details) else: - LOG.debug("empty payload of type %s" % ctype) + LOG.debug("Empty payload of type %s", content_type) # Callback is a function that will be called with @@ -212,7 +225,10 @@ def walk(msg, callback, data): if not filename: filename = PART_FN_TPL % (partnum) - callback(data, ctype, filename, part.get_payload(decode=True)) + headers = dict(part) + LOG.debug(headers) + headers['Content-Type'] = ctype + callback(data, filename, part.get_payload(decode=True), headers) partnum = partnum + 1 diff --git a/cloudinit/handlers/boot_hook.py b/cloudinit/handlers/boot_hook.py index 456b8020..1848ce2c 100644 --- a/cloudinit/handlers/boot_hook.py +++ b/cloudinit/handlers/boot_hook.py @@ -29,6 +29,7 @@ from cloudinit import util from cloudinit.settings import (PER_ALWAYS) LOG = logging.getLogger(__name__) +BOOTHOOK_PREFIX = "#cloud-boothook" class BootHookPartHandler(handlers.Handler): @@ -41,22 +42,19 @@ class BootHookPartHandler(handlers.Handler): def list_types(self): return [ - handlers.type_from_starts_with("#cloud-boothook"), + handlers.type_from_starts_with(BOOTHOOK_PREFIX), ] def _write_part(self, payload, filename): filename = util.clean_filename(filename) - payload = util.dos2unix(payload) - prefix = "#cloud-boothook" - start = 0 - if payload.startswith(prefix): - start = len(prefix) + 1 filepath = os.path.join(self.boothook_dir, filename) - contents = payload[start:] - util.write_file(filepath, contents, 0700) + contents = util.strip_prefix_suffix(util.dos2unix(payload), + prefix=BOOTHOOK_PREFIX) + util.write_file(filepath, contents.lstrip(), 0700) return filepath - def _handle_part(self, _data, ctype, filename, payload, _frequency): + def handle_part(self, _data, ctype, filename, # pylint: disable=W0221 + payload, frequency): # pylint: disable=W0613 if ctype in handlers.CONTENT_SIGNALS: return @@ -69,5 +67,5 @@ class BootHookPartHandler(handlers.Handler): except util.ProcessExecutionError: util.logexc(LOG, "Boothooks script %s execution error", filepath) except Exception: - util.logexc(LOG, ("Boothooks unknown " - "error when running %s"), filepath) + util.logexc(LOG, "Boothooks unknown error when running %s", + filepath) diff --git a/cloudinit/handlers/cloud_config.py b/cloudinit/handlers/cloud_config.py index f6d95244..34a73115 100644 --- a/cloudinit/handlers/cloud_config.py +++ b/cloudinit/handlers/cloud_config.py @@ -20,43 +20,143 @@ # You should have received a copy of the GNU General Public License # along with this program. If not, see <http://www.gnu.org/licenses/>. +import jsonpatch + from cloudinit import handlers from cloudinit import log as logging +from cloudinit import mergers from cloudinit import util from cloudinit.settings import (PER_ALWAYS) LOG = logging.getLogger(__name__) +MERGE_HEADER = 'Merge-Type' + +# Due to the way the loading of yaml configuration was done previously, +# where previously each cloud config part was appended to a larger yaml +# file and then finally that file was loaded as one big yaml file we need +# to mimic that behavior by altering the default strategy to be replacing +# keys of prior merges. +# +# +# For example +# #file 1 +# a: 3 +# #file 2 +# a: 22 +# #combined file (comments not included) +# a: 3 +# a: 22 +# +# This gets loaded into yaml with final result {'a': 22} +DEF_MERGERS = mergers.string_extract_mergers('dict(replace)+list()+str()') +CLOUD_PREFIX = "#cloud-config" +JSONP_PREFIX = "#cloud-config-jsonp" + +# The file header -> content types this module will handle. +CC_TYPES = { + JSONP_PREFIX: handlers.type_from_starts_with(JSONP_PREFIX), + CLOUD_PREFIX: handlers.type_from_starts_with(CLOUD_PREFIX), +} + class CloudConfigPartHandler(handlers.Handler): def __init__(self, paths, **_kwargs): - handlers.Handler.__init__(self, PER_ALWAYS) - self.cloud_buf = [] + handlers.Handler.__init__(self, PER_ALWAYS, version=3) + self.cloud_buf = None self.cloud_fn = paths.get_ipath("cloud_config") + self.file_names = [] def list_types(self): - return [ - handlers.type_from_starts_with("#cloud-config"), - ] + return list(CC_TYPES.values()) - def _write_cloud_config(self, buf): + def _write_cloud_config(self): if not self.cloud_fn: return - lines = [str(b) for b in buf] - payload = "\n".join(lines) - util.write_file(self.cloud_fn, payload, 0600) + # Capture which files we merged from... + file_lines = [] + if self.file_names: + file_lines.append("# from %s files" % (len(self.file_names))) + for fn in self.file_names: + if not fn: + fn = '?' + file_lines.append("# %s" % (fn)) + file_lines.append("") + if self.cloud_buf is not None: + # Something was actually gathered.... + lines = [ + CLOUD_PREFIX, + '', + ] + lines.extend(file_lines) + lines.append(util.yaml_dumps(self.cloud_buf)) + else: + lines = [] + util.write_file(self.cloud_fn, "\n".join(lines), 0600) + + def _extract_mergers(self, payload, headers): + merge_header_headers = '' + for h in [MERGE_HEADER, 'X-%s' % (MERGE_HEADER)]: + tmp_h = headers.get(h, '') + if tmp_h: + merge_header_headers = tmp_h + break + # Select either the merge-type from the content + # or the merge type from the headers or default to our own set + # if neither exists (or is empty) from the later. + payload_yaml = util.load_yaml(payload) + mergers_yaml = mergers.dict_extract_mergers(payload_yaml) + mergers_header = mergers.string_extract_mergers(merge_header_headers) + all_mergers = [] + all_mergers.extend(mergers_yaml) + all_mergers.extend(mergers_header) + if not all_mergers: + all_mergers = DEF_MERGERS + return (payload_yaml, all_mergers) + + def _merge_patch(self, payload): + # JSON doesn't handle comments in this manner, so ensure that + # if we started with this 'type' that we remove it before + # attempting to load it as json (which the jsonpatch library will + # attempt to do). + payload = payload.lstrip() + payload = util.strip_prefix_suffix(payload, prefix=JSONP_PREFIX) + patch = jsonpatch.JsonPatch.from_string(payload) + LOG.debug("Merging by applying json patch %s", patch) + self.cloud_buf = patch.apply(self.cloud_buf, in_place=False) - def _handle_part(self, _data, ctype, filename, payload, _frequency): + def _merge_part(self, payload, headers): + (payload_yaml, my_mergers) = self._extract_mergers(payload, headers) + LOG.debug("Merging by applying %s", my_mergers) + merger = mergers.construct(my_mergers) + self.cloud_buf = merger.merge(self.cloud_buf, payload_yaml) + + def _reset(self): + self.file_names = [] + self.cloud_buf = None + + def handle_part(self, _data, ctype, filename, # pylint: disable=W0221 + payload, _frequency, headers): # pylint: disable=W0613 if ctype == handlers.CONTENT_START: - self.cloud_buf = [] + self._reset() return if ctype == handlers.CONTENT_END: - self._write_cloud_config(self.cloud_buf) - self.cloud_buf = [] + self._write_cloud_config() + self._reset() return - - filename = util.clean_filename(filename) - if not filename: - filename = '??' - self.cloud_buf.extend(["#%s" % (filename), str(payload)]) + try: + # First time through, merge with an empty dict... + if self.cloud_buf is None or not self.file_names: + self.cloud_buf = {} + if ctype == CC_TYPES[JSONP_PREFIX]: + self._merge_patch(payload) + else: + self._merge_part(payload, headers) + # Ensure filename is ok to store + for i in ("\n", "\r", "\t"): + filename = filename.replace(i, " ") + self.file_names.append(filename.strip()) + except: + util.logexc(LOG, "Failed at merging in cloud config part from %s", + filename) diff --git a/cloudinit/handlers/shell_script.py b/cloudinit/handlers/shell_script.py index 6c5c11ca..62289d98 100644 --- a/cloudinit/handlers/shell_script.py +++ b/cloudinit/handlers/shell_script.py @@ -29,6 +29,7 @@ from cloudinit import util from cloudinit.settings import (PER_ALWAYS) LOG = logging.getLogger(__name__) +SHELL_PREFIX = "#!" class ShellScriptPartHandler(handlers.Handler): @@ -38,10 +39,11 @@ class ShellScriptPartHandler(handlers.Handler): def list_types(self): return [ - handlers.type_from_starts_with("#!"), + handlers.type_from_starts_with(SHELL_PREFIX), ] - def _handle_part(self, _data, ctype, filename, payload, _frequency): + def handle_part(self, _data, ctype, filename, # pylint: disable=W0221 + payload, frequency): # pylint: disable=W0613 if ctype in handlers.CONTENT_SIGNALS: # TODO(harlowja): maybe delete existing things here return diff --git a/cloudinit/handlers/upstart_job.py b/cloudinit/handlers/upstart_job.py index 4684f7f2..bac4cad2 100644 --- a/cloudinit/handlers/upstart_job.py +++ b/cloudinit/handlers/upstart_job.py @@ -22,6 +22,7 @@ import os +import re from cloudinit import handlers from cloudinit import log as logging @@ -30,6 +31,7 @@ from cloudinit import util from cloudinit.settings import (PER_INSTANCE) LOG = logging.getLogger(__name__) +UPSTART_PREFIX = "#upstart-job" class UpstartJobPartHandler(handlers.Handler): @@ -39,10 +41,11 @@ class UpstartJobPartHandler(handlers.Handler): def list_types(self): return [ - handlers.type_from_starts_with("#upstart-job"), + handlers.type_from_starts_with(UPSTART_PREFIX), ] - def _handle_part(self, _data, ctype, filename, payload, frequency): + def handle_part(self, _data, ctype, filename, # pylint: disable=W0221 + payload, frequency): if ctype in handlers.CONTENT_SIGNALS: return @@ -65,6 +68,53 @@ class UpstartJobPartHandler(handlers.Handler): path = os.path.join(self.upstart_dir, filename) util.write_file(path, payload, 0644) - # if inotify support is not present in the root filesystem - # (overlayroot) then we need to tell upstart to re-read /etc - util.subp(["initctl", "reload-configuration"], capture=False) + if SUITABLE_UPSTART: + util.subp(["initctl", "reload-configuration"], capture=False) + + +def _has_suitable_upstart(): + # (LP: #1124384) + # a bug in upstart means that invoking reload-configuration + # at this stage in boot causes havoc. So, try to determine if upstart + # is installed, and reloading configuration is OK. + if not os.path.exists("/sbin/initctl"): + return False + try: + (version_out, _err) = util.subp(["initctl", "version"]) + except: + util.logexc(LOG, "initctl version failed") + return False + + # expecting 'initctl version' to output something like: init (upstart X.Y) + if re.match("upstart 1.[0-7][)]", version_out): + return False + if "upstart 0." in version_out: + return False + elif "upstart 1.8" in version_out: + if not os.path.exists("/usr/bin/dpkg-query"): + return False + try: + (dpkg_ver, _err) = util.subp(["dpkg-query", + "--showformat=${Version}", + "--show", "upstart"], rcs=[0, 1]) + except Exception: + util.logexc(LOG, "dpkg-query failed") + return False + + try: + good = "1.8-0ubuntu1.2" + util.subp(["dpkg", "--compare-versions", dpkg_ver, "ge", good]) + return True + except util.ProcessExecutionError as e: + if e.exit_code is 1: + pass + else: + util.logexc(LOG, "dpkg --compare-versions failed [%s]", + e.exit_code) + except Exception as e: + util.logexc(LOG, "dpkg --compare-versions failed") + return False + else: + return True + +SUITABLE_UPSTART = _has_suitable_upstart() |