From a5727fe1477c9cc4288d1ac41f70bd1ab7d7928a Mon Sep 17 00:00:00 2001 From: Ben Howard Date: Wed, 8 Jan 2014 17:16:24 -0700 Subject: Significant re-working of the userdata handling and introduction of vendordata. Vendordata is a datasource provided userdata-like blob that is parsed similiarly to userdata, execept at the user's pleasure. cloudinit/config/cc_scripts_vendor.py: added vendor script cloud config cloudinit/config/cc_vendor_scripts_per_boot.py: added vendor per boot cloud config cloudinit/config/cc_vendor_scripts_per_instance.py: added vendor per instance vendor cloud config cloudinit/config/cc_vendor_scripts_per_once.py: added per once vendor cloud config script doc/examples/cloud-config-vendor-data.txt: documentation of vendor-data examples doc/vendordata.txt: documentation of vendordata for vendors (RENAMED) tests/unittests/test_userdata.py => tests/unittests/test_userdata.py TO: tests/unittests/test_userdata.py => tests/unittests/test_data.py: userdata test cases are not expanded to confirm superiority over vendor data. bin/cloud-init: change instances of 'consume_userdata' to 'consume_data' cloudinit/handlers/cloud_config.py: Added vendor script handling to default cloud-config modules cloudinit/handlers/shell_script.py: Added ability to change the path key to support vendor provided 'vendor-scripts'. Defaults to 'script'. cloudinit/helpers.py: - Changed ConfigMerger to include handling of vendordata. - Changed helpers to include paths for vendordata. cloudinit/sources/__init__.py: Added functions for helping vendordata - get_vendordata_raw(): returns vendordata unprocessed - get_vendordata(): returns vendordata through userdata processor - has_vendordata(): indicator if vendordata is present - consume_vendordata(): datasource directive for indicating explict user approval of vendordata consumption. Defaults to 'false' cloudinit/stages.py: Re-jiggered for handling of vendordata - _initial_subdirs(): added vendor script definition - update(): added self._store_vendordata() - [ADDED] _store_vendordata(): store vendordata - _get_default_handlers(): modified to allow for filtering which handlers will run against vendordata - [ADDED] _do_handlers(): moved logic from consume_userdata to _do_handlers(). This allows _consume_vendordata() and _consume_userdata() to use the same code path. - [RENAMED] consume_userdata() to _consume_userdata() - [ADDED] _consume_vendordata() for handling vendordata - run after userdata to get user cloud-config - uses ConfigMerger to get the configuration from the instance perspective about whether or not to use vendordata - [ADDED] consume_data() to call _consume_{user,vendor}data cloudinit/util.py: - [ADDED] get_nested_option_as_list() used by cc_vendor* for getting a nested value from a dict and returned as a list - runparts(): added 'exe_prefix' for running exe with a prefix, used by cc_vendor* config/cloud.cfg: Added vendor script execution as default tests/unittests/test_runs/test_merge_run.py: changed consume_userdata() to consume_data() tests/unittests/test_runs/test_simple_run.py: changed consume_userdata() to consume_data() --- cloudinit/handlers/cloud_config.py | 2 ++ cloudinit/handlers/shell_script.py | 2 ++ 2 files changed, 4 insertions(+) (limited to 'cloudinit/handlers') diff --git a/cloudinit/handlers/cloud_config.py b/cloudinit/handlers/cloud_config.py index 34a73115..4232700f 100644 --- a/cloudinit/handlers/cloud_config.py +++ b/cloudinit/handlers/cloud_config.py @@ -66,6 +66,8 @@ class CloudConfigPartHandler(handlers.Handler): handlers.Handler.__init__(self, PER_ALWAYS, version=3) self.cloud_buf = None self.cloud_fn = paths.get_ipath("cloud_config") + if 'cloud_config_path' in _kwargs: + self.cloud_fn = paths.get_ipath(_kwargs["cloud_config_path"]) self.file_names = [] def list_types(self): diff --git a/cloudinit/handlers/shell_script.py b/cloudinit/handlers/shell_script.py index 62289d98..30c1ed89 100644 --- a/cloudinit/handlers/shell_script.py +++ b/cloudinit/handlers/shell_script.py @@ -36,6 +36,8 @@ class ShellScriptPartHandler(handlers.Handler): def __init__(self, paths, **_kwargs): handlers.Handler.__init__(self, PER_ALWAYS) self.script_dir = paths.get_ipath_cur('scripts') + if 'script_path' in _kwargs: + self.script_dir = paths.get_ipath_cur(_kwargs['script_path']) def list_types(self): return [ -- cgit v1.2.3 From 8209b21fc29c7d8585b8925a4deb929639797f9b Mon Sep 17 00:00:00 2001 From: Scott Moser Date: Thu, 16 Jan 2014 16:57:21 -0500 Subject: simplify consume_vendordata, move exclusion, consume_vendordata per instance this simplifies consume_vendordata a bit, changes consume_vendordata to be by default "PER_INSTANCE" (like userdata). The other thing we do here is move the exlusion to be handled when walking the data. The benefit of doing it then is that we can exclude part-handlers. Without the ability to exlude part handlers, exclusion is basically useless (since part-handlers could accomplish anything they wanted). --- cloudinit/handlers/__init__.py | 4 ++ cloudinit/stages.py | 130 +++++++++++++++-------------------------- 2 files changed, 52 insertions(+), 82 deletions(-) (limited to 'cloudinit/handlers') diff --git a/cloudinit/handlers/__init__.py b/cloudinit/handlers/__init__.py index 2ddc75f4..059d7495 100644 --- a/cloudinit/handlers/__init__.py +++ b/cloudinit/handlers/__init__.py @@ -187,6 +187,10 @@ def _escape_string(text): def walker_callback(data, filename, payload, headers): content_type = headers['Content-Type'] + if content_type in data.get('excluded'): + LOG.debug('content_type "%s" is excluded', content_type) + return + if content_type in PART_CONTENT_TYPES: walker_handle_handler(data, content_type, filename, payload) return diff --git a/cloudinit/stages.py b/cloudinit/stages.py index 043b3257..19fbe706 100644 --- a/cloudinit/stages.py +++ b/cloudinit/stages.py @@ -338,56 +338,40 @@ class Init(object): processed_vd = "%s" % (self.datasource.get_vendordata()) util.write_file(self._get_ipath('vendordata'), processed_vd, 0600) - def _get_default_handlers(self, user_data=False, vendor_data=False, - excluded=None): - opts = { + def _default_handlers(self, opts=None): + if opts is None: + opts = {} + + opts.update({ 'paths': self.paths, 'datasource': self.datasource, - } - - def conditional_get(cls, mod): - cls_name = cls.__name__.split('.')[-1] - _mod = getattr(cls, mod) - if not excluded: - return _mod(**opts) - - if cls_name not in excluded: - _mod = getattr(cls, mod) - return _mod(**opts) - + }) # TODO(harlowja) Hmmm, should we dynamically import these?? def_handlers = [ - conditional_get(bh_part, 'BootHookPartHandler'), - conditional_get(up_part, 'UpstartJobPartHandler'), + cc_part.CloudConfigPartHandler(**opts), + ss_part.ShellScriptPartHandler(**opts), + bh_part.BootHookPartHandler(**opts), + up_part.UpstartJobPartHandler(**opts), ] - - # Add in the shell script part handler - if user_data: - def_handlers.extend([ - conditional_get(cc_part, 'CloudConfigPartHandler'), - conditional_get(ss_part, 'ShellScriptPartHandler')]) - - # This changes the path for the vendor script execution - if vendor_data: - opts['script_path'] = "vendor_scripts" - opts['cloud_config_path'] = "vendor_cloud_config" - def_handlers.extend([ - conditional_get(cc_part, 'CloudConfigPartHandler'), - conditional_get(ss_part, 'ShellScriptPartHandler')]) - - return [x for x in def_handlers if x is not None] + return def_handlers def _default_userdata_handlers(self): - return self._get_default_handlers(user_data=True) + return self._default_handlers() - def _default_vendordata_handlers(self, excluded=None): - return self._get_default_handlers(vendor_data=True, excluded=excluded) + def _default_vendordata_handlers(self): + return self._default_handlers( + opts={'script_path': 'vendor_scripts', + 'cloud_config_path': 'vendor_cloud_config'}) - def _do_handlers(self, data_msg, c_handlers_list, frequency): + def _do_handlers(self, data_msg, c_handlers_list, frequency, + excluded=None): """ Generalized handlers suitable for use with either vendordata or userdata """ + if excluded is None: + excluded = [] + cdir = self.paths.get_cpath("handlers") idir = self._get_ipath("handlers") @@ -450,7 +434,7 @@ class Init(object): handlers.call_begin(mod, data, frequency) c_handlers.initialized.append(mod) - def walk_handlers(): + def walk_handlers(excluded): # Walk the user data part_data = { 'handlers': c_handlers, @@ -463,9 +447,9 @@ class Init(object): # to help write there contents to files with numbered # names... 'handlercount': 0, + 'excluded': excluded, } - handlers.walk(data_msg, handlers.walker_callback, - data=part_data) + handlers.walk(data_msg, handlers.walker_callback, data=part_data) def finalize_handlers(): # Give callbacks opportunity to finalize @@ -482,7 +466,7 @@ class Init(object): try: init_handlers() - walk_handlers() + walk_handlers(excluded) finally: finalize_handlers() @@ -503,67 +487,49 @@ class Init(object): # objects before the load of the userdata happened, # this is expected. - def _consume_vendordata(self, frequency=PER_ALWAYS): + def _consume_vendordata(self, frequency=PER_INSTANCE): """ Consume the vendordata and run the part handlers on it """ - if not self.datasource.has_vendordata(): - LOG.info("datasource did not provide vendor data") + # User-data should have been consumed first. + # So we merge the other available cloud-configs (everything except + # vendor provided), and check whether or not we should consume + # vendor data at all. That gives user or system a chance to override. + if not self.datasource.get_vendordata_raw(): + LOG.debug("no vendordata from datasource") return - # User-data should have been consumed first. If it has, then we can - # read it and simply parse it. This means that the datasource can - # define if the vendordata can be consumed too....i.e this method - # gives us a lot of flexibility. _cc_merger = helpers.ConfigMerger(paths=self._paths, datasource=self.datasource, additional_fns=[], base_cfg=self.cfg, include_vendor=False) - _cc = _cc_merger.cfg - - if not self.datasource.consume_vendordata(): - if not isinstance(_cc, dict): - LOG.info(("userdata does explicitly allow vendordata " - "consumption")) - return - - if 'vendor_data' not in _cc: - LOG.info(("no 'vendor_data' directive found in the" - "conf files. Skipping consumption of vendordata")) - return + vdcfg = _cc_merger.cfg.get('vendor_data', {}) - # This allows for the datasource to signal explicit conditions when - # when the user has opted in to user-data - if self.datasource.consume_vendordata(): - LOG.info(("datasource has indicated that vendordata that user" - " opted-in via another channel")) + if not isinstance(vdcfg, dict): + vdcfg = {'enabled': False} + LOG.warn("invalid 'vendor_data' setting. resetting to: %s", vdcfg) - vdc = _cc.get('vendor_data') - no_handlers = None - if isinstance(vdc, dict): - enabled = vdc.get('enabled') - no_handlers = vdc.get('no_run') + if not util.is_true(vdcfg.get('enabled')): + LOG.debug("vendordata consumption is disabled.") + return - if enabled is None: - LOG.info("vendordata will not be consumed: user has not opted-in") - return - elif util.is_false(enabled): - LOG.info("user has requested NO vendordata consumption") - return + enabled = vdc.get('enabled') + no_handlers = vdc.get('disabled_handlers', None) - LOG.info("vendor data will be consumed") + LOG.debug("vendor data will be consumed. disabled_handlers=%s", + no_handlers) # Ensure vendordata source fetched before activation (just incase) - vendor_data_msg = self.datasource.get_vendordata(True) + vendor_data_msg = self.datasource.get_vendordata() # This keeps track of all the active handlers, while excluding what the # users doesn't want run, i.e. boot_hook, cloud_config, shell_script - c_handlers_list = self._default_vendordata_handlers( - excluded=no_handlers) + c_handlers_list = self._default_vendordata_handlers() # Run the handlers - self._do_handlers(vendor_data_msg, c_handlers_list, frequency) + self._do_handlers(vendor_data_msg, c_handlers_list, frequency, + excluded=no_handlers) def _consume_userdata(self, frequency=PER_INSTANCE): """ @@ -574,7 +540,7 @@ class Init(object): user_data_msg = self.datasource.get_userdata(True) # This keeps track of all the active handlers - c_handlers_list = self._default_userdata_handlers() + c_handlers_list = self._default_handlers() # Run the handlers self._do_handlers(user_data_msg, c_handlers_list, frequency) -- cgit v1.2.3