From cad4b0c217427f5497cc32fe9e19bd3e5071a131 Mon Sep 17 00:00:00 2001 From: Joshua Harlow Date: Mon, 11 Jun 2012 17:12:00 -0700 Subject: Add this file, which contains the main init stage, and the handlers stage (as seperate objects). --- cloudinit/stages.py | 411 ++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 411 insertions(+) create mode 100644 cloudinit/stages.py (limited to 'cloudinit/stages.py') diff --git a/cloudinit/stages.py b/cloudinit/stages.py new file mode 100644 index 00000000..55ad143d --- /dev/null +++ b/cloudinit/stages.py @@ -0,0 +1,411 @@ +# vi: ts=4 expandtab +# +# Copyright (C) 2012 Canonical Ltd. +# Copyright (C) 2012 Hewlett-Packard Development Company, L.P. +# Copyright (C) 2012 Yahoo! Inc. +# +# Author: Scott Moser +# Author: Juerg Haefliger +# Author: Joshua Harlow +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 3, as +# published by the Free Software Foundation. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +import cPickle as pickle + +import copy +import os +import sys + +try: + from configobj import ConfigObj +except ImportError: + ConfigObj = None + +from cloudinit.settings import (PER_INSTANCE) +from cloudinit.settings import (OLD_CLOUD_CONFIG) + +from cloudinit import cloud +from cloudinit import distros +from cloudinit import handlers +from cloudinit import helpers +from cloudinit import importer +from cloudinit import log as logging +from cloudinit import sources +from cloudinit import util + +from cloudinit import user_data as ud + +LOG = logging.getLogger(__name__) + + +class Init(object): + def __init__(self, ds_deps=None): + self.datasource = None + if ds_deps: + self.ds_deps = ds_deps + else: + self.ds_deps = [sources.DEP_FILESYSTEM, sources.DEP_NETWORK] + # Created on first use + self.cached_cfg = None + self.cached_distro = None + self.cached_paths = None + + def _read_cfg_old(self): + # Support reading the old ConfigObj format file and merging + # it into the yaml dictionary + if not ConfigObj: + return {} + old_cfg = ConfigObj(OLD_CLOUD_CONFIG) + return dict(old_cfg) + + @property + def cfg(self): + if self.cached_cfg is None: + self.cached_cfg = self._read_cfg() + return self.cached_cfg + + @property + def paths(self): + if not self.cached_paths: + sys_info = self.cfg.get('system_info', {}) + self.cached_paths = helpers.Paths(copy.deepcopy(sys_info), + self.datasource) + return self.cached_paths + + def _initial_subdirs(self): + c_dir = self.paths.cloud_dir + initial_dirs = [ + os.path.join(c_dir, 'scripts'), + os.path.join(c_dir, 'scripts', 'per-instance'), + os.path.join(c_dir, 'scripts', 'per-once'), + os.path.join(c_dir, 'scripts', 'per-boot'), + os.path.join(c_dir, 'seed'), + os.path.join(c_dir, 'instances'), + os.path.join(c_dir, 'handlers'), + os.path.join(c_dir, 'sem'), + os.path.join(c_dir, 'data'), + ] + return initial_dirs + + def purge_cache(self, rmcur=True): + rmlist = [] + rmlist.append(self.paths.boot_finished) + if rmcur: + rmlist.append(self.paths.instance_link) + for f in rmlist: + util.del_file(f) + return len(rmlist) + + def initialize(self): + self._initialize_filesystem() + + def _initialize_filesystem(self): + util.ensure_dirs(self._initial_subdirs()) + log_file = util.get_cfg_option_str(self.cfg, 'def_log_file') + perms = util.get_cfg_option_str(self.cfg, 'syslog_fix_perms') + if log_file: + util.ensure_file(log_file) + if perms: + (u, g) = perms.split(':', 1) + if u == "-1" or u == "None": + u = None + if g == "-1" or g == "None": + g = None + util.chownbyname(log_file, u, g) + + def _read_cfg(self): + # Deep copy so that + b_config = util.get_builtin_cfg() + try: + conf = util.get_base_cfg() + except Exception: + conf = b_config + return util.mergedict(conf, self._read_cfg_old()) + + def _restore_from_cache(self): + pickled_fn = self.paths.get_ipath_cur('obj_pkl') + try: + # we try to restore from a current link and static path + # by using the instance link, if purge_cache was called + # the file wont exist + return pickle.loads(util.load_file(pickled_fn)) + except Exception as e: + LOG.exception(("Failed loading pickled datasource from" + " %s due to: %s"), pickled_fn, e) + return None + + def _write_to_cache(self): + pickled_fn = self.paths.get_ipath_cur("obj_pkl") + try: + contents = pickle.dumps(self.datasource) + util.write_file(pickled_fn, contents, mode=0400) + except Exception as e: + LOG.exception(("Failed pickling datasource to" + " %s due to: %s"), pickled_fn, e) + return False + + def _get_datasources(self): + # Any config provided??? + pkg_list = self.cfg.get('datasource_pkg_list') or [] + # Add the defaults at the end + for n in [util.obj_name(sources), '']: + if n not in pkg_list: + pkg_list.append(n) + cfg_list = self.cfg.get('datasource_list') or [] + return (cfg_list, pkg_list) + + def _get_data_source(self): + if self.datasource: + return self.datasource + ds = self._restore_from_cache() + if ds: + LOG.debug("Restored from cache datasource: %s" % ds) + else: + (cfg_list, pkg_list) = self._get_datasources() + # Deep copy so that handlers can not modify (which will + # affect handlers down the line...) + sys_cfg = copy.deepcopy(self.cfg) + ds_deps = copy.deepcopy(self.ds_deps) + distro = distros.fetch(sys_cfg, + cloud.Cloud(self.datasource, + self.paths, sys_cfg)) + (ds, dsname) = sources.find_source(sys_cfg, distro, self.paths, + ds_deps, cfg_list, pkg_list) + LOG.debug("Loaded datasource %s - %s", dsname, ds) + self.datasource = ds + if self.cached_paths: + self.cached_paths.datasource = ds + return ds + + def _reflect_cur_instance(self): + # Ensure we are hooked into the right symlink for the current instance + idir = self.paths.get_ipath() + util.del_file(self.paths.instance_link) + util.sym_link(idir, self.paths.instance_link) + + # Ensures these dirs exist + dir_list = [] + for d in ["handlers", "scripts", "sem"]: + dir_list.append(os.path.join(idir, d)) + util.ensure_dirs(dir_list) + + # Write out information on what is being used for the current instance + # and what may have been used for a previous instance... + dp = self.paths.get_cpath('data') + + # Write what the datasource was and is.. + ds = "%s: %s" % (util.obj_name(self.datasource), self.datasource) + previous_ds = '' + ds_fn = os.path.join(idir, 'datasource') + try: + previous_ds = util.load_file(ds_fn).strip() + except Exception: + pass + if not previous_ds: + # TODO: ?? is this right + previous_ds = ds + util.write_file(ds_fn, "%s\n" % ds) + util.write_file(os.path.join(dp, 'previous-datasource'), + "%s\n" % (previous_ds)) + + # What the instance id was and is... + iid = self.datasource.get_instance_id() + previous_iid = '' + p_iid_fn = os.path.join(dp, 'previous-instance-id') + c_iid_fn = os.path.join(dp, 'instance-id') + try: + previous_iid = util.load_file(p_iid_fn).strip() + except Exception: + pass + if not previous_iid: + # TODO: ?? is this right + previous_iid = iid + util.write_file(c_iid_fn, "%s\n" % iid) + util.write_file(p_iid_fn, "%s\n" % previous_iid) + + def fetch(self): + return self._get_data_source() + + def instancify(self): + self._reflect_cur_instance() + + def update(self): + self._write_to_cache() + self._store_userdata() + + def _store_userdata(self): + raw_ud = "%s" % (self.datasource.get_userdata_raw()) + util.write_file(self.paths.get_ipath('userdata_raw'), raw_ud, 0600) + processed_ud = "%s" % (self.datasource.get_userdata()) + util.write_file(self.paths.get_ipath('userdata'), processed_ud, 0600) + + def consume(self, frequency=PER_INSTANCE): + cdir = self.paths.get_cpath("handlers") + idir = self.paths.get_ipath("handlers") + + # Add the path to the plugins dir to the top of our list for import + # instance dir should be read before cloud-dir + sys.path.insert(0, cdir) + sys.path.insert(0, idir) + + # This keeps track of all the active handlers + c_handlers = helpers.ContentHandlers(self.paths) + + # Add handlers in cdir + potential_handlers = util.find_modules(cdir) + for (fname, modname) in potential_handlers.iteritems(): + try: + mod = ud.fixup_module(importer.import_module(modname)) + types = c_handlers.register(mod) + LOG.debug("Added handler for [%s] from %s", types, fname) + except: + LOG.exception("Failed to register handler from %s", fname) + + def_handlers = c_handlers.register_defaults() + if def_handlers: + LOG.debug("Registered default handlers for [%s]", def_handlers) + + # Form our cloud proxy + data = cloud.Cloud(self.datasource, + self.paths, copy.deepcopy(self.cfg)) + + # Init the handlers first + # Ensure userdata fetched before activation + called = [] + for (_mtype, mod) in c_handlers.iteritems(): + if mod in called: + continue + ud.call_begin(mod, data, frequency) + called.append(mod) + + # Walk the user data + part_data = { + 'handlers': c_handlers, + 'handlerdir': idir, + 'data': data, + 'frequency': frequency, + 'handlercount': 0, + } + ud.walk(data.get_userdata(), ud.walker_callback, data=part_data) + + # Give callbacks opportunity to finalize + called = [] + for (_mtype, mod) in c_handlers.iteritems(): + if mod in called: + continue + ud.call_end(mod, data, frequency) + called.append(mod) + + +class Handlers(object): + def __init__(self, datasource, h_cloud, cfgfile=None, basecfg=None): + self.datasource = datasource + self.cfgfile = cfgfile + self.basecfg = basecfg + self.h_cloud = h_cloud + self.cachedcfg = None + + @property + def cfg(self): + if self.cachedcfg is None: + self.cachedcfg = self._get_config(self.cfgfile) + return self.cachedcfg + + def _get_config(self, cfgfile): + mcfg = None + + if self.cfgfile: + try: + mcfg = util.read_conf(cfgfile) + except: + LOG.exception(("Failed loading of cloud config '%s'. " + "Continuing with an empty config."), cfgfile) + if not mcfg: + mcfg = {} + + ds_cfg = None + try: + ds_cfg = self.datasource.get_config_obj() + except: + LOG.exception("Failed loading of datasource config.") + if not ds_cfg: + ds_cfg = {} + + mcfg = util.mergedict(mcfg, ds_cfg) + if self.basecfg: + return util.mergedict(mcfg, self.basecfg) + else: + return mcfg + + + def _read_modules(self, name): + module_list = [] + if name not in self.cfg: + return module_list + cfg_mods = self.cfg[name] + # Create 'module_list', an array of arrays + # Where array[0] = module name + # array[1] = frequency + # array[2:] = arguments + for item in cfg_mods: + if not item: + continue + if isinstance(item, str): + module_list.append([item]) + elif isinstance(item, list): + module_list.append(item) + else: + raise TypeError("Failed to read '%s' item in config") + return module_list + + def _form_modules(self, raw_mods): + mostly_mods = [] + for raw_mod in raw_mods: + raw_name = raw_mod[0] + freq = None + run_args = None + if len(raw_mod) > 1: + freq = raw_mod[1] + if len(raw_mod) > 2: + run_args = raw_mod[2:] + if not run_args: + run_args = [] + mod_name = handlers.form_module_name(raw_name) + if not mod_name: + continue + mod = handlers.fixup_module(importer.import_module(mod_name)) + mostly_mods.append([mod, raw_name, freq, run_args]) + return mostly_mods + + def _run_modules(self, mostly_mods): + failures = [] + for (mod, name, freq, args) in mostly_mods: + try: + if not freq: + freq = mod.frequency + if not freq: + freq = PER_INSTANCE + func_args = [name, copy.deepcopy(self.cfg), + self.h_cloud, LOG, + args] + run_name = "config-" + name + self.h_cloud.run(run_name, mod.handle, func_args, freq=freq) + except: + LOG.exception("Running %s failed", mod) + failures.append(name) + return failures + + def run(self, name): + raw_mods = self._read_modules(name) + mostly_mods = self._form_modules(raw_mods) + return self._run_modules(mostly_mods) -- cgit v1.2.3 From dab0b7c7ebcc92c772bfadce334ba118955f5a59 Mon Sep 17 00:00:00 2001 From: Joshua Harlow Date: Mon, 11 Jun 2012 18:02:32 -0700 Subject: Fix logging + fetch user data before user data content handlers are activated. --- cloudinit/stages.py | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) (limited to 'cloudinit/stages.py') diff --git a/cloudinit/stages.py b/cloudinit/stages.py index 55ad143d..2cb9d0ec 100644 --- a/cloudinit/stages.py +++ b/cloudinit/stages.py @@ -124,7 +124,6 @@ class Init(object): util.chownbyname(log_file, u, g) def _read_cfg(self): - # Deep copy so that b_config = util.get_builtin_cfg() try: conf = util.get_base_cfg() @@ -279,8 +278,10 @@ class Init(object): data = cloud.Cloud(self.datasource, self.paths, copy.deepcopy(self.cfg)) - # Init the handlers first # Ensure userdata fetched before activation + ud_obj = data.get_userdata() + + # Init the handlers first called = [] for (_mtype, mod) in c_handlers.iteritems(): if mod in called: @@ -294,9 +295,12 @@ class Init(object): 'handlerdir': idir, 'data': data, 'frequency': frequency, + # This will be used when new handlers are found + # to help write there contents to files with numbered + # names... 'handlercount': 0, } - ud.walk(data.get_userdata(), ud.walker_callback, data=part_data) + ud.walk(ud_obj, ud.walker_callback, data=part_data) # Give callbacks opportunity to finalize called = [] -- cgit v1.2.3 From 38f5c116dded3d9ed9568c2a06f071b2a88bdb2e Mon Sep 17 00:00:00 2001 From: Joshua Harlow Date: Fri, 15 Jun 2012 18:31:52 -0700 Subject: Cleanup and renaming that should mostly complete this new module. 1. Renamed config to modules, then renamed to transforms 2. Enabled checking of transform frequencies before usage (warning the user that something is bad) 3. Adjusting how the cloud object is formed to comply with its new api. --- cloudinit/stages.py | 209 +++++++++++++++++++++++++++++++--------------------- 1 file changed, 126 insertions(+), 83 deletions(-) (limited to 'cloudinit/stages.py') diff --git a/cloudinit/stages.py b/cloudinit/stages.py index 2cb9d0ec..2615d59f 100644 --- a/cloudinit/stages.py +++ b/cloudinit/stages.py @@ -31,12 +31,12 @@ try: except ImportError: ConfigObj = None -from cloudinit.settings import (PER_INSTANCE) +from cloudinit.settings import (PER_INSTANCE, FREQUENCIES) from cloudinit.settings import (OLD_CLOUD_CONFIG) from cloudinit import cloud from cloudinit import distros -from cloudinit import handlers +from cloudinit import modules from cloudinit import helpers from cloudinit import importer from cloudinit import log as logging @@ -50,15 +50,16 @@ LOG = logging.getLogger(__name__) class Init(object): def __init__(self, ds_deps=None): - self.datasource = None if ds_deps: self.ds_deps = ds_deps else: self.ds_deps = [sources.DEP_FILESYSTEM, sources.DEP_NETWORK] # Created on first use - self.cached_cfg = None - self.cached_distro = None - self.cached_paths = None + self._cfg = None + self._paths = None + self._distro = None + # Created only when a fetch occurs + self.datasource = None def _read_cfg_old(self): # Support reading the old ConfigObj format file and merging @@ -68,23 +69,40 @@ class Init(object): old_cfg = ConfigObj(OLD_CLOUD_CONFIG) return dict(old_cfg) + @property + def distro(self): + if not self._distro: + d_cfg = util.get_cfg_by_path(self.cfg, ('system_info'), {}) + # Ensure not modified indirectly + d_cfg = copy.deepcopy(d_cfg) + d_cfg.pop('paths', None) + distro_cls = distros.fetch(sys_cfg.pop('distro', 'ubuntu')) + LOG.debug("Using distro class %s", distro_cls) + distro = distro_cls(d_cfg, helpers.Runners(self.paths)) + self._distro = distro + return self._distro + @property def cfg(self): - if self.cached_cfg is None: - self.cached_cfg = self._read_cfg() - return self.cached_cfg + # None check so that we don't keep on re-loading if empty + if self._cfg is None: + self._cfg = self._read_cfg() + LOG.debug("Loading init config %s", self._cfg) + return self._cfg @property def paths(self): - if not self.cached_paths: - sys_info = self.cfg.get('system_info', {}) - self.cached_paths = helpers.Paths(copy.deepcopy(sys_info), - self.datasource) - return self.cached_paths + if not self._paths: + path_info = util.get_cfg_by_path(self.cfg, ('system_info', 'paths'), {}) + # Ensure not modified indirectly + path_info = copy.deepcopy(path_info) + self._paths = helpers.Paths(path_info, self.datasource) + return self._paths def _initial_subdirs(self): c_dir = self.paths.cloud_dir initial_dirs = [ + c_dir, os.path.join(c_dir, 'scripts'), os.path.join(c_dir, 'scripts', 'per-instance'), os.path.join(c_dir, 'scripts', 'per-once'), @@ -139,8 +157,8 @@ class Init(object): # the file wont exist return pickle.loads(util.load_file(pickled_fn)) except Exception as e: - LOG.exception(("Failed loading pickled datasource from" - " %s due to: %s"), pickled_fn, e) + util.logexc(LOG, "Failed loading pickled datasource from %s", + pickled_fn) return None def _write_to_cache(self): @@ -149,8 +167,7 @@ class Init(object): contents = pickle.dumps(self.datasource) util.write_file(pickled_fn, contents, mode=0400) except Exception as e: - LOG.exception(("Failed pickling datasource to" - " %s due to: %s"), pickled_fn, e) + util.logexc(LOG, "Failed pickling datasource to %s", pickled_fn) return False def _get_datasources(self): @@ -171,19 +188,17 @@ class Init(object): LOG.debug("Restored from cache datasource: %s" % ds) else: (cfg_list, pkg_list) = self._get_datasources() - # Deep copy so that handlers can not modify (which will - # affect handlers down the line...) + # Deep copy so that user-data handlers can not modify + # (which will affect user-data handlers down the line...) sys_cfg = copy.deepcopy(self.cfg) ds_deps = copy.deepcopy(self.ds_deps) - distro = distros.fetch(sys_cfg, - cloud.Cloud(self.datasource, - self.paths, sys_cfg)) - (ds, dsname) = sources.find_source(sys_cfg, distro, self.paths, + (ds, dsname) = sources.find_source(sys_cfg, self.distro, self.paths, ds_deps, cfg_list, pkg_list) LOG.debug("Loaded datasource %s - %s", dsname, ds) self.datasource = ds - if self.cached_paths: - self.cached_paths.datasource = ds + # Ensure we adjust our path members datasource + # now that we have one (thus allowing ipath to be used) + self.paths.datasource = ds return ds def _reflect_cur_instance(self): @@ -231,12 +246,19 @@ class Init(object): previous_iid = iid util.write_file(c_iid_fn, "%s\n" % iid) util.write_file(p_iid_fn, "%s\n" % previous_iid) + return iid def fetch(self): return self._get_data_source() def instancify(self): - self._reflect_cur_instance() + return self._reflect_cur_instance() + + def cloudify(self): + # Form the needed options to cloudify our members + return cloud.Cloud(self.datasource, + self.paths, self.cfg, + self.distro, helpers.Runners(self.paths)) def update(self): self._write_to_cache() @@ -266,24 +288,24 @@ class Init(object): try: mod = ud.fixup_module(importer.import_module(modname)) types = c_handlers.register(mod) - LOG.debug("Added handler for [%s] from %s", types, fname) + LOG.debug("Added handler for %s from %s", types, fname) except: - LOG.exception("Failed to register handler from %s", fname) + util.logexc(LOG, "Failed to register handler from %s", fname) def_handlers = c_handlers.register_defaults() if def_handlers: - LOG.debug("Registered default handlers for [%s]", def_handlers) + LOG.debug("Registered default handlers for %s", def_handlers) - # Form our cloud proxy - data = cloud.Cloud(self.datasource, - self.paths, copy.deepcopy(self.cfg)) - # Ensure userdata fetched before activation - ud_obj = data.get_userdata() + # Ensure userdata fetched before activation (just incase) + ud_obj = self.datasource.get_userdata() + + # Form our cloud interface + data = self.cloudify() # Init the handlers first called = [] - for (_mtype, mod) in c_handlers.iteritems(): + for (_ctype, mod) in c_handlers.iteritems(): if mod in called: continue ud.call_begin(mod, data, frequency) @@ -304,26 +326,28 @@ class Init(object): # Give callbacks opportunity to finalize called = [] - for (_mtype, mod) in c_handlers.iteritems(): + for (_ctype, mod) in c_handlers.iteritems(): if mod in called: continue ud.call_end(mod, data, frequency) called.append(mod) -class Handlers(object): - def __init__(self, datasource, h_cloud, cfgfile=None, basecfg=None): - self.datasource = datasource +class Transforms(object): + def __init__(self, cloudobj, cfgfile=None): + self.datasource = cloudobj.datasource self.cfgfile = cfgfile - self.basecfg = basecfg - self.h_cloud = h_cloud - self.cachedcfg = None + self.basecfg = copy.deepcopy(cloudobj.cfg) + self.cloudobj = cloudobj + # Created on first use + self._cachedcfg = None @property def cfg(self): - if self.cachedcfg is None: - self.cachedcfg = self._get_config(self.cfgfile) - return self.cachedcfg + if self._cachedcfg is None: + self._cachedcfg = self._get_config(self.cfgfile) + LOG.debug("Loading module config %s", self._cachedcfg) + return self._cachedcfg def _get_config(self, cfgfile): mcfg = None @@ -332,8 +356,8 @@ class Handlers(object): try: mcfg = util.read_conf(cfgfile) except: - LOG.exception(("Failed loading of cloud config '%s'. " - "Continuing with an empty config."), cfgfile) + util.logexc(LOG, ("Failed loading of cloud config '%s'. " + "Continuing with an empty config."), cfgfile) if not mcfg: mcfg = {} @@ -341,7 +365,7 @@ class Handlers(object): try: ds_cfg = self.datasource.get_config_obj() except: - LOG.exception("Failed loading of datasource config.") + util.logexc(LOG, "Failed loading of datasource config object.") if not ds_cfg: ds_cfg = {} @@ -352,64 +376,83 @@ class Handlers(object): return mcfg - def _read_modules(self, name): + def _read_transforms(self, name): module_list = [] if name not in self.cfg: return module_list cfg_mods = self.cfg[name] - # Create 'module_list', an array of arrays - # Where array[0] = module name - # array[1] = frequency - # array[2:] = arguments + # Create 'module_list', an array of hashes + # Where hash['mod'] = module name + # hash['freq'] = frequency + # hash['args'] = arguments for item in cfg_mods: if not item: continue - if isinstance(item, str): - module_list.append([item]) - elif isinstance(item, list): - module_list.append(item) + if isinstance(item, (str, basestring)): + module_list.append({ + 'mod': item.strip(), + }) + elif isinstance(item, (list)): + contents = {} + # Meant to fall through... + if len(item) >= 1: + contents['mod'] = item[0].strip() + if len(item) >= 2: + contents['freq'] = item[1].strip() + if len(item) >= 3: + contents['args'] = item[2:] + if contents: + module_list.append(contents) else: - raise TypeError("Failed to read '%s' item in config") + raise TypeError(("Failed to read '%s' item in config," + " unknown type %s") % + (item, util.obj_name(item))) return module_list - def _form_modules(self, raw_mods): + def _transforms_modules(self, raw_mods): mostly_mods = [] for raw_mod in raw_mods: - raw_name = raw_mod[0] - freq = None - run_args = None - if len(raw_mod) > 1: - freq = raw_mod[1] - if len(raw_mod) > 2: - run_args = raw_mod[2:] - if not run_args: - run_args = [] - mod_name = handlers.form_module_name(raw_name) + raw_name = raw_mod['mod'] + freq = raw_mod.get('freq') + run_args = raw_mod.get('args') or [] + mod_name = modules.form_module_name(raw_name) if not mod_name: continue - mod = handlers.fixup_module(importer.import_module(mod_name)) + if freq and freq not in FREQUENCIES: + LOG.warn("Config specified module %s has an unknown frequency %s", raw_name, freq) + # Reset it so when ran it will get set to a known value + freq = None + mod = modules.fixup_module(importer.import_module(mod_name)) mostly_mods.append([mod, raw_name, freq, run_args]) return mostly_mods - def _run_modules(self, mostly_mods): + def _run_transforms(self, mostly_mods): failures = [] for (mod, name, freq, args) in mostly_mods: try: + # Try the modules frequency, otherwise fallback to a known one if not freq: freq = mod.frequency - if not freq: + if not freq in FREQUENCIES: freq = PER_INSTANCE + worked_distros = mod.distros + if worked_distros and self.cloud.distro.name() not in worked_distros: + LOG.warn(("Module %s is verified on %s distros" + " but not on %s distro. It may or may not work" + " correctly."), name, worked_distros, + self.cloud.distro.name()) + # Deep copy the config so that modules can't alter it func_args = [name, copy.deepcopy(self.cfg), - self.h_cloud, LOG, - args] - run_name = "config-" + name - self.h_cloud.run(run_name, mod.handle, func_args, freq=freq) - except: - LOG.exception("Running %s failed", mod) - failures.append(name) + self.cloudobj, LOG, args] + # This name will affect the semphapore name created + run_name = "config-%s" % (name) + self.cloudobj.run(run_name, mod.handle, func_args, freq=freq) + except Exception as e: + util.logexc(LOG, "Running %s failed", mod) + failures.append((name, e)) return failures def run(self, name): - raw_mods = self._read_modules(name) - mostly_mods = self._form_modules(raw_mods) - return self._run_modules(mostly_mods) + raw_mods = self._read_transforms(name) + mostly_mods = self._transforms_modules(raw_mods) + return self._run_transforms(mostly_mods) -- cgit v1.2.3 From 450261a1fcf1f8929a2f7a25c2c278ba40689289 Mon Sep 17 00:00:00 2001 From: Joshua Harlow Date: Fri, 15 Jun 2012 21:33:55 -0700 Subject: Fixups to ensure that pylint does not find anything major wrong. --- cloudinit/cloud.py | 2 - cloudinit/distros/__init__.py | 21 +++--- cloudinit/distros/ubuntu.py | 23 +++--- cloudinit/handlers/boot_hook.py | 14 ++-- cloudinit/handlers/cloud_config.py | 4 +- cloudinit/handlers/shell_script.py | 5 +- cloudinit/handlers/upstart_job.py | 7 +- cloudinit/helpers.py | 41 ++--------- cloudinit/log.py | 2 +- cloudinit/sources/DataSourceCloudStack.py | 2 +- cloudinit/sources/DataSourceConfigDrive.py | 4 +- cloudinit/sources/DataSourceMAAS.py | 14 ++-- cloudinit/sources/DataSourceNoCloud.py | 3 +- cloudinit/sources/DataSourceOVF.py | 12 +-- cloudinit/sources/__init__.py | 23 +++--- cloudinit/stages.py | 98 ++++++++++++++++--------- cloudinit/transforms/__init__.py | 2 +- cloudinit/transforms/cc_apt_update_upgrade.py | 12 ++- cloudinit/transforms/cc_bootcmd.py | 8 +- cloudinit/transforms/cc_byobu.py | 2 +- cloudinit/transforms/cc_ca_certs.py | 13 +++- cloudinit/transforms/cc_chef.py | 3 +- cloudinit/transforms/cc_disable_ec2_metadata.py | 3 +- cloudinit/transforms/cc_final_message.py | 4 +- cloudinit/transforms/cc_foo.py | 6 +- cloudinit/transforms/cc_grub_dpkg.py | 4 +- cloudinit/transforms/cc_keys_to_console.py | 23 ++++-- cloudinit/transforms/cc_landscape.py | 3 +- cloudinit/transforms/cc_locale.py | 2 +- cloudinit/transforms/cc_mcollective.py | 9 +-- cloudinit/transforms/cc_mounts.py | 15 ++-- cloudinit/transforms/cc_phone_home.py | 36 ++++++--- cloudinit/transforms/cc_puppet.py | 17 +++-- cloudinit/transforms/cc_resizefs.py | 38 ++++++---- cloudinit/transforms/cc_rightscale_userdata.py | 16 ++-- cloudinit/transforms/cc_rsyslog.py | 14 ++-- cloudinit/transforms/cc_runcmd.py | 9 ++- cloudinit/transforms/cc_salt_minion.py | 7 +- cloudinit/transforms/cc_scripts_per_boot.py | 5 +- cloudinit/transforms/cc_scripts_per_instance.py | 5 +- cloudinit/transforms/cc_scripts_per_once.py | 5 +- cloudinit/transforms/cc_scripts_user.py | 9 ++- cloudinit/transforms/cc_set_hostname.py | 2 +- cloudinit/transforms/cc_set_passwords.py | 10 ++- cloudinit/transforms/cc_ssh.py | 22 +++--- cloudinit/transforms/cc_ssh_import_id.py | 8 +- cloudinit/transforms/cc_timezone.py | 8 +- cloudinit/transforms/cc_update_etc_hosts.py | 22 ++++-- cloudinit/transforms/cc_update_hostname.py | 4 +- cloudinit/transforms/cc_welcome.py | 6 +- cloudinit/user_data.py | 13 ++-- cloudinit/util.py | 48 ++++++------ 52 files changed, 388 insertions(+), 300 deletions(-) (limited to 'cloudinit/stages.py') diff --git a/cloudinit/cloud.py b/cloudinit/cloud.py index b2dfc749..8372d123 100644 --- a/cloudinit/cloud.py +++ b/cloudinit/cloud.py @@ -23,8 +23,6 @@ import copy import os -from cloudinit import distros -from cloudinit import helpers from cloudinit import log as logging LOG = logging.getLogger(__name__) diff --git a/cloudinit/distros/__init__.py b/cloudinit/distros/__init__.py index 90607668..fd4c70c1 100644 --- a/cloudinit/distros/__init__.py +++ b/cloudinit/distros/__init__.py @@ -20,29 +20,32 @@ # You should have received a copy of the GNU General Public License # along with this program. If not, see . +from StringIO import StringIO + import abc import copy from cloudinit import importer +from cloudinit import log as logging from cloudinit import util -from StringIO import StringIO - # TODO: Make this via config?? IFACE_ACTIONS = { 'up': ['ifup', '--all'], 'down': ['ifdown', '--all'], } +LOG = logging.getLogger(__name__) + class Distro(object): __metaclass__ = abc.ABCMeta - def __init__(self, cfg, runner): + def __init__(self, name, cfg, runner): self._runner = runner - self._cfg = util.get_cfg_by_path(cfg, ('system_info', ), {}) - self.name = self._cfg.pop("distro", 'generic') + self._cfg = cfg + self.name = name @abc.abstractmethod def install_packages(self, pkglist): @@ -135,10 +138,9 @@ class Distro(object): action, cmd) (_out, err) = util.subp(cmd) if len(err): - LOG.warn("Running %s resulted in stderr output: %s", - IF_UP_CMD, err) + LOG.warn("Running %s resulted in stderr output: %s", cmd, err) return True - except util.ProcessExecutionError as exc: + except util.ProcessExecutionError: util.logexc(LOG, "Running %s failed", cmd) return False @@ -152,7 +154,8 @@ def fetch(distro_name, mods=(__name__, )): except RuntimeError: pass if not mod: - raise RuntimeError("No distribution found for distro %s" % (distro_name)) + raise RuntimeError("No distribution found for distro %s" + % (distro_name)) distro_cls = getattr(mod, 'Distro') return distro_cls diff --git a/cloudinit/distros/ubuntu.py b/cloudinit/distros/ubuntu.py index 6b0aff47..9252a1c4 100644 --- a/cloudinit/distros/ubuntu.py +++ b/cloudinit/distros/ubuntu.py @@ -20,17 +20,13 @@ # You should have received a copy of the GNU General Public License # along with this program. If not, see . -from StringIO import StringIO - import os -import socket from cloudinit import distros from cloudinit import log as logging -from cloudinit import templater from cloudinit import util -from cloudinit.settings import (PER_INSTANCE) +from cloudinit.settings import PER_INSTANCE LOG = logging.getLogger(__name__) @@ -65,9 +61,11 @@ class Distro(distros.Distro): try: util.write_file(fn, "%s\n" % hostname, 0644) except: - util.logexc(LOG, "Failed to write hostname %s to %s", hostname, fn) - if hostname_in_etc and hostname_prev and hostname_in_etc != hostname_prev: - LOG.debug(("%s differs from /etc/hostname." + util.logexc(LOG, "Failed to write hostname %s to %s", + hostname, fn) + if (hostname_in_etc and hostname_prev and + hostname_in_etc != hostname_prev): + LOG.debug(("%s differs from /etc/hostname." " Assuming user maintained hostname."), prev_file) if "/etc/hostname" in update_files: LOG.debug("Setting hostname to %s", hostname) @@ -91,7 +89,8 @@ class Distro(distros.Distro): def set_timezone(self, tz): tz_file = os.path.join("/usr/share/zoneinfo", tz) if not os.path.isfile(tz_file): - raise Exception("Invalid timezone %s, no file found at %s" % (tz, tz_file)) + raise Exception(("Invalid timezone %s," + " no file found at %s") % (tz, tz_file)) tz_contents = "%s\n" % tz util.write_file("/etc/timezone", tz_contents) # TODO, this should be in a rhel distro subclass?? @@ -101,9 +100,6 @@ class Distro(distros.Distro): # This ensures that the correct tz will be used for the system util.copy(tz_file, "/etc/localtime") - def name(self): - return "ubuntu" - # apt_get top level command (install, update...), and args to pass it def _apt_get(self, tlc, args=None): e = os.environ.copy() @@ -116,4 +112,5 @@ class Distro(distros.Distro): util.subp(cmd, env=e, capture=False) def _update_package_sources(self): - self.runner.run("update-sources", self._apt_get, ["update"], freq=PER_INSTANCE) \ No newline at end of file + self._runner.run("update-sources", self._apt_get, + ["update"], freq=PER_INSTANCE) \ No newline at end of file diff --git a/cloudinit/handlers/boot_hook.py b/cloudinit/handlers/boot_hook.py index c75aeb72..b3aab366 100644 --- a/cloudinit/handlers/boot_hook.py +++ b/cloudinit/handlers/boot_hook.py @@ -32,9 +32,9 @@ LOG = logging.getLogger(__name__) class BootHookPartHandler(ud.PartHandler): - def __init__(self, boothook_dir, instance_id): + def __init__(self, paths, instance_id, **_kwargs): ud.PartHandler.__init__(self, PER_ALWAYS) - self.boothook_dir = boothook_dir + self.boothook_dir = paths.get_ipath("boothooks") self.instance_id = instance_id def list_types(self): @@ -54,13 +54,15 @@ class BootHookPartHandler(ud.PartHandler): start = len(prefix) + 1 filepath = os.path.join(self.boothook_dir, filename) - util.write_file(filepath, payload[start:], 0700) + contents = payload[start:] + util.write_file(filepath, contents, 0700) try: env = os.environ.copy() - env['INSTANCE_ID'] = str(self.instance_id) + if self.instance_id: + env['INSTANCE_ID'] = str(self.instance_id) util.subp([filepath], env=env) - except util.ProcessExecutionError as e: + except util.ProcessExecutionError: util.logexc(LOG, "Boothooks script %s execution error", filepath) - except Exception as e: + except Exception: util.logexc(LOG, ("Boothooks unknown " "error when running %s"), filepath) diff --git a/cloudinit/handlers/cloud_config.py b/cloudinit/handlers/cloud_config.py index f0e88eeb..12d1bd96 100644 --- a/cloudinit/handlers/cloud_config.py +++ b/cloudinit/handlers/cloud_config.py @@ -30,10 +30,10 @@ LOG = logging.getLogger(__name__) class CloudConfigPartHandler(ud.PartHandler): - def __init__(self, cloud_fn): + def __init__(self, paths, **_kwargs): ud.PartHandler.__init__(self, PER_ALWAYS) self.cloud_buf = [] - self.cloud_fn = cloud_fn + self.cloud_fn = paths.get_ipath("cloud_config") def list_types(self): return [ diff --git a/cloudinit/handlers/shell_script.py b/cloudinit/handlers/shell_script.py index 564e4623..f6e2ef16 100644 --- a/cloudinit/handlers/shell_script.py +++ b/cloudinit/handlers/shell_script.py @@ -32,10 +32,9 @@ LOG = logging.getLogger(__name__) class ShellScriptPartHandler(ud.PartHandler): - - def __init__(self, script_dir): + def __init__(self, paths, **_kwargs): ud.PartHandler.__init__(self, PER_ALWAYS) - self.script_dir = script_dir + self.script_dir = paths.get_ipath_cur('scripts') def list_types(self): return [ diff --git a/cloudinit/handlers/upstart_job.py b/cloudinit/handlers/upstart_job.py index 568a644a..059a4851 100644 --- a/cloudinit/handlers/upstart_job.py +++ b/cloudinit/handlers/upstart_job.py @@ -33,9 +33,9 @@ LOG = logging.getLogger(__name__) class UpstartJobPartHandler(ud.PartHandler): - def __init__(self, upstart_dir): + def __init__(self, paths, **_kwargs): ud.PartHandler.__init__(self, PER_INSTANCE) - self.upstart_dir = upstart_dir + self.upstart_dir = paths.upstart_conf_d def list_types(self): return [ @@ -46,6 +46,9 @@ class UpstartJobPartHandler(ud.PartHandler): if ctype in ud.CONTENT_SIGNALS: return + if not self.upstart_dir: + return + filename = util.clean_filename(filename) (_name, ext) = os.path.splitext(filename) if not ext: diff --git a/cloudinit/helpers.py b/cloudinit/helpers.py index 2ecda3e9..c276a54c 100644 --- a/cloudinit/helpers.py +++ b/cloudinit/helpers.py @@ -30,11 +30,6 @@ from cloudinit.settings import (PER_INSTANCE, PER_ALWAYS, PER_ONCE) from cloudinit import log as logging from cloudinit import util -from cloudinit.user_data import boot_hook as bh_part -from cloudinit.user_data import cloud_config as cc_part -from cloudinit.user_data import shell_script as ss_part -from cloudinit.user_data import upstart_job as up_part - LOG = logging.getLogger(__name__) @@ -77,7 +72,7 @@ class FileSemaphores(object): sem_file = self._get_path(name, freq) try: util.del_file(sem_file) - except (IOError, OSError) as e: + except (IOError, OSError): util.logexc(LOG, "Failed deleting semaphore %s", sem_file) return False return True @@ -99,7 +94,7 @@ class FileSemaphores(object): contents = "%s: %s\n" % (os.getpid(), time()) try: util.write_file(sem_file, contents) - except (IOError, OSError) as e: + except (IOError, OSError): util.logexc(LOG, "Failed writing semaphore file %s", sem_file) return None return sem_file @@ -162,9 +157,10 @@ class Runners(object): class ContentHandlers(object): - def __init__(self, paths): + def __init__(self, paths, iid=None): self.paths = paths self.registered = {} + self.iid = iid def __contains__(self, item): return self.is_registered(item) @@ -191,34 +187,9 @@ class ContentHandlers(object): def iteritems(self): return self.registered.iteritems() - def _get_default_handlers(self): - def_handlers = [] - - cc_path = self.paths.get_ipath("cloud_config") - if cc_path: - cc_h = cc_part.CloudConfigPartHandler(cc_path) - def_handlers.append(cc_h) - - sc_path = self.paths.get_ipath_cur('scripts') - if sc_path: - ss_h = ss_part.ShellScriptPartHandler(sc_path) - def_handlers.append(ss_h) - - bh_path = self.paths.get_ipath("boothooks") - if bh_path: - bh_h = bh_part.BootHookPartHandler(bh_path) - def_handlers.append(bh_h) - - upstart_pth = self.paths.upstart_conf_d - if upstart_pth: - up_h = up_part.UpstartJobPartHandler(upstart_pth) - def_handlers.append(up_h) - - return def_handlers - - def register_defaults(self): + def register_defaults(self, defs): registered = set() - for mod in self._get_default_handlers(): + for mod in defs: for t in mod.list_types(): if not self.is_registered(t): self.registered[t] = mod diff --git a/cloudinit/log.py b/cloudinit/log.py index c247eb9e..5fcb77ef 100644 --- a/cloudinit/log.py +++ b/cloudinit/log.py @@ -56,7 +56,7 @@ def setupBasicLogging(): cfile = logging.FileHandler('/var/log/cloud-init.log') cfile.setFormatter(logging.Formatter(DEF_CON_FORMAT)) cfile.setLevel(DEBUG) - root.addHandle(cfile) + root.addHandler(cfile) except (IOError, OSError): # Likely that u can't write to that file... # Make console now have DEBUG?? diff --git a/cloudinit/sources/DataSourceCloudStack.py b/cloudinit/sources/DataSourceCloudStack.py index 791df68f..27217e65 100644 --- a/cloudinit/sources/DataSourceCloudStack.py +++ b/cloudinit/sources/DataSourceCloudStack.py @@ -79,7 +79,7 @@ class DataSourceCloudStack(sources.DataSource): tot_time = (time.time() - start) LOG.debug("Crawl of metadata service took %s", int(tot_time)) return True - except Exception as e: + except Exception: util.logexc(LOG, ('Failed fetching from metadata ' 'service %s'), self.metadata_address) return False diff --git a/cloudinit/sources/DataSourceConfigDrive.py b/cloudinit/sources/DataSourceConfigDrive.py index 176b62b0..7450572f 100644 --- a/cloudinit/sources/DataSourceConfigDrive.py +++ b/cloudinit/sources/DataSourceConfigDrive.py @@ -51,8 +51,8 @@ class DataSourceConfigDrive(sources.DataSource): self.seed_dir = os.path.join(paths.seed_dir, 'config_drive') def __str__(self): - mstr = "%s[%s]" % (util.obj_name(self), self.dsmode) - mstr += " [seed=%s]" % (self.seed) + mstr = "%s [%s]" % (util.obj_name(self), self.dsmode) + mstr += "[seed=%s]" % (self.seed) return mstr def get_data(self): diff --git a/cloudinit/sources/DataSourceMAAS.py b/cloudinit/sources/DataSourceMAAS.py index 27196265..9e639649 100644 --- a/cloudinit/sources/DataSourceMAAS.py +++ b/cloudinit/sources/DataSourceMAAS.py @@ -18,9 +18,9 @@ # You should have received a copy of the GNU General Public License # along with this program. If not, see . -import os import errno import oauth.oauth as oauth +import os import time import urllib2 @@ -48,7 +48,7 @@ class DataSourceMAAS(sources.DataSource): self.seed_dir = os.path.join(paths.seed_dir, 'maas') def __str__(self): - return "%s[%s]" % (util.obj_name(self), self.base_url) + return "%s [%s]" % (util.obj_name(self), self.base_url) def get_data(self): mcfg = self.ds_cfg @@ -122,9 +122,10 @@ class DataSourceMAAS(sources.DataSource): starttime = time.time() check_url = "%s/%s/meta-data/instance-id" % (url, MD_VERSION) - url = util.wait_for_url(urls=[check_url], max_wait=max_wait, - timeout=timeout, status_cb=LOG.warn, - headers_cb=self.md_headers) + urls = [check_url] + url = uhelp.wait_for_url(urls=urls, max_wait=max_wait, + timeout=timeout, status_cb=LOG.warn, + headers_cb=self.md_headers) if url: LOG.info("Using metadata source: '%s'", url) @@ -185,7 +186,8 @@ def read_maas_seed_url(seed_url, header_cb=None, timeout=None, headers = {} try: (resp, sc) = uhelp.readurl(url, headers=headers, timeout=timeout) - md[name] = resp + if uhelp.ok_http_code(sc): + md[name] = resp except urllib2.HTTPError as e: if e.code != 404: raise diff --git a/cloudinit/sources/DataSourceNoCloud.py b/cloudinit/sources/DataSourceNoCloud.py index 84d0f99d..2b016d1c 100644 --- a/cloudinit/sources/DataSourceNoCloud.py +++ b/cloudinit/sources/DataSourceNoCloud.py @@ -106,7 +106,8 @@ class DataSourceNoCloud(sources.DataSource): if e.errno != errno.ENOENT: raise except util.MountFailedError: - util.logexc(LOG, "Failed to mount %s when looking for seed", dev) + util.logexc(LOG, ("Failed to mount %s" + " when looking for data"), dev) # There was no indication on kernel cmdline or data # in the seeddir suggesting this handler should be used. diff --git a/cloudinit/sources/DataSourceOVF.py b/cloudinit/sources/DataSourceOVF.py index bb0f46c2..258d8d03 100644 --- a/cloudinit/sources/DataSourceOVF.py +++ b/cloudinit/sources/DataSourceOVF.py @@ -21,10 +21,10 @@ # along with this program. If not, see . from xml.dom import minidom + import base64 import os import re -import tempfile from cloudinit import log as logging from cloudinit import sources @@ -51,7 +51,7 @@ class DataSourceOVF(sources.DataSource): ud = "" defaults = { - "instance-id": "iid-dsovf" + "instance-id": "iid-dsovf", } (seedfile, contents) = get_ovf_env(self.paths.seed_dir) @@ -198,7 +198,7 @@ def transport_iso9660(require_iso=True): for dev in devs: fullp = os.path.join("/dev/", dev) - if (fullp in mounted or + if (fullp in mounts or not cdmatch.match(dev) or os.path.isdir(fullp)): continue @@ -210,7 +210,8 @@ def transport_iso9660(require_iso=True): continue try: - (fname, contents) = utils.mount_cb(fullp, get_ovf_env, mtype="iso9660") + (fname, contents) = util.mount_cb(fullp, + get_ovf_env, mtype="iso9660") except util.MountFailedError: util.logexc(LOG, "Failed mounting %s", fullp) continue @@ -265,7 +266,8 @@ def get_properties(contents): raise XmlError("No 'PropertySection's") props = {} - propElems = find_child(propSections[0], lambda n: n.localName == "Property") + propElems = find_child(propSections[0], + (lambda n: n.localName == "Property")) for elem in propElems: key = elem.attributes.getNamedItemNS(envNsURI, "key").value diff --git a/cloudinit/sources/__init__.py b/cloudinit/sources/__init__.py index 08669f5d..beb0f3d7 100644 --- a/cloudinit/sources/__init__.py +++ b/cloudinit/sources/__init__.py @@ -22,10 +22,9 @@ from cloudinit import importer from cloudinit import log as logging +from cloudinit import user_data as ud from cloudinit import util -from cloudinit.user_data import processor as ud_proc - DEP_FILESYSTEM = "FILESYSTEM" DEP_NETWORK = "NETWORK" DS_PREFIX = 'DataSource' @@ -42,7 +41,6 @@ class DataSource(object): self.sys_cfg = sys_cfg self.distro = distro self.paths = paths - self.userdata_proc = ud_proc.UserDataProcessor(paths) self.userdata = None self.metadata = None self.userdata_raw = None @@ -55,7 +53,7 @@ class DataSource(object): def get_userdata(self): if self.userdata is None: raw_data = self.get_userdata_raw() - self.userdata = self.userdata_proc.process(raw_data) + self.userdata = ud.UserDataProcessor(self.paths).process(raw_data) return self.userdata def get_userdata_raw(self): @@ -73,7 +71,7 @@ class DataSource(object): if not self.metadata or 'public-keys' not in self.metadata: return keys - if isinstance(self.metadata['public-keys'], (str)): + if isinstance(self.metadata['public-keys'], (basestring, str)): return str(self.metadata['public-keys']).splitlines() if isinstance(self.metadata['public-keys'], (list, set)): @@ -84,11 +82,12 @@ class DataSource(object): # lp:506332 uec metadata service responds with # data that makes boto populate a string for 'klist' rather # than a list. - if isinstance(klist, (str)): + if isinstance(klist, (str, basestring)): klist = [klist] if isinstance(klist, (list, set)): for pkey in klist: - # there is an empty string at the end of the keylist, trim it + # There is an empty string at + # the end of the keylist, trim it if pkey: keys.append(pkey) @@ -159,13 +158,14 @@ def find_source(sys_cfg, distro, paths, ds_deps, cfg_list, pkg_list): ds_list = list_sources(cfg_list, ds_deps, pkg_list) ds_names = [util.obj_name(f) for f in ds_list] LOG.info("Searching for data source in: %s", ds_names) + for cls in ds_list: ds = util.obj_name(cls) try: s = cls(distro, sys_cfg, paths) if s.get_data(): return (s, ds) - except Exception as e: + except Exception: util.logexc(LOG, "Getting data from %s failed", ds) msg = "Did not find any data source, searched classes: %s" % (ds_names) @@ -178,7 +178,8 @@ def find_source(sys_cfg, distro, paths, ds_deps, cfg_list, pkg_list): # return an ordered list of classes that match def list_sources(cfg_list, depends, pkg_list): src_list = [] - LOG.info("Looking for for data source in: %s, %s that match %s", cfg_list, pkg_list, depends) + LOG.info(("Looking for for data source in: %s," + " %s that matches %s"), cfg_list, pkg_list, depends) for ds_coll in cfg_list: ds_name = str(ds_coll) if not ds_name.startswith(DS_PREFIX): @@ -201,8 +202,8 @@ def list_sources(cfg_list, depends, pkg_list): if not cls_matches: continue src_list.extend(cls_matches) - LOG.debug("Found a match for data source %s in %s with matches %s", - ds_name, mod, cls_matches) + LOG.debug(("Found a match for data source %s" + " in %s with matches %s"), ds_name, mod, cls_matches) break return src_list diff --git a/cloudinit/stages.py b/cloudinit/stages.py index 2615d59f..b9076881 100644 --- a/cloudinit/stages.py +++ b/cloudinit/stages.py @@ -31,19 +31,23 @@ try: except ImportError: ConfigObj = None -from cloudinit.settings import (PER_INSTANCE, FREQUENCIES) from cloudinit.settings import (OLD_CLOUD_CONFIG) +from cloudinit.settings import (PER_INSTANCE, FREQUENCIES) + +from cloudinit.handlers import boot_hook as bh_part +from cloudinit.handlers import cloud_config as cc_part +from cloudinit.handlers import shell_script as ss_part +from cloudinit.handlers import upstart_job as up_part from cloudinit import cloud from cloudinit import distros -from cloudinit import modules from cloudinit import helpers from cloudinit import importer from cloudinit import log as logging from cloudinit import sources -from cloudinit import util - +from cloudinit import transforms from cloudinit import user_data as ud +from cloudinit import util LOG = logging.getLogger(__name__) @@ -73,12 +77,19 @@ class Init(object): def distro(self): if not self._distro: d_cfg = util.get_cfg_by_path(self.cfg, ('system_info'), {}) + # Ensure its a dictionary + if not isinstance(d_cfg, (dict)): + d_cfg = {} # Ensure not modified indirectly d_cfg = copy.deepcopy(d_cfg) + # Remove this since its path config, not distro config d_cfg.pop('paths', None) - distro_cls = distros.fetch(sys_cfg.pop('distro', 'ubuntu')) + # Try to find the right class to use + distro_name = d_cfg.pop('distro', 'ubuntu') + distro_cls = distros.fetch(distro_name) LOG.debug("Using distro class %s", distro_cls) - distro = distro_cls(d_cfg, helpers.Runners(self.paths)) + distro = distro_cls(distro_name, d_cfg, + helpers.Runners(self.paths)) self._distro = distro return self._distro @@ -93,7 +104,8 @@ class Init(object): @property def paths(self): if not self._paths: - path_info = util.get_cfg_by_path(self.cfg, ('system_info', 'paths'), {}) + path_info = util.get_cfg_by_path(self.cfg, + ('system_info', 'paths'), {}) # Ensure not modified indirectly path_info = copy.deepcopy(path_info) self._paths = helpers.Paths(path_info, self.datasource) @@ -156,7 +168,7 @@ class Init(object): # by using the instance link, if purge_cache was called # the file wont exist return pickle.loads(util.load_file(pickled_fn)) - except Exception as e: + except Exception: util.logexc(LOG, "Failed loading pickled datasource from %s", pickled_fn) return None @@ -166,7 +178,7 @@ class Init(object): try: contents = pickle.dumps(self.datasource) util.write_file(pickled_fn, contents, mode=0400) - except Exception as e: + except Exception: util.logexc(LOG, "Failed pickling datasource to %s", pickled_fn) return False @@ -192,7 +204,8 @@ class Init(object): # (which will affect user-data handlers down the line...) sys_cfg = copy.deepcopy(self.cfg) ds_deps = copy.deepcopy(self.ds_deps) - (ds, dsname) = sources.find_source(sys_cfg, self.distro, self.paths, + (ds, dsname) = sources.find_source(sys_cfg, self.distro, + self.paths, ds_deps, cfg_list, pkg_list) LOG.debug("Loaded datasource %s - %s", dsname, ds) self.datasource = ds @@ -270,6 +283,20 @@ class Init(object): processed_ud = "%s" % (self.datasource.get_userdata()) util.write_file(self.paths.get_ipath('userdata'), processed_ud, 0600) + def _default_userdata_handlers(self): + opts = { + 'paths': self.paths, + 'instance_id': self.datasource.get_instance_id(), + } + # TODO Hmmm, should we dynamically import these?? + def_handlers = [ + cc_part.CloudConfigPartHandler(**opts), + ss_part.ShellScriptPartHandler(**opts), + bh_part.BootHookPartHandler(**opts), + up_part.UpstartJobPartHandler(**opts), + ] + return def_handlers + def consume(self, frequency=PER_INSTANCE): cdir = self.paths.get_cpath("handlers") idir = self.paths.get_ipath("handlers") @@ -279,8 +306,11 @@ class Init(object): sys.path.insert(0, cdir) sys.path.insert(0, idir) + # Ensure datasource fetched before activation (just incase) + ud_obj = self.datasource.get_userdata() + # This keeps track of all the active handlers - c_handlers = helpers.ContentHandlers(self.paths) + c_handlers = helpers.ContentHandlers(paths=self.paths) # Add handlers in cdir potential_handlers = util.find_modules(cdir) @@ -292,13 +322,10 @@ class Init(object): except: util.logexc(LOG, "Failed to register handler from %s", fname) - def_handlers = c_handlers.register_defaults() - if def_handlers: - LOG.debug("Registered default handlers for %s", def_handlers) - - - # Ensure userdata fetched before activation (just incase) - ud_obj = self.datasource.get_userdata() + def_handlers = self._default_userdata_handlers() + applied_def_handlers = c_handlers.register_defaults(def_handlers) + if applied_def_handlers: + LOG.debug("Registered default handlers: %s", applied_def_handlers) # Form our cloud interface data = self.cloudify() @@ -334,11 +361,11 @@ class Init(object): class Transforms(object): - def __init__(self, cloudobj, cfgfile=None): - self.datasource = cloudobj.datasource + def __init__(self, init, cfgfile=None): + self.datasource = init.fetch() self.cfgfile = cfgfile - self.basecfg = copy.deepcopy(cloudobj.cfg) - self.cloudobj = cloudobj + self.basecfg = copy.deepcopy(init.cfg) + self.init = init # Created on first use self._cachedcfg = None @@ -409,25 +436,28 @@ class Transforms(object): (item, util.obj_name(item))) return module_list - def _transforms_modules(self, raw_mods): + def _fixup_transforms(self, raw_mods): mostly_mods = [] for raw_mod in raw_mods: raw_name = raw_mod['mod'] freq = raw_mod.get('freq') run_args = raw_mod.get('args') or [] - mod_name = modules.form_module_name(raw_name) + mod_name = transforms.form_module_name(raw_name) if not mod_name: continue if freq and freq not in FREQUENCIES: - LOG.warn("Config specified module %s has an unknown frequency %s", raw_name, freq) + LOG.warn(("Config specified transform %s" + " has an unknown frequency %s"), raw_name, freq) # Reset it so when ran it will get set to a known value freq = None - mod = modules.fixup_module(importer.import_module(mod_name)) + mod = transforms.fixup_module(importer.import_module(mod_name)) mostly_mods.append([mod, raw_name, freq, run_args]) return mostly_mods def _run_transforms(self, mostly_mods): failures = [] + d_name = self.init.distro.name + c_cloud = self.init.cloudify() for (mod, name, freq, args) in mostly_mods: try: # Try the modules frequency, otherwise fallback to a known one @@ -436,17 +466,17 @@ class Transforms(object): if not freq in FREQUENCIES: freq = PER_INSTANCE worked_distros = mod.distros - if worked_distros and self.cloud.distro.name() not in worked_distros: - LOG.warn(("Module %s is verified on %s distros" + if (worked_distros and d_name not in worked_distros): + LOG.warn(("Transform %s is verified on %s distros" " but not on %s distro. It may or may not work" - " correctly."), name, worked_distros, - self.cloud.distro.name()) + " correctly."), name, worked_distros, d_name) # Deep copy the config so that modules can't alter it + # Use the transforms logger and not our own func_args = [name, copy.deepcopy(self.cfg), - self.cloudobj, LOG, args] - # This name will affect the semphapore name created + c_cloud, transforms.LOG, args] + # This name will affect the semaphore name created run_name = "config-%s" % (name) - self.cloudobj.run(run_name, mod.handle, func_args, freq=freq) + c_cloud.run(run_name, mod.handle, func_args, freq=freq) except Exception as e: util.logexc(LOG, "Running %s failed", mod) failures.append((name, e)) @@ -454,5 +484,5 @@ class Transforms(object): def run(self, name): raw_mods = self._read_transforms(name) - mostly_mods = self._transforms_modules(raw_mods) + mostly_mods = self._fixup_transforms(raw_mods) return self._run_transforms(mostly_mods) diff --git a/cloudinit/transforms/__init__.py b/cloudinit/transforms/__init__.py index 8275b375..40affc4b 100644 --- a/cloudinit/transforms/__init__.py +++ b/cloudinit/transforms/__init__.py @@ -44,7 +44,7 @@ def fixup_module(mod, def_freq=PER_INSTANCE): else: freq = mod.frequency if freq and freq not in FREQUENCIES: - LOG.warn("Module %s has an unknown frequency %s", mod, freq) + LOG.warn("Transform %s has an unknown frequency %s", mod, freq) if not hasattr(mod, 'handle'): def empty_handle(_name, _cfg, _cloud, _log, _args): pass diff --git a/cloudinit/transforms/cc_apt_update_upgrade.py b/cloudinit/transforms/cc_apt_update_upgrade.py index c4a543ed..a4e058c6 100644 --- a/cloudinit/transforms/cc_apt_update_upgrade.py +++ b/cloudinit/transforms/cc_apt_update_upgrade.py @@ -71,7 +71,7 @@ def handle(_name, cfg, cloud, log, _args): except: util.logexc(log, "Failed to run debconf-set-selections") - pkglist = util.get_cfg_option_list_or_str(cfg, 'packages', []) + pkglist = util.get_cfg_option_list(cfg, 'packages', []) errors = [] if update or len(pkglist) or upgrade: @@ -96,7 +96,9 @@ def handle(_name, cfg, cloud, log, _args): errors.append(e) if len(errors): - raise errors[0] + log.warn("%s failed with exceptions, re-raising the last one", + len(errors)) + raise errors[-1] def mirror2lists_fileprefix(mirror): @@ -186,7 +188,8 @@ def add_sources(srclist, template_params=None): try: util.write_file(ent['filename'], source + "\n", omode="ab") except: - errorlist.append([source, "failed write to file %s" % ent['filename']]) + errorlist.append([source, + "failed write to file %s" % ent['filename']]) return errorlist @@ -219,9 +222,10 @@ def find_apt_mirror(cloud, cfg): doms.extend((".localdomain", "",)) mirror_list = [] + distro = cloud.distro.name mirrorfmt = "http://%s-mirror%s/%s" % (distro, "%s", distro) for post in doms: - mirror_list.append(mirrorfmt % post) + mirror_list.append(mirrorfmt % (post)) mirror = util.search_for_mirror(mirror_list) diff --git a/cloudinit/transforms/cc_bootcmd.py b/cloudinit/transforms/cc_bootcmd.py index a2efad32..80afb5e7 100644 --- a/cloudinit/transforms/cc_bootcmd.py +++ b/cloudinit/transforms/cc_bootcmd.py @@ -30,7 +30,8 @@ frequency = PER_ALWAYS def handle(name, cfg, cloud, log, _args): if "bootcmd" not in cfg: - log.debug("Skipping module named %s, no 'bootcomd' key in configuration", name) + log.debug(("Skipping transform named %s," + " no 'bootcomd' key in configuration"), name) return with tempfile.NamedTemporaryFile(suffix=".sh") as tmpf: @@ -39,7 +40,7 @@ def handle(name, cfg, cloud, log, _args): tmpf.write(content) tmpf.flush() except: - log.warn("Failed to shellify bootcmd") + util.logexc(log, "Failed to shellify bootcmd") raise try: @@ -48,5 +49,6 @@ def handle(name, cfg, cloud, log, _args): cmd = ['/bin/sh', tmpf.name] util.subp(cmd, env=env, capture=False) except: - log.warn("Failed to run commands from bootcmd") + util.logexc(log, + ("Failed to run bootcmd transform %s"), name) raise diff --git a/cloudinit/transforms/cc_byobu.py b/cloudinit/transforms/cc_byobu.py index 38586174..741aa934 100644 --- a/cloudinit/transforms/cc_byobu.py +++ b/cloudinit/transforms/cc_byobu.py @@ -30,7 +30,7 @@ def handle(name, cfg, _cloud, log, args): value = util.get_cfg_option_str(cfg, "byobu_by_default", "") if not value: - log.debug("Skipping module named %s, no 'byobu' values found", name) + log.debug("Skipping transform named %s, no 'byobu' values found", name) return if value == "user" or value == "system": diff --git a/cloudinit/transforms/cc_ca_certs.py b/cloudinit/transforms/cc_ca_certs.py index 8ca9a200..e0802bfe 100644 --- a/cloudinit/transforms/cc_ca_certs.py +++ b/cloudinit/transforms/cc_ca_certs.py @@ -23,6 +23,8 @@ CA_CERT_FILENAME = "cloud-init-ca-certs.crt" CA_CERT_CONFIG = "/etc/ca-certificates.conf" CA_CERT_SYSTEM_PATH = "/etc/ssl/certs/" +distros = ['ubuntu'] + def update_ca_certs(): """ @@ -70,22 +72,25 @@ def handle(name, cfg, _cloud, log, _args): """ # If there isn't a ca-certs section in the configuration don't do anything if "ca-certs" not in cfg: - log.debug("Skipping module named %s, no 'ca-certs' key in configuration", name) + log.debug(("Skipping transform named %s," + " no 'ca-certs' key in configuration"), name) return + ca_cert_cfg = cfg['ca-certs'] # If there is a remove-defaults option set to true, remove the system # default trusted CA certs first. if ca_cert_cfg.get("remove-defaults", False): - log.debug("removing default certificates") + log.debug("Removing default certificates") remove_default_ca_certs() # If we are given any new trusted CA certs to add, add them. if "trusted" in ca_cert_cfg: - trusted_certs = util.get_cfg_option_list_or_str(ca_cert_cfg, "trusted") + trusted_certs = util.get_cfg_option_list(ca_cert_cfg, "trusted") if trusted_certs: - log.debug("adding %d certificates" % len(trusted_certs)) + log.debug("Adding %d certificates" % len(trusted_certs)) add_ca_certs(trusted_certs) # Update the system with the new cert configuration. + log.debug("Updating certificates") update_ca_certs() diff --git a/cloudinit/transforms/cc_chef.py b/cloudinit/transforms/cc_chef.py index 12c2f539..473e5f8b 100644 --- a/cloudinit/transforms/cc_chef.py +++ b/cloudinit/transforms/cc_chef.py @@ -31,7 +31,8 @@ def handle(name, cfg, cloud, log, _args): # If there isn't a chef key in the configuration don't do anything if 'chef' not in cfg: - log.debug("Skipping module named %s, no 'chef' key in configuration", name) + log.debug(("Skipping transform named %s," + " no 'chef' key in configuration"), name) return chef_cfg = cfg['chef'] diff --git a/cloudinit/transforms/cc_disable_ec2_metadata.py b/cloudinit/transforms/cc_disable_ec2_metadata.py index 4d2a7f55..3c0dd57b 100644 --- a/cloudinit/transforms/cc_disable_ec2_metadata.py +++ b/cloudinit/transforms/cc_disable_ec2_metadata.py @@ -28,5 +28,6 @@ reject_cmd = ['route', 'add', '-host', '169.254.169.254', 'reject'] def handle(_name, cfg, _cloud, _log, _args): - if util.get_cfg_option_bool(cfg, "disable_ec2_metadata", False): + disabled = util.get_cfg_option_bool(cfg, "disable_ec2_metadata", False) + if disabled: util.subp(reject_cmd) diff --git a/cloudinit/transforms/cc_final_message.py b/cloudinit/transforms/cc_final_message.py index dc4ae34c..c257b6d0 100644 --- a/cloudinit/transforms/cc_final_message.py +++ b/cloudinit/transforms/cc_final_message.py @@ -32,7 +32,7 @@ final_message_def = ("Cloud-init v. {{version}} finished at {{timestamp}}." " Up {{uptime}} seconds.") -def handle(name, cfg, cloud, log, args): +def handle(_name, cfg, cloud, log, args): msg_in = None if len(args) != 0: @@ -60,7 +60,7 @@ def handle(name, cfg, cloud, log, args): # Use stdout, stderr or the logger?? content = templater.render_string(msg_in, subs) sys.stderr.write("%s\n" % (content)) - except Exception as e: + except Exception: util.logexc(log, "Failed to render final message template") boot_fin_fn = cloud.paths.boot_finished diff --git a/cloudinit/transforms/cc_foo.py b/cloudinit/transforms/cc_foo.py index 8007f981..99135704 100644 --- a/cloudinit/transforms/cc_foo.py +++ b/cloudinit/transforms/cc_foo.py @@ -45,8 +45,8 @@ from cloudinit.settings import PER_INSTANCE # informational purposes. If non existent all distros are assumed and # no warning occurs. -frequency = settings.PER_INSTANCE +frequency = PER_INSTANCE -def handle(name, _cfg, _cloud, _log, _args): - print("Hi from %s" % (name)) +def handle(name, _cfg, _cloud, log, _args): + log.debug("Hi from transform %s", name) diff --git a/cloudinit/transforms/cc_grub_dpkg.py b/cloudinit/transforms/cc_grub_dpkg.py index c048d5cc..02f05ce3 100644 --- a/cloudinit/transforms/cc_grub_dpkg.py +++ b/cloudinit/transforms/cc_grub_dpkg.py @@ -54,9 +54,9 @@ def handle(_name, cfg, _cloud, log, _args): # now idevs and idevs_empty are set to determined values # or, those set by user - dconf_sel = ("grub-pc grub-pc/install_devices string %s\n" + dconf_sel = (("grub-pc grub-pc/install_devices string %s\n" "grub-pc grub-pc/install_devices_empty boolean %s\n") % - (idevs, idevs_empty) + (idevs, idevs_empty)) log.debug("Setting grub debconf-set-selections with '%s','%s'" % (idevs, idevs_empty)) diff --git a/cloudinit/transforms/cc_keys_to_console.py b/cloudinit/transforms/cc_keys_to_console.py index 2f2a5297..e974375f 100644 --- a/cloudinit/transforms/cc_keys_to_console.py +++ b/cloudinit/transforms/cc_keys_to_console.py @@ -18,23 +18,34 @@ # You should have received a copy of the GNU General Public License # along with this program. If not, see . +import os + from cloudinit.settings import PER_INSTANCE from cloudinit import util frequency = PER_INSTANCE +# This is a tool that cloud init provides +helper_tool = '/usr/lib/cloud-init/write-ssh-key-fingerprints' + -def handle(_name, cfg, _cloud, log, _args): - cmd = ['/usr/lib/cloud-init/write-ssh-key-fingerprints'] - fp_blacklist = util.get_cfg_option_list_or_str(cfg, +def handle(name, cfg, _cloud, log, _args): + if not os.path.exists(helper_tool): + log.warn(("Unable to activate transform %s," + " helper tool not found at %s"), name, helper_tool) + return + + fp_blacklist = util.get_cfg_option_list(cfg, "ssh_fp_console_blacklist", []) - key_blacklist = util.get_cfg_option_list_or_str(cfg, + key_blacklist = util.get_cfg_option_list(cfg, "ssh_key_console_blacklist", ["ssh-dss"]) + try: + cmd = [helper_tool] cmd.append(','.join(fp_blacklist)) cmd.append(','.join(key_blacklist)) - (stdout, stderr) = util.subp(cmd) + (stdout, _stderr) = util.subp(cmd) util.write_file('/dev/console', stdout) except: - log.warn("Writing keys to console failed!") + log.warn("Writing keys to /dev/console failed!") raise diff --git a/cloudinit/transforms/cc_landscape.py b/cloudinit/transforms/cc_landscape.py index 48491992..19948d0e 100644 --- a/cloudinit/transforms/cc_landscape.py +++ b/cloudinit/transforms/cc_landscape.py @@ -55,7 +55,8 @@ def handle(name, cfg, _cloud, log, _args): /etc/landscape/client.conf """ if not ConfigObj: - log.warn("'ConfigObj' support not enabled, running %s disabled", name) + log.warn(("'ConfigObj' support not available," + " running transform %s disabled"), name) return ls_cloudcfg = cfg.get("landscape", {}) diff --git a/cloudinit/transforms/cc_locale.py b/cloudinit/transforms/cc_locale.py index 3fb4c5d9..7f273123 100644 --- a/cloudinit/transforms/cc_locale.py +++ b/cloudinit/transforms/cc_locale.py @@ -49,7 +49,7 @@ def handle(name, cfg, cloud, log, args): "/etc/default/locale") if not locale: - log.debug(("Skipping module named %s, " + log.debug(("Skipping transform named %s, " "no 'locale' configuration found"), name) return diff --git a/cloudinit/transforms/cc_mcollective.py b/cloudinit/transforms/cc_mcollective.py index aeeda9d2..5464fe8c 100644 --- a/cloudinit/transforms/cc_mcollective.py +++ b/cloudinit/transforms/cc_mcollective.py @@ -19,13 +19,10 @@ # You should have received a copy of the GNU General Public License # along with this program. If not, see . -from ConfigParser import ConfigParser from StringIO import StringIO -import os - +from cloudinit import cfg as config from cloudinit import util -from cloudinit import cfg pubcert_file = "/etc/mcollective/ssl/server-public.pem" pricert_file = "/etc/mcollective/ssl/server-private.pem" @@ -35,7 +32,7 @@ def handle(name, cfg, cloud, log, _args): # If there isn't a mcollective key in the configuration don't do anything if 'mcollective' not in cfg: - log.debug(("Skipping module named %s, " + log.debug(("Skipping transform named %s, " "no 'mcollective' key in configuration"), name) return @@ -47,7 +44,7 @@ def handle(name, cfg, cloud, log, _args): # ... and then update the mcollective configuration if 'conf' in mcollective_cfg: # Create object for reading server.cfg values - mcollective_config = cfg.DefaultingConfigParser() + mcollective_config = config.DefaultingConfigParser() # Read server.cfg values from original file in order to be able to mix # the rest up old_contents = util.load_file('/etc/mcollective/server.cfg') diff --git a/cloudinit/transforms/cc_mounts.py b/cloudinit/transforms/cc_mounts.py index babcbda1..44182b87 100644 --- a/cloudinit/transforms/cc_mounts.py +++ b/cloudinit/transforms/cc_mounts.py @@ -20,7 +20,6 @@ from string import whitespace # pylint: disable=W0402 -import os import re from cloudinit import util @@ -28,7 +27,7 @@ from cloudinit import util # shortname matches 'sda', 'sda1', 'xvda', 'hda', 'sdb', xvdb, vda, vdd1 shortname_filter = r"^[x]{0,1}[shv]d[a-z][0-9]*$" shortname = re.compile(shortname_filter) -ws = re.compile("[%s]+" % whitespace) +ws = re.compile("[%s]+" % (whitespace)) def is_mdname(name): @@ -65,13 +64,14 @@ def handle(_name, cfg, cloud, log, _args): continue startname = str(cfgmnt[i][0]) - LOG.debug("Attempting to determine the real name of %s", startname) + log.debug("Attempting to determine the real name of %s", startname) # workaround, allow user to specify 'ephemeral' # rather than more ec2 correct 'ephemeral0' if startname == "ephemeral": cfgmnt[i][0] = "ephemeral0" - log.debug("Adjusted mount option %s name from ephemeral to ephemeral0", (i + 1)) + log.debug(("Adjusted mount option %s " + "name from ephemeral to ephemeral0"), (i + 1)) if is_mdname(startname): newname = cloud.device_name_to_device(startname) @@ -136,7 +136,8 @@ def handle(_name, cfg, cloud, log, _args): break if cfgmnt_has: - log.debug("Not including %s, already previously included", startname) + log.debug(("Not including %s, already" + " previously included"), startname) continue cfgmnt.append(defmnt) @@ -159,7 +160,7 @@ def handle(_name, cfg, cloud, log, _args): dirs = [] for line in actlist: # write 'comment' in the fs_mntops, entry, claiming this - line[3] = "%s,comment=cloudconfig" % line[3] + line[3] = "%s,%s" % (line[3], comment) if line[2] == "swap": needswap = True if line[1].startswith("/"): @@ -168,7 +169,7 @@ def handle(_name, cfg, cloud, log, _args): fstab_lines = [] fstab = util.load_file("/etc/fstab") - for line in fstab.read().splitlines(): + for line in fstab.splitlines(): try: toks = ws.split(line) if toks[3].find(comment) != -1: diff --git a/cloudinit/transforms/cc_phone_home.py b/cloudinit/transforms/cc_phone_home.py index 36af6dfa..98ff2b85 100644 --- a/cloudinit/transforms/cc_phone_home.py +++ b/cloudinit/transforms/cc_phone_home.py @@ -24,9 +24,8 @@ from cloudinit import util from cloudinit.settings import PER_INSTANCE -from time import sleep - frequency = PER_INSTANCE + post_list_all = ['pub_key_dsa', 'pub_key_rsa', 'pub_key_ecdsa', 'instance_id', 'hostname'] @@ -49,7 +48,7 @@ def handle(name, cfg, cloud, log, args): ph_cfg = cfg['phone_home'] if 'url' not in ph_cfg: - log.warn(("Skipping module named %s, " + log.warn(("Skipping transform named %s, " "no 'url' found in 'phone_home' configuration"), name) return @@ -60,7 +59,8 @@ def handle(name, cfg, cloud, log, args): tries = int(tries) except: tries = 10 - util.logexc(log, "Configuration entry 'tries' is not an integer, using %s", tries) + util.logexc(log, ("Configuration entry 'tries'" + " is not an integer, using %s instead"), tries) if post_list == "all": post_list = post_list_all @@ -75,23 +75,37 @@ def handle(name, cfg, cloud, log, args): 'pub_key_ecdsa': '/etc/ssh/ssh_host_ecdsa_key.pub', } - for n, path in pubkeys.iteritems(): + for (n, path) in pubkeys.iteritems(): try: all_keys[n] = util.load_file(path) except: - util.logexc(log, "%s: failed to open, can not phone home that data", path) + util.logexc(log, ("%s: failed to open, can not" + " phone home that data"), path) submit_keys = {} for k in post_list: if k in all_keys: submit_keys[k] = all_keys[k] else: - submit_keys[k] = "N/A" - log.warn("Requested key %s from 'post' configuration list not available", k) + submit_keys[k] = None + log.warn(("Requested key %s from 'post'" + " configuration list not available"), k) - url = templater.render_string(url, {'INSTANCE_ID': all_keys['instance_id']}) + # Get them read to be posted + real_submit_keys = {} + for (k, v) in submit_keys.iteritems(): + if v is None: + real_submit_keys[k] = 'N/A' + else: + real_submit_keys[k] = str(v) + # Incase the url is parameterized + url_params = { + 'INSTANCE_ID': all_keys['instance_id'], + } + url = templater.render_string(url, url_params) try: - uhelp.readurl(url, data=submit_keys, retries=tries, sec_between=3) + uhelp.readurl(url, data=real_submit_keys, retries=tries, sec_between=3) except: - util.logexc(log, "Failed to post phone home data to %s in %s tries", url, tries) + util.logexc(log, ("Failed to post phone home data to" + " %s in %s tries"), url, tries) diff --git a/cloudinit/transforms/cc_puppet.py b/cloudinit/transforms/cc_puppet.py index 0a21a929..76cc9732 100644 --- a/cloudinit/transforms/cc_puppet.py +++ b/cloudinit/transforms/cc_puppet.py @@ -24,31 +24,32 @@ import os import pwd import socket +from cloudinit import cfg as config from cloudinit import util -from cloudinit import cfg def handle(name, cfg, cloud, log, _args): # If there isn't a puppet key in the configuration don't do anything if 'puppet' not in cfg: - log.debug(("Skipping module named %s," + log.debug(("Skipping transform named %s," " no 'puppet' configuration found"), name) return puppet_cfg = cfg['puppet'] # Start by installing the puppet package ... - cloud.distro.install_packages(("puppet",)) + cloud.distro.install_packages(["puppet"]) # ... and then update the puppet configuration if 'conf' in puppet_cfg: # Add all sections from the conf object to puppet.conf contents = util.load_file('/etc/puppet/puppet.conf') # Create object for reading puppet.conf values - puppet_config = cfg.DefaultingConfigParser() + puppet_config = config.DefaultingConfigParser() # Read puppet.conf values from original file in order to be able to # mix the rest up. First clean them up (TODO is this really needed??) - cleaned_contents = '\n'.join([i.lstrip() for i in contents.splitlines()]) + cleaned_lines = [i.lstrip() for i in contents.splitlines()] + cleaned_contents = '\n'.join(cleaned_lines) puppet_config.readfp(StringIO(cleaned_contents), filename='/etc/puppet/puppet.conf') for (cfg_name, cfg) in puppet_cfg['conf'].iteritems(): @@ -81,7 +82,8 @@ def handle(name, cfg, cloud, log, _args): puppet_config.set(cfg_name, o, v) # We got all our config as wanted we'll rename # the previous puppet.conf and create our new one - util.rename('/etc/puppet/puppet.conf', '/etc/puppet/puppet.conf.old') + util.rename('/etc/puppet/puppet.conf', + '/etc/puppet/puppet.conf.old') contents = puppet_config.stringify() util.write_file('/etc/puppet/puppet.conf', contents) @@ -91,7 +93,8 @@ def handle(name, cfg, cloud, log, _args): '-e', 's/^START=.*/START=yes/', '/etc/default/puppet'], capture=False) elif os.path.exists('/bin/systemctl'): - util.subp(['/bin/systemctl', 'enable', 'puppet.service'], capture=False) + util.subp(['/bin/systemctl', 'enable', 'puppet.service'], + capture=False) elif os.path.exists('/sbin/chkconfig'): util.subp(['/sbin/chkconfig', 'puppet', 'on'], capture=False) else: diff --git a/cloudinit/transforms/cc_resizefs.py b/cloudinit/transforms/cc_resizefs.py index daaf4da9..fe012417 100644 --- a/cloudinit/transforms/cc_resizefs.py +++ b/cloudinit/transforms/cc_resizefs.py @@ -18,11 +18,8 @@ # You should have received a copy of the GNU General Public License # along with this program. If not, see . -import errno import os import stat -import sys -import tempfile import time from cloudinit import util @@ -46,15 +43,18 @@ def nodeify_path(devpth, where, log): if util.is_container(): log.debug("Inside container, ignoring mknod failure in resizefs") return - log.warn("Failed to make device node to resize %s at %s", where, devpth) + log.warn("Failed to make device node to resize %s at %s", + where, devpth) raise def get_fs_type(st_dev, path, log): try: - fs_type = util.find_devs_with(tag='TYPE', oformat='value', + dev_entries = util.find_devs_with(tag='TYPE', oformat='value', no_cache=True, path=path) - return fs_type + if not dev_entries: + return None + return dev_entries[0].strip() except util.ProcessExecutionError: util.logexc(log, ("Failed to get filesystem type" " of maj=%s, min=%s for path %s"), @@ -69,12 +69,16 @@ def handle(name, cfg, _cloud, log, args): resize_root = util.get_cfg_option_str(cfg, "resize_rootfs", True) if not util.translate_bool(resize_root): - log.debug("Skipping module named %s, resizing disabled", name) + log.debug("Skipping transform named %s, resizing disabled", name) return # TODO is the directory ok to be used?? resize_root_d = util.get_cfg_option_str(cfg, "resize_rootfs_tmp", "/run") util.ensure_dir(resize_root_d) + + # TODO: allow what is to be resized to + # be configurable?? + resize_what = "/" with util.SilentTemporaryFile(prefix="cloudinit.resizefs.", dir=resize_root_d, delete=True) as tfh: devpth = tfh.name @@ -86,23 +90,25 @@ def handle(name, cfg, _cloud, log, args): # auto deletion tfh.unlink_now() - # TODO: allow what is to be resized to - # be configurable?? - st_dev = nodeify_path(devpth, "/", log) - fs_type = get_fs_type(st_dev, devpath, log) + st_dev = nodeify_path(devpth, resize_what, log) + fs_type = get_fs_type(st_dev, devpth, log) + if not fs_type: + log.warn("Could not determine filesystem type of %s", resize_what) + return resizer = None - fstype_lc = fstype.lower() + fstype_lc = fs_type.lower() for (pfix, root_cmd) in resize_fs_prefixes_cmds: if fstype_lc.startswith(pfix): resizer = root_cmd break if not resizer: - log.warn("Not resizing unknown filesystem type %s", fs_type) + log.warn("Not resizing unknown filesystem type %s for %s", + fs_type, resize_what) return - log.debug("Resizing using %s", resizer) + log.debug("Resizing %s (%s) using %s", resize_what, fs_type, resizer) resize_cmd = [resizer, devpth] if resize_root == "noblock": @@ -125,8 +131,8 @@ def do_resize(resize_cmd, log): start = time.time() try: util.subp(resize_cmd) - except util.ProcessExecutionError as e: - util.logexc(log, "Failed to resize filesystem (using %s)", resize_cmd) + except util.ProcessExecutionError: + util.logexc(log, "Failed to resize filesystem (cmd=%s)", resize_cmd) raise tot_time = int(time.time() - start) log.debug("Resizing took %s seconds", tot_time) diff --git a/cloudinit/transforms/cc_rightscale_userdata.py b/cloudinit/transforms/cc_rightscale_userdata.py index cde11b54..40d76c89 100644 --- a/cloudinit/transforms/cc_rightscale_userdata.py +++ b/cloudinit/transforms/cc_rightscale_userdata.py @@ -53,16 +53,19 @@ def handle(name, _cfg, cloud, log, _args): try: ud = cloud.get_userdata_raw() except: - log.warn("Failed to get raw userdata in module %s", name) + log.warn("Failed to get raw userdata in transform %s", name) return try: mdict = parse_qs(ud) if not mdict or not my_hookname in mdict: - log.debug("Skipping module %s, did not find %s in parsed raw userdata", name, my_hookname) + log.debug(("Skipping transform %s, " + "did not find %s in parsed" + " raw userdata"), name, my_hookname) return except: - log.warn("Failed to parse query string %s into a dictionary", ud) + util.logexc(log, ("Failed to parse query string %s" + " into a dictionary"), ud) raise wrote_fns = [] @@ -83,7 +86,8 @@ def handle(name, _cfg, cloud, log, _args): wrote_fns.append(fname) except Exception as e: captured_excps.append(e) - util.logexc(log, "%s failed to read %s and write %s", my_name, url, fname) + util.logexc(log, "%s failed to read %s and write %s", + my_name, url, fname) if wrote_fns: log.debug("Wrote out rightscale userdata to %s files", len(wrote_fns)) @@ -93,6 +97,6 @@ def handle(name, _cfg, cloud, log, _args): log.debug("%s urls were skipped or failed", skipped) if captured_excps: - log.warn("%s failed with exceptions, re-raising the last one", len(captured_excps)) + log.warn("%s failed with exceptions, re-raising the last one", + len(captured_excps)) raise captured_excps[-1] - diff --git a/cloudinit/transforms/cc_rsyslog.py b/cloudinit/transforms/cc_rsyslog.py index ccbe68ff..71b74711 100644 --- a/cloudinit/transforms/cc_rsyslog.py +++ b/cloudinit/transforms/cc_rsyslog.py @@ -36,7 +36,8 @@ def handle(name, cfg, cloud, log, _args): # process 'rsyslog' if not 'rsyslog' in cfg: - log.debug("Skipping module named %s, no 'rsyslog' key in configuration", name) + log.debug(("Skipping transform named %s," + " no 'rsyslog' key in configuration"), name) return def_dir = cfg.get('rsyslog_dir', DEF_DIR) @@ -62,15 +63,16 @@ def handle(name, cfg, cloud, log, _args): if not filename.startswith("/"): filename = os.path.join(def_dir, filename) + # Truncate filename first time you see it omode = "ab" - # truncate filename first time you see it if filename not in files: omode = "wb" files.append(filename) try: - util.write_file(filename, content + "\n", omode=omode) - except Exception as e: + contents = "%s\n" % (content) + util.write_file(filename, contents, omode=omode) + except Exception: util.logexc(log, "Failed to write to %s", filename) # Attempt to restart syslogd @@ -87,8 +89,8 @@ def handle(name, cfg, cloud, log, _args): log.debug("Restarting rsyslog") util.subp(['service', 'rsyslog', 'restart']) restarted = True - except Exception as e: - util.logexc("Failed restarting rsyslog") + except Exception: + util.logexc(log, "Failed restarting rsyslog") if restarted: # This only needs to run if we *actually* restarted diff --git a/cloudinit/transforms/cc_runcmd.py b/cloudinit/transforms/cc_runcmd.py index 19c0e721..31a254a5 100644 --- a/cloudinit/transforms/cc_runcmd.py +++ b/cloudinit/transforms/cc_runcmd.py @@ -25,13 +25,14 @@ from cloudinit import util def handle(name, cfg, cloud, log, _args): if "runcmd" not in cfg: - log.debug("Skipping module named %s, no 'runcmd' key in configuration", name) + log.debug(("Skipping transform named %s," + " no 'runcmd' key in configuration"), name) return - outfile = os.path.join(cloud.get_ipath('scripts'), "runcmd") + out_fn = os.path.join(cloud.get_ipath('scripts'), "runcmd") cmd = cfg["runcmd"] try: content = util.shellify(cmd) - util.write_file(outfile, content, 0700) + util.write_file(out_fn, content, 0700) except: - util.logexc(log, "Failed to shellify %s into file %s", cmd, outfile) + util.logexc(log, "Failed to shellify %s into file %s", cmd, out_fn) diff --git a/cloudinit/transforms/cc_salt_minion.py b/cloudinit/transforms/cc_salt_minion.py index 47cbc194..d05d2a1e 100644 --- a/cloudinit/transforms/cc_salt_minion.py +++ b/cloudinit/transforms/cc_salt_minion.py @@ -21,16 +21,17 @@ from cloudinit import util # Note: see http://saltstack.org/topics/installation/ -def handle(name, cfg, cloud, _log, _args): +def handle(name, cfg, cloud, log, _args): # If there isn't a salt key in the configuration don't do anything if 'salt_minion' not in cfg: - log.debug("Skipping module named %s, no 'salt_minion' key in configuration", name) + log.debug(("Skipping transform named %s," + " no 'salt_minion' key in configuration"), name) return salt_cfg = cfg['salt_minion'] # Start by installing the salt package ... - cloud.distro.install_packages(("salt",)) + cloud.distro.install_packages(["salt"]) # Ensure we can configure files at the right dir config_dir = salt_cfg.get("config_dir", '/etc/salt') diff --git a/cloudinit/transforms/cc_scripts_per_boot.py b/cloudinit/transforms/cc_scripts_per_boot.py index bcdf4400..364e1d02 100644 --- a/cloudinit/transforms/cc_scripts_per_boot.py +++ b/cloudinit/transforms/cc_scripts_per_boot.py @@ -29,12 +29,13 @@ frequency = PER_ALWAYS script_subdir = 'per-boot' -def handle(_name, _cfg, cloud, log, _args): +def handle(name, _cfg, cloud, log, _args): # Comes from the following: # https://forums.aws.amazon.com/thread.jspa?threadID=96918 runparts_path = os.path.join(cloud.get_cpath(), 'scripts', script_subdir) try: util.runparts(runparts_path) except: - log.warn("Failed to run-parts(%s) in %s", script_subdir, runparts_path) + log.warn("Failed to run transform %s (%s in %s)", + name, script_subdir, runparts_path) raise diff --git a/cloudinit/transforms/cc_scripts_per_instance.py b/cloudinit/transforms/cc_scripts_per_instance.py index 8d6609a1..d75ab47d 100644 --- a/cloudinit/transforms/cc_scripts_per_instance.py +++ b/cloudinit/transforms/cc_scripts_per_instance.py @@ -29,12 +29,13 @@ frequency = PER_INSTANCE script_subdir = 'per-instance' -def handle(_name, _cfg, cloud, log, _args): +def handle(name, _cfg, cloud, log, _args): # Comes from the following: # https://forums.aws.amazon.com/thread.jspa?threadID=96918 runparts_path = os.path.join(cloud.get_cpath(), 'scripts', script_subdir) try: util.runparts(runparts_path) except: - log.warn("Failed to run-parts(%s) in %s", script_subdir, runparts_path) + log.warn("Failed to run transform %s (%s in %s)", + name, script_subdir, runparts_path) raise diff --git a/cloudinit/transforms/cc_scripts_per_once.py b/cloudinit/transforms/cc_scripts_per_once.py index dbcec05d..80f8c325 100644 --- a/cloudinit/transforms/cc_scripts_per_once.py +++ b/cloudinit/transforms/cc_scripts_per_once.py @@ -29,12 +29,13 @@ frequency = PER_ONCE script_subdir = 'per-once' -def handle(_name, _cfg, cloud, log, _args): +def handle(name, _cfg, cloud, log, _args): # Comes from the following: # https://forums.aws.amazon.com/thread.jspa?threadID=96918 runparts_path = os.path.join(cloud.get_cpath(), 'scripts', script_subdir) try: util.runparts(runparts_path) except: - log.warn("Failed to run-parts(%s) in %s", script_subdir, runparts_path) + log.warn("Failed to run transform %s (%s in %s)", + name, script_subdir, runparts_path) raise diff --git a/cloudinit/transforms/cc_scripts_user.py b/cloudinit/transforms/cc_scripts_user.py index 1e438ee6..f4fe3a2a 100644 --- a/cloudinit/transforms/cc_scripts_user.py +++ b/cloudinit/transforms/cc_scripts_user.py @@ -26,14 +26,17 @@ from cloudinit.settings import PER_INSTANCE frequency = PER_INSTANCE +script_subdir = 'scripts' -def handle(_name, _cfg, cloud, log, _args): + +def handle(name, _cfg, cloud, log, _args): # This is written to by the user data handlers # Ie, any custom shell scripts that come down # go here... - runparts_path = os.path.join(cloud.get_ipath_cur(), "scripts") + runparts_path = os.path.join(cloud.get_ipath_cur(), script_subdir) try: util.runparts(runparts_path) except: - log.warn("Failed to run-parts(%s) in %s", "user-data", runparts_path) + log.warn("Failed to run transform %s (%s in %s)", + name, script_subdir, runparts_path) raise diff --git a/cloudinit/transforms/cc_set_hostname.py b/cloudinit/transforms/cc_set_hostname.py index fa2b59c2..3ac8a8fa 100644 --- a/cloudinit/transforms/cc_set_hostname.py +++ b/cloudinit/transforms/cc_set_hostname.py @@ -24,7 +24,7 @@ from cloudinit import util def handle(name, cfg, cloud, log, _args): if util.get_cfg_option_bool(cfg, "preserve_hostname", False): log.debug(("Configuration option 'preserve_hostname' is set," - " not setting the hostname in %s"), name) + " not setting the hostname in transform %s"), name) return (hostname, _fqdn) = util.get_hostname_fqdn(cfg, cloud) diff --git a/cloudinit/transforms/cc_set_passwords.py b/cloudinit/transforms/cc_set_passwords.py index 4f2cdb97..c0cc4e84 100644 --- a/cloudinit/transforms/cc_set_passwords.py +++ b/cloudinit/transforms/cc_set_passwords.py @@ -22,7 +22,7 @@ import sys from cloudinit import util -from string import letters, digits +from string import letters, digits # pylint: disable=W0402 # We are removing certain 'painful' letters/numbers pw_set = (letters.translate(None, 'loLOI') + @@ -71,11 +71,13 @@ def handle(_name, cfg, cloud, log, args): util.subp(['chpasswd'], ch_in) except Exception as e: errors.append(e) - util.logexc(log, "Failed to set passwords with chpasswd for %s", users) + util.logexc(log, + "Failed to set passwords with chpasswd for %s", users) if len(randlist): - sys.stderr.write("%s\n%s\n" % ("Set the following 'random' passwords\n", - '\n'.join(randlist))) + blurb = ("Set the following 'random' passwords\n", + '\n'.join(randlist)) + sys.stderr.write("%s\n%s\n" % blurb) if expire: expired_users = [] diff --git a/cloudinit/transforms/cc_ssh.py b/cloudinit/transforms/cc_ssh.py index db6848d9..3c2b3622 100644 --- a/cloudinit/transforms/cc_ssh.py +++ b/cloudinit/transforms/cc_ssh.py @@ -65,8 +65,7 @@ def handle(_name, cfg, cloud, log, _args): tgt_fn = key2file[key][0] tgt_perms = key2file[key][1] util.write_file(tgt_fn, val, tgt_perms) - - cmd = 'o=$(ssh-keygen -yf "%s") && echo "$o" root@localhost > "%s"' + for priv, pub in priv2pub.iteritems(): if pub in cfg['ssh_keys'] or not priv in cfg['ssh_keys']: continue @@ -78,11 +77,15 @@ def handle(_name, cfg, cloud, log, _args): util.subp(cmd, capture=False) log.debug("Generated a key for %s from %s", pair[0], pair[1]) except: - util.logexc(log, "Failed generated a key for %s from %s", pair[0], pair[1]) + util.logexc(log, ("Failed generated a key" + " for %s from %s"), pair[0], pair[1]) else: # if not, generate them - for keytype in util.get_cfg_option_list_or_str(cfg, 'ssh_genkeytypes', generate_keys): - keyfile = '/etc/ssh/ssh_host_%s_key' % keytype + genkeys = util.get_cfg_option_list(cfg, + 'ssh_genkeytypes', + generate_keys) + for keytype in genkeys: + keyfile = '/etc/ssh/ssh_host_%s_key' % (keytype) if not os.path.exists(keyfile): cmd = ['ssh-keygen', '-t', keytype, '-N', '', '-f', keyfile] try: @@ -90,26 +93,27 @@ def handle(_name, cfg, cloud, log, _args): with util.SeLinuxGuard("/etc/ssh", recursive=True): util.subp(cmd, capture=False) except: - util.logexc(log, "Failed generating key type %s to file %s", keytype, keyfile) + util.logexc(log, ("Failed generating key type" + " %s to file %s"), keytype, keyfile) try: user = util.get_cfg_option_str(cfg, 'user') disable_root = util.get_cfg_option_bool(cfg, "disable_root", True) disable_root_opts = util.get_cfg_option_str(cfg, "disable_root_opts", - DISABLE_ROOT_OPTS) + DISABLE_ROOT_OPTS) keys = cloud.get_public_ssh_keys() or [] if "ssh_authorized_keys" in cfg: cfgkeys = cfg["ssh_authorized_keys"] keys.extend(cfgkeys) - apply_credentials(keys, user, disable_root, disable_root_opts, log) + apply_credentials(keys, user, disable_root, disable_root_opts) except: util.logexc(log, "Applying ssh credentials failed!") def apply_credentials(keys, user, disable_root, - disable_root_opts=DISABLE_ROOT_OPTS, log=None): + disable_root_opts=DISABLE_ROOT_OPTS): keys = set(keys) if user: diff --git a/cloudinit/transforms/cc_ssh_import_id.py b/cloudinit/transforms/cc_ssh_import_id.py index 019413d4..d57e4665 100644 --- a/cloudinit/transforms/cc_ssh_import_id.py +++ b/cloudinit/transforms/cc_ssh_import_id.py @@ -33,10 +33,14 @@ def handle(name, cfg, _cloud, log, args): ids = args[1:] else: user = util.get_cfg_option_str(cfg, "user", "ubuntu") - ids = util.get_cfg_option_list_or_str(cfg, "ssh_import_id", []) + ids = util.get_cfg_option_list(cfg, "ssh_import_id", []) if len(ids) == 0: - log.debug("Skipping module named %s, no ids found to import", name) + log.debug("Skipping transform named %s, no ids found to import", name) + return + + if not user: + log.debug("Skipping transform named %s, no user found to import", name) return cmd = ["sudo", "-Hu", user, "ssh-import-id"] + ids diff --git a/cloudinit/transforms/cc_timezone.py b/cloudinit/transforms/cc_timezone.py index 02cbf2dc..747c436c 100644 --- a/cloudinit/transforms/cc_timezone.py +++ b/cloudinit/transforms/cc_timezone.py @@ -18,20 +18,22 @@ # You should have received a copy of the GNU General Public License # along with this program. If not, see . +from cloudinit import util + from cloudinit.settings import PER_INSTANCE frequency = PER_INSTANCE -def handle(_name, cfg, cloud, log, args): +def handle(name, cfg, cloud, log, args): if len(args) != 0: timezone = args[0] else: timezone = util.get_cfg_option_str(cfg, "timezone", False) if not timezone: - log.debug("Skipping module named %s, no 'timezone' specified", name) + log.debug("Skipping transform named %s, no 'timezone' specified", name) return - + # Let the distro handle settings its timezone cloud.distro.set_timezone(timezone) diff --git a/cloudinit/transforms/cc_update_etc_hosts.py b/cloudinit/transforms/cc_update_etc_hosts.py index 361097a6..d0e56183 100644 --- a/cloudinit/transforms/cc_update_etc_hosts.py +++ b/cloudinit/transforms/cc_update_etc_hosts.py @@ -30,22 +30,30 @@ def handle(name, cfg, cloud, log, _args): manage_hosts = util.get_cfg_option_str(cfg, "manage_etc_hosts", False) if util.translate_bool(manage_hosts, addons=['template']): (hostname, fqdn) = util.get_hostname_fqdn(cfg, cloud) - # Render from template file if not hostname: - log.warn("Option 'manage_etc_hosts' was set, but no hostname was found") + log.warn(("Option 'manage_etc_hosts' was set," + " but no hostname was found")) return - tpl_fn_name = cloud.get_template_filename("hosts.%s" % (cloud.distro.name())) + + # Render from a template file + distro_n = cloud.distro.name + tpl_fn_name = cloud.get_template_filename("hosts.%s" % (distro_n)) if not tpl_fn_name: - raise Exception("No hosts template could be found for distro %s" % (cloud.distro.name())) + raise Exception(("No hosts template could be" + " found for distro %s") % (distro_n)) + templater.render_to_file(tpl_fn_name, '/etc/hosts', {'hostname': hostname, 'fqdn': fqdn}) + elif manage_hosts == "localhost": - log.debug("Managing localhost in /etc/hosts") (hostname, fqdn) = util.get_hostname_fqdn(cfg, cloud) if not hostname: - log.warn("Option 'manage_etc_hosts' was set, but no hostname was found") + log.warn(("Option 'manage_etc_hosts' was set," + " but no hostname was found")) return + + log.debug("Managing localhost in /etc/hosts") cloud.distro.update_etc_hosts(hostname, fqdn) else: log.debug(("Configuration option 'manage_etc_hosts' is not set," - " not managing /etc/hosts in %s"), name) + " not managing /etc/hosts in transform %s"), name) diff --git a/cloudinit/transforms/cc_update_hostname.py b/cloudinit/transforms/cc_update_hostname.py index 439bdcb3..58444fab 100644 --- a/cloudinit/transforms/cc_update_hostname.py +++ b/cloudinit/transforms/cc_update_hostname.py @@ -18,6 +18,8 @@ # You should have received a copy of the GNU General Public License # along with this program. If not, see . +import os + from cloudinit import util from cloudinit.settings import PER_ALWAYS @@ -27,7 +29,7 @@ frequency = PER_ALWAYS def handle(name, cfg, cloud, log, _args): if util.get_cfg_option_bool(cfg, "preserve_hostname", False): log.debug(("Configuration option 'preserve_hostname' is set," - " not updating the hostname in %s"), name) + " not updating the hostname in transform %s"), name) return (hostname, _fqdn) = util.get_hostname_fqdn(cfg, cloud) diff --git a/cloudinit/transforms/cc_welcome.py b/cloudinit/transforms/cc_welcome.py index 0db71125..04691d21 100644 --- a/cloudinit/transforms/cc_welcome.py +++ b/cloudinit/transforms/cc_welcome.py @@ -35,9 +35,9 @@ welcome_message_def = ("Cloud-init v. {{version}} starting stage {{stage}} at " frequency = PER_ALWAYS -def handle(name, cfg, cloud, log, args): +def handle(_name, cfg, cloud, log, args): - welcome_msg = util.get_cfg_option_str(cfg, "welcome_msg"): + welcome_msg = util.get_cfg_option_str(cfg, "welcome_msg") if not welcome_msg: tpl_fn = cloud.get_template_filename("welcome_msg") if tpl_fn: @@ -54,7 +54,7 @@ def handle(name, cfg, cloud, log, args): 'stage': stage, 'version': version.version_string(), 'uptime': util.uptime(), - 'timestamp', util.time_rfc2822(), + 'timestamp': util.time_rfc2822(), } try: contents = templater.render_string(welcome_msg, tpl_params) diff --git a/cloudinit/user_data.py b/cloudinit/user_data.py index 64fc2734..9915b8b0 100644 --- a/cloudinit/user_data.py +++ b/cloudinit/user_data.py @@ -22,14 +22,15 @@ import os -import glob import email - +from email.mime.multipart import MIMEMultipart +from email.mime.text import MIMEText from email.mime.base import MIMEBase from cloudinit import importer from cloudinit import log as logging +from cloudinit import url_helper from cloudinit import util from cloudinit.settings import (PER_ALWAYS, PER_INSTANCE, FREQUENCIES) @@ -86,7 +87,7 @@ class UserDataProcessor(object): self.paths = paths def process(self, blob): - base_msg = ud.convert_string(blob) + base_msg = convert_string(blob) process_msg = MIMEMultipart() self._process_msg(base_msg, process_msg) return process_msg @@ -105,7 +106,7 @@ class UserDataProcessor(object): ctype_orig = UNDEF_TYPE if ctype_orig in TYPE_NEEDED: - ctype = ud.type_from_starts_with(payload) + ctype = type_from_starts_with(payload) if ctype is None: ctype = ctype_orig @@ -158,7 +159,7 @@ class UserDataProcessor(object): if not url_helper.ok_http_code(st): content = '' - new_msg = ud.convert_string(content) + new_msg = convert_string(content) self._process_msg(new_msg, append_msg) def _explode_archive(self, archive, append_msg): @@ -179,7 +180,7 @@ class UserDataProcessor(object): content = ent.get('content', '') mtype = ent.get('type') if not mtype: - mtype = ud.type_from_starts_with(content, ARCHIVE_UNDEF_TYPE) + mtype = type_from_starts_with(content, ARCHIVE_UNDEF_TYPE) maintype, subtype = mtype.split('/', 1) if maintype == "text": diff --git a/cloudinit/util.py b/cloudinit/util.py index 7259d933..1f884df8 100644 --- a/cloudinit/util.py +++ b/cloudinit/util.py @@ -22,8 +22,8 @@ from StringIO import StringIO +import copy as obj_copy import contextlib -import copy import errno import glob import grp @@ -35,12 +35,11 @@ import pwd import random import shutil import socket -import string +import string # pylint: disable=W0402 import subprocess import sys import tempfile import time -import traceback import types import urlparse @@ -171,7 +170,8 @@ def fork_cb(child_cb, *args): child_cb(*args) os._exit(0) # pylint: disable=W0212 except: - logexc(LOG, "Failed forking and calling callback %s", obj_name(child_cb)) + logexc(LOG, ("Failed forking and" + " calling callback %s"), obj_name(child_cb)) os._exit(1) # pylint: disable=W0212 else: LOG.debug("Forked child %s who will run callback %s", @@ -549,10 +549,11 @@ def load_yaml(blob, default=None, allowed=(dict,)): converted = yaml.load(blob) if not isinstance(converted, allowed): # Yes this will just be caught, but thats ok for now... - raise TypeError("Yaml load allows %s root types, but got %s instead" % + raise TypeError(("Yaml load allows %s root types," + " but got %s instead") % (allowed, obj_name(converted))) loaded = converted - except (yaml.YAMLError, TypeError, ValueError) as exc: + except (yaml.YAMLError, TypeError, ValueError): logexc(LOG, "Failed loading yaml blob") return loaded @@ -833,15 +834,12 @@ def find_devs_with(criteria=None, oformat='device', options.append(path) cmd = blk_id_cmd + options (out, _err) = subp(cmd) - if path: - return out.strip() - else: - entries = [] - for line in out.splitlines(): - line = line.strip() - if line: - entries.append(line) - return entries + entries = [] + for line in out.splitlines(): + line = line.strip() + if line: + entries.append(line) + return entries def load_file(fname, read_cb=None, quiet=False): @@ -1109,7 +1107,7 @@ def mount_cb(device, callback, data=None, rw=False, mtype=None): def get_builtin_cfg(): # Deep copy so that others can't modify - return copy.deepcopy(CFG_BUILTIN) + return obj_copy.deepcopy(CFG_BUILTIN) def sym_link(source, link): @@ -1140,16 +1138,14 @@ def time_rfc2822(): def uptime(): + uptime_str = '??' try: - uptimef = load_file("/proc/uptime").strip() - if not uptimef: - uptime = 'na' - else: - uptime = uptimef.split()[0] + contents = load_file("/proc/uptime").strip() + if contents: + uptime_str = contents.split()[0] except: logexc(LOG, "Unable to read uptime from /proc/uptime") - uptime = '??' - return uptime + return uptime_str def ensure_file(path): @@ -1261,7 +1257,8 @@ def shellify(cmdlist, add_header=True): content = "%s%s\n" % (content, args) else: raise RuntimeError(("Unable to shellify type %s" - " which is not a list or string") % (obj_name(args))) + " which is not a list or string") + % (obj_name(args))) LOG.debug("Shellified %s to %s", cmdlist, content) return content @@ -1275,8 +1272,7 @@ def is_container(): try: # try to run a helper program. if it returns true/zero # then we're inside a container. otherwise, no - cmd = [helper] - subp(cmd, allowed_rc=[0]) + subp([helper]) return True except (IOError, OSError): pass -- cgit v1.2.3 From 95573f9d7aa53d9f4c5ba5e969775f9bc59cb5ae Mon Sep 17 00:00:00 2001 From: Joshua Harlow Date: Sat, 16 Jun 2012 08:55:39 -0700 Subject: Adjust how config is extracted in that it now can be extracted via 3 different modes. 1. Restricted - which doesn't give back the system info (used by handlers/transforms/public cfg api) 2. System - which only gives back the system info (used by distro class) 3. Paths - gives back only the system/path info (used by the path class) --- cloudinit/stages.py | 29 ++++++++++++++++------------- 1 file changed, 16 insertions(+), 13 deletions(-) (limited to 'cloudinit/stages.py') diff --git a/cloudinit/stages.py b/cloudinit/stages.py index b9076881..63b7cf12 100644 --- a/cloudinit/stages.py +++ b/cloudinit/stages.py @@ -76,15 +76,8 @@ class Init(object): @property def distro(self): if not self._distro: - d_cfg = util.get_cfg_by_path(self.cfg, ('system_info'), {}) - # Ensure its a dictionary - if not isinstance(d_cfg, (dict)): - d_cfg = {} - # Ensure not modified indirectly - d_cfg = copy.deepcopy(d_cfg) - # Remove this since its path config, not distro config - d_cfg.pop('paths', None) # Try to find the right class to use + d_cfg = self._extract_cfg('system') distro_name = d_cfg.pop('distro', 'ubuntu') distro_cls = distros.fetch(distro_name) LOG.debug("Using distro class %s", distro_cls) @@ -95,19 +88,29 @@ class Init(object): @property def cfg(self): + return self._extract_cfg('restricted') + + def _extract_cfg(self, restriction): # None check so that we don't keep on re-loading if empty if self._cfg is None: self._cfg = self._read_cfg() LOG.debug("Loading init config %s", self._cfg) - return self._cfg + # Nobody gets the real config + ocfg = copy.deepcopy(self._cfg) + if restriction == 'restricted': + ocfg.pop('system_info', None) + elif restriction == 'system': + ocfg = util.get_cfg_by_path(ocfg, ('system_info',), {}) + elif restriction == 'paths': + ocfg = util.get_cfg_by_path(ocfg, ('system_info', 'paths'), {}) + if not isinstance(ocfg, (dict)): + ocfg = {} + return ocfg @property def paths(self): if not self._paths: - path_info = util.get_cfg_by_path(self.cfg, - ('system_info', 'paths'), {}) - # Ensure not modified indirectly - path_info = copy.deepcopy(path_info) + path_info = self._extract_cfg('paths') self._paths = helpers.Paths(path_info, self.datasource) return self._paths -- cgit v1.2.3 From 1b5e8117d862743f8f66cd27462965cc6c89af90 Mon Sep 17 00:00:00 2001 From: Joshua Harlow Date: Sat, 16 Jun 2012 09:31:52 -0700 Subject: Rename to fixup transform/fixup handler/form transform name instead of form mod* --- cloudinit/stages.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) (limited to 'cloudinit/stages.py') diff --git a/cloudinit/stages.py b/cloudinit/stages.py index 63b7cf12..9c772243 100644 --- a/cloudinit/stages.py +++ b/cloudinit/stages.py @@ -319,7 +319,7 @@ class Init(object): potential_handlers = util.find_modules(cdir) for (fname, modname) in potential_handlers.iteritems(): try: - mod = ud.fixup_module(importer.import_module(modname)) + mod = ud.fixup_handler(importer.import_module(modname)) types = c_handlers.register(mod) LOG.debug("Added handler for %s from %s", types, fname) except: @@ -445,7 +445,7 @@ class Transforms(object): raw_name = raw_mod['mod'] freq = raw_mod.get('freq') run_args = raw_mod.get('args') or [] - mod_name = transforms.form_module_name(raw_name) + mod_name = transforms.form_transform_name(raw_name) if not mod_name: continue if freq and freq not in FREQUENCIES: @@ -453,7 +453,7 @@ class Transforms(object): " has an unknown frequency %s"), raw_name, freq) # Reset it so when ran it will get set to a known value freq = None - mod = transforms.fixup_module(importer.import_module(mod_name)) + mod = transforms.fixup_transform(importer.import_module(mod_name)) mostly_mods.append([mod, raw_name, freq, run_args]) return mostly_mods -- cgit v1.2.3 From 1e6455899af5532314d7244bbc660a0e34a84eff Mon Sep 17 00:00:00 2001 From: Joshua Harlow Date: Sat, 16 Jun 2012 09:51:45 -0700 Subject: Content handlers does not need the paths variable anymore so removed that. --- cloudinit/helpers.py | 3 +-- cloudinit/stages.py | 2 +- 2 files changed, 2 insertions(+), 3 deletions(-) (limited to 'cloudinit/stages.py') diff --git a/cloudinit/helpers.py b/cloudinit/helpers.py index 5e47794b..76faa1d5 100644 --- a/cloudinit/helpers.py +++ b/cloudinit/helpers.py @@ -160,8 +160,7 @@ class Runners(object): class ContentHandlers(object): - def __init__(self, paths): - self.paths = paths + def __init__(self): self.registered = {} def __contains__(self, item): diff --git a/cloudinit/stages.py b/cloudinit/stages.py index 9c772243..5dc289ea 100644 --- a/cloudinit/stages.py +++ b/cloudinit/stages.py @@ -313,7 +313,7 @@ class Init(object): ud_obj = self.datasource.get_userdata() # This keeps track of all the active handlers - c_handlers = helpers.ContentHandlers(paths=self.paths) + c_handlers = helpers.ContentHandlers() # Add handlers in cdir potential_handlers = util.find_modules(cdir) -- cgit v1.2.3 From 1ddc33d3091d5518e29500d7db434d74182ce0f1 Mon Sep 17 00:00:00 2001 From: Joshua Harlow Date: Sat, 16 Jun 2012 10:58:30 -0700 Subject: After the moving of items to handler/__init__ reflect those references here. --- cloudinit/stages.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) (limited to 'cloudinit/stages.py') diff --git a/cloudinit/stages.py b/cloudinit/stages.py index 5dc289ea..9d8ff2bb 100644 --- a/cloudinit/stages.py +++ b/cloudinit/stages.py @@ -34,6 +34,7 @@ except ImportError: from cloudinit.settings import (OLD_CLOUD_CONFIG) from cloudinit.settings import (PER_INSTANCE, FREQUENCIES) +from cloudinit import handlers from cloudinit.handlers import boot_hook as bh_part from cloudinit.handlers import cloud_config as cc_part from cloudinit.handlers import shell_script as ss_part @@ -46,7 +47,6 @@ from cloudinit import importer from cloudinit import log as logging from cloudinit import sources from cloudinit import transforms -from cloudinit import user_data as ud from cloudinit import util LOG = logging.getLogger(__name__) @@ -319,7 +319,7 @@ class Init(object): potential_handlers = util.find_modules(cdir) for (fname, modname) in potential_handlers.iteritems(): try: - mod = ud.fixup_handler(importer.import_module(modname)) + mod = handlers.fixup_handler(importer.import_module(modname)) types = c_handlers.register(mod) LOG.debug("Added handler for %s from %s", types, fname) except: @@ -338,7 +338,7 @@ class Init(object): for (_ctype, mod) in c_handlers.iteritems(): if mod in called: continue - ud.call_begin(mod, data, frequency) + handlers.call_begin(mod, data, frequency) called.append(mod) # Walk the user data @@ -352,14 +352,14 @@ class Init(object): # names... 'handlercount': 0, } - ud.walk(ud_obj, ud.walker_callback, data=part_data) + handlers.walk(ud_obj, handlers.walker_callback, data=part_data) # Give callbacks opportunity to finalize called = [] for (_ctype, mod) in c_handlers.iteritems(): if mod in called: continue - ud.call_end(mod, data, frequency) + handlers.call_end(mod, data, frequency) called.append(mod) @@ -481,7 +481,7 @@ class Transforms(object): run_name = "config-%s" % (name) c_cloud.run(run_name, mod.handle, func_args, freq=freq) except Exception as e: - util.logexc(LOG, "Running %s failed", mod) + util.logexc(LOG, "Running %s (%s) failed", name, mod) failures.append((name, e)) return failures -- cgit v1.2.3 From 1a803c9f1f55095e1cf69bd4ca12bda0299b64b4 Mon Sep 17 00:00:00 2001 From: Joshua Harlow Date: Sat, 16 Jun 2012 12:33:31 -0700 Subject: Have the top level distro class take paths instead of a runner. This allows the following: 1. Let the ubuntu subclass construct its own runner with those paths (since not every subclass may want it) Adjust the base class + subclass to reflect this, adjust stages as well to reflect the constructor changes. --- cloudinit/distros/__init__.py | 4 ++-- cloudinit/distros/ubuntu.py | 8 ++++++++ cloudinit/stages.py | 12 +++++------- 3 files changed, 15 insertions(+), 9 deletions(-) (limited to 'cloudinit/stages.py') diff --git a/cloudinit/distros/__init__.py b/cloudinit/distros/__init__.py index e85e702e..0ee7f06b 100644 --- a/cloudinit/distros/__init__.py +++ b/cloudinit/distros/__init__.py @@ -42,8 +42,8 @@ class Distro(object): __metaclass__ = abc.ABCMeta - def __init__(self, name, cfg, runner): - self._runner = runner + def __init__(self, name, cfg, paths): + self._paths = paths self._cfg = cfg self.name = name diff --git a/cloudinit/distros/ubuntu.py b/cloudinit/distros/ubuntu.py index ad12400a..9b743b55 100644 --- a/cloudinit/distros/ubuntu.py +++ b/cloudinit/distros/ubuntu.py @@ -23,6 +23,7 @@ import os from cloudinit import distros +from cloudinit import helpers from cloudinit import log as logging from cloudinit import util @@ -33,6 +34,13 @@ LOG = logging.getLogger(__name__) class Distro(distros.Distro): + def __init__(self, name, cfg, paths): + distros.Distro.__init__(self, name, cfg, paths) + # This will be used to restrict certain + # calls from repeatly happening (when they + # should only happen say once per instance...) + self._runner = helpers.Runners(paths) + def install_packages(self, pkglist): self._update_package_sources() self._apt_get('install', pkglist) diff --git a/cloudinit/stages.py b/cloudinit/stages.py index 9d8ff2bb..8fa9d6d3 100644 --- a/cloudinit/stages.py +++ b/cloudinit/stages.py @@ -77,13 +77,11 @@ class Init(object): def distro(self): if not self._distro: # Try to find the right class to use - d_cfg = self._extract_cfg('system') - distro_name = d_cfg.pop('distro', 'ubuntu') - distro_cls = distros.fetch(distro_name) - LOG.debug("Using distro class %s", distro_cls) - distro = distro_cls(distro_name, d_cfg, - helpers.Runners(self.paths)) - self._distro = distro + scfg = self._extract_cfg('system') + name = scfg.pop('distro', 'ubuntu') + cls = distros.fetch(name) + LOG.debug("Using distro class %s", cls) + self._distro = cls(name, scfg, self.paths) return self._distro @property -- cgit v1.2.3 From 457b9998d760150efb17658f9a0fd4816417577e Mon Sep 17 00:00:00 2001 From: Joshua Harlow Date: Sat, 16 Jun 2012 13:11:22 -0700 Subject: Pass in the datasource as a option, instead of the lower level instance id. This allows for others to use datasource functions if they desire to instead of being restricted. +1 for future use ;) --- cloudinit/handlers/boot_hook.py | 6 ++++-- cloudinit/stages.py | 2 +- 2 files changed, 5 insertions(+), 3 deletions(-) (limited to 'cloudinit/stages.py') diff --git a/cloudinit/handlers/boot_hook.py b/cloudinit/handlers/boot_hook.py index 10f60b8d..fa675f09 100644 --- a/cloudinit/handlers/boot_hook.py +++ b/cloudinit/handlers/boot_hook.py @@ -32,10 +32,12 @@ LOG = logging.getLogger(__name__) class BootHookPartHandler(handlers.Handler): - def __init__(self, paths, instance_id, **_kwargs): + def __init__(self, paths, datasource, **_kwargs): handlers.Handler.__init__(self, PER_ALWAYS) self.boothook_dir = paths.get_ipath("boothooks") - self.instance_id = instance_id + self.instance_id = None + if datasource: + self.instance_id = datasource.get_instance_id() def list_types(self): return [ diff --git a/cloudinit/stages.py b/cloudinit/stages.py index 8fa9d6d3..2931830c 100644 --- a/cloudinit/stages.py +++ b/cloudinit/stages.py @@ -287,7 +287,7 @@ class Init(object): def _default_userdata_handlers(self): opts = { 'paths': self.paths, - 'instance_id': self.datasource.get_instance_id(), + 'datasource': self.datasource, } # TODO Hmmm, should we dynamically import these?? def_handlers = [ -- cgit v1.2.3 From a1f20dff76b00c7502f005c2b597f49c89cc2a09 Mon Sep 17 00:00:00 2001 From: Joshua Harlow Date: Sat, 16 Jun 2012 13:23:32 -0700 Subject: Check instance id against none, and not just empty/false/0/none since 0 or empty might be valid --- cloudinit/handlers/boot_hook.py | 2 +- cloudinit/stages.py | 8 ++++---- 2 files changed, 5 insertions(+), 5 deletions(-) (limited to 'cloudinit/stages.py') diff --git a/cloudinit/handlers/boot_hook.py b/cloudinit/handlers/boot_hook.py index fa675f09..456b8020 100644 --- a/cloudinit/handlers/boot_hook.py +++ b/cloudinit/handlers/boot_hook.py @@ -63,7 +63,7 @@ class BootHookPartHandler(handlers.Handler): filepath = self._write_part(payload, filename) try: env = os.environ.copy() - if self.instance_id: + if self.instance_id is not None: env['INSTANCE_ID'] = str(self.instance_id) util.subp([filepath], env=env) except util.ProcessExecutionError: diff --git a/cloudinit/stages.py b/cloudinit/stages.py index 2931830c..c2d78a78 100644 --- a/cloudinit/stages.py +++ b/cloudinit/stages.py @@ -233,13 +233,13 @@ class Init(object): # Write what the datasource was and is.. ds = "%s: %s" % (util.obj_name(self.datasource), self.datasource) - previous_ds = '' + previous_ds = None ds_fn = os.path.join(idir, 'datasource') try: previous_ds = util.load_file(ds_fn).strip() except Exception: pass - if not previous_ds: + if previous_ds is None: # TODO: ?? is this right previous_ds = ds util.write_file(ds_fn, "%s\n" % ds) @@ -248,14 +248,14 @@ class Init(object): # What the instance id was and is... iid = self.datasource.get_instance_id() - previous_iid = '' + previous_iid = None p_iid_fn = os.path.join(dp, 'previous-instance-id') c_iid_fn = os.path.join(dp, 'instance-id') try: previous_iid = util.load_file(p_iid_fn).strip() except Exception: pass - if not previous_iid: + if previous_iid is None: # TODO: ?? is this right previous_iid = iid util.write_file(c_iid_fn, "%s\n" % iid) -- cgit v1.2.3 From 9249e20d87ad7827405dfd04ec98c72955044d7f Mon Sep 17 00:00:00 2001 From: Joshua Harlow Date: Sat, 16 Jun 2012 13:25:15 -0700 Subject: Revert that last change for this file, it should be a string that is non-empty so checking against that is fine --- cloudinit/stages.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'cloudinit/stages.py') diff --git a/cloudinit/stages.py b/cloudinit/stages.py index c2d78a78..05b59852 100644 --- a/cloudinit/stages.py +++ b/cloudinit/stages.py @@ -239,7 +239,7 @@ class Init(object): previous_ds = util.load_file(ds_fn).strip() except Exception: pass - if previous_ds is None: + if not previous_ds: # TODO: ?? is this right previous_ds = ds util.write_file(ds_fn, "%s\n" % ds) @@ -255,7 +255,7 @@ class Init(object): previous_iid = util.load_file(p_iid_fn).strip() except Exception: pass - if previous_iid is None: + if not previous_iid: # TODO: ?? is this right previous_iid = iid util.write_file(c_iid_fn, "%s\n" % iid) -- cgit v1.2.3 From fc026d6e7dd51ea8c04e081c821e086f89bf6291 Mon Sep 17 00:00:00 2001 From: Joshua Harlow Date: Sat, 16 Jun 2012 13:27:10 -0700 Subject: Should be loaded not loading --- cloudinit/stages.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'cloudinit/stages.py') diff --git a/cloudinit/stages.py b/cloudinit/stages.py index 05b59852..b0eef928 100644 --- a/cloudinit/stages.py +++ b/cloudinit/stages.py @@ -92,7 +92,7 @@ class Init(object): # None check so that we don't keep on re-loading if empty if self._cfg is None: self._cfg = self._read_cfg() - LOG.debug("Loading init config %s", self._cfg) + LOG.debug("Loaded init config %s", self._cfg) # Nobody gets the real config ocfg = copy.deepcopy(self._cfg) if restriction == 'restricted': -- cgit v1.2.3 From 1154876da44e1fd4516807637c317ce9828fe08b Mon Sep 17 00:00:00 2001 From: Joshua Harlow Date: Sat, 16 Jun 2012 13:31:48 -0700 Subject: Fix logic issue with reading previous file for previous instance id, when the current file should be looked at instead --- cloudinit/stages.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) (limited to 'cloudinit/stages.py') diff --git a/cloudinit/stages.py b/cloudinit/stages.py index b0eef928..ddac2ac2 100644 --- a/cloudinit/stages.py +++ b/cloudinit/stages.py @@ -249,17 +249,17 @@ class Init(object): # What the instance id was and is... iid = self.datasource.get_instance_id() previous_iid = None - p_iid_fn = os.path.join(dp, 'previous-instance-id') c_iid_fn = os.path.join(dp, 'instance-id') try: - previous_iid = util.load_file(p_iid_fn).strip() + previous_iid = util.load_file(c_iid_fn).strip() except Exception: pass if not previous_iid: # TODO: ?? is this right previous_iid = iid util.write_file(c_iid_fn, "%s\n" % iid) - util.write_file(p_iid_fn, "%s\n" % previous_iid) + util.write_file(os.path.join(dp, 'previous-instance-id'), + "%s\n" % previous_iid) return iid def fetch(self): -- cgit v1.2.3 From a5e15033e70ec3cea877879d0070480f6a7efe49 Mon Sep 17 00:00:00 2001 From: Joshua Harlow Date: Sat, 16 Jun 2012 13:33:06 -0700 Subject: Comments about checking if this is right, no longer needed --- cloudinit/stages.py | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) (limited to 'cloudinit/stages.py') diff --git a/cloudinit/stages.py b/cloudinit/stages.py index ddac2ac2..5a00eae3 100644 --- a/cloudinit/stages.py +++ b/cloudinit/stages.py @@ -240,7 +240,6 @@ class Init(object): except Exception: pass if not previous_ds: - # TODO: ?? is this right previous_ds = ds util.write_file(ds_fn, "%s\n" % ds) util.write_file(os.path.join(dp, 'previous-datasource'), @@ -249,17 +248,16 @@ class Init(object): # What the instance id was and is... iid = self.datasource.get_instance_id() previous_iid = None - c_iid_fn = os.path.join(dp, 'instance-id') + iid_fn = os.path.join(dp, 'instance-id') try: - previous_iid = util.load_file(c_iid_fn).strip() + previous_iid = util.load_file(iid_fn).strip() except Exception: pass if not previous_iid: - # TODO: ?? is this right previous_iid = iid - util.write_file(c_iid_fn, "%s\n" % iid) + util.write_file(iid_fn, "%s\n" % iid) util.write_file(os.path.join(dp, 'previous-instance-id'), - "%s\n" % previous_iid) + "%s\n" % (previous_iid)) return iid def fetch(self): -- cgit v1.2.3 From 71e8036e6534afbb871e864dba988541652dc400 Mon Sep 17 00:00:00 2001 From: Joshua Harlow Date: Sat, 16 Jun 2012 20:04:13 -0700 Subject: Not config anymore, now a transform --- cloudinit/stages.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'cloudinit/stages.py') diff --git a/cloudinit/stages.py b/cloudinit/stages.py index 5a00eae3..24a4d23f 100644 --- a/cloudinit/stages.py +++ b/cloudinit/stages.py @@ -474,7 +474,7 @@ class Transforms(object): func_args = [name, copy.deepcopy(self.cfg), c_cloud, transforms.LOG, args] # This name will affect the semaphore name created - run_name = "config-%s" % (name) + run_name = "transform-%s" % (name) c_cloud.run(run_name, mod.handle, func_args, freq=freq) except Exception as e: util.logexc(LOG, "Running %s (%s) failed", name, mod) -- cgit v1.2.3 From 0bfb3f5e5b4475c1dcbdb5db95bfce97d97ffc6a Mon Sep 17 00:00:00 2001 From: Joshua Harlow Date: Sat, 16 Jun 2012 20:16:19 -0700 Subject: Rename user data obj, which is really a user data message object and add comments as to what the data items are used for --- cloudinit/stages.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) (limited to 'cloudinit/stages.py') diff --git a/cloudinit/stages.py b/cloudinit/stages.py index 24a4d23f..70f2bcc9 100644 --- a/cloudinit/stages.py +++ b/cloudinit/stages.py @@ -306,7 +306,7 @@ class Init(object): sys.path.insert(0, idir) # Ensure datasource fetched before activation (just incase) - ud_obj = self.datasource.get_userdata() + user_data_msg = self.datasource.get_userdata() # This keeps track of all the active handlers c_handlers = helpers.ContentHandlers() @@ -340,15 +340,17 @@ class Init(object): # Walk the user data part_data = { 'handlers': c_handlers, + # Any new handlers that are encountered get writen here 'handlerdir': idir, - 'data': data, + 'data': data, + # The default frequency if handlers don't have one 'frequency': frequency, # This will be used when new handlers are found # to help write there contents to files with numbered # names... 'handlercount': 0, } - handlers.walk(ud_obj, handlers.walker_callback, data=part_data) + handlers.walk(user_data_msg, handlers.walker_callback, data=part_data) # Give callbacks opportunity to finalize called = [] -- cgit v1.2.3 From a2e75f6d01688026b7ccfe5d71a11fff6205a287 Mon Sep 17 00:00:00 2001 From: Joshua Harlow Date: Sat, 16 Jun 2012 20:44:58 -0700 Subject: 1. Fix the welcome_msg name to welcome_message (oops) 2. Adjust that name in the cloud.cfg 3. Add in the ability to specify a dictionary that is the transform instead of a list which makes the config look nicer when users have custom transforms with arguments and such. --- cloudinit/stages.py | 12 +++++++ cloudinit/transforms/welcome_message.py | 64 +++++++++++++++++++++++++++++++++ cloudinit/transforms/welcome_msg.py | 64 --------------------------------- config/cloud.cfg | 12 ++++--- 4 files changed, 84 insertions(+), 68 deletions(-) create mode 100644 cloudinit/transforms/welcome_message.py delete mode 100644 cloudinit/transforms/welcome_msg.py (limited to 'cloudinit/stages.py') diff --git a/cloudinit/stages.py b/cloudinit/stages.py index 70f2bcc9..ba6cb915 100644 --- a/cloudinit/stages.py +++ b/cloudinit/stages.py @@ -431,6 +431,18 @@ class Transforms(object): contents['args'] = item[2:] if contents: module_list.append(contents) + elif isinstance(item, (dict)): + contents = {} + valid = False + if 'name' in item: + contents['mod'] = item['name'].strip() + valid = True + if 'frequency' in item: + contents['freq'] = item['frequency'].strip() + if 'args' in item: + contents['args'] = item['args'] or [] + if contents and valid: + module_list.append(contents) else: raise TypeError(("Failed to read '%s' item in config," " unknown type %s") % diff --git a/cloudinit/transforms/welcome_message.py b/cloudinit/transforms/welcome_message.py new file mode 100644 index 00000000..04691d21 --- /dev/null +++ b/cloudinit/transforms/welcome_message.py @@ -0,0 +1,64 @@ +# vi: ts=4 expandtab +# +# Copyright (C) 2012 Canonical Ltd. +# Copyright (C) 2012 Hewlett-Packard Development Company, L.P. +# Copyright (C) 2012 Yahoo! Inc. +# +# Author: Scott Moser +# Author: Juerg Haefliger +# Author: Joshua Harlow +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 3, as +# published by the Free Software Foundation. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +from cloudinit.settings import PER_ALWAYS + +from cloudinit import templater +from cloudinit import util +from cloudinit import version + +import sys + +welcome_message_def = ("Cloud-init v. {{version}} starting stage {{stage}} at " + "{{timestamp}}. Up {{uptime}} seconds.") + + +frequency = PER_ALWAYS + + +def handle(_name, cfg, cloud, log, args): + + welcome_msg = util.get_cfg_option_str(cfg, "welcome_msg") + if not welcome_msg: + tpl_fn = cloud.get_template_filename("welcome_msg") + if tpl_fn: + welcome_msg = util.load_file(tpl_fn) + + if not welcome_msg: + welcome_msg = welcome_message_def + + stage = "??" + if args: + stage = args[0] + + tpl_params = { + 'stage': stage, + 'version': version.version_string(), + 'uptime': util.uptime(), + 'timestamp': util.time_rfc2822(), + } + try: + contents = templater.render_string(welcome_msg, tpl_params) + # TODO use log or sys.stderr?? + sys.stderr.write("%s\n" % (contents)) + except: + util.logexc(log, "Failed to render welcome message template") diff --git a/cloudinit/transforms/welcome_msg.py b/cloudinit/transforms/welcome_msg.py deleted file mode 100644 index 04691d21..00000000 --- a/cloudinit/transforms/welcome_msg.py +++ /dev/null @@ -1,64 +0,0 @@ -# vi: ts=4 expandtab -# -# Copyright (C) 2012 Canonical Ltd. -# Copyright (C) 2012 Hewlett-Packard Development Company, L.P. -# Copyright (C) 2012 Yahoo! Inc. -# -# Author: Scott Moser -# Author: Juerg Haefliger -# Author: Joshua Harlow -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License version 3, as -# published by the Free Software Foundation. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program. If not, see . - -from cloudinit.settings import PER_ALWAYS - -from cloudinit import templater -from cloudinit import util -from cloudinit import version - -import sys - -welcome_message_def = ("Cloud-init v. {{version}} starting stage {{stage}} at " - "{{timestamp}}. Up {{uptime}} seconds.") - - -frequency = PER_ALWAYS - - -def handle(_name, cfg, cloud, log, args): - - welcome_msg = util.get_cfg_option_str(cfg, "welcome_msg") - if not welcome_msg: - tpl_fn = cloud.get_template_filename("welcome_msg") - if tpl_fn: - welcome_msg = util.load_file(tpl_fn) - - if not welcome_msg: - welcome_msg = welcome_message_def - - stage = "??" - if args: - stage = args[0] - - tpl_params = { - 'stage': stage, - 'version': version.version_string(), - 'uptime': util.uptime(), - 'timestamp': util.time_rfc2822(), - } - try: - contents = templater.render_string(welcome_msg, tpl_params) - # TODO use log or sys.stderr?? - sys.stderr.write("%s\n" % (contents)) - except: - util.logexc(log, "Failed to render welcome message template") diff --git a/config/cloud.cfg b/config/cloud.cfg index 147e0500..0e431962 100644 --- a/config/cloud.cfg +++ b/config/cloud.cfg @@ -18,9 +18,13 @@ preserve_hostname: false # timeout: 5 # (defaults to 50 seconds) # max_wait: 10 # (defaults to 120 seconds) -# The transform modules that run in the 'init' stage +# The transform that run in the 'init' stage cloud_init_modules: - - welcome_msg +# This is the hash way of specifying a transform + - name: welcome_message +# This argument list will get passed to the transform when activated + args: + - init - bootcmd - resizefs - set_hostname @@ -30,7 +34,7 @@ cloud_init_modules: - rsyslog - ssh - # The transform modules that run in the 'config' stage + # The transforms that run in the 'config' stage cloud_config_modules: - mounts - ssh-import-id @@ -49,7 +53,7 @@ cloud_config_modules: - runcmd - byobu -# The transform modules that run in the 'final' stage +# The transforms that run in the 'final' stage cloud_final_modules: - rightscale_userdata - scripts-per-once -- cgit v1.2.3 From b84432fdd390e3bd239dda997e6fa8fad086e2bd Mon Sep 17 00:00:00 2001 From: Joshua Harlow Date: Mon, 18 Jun 2012 17:32:03 -0700 Subject: 1. Revert the 'transform-' template back to 'config-' template for now. 2. Make the init class have a public 'read_cfg' which can be used to force cfg loading to occur (instead of having to go through the cfg property) a. This is a more 'public' way of forcing config to load, without exposing the config itself. --- cloudinit/stages.py | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) (limited to 'cloudinit/stages.py') diff --git a/cloudinit/stages.py b/cloudinit/stages.py index ba6cb915..a713bd1f 100644 --- a/cloudinit/stages.py +++ b/cloudinit/stages.py @@ -89,10 +89,8 @@ class Init(object): return self._extract_cfg('restricted') def _extract_cfg(self, restriction): - # None check so that we don't keep on re-loading if empty - if self._cfg is None: - self._cfg = self._read_cfg() - LOG.debug("Loaded init config %s", self._cfg) + # Ensure actually read + self.read_cfg() # Nobody gets the real config ocfg = copy.deepcopy(self._cfg) if restriction == 'restricted': @@ -154,6 +152,12 @@ class Init(object): g = None util.chownbyname(log_file, u, g) + def read_cfg(self): + # None check so that we don't keep on re-loading if empty + if self._cfg is None: + self._cfg = self._read_cfg() + LOG.debug("Loaded init config %s", self._cfg) + def _read_cfg(self): b_config = util.get_builtin_cfg() try: @@ -488,7 +492,7 @@ class Transforms(object): func_args = [name, copy.deepcopy(self.cfg), c_cloud, transforms.LOG, args] # This name will affect the semaphore name created - run_name = "transform-%s" % (name) + run_name = "config-%s" % (name) c_cloud.run(run_name, mod.handle, func_args, freq=freq) except Exception as e: util.logexc(LOG, "Running %s (%s) failed", name, mod) -- cgit v1.2.3 From 55771ac86ae46232aabc9ec7419e9da0782f9986 Mon Sep 17 00:00:00 2001 From: Joshua Harlow Date: Mon, 18 Jun 2012 21:08:21 -0700 Subject: 1. Cleanup the rm cur instance link variable name to match more of what it is 2. Allow on config reading the ability to pass in config files that over-ride the search for configs (useful for testing...) --- cloudinit/stages.py | 31 ++++++++++++++++++------------- 1 file changed, 18 insertions(+), 13 deletions(-) (limited to 'cloudinit/stages.py') diff --git a/cloudinit/stages.py b/cloudinit/stages.py index a713bd1f..2c610ce5 100644 --- a/cloudinit/stages.py +++ b/cloudinit/stages.py @@ -126,14 +126,14 @@ class Init(object): ] return initial_dirs - def purge_cache(self, rmcur=True): - rmlist = [] + def purge_cache(self, rm_instance_lnk=True): + rm_list = [] rmlist.append(self.paths.boot_finished) - if rmcur: - rmlist.append(self.paths.instance_link) - for f in rmlist: + if rm_instance_lnk: + rm_list.append(self.paths.instance_link) + for f in rm_list: util.del_file(f) - return len(rmlist) + return len(rm_list) def initialize(self): self._initialize_filesystem() @@ -152,19 +152,24 @@ class Init(object): g = None util.chownbyname(log_file, u, g) - def read_cfg(self): + def read_cfg(self, extra_fns=None): # None check so that we don't keep on re-loading if empty if self._cfg is None: - self._cfg = self._read_cfg() + self._cfg = self._read_cfg(extra_fns) LOG.debug("Loaded init config %s", self._cfg) - def _read_cfg(self): - b_config = util.get_builtin_cfg() + def _read_cfg(self, extra_fns): + builtin_cfg = util.get_builtin_cfg() try: - conf = util.get_base_cfg() + conf = util.get_base_cfg(builtin=builtin_cfg) except Exception: - conf = b_config - return util.mergedict(conf, self._read_cfg_old()) + conf = builtin_cfg + m_cfg = util.mergedict(conf, self._read_cfg_old()) + if extra_fns: + for fn in extra_fns: + # Any extras over-ride the existing configs + m_cfg = util.mergedict(util.read_conf(fn), m_cfg) + return m_cfg def _restore_from_cache(self): pickled_fn = self.paths.get_ipath_cur('obj_pkl') -- cgit v1.2.3 From f06b5cbf6187dd90b90cc24d3f3c087fe51a6d07 Mon Sep 17 00:00:00 2001 From: Joshua Harlow Date: Tue, 19 Jun 2012 11:06:32 -0700 Subject: Fix pylint error about the rmlist variable name being mis-named. --- cloudinit/stages.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'cloudinit/stages.py') diff --git a/cloudinit/stages.py b/cloudinit/stages.py index 2c610ce5..6eb211db 100644 --- a/cloudinit/stages.py +++ b/cloudinit/stages.py @@ -128,7 +128,7 @@ class Init(object): def purge_cache(self, rm_instance_lnk=True): rm_list = [] - rmlist.append(self.paths.boot_finished) + rm_list.append(self.paths.boot_finished) if rm_instance_lnk: rm_list.append(self.paths.instance_link) for f in rm_list: -- cgit v1.2.3 From e731e0b9a06f27d705a0635e848e68f00f2b16cc Mon Sep 17 00:00:00 2001 From: Joshua Harlow Date: Tue, 19 Jun 2012 15:39:54 -0700 Subject: 1. Cleanup variable names to match more of the pythonic underscore pattern 2. Seperate config loading from the actual final 'merging' process. a. A util function will now merge multiple config dictionaries after they have all been loaded instead of loading and merging at the same time, which can get confusing to follow. --- cloudinit/stages.py | 87 +++++++++++++++++++++++++++++------------------------ cloudinit/util.py | 11 +++++++ 2 files changed, 58 insertions(+), 40 deletions(-) (limited to 'cloudinit/stages.py') diff --git a/cloudinit/stages.py b/cloudinit/stages.py index 6eb211db..558de035 100644 --- a/cloudinit/stages.py +++ b/cloudinit/stages.py @@ -156,20 +156,27 @@ class Init(object): # None check so that we don't keep on re-loading if empty if self._cfg is None: self._cfg = self._read_cfg(extra_fns) - LOG.debug("Loaded init config %s", self._cfg) + LOG.debug("Loaded %s config %s", util.obj_name(self), self._cfg) def _read_cfg(self, extra_fns): - builtin_cfg = util.get_builtin_cfg() - try: - conf = util.get_base_cfg(builtin=builtin_cfg) - except Exception: - conf = builtin_cfg - m_cfg = util.mergedict(conf, self._read_cfg_old()) + # Read extra files provided (if any) + i_cfgs = [] if extra_fns: for fn in extra_fns: - # Any extras over-ride the existing configs - m_cfg = util.mergedict(util.read_conf(fn), m_cfg) - return m_cfg + try: + fn_cfg = util.read_conf(fn) + i_cfgs.append(fn_cfg) + except: + util.logexc(LOG, ("Failed loading of additional" + " configuration from %s"), fn) + # Now read in the built-in + base + old + try: + conf = util.get_base_cfg(builtin=util.get_builtin_cfg()) + except Exception: + conf = util.get_builtin_cfg() + i_cfgs.append(conf) + i_cfgs.append(self._read_cfg_old()) + return util.mergemanydict(i_cfgs) def _restore_from_cache(self): pickled_fn = self.paths.get_ipath_cur('obj_pkl') @@ -371,46 +378,46 @@ class Init(object): class Transforms(object): - def __init__(self, init, cfgfile=None): + def __init__(self, init, cfg_files=None): self.datasource = init.fetch() - self.cfgfile = cfgfile - self.basecfg = copy.deepcopy(init.cfg) + self.cfg_files = cfg_files + self.base_cfg = copy.deepcopy(init.cfg) self.init = init # Created on first use - self._cachedcfg = None + self._cached_cfg = None @property def cfg(self): - if self._cachedcfg is None: - self._cachedcfg = self._get_config(self.cfgfile) - LOG.debug("Loading module config %s", self._cachedcfg) - return self._cachedcfg + # None check to avoid empty case + if self._cached_cfg is None: + self._cached_cfg = self._get_config() + LOG.debug("Loading %s config %s", + util.obj_name(self), self._cached_cfg) + return self._cached_cfg + + def _get_config(self): + t_cfgs = [] + if self.cfg_files: + for fn in self.cfg_files: + try: + t_cfgs.append(util.read_conf(fn)) + except: + util.logexc(LOG, ("Failed loading of configuration" + " from %s"), fn) - def _get_config(self, cfgfile): - mcfg = None - - if self.cfgfile: + if self.datasource: try: - mcfg = util.read_conf(cfgfile) + d_cfg = self.datasource.get_config_obj() + if d_cfg: + t_cfgs.append(d_cfg) except: - util.logexc(LOG, ("Failed loading of cloud config '%s'. " - "Continuing with an empty config."), cfgfile) - if not mcfg: - mcfg = {} + util.logexc(LOG, ("Failed loading of datasource" + " config object from %s"), self.datasource) + + if self.base_cfg: + t_cfgs.append(self.base_cfg) - ds_cfg = None - try: - ds_cfg = self.datasource.get_config_obj() - except: - util.logexc(LOG, "Failed loading of datasource config object.") - if not ds_cfg: - ds_cfg = {} - - mcfg = util.mergedict(mcfg, ds_cfg) - if self.basecfg: - return util.mergedict(mcfg, self.basecfg) - else: - return mcfg + return util.mergemanydict(t_cfgs) def _read_transforms(self, name): diff --git a/cloudinit/util.py b/cloudinit/util.py index 164bcea8..91d20a76 100644 --- a/cloudinit/util.py +++ b/cloudinit/util.py @@ -360,6 +360,7 @@ def get_cfg_by_path(yobj, keyp, default=None): def fixup_output(cfg, mode): (outfmt, errfmt) = get_output_cfg(cfg, mode) redirect_output(outfmt, errfmt) + return (outfmt, errfmt) # redirect_output(outfmt, errfmt, orig_out, orig_err) @@ -448,6 +449,16 @@ def obj_name(obj): return obj_name(obj.__class__) +def mergemanydict(srcs, reverse=False): + if reverse: + srcs = reversed(srcs) + m_cfg = {} + for a_cfg in srcs: + if a_cfg: + m_cfg = mergedict(m_cfg, a_cfg) + return m_cfg + + def mergedict(src, cand): """ Merge values from C{cand} into C{src}. -- cgit v1.2.3 From 8688ce8a440973978e413584856e55dd6df693bc Mon Sep 17 00:00:00 2001 From: Joshua Harlow Date: Tue, 19 Jun 2012 15:59:46 -0700 Subject: Add a return statement that will collect the failures and the amount of transforms ran. --- cloudinit/stages.py | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) (limited to 'cloudinit/stages.py') diff --git a/cloudinit/stages.py b/cloudinit/stages.py index 558de035..d3e61ddc 100644 --- a/cloudinit/stages.py +++ b/cloudinit/stages.py @@ -156,7 +156,7 @@ class Init(object): # None check so that we don't keep on re-loading if empty if self._cfg is None: self._cfg = self._read_cfg(extra_fns) - LOG.debug("Loaded %s config %s", util.obj_name(self), self._cfg) + LOG.debug("Loaded 'init' config %s", self._cfg) def _read_cfg(self, extra_fns): # Read extra files provided (if any) @@ -391,8 +391,7 @@ class Transforms(object): # None check to avoid empty case if self._cached_cfg is None: self._cached_cfg = self._get_config() - LOG.debug("Loading %s config %s", - util.obj_name(self), self._cached_cfg) + LOG.debug("Loading 'transform' config %s", self._cached_cfg) return self._cached_cfg def _get_config(self): @@ -487,6 +486,7 @@ class Transforms(object): failures = [] d_name = self.init.distro.name c_cloud = self.init.cloudify() + am_ran = 0 for (mod, name, freq, args) in mostly_mods: try: # Try the modules frequency, otherwise fallback to a known one @@ -503,13 +503,15 @@ class Transforms(object): # Use the transforms logger and not our own func_args = [name, copy.deepcopy(self.cfg), c_cloud, transforms.LOG, args] + # Mark it as having started running + am_ran += 1 # This name will affect the semaphore name created run_name = "config-%s" % (name) c_cloud.run(run_name, mod.handle, func_args, freq=freq) except Exception as e: util.logexc(LOG, "Running %s (%s) failed", name, mod) failures.append((name, e)) - return failures + return (am_ran, failures) def run(self, name): raw_mods = self._read_transforms(name) -- cgit v1.2.3 From f328abcb49024c54bc77379ae78f4401015f7f31 Mon Sep 17 00:00:00 2001 From: Joshua Harlow Date: Tue, 19 Jun 2012 20:57:12 -0700 Subject: Fix comparison to none, instead of empty in ds_deps assignment check. --- cloudinit/stages.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'cloudinit/stages.py') diff --git a/cloudinit/stages.py b/cloudinit/stages.py index d3e61ddc..95ac5313 100644 --- a/cloudinit/stages.py +++ b/cloudinit/stages.py @@ -54,7 +54,7 @@ LOG = logging.getLogger(__name__) class Init(object): def __init__(self, ds_deps=None): - if ds_deps: + if ds_deps is not None: self.ds_deps = ds_deps else: self.ds_deps = [sources.DEP_FILESYSTEM, sources.DEP_NETWORK] -- cgit v1.2.3 From bd5075dc77219df36cd1209582d3e004a6d96449 Mon Sep 17 00:00:00 2001 From: Joshua Harlow Date: Tue, 19 Jun 2012 21:38:43 -0700 Subject: 1. Don't force the datasource to always fetch in construction (sometimes not wanted) 2. Add a run single transform function that can be used by the run single main entrypoint action 3. Add a find transform function to be used by the run single action to determine if a transform name is valid --- cloudinit/stages.py | 31 +++++++++++++++++++++++++++---- 1 file changed, 27 insertions(+), 4 deletions(-) (limited to 'cloudinit/stages.py') diff --git a/cloudinit/stages.py b/cloudinit/stages.py index 95ac5313..7e64d3cd 100644 --- a/cloudinit/stages.py +++ b/cloudinit/stages.py @@ -379,7 +379,7 @@ class Init(object): class Transforms(object): def __init__(self, init, cfg_files=None): - self.datasource = init.fetch() + self.datasource = init.datasource self.cfg_files = cfg_files self.base_cfg = copy.deepcopy(init.cfg) self.init = init @@ -418,7 +418,6 @@ class Transforms(object): return util.mergemanydict(t_cfgs) - def _read_transforms(self, name): module_list = [] if name not in self.cfg: @@ -513,7 +512,31 @@ class Transforms(object): failures.append((name, e)) return (am_ran, failures) - def run(self, name): - raw_mods = self._read_transforms(name) + def find_transform(self, tr_name, sections): + found_where = [] + for n in sections: + mods = self._read_transforms(n) + for mod_info in mods: + if mod_info.get('mod') == tr_name: + found_where.append(n) + return found_where + + def run_single(self, tr_name, section): + mods = self._read_transforms(section) + mod_tr = None + for mod_info in mods: + if mod_info.get('mod') == tr_name: + mod_tr = mod_info + break + if not mod_tr: + # Nothing to run, does that transform exist there?? + return (0, 0) + else: + raw_mods = [mod_tr] + mostly_mods = self._fixup_transforms(raw_mods) + return self._run_transforms(mostly_mods) + + def run_section(self, section_name): + raw_mods = self.read_transforms(section_name) mostly_mods = self._fixup_transforms(raw_mods) return self._run_transforms(mostly_mods) -- cgit v1.2.3 From d57681d5f1906e6187e7ca6381f6bd2d37dab3da Mon Sep 17 00:00:00 2001 From: Joshua Harlow Date: Tue, 19 Jun 2012 21:44:51 -0700 Subject: Allow the run single to pass in a set of args and a frequency that will replace the initial sections args and freqency if provided. If not provided then no replacement occurs. --- cloudinit/stages.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) (limited to 'cloudinit/stages.py') diff --git a/cloudinit/stages.py b/cloudinit/stages.py index 7e64d3cd..0c3d5915 100644 --- a/cloudinit/stages.py +++ b/cloudinit/stages.py @@ -521,7 +521,7 @@ class Transforms(object): found_where.append(n) return found_where - def run_single(self, tr_name, section): + def run_single(self, tr_name, section, args=None, freq=None): mods = self._read_transforms(section) mod_tr = None for mod_info in mods: @@ -532,6 +532,12 @@ class Transforms(object): # Nothing to run, does that transform exist there?? return (0, 0) else: + # Adjust the module + if args: + mod_tr['args'] = args + if freq: + mod_tr['freq'] = freq + # Now resume doing the normal fixups and running raw_mods = [mod_tr] mostly_mods = self._fixup_transforms(raw_mods) return self._run_transforms(mostly_mods) -- cgit v1.2.3 From 5859fd60454016e5e7b50a9d9d665e5aeb2da3b1 Mon Sep 17 00:00:00 2001 From: harlowja Date: Wed, 20 Jun 2012 00:36:43 -0700 Subject: Change this function name back after made it back to protected after making the run single transform mode work --- cloudinit/stages.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'cloudinit/stages.py') diff --git a/cloudinit/stages.py b/cloudinit/stages.py index 0c3d5915..f39b6532 100644 --- a/cloudinit/stages.py +++ b/cloudinit/stages.py @@ -543,6 +543,6 @@ class Transforms(object): return self._run_transforms(mostly_mods) def run_section(self, section_name): - raw_mods = self.read_transforms(section_name) + raw_mods = self._read_transforms(section_name) mostly_mods = self._fixup_transforms(raw_mods) return self._run_transforms(mostly_mods) -- cgit v1.2.3 From 6daf4e670d1eb19a87f2f5175b8afe26ada77d9b Mon Sep 17 00:00:00 2001 From: Joshua Harlow Date: Wed, 20 Jun 2012 12:13:14 -0700 Subject: Add the ability to only fetch a local datasource instead of also trying to fetch an offical datasource from an external source. --- cloudinit/stages.py | 19 ++++++++++--------- 1 file changed, 10 insertions(+), 9 deletions(-) (limited to 'cloudinit/stages.py') diff --git a/cloudinit/stages.py b/cloudinit/stages.py index f39b6532..334d5004 100644 --- a/cloudinit/stages.py +++ b/cloudinit/stages.py @@ -209,13 +209,13 @@ class Init(object): cfg_list = self.cfg.get('datasource_list') or [] return (cfg_list, pkg_list) - def _get_data_source(self): + def _get_data_source(self, local_only=False): if self.datasource: return self.datasource ds = self._restore_from_cache() if ds: - LOG.debug("Restored from cache datasource: %s" % ds) - else: + LOG.debug("Restored from cache, datasource: %s", ds) + if not ds and not local_only: (cfg_list, pkg_list) = self._get_datasources() # Deep copy so that user-data handlers can not modify # (which will affect user-data handlers down the line...) @@ -225,10 +225,11 @@ class Init(object): self.paths, ds_deps, cfg_list, pkg_list) LOG.debug("Loaded datasource %s - %s", dsname, ds) - self.datasource = ds - # Ensure we adjust our path members datasource - # now that we have one (thus allowing ipath to be used) - self.paths.datasource = ds + if ds: + self.datasource = ds + # Ensure we adjust our path members datasource + # now that we have one (thus allowing ipath to be used) + self.paths.datasource = ds return ds def _reflect_cur_instance(self): @@ -276,8 +277,8 @@ class Init(object): "%s\n" % (previous_iid)) return iid - def fetch(self): - return self._get_data_source() + def fetch(self, local_only=False): + return self._get_data_source(local_only) def instancify(self): return self._reflect_cur_instance() -- cgit v1.2.3 From 5c5525b35970611a5c4dbd44c5c6b6e2d57556cb Mon Sep 17 00:00:00 2001 From: Joshua Harlow Date: Wed, 20 Jun 2012 16:40:39 -0700 Subject: 1. Rename to modules instead of transforms 2. Run single will now attempt to run a module of a given name, if it can find it (no restrictions on config) --- cloudinit/stages.py | 80 ++++++++++++++++++++++------------------------------- 1 file changed, 33 insertions(+), 47 deletions(-) (limited to 'cloudinit/stages.py') diff --git a/cloudinit/stages.py b/cloudinit/stages.py index 334d5004..ae6e2de5 100644 --- a/cloudinit/stages.py +++ b/cloudinit/stages.py @@ -35,18 +35,20 @@ from cloudinit.settings import (OLD_CLOUD_CONFIG) from cloudinit.settings import (PER_INSTANCE, FREQUENCIES) from cloudinit import handlers + +# Default handlers (used if not overridden) from cloudinit.handlers import boot_hook as bh_part from cloudinit.handlers import cloud_config as cc_part from cloudinit.handlers import shell_script as ss_part from cloudinit.handlers import upstart_job as up_part from cloudinit import cloud +from cloudinit import config from cloudinit import distros from cloudinit import helpers from cloudinit import importer from cloudinit import log as logging from cloudinit import sources -from cloudinit import transforms from cloudinit import util LOG = logging.getLogger(__name__) @@ -319,8 +321,10 @@ class Init(object): # Add the path to the plugins dir to the top of our list for import # instance dir should be read before cloud-dir - sys.path.insert(0, cdir) - sys.path.insert(0, idir) + if cdir and cdir not in sys.path: + sys.path.insert(0, cdir) + if idir and idir not in sys.path: + sys.path.insert(0, idir) # Ensure datasource fetched before activation (just incase) user_data_msg = self.datasource.get_userdata() @@ -378,7 +382,7 @@ class Init(object): called.append(mod) -class Transforms(object): +class Modules(object): def __init__(self, init, cfg_files=None): self.datasource = init.datasource self.cfg_files = cfg_files @@ -392,7 +396,7 @@ class Transforms(object): # None check to avoid empty case if self._cached_cfg is None: self._cached_cfg = self._get_config() - LOG.debug("Loading 'transform' config %s", self._cached_cfg) + LOG.debug("Loading 'module' config %s", self._cached_cfg) return self._cached_cfg def _get_config(self): @@ -419,7 +423,7 @@ class Transforms(object): return util.mergemanydict(t_cfgs) - def _read_transforms(self, name): + def _read_modules(self, name): module_list = [] if name not in self.cfg: return module_list @@ -464,28 +468,28 @@ class Transforms(object): (item, util.obj_name(item))) return module_list - def _fixup_transforms(self, raw_mods): + def _fixup_modules(self, raw_mods): mostly_mods = [] for raw_mod in raw_mods: raw_name = raw_mod['mod'] freq = raw_mod.get('freq') run_args = raw_mod.get('args') or [] - mod_name = transforms.form_transform_name(raw_name) + mod_name = config.form_module_name(raw_name) if not mod_name: continue if freq and freq not in FREQUENCIES: - LOG.warn(("Config specified transform %s" + LOG.warn(("Config specified module %s" " has an unknown frequency %s"), raw_name, freq) # Reset it so when ran it will get set to a known value freq = None - mod = transforms.fixup_transform(importer.import_module(mod_name)) + mod = config.fixup_module(importer.import_module(mod_name)) mostly_mods.append([mod, raw_name, freq, run_args]) return mostly_mods - def _run_transforms(self, mostly_mods): + def _run_modules(self, mostly_mods): failures = [] d_name = self.init.distro.name - c_cloud = self.init.cloudify() + cc = self.init.cloudify() am_ran = 0 for (mod, name, freq, args) in mostly_mods: try: @@ -496,54 +500,36 @@ class Transforms(object): freq = PER_INSTANCE worked_distros = mod.distros if (worked_distros and d_name not in worked_distros): - LOG.warn(("Transform %s is verified on %s distros" + LOG.warn(("Module %s is verified on %s distros" " but not on %s distro. It may or may not work" " correctly."), name, worked_distros, d_name) # Deep copy the config so that modules can't alter it # Use the transforms logger and not our own func_args = [name, copy.deepcopy(self.cfg), - c_cloud, transforms.LOG, args] + cc, config.LOG, args] # Mark it as having started running am_ran += 1 # This name will affect the semaphore name created run_name = "config-%s" % (name) - c_cloud.run(run_name, mod.handle, func_args, freq=freq) + cc.run(run_name, mod.handle, func_args, freq=freq) except Exception as e: util.logexc(LOG, "Running %s (%s) failed", name, mod) failures.append((name, e)) return (am_ran, failures) - def find_transform(self, tr_name, sections): - found_where = [] - for n in sections: - mods = self._read_transforms(n) - for mod_info in mods: - if mod_info.get('mod') == tr_name: - found_where.append(n) - return found_where - - def run_single(self, tr_name, section, args=None, freq=None): - mods = self._read_transforms(section) - mod_tr = None - for mod_info in mods: - if mod_info.get('mod') == tr_name: - mod_tr = mod_info - break - if not mod_tr: - # Nothing to run, does that transform exist there?? - return (0, 0) - else: - # Adjust the module - if args: - mod_tr['args'] = args - if freq: - mod_tr['freq'] = freq - # Now resume doing the normal fixups and running - raw_mods = [mod_tr] - mostly_mods = self._fixup_transforms(raw_mods) - return self._run_transforms(mostly_mods) + def run_single(self, mod_name, args=None, freq=None): + # Form the users module 'specs' + mod_to_be = { + 'mod': mod_name, + 'args': args, + 'freq': freq, + } + # Now resume doing the normal fixups and running + raw_mods = [mod_to_be] + mostly_mods = self._fixup_modules(raw_mods) + return self._run_modules(mostly_mods) def run_section(self, section_name): - raw_mods = self._read_transforms(section_name) - mostly_mods = self._fixup_transforms(raw_mods) - return self._run_transforms(mostly_mods) + raw_mods = self._read_modules(section_name) + mostly_mods = self._fixup_modules(raw_mods) + return self._run_modules(mostly_mods) -- cgit v1.2.3 From ec4bdc4fb8d8d3a8f8b4f498eb47eac740485ede Mon Sep 17 00:00:00 2001 From: Joshua Harlow Date: Wed, 20 Jun 2012 17:13:55 -0700 Subject: Massive pylint + pep8 fixups! --- bin/cloud-init | 20 +++++++-------- cloudinit/cloud.py | 3 ++- cloudinit/config/__init__.py | 2 +- cloudinit/config/cc_chef.py | 14 +++++------ cloudinit/config/cc_disable_ec2_metadata.py | 4 +-- cloudinit/config/cc_final_message.py | 8 +++--- cloudinit/config/cc_foo.py | 8 +++--- cloudinit/config/cc_keys_to_console.py | 8 +++--- cloudinit/config/cc_landscape.py | 2 +- cloudinit/config/cc_mcollective.py | 4 +-- cloudinit/config/cc_mounts.py | 13 +++++----- cloudinit/config/cc_phone_home.py | 11 +++++--- cloudinit/config/cc_puppet.py | 2 +- cloudinit/config/cc_resizefs.py | 10 ++++---- cloudinit/config/cc_salt_minion.py | 2 +- cloudinit/config/cc_scripts_per_boot.py | 6 ++--- cloudinit/config/cc_scripts_per_instance.py | 6 ++--- cloudinit/config/cc_scripts_per_once.py | 6 ++--- cloudinit/config/cc_scripts_user.py | 6 ++--- cloudinit/config/cc_set_passwords.py | 4 +-- cloudinit/config/cc_ssh.py | 39 +++++++++++++++-------------- cloudinit/distros/__init__.py | 1 - cloudinit/distros/rhel.py | 14 +++++------ cloudinit/distros/ubuntu.py | 6 ++--- cloudinit/handlers/__init__.py | 8 +++--- cloudinit/helpers.py | 6 ++--- cloudinit/log.py | 2 -- cloudinit/settings.py | 2 +- cloudinit/sources/DataSourceCloudStack.py | 2 +- cloudinit/sources/DataSourceConfigDrive.py | 2 +- cloudinit/sources/DataSourceEc2.py | 6 ++--- cloudinit/sources/DataSourceMAAS.py | 1 + cloudinit/sources/DataSourceNoCloud.py | 2 +- cloudinit/ssh_util.py | 5 ++-- cloudinit/stages.py | 6 ++--- cloudinit/url_helper.py | 14 +++++------ cloudinit/user_data.py | 28 +++++++++------------ cloudinit/util.py | 37 ++++++++++++++------------- 38 files changed, 159 insertions(+), 161 deletions(-) (limited to 'cloudinit/stages.py') diff --git a/bin/cloud-init b/bin/cloud-init index 032d5f39..c1788ef4 100755 --- a/bin/cloud-init +++ b/bin/cloud-init @@ -75,6 +75,7 @@ def welcome(action): sys.stderr.flush() LOG.info(welcome_msg) + def extract_fns(args): # Files are already opened so lets just pass that along # since it would of broke if it couldn't have @@ -329,11 +330,11 @@ def main_single(name, args): def main(): parser = argparse.ArgumentParser() - + # Top level args - parser.add_argument('--version', '-v', action='version', + parser.add_argument('--version', '-v', action='version', version='%(prog)s ' + (version.version_string())) - parser.add_argument('--file', '-f', action='append', + parser.add_argument('--file', '-f', action='append', dest='files', help=('additional yaml configuration' ' files to use'), @@ -345,18 +346,18 @@ def main(): subparsers = parser.add_subparsers() # Each action and its sub-options (if any) - parser_init = subparsers.add_parser('init', + parser_init = subparsers.add_parser('init', help=('initializes cloud-init and' ' performs initial modules')) parser_init.add_argument("--local", '-l', action='store_true', help="start in local mode (default: %(default)s)", default=False) - # This is used so that we can know which action is selected + + # This is used so that we can know which action is selected + # the functor to use to run this subcommand parser_init.set_defaults(action=('init', main_init)) # These settings are used for the 'config' and 'final' stages - parser_mod = subparsers.add_parser('modules', + parser_mod = subparsers.add_parser('modules', help=('activates modules ' 'using a given configuration key')) parser_mod.add_argument("--mode", '-m', action='store', @@ -368,7 +369,7 @@ def main(): # These settings are used when you want to query information # stored in the cloud-init data objects/directories/files - parser_query = subparsers.add_parser('query', + parser_query = subparsers.add_parser('query', help=('query information stored ' 'in cloud-init')) parser_query.add_argument("--name", '-n', action="store", @@ -378,7 +379,7 @@ def main(): parser_query.set_defaults(action=('query', main_query)) # This subcommand allows you to run a single module - parser_single = subparsers.add_parser('single', + parser_single = subparsers.add_parser('single', help=('run a single module ')) parser_single.set_defaults(action=('single', main_single)) parser_single.add_argument("--name", '-n', action="store", @@ -394,10 +395,10 @@ def main(): ' pass to this module')) parser_single.set_defaults(action=('single', main_single)) - args = parser.parse_args() # Setup basic logging to start (until reinitialized) + # iff in debug mode... if args.debug: logging.setupBasicLogging() @@ -407,4 +408,3 @@ def main(): if __name__ == '__main__': sys.exit(main()) - diff --git a/cloudinit/cloud.py b/cloudinit/cloud.py index 90679202..6cdcb76a 100644 --- a/cloudinit/cloud.py +++ b/cloudinit/cloud.py @@ -38,6 +38,7 @@ LOG = logging.getLogger(__name__) # as providing a backwards compatible object that can be maintained # while the stages/other objects can be worked on independently... + class Cloud(object): def __init__(self, datasource, paths, cfg, distro, runners): self.datasource = datasource @@ -71,7 +72,7 @@ class Cloud(object): # The rest of thes are just useful proxies def get_userdata(self): return self.datasource.get_userdata() - + def get_instance_id(self): return self.datasource.get_instance_id() diff --git a/cloudinit/config/__init__.py b/cloudinit/config/__init__.py index 74e2f275..02e32462 100644 --- a/cloudinit/config/__init__.py +++ b/cloudinit/config/__init__.py @@ -25,7 +25,7 @@ from cloudinit import log as logging LOG = logging.getLogger(__name__) -# This prefix is used to make it less +# This prefix is used to make it less # of a change that when importing # we will not find something else with the same # name in the lookup path... diff --git a/cloudinit/config/cc_chef.py b/cloudinit/config/cc_chef.py index 4e8ef346..74af2a7e 100644 --- a/cloudinit/config/cc_chef.py +++ b/cloudinit/config/cc_chef.py @@ -24,7 +24,7 @@ import os from cloudinit import templater from cloudinit import util -ruby_version_default = "1.8" +RUBY_VERSION_DEFAULT = "1.8" def handle(name, cfg, cloud, log, _args): @@ -38,11 +38,11 @@ def handle(name, cfg, cloud, log, _args): # Ensure the chef directories we use exist c_dirs = [ - '/etc/chef', - '/var/log/chef', - '/var/lib/chef', - '/var/cache/chef', - '/var/backups/chef', + '/etc/chef', + '/var/log/chef', + '/var/lib/chef', + '/var/cache/chef', + '/var/backups/chef', '/var/run/chef', ] for d in c_dirs: @@ -92,7 +92,7 @@ def handle(name, cfg, cloud, log, _args): # this will install and run the chef-client from gems chef_version = util.get_cfg_option_str(chef_cfg, 'version', None) ruby_version = util.get_cfg_option_str(chef_cfg, 'ruby_version', - ruby_version_default) + RUBY_VERSION_DEFAULT) install_chef_from_gems(cloud.distro, ruby_version, chef_version) # and finally, run chef-client log.debug('Running chef-client') diff --git a/cloudinit/config/cc_disable_ec2_metadata.py b/cloudinit/config/cc_disable_ec2_metadata.py index c7d26029..62cca7cc 100644 --- a/cloudinit/config/cc_disable_ec2_metadata.py +++ b/cloudinit/config/cc_disable_ec2_metadata.py @@ -24,13 +24,13 @@ from cloudinit.settings import PER_ALWAYS frequency = PER_ALWAYS -reject_cmd = ['route', 'add', '-host', '169.254.169.254', 'reject'] +REJECT_CMD = ['route', 'add', '-host', '169.254.169.254', 'reject'] def handle(name, cfg, _cloud, log, _args): disabled = util.get_cfg_option_bool(cfg, "disable_ec2_metadata", False) if disabled: - util.subp(reject_cmd) + util.subp(REJECT_CMD) else: log.debug(("Skipping transform named %s," " disabling the ec2 route not enabled"), name) diff --git a/cloudinit/config/cc_final_message.py b/cloudinit/config/cc_final_message.py index c257b6d0..fd59aa1e 100644 --- a/cloudinit/config/cc_final_message.py +++ b/cloudinit/config/cc_final_message.py @@ -28,7 +28,7 @@ from cloudinit.settings import PER_ALWAYS frequency = PER_ALWAYS -final_message_def = ("Cloud-init v. {{version}} finished at {{timestamp}}." +FINAL_MESSAGE_DEF = ("Cloud-init v. {{version}} finished at {{timestamp}}." " Up {{uptime}} seconds.") @@ -39,21 +39,21 @@ def handle(_name, cfg, cloud, log, args): msg_in = args[0] else: msg_in = util.get_cfg_option_str(cfg, "final_message") - + if not msg_in: template_fn = cloud.get_template_filename('final_message') if template_fn: msg_in = util.load_file(template_fn) if not msg_in: - msg_in = final_message_def + msg_in = FINAL_MESSAGE_DEF uptime = util.uptime() ts = util.time_rfc2822() cver = version.version_string() try: subs = { - 'uptime': uptime, + 'uptime': uptime, 'timestamp': ts, 'version': cver, } diff --git a/cloudinit/config/cc_foo.py b/cloudinit/config/cc_foo.py index 99135704..e81e7faa 100644 --- a/cloudinit/config/cc_foo.py +++ b/cloudinit/config/cc_foo.py @@ -30,19 +30,19 @@ from cloudinit.settings import PER_INSTANCE # as well as any datasource provided configuration # c) A cloud object that can be used to access various # datasource and paths for the given distro and data provided -# by the various datasource instance types. +# by the various datasource instance types. # d) A argument list that may or may not be empty to this module. # Typically those are from module configuration where the module # is defined with some extra configuration that will eventually # be translated from yaml into arguments to this module. # 2. A optional 'frequency' that defines how often this module should be ran. -# Typically one of PER_INSTANCE, PER_ALWAYS, PER_ONCE. If not -# provided PER_INSTANCE will be assumed. +# Typically one of PER_INSTANCE, PER_ALWAYS, PER_ONCE. If not +# provided PER_INSTANCE will be assumed. # See settings.py for these constants. # 3. A optional 'distros' array/set/tuple that defines the known distros # this module will work with (if not all of them). This is used to write # a warning out if a module is being ran on a untested distribution for -# informational purposes. If non existent all distros are assumed and +# informational purposes. If non existent all distros are assumed and # no warning occurs. frequency = PER_INSTANCE diff --git a/cloudinit/config/cc_keys_to_console.py b/cloudinit/config/cc_keys_to_console.py index 40758198..a8fb3ba7 100644 --- a/cloudinit/config/cc_keys_to_console.py +++ b/cloudinit/config/cc_keys_to_console.py @@ -26,13 +26,13 @@ from cloudinit import util frequency = PER_INSTANCE # This is a tool that cloud init provides -helper_tool = '/usr/lib/cloud-init/write-ssh-key-fingerprints' +HELPER_TOOL = '/usr/lib/cloud-init/write-ssh-key-fingerprints' def handle(name, cfg, cloud, log, _args): - if not os.path.exists(helper_tool): + if not os.path.exists(HELPER_TOOL): log.warn(("Unable to activate transform %s," - " helper tool not found at %s"), name, helper_tool) + " helper tool not found at %s"), name, HELPER_TOOL) return fp_blacklist = util.get_cfg_option_list(cfg, @@ -42,7 +42,7 @@ def handle(name, cfg, cloud, log, _args): ["ssh-dss"]) try: - cmd = [helper_tool] + cmd = [HELPER_TOOL] cmd.append(','.join(fp_blacklist)) cmd.append(','.join(key_blacklist)) (stdout, _stderr) = util.subp(cmd) diff --git a/cloudinit/config/cc_landscape.py b/cloudinit/config/cc_landscape.py index 29ce41b9..599276a7 100644 --- a/cloudinit/config/cc_landscape.py +++ b/cloudinit/config/cc_landscape.py @@ -62,7 +62,7 @@ def handle(name, cfg, cloud, log, _args): ls_cloudcfg = cfg.get("landscape", {}) if not isinstance(ls_cloudcfg, dict): - raise Exception(("'landscape' key existed in config," + raise Exception(("'landscape' key existed in config," " but not a dictionary type," " is a %s instead"), util.obj_name(ls_cloudcfg)) diff --git a/cloudinit/config/cc_mcollective.py b/cloudinit/config/cc_mcollective.py index 4cec6494..ba5e13ca 100644 --- a/cloudinit/config/cc_mcollective.py +++ b/cloudinit/config/cc_mcollective.py @@ -52,7 +52,7 @@ def handle(name, cfg, cloud, log, _args): # It doesn't contain any sections so just add one temporarily # Use a hash id based off the contents, # just incase of conflicts... (try to not have any...) - # This is so that an error won't occur when reading (and no + # This is so that an error won't occur when reading (and no # sections exist in the file) section_tpl = "[nullsection_%s]" attempts = 0 @@ -85,7 +85,7 @@ def handle(name, cfg, cloud, log, _args): # the previous server.cfg and create our new one old_fn = "%s.old" % (server_cfg_fn) util.rename(server_cfg_fn, old_fn) - # Now we got the whole file, write to disk except the section + # Now we got the whole file, write to disk except the section # we added so that config parser won't error out when trying to read. # Note below, that we've just used ConfigParser because it generally # works. Below, we remove the initial 'nullsection' header. diff --git a/cloudinit/config/cc_mounts.py b/cloudinit/config/cc_mounts.py index 700fbc44..ab097c2a 100644 --- a/cloudinit/config/cc_mounts.py +++ b/cloudinit/config/cc_mounts.py @@ -24,10 +24,10 @@ import re from cloudinit import util -# shortname matches 'sda', 'sda1', 'xvda', 'hda', 'sdb', xvdb, vda, vdd1 -shortname_filter = r"^[x]{0,1}[shv]d[a-z][0-9]*$" -shortname = re.compile(shortname_filter) -ws = re.compile("[%s]+" % (whitespace)) +# Shortname matches 'sda', 'sda1', 'xvda', 'hda', 'sdb', xvdb, vda, vdd1 +SHORTNAME_FILTER = r"^[x]{0,1}[shv]d[a-z][0-9]*$" +SHORTNAME = re.compile(SHORTNAME_FILTER) +WS = re.compile("[%s]+" % (whitespace)) def is_mdname(name): @@ -55,7 +55,6 @@ def handle(_name, cfg, cloud, log, _args): if "mounts" in cfg: cfgmnt = cfg["mounts"] - for i in range(len(cfgmnt)): # skip something that wasn't a list if not isinstance(cfgmnt[i], list): @@ -85,7 +84,7 @@ def handle(_name, cfg, cloud, log, _args): cfgmnt[i][0] = renamed log.debug("Mapped metadata name %s to %s", startname, renamed) else: - if shortname.match(startname): + if SHORTNAME.match(startname): renamed = "/dev/%s" % startname log.debug("Mapped shortname name %s to %s", startname, renamed) cfgmnt[i][0] = renamed @@ -171,7 +170,7 @@ def handle(_name, cfg, cloud, log, _args): fstab = util.load_file(cloud.paths.join(True, "/etc/fstab")) for line in fstab.splitlines(): try: - toks = ws.split(line) + toks = WS.split(line) if toks[3].find(comment) != -1: continue except: diff --git a/cloudinit/config/cc_phone_home.py b/cloudinit/config/cc_phone_home.py index a8752527..dcb07b66 100644 --- a/cloudinit/config/cc_phone_home.py +++ b/cloudinit/config/cc_phone_home.py @@ -26,8 +26,13 @@ from cloudinit.settings import PER_INSTANCE frequency = PER_INSTANCE -post_list_all = ['pub_key_dsa', 'pub_key_rsa', 'pub_key_ecdsa', - 'instance_id', 'hostname'] +POST_LIST_ALL = [ + 'pub_key_dsa', + 'pub_key_rsa', + 'pub_key_ecdsa', + 'instance_id', + 'hostname' +] # phone_home: @@ -63,7 +68,7 @@ def handle(name, cfg, cloud, log, args): " is not an integer, using %s instead"), tries) if post_list == "all": - post_list = post_list_all + post_list = POST_LIST_ALL all_keys = {} all_keys['instance_id'] = cloud.get_instance_id() diff --git a/cloudinit/config/cc_puppet.py b/cloudinit/config/cc_puppet.py index 5fb88bf2..5154efba 100644 --- a/cloudinit/config/cc_puppet.py +++ b/cloudinit/config/cc_puppet.py @@ -63,7 +63,7 @@ def handle(name, cfg, cloud, log, _args): util.ensure_dir(pp_ssl_dir, 0771) util.chownbyid(pp_ssl_dir, pwd.getpwnam('puppet').pw_uid, 0) - pp_ssl_certs = cloud.paths.join(False, + pp_ssl_certs = cloud.paths.join(False, '/var/lib/puppet/ssl/certs/') util.ensure_dir(pp_ssl_certs) util.chownbyid(pp_ssl_certs, diff --git a/cloudinit/config/cc_resizefs.py b/cloudinit/config/cc_resizefs.py index 1690094a..c019989e 100644 --- a/cloudinit/config/cc_resizefs.py +++ b/cloudinit/config/cc_resizefs.py @@ -27,7 +27,7 @@ from cloudinit.settings import PER_ALWAYS frequency = PER_ALWAYS -resize_fs_prefixes_cmds = [ +RESIZE_FS_PREFIXES_CMDS = [ ('ext', 'resize2fs'), ('xfs', 'xfs_growfs'), ] @@ -89,16 +89,16 @@ def handle(name, cfg, cloud, log, args): # occurs this temporary file will still benefit from # auto deletion tfh.unlink_now() - + st_dev = nodeify_path(devpth, resize_what, log) fs_type = get_fs_type(st_dev, devpth, log) if not fs_type: log.warn("Could not determine filesystem type of %s", resize_what) return - + resizer = None fstype_lc = fs_type.lower() - for (pfix, root_cmd) in resize_fs_prefixes_cmds: + for (pfix, root_cmd) in RESIZE_FS_PREFIXES_CMDS: if fstype_lc.startswith(pfix): resizer = root_cmd break @@ -112,7 +112,7 @@ def handle(name, cfg, cloud, log, args): resize_cmd = [resizer, devpth] if resize_root == "noblock": - # Fork to a child that will run + # Fork to a child that will run # the resize command util.fork_cb(do_resize, resize_cmd, log) # Don't delete the file now in the parent diff --git a/cloudinit/config/cc_salt_minion.py b/cloudinit/config/cc_salt_minion.py index 16f5286d..986e6db6 100644 --- a/cloudinit/config/cc_salt_minion.py +++ b/cloudinit/config/cc_salt_minion.py @@ -32,7 +32,7 @@ def handle(name, cfg, cloud, log, _args): # Start by installing the salt package ... cloud.distro.install_packages(["salt"]) - + # Ensure we can configure files at the right dir config_dir = salt_cfg.get("config_dir", '/etc/salt') config_dir = cloud.paths.join(False, config_dir) diff --git a/cloudinit/config/cc_scripts_per_boot.py b/cloudinit/config/cc_scripts_per_boot.py index 364e1d02..d3c47442 100644 --- a/cloudinit/config/cc_scripts_per_boot.py +++ b/cloudinit/config/cc_scripts_per_boot.py @@ -26,16 +26,16 @@ from cloudinit.settings import PER_ALWAYS frequency = PER_ALWAYS -script_subdir = 'per-boot' +SCRIPT_SUBDIR = 'per-boot' def handle(name, _cfg, cloud, log, _args): # Comes from the following: # https://forums.aws.amazon.com/thread.jspa?threadID=96918 - runparts_path = os.path.join(cloud.get_cpath(), 'scripts', script_subdir) + runparts_path = os.path.join(cloud.get_cpath(), 'scripts', SCRIPT_SUBDIR) try: util.runparts(runparts_path) except: log.warn("Failed to run transform %s (%s in %s)", - name, script_subdir, runparts_path) + name, SCRIPT_SUBDIR, runparts_path) raise diff --git a/cloudinit/config/cc_scripts_per_instance.py b/cloudinit/config/cc_scripts_per_instance.py index d75ab47d..8e428ac2 100644 --- a/cloudinit/config/cc_scripts_per_instance.py +++ b/cloudinit/config/cc_scripts_per_instance.py @@ -26,16 +26,16 @@ from cloudinit.settings import PER_INSTANCE frequency = PER_INSTANCE -script_subdir = 'per-instance' +SCRIPT_SUBDIR = 'per-instance' def handle(name, _cfg, cloud, log, _args): # Comes from the following: # https://forums.aws.amazon.com/thread.jspa?threadID=96918 - runparts_path = os.path.join(cloud.get_cpath(), 'scripts', script_subdir) + runparts_path = os.path.join(cloud.get_cpath(), 'scripts', SCRIPT_SUBDIR) try: util.runparts(runparts_path) except: log.warn("Failed to run transform %s (%s in %s)", - name, script_subdir, runparts_path) + name, SCRIPT_SUBDIR, runparts_path) raise diff --git a/cloudinit/config/cc_scripts_per_once.py b/cloudinit/config/cc_scripts_per_once.py index 80f8c325..e7a29a44 100644 --- a/cloudinit/config/cc_scripts_per_once.py +++ b/cloudinit/config/cc_scripts_per_once.py @@ -26,16 +26,16 @@ from cloudinit.settings import PER_ONCE frequency = PER_ONCE -script_subdir = 'per-once' +SCRIPT_SUBDIR = 'per-once' def handle(name, _cfg, cloud, log, _args): # Comes from the following: # https://forums.aws.amazon.com/thread.jspa?threadID=96918 - runparts_path = os.path.join(cloud.get_cpath(), 'scripts', script_subdir) + runparts_path = os.path.join(cloud.get_cpath(), 'scripts', SCRIPT_SUBDIR) try: util.runparts(runparts_path) except: log.warn("Failed to run transform %s (%s in %s)", - name, script_subdir, runparts_path) + name, SCRIPT_SUBDIR, runparts_path) raise diff --git a/cloudinit/config/cc_scripts_user.py b/cloudinit/config/cc_scripts_user.py index f4fe3a2a..1ff05aae 100644 --- a/cloudinit/config/cc_scripts_user.py +++ b/cloudinit/config/cc_scripts_user.py @@ -26,17 +26,17 @@ from cloudinit.settings import PER_INSTANCE frequency = PER_INSTANCE -script_subdir = 'scripts' +SCRIPT_SUBDIR = 'scripts' def handle(name, _cfg, cloud, log, _args): # This is written to by the user data handlers # Ie, any custom shell scripts that come down # go here... - runparts_path = os.path.join(cloud.get_ipath_cur(), script_subdir) + runparts_path = os.path.join(cloud.get_ipath_cur(), SCRIPT_SUBDIR) try: util.runparts(runparts_path) except: log.warn("Failed to run transform %s (%s in %s)", - name, script_subdir, runparts_path) + name, SCRIPT_SUBDIR, runparts_path) raise diff --git a/cloudinit/config/cc_set_passwords.py b/cloudinit/config/cc_set_passwords.py index e7049f22..ce17f357 100644 --- a/cloudinit/config/cc_set_passwords.py +++ b/cloudinit/config/cc_set_passwords.py @@ -25,7 +25,7 @@ from cloudinit import util from string import letters, digits # pylint: disable=W0402 # We are removing certain 'painful' letters/numbers -pw_set = (letters.translate(None, 'loLOI') + +PW_SET = (letters.translate(None, 'loLOI') + digits.translate(None, '01')) @@ -148,4 +148,4 @@ def handle(_name, cfg, cloud, log, args): def rand_user_password(pwlen=9): - return util.rand_str(pwlen, select_from=pw_set) + return util.rand_str(pwlen, select_from=PW_SET) diff --git a/cloudinit/config/cc_ssh.py b/cloudinit/config/cc_ssh.py index e5e99560..4019ae90 100644 --- a/cloudinit/config/cc_ssh.py +++ b/cloudinit/config/cc_ssh.py @@ -24,11 +24,11 @@ import glob from cloudinit import util from cloudinit import ssh_util -DISABLE_ROOT_OPTS = ( "no-port-forwarding,no-agent-forwarding," -"no-X11-forwarding,command=\"echo \'Please login as the user \\\"$USER\\\" " +DISABLE_ROOT_OPTS = ("no-port-forwarding,no-agent-forwarding," +"no-X11-forwarding,command=\"echo \'Please login as the user \\\"$USER\\\" " "rather than the user \\\"root\\\".\';echo;sleep 10\"") -key2file = { +KEY_2_FILE = { "rsa_private": ("/etc/ssh/ssh_host_rsa_key", 0600), "rsa_public": ("/etc/ssh/ssh_host_rsa_key.pub", 0644), "dsa_private": ("/etc/ssh/ssh_host_dsa_key", 0600), @@ -37,15 +37,17 @@ key2file = { "ecdsa_public": ("/etc/ssh/ssh_host_ecdsa_key.pub", 0644), } -priv2pub = { - 'rsa_private': 'rsa_public', +PRIV_2_PUB = { + 'rsa_private': 'rsa_public', 'dsa_private': 'dsa_public', 'ecdsa_private': 'ecdsa_public', } -key_gen_tpl = 'o=$(ssh-keygen -yf "%s") && echo "$o" root@localhost > "%s"' +KEY_GEN_TPL = 'o=$(ssh-keygen -yf "%s") && echo "$o" root@localhost > "%s"' -generate_keys = ['rsa', 'dsa', 'ecdsa'] +GENERATE_KEY_NAMES = ['rsa', 'dsa', 'ecdsa'] + +KEY_FILE_TPL = '/etc/ssh/ssh_host_%s_key' def handle(_name, cfg, cloud, log, _args): @@ -58,21 +60,21 @@ def handle(_name, cfg, cloud, log, _args): util.del_file(f) except: util.logexc(log, "Failed deleting key file %s", f) - + if "ssh_keys" in cfg: # if there are keys in cloud-config, use them for (key, val) in cfg["ssh_keys"].iteritems(): - if key in key2file: - tgt_fn = key2file[key][0] - tgt_perms = key2file[key][1] + if key in KEY_2_FILE: + tgt_fn = KEY_2_FILE[key][0] + tgt_perms = KEY_2_FILE[key][1] util.write_file(cloud.paths.join(False, tgt_fn), val, tgt_perms) - for (priv, pub) in priv2pub.iteritems(): + for (priv, pub) in PRIV_2_PUB.iteritems(): if pub in cfg['ssh_keys'] or not priv in cfg['ssh_keys']: continue - pair = (key2file[priv][0], key2file[pub][0]) - cmd = ['sh', '-xc', key_gen_tpl % pair] + pair = (KEY_2_FILE[priv][0], KEY_2_FILE[pub][0]) + cmd = ['sh', '-xc', KEY_GEN_TPL % pair] try: # TODO: Is this guard needed? with util.SeLinuxGuard("/etc/ssh", recursive=True): @@ -84,12 +86,11 @@ def handle(_name, cfg, cloud, log, _args): else: # if not, generate them genkeys = util.get_cfg_option_list(cfg, - 'ssh_genkeytypes', - generate_keys) + 'ssh_genkeytypes', + GENERATE_KEY_NAMES) for keytype in genkeys: - keyfile = '/etc/ssh/ssh_host_%s_key' % (keytype) - keyfile = cloud.paths.join(False, keyfile) - util.ensure_dir(os.path.dirname(keyfile)) + keyfile = cloud.paths.join(False, KEY_FILE_TPL % (keytype)) + util.ensure_dir(os.path.dirname(keyfile)) if not os.path.exists(keyfile): cmd = ['ssh-keygen', '-t', keytype, '-N', '', '-f', keyfile] try: diff --git a/cloudinit/distros/__init__.py b/cloudinit/distros/__init__.py index 45dd85ec..25a60c52 100644 --- a/cloudinit/distros/__init__.py +++ b/cloudinit/distros/__init__.py @@ -157,4 +157,3 @@ def fetch(distro_name, mods=(__name__, )): % (distro_name)) distro_cls = getattr(mod, 'Distro') return distro_cls - diff --git a/cloudinit/distros/rhel.py b/cloudinit/distros/rhel.py index b67ae5b8..5cbefa6e 100644 --- a/cloudinit/distros/rhel.py +++ b/cloudinit/distros/rhel.py @@ -35,7 +35,7 @@ class Distro(distros.Distro): def __init__(self, name, cfg, paths): distros.Distro.__init__(self, name, cfg, paths) - + def install_packages(self, pkglist): self.package_command('install', pkglist) @@ -210,12 +210,12 @@ class Distro(distros.Distro): def package_command(self, command, args=None): cmd = ['yum'] # If enabled, then yum will be tolerant of errors on the command line - # with regard to packages. - # For example: if you request to install foo, bar and baz and baz is + # with regard to packages. + # For example: if you request to install foo, bar and baz and baz is # installed; yum won't error out complaining that baz is already - # installed. + # installed. cmd.append("-t") - # Determines whether or not yum prompts for confirmation + # Determines whether or not yum prompts for confirmation # of critical actions. We don't want to prompt... cmd.append("-y") cmd.append(command) @@ -223,8 +223,8 @@ class Distro(distros.Distro): cmd.extend(args) # Allow the output of this to flow outwards (ie not be captured) util.subp(cmd, capture=False) - - + + # This is a util function to translate a ubuntu /etc/network/interfaces 'blob' # to a rhel equiv. that can then be written to /etc/sysconfig/network-scripts/ # TODO remove when we have python-netcf active... diff --git a/cloudinit/distros/ubuntu.py b/cloudinit/distros/ubuntu.py index 5a1b572e..fd7b7b8d 100644 --- a/cloudinit/distros/ubuntu.py +++ b/cloudinit/distros/ubuntu.py @@ -36,11 +36,11 @@ class Distro(distros.Distro): def __init__(self, name, cfg, paths): distros.Distro.__init__(self, name, cfg, paths) - # This will be used to restrict certain + # This will be used to restrict certain # calls from repeatly happening (when they # should only happen say once per instance...) self._runner = helpers.Runners(paths) - + def install_packages(self, pkglist): self._update_package_sources() self.package_command('install', pkglist) @@ -131,4 +131,4 @@ class Distro(distros.Distro): def _update_package_sources(self): self._runner.run("update-sources", self.package_command, - ["update"], freq=PER_INSTANCE) \ No newline at end of file + ["update"], freq=PER_INSTANCE) diff --git a/cloudinit/handlers/__init__.py b/cloudinit/handlers/__init__.py index c6f2119c..d52b1cba 100644 --- a/cloudinit/handlers/__init__.py +++ b/cloudinit/handlers/__init__.py @@ -104,7 +104,7 @@ def run_part(mod, data, ctype, filename, payload, frequency): except: mod_ver = 1 try: - LOG.debug("Calling handler %s (%s, %s, %s) with frequency %s", + LOG.debug("Calling handler %s (%s, %s, %s) with frequency %s", mod, ctype, filename, mod_ver, frequency) if mod_ver >= 2: # Treat as v. 2 which does get a frequency @@ -114,7 +114,7 @@ def run_part(mod, data, ctype, filename, payload, frequency): mod.handle_part(data, ctype, filename, payload) except: util.logexc(LOG, ("Failed calling handler %s (%s, %s, %s)" - " with frequency %s"), + " with frequency %s"), mod, ctype, filename, mod_ver, frequency) @@ -178,7 +178,7 @@ def walker_callback(pdata, ctype, filename, payload): payload, pdata['frequency']) -# Callback is a function that will be called with +# Callback is a function that will be called with # (data, content_type, filename, payload) def walk(msg, callback, data): partnum = 0 @@ -226,5 +226,3 @@ def type_from_starts_with(payload, default=None): if payload_lc.startswith(text): return INCLUSION_TYPES_MAP[text] return default - - diff --git a/cloudinit/helpers.py b/cloudinit/helpers.py index 45633e0f..4447d1ee 100644 --- a/cloudinit/helpers.py +++ b/cloudinit/helpers.py @@ -84,7 +84,7 @@ class FileSemaphores(object): try: util.del_dir(self.sem_path) except (IOError, OSError): - util.logexc(LOG, "Failed deleting semaphore directory %s", + util.logexc(LOG, "Failed deleting semaphore directory %s", self.sem_path) def _acquire(self, name, freq): @@ -212,7 +212,7 @@ class Paths(object): self.cfgs = path_cfgs # Populate all the initial paths self.cloud_dir = self.join(False, - path_cfgs.get('cloud_dir', + path_cfgs.get('cloud_dir', '/var/lib/cloud')) self.instance_link = os.path.join(self.cloud_dir, 'instance') self.boot_finished = os.path.join(self.instance_link, "boot-finished") @@ -237,7 +237,7 @@ class Paths(object): # Set when a datasource becomes active self.datasource = ds - # joins the paths but also appends a read + # joins the paths but also appends a read # or write root if available def join(self, read_only, *paths): if read_only: diff --git a/cloudinit/log.py b/cloudinit/log.py index 478946f8..fc1428a2 100644 --- a/cloudinit/log.py +++ b/cloudinit/log.py @@ -20,7 +20,6 @@ # You should have received a copy of the GNU General Public License # along with this program. If not, see . - import logging import logging.handlers import logging.config @@ -53,7 +52,6 @@ def setupBasicLogging(): root.setLevel(DEBUG) - def setupLogging(cfg=None): # See if the config provides any logging conf... if not cfg: diff --git a/cloudinit/settings.py b/cloudinit/settings.py index 8a1eaeb3..fac9b862 100644 --- a/cloudinit/settings.py +++ b/cloudinit/settings.py @@ -47,7 +47,7 @@ CFG_BUILTIN = { 'paths': { 'cloud_dir': '/var/lib/cloud', 'templates_dir': '/etc/cloud/templates/', - }, + }, 'distro': 'ubuntu', }, } diff --git a/cloudinit/sources/DataSourceCloudStack.py b/cloudinit/sources/DataSourceCloudStack.py index b1817654..83c577e6 100644 --- a/cloudinit/sources/DataSourceCloudStack.py +++ b/cloudinit/sources/DataSourceCloudStack.py @@ -121,7 +121,7 @@ class DataSourceCloudStack(sources.DataSource): None, self.metadata_address) self.metadata = boto_utils.get_instance_metadata(self.api_ver, self.metadata_address) - LOG.debug("Crawl of metadata service took %s seconds", + LOG.debug("Crawl of metadata service took %s seconds", int(time.time() - start_time)) return True except Exception: diff --git a/cloudinit/sources/DataSourceConfigDrive.py b/cloudinit/sources/DataSourceConfigDrive.py index 5da1ffea..9905dad4 100644 --- a/cloudinit/sources/DataSourceConfigDrive.py +++ b/cloudinit/sources/DataSourceConfigDrive.py @@ -36,7 +36,7 @@ CFG_DRIVE_FILES = [ "meta.js", ] DEFAULT_METADATA = { - "instance-id": DEFAULT_IID, + "instance-id": DEFAULT_IID, "dsmode": DEFAULT_MODE, } CFG_DRIVE_DEV_ENV = 'CLOUD_INIT_CONFIG_DRIVE_DEVICE' diff --git a/cloudinit/sources/DataSourceEc2.py b/cloudinit/sources/DataSourceEc2.py index 55447102..0598dfa2 100644 --- a/cloudinit/sources/DataSourceEc2.py +++ b/cloudinit/sources/DataSourceEc2.py @@ -38,7 +38,7 @@ DEF_MD_URL = "http://169.254.169.254" DEF_MD_VERSION = '2009-04-04' # Default metadata urls that will be used if none are provided -# They will be checked for 'resolveability' and some of the +# They will be checked for 'resolveability' and some of the # following may be discarded if they do not resolve DEF_MD_URLS = [DEF_MD_URL, "http://instance-data:8773"] @@ -69,7 +69,7 @@ class DataSourceEc2(sources.DataSource): None, self.metadata_address) self.metadata = boto_utils.get_instance_metadata(self.api_ver, self.metadata_address) - LOG.debug("Crawl of metadata service took %s seconds", + LOG.debug("Crawl of metadata service took %s seconds", int(time.time() - start_time)) return True except Exception: @@ -201,7 +201,7 @@ class DataSourceEc2(sources.DataSource): return None # Example: - # 'block-device-mapping': + # 'block-device-mapping': # {'ami': '/dev/sda1', # 'ephemeral0': '/dev/sdb', # 'root': '/dev/sda1'} diff --git a/cloudinit/sources/DataSourceMAAS.py b/cloudinit/sources/DataSourceMAAS.py index bb8fbac1..104e7a54 100644 --- a/cloudinit/sources/DataSourceMAAS.py +++ b/cloudinit/sources/DataSourceMAAS.py @@ -251,6 +251,7 @@ datasources = [ (DataSourceMAAS, (sources.DEP_FILESYSTEM, sources.DEP_NETWORK)), ] + # Return a list of data sources that match this set of dependencies def get_datasource_list(depends): return sources.list_from_depends(depends, datasources) diff --git a/cloudinit/sources/DataSourceNoCloud.py b/cloudinit/sources/DataSourceNoCloud.py index 2b016d1c..8499a97c 100644 --- a/cloudinit/sources/DataSourceNoCloud.py +++ b/cloudinit/sources/DataSourceNoCloud.py @@ -154,7 +154,7 @@ class DataSourceNoCloud(sources.DataSource): (self.dsmode in ("local", seeded_interfaces))): LOG.info("Updating network interfaces from %s", self) self.distro.apply_network(md['network-interfaces']) - + if md['dsmode'] == self.dsmode: self.seed = ",".join(found) self.metadata = md diff --git a/cloudinit/ssh_util.py b/cloudinit/ssh_util.py index fef3d40f..45dd5535 100644 --- a/cloudinit/ssh_util.py +++ b/cloudinit/ssh_util.py @@ -103,10 +103,10 @@ class AuthKeyLineParser(object): elif curc == '"': quoted = not quoted i = i + 1 - + options = ent[0:i] options_lst = [] - + # Now use a csv parser to pull the options # out of the above string that we just found an endpoint for. # @@ -211,7 +211,6 @@ def update_authorized_keys(fname, keys): def setup_user_keys(keys, user, key_prefix, paths): - # Make sure the users .ssh dir is setup accordingly pwent = pwd.getpwnam(user) ssh_dir = os.path.join(pwent.pw_dir, '.ssh') diff --git a/cloudinit/stages.py b/cloudinit/stages.py index ae6e2de5..84a965c2 100644 --- a/cloudinit/stages.py +++ b/cloudinit/stages.py @@ -287,7 +287,7 @@ class Init(object): def cloudify(self): # Form the needed options to cloudify our members - return cloud.Cloud(self.datasource, + return cloud.Cloud(self.datasource, self.paths, self.cfg, self.distro, helpers.Runners(self.paths)) @@ -318,7 +318,7 @@ class Init(object): def consume(self, frequency=PER_INSTANCE): cdir = self.paths.get_cpath("handlers") idir = self.paths.get_ipath("handlers") - + # Add the path to the plugins dir to the top of our list for import # instance dir should be read before cloud-dir if cdir and cdir not in sys.path: @@ -417,7 +417,7 @@ class Modules(object): except: util.logexc(LOG, ("Failed loading of datasource" " config object from %s"), self.datasource) - + if self.base_cfg: t_cfgs.append(self.base_cfg) diff --git a/cloudinit/url_helper.py b/cloudinit/url_helper.py index 1c583eba..223278ce 100644 --- a/cloudinit/url_helper.py +++ b/cloudinit/url_helper.py @@ -47,11 +47,11 @@ class UrlResponse(object): @property def contents(self): return self._contents - + @property def headers(self): return self._headers - + def __str__(self): if not self.contents: return '' @@ -66,7 +66,7 @@ class UrlResponse(object): return True else: return False - + def readurl(url, data=None, timeout=None, retries=0, sec_between=1, headers=None): @@ -89,8 +89,8 @@ def readurl(url, data=None, timeout=None, excepts = [] LOG.info(("Attempting to open '%s' with %s attempts" - " (%s retries, timeout=%s) to be performed"), - url, attempts, retries, timeout) + " (%s retries, timeout=%s) to be performed"), + url, attempts, retries, timeout) open_args = {} if timeout is not None: open_args['timeout'] = int(timeout) @@ -112,7 +112,7 @@ def readurl(url, data=None, timeout=None, excepts.append(e) except urllib2.URLError as e: # This can be a message string or - # another exception instance + # another exception instance # (socket.error for remote URLs, OSError for local URLs). if (isinstance(e.reason, (OSError)) and e.reason.errno == errno.ENOENT): @@ -128,7 +128,7 @@ def readurl(url, data=None, timeout=None, # Didn't work out LOG.warn("Failed reading from %s after %s attempts", url, attempts) - + # It must of errored at least once for code # to get here so re-raise the last error LOG.debug("%s errors occured, re-raising the last one", len(excepts)) diff --git a/cloudinit/user_data.py b/cloudinit/user_data.py index b7902d44..4babb8e5 100644 --- a/cloudinit/user_data.py +++ b/cloudinit/user_data.py @@ -65,33 +65,33 @@ class UserDataProcessor(object): # multipart/* are just containers if part.get_content_maintype() == 'multipart': continue - + ctype = None ctype_orig = part.get_content_type() payload = part.get_payload(decode=True) - + if not ctype_orig: ctype_orig = UNDEF_TYPE - + if ctype_orig in TYPE_NEEDED: ctype = handlers.type_from_starts_with(payload) - + if ctype is None: ctype = ctype_orig - + if ctype in INCLUDE_TYPES: self._do_include(payload, append_msg) continue - + if ctype in ARCHIVE_TYPES: self._explode_archive(payload, append_msg) continue - + if 'Content-Type' in base_msg: base_msg.replace_header('Content-Type', ctype) else: base_msg['Content-Type'] = ctype - + self._attach_part(append_msg, part) def _get_include_once_filename(self, entry): @@ -108,8 +108,8 @@ class UserDataProcessor(object): lc_line = line.lower() if lc_line.startswith("#include-once"): line = line[len("#include-once"):].lstrip() - # Every following include will now - # not be refetched.... but will be + # Every following include will now + # not be refetched.... but will be # re-read from a local urlcache (if it worked) include_once_on = True elif lc_line.startswith("#include"): @@ -190,10 +190,10 @@ class UserDataProcessor(object): """ if ATTACHMENT_FIELD not in outer_msg: outer_msg[ATTACHMENT_FIELD] = '0' - + if new_count is not None: outer_msg.replace_header(ATTACHMENT_FIELD, str(new_count)) - + fetched_count = 0 try: fetched_count = int(outer_msg.get(ATTACHMENT_FIELD)) @@ -234,7 +234,3 @@ def convert_string(raw_data, headers=None): msg = MIMEBase(maintype, subtype, *headers) msg.set_payload(data) return msg - - - - diff --git a/cloudinit/util.py b/cloudinit/util.py index 91d20a76..56c01fab 100644 --- a/cloudinit/util.py +++ b/cloudinit/util.py @@ -35,7 +35,7 @@ import pwd import random import shutil import socket -import string # pylint: disable=W0402 +import string # pylint: disable=W0402 import subprocess import sys import tempfile @@ -153,13 +153,15 @@ def SilentTemporaryFile(**kwargs): # file to unlink has been unlinked elsewhere.. LOG.debug("Created temporary file %s", fh.name) fh.unlink = del_file - # Add a new method that will unlink + + # Add a new method that will unlink # right 'now' but still lets the exit # method attempt to remove it (which will # not throw due to our del file being quiet # about files that are not there) def unlink_now(): fh.unlink(fh.name) + setattr(fh, 'unlink_now', unlink_now) return fh @@ -199,7 +201,7 @@ def is_false_str(val, addons=None): def translate_bool(val, addons=None): if not val: - # This handles empty lists and false and + # This handles empty lists and false and # other things that python believes are false return False # If its already a boolean skip @@ -214,7 +216,6 @@ def rand_str(strlen=32, select_from=None): return "".join([random.choice(select_from) for _x in range(0, strlen)]) - def read_conf(fname): try: return load_yaml(load_file(fname), default={}) @@ -275,7 +276,7 @@ def is_ipv4(instr): def merge_base_cfg(cfgfile, cfg_builtin=None): syscfg = read_conf_with_confd(cfgfile) - + kern_contents = read_cc_from_cmdline() kerncfg = {} if kern_contents: @@ -575,7 +576,7 @@ def load_yaml(blob, default=None, allowed=(dict,)): try: blob = str(blob) LOG.debug(("Attempting to load yaml from string " - "of length %s with allowed root types %s"), + "of length %s with allowed root types %s"), len(blob), allowed) converted = yaml.load(blob) if not isinstance(converted, allowed): @@ -625,7 +626,7 @@ def read_conf_d(confd): # remove anything not ending in '.cfg' confs = [f for f in confs if f.endswith(".cfg")] - + # remove anything not a file confs = [f for f in confs if os.path.isfile(os.path.join(confd, f))] @@ -726,9 +727,9 @@ def get_fqdn_from_hosts(hostname, filename="/etc/hosts"): """ For each host a single line should be present with the following information: - - IP_address canonical_hostname [aliases...] - + + IP_address canonical_hostname [aliases...] + Fields of the entry are separated by any number of blanks and/or tab characters. Text from a "#" character until the end of the line is a comment, and is ignored. Host names may contain only alphanumeric @@ -747,7 +748,7 @@ def get_fqdn_from_hosts(hostname, filename="/etc/hosts"): if not line: continue - # If there there is less than 3 entries + # If there there is less than 3 entries # (IP_address, canonical_hostname, alias) # then ignore this line toks = line.split() @@ -829,7 +830,7 @@ def close_stdin(): os.dup2(fp.fileno(), sys.stdin.fileno()) -def find_devs_with(criteria=None, oformat='device', +def find_devs_with(criteria=None, oformat='device', tag=None, no_cache=False, path=None): """ find devices matching given criteria (via blkid) @@ -841,23 +842,23 @@ def find_devs_with(criteria=None, oformat='device', blk_id_cmd = ['blkid'] options = [] if criteria: - # Search for block devices with tokens named NAME that + # Search for block devices with tokens named NAME that # have the value 'value' and display any devices which are found. # Common values for NAME include TYPE, LABEL, and UUID. # If there are no devices specified on the command line, - # all block devices will be searched; otherwise, + # all block devices will be searched; otherwise, # only search the devices specified by the user. options.append("-t%s" % (criteria)) if tag: # For each (specified) device, show only the tags that match tag. options.append("-s%s" % (tag)) if no_cache: - # If you want to start with a clean cache - # (i.e. don't report devices previously scanned + # If you want to start with a clean cache + # (i.e. don't report devices previously scanned # but not necessarily available at this time), specify /dev/null. options.extend(["-c", "/dev/null"]) if oformat: - # Display blkid's output using the specified format. + # Display blkid's output using the specified format. # The format parameter may be: # full, value, list, device, udev, export options.append('-o%s' % (oformat)) @@ -1104,7 +1105,7 @@ def mounts(): (dev, mp, fstype, opts, _freq, _passno) = mpline.split() except: continue - # If the name of the mount point contains spaces these + # If the name of the mount point contains spaces these # can be escaped as '\040', so undo that.. mp = mp.replace("\\040", " ") mounted[dev] = { -- cgit v1.2.3 From 95e0fa29af3656c1011c41ab0f35dc4e9317269c Mon Sep 17 00:00:00 2001 From: Joshua Harlow Date: Wed, 20 Jun 2012 23:40:00 -0700 Subject: 1. Add an importer function that will search for a given module in a set of search module 'prefixes' that also has a potential set of required attributes. 2. Use this new importer to find the distro class, the userdata handler modules, the config modules and the datasource modules, if none can be found error out accordingly. --- cloudinit/config/__init__.py | 4 ---- cloudinit/distros/__init__.py | 23 ++++++++++------------ cloudinit/handlers/__init__.py | 8 -------- cloudinit/importer.py | 44 +++++++++++++++++++++++++++++++++--------- cloudinit/sources/__init__.py | 34 +++++++++++--------------------- cloudinit/stages.py | 22 +++++++++++++++++---- 6 files changed, 74 insertions(+), 61 deletions(-) (limited to 'cloudinit/stages.py') diff --git a/cloudinit/config/__init__.py b/cloudinit/config/__init__.py index 02e32462..ab13045f 100644 --- a/cloudinit/config/__init__.py +++ b/cloudinit/config/__init__.py @@ -51,10 +51,6 @@ def fixup_module(mod, def_freq=PER_INSTANCE): freq = mod.frequency if freq and freq not in FREQUENCIES: LOG.warn("Module %s has an unknown frequency %s", mod, freq) - if not hasattr(mod, 'handle'): - def empty_handle(_name, _cfg, _cloud, _log, _args): - pass - setattr(mod, 'handle', empty_handle) if not hasattr(mod, 'distros'): setattr(mod, 'distros', None) return mod diff --git a/cloudinit/distros/__init__.py b/cloudinit/distros/__init__.py index 25a60c52..e0ef6ee0 100644 --- a/cloudinit/distros/__init__.py +++ b/cloudinit/distros/__init__.py @@ -144,16 +144,13 @@ class Distro(object): return False -def fetch(distro_name, mods=(__name__, )): - mod = None - for m in mods: - mod_name = "%s.%s" % (m, distro_name) - try: - mod = importer.import_module(mod_name) - except ImportError: - pass - if not mod: - raise RuntimeError("No distribution found for distro %s" - % (distro_name)) - distro_cls = getattr(mod, 'Distro') - return distro_cls +def fetch(name): + locs = importer.find_module(name, + ['', __name__], + ['Distro']) + if not locs: + raise ImportError("No distribution found for distro %s" + % (name)) + mod = importer.import_module(locs[0]) + cls = getattr(mod, 'Distro') + return cls diff --git a/cloudinit/handlers/__init__.py b/cloudinit/handlers/__init__.py index d52b1cba..0f7432e5 100644 --- a/cloudinit/handlers/__init__.py +++ b/cloudinit/handlers/__init__.py @@ -202,20 +202,12 @@ def walk(msg, callback, data): def fixup_handler(mod, def_freq=PER_INSTANCE): if not hasattr(mod, "handler_version"): setattr(mod, "handler_version", 1) - if not hasattr(mod, 'list_types'): - def empty_types(): - return [] - setattr(mod, 'list_types', empty_types) if not hasattr(mod, 'frequency'): setattr(mod, 'frequency', def_freq) else: freq = mod.frequency if freq and freq not in FREQUENCIES: LOG.warn("Handler %s has an unknown frequency %s", mod, freq) - if not hasattr(mod, 'handle_part'): - def empty_handler(_data, _ctype, _filename, _payload): - pass - setattr(mod, 'handle_part', empty_handler) return mod diff --git a/cloudinit/importer.py b/cloudinit/importer.py index a36b87bc..71cf2726 100644 --- a/cloudinit/importer.py +++ b/cloudinit/importer.py @@ -23,17 +23,43 @@ import sys from cloudinit import log as logging -from cloudinit import util LOG = logging.getLogger(__name__) -# Simple wrapper that allows us to add more logging in... def import_module(module_name): - try: - LOG.debug("Attempting to import module %s", module_name) - __import__(module_name) - return sys.modules[module_name] - except: - util.logexc(LOG, 'Failed at importing %s', module_name) - raise + __import__(module_name) + return sys.modules[module_name] + + +def find_module(base_name, search_paths, required_attrs=None): + found_places = [] + if not required_attrs: + required_attrs = [] + real_paths = [] + for path in search_paths: + real_path = [] + if path: + real_path.extend(path.split(".")) + real_path.append(base_name) + full_path = '.'.join(real_path) + real_paths.append(full_path) + LOG.debug("Looking for modules %s that have attributes %s", + real_paths, required_attrs) + for full_path in real_paths: + mod = None + try: + mod = import_module(full_path) + except ImportError: + pass + if not mod: + continue + found_attrs = 0 + for attr in required_attrs: + if hasattr(mod, attr): + found_attrs += 1 + if found_attrs == len(required_attrs): + found_places.append(full_path) + LOG.debug("Found %s with attributes %s in %s", base_name, + required_attrs, found_places) + return found_places diff --git a/cloudinit/sources/__init__.py b/cloudinit/sources/__init__.py index 8ab7cf54..42e924b0 100644 --- a/cloudinit/sources/__init__.py +++ b/cloudinit/sources/__init__.py @@ -191,31 +191,19 @@ def list_sources(cfg_list, depends, pkg_list): LOG.info(("Looking for for data source in: %s," " via packages %s that matches dependencies %s"), cfg_list, pkg_list, depends) - for ds_coll in cfg_list: - ds_name = str(ds_coll) + for ds_name in cfg_list: if not ds_name.startswith(DS_PREFIX): ds_name = '%s%s' % (DS_PREFIX, ds_name) - for pkg in pkg_list: - pkg_name = [] - if pkg: - # Any package name given, this affects - # the lookup path - pkg_name.append(str(pkg)) - pkg_name.append(ds_name) - try: - mod = importer.import_module(".".join(pkg_name)) - except ImportError: - continue - lister = getattr(mod, "get_datasource_list", None) - if not lister: - continue - cls_matches = lister(depends) - if not cls_matches: - continue - src_list.extend(cls_matches) - LOG.debug(("Found a match" - " in %s with matches %s"), mod, cls_matches) - break + m_locs = importer.find_module(ds_name, + pkg_list, + ['get_datasource_list']) + for m_loc in m_locs: + mod = importer.import_module(m_loc) + lister = getattr(mod, "get_datasource_list") + matches = lister(depends) + if matches: + src_list.extend(matches) + break return src_list diff --git a/cloudinit/stages.py b/cloudinit/stages.py index 84a965c2..1997301a 100644 --- a/cloudinit/stages.py +++ b/cloudinit/stages.py @@ -205,7 +205,7 @@ class Init(object): # Any config provided??? pkg_list = self.cfg.get('datasource_pkg_list') or [] # Add the defaults at the end - for n in [util.obj_name(sources), '']: + for n in ['', util.obj_name(sources)]: if n not in pkg_list: pkg_list.append(n) cfg_list = self.cfg.get('datasource_list') or [] @@ -334,9 +334,17 @@ class Init(object): # Add handlers in cdir potential_handlers = util.find_modules(cdir) - for (fname, modname) in potential_handlers.iteritems(): + for (fname, mod_name) in potential_handlers.iteritems(): try: - mod = handlers.fixup_handler(importer.import_module(modname)) + mod_locs = importer.find_module(mod_name, [''], + ['list_types', + 'handle_part']) + if not mod_locs: + LOG.warn(("Could not find a valid user-data handler" + " named %s in file %s"), mod_name, fname) + continue + mod = importer.import_module(mod_locs[0]) + mod = handlers.fixup_handler(mod) types = c_handlers.register(mod) LOG.debug("Added handler for %s from %s", types, fname) except: @@ -482,7 +490,13 @@ class Modules(object): " has an unknown frequency %s"), raw_name, freq) # Reset it so when ran it will get set to a known value freq = None - mod = config.fixup_module(importer.import_module(mod_name)) + mod_locs = importer.find_module(mod_name, + ['', util.obj_name(config)], + ['handle']) + if not mod_locs: + LOG.warn("Could not find module named %s", mod_name) + continue + mod = config.fixup_module(importer.import_module(mod_locs[0])) mostly_mods.append([mod, raw_name, freq, run_args]) return mostly_mods -- cgit v1.2.3 From 0c83ff5cb81b6de3028cf90e7dd6aba565682dbf Mon Sep 17 00:00:00 2001 From: harlowja Date: Thu, 21 Jun 2012 09:12:16 -0700 Subject: 1. Renames for debug message from 'transform' to 'module' 2. Fixing up more cloud.path.joins found to use the right ro/rw filename --- cloudinit/cloud.py | 4 ++-- cloudinit/config/cc_apt_pipelining.py | 6 ++++-- cloudinit/config/cc_bootcmd.py | 4 ++-- cloudinit/config/cc_byobu.py | 2 +- cloudinit/config/cc_ca_certs.py | 2 +- cloudinit/config/cc_chef.py | 2 +- cloudinit/config/cc_disable_ec2_metadata.py | 2 +- cloudinit/config/cc_foo.py | 2 +- cloudinit/config/cc_keys_to_console.py | 2 +- cloudinit/config/cc_landscape.py | 11 ++++++++--- cloudinit/config/cc_locale.py | 2 +- cloudinit/config/cc_mcollective.py | 2 +- cloudinit/config/cc_mounts.py | 3 ++- cloudinit/config/cc_phone_home.py | 2 +- cloudinit/config/cc_puppet.py | 12 +++++++----- cloudinit/config/cc_resizefs.py | 2 +- cloudinit/config/cc_rightscale_userdata.py | 6 +++--- cloudinit/config/cc_rsyslog.py | 2 +- cloudinit/config/cc_runcmd.py | 2 +- cloudinit/config/cc_salt_minion.py | 10 +++++----- cloudinit/config/cc_scripts_per_boot.py | 2 +- cloudinit/config/cc_scripts_per_instance.py | 2 +- cloudinit/config/cc_scripts_per_once.py | 2 +- cloudinit/config/cc_scripts_user.py | 2 +- cloudinit/config/cc_set_hostname.py | 2 +- cloudinit/config/cc_set_passwords.py | 4 +++- cloudinit/config/cc_ssh_import_id.py | 4 ++-- cloudinit/config/cc_timezone.py | 2 +- cloudinit/config/cc_update_etc_hosts.py | 2 +- cloudinit/config/cc_update_hostname.py | 2 +- cloudinit/stages.py | 2 +- 31 files changed, 59 insertions(+), 47 deletions(-) (limited to 'cloudinit/stages.py') diff --git a/cloudinit/cloud.py b/cloudinit/cloud.py index 6cdcb76a..22d9167e 100644 --- a/cloudinit/cloud.py +++ b/cloudinit/cloud.py @@ -29,7 +29,7 @@ LOG = logging.getLogger(__name__) # This class is the high level wrapper that provides # access to cloud-init objects without exposing the stage objects -# to handler and or transform manipulation. It allows for cloud +# to handler and or module manipulation. It allows for cloud # init to restrict what those types of user facing code may see # and or adjust (which helps avoid code messing with each other) # @@ -47,7 +47,7 @@ class Cloud(object): self._cfg = cfg self._runners = runners - # If a transform manipulates logging or logging services + # If a 'user' manipulates logging or logging services # it is typically useful to cause the logging to be # setup again. def cycle_logging(self): diff --git a/cloudinit/config/cc_apt_pipelining.py b/cloudinit/config/cc_apt_pipelining.py index f460becb..3426099e 100644 --- a/cloudinit/config/cc_apt_pipelining.py +++ b/cloudinit/config/cc_apt_pipelining.py @@ -25,6 +25,9 @@ distros = ['ubuntu', 'debian'] DEFAULT_FILE = "/etc/apt/apt.conf.d/90cloud-init-pipelining" +APT_PIPE_TPL = ("//Written by cloud-init per 'apt_pipelining'\n" + 'Acquire::http::Pipeline-Depth "%s";\n') + # Acquire::http::Pipeline-Depth can be a value # from 0 to 5 indicating how many outstanding requests APT should send. # A value of zero MUST be specified if the remote host does not properly linger @@ -49,8 +52,7 @@ def handle(_name, cfg, cloud, log, _args): def write_apt_snippet(cloud, setting, log, f_name): """ Writes f_name with apt pipeline depth 'setting' """ - file_contents = ("//Written by cloud-init per 'apt_pipelining'\n" - 'Acquire::http::Pipeline-Depth "%s";\n') % (setting) + file_contents = APT_PIPE_TPL % (setting) util.write_file(cloud.paths.join(False, f_name), file_contents) diff --git a/cloudinit/config/cc_bootcmd.py b/cloudinit/config/cc_bootcmd.py index 635e3a1f..89ccf3f1 100644 --- a/cloudinit/config/cc_bootcmd.py +++ b/cloudinit/config/cc_bootcmd.py @@ -30,7 +30,7 @@ frequency = PER_ALWAYS def handle(name, cfg, cloud, log, _args): if "bootcmd" not in cfg: - log.debug(("Skipping transform named %s," + log.debug(("Skipping module named %s," " no 'bootcmd' key in configuration"), name) return @@ -52,5 +52,5 @@ def handle(name, cfg, cloud, log, _args): util.subp(cmd, env=env, capture=False) except: util.logexc(log, - ("Failed to run bootcmd transform %s"), name) + ("Failed to run bootcmd module %s"), name) raise diff --git a/cloudinit/config/cc_byobu.py b/cloudinit/config/cc_byobu.py index 741aa934..38586174 100644 --- a/cloudinit/config/cc_byobu.py +++ b/cloudinit/config/cc_byobu.py @@ -30,7 +30,7 @@ def handle(name, cfg, _cloud, log, args): value = util.get_cfg_option_str(cfg, "byobu_by_default", "") if not value: - log.debug("Skipping transform named %s, no 'byobu' values found", name) + log.debug("Skipping module named %s, no 'byobu' values found", name) return if value == "user" or value == "system": diff --git a/cloudinit/config/cc_ca_certs.py b/cloudinit/config/cc_ca_certs.py index 56c41561..3221a587 100644 --- a/cloudinit/config/cc_ca_certs.py +++ b/cloudinit/config/cc_ca_certs.py @@ -75,7 +75,7 @@ def handle(name, cfg, cloud, log, _args): """ # If there isn't a ca-certs section in the configuration don't do anything if "ca-certs" not in cfg: - log.debug(("Skipping transform named %s," + log.debug(("Skipping module named %s," " no 'ca-certs' key in configuration"), name) return diff --git a/cloudinit/config/cc_chef.py b/cloudinit/config/cc_chef.py index 74af2a7e..d8bd85f8 100644 --- a/cloudinit/config/cc_chef.py +++ b/cloudinit/config/cc_chef.py @@ -31,7 +31,7 @@ def handle(name, cfg, cloud, log, _args): # If there isn't a chef key in the configuration don't do anything if 'chef' not in cfg: - log.debug(("Skipping transform named %s," + log.debug(("Skipping module named %s," " no 'chef' key in configuration"), name) return chef_cfg = cfg['chef'] diff --git a/cloudinit/config/cc_disable_ec2_metadata.py b/cloudinit/config/cc_disable_ec2_metadata.py index 62cca7cc..a7c6a75b 100644 --- a/cloudinit/config/cc_disable_ec2_metadata.py +++ b/cloudinit/config/cc_disable_ec2_metadata.py @@ -32,5 +32,5 @@ def handle(name, cfg, _cloud, log, _args): if disabled: util.subp(REJECT_CMD) else: - log.debug(("Skipping transform named %s," + log.debug(("Skipping module named %s," " disabling the ec2 route not enabled"), name) diff --git a/cloudinit/config/cc_foo.py b/cloudinit/config/cc_foo.py index e81e7faa..95aab4dd 100644 --- a/cloudinit/config/cc_foo.py +++ b/cloudinit/config/cc_foo.py @@ -49,4 +49,4 @@ frequency = PER_INSTANCE def handle(name, _cfg, _cloud, log, _args): - log.debug("Hi from transform %s", name) + log.debug("Hi from module %s", name) diff --git a/cloudinit/config/cc_keys_to_console.py b/cloudinit/config/cc_keys_to_console.py index a8fb3ba7..d4c877f7 100644 --- a/cloudinit/config/cc_keys_to_console.py +++ b/cloudinit/config/cc_keys_to_console.py @@ -31,7 +31,7 @@ HELPER_TOOL = '/usr/lib/cloud-init/write-ssh-key-fingerprints' def handle(name, cfg, cloud, log, _args): if not os.path.exists(HELPER_TOOL): - log.warn(("Unable to activate transform %s," + log.warn(("Unable to activate module %s," " helper tool not found at %s"), name, HELPER_TOOL) return diff --git a/cloudinit/config/cc_landscape.py b/cloudinit/config/cc_landscape.py index 599276a7..d45c9203 100644 --- a/cloudinit/config/cc_landscape.py +++ b/cloudinit/config/cc_landscape.py @@ -56,7 +56,7 @@ def handle(name, cfg, cloud, log, _args): """ if not ConfigObj: log.warn(("'ConfigObj' support not available," - " running transform %s disabled"), name) + " running module %s disabled"), name) return ls_cloudcfg = cfg.get("landscape", {}) @@ -66,9 +66,14 @@ def handle(name, cfg, cloud, log, _args): " but not a dictionary type," " is a %s instead"), util.obj_name(ls_cloudcfg)) - lsc_client_fn = cloud.paths.join(True, LSC_CLIENT_CFG_FILE) - merged = merge_together([LSC_BUILTIN_CFG, lsc_client_fn, ls_cloudcfg]) + merge_data = [ + LSC_BUILTIN_CFG, + cloud.paths.join(True, LSC_CLIENT_CFG_FILE), + ls_cloudcfg, + ] + merged = merge_together(merge_data) + lsc_client_fn = cloud.paths.join(False, LSC_CLIENT_CFG_FILE) lsc_dir = cloud.paths.join(False, os.path.dirname(lsc_client_fn)) if not os.path.isdir(lsc_dir): util.ensure_dir(lsc_dir) diff --git a/cloudinit/config/cc_locale.py b/cloudinit/config/cc_locale.py index 7f273123..3fb4c5d9 100644 --- a/cloudinit/config/cc_locale.py +++ b/cloudinit/config/cc_locale.py @@ -49,7 +49,7 @@ def handle(name, cfg, cloud, log, args): "/etc/default/locale") if not locale: - log.debug(("Skipping transform named %s, " + log.debug(("Skipping module named %s, " "no 'locale' configuration found"), name) return diff --git a/cloudinit/config/cc_mcollective.py b/cloudinit/config/cc_mcollective.py index 3fec6729..36a4cade 100644 --- a/cloudinit/config/cc_mcollective.py +++ b/cloudinit/config/cc_mcollective.py @@ -32,7 +32,7 @@ def handle(name, cfg, cloud, log, _args): # If there isn't a mcollective key in the configuration don't do anything if 'mcollective' not in cfg: - log.debug(("Skipping transform named %s, " + log.debug(("Skipping module named %s, " "no 'mcollective' key in configuration"), name) return diff --git a/cloudinit/config/cc_mounts.py b/cloudinit/config/cc_mounts.py index ab097c2a..d3dcf7af 100644 --- a/cloudinit/config/cc_mounts.py +++ b/cloudinit/config/cc_mounts.py @@ -188,8 +188,9 @@ def handle(_name, cfg, cloud, log, _args): util.logexc(log, "Activating swap via 'swapon -a' failed") for d in dirs: + real_dir = cloud.paths.join(False, d) try: - util.ensure_dir(cloud.paths.join(False, d)) + util.ensure_dir(real_dir) except: util.logexc(log, "Failed to make '%s' config-mount", d) diff --git a/cloudinit/config/cc_phone_home.py b/cloudinit/config/cc_phone_home.py index dcb07b66..d929eb64 100644 --- a/cloudinit/config/cc_phone_home.py +++ b/cloudinit/config/cc_phone_home.py @@ -53,7 +53,7 @@ def handle(name, cfg, cloud, log, args): ph_cfg = cfg['phone_home'] if 'url' not in ph_cfg: - log.warn(("Skipping transform named %s, " + log.warn(("Skipping module named %s, " "no 'url' found in 'phone_home' configuration"), name) return diff --git a/cloudinit/config/cc_puppet.py b/cloudinit/config/cc_puppet.py index 5154efba..467c1496 100644 --- a/cloudinit/config/cc_puppet.py +++ b/cloudinit/config/cc_puppet.py @@ -31,7 +31,7 @@ from cloudinit import util def handle(name, cfg, cloud, log, _args): # If there isn't a puppet key in the configuration don't do anything if 'puppet' not in cfg: - log.debug(("Skipping transform named %s," + log.debug(("Skipping module named %s," " no 'puppet' configuration found"), name) return @@ -43,7 +43,7 @@ def handle(name, cfg, cloud, log, _args): # ... and then update the puppet configuration if 'conf' in puppet_cfg: # Add all sections from the conf object to puppet.conf - puppet_conf_fn = cloud.paths.join(False, '/etc/puppet/puppet.conf') + puppet_conf_fn = cloud.paths.join(True, '/etc/puppet/puppet.conf') contents = util.load_file(puppet_conf_fn) # Create object for reading puppet.conf values puppet_config = helpers.DefaultingConfigParser() @@ -89,9 +89,11 @@ def handle(name, cfg, cloud, log, _args): puppet_config.set(cfg_name, o, v) # We got all our config as wanted we'll rename # the previous puppet.conf and create our new one - puppet_conf_old_fn = "%s.old" % (puppet_conf_fn) - util.rename(puppet_conf_fn, puppet_conf_old_fn) - util.write_file(puppet_conf_fn, puppet_config.stringify()) + conf_old_fn = cloud.paths.join(False, + '/etc/puppet/puppet.conf.old') + util.rename(puppet_conf_fn, conf_old_fn) + puppet_conf_rw = cloud.paths.join(False, '/etc/puppet/puppet.conf') + util.write_file(puppet_conf_rw, puppet_config.stringify()) # Set puppet to automatically start if os.path.exists('/etc/default/puppet'): diff --git a/cloudinit/config/cc_resizefs.py b/cloudinit/config/cc_resizefs.py index c019989e..7e1428e9 100644 --- a/cloudinit/config/cc_resizefs.py +++ b/cloudinit/config/cc_resizefs.py @@ -69,7 +69,7 @@ def handle(name, cfg, cloud, log, args): resize_root = util.get_cfg_option_str(cfg, "resize_rootfs", True) if not util.translate_bool(resize_root): - log.debug("Skipping transform named %s, resizing disabled", name) + log.debug("Skipping module named %s, resizing disabled", name) return # TODO is the directory ok to be used?? diff --git a/cloudinit/config/cc_rightscale_userdata.py b/cloudinit/config/cc_rightscale_userdata.py index 8385e281..7a134569 100644 --- a/cloudinit/config/cc_rightscale_userdata.py +++ b/cloudinit/config/cc_rightscale_userdata.py @@ -53,13 +53,13 @@ def handle(name, _cfg, cloud, log, _args): try: ud = cloud.get_userdata_raw() except: - log.warn("Failed to get raw userdata in transform %s", name) + log.warn("Failed to get raw userdata in module %s", name) return try: mdict = parse_qs(ud) if not mdict or not MY_HOOKNAME in mdict: - log.debug(("Skipping transform %s, " + log.debug(("Skipping module %s, " "did not find %s in parsed" " raw userdata"), name, MY_HOOKNAME) return @@ -73,7 +73,7 @@ def handle(name, _cfg, cloud, log, _args): # These will eventually be then ran by the cc_scripts_user # TODO: maybe this should just be a new user data handler?? - # Instead of a late transform that acts like a user data handler? + # Instead of a late module that acts like a user data handler? scripts_d = cloud.get_ipath_cur('scripts') urls = mdict[MY_HOOKNAME] for (i, url) in enumerate(urls): diff --git a/cloudinit/config/cc_rsyslog.py b/cloudinit/config/cc_rsyslog.py index f2c1de1e..78327526 100644 --- a/cloudinit/config/cc_rsyslog.py +++ b/cloudinit/config/cc_rsyslog.py @@ -36,7 +36,7 @@ def handle(name, cfg, cloud, log, _args): # process 'rsyslog' if not 'rsyslog' in cfg: - log.debug(("Skipping transform named %s," + log.debug(("Skipping module named %s," " no 'rsyslog' key in configuration"), name) return diff --git a/cloudinit/config/cc_runcmd.py b/cloudinit/config/cc_runcmd.py index f121484b..65064cfb 100644 --- a/cloudinit/config/cc_runcmd.py +++ b/cloudinit/config/cc_runcmd.py @@ -25,7 +25,7 @@ from cloudinit import util def handle(name, cfg, cloud, log, _args): if "runcmd" not in cfg: - log.debug(("Skipping transform named %s," + log.debug(("Skipping module named %s," " no 'runcmd' key in configuration"), name) return diff --git a/cloudinit/config/cc_salt_minion.py b/cloudinit/config/cc_salt_minion.py index 986e6db6..ff90d07a 100644 --- a/cloudinit/config/cc_salt_minion.py +++ b/cloudinit/config/cc_salt_minion.py @@ -24,7 +24,7 @@ from cloudinit import util def handle(name, cfg, cloud, log, _args): # If there isn't a salt key in the configuration don't do anything if 'salt_minion' not in cfg: - log.debug(("Skipping transform named %s," + log.debug(("Skipping module named %s," " no 'salt_minion' key in configuration"), name) return @@ -34,8 +34,8 @@ def handle(name, cfg, cloud, log, _args): cloud.distro.install_packages(["salt"]) # Ensure we can configure files at the right dir - config_dir = salt_cfg.get("config_dir", '/etc/salt') - config_dir = cloud.paths.join(False, config_dir) + config_dir = cloud.paths.join(False, salt_cfg.get("config_dir", + '/etc/salt')) util.ensure_dir(config_dir) # ... and then update the salt configuration @@ -47,8 +47,8 @@ def handle(name, cfg, cloud, log, _args): # ... copy the key pair if specified if 'public_key' in salt_cfg and 'private_key' in salt_cfg: - pki_dir = salt_cfg.get('pki_dir', '/etc/salt/pki') - pki_dir = cloud.paths.join(pki_dir) + pki_dir = cloud.paths.join(False, salt_cfg.get('pki_dir', + '/etc/salt/pki')) with util.umask(077): util.ensure_dir(pki_dir) pub_name = os.path.join(pki_dir, 'minion.pub') diff --git a/cloudinit/config/cc_scripts_per_boot.py b/cloudinit/config/cc_scripts_per_boot.py index d3c47442..42b987eb 100644 --- a/cloudinit/config/cc_scripts_per_boot.py +++ b/cloudinit/config/cc_scripts_per_boot.py @@ -36,6 +36,6 @@ def handle(name, _cfg, cloud, log, _args): try: util.runparts(runparts_path) except: - log.warn("Failed to run transform %s (%s in %s)", + log.warn("Failed to run module %s (%s in %s)", name, SCRIPT_SUBDIR, runparts_path) raise diff --git a/cloudinit/config/cc_scripts_per_instance.py b/cloudinit/config/cc_scripts_per_instance.py index 8e428ac2..b5d71c13 100644 --- a/cloudinit/config/cc_scripts_per_instance.py +++ b/cloudinit/config/cc_scripts_per_instance.py @@ -36,6 +36,6 @@ def handle(name, _cfg, cloud, log, _args): try: util.runparts(runparts_path) except: - log.warn("Failed to run transform %s (%s in %s)", + log.warn("Failed to run module %s (%s in %s)", name, SCRIPT_SUBDIR, runparts_path) raise diff --git a/cloudinit/config/cc_scripts_per_once.py b/cloudinit/config/cc_scripts_per_once.py index e7a29a44..d77d36d5 100644 --- a/cloudinit/config/cc_scripts_per_once.py +++ b/cloudinit/config/cc_scripts_per_once.py @@ -36,6 +36,6 @@ def handle(name, _cfg, cloud, log, _args): try: util.runparts(runparts_path) except: - log.warn("Failed to run transform %s (%s in %s)", + log.warn("Failed to run module %s (%s in %s)", name, SCRIPT_SUBDIR, runparts_path) raise diff --git a/cloudinit/config/cc_scripts_user.py b/cloudinit/config/cc_scripts_user.py index 1ff05aae..5c53014f 100644 --- a/cloudinit/config/cc_scripts_user.py +++ b/cloudinit/config/cc_scripts_user.py @@ -37,6 +37,6 @@ def handle(name, _cfg, cloud, log, _args): try: util.runparts(runparts_path) except: - log.warn("Failed to run transform %s (%s in %s)", + log.warn("Failed to run module %s (%s in %s)", name, SCRIPT_SUBDIR, runparts_path) raise diff --git a/cloudinit/config/cc_set_hostname.py b/cloudinit/config/cc_set_hostname.py index 3ac8a8fa..b0f27ebf 100644 --- a/cloudinit/config/cc_set_hostname.py +++ b/cloudinit/config/cc_set_hostname.py @@ -24,7 +24,7 @@ from cloudinit import util def handle(name, cfg, cloud, log, _args): if util.get_cfg_option_bool(cfg, "preserve_hostname", False): log.debug(("Configuration option 'preserve_hostname' is set," - " not setting the hostname in transform %s"), name) + " not setting the hostname in module %s"), name) return (hostname, _fqdn) = util.get_hostname_fqdn(cfg, cloud) diff --git a/cloudinit/config/cc_set_passwords.py b/cloudinit/config/cc_set_passwords.py index ce17f357..eb68ddfe 100644 --- a/cloudinit/config/cc_set_passwords.py +++ b/cloudinit/config/cc_set_passwords.py @@ -106,7 +106,9 @@ def handle(_name, cfg, cloud, log, args): replacement = "PasswordAuthentication %s" % (pw_auth) # See http://linux.die.net/man/5/sshd_config - old_lines = util.load_file('/etc/ssh/sshd_config').splitlines() + conf_fn = cloud.paths.join(True, '/etc/ssh/sshd_config') + # Todo: use the common ssh_util function for this parsing... + old_lines = util.load_file(conf_fn).splitlines() for i, line in enumerate(old_lines): if not line.strip() or line.startswith("#"): new_lines.append(line) diff --git a/cloudinit/config/cc_ssh_import_id.py b/cloudinit/config/cc_ssh_import_id.py index d57e4665..c58b28ec 100644 --- a/cloudinit/config/cc_ssh_import_id.py +++ b/cloudinit/config/cc_ssh_import_id.py @@ -36,11 +36,11 @@ def handle(name, cfg, _cloud, log, args): ids = util.get_cfg_option_list(cfg, "ssh_import_id", []) if len(ids) == 0: - log.debug("Skipping transform named %s, no ids found to import", name) + log.debug("Skipping module named %s, no ids found to import", name) return if not user: - log.debug("Skipping transform named %s, no user found to import", name) + log.debug("Skipping module named %s, no user found to import", name) return cmd = ["sudo", "-Hu", user, "ssh-import-id"] + ids diff --git a/cloudinit/config/cc_timezone.py b/cloudinit/config/cc_timezone.py index 747c436c..b9eb85b2 100644 --- a/cloudinit/config/cc_timezone.py +++ b/cloudinit/config/cc_timezone.py @@ -32,7 +32,7 @@ def handle(name, cfg, cloud, log, args): timezone = util.get_cfg_option_str(cfg, "timezone", False) if not timezone: - log.debug("Skipping transform named %s, no 'timezone' specified", name) + log.debug("Skipping module named %s, no 'timezone' specified", name) return # Let the distro handle settings its timezone diff --git a/cloudinit/config/cc_update_etc_hosts.py b/cloudinit/config/cc_update_etc_hosts.py index 75615db1..6820ac4f 100644 --- a/cloudinit/config/cc_update_etc_hosts.py +++ b/cloudinit/config/cc_update_etc_hosts.py @@ -57,4 +57,4 @@ def handle(name, cfg, cloud, log, _args): cloud.distro.update_etc_hosts(hostname, fqdn) else: log.debug(("Configuration option 'manage_etc_hosts' is not set," - " not managing /etc/hosts in transform %s"), name) + " not managing /etc/hosts in module %s"), name) diff --git a/cloudinit/config/cc_update_hostname.py b/cloudinit/config/cc_update_hostname.py index 58444fab..b84a1a06 100644 --- a/cloudinit/config/cc_update_hostname.py +++ b/cloudinit/config/cc_update_hostname.py @@ -29,7 +29,7 @@ frequency = PER_ALWAYS def handle(name, cfg, cloud, log, _args): if util.get_cfg_option_bool(cfg, "preserve_hostname", False): log.debug(("Configuration option 'preserve_hostname' is set," - " not updating the hostname in transform %s"), name) + " not updating the hostname in module %s"), name) return (hostname, _fqdn) = util.get_hostname_fqdn(cfg, cloud) diff --git a/cloudinit/stages.py b/cloudinit/stages.py index 1997301a..9f28c2e8 100644 --- a/cloudinit/stages.py +++ b/cloudinit/stages.py @@ -518,7 +518,7 @@ class Modules(object): " but not on %s distro. It may or may not work" " correctly."), name, worked_distros, d_name) # Deep copy the config so that modules can't alter it - # Use the transforms logger and not our own + # Use the configs logger and not our own func_args = [name, copy.deepcopy(self.cfg), cc, config.LOG, args] # Mark it as having started running -- cgit v1.2.3 From a588b9b966e51c9b9778b0082c8c5b6ab10f2677 Mon Sep 17 00:00:00 2001 From: Joshua Harlow Date: Thu, 21 Jun 2012 20:14:50 -0700 Subject: 1. Go through a single protected get ipath method that will throw if there is not an active datasource (ie the user has done an out of order call to a function that needs the datasource to exist) 2. Add in a '_get_instance_subdirs' method that can be over-ridden in the future if more subdirs are needed. --- cloudinit/stages.py | 26 ++++++++++++++++++++------ 1 file changed, 20 insertions(+), 6 deletions(-) (limited to 'cloudinit/stages.py') diff --git a/cloudinit/stages.py b/cloudinit/stages.py index 9f28c2e8..25f13fd4 100644 --- a/cloudinit/stages.py +++ b/cloudinit/stages.py @@ -234,15 +234,29 @@ class Init(object): self.paths.datasource = ds return ds + def _get_instance_subdirs(self): + return ['handlers', 'scripts', 'sems'] + + def _get_ipath(self, subname=None): + # Force a check to see if anything + # actually comes back, if not + # then a datasource has not been assigned... + instance_dir = self.paths.get_ipath(subname) + if not instance_dir: + raise RuntimeError(("No instance directory is available." + " Has a datasource been fetched??")) + return instance_dir + def _reflect_cur_instance(self): - # Ensure we are hooked into the right symlink for the current instance - idir = self.paths.get_ipath() + # Remove the old symlink and attach a new one so + # that further reads/writes connect into the right location + idir = self._get_ipath() util.del_file(self.paths.instance_link) util.sym_link(idir, self.paths.instance_link) # Ensures these dirs exist dir_list = [] - for d in ["handlers", "scripts", "sem"]: + for d in self._get_instance_subdirs(): dir_list.append(os.path.join(idir, d)) util.ensure_dirs(dir_list) @@ -297,9 +311,9 @@ class Init(object): def _store_userdata(self): raw_ud = "%s" % (self.datasource.get_userdata_raw()) - util.write_file(self.paths.get_ipath('userdata_raw'), raw_ud, 0600) + util.write_file(self._get_ipath('userdata_raw'), raw_ud, 0600) processed_ud = "%s" % (self.datasource.get_userdata()) - util.write_file(self.paths.get_ipath('userdata'), processed_ud, 0600) + util.write_file(self._get_ipath('userdata'), processed_ud, 0600) def _default_userdata_handlers(self): opts = { @@ -317,7 +331,7 @@ class Init(object): def consume(self, frequency=PER_INSTANCE): cdir = self.paths.get_cpath("handlers") - idir = self.paths.get_ipath("handlers") + idir = self._get_ipath("handlers") # Add the path to the plugins dir to the top of our list for import # instance dir should be read before cloud-dir -- cgit v1.2.3 From ba5fb03646f6318a0ace286da746b4bb32f75d5a Mon Sep 17 00:00:00 2001 From: harlowja Date: Thu, 21 Jun 2012 23:35:07 -0700 Subject: Fixup python selinux guards, only try to restore after we check if its useful to restore, fix test to work with selinux enabled sysystems --- cloudinit/stages.py | 20 ++++++++++++-------- cloudinit/util.py | 26 +++++++++++++++++--------- tests/unittests/test_util.py | 13 ++++++------- 3 files changed, 35 insertions(+), 24 deletions(-) (limited to 'cloudinit/stages.py') diff --git a/cloudinit/stages.py b/cloudinit/stages.py index 25f13fd4..cf5e6924 100644 --- a/cloudinit/stages.py +++ b/cloudinit/stages.py @@ -221,11 +221,12 @@ class Init(object): (cfg_list, pkg_list) = self._get_datasources() # Deep copy so that user-data handlers can not modify # (which will affect user-data handlers down the line...) - sys_cfg = copy.deepcopy(self.cfg) - ds_deps = copy.deepcopy(self.ds_deps) - (ds, dsname) = sources.find_source(sys_cfg, self.distro, + (ds, dsname) = sources.find_source(self.cfg, + self.distro, self.paths, - ds_deps, cfg_list, pkg_list) + copy.deepcopy(self.ds_deps), + cfg_list, + pkg_list) LOG.debug("Loaded datasource %s - %s", dsname, ds) if ds: self.datasource = ds @@ -408,7 +409,7 @@ class Modules(object): def __init__(self, init, cfg_files=None): self.datasource = init.datasource self.cfg_files = cfg_files - self.base_cfg = copy.deepcopy(init.cfg) + self.base_cfg = init.cfg self.init = init # Created on first use self._cached_cfg = None @@ -419,7 +420,8 @@ class Modules(object): if self._cached_cfg is None: self._cached_cfg = self._get_config() LOG.debug("Loading 'module' config %s", self._cached_cfg) - return self._cached_cfg + # Only give out a copy so that others can't modify this... + return copy.deepcopy(self._cached_cfg) def _get_config(self): t_cfgs = [] @@ -531,9 +533,11 @@ class Modules(object): LOG.warn(("Module %s is verified on %s distros" " but not on %s distro. It may or may not work" " correctly."), name, worked_distros, d_name) - # Deep copy the config so that modules can't alter it # Use the configs logger and not our own - func_args = [name, copy.deepcopy(self.cfg), + # TODO: possibly check the module + # for having a LOG attr and just give it back + # its own logger? + func_args = [name, self.cfg, cc, config.LOG, args] # Mark it as having started running am_ran += 1 diff --git a/cloudinit/util.py b/cloudinit/util.py index 3aa4e462..332b8379 100644 --- a/cloudinit/util.py +++ b/cloudinit/util.py @@ -35,6 +35,7 @@ import pwd import random import shutil import socket +import stat import string # pylint: disable=W0402 import subprocess import sys @@ -132,14 +133,24 @@ class SeLinuxGuard(object): self.enabled = True def __enter__(self): - # TODO: Should we try to engage selinux here?? return self.enabled def __exit__(self, excp_type, excp_value, excp_traceback): if self.enabled: - LOG.debug("Restoring selinux mode for %s (recursive=%s)", - self.path, self.recursive) - selinux.restorecon(self.path, recursive=self.recursive) + path = os.path.realpath(os.path.expanduser(self.path)) + do_restore = False + try: + # See if even worth restoring?? + stats = os.lstat(path) + if stat.ST_MODE in stats: + selinux.matchpathcon(path, stats[stat.ST_MODE]) + do_restore = True + except OSError: + pass + if do_restore: + LOG.debug("Restoring selinux mode for %s (recursive=%s)", + path, self.recursive) + selinux.restorecon(path, recursive=self.recursive) class MountFailedError(Exception): @@ -1067,8 +1078,7 @@ def ensure_dir(path, mode=None): if not os.path.isdir(path): # Make the dir and adjust the mode LOG.debug("Ensuring directory exists at path %s", path) - # TODO: check if guard needed?? - with SeLinuxGuard(path=os.path.dirname(path)): + with SeLinuxGuard(os.path.dirname(path), recursive=True): os.makedirs(path) chmod(path, mode) else: @@ -1222,8 +1232,7 @@ def chmod(path, mode): if path and real_mode: LOG.debug("Adjusting the permissions of %s (perms=%o)", path, real_mode) - # TODO: check if guard needed?? - with SeLinuxGuard(path=path): + with SeLinuxGuard(path): os.chmod(path, real_mode) @@ -1239,7 +1248,6 @@ def write_file(filename, content, mode=0644, omode="wb"): """ ensure_dir(os.path.dirname(filename)) LOG.debug("Writing to %s - %s, %s bytes", filename, omode, len(content)) - # TODO: check if guard needed?? with SeLinuxGuard(path=filename): with open(filename, omode) as fh: fh.write(content) diff --git a/tests/unittests/test_util.py b/tests/unittests/test_util.py index ba565b29..3be6e186 100644 --- a/tests/unittests/test_util.py +++ b/tests/unittests/test_util.py @@ -71,7 +71,7 @@ class TestGetCfgOptionListOrStr(TestCase): """None is returned if key is not found and no default given.""" config = {} result = util.get_cfg_option_list(config, "key") - self.assertIsNone(result) + self.assertEqual(None, result) def test_not_found_with_default(self): """Default is returned if key is not found.""" @@ -166,14 +166,13 @@ class TestWriteFile(MockerTestCase): "selinux.restorecon", passthrough=False) mock_is_selinux_enabled = self.mocker.replace( "selinux.is_selinux_enabled", passthrough=False) - mock_is_selinux_enabled.result(True) - mock_restorecon(path) + mock_is_selinux_enabled() + self.mocker.result(True) + mock_restorecon("/etc/hosts", recursive=False) + self.mocker.result(True) self.mocker.replay() - old = util.HAVE_LIBSELINUX - util.HAVE_LIBSELINUX = True - with util.SeLinuxGuard(self.tmp) as is_on: + with util.SeLinuxGuard("/etc/hosts") as is_on: self.assertTrue(is_on) - util.HAVE_LIBSELINUX = old except ImportError: pass -- cgit v1.2.3 From 2a2dc725670f8ff3c6024332302ff9c718ff27f2 Mon Sep 17 00:00:00 2001 From: harlowja Date: Fri, 22 Jun 2012 00:04:07 -0700 Subject: 1. Return which modules ran from the run module function 2. Use that list in the main binary & adjust related comparisions --- bin/cloud-init | 16 +++++++++------- cloudinit/stages.py | 10 ++++++---- 2 files changed, 15 insertions(+), 11 deletions(-) (limited to 'cloudinit/stages.py') diff --git a/bin/cloud-init b/bin/cloud-init index aff8f967..68c7ba76 100755 --- a/bin/cloud-init +++ b/bin/cloud-init @@ -91,8 +91,8 @@ def extract_fns(args): def run_module_section(mods, action_name, section): full_section_name = MOD_SECTION_TPL % (section) - (ran_am, failures) = mods.run_section(full_section_name) - total_attempted = ran_am + len(failures) + (which_ran, failures) = mods.run_section(full_section_name) + total_attempted = len(which_ran) + len(failures) if total_attempted == 0: msg = ("No '%s' modules to run" " under section '%s'") % (action_name, full_section_name) @@ -100,7 +100,8 @@ def run_module_section(mods, action_name, section): LOG.debug(msg) return 0 else: - LOG.debug("Ran %s modules with %s failures", ran_am, len(failures)) + LOG.debug("Ran %s modules with %s failures", + len(which_ran), len(failures)) return len(failures) @@ -316,16 +317,17 @@ def main_single(name, args): logging.resetLogging() logging.setupLogging(mods.cfg) # Stage 4 - (run_am, failures) = mods.run_single(mod_name, - mod_args, - mod_freq) + (which_ran, failures) = mods.run_single(mod_name, + mod_args, + mod_freq) if failures: LOG.warn("Ran %s but it failed!", mod_name) return 1 - elif run_am == 0: + elif not which_ran: LOG.warn("Did not run %s, does it exist?", mod_name) return 1 else: + # Guess it worked return 0 diff --git a/cloudinit/stages.py b/cloudinit/stages.py index cf5e6924..9481db83 100644 --- a/cloudinit/stages.py +++ b/cloudinit/stages.py @@ -517,10 +517,12 @@ class Modules(object): return mostly_mods def _run_modules(self, mostly_mods): - failures = [] d_name = self.init.distro.name cc = self.init.cloudify() - am_ran = 0 + # Return which ones ran + # and which ones failed + the exception of why it failed + failures = [] + which_ran = [] for (mod, name, freq, args) in mostly_mods: try: # Try the modules frequency, otherwise fallback to a known one @@ -540,14 +542,14 @@ class Modules(object): func_args = [name, self.cfg, cc, config.LOG, args] # Mark it as having started running - am_ran += 1 + which_ran.append(name) # This name will affect the semaphore name created run_name = "config-%s" % (name) cc.run(run_name, mod.handle, func_args, freq=freq) except Exception as e: util.logexc(LOG, "Running %s (%s) failed", name, mod) failures.append((name, e)) - return (am_ran, failures) + return (which_ran, failures) def run_single(self, mod_name, args=None, freq=None): # Form the users module 'specs' -- cgit v1.2.3 From 7563328f1f08290a0d63e9e9e6da16b01ab4b5b8 Mon Sep 17 00:00:00 2001 From: harlowja Date: Fri, 22 Jun 2012 08:50:51 -0700 Subject: ConfigObj is a requirement now, so no need to see if the import worked or failed. --- cloudinit/stages.py | 10 ++-------- 1 file changed, 2 insertions(+), 8 deletions(-) (limited to 'cloudinit/stages.py') diff --git a/cloudinit/stages.py b/cloudinit/stages.py index 9481db83..595f7aea 100644 --- a/cloudinit/stages.py +++ b/cloudinit/stages.py @@ -26,10 +26,7 @@ import copy import os import sys -try: - from configobj import ConfigObj -except ImportError: - ConfigObj = None +from configobj import ConfigObj from cloudinit.settings import (OLD_CLOUD_CONFIG) from cloudinit.settings import (PER_INSTANCE, FREQUENCIES) @@ -68,10 +65,7 @@ class Init(object): self.datasource = None def _read_cfg_old(self): - # Support reading the old ConfigObj format file and merging - # it into the yaml dictionary - if not ConfigObj: - return {} + # Support reading the old ConfigObj format file old_cfg = ConfigObj(OLD_CLOUD_CONFIG) return dict(old_cfg) -- cgit v1.2.3 From e69068d4b431ad237ff7e57795759fe8025a4bd7 Mon Sep 17 00:00:00 2001 From: Joshua Harlow Date: Fri, 22 Jun 2012 12:36:01 -0700 Subject: Remove support for older ec2 configobj merging. --- cloudinit/stages.py | 11 +---------- 1 file changed, 1 insertion(+), 10 deletions(-) (limited to 'cloudinit/stages.py') diff --git a/cloudinit/stages.py b/cloudinit/stages.py index 595f7aea..cfe1c071 100644 --- a/cloudinit/stages.py +++ b/cloudinit/stages.py @@ -26,9 +26,6 @@ import copy import os import sys -from configobj import ConfigObj - -from cloudinit.settings import (OLD_CLOUD_CONFIG) from cloudinit.settings import (PER_INSTANCE, FREQUENCIES) from cloudinit import handlers @@ -64,11 +61,6 @@ class Init(object): # Created only when a fetch occurs self.datasource = None - def _read_cfg_old(self): - # Support reading the old ConfigObj format file - old_cfg = ConfigObj(OLD_CLOUD_CONFIG) - return dict(old_cfg) - @property def distro(self): if not self._distro: @@ -165,13 +157,12 @@ class Init(object): except: util.logexc(LOG, ("Failed loading of additional" " configuration from %s"), fn) - # Now read in the built-in + base + old + # Now read in the built-in + base try: conf = util.get_base_cfg(builtin=util.get_builtin_cfg()) except Exception: conf = util.get_builtin_cfg() i_cfgs.append(conf) - i_cfgs.append(self._read_cfg_old()) return util.mergemanydict(i_cfgs) def _restore_from_cache(self): -- cgit v1.2.3 From a0f42d8f96248d519e70a86e470868616dea3d6e Mon Sep 17 00:00:00 2001 From: Joshua Harlow Date: Fri, 22 Jun 2012 23:43:48 -0700 Subject: Don't complete the update if there is no datasource or if writing to cache fails --- cloudinit/stages.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) (limited to 'cloudinit/stages.py') diff --git a/cloudinit/stages.py b/cloudinit/stages.py index cfe1c071..3ed13009 100644 --- a/cloudinit/stages.py +++ b/cloudinit/stages.py @@ -178,10 +178,13 @@ class Init(object): return None def _write_to_cache(self): + if not self.datasource: + return False pickled_fn = self.paths.get_ipath_cur("obj_pkl") try: contents = pickle.dumps(self.datasource) util.write_file(pickled_fn, contents, mode=0400) + return True except Exception: util.logexc(LOG, "Failed pickling datasource to %s", pickled_fn) return False @@ -292,7 +295,8 @@ class Init(object): self.distro, helpers.Runners(self.paths)) def update(self): - self._write_to_cache() + if not self._write_to_cache(): + return self._store_userdata() def _store_userdata(self): -- cgit v1.2.3 From 4e938eef6581769c086e417a24206274f26b8a95 Mon Sep 17 00:00:00 2001 From: harlowja Date: Sat, 23 Jun 2012 15:00:41 -0700 Subject: 1. Use a common config merging class now in helpers that will be the central point of config fetching for both the 'init' stage and the 'module' stages a. This helps those who want to understand exactly what configs are fetched and how the merging occurs. --- cloudinit/helpers.py | 82 +++++++++++++++++++++++++++++++++++++++++++++++++++- cloudinit/stages.py | 59 ++++++++++--------------------------- 2 files changed, 96 insertions(+), 45 deletions(-) (limited to 'cloudinit/stages.py') diff --git a/cloudinit/helpers.py b/cloudinit/helpers.py index 7a2ac6b8..0f86285f 100644 --- a/cloudinit/helpers.py +++ b/cloudinit/helpers.py @@ -28,7 +28,8 @@ import os from ConfigParser import (NoSectionError, NoOptionError, RawConfigParser) -from cloudinit.settings import (PER_INSTANCE, PER_ALWAYS, PER_ONCE) +from cloudinit.settings import (PER_INSTANCE, PER_ALWAYS, PER_ONCE, + CFG_ENV_NAME) from cloudinit import log as logging from cloudinit import util @@ -176,6 +177,85 @@ class Runners(object): return (True, results) +class ConfigMerger(object): + def __init__(self, paths=None, datasource=None, + additional_fns=None, base_cfg=None): + self._paths = paths + self._ds = datasource + self._fns = additional_fns + self._base_cfg = base_cfg + # Created on first use + self._cfg = None + + def _get_datasource_configs(self): + d_cfgs = [] + if self._ds: + try: + ds_cfg = self._ds.get_config_obj() + if ds_cfg and isinstance(ds_cfg, (dict)): + d_cfgs.append(ds_cfg) + except: + util.logexc(LOG, ("Failed loading of datasource" + " config object from %s"), self._ds) + return d_cfgs + + def _get_env_configs(self): + e_cfgs = [] + if CFG_ENV_NAME in os.environ: + e_fn = os.environ[CFG_ENV_NAME] + try: + e_cfgs.append(util.read_conf(e_fn)) + except: + util.logexc(LOG, ('Failed loading of env. config' + ' from %s'), e_fn) + return e_cfgs + + def _get_instance_configs(self): + i_cfgs = [] + # If cloud-config was written, pick it up as + # a configuration file to use when running... + if not self._paths: + return i_cfgs + cc_fn = self._paths.get_ipath_cur('cloud_config') + if cc_fn and os.path.isfile(cc_fn): + try: + i_cfgs.append(util.read_conf(cc_fn)) + except: + util.logexc(LOG, ('Failed loading of cloud-config' + ' from %s'), cc_fn) + return i_cfgs + + def _read_cfg(self): + # Input config files override + # env config files which + # override instance configs + # which override datasource + # configs which override + # base configuration + cfgs = [] + if self._fns: + for c_fn in self._fns: + try: + cfgs.append(util.read_conf(c_fn)) + except: + util.logexc(LOG, ("Failed loading of configuration" + " from %s"), c_fn) + + cfgs.extend(self._get_env_configs()) + cfgs.extend(self._get_instance_configs()) + cfgs.extend(self._get_datasource_configs()) + if self._base_cfg: + cfgs.append(self._base_cfg) + return util.mergemanydict(cfgs) + + @property + def cfg(self): + # None check to avoid empty case causing re-reading + if self._cfg is None: + self._cfg = self._read_cfg() + return self._cfg + + class ContentHandlers(object): def __init__(self): diff --git a/cloudinit/stages.py b/cloudinit/stages.py index cfe1c071..1846dd4d 100644 --- a/cloudinit/stages.py +++ b/cloudinit/stages.py @@ -147,23 +147,16 @@ class Init(object): LOG.debug("Loaded 'init' config %s", self._cfg) def _read_cfg(self, extra_fns): - # Read extra files provided (if any) - i_cfgs = [] - if extra_fns: - for fn in extra_fns: - try: - fn_cfg = util.read_conf(fn) - i_cfgs.append(fn_cfg) - except: - util.logexc(LOG, ("Failed loading of additional" - " configuration from %s"), fn) - # Now read in the built-in + base try: - conf = util.get_base_cfg(builtin=util.get_builtin_cfg()) + base_conf = util.get_base_cfg(builtin=util.get_builtin_cfg()) except Exception: - conf = util.get_builtin_cfg() - i_cfgs.append(conf) - return util.mergemanydict(i_cfgs) + base_conf = util.get_builtin_cfg() + no_cfg_pths = helpers.Paths({}, self.datasource) + merger = helpers.ConfigMerger(paths=no_cfg_pths, + datasource=self.datasource, + additional_fns=extra_fns, + base_cfg=base_conf) + return merger.cfg def _restore_from_cache(self): pickled_fn = self.paths.get_ipath_cur('obj_pkl') @@ -392,46 +385,24 @@ class Init(object): class Modules(object): def __init__(self, init, cfg_files=None): - self.datasource = init.datasource - self.cfg_files = cfg_files - self.base_cfg = init.cfg self.init = init + self.cfg_files = cfg_files # Created on first use self._cached_cfg = None @property def cfg(self): - # None check to avoid empty case + # None check to avoid empty case causing re-reading if self._cached_cfg is None: - self._cached_cfg = self._get_config() + merger = helpers.ConfigMerger(paths=self.init.paths, + datasource=self.init.datasource, + fns=self.cfg_files, + base_cfg=self.init.cfg) + self._cached_cfg = merger.cfg LOG.debug("Loading 'module' config %s", self._cached_cfg) # Only give out a copy so that others can't modify this... return copy.deepcopy(self._cached_cfg) - def _get_config(self): - t_cfgs = [] - if self.cfg_files: - for fn in self.cfg_files: - try: - t_cfgs.append(util.read_conf(fn)) - except: - util.logexc(LOG, ("Failed loading of configuration" - " from %s"), fn) - - if self.datasource: - try: - d_cfg = self.datasource.get_config_obj() - if d_cfg: - t_cfgs.append(d_cfg) - except: - util.logexc(LOG, ("Failed loading of datasource" - " config object from %s"), self.datasource) - - if self.base_cfg: - t_cfgs.append(self.base_cfg) - - return util.mergemanydict(t_cfgs) - def _read_modules(self, name): module_list = [] if name not in self.cfg: -- cgit v1.2.3 From cb7274fa1ded413b0c5a19152ddf6e791aba98cf Mon Sep 17 00:00:00 2001 From: Joshua Harlow Date: Thu, 28 Jun 2012 15:13:19 -0700 Subject: 1. Update with smosers code review and comments (and put some of those comments into the files) 2. Rename consume() to consume_userdata() as it helps in figuring out what this does. 3. Fixup the tests due to #2 --- bin/cloud-init | 49 ++++++++++++++++++++++++++++------------ cloudinit/stages.py | 19 ++++++++-------- tests/unittests/test_userdata.py | 10 ++++---- 3 files changed, 49 insertions(+), 29 deletions(-) (limited to 'cloudinit/stages.py') diff --git a/bin/cloud-init b/bin/cloud-init index 0b879876..d3ef092f 100755 --- a/bin/cloud-init +++ b/bin/cloud-init @@ -34,13 +34,15 @@ if os.path.exists(os.path.join(possible_topdir, "cloudinit", "__init__.py")): from cloudinit import log as logging from cloudinit import netinfo -from cloudinit import settings from cloudinit import sources from cloudinit import stages from cloudinit import templater from cloudinit import util from cloudinit import version +from cloudinit.settings import (PER_INSTANCE, PER_ALWAYS, PER_ONCE, + CLOUD_CONFIG) + # Module section template MOD_SECTION_TPL = "cloud_%s_modules" @@ -54,9 +56,9 @@ QUERY_DATA_TYPES = [ # Frequency shortname to full name FREQ_SHORT_NAMES = { - 'instance': settings.PER_INSTANCE, - 'always': settings.PER_ALWAYS, - 'once': settings.PER_ONCE, + 'instance': PER_INSTANCE, + 'always': PER_ALWAYS, + 'once': PER_ONCE, } LOG = logging.getLogger() @@ -111,8 +113,15 @@ def main_init(name, args): deps = [sources.DEP_FILESYSTEM] if not args.local: - # TODO: What is this for?? - root_name = "%s.d" % (settings.CLOUD_CONFIG) + # See doc/kernel-cmdline.txt + # + # This is used in maas datasource, in "ephemeral" (read-only root) + # environment where the instance netboots to iscsi ro root. + # and the entity that controls the pxe config has to configure + # the maas datasource. + # + # Could be used elsewhere, only works on network based (not local). + root_name = "%s.d" % (CLOUD_CONFIG) target_fn = os.path.join(root_name, "91_kernel_cmdline_url.cfg") util.read_write_cmdline_url(target_fn) @@ -194,22 +203,34 @@ def main_init(name, args): init.fetch() except sources.DataSourceNotFoundException: util.logexc(LOG, "No instance datasource found!") - # TODO: Return 0 or 1?? - return 1 + # In the case of cloud-init (net mode) it is a bit + # more likely that the user would consider it + # failure if nothing was found. When using + # upstart it will also mentions job failure + # in console log if exit code is != 0. + if args.local: + return 0 + else: + return 1 # Stage 6 iid = init.instancify() LOG.debug("%s will now be targeting instance id: %s", name, iid) init.update() # Stage 7 try: + # Attempt to consume the data per instance. + # This may run user-data handlers and/or perform + # url downloads and such as needed. (ran, _results) = init.cloudify().run('consume_userdata', - init.consume, - args=[settings.PER_INSTANCE], - freq=settings.PER_INSTANCE) + init.consume_userdata, + args=[PER_INSTANCE], + freq=PER_INSTANCE) if not ran: - # Just consume anything that is set to run per - # always if nothing ran in the per instance section - init.consume(settings.PER_ALWAYS) + # Just consume anything that is set to run per-always + # if nothing ran in the per-instance code + # + # TODO: should this always happen?? (even if the above runs?) + init.consume_userdata(PER_ALWAYS) except Exception: util.logexc(LOG, "Consuming user data failed!") return 1 diff --git a/cloudinit/stages.py b/cloudinit/stages.py index 47f6e188..6689e4c9 100644 --- a/cloudinit/stages.py +++ b/cloudinit/stages.py @@ -192,13 +192,13 @@ class Init(object): cfg_list = self.cfg.get('datasource_list') or [] return (cfg_list, pkg_list) - def _get_data_source(self, local_only=False): + def _get_data_source(self): if self.datasource: return self.datasource ds = self._restore_from_cache() if ds: LOG.debug("Restored from cache, datasource: %s", ds) - if not ds and not local_only: + if not ds: (cfg_list, pkg_list) = self._get_datasources() # Deep copy so that user-data handlers can not modify # (which will affect user-data handlers down the line...) @@ -209,11 +209,10 @@ class Init(object): cfg_list, pkg_list) LOG.debug("Loaded datasource %s - %s", dsname, ds) - if ds: - self.datasource = ds - # Ensure we adjust our path members datasource - # now that we have one (thus allowing ipath to be used) - self.paths.datasource = ds + self.datasource = ds + # Ensure we adjust our path members datasource + # now that we have one (thus allowing ipath to be used) + self.paths.datasource = ds return ds def _get_instance_subdirs(self): @@ -275,8 +274,8 @@ class Init(object): "%s\n" % (previous_iid)) return iid - def fetch(self, local_only=False): - return self._get_data_source(local_only) + def fetch(self): + return self._get_data_source() def instancify(self): return self._reflect_cur_instance() @@ -312,7 +311,7 @@ class Init(object): ] return def_handlers - def consume(self, frequency=PER_INSTANCE): + def consume_userdata(self, frequency=PER_INSTANCE): cdir = self.paths.get_cpath("handlers") idir = self._get_ipath("handlers") diff --git a/tests/unittests/test_userdata.py b/tests/unittests/test_userdata.py index eeddde7d..861642b6 100644 --- a/tests/unittests/test_userdata.py +++ b/tests/unittests/test_userdata.py @@ -68,7 +68,7 @@ class TestConsumeUserData(MockerTestCase): log_file = self.capture_log(logging.WARNING) ci.fetch() - ci.consume() + ci.consume_userdata() self.assertIn( "Unhandled non-multipart (text/x-not-multipart) userdata:", log_file.getvalue()) @@ -85,7 +85,7 @@ class TestConsumeUserData(MockerTestCase): log_file = self.capture_log(logging.WARNING) ci.fetch() - ci.consume() + ci.consume_userdata() self.assertIn( "Unhandled unknown content-type (text/plain)", log_file.getvalue()) @@ -104,7 +104,7 @@ class TestConsumeUserData(MockerTestCase): log_file = self.capture_log(logging.WARNING) ci.fetch() - ci.consume() + ci.consume_userdata() self.assertEqual("", log_file.getvalue()) def test_mime_text_x_shellscript(self): @@ -122,7 +122,7 @@ class TestConsumeUserData(MockerTestCase): log_file = self.capture_log(logging.WARNING) ci.fetch() - ci.consume() + ci.consume_userdata() self.assertEqual("", log_file.getvalue()) def test_mime_text_plain_shell(self): @@ -140,5 +140,5 @@ class TestConsumeUserData(MockerTestCase): log_file = self.capture_log(logging.WARNING) ci.fetch() - ci.consume() + ci.consume_userdata() self.assertEqual("", log_file.getvalue()) -- cgit v1.2.3 From 5ec0f4d0e7e00b9124b85a8b9f9d0ce94fa06388 Mon Sep 17 00:00:00 2001 From: Joshua Harlow Date: Fri, 29 Jun 2012 12:03:05 -0700 Subject: Fix the variable name, should have been 'additional_fns' --- cloudinit/stages.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'cloudinit/stages.py') diff --git a/cloudinit/stages.py b/cloudinit/stages.py index 6689e4c9..4fcc66e4 100644 --- a/cloudinit/stages.py +++ b/cloudinit/stages.py @@ -399,7 +399,7 @@ class Modules(object): if self._cached_cfg is None: merger = helpers.ConfigMerger(paths=self.init.paths, datasource=self.init.datasource, - fns=self.cfg_files, + additional_fns=self.cfg_files, base_cfg=self.init.cfg) self._cached_cfg = merger.cfg LOG.debug("Loading 'module' config %s", self._cached_cfg) -- cgit v1.2.3 From 197d7d4fee1d402cda0e2054ae48247b4c361f53 Mon Sep 17 00:00:00 2001 From: Joshua Harlow Date: Fri, 29 Jun 2012 12:18:54 -0700 Subject: Cleanup the pickling. 1. When loading the pickled file, don't log if it isn't there a. Do log though if it is there and unpickling fails 2. On writing log if pickling fails and log if pickle writing fails. --- cloudinit/stages.py | 30 +++++++++++++++++++++--------- 1 file changed, 21 insertions(+), 9 deletions(-) (limited to 'cloudinit/stages.py') diff --git a/cloudinit/stages.py b/cloudinit/stages.py index 4fcc66e4..ef89e77c 100644 --- a/cloudinit/stages.py +++ b/cloudinit/stages.py @@ -159,15 +159,23 @@ class Init(object): return merger.cfg def _restore_from_cache(self): + # We try to restore from a current link and static path + # by using the instance link, if purge_cache was called + # the file wont exist. pickled_fn = self.paths.get_ipath_cur('obj_pkl') + pickle_contents = None try: - # we try to restore from a current link and static path - # by using the instance link, if purge_cache was called - # the file wont exist - return pickle.loads(util.load_file(pickled_fn)) + pickle_contents = util.load_file(pickled_fn) except Exception: - util.logexc(LOG, "Failed loading pickled datasource from %s", - pickled_fn) + pass + # This is expected so just return nothing + # successfully loaded... + if not pickle_contents: + return None + try: + return pickle.loads(pickle_contents) + except Exception: + util.logexc(LOG, "Failed loading pickled blob from %s", pickled_fn) return None def _write_to_cache(self): @@ -175,12 +183,16 @@ class Init(object): return False pickled_fn = self.paths.get_ipath_cur("obj_pkl") try: - contents = pickle.dumps(self.datasource) - util.write_file(pickled_fn, contents, mode=0400) - return True + pk_contents = pickle.dumps(self.datasource) + except Exception: + util.logexc(LOG, "Failed pickling datasource %s", self.datasource) + return False + try: + util.write_file(pickled_fn, pk_contents, mode=0400) except Exception: util.logexc(LOG, "Failed pickling datasource to %s", pickled_fn) return False + return True def _get_datasources(self): # Any config provided??? -- cgit v1.2.3 From 196c5e770ed1613803646a0177e144a3a4c2813e Mon Sep 17 00:00:00 2001 From: Joshua Harlow Date: Fri, 29 Jun 2012 14:01:26 -0700 Subject: Remove logging of module/init configs --- cloudinit/stages.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'cloudinit/stages.py') diff --git a/cloudinit/stages.py b/cloudinit/stages.py index ef89e77c..2f175934 100644 --- a/cloudinit/stages.py +++ b/cloudinit/stages.py @@ -144,7 +144,7 @@ class Init(object): # None check so that we don't keep on re-loading if empty if self._cfg is None: self._cfg = self._read_cfg(extra_fns) - LOG.debug("Loaded 'init' config %s", self._cfg) + # LOG.debug("Loaded 'init' config %s", self._cfg) def _read_cfg(self, extra_fns): try: @@ -414,7 +414,7 @@ class Modules(object): additional_fns=self.cfg_files, base_cfg=self.init.cfg) self._cached_cfg = merger.cfg - LOG.debug("Loading 'module' config %s", self._cached_cfg) + # LOG.debug("Loading 'module' config %s", self._cached_cfg) # Only give out a copy so that others can't modify this... return copy.deepcopy(self._cached_cfg) -- cgit v1.2.3 From 25712b40dfbe197c1ad8bdc783a2235e3c87c7de Mon Sep 17 00:00:00 2001 From: harlowja Date: Sun, 1 Jul 2012 12:08:08 -0700 Subject: 1. Rename util functions to is_true and is_false 2. Move the config loading functions to where they are used (in stages) 3. Adjust cc_set_passwords to use the is_true and is_false renamed functions 4. Adjust the init stage to have a _read_base_config function used to load the base 'initial' configuration from the following locations a. Kernel cmdline b. Conf.d location (+ the cloud.cfg location) c. Built-in configuration --- cloudinit/config/cc_set_passwords.py | 4 +-- cloudinit/stages.py | 27 +++++++++++------ cloudinit/util.py | 56 +++++++++++------------------------- 3 files changed, 37 insertions(+), 50 deletions(-) (limited to 'cloudinit/stages.py') diff --git a/cloudinit/config/cc_set_passwords.py b/cloudinit/config/cc_set_passwords.py index 5b72224b..ab266741 100644 --- a/cloudinit/config/cc_set_passwords.py +++ b/cloudinit/config/cc_set_passwords.py @@ -96,9 +96,9 @@ def handle(_name, cfg, cloud, log, args): pw_auth = None if 'ssh_pwauth' in cfg: change_pwauth = True - if util.is_true_str(cfg['ssh_pwauth']): + if util.is_true(cfg['ssh_pwauth']): pw_auth = 'yes' - if util.is_false_str(cfg['ssh_pwauth']): + if util.is_false(cfg['ssh_pwauth']): pw_auth = 'no' if change_pwauth: diff --git a/cloudinit/stages.py b/cloudinit/stages.py index 2f175934..79663b27 100644 --- a/cloudinit/stages.py +++ b/cloudinit/stages.py @@ -26,7 +26,7 @@ import copy import os import sys -from cloudinit.settings import (PER_INSTANCE, FREQUENCIES) +from cloudinit.settings import (PER_INSTANCE, FREQUENCIES, CLOUD_CONFIG) from cloudinit import handlers @@ -146,16 +146,27 @@ class Init(object): self._cfg = self._read_cfg(extra_fns) # LOG.debug("Loaded 'init' config %s", self._cfg) + def _read_base_cfg(self): + base_cfgs = [] + default_cfg = util.get_builtin_cfg() + kern_contents = util.read_cc_from_cmdline() + # Kernel/cmdline parameters override system config + if kern_contents: + base_cfgs.append(util.load_yaml(kern_contents, default={})) + # Anything in your conf.d location?? + if os.path.isfile(CLOUD_CONFIG): + base_cfgs.append(util.read_conf_with_confd(CLOUD_CONFIG)) + # And finally the default gets to play + if default_cfg: + base_cfgs.append(default_cfg) + return util.mergemanydict(base_cfgs) + def _read_cfg(self, extra_fns): - try: - base_conf = util.get_base_cfg(builtin=util.get_builtin_cfg()) - except Exception: - base_conf = util.get_builtin_cfg() - no_cfg_pths = helpers.Paths({}, self.datasource) - merger = helpers.ConfigMerger(paths=no_cfg_pths, + no_cfg_paths = helpers.Paths({}, self.datasource) + merger = helpers.ConfigMerger(paths=no_cfg_paths, datasource=self.datasource, additional_fns=extra_fns, - base_cfg=base_conf) + base_cfg=self._read_base_cfg()) return merger.cfg def _restore_from_cache(self): diff --git a/cloudinit/util.py b/cloudinit/util.py index 0c592656..f07d22e7 100644 --- a/cloudinit/util.py +++ b/cloudinit/util.py @@ -50,7 +50,7 @@ from cloudinit import importer from cloudinit import log as logging from cloudinit import url_helper as uhelp -from cloudinit.settings import (CFG_BUILTIN, CLOUD_CONFIG) +from cloudinit.settings import (CFG_BUILTIN) LOG = logging.getLogger(__name__) @@ -192,7 +192,9 @@ def fork_cb(child_cb, *args): fid, obj_name(child_cb)) -def is_true_str(val, addons=None): +def is_true(val, addons=None): + if isinstance(val, (bool)): + return val is True check_set = ['true', '1', 'on', 'yes'] if addons: check_set = check_set + addons @@ -201,7 +203,9 @@ def is_true_str(val, addons=None): return False -def is_false_str(val, addons=None): +def is_false(val, addons=None): + if isinstance(val, (bool)): + return val is False check_set = ['off', '0', 'no', 'false'] if addons: check_set = check_set + addons @@ -218,7 +222,7 @@ def translate_bool(val, addons=None): # If its already a boolean skip if isinstance(val, (bool)): return val - return is_true_str(val, addons) + return is_true(val, addons) def rand_str(strlen=32, select_from=None): @@ -285,29 +289,6 @@ def is_ipv4(instr): return (len(toks) == 4) -def merge_base_cfg(cfgfile, cfg_builtin=None): - syscfg = read_conf_with_confd(cfgfile) - - kern_contents = read_cc_from_cmdline() - kerncfg = {} - if kern_contents: - kerncfg = load_yaml(kern_contents, default={}) - - # Kernel parameters override system config - if kerncfg: - combined = mergedict(kerncfg, syscfg) - else: - combined = syscfg - - if cfg_builtin: - # Combined over-ride anything builtin - fin = mergedict(combined, cfg_builtin) - else: - fin = combined - - return fin - - def get_cfg_option_bool(yobj, key, default=False): if key not in yobj: return default @@ -622,15 +603,17 @@ def read_seeded(base="", ext="", timeout=5, retries=10, file_retries=0): def read_conf_d(confd): - # get reverse sorted list (later trumps newer) + # Get reverse sorted list (later trumps newer) confs = sorted(os.listdir(confd), reverse=True) - # remove anything not ending in '.cfg' + # Remove anything not ending in '.cfg' confs = [f for f in confs if f.endswith(".cfg")] - # remove anything not a file - confs = [f for f in confs if os.path.isfile(os.path.join(confd, f))] + # Remove anything not a file + confs = [f for f in confs + if os.path.isfile(os.path.join(confd, f))] + # Load them all so that they can be merged cfgs = [] for fn in confs: cfgs.append(read_conf(os.path.join(confd, fn))) @@ -658,7 +641,8 @@ def read_conf_with_confd(cfgfile): return cfg # Conf.d settings override input configuration - return mergedict(read_conf_d(confd), cfg) + confd_cfg = read_conf_d(confd) + return mergedict(confd_cfg, cfg) def read_cc_from_cmdline(cmdline=None): @@ -1076,14 +1060,6 @@ def ensure_dir(path, mode=None): chmod(path, mode) -def get_base_cfg(cfg_path=None, builtin=None): - if not cfg_path: - cfg_path = CLOUD_CONFIG - if not builtin: - builtin = get_builtin_cfg() - return merge_base_cfg(cfg_path, builtin) - - @contextlib.contextmanager def unmounter(umount): try: -- cgit v1.2.3 From 9e1ffb27f4f316638a1fe83678c19f7215a8ef4b Mon Sep 17 00:00:00 2001 From: harlowja Date: Sun, 1 Jul 2012 12:11:21 -0700 Subject: Always read fro the cloud config location + conf.d locations --- cloudinit/stages.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'cloudinit/stages.py') diff --git a/cloudinit/stages.py b/cloudinit/stages.py index 79663b27..8fd6aa5d 100644 --- a/cloudinit/stages.py +++ b/cloudinit/stages.py @@ -154,8 +154,8 @@ class Init(object): if kern_contents: base_cfgs.append(util.load_yaml(kern_contents, default={})) # Anything in your conf.d location?? - if os.path.isfile(CLOUD_CONFIG): - base_cfgs.append(util.read_conf_with_confd(CLOUD_CONFIG)) + # or the 'default' cloud.cfg location??? + base_cfgs.append(util.read_conf_with_confd(CLOUD_CONFIG)) # And finally the default gets to play if default_cfg: base_cfgs.append(default_cfg) -- cgit v1.2.3