From 154ad87b29344ea4d29d92f8559f61bb6efe6530 Mon Sep 17 00:00:00 2001 From: Joshua Harlow Date: Thu, 8 Nov 2012 16:30:57 -0800 Subject: Ensure that at needed stages the local variables of the init class are reset so that when they are regenerated that they will use the updated data instead of using previous data (since they weren't reset). LP: #1076811 --- cloudinit/stages.py | 31 ++++++++++++++++++++++++++++--- 1 file changed, 28 insertions(+), 3 deletions(-) (limited to 'cloudinit/stages.py') diff --git a/cloudinit/stages.py b/cloudinit/stages.py index 4ed1a750..e0cf1cbe 100644 --- a/cloudinit/stages.py +++ b/cloudinit/stages.py @@ -36,6 +36,8 @@ from cloudinit.handlers import cloud_config as cc_part from cloudinit.handlers import shell_script as ss_part from cloudinit.handlers import upstart_job as up_part +from cloudinit.sources import DataSourceNone + from cloudinit import cloud from cloudinit import config from cloudinit import distros @@ -58,8 +60,16 @@ class Init(object): self._cfg = None self._paths = None self._distro = None - # Created only when a fetch occurs - self.datasource = None + # Changed only when a fetch occurs + self.datasource = DataSourceNone.DataSourceNone({}, None, None) + + def _reset(self, ds=False): + # Recreated on access + self._cfg = None + self._paths = None + self._distro = None + if ds: + self.datasource = DataSourceNone.DataSourceNone({}, None, None) @property def distro(self): @@ -236,7 +246,7 @@ class Init(object): self.datasource = ds # Ensure we adjust our path members datasource # now that we have one (thus allowing ipath to be used) - self.paths.datasource = ds + self._reset() return ds def _get_instance_subdirs(self): @@ -296,6 +306,10 @@ class Init(object): util.write_file(iid_fn, "%s\n" % iid) util.write_file(os.path.join(dp, 'previous-instance-id'), "%s\n" % (previous_iid)) + # Ensure needed components are regenerated + # after change of instance which may cause + # change of configuration + self._reset() return iid def fetch(self): @@ -409,6 +423,17 @@ class Init(object): handlers.call_end(mod, data, frequency) called.append(mod) + # Perform post-consumption adjustments so that + # modules that run during the init stage reflect + # this consumed set. + # + # They will be recreated on future access... + self._reset() + # Note(harlowja): the 'active' datasource will have + # references to the previous config, distro, paths + # objects before the load of the userdata happened, + # this is expected. + class Modules(object): def __init__(self, init, cfg_files=None): -- cgit v1.2.3 From fe68049611e12591a632d5264d9ba1b67b0cfa0d Mon Sep 17 00:00:00 2001 From: Scott Moser Date: Mon, 12 Nov 2012 13:40:57 -0500 Subject: stages.py: fix issue that resulted in broken data source searching This just replaces the portions of stages.py that were checking or setting self.datasource to a DataSourceNone to use a static/global datasource. And then, check for None is done against that. --- cloudinit/stages.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) (limited to 'cloudinit/stages.py') diff --git a/cloudinit/stages.py b/cloudinit/stages.py index e0cf1cbe..9c231994 100644 --- a/cloudinit/stages.py +++ b/cloudinit/stages.py @@ -49,6 +49,7 @@ from cloudinit import util LOG = logging.getLogger(__name__) +NULL_DATA_SOURCE = DataSourceNone.DataSourceNone({}, None, None) class Init(object): def __init__(self, ds_deps=None): @@ -61,7 +62,7 @@ class Init(object): self._paths = None self._distro = None # Changed only when a fetch occurs - self.datasource = DataSourceNone.DataSourceNone({}, None, None) + self.datasource = NULL_DATA_SOURCE def _reset(self, ds=False): # Recreated on access @@ -69,7 +70,7 @@ class Init(object): self._paths = None self._distro = None if ds: - self.datasource = DataSourceNone.DataSourceNone({}, None, None) + self.datasource = NULL_DATA_SOURCE @property def distro(self): @@ -201,7 +202,7 @@ class Init(object): return None def _write_to_cache(self): - if not self.datasource: + if self.datasource is NULL_DATA_SOURCE: return False pickled_fn = self.paths.get_ipath_cur("obj_pkl") try: @@ -227,7 +228,7 @@ class Init(object): return (cfg_list, pkg_list) def _get_data_source(self): - if self.datasource: + if self.datasource is not NULL_DATA_SOURCE: return self.datasource ds = self._restore_from_cache() if ds: -- cgit v1.2.3 From 90af12849c373d3b729c4d7d84dbb69c2ecd885a Mon Sep 17 00:00:00 2001 From: Scott Moser Date: Mon, 12 Nov 2012 15:26:19 -0500 Subject: change the "null datasource" for stages.py to None Previous commit set up an object for checking "None" in the stages class. Here, change that to be None. The benefit of the DataSourceNone was so that the stages was generally functional even with that "None" object. The negative affect of that is that you could go down the error path much longer when using it. If we use None type, then self.datasource. will fail immediately and loudly. --- cloudinit/stages.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'cloudinit/stages.py') diff --git a/cloudinit/stages.py b/cloudinit/stages.py index 9c231994..e83d72da 100644 --- a/cloudinit/stages.py +++ b/cloudinit/stages.py @@ -49,7 +49,7 @@ from cloudinit import util LOG = logging.getLogger(__name__) -NULL_DATA_SOURCE = DataSourceNone.DataSourceNone({}, None, None) +NULL_DATA_SOURCE = None class Init(object): def __init__(self, ds_deps=None): -- cgit v1.2.3 From 9c64c5d0e01e48612fe37d3304b1f6eb70181cae Mon Sep 17 00:00:00 2001 From: Scott Moser Date: Mon, 12 Nov 2012 22:11:34 -0500 Subject: pylint and pep8 fixes --- cloudinit/stages.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) (limited to 'cloudinit/stages.py') diff --git a/cloudinit/stages.py b/cloudinit/stages.py index e83d72da..94a267df 100644 --- a/cloudinit/stages.py +++ b/cloudinit/stages.py @@ -36,8 +36,6 @@ from cloudinit.handlers import cloud_config as cc_part from cloudinit.handlers import shell_script as ss_part from cloudinit.handlers import upstart_job as up_part -from cloudinit.sources import DataSourceNone - from cloudinit import cloud from cloudinit import config from cloudinit import distros @@ -51,6 +49,7 @@ LOG = logging.getLogger(__name__) NULL_DATA_SOURCE = None + class Init(object): def __init__(self, ds_deps=None): if ds_deps is not None: -- cgit v1.2.3 From 5c5041367a8630543d84a9edf9dd4321f0c69718 Mon Sep 17 00:00:00 2001 From: Scott Moser Date: Thu, 6 Dec 2012 09:36:57 -0500 Subject: cloudinit/stages.py: separate _read_base_cfg() into static function The Init._read_base_cfg() was really a static function, this just moves it to its own static function. Its not used anywhere else at the moment. --- cloudinit/stages.py | 37 +++++++++++++++++++++---------------- 1 file changed, 21 insertions(+), 16 deletions(-) (limited to 'cloudinit/stages.py') diff --git a/cloudinit/stages.py b/cloudinit/stages.py index 94a267df..d9391f39 100644 --- a/cloudinit/stages.py +++ b/cloudinit/stages.py @@ -157,27 +157,12 @@ class Init(object): self._cfg = self._read_cfg(extra_fns) # LOG.debug("Loaded 'init' config %s", self._cfg) - def _read_base_cfg(self): - base_cfgs = [] - default_cfg = util.get_builtin_cfg() - kern_contents = util.read_cc_from_cmdline() - # Kernel/cmdline parameters override system config - if kern_contents: - base_cfgs.append(util.load_yaml(kern_contents, default={})) - # Anything in your conf.d location?? - # or the 'default' cloud.cfg location??? - base_cfgs.append(util.read_conf_with_confd(CLOUD_CONFIG)) - # And finally the default gets to play - if default_cfg: - base_cfgs.append(default_cfg) - return util.mergemanydict(base_cfgs) - def _read_cfg(self, extra_fns): no_cfg_paths = helpers.Paths({}, self.datasource) merger = helpers.ConfigMerger(paths=no_cfg_paths, datasource=self.datasource, additional_fns=extra_fns, - base_cfg=self._read_base_cfg()) + base_cfg=fetch_base_config()) return merger.cfg def _restore_from_cache(self): @@ -575,3 +560,23 @@ class Modules(object): raw_mods = self._read_modules(section_name) mostly_mods = self._fixup_modules(raw_mods) return self._run_modules(mostly_mods) + + +def fetch_base_config(): + base_cfgs = [] + default_cfg = util.get_builtin_cfg() + kern_contents = util.read_cc_from_cmdline() + + # Kernel/cmdline parameters override system config + if kern_contents: + base_cfgs.append(util.load_yaml(kern_contents, default={})) + + # Anything in your conf.d location?? + # or the 'default' cloud.cfg location??? + base_cfgs.append(util.read_conf_with_confd(CLOUD_CONFIG)) + + # And finally the default gets to play + if default_cfg: + base_cfgs.append(default_cfg) + + return util.mergemanydict(base_cfgs) -- cgit v1.2.3 From c196afccda0293613e3586922347749c33dfddbf Mon Sep 17 00:00:00 2001 From: Joshua Harlow Date: Mon, 17 Dec 2012 08:41:11 -0500 Subject: ensure a datasource's 'distro' and sys_cfg are updated After parsing and merging datasource's config, the changes in were not making it into the datasource's 'distro. The end result was that the when a config module was called, it's 'cloud' argument would be updated in 'cloud.distro', but not in 'cloud.datasource.distro'. This path was required for getting mirror settings to take affect, because they include information from the datasource. Ie: cc_apt_configure had mirror_info = cloud.datasource.get_package_mirror_info() the datasource then used *its* copy of sys_cfg to call self.distro.get_package_mirror_info and *that* distro's sys_cfg had not been updated. LP: #1090482 --- ChangeLog | 2 ++ cloudinit/stages.py | 20 +++++++++++++------- 2 files changed, 15 insertions(+), 7 deletions(-) (limited to 'cloudinit/stages.py') diff --git a/ChangeLog b/ChangeLog index e4bcdd76..64a6e618 100644 --- a/ChangeLog +++ b/ChangeLog @@ -9,6 +9,8 @@ option (LP: #1083715) - make install of puppet configurable (LP: #1090205) [Craig Tracey] - support omnibus installer for chef [Anatoliy Dobrosynets] + - fix bug where cloud-config in user-data could not modify system_info + settings (LP: #1090482) 0.7.1: - sysvinit: fix missing dependency in cloud-init job for RHEL 5.6 - config-drive: map hostname to local-hostname (LP: #1061964) diff --git a/cloudinit/stages.py b/cloudinit/stages.py index d9391f39..8d3213b4 100644 --- a/cloudinit/stages.py +++ b/cloudinit/stages.py @@ -63,23 +63,29 @@ class Init(object): # Changed only when a fetch occurs self.datasource = NULL_DATA_SOURCE - def _reset(self, ds=False): + def _reset(self, reset_ds=False): # Recreated on access self._cfg = None self._paths = None self._distro = None - if ds: + if reset_ds: self.datasource = NULL_DATA_SOURCE @property def distro(self): if not self._distro: # Try to find the right class to use - scfg = self._extract_cfg('system') - name = scfg.pop('distro', 'ubuntu') - cls = distros.fetch(name) - LOG.debug("Using distro class %s", cls) - self._distro = cls(name, scfg, self.paths) + system_config = self._extract_cfg('system') + distro_name = system_config.pop('distro', 'ubuntu') + distro_cls = distros.fetch(distro_name) + LOG.debug("Using distro class %s", distro_cls) + self._distro = distro_cls(distro_name, system_config, self.paths) + # If we have an active datasource we need to adjust + # said datasource and move its distro/system config + # from whatever it was to a new set... + if self.datasource is not NULL_DATA_SOURCE: + self.datasource.distro = self._distro + self.datasource.sys_cfg = system_config return self._distro @property -- cgit v1.2.3 From 361738c6a9a14e32bd2123828fab8d8b70c6bc3a Mon Sep 17 00:00:00 2001 From: ctracey Date: Tue, 15 Jan 2013 16:08:43 -0500 Subject: add support for operating system families often it is convenient to classify a distro as being part of an operating system family. for instance, file templates may be identical for both debian and ubuntu, but to support this under the current templating code, one would need multiple templates for the same code. similarly, configuration handlers often fall into the same bucket: the configuraton is known to work/has been tested on a particular family of operating systems. right now this is handled with a declaration like: distros = ['fedora', 'rhel'] this fix seeks to address both of these issues. it allows for the simplification of the above line to: osfamilies = ['redhat'] and provides a mechanism for operating system family templates. --- cloudinit/config/__init__.py | 4 +++- cloudinit/distros/__init__.py | 16 +++++++++++++++- cloudinit/distros/debian.py | 1 + cloudinit/distros/rhel.py | 1 + cloudinit/stages.py | 9 +++++++-- 5 files changed, 27 insertions(+), 4 deletions(-) (limited to 'cloudinit/stages.py') diff --git a/cloudinit/config/__init__.py b/cloudinit/config/__init__.py index 69a8cc68..d57453be 100644 --- a/cloudinit/config/__init__.py +++ b/cloudinit/config/__init__.py @@ -52,5 +52,7 @@ def fixup_module(mod, def_freq=PER_INSTANCE): if freq and freq not in FREQUENCIES: LOG.warn("Module %s has an unknown frequency %s", mod, freq) if not hasattr(mod, 'distros'): - setattr(mod, 'distros', None) + setattr(mod, 'distros', []) + if not hasattr(mod, 'osfamilies'): + setattr(mod, 'osfamilies', []) return mod diff --git a/cloudinit/distros/__init__.py b/cloudinit/distros/__init__.py index 38b2f829..ff325b40 100644 --- a/cloudinit/distros/__init__.py +++ b/cloudinit/distros/__init__.py @@ -35,6 +35,11 @@ from cloudinit import util from cloudinit.distros.parsers import hosts +OSFAMILIES = { + 'debian': ['debian', 'ubuntu'], + 'redhat': ['fedora', 'rhel'] +} + LOG = logging.getLogger(__name__) @@ -143,6 +148,16 @@ class Distro(object): def _select_hostname(self, hostname, fqdn): raise NotImplementedError() + @staticmethod + def expand_osfamily(family_list): + distros = [] + for family in family_list: + if not family in OSFAMILIES: + raise ValueError("No distibutions found for osfamily %s" + % (family)) + distros.extend(OSFAMILIES[family]) + return distros + def update_hostname(self, hostname, fqdn, prev_hostname_fn): applying_hostname = hostname @@ -515,7 +530,6 @@ def _get_package_mirror_info(mirror_info, availability_zone=None, return results - def _get_arch_package_mirror_info(package_mirrors, arch): # pull out the specific arch from a 'package_mirrors' config option default = None diff --git a/cloudinit/distros/debian.py b/cloudinit/distros/debian.py index 7422f4f0..49b73477 100644 --- a/cloudinit/distros/debian.py +++ b/cloudinit/distros/debian.py @@ -48,6 +48,7 @@ class Distro(distros.Distro): # calls from repeatly happening (when they # should only happen say once per instance...) self._runner = helpers.Runners(paths) + self.osfamily = 'debian' def apply_locale(self, locale, out_fn=None): if not out_fn: diff --git a/cloudinit/distros/rhel.py b/cloudinit/distros/rhel.py index bc0877d5..e65be8d7 100644 --- a/cloudinit/distros/rhel.py +++ b/cloudinit/distros/rhel.py @@ -60,6 +60,7 @@ class Distro(distros.Distro): # calls from repeatly happening (when they # should only happen say once per instance...) self._runner = helpers.Runners(paths) + self.osfamily = 'redhat' def install_packages(self, pkglist): self.package_command('install', pkglist) diff --git a/cloudinit/stages.py b/cloudinit/stages.py index 8d3213b4..d7d1dea0 100644 --- a/cloudinit/stages.py +++ b/cloudinit/stages.py @@ -529,11 +529,16 @@ class Modules(object): freq = mod.frequency if not freq in FREQUENCIES: freq = PER_INSTANCE - worked_distros = mod.distros + + worked_distros = set(mod.distros) + worked_distros.update( + distros.Distro.expand_osfamily(mod.osfamilies)) + if (worked_distros and d_name not in worked_distros): LOG.warn(("Module %s is verified on %s distros" " but not on %s distro. It may or may not work" - " correctly."), name, worked_distros, d_name) + " correctly."), name, list(worked_distros), + d_name) # Use the configs logger and not our own # TODO(harlowja): possibly check the module # for having a LOG attr and just give it back -- cgit v1.2.3