From 15a33d190f2a9247accf8834b005521c615cb6b3 Mon Sep 17 00:00:00 2001 From: Joshua Harlow Date: Sat, 5 Jan 2013 10:04:58 -0800 Subject: Make which fields are redacted come from a field array. LP: #1096417 --- cloudinit/distros/__init__.py | 18 +++++++++++------- 1 file changed, 11 insertions(+), 7 deletions(-) (limited to 'cloudinit/distros') diff --git a/cloudinit/distros/__init__.py b/cloudinit/distros/__init__.py index 6a684b89..8a3e0570 100644 --- a/cloudinit/distros/__init__.py +++ b/cloudinit/distros/__init__.py @@ -297,22 +297,26 @@ class Distro(object): "no_create_home": "-M", } + redact_fields = ['passwd'] + # Now check the value and create the command for option in kwargs: value = kwargs[option] if option in adduser_opts and value \ and isinstance(value, str): adduser_cmd.extend([adduser_opts[option], value]) - - # Redact the password field from the logs - if option != "password": - x_adduser_cmd.extend([adduser_opts[option], value]) - else: + # Redact certain fields from the logs + if option in redact_fields: x_adduser_cmd.extend([adduser_opts[option], 'REDACTED']) - + else: + x_adduser_cmd.extend([adduser_opts[option], value]) elif option in adduser_opts_flags and value: adduser_cmd.append(adduser_opts_flags[option]) - x_adduser_cmd.append(adduser_opts_flags[option]) + # Redact certain fields from the logs + if option in redact_fields: + x_adduser_cmd.append('REDACTED') + else: + x_adduser_cmd.append(adduser_opts_flags[option]) # Default to creating home directory unless otherwise directed # Also, we do not create home directories for system users. -- cgit v1.2.3 From 6cfd12c96608eb5fd086da49c4c685635e40e6e0 Mon Sep 17 00:00:00 2001 From: Joshua Harlow Date: Sat, 5 Jan 2013 10:18:01 -0800 Subject: Fix the password locking logic. Instead of only not locking when system is present the logic should handle the correct case when lock password is set and system is not present. LP: #1096423 --- cloudinit/distros/__init__.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) (limited to 'cloudinit/distros') diff --git a/cloudinit/distros/__init__.py b/cloudinit/distros/__init__.py index 6a684b89..be32757d 100644 --- a/cloudinit/distros/__init__.py +++ b/cloudinit/distros/__init__.py @@ -335,9 +335,10 @@ class Distro(object): self.set_passwd(name, kwargs['plain_text_passwd']) # Default locking down the account. - if ('lock_passwd' not in kwargs and - ('lock_passwd' in kwargs and kwargs['lock_passwd']) or - 'system' not in kwargs): + # + # Which means if lock_passwd is False (on non-existent its true) + # then lock or if system is True (on non-existent its false) then lock. + if (kwargs.get('lock_passwd', True) or kwargs.get('system', False)): try: util.subp(['passwd', '--lock', name]) except Exception as e: -- cgit v1.2.3 From 361738c6a9a14e32bd2123828fab8d8b70c6bc3a Mon Sep 17 00:00:00 2001 From: ctracey Date: Tue, 15 Jan 2013 16:08:43 -0500 Subject: add support for operating system families often it is convenient to classify a distro as being part of an operating system family. for instance, file templates may be identical for both debian and ubuntu, but to support this under the current templating code, one would need multiple templates for the same code. similarly, configuration handlers often fall into the same bucket: the configuraton is known to work/has been tested on a particular family of operating systems. right now this is handled with a declaration like: distros = ['fedora', 'rhel'] this fix seeks to address both of these issues. it allows for the simplification of the above line to: osfamilies = ['redhat'] and provides a mechanism for operating system family templates. --- cloudinit/config/__init__.py | 4 +++- cloudinit/distros/__init__.py | 16 +++++++++++++++- cloudinit/distros/debian.py | 1 + cloudinit/distros/rhel.py | 1 + cloudinit/stages.py | 9 +++++++-- 5 files changed, 27 insertions(+), 4 deletions(-) (limited to 'cloudinit/distros') diff --git a/cloudinit/config/__init__.py b/cloudinit/config/__init__.py index 69a8cc68..d57453be 100644 --- a/cloudinit/config/__init__.py +++ b/cloudinit/config/__init__.py @@ -52,5 +52,7 @@ def fixup_module(mod, def_freq=PER_INSTANCE): if freq and freq not in FREQUENCIES: LOG.warn("Module %s has an unknown frequency %s", mod, freq) if not hasattr(mod, 'distros'): - setattr(mod, 'distros', None) + setattr(mod, 'distros', []) + if not hasattr(mod, 'osfamilies'): + setattr(mod, 'osfamilies', []) return mod diff --git a/cloudinit/distros/__init__.py b/cloudinit/distros/__init__.py index 38b2f829..ff325b40 100644 --- a/cloudinit/distros/__init__.py +++ b/cloudinit/distros/__init__.py @@ -35,6 +35,11 @@ from cloudinit import util from cloudinit.distros.parsers import hosts +OSFAMILIES = { + 'debian': ['debian', 'ubuntu'], + 'redhat': ['fedora', 'rhel'] +} + LOG = logging.getLogger(__name__) @@ -143,6 +148,16 @@ class Distro(object): def _select_hostname(self, hostname, fqdn): raise NotImplementedError() + @staticmethod + def expand_osfamily(family_list): + distros = [] + for family in family_list: + if not family in OSFAMILIES: + raise ValueError("No distibutions found for osfamily %s" + % (family)) + distros.extend(OSFAMILIES[family]) + return distros + def update_hostname(self, hostname, fqdn, prev_hostname_fn): applying_hostname = hostname @@ -515,7 +530,6 @@ def _get_package_mirror_info(mirror_info, availability_zone=None, return results - def _get_arch_package_mirror_info(package_mirrors, arch): # pull out the specific arch from a 'package_mirrors' config option default = None diff --git a/cloudinit/distros/debian.py b/cloudinit/distros/debian.py index 7422f4f0..49b73477 100644 --- a/cloudinit/distros/debian.py +++ b/cloudinit/distros/debian.py @@ -48,6 +48,7 @@ class Distro(distros.Distro): # calls from repeatly happening (when they # should only happen say once per instance...) self._runner = helpers.Runners(paths) + self.osfamily = 'debian' def apply_locale(self, locale, out_fn=None): if not out_fn: diff --git a/cloudinit/distros/rhel.py b/cloudinit/distros/rhel.py index bc0877d5..e65be8d7 100644 --- a/cloudinit/distros/rhel.py +++ b/cloudinit/distros/rhel.py @@ -60,6 +60,7 @@ class Distro(distros.Distro): # calls from repeatly happening (when they # should only happen say once per instance...) self._runner = helpers.Runners(paths) + self.osfamily = 'redhat' def install_packages(self, pkglist): self.package_command('install', pkglist) diff --git a/cloudinit/stages.py b/cloudinit/stages.py index 8d3213b4..d7d1dea0 100644 --- a/cloudinit/stages.py +++ b/cloudinit/stages.py @@ -529,11 +529,16 @@ class Modules(object): freq = mod.frequency if not freq in FREQUENCIES: freq = PER_INSTANCE - worked_distros = mod.distros + + worked_distros = set(mod.distros) + worked_distros.update( + distros.Distro.expand_osfamily(mod.osfamilies)) + if (worked_distros and d_name not in worked_distros): LOG.warn(("Module %s is verified on %s distros" " but not on %s distro. It may or may not work" - " correctly."), name, worked_distros, d_name) + " correctly."), name, list(worked_distros), + d_name) # Use the configs logger and not our own # TODO(harlowja): possibly check the module # for having a LOG attr and just give it back -- cgit v1.2.3 From 5d4f4df6804995d74e7962f60dcd72b26bcac69b Mon Sep 17 00:00:00 2001 From: ctracey Date: Tue, 15 Jan 2013 16:25:20 -0500 Subject: cleanup a pep8 failure accidentally removed a line between two functions. --- cloudinit/distros/__init__.py | 1 + 1 file changed, 1 insertion(+) (limited to 'cloudinit/distros') diff --git a/cloudinit/distros/__init__.py b/cloudinit/distros/__init__.py index ff325b40..5a2092c0 100644 --- a/cloudinit/distros/__init__.py +++ b/cloudinit/distros/__init__.py @@ -530,6 +530,7 @@ def _get_package_mirror_info(mirror_info, availability_zone=None, return results + def _get_arch_package_mirror_info(package_mirrors, arch): # pull out the specific arch from a 'package_mirrors' config option default = None -- cgit v1.2.3 From baefd17a9d997e11f85bf89d9337c2d40748bc37 Mon Sep 17 00:00:00 2001 From: Joshua Harlow Date: Fri, 18 Jan 2013 10:57:20 -0800 Subject: Adjust how the legacy user: XYZ config alters the normalized user list Previously if a legacy user: XYZ entry was found, XYZ would not automatically be promoted to the default user but would instead just be added on as a new entry to the normalized user list. It appears the behavior that is wanted is for the XYZ entry to be added on as the default user (thus overriding a distro provided default user), which better matches how the code previous worked. LP: #1100920 --- cloudinit/distros/__init__.py | 67 +++++++++++++++------- .../test_distros/test_user_data_normalize.py | 10 +++- 2 files changed, 52 insertions(+), 25 deletions(-) (limited to 'cloudinit/distros') diff --git a/cloudinit/distros/__init__.py b/cloudinit/distros/__init__.py index 38b2f829..c74be4e2 100644 --- a/cloudinit/distros/__init__.py +++ b/cloudinit/distros/__init__.py @@ -705,41 +705,64 @@ def _normalize_users(u_cfg, def_user_cfg=None): def normalize_users_groups(cfg, distro): if not cfg: cfg = {} + users = {} groups = {} if 'groups' in cfg: groups = _normalize_groups(cfg['groups']) - # Handle the previous style of doing this... + # Handle the previous style of doing this where the first user + # overrides the concept of the default user if provided in the user: XYZ + # format. old_user = None if 'user' in cfg and cfg['user']: - old_user = str(cfg['user']) - if not 'users' in cfg: - cfg['users'] = old_user + old_user = cfg['user'] + # Translate it into the format that is more useful + # going forward + if isinstance(old_user, (basestring, str)): + old_user = { + 'name': old_user, + } + if not isinstance(old_user, (dict)): + LOG.warn(("Format for 'user:' key must be a string or " + "dictionary and not %s"), util.obj_name(old_user)) old_user = None - if 'users' in cfg: - default_user_config = None + + default_user_config = None + if not old_user: + # If no old user format, then assume the distro + # provides what the 'default' user maps to, but notice + # that if this is provided, we won't automatically inject + # a 'default' user into the users list, while if a old user + # format is provided we will. try: default_user_config = distro.get_default_user() except NotImplementedError: LOG.warn(("Distro has not implemented default user " "access. No default user will be normalized.")) - base_users = cfg['users'] - if old_user: - if isinstance(base_users, (list)): - if len(base_users): - # The old user replaces user[0] - base_users[0] = {'name': old_user} - else: - # Just add it on at the end... - base_users.append({'name': old_user}) - elif isinstance(base_users, (dict)): - if old_user not in base_users: - base_users[old_user] = True - elif isinstance(base_users, (str, basestring)): - # Just append it on to be re-parsed later - base_users += ",%s" % (old_user) - users = _normalize_users(base_users, default_user_config) + else: + default_user_config = dict(old_user) + + base_users = cfg.get('users', []) + if not isinstance(base_users, (list, dict, str, basestring)): + LOG.warn(("Format for 'users:' key must be a comma separated string" + " or a dictionary or a list and not %s"), + util.obj_name(base_users)) + base_users = [] + + if old_user: + # Ensure that when user: is provided that this user + # always gets added (as the default user) + if isinstance(base_users, (list)): + # Just add it on at the end... + base_users.append({'name': 'default'}) + elif isinstance(base_users, (dict)): + base_users['default'] = base_users.get('default', True) + elif isinstance(base_users, (str, basestring)): + # Just append it on to be re-parsed later + base_users += ",default" + + users = _normalize_users(base_users, default_user_config) return (users, groups) diff --git a/tests/unittests/test_distros/test_user_data_normalize.py b/tests/unittests/test_distros/test_user_data_normalize.py index 5d9d4311..50398c74 100644 --- a/tests/unittests/test_distros/test_user_data_normalize.py +++ b/tests/unittests/test_distros/test_user_data_normalize.py @@ -173,26 +173,29 @@ class TestUGNormalize(MockerTestCase): 'users': 'default' } (users, _groups) = self._norm(ug_cfg, distro) - self.assertIn('bob', users) + self.assertNotIn('bob', users) # Bob is not the default now, zetta is self.assertIn('zetta', users) + self.assertTrue(users['zetta']['default']) self.assertNotIn('default', users) ug_cfg = { 'user': 'zetta', 'users': 'default, joe' } (users, _groups) = self._norm(ug_cfg, distro) - self.assertIn('bob', users) + self.assertNotIn('bob', users) # Bob is not the default now, zetta is self.assertIn('joe', users) self.assertIn('zetta', users) + self.assertTrue(users['zetta']['default']) self.assertNotIn('default', users) ug_cfg = { 'user': 'zetta', 'users': ['bob', 'joe'] } (users, _groups) = self._norm(ug_cfg, distro) - self.assertNotIn('bob', users) + self.assertIn('bob', users) self.assertIn('joe', users) self.assertIn('zetta', users) + self.assertTrue(users['zetta']['default']) ug_cfg = { 'user': 'zetta', 'users': { @@ -204,6 +207,7 @@ class TestUGNormalize(MockerTestCase): self.assertIn('bob', users) self.assertIn('joe', users) self.assertIn('zetta', users) + self.assertTrue(users['zetta']['default']) ug_cfg = { 'user': 'zetta', } -- cgit v1.2.3 From 06ca24c39289f2d1f0f3f810abf155043a36d2f2 Mon Sep 17 00:00:00 2001 From: harlowja Date: Sat, 19 Jan 2013 17:51:24 -0800 Subject: Merge the old user style with the distro provided config. When the old user: style entry is found, don't forget that we need to use the distro settings that are provided but override the name with the new name, this is now accomplished by merging them together in the correct order (using the standard cloud-init merging algo). --- cloudinit/distros/__init__.py | 42 +++++++++++++++++++++++------------------- 1 file changed, 23 insertions(+), 19 deletions(-) (limited to 'cloudinit/distros') diff --git a/cloudinit/distros/__init__.py b/cloudinit/distros/__init__.py index c74be4e2..ddea8417 100644 --- a/cloudinit/distros/__init__.py +++ b/cloudinit/distros/__init__.py @@ -714,7 +714,7 @@ def normalize_users_groups(cfg, distro): # Handle the previous style of doing this where the first user # overrides the concept of the default user if provided in the user: XYZ # format. - old_user = None + old_user = {} if 'user' in cfg and cfg['user']: old_user = cfg['user'] # Translate it into the format that is more useful @@ -724,28 +724,32 @@ def normalize_users_groups(cfg, distro): 'name': old_user, } if not isinstance(old_user, (dict)): - LOG.warn(("Format for 'user:' key must be a string or " + LOG.warn(("Format for 'user' key must be a string or " "dictionary and not %s"), util.obj_name(old_user)) - old_user = None - - default_user_config = None - if not old_user: - # If no old user format, then assume the distro - # provides what the 'default' user maps to, but notice - # that if this is provided, we won't automatically inject - # a 'default' user into the users list, while if a old user - # format is provided we will. - try: - default_user_config = distro.get_default_user() - except NotImplementedError: - LOG.warn(("Distro has not implemented default user " - "access. No default user will be normalized.")) - else: - default_user_config = dict(old_user) + old_user = {} + + # If no old user format, then assume the distro + # provides what the 'default' user maps to, but notice + # that if this is provided, we won't automatically inject + # a 'default' user into the users list, while if a old user + # format is provided we will. + distro_user_config = {} + try: + distro_user_config = distro.get_default_user() + except NotImplementedError: + LOG.warn(("Distro has not implemented default user " + "access. No distribution provided default user" + " will be normalized.")) + + # Merge the old user (which may just be an empty dict when not + # present with the distro provided default user configuration so + # that the old user style picks up all the distribution specific + # attributes (if any) + default_user_config = util.mergemanydict([old_user, distro_user_config]) base_users = cfg.get('users', []) if not isinstance(base_users, (list, dict, str, basestring)): - LOG.warn(("Format for 'users:' key must be a comma separated string" + LOG.warn(("Format for 'users' key must be a comma separated string" " or a dictionary or a list and not %s"), util.obj_name(base_users)) base_users = [] -- cgit v1.2.3 From dc3ebfe2416028b78b6a846e939201d894b2c9b6 Mon Sep 17 00:00:00 2001 From: Craig Tracey Date: Sun, 27 Jan 2013 21:48:03 -0500 Subject: Adding package versioning logic to package_command This change adds the ability to provide specific package versions to Distro.install_packages and subsequently Distro.package_command. In order to effectively use Distro.install_packages, one is now able to pass a variety of formats in order to easily manage package requirements. These are examples of what can be passed: - "package" - ["package1","package2"] - ("package",) - ("package", "version") - [("package1",)("package2",)] - [("package1", "version1"),("package2","version2")] This change also adds the option to install a specific version for the puppet configuration module. This is especially important here as successful puppet deployments are highly reliant on specific puppet versions. --- cloudinit/config/cc_landscape.py | 2 +- cloudinit/config/cc_puppet.py | 10 ++++++++-- cloudinit/config/cc_salt_minion.py | 2 +- cloudinit/distros/debian.py | 17 +++++++++++++---- cloudinit/distros/rhel.py | 16 ++++++++++++---- cloudinit/util.py | 23 +++++++++++++++++++++++ 6 files changed, 58 insertions(+), 12 deletions(-) (limited to 'cloudinit/distros') diff --git a/cloudinit/config/cc_landscape.py b/cloudinit/config/cc_landscape.py index 02610dd0..2efdff79 100644 --- a/cloudinit/config/cc_landscape.py +++ b/cloudinit/config/cc_landscape.py @@ -62,7 +62,7 @@ def handle(_name, cfg, cloud, log, _args): if not ls_cloudcfg: return - cloud.distro.install_packages(["landscape-client"]) + cloud.distro.install_packages(('landscape-client',)) merge_data = [ LSC_BUILTIN_CFG, diff --git a/cloudinit/config/cc_puppet.py b/cloudinit/config/cc_puppet.py index e9a0a0f4..471a1a8a 100644 --- a/cloudinit/config/cc_puppet.py +++ b/cloudinit/config/cc_puppet.py @@ -59,8 +59,14 @@ def handle(name, cfg, cloud, log, _args): # Start by installing the puppet package if necessary... install = util.get_cfg_option_bool(puppet_cfg, 'install', True) - if install: - cloud.distro.install_packages(["puppet"]) + version = util.get_cfg_option_str(puppet_cfg, 'version', None) + if not install and version: + log.warn(("Puppet install set false but version supplied," + " doing nothing.")) + elif install: + log.debug(("Attempting to install puppet %s,"), + version if version else 'latest') + cloud.distro.install_packages(('puppet', version)) # ... and then update the puppet configuration if 'conf' in puppet_cfg: diff --git a/cloudinit/config/cc_salt_minion.py b/cloudinit/config/cc_salt_minion.py index f3eede18..53013dcb 100644 --- a/cloudinit/config/cc_salt_minion.py +++ b/cloudinit/config/cc_salt_minion.py @@ -31,7 +31,7 @@ def handle(name, cfg, cloud, log, _args): salt_cfg = cfg['salt_minion'] # Start by installing the salt package ... - cloud.distro.install_packages(["salt-minion"]) + cloud.distro.install_packages(('salt-minion',)) # Ensure we can configure files at the right dir config_dir = salt_cfg.get("config_dir", '/etc/salt') diff --git a/cloudinit/distros/debian.py b/cloudinit/distros/debian.py index 49b73477..1a8e927b 100644 --- a/cloudinit/distros/debian.py +++ b/cloudinit/distros/debian.py @@ -65,7 +65,7 @@ class Distro(distros.Distro): def install_packages(self, pkglist): self.update_package_sources() - self.package_command('install', pkglist) + self.package_command('install', pkgs=pkglist) def _write_network(self, settings): util.write_file(self.network_conf_fn, settings) @@ -142,15 +142,24 @@ class Distro(distros.Distro): # This ensures that the correct tz will be used for the system util.copy(tz_file, self.tz_local_fn) - def package_command(self, command, args=None): + def package_command(self, command, args=None, pkgs=[]): e = os.environ.copy() # See: http://tiny.cc/kg91fw # Or: http://tiny.cc/mh91fw e['DEBIAN_FRONTEND'] = 'noninteractive' cmd = ['apt-get', '--option', 'Dpkg::Options::=--force-confold', - '--assume-yes', '--quiet', command] - if args: + '--assume-yes', '--quiet'] + + if args and isinstance(args, str): + cmd.append(args) + elif args and isinstance(args, list): cmd.extend(args) + + cmd.append(command) + + pkglist = util.expand_package_list('%s=%s', pkgs) + cmd.extend(pkglist) + # Allow the output of this to flow outwards (ie not be captured) util.subp(cmd, env=e, capture=False) diff --git a/cloudinit/distros/rhel.py b/cloudinit/distros/rhel.py index e65be8d7..2f91e386 100644 --- a/cloudinit/distros/rhel.py +++ b/cloudinit/distros/rhel.py @@ -63,7 +63,7 @@ class Distro(distros.Distro): self.osfamily = 'redhat' def install_packages(self, pkglist): - self.package_command('install', pkglist) + self.package_command('install', pkgs=pkglist) def _adjust_resolve(self, dns_servers, search_servers): try: @@ -208,7 +208,7 @@ class Distro(distros.Distro): # This ensures that the correct tz will be used for the system util.copy(tz_file, self.tz_local_fn) - def package_command(self, command, args=None): + def package_command(self, command, args=None, pkgs=[]): cmd = ['yum'] # If enabled, then yum will be tolerant of errors on the command line # with regard to packages. @@ -219,9 +219,17 @@ class Distro(distros.Distro): # Determines whether or not yum prompts for confirmation # of critical actions. We don't want to prompt... cmd.append("-y") - cmd.append(command) - if args: + + if args and isinstance(args, str): + cmd.append(args) + elif args and isinstance(args, list): cmd.extend(args) + + cmd.append(command) + + pkglist = util.expand_package_list('%s-%s', pkgs) + cmd.extend(pkglist) + # Allow the output of this to flow outwards (ie not be captured) util.subp(cmd, capture=False) diff --git a/cloudinit/util.py b/cloudinit/util.py index c0ea8d91..c9c5f794 100644 --- a/cloudinit/util.py +++ b/cloudinit/util.py @@ -1560,3 +1560,26 @@ def is_partition(device): device = device[5:] return os.path.isfile("/sys/class/block/%s/partition" % device) + + +def expand_package_list(version_fmt, pkgs): + # we will accept tuples, lists of tuples, or just plain lists + if not isinstance(pkgs, list): + pkgs = [pkgs] + + pkglist = [] + for pkg in pkgs: + if isinstance(pkg, str): + pkglist.append(pkg) + continue + + if len(pkg) < 1 or len(pkg) > 2: + raise RuntimeError("Invalid package_command tuple.") + + if len(pkg) == 2 and pkg[1]: + pkglist.append(version_fmt % pkg) + continue + + pkglist.append(pkg[0]) + + return pkglist -- cgit v1.2.3