From c4aeba3f54eca687dd11837d1ec59de9f82c1cf6 Mon Sep 17 00:00:00 2001 From: Scott Moser Date: Fri, 30 Sep 2016 11:13:51 -0400 Subject: tests: silence the Cheetah UserWarning about NameMapper C version. This silences a warning made by Cheetah in pip installed environments: UserWarning: You don't have the C version of NameMapper installed! I'm disabling Cheetah's useStackFrames option ... The reason for the monkey patching is that the warning goes to stderr during nose and breaks up its expected output. The side affect of it is that tests would run with Cheetah's 'useStackFrames' enabled which is "painfully slow with the Python version of NameMapper". --- tests/unittests/__init__.py | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/tests/unittests/__init__.py b/tests/unittests/__init__.py index e69de29b..1b34b5af 100644 --- a/tests/unittests/__init__.py +++ b/tests/unittests/__init__.py @@ -0,0 +1,9 @@ +try: + # For test cases, avoid the following UserWarning to stderr: + # You don't have the C version of NameMapper installed ... + from Cheetah import NameMapper as _nm + _nm.C_VERSION = True +except ImportError: + pass + +# vi: ts=4 expandtab -- cgit v1.2.3 From 808edb127507d91ecee1834aaf5cf1f000cd6e28 Mon Sep 17 00:00:00 2001 From: Scott Moser Date: Tue, 20 Sep 2016 10:27:54 -0400 Subject: MAAS: improve the main of datasource to look at kernel cmdline config. This just looks in one other maas related path for a config file. The file '91_kernel_cmdline_url' is written by cloud-init when it gets a cloud-config-url parameter. Also now we read the config even if a url is specified to potentially fill in credentials. --- cloudinit/sources/DataSourceMAAS.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/cloudinit/sources/DataSourceMAAS.py b/cloudinit/sources/DataSourceMAAS.py index ab93c0a2..81abcd47 100644 --- a/cloudinit/sources/DataSourceMAAS.py +++ b/cloudinit/sources/DataSourceMAAS.py @@ -310,12 +310,12 @@ if __name__ == "__main__": creds = {'consumer_key': args.ckey, 'token_key': args.tkey, 'token_secret': args.tsec, 'consumer_secret': args.csec} - maaspkg_cfg = "/etc/cloud/cloud.cfg.d/90_dpkg_maas.cfg" - if (args.config is None and args.url is None and - os.path.exists(maaspkg_cfg) and - os.access(maaspkg_cfg, os.R_OK)): - sys.stderr.write("Used config in %s.\n" % maaspkg_cfg) - args.config = maaspkg_cfg + if args.config is None: + for fname in ('91_kernel_cmdline_url', '90_dpkg_maas'): + fpath = "/etc/cloud/cloud.cfg.d/" + fname + ".cfg" + if os.path.exists(fpath) and os.access(fpath, os.R_OK): + sys.stderr.write("Used config in %s.\n" % fpath) + args.config = fpath if args.config: cfg = util.read_conf(args.config) -- cgit v1.2.3 From e772f52bf77ef8e01db934167a974ca31b7cd61f Mon Sep 17 00:00:00 2001 From: Matthew Thode Date: Fri, 7 Oct 2016 10:57:28 -0500 Subject: update Gentoo initscripts to run in the correct order cloud-init-local needs to be run before net (and in the boot runlevel) as it sets up networking config and adds it to the default runlevel. cloud-init-local needs to be run in the boot runlevel because it modifies services in the default runlevel. When a runlevel is started it is cached, so modifications that happen to the current runlevel while you are in it are not acted upon. cloud-init needs to run after net, it does not need net because we still want it to set up other things if it can. --- sysvinit/gentoo/cloud-init | 1 + sysvinit/gentoo/cloud-init-local | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/sysvinit/gentoo/cloud-init b/sysvinit/gentoo/cloud-init index 5afc0f2e..531a715d 100644 --- a/sysvinit/gentoo/cloud-init +++ b/sysvinit/gentoo/cloud-init @@ -2,6 +2,7 @@ # add depends for network, dns, fs etc depend() { after cloud-init-local + after net before cloud-config provide cloud-init } diff --git a/sysvinit/gentoo/cloud-init-local b/sysvinit/gentoo/cloud-init-local index 9bd0b569..0f8cf65c 100644 --- a/sysvinit/gentoo/cloud-init-local +++ b/sysvinit/gentoo/cloud-init-local @@ -2,7 +2,7 @@ depend() { after localmount - after netmount + before net before cloud-init provide cloud-init-local } -- cgit v1.2.3 From 7ae201166402fbf2e6c1632028be956a954835ef Mon Sep 17 00:00:00 2001 From: Scott Moser Date: Tue, 18 Oct 2016 12:30:38 -0400 Subject: DigitalOcean: enable usage of data source by default. Just add DigitalOcean to the list of datasources that are used if there is no 'datasource_list' provided in config. --- cloudinit/settings.py | 1 + 1 file changed, 1 insertion(+) diff --git a/cloudinit/settings.py b/cloudinit/settings.py index 8c258ea1..a9682716 100644 --- a/cloudinit/settings.py +++ b/cloudinit/settings.py @@ -32,6 +32,7 @@ CFG_BUILTIN = { 'NoCloud', 'ConfigDrive', 'OpenNebula', + 'DigitalOcean', 'Azure', 'AltCloud', 'OVF', -- cgit v1.2.3 From f0747c4b4cf073273e11d383f0354257be7276ed Mon Sep 17 00:00:00 2001 From: Joshua Harlow Date: Fri, 30 Sep 2016 22:30:48 -0700 Subject: Move user/group functions to new ug_util file The amount of code to do user and group normalization and extraction deserves its own file so move the code that does this to a new file and update references to the old location. This removes some of the funkyness done in config modules to avoid namespace and attribute clashes as well. --- cloudinit/config/cc_byobu.py | 11 +- cloudinit/config/cc_set_passwords.py | 10 +- cloudinit/config/cc_ssh.py | 10 +- cloudinit/config/cc_ssh_authkey_fingerprints.py | 8 +- cloudinit/config/cc_ssh_import_id.py | 8 +- cloudinit/distros/__init__.py | 269 ------------------ cloudinit/distros/ug_util.py | 299 +++++++++++++++++++++ .../test_distros/test_user_data_normalize.py | 5 +- 8 files changed, 315 insertions(+), 305 deletions(-) mode change 100644 => 100755 cloudinit/config/cc_byobu.py mode change 100644 => 100755 cloudinit/config/cc_set_passwords.py mode change 100644 => 100755 cloudinit/config/cc_ssh.py mode change 100644 => 100755 cloudinit/config/cc_ssh_authkey_fingerprints.py mode change 100644 => 100755 cloudinit/config/cc_ssh_import_id.py mode change 100644 => 100755 cloudinit/distros/__init__.py create mode 100755 cloudinit/distros/ug_util.py mode change 100644 => 100755 tests/unittests/test_distros/test_user_data_normalize.py diff --git a/cloudinit/config/cc_byobu.py b/cloudinit/config/cc_byobu.py old mode 100644 new mode 100755 index 1f00dd90..4a616e26 --- a/cloudinit/config/cc_byobu.py +++ b/cloudinit/config/cc_byobu.py @@ -50,12 +50,7 @@ Valid configuration options for this module are: byobu_by_default: """ - -# Ensure this is aliased to a name not 'distros' -# since the module attribute 'distros' -# is a list of distros that are supported, not a sub-module -from cloudinit import distros as ds - +from cloudinit.distros import ug_util from cloudinit import util distros = ['ubuntu', 'debian'] @@ -94,8 +89,8 @@ def handle(name, cfg, cloud, log, args): shcmd = "" if mod_user: - (users, _groups) = ds.normalize_users_groups(cfg, cloud.distro) - (user, _user_config) = ds.extract_default(users) + (users, _groups) = ug_util.normalize_users_groups(cfg, cloud.distro) + (user, _user_config) = ug_util.extract_default(users) if not user: log.warn(("No default byobu user provided, " "can not launch %s for the default user"), bl_inst) diff --git a/cloudinit/config/cc_set_passwords.py b/cloudinit/config/cc_set_passwords.py old mode 100644 new mode 100755 index 94716017..6fc00517 --- a/cloudinit/config/cc_set_passwords.py +++ b/cloudinit/config/cc_set_passwords.py @@ -66,11 +66,7 @@ enabled, disabled, or left to system defaults using ``ssh_pwauth``. import sys -# Ensure this is aliased to a name not 'distros' -# since the module attribute 'distros' -# is a list of distros that are supported, not a sub-module -from cloudinit import distros as ds - +from cloudinit.distros import ug_util from cloudinit import ssh_util from cloudinit import util @@ -99,8 +95,8 @@ def handle(_name, cfg, cloud, log, args): expire = util.get_cfg_option_bool(chfg, 'expire', expire) if not plist and password: - (users, _groups) = ds.normalize_users_groups(cfg, cloud.distro) - (user, _user_config) = ds.extract_default(users) + (users, _groups) = ug_util.normalize_users_groups(cfg, cloud.distro) + (user, _user_config) = ug_util.extract_default(users) if user: plist = "%s:%s" % (user, password) else: diff --git a/cloudinit/config/cc_ssh.py b/cloudinit/config/cc_ssh.py old mode 100644 new mode 100755 index 6138fb53..576fa58a --- a/cloudinit/config/cc_ssh.py +++ b/cloudinit/config/cc_ssh.py @@ -109,11 +109,7 @@ import glob import os import sys -# Ensure this is aliased to a name not 'distros' -# since the module attribute 'distros' -# is a list of distros that are supported, not a sub-module -from cloudinit import distros as ds - +from cloudinit.distros import ug_util from cloudinit import ssh_util from cloudinit import util @@ -197,8 +193,8 @@ def handle(_name, cfg, cloud, log, _args): "file %s", keytype, keyfile) try: - (users, _groups) = ds.normalize_users_groups(cfg, cloud.distro) - (user, _user_config) = ds.extract_default(users) + (users, _groups) = ug_util.normalize_users_groups(cfg, cloud.distro) + (user, _user_config) = ug_util.extract_default(users) disable_root = util.get_cfg_option_bool(cfg, "disable_root", True) disable_root_opts = util.get_cfg_option_str(cfg, "disable_root_opts", DISABLE_ROOT_OPTS) diff --git a/cloudinit/config/cc_ssh_authkey_fingerprints.py b/cloudinit/config/cc_ssh_authkey_fingerprints.py old mode 100644 new mode 100755 index 6f3d0ee2..7eeb0f84 --- a/cloudinit/config/cc_ssh_authkey_fingerprints.py +++ b/cloudinit/config/cc_ssh_authkey_fingerprints.py @@ -42,11 +42,7 @@ import hashlib from prettytable import PrettyTable -# Ensure this is aliased to a name not 'distros' -# since the module attribute 'distros' -# is a list of distros that are supported, not a sub-module -from cloudinit import distros as ds - +from cloudinit.distros import ug_util from cloudinit import ssh_util from cloudinit import util @@ -119,7 +115,7 @@ def handle(name, cfg, cloud, log, _args): return hash_meth = util.get_cfg_option_str(cfg, "authkey_hash", "md5") - (users, _groups) = ds.normalize_users_groups(cfg, cloud.distro) + (users, _groups) = ug_util.normalize_users_groups(cfg, cloud.distro) for (user_name, _cfg) in users.items(): (key_fn, key_entries) = ssh_util.extract_authorized_keys(user_name) _pprint_key_entries(user_name, key_fn, diff --git a/cloudinit/config/cc_ssh_import_id.py b/cloudinit/config/cc_ssh_import_id.py old mode 100644 new mode 100755 index 99359c87..1be96dc5 --- a/cloudinit/config/cc_ssh_import_id.py +++ b/cloudinit/config/cc_ssh_import_id.py @@ -42,11 +42,7 @@ either ``lp:`` for launchpad or ``gh:`` for github to the username. - lp:user """ -# Ensure this is aliased to a name not 'distros' -# since the module attribute 'distros' -# is a list of distros that are supported, not a sub-module -from cloudinit import distros as ds - +from cloudinit.distros import ug_util from cloudinit import util import pwd @@ -67,7 +63,7 @@ def handle(_name, cfg, cloud, log, args): return # import for cloudinit created users - (users, _groups) = ds.normalize_users_groups(cfg, cloud.distro) + (users, _groups) = ug_util.normalize_users_groups(cfg, cloud.distro) elist = [] for (user, user_cfg) in users.items(): import_ids = [] diff --git a/cloudinit/distros/__init__.py b/cloudinit/distros/__init__.py old mode 100644 new mode 100755 index b1192e84..78adf5f9 --- a/cloudinit/distros/__init__.py +++ b/cloudinit/distros/__init__.py @@ -685,275 +685,6 @@ def _get_arch_package_mirror_info(package_mirrors, arch): return default -# Normalizes a input group configuration -# which can be a comma seperated list of -# group names, or a list of group names -# or a python dictionary of group names -# to a list of members of that group. -# -# The output is a dictionary of group -# names => members of that group which -# is the standard form used in the rest -# of cloud-init -def _normalize_groups(grp_cfg): - if isinstance(grp_cfg, six.string_types): - grp_cfg = grp_cfg.strip().split(",") - if isinstance(grp_cfg, list): - c_grp_cfg = {} - for i in grp_cfg: - if isinstance(i, dict): - for k, v in i.items(): - if k not in c_grp_cfg: - if isinstance(v, list): - c_grp_cfg[k] = list(v) - elif isinstance(v, six.string_types): - c_grp_cfg[k] = [v] - else: - raise TypeError("Bad group member type %s" % - type_utils.obj_name(v)) - else: - if isinstance(v, list): - c_grp_cfg[k].extend(v) - elif isinstance(v, six.string_types): - c_grp_cfg[k].append(v) - else: - raise TypeError("Bad group member type %s" % - type_utils.obj_name(v)) - elif isinstance(i, six.string_types): - if i not in c_grp_cfg: - c_grp_cfg[i] = [] - else: - raise TypeError("Unknown group name type %s" % - type_utils.obj_name(i)) - grp_cfg = c_grp_cfg - groups = {} - if isinstance(grp_cfg, dict): - for (grp_name, grp_members) in grp_cfg.items(): - groups[grp_name] = util.uniq_merge_sorted(grp_members) - else: - raise TypeError(("Group config must be list, dict " - " or string types only and not %s") % - type_utils.obj_name(grp_cfg)) - return groups - - -# Normalizes a input group configuration -# which can be a comma seperated list of -# user names, or a list of string user names -# or a list of dictionaries with components -# that define the user config + 'name' (if -# a 'name' field does not exist then the -# default user is assumed to 'own' that -# configuration. -# -# The output is a dictionary of user -# names => user config which is the standard -# form used in the rest of cloud-init. Note -# the default user will have a special config -# entry 'default' which will be marked as true -# all other users will be marked as false. -def _normalize_users(u_cfg, def_user_cfg=None): - if isinstance(u_cfg, dict): - ad_ucfg = [] - for (k, v) in u_cfg.items(): - if isinstance(v, (bool, int, float) + six.string_types): - if util.is_true(v): - ad_ucfg.append(str(k)) - elif isinstance(v, dict): - v['name'] = k - ad_ucfg.append(v) - else: - raise TypeError(("Unmappable user value type %s" - " for key %s") % (type_utils.obj_name(v), k)) - u_cfg = ad_ucfg - elif isinstance(u_cfg, six.string_types): - u_cfg = util.uniq_merge_sorted(u_cfg) - - users = {} - for user_config in u_cfg: - if isinstance(user_config, (list,) + six.string_types): - for u in util.uniq_merge(user_config): - if u and u not in users: - users[u] = {} - elif isinstance(user_config, dict): - if 'name' in user_config: - n = user_config.pop('name') - prev_config = users.get(n) or {} - users[n] = util.mergemanydict([prev_config, - user_config]) - else: - # Assume the default user then - prev_config = users.get('default') or {} - users['default'] = util.mergemanydict([prev_config, - user_config]) - else: - raise TypeError(("User config must be dictionary/list " - " or string types only and not %s") % - type_utils.obj_name(user_config)) - - # Ensure user options are in the right python friendly format - if users: - c_users = {} - for (uname, uconfig) in users.items(): - c_uconfig = {} - for (k, v) in uconfig.items(): - k = k.replace('-', '_').strip() - if k: - c_uconfig[k] = v - c_users[uname] = c_uconfig - users = c_users - - # Fixup the default user into the real - # default user name and replace it... - def_user = None - if users and 'default' in users: - def_config = users.pop('default') - if def_user_cfg: - # Pickup what the default 'real name' is - # and any groups that are provided by the - # default config - def_user_cfg = def_user_cfg.copy() - def_user = def_user_cfg.pop('name') - def_groups = def_user_cfg.pop('groups', []) - # Pickup any config + groups for that user name - # that we may have previously extracted - parsed_config = users.pop(def_user, {}) - parsed_groups = parsed_config.get('groups', []) - # Now merge our extracted groups with - # anything the default config provided - users_groups = util.uniq_merge_sorted(parsed_groups, def_groups) - parsed_config['groups'] = ",".join(users_groups) - # The real config for the default user is the - # combination of the default user config provided - # by the distro, the default user config provided - # by the above merging for the user 'default' and - # then the parsed config from the user's 'real name' - # which does not have to be 'default' (but could be) - users[def_user] = util.mergemanydict([def_user_cfg, - def_config, - parsed_config]) - - # Ensure that only the default user that we - # found (if any) is actually marked as being - # the default user - if users: - for (uname, uconfig) in users.items(): - if def_user and uname == def_user: - uconfig['default'] = True - else: - uconfig['default'] = False - - return users - - -# Normalizes a set of user/users and group -# dictionary configuration into a useable -# format that the rest of cloud-init can -# understand using the default user -# provided by the input distrobution (if any) -# to allow for mapping of the 'default' user. -# -# Output is a dictionary of group names -> [member] (list) -# and a dictionary of user names -> user configuration (dict) -# -# If 'user' exists it will override -# the 'users'[0] entry (if a list) otherwise it will -# just become an entry in the returned dictionary (no override) -def normalize_users_groups(cfg, distro): - if not cfg: - cfg = {} - - users = {} - groups = {} - if 'groups' in cfg: - groups = _normalize_groups(cfg['groups']) - - # Handle the previous style of doing this where the first user - # overrides the concept of the default user if provided in the user: XYZ - # format. - old_user = {} - if 'user' in cfg and cfg['user']: - old_user = cfg['user'] - # Translate it into the format that is more useful - # going forward - if isinstance(old_user, six.string_types): - old_user = { - 'name': old_user, - } - if not isinstance(old_user, dict): - LOG.warn(("Format for 'user' key must be a string or " - "dictionary and not %s"), type_utils.obj_name(old_user)) - old_user = {} - - # If no old user format, then assume the distro - # provides what the 'default' user maps to, but notice - # that if this is provided, we won't automatically inject - # a 'default' user into the users list, while if a old user - # format is provided we will. - distro_user_config = {} - try: - distro_user_config = distro.get_default_user() - except NotImplementedError: - LOG.warn(("Distro has not implemented default user " - "access. No distribution provided default user" - " will be normalized.")) - - # Merge the old user (which may just be an empty dict when not - # present with the distro provided default user configuration so - # that the old user style picks up all the distribution specific - # attributes (if any) - default_user_config = util.mergemanydict([old_user, distro_user_config]) - - base_users = cfg.get('users', []) - if not isinstance(base_users, (list, dict) + six.string_types): - LOG.warn(("Format for 'users' key must be a comma separated string" - " or a dictionary or a list and not %s"), - type_utils.obj_name(base_users)) - base_users = [] - - if old_user: - # Ensure that when user: is provided that this user - # always gets added (as the default user) - if isinstance(base_users, list): - # Just add it on at the end... - base_users.append({'name': 'default'}) - elif isinstance(base_users, dict): - base_users['default'] = dict(base_users).get('default', True) - elif isinstance(base_users, six.string_types): - # Just append it on to be re-parsed later - base_users += ",default" - - users = _normalize_users(base_users, default_user_config) - return (users, groups) - - -# Given a user dictionary config it will -# extract the default user name and user config -# from that list and return that tuple or -# return (None, None) if no default user is -# found in the given input -def extract_default(users, default_name=None, default_config=None): - if not users: - users = {} - - def safe_find(entry): - config = entry[1] - if not config or 'default' not in config: - return False - else: - return config['default'] - - tmp_users = users.items() - tmp_users = dict(filter(safe_find, tmp_users)) - if not tmp_users: - return (default_name, default_config) - else: - name = list(tmp_users)[0] - config = tmp_users[name] - config.pop('default', None) - return (name, config) - - def fetch(name): locs, looked_locs = importer.find_module(name, ['', __name__], ['Distro']) if not locs: diff --git a/cloudinit/distros/ug_util.py b/cloudinit/distros/ug_util.py new file mode 100755 index 00000000..99301530 --- /dev/null +++ b/cloudinit/distros/ug_util.py @@ -0,0 +1,299 @@ +# vi: ts=4 expandtab +# +# Copyright (C) 2012 Canonical Ltd. +# Copyright (C) 2012, 2013 Hewlett-Packard Development Company, L.P. +# Copyright (C) 2012 Yahoo! Inc. +# +# Author: Scott Moser +# Author: Juerg Haefliger +# Author: Joshua Harlow +# Author: Ben Howard +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 3, as +# published by the Free Software Foundation. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +import six + +from cloudinit import log as logging +from cloudinit import type_utils +from cloudinit import util + +LOG = logging.getLogger(__name__) + + +# Normalizes a input group configuration +# which can be a comma seperated list of +# group names, or a list of group names +# or a python dictionary of group names +# to a list of members of that group. +# +# The output is a dictionary of group +# names => members of that group which +# is the standard form used in the rest +# of cloud-init +def _normalize_groups(grp_cfg): + if isinstance(grp_cfg, six.string_types): + grp_cfg = grp_cfg.strip().split(",") + if isinstance(grp_cfg, list): + c_grp_cfg = {} + for i in grp_cfg: + if isinstance(i, dict): + for k, v in i.items(): + if k not in c_grp_cfg: + if isinstance(v, list): + c_grp_cfg[k] = list(v) + elif isinstance(v, six.string_types): + c_grp_cfg[k] = [v] + else: + raise TypeError("Bad group member type %s" % + type_utils.obj_name(v)) + else: + if isinstance(v, list): + c_grp_cfg[k].extend(v) + elif isinstance(v, six.string_types): + c_grp_cfg[k].append(v) + else: + raise TypeError("Bad group member type %s" % + type_utils.obj_name(v)) + elif isinstance(i, six.string_types): + if i not in c_grp_cfg: + c_grp_cfg[i] = [] + else: + raise TypeError("Unknown group name type %s" % + type_utils.obj_name(i)) + grp_cfg = c_grp_cfg + groups = {} + if isinstance(grp_cfg, dict): + for (grp_name, grp_members) in grp_cfg.items(): + groups[grp_name] = util.uniq_merge_sorted(grp_members) + else: + raise TypeError(("Group config must be list, dict " + " or string types only and not %s") % + type_utils.obj_name(grp_cfg)) + return groups + + +# Normalizes a input group configuration +# which can be a comma seperated list of +# user names, or a list of string user names +# or a list of dictionaries with components +# that define the user config + 'name' (if +# a 'name' field does not exist then the +# default user is assumed to 'own' that +# configuration. +# +# The output is a dictionary of user +# names => user config which is the standard +# form used in the rest of cloud-init. Note +# the default user will have a special config +# entry 'default' which will be marked as true +# all other users will be marked as false. +def _normalize_users(u_cfg, def_user_cfg=None): + if isinstance(u_cfg, dict): + ad_ucfg = [] + for (k, v) in u_cfg.items(): + if isinstance(v, (bool, int, float) + six.string_types): + if util.is_true(v): + ad_ucfg.append(str(k)) + elif isinstance(v, dict): + v['name'] = k + ad_ucfg.append(v) + else: + raise TypeError(("Unmappable user value type %s" + " for key %s") % (type_utils.obj_name(v), k)) + u_cfg = ad_ucfg + elif isinstance(u_cfg, six.string_types): + u_cfg = util.uniq_merge_sorted(u_cfg) + + users = {} + for user_config in u_cfg: + if isinstance(user_config, (list,) + six.string_types): + for u in util.uniq_merge(user_config): + if u and u not in users: + users[u] = {} + elif isinstance(user_config, dict): + if 'name' in user_config: + n = user_config.pop('name') + prev_config = users.get(n) or {} + users[n] = util.mergemanydict([prev_config, + user_config]) + else: + # Assume the default user then + prev_config = users.get('default') or {} + users['default'] = util.mergemanydict([prev_config, + user_config]) + else: + raise TypeError(("User config must be dictionary/list " + " or string types only and not %s") % + type_utils.obj_name(user_config)) + + # Ensure user options are in the right python friendly format + if users: + c_users = {} + for (uname, uconfig) in users.items(): + c_uconfig = {} + for (k, v) in uconfig.items(): + k = k.replace('-', '_').strip() + if k: + c_uconfig[k] = v + c_users[uname] = c_uconfig + users = c_users + + # Fixup the default user into the real + # default user name and replace it... + def_user = None + if users and 'default' in users: + def_config = users.pop('default') + if def_user_cfg: + # Pickup what the default 'real name' is + # and any groups that are provided by the + # default config + def_user_cfg = def_user_cfg.copy() + def_user = def_user_cfg.pop('name') + def_groups = def_user_cfg.pop('groups', []) + # Pickup any config + groups for that user name + # that we may have previously extracted + parsed_config = users.pop(def_user, {}) + parsed_groups = parsed_config.get('groups', []) + # Now merge our extracted groups with + # anything the default config provided + users_groups = util.uniq_merge_sorted(parsed_groups, def_groups) + parsed_config['groups'] = ",".join(users_groups) + # The real config for the default user is the + # combination of the default user config provided + # by the distro, the default user config provided + # by the above merging for the user 'default' and + # then the parsed config from the user's 'real name' + # which does not have to be 'default' (but could be) + users[def_user] = util.mergemanydict([def_user_cfg, + def_config, + parsed_config]) + + # Ensure that only the default user that we + # found (if any) is actually marked as being + # the default user + if users: + for (uname, uconfig) in users.items(): + if def_user and uname == def_user: + uconfig['default'] = True + else: + uconfig['default'] = False + + return users + + +# Normalizes a set of user/users and group +# dictionary configuration into a useable +# format that the rest of cloud-init can +# understand using the default user +# provided by the input distrobution (if any) +# to allow for mapping of the 'default' user. +# +# Output is a dictionary of group names -> [member] (list) +# and a dictionary of user names -> user configuration (dict) +# +# If 'user' exists it will override +# the 'users'[0] entry (if a list) otherwise it will +# just become an entry in the returned dictionary (no override) +def normalize_users_groups(cfg, distro): + if not cfg: + cfg = {} + + users = {} + groups = {} + if 'groups' in cfg: + groups = _normalize_groups(cfg['groups']) + + # Handle the previous style of doing this where the first user + # overrides the concept of the default user if provided in the user: XYZ + # format. + old_user = {} + if 'user' in cfg and cfg['user']: + old_user = cfg['user'] + # Translate it into the format that is more useful + # going forward + if isinstance(old_user, six.string_types): + old_user = { + 'name': old_user, + } + if not isinstance(old_user, dict): + LOG.warn(("Format for 'user' key must be a string or " + "dictionary and not %s"), type_utils.obj_name(old_user)) + old_user = {} + + # If no old user format, then assume the distro + # provides what the 'default' user maps to, but notice + # that if this is provided, we won't automatically inject + # a 'default' user into the users list, while if a old user + # format is provided we will. + distro_user_config = {} + try: + distro_user_config = distro.get_default_user() + except NotImplementedError: + LOG.warn(("Distro has not implemented default user " + "access. No distribution provided default user" + " will be normalized.")) + + # Merge the old user (which may just be an empty dict when not + # present with the distro provided default user configuration so + # that the old user style picks up all the distribution specific + # attributes (if any) + default_user_config = util.mergemanydict([old_user, distro_user_config]) + + base_users = cfg.get('users', []) + if not isinstance(base_users, (list, dict) + six.string_types): + LOG.warn(("Format for 'users' key must be a comma separated string" + " or a dictionary or a list and not %s"), + type_utils.obj_name(base_users)) + base_users = [] + + if old_user: + # Ensure that when user: is provided that this user + # always gets added (as the default user) + if isinstance(base_users, list): + # Just add it on at the end... + base_users.append({'name': 'default'}) + elif isinstance(base_users, dict): + base_users['default'] = dict(base_users).get('default', True) + elif isinstance(base_users, six.string_types): + # Just append it on to be re-parsed later + base_users += ",default" + + users = _normalize_users(base_users, default_user_config) + return (users, groups) + + +# Given a user dictionary config it will +# extract the default user name and user config +# from that list and return that tuple or +# return (None, None) if no default user is +# found in the given input +def extract_default(users, default_name=None, default_config=None): + if not users: + users = {} + + def safe_find(entry): + config = entry[1] + if not config or 'default' not in config: + return False + else: + return config['default'] + + tmp_users = users.items() + tmp_users = dict(filter(safe_find, tmp_users)) + if not tmp_users: + return (default_name, default_config) + else: + name = list(tmp_users)[0] + config = tmp_users[name] + config.pop('default', None) + return (name, config) diff --git a/tests/unittests/test_distros/test_user_data_normalize.py b/tests/unittests/test_distros/test_user_data_normalize.py old mode 100644 new mode 100755 index a887a930..b24888fc --- a/tests/unittests/test_distros/test_user_data_normalize.py +++ b/tests/unittests/test_distros/test_user_data_normalize.py @@ -1,4 +1,5 @@ from cloudinit import distros +from cloudinit.distros import ug_util from cloudinit import helpers from cloudinit import settings @@ -29,7 +30,7 @@ class TestUGNormalize(TestCase): return distro def _norm(self, cfg, distro): - return distros.normalize_users_groups(cfg, distro) + return ug_util.normalize_users_groups(cfg, distro) def test_group_dict(self): distro = self._make_distro('ubuntu') @@ -236,7 +237,7 @@ class TestUGNormalize(TestCase): } (users, _groups) = self._norm(ug_cfg, distro) self.assertIn('bob', users) - (name, config) = distros.extract_default(users) + (name, config) = ug_util.extract_default(users) self.assertEqual(name, 'bob') expected_config = {} def_config = None -- cgit v1.2.3 From e8730078df8c99696b1b684e09c803eef7c4926c Mon Sep 17 00:00:00 2001 From: Scott Moser Date: Fri, 30 Sep 2016 15:53:42 -0400 Subject: Fix python2.6 things found running in centos 6. This gets the tests running in centos 6. * ProcessExecutionError: remove setting of .message Nothing in cloud-init seems to use .message anywhere, so it does not seem necessary. The reason to change it is that on 2.6 it spits out: cloudinit/util.py:286: DeprecationWarning: BaseException.message * tox.ini: add a centos6 environment the tox versions listed here replicate a centos6 install with packages from EPEL. You will still need a python2.6 to run this env so we do not enable it by default. --- cloudinit/sources/DataSourceAltCloud.py | 6 ++---- cloudinit/sources/helpers/azure.py | 2 +- cloudinit/util.py | 7 ++----- tests/unittests/test_handler/test_handler_apt_conf_v1.py | 2 +- tests/unittests/test_util.py | 2 +- tox.ini | 16 ++++++++++++++++ 6 files changed, 23 insertions(+), 12 deletions(-) diff --git a/cloudinit/sources/DataSourceAltCloud.py b/cloudinit/sources/DataSourceAltCloud.py index 48136f7c..20345389 100644 --- a/cloudinit/sources/DataSourceAltCloud.py +++ b/cloudinit/sources/DataSourceAltCloud.py @@ -195,8 +195,7 @@ class DataSourceAltCloud(sources.DataSource): (cmd_out, _err) = util.subp(cmd) LOG.debug(('Command: %s\nOutput%s') % (' '.join(cmd), cmd_out)) except ProcessExecutionError as _err: - util.logexc(LOG, 'Failed command: %s\n%s', ' '.join(cmd), - _err.message) + util.logexc(LOG, 'Failed command: %s\n%s', ' '.join(cmd), _err) return False except OSError as _err: util.logexc(LOG, 'Failed command: %s\n%s', ' '.join(cmd), _err) @@ -211,8 +210,7 @@ class DataSourceAltCloud(sources.DataSource): (cmd_out, _err) = util.subp(cmd) LOG.debug(('Command: %s\nOutput%s') % (' '.join(cmd), cmd_out)) except ProcessExecutionError as _err: - util.logexc(LOG, 'Failed command: %s\n%s', ' '.join(cmd), - _err.message) + util.logexc(LOG, 'Failed command: %s\n%s', ' '.join(cmd), _err) return False except OSError as _err: util.logexc(LOG, 'Failed command: %s\n%s', ' '.join(cmd), diff --git a/cloudinit/sources/helpers/azure.py b/cloudinit/sources/helpers/azure.py index 689ed4cc..1b3e9b70 100644 --- a/cloudinit/sources/helpers/azure.py +++ b/cloudinit/sources/helpers/azure.py @@ -232,7 +232,7 @@ class WALinuxAgentShim(object): def _get_value_from_leases_file(fallback_lease_file): leases = [] content = util.load_file(fallback_lease_file) - LOG.debug("content is {}".format(content)) + LOG.debug("content is %s", content) for line in content.splitlines(): if 'unknown-245' in line: # Example line from Ubuntu diff --git a/cloudinit/util.py b/cloudinit/util.py index eb3e5899..4cff83c5 100644 --- a/cloudinit/util.py +++ b/cloudinit/util.py @@ -199,7 +199,7 @@ def fully_decoded_payload(part): encoding = charset.input_codec else: encoding = 'utf-8' - return cte_payload.decode(encoding, errors='surrogateescape') + return cte_payload.decode(encoding, 'surrogateescape') return cte_payload @@ -282,9 +282,6 @@ class ProcessExecutionError(IOError): 'reason': self.reason, } IOError.__init__(self, message) - # For backward compatibility with Python 2. - if not hasattr(self, 'message'): - self.message = message class SeLinuxGuard(object): @@ -1821,7 +1818,7 @@ def subp(args, data=None, rcs=None, env=None, capture=True, shell=False, def ldecode(data, m='utf-8'): if not isinstance(data, bytes): return data - return data.decode(m, errors=decode) + return data.decode(m, decode) out = ldecode(out) err = ldecode(err) diff --git a/tests/unittests/test_handler/test_handler_apt_conf_v1.py b/tests/unittests/test_handler/test_handler_apt_conf_v1.py index 45714efd..64acc3e0 100644 --- a/tests/unittests/test_handler/test_handler_apt_conf_v1.py +++ b/tests/unittests/test_handler/test_handler_apt_conf_v1.py @@ -118,7 +118,7 @@ class TestConversion(TestCase): def test_convert_with_apt_mirror(self): mirror = 'http://my.mirror/ubuntu' f = cc_apt_configure.convert_to_v3_apt_format({'apt_mirror': mirror}) - self.assertIn(mirror, {m['uri'] for m in f['apt']['primary']}) + self.assertIn(mirror, set(m['uri'] for m in f['apt']['primary'])) def test_no_old_content(self): mirror = 'http://my.mirror/ubuntu' diff --git a/tests/unittests/test_util.py b/tests/unittests/test_util.py index fc6b9d40..881509aa 100644 --- a/tests/unittests/test_util.py +++ b/tests/unittests/test_util.py @@ -553,7 +553,7 @@ class TestSubp(helpers.TestCase): def test_subp_decode_invalid_utf8_replaces(self): (out, _err) = util.subp(self.stdin2out, capture=True, data=self.utf8_invalid) - expected = self.utf8_invalid.decode('utf-8', errors='replace') + expected = self.utf8_invalid.decode('utf-8', 'replace') self.assertEqual(out, expected) def test_subp_decode_strict_raises(self): diff --git a/tox.ini b/tox.ini index 729de2a6..277858ed 100644 --- a/tox.ini +++ b/tox.ini @@ -59,3 +59,19 @@ deps = pyflakes==1.1.0 flake8==2.5.4 hacking==0.10.2 + +[testenv:centos6] +basepython = python2.6 +commands = nosetests {posargs:tests} +deps = + # requirements + argparse==1.2.1 + jinja2==2.2.1 + pyyaml==3.10 + PrettyTable==0.7.2 + oauthlib==0.6.0 + configobj==4.6.0 + requests==2.6.0 + jsonpatch==1.2 + six==1.9.0 + -r{toxinidir}/test-requirements.txt -- cgit v1.2.3 From 9972d246947f1a6ec102b978b99b26acc43133ec Mon Sep 17 00:00:00 2001 From: Scott Moser Date: Thu, 29 Sep 2016 12:45:02 -0400 Subject: OpenNebula: replace 'ip' parsing with cloudinit.net usage. Replace the parsing of 'ip' to get a link and mac address list in OpenNebula's datasource with usage of cloudinit.net. This makes test cases there not depend on 'ip' availability and also uses common code. --- cloudinit/sources/DataSourceOpenNebula.py | 34 ++++++++++------------ tests/unittests/test_datasource/test_opennebula.py | 23 +++++++-------- 2 files changed, 25 insertions(+), 32 deletions(-) diff --git a/cloudinit/sources/DataSourceOpenNebula.py b/cloudinit/sources/DataSourceOpenNebula.py index 635a836c..ba5f3f92 100644 --- a/cloudinit/sources/DataSourceOpenNebula.py +++ b/cloudinit/sources/DataSourceOpenNebula.py @@ -30,6 +30,7 @@ import re import string from cloudinit import log as logging +from cloudinit import net from cloudinit import sources from cloudinit import util @@ -120,17 +121,11 @@ class BrokenContextDiskDir(Exception): class OpenNebulaNetwork(object): - REG_DEV_MAC = re.compile( - r'^\d+: (eth\d+):.*?link\/ether (..:..:..:..:..:..) ?', - re.MULTILINE | re.DOTALL) - - def __init__(self, ip, context): - self.ip = ip + def __init__(self, context, system_nics_by_mac=None): self.context = context - self.ifaces = self.get_ifaces() - - def get_ifaces(self): - return self.REG_DEV_MAC.findall(self.ip) + if system_nics_by_mac is None: + system_nics_by_mac = get_physical_nics_by_mac() + self.ifaces = system_nics_by_mac def mac2ip(self, mac): components = mac.split(':')[2:] @@ -188,9 +183,7 @@ class OpenNebulaNetwork(object): conf.append('iface lo inet loopback') conf.append('') - for i in self.ifaces: - dev = i[0] - mac = i[1] + for mac, dev in self.ifaces.items(): ip_components = self.mac2ip(mac) conf.append('auto ' + dev) @@ -405,16 +398,19 @@ def read_context_disk_dir(source_dir, asuser=None): # generate static /etc/network/interfaces # only if there are any required context variables # http://opennebula.org/documentation:rel3.8:cong#network_configuration - for k in context: - if re.match(r'^ETH\d+_IP$', k): - (out, _) = util.subp(['ip', 'link']) - net = OpenNebulaNetwork(out, context) - results['network-interfaces'] = net.gen_conf() - break + ipaddr_keys = [k for k in context if re.match(r'^ETH\d+_IP$', k)] + if ipaddr_keys: + onet = OpenNebulaNetwork(context) + results['network-interfaces'] = onet.gen_conf() return results +def get_physical_nics_by_mac(): + devs = net.get_interfaces_by_mac() + return dict([(m, n) for m, n in devs.items() if net.is_physical(n)]) + + # Legacy: Must be present in case we load an old pkl object DataSourceOpenNebulaNet = DataSourceOpenNebula diff --git a/tests/unittests/test_datasource/test_opennebula.py b/tests/unittests/test_datasource/test_opennebula.py index d796f030..ce5b5550 100644 --- a/tests/unittests/test_datasource/test_opennebula.py +++ b/tests/unittests/test_datasource/test_opennebula.py @@ -1,7 +1,7 @@ from cloudinit import helpers from cloudinit.sources import DataSourceOpenNebula as ds from cloudinit import util -from ..helpers import TestCase, populate_dir +from ..helpers import mock, populate_dir, TestCase import os import pwd @@ -31,12 +31,7 @@ SSH_KEY = 'ssh-rsa AAAAB3NzaC1....sIkJhq8wdX+4I3A4cYbYP ubuntu@server-460-%i' HOSTNAME = 'foo.example.com' PUBLIC_IP = '10.0.0.3' -CMD_IP_OUT = '''\ -1: lo: mtu 16436 qdisc noqueue state UNKNOWN - link/loopback 00:00:00:00:00:00 brd 00:00:00:00:00:00 -2: eth0: mtu 1500 qdisc mq state UP qlen 1000 - link/ether 02:00:0a:12:01:01 brd ff:ff:ff:ff:ff:ff -''' +DS_PATH = "cloudinit.sources.DataSourceOpenNebula" class TestOpenNebulaDataSource(TestCase): @@ -233,18 +228,19 @@ class TestOpenNebulaDataSource(TestCase): class TestOpenNebulaNetwork(unittest.TestCase): - def setUp(self): - super(TestOpenNebulaNetwork, self).setUp() + system_nics = {'02:00:0a:12:01:01': 'eth0'} def test_lo(self): - net = ds.OpenNebulaNetwork('', {}) + net = ds.OpenNebulaNetwork(context={}, system_nics_by_mac={}) self.assertEqual(net.gen_conf(), u'''\ auto lo iface lo inet loopback ''') - def test_eth0(self): - net = ds.OpenNebulaNetwork(CMD_IP_OUT, {}) + @mock.patch(DS_PATH + ".get_physical_nics_by_mac") + def test_eth0(self, m_get_phys_by_mac): + m_get_phys_by_mac.return_value = self.system_nics + net = ds.OpenNebulaNetwork({}) self.assertEqual(net.gen_conf(), u'''\ auto lo iface lo inet loopback @@ -267,7 +263,8 @@ iface eth0 inet static 'ETH0_DNS': '1.2.3.6 1.2.3.7' } - net = ds.OpenNebulaNetwork(CMD_IP_OUT, context) + net = ds.OpenNebulaNetwork(context, + system_nics_by_mac=self.system_nics) self.assertEqual(net.gen_conf(), u'''\ auto lo iface lo inet loopback -- cgit v1.2.3 From ba0adb9b5100735358a76fdee7b251dba224a4cd Mon Sep 17 00:00:00 2001 From: Jim Gorz Date: Fri, 23 Sep 2016 09:57:00 -0700 Subject: Fix sshd restarts for rhel distros. Set the default value for 'ssh_svcname' in rhel distros. This means that it no longer needs to be set in system_info config. --- cloudinit/distros/rhel.py | 1 + 1 file changed, 1 insertion(+) diff --git a/cloudinit/distros/rhel.py b/cloudinit/distros/rhel.py index 1aa42d75..e574e1b9 100644 --- a/cloudinit/distros/rhel.py +++ b/cloudinit/distros/rhel.py @@ -62,6 +62,7 @@ class Distro(distros.Distro): self._runner = helpers.Runners(paths) self.osfamily = 'redhat' self._net_renderer = sysconfig.Renderer() + cfg['ssh_svcname'] = 'sshd' def install_packages(self, pkglist): self.package_command('install', pkgs=pkglist) -- cgit v1.2.3 From d8534561ba76db25b6fc0044eb1bfda63686e859 Mon Sep 17 00:00:00 2001 From: Ryan Harper Date: Thu, 1 Sep 2016 15:49:20 -0500 Subject: Add support for snap create-user on Ubuntu Core images. Ubuntu Core images use the `snap create-user` to add users to an Ubuntu Core system. Add support for creating snap users by adding a key to the users dictionary. users: - name: bob snapuser: bob@bobcom.io Or via the 'snappy' dictionary: snappy: email: bob@bobcom.io Users may also create a snap user without contacting the SSO by providing a 'system-user' assertion by importing them into snapd. Additionally, Ubuntu Core systems have a read-only /etc/passwd such that the normal useradd/groupadd commands do not function without an additional flag, '--extrausers', which redirects the pwd to /var/lib/extrausers. Move the system_is_snappy() check from cc_snappy module to util for re-use and then update the Distro class to append '--extrausers' if the system is Ubuntu Core. --- cloudinit/config/cc_snap_config.py | 184 +++++++++++++ cloudinit/config/cc_snappy.py | 18 +- cloudinit/distros/__init__.py | 35 +++ cloudinit/util.py | 12 + config/cloud.cfg | 1 + doc/examples/cloud-config-user-groups.txt | 8 + .../test_distros/test_user_data_normalize.py | 65 +++++ .../unittests/test_handler/test_handler_snappy.py | 293 ++++++++++++++++++++- 8 files changed, 601 insertions(+), 15 deletions(-) create mode 100644 cloudinit/config/cc_snap_config.py diff --git a/cloudinit/config/cc_snap_config.py b/cloudinit/config/cc_snap_config.py new file mode 100644 index 00000000..275a2d09 --- /dev/null +++ b/cloudinit/config/cc_snap_config.py @@ -0,0 +1,184 @@ +# vi: ts=4 expandtab +# +# Copyright (C) 2016 Canonical Ltd. +# +# Author: Ryan Harper +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 3, as +# published by the Free Software Foundation. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +""" +Snappy +------ +**Summary:** snap_config modules allows configuration of snapd. + +This module uses the same ``snappy`` namespace for configuration but +acts only only a subset of the configuration. + +If ``assertions`` is set and the user has included a list of assertions +then cloud-init will collect the assertions into a single assertion file +and invoke ``snap ack `` which will attempt +to load the provided assertions into the snapd assertion database. + +If ``email`` is set, this value is used to create an authorized user for +contacting and installing snaps from the Ubuntu Store. This is done by +calling ``snap create-user`` command. + +If ``known`` is set to True, then it is expected the user also included +an assertion of type ``system-user``. When ``snap create-user`` is called +cloud-init will append '--known' flag which instructs snapd to look for +a system-user assertion with the details. If ``known`` is not set, then +``snap create-user`` will contact the Ubuntu SSO for validating and importing +a system-user for the instance. + +.. note:: + If the system is already managed, then cloud-init will not attempt to + create a system-user. + +**Internal name:** ``cc_snap_config`` + +**Module frequency:** per instance + +**Supported distros:** any with 'snapd' available + +**Config keys**:: + + #cloud-config + snappy: + assertions: + - | + + - | + + email: user@user.org + known: true + +""" + +from cloudinit import log as logging +from cloudinit.settings import PER_INSTANCE +from cloudinit import util + +LOG = logging.getLogger(__name__) + +frequency = PER_INSTANCE +SNAPPY_CMD = "snap" +ASSERTIONS_FILE = "/var/lib/cloud/instance/snapd.assertions" + + +""" +snappy: + assertions: + - | + + - | + + email: foo@foo.io + known: true +""" + + +def add_assertions(assertions=None): + """Import list of assertions. + + Import assertions by concatenating each assertion into a + string separated by a '\n'. Write this string to a instance file and + then invoke `snap ack /path/to/file` and check for errors. + If snap exits 0, then all assertions are imported. + """ + if not assertions: + assertions = [] + + if not isinstance(assertions, list): + raise ValueError('assertion parameter was not a list: %s', assertions) + + snap_cmd = [SNAPPY_CMD, 'ack'] + combined = "\n".join(assertions) + if len(combined) == 0: + raise ValueError("Assertion list is empty") + + for asrt in assertions: + LOG.debug('Acking: %s', asrt.split('\n')[0:2]) + + util.write_file(ASSERTIONS_FILE, combined.encode('utf-8')) + util.subp(snap_cmd + [ASSERTIONS_FILE], capture=True) + + +def add_snap_user(cfg=None): + """Add a snap system-user if provided with email under snappy config. + + - Check that system is not already managed. + - Check that if using a system-user assertion, that it's + imported into snapd. + + Returns a dictionary to be passed to Distro.create_user + """ + + if not cfg: + cfg = {} + + if not isinstance(cfg, dict): + raise ValueError('configuration parameter was not a dict: %s', cfg) + + snapuser = cfg.get('email', None) + if not snapuser: + return + + usercfg = { + 'snapuser': snapuser, + 'known': cfg.get('known', False), + } + + # query if we're already registered + out, _ = util.subp([SNAPPY_CMD, 'managed'], capture=True) + if out.strip() == "true": + LOG.warning('This device is already managed. ' + 'Skipping system-user creation') + return + + if usercfg.get('known'): + # Check that we imported a system-user assertion + out, _ = util.subp([SNAPPY_CMD, 'known', 'system-user'], + capture=True) + if len(out) == 0: + LOG.error('Missing "system-user" assertion. ' + 'Check "snappy" user-data assertions.') + return + + return usercfg + + +def handle(name, cfg, cloud, log, args): + cfgin = cfg.get('snappy') + if not cfgin: + LOG.debug('No snappy config provided, skipping') + return + + if not(util.system_is_snappy()): + LOG.debug("%s: system not snappy", name) + return + + assertions = cfgin.get('assertions', []) + if len(assertions) > 0: + LOG.debug('Importing user-provided snap assertions') + add_assertions(assertions) + + # Create a snap user if requested. + # Snap systems contact the store with a user's email + # and extract information needed to create a local user. + # A user may provide a 'system-user' assertion which includes + # the required information. Using such an assertion to create + # a local user requires specifying 'known: true' in the supplied + # user-data. + usercfg = add_snap_user(cfg=cfgin) + if usercfg: + cloud.distro.create_user(usercfg.get('snapuser'), **usercfg) diff --git a/cloudinit/config/cc_snappy.py b/cloudinit/config/cc_snappy.py index 36db9e67..e03ec483 100644 --- a/cloudinit/config/cc_snappy.py +++ b/cloudinit/config/cc_snappy.py @@ -257,24 +257,14 @@ def disable_enable_ssh(enabled): util.write_file(not_to_be_run, "cloud-init\n") -def system_is_snappy(): - # channel.ini is configparser loadable. - # snappy will move to using /etc/system-image/config.d/*.ini - # this is certainly not a perfect test, but good enough for now. - content = util.load_file("/etc/system-image/channel.ini", quiet=True) - if 'ubuntu-core' in content.lower(): - return True - if os.path.isdir("/etc/system-image/config.d/"): - return True - return False - - def set_snappy_command(): global SNAPPY_CMD if util.which("snappy-go"): SNAPPY_CMD = "snappy-go" - else: + elif util.which("snappy"): SNAPPY_CMD = "snappy" + else: + SNAPPY_CMD = "snap" LOG.debug("snappy command is '%s'", SNAPPY_CMD) @@ -289,7 +279,7 @@ def handle(name, cfg, cloud, log, args): LOG.debug("%s: System is not snappy. disabling", name) return - if sys_snappy.lower() == "auto" and not(system_is_snappy()): + if sys_snappy.lower() == "auto" and not(util.system_is_snappy()): LOG.debug("%s: 'auto' mode, and system not snappy", name) return diff --git a/cloudinit/distros/__init__.py b/cloudinit/distros/__init__.py index 78adf5f9..4a726430 100755 --- a/cloudinit/distros/__init__.py +++ b/cloudinit/distros/__init__.py @@ -367,6 +367,9 @@ class Distro(object): adduser_cmd = ['useradd', name] log_adduser_cmd = ['useradd', name] + if util.system_is_snappy(): + adduser_cmd.append('--extrausers') + log_adduser_cmd.append('--extrausers') # Since we are creating users, we want to carefully validate the # inputs. If something goes wrong, we can end up with a system @@ -445,6 +448,32 @@ class Distro(object): util.logexc(LOG, "Failed to create user %s", name) raise e + def add_snap_user(self, name, **kwargs): + """ + Add a snappy user to the system using snappy tools + """ + + snapuser = kwargs.get('snapuser') + known = kwargs.get('known', False) + adduser_cmd = ["snap", "create-user", "--sudoer", "--json"] + if known: + adduser_cmd.append("--known") + adduser_cmd.append(snapuser) + + # Run the command + LOG.debug("Adding snap user %s", name) + try: + (out, err) = util.subp(adduser_cmd, logstring=adduser_cmd, + capture=True) + LOG.debug("snap create-user returned: %s:%s", out, err) + jobj = util.load_json(out) + username = jobj.get('username', None) + except Exception as e: + util.logexc(LOG, "Failed to create snap user %s", name) + raise e + + return username + def create_user(self, name, **kwargs): """ Creates users for the system using the GNU passwd tools. This @@ -452,6 +481,10 @@ class Distro(object): distros where useradd is not desirable or not available. """ + # Add a snap user, if requested + if 'snapuser' in kwargs: + return self.add_snap_user(name, **kwargs) + # Add the user self.add_user(name, **kwargs) @@ -602,6 +635,8 @@ class Distro(object): def create_group(self, name, members=None): group_add_cmd = ['groupadd', name] + if util.system_is_snappy(): + group_add_cmd.append('--extrausers') if not members: members = [] diff --git a/cloudinit/util.py b/cloudinit/util.py index 4cff83c5..4b3fd0cb 100644 --- a/cloudinit/util.py +++ b/cloudinit/util.py @@ -2374,3 +2374,15 @@ def get_installed_packages(target=None): pkgs_inst.add(re.sub(":.*", "", pkg)) return pkgs_inst + + +def system_is_snappy(): + # channel.ini is configparser loadable. + # snappy will move to using /etc/system-image/config.d/*.ini + # this is certainly not a perfect test, but good enough for now. + content = load_file("/etc/system-image/channel.ini", quiet=True) + if 'ubuntu-core' in content.lower(): + return True + if os.path.isdir("/etc/system-image/config.d/"): + return True + return False diff --git a/config/cloud.cfg b/config/cloud.cfg index d608dc86..1b93e7f9 100644 --- a/config/cloud.cfg +++ b/config/cloud.cfg @@ -45,6 +45,7 @@ cloud_config_modules: # Emit the cloud config ready event # this can be used by upstart jobs for 'start on cloud-config'. - emit_upstart + - snap_config - ssh-import-id - locale - set-passwords diff --git a/doc/examples/cloud-config-user-groups.txt b/doc/examples/cloud-config-user-groups.txt index 0e8ed243..9c5202f5 100644 --- a/doc/examples/cloud-config-user-groups.txt +++ b/doc/examples/cloud-config-user-groups.txt @@ -30,6 +30,7 @@ users: gecos: Magic Cloud App Daemon User inactive: true system: true + - snapuser: joe@joeuser.io # Valid Values: # name: The user's login name @@ -80,6 +81,13 @@ users: # cloud-init does not parse/check the syntax of the sudo # directive. # system: Create the user as a system user. This means no home directory. +# snapuser: Create a Snappy (Ubuntu-Core) user via the snap create-user +# command available on Ubuntu systems. If the user has an account +# on the Ubuntu SSO, specifying the email will allow snap to +# request a username and any public ssh keys and will import +# these into the system with username specifed by SSO account. +# If 'username' is not set in SSO, then username will be the +# shortname before the email domain. # # Default user creation: diff --git a/tests/unittests/test_distros/test_user_data_normalize.py b/tests/unittests/test_distros/test_user_data_normalize.py index b24888fc..33bf922d 100755 --- a/tests/unittests/test_distros/test_user_data_normalize.py +++ b/tests/unittests/test_distros/test_user_data_normalize.py @@ -4,6 +4,7 @@ from cloudinit import helpers from cloudinit import settings from ..helpers import TestCase +import mock bcfg = { @@ -296,3 +297,67 @@ class TestUGNormalize(TestCase): self.assertIn('bob', users) self.assertEqual({'default': False}, users['joe']) self.assertEqual({'default': False}, users['bob']) + + @mock.patch('cloudinit.util.subp') + def test_create_snap_user(self, mock_subp): + mock_subp.side_effect = [('{"username": "joe", "ssh-key-count": 1}\n', + '')] + distro = self._make_distro('ubuntu') + ug_cfg = { + 'users': [ + {'name': 'joe', 'snapuser': 'joe@joe.com'}, + ], + } + (users, _groups) = self._norm(ug_cfg, distro) + for (user, config) in users.items(): + print('user=%s config=%s' % (user, config)) + username = distro.create_user(user, **config) + + snapcmd = ['snap', 'create-user', '--sudoer', '--json', 'joe@joe.com'] + mock_subp.assert_called_with(snapcmd, capture=True, logstring=snapcmd) + self.assertEqual(username, 'joe') + + @mock.patch('cloudinit.util.subp') + def test_create_snap_user_known(self, mock_subp): + mock_subp.side_effect = [('{"username": "joe", "ssh-key-count": 1}\n', + '')] + distro = self._make_distro('ubuntu') + ug_cfg = { + 'users': [ + {'name': 'joe', 'snapuser': 'joe@joe.com', 'known': True}, + ], + } + (users, _groups) = self._norm(ug_cfg, distro) + for (user, config) in users.items(): + print('user=%s config=%s' % (user, config)) + username = distro.create_user(user, **config) + + snapcmd = ['snap', 'create-user', '--sudoer', '--json', '--known', + 'joe@joe.com'] + mock_subp.assert_called_with(snapcmd, capture=True, logstring=snapcmd) + self.assertEqual(username, 'joe') + + @mock.patch('cloudinit.util.system_is_snappy') + @mock.patch('cloudinit.util.is_group') + @mock.patch('cloudinit.util.subp') + def test_add_user_on_snappy_system(self, mock_subp, mock_isgrp, + mock_snappy): + mock_isgrp.return_value = False + mock_subp.return_value = True + mock_snappy.return_value = True + distro = self._make_distro('ubuntu') + ug_cfg = { + 'users': [ + {'name': 'joe', 'groups': 'users', 'create_groups': True}, + ], + } + (users, _groups) = self._norm(ug_cfg, distro) + for (user, config) in users.items(): + print('user=%s config=%s' % (user, config)) + distro.add_user(user, **config) + + groupcmd = ['groupadd', 'users', '--extrausers'] + addcmd = ['useradd', 'joe', '--extrausers', '--groups', 'users', '-m'] + + mock_subp.assert_any_call(groupcmd) + mock_subp.assert_any_call(addcmd, logstring=addcmd) diff --git a/tests/unittests/test_handler/test_handler_snappy.py b/tests/unittests/test_handler/test_handler_snappy.py index 57dce1bc..e320dd82 100644 --- a/tests/unittests/test_handler/test_handler_snappy.py +++ b/tests/unittests/test_handler/test_handler_snappy.py @@ -1,14 +1,22 @@ from cloudinit.config.cc_snappy import ( makeop, get_package_ops, render_snap_op) -from cloudinit import util +from cloudinit.config.cc_snap_config import ( + add_assertions, add_snap_user, ASSERTIONS_FILE) +from cloudinit import (distros, helpers, cloud, util) +from cloudinit.config.cc_snap_config import handle as snap_handle +from cloudinit.sources import DataSourceNone +from ..helpers import FilesystemMockingTestCase, mock from .. import helpers as t_help +import logging import os import shutil import tempfile +import textwrap import yaml +LOG = logging.getLogger(__name__) ALLOWED = (dict, list, int, str) @@ -287,6 +295,289 @@ class TestInstallPackages(t_help.TestCase): self.assertEqual(yaml.safe_load(mydata), data_found) +class TestSnapConfig(FilesystemMockingTestCase): + + SYSTEM_USER_ASSERTION = textwrap.dedent(""" + type: system-user + authority-id: LqvZQdfyfGlYvtep4W6Oj6pFXP9t1Ksp + brand-id: LqvZQdfyfGlYvtep4W6Oj6pFXP9t1Ksp + email: foo@bar.com + password: $6$E5YiAuMIPAwX58jG$miomhVNui/vf7f/3ctB/f0RWSKFxG0YXzrJ9rtJ1ikvzt + series: + - 16 + since: 2016-09-10T16:34:00+03:00 + until: 2017-11-10T16:34:00+03:00 + username: baz + sign-key-sha3-384: RuVvnp4n52GilycjfbbTCI3_L8Y6QlIE75wxMc0KzGV3AUQqVd9GuXoj + + AcLBXAQAAQoABgUCV/UU1wAKCRBKnlMoJQLkZVeLD/9/+hIeVywtzsDA3oxl+P+u9D13y9s6svP + Jd6Wnf4FTw6sq1GjBE4ZA7lrwSaRCUJ9Vcsvf2q9OGPY7mOb2TBxaDe0PbUMjrSrqllSSQwhpNI + zG+NxkkKuxsUmLzFa+k9m6cyojNbw5LFhQZBQCGlr3JYqC0tIREq/UsZxj+90TUC87lDJwkU8GF + s4CR+rejZj4itIcDcVxCSnJH6hv6j2JrJskJmvObqTnoOlcab+JXdamXqbldSP3UIhWoyVjqzkj + +to7mXgx+cCUA9+ngNCcfUG+1huGGTWXPCYkZ78HvErcRlIdeo4d3xwtz1cl/w3vYnq9og1XwsP + Yfetr3boig2qs1Y+j/LpsfYBYncgWjeDfAB9ZZaqQz/oc8n87tIPZDJHrusTlBfop8CqcM4xsKS + d+wnEY8e/F24mdSOYmS1vQCIDiRU3MKb6x138Ud6oHXFlRBbBJqMMctPqWDunWzb5QJ7YR0I39q + BrnEqv5NE0G7w6HOJ1LSPG5Hae3P4T2ea+ATgkb03RPr3KnXnzXg4TtBbW1nytdlgoNc/BafE1H + f3NThcq9gwX4xWZ2PAWnqVPYdDMyCtzW3Ck+o6sIzx+dh4gDLPHIi/6TPe/pUuMop9CBpWwez7V + v1z+1+URx6Xlq3Jq18y5pZ6fY3IDJ6km2nQPMzcm4Q==""") + + ACCOUNT_ASSERTION = textwrap.dedent(""" + type: account-key + authority-id: canonical + revision: 2 + public-key-sha3-384: BWDEoaqyr25nF5SNCvEv2v7QnM9QsfCc0PBMYD_i2NGSQ32EF2d4D0 + account-id: canonical + name: store + since: 2016-04-01T00:00:00.0Z + body-length: 717 + sign-key-sha3-384: -CvQKAwRQ5h3Ffn10FILJoEZUXOv6km9FwA80-Rcj-f-6jadQ89VRswH + + AcbBTQRWhcGAARAA0KKYYQWuHOrsFVi4p4l7ZzSvX7kLgJFFeFgOkzdWKBTHEnsMKjl5mefFe9j + qe8NlmJdfY7BenP7XeBtwKp700H/t9lLrZbpTNAPHXYxEWFJp5bPqIcJYBZ+29oLVLN1Tc5X482 + vCiDqL8+pPYqBrK2fNlyPlNNSum9wI70rDDL4r6FVvr+osTnGejibdV8JphWX+lrSQDnRSdM8KJ + UM43vTgLGTi9W54oRhsA2OFexRfRksTrnqGoonCjqX5wO3OFSaMDzMsO2MJ/hPfLgDqw53qjzuK + Iec9OL3k5basvu2cj5u9tKwVFDsCKK2GbKUsWWpx2KTpOifmhmiAbzkTHbH9KaoMS7p0kJwhTQG + o9aJ9VMTWHJc/NCBx7eu451u6d46sBPCXS/OMUh2766fQmoRtO1OwCTxsRKG2kkjbMn54UdFULl + VfzvyghMNRKIezsEkmM8wueTqGUGZWa6CEZqZKwhe/PROxOPYzqtDH18XZknbU1n5lNb7vNfem9 + 2ai+3+JyFnW9UhfvpVF7gzAgdyCqNli4C6BIN43uwoS8HkykocZS/+Gv52aUQ/NZ8BKOHLw+7an + Q0o8W9ltSLZbEMxFIPSN0stiZlkXAp6DLyvh1Y4wXSynDjUondTpej2fSvSlCz/W5v5V7qA4nIc + vUvV7RjVzv17ut0AEQEAAQ== + + AcLDXAQAAQoABgUCV83k9QAKCRDUpVvql9g3IBT8IACKZ7XpiBZ3W4lqbPssY6On81WmxQLtvsM + WTp6zZpl/wWOSt2vMNUk9pvcmrNq1jG9CuhDfWFLGXEjcrrmVkN3YuCOajMSPFCGrxsIBLSRt/b + nrKykdLAAzMfG8rP1d82bjFFiIieE+urQ0Kcv09Jtdvavq3JT1Tek5mFyyfhHNlQEKOzWqmRWiL + 3c3VOZUs1ZD8TSlnuq/x+5T0X0YtOyGjSlVxk7UybbyMNd6MZfNaMpIG4x+mxD3KHFtBAC7O6kL + eX3i6j5nCY5UABfA3DZEAkWP4zlmdBEOvZ9t293NaDdOpzsUHRkoi0Zez/9BHQ/kwx/uNc2WqrY + inCmu16JGNeXqsyinnLl7Ghn2RwhvDMlLxF6RTx8xdx1yk6p3PBTwhZMUvuZGjUtN/AG8BmVJQ1 + rsGSRkkSywvnhVJRB2sudnrMBmNS2goJbzSbmJnOlBrd2WsV0T9SgNMWZBiov3LvU4o2SmAb6b+ + rYwh8H5QHcuuYJuxDjFhPswIp6Wes5T6hUicf3SWtObcDS4HSkVS4ImBjjX9YgCuFy7QdnooOWE + aPvkRw3XCVeYq0K6w9GRsk1YFErD4XmXXZjDYY650MX9v42Sz5MmphHV8jdIY5ssbadwFSe2rCQ + 6UX08zy7RsIb19hTndE6ncvSNDChUR9eEnCm73eYaWTWTnq1cxdVP/s52r8uss++OYOkPWqh5nO + haRn7INjH/yZX4qXjNXlTjo0PnHH0q08vNKDwLhxS+D9du+70FeacXFyLIbcWllSbJ7DmbumGpF + yYbtj3FDDPzachFQdIG3lSt+cSUGeyfSs6wVtc3cIPka/2Urx7RprfmoWSI6+a5NcLdj0u2z8O9 + HxeIgxDpg/3gT8ZIuFKePMcLDM19Fh/p0ysCsX+84B9chNWtsMSmIaE57V+959MVtsLu7SLb9gi + skrju0pQCwsu2wHMLTNd1f3PTHmrr49hxetTus07HSQUApMtAGKzQilF5zqFjbyaTd4xgQbd+PK + CjFyzQTDOcUhXpuUGt/IzlqiFfsCsmbj2K4KdSNYMlqIgZ3Azu8KvZLIhsyN7v5vNIZSPfEbjde + ClU9r0VRiJmtYBUjcSghD9LWn+yRLwOxhfQVjm0cBwIt5R/yPF/qC76yIVuWUtM5Y2/zJR1J8OF + qWchvlImHtvDzS9FQeLyzJAOjvZ2CnWp2gILgUz0WQdOk1Dq8ax7KS9BQ42zxw9EZAEPw3PEFqR + IQsRTONp+iVS8YxSmoYZjDlCgRMWUmawez/Fv5b9Fb/XkO5Eq4e+KfrpUujXItaipb+tV8h5v3t + oG3Ie3WOHrVjCLXIdYslpL1O4nadqR6Xv58pHj6k""") + + test_assertions = [ACCOUNT_ASSERTION, SYSTEM_USER_ASSERTION] + + def setUp(self): + super(TestSnapConfig, self).setUp() + self.subp = util.subp + self.new_root = tempfile.mkdtemp() + self.addCleanup(shutil.rmtree, self.new_root) + + def _get_cloud(self, distro, metadata=None): + self.patchUtils(self.new_root) + paths = helpers.Paths({}) + cls = distros.fetch(distro) + mydist = cls(distro, {}, paths) + myds = DataSourceNone.DataSourceNone({}, mydist, paths) + if metadata: + myds.metadata.update(metadata) + return cloud.Cloud(myds, paths, {}, mydist, None) + + @mock.patch('cloudinit.util.write_file') + @mock.patch('cloudinit.util.subp') + def test_snap_config_add_assertions(self, msubp, mwrite): + add_assertions(self.test_assertions) + + combined = "\n".join(self.test_assertions) + mwrite.assert_any_call(ASSERTIONS_FILE, combined.encode('utf-8')) + msubp.assert_called_with(['snap', 'ack', ASSERTIONS_FILE], + capture=True) + + def test_snap_config_add_assertions_empty(self): + self.assertRaises(ValueError, add_assertions, []) + + def test_add_assertions_nonlist(self): + self.assertRaises(ValueError, add_assertions, {}) + + @mock.patch('cloudinit.util.write_file') + @mock.patch('cloudinit.util.subp') + def test_snap_config_add_assertions_ack_fails(self, msubp, mwrite): + msubp.side_effect = [util.ProcessExecutionError("Invalid assertion")] + self.assertRaises(util.ProcessExecutionError, add_assertions, + self.test_assertions) + + @mock.patch('cloudinit.config.cc_snap_config.add_assertions') + @mock.patch('cloudinit.config.cc_snap_config.util') + def test_snap_config_handle_no_config(self, mock_util, mock_add): + cfg = {} + cc = self._get_cloud('ubuntu') + cc.distro = mock.MagicMock() + cc.distro.name = 'ubuntu' + mock_util.which.return_value = None + snap_handle('snap_config', cfg, cc, LOG, None) + mock_add.assert_not_called() + + def test_snap_config_add_snap_user_no_config(self): + usercfg = add_snap_user(cfg=None) + self.assertEqual(usercfg, None) + + def test_snap_config_add_snap_user_not_dict(self): + cfg = ['foobar'] + self.assertRaises(ValueError, add_snap_user, cfg) + + def test_snap_config_add_snap_user_no_email(self): + cfg = {'assertions': [], 'known': True} + usercfg = add_snap_user(cfg=cfg) + self.assertEqual(usercfg, None) + + @mock.patch('cloudinit.config.cc_snap_config.util') + def test_snap_config_add_snap_user_email_only(self, mock_util): + email = 'janet@planetjanet.org' + cfg = {'email': email} + mock_util.which.return_value = None + mock_util.system_is_snappy.return_value = True + mock_util.subp.side_effect = [ + ("false\n", ""), # snap managed + ] + + usercfg = add_snap_user(cfg=cfg) + + self.assertEqual(usercfg, {'snapuser': email, 'known': False}) + + @mock.patch('cloudinit.config.cc_snap_config.util') + def test_snap_config_add_snap_user_email_known(self, mock_util): + email = 'janet@planetjanet.org' + known = True + cfg = {'email': email, 'known': known} + mock_util.which.return_value = None + mock_util.system_is_snappy.return_value = True + mock_util.subp.side_effect = [ + ("false\n", ""), # snap managed + (self.SYSTEM_USER_ASSERTION, ""), # snap known system-user + ] + + usercfg = add_snap_user(cfg=cfg) + + self.assertEqual(usercfg, {'snapuser': email, 'known': known}) + + @mock.patch('cloudinit.config.cc_snap_config.add_assertions') + @mock.patch('cloudinit.config.cc_snap_config.util') + def test_snap_config_handle_system_not_snappy(self, mock_util, mock_add): + cfg = {'snappy': {'assertions': self.test_assertions}} + cc = self._get_cloud('ubuntu') + cc.distro = mock.MagicMock() + cc.distro.name = 'ubuntu' + mock_util.which.return_value = None + mock_util.system_is_snappy.return_value = False + + snap_handle('snap_config', cfg, cc, LOG, None) + + mock_add.assert_not_called() + + @mock.patch('cloudinit.config.cc_snap_config.add_assertions') + @mock.patch('cloudinit.config.cc_snap_config.util') + def test_snap_config_handle_snapuser(self, mock_util, mock_add): + email = 'janet@planetjanet.org' + cfg = { + 'snappy': { + 'assertions': self.test_assertions, + 'email': email, + } + } + cc = self._get_cloud('ubuntu') + cc.distro = mock.MagicMock() + cc.distro.name = 'ubuntu' + mock_util.which.return_value = None + mock_util.system_is_snappy.return_value = True + mock_util.subp.side_effect = [ + ("false\n", ""), # snap managed + ] + + snap_handle('snap_config', cfg, cc, LOG, None) + + mock_add.assert_called_with(self.test_assertions) + usercfg = {'snapuser': email, 'known': False} + cc.distro.create_user.assert_called_with(email, **usercfg) + + @mock.patch('cloudinit.config.cc_snap_config.add_assertions') + @mock.patch('cloudinit.config.cc_snap_config.util') + def test_snap_config_handle_snapuser_known(self, mock_util, mock_add): + email = 'janet@planetjanet.org' + cfg = { + 'snappy': { + 'assertions': self.test_assertions, + 'email': email, + 'known': True, + } + } + cc = self._get_cloud('ubuntu') + cc.distro = mock.MagicMock() + cc.distro.name = 'ubuntu' + mock_util.which.return_value = None + mock_util.system_is_snappy.return_value = True + mock_util.subp.side_effect = [ + ("false\n", ""), # snap managed + (self.SYSTEM_USER_ASSERTION, ""), # snap known system-user + ] + + snap_handle('snap_config', cfg, cc, LOG, None) + + mock_add.assert_called_with(self.test_assertions) + usercfg = {'snapuser': email, 'known': True} + cc.distro.create_user.assert_called_with(email, **usercfg) + + @mock.patch('cloudinit.config.cc_snap_config.add_assertions') + @mock.patch('cloudinit.config.cc_snap_config.util') + def test_snap_config_handle_snapuser_known_managed(self, mock_util, + mock_add): + email = 'janet@planetjanet.org' + cfg = { + 'snappy': { + 'assertions': self.test_assertions, + 'email': email, + 'known': True, + } + } + cc = self._get_cloud('ubuntu') + cc.distro = mock.MagicMock() + cc.distro.name = 'ubuntu' + mock_util.which.return_value = None + mock_util.system_is_snappy.return_value = True + mock_util.subp.side_effect = [ + ("true\n", ""), # snap managed + ] + + snap_handle('snap_config', cfg, cc, LOG, None) + + mock_add.assert_called_with(self.test_assertions) + cc.distro.create_user.assert_not_called() + + @mock.patch('cloudinit.config.cc_snap_config.add_assertions') + @mock.patch('cloudinit.config.cc_snap_config.util') + def test_snap_config_handle_snapuser_known_no_assertion(self, mock_util, + mock_add): + email = 'janet@planetjanet.org' + cfg = { + 'snappy': { + 'assertions': [self.ACCOUNT_ASSERTION], + 'email': email, + 'known': True, + } + } + cc = self._get_cloud('ubuntu') + cc.distro = mock.MagicMock() + cc.distro.name = 'ubuntu' + mock_util.which.return_value = None + mock_util.system_is_snappy.return_value = True + mock_util.subp.side_effect = [ + ("true\n", ""), # snap managed + ("", ""), # snap known system-user + ] + + snap_handle('snap_config', cfg, cc, LOG, None) + + mock_add.assert_called_with([self.ACCOUNT_ASSERTION]) + cc.distro.create_user.assert_not_called() + + def makeop_tmpd(tmpd, op, name, config=None, path=None, cfgfile=None): if cfgfile: cfgfile = os.path.sep.join([tmpd, cfgfile]) -- cgit v1.2.3 From f6ae1f9cb1495b14623eed60bef5afbdec85f607 Mon Sep 17 00:00:00 2001 From: Wesley Wiedenmeier Date: Sat, 8 Oct 2016 18:54:47 -0500 Subject: Add documentation for logging features. Update the summary of rsyslog module and add logging.rst to docs. --- cloudinit/config/cc_rsyslog.py | 2 + doc/rtd/index.rst | 1 + doc/rtd/topics/logging.rst | 175 +++++++++++++++++++++++++++++++++++++++++ 3 files changed, 178 insertions(+) create mode 100644 doc/rtd/topics/logging.rst diff --git a/cloudinit/config/cc_rsyslog.py b/cloudinit/config/cc_rsyslog.py index 48f18620..1c12e567 100644 --- a/cloudinit/config/cc_rsyslog.py +++ b/cloudinit/config/cc_rsyslog.py @@ -18,6 +18,8 @@ # You should have received a copy of the GNU General Public License # along with this program. If not, see . """ +.. _cc_rsyslog: + Rsyslog ------- **Summary:** configure system loggig via rsyslog diff --git a/doc/rtd/index.rst b/doc/rtd/index.rst index fe04b1a9..f8ff3c9f 100644 --- a/doc/rtd/index.rst +++ b/doc/rtd/index.rst @@ -23,6 +23,7 @@ Summary topics/dir_layout topics/examples topics/datasources + topics/logging topics/modules topics/merging topics/moreinfo diff --git a/doc/rtd/topics/logging.rst b/doc/rtd/topics/logging.rst new file mode 100644 index 00000000..b010aa96 --- /dev/null +++ b/doc/rtd/topics/logging.rst @@ -0,0 +1,175 @@ +======= +Logging +======= +Cloud-init supports both local and remote logging configurable through python's +built-in logging configuration and through the cloud-init rsyslog module. + +Command Output +-------------- +Cloud-init can redirect its stdout and stderr based on config given under the +``output`` config key. The output of any commands run by cloud-init and any +user or vendor scripts provided will also be included here. The ``output`` +key accepts a dictionary for configuration. Output files may be specified +individually for each stage (``init``, ``config``, and ``final``), or a single +key ``all`` may be used to specify output for all stages. + +The output for each stage may be specified as a dictionary of ``output`` and +``error`` keys, for stdout and stderr respectively, as a tuple with stdout +first and stderr second, or as a single string to use for both. The strings +passed to all of these keys are handled by the system shell, so any form of +redirection that can be used in bash is valid, including piping cloud-init's +output to ``tee``, or ``logger``. If only a filename is provided, cloud-init +will append its output to the file as though ``>>`` was specified. + +By default, cloud-init loads its output configuration from +``/etc/cloud/coud.cfg.d/05_logging.cfg``. The default config directs both +stdout and stderr from all cloud-init stages to +``/var/log/cloud-init-output.log``. The default config is given as :: + + output: { all: "| tee -a /var/log/cloud-init-output.log" } + +For a more complex example, the following configuration would output the init +stage to ``/var/log/cloud-init.out`` and ``/var/log/cloud-init.err``, for +stdout and stderr respectively, replacing anything that was previously there. +For the config stage, it would pipe both stdout and stderr through +``tee -a /var/log/cloud-config.log``. For the final stage it would append the +output of stdout and stderr to ``/var/log/cloud-final.out`` and +``/var/log/cloud-final.err`` respectively. :: + + output: + init: + output: "> /var/log/cloud-init.out" + error: "> /var/log/cloud-init.err" + config: "tee -a /var/log/cloud-config.log" + final: + - ">> /var/log/cloud-final.out" + - "/var/log/cloud-final.err" + +Python Logging +-------------- +Cloud-init uses the python logging module, and can accept config for this +module using the standard python fileConfig format. Cloud-init looks for config +for the logging module under the ``logcfg`` key. + +.. note:: + the logging configuration is not yaml, it is python ``fileConfig`` format, + and is passed through directly to the python logging module. please use the + correct syntax for a multi-line string in yaml. + +By default, cloud-init uses the logging configuration provided in +``/etc/cloud/cloud.cfg.d/05_logging.cfg``. The default python logging +configuration writes all cloud-init events with a priority of ``WARNING`` or +higher to console, and writes all events with a level of ``DEBUG`` or higher +to ``/var/log/cloud-init.log`` and via syslog. + +Python's fileConfig format consists of sections with headings in the format +``[title]`` and key value pairs in each section. Configuration for python +logging must contain the sections ``[loggers]``, ``[handlers]``, and +``[formatters]``, which name the entities of their respective types that will +be defined. The section name for each defined logger, handler and formatter +will start with its type, followed by an underscore (``_``) and the name of the +entity. For example, if a logger was specified with the name ``log01``, config +for the logger would be in the section ``[logger_log01]``. + +Logger config entries contain basic logging set up. They may specify a list of +handlers to send logging events to as well as the lowest priority level of +events to handle. A logger named ``root`` must be specified and its +configuration (under ``[logger_root]``) must contain a level and a list of +handlers. A level entry can be any of the following: ``DEBUG``, ``INFO``, +``WARNING``, ``ERROR``, ``CRITICAL``, or ``NOTSET``. For the ``root`` logger +the ``NOTSET`` option will allow all logging events to be recorded. + +Each configured handler must specify a class under the python's ``logging`` +package namespace. A handler may specify a message formatter to use, a priority +level, and arguments for the handler class. Common handlers are +``StreamHandler``, which handles stream redirects (i.e. logging to stderr), +and ``FileHandler`` which outputs to a log file. The logging module also +supports logging over net sockets, over http, via smtp, and additional +complex configurations. For full details about the handlers available for +python logging, please see the documentation for `python logging handlers`_. + +Log messages are formatted using the ``logging.Formatter`` class, which is +configured using ``formatter`` config entities. A default format of +``%(message)s`` is given if no formatter configs are specified. Formatter +config entities accept a format string which supports variable replacements. +These may also accept a ``datefmt`` string which may be used to configure the +timestamp used in the log messages. The format variables ``%(asctime)s``, +``%(levelname)s`` and ``%(message)s`` are commonly used and represent the +timestamp, the priority level of the event and the event message. For +additional information on logging formatters see `python logging formatters`_. + +.. note:: + by default the format string used in the logging formatter are in python's + old style ``%s`` form. the ``str.format()`` and ``string.Template`` styles + can also be used by using ``{`` or ``$`` in place of ``%`` by setting the + ``style`` parameter in formatter config. + +A simple, but functional python logging configuration for cloud-init is below. +It will log all messages of priority ``DEBUG`` or higher both stderr and +``/tmp/my.log`` using a ``StreamHandler`` and a ``FileHandler``, using +the default format string ``%(message)s``:: + + logcfg: | + [loggers] + keys=root,cloudinit + [handlers] + keys=ch,cf + [formatters] + keys= + [logger_root] + level=DEBUG + handlers= + [logger_cloudinit] + level=DEBUG + qualname=cloudinit + handlers=ch,cf + [handler_ch] + class=StreamHandler + level=DEBUG + args=(sys.stderr,) + [handler_cf] + class=FileHandler + level=DEBUG + args=('/tmp/my.log',) + +For additional information about configuring python's logging module, please +see the documentation for `python logging config`_. + +Rsyslog Module +-------------- +Cloud-init's ``cc_rsyslog`` module allows for fully customizable rsyslog +configuration under the ``rsyslog`` config key. The simplest way to +use the rsyslog module is by specifying remote servers under the ``remotes`` +key in ``rsyslog`` config. The ``remotes`` key takes a dictionary where each +key represents the name of an rsyslog server and each value is the +configuration for that server. The format for server config is: + + - optional filter for log messages (defaults to ``*.*``) + - optional leading ``@`` or ``@@``, indicating udp and tcp respectively + (defaults to ``@``, for udp) + - ipv4 or ipv6 hostname or address. ipv6 addresses must be in ``[::1]`` + format, (e.g. ``@[fd00::1]:514``) + - optional port number (defaults to ``514``) + +For example, to send logging to an rsyslog server named ``log_serv`` with +address ``10.0.4.1``, using port number ``514``, over udp, with all log +messages enabled one could use either of the following. + +With all options specified:: + + rsyslog: + remotes: + log_serv: "*.* @10.0.4.1:514" + +With defaults used:: + + rsyslog: + remotes: + log_serv: "10.0.4.1" + + +For more information on rsyslog configuration, see :ref:`cc_rsyslog`. + +.. _python logging config: https://docs.python.org/3/library/logging.config.html#configuration-file-format +.. _python logging handlers: https://docs.python.org/3/library/logging.handlers.html +.. _python logging formatters: https://docs.python.org/3/library/logging.html#formatter-objects -- cgit v1.2.3 From 1e55f4127f356b930e2c1ad36dcb6bed24f3beb2 Mon Sep 17 00:00:00 2001 From: Scott Moser Date: Mon, 24 Oct 2016 10:31:21 -0400 Subject: unittests: do not read system /etc/cloud/cloud.cfg.d Many of the unit tests in test_data would inadvertantly read the system's /etc/cloud/cloud.cfg and /etc/cloud/cloud.cfg.d. This was first noticed on a system deployed by MAAS, where files in /etc/cloud/cloud.cfg.d/ are root read-only. This changes those tests to actually make use of FilesystemMockingTestCase functionality and adds 'reRoot()' to that class which is easier to use for at least this use case. LP: #1635350 --- tests/unittests/helpers.py | 8 ++++++++ tests/unittests/test_data.py | 45 +++++++++++++++----------------------------- 2 files changed, 23 insertions(+), 30 deletions(-) diff --git a/tests/unittests/helpers.py b/tests/unittests/helpers.py index 1cdc05a1..a2355a79 100644 --- a/tests/unittests/helpers.py +++ b/tests/unittests/helpers.py @@ -205,6 +205,14 @@ class FilesystemMockingTestCase(ResourceUsingTestCase): self.patched_funcs.enter_context( mock.patch.object(sys, 'stderr', stderr)) + def reRoot(self, root=None): + if root is None: + root = tempfile.mkdtemp() + self.addCleanup(shutil.rmtree, root) + self.patchUtils(root) + self.patchOS(root) + return root + def import_httpretty(): """Import HTTPretty and monkey patch Python 3.4 issue. diff --git a/tests/unittests/test_data.py b/tests/unittests/test_data.py index 13db8a4c..55d9b93f 100644 --- a/tests/unittests/test_data.py +++ b/tests/unittests/test_data.py @@ -3,8 +3,6 @@ import gzip import logging import os -import shutil -import tempfile try: from unittest import mock @@ -98,10 +96,7 @@ class TestConsumeUserData(helpers.FilesystemMockingTestCase): ci = stages.Init() ci.datasource = FakeDataSource(blob) - new_root = tempfile.mkdtemp() - self.addCleanup(shutil.rmtree, new_root) - self.patchUtils(new_root) - self.patchOS(new_root) + self.reRoot() ci.fetch() ci.consume_data() cc_contents = util.load_file(ci.paths.get_ipath("cloud_config")) @@ -127,9 +122,7 @@ class TestConsumeUserData(helpers.FilesystemMockingTestCase): { "op": "add", "path": "/foo", "value": "quxC" } ] ''' - new_root = tempfile.mkdtemp() - self.addCleanup(shutil.rmtree, new_root) - self._patchIn(new_root) + self.reRoot() initer = stages.Init() initer.datasource = FakeDataSource(user_blob, vendordata=vendor_blob) initer.read_cfg() @@ -167,9 +160,7 @@ class TestConsumeUserData(helpers.FilesystemMockingTestCase): { "op": "add", "path": "/foo", "value": "quxC" } ] ''' - new_root = tempfile.mkdtemp() - self.addCleanup(shutil.rmtree, new_root) - self._patchIn(new_root) + self.reRoot() initer = stages.Init() initer.datasource = FakeDataSource(user_blob, vendordata=vendor_blob) initer.read_cfg() @@ -212,12 +203,9 @@ c: d message.attach(message_cc) message.attach(message_jp) + self.reRoot() ci = stages.Init() ci.datasource = FakeDataSource(str(message)) - new_root = tempfile.mkdtemp() - self.addCleanup(shutil.rmtree, new_root) - self.patchUtils(new_root) - self.patchOS(new_root) ci.fetch() ci.consume_data() cc_contents = util.load_file(ci.paths.get_ipath("cloud_config")) @@ -245,9 +233,7 @@ name: user run: - z ''' - new_root = tempfile.mkdtemp() - self.addCleanup(shutil.rmtree, new_root) - self._patchIn(new_root) + self.reRoot() initer = stages.Init() initer.datasource = FakeDataSource(user_blob, vendordata=vendor_blob) initer.read_cfg() @@ -281,9 +267,7 @@ vendor_data: enabled: True prefix: /bin/true ''' - new_root = tempfile.mkdtemp() - self.addCleanup(shutil.rmtree, new_root) - self._patchIn(new_root) + new_root = self.reRoot() initer = stages.Init() initer.datasource = FakeDataSource(user_blob, vendordata=vendor_blob) initer.read_cfg() @@ -342,10 +326,7 @@ p: 1 paths = c_helpers.Paths({}, ds=FakeDataSource('')) cloud_cfg = handlers.cloud_config.CloudConfigPartHandler(paths) - new_root = tempfile.mkdtemp() - self.addCleanup(shutil.rmtree, new_root) - self.patchUtils(new_root) - self.patchOS(new_root) + self.reRoot() cloud_cfg.handle_part(None, handlers.CONTENT_START, None, None, None, None) for i, m in enumerate(messages): @@ -365,6 +346,7 @@ p: 1 def test_unhandled_type_warning(self): """Raw text without magic is ignored but shows warning.""" + self.reRoot() ci = stages.Init() data = "arbitrary text\n" ci.datasource = FakeDataSource(data) @@ -402,10 +384,7 @@ c: 4 message.attach(gzip_part(base_content2)) ci = stages.Init() ci.datasource = FakeDataSource(str(message)) - new_root = tempfile.mkdtemp() - self.addCleanup(shutil.rmtree, new_root) - self.patchUtils(new_root) - self.patchOS(new_root) + self.reRoot() ci.fetch() ci.consume_data() contents = util.load_file(ci.paths.get_ipath("cloud_config")) @@ -418,6 +397,7 @@ c: 4 def test_mime_text_plain(self): """Mime message of type text/plain is ignored but shows warning.""" + self.reRoot() ci = stages.Init() message = MIMEBase("text", "plain") message.set_payload("Just text") @@ -435,6 +415,7 @@ c: 4 def test_shellscript(self): """Raw text starting #!/bin/sh is treated as script.""" + self.reRoot() ci = stages.Init() script = "#!/bin/sh\necho hello\n" ci.datasource = FakeDataSource(script) @@ -453,6 +434,7 @@ c: 4 def test_mime_text_x_shellscript(self): """Mime message of type text/x-shellscript is treated as script.""" + self.reRoot() ci = stages.Init() script = "#!/bin/sh\necho hello\n" message = MIMEBase("text", "x-shellscript") @@ -473,6 +455,7 @@ c: 4 def test_mime_text_plain_shell(self): """Mime type text/plain starting #!/bin/sh is treated as script.""" + self.reRoot() ci = stages.Init() script = "#!/bin/sh\necho hello\n" message = MIMEBase("text", "plain") @@ -493,6 +476,7 @@ c: 4 def test_mime_application_octet_stream(self): """Mime type application/octet-stream is ignored but shows warning.""" + self.reRoot() ci = stages.Init() message = MIMEBase("application", "octet-stream") message.set_payload(b'\xbf\xe6\xb2\xc3\xd3\xba\x13\xa4\xd8\xa1\xcc') @@ -516,6 +500,7 @@ c: 4 {'content': non_decodable}] message = b'#cloud-config-archive\n' + util.yaml_dumps(data).encode() + self.reRoot() ci = stages.Init() ci.datasource = FakeDataSource(message) -- cgit v1.2.3 From 29348af1c889931e8973f8fc8cb090c063316f7a Mon Sep 17 00:00:00 2001 From: Scott Moser Date: Fri, 21 Oct 2016 13:54:51 -0400 Subject: disk-config: udev settle after partitioning in gpt format. The function exec_mkpart_gpt was simply not waiting for udev events to flush after calling sgdisk. The corresponding function exec_mkpart_mbr already did. This should fix a transient failure where mkfs would fail with 'not a block device'. LP: #1626243 --- cloudinit/config/cc_disk_setup.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/cloudinit/config/cc_disk_setup.py b/cloudinit/config/cc_disk_setup.py index efa7a226..0c4b794d 100644 --- a/cloudinit/config/cc_disk_setup.py +++ b/cloudinit/config/cc_disk_setup.py @@ -716,6 +716,8 @@ def exec_mkpart_gpt(device, layout): LOG.warn("Failed to partition device %s" % device) raise + read_parttbl(device) + def exec_mkpart(table_type, device, layout): """ -- cgit v1.2.3