From 1f3a225af78dbfbff75c3faad28a5dc8cad0d1e3 Mon Sep 17 00:00:00 2001 From: Paride Legovini Date: Thu, 27 Aug 2020 17:20:35 +0200 Subject: LXD: detach network from profile before deleting it (#542) * LXD: detach network from profile before deleting it When cleaning up the bridge network created by default by LXD as part of the `lxd init` process detach the network its profile before deleting it. LXD will otherwise refuse to delete it with error: Error: The network is currently in use. Discussion with LXD upstream: https://github.com/lxc/lxd/issues/7804. LP: #1776958 * LXD bridge deletion: fail if bridge exists but can't be deleted * LXD bridge deletion: remove useless failure logging --- cloudinit/config/cc_lxd.py | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) (limited to 'cloudinit/config') diff --git a/cloudinit/config/cc_lxd.py b/cloudinit/config/cc_lxd.py index 7129c9c6..486037d9 100644 --- a/cloudinit/config/cc_lxd.py +++ b/cloudinit/config/cc_lxd.py @@ -283,14 +283,18 @@ def maybe_cleanup_default(net_name, did_init, create, attach, fail_assume_enoent = "failed. Assuming it did not exist." succeeded = "succeeded." if create: - msg = "Deletion of lxd network '%s' %s" + msg = "Detach of lxd network '%s' from profile '%s' %s" try: - _lxc(["network", "delete", net_name]) - LOG.debug(msg, net_name, succeeded) + _lxc(["network", "detach-profile", net_name, profile]) + LOG.debug(msg, net_name, profile, succeeded) except subp.ProcessExecutionError as e: if e.exit_code != 1: raise e - LOG.debug(msg, net_name, fail_assume_enoent) + LOG.debug(msg, net_name, profile, fail_assume_enoent) + else: + msg = "Deletion of lxd network '%s' %s" + _lxc(["network", "delete", net_name]) + LOG.debug(msg, net_name, succeeded) if attach: msg = "Removal of device '%s' from profile '%s' %s" -- cgit v1.2.3 From 6d332e5c8dbfb6521a530b1fa49d73da51efff96 Mon Sep 17 00:00:00 2001 From: Emmanuel Thomé Date: Tue, 15 Sep 2020 21:51:52 +0200 Subject: create a shutdown_command method in distro classes (#567) Under FreeBSD, we want to use "shutdown -p" for poweroff. Alpine Linux also has some specificities. We choose to define a method that returns the shutdown command line to use, rather than a method that actually does the shutdown. This makes it easier to have the tests in test_handler_power_state do their verifications. Two tests are added for the special behaviours that are known so far. --- cloudinit/config/cc_power_state_change.py | 56 +++------------------ cloudinit/distros/__init__.py | 19 +++++++ cloudinit/distros/alpine.py | 26 ++++++++++ cloudinit/distros/bsd.py | 4 ++ .../test_handler/test_handler_power_state.py | 58 ++++++++++++++++------ tools/.github-cla-signers | 1 + 6 files changed, 102 insertions(+), 62 deletions(-) (limited to 'cloudinit/config') diff --git a/cloudinit/config/cc_power_state_change.py b/cloudinit/config/cc_power_state_change.py index 6fcb8a7d..b0cfafcd 100644 --- a/cloudinit/config/cc_power_state_change.py +++ b/cloudinit/config/cc_power_state_change.py @@ -117,7 +117,7 @@ def check_condition(cond, log=None): def handle(_name, cfg, cloud, log, _args): try: - (args, timeout, condition) = load_power_state(cfg, cloud.distro.name) + (args, timeout, condition) = load_power_state(cfg, cloud.distro) if args is None: log.debug("no power_state provided. doing nothing") return @@ -144,19 +144,7 @@ def handle(_name, cfg, cloud, log, _args): condition, execmd, [args, devnull_fp]) -def convert_delay(delay, fmt=None, scale=None): - if not fmt: - fmt = "+%s" - if not scale: - scale = 1 - - if delay != "now": - delay = fmt % int(int(delay) * int(scale)) - - return delay - - -def load_power_state(cfg, distro_name): +def load_power_state(cfg, distro): # returns a tuple of shutdown_command, timeout # shutdown_command is None if no config found pstate = cfg.get('power_state') @@ -167,44 +155,16 @@ def load_power_state(cfg, distro_name): if not isinstance(pstate, dict): raise TypeError("power_state is not a dict.") - opt_map = {'halt': '-H', 'poweroff': '-P', 'reboot': '-r'} - + modes_ok = ['halt', 'poweroff', 'reboot'] mode = pstate.get("mode") - if mode not in opt_map: + if mode not in distro.shutdown_options_map: raise TypeError( "power_state[mode] required, must be one of: %s. found: '%s'." % - (','.join(opt_map.keys()), mode)) - - delay = pstate.get("delay", "now") - message = pstate.get("message") - scale = 1 - fmt = "+%s" - command = ["shutdown", opt_map[mode]] - - if distro_name == 'alpine': - # Convert integer 30 or string '30' to '1800' (seconds) as Alpine's - # halt/poweroff/reboot commands take seconds rather than minutes. - scale = 60 - # No "+" in front of delay value as not supported by Alpine's commands. - fmt = "%s" - if delay == "now": - # Alpine's commands do not understand "now". - delay = "0" - command = [mode, "-d"] - # Alpine's commands don't support a message. - message = None - - try: - delay = convert_delay(delay, fmt=fmt, scale=scale) - except ValueError as e: - raise TypeError( - "power_state[delay] must be 'now' or '+m' (minutes)." - " found '%s'." % delay - ) from e + (','.join(modes_ok), mode)) - args = command + [delay] - if message: - args.append(message) + args = distro.shutdown_command(mode=mode, + delay=pstate.get("delay", "now"), + message=pstate.get("message")) try: timeout = float(pstate.get('timeout', 30.0)) diff --git a/cloudinit/distros/__init__.py b/cloudinit/distros/__init__.py index 2bd9bae8..fac8cf67 100755 --- a/cloudinit/distros/__init__.py +++ b/cloudinit/distros/__init__.py @@ -73,6 +73,9 @@ class Distro(metaclass=abc.ABCMeta): renderer_configs = {} _preferred_ntp_clients = None networking_cls = LinuxNetworking + # This is used by self.shutdown_command(), and can be overridden in + # subclasses + shutdown_options_map = {'halt': '-H', 'poweroff': '-P', 'reboot': '-r'} def __init__(self, name, cfg, paths): self._paths = paths @@ -750,6 +753,22 @@ class Distro(metaclass=abc.ABCMeta): subp.subp(['usermod', '-a', '-G', name, member]) LOG.info("Added user '%s' to group '%s'", member, name) + def shutdown_command(self, *, mode, delay, message): + # called from cc_power_state_change.load_power_state + command = ["shutdown", self.shutdown_options_map[mode]] + try: + if delay != "now": + delay = "+%d" % int(delay) + except ValueError as e: + raise TypeError( + "power_state[delay] must be 'now' or '+m' (minutes)." + " found '%s'." % (delay,) + ) from e + args = command + [delay] + if message: + args.append(message) + return args + def _apply_hostname_transformations_to_url(url: str, transformations: list): """ diff --git a/cloudinit/distros/alpine.py b/cloudinit/distros/alpine.py index e42443fc..e92ff3fb 100644 --- a/cloudinit/distros/alpine.py +++ b/cloudinit/distros/alpine.py @@ -162,4 +162,30 @@ class Distro(distros.Distro): return self._preferred_ntp_clients + def shutdown_command(self, mode='poweroff', delay='now', message=None): + # called from cc_power_state_change.load_power_state + # Alpine has halt/poweroff/reboot, with the following specifics: + # - we use them rather than the generic "shutdown" + # - delay is given with "-d [integer]" + # - the integer is in seconds, cannot be "now", and takes no "+" + # - no message is supported (argument ignored, here) + + command = [mode, "-d"] + + # Convert delay from minutes to seconds, as Alpine's + # halt/poweroff/reboot commands take seconds rather than minutes. + if delay == "now": + # Alpine's commands do not understand "now". + command += ['0'] + else: + try: + command.append(str(int(delay) * 60)) + except ValueError as e: + raise TypeError( + "power_state[delay] must be 'now' or '+m' (minutes)." + " found '%s'." % (delay,) + ) from e + + return command + # vi: ts=4 expandtab diff --git a/cloudinit/distros/bsd.py b/cloudinit/distros/bsd.py index 2ed7a7d5..f717a667 100644 --- a/cloudinit/distros/bsd.py +++ b/cloudinit/distros/bsd.py @@ -17,6 +17,10 @@ class BSD(distros.Distro): hostname_conf_fn = '/etc/rc.conf' rc_conf_fn = "/etc/rc.conf" + # This differs from the parent Distro class, which has -P for + # poweroff. + shutdown_options_map = {'halt': '-H', 'poweroff': '-p', 'reboot': '-r'} + # Set in BSD distro subclasses group_add_cmd_prefix = [] pkg_cmd_install_prefix = [] diff --git a/tests/unittests/test_handler/test_handler_power_state.py b/tests/unittests/test_handler/test_handler_power_state.py index 93b24fdc..4ac49424 100644 --- a/tests/unittests/test_handler/test_handler_power_state.py +++ b/tests/unittests/test_handler/test_handler_power_state.py @@ -4,72 +4,102 @@ import sys from cloudinit.config import cc_power_state_change as psc +from cloudinit import distros +from cloudinit import helpers + from cloudinit.tests import helpers as t_help from cloudinit.tests.helpers import mock class TestLoadPowerState(t_help.TestCase): + def setUp(self): + super(TestLoadPowerState, self).setUp() + cls = distros.fetch('ubuntu') + paths = helpers.Paths({}) + self.dist = cls('ubuntu', {}, paths) + def test_no_config(self): # completely empty config should mean do nothing - (cmd, _timeout, _condition) = psc.load_power_state({}, 'ubuntu') + (cmd, _timeout, _condition) = psc.load_power_state({}, self.dist) self.assertIsNone(cmd) def test_irrelevant_config(self): # no power_state field in config should return None for cmd (cmd, _timeout, _condition) = psc.load_power_state({'foo': 'bar'}, - 'ubuntu') + self.dist) self.assertIsNone(cmd) def test_invalid_mode(self): + cfg = {'power_state': {'mode': 'gibberish'}} - self.assertRaises(TypeError, psc.load_power_state, cfg, 'ubuntu') + self.assertRaises(TypeError, psc.load_power_state, cfg, self.dist) cfg = {'power_state': {'mode': ''}} - self.assertRaises(TypeError, psc.load_power_state, cfg, 'ubuntu') + self.assertRaises(TypeError, psc.load_power_state, cfg, self.dist) def test_empty_mode(self): cfg = {'power_state': {'message': 'goodbye'}} - self.assertRaises(TypeError, psc.load_power_state, cfg, 'ubuntu') + self.assertRaises(TypeError, psc.load_power_state, cfg, self.dist) def test_valid_modes(self): cfg = {'power_state': {}} for mode in ('halt', 'poweroff', 'reboot'): cfg['power_state']['mode'] = mode - check_lps_ret(psc.load_power_state(cfg, 'ubuntu'), mode=mode) + check_lps_ret(psc.load_power_state(cfg, self.dist), mode=mode) def test_invalid_delay(self): cfg = {'power_state': {'mode': 'poweroff', 'delay': 'goodbye'}} - self.assertRaises(TypeError, psc.load_power_state, cfg, 'ubuntu') + self.assertRaises(TypeError, psc.load_power_state, cfg, self.dist) def test_valid_delay(self): cfg = {'power_state': {'mode': 'poweroff', 'delay': ''}} for delay in ("now", "+1", "+30"): cfg['power_state']['delay'] = delay - check_lps_ret(psc.load_power_state(cfg, 'ubuntu')) + check_lps_ret(psc.load_power_state(cfg, self.dist)) def test_message_present(self): cfg = {'power_state': {'mode': 'poweroff', 'message': 'GOODBYE'}} - ret = psc.load_power_state(cfg, 'ubuntu') - check_lps_ret(psc.load_power_state(cfg, 'ubuntu')) + ret = psc.load_power_state(cfg, self.dist) + check_lps_ret(psc.load_power_state(cfg, self.dist)) self.assertIn(cfg['power_state']['message'], ret[0]) def test_no_message(self): # if message is not present, then no argument should be passed for it cfg = {'power_state': {'mode': 'poweroff'}} - (cmd, _timeout, _condition) = psc.load_power_state(cfg, 'ubuntu') + (cmd, _timeout, _condition) = psc.load_power_state(cfg, self.dist) self.assertNotIn("", cmd) - check_lps_ret(psc.load_power_state(cfg, 'ubuntu')) + check_lps_ret(psc.load_power_state(cfg, self.dist)) self.assertTrue(len(cmd) == 3) def test_condition_null_raises(self): cfg = {'power_state': {'mode': 'poweroff', 'condition': None}} - self.assertRaises(TypeError, psc.load_power_state, cfg, 'ubuntu') + self.assertRaises(TypeError, psc.load_power_state, cfg, self.dist) def test_condition_default_is_true(self): cfg = {'power_state': {'mode': 'poweroff'}} - _cmd, _timeout, cond = psc.load_power_state(cfg, 'ubuntu') + _cmd, _timeout, cond = psc.load_power_state(cfg, self.dist) self.assertEqual(cond, True) + def test_freebsd_poweroff_uses_lowercase_p(self): + cls = distros.fetch('freebsd') + paths = helpers.Paths({}) + freebsd = cls('freebsd', {}, paths) + cfg = {'power_state': {'mode': 'poweroff'}} + ret = psc.load_power_state(cfg, freebsd) + self.assertIn('-p', ret[0]) + + def test_alpine_delay(self): + # alpine takes delay in seconds. + cls = distros.fetch('alpine') + paths = helpers.Paths({}) + alpine = cls('alpine', {}, paths) + cfg = {'power_state': {'mode': 'poweroff', 'delay': ''}} + for delay, value in (('now', 0), ("+1", 60), ("+30", 1800)): + cfg['power_state']['delay'] = delay + ret = psc.load_power_state(cfg, alpine) + self.assertEqual('-d', ret[0][1]) + self.assertEqual(str(value), ret[0][2]) + class TestCheckCondition(t_help.TestCase): def cmd_with_exit(self, rc): diff --git a/tools/.github-cla-signers b/tools/.github-cla-signers index aa946511..f01e9b66 100644 --- a/tools/.github-cla-signers +++ b/tools/.github-cla-signers @@ -6,6 +6,7 @@ candlerb dermotbradley dhensby eandersson +emmanuelthome izzyleung johnsonshi jqueuniet -- cgit v1.2.3 From 09a0dfb1759a4f747415c8cd6b66c63cc4374e5d Mon Sep 17 00:00:00 2001 From: Wacław Schiller Date: Mon, 21 Sep 2020 18:01:35 +0200 Subject: Fix typo in disk_setup module's description (#579) --- cloudinit/config/cc_disk_setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'cloudinit/config') diff --git a/cloudinit/config/cc_disk_setup.py b/cloudinit/config/cc_disk_setup.py index a7bdc703..d1200694 100644 --- a/cloudinit/config/cc_disk_setup.py +++ b/cloudinit/config/cc_disk_setup.py @@ -35,7 +35,7 @@ either a size or a list containing a size and the numerical value for a partition type. The size for partitions is specified in **percentage** of disk space, not in bytes (e.g. a size of 33 would take up 1/3 of the disk space). The ``overwrite`` option controls whether this module tries to be safe about -writing partition talbes or not. If ``overwrite: false`` is set, the device +writing partition tables or not. If ``overwrite: false`` is set, the device will be checked for a partition table and for a file system and if either is found, the operation will be skipped. If ``overwrite: true`` is set, no checks will be performed. -- cgit v1.2.3 From d2e1b315f1fbd325a62434c7d46bc8ea41417333 Mon Sep 17 00:00:00 2001 From: Daniel Watkins Date: Tue, 22 Sep 2020 09:00:48 -0400 Subject: cc_users_groups: minor doc formatting fix (#577) Co-authored-by: Rick Harding --- cloudinit/config/cc_users_groups.py | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) (limited to 'cloudinit/config') diff --git a/cloudinit/config/cc_users_groups.py b/cloudinit/config/cc_users_groups.py index 426498a3..d4e923ef 100644 --- a/cloudinit/config/cc_users_groups.py +++ b/cloudinit/config/cc_users_groups.py @@ -80,10 +80,9 @@ config keys for an entry in ``users`` are as follows: .. note:: Most of these configuration options will not be honored if the user - already exists. Following options are the exceptions and they are - applicable on already-existing users: - - 'plain_text_passwd', 'hashed_passwd', 'lock_passwd', 'sudo', - 'ssh_authorized_keys', 'ssh_redirect_user'. + already exists. The following options are the exceptions; they are applied + to already-existing users: ``plain_text_passwd``, ``hashed_passwd``, + ``lock_passwd``, ``sudo``, ``ssh_authorized_keys``, ``ssh_redirect_user``. **Internal name:** ``cc_users_groups`` -- cgit v1.2.3 From 9d9f4f323fdf978b91c4df86534b533e582da923 Mon Sep 17 00:00:00 2001 From: Wacław Schiller Date: Tue, 22 Sep 2020 16:52:05 +0200 Subject: Fix typo in resolv_conf module's description (#578) --- cloudinit/config/cc_resolv_conf.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'cloudinit/config') diff --git a/cloudinit/config/cc_resolv_conf.py b/cloudinit/config/cc_resolv_conf.py index 519e66eb..7beb11ca 100644 --- a/cloudinit/config/cc_resolv_conf.py +++ b/cloudinit/config/cc_resolv_conf.py @@ -14,7 +14,7 @@ Resolv Conf This module is intended to manage resolv.conf in environments where early configuration of resolv.conf is necessary for further bootstrapping and/or where configuration management such as puppet or chef own dns configuration. -As Debian/Ubuntu will, by default, utilize resovlconf, and similarly RedHat +As Debian/Ubuntu will, by default, utilize resolvconf, and similarly RedHat will use sysconfig, this module is likely to be of little use unless those are configured correctly. -- cgit v1.2.3 From 33c6d5cda8773b383bdec881c4e67f0d6c12ebd6 Mon Sep 17 00:00:00 2001 From: Scott Moser Date: Thu, 1 Oct 2020 12:34:50 -0400 Subject: Fix name of ntp and chrony service on CentOS and RHEL. (#589) The service installed by the CentOS and RHEL 'ntp' package is ntpd.service not ntp.service Fix that for those two distros. Also fix chrony service from 'chrony' to 'chronyd'. LP: #1897915 --- cloudinit/config/cc_ntp.py | 8 ++++++++ 1 file changed, 8 insertions(+) (limited to 'cloudinit/config') diff --git a/cloudinit/config/cc_ntp.py b/cloudinit/config/cc_ntp.py index 3d7279d6..e183993f 100644 --- a/cloudinit/config/cc_ntp.py +++ b/cloudinit/config/cc_ntp.py @@ -80,6 +80,14 @@ DISTRO_CLIENT_CONFIG = { 'confpath': '/etc/chrony/chrony.conf', }, }, + 'rhel': { + 'ntp': { + 'service_name': 'ntpd', + }, + 'chrony': { + 'service_name': 'chronyd', + }, + }, 'opensuse': { 'chrony': { 'service_name': 'chronyd', -- cgit v1.2.3 From 5f8a2bbc5f26c7abafbc9bd3d1b1b655ffdcc1ae Mon Sep 17 00:00:00 2001 From: Daniel Watkins Date: Tue, 20 Oct 2020 17:13:37 -0400 Subject: cc_mounts: correctly fallback to dd if fallocate fails (#585) `create_swap()` was previously catching and not re-raising the ProcessExecutionError that indicated swap creation failure; this meant that the fallback logic could never be triggered. This commit adds the required re-raise (as well as removing a duplicated log message). LP: #1897099 --- cloudinit/config/cc_mounts.py | 8 ++++---- cloudinit/config/tests/test_mounts.py | 33 +++++++++++++++++++++++++++++++++ 2 files changed, 37 insertions(+), 4 deletions(-) (limited to 'cloudinit/config') diff --git a/cloudinit/config/cc_mounts.py b/cloudinit/config/cc_mounts.py index 54f2f878..c22d1698 100644 --- a/cloudinit/config/cc_mounts.py +++ b/cloudinit/config/cc_mounts.py @@ -255,8 +255,9 @@ def create_swapfile(fname: str, size: str) -> None: try: subp.subp(cmd, capture=True) except subp.ProcessExecutionError as e: - LOG.warning(errmsg, fname, size, method, e) + LOG.info(errmsg, fname, size, method, e) util.del_file(fname) + raise swap_dir = os.path.dirname(fname) util.ensure_dir(swap_dir) @@ -269,9 +270,8 @@ def create_swapfile(fname: str, size: str) -> None: else: try: create_swap(fname, size, "fallocate") - except subp.ProcessExecutionError as e: - LOG.warning(errmsg, fname, size, "dd", e) - LOG.warning("Will attempt with dd.") + except subp.ProcessExecutionError: + LOG.info("fallocate swap creation failed, will attempt with dd") create_swap(fname, size, "dd") if os.path.exists(fname): diff --git a/cloudinit/config/tests/test_mounts.py b/cloudinit/config/tests/test_mounts.py index 764a33e3..56510fd6 100644 --- a/cloudinit/config/tests/test_mounts.py +++ b/cloudinit/config/tests/test_mounts.py @@ -4,6 +4,7 @@ from unittest import mock import pytest from cloudinit.config.cc_mounts import create_swapfile +from cloudinit.subp import ProcessExecutionError M_PATH = 'cloudinit.config.cc_mounts.' @@ -26,3 +27,35 @@ class TestCreateSwapfile: create_swapfile(fname, '') assert mock.call(['mkswap', fname]) in m_subp.call_args_list + + @mock.patch(M_PATH + "util.get_mount_info") + @mock.patch(M_PATH + "subp.subp") + def test_fallback_from_fallocate_to_dd( + self, m_subp, m_get_mount_info, caplog, tmpdir + ): + swap_file = tmpdir.join("swap-file") + fname = str(swap_file) + + def subp_side_effect(cmd, *args, **kwargs): + # Mock fallocate failing, to initiate fallback + if cmd[0] == "fallocate": + raise ProcessExecutionError() + + m_subp.side_effect = subp_side_effect + # Use ext4 so both fallocate and dd are valid swap creation methods + m_get_mount_info.return_value = (mock.ANY, "ext4") + + create_swapfile(fname, "") + + cmds = [args[0][0] for args, _kwargs in m_subp.call_args_list] + assert "fallocate" in cmds, "fallocate was not called" + assert "dd" in cmds, "fallocate failure did not fallback to dd" + + assert cmds.index("dd") > cmds.index( + "fallocate" + ), "dd ran before fallocate" + + assert mock.call(["mkswap", fname]) in m_subp.call_args_list + + msg = "fallocate swap creation failed, will attempt with dd" + assert msg in caplog.text -- cgit v1.2.3 From f99d4f96b00a9cfec1c721d364cbfd728674e5dc Mon Sep 17 00:00:00 2001 From: Aman306 <45781773+Aman306@users.noreply.github.com> Date: Wed, 28 Oct 2020 23:36:09 +0530 Subject: Add config modules for controlling IBM PowerVM RMC. (#584) Reliable Scalable Cluster Technology (RSCT) is a set of software components that together provide a comprehensive clustering environment(RAS features) for IBM PowerVM based virtual machines. RSCT includes the Resource Monitoring and Control (RMC) subsystem. RMC is a generalized framework used for managing, monitoring, and manipulating resources. RMC runs as a daemon process on individual machines and needs creation of unique node id and restarts during VM boot. LP: #1895979 Co-authored-by: Scott Moser --- cloudinit/config/cc_refresh_rmc_and_interface.py | 159 +++++++++++++++++++++ cloudinit/config/cc_reset_rmc.py | 143 ++++++++++++++++++ config/cloud.cfg.tmpl | 2 + .../test_handler_refresh_rmc_and_interface.py | 109 ++++++++++++++ tools/.github-cla-signers | 1 + 5 files changed, 414 insertions(+) create mode 100644 cloudinit/config/cc_refresh_rmc_and_interface.py create mode 100644 cloudinit/config/cc_reset_rmc.py create mode 100644 tests/unittests/test_handler/test_handler_refresh_rmc_and_interface.py (limited to 'cloudinit/config') diff --git a/cloudinit/config/cc_refresh_rmc_and_interface.py b/cloudinit/config/cc_refresh_rmc_and_interface.py new file mode 100644 index 00000000..146758ad --- /dev/null +++ b/cloudinit/config/cc_refresh_rmc_and_interface.py @@ -0,0 +1,159 @@ +# (c) Copyright IBM Corp. 2020 All Rights Reserved +# +# Author: Aman Kumar Sinha +# +# This file is part of cloud-init. See LICENSE file for license information. + +""" +Refresh IPv6 interface and RMC +------------------------------ +**Summary:** Ensure Network Manager is not managing IPv6 interface + +This module is IBM PowerVM Hypervisor specific + +Reliable Scalable Cluster Technology (RSCT) is a set of software components +that together provide a comprehensive clustering environment(RAS features) +for IBM PowerVM based virtual machines. RSCT includes the Resource +Monitoring and Control (RMC) subsystem. RMC is a generalized framework used +for managing, monitoring, and manipulating resources. RMC runs as a daemon +process on individual machines and needs creation of unique node id and +restarts during VM boot. +More details refer +https://www.ibm.com/support/knowledgecenter/en/SGVKBA_3.2/admin/bl503_ovrv.htm + +This module handles +- Refreshing RMC +- Disabling NetworkManager from handling IPv6 interface, as IPv6 interface + is used for communication between RMC daemon and PowerVM hypervisor. + +**Internal name:** ``cc_refresh_rmc_and_interface`` + +**Module frequency:** per always + +**Supported distros:** RHEL + +""" + +from cloudinit import log as logging +from cloudinit.settings import PER_ALWAYS +from cloudinit import util +from cloudinit import subp +from cloudinit import netinfo + +import errno + +frequency = PER_ALWAYS + +LOG = logging.getLogger(__name__) +# Ensure that /opt/rsct/bin has been added to standard PATH of the +# distro. The symlink to rmcctrl is /usr/sbin/rsct/bin/rmcctrl . +RMCCTRL = 'rmcctrl' + + +def handle(name, _cfg, _cloud, _log, _args): + if not subp.which(RMCCTRL): + LOG.debug("No '%s' in path, disabled", RMCCTRL) + return + + LOG.debug( + 'Making the IPv6 up explicitly. ' + 'Ensuring IPv6 interface is not being handled by NetworkManager ' + 'and it is restarted to re-establish the communication with ' + 'the hypervisor') + + ifaces = find_ipv6_ifaces() + + # Setting NM_CONTROLLED=no for IPv6 interface + # making it down and up + + if len(ifaces) == 0: + LOG.debug("Did not find any interfaces with ipv6 addresses.") + else: + for iface in ifaces: + refresh_ipv6(iface) + disable_ipv6(sysconfig_path(iface)) + restart_network_manager() + + +def find_ipv6_ifaces(): + info = netinfo.netdev_info() + ifaces = [] + for iface, data in info.items(): + if iface == "lo": + LOG.debug('Skipping localhost interface') + if len(data.get("ipv4", [])) != 0: + # skip this interface, as it has ipv4 addrs + continue + ifaces.append(iface) + return ifaces + + +def refresh_ipv6(interface): + # IPv6 interface is explicitly brought up, subsequent to which the + # RMC services are restarted to re-establish the communication with + # the hypervisor. + subp.subp(['ip', 'link', 'set', interface, 'down']) + subp.subp(['ip', 'link', 'set', interface, 'up']) + + +def sysconfig_path(iface): + return '/etc/sysconfig/network-scripts/ifcfg-' + iface + + +def restart_network_manager(): + subp.subp(['systemctl', 'restart', 'NetworkManager']) + + +def disable_ipv6(iface_file): + # Ensuring that the communication b/w the hypervisor and VM is not + # interrupted due to NetworkManager. For this purpose, as part of + # this function, the NM_CONTROLLED is explicitly set to No for IPV6 + # interface and NetworkManager is restarted. + try: + contents = util.load_file(iface_file) + except IOError as e: + if e.errno == errno.ENOENT: + LOG.debug("IPv6 interface file %s does not exist\n", + iface_file) + else: + raise e + + if 'IPV6INIT' not in contents: + LOG.debug("Interface file %s did not have IPV6INIT", iface_file) + return + + LOG.debug("Editing interface file %s ", iface_file) + + # Dropping any NM_CONTROLLED or IPV6 lines from IPv6 interface file. + lines = contents.splitlines() + lines = [line for line in lines if not search(line)] + lines.append("NM_CONTROLLED=no") + + with open(iface_file, "w") as fp: + fp.write("\n".join(lines) + "\n") + + +def search(contents): + # Search for any NM_CONTROLLED or IPV6 lines in IPv6 interface file. + return( + contents.startswith("IPV6ADDR") or + contents.startswith("IPADDR6") or + contents.startswith("IPV6INIT") or + contents.startswith("NM_CONTROLLED")) + + +def refresh_rmc(): + # To make a healthy connection between RMC daemon and hypervisor we + # refresh RMC. With refreshing RMC we are ensuring that making IPv6 + # down and up shouldn't impact communication between RMC daemon and + # hypervisor. + # -z : stop Resource Monitoring & Control subsystem and all resource + # managers, but the command does not return control to the user + # until the subsystem and all resource managers are stopped. + # -s : start Resource Monitoring & Control subsystem. + try: + subp.subp([RMCCTRL, '-z']) + subp.subp([RMCCTRL, '-s']) + except Exception: + util.logexc(LOG, 'Failed to refresh the RMC subsystem.') + raise diff --git a/cloudinit/config/cc_reset_rmc.py b/cloudinit/config/cc_reset_rmc.py new file mode 100644 index 00000000..1cd72774 --- /dev/null +++ b/cloudinit/config/cc_reset_rmc.py @@ -0,0 +1,143 @@ +# (c) Copyright IBM Corp. 2020 All Rights Reserved +# +# Author: Aman Kumar Sinha +# +# This file is part of cloud-init. See LICENSE file for license information. + + +""" +Reset RMC +------------ +**Summary:** reset rsct node id + +Reset RMC module is IBM PowerVM Hypervisor specific + +Reliable Scalable Cluster Technology (RSCT) is a set of software components, +that together provide a comprehensive clustering environment (RAS features) +for IBM PowerVM based virtual machines. RSCT includes the Resource monitoring +and control (RMC) subsystem. RMC is a generalized framework used for managing, +monitoring, and manipulating resources. RMC runs as a daemon process on +individual machines and needs creation of unique node id and restarts +during VM boot. +More details refer +https://www.ibm.com/support/knowledgecenter/en/SGVKBA_3.2/admin/bl503_ovrv.htm + +This module handles +- creation of the unique RSCT node id to every instance/virtual machine + and ensure once set, it isn't changed subsequently by cloud-init. + In order to do so, it restarts RSCT service. + +Prerequisite of using this module is to install RSCT packages. + +**Internal name:** ``cc_reset_rmc`` + +**Module frequency:** per instance + +**Supported distros:** rhel, sles and ubuntu + +""" +import os + +from cloudinit import log as logging +from cloudinit.settings import PER_INSTANCE +from cloudinit import util +from cloudinit import subp + +frequency = PER_INSTANCE + +# RMCCTRL is expected to be in system PATH (/opt/rsct/bin) +# The symlink for RMCCTRL and RECFGCT are +# /usr/sbin/rsct/bin/rmcctrl and +# /usr/sbin/rsct/install/bin/recfgct respectively. +RSCT_PATH = '/opt/rsct/install/bin' +RMCCTRL = 'rmcctrl' +RECFGCT = 'recfgct' + +LOG = logging.getLogger(__name__) + +NODE_ID_FILE = '/etc/ct_node_id' + + +def handle(name, _cfg, cloud, _log, _args): + # Ensuring node id has to be generated only once during first boot + if cloud.datasource.platform_type == 'none': + LOG.debug('Skipping creation of new ct_node_id node') + return + + if not os.path.isdir(RSCT_PATH): + LOG.debug("module disabled, RSCT_PATH not present") + return + + orig_path = os.environ.get('PATH') + try: + add_path(orig_path) + reset_rmc() + finally: + if orig_path: + os.environ['PATH'] = orig_path + else: + del os.environ['PATH'] + + +def reconfigure_rsct_subsystems(): + # Reconfigure the RSCT subsystems, which includes removing all RSCT data + # under the /var/ct directory, generating a new node ID, and making it + # appear as if the RSCT components were just installed + try: + out = subp.subp([RECFGCT])[0] + LOG.debug(out.strip()) + return out + except subp.ProcessExecutionError: + util.logexc(LOG, 'Failed to reconfigure the RSCT subsystems.') + raise + + +def get_node_id(): + try: + fp = util.load_file(NODE_ID_FILE) + node_id = fp.split('\n')[0] + return node_id + except Exception: + util.logexc(LOG, 'Failed to get node ID from file %s.' % NODE_ID_FILE) + raise + + +def add_path(orig_path): + # Adding the RSCT_PATH to env standard path + # So thet cloud init automatically find and + # run RECFGCT to create new node_id. + suff = ":" + orig_path if orig_path else "" + os.environ['PATH'] = RSCT_PATH + suff + return os.environ['PATH'] + + +def rmcctrl(): + # Stop the RMC subsystem and all resource managers so that we can make + # some changes to it + try: + return subp.subp([RMCCTRL, '-z']) + except Exception: + util.logexc(LOG, 'Failed to stop the RMC subsystem.') + raise + + +def reset_rmc(): + LOG.debug('Attempting to reset RMC.') + + node_id_before = get_node_id() + LOG.debug('Node ID at beginning of module: %s', node_id_before) + + # Stop the RMC subsystem and all resource managers so that we can make + # some changes to it + rmcctrl() + reconfigure_rsct_subsystems() + + node_id_after = get_node_id() + LOG.debug('Node ID at end of module: %s', node_id_after) + + # Check if new node ID is generated or not + # by comparing old and new node ID + if node_id_after == node_id_before: + msg = 'New node ID did not get generated.' + LOG.error(msg) + raise Exception(msg) diff --git a/config/cloud.cfg.tmpl b/config/cloud.cfg.tmpl index 2beb9b0c..7171aaa5 100644 --- a/config/cloud.cfg.tmpl +++ b/config/cloud.cfg.tmpl @@ -135,6 +135,8 @@ cloud_final_modules: - chef - mcollective - salt-minion + - reset_rmc + - refresh_rmc_and_interface - rightscale_userdata - scripts-vendor - scripts-per-once diff --git a/tests/unittests/test_handler/test_handler_refresh_rmc_and_interface.py b/tests/unittests/test_handler/test_handler_refresh_rmc_and_interface.py new file mode 100644 index 00000000..e13b7793 --- /dev/null +++ b/tests/unittests/test_handler/test_handler_refresh_rmc_and_interface.py @@ -0,0 +1,109 @@ +from cloudinit.config import cc_refresh_rmc_and_interface as ccrmci + +from cloudinit import util + +from cloudinit.tests import helpers as t_help +from cloudinit.tests.helpers import mock + +from textwrap import dedent +import logging + +LOG = logging.getLogger(__name__) +MPATH = "cloudinit.config.cc_refresh_rmc_and_interface" +NET_INFO = { + 'lo': {'ipv4': [{'ip': '127.0.0.1', + 'bcast': '', 'mask': '255.0.0.0', + 'scope': 'host'}], + 'ipv6': [{'ip': '::1/128', + 'scope6': 'host'}], 'hwaddr': '', + 'up': 'True'}, + 'env2': {'ipv4': [{'ip': '8.0.0.19', + 'bcast': '8.0.0.255', 'mask': '255.255.255.0', + 'scope': 'global'}], + 'ipv6': [{'ip': 'fe80::f896:c2ff:fe81:8220/64', + 'scope6': 'link'}], 'hwaddr': 'fa:96:c2:81:82:20', + 'up': 'True'}, + 'env3': {'ipv4': [{'ip': '90.0.0.14', + 'bcast': '90.0.0.255', 'mask': '255.255.255.0', + 'scope': 'global'}], + 'ipv6': [{'ip': 'fe80::f896:c2ff:fe81:8221/64', + 'scope6': 'link'}], 'hwaddr': 'fa:96:c2:81:82:21', + 'up': 'True'}, + 'env4': {'ipv4': [{'ip': '9.114.23.7', + 'bcast': '9.114.23.255', 'mask': '255.255.255.0', + 'scope': 'global'}], + 'ipv6': [{'ip': 'fe80::f896:c2ff:fe81:8222/64', + 'scope6': 'link'}], 'hwaddr': 'fa:96:c2:81:82:22', + 'up': 'True'}, + 'env5': {'ipv4': [], + 'ipv6': [{'ip': 'fe80::9c26:c3ff:fea4:62c8/64', + 'scope6': 'link'}], 'hwaddr': '42:20:86:df:fa:4c', + 'up': 'True'}} + + +class TestRsctNodeFile(t_help.CiTestCase): + def test_disable_ipv6_interface(self): + """test parsing of iface files.""" + fname = self.tmp_path("iface-eth5") + util.write_file(fname, dedent("""\ + BOOTPROTO=static + DEVICE=eth5 + HWADDR=42:20:86:df:fa:4c + IPV6INIT=yes + IPADDR6=fe80::9c26:c3ff:fea4:62c8/64 + IPV6ADDR=fe80::9c26:c3ff:fea4:62c8/64 + NM_CONTROLLED=yes + ONBOOT=yes + STARTMODE=auto + TYPE=Ethernet + USERCTL=no + """)) + + ccrmci.disable_ipv6(fname) + self.assertEqual(dedent("""\ + BOOTPROTO=static + DEVICE=eth5 + HWADDR=42:20:86:df:fa:4c + ONBOOT=yes + STARTMODE=auto + TYPE=Ethernet + USERCTL=no + NM_CONTROLLED=no + """), util.load_file(fname)) + + @mock.patch(MPATH + '.refresh_rmc') + @mock.patch(MPATH + '.restart_network_manager') + @mock.patch(MPATH + '.disable_ipv6') + @mock.patch(MPATH + '.refresh_ipv6') + @mock.patch(MPATH + '.netinfo.netdev_info') + @mock.patch(MPATH + '.subp.which') + def test_handle(self, m_refresh_rmc, + m_netdev_info, m_refresh_ipv6, m_disable_ipv6, + m_restart_nm, m_which): + """Basic test of handle.""" + m_netdev_info.return_value = NET_INFO + m_which.return_value = '/opt/rsct/bin/rmcctrl' + ccrmci.handle( + "refresh_rmc_and_interface", None, None, None, None) + self.assertEqual(1, m_netdev_info.call_count) + m_refresh_ipv6.assert_called_with('env5') + m_disable_ipv6.assert_called_with( + '/etc/sysconfig/network-scripts/ifcfg-env5') + self.assertEqual(1, m_restart_nm.call_count) + self.assertEqual(1, m_refresh_rmc.call_count) + + @mock.patch(MPATH + '.netinfo.netdev_info') + def test_find_ipv6(self, m_netdev_info): + """find_ipv6_ifaces parses netdev_info returning those with ipv6""" + m_netdev_info.return_value = NET_INFO + found = ccrmci.find_ipv6_ifaces() + self.assertEqual(['env5'], found) + + @mock.patch(MPATH + '.subp.subp') + def test_refresh_ipv6(self, m_subp): + """refresh_ipv6 should ip down and up the interface.""" + iface = "myeth0" + ccrmci.refresh_ipv6(iface) + m_subp.assert_has_calls([ + mock.call(['ip', 'link', 'set', iface, 'down']), + mock.call(['ip', 'link', 'set', iface, 'up'])]) diff --git a/tools/.github-cla-signers b/tools/.github-cla-signers index b7e9dc02..475f0872 100644 --- a/tools/.github-cla-signers +++ b/tools/.github-cla-signers @@ -1,5 +1,6 @@ ader1990 AlexBaranowski +Aman306 beezly bipinbachhao BirknerAlex -- cgit v1.2.3 From f8c84aeead77b7e508644d94889ee701f20e8d31 Mon Sep 17 00:00:00 2001 From: dermotbradley Date: Fri, 30 Oct 2020 20:12:38 +0000 Subject: Correct documentation and testcase data for some user-data YAML (#618) For cc_users_groups the user setting "expiredate" must be quoted in order for the relevant flag and value to be then passed to the useradd command. It its vaiue is not quoted then it is treated as Python type datetime.date and in `cloudinit/distros/__init__.py` the below "is it a string" condition fails and so no "--expiredate" parameter is passed to useradd and therefore it has no effect: ``` if key in useradd_opts and val and isinstance(val, str): useradd_cmd.extend([useradd_opts[key], val]) ``` For cc_users_groups, the user setting "inactive" does not actually disable accounts, the useradd "--inactive" option actually defines the number of days after password expiry that users can still login. So I have changed the docs to show it taking a quoted value of days (which works with the current code) rather than a boolean value. The quotes are necessary, like expiredate above, so that the value is also passed to the useradd command. For cc_power_state_change.py the "delay" setting value needs to have quotes around it as otherwise its leading plus sign will be stripped off. --- cloudinit/config/cc_power_state_change.py | 2 +- cloudinit/config/cc_users_groups.py | 9 +++++---- doc/examples/cloud-config-power-state.txt | 2 +- doc/examples/cloud-config-user-groups.txt | 7 ++++--- doc/examples/cloud-config.txt | 4 ++-- tests/cloud_tests/testcases/examples/including_user_groups.yaml | 4 ++-- tests/cloud_tests/testcases/modules/user_groups.yaml | 4 ++-- tests/data/merge_sources/expected10.yaml | 2 +- tests/data/merge_sources/expected7.yaml | 6 +++--- tests/data/merge_sources/source10-1.yaml | 2 +- tests/data/merge_sources/source7-1.yaml | 4 ++-- tests/data/merge_sources/source7-2.yaml | 2 +- 12 files changed, 25 insertions(+), 23 deletions(-) (limited to 'cloudinit/config') diff --git a/cloudinit/config/cc_power_state_change.py b/cloudinit/config/cc_power_state_change.py index b0cfafcd..5780a7e9 100644 --- a/cloudinit/config/cc_power_state_change.py +++ b/cloudinit/config/cc_power_state_change.py @@ -22,7 +22,7 @@ The ``delay`` key specifies a duration to be added onto any shutdown command used. Therefore, if a 5 minute delay and a 120 second shutdown are specified, the maximum amount of time between cloud-init starting and the system shutting down is 7 minutes, and the minimum amount of time is 5 minutes. The ``delay`` -key must have an argument in either the form ``+5`` for 5 minutes or ``now`` +key must have an argument in either the form ``'+5'`` for 5 minutes or ``now`` for immediate shutdown. Optionally, a command can be run to determine whether or not diff --git a/cloudinit/config/cc_users_groups.py b/cloudinit/config/cc_users_groups.py index d4e923ef..ac4a4410 100644 --- a/cloudinit/config/cc_users_groups.py +++ b/cloudinit/config/cc_users_groups.py @@ -26,13 +26,14 @@ entry of the ``users`` list. Each entry in the ``users`` list, other than a config keys for an entry in ``users`` are as follows: - ``name``: The user's login name - - ``expiredate``: Optional. Date on which the user's login will be + - ``expiredate``: Optional. Date on which the user's account will be disabled. Default: none - ``gecos``: Optional. Comment about the user, usually a comma-separated string of real name and contact information. Default: none - ``groups``: Optional. Additional groups to add the user to. Default: none - ``homedir``: Optional. Home dir for user. Default is ``/home/`` - - ``inactive``: Optional. Mark user inactive. Default: false + - ``inactive``: Optional. Number of days after a password expires until + the account is permanently disabled. Default: none - ``lock_passwd``: Optional. Disable password login. Default: true - ``no_create_home``: Optional. Do not create home directory. Default: false @@ -102,11 +103,11 @@ config keys for an entry in ``users`` are as follows: - name: sudo: false - name: - expiredate: + expiredate: '' gecos: groups: homedir: - inactive: + inactive: '' lock_passwd: no_create_home: no_log_init: diff --git a/doc/examples/cloud-config-power-state.txt b/doc/examples/cloud-config-power-state.txt index 9cd56814..002707ec 100644 --- a/doc/examples/cloud-config-power-state.txt +++ b/doc/examples/cloud-config-power-state.txt @@ -18,7 +18,7 @@ # when 'timeout' seconds have elapsed. # # delay: form accepted by shutdown. default is 'now'. other format -# accepted is +m (m in minutes) +# accepted is '+m' (m in minutes) # mode: required. must be one of 'poweroff', 'halt', 'reboot' # message: provided as the message argument to 'shutdown'. default is none. # timeout: the amount of time to give the cloud-init process to finish diff --git a/doc/examples/cloud-config-user-groups.txt b/doc/examples/cloud-config-user-groups.txt index b593cdd1..4a5a7e20 100644 --- a/doc/examples/cloud-config-user-groups.txt +++ b/doc/examples/cloud-config-user-groups.txt @@ -19,7 +19,7 @@ users: primary_group: foobar groups: users selinux_user: staff_u - expiredate: 2012-09-01 + expiredate: '2012-09-01' ssh_import_id: foobar lock_passwd: false passwd: $6$j212wezy$7H/1LT4f9/N3wpgNunhsIqtMj62OKiS3nyNwuizouQc3u7MbYCarYeAHWYPYb2FT.lbioDm2RrkJPb9BZMN1O/ @@ -34,7 +34,7 @@ users: - - name: cloudy gecos: Magic Cloud App Daemon User - inactive: true + inactive: '5' system: true - name: fizzbuzz sudo: False @@ -47,6 +47,7 @@ users: # Valid Values: # name: The user's login name +# expiredate: Date on which the user's account will be disabled. # gecos: The user name's real name, i.e. "Bob B. Smith" # homedir: Optional. Set to the local path you want to use. Defaults to # /home/ @@ -57,7 +58,7 @@ users: # "staff_u". When this is omitted the system will select the default # SELinux user. # lock_passwd: Defaults to true. Lock the password to disable password login -# inactive: Create the user as inactive +# inactive: Number of days after password expires until account is disabled # passwd: The hash -- not the password itself -- of the password you want # to use for this user. You can generate a safe hash via: # mkpasswd --method=SHA-512 --rounds=4096 diff --git a/doc/examples/cloud-config.txt b/doc/examples/cloud-config.txt index f3ae5e68..de9a0f87 100644 --- a/doc/examples/cloud-config.txt +++ b/doc/examples/cloud-config.txt @@ -518,10 +518,10 @@ manual_cache_clean: False # syslog being taken down while cloud-init is running. # # delay: form accepted by shutdown. default is 'now'. other format -# accepted is +m (m in minutes) +# accepted is '+m' (m in minutes) # mode: required. must be one of 'poweroff', 'halt', 'reboot' # message: provided as the message argument to 'shutdown'. default is none. power_state: - delay: 30 + delay: '+30' mode: poweroff message: Bye Bye diff --git a/tests/cloud_tests/testcases/examples/including_user_groups.yaml b/tests/cloud_tests/testcases/examples/including_user_groups.yaml index 77528d98..86e392dd 100644 --- a/tests/cloud_tests/testcases/examples/including_user_groups.yaml +++ b/tests/cloud_tests/testcases/examples/including_user_groups.yaml @@ -18,7 +18,7 @@ cloud_config: | gecos: Foo B. Bar primary_group: foobar groups: users - expiredate: 2038-01-19 + expiredate: '2038-01-19' lock_passwd: false passwd: $6$j212wezy$7H/1LT4f9/N3wpgNunhsIqtMj62OKiS3nyNwuizouQc3u7MbYCarYeAHWYPYb2FT.lbioDm2RrkJPb9BZMN1O/ - name: barfoo @@ -28,7 +28,7 @@ cloud_config: | lock_passwd: true - name: cloudy gecos: Magic Cloud App Daemon User - inactive: true + inactive: '5' system: true collect_scripts: group_ubuntu: | diff --git a/tests/cloud_tests/testcases/modules/user_groups.yaml b/tests/cloud_tests/testcases/modules/user_groups.yaml index 675dfb8c..91b0e281 100644 --- a/tests/cloud_tests/testcases/modules/user_groups.yaml +++ b/tests/cloud_tests/testcases/modules/user_groups.yaml @@ -17,7 +17,7 @@ cloud_config: | gecos: Foo B. Bar primary_group: foobar groups: users - expiredate: 2038-01-19 + expiredate: '2038-01-19' lock_passwd: false passwd: $6$j212wezy$7H/1LT4f9/N3wpgNunhsIqtMj62OKiS3nyNwuizouQc3u7MbYCarYeAHWYPYb2FT.lbioDm2RrkJPb9BZMN1O/ - name: barfoo @@ -27,7 +27,7 @@ cloud_config: | lock_passwd: true - name: cloudy gecos: Magic Cloud App Daemon User - inactive: true + inactive: '5' system: true collect_scripts: group_ubuntu: | diff --git a/tests/data/merge_sources/expected10.yaml b/tests/data/merge_sources/expected10.yaml index b865db16..e9f88f7b 100644 --- a/tests/data/merge_sources/expected10.yaml +++ b/tests/data/merge_sources/expected10.yaml @@ -1,7 +1,7 @@ #cloud-config power_state: - delay: 30 + delay: '+30' mode: poweroff message: [Bye, Bye, Pew, Pew] diff --git a/tests/data/merge_sources/expected7.yaml b/tests/data/merge_sources/expected7.yaml index d32988e8..8186d13a 100644 --- a/tests/data/merge_sources/expected7.yaml +++ b/tests/data/merge_sources/expected7.yaml @@ -7,7 +7,7 @@ users: primary_group: foobar groups: users selinux_user: staff_u - expiredate: 2012-09-01 + expiredate: '2012-09-01' ssh_import_id: foobar lock-passwd: false passwd: $6$j212wezy$7H/1LT4f9/N3wpgNunhsIqtMj62OKiS3nyNwuizouQc3u7MbYCarYeAHWYPYb2FT.lbioDm2RrkJPb9BZMN1O/ @@ -22,7 +22,7 @@ users: - - name: cloudy gecos: Magic Cloud App Daemon User - inactive: true + inactive: '5' system: true - bob - joe @@ -32,7 +32,7 @@ users: primary_group: foobar groups: users selinux_user: staff_u - expiredate: 2012-09-01 + expiredate: '2012-09-01' ssh_import_id: foobar lock-passwd: false passwd: $6$j212wezy$7H/1LT4f9/N3wpgNunhsIqtMj62OKiS3nyNwuizouQc3u7MbYCarYeAHWYPYb2FT.lbioDm2RrkJPb9BZMN1O/ diff --git a/tests/data/merge_sources/source10-1.yaml b/tests/data/merge_sources/source10-1.yaml index 6ae72a13..36fd336d 100644 --- a/tests/data/merge_sources/source10-1.yaml +++ b/tests/data/merge_sources/source10-1.yaml @@ -1,6 +1,6 @@ #cloud-config power_state: - delay: 30 + delay: '+30' mode: poweroff message: [Bye, Bye] diff --git a/tests/data/merge_sources/source7-1.yaml b/tests/data/merge_sources/source7-1.yaml index 6405fc9b..ec93079f 100644 --- a/tests/data/merge_sources/source7-1.yaml +++ b/tests/data/merge_sources/source7-1.yaml @@ -7,7 +7,7 @@ users: primary_group: foobar groups: users selinux_user: staff_u - expiredate: 2012-09-01 + expiredate: '2012-09-01' ssh_import_id: foobar lock-passwd: false passwd: $6$j212wezy$7H/1LT4f9/N3wpgNunhsIqtMj62OKiS3nyNwuizouQc3u7MbYCarYeAHWYPYb2FT.lbioDm2RrkJPb9BZMN1O/ @@ -22,6 +22,6 @@ users: - - name: cloudy gecos: Magic Cloud App Daemon User - inactive: true + inactive: '5' system: true diff --git a/tests/data/merge_sources/source7-2.yaml b/tests/data/merge_sources/source7-2.yaml index 0cd28978..0c02abff 100644 --- a/tests/data/merge_sources/source7-2.yaml +++ b/tests/data/merge_sources/source7-2.yaml @@ -9,7 +9,7 @@ users: primary_group: foobar groups: users selinux_user: staff_u - expiredate: 2012-09-01 + expiredate: '2012-09-01' ssh_import_id: foobar lock-passwd: false passwd: $6$j212wezy$7H/1LT4f9/N3wpgNunhsIqtMj62OKiS3nyNwuizouQc3u7MbYCarYeAHWYPYb2FT.lbioDm2RrkJPb9BZMN1O/ -- cgit v1.2.3 From 2c4500394b2e3b3074468fffc4d91b404304bc39 Mon Sep 17 00:00:00 2001 From: Mina Galić Date: Mon, 2 Nov 2020 22:13:02 +0100 Subject: remove unnecessary reboot from gpart resize (#646) a reboot after `gpart resize` hasn't been necessary since ca FreeBSD 9.2 Co-authored-by: Rick Harding --- cloudinit/config/cc_growpart.py | 4 ---- 1 file changed, 4 deletions(-) (limited to 'cloudinit/config') diff --git a/cloudinit/config/cc_growpart.py b/cloudinit/config/cc_growpart.py index 237c3d02..621df0ac 100644 --- a/cloudinit/config/cc_growpart.py +++ b/cloudinit/config/cc_growpart.py @@ -196,10 +196,6 @@ class ResizeGpart(object): util.logexc(LOG, "Failed: gpart resize -i %s %s", partnum, diskdev) raise ResizeFailedException(e) from e - # Since growing the FS requires a reboot, make sure we reboot - # first when this module has finished. - open('/var/run/reboot-required', 'a').close() - return (before, get_size(partdev)) -- cgit v1.2.3 From 8dfd8801ab1329efe066876c037f71a73dcf3de1 Mon Sep 17 00:00:00 2001 From: Shane Frasier Date: Tue, 3 Nov 2020 12:12:42 -0500 Subject: Make some language improvements in growpart documentation (#649) * Fix awkward English in sentence * Add the missing word "the" * Fix misspelling * Add @jsf9k as a contributor Co-authored-by: Rick Harding --- cloudinit/config/cc_growpart.py | 11 ++++++----- tools/.github-cla-signers | 1 + 2 files changed, 7 insertions(+), 5 deletions(-) (limited to 'cloudinit/config') diff --git a/cloudinit/config/cc_growpart.py b/cloudinit/config/cc_growpart.py index 621df0ac..9f338ad1 100644 --- a/cloudinit/config/cc_growpart.py +++ b/cloudinit/config/cc_growpart.py @@ -16,12 +16,13 @@ This is useful for cloud instances with a larger amount of disk space available than the pristine image uses, as it allows the instance to automatically make use of the extra space. -The devices run growpart on are specified as a list under the ``devices`` key. -Each entry in the devices list can be either the path to the device's -mountpoint in the filesystem or a path to the block device in ``/dev``. +The devices on which to run growpart are specified as a list under the +``devices`` key. Each entry in the devices list can be either the path to the +device's mountpoint in the filesystem or a path to the block device in +``/dev``. The utility to use for resizing can be selected using the ``mode`` config key. -If ``mode`` key is set to ``auto``, then any available utility (either +If the ``mode`` key is set to ``auto``, then any available utility (either ``growpart`` or BSD ``gpart``) will be used. If neither utility is available, no error will be raised. If ``mode`` is set to ``growpart``, then the ``growpart`` utility will be used. If this utility is not available on the @@ -34,7 +35,7 @@ where one tool is able to function and the other is not. The default configuration for both should work for most cloud instances. To explicitly prevent ``cloud-initramfs-tools`` from running ``growroot``, the file ``/etc/growroot-disabled`` can be created. By default, both ``growroot`` and -``cc_growpart`` will check for the existance of this file and will not run if +``cc_growpart`` will check for the existence of this file and will not run if it is present. However, this file can be ignored for ``cc_growpart`` by setting ``ignore_growroot_disabled`` to ``true``. For more information on ``cloud-initramfs-tools`` see: https://launchpad.net/cloud-initramfs-tools diff --git a/tools/.github-cla-signers b/tools/.github-cla-signers index fc05f982..2090dc2a 100644 --- a/tools/.github-cla-signers +++ b/tools/.github-cla-signers @@ -12,6 +12,7 @@ emmanuelthome izzyleung johnsonshi jqueuniet +jsf9k landon912 lucasmoura manuelisimo -- cgit v1.2.3 From f680114446a5a20ce88f3d10d966811a774c8e8f Mon Sep 17 00:00:00 2001 From: Chad Smith Date: Wed, 18 Nov 2020 07:23:44 -0700 Subject: cli: add --system param to allow validating system user-data on a machine (#575) Allow root user to validate the userdata provided to the launched machine using `cloud-init devel schema --system` --- cloudinit/config/schema.py | 41 ++++++++--- doc/rtd/topics/faq.rst | 6 +- tests/unittests/test_cli.py | 2 +- tests/unittests/test_handler/test_schema.py | 109 ++++++++++++++++++++-------- 4 files changed, 114 insertions(+), 44 deletions(-) (limited to 'cloudinit/config') diff --git a/cloudinit/config/schema.py b/cloudinit/config/schema.py index 8a966aee..456bab2c 100644 --- a/cloudinit/config/schema.py +++ b/cloudinit/config/schema.py @@ -1,6 +1,7 @@ # This file is part of cloud-init. See LICENSE file for license information. """schema.py: Set of module functions for processing cloud-config schema.""" +from cloudinit.cmd.devel import read_cfg_paths from cloudinit import importer from cloudinit.util import find_modules, load_file @@ -173,7 +174,8 @@ def annotated_cloudconfig_file(cloudconfig, original_content, schema_errors): def validate_cloudconfig_file(config_path, schema, annotate=False): """Validate cloudconfig file adheres to a specific jsonschema. - @param config_path: Path to the yaml cloud-config file to parse. + @param config_path: Path to the yaml cloud-config file to parse, or None + to default to system userdata from Paths object. @param schema: Dict describing a valid jsonschema to validate against. @param annotate: Boolean set True to print original config file with error annotations on the offending lines. @@ -181,9 +183,24 @@ def validate_cloudconfig_file(config_path, schema, annotate=False): @raises SchemaValidationError containing any of schema_errors encountered. @raises RuntimeError when config_path does not exist. """ - if not os.path.exists(config_path): - raise RuntimeError('Configfile {0} does not exist'.format(config_path)) - content = load_file(config_path, decode=False) + if config_path is None: + # Use system's raw userdata path + if os.getuid() != 0: + raise RuntimeError( + "Unable to read system userdata as non-root user." + " Try using sudo" + ) + paths = read_cfg_paths() + user_data_file = paths.get_ipath_cur("userdata_raw") + content = load_file(user_data_file, decode=False) + else: + if not os.path.exists(config_path): + raise RuntimeError( + 'Configfile {0} does not exist'.format( + config_path + ) + ) + content = load_file(config_path, decode=False) if not content.startswith(CLOUD_CONFIG_HEADER): errors = ( ('format-l1.c1', 'File {0} needs to begin with "{1}"'.format( @@ -425,6 +442,8 @@ def get_parser(parser=None): description='Validate cloud-config files or document schema') parser.add_argument('-c', '--config-file', help='Path of the cloud-config yaml file to validate') + parser.add_argument('--system', action='store_true', default=False, + help='Validate the system cloud-config userdata') parser.add_argument('-d', '--docs', nargs='+', help=('Print schema module docs. Choices: all or' ' space-delimited cc_names.')) @@ -435,11 +454,11 @@ def get_parser(parser=None): def handle_schema_args(name, args): """Handle provided schema args and perform the appropriate actions.""" - exclusive_args = [args.config_file, args.docs] - if not any(exclusive_args) or all(exclusive_args): - error('Expected either --config-file argument or --docs') + exclusive_args = [args.config_file, args.docs, args.system] + if len([arg for arg in exclusive_args if arg]) != 1: + error('Expected one of --config-file, --system or --docs arguments') full_schema = get_schema() - if args.config_file: + if args.config_file or args.system: try: validate_cloudconfig_file( args.config_file, full_schema, args.annotate) @@ -449,7 +468,11 @@ def handle_schema_args(name, args): except RuntimeError as e: error(str(e)) else: - print("Valid cloud-config file {0}".format(args.config_file)) + if args.config_file is None: + cfg_name = "system userdata" + else: + cfg_name = args.config_file + print("Valid cloud-config:", cfg_name) elif args.docs: schema_ids = [subschema['id'] for subschema in full_schema['allOf']] schema_ids += ['all'] diff --git a/doc/rtd/topics/faq.rst b/doc/rtd/topics/faq.rst index d08914b5..27fabf15 100644 --- a/doc/rtd/topics/faq.rst +++ b/doc/rtd/topics/faq.rst @@ -141,12 +141,12 @@ that can validate your user data offline. .. _validate-yaml.py: https://github.com/canonical/cloud-init/blob/master/tools/validate-yaml.py -Another option is to run the following on an instance when debugging: +Another option is to run the following on an instance to debug userdata +provided to the system: .. code-block:: shell-session - $ sudo cloud-init query userdata > user-data.yaml - $ cloud-init devel schema -c user-data.yaml --annotate + $ cloud-init devel schema --system --annotate As launching instances in the cloud can cost money and take a bit longer, sometimes it is easier to launch instances locally using Multipass or LXD: diff --git a/tests/unittests/test_cli.py b/tests/unittests/test_cli.py index dcf0fe5a..74f85959 100644 --- a/tests/unittests/test_cli.py +++ b/tests/unittests/test_cli.py @@ -214,7 +214,7 @@ class TestCLI(test_helpers.FilesystemMockingTestCase): self.assertEqual(1, exit_code) # Known whitebox output from schema subcommand self.assertEqual( - 'Expected either --config-file argument or --docs\n', + 'Expected one of --config-file, --system or --docs arguments\n', self.stderr.getvalue()) def test_wb_devel_schema_subcommand_doc_content(self): diff --git a/tests/unittests/test_handler/test_schema.py b/tests/unittests/test_handler/test_schema.py index 44292571..15aa77bb 100644 --- a/tests/unittests/test_handler/test_schema.py +++ b/tests/unittests/test_handler/test_schema.py @@ -9,9 +9,9 @@ from cloudinit.util import write_file from cloudinit.tests.helpers import CiTestCase, mock, skipUnlessJsonSchema from copy import copy +import itertools import os import pytest -from io import StringIO from pathlib import Path from textwrap import dedent from yaml import safe_load @@ -400,50 +400,97 @@ class AnnotatedCloudconfigFileTest(CiTestCase): annotated_cloudconfig_file(parsed_config, content, schema_errors)) -class MainTest(CiTestCase): +class TestMain: - def test_main_missing_args(self): + exclusive_combinations = itertools.combinations( + ["--system", "--docs all", "--config-file something"], 2 + ) + + @pytest.mark.parametrize("params", exclusive_combinations) + def test_main_exclusive_args(self, params, capsys): + """Main exits non-zero and error on required exclusive args.""" + params = list(itertools.chain(*[a.split() for a in params])) + with mock.patch('sys.argv', ['mycmd'] + params): + with pytest.raises(SystemExit) as context_manager: + main() + assert 1 == context_manager.value.code + + _out, err = capsys.readouterr() + expected = ( + 'Expected one of --config-file, --system or --docs arguments\n' + ) + assert expected == err + + def test_main_missing_args(self, capsys): """Main exits non-zero and reports an error on missing parameters.""" with mock.patch('sys.argv', ['mycmd']): - with mock.patch('sys.stderr', new_callable=StringIO) as m_stderr: - with self.assertRaises(SystemExit) as context_manager: - main() - self.assertEqual(1, context_manager.exception.code) - self.assertEqual( - 'Expected either --config-file argument or --docs\n', - m_stderr.getvalue()) + with pytest.raises(SystemExit) as context_manager: + main() + assert 1 == context_manager.value.code + + _out, err = capsys.readouterr() + expected = ( + 'Expected one of --config-file, --system or --docs arguments\n' + ) + assert expected == err - def test_main_absent_config_file(self): + def test_main_absent_config_file(self, capsys): """Main exits non-zero when config file is absent.""" myargs = ['mycmd', '--annotate', '--config-file', 'NOT_A_FILE'] with mock.patch('sys.argv', myargs): - with mock.patch('sys.stderr', new_callable=StringIO) as m_stderr: - with self.assertRaises(SystemExit) as context_manager: - main() - self.assertEqual(1, context_manager.exception.code) - self.assertEqual( - 'Configfile NOT_A_FILE does not exist\n', - m_stderr.getvalue()) + with pytest.raises(SystemExit) as context_manager: + main() + assert 1 == context_manager.value.code + _out, err = capsys.readouterr() + assert 'Configfile NOT_A_FILE does not exist\n' == err - def test_main_prints_docs(self): + def test_main_prints_docs(self, capsys): """When --docs parameter is provided, main generates documentation.""" myargs = ['mycmd', '--docs', 'all'] with mock.patch('sys.argv', myargs): - with mock.patch('sys.stdout', new_callable=StringIO) as m_stdout: - self.assertEqual(0, main(), 'Expected 0 exit code') - self.assertIn('\nNTP\n---\n', m_stdout.getvalue()) - self.assertIn('\nRuncmd\n------\n', m_stdout.getvalue()) + assert 0 == main(), 'Expected 0 exit code' + out, _err = capsys.readouterr() + assert '\nNTP\n---\n' in out + assert '\nRuncmd\n------\n' in out - def test_main_validates_config_file(self): + def test_main_validates_config_file(self, tmpdir, capsys): """When --config-file parameter is provided, main validates schema.""" - myyaml = self.tmp_path('my.yaml') - myargs = ['mycmd', '--config-file', myyaml] - write_file(myyaml, b'#cloud-config\nntp:') # shortest ntp schema + myyaml = tmpdir.join('my.yaml') + myargs = ['mycmd', '--config-file', myyaml.strpath] + myyaml.write(b'#cloud-config\nntp:') # shortest ntp schema with mock.patch('sys.argv', myargs): - with mock.patch('sys.stdout', new_callable=StringIO) as m_stdout: - self.assertEqual(0, main(), 'Expected 0 exit code') - self.assertIn( - 'Valid cloud-config file {0}'.format(myyaml), m_stdout.getvalue()) + assert 0 == main(), 'Expected 0 exit code' + out, _err = capsys.readouterr() + assert 'Valid cloud-config: {0}\n'.format(myyaml) == out + + @mock.patch('cloudinit.config.schema.read_cfg_paths') + @mock.patch('cloudinit.config.schema.os.getuid', return_value=0) + def test_main_validates_system_userdata( + self, m_getuid, m_read_cfg_paths, capsys, paths + ): + """When --system is provided, main validates system userdata.""" + m_read_cfg_paths.return_value = paths + ud_file = paths.get_ipath_cur("userdata_raw") + write_file(ud_file, b'#cloud-config\nntp:') + myargs = ['mycmd', '--system'] + with mock.patch('sys.argv', myargs): + assert 0 == main(), 'Expected 0 exit code' + out, _err = capsys.readouterr() + assert 'Valid cloud-config: system userdata\n' == out + + @mock.patch('cloudinit.config.schema.os.getuid', return_value=1000) + def test_main_system_userdata_requires_root(self, m_getuid, capsys, paths): + """Non-root user can't use --system param""" + myargs = ['mycmd', '--system'] + with mock.patch('sys.argv', myargs): + with pytest.raises(SystemExit) as context_manager: + main() + assert 1 == context_manager.value.code + _out, err = capsys.readouterr() + expected = ( + 'Unable to read system userdata as non-root user. Try using sudo\n' + ) + assert expected == err class CloudTestsIntegrationTest(CiTestCase): -- cgit v1.2.3 From 12ef7541c2d0c6b2cd510f95fda53ca9c8333064 Mon Sep 17 00:00:00 2001 From: Mina Galić Date: Thu, 19 Nov 2020 23:19:16 +0100 Subject: cc_resizefs on FreeBSD: Fix _can_skip_ufs_resize (#655) On FreeBSD, if a UFS has trim: (-t) or MAC multilabel: (-l) flag, resize FS fail, because the _can_skip_ufs_resize check gets tripped up by the missing options. This was reported at FreeBSD Bugzilla: https://bugs.freebsd.org/bugzilla/show_bug.cgi?id=250496 and as LP: #1901958 Rather than fixing the parser as in the patches proposed there (and attempted in #636) this pull-request rips out all of it, and simplifies the code. We now use `growfs -N` and check if that returns an error. If it returns the correct kind of error, we can skip the resize, because we either are at the correct size, or the filesystem in question is broken or not UFS. If it returns the wrong kind of error, we just re-raise it. LP: #1901958 --- cloudinit/config/cc_resizefs.py | 68 +++++----------------- .../test_handler/test_handler_resizefs.py | 55 +++++++++-------- 2 files changed, 42 insertions(+), 81 deletions(-) (limited to 'cloudinit/config') diff --git a/cloudinit/config/cc_resizefs.py b/cloudinit/config/cc_resizefs.py index 978d2ee0..9afbb847 100644 --- a/cloudinit/config/cc_resizefs.py +++ b/cloudinit/config/cc_resizefs.py @@ -9,10 +9,7 @@ """Resizefs: cloud-config module which resizes the filesystem""" import errno -import getopt import os -import re -import shlex import stat from textwrap import dedent @@ -88,56 +85,23 @@ def _resize_zfs(mount_point, devpth): return ('zpool', 'online', '-e', mount_point, devpth) -def _get_dumpfs_output(mount_point): - return subp.subp(['dumpfs', '-m', mount_point])[0] - - -def _get_gpart_output(part): - return subp.subp(['gpart', 'show', part])[0] - - def _can_skip_resize_ufs(mount_point, devpth): - # extract the current fs sector size - """ - # dumpfs -m / - # newfs command for / (/dev/label/rootfs) - newfs -L rootf -O 2 -U -a 4 -b 32768 -d 32768 -e 4096 -f 4096 -g 16384 - -h 64 -i 8192 -j -k 6408 -m 8 -o time -s 58719232 /dev/label/rootf - """ - cur_fs_sz = None - frag_sz = None - dumpfs_res = _get_dumpfs_output(mount_point) - for line in dumpfs_res.splitlines(): - if not line.startswith('#'): - newfs_cmd = shlex.split(line) - opt_value = 'O:Ua:s:b:d:e:f:g:h:i:jk:m:o:L:' - optlist, _args = getopt.getopt(newfs_cmd[1:], opt_value) - for o, a in optlist: - if o == "-s": - cur_fs_sz = int(a) - if o == "-f": - frag_sz = int(a) - # check the current partition size - # Example output from `gpart show /dev/da0`: - # => 40 62914480 da0 GPT (30G) - # 40 1024 1 freebsd-boot (512K) - # 1064 58719232 2 freebsd-ufs (28G) - # 58720296 3145728 3 freebsd-swap (1.5G) - # 61866024 1048496 - free - (512M) - expect_sz = None - m = re.search('^(/dev/.+)p([0-9])$', devpth) - gpart_res = _get_gpart_output(m.group(1)) - for line in gpart_res.splitlines(): - if re.search(r"freebsd-ufs", line): - fields = line.split() - expect_sz = int(fields[1]) - # Normalize the gpart sector size, - # because the size is not exactly the same as fs size. - normal_expect_sz = (expect_sz - expect_sz % (frag_sz / 512)) - if normal_expect_sz == cur_fs_sz: - return True - else: - return False + # possible errors cases on the code-path to growfs -N following: + # https://github.com/freebsd/freebsd/blob/HEAD/sbin/growfs/growfs.c + # This is the "good" error: + skip_start = "growfs: requested size" + skip_contain = "is not larger than the current filesystem size" + # growfs exits with 1 for almost all cases up to this one. + # This means we can't just use rcs=[0, 1] as subp parameter: + try: + subp.subp(['growfs', '-N', devpth]) + except subp.ProcessExecutionError as e: + if e.stderr.startswith(skip_start) and skip_contain in e.stderr: + # This FS is already at the desired size + return True + else: + raise e + return False # Do not use a dictionary as these commands should be able to be used diff --git a/tests/unittests/test_handler/test_handler_resizefs.py b/tests/unittests/test_handler/test_handler_resizefs.py index db9a0414..28d55072 100644 --- a/tests/unittests/test_handler/test_handler_resizefs.py +++ b/tests/unittests/test_handler/test_handler_resizefs.py @@ -6,8 +6,8 @@ from cloudinit.config.cc_resizefs import ( from collections import namedtuple import logging -import textwrap +from cloudinit.subp import ProcessExecutionError from cloudinit.tests.helpers import ( CiTestCase, mock, skipUnlessJsonSchema, util, wrap_and_call) @@ -22,44 +22,41 @@ class TestResizefs(CiTestCase): super(TestResizefs, self).setUp() self.name = "resizefs" - @mock.patch('cloudinit.config.cc_resizefs._get_dumpfs_output') - @mock.patch('cloudinit.config.cc_resizefs._get_gpart_output') - def test_skip_ufs_resize(self, gpart_out, dumpfs_out): + @mock.patch('cloudinit.subp.subp') + def test_skip_ufs_resize(self, m_subp): fs_type = "ufs" resize_what = "/" devpth = "/dev/da0p2" - dumpfs_out.return_value = ( - "# newfs command for / (/dev/label/rootfs)\n" - "newfs -O 2 -U -a 4 -b 32768 -d 32768 -e 4096 " - "-f 4096 -g 16384 -h 64 -i 8192 -j -k 6408 -m 8 " - "-o time -s 58719232 /dev/label/rootfs\n") - gpart_out.return_value = textwrap.dedent("""\ - => 40 62914480 da0 GPT (30G) - 40 1024 1 freebsd-boot (512K) - 1064 58719232 2 freebsd-ufs (28G) - 58720296 3145728 3 freebsd-swap (1.5G) - 61866024 1048496 - free - (512M) - """) + err = ("growfs: requested size 2.0GB is not larger than the " + "current filesystem size 2.0GB\n") + exception = ProcessExecutionError(stderr=err, exit_code=1) + m_subp.side_effect = exception res = can_skip_resize(fs_type, resize_what, devpth) self.assertTrue(res) - @mock.patch('cloudinit.config.cc_resizefs._get_dumpfs_output') - @mock.patch('cloudinit.config.cc_resizefs._get_gpart_output') - def test_skip_ufs_resize_roundup(self, gpart_out, dumpfs_out): + @mock.patch('cloudinit.subp.subp') + def test_cannot_skip_ufs_resize(self, m_subp): fs_type = "ufs" resize_what = "/" devpth = "/dev/da0p2" - dumpfs_out.return_value = ( - "# newfs command for / (/dev/label/rootfs)\n" - "newfs -O 2 -U -a 4 -b 32768 -d 32768 -e 4096 " - "-f 4096 -g 16384 -h 64 -i 8192 -j -k 368 -m 8 " - "-o time -s 297080 /dev/label/rootfs\n") - gpart_out.return_value = textwrap.dedent("""\ - => 34 297086 da0 GPT (145M) - 34 297086 1 freebsd-ufs (145M) - """) + m_subp.return_value = ( + ("stdout: super-block backups (for fsck_ffs -b #) at:\n\n"), + ("growfs: no room to allocate last cylinder group; " + "leaving 364KB unused\n") + ) res = can_skip_resize(fs_type, resize_what, devpth) - self.assertTrue(res) + self.assertFalse(res) + + @mock.patch('cloudinit.subp.subp') + def test_cannot_skip_ufs_growfs_exception(self, m_subp): + fs_type = "ufs" + resize_what = "/" + devpth = "/dev/da0p2" + err = "growfs: /dev/da0p2 is not clean - run fsck.\n" + exception = ProcessExecutionError(stderr=err, exit_code=1) + m_subp.side_effect = exception + with self.assertRaises(ProcessExecutionError): + can_skip_resize(fs_type, resize_what, devpth) def test_can_skip_resize_ext(self): self.assertFalse(can_skip_resize('ext', '/', '/dev/sda1')) -- cgit v1.2.3 From 66b4be8b6da188a0667bd8c86a25155b6f4f3f6c Mon Sep 17 00:00:00 2001 From: Jonathan Lung Date: Fri, 20 Nov 2020 15:59:51 -0500 Subject: Support configuring SSH host certificates. (#660) Existing config writes keys to /etc/ssh after deleting files matching a glob that includes certificate files. Since sshd looks for certificates in the same directory as the keys, a host certificate must be placed in this directory. This update enables the certificate's contents to be specified along with the keys. Co-authored-by: jonathan lung Co-authored-by: jonathan lung --- cloudinit/config/cc_ssh.py | 31 +++++++--- cloudinit/config/tests/test_ssh.py | 68 ++++++++++++++++++++-- .../modules/test_ssh_keys_provided.py | 13 +++++ tools/.github-cla-signers | 1 + 4 files changed, 101 insertions(+), 12 deletions(-) (limited to 'cloudinit/config') diff --git a/cloudinit/config/cc_ssh.py b/cloudinit/config/cc_ssh.py index 9b2a333a..05a16dbc 100755 --- a/cloudinit/config/cc_ssh.py +++ b/cloudinit/config/cc_ssh.py @@ -83,8 +83,9 @@ enabled by default. Host keys can be added using the ``ssh_keys`` configuration key. The argument to this config key should be a dictionary entries for the public and private keys of each desired key type. Entries in the ``ssh_keys`` config dict should -have keys in the format ``_private`` and ``_public``, -e.g. ``rsa_private: `` and ``rsa_public: ``. See below for supported +have keys in the format ``_private``, ``_public``, and, +optionally, ``_certificate``, e.g. ``rsa_private: ``, +``rsa_public: ``, and ``rsa_certificate: ``. See below for supported key types. Not all key types have to be specified, ones left unspecified will not be used. If this config option is used, then no keys will be generated. @@ -94,7 +95,8 @@ not be used. If this config option is used, then no keys will be generated. secure .. note:: - to specify multiline private host keys, use yaml multiline syntax + to specify multiline private host keys and certificates, use yaml + multiline syntax If no host keys are specified using ``ssh_keys``, then keys will be generated using ``ssh-keygen``. By default one public/private pair of each supported @@ -128,12 +130,17 @@ config flags are: ... -----END RSA PRIVATE KEY----- rsa_public: ssh-rsa AAAAB3NzaC1yc2EAAAABIwAAAGEAoPRhIfLvedSDKw7Xd ... + rsa_certificate: | + ssh-rsa-cert-v01@openssh.com AAAAIHNzaC1lZDI1NTE5LWNlcnQt ... dsa_private: | -----BEGIN DSA PRIVATE KEY----- MIIBxwIBAAJhAKD0YSHy73nUgysO13XsJmd4fHiFyQ+00R7VVu2iV9Qco ... -----END DSA PRIVATE KEY----- dsa_public: ssh-dsa AAAAB3NzaC1yc2EAAAABIwAAAGEAoPRhIfLvedSDKw7Xd ... + dsa_certificate: | + ssh-dsa-cert-v01@openssh.com AAAAIHNzaC1lZDI1NTE5LWNlcnQt ... + ssh_genkeytypes: disable_root: disable_root_opts: @@ -169,6 +176,8 @@ for k in GENERATE_KEY_NAMES: CONFIG_KEY_TO_FILE.update({"%s_private" % k: (KEY_FILE_TPL % k, 0o600)}) CONFIG_KEY_TO_FILE.update( {"%s_public" % k: (KEY_FILE_TPL % k + ".pub", 0o600)}) + CONFIG_KEY_TO_FILE.update( + {"%s_certificate" % k: (KEY_FILE_TPL % k + "-cert.pub", 0o600)}) PRIV_TO_PUB["%s_private" % k] = "%s_public" % k KEY_GEN_TPL = 'o=$(ssh-keygen -yf "%s") && echo "$o" root@localhost > "%s"' @@ -186,12 +195,18 @@ def handle(_name, cfg, cloud, log, _args): util.logexc(log, "Failed deleting key file %s", f) if "ssh_keys" in cfg: - # if there are keys in cloud-config, use them + # if there are keys and/or certificates in cloud-config, use them for (key, val) in cfg["ssh_keys"].items(): - if key in CONFIG_KEY_TO_FILE: - tgt_fn = CONFIG_KEY_TO_FILE[key][0] - tgt_perms = CONFIG_KEY_TO_FILE[key][1] - util.write_file(tgt_fn, val, tgt_perms) + # skip entry if unrecognized + if key not in CONFIG_KEY_TO_FILE: + continue + tgt_fn = CONFIG_KEY_TO_FILE[key][0] + tgt_perms = CONFIG_KEY_TO_FILE[key][1] + util.write_file(tgt_fn, val, tgt_perms) + # set server to present the most recently identified certificate + if '_certificate' in key: + cert_config = {'HostCertificate': tgt_fn} + ssh_util.update_ssh_config(cert_config) for (priv, pub) in PRIV_TO_PUB.items(): if pub in cfg['ssh_keys'] or priv not in cfg['ssh_keys']: diff --git a/cloudinit/config/tests/test_ssh.py b/cloudinit/config/tests/test_ssh.py index 0c554414..87ccdb60 100644 --- a/cloudinit/config/tests/test_ssh.py +++ b/cloudinit/config/tests/test_ssh.py @@ -10,6 +10,8 @@ import logging LOG = logging.getLogger(__name__) MODPATH = "cloudinit.config.cc_ssh." +KEY_NAMES_NO_DSA = [name for name in cc_ssh.GENERATE_KEY_NAMES + if name not in 'dsa'] @mock.patch(MODPATH + "ssh_util.setup_user_keys") @@ -25,7 +27,7 @@ class TestHandleSsh(CiTestCase): } self.test_hostkey_files = [] hostkey_tmpdir = self.tmp_dir() - for key_type in ['dsa', 'ecdsa', 'ed25519', 'rsa']: + for key_type in cc_ssh.GENERATE_KEY_NAMES: key_data = self.test_hostkeys[key_type] filename = 'ssh_host_%s_key.pub' % key_type filepath = os.path.join(hostkey_tmpdir, filename) @@ -223,7 +225,7 @@ class TestHandleSsh(CiTestCase): cfg = {} expected_call = [self.test_hostkeys[key_type] for key_type - in ['ecdsa', 'ed25519', 'rsa']] + in KEY_NAMES_NO_DSA] cc_ssh.handle("name", cfg, cloud, LOG, None) self.assertEqual([mock.call(expected_call)], cloud.datasource.publish_host_keys.call_args_list) @@ -252,7 +254,7 @@ class TestHandleSsh(CiTestCase): cfg = {'ssh_publish_hostkeys': {'enabled': True}} expected_call = [self.test_hostkeys[key_type] for key_type - in ['ecdsa', 'ed25519', 'rsa']] + in KEY_NAMES_NO_DSA] cc_ssh.handle("name", cfg, cloud, LOG, None) self.assertEqual([mock.call(expected_call)], cloud.datasource.publish_host_keys.call_args_list) @@ -339,7 +341,65 @@ class TestHandleSsh(CiTestCase): cfg = {'ssh_publish_hostkeys': {'enabled': True, 'blacklist': []}} expected_call = [self.test_hostkeys[key_type] for key_type - in ['dsa', 'ecdsa', 'ed25519', 'rsa']] + in cc_ssh.GENERATE_KEY_NAMES] cc_ssh.handle("name", cfg, cloud, LOG, None) self.assertEqual([mock.call(expected_call)], cloud.datasource.publish_host_keys.call_args_list) + + @mock.patch(MODPATH + "ug_util.normalize_users_groups") + @mock.patch(MODPATH + "util.write_file") + def test_handle_ssh_keys_in_cfg(self, m_write_file, m_nug, m_setup_keys): + """Test handle with ssh keys and certificate.""" + # Populate a config dictionary to pass to handle() as well + # as the expected file-writing calls. + cfg = {"ssh_keys": {}} + + expected_calls = [] + for key_type in cc_ssh.GENERATE_KEY_NAMES: + private_name = "{}_private".format(key_type) + public_name = "{}_public".format(key_type) + cert_name = "{}_certificate".format(key_type) + + # Actual key contents don"t have to be realistic + private_value = "{}_PRIVATE_KEY".format(key_type) + public_value = "{}_PUBLIC_KEY".format(key_type) + cert_value = "{}_CERT_KEY".format(key_type) + + cfg["ssh_keys"][private_name] = private_value + cfg["ssh_keys"][public_name] = public_value + cfg["ssh_keys"][cert_name] = cert_value + + expected_calls.extend([ + mock.call( + '/etc/ssh/ssh_host_{}_key'.format(key_type), + private_value, + 384 + ), + mock.call( + '/etc/ssh/ssh_host_{}_key.pub'.format(key_type), + public_value, + 384 + ), + mock.call( + '/etc/ssh/ssh_host_{}_key-cert.pub'.format(key_type), + cert_value, + 384 + ), + mock.call( + '/etc/ssh/sshd_config', + ('HostCertificate /etc/ssh/ssh_host_{}_key-cert.pub' + '\n'.format(key_type)), + preserve_mode=True + ) + ]) + + # Run the handler. + m_nug.return_value = ([], {}) + with mock.patch(MODPATH + 'ssh_util.parse_ssh_config', + return_value=[]): + cc_ssh.handle("name", cfg, self.tmp_cloud(distro='ubuntu'), + LOG, None) + + # Check that all expected output has been done. + for call_ in expected_calls: + self.assertIn(call_, m_write_file.call_args_list) diff --git a/tests/integration_tests/modules/test_ssh_keys_provided.py b/tests/integration_tests/modules/test_ssh_keys_provided.py index dc6d2fc1..27d193c1 100644 --- a/tests/integration_tests/modules/test_ssh_keys_provided.py +++ b/tests/integration_tests/modules/test_ssh_keys_provided.py @@ -45,6 +45,7 @@ ssh_keys: A3tFPEOxauXpzCt8f8eXsz0WQXAgIKW2h8zu5QHjomioU3i27mtE -----END RSA PRIVATE KEY----- rsa_public: ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQC0/Ho+o3eJISydO2JvIgTLnZOtrxPl+fSvJfKDjoOLY0HB2eOjy2s2/2N6d9X9SGZ4+XnyVeNPjfBXw4IyXoqxhfIF16Azfk022iejgjiYssoUxH31M60OfqJhxo16dWEXdkKP1nac06VOt1zS5yEeooyvEuMJEJSsv3VR/7GKhMX3TVhEz5moLmVP3bIAvvoXio8X4urVC1R819QjDC86nlxwNks/GKPRi/IHO5tjJ72Eke7KNsm/vxHgkdX4vZaHNKhfdb/pavFXN5eoUaofz3hxw5oL/u2epI/pXyUhDp8Tb5wO6slykzcIlGCSd0YeO1TnljvViRx0uSxIy97N root@xenial-lxd + rsa_certificate: ssh-rsa-cert-v01@openssh.com AAAAHHNzaC1yc2EtY2VydC12MDFAb3BlbnNzaC5jb20AAAAgMpgBP4Phn3L8I7Vqh7lmHKcOfIokEvSEbHDw83Y3JloAAAADAQABAAABAQC0/Ho+o3eJISydO2JvIgTLnZOtrxPl+fSvJfKDjoOLY0HB2eOjy2s2/2N6d9X9SGZ4+XnyVeNPjfBXw4IyXoqxhfIF16Azfk022iejgjiYssoUxH31M60OfqJhxo16dWEXdkKP1nac06VOt1zS5yEeooyvEuMJEJSsv3VR/7GKhMX3TVhEz5moLmVP3bIAvvoXio8X4urVC1R819QjDC86nlxwNks/GKPRi/IHO5tjJ72Eke7KNsm/vxHgkdX4vZaHNKhfdb/pavFXN5eoUaofz3hxw5oL/u2epI/pXyUhDp8Tb5wO6slykzcIlGCSd0YeO1TnljvViRx0uSxIy97NAAAAAAAAAAAAAAACAAAACnhlbmlhbC1seGQAAAAAAAAAAF+vVEIAAAAAYY83bgAAAAAAAAAAAAAAAAAAADMAAAALc3NoLWVkMjU1MTkAAAAgz4SlDwbq53ZrRsnS6ISdwxgFDRpnEX44K8jFmLpI9NAAAABTAAAAC3NzaC1lZDI1NTE5AAAAQMWpiRWKNMFvRX0g6OQOELMqDhtNBpkIN92IyO25qiY2oDSd1NyVme6XnGDFt8CS7z5NufV04doP4aacLOBbQww= root@xenial-lxd dsa_private: | -----BEGIN DSA PRIVATE KEY----- MIIBuwIBAAKBgQD5Fstc23IVSDe6k4DNP8smPKuEWUvHDTOGaXrhOVAfzZ6+jklP @@ -108,6 +109,18 @@ class TestSshKeysProvided: "4DOkqNiUGl80Zp1RgZNohHUXlJMtAbrIlAVEk+mTmg7vjfyp2un" "RQvLZpMRdywBm") in out + def test_ssh_rsa_certificate_provided(self, class_client): + """Test rsa certificate was imported.""" + out = class_client.read_from_file("/etc/ssh/ssh_host_rsa_key-cert.pub") + assert ( + "AAAAHHNzaC1yc2EtY2VydC12MDFAb3BlbnNzaC5jb20AAAAgMpg" + "BP4Phn3L8I7Vqh7lmHKcOfIokEvSEbHDw83Y3JloAAAAD") in out + + def test_ssh_certificate_updated_sshd_config(self, class_client): + """Test ssh certificate was added to /etc/ssh/sshd_config.""" + out = class_client.read_from_file("/etc/ssh/sshd_config").strip() + assert "HostCertificate /etc/ssh/ssh_host_rsa_key-cert.pub" in out + def test_ssh_ecdsa_keys_provided(self, class_client): """Test ecdsa public key was imported.""" out = class_client.read_from_file("/etc/ssh/ssh_host_ecdsa_key.pub") diff --git a/tools/.github-cla-signers b/tools/.github-cla-signers index b2464768..9b594a44 100644 --- a/tools/.github-cla-signers +++ b/tools/.github-cla-signers @@ -15,6 +15,7 @@ jqueuniet jsf9k landon912 lucasmoura +lungj manuelisimo marlluslustosa matthewruffell -- cgit v1.2.3 From c6bcb8df28daa234686a563549681082eb3283a1 Mon Sep 17 00:00:00 2001 From: zsdc Date: Fri, 25 Dec 2020 18:36:39 +0200 Subject: T2117: Updates from 20.4 copied to resizefs_vyos This commit does not change any actually used in VyOS logic. It only copies changes from the upstream 20.4 to stay closer to the upstream code. --- cloudinit/config/cc_resizefs_vyos.py | 68 +++++----------------- .../test_handler/test_handler_resizefs_vyos.py | 55 +++++++++-------- 2 files changed, 42 insertions(+), 81 deletions(-) (limited to 'cloudinit/config') diff --git a/cloudinit/config/cc_resizefs_vyos.py b/cloudinit/config/cc_resizefs_vyos.py index f5555afc..f8eb84fe 100644 --- a/cloudinit/config/cc_resizefs_vyos.py +++ b/cloudinit/config/cc_resizefs_vyos.py @@ -9,10 +9,7 @@ """Resizefs: cloud-config module which resizes the filesystem""" import errno -import getopt import os -import re -import shlex import stat from textwrap import dedent @@ -101,56 +98,23 @@ def _resize_zfs(mount_point, devpth): return ('zpool', 'online', '-e', mount_point, devpth) -def _get_dumpfs_output(mount_point): - return subp.subp(['dumpfs', '-m', mount_point])[0] - - -def _get_gpart_output(part): - return subp.subp(['gpart', 'show', part])[0] - - def _can_skip_resize_ufs(mount_point, devpth): - # extract the current fs sector size - """ - # dumpfs -m / - # newfs command for / (/dev/label/rootfs) - newfs -L rootf -O 2 -U -a 4 -b 32768 -d 32768 -e 4096 -f 4096 -g 16384 - -h 64 -i 8192 -j -k 6408 -m 8 -o time -s 58719232 /dev/label/rootf - """ - cur_fs_sz = None - frag_sz = None - dumpfs_res = _get_dumpfs_output(mount_point) - for line in dumpfs_res.splitlines(): - if not line.startswith('#'): - newfs_cmd = shlex.split(line) - opt_value = 'O:Ua:s:b:d:e:f:g:h:i:jk:m:o:L:' - optlist, _args = getopt.getopt(newfs_cmd[1:], opt_value) - for o, a in optlist: - if o == "-s": - cur_fs_sz = int(a) - if o == "-f": - frag_sz = int(a) - # check the current partition size - # Example output from `gpart show /dev/da0`: - # => 40 62914480 da0 GPT (30G) - # 40 1024 1 freebsd-boot (512K) - # 1064 58719232 2 freebsd-ufs (28G) - # 58720296 3145728 3 freebsd-swap (1.5G) - # 61866024 1048496 - free - (512M) - expect_sz = None - m = re.search('^(/dev/.+)p([0-9])$', devpth) - gpart_res = _get_gpart_output(m.group(1)) - for line in gpart_res.splitlines(): - if re.search(r"freebsd-ufs", line): - fields = line.split() - expect_sz = int(fields[1]) - # Normalize the gpart sector size, - # because the size is not exactly the same as fs size. - normal_expect_sz = (expect_sz - expect_sz % (frag_sz / 512)) - if normal_expect_sz == cur_fs_sz: - return True - else: - return False + # possible errors cases on the code-path to growfs -N following: + # https://github.com/freebsd/freebsd/blob/HEAD/sbin/growfs/growfs.c + # This is the "good" error: + skip_start = "growfs: requested size" + skip_contain = "is not larger than the current filesystem size" + # growfs exits with 1 for almost all cases up to this one. + # This means we can't just use rcs=[0, 1] as subp parameter: + try: + subp.subp(['growfs', '-N', devpth]) + except subp.ProcessExecutionError as e: + if e.stderr.startswith(skip_start) and skip_contain in e.stderr: + # This FS is already at the desired size + return True + else: + raise e + return False # Do not use a dictionary as these commands should be able to be used diff --git a/tests/unittests/test_handler/test_handler_resizefs_vyos.py b/tests/unittests/test_handler/test_handler_resizefs_vyos.py index d1fafff7..c18ab1ea 100644 --- a/tests/unittests/test_handler/test_handler_resizefs_vyos.py +++ b/tests/unittests/test_handler/test_handler_resizefs_vyos.py @@ -6,8 +6,8 @@ from cloudinit.config.cc_resizefs_vyos import ( from collections import namedtuple import logging -import textwrap +from cloudinit.subp import ProcessExecutionError from cloudinit.tests.helpers import ( CiTestCase, mock, skipUnlessJsonSchema, util, wrap_and_call) @@ -22,44 +22,41 @@ class TestResizefs(CiTestCase): super(TestResizefs, self).setUp() self.name = "resizefs" - @mock.patch('cloudinit.config.cc_resizefs_vyos._get_dumpfs_output') - @mock.patch('cloudinit.config.cc_resizefs_vyos._get_gpart_output') - def test_skip_ufs_resize(self, gpart_out, dumpfs_out): + @mock.patch('cloudinit.subp.subp') + def test_skip_ufs_resize(self, m_subp): fs_type = "ufs" resize_what = "/" devpth = "/dev/da0p2" - dumpfs_out.return_value = ( - "# newfs command for / (/dev/label/rootfs)\n" - "newfs -O 2 -U -a 4 -b 32768 -d 32768 -e 4096 " - "-f 4096 -g 16384 -h 64 -i 8192 -j -k 6408 -m 8 " - "-o time -s 58719232 /dev/label/rootfs\n") - gpart_out.return_value = textwrap.dedent("""\ - => 40 62914480 da0 GPT (30G) - 40 1024 1 freebsd-boot (512K) - 1064 58719232 2 freebsd-ufs (28G) - 58720296 3145728 3 freebsd-swap (1.5G) - 61866024 1048496 - free - (512M) - """) + err = ("growfs: requested size 2.0GB is not larger than the " + "current filesystem size 2.0GB\n") + exception = ProcessExecutionError(stderr=err, exit_code=1) + m_subp.side_effect = exception res = can_skip_resize(fs_type, resize_what, devpth) self.assertTrue(res) - @mock.patch('cloudinit.config.cc_resizefs_vyos._get_dumpfs_output') - @mock.patch('cloudinit.config.cc_resizefs_vyos._get_gpart_output') - def test_skip_ufs_resize_roundup(self, gpart_out, dumpfs_out): + @mock.patch('cloudinit.subp.subp') + def test_cannot_skip_ufs_resize(self, m_subp): fs_type = "ufs" resize_what = "/" devpth = "/dev/da0p2" - dumpfs_out.return_value = ( - "# newfs command for / (/dev/label/rootfs)\n" - "newfs -O 2 -U -a 4 -b 32768 -d 32768 -e 4096 " - "-f 4096 -g 16384 -h 64 -i 8192 -j -k 368 -m 8 " - "-o time -s 297080 /dev/label/rootfs\n") - gpart_out.return_value = textwrap.dedent("""\ - => 34 297086 da0 GPT (145M) - 34 297086 1 freebsd-ufs (145M) - """) + m_subp.return_value = ( + ("stdout: super-block backups (for fsck_ffs -b #) at:\n\n"), + ("growfs: no room to allocate last cylinder group; " + "leaving 364KB unused\n") + ) res = can_skip_resize(fs_type, resize_what, devpth) - self.assertTrue(res) + self.assertFalse(res) + + @mock.patch('cloudinit.subp.subp') + def test_cannot_skip_ufs_growfs_exception(self, m_subp): + fs_type = "ufs" + resize_what = "/" + devpth = "/dev/da0p2" + err = "growfs: /dev/da0p2 is not clean - run fsck.\n" + exception = ProcessExecutionError(stderr=err, exit_code=1) + m_subp.side_effect = exception + with self.assertRaises(ProcessExecutionError): + can_skip_resize(fs_type, resize_what, devpth) def test_can_skip_resize_ext(self): self.assertFalse(can_skip_resize('ext', '/', '/dev/sda1')) -- cgit v1.2.3