From 1f3a225af78dbfbff75c3faad28a5dc8cad0d1e3 Mon Sep 17 00:00:00 2001 From: Paride Legovini Date: Thu, 27 Aug 2020 17:20:35 +0200 Subject: LXD: detach network from profile before deleting it (#542) * LXD: detach network from profile before deleting it When cleaning up the bridge network created by default by LXD as part of the `lxd init` process detach the network its profile before deleting it. LXD will otherwise refuse to delete it with error: Error: The network is currently in use. Discussion with LXD upstream: https://github.com/lxc/lxd/issues/7804. LP: #1776958 * LXD bridge deletion: fail if bridge exists but can't be deleted * LXD bridge deletion: remove useless failure logging --- tests/unittests/test_handler/test_handler_lxd.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'tests/unittests/test_handler') diff --git a/tests/unittests/test_handler/test_handler_lxd.py b/tests/unittests/test_handler/test_handler_lxd.py index 21011204..b2181992 100644 --- a/tests/unittests/test_handler/test_handler_lxd.py +++ b/tests/unittests/test_handler/test_handler_lxd.py @@ -214,7 +214,7 @@ class TestLxdMaybeCleanupDefault(t_help.CiTestCase): """deletion of network should occur if create is True.""" cc_lxd.maybe_cleanup_default( net_name=self.defnet, did_init=True, create=True, attach=False) - m_lxc.assert_called_once_with(["network", "delete", self.defnet]) + m_lxc.assert_called_with(["network", "delete", self.defnet]) @mock.patch("cloudinit.config.cc_lxd._lxc") def test_device_removed_if_attach_true(self, m_lxc): -- cgit v1.2.3 From 6d332e5c8dbfb6521a530b1fa49d73da51efff96 Mon Sep 17 00:00:00 2001 From: Emmanuel Thomé Date: Tue, 15 Sep 2020 21:51:52 +0200 Subject: create a shutdown_command method in distro classes (#567) Under FreeBSD, we want to use "shutdown -p" for poweroff. Alpine Linux also has some specificities. We choose to define a method that returns the shutdown command line to use, rather than a method that actually does the shutdown. This makes it easier to have the tests in test_handler_power_state do their verifications. Two tests are added for the special behaviours that are known so far. --- cloudinit/config/cc_power_state_change.py | 56 +++------------------ cloudinit/distros/__init__.py | 19 +++++++ cloudinit/distros/alpine.py | 26 ++++++++++ cloudinit/distros/bsd.py | 4 ++ .../test_handler/test_handler_power_state.py | 58 ++++++++++++++++------ tools/.github-cla-signers | 1 + 6 files changed, 102 insertions(+), 62 deletions(-) (limited to 'tests/unittests/test_handler') diff --git a/cloudinit/config/cc_power_state_change.py b/cloudinit/config/cc_power_state_change.py index 6fcb8a7d..b0cfafcd 100644 --- a/cloudinit/config/cc_power_state_change.py +++ b/cloudinit/config/cc_power_state_change.py @@ -117,7 +117,7 @@ def check_condition(cond, log=None): def handle(_name, cfg, cloud, log, _args): try: - (args, timeout, condition) = load_power_state(cfg, cloud.distro.name) + (args, timeout, condition) = load_power_state(cfg, cloud.distro) if args is None: log.debug("no power_state provided. doing nothing") return @@ -144,19 +144,7 @@ def handle(_name, cfg, cloud, log, _args): condition, execmd, [args, devnull_fp]) -def convert_delay(delay, fmt=None, scale=None): - if not fmt: - fmt = "+%s" - if not scale: - scale = 1 - - if delay != "now": - delay = fmt % int(int(delay) * int(scale)) - - return delay - - -def load_power_state(cfg, distro_name): +def load_power_state(cfg, distro): # returns a tuple of shutdown_command, timeout # shutdown_command is None if no config found pstate = cfg.get('power_state') @@ -167,44 +155,16 @@ def load_power_state(cfg, distro_name): if not isinstance(pstate, dict): raise TypeError("power_state is not a dict.") - opt_map = {'halt': '-H', 'poweroff': '-P', 'reboot': '-r'} - + modes_ok = ['halt', 'poweroff', 'reboot'] mode = pstate.get("mode") - if mode not in opt_map: + if mode not in distro.shutdown_options_map: raise TypeError( "power_state[mode] required, must be one of: %s. found: '%s'." % - (','.join(opt_map.keys()), mode)) - - delay = pstate.get("delay", "now") - message = pstate.get("message") - scale = 1 - fmt = "+%s" - command = ["shutdown", opt_map[mode]] - - if distro_name == 'alpine': - # Convert integer 30 or string '30' to '1800' (seconds) as Alpine's - # halt/poweroff/reboot commands take seconds rather than minutes. - scale = 60 - # No "+" in front of delay value as not supported by Alpine's commands. - fmt = "%s" - if delay == "now": - # Alpine's commands do not understand "now". - delay = "0" - command = [mode, "-d"] - # Alpine's commands don't support a message. - message = None - - try: - delay = convert_delay(delay, fmt=fmt, scale=scale) - except ValueError as e: - raise TypeError( - "power_state[delay] must be 'now' or '+m' (minutes)." - " found '%s'." % delay - ) from e + (','.join(modes_ok), mode)) - args = command + [delay] - if message: - args.append(message) + args = distro.shutdown_command(mode=mode, + delay=pstate.get("delay", "now"), + message=pstate.get("message")) try: timeout = float(pstate.get('timeout', 30.0)) diff --git a/cloudinit/distros/__init__.py b/cloudinit/distros/__init__.py index 2bd9bae8..fac8cf67 100755 --- a/cloudinit/distros/__init__.py +++ b/cloudinit/distros/__init__.py @@ -73,6 +73,9 @@ class Distro(metaclass=abc.ABCMeta): renderer_configs = {} _preferred_ntp_clients = None networking_cls = LinuxNetworking + # This is used by self.shutdown_command(), and can be overridden in + # subclasses + shutdown_options_map = {'halt': '-H', 'poweroff': '-P', 'reboot': '-r'} def __init__(self, name, cfg, paths): self._paths = paths @@ -750,6 +753,22 @@ class Distro(metaclass=abc.ABCMeta): subp.subp(['usermod', '-a', '-G', name, member]) LOG.info("Added user '%s' to group '%s'", member, name) + def shutdown_command(self, *, mode, delay, message): + # called from cc_power_state_change.load_power_state + command = ["shutdown", self.shutdown_options_map[mode]] + try: + if delay != "now": + delay = "+%d" % int(delay) + except ValueError as e: + raise TypeError( + "power_state[delay] must be 'now' or '+m' (minutes)." + " found '%s'." % (delay,) + ) from e + args = command + [delay] + if message: + args.append(message) + return args + def _apply_hostname_transformations_to_url(url: str, transformations: list): """ diff --git a/cloudinit/distros/alpine.py b/cloudinit/distros/alpine.py index e42443fc..e92ff3fb 100644 --- a/cloudinit/distros/alpine.py +++ b/cloudinit/distros/alpine.py @@ -162,4 +162,30 @@ class Distro(distros.Distro): return self._preferred_ntp_clients + def shutdown_command(self, mode='poweroff', delay='now', message=None): + # called from cc_power_state_change.load_power_state + # Alpine has halt/poweroff/reboot, with the following specifics: + # - we use them rather than the generic "shutdown" + # - delay is given with "-d [integer]" + # - the integer is in seconds, cannot be "now", and takes no "+" + # - no message is supported (argument ignored, here) + + command = [mode, "-d"] + + # Convert delay from minutes to seconds, as Alpine's + # halt/poweroff/reboot commands take seconds rather than minutes. + if delay == "now": + # Alpine's commands do not understand "now". + command += ['0'] + else: + try: + command.append(str(int(delay) * 60)) + except ValueError as e: + raise TypeError( + "power_state[delay] must be 'now' or '+m' (minutes)." + " found '%s'." % (delay,) + ) from e + + return command + # vi: ts=4 expandtab diff --git a/cloudinit/distros/bsd.py b/cloudinit/distros/bsd.py index 2ed7a7d5..f717a667 100644 --- a/cloudinit/distros/bsd.py +++ b/cloudinit/distros/bsd.py @@ -17,6 +17,10 @@ class BSD(distros.Distro): hostname_conf_fn = '/etc/rc.conf' rc_conf_fn = "/etc/rc.conf" + # This differs from the parent Distro class, which has -P for + # poweroff. + shutdown_options_map = {'halt': '-H', 'poweroff': '-p', 'reboot': '-r'} + # Set in BSD distro subclasses group_add_cmd_prefix = [] pkg_cmd_install_prefix = [] diff --git a/tests/unittests/test_handler/test_handler_power_state.py b/tests/unittests/test_handler/test_handler_power_state.py index 93b24fdc..4ac49424 100644 --- a/tests/unittests/test_handler/test_handler_power_state.py +++ b/tests/unittests/test_handler/test_handler_power_state.py @@ -4,72 +4,102 @@ import sys from cloudinit.config import cc_power_state_change as psc +from cloudinit import distros +from cloudinit import helpers + from cloudinit.tests import helpers as t_help from cloudinit.tests.helpers import mock class TestLoadPowerState(t_help.TestCase): + def setUp(self): + super(TestLoadPowerState, self).setUp() + cls = distros.fetch('ubuntu') + paths = helpers.Paths({}) + self.dist = cls('ubuntu', {}, paths) + def test_no_config(self): # completely empty config should mean do nothing - (cmd, _timeout, _condition) = psc.load_power_state({}, 'ubuntu') + (cmd, _timeout, _condition) = psc.load_power_state({}, self.dist) self.assertIsNone(cmd) def test_irrelevant_config(self): # no power_state field in config should return None for cmd (cmd, _timeout, _condition) = psc.load_power_state({'foo': 'bar'}, - 'ubuntu') + self.dist) self.assertIsNone(cmd) def test_invalid_mode(self): + cfg = {'power_state': {'mode': 'gibberish'}} - self.assertRaises(TypeError, psc.load_power_state, cfg, 'ubuntu') + self.assertRaises(TypeError, psc.load_power_state, cfg, self.dist) cfg = {'power_state': {'mode': ''}} - self.assertRaises(TypeError, psc.load_power_state, cfg, 'ubuntu') + self.assertRaises(TypeError, psc.load_power_state, cfg, self.dist) def test_empty_mode(self): cfg = {'power_state': {'message': 'goodbye'}} - self.assertRaises(TypeError, psc.load_power_state, cfg, 'ubuntu') + self.assertRaises(TypeError, psc.load_power_state, cfg, self.dist) def test_valid_modes(self): cfg = {'power_state': {}} for mode in ('halt', 'poweroff', 'reboot'): cfg['power_state']['mode'] = mode - check_lps_ret(psc.load_power_state(cfg, 'ubuntu'), mode=mode) + check_lps_ret(psc.load_power_state(cfg, self.dist), mode=mode) def test_invalid_delay(self): cfg = {'power_state': {'mode': 'poweroff', 'delay': 'goodbye'}} - self.assertRaises(TypeError, psc.load_power_state, cfg, 'ubuntu') + self.assertRaises(TypeError, psc.load_power_state, cfg, self.dist) def test_valid_delay(self): cfg = {'power_state': {'mode': 'poweroff', 'delay': ''}} for delay in ("now", "+1", "+30"): cfg['power_state']['delay'] = delay - check_lps_ret(psc.load_power_state(cfg, 'ubuntu')) + check_lps_ret(psc.load_power_state(cfg, self.dist)) def test_message_present(self): cfg = {'power_state': {'mode': 'poweroff', 'message': 'GOODBYE'}} - ret = psc.load_power_state(cfg, 'ubuntu') - check_lps_ret(psc.load_power_state(cfg, 'ubuntu')) + ret = psc.load_power_state(cfg, self.dist) + check_lps_ret(psc.load_power_state(cfg, self.dist)) self.assertIn(cfg['power_state']['message'], ret[0]) def test_no_message(self): # if message is not present, then no argument should be passed for it cfg = {'power_state': {'mode': 'poweroff'}} - (cmd, _timeout, _condition) = psc.load_power_state(cfg, 'ubuntu') + (cmd, _timeout, _condition) = psc.load_power_state(cfg, self.dist) self.assertNotIn("", cmd) - check_lps_ret(psc.load_power_state(cfg, 'ubuntu')) + check_lps_ret(psc.load_power_state(cfg, self.dist)) self.assertTrue(len(cmd) == 3) def test_condition_null_raises(self): cfg = {'power_state': {'mode': 'poweroff', 'condition': None}} - self.assertRaises(TypeError, psc.load_power_state, cfg, 'ubuntu') + self.assertRaises(TypeError, psc.load_power_state, cfg, self.dist) def test_condition_default_is_true(self): cfg = {'power_state': {'mode': 'poweroff'}} - _cmd, _timeout, cond = psc.load_power_state(cfg, 'ubuntu') + _cmd, _timeout, cond = psc.load_power_state(cfg, self.dist) self.assertEqual(cond, True) + def test_freebsd_poweroff_uses_lowercase_p(self): + cls = distros.fetch('freebsd') + paths = helpers.Paths({}) + freebsd = cls('freebsd', {}, paths) + cfg = {'power_state': {'mode': 'poweroff'}} + ret = psc.load_power_state(cfg, freebsd) + self.assertIn('-p', ret[0]) + + def test_alpine_delay(self): + # alpine takes delay in seconds. + cls = distros.fetch('alpine') + paths = helpers.Paths({}) + alpine = cls('alpine', {}, paths) + cfg = {'power_state': {'mode': 'poweroff', 'delay': ''}} + for delay, value in (('now', 0), ("+1", 60), ("+30", 1800)): + cfg['power_state']['delay'] = delay + ret = psc.load_power_state(cfg, alpine) + self.assertEqual('-d', ret[0][1]) + self.assertEqual(str(value), ret[0][2]) + class TestCheckCondition(t_help.TestCase): def cmd_with_exit(self, rc): diff --git a/tools/.github-cla-signers b/tools/.github-cla-signers index aa946511..f01e9b66 100644 --- a/tools/.github-cla-signers +++ b/tools/.github-cla-signers @@ -6,6 +6,7 @@ candlerb dermotbradley dhensby eandersson +emmanuelthome izzyleung johnsonshi jqueuniet -- cgit v1.2.3 From f99d4f96b00a9cfec1c721d364cbfd728674e5dc Mon Sep 17 00:00:00 2001 From: Aman306 <45781773+Aman306@users.noreply.github.com> Date: Wed, 28 Oct 2020 23:36:09 +0530 Subject: Add config modules for controlling IBM PowerVM RMC. (#584) Reliable Scalable Cluster Technology (RSCT) is a set of software components that together provide a comprehensive clustering environment(RAS features) for IBM PowerVM based virtual machines. RSCT includes the Resource Monitoring and Control (RMC) subsystem. RMC is a generalized framework used for managing, monitoring, and manipulating resources. RMC runs as a daemon process on individual machines and needs creation of unique node id and restarts during VM boot. LP: #1895979 Co-authored-by: Scott Moser --- cloudinit/config/cc_refresh_rmc_and_interface.py | 159 +++++++++++++++++++++ cloudinit/config/cc_reset_rmc.py | 143 ++++++++++++++++++ config/cloud.cfg.tmpl | 2 + .../test_handler_refresh_rmc_and_interface.py | 109 ++++++++++++++ tools/.github-cla-signers | 1 + 5 files changed, 414 insertions(+) create mode 100644 cloudinit/config/cc_refresh_rmc_and_interface.py create mode 100644 cloudinit/config/cc_reset_rmc.py create mode 100644 tests/unittests/test_handler/test_handler_refresh_rmc_and_interface.py (limited to 'tests/unittests/test_handler') diff --git a/cloudinit/config/cc_refresh_rmc_and_interface.py b/cloudinit/config/cc_refresh_rmc_and_interface.py new file mode 100644 index 00000000..146758ad --- /dev/null +++ b/cloudinit/config/cc_refresh_rmc_and_interface.py @@ -0,0 +1,159 @@ +# (c) Copyright IBM Corp. 2020 All Rights Reserved +# +# Author: Aman Kumar Sinha +# +# This file is part of cloud-init. See LICENSE file for license information. + +""" +Refresh IPv6 interface and RMC +------------------------------ +**Summary:** Ensure Network Manager is not managing IPv6 interface + +This module is IBM PowerVM Hypervisor specific + +Reliable Scalable Cluster Technology (RSCT) is a set of software components +that together provide a comprehensive clustering environment(RAS features) +for IBM PowerVM based virtual machines. RSCT includes the Resource +Monitoring and Control (RMC) subsystem. RMC is a generalized framework used +for managing, monitoring, and manipulating resources. RMC runs as a daemon +process on individual machines and needs creation of unique node id and +restarts during VM boot. +More details refer +https://www.ibm.com/support/knowledgecenter/en/SGVKBA_3.2/admin/bl503_ovrv.htm + +This module handles +- Refreshing RMC +- Disabling NetworkManager from handling IPv6 interface, as IPv6 interface + is used for communication between RMC daemon and PowerVM hypervisor. + +**Internal name:** ``cc_refresh_rmc_and_interface`` + +**Module frequency:** per always + +**Supported distros:** RHEL + +""" + +from cloudinit import log as logging +from cloudinit.settings import PER_ALWAYS +from cloudinit import util +from cloudinit import subp +from cloudinit import netinfo + +import errno + +frequency = PER_ALWAYS + +LOG = logging.getLogger(__name__) +# Ensure that /opt/rsct/bin has been added to standard PATH of the +# distro. The symlink to rmcctrl is /usr/sbin/rsct/bin/rmcctrl . +RMCCTRL = 'rmcctrl' + + +def handle(name, _cfg, _cloud, _log, _args): + if not subp.which(RMCCTRL): + LOG.debug("No '%s' in path, disabled", RMCCTRL) + return + + LOG.debug( + 'Making the IPv6 up explicitly. ' + 'Ensuring IPv6 interface is not being handled by NetworkManager ' + 'and it is restarted to re-establish the communication with ' + 'the hypervisor') + + ifaces = find_ipv6_ifaces() + + # Setting NM_CONTROLLED=no for IPv6 interface + # making it down and up + + if len(ifaces) == 0: + LOG.debug("Did not find any interfaces with ipv6 addresses.") + else: + for iface in ifaces: + refresh_ipv6(iface) + disable_ipv6(sysconfig_path(iface)) + restart_network_manager() + + +def find_ipv6_ifaces(): + info = netinfo.netdev_info() + ifaces = [] + for iface, data in info.items(): + if iface == "lo": + LOG.debug('Skipping localhost interface') + if len(data.get("ipv4", [])) != 0: + # skip this interface, as it has ipv4 addrs + continue + ifaces.append(iface) + return ifaces + + +def refresh_ipv6(interface): + # IPv6 interface is explicitly brought up, subsequent to which the + # RMC services are restarted to re-establish the communication with + # the hypervisor. + subp.subp(['ip', 'link', 'set', interface, 'down']) + subp.subp(['ip', 'link', 'set', interface, 'up']) + + +def sysconfig_path(iface): + return '/etc/sysconfig/network-scripts/ifcfg-' + iface + + +def restart_network_manager(): + subp.subp(['systemctl', 'restart', 'NetworkManager']) + + +def disable_ipv6(iface_file): + # Ensuring that the communication b/w the hypervisor and VM is not + # interrupted due to NetworkManager. For this purpose, as part of + # this function, the NM_CONTROLLED is explicitly set to No for IPV6 + # interface and NetworkManager is restarted. + try: + contents = util.load_file(iface_file) + except IOError as e: + if e.errno == errno.ENOENT: + LOG.debug("IPv6 interface file %s does not exist\n", + iface_file) + else: + raise e + + if 'IPV6INIT' not in contents: + LOG.debug("Interface file %s did not have IPV6INIT", iface_file) + return + + LOG.debug("Editing interface file %s ", iface_file) + + # Dropping any NM_CONTROLLED or IPV6 lines from IPv6 interface file. + lines = contents.splitlines() + lines = [line for line in lines if not search(line)] + lines.append("NM_CONTROLLED=no") + + with open(iface_file, "w") as fp: + fp.write("\n".join(lines) + "\n") + + +def search(contents): + # Search for any NM_CONTROLLED or IPV6 lines in IPv6 interface file. + return( + contents.startswith("IPV6ADDR") or + contents.startswith("IPADDR6") or + contents.startswith("IPV6INIT") or + contents.startswith("NM_CONTROLLED")) + + +def refresh_rmc(): + # To make a healthy connection between RMC daemon and hypervisor we + # refresh RMC. With refreshing RMC we are ensuring that making IPv6 + # down and up shouldn't impact communication between RMC daemon and + # hypervisor. + # -z : stop Resource Monitoring & Control subsystem and all resource + # managers, but the command does not return control to the user + # until the subsystem and all resource managers are stopped. + # -s : start Resource Monitoring & Control subsystem. + try: + subp.subp([RMCCTRL, '-z']) + subp.subp([RMCCTRL, '-s']) + except Exception: + util.logexc(LOG, 'Failed to refresh the RMC subsystem.') + raise diff --git a/cloudinit/config/cc_reset_rmc.py b/cloudinit/config/cc_reset_rmc.py new file mode 100644 index 00000000..1cd72774 --- /dev/null +++ b/cloudinit/config/cc_reset_rmc.py @@ -0,0 +1,143 @@ +# (c) Copyright IBM Corp. 2020 All Rights Reserved +# +# Author: Aman Kumar Sinha +# +# This file is part of cloud-init. See LICENSE file for license information. + + +""" +Reset RMC +------------ +**Summary:** reset rsct node id + +Reset RMC module is IBM PowerVM Hypervisor specific + +Reliable Scalable Cluster Technology (RSCT) is a set of software components, +that together provide a comprehensive clustering environment (RAS features) +for IBM PowerVM based virtual machines. RSCT includes the Resource monitoring +and control (RMC) subsystem. RMC is a generalized framework used for managing, +monitoring, and manipulating resources. RMC runs as a daemon process on +individual machines and needs creation of unique node id and restarts +during VM boot. +More details refer +https://www.ibm.com/support/knowledgecenter/en/SGVKBA_3.2/admin/bl503_ovrv.htm + +This module handles +- creation of the unique RSCT node id to every instance/virtual machine + and ensure once set, it isn't changed subsequently by cloud-init. + In order to do so, it restarts RSCT service. + +Prerequisite of using this module is to install RSCT packages. + +**Internal name:** ``cc_reset_rmc`` + +**Module frequency:** per instance + +**Supported distros:** rhel, sles and ubuntu + +""" +import os + +from cloudinit import log as logging +from cloudinit.settings import PER_INSTANCE +from cloudinit import util +from cloudinit import subp + +frequency = PER_INSTANCE + +# RMCCTRL is expected to be in system PATH (/opt/rsct/bin) +# The symlink for RMCCTRL and RECFGCT are +# /usr/sbin/rsct/bin/rmcctrl and +# /usr/sbin/rsct/install/bin/recfgct respectively. +RSCT_PATH = '/opt/rsct/install/bin' +RMCCTRL = 'rmcctrl' +RECFGCT = 'recfgct' + +LOG = logging.getLogger(__name__) + +NODE_ID_FILE = '/etc/ct_node_id' + + +def handle(name, _cfg, cloud, _log, _args): + # Ensuring node id has to be generated only once during first boot + if cloud.datasource.platform_type == 'none': + LOG.debug('Skipping creation of new ct_node_id node') + return + + if not os.path.isdir(RSCT_PATH): + LOG.debug("module disabled, RSCT_PATH not present") + return + + orig_path = os.environ.get('PATH') + try: + add_path(orig_path) + reset_rmc() + finally: + if orig_path: + os.environ['PATH'] = orig_path + else: + del os.environ['PATH'] + + +def reconfigure_rsct_subsystems(): + # Reconfigure the RSCT subsystems, which includes removing all RSCT data + # under the /var/ct directory, generating a new node ID, and making it + # appear as if the RSCT components were just installed + try: + out = subp.subp([RECFGCT])[0] + LOG.debug(out.strip()) + return out + except subp.ProcessExecutionError: + util.logexc(LOG, 'Failed to reconfigure the RSCT subsystems.') + raise + + +def get_node_id(): + try: + fp = util.load_file(NODE_ID_FILE) + node_id = fp.split('\n')[0] + return node_id + except Exception: + util.logexc(LOG, 'Failed to get node ID from file %s.' % NODE_ID_FILE) + raise + + +def add_path(orig_path): + # Adding the RSCT_PATH to env standard path + # So thet cloud init automatically find and + # run RECFGCT to create new node_id. + suff = ":" + orig_path if orig_path else "" + os.environ['PATH'] = RSCT_PATH + suff + return os.environ['PATH'] + + +def rmcctrl(): + # Stop the RMC subsystem and all resource managers so that we can make + # some changes to it + try: + return subp.subp([RMCCTRL, '-z']) + except Exception: + util.logexc(LOG, 'Failed to stop the RMC subsystem.') + raise + + +def reset_rmc(): + LOG.debug('Attempting to reset RMC.') + + node_id_before = get_node_id() + LOG.debug('Node ID at beginning of module: %s', node_id_before) + + # Stop the RMC subsystem and all resource managers so that we can make + # some changes to it + rmcctrl() + reconfigure_rsct_subsystems() + + node_id_after = get_node_id() + LOG.debug('Node ID at end of module: %s', node_id_after) + + # Check if new node ID is generated or not + # by comparing old and new node ID + if node_id_after == node_id_before: + msg = 'New node ID did not get generated.' + LOG.error(msg) + raise Exception(msg) diff --git a/config/cloud.cfg.tmpl b/config/cloud.cfg.tmpl index 2beb9b0c..7171aaa5 100644 --- a/config/cloud.cfg.tmpl +++ b/config/cloud.cfg.tmpl @@ -135,6 +135,8 @@ cloud_final_modules: - chef - mcollective - salt-minion + - reset_rmc + - refresh_rmc_and_interface - rightscale_userdata - scripts-vendor - scripts-per-once diff --git a/tests/unittests/test_handler/test_handler_refresh_rmc_and_interface.py b/tests/unittests/test_handler/test_handler_refresh_rmc_and_interface.py new file mode 100644 index 00000000..e13b7793 --- /dev/null +++ b/tests/unittests/test_handler/test_handler_refresh_rmc_and_interface.py @@ -0,0 +1,109 @@ +from cloudinit.config import cc_refresh_rmc_and_interface as ccrmci + +from cloudinit import util + +from cloudinit.tests import helpers as t_help +from cloudinit.tests.helpers import mock + +from textwrap import dedent +import logging + +LOG = logging.getLogger(__name__) +MPATH = "cloudinit.config.cc_refresh_rmc_and_interface" +NET_INFO = { + 'lo': {'ipv4': [{'ip': '127.0.0.1', + 'bcast': '', 'mask': '255.0.0.0', + 'scope': 'host'}], + 'ipv6': [{'ip': '::1/128', + 'scope6': 'host'}], 'hwaddr': '', + 'up': 'True'}, + 'env2': {'ipv4': [{'ip': '8.0.0.19', + 'bcast': '8.0.0.255', 'mask': '255.255.255.0', + 'scope': 'global'}], + 'ipv6': [{'ip': 'fe80::f896:c2ff:fe81:8220/64', + 'scope6': 'link'}], 'hwaddr': 'fa:96:c2:81:82:20', + 'up': 'True'}, + 'env3': {'ipv4': [{'ip': '90.0.0.14', + 'bcast': '90.0.0.255', 'mask': '255.255.255.0', + 'scope': 'global'}], + 'ipv6': [{'ip': 'fe80::f896:c2ff:fe81:8221/64', + 'scope6': 'link'}], 'hwaddr': 'fa:96:c2:81:82:21', + 'up': 'True'}, + 'env4': {'ipv4': [{'ip': '9.114.23.7', + 'bcast': '9.114.23.255', 'mask': '255.255.255.0', + 'scope': 'global'}], + 'ipv6': [{'ip': 'fe80::f896:c2ff:fe81:8222/64', + 'scope6': 'link'}], 'hwaddr': 'fa:96:c2:81:82:22', + 'up': 'True'}, + 'env5': {'ipv4': [], + 'ipv6': [{'ip': 'fe80::9c26:c3ff:fea4:62c8/64', + 'scope6': 'link'}], 'hwaddr': '42:20:86:df:fa:4c', + 'up': 'True'}} + + +class TestRsctNodeFile(t_help.CiTestCase): + def test_disable_ipv6_interface(self): + """test parsing of iface files.""" + fname = self.tmp_path("iface-eth5") + util.write_file(fname, dedent("""\ + BOOTPROTO=static + DEVICE=eth5 + HWADDR=42:20:86:df:fa:4c + IPV6INIT=yes + IPADDR6=fe80::9c26:c3ff:fea4:62c8/64 + IPV6ADDR=fe80::9c26:c3ff:fea4:62c8/64 + NM_CONTROLLED=yes + ONBOOT=yes + STARTMODE=auto + TYPE=Ethernet + USERCTL=no + """)) + + ccrmci.disable_ipv6(fname) + self.assertEqual(dedent("""\ + BOOTPROTO=static + DEVICE=eth5 + HWADDR=42:20:86:df:fa:4c + ONBOOT=yes + STARTMODE=auto + TYPE=Ethernet + USERCTL=no + NM_CONTROLLED=no + """), util.load_file(fname)) + + @mock.patch(MPATH + '.refresh_rmc') + @mock.patch(MPATH + '.restart_network_manager') + @mock.patch(MPATH + '.disable_ipv6') + @mock.patch(MPATH + '.refresh_ipv6') + @mock.patch(MPATH + '.netinfo.netdev_info') + @mock.patch(MPATH + '.subp.which') + def test_handle(self, m_refresh_rmc, + m_netdev_info, m_refresh_ipv6, m_disable_ipv6, + m_restart_nm, m_which): + """Basic test of handle.""" + m_netdev_info.return_value = NET_INFO + m_which.return_value = '/opt/rsct/bin/rmcctrl' + ccrmci.handle( + "refresh_rmc_and_interface", None, None, None, None) + self.assertEqual(1, m_netdev_info.call_count) + m_refresh_ipv6.assert_called_with('env5') + m_disable_ipv6.assert_called_with( + '/etc/sysconfig/network-scripts/ifcfg-env5') + self.assertEqual(1, m_restart_nm.call_count) + self.assertEqual(1, m_refresh_rmc.call_count) + + @mock.patch(MPATH + '.netinfo.netdev_info') + def test_find_ipv6(self, m_netdev_info): + """find_ipv6_ifaces parses netdev_info returning those with ipv6""" + m_netdev_info.return_value = NET_INFO + found = ccrmci.find_ipv6_ifaces() + self.assertEqual(['env5'], found) + + @mock.patch(MPATH + '.subp.subp') + def test_refresh_ipv6(self, m_subp): + """refresh_ipv6 should ip down and up the interface.""" + iface = "myeth0" + ccrmci.refresh_ipv6(iface) + m_subp.assert_has_calls([ + mock.call(['ip', 'link', 'set', iface, 'down']), + mock.call(['ip', 'link', 'set', iface, 'up'])]) diff --git a/tools/.github-cla-signers b/tools/.github-cla-signers index b7e9dc02..475f0872 100644 --- a/tools/.github-cla-signers +++ b/tools/.github-cla-signers @@ -1,5 +1,6 @@ ader1990 AlexBaranowski +Aman306 beezly bipinbachhao BirknerAlex -- cgit v1.2.3 From f680114446a5a20ce88f3d10d966811a774c8e8f Mon Sep 17 00:00:00 2001 From: Chad Smith Date: Wed, 18 Nov 2020 07:23:44 -0700 Subject: cli: add --system param to allow validating system user-data on a machine (#575) Allow root user to validate the userdata provided to the launched machine using `cloud-init devel schema --system` --- cloudinit/config/schema.py | 41 ++++++++--- doc/rtd/topics/faq.rst | 6 +- tests/unittests/test_cli.py | 2 +- tests/unittests/test_handler/test_schema.py | 109 ++++++++++++++++++++-------- 4 files changed, 114 insertions(+), 44 deletions(-) (limited to 'tests/unittests/test_handler') diff --git a/cloudinit/config/schema.py b/cloudinit/config/schema.py index 8a966aee..456bab2c 100644 --- a/cloudinit/config/schema.py +++ b/cloudinit/config/schema.py @@ -1,6 +1,7 @@ # This file is part of cloud-init. See LICENSE file for license information. """schema.py: Set of module functions for processing cloud-config schema.""" +from cloudinit.cmd.devel import read_cfg_paths from cloudinit import importer from cloudinit.util import find_modules, load_file @@ -173,7 +174,8 @@ def annotated_cloudconfig_file(cloudconfig, original_content, schema_errors): def validate_cloudconfig_file(config_path, schema, annotate=False): """Validate cloudconfig file adheres to a specific jsonschema. - @param config_path: Path to the yaml cloud-config file to parse. + @param config_path: Path to the yaml cloud-config file to parse, or None + to default to system userdata from Paths object. @param schema: Dict describing a valid jsonschema to validate against. @param annotate: Boolean set True to print original config file with error annotations on the offending lines. @@ -181,9 +183,24 @@ def validate_cloudconfig_file(config_path, schema, annotate=False): @raises SchemaValidationError containing any of schema_errors encountered. @raises RuntimeError when config_path does not exist. """ - if not os.path.exists(config_path): - raise RuntimeError('Configfile {0} does not exist'.format(config_path)) - content = load_file(config_path, decode=False) + if config_path is None: + # Use system's raw userdata path + if os.getuid() != 0: + raise RuntimeError( + "Unable to read system userdata as non-root user." + " Try using sudo" + ) + paths = read_cfg_paths() + user_data_file = paths.get_ipath_cur("userdata_raw") + content = load_file(user_data_file, decode=False) + else: + if not os.path.exists(config_path): + raise RuntimeError( + 'Configfile {0} does not exist'.format( + config_path + ) + ) + content = load_file(config_path, decode=False) if not content.startswith(CLOUD_CONFIG_HEADER): errors = ( ('format-l1.c1', 'File {0} needs to begin with "{1}"'.format( @@ -425,6 +442,8 @@ def get_parser(parser=None): description='Validate cloud-config files or document schema') parser.add_argument('-c', '--config-file', help='Path of the cloud-config yaml file to validate') + parser.add_argument('--system', action='store_true', default=False, + help='Validate the system cloud-config userdata') parser.add_argument('-d', '--docs', nargs='+', help=('Print schema module docs. Choices: all or' ' space-delimited cc_names.')) @@ -435,11 +454,11 @@ def get_parser(parser=None): def handle_schema_args(name, args): """Handle provided schema args and perform the appropriate actions.""" - exclusive_args = [args.config_file, args.docs] - if not any(exclusive_args) or all(exclusive_args): - error('Expected either --config-file argument or --docs') + exclusive_args = [args.config_file, args.docs, args.system] + if len([arg for arg in exclusive_args if arg]) != 1: + error('Expected one of --config-file, --system or --docs arguments') full_schema = get_schema() - if args.config_file: + if args.config_file or args.system: try: validate_cloudconfig_file( args.config_file, full_schema, args.annotate) @@ -449,7 +468,11 @@ def handle_schema_args(name, args): except RuntimeError as e: error(str(e)) else: - print("Valid cloud-config file {0}".format(args.config_file)) + if args.config_file is None: + cfg_name = "system userdata" + else: + cfg_name = args.config_file + print("Valid cloud-config:", cfg_name) elif args.docs: schema_ids = [subschema['id'] for subschema in full_schema['allOf']] schema_ids += ['all'] diff --git a/doc/rtd/topics/faq.rst b/doc/rtd/topics/faq.rst index d08914b5..27fabf15 100644 --- a/doc/rtd/topics/faq.rst +++ b/doc/rtd/topics/faq.rst @@ -141,12 +141,12 @@ that can validate your user data offline. .. _validate-yaml.py: https://github.com/canonical/cloud-init/blob/master/tools/validate-yaml.py -Another option is to run the following on an instance when debugging: +Another option is to run the following on an instance to debug userdata +provided to the system: .. code-block:: shell-session - $ sudo cloud-init query userdata > user-data.yaml - $ cloud-init devel schema -c user-data.yaml --annotate + $ cloud-init devel schema --system --annotate As launching instances in the cloud can cost money and take a bit longer, sometimes it is easier to launch instances locally using Multipass or LXD: diff --git a/tests/unittests/test_cli.py b/tests/unittests/test_cli.py index dcf0fe5a..74f85959 100644 --- a/tests/unittests/test_cli.py +++ b/tests/unittests/test_cli.py @@ -214,7 +214,7 @@ class TestCLI(test_helpers.FilesystemMockingTestCase): self.assertEqual(1, exit_code) # Known whitebox output from schema subcommand self.assertEqual( - 'Expected either --config-file argument or --docs\n', + 'Expected one of --config-file, --system or --docs arguments\n', self.stderr.getvalue()) def test_wb_devel_schema_subcommand_doc_content(self): diff --git a/tests/unittests/test_handler/test_schema.py b/tests/unittests/test_handler/test_schema.py index 44292571..15aa77bb 100644 --- a/tests/unittests/test_handler/test_schema.py +++ b/tests/unittests/test_handler/test_schema.py @@ -9,9 +9,9 @@ from cloudinit.util import write_file from cloudinit.tests.helpers import CiTestCase, mock, skipUnlessJsonSchema from copy import copy +import itertools import os import pytest -from io import StringIO from pathlib import Path from textwrap import dedent from yaml import safe_load @@ -400,50 +400,97 @@ class AnnotatedCloudconfigFileTest(CiTestCase): annotated_cloudconfig_file(parsed_config, content, schema_errors)) -class MainTest(CiTestCase): +class TestMain: - def test_main_missing_args(self): + exclusive_combinations = itertools.combinations( + ["--system", "--docs all", "--config-file something"], 2 + ) + + @pytest.mark.parametrize("params", exclusive_combinations) + def test_main_exclusive_args(self, params, capsys): + """Main exits non-zero and error on required exclusive args.""" + params = list(itertools.chain(*[a.split() for a in params])) + with mock.patch('sys.argv', ['mycmd'] + params): + with pytest.raises(SystemExit) as context_manager: + main() + assert 1 == context_manager.value.code + + _out, err = capsys.readouterr() + expected = ( + 'Expected one of --config-file, --system or --docs arguments\n' + ) + assert expected == err + + def test_main_missing_args(self, capsys): """Main exits non-zero and reports an error on missing parameters.""" with mock.patch('sys.argv', ['mycmd']): - with mock.patch('sys.stderr', new_callable=StringIO) as m_stderr: - with self.assertRaises(SystemExit) as context_manager: - main() - self.assertEqual(1, context_manager.exception.code) - self.assertEqual( - 'Expected either --config-file argument or --docs\n', - m_stderr.getvalue()) + with pytest.raises(SystemExit) as context_manager: + main() + assert 1 == context_manager.value.code + + _out, err = capsys.readouterr() + expected = ( + 'Expected one of --config-file, --system or --docs arguments\n' + ) + assert expected == err - def test_main_absent_config_file(self): + def test_main_absent_config_file(self, capsys): """Main exits non-zero when config file is absent.""" myargs = ['mycmd', '--annotate', '--config-file', 'NOT_A_FILE'] with mock.patch('sys.argv', myargs): - with mock.patch('sys.stderr', new_callable=StringIO) as m_stderr: - with self.assertRaises(SystemExit) as context_manager: - main() - self.assertEqual(1, context_manager.exception.code) - self.assertEqual( - 'Configfile NOT_A_FILE does not exist\n', - m_stderr.getvalue()) + with pytest.raises(SystemExit) as context_manager: + main() + assert 1 == context_manager.value.code + _out, err = capsys.readouterr() + assert 'Configfile NOT_A_FILE does not exist\n' == err - def test_main_prints_docs(self): + def test_main_prints_docs(self, capsys): """When --docs parameter is provided, main generates documentation.""" myargs = ['mycmd', '--docs', 'all'] with mock.patch('sys.argv', myargs): - with mock.patch('sys.stdout', new_callable=StringIO) as m_stdout: - self.assertEqual(0, main(), 'Expected 0 exit code') - self.assertIn('\nNTP\n---\n', m_stdout.getvalue()) - self.assertIn('\nRuncmd\n------\n', m_stdout.getvalue()) + assert 0 == main(), 'Expected 0 exit code' + out, _err = capsys.readouterr() + assert '\nNTP\n---\n' in out + assert '\nRuncmd\n------\n' in out - def test_main_validates_config_file(self): + def test_main_validates_config_file(self, tmpdir, capsys): """When --config-file parameter is provided, main validates schema.""" - myyaml = self.tmp_path('my.yaml') - myargs = ['mycmd', '--config-file', myyaml] - write_file(myyaml, b'#cloud-config\nntp:') # shortest ntp schema + myyaml = tmpdir.join('my.yaml') + myargs = ['mycmd', '--config-file', myyaml.strpath] + myyaml.write(b'#cloud-config\nntp:') # shortest ntp schema with mock.patch('sys.argv', myargs): - with mock.patch('sys.stdout', new_callable=StringIO) as m_stdout: - self.assertEqual(0, main(), 'Expected 0 exit code') - self.assertIn( - 'Valid cloud-config file {0}'.format(myyaml), m_stdout.getvalue()) + assert 0 == main(), 'Expected 0 exit code' + out, _err = capsys.readouterr() + assert 'Valid cloud-config: {0}\n'.format(myyaml) == out + + @mock.patch('cloudinit.config.schema.read_cfg_paths') + @mock.patch('cloudinit.config.schema.os.getuid', return_value=0) + def test_main_validates_system_userdata( + self, m_getuid, m_read_cfg_paths, capsys, paths + ): + """When --system is provided, main validates system userdata.""" + m_read_cfg_paths.return_value = paths + ud_file = paths.get_ipath_cur("userdata_raw") + write_file(ud_file, b'#cloud-config\nntp:') + myargs = ['mycmd', '--system'] + with mock.patch('sys.argv', myargs): + assert 0 == main(), 'Expected 0 exit code' + out, _err = capsys.readouterr() + assert 'Valid cloud-config: system userdata\n' == out + + @mock.patch('cloudinit.config.schema.os.getuid', return_value=1000) + def test_main_system_userdata_requires_root(self, m_getuid, capsys, paths): + """Non-root user can't use --system param""" + myargs = ['mycmd', '--system'] + with mock.patch('sys.argv', myargs): + with pytest.raises(SystemExit) as context_manager: + main() + assert 1 == context_manager.value.code + _out, err = capsys.readouterr() + expected = ( + 'Unable to read system userdata as non-root user. Try using sudo\n' + ) + assert expected == err class CloudTestsIntegrationTest(CiTestCase): -- cgit v1.2.3 From 12ef7541c2d0c6b2cd510f95fda53ca9c8333064 Mon Sep 17 00:00:00 2001 From: Mina Galić Date: Thu, 19 Nov 2020 23:19:16 +0100 Subject: cc_resizefs on FreeBSD: Fix _can_skip_ufs_resize (#655) On FreeBSD, if a UFS has trim: (-t) or MAC multilabel: (-l) flag, resize FS fail, because the _can_skip_ufs_resize check gets tripped up by the missing options. This was reported at FreeBSD Bugzilla: https://bugs.freebsd.org/bugzilla/show_bug.cgi?id=250496 and as LP: #1901958 Rather than fixing the parser as in the patches proposed there (and attempted in #636) this pull-request rips out all of it, and simplifies the code. We now use `growfs -N` and check if that returns an error. If it returns the correct kind of error, we can skip the resize, because we either are at the correct size, or the filesystem in question is broken or not UFS. If it returns the wrong kind of error, we just re-raise it. LP: #1901958 --- cloudinit/config/cc_resizefs.py | 68 +++++----------------- .../test_handler/test_handler_resizefs.py | 55 +++++++++-------- 2 files changed, 42 insertions(+), 81 deletions(-) (limited to 'tests/unittests/test_handler') diff --git a/cloudinit/config/cc_resizefs.py b/cloudinit/config/cc_resizefs.py index 978d2ee0..9afbb847 100644 --- a/cloudinit/config/cc_resizefs.py +++ b/cloudinit/config/cc_resizefs.py @@ -9,10 +9,7 @@ """Resizefs: cloud-config module which resizes the filesystem""" import errno -import getopt import os -import re -import shlex import stat from textwrap import dedent @@ -88,56 +85,23 @@ def _resize_zfs(mount_point, devpth): return ('zpool', 'online', '-e', mount_point, devpth) -def _get_dumpfs_output(mount_point): - return subp.subp(['dumpfs', '-m', mount_point])[0] - - -def _get_gpart_output(part): - return subp.subp(['gpart', 'show', part])[0] - - def _can_skip_resize_ufs(mount_point, devpth): - # extract the current fs sector size - """ - # dumpfs -m / - # newfs command for / (/dev/label/rootfs) - newfs -L rootf -O 2 -U -a 4 -b 32768 -d 32768 -e 4096 -f 4096 -g 16384 - -h 64 -i 8192 -j -k 6408 -m 8 -o time -s 58719232 /dev/label/rootf - """ - cur_fs_sz = None - frag_sz = None - dumpfs_res = _get_dumpfs_output(mount_point) - for line in dumpfs_res.splitlines(): - if not line.startswith('#'): - newfs_cmd = shlex.split(line) - opt_value = 'O:Ua:s:b:d:e:f:g:h:i:jk:m:o:L:' - optlist, _args = getopt.getopt(newfs_cmd[1:], opt_value) - for o, a in optlist: - if o == "-s": - cur_fs_sz = int(a) - if o == "-f": - frag_sz = int(a) - # check the current partition size - # Example output from `gpart show /dev/da0`: - # => 40 62914480 da0 GPT (30G) - # 40 1024 1 freebsd-boot (512K) - # 1064 58719232 2 freebsd-ufs (28G) - # 58720296 3145728 3 freebsd-swap (1.5G) - # 61866024 1048496 - free - (512M) - expect_sz = None - m = re.search('^(/dev/.+)p([0-9])$', devpth) - gpart_res = _get_gpart_output(m.group(1)) - for line in gpart_res.splitlines(): - if re.search(r"freebsd-ufs", line): - fields = line.split() - expect_sz = int(fields[1]) - # Normalize the gpart sector size, - # because the size is not exactly the same as fs size. - normal_expect_sz = (expect_sz - expect_sz % (frag_sz / 512)) - if normal_expect_sz == cur_fs_sz: - return True - else: - return False + # possible errors cases on the code-path to growfs -N following: + # https://github.com/freebsd/freebsd/blob/HEAD/sbin/growfs/growfs.c + # This is the "good" error: + skip_start = "growfs: requested size" + skip_contain = "is not larger than the current filesystem size" + # growfs exits with 1 for almost all cases up to this one. + # This means we can't just use rcs=[0, 1] as subp parameter: + try: + subp.subp(['growfs', '-N', devpth]) + except subp.ProcessExecutionError as e: + if e.stderr.startswith(skip_start) and skip_contain in e.stderr: + # This FS is already at the desired size + return True + else: + raise e + return False # Do not use a dictionary as these commands should be able to be used diff --git a/tests/unittests/test_handler/test_handler_resizefs.py b/tests/unittests/test_handler/test_handler_resizefs.py index db9a0414..28d55072 100644 --- a/tests/unittests/test_handler/test_handler_resizefs.py +++ b/tests/unittests/test_handler/test_handler_resizefs.py @@ -6,8 +6,8 @@ from cloudinit.config.cc_resizefs import ( from collections import namedtuple import logging -import textwrap +from cloudinit.subp import ProcessExecutionError from cloudinit.tests.helpers import ( CiTestCase, mock, skipUnlessJsonSchema, util, wrap_and_call) @@ -22,44 +22,41 @@ class TestResizefs(CiTestCase): super(TestResizefs, self).setUp() self.name = "resizefs" - @mock.patch('cloudinit.config.cc_resizefs._get_dumpfs_output') - @mock.patch('cloudinit.config.cc_resizefs._get_gpart_output') - def test_skip_ufs_resize(self, gpart_out, dumpfs_out): + @mock.patch('cloudinit.subp.subp') + def test_skip_ufs_resize(self, m_subp): fs_type = "ufs" resize_what = "/" devpth = "/dev/da0p2" - dumpfs_out.return_value = ( - "# newfs command for / (/dev/label/rootfs)\n" - "newfs -O 2 -U -a 4 -b 32768 -d 32768 -e 4096 " - "-f 4096 -g 16384 -h 64 -i 8192 -j -k 6408 -m 8 " - "-o time -s 58719232 /dev/label/rootfs\n") - gpart_out.return_value = textwrap.dedent("""\ - => 40 62914480 da0 GPT (30G) - 40 1024 1 freebsd-boot (512K) - 1064 58719232 2 freebsd-ufs (28G) - 58720296 3145728 3 freebsd-swap (1.5G) - 61866024 1048496 - free - (512M) - """) + err = ("growfs: requested size 2.0GB is not larger than the " + "current filesystem size 2.0GB\n") + exception = ProcessExecutionError(stderr=err, exit_code=1) + m_subp.side_effect = exception res = can_skip_resize(fs_type, resize_what, devpth) self.assertTrue(res) - @mock.patch('cloudinit.config.cc_resizefs._get_dumpfs_output') - @mock.patch('cloudinit.config.cc_resizefs._get_gpart_output') - def test_skip_ufs_resize_roundup(self, gpart_out, dumpfs_out): + @mock.patch('cloudinit.subp.subp') + def test_cannot_skip_ufs_resize(self, m_subp): fs_type = "ufs" resize_what = "/" devpth = "/dev/da0p2" - dumpfs_out.return_value = ( - "# newfs command for / (/dev/label/rootfs)\n" - "newfs -O 2 -U -a 4 -b 32768 -d 32768 -e 4096 " - "-f 4096 -g 16384 -h 64 -i 8192 -j -k 368 -m 8 " - "-o time -s 297080 /dev/label/rootfs\n") - gpart_out.return_value = textwrap.dedent("""\ - => 34 297086 da0 GPT (145M) - 34 297086 1 freebsd-ufs (145M) - """) + m_subp.return_value = ( + ("stdout: super-block backups (for fsck_ffs -b #) at:\n\n"), + ("growfs: no room to allocate last cylinder group; " + "leaving 364KB unused\n") + ) res = can_skip_resize(fs_type, resize_what, devpth) - self.assertTrue(res) + self.assertFalse(res) + + @mock.patch('cloudinit.subp.subp') + def test_cannot_skip_ufs_growfs_exception(self, m_subp): + fs_type = "ufs" + resize_what = "/" + devpth = "/dev/da0p2" + err = "growfs: /dev/da0p2 is not clean - run fsck.\n" + exception = ProcessExecutionError(stderr=err, exit_code=1) + m_subp.side_effect = exception + with self.assertRaises(ProcessExecutionError): + can_skip_resize(fs_type, resize_what, devpth) def test_can_skip_resize_ext(self): self.assertFalse(can_skip_resize('ext', '/', '/dev/sda1')) -- cgit v1.2.3 From c6bcb8df28daa234686a563549681082eb3283a1 Mon Sep 17 00:00:00 2001 From: zsdc Date: Fri, 25 Dec 2020 18:36:39 +0200 Subject: T2117: Updates from 20.4 copied to resizefs_vyos This commit does not change any actually used in VyOS logic. It only copies changes from the upstream 20.4 to stay closer to the upstream code. --- cloudinit/config/cc_resizefs_vyos.py | 68 +++++----------------- .../test_handler/test_handler_resizefs_vyos.py | 55 +++++++++-------- 2 files changed, 42 insertions(+), 81 deletions(-) (limited to 'tests/unittests/test_handler') diff --git a/cloudinit/config/cc_resizefs_vyos.py b/cloudinit/config/cc_resizefs_vyos.py index f5555afc..f8eb84fe 100644 --- a/cloudinit/config/cc_resizefs_vyos.py +++ b/cloudinit/config/cc_resizefs_vyos.py @@ -9,10 +9,7 @@ """Resizefs: cloud-config module which resizes the filesystem""" import errno -import getopt import os -import re -import shlex import stat from textwrap import dedent @@ -101,56 +98,23 @@ def _resize_zfs(mount_point, devpth): return ('zpool', 'online', '-e', mount_point, devpth) -def _get_dumpfs_output(mount_point): - return subp.subp(['dumpfs', '-m', mount_point])[0] - - -def _get_gpart_output(part): - return subp.subp(['gpart', 'show', part])[0] - - def _can_skip_resize_ufs(mount_point, devpth): - # extract the current fs sector size - """ - # dumpfs -m / - # newfs command for / (/dev/label/rootfs) - newfs -L rootf -O 2 -U -a 4 -b 32768 -d 32768 -e 4096 -f 4096 -g 16384 - -h 64 -i 8192 -j -k 6408 -m 8 -o time -s 58719232 /dev/label/rootf - """ - cur_fs_sz = None - frag_sz = None - dumpfs_res = _get_dumpfs_output(mount_point) - for line in dumpfs_res.splitlines(): - if not line.startswith('#'): - newfs_cmd = shlex.split(line) - opt_value = 'O:Ua:s:b:d:e:f:g:h:i:jk:m:o:L:' - optlist, _args = getopt.getopt(newfs_cmd[1:], opt_value) - for o, a in optlist: - if o == "-s": - cur_fs_sz = int(a) - if o == "-f": - frag_sz = int(a) - # check the current partition size - # Example output from `gpart show /dev/da0`: - # => 40 62914480 da0 GPT (30G) - # 40 1024 1 freebsd-boot (512K) - # 1064 58719232 2 freebsd-ufs (28G) - # 58720296 3145728 3 freebsd-swap (1.5G) - # 61866024 1048496 - free - (512M) - expect_sz = None - m = re.search('^(/dev/.+)p([0-9])$', devpth) - gpart_res = _get_gpart_output(m.group(1)) - for line in gpart_res.splitlines(): - if re.search(r"freebsd-ufs", line): - fields = line.split() - expect_sz = int(fields[1]) - # Normalize the gpart sector size, - # because the size is not exactly the same as fs size. - normal_expect_sz = (expect_sz - expect_sz % (frag_sz / 512)) - if normal_expect_sz == cur_fs_sz: - return True - else: - return False + # possible errors cases on the code-path to growfs -N following: + # https://github.com/freebsd/freebsd/blob/HEAD/sbin/growfs/growfs.c + # This is the "good" error: + skip_start = "growfs: requested size" + skip_contain = "is not larger than the current filesystem size" + # growfs exits with 1 for almost all cases up to this one. + # This means we can't just use rcs=[0, 1] as subp parameter: + try: + subp.subp(['growfs', '-N', devpth]) + except subp.ProcessExecutionError as e: + if e.stderr.startswith(skip_start) and skip_contain in e.stderr: + # This FS is already at the desired size + return True + else: + raise e + return False # Do not use a dictionary as these commands should be able to be used diff --git a/tests/unittests/test_handler/test_handler_resizefs_vyos.py b/tests/unittests/test_handler/test_handler_resizefs_vyos.py index d1fafff7..c18ab1ea 100644 --- a/tests/unittests/test_handler/test_handler_resizefs_vyos.py +++ b/tests/unittests/test_handler/test_handler_resizefs_vyos.py @@ -6,8 +6,8 @@ from cloudinit.config.cc_resizefs_vyos import ( from collections import namedtuple import logging -import textwrap +from cloudinit.subp import ProcessExecutionError from cloudinit.tests.helpers import ( CiTestCase, mock, skipUnlessJsonSchema, util, wrap_and_call) @@ -22,44 +22,41 @@ class TestResizefs(CiTestCase): super(TestResizefs, self).setUp() self.name = "resizefs" - @mock.patch('cloudinit.config.cc_resizefs_vyos._get_dumpfs_output') - @mock.patch('cloudinit.config.cc_resizefs_vyos._get_gpart_output') - def test_skip_ufs_resize(self, gpart_out, dumpfs_out): + @mock.patch('cloudinit.subp.subp') + def test_skip_ufs_resize(self, m_subp): fs_type = "ufs" resize_what = "/" devpth = "/dev/da0p2" - dumpfs_out.return_value = ( - "# newfs command for / (/dev/label/rootfs)\n" - "newfs -O 2 -U -a 4 -b 32768 -d 32768 -e 4096 " - "-f 4096 -g 16384 -h 64 -i 8192 -j -k 6408 -m 8 " - "-o time -s 58719232 /dev/label/rootfs\n") - gpart_out.return_value = textwrap.dedent("""\ - => 40 62914480 da0 GPT (30G) - 40 1024 1 freebsd-boot (512K) - 1064 58719232 2 freebsd-ufs (28G) - 58720296 3145728 3 freebsd-swap (1.5G) - 61866024 1048496 - free - (512M) - """) + err = ("growfs: requested size 2.0GB is not larger than the " + "current filesystem size 2.0GB\n") + exception = ProcessExecutionError(stderr=err, exit_code=1) + m_subp.side_effect = exception res = can_skip_resize(fs_type, resize_what, devpth) self.assertTrue(res) - @mock.patch('cloudinit.config.cc_resizefs_vyos._get_dumpfs_output') - @mock.patch('cloudinit.config.cc_resizefs_vyos._get_gpart_output') - def test_skip_ufs_resize_roundup(self, gpart_out, dumpfs_out): + @mock.patch('cloudinit.subp.subp') + def test_cannot_skip_ufs_resize(self, m_subp): fs_type = "ufs" resize_what = "/" devpth = "/dev/da0p2" - dumpfs_out.return_value = ( - "# newfs command for / (/dev/label/rootfs)\n" - "newfs -O 2 -U -a 4 -b 32768 -d 32768 -e 4096 " - "-f 4096 -g 16384 -h 64 -i 8192 -j -k 368 -m 8 " - "-o time -s 297080 /dev/label/rootfs\n") - gpart_out.return_value = textwrap.dedent("""\ - => 34 297086 da0 GPT (145M) - 34 297086 1 freebsd-ufs (145M) - """) + m_subp.return_value = ( + ("stdout: super-block backups (for fsck_ffs -b #) at:\n\n"), + ("growfs: no room to allocate last cylinder group; " + "leaving 364KB unused\n") + ) res = can_skip_resize(fs_type, resize_what, devpth) - self.assertTrue(res) + self.assertFalse(res) + + @mock.patch('cloudinit.subp.subp') + def test_cannot_skip_ufs_growfs_exception(self, m_subp): + fs_type = "ufs" + resize_what = "/" + devpth = "/dev/da0p2" + err = "growfs: /dev/da0p2 is not clean - run fsck.\n" + exception = ProcessExecutionError(stderr=err, exit_code=1) + m_subp.side_effect = exception + with self.assertRaises(ProcessExecutionError): + can_skip_resize(fs_type, resize_what, devpth) def test_can_skip_resize_ext(self): self.assertFalse(can_skip_resize('ext', '/', '/dev/sda1')) -- cgit v1.2.3