Initial hotplug support (#936)

Adds a udev script which will invoke a hotplug hook script on all net
add events. The script will write some udev arguments to a systemd FIFO
socket (to ensure we have only instance of cloud-init running at a
time), which is then read by a new service that calls a new 'cloud-init
devel hotplug-hook' command to handle the new event.

This hotplug-hook command will:
- Fetch the pickled datsource
- Verify that the hotplug event is supported/enabled
- Update the metadata for the datasource
- Ensure the hotplugged device exists within the datasource
- Apply the config change on the datasource metadata
- Bring up the new interface (or apply global network configuration)
- Save the updated metadata back to the pickle cache

Also scattered in some unrelated typing where helpful

10 files changed, 412 insertions, 48 deletions

diff --git a/cloudinit/cmd/devel/hotplug_hook.py b/cloudinit/cmd/devel/hotplug_hook.py
new file mode 100644
index 00000000..0282f24a
--- /dev/null
+++ b/cloudinit/cmd/devel/hotplug_hook.py
@@ -0,0 +1,236 @@
+# This file is part of cloud-init. See LICENSE file for license information.
+"""Handle reconfiguration on hotplug events"""
+import abc
+import argparse
+import os
+import time
+
+from cloudinit import log
+from cloudinit import reporting
+from cloudinit.event import EventScope, EventType
+from cloudinit.net import activators, read_sys_net_safe
+from cloudinit.net.network_state import parse_net_config_data
+from cloudinit.reporting import events
+from cloudinit.stages import Init
+from cloudinit.sources import DataSource
+
+
+LOG = log.getLogger(__name__)
+NAME = 'hotplug-hook'
+
+
+def get_parser(parser=None):
+    """Build or extend an arg parser for hotplug-hook utility.
+
+    @param parser: Optional existing ArgumentParser instance representing the
+        subcommand which will be extended to support the args of this utility.
+
+    @returns: ArgumentParser with proper argument configuration.
+    """
+    if not parser:
+        parser = argparse.ArgumentParser(prog=NAME, description=__doc__)
+
+    parser.description = __doc__
+    parser.add_argument("-d", "--devpath", required=True,
+                        metavar="PATH",
+                        help="sysfs path to hotplugged device")
+    parser.add_argument("-s", "--subsystem", required=True,
+                        help="subsystem to act on",
+                        choices=['net'])
+    parser.add_argument("-u", "--udevaction", required=True,
+                        help="action to take",
+                        choices=['add', 'remove'])
+
+    return parser
+
+
+class UeventHandler(abc.ABC):
+    def __init__(self, id, datasource, devpath, action, success_fn):
+        self.id = id
+        self.datasource = datasource  # type: DataSource
+        self.devpath = devpath
+        self.action = action
+        self.success_fn = success_fn
+
+    @abc.abstractmethod
+    def apply(self):
+        raise NotImplementedError()
+
+    @property
+    @abc.abstractmethod
+    def config(self):
+        raise NotImplementedError()
+
+    @abc.abstractmethod
+    def device_detected(self) -> bool:
+        raise NotImplementedError()
+
+    def detect_hotplugged_device(self):
+        detect_presence = None
+        if self.action == 'add':
+            detect_presence = True
+        elif self.action == 'remove':
+            detect_presence = False
+        else:
+            raise ValueError('Unknown action: %s' % self.action)
+
+        if detect_presence != self.device_detected():
+            raise RuntimeError(
+                'Failed to detect %s in updated metadata' % self.id)
+
+    def success(self):
+        return self.success_fn()
+
+    def update_metadata(self):
+        result = self.datasource.update_metadata_if_supported([
+            EventType.HOTPLUG])
+        if not result:
+            raise RuntimeError(
+                'Datasource %s not updated for '
+                'event %s' % (self.datasource, EventType.HOTPLUG)
+            )
+        return result
+
+
+class NetHandler(UeventHandler):
+    def __init__(self, datasource, devpath, action, success_fn):
+        # convert devpath to mac address
+        id = read_sys_net_safe(os.path.basename(devpath), 'address')
+        super().__init__(id, datasource, devpath, action, success_fn)
+
+    def apply(self):
+        self.datasource.distro.apply_network_config(
+            self.config,
+            bring_up=False,
+        )
+        interface_name = os.path.basename(self.devpath)
+        activator = activators.select_activator()
+        if self.action == 'add':
+            if not activator.bring_up_interface(interface_name):
+                raise RuntimeError(
+                    'Failed to bring up device: {}'.format(self.devpath))
+        elif self.action == 'remove':
+            if not activator.bring_down_interface(interface_name):
+                raise RuntimeError(
+                    'Failed to bring down device: {}'.format(self.devpath))
+
+    @property
+    def config(self):
+        return self.datasource.network_config
+
+    def device_detected(self) -> bool:
+        netstate = parse_net_config_data(self.config)
+        found = [
+            iface for iface in netstate.iter_interfaces()
+            if iface.get('mac_address') == self.id
+        ]
+        LOG.debug('Ifaces with ID=%s : %s', self.id, found)
+        return len(found) > 0
+
+
+SUBSYSTEM_PROPERTES_MAP = {
+    'net': (NetHandler, EventScope.NETWORK),
+}
+
+
+def handle_hotplug(
+    hotplug_init: Init, devpath, subsystem, udevaction
+):
+    handler_cls, event_scope = SUBSYSTEM_PROPERTES_MAP.get(
+        subsystem, (None, None)
+    )
+    if handler_cls is None:
+        raise Exception(
+            'hotplug-hook: cannot handle events for subsystem: {}'.format(
+                subsystem))
+
+    LOG.debug('Fetching datasource')
+    datasource = hotplug_init.fetch(existing="trust")
+
+    if not hotplug_init.update_event_enabled(
+        event_source_type=EventType.HOTPLUG,
+        scope=EventScope.NETWORK
+    ):
+        LOG.debug('hotplug not enabled for event of type %s', event_scope)
+        return
+
+    LOG.debug('Creating %s event handler', subsystem)
+    event_handler = handler_cls(
+        datasource=datasource,
+        devpath=devpath,
+        action=udevaction,
+        success_fn=hotplug_init._write_to_cache
+    )  # type: UeventHandler
+    wait_times = [1, 3, 5, 10, 30]
+    for attempt, wait in enumerate(wait_times):
+        LOG.debug(
+            'subsystem=%s update attempt %s/%s',
+            subsystem,
+            attempt,
+            len(wait_times)
+        )
+        try:
+            LOG.debug('Refreshing metadata')
+            event_handler.update_metadata()
+            LOG.debug('Detecting device in updated metadata')
+            event_handler.detect_hotplugged_device()
+            LOG.debug('Applying config change')
+            event_handler.apply()
+            LOG.debug('Updating cache')
+            event_handler.success()
+            break
+        except Exception as e:
+            LOG.debug('Exception while processing hotplug event. %s', e)
+            time.sleep(wait)
+            last_exception = e
+    else:
+        raise last_exception  # type: ignore
+
+
+def handle_args(name, args):
+    # Note that if an exception happens between now and when logging is
+    # setup, we'll only see it in the journal
+    hotplug_reporter = events.ReportEventStack(
+        name, __doc__, reporting_enabled=True
+    )
+
+    hotplug_init = Init(ds_deps=[], reporter=hotplug_reporter)
+    hotplug_init.read_cfg()
+
+    log.setupLogging(hotplug_init.cfg)
+    if 'reporting' in hotplug_init.cfg:
+        reporting.update_configuration(hotplug_init.cfg.get('reporting'))
+
+    # Logging isn't going to be setup until now
+    LOG.debug(
+        '%s called with the following arguments: {udevaction: %s, '
+        'subsystem: %s, devpath: %s}',
+        name, args.udevaction, args.subsystem, args.devpath
+    )
+    LOG.debug(
+        '%s called with the following arguments:\n'
+        'udevaction: %s\n'
+        'subsystem: %s\n'
+        'devpath: %s',
+        name, args.udevaction, args.subsystem, args.devpath
+    )
+
+    with hotplug_reporter:
+        try:
+            handle_hotplug(
+                hotplug_init=hotplug_init,
+                devpath=args.devpath,
+                subsystem=args.subsystem,
+                udevaction=args.udevaction,
+            )
+        except Exception:
+            LOG.exception('Received fatal exception handling hotplug!')
+            raise
+
+    LOG.debug('Exiting hotplug handler')
+    reporting.flush_events()
+
+
+if __name__ == '__main__':
+    args = get_parser().parse_args()
+    handle_args(NAME, args)
diff --git a/cloudinit/cmd/devel/parser.py b/cloudinit/cmd/devel/parser.py
index 1a3c46a4..be304630 100644
--- a/cloudinit/cmd/devel/parser.py
+++ b/cloudinit/cmd/devel/parser.py
@@ -7,6 +7,7 @@
 import argparse
 from cloudinit.config import schema
 
+from . import hotplug_hook
 from . import net_convert
 from . import render
 from . import make_mime
@@ -21,6 +22,8 @@ def get_parser(parser=None):
     subparsers.required = True
 
     subcmds = [
+        (hotplug_hook.NAME, hotplug_hook.__doc__,
+         hotplug_hook.get_parser, hotplug_hook.handle_args),
         ('schema', 'Validate cloud-config files for document schema',
          schema.get_parser, schema.handle_schema_args),
         (net_convert.NAME, net_convert.__doc__,
diff --git a/cloudinit/distros/__init__.py b/cloudinit/distros/__init__.py
index 2caa8bc2..7bdf2197 100755
--- a/cloudinit/distros/__init__.py
+++ b/cloudinit/distros/__init__.py
@@ -206,8 +206,15 @@ class Distro(persistence.CloudInitPickleMixin, metaclass=abc.ABCMeta):
     def generate_fallback_config(self):
         return net.generate_fallback_config()
 
-    def apply_network_config(self, netconfig, bring_up=False):
-        # apply network config netconfig
+    def apply_network_config(self, netconfig, bring_up=False) -> bool:
+        """Apply the network config.
+
+        If bring_up is True, attempt to bring up the passed in devices. If
+        devices is None, attempt to bring up devices returned by
+        _write_network_config.
+
+        Returns True if any devices failed to come up, otherwise False.
+        """
         # This method is preferred to apply_network which only takes
         # a much less complete network config format (interfaces(5)).
         network_state = parse_net_config_data(netconfig)
diff --git a/cloudinit/event.py b/cloudinit/event.py
index 76a0afc6..53ad4c25 100644
--- a/cloudinit/event.py
+++ b/cloudinit/event.py
@@ -29,6 +29,7 @@ class EventType(Enum):
     BOOT = "boot"
     BOOT_NEW_INSTANCE = "boot-new-instance"
     BOOT_LEGACY = "boot-legacy"
+    HOTPLUG = 'hotplug'
 
     def __str__(self):  # pylint: disable=invalid-str-returned
         return self.value
diff --git a/cloudinit/net/activators.py b/cloudinit/net/activators.py
index 34fee3bf..84aaafc9 100644
--- a/cloudinit/net/activators.py
+++ b/cloudinit/net/activators.py
@@ -15,31 +15,80 @@ from cloudinit.net.sysconfig import NM_CFG_FILE
 LOG = logging.getLogger(__name__)
 
 
+def _alter_interface(cmd, device_name) -> bool:
+    LOG.debug("Attempting command %s for device %s", cmd, device_name)
+    try:
+        (_out, err) = subp.subp(cmd)
+        if len(err):
+            LOG.warning("Running %s resulted in stderr output: %s",
+                        cmd, err)
+        return True
+    except subp.ProcessExecutionError:
+        util.logexc(LOG, "Running interface command %s failed", cmd)
+        return False
+
+
 class NetworkActivator(ABC):
     @staticmethod
     @abstractmethod
     def available() -> bool:
+        """Return True if activator is available, otherwise return False."""
         raise NotImplementedError()
 
     @staticmethod
     @abstractmethod
     def bring_up_interface(device_name: str) -> bool:
+        """Bring up interface.
+
+        Return True is successful, otherwise return False
+        """
+        raise NotImplementedError()
+
+    @staticmethod
+    @abstractmethod
+    def bring_down_interface(device_name: str) -> bool:
+        """Bring down interface.
+
+        Return True is successful, otherwise return False
+        """
         raise NotImplementedError()
 
     @classmethod
     def bring_up_interfaces(cls, device_names: Iterable[str]) -> bool:
-        all_succeeded = True
-        for device in device_names:
-            if not cls.bring_up_interface(device):
-                all_succeeded = False
-        return all_succeeded
+        """Bring up specified list of interfaces.
+
+        Return True is successful, otherwise return False
+        """
+        return all(cls.bring_up_interface(device) for device in device_names)
 
     @classmethod
     def bring_up_all_interfaces(cls, network_state: NetworkState) -> bool:
+        """Bring up all interfaces.
+
+        Return True is successful, otherwise return False
+        """
         return cls.bring_up_interfaces(
             [i['name'] for i in network_state.iter_interfaces()]
         )
 
+    @classmethod
+    def bring_down_interfaces(cls, device_names: Iterable[str]) -> bool:
+        """Bring down specified list of interfaces.
+
+        Return True is successful, otherwise return False
+        """
+        return all(cls.bring_down_interface(device) for device in device_names)
+
+    @classmethod
+    def bring_down_all_interfaces(cls, network_state: NetworkState) -> bool:
+        """Bring down all interfaces.
+
+        Return True is successful, otherwise return False
+        """
+        return cls.bring_down_interfaces(
+            [i['name'] for i in network_state.iter_interfaces()]
+        )
+
 
 class IfUpDownActivator(NetworkActivator):
     # Note that we're not overriding bring_up_interfaces to pass something
@@ -53,24 +102,27 @@ class IfUpDownActivator(NetworkActivator):
 
     @staticmethod
     def bring_up_interface(device_name: str) -> bool:
-        """Bring up interface using ifup."""
+        """Bring up interface using ifup.
+
+        Return True is successful, otherwise return False
+        """
         cmd = ['ifup', device_name]
-        LOG.debug("Attempting to run bring up interface %s using command %s",
-                  device_name, cmd)
-        try:
-            (_out, err) = subp.subp(cmd)
-            if len(err):
-                LOG.warning("Running %s resulted in stderr output: %s",
-                            cmd, err)
-            return True
-        except subp.ProcessExecutionError:
-            util.logexc(LOG, "Running interface command %s failed", cmd)
-            return False
+        return _alter_interface(cmd, device_name)
+
+    @staticmethod
+    def bring_down_interface(device_name: str) -> bool:
+        """Bring up interface using ifup.
+
+        Return True is successful, otherwise return False
+        """
+        cmd = ['ifdown', device_name]
+        return _alter_interface(cmd, device_name)
 
 
 class NetworkManagerActivator(NetworkActivator):
     @staticmethod
     def available(target=None) -> bool:
+        """ Return true if network manager can be used on this system."""
         config_present = os.path.isfile(
             subp.target_path(target, path=NM_CFG_FILE)
         )
@@ -79,44 +131,86 @@ class NetworkManagerActivator(NetworkActivator):
 
     @staticmethod
     def bring_up_interface(device_name: str) -> bool:
-        try:
-            subp.subp(['nmcli', 'connection', 'up', device_name])
-        except subp.ProcessExecutionError:
-            util.logexc(LOG, "nmcli failed to bring up {}".format(device_name))
-            return False
-        return True
+        """Bring up interface using nmcli.
+
+        Return True is successful, otherwise return False
+        """
+        cmd = ['nmcli', 'connection', 'up', 'ifname', device_name]
+        return _alter_interface(cmd, device_name)
+
+    @staticmethod
+    def bring_down_interface(device_name: str) -> bool:
+        """Bring down interface using nmcli.
+
+        Return True is successful, otherwise return False
+        """
+        cmd = ['nmcli', 'connection', 'down', device_name]
+        return _alter_interface(cmd, device_name)
 
 
 class NetplanActivator(NetworkActivator):
+    NETPLAN_CMD = ['netplan', 'apply']
+
     @staticmethod
     def available(target=None) -> bool:
+        """ Return true if netplan can be used on this system."""
         return netplan_available(target=target)
 
     @staticmethod
-    def _apply_netplan():
-        LOG.debug('Applying current netplan config')
-        try:
-            subp.subp(['netplan', 'apply'], capture=True)
-        except subp.ProcessExecutionError:
-            util.logexc(LOG, "netplan apply failed")
-            return False
-        return True
-
-    @staticmethod
     def bring_up_interface(device_name: str) -> bool:
+        """Apply netplan config.
+
+        Return True is successful, otherwise return False
+        """
         LOG.debug("Calling 'netplan apply' rather than "
-                  "bringing up individual interfaces")
-        return NetplanActivator._apply_netplan()
+                  "altering individual interfaces")
+        return _alter_interface(NetplanActivator.NETPLAN_CMD, 'all')
 
     @staticmethod
     def bring_up_interfaces(device_names: Iterable[str]) -> bool:
+        """Apply netplan config.
+
+        Return True is successful, otherwise return False
+        """
         LOG.debug("Calling 'netplan apply' rather than "
-                  "bringing up individual interfaces")
-        return NetplanActivator._apply_netplan()
+                  "altering individual interfaces")
+        return _alter_interface(NetplanActivator.NETPLAN_CMD, 'all')
 
     @staticmethod
     def bring_up_all_interfaces(network_state: NetworkState) -> bool:
-        return NetplanActivator._apply_netplan()
+        """Apply netplan config.
+
+        Return True is successful, otherwise return False
+        """
+        return _alter_interface(NetplanActivator.NETPLAN_CMD, 'all')
+
+    @staticmethod
+    def bring_down_interface(device_name: str) -> bool:
+        """Apply netplan config.
+
+        Return True is successful, otherwise return False
+        """
+        LOG.debug("Calling 'netplan apply' rather than "
+                  "altering individual interfaces")
+        return _alter_interface(NetplanActivator.NETPLAN_CMD, 'all')
+
+    @staticmethod
+    def bring_down_interfaces(device_names: Iterable[str]) -> bool:
+        """Apply netplan config.
+
+        Return True is successful, otherwise return False
+        """
+        LOG.debug("Calling 'netplan apply' rather than "
+                  "altering individual interfaces")
+        return _alter_interface(NetplanActivator.NETPLAN_CMD, 'all')
+
+    @staticmethod
+    def bring_down_all_interfaces(network_state: NetworkState) -> bool:
+        """Apply netplan config.
+
+        Return True is successful, otherwise return False
+        """
+        return _alter_interface(NetplanActivator.NETPLAN_CMD, 'all')
 
 
 # This section is mostly copied and pasted from renderers.py. An abstract
@@ -153,4 +247,6 @@ def select_activator(priority=None, target=None) -> Type[NetworkActivator]:
         raise RuntimeError(
             "No available network activators found%s. Searched "
             "through list: %s" % (tmsg, priority))
-    return found[0]
+    selected = found[0]
+    LOG.debug('Using selected activator: %s', selected)
+    return selected
diff --git a/cloudinit/sources/DataSourceConfigDrive.py b/cloudinit/sources/DataSourceConfigDrive.py
index 62756cf7..19c8d126 100644
--- a/cloudinit/sources/DataSourceConfigDrive.py
+++ b/cloudinit/sources/DataSourceConfigDrive.py
@@ -12,9 +12,8 @@ from cloudinit import log as logging
 from cloudinit import sources
 from cloudinit import subp
 from cloudinit import util
-
+from cloudinit.event import EventScope, EventType
 from cloudinit.net import eni
-
 from cloudinit.sources.DataSourceIBMCloud import get_ibm_platform
 from cloudinit.sources.helpers import openstack
 
@@ -37,6 +36,13 @@ class DataSourceConfigDrive(openstack.SourceMixin, sources.DataSource):
 
     dsname = 'ConfigDrive'
 
+    supported_update_events = {EventScope.NETWORK: {
+        EventType.BOOT_NEW_INSTANCE,
+        EventType.BOOT,
+        EventType.BOOT_LEGACY,
+        EventType.HOTPLUG,
+    }}
+
     def __init__(self, sys_cfg, distro, paths):
         super(DataSourceConfigDrive, self).__init__(sys_cfg, distro, paths)
         self.source = None
diff --git a/cloudinit/sources/DataSourceEc2.py b/cloudinit/sources/DataSourceEc2.py
index 8a7f7c60..700437b0 100644
--- a/cloudinit/sources/DataSourceEc2.py
+++ b/cloudinit/sources/DataSourceEc2.py
@@ -76,6 +76,13 @@ class DataSourceEc2(sources.DataSource):
     # Whether we want to get network configuration from the metadata service.
     perform_dhcp_setup = False
 
+    supported_update_events = {EventScope.NETWORK: {
+        EventType.BOOT_NEW_INSTANCE,
+        EventType.BOOT,
+        EventType.BOOT_LEGACY,
+        EventType.HOTPLUG,
+    }}
+
     def __init__(self, sys_cfg, distro, paths):
         super(DataSourceEc2, self).__init__(sys_cfg, distro, paths)
         self.metadata_address = None
diff --git a/cloudinit/sources/DataSourceOpenStack.py b/cloudinit/sources/DataSourceOpenStack.py
index 619a171e..a85b71d7 100644
--- a/cloudinit/sources/DataSourceOpenStack.py
+++ b/cloudinit/sources/DataSourceOpenStack.py
@@ -8,11 +8,11 @@ import time
 
 from cloudinit import dmi
 from cloudinit import log as logging
-from cloudinit.net.dhcp import EphemeralDHCPv4, NoDHCPLeaseError
 from cloudinit import sources
 from cloudinit import url_helper
 from cloudinit import util
-
+from cloudinit.event import EventScope, EventType
+from cloudinit.net.dhcp import EphemeralDHCPv4, NoDHCPLeaseError
 from cloudinit.sources.helpers import openstack
 from cloudinit.sources import DataSourceOracle as oracle
 
@@ -46,6 +46,13 @@ class DataSourceOpenStack(openstack.SourceMixin, sources.DataSource):
     # Whether we want to get network configuration from the metadata service.
     perform_dhcp_setup = False
 
+    supported_update_events = {EventScope.NETWORK: {
+        EventType.BOOT_NEW_INSTANCE,
+        EventType.BOOT,
+        EventType.BOOT_LEGACY,
+        EventType.HOTPLUG
+    }}
+
     def __init__(self, sys_cfg, distro, paths):
         super(DataSourceOpenStack, self).__init__(sys_cfg, distro, paths)
         self.metadata_address = None
diff --git a/cloudinit/sources/__init__.py b/cloudinit/sources/__init__.py
index 9d25b0ee..bf6bf139 100644
--- a/cloudinit/sources/__init__.py
+++ b/cloudinit/sources/__init__.py
@@ -23,6 +23,7 @@ from cloudinit import type_utils
 from cloudinit import user_data as ud
 from cloudinit import util
 from cloudinit.atomic_helper import write_json
+from cloudinit.distros import Distro
 from cloudinit.event import EventScope, EventType
 from cloudinit.filters import launch_index
 from cloudinit.persistence import CloudInitPickleMixin
@@ -215,7 +216,7 @@ class DataSource(CloudInitPickleMixin, metaclass=abc.ABCMeta):
 
     _ci_pkl_version = 1
 
-    def __init__(self, sys_cfg, distro, paths, ud_proc=None):
+    def __init__(self, sys_cfg, distro: Distro, paths, ud_proc=None):
         self.sys_cfg = sys_cfg
         self.distro = distro
         self.paths = paths
diff --git a/cloudinit/stages.py b/cloudinit/stages.py
index 06e0d9b1..bc164fa0 100644
--- a/cloudinit/stages.py
+++ b/cloudinit/stages.py
@@ -241,7 +241,7 @@ class Init(object):
             else:
                 return (None, "cache invalid in datasource: %s" % ds)
 
-    def _get_data_source(self, existing):
+    def _get_data_source(self, existing) -> sources.DataSource:
         if self.datasource is not NULL_DATA_SOURCE:
             return self.datasource
 
@@ -267,7 +267,7 @@ class Init(object):
                                                cfg_list,
                                                pkg_list, self.reporter)
             LOG.info("Loaded datasource %s - %s", dsname, ds)
-        self.datasource = ds
+        self.datasource = ds  # type: sources.DataSource
         # Ensure we adjust our path members datasource
         # now that we have one (thus allowing ipath to be used)
         self._reset()