Allow user control over update events (#834)

Control is currently limited to boot events, though this should
allow us to more easily incorporate HOTPLUG support. Disabling
'instance-first-boot' is not supported as we apply networking config
too early in boot to have processed userdata (along with the fact
that this would be a pretty big foot-gun).

The concept of update events on datasource has been split into
supported update events and default update events. Defaults will be
used if there is no user-defined update events, but user-defined
events won't be supplied if they aren't supported.
When applying the networking config, we now check to see if the event
is supported by the datasource as well as if it is enabled.

Configuration looks like:
updates:
  network:
    when: ['boot']

17 files changed, 545 insertions, 82 deletions

diff --git a/cloudinit/event.py b/cloudinit/event.py
index f7b311fb..76a0afc6 100644
--- a/cloudinit/event.py
+++ b/cloudinit/event.py
@@ -1,17 +1,72 @@
 # This file is part of cloud-init. See LICENSE file for license information.
-
 """Classes and functions related to event handling."""
 
+from enum import Enum
+from typing import Dict, Set
+
+from cloudinit import log as logging
+
+LOG = logging.getLogger(__name__)
+
 
-# Event types which can generate maintenance requests for cloud-init.
-class EventType(object):
-    BOOT = "System boot"
-    BOOT_NEW_INSTANCE = "New instance first boot"
+class EventScope(Enum):
+    # NETWORK is currently the only scope, but we want to leave room to
+    # grow other scopes (e.g., STORAGE) without having to make breaking
+    # changes to the user config
+    NETWORK = 'network'
 
-    # TODO: Cloud-init will grow support for the follow event types:
-    # UDEV
+    def __str__(self):  # pylint: disable=invalid-str-returned
+        return self.value
+
+
+class EventType(Enum):
+    """Event types which can generate maintenance requests for cloud-init."""
+    # Cloud-init should grow support for the follow event types:
+    # HOTPLUG
     # METADATA_CHANGE
     # USER_REQUEST
 
+    BOOT = "boot"
+    BOOT_NEW_INSTANCE = "boot-new-instance"
+    BOOT_LEGACY = "boot-legacy"
+
+    def __str__(self):  # pylint: disable=invalid-str-returned
+        return self.value
+
+
+def userdata_to_events(user_config: dict) -> Dict[EventScope, Set[EventType]]:
+    """Convert userdata into update config format defined on datasource.
+
+    Userdata is in the form of (e.g):
+    {'network': {'when': ['boot']}}
+
+    DataSource config is in the form of:
+    {EventScope.Network: {EventType.BOOT}}
+
+    Take the first and return the second
+    """
+    update_config = {}
+    for scope, scope_list in user_config.items():
+        try:
+            new_scope = EventScope(scope)
+        except ValueError as e:
+            LOG.warning(
+                "%s! Update data will be ignored for '%s' scope",
+                str(e),
+                scope,
+            )
+            continue
+        try:
+            new_values = [EventType(x) for x in scope_list['when']]
+        except ValueError as e:
+            LOG.warning(
+                "%s! Update data will be ignored for '%s' scope",
+                str(e),
+                scope,
+            )
+            new_values = []
+        update_config[new_scope] = set(new_values)
+
+    return update_config
 
 # vi: ts=4 expandtab
diff --git a/cloudinit/sources/DataSourceAzure.py b/cloudinit/sources/DataSourceAzure.py
index 2f3390c3..dcdf9f8f 100755
--- a/cloudinit/sources/DataSourceAzure.py
+++ b/cloudinit/sources/DataSourceAzure.py
@@ -22,7 +22,7 @@ import requests
 from cloudinit import dmi
 from cloudinit import log as logging
 from cloudinit import net
-from cloudinit.event import EventType
+from cloudinit.event import EventScope, EventType
 from cloudinit.net import device_driver
 from cloudinit.net.dhcp import EphemeralDHCPv4
 from cloudinit import sources
@@ -338,6 +338,13 @@ def temporary_hostname(temp_hostname, cfg, hostname_command='hostname'):
 class DataSourceAzure(sources.DataSource):
 
     dsname = 'Azure'
+    # Regenerate network config new_instance boot and every boot
+    default_update_events = {EventScope.NETWORK: {
+        EventType.BOOT_NEW_INSTANCE,
+        EventType.BOOT,
+        EventType.BOOT_LEGACY
+    }}
+
     _negotiated = False
     _metadata_imds = sources.UNSET
     _ci_pkl_version = 1
@@ -352,8 +359,6 @@ class DataSourceAzure(sources.DataSource):
             BUILTIN_DS_CONFIG])
         self.dhclient_lease_file = self.ds_cfg.get('dhclient_lease_file')
         self._network_config = None
-        # Regenerate network config new_instance boot and every boot
-        self.update_events['network'].add(EventType.BOOT)
         self._ephemeral_dhcp_ctx = None
         self.failed_desired_api_version = False
         self.iso_dev = None
@@ -2309,8 +2314,8 @@ def maybe_remove_ubuntu_network_config_scripts(paths=None):
                 LOG.info(
                     'Removing Ubuntu extended network scripts because'
                     ' cloud-init updates Azure network configuration on the'
-                    ' following event: %s.',
-                    EventType.BOOT)
+                    ' following events: %s.',
+                    [EventType.BOOT.value, EventType.BOOT_LEGACY.value])
                 logged = True
             if os.path.isdir(path):
                 util.del_dir(path)
diff --git a/cloudinit/sources/DataSourceEc2.py b/cloudinit/sources/DataSourceEc2.py
index a2105dc7..8a7f7c60 100644
--- a/cloudinit/sources/DataSourceEc2.py
+++ b/cloudinit/sources/DataSourceEc2.py
@@ -8,6 +8,7 @@
 #
 # This file is part of cloud-init. See LICENSE file for license information.
 
+import copy
 import os
 import time
 
@@ -20,7 +21,7 @@ from cloudinit import sources
 from cloudinit import url_helper as uhelp
 from cloudinit import util
 from cloudinit import warnings
-from cloudinit.event import EventType
+from cloudinit.event import EventScope, EventType
 
 LOG = logging.getLogger(__name__)
 
@@ -426,7 +427,12 @@ class DataSourceEc2(sources.DataSource):
             # Non-VPC (aka Classic) Ec2 instances need to rewrite the
             # network config file every boot due to MAC address change.
             if self.is_classic_instance():
-                self.update_events['network'].add(EventType.BOOT)
+                self.default_update_events = copy.deepcopy(
+                    self.default_update_events)
+                self.default_update_events[EventScope.NETWORK].add(
+                    EventType.BOOT)
+                self.default_update_events[EventScope.NETWORK].add(
+                    EventType.BOOT_LEGACY)
         else:
             LOG.warning("Metadata 'network' key not valid: %s.", net_md)
         self._network_config = result
diff --git a/cloudinit/sources/DataSourceRbxCloud.py b/cloudinit/sources/DataSourceRbxCloud.py
index 0b8994bf..bb69e998 100644
--- a/cloudinit/sources/DataSourceRbxCloud.py
+++ b/cloudinit/sources/DataSourceRbxCloud.py
@@ -17,7 +17,7 @@ from cloudinit import log as logging
 from cloudinit import sources
 from cloudinit import subp
 from cloudinit import util
-from cloudinit.event import EventType
+from cloudinit.event import EventScope, EventType
 
 LOG = logging.getLogger(__name__)
 ETC_HOSTS = '/etc/hosts'
@@ -206,10 +206,11 @@ def read_user_data_callback(mount_dir):
 
 class DataSourceRbxCloud(sources.DataSource):
     dsname = "RbxCloud"
-    update_events = {'network': [
+    default_update_events = {EventScope.NETWORK: {
         EventType.BOOT_NEW_INSTANCE,
-        EventType.BOOT
-    ]}
+        EventType.BOOT,
+        EventType.BOOT_LEGACY
+    }}
 
     def __init__(self, sys_cfg, distro, paths):
         sources.DataSource.__init__(self, sys_cfg, distro, paths)
diff --git a/cloudinit/sources/DataSourceScaleway.py b/cloudinit/sources/DataSourceScaleway.py
index 41be7665..7b8974a2 100644
--- a/cloudinit/sources/DataSourceScaleway.py
+++ b/cloudinit/sources/DataSourceScaleway.py
@@ -31,8 +31,8 @@ from cloudinit import sources
 from cloudinit import url_helper
 from cloudinit import util
 from cloudinit import net
+from cloudinit.event import EventScope, EventType
 from cloudinit.net.dhcp import EphemeralDHCPv4, NoDHCPLeaseError
-from cloudinit.event import EventType
 
 LOG = logging.getLogger(__name__)
 
@@ -172,7 +172,13 @@ def query_data_api(api_type, api_address, retries, timeout):
 
 class DataSourceScaleway(sources.DataSource):
     dsname = "Scaleway"
-    update_events = {'network': [EventType.BOOT_NEW_INSTANCE, EventType.BOOT]}
+    default_update_events = {
+        EventScope.NETWORK: {
+            EventType.BOOT_NEW_INSTANCE,
+            EventType.BOOT,
+            EventType.BOOT_LEGACY
+        }
+    }
 
     def __init__(self, sys_cfg, distro, paths):
         super(DataSourceScaleway, self).__init__(sys_cfg, distro, paths)
diff --git a/cloudinit/sources/DataSourceSmartOS.py b/cloudinit/sources/DataSourceSmartOS.py
index fd292baa..9b16bf8d 100644
--- a/cloudinit/sources/DataSourceSmartOS.py
+++ b/cloudinit/sources/DataSourceSmartOS.py
@@ -36,7 +36,7 @@ from cloudinit import serial
 from cloudinit import sources
 from cloudinit import subp
 from cloudinit import util
-from cloudinit.event import EventType
+from cloudinit.event import EventScope, EventType
 
 LOG = logging.getLogger(__name__)
 
@@ -170,6 +170,11 @@ class DataSourceSmartOS(sources.DataSource):
 
     smartos_type = sources.UNSET
     md_client = sources.UNSET
+    default_update_events = {EventScope.NETWORK: {
+        EventType.BOOT_NEW_INSTANCE,
+        EventType.BOOT,
+        EventType.BOOT_LEGACY
+    }}
 
     def __init__(self, sys_cfg, distro, paths):
         sources.DataSource.__init__(self, sys_cfg, distro, paths)
@@ -181,7 +186,6 @@ class DataSourceSmartOS(sources.DataSource):
         self.metadata = {}
         self.network_data = None
         self._network_config = None
-        self.update_events['network'].add(EventType.BOOT)
 
         self.script_base_d = os.path.join(self.paths.get_cpath("scripts"))
 
diff --git a/cloudinit/sources/__init__.py b/cloudinit/sources/__init__.py
index 7d74f8d9..a07c4b4f 100644
--- a/cloudinit/sources/__init__.py
+++ b/cloudinit/sources/__init__.py
@@ -13,6 +13,7 @@ import copy
 import json
 import os
 from collections import namedtuple
+from typing import Dict, List
 
 from cloudinit import dmi
 from cloudinit import importer
@@ -22,7 +23,7 @@ from cloudinit import type_utils
 from cloudinit import user_data as ud
 from cloudinit import util
 from cloudinit.atomic_helper import write_json
-from cloudinit.event import EventType
+from cloudinit.event import EventScope, EventType
 from cloudinit.filters import launch_index
 from cloudinit.persistence import CloudInitPickleMixin
 from cloudinit.reporting import events
@@ -175,12 +176,23 @@ class DataSource(CloudInitPickleMixin, metaclass=abc.ABCMeta):
 
     # The datasource defines a set of supported EventTypes during which
     # the datasource can react to changes in metadata and regenerate
-    # network configuration on metadata changes.
-    # A datasource which supports writing network config on each system boot
-    # would call update_events['network'].add(EventType.BOOT).
+    # network configuration on metadata changes. These are defined in
+    # `supported_network_events`.
+    # The datasource also defines a set of default EventTypes that the
+    # datasource can react to. These are the event types that will be used
+    # if not overridden by the user.
+    # A datasource requiring to write network config on each system boot
+    # would call default_update_events['network'].add(EventType.BOOT).
 
     # Default: generate network config on new instance id (first boot).
-    update_events = {'network': set([EventType.BOOT_NEW_INSTANCE])}
+    supported_update_events = {EventScope.NETWORK: {
+        EventType.BOOT_NEW_INSTANCE,
+        EventType.BOOT,
+        EventType.BOOT_LEGACY,
+    }}
+    default_update_events = {EventScope.NETWORK: {
+        EventType.BOOT_NEW_INSTANCE,
+    }}
 
     # N-tuple listing default values for any metadata-related class
     # attributes cached on an instance by a process_data runs. These attribute
@@ -648,10 +660,12 @@ class DataSource(CloudInitPickleMixin, metaclass=abc.ABCMeta):
     def get_package_mirror_info(self):
         return self.distro.get_package_mirror_info(data_source=self)
 
-    def update_metadata(self, source_event_types):
+    def update_metadata_if_supported(
+        self, source_event_types: List[EventType]
+    ) -> bool:
         """Refresh cached metadata if the datasource supports this event.
 
-        The datasource has a list of update_events which
+        The datasource has a list of supported_update_events which
         trigger refreshing all cached metadata as well as refreshing the
         network configuration.
 
@@ -661,9 +675,9 @@ class DataSource(CloudInitPickleMixin, metaclass=abc.ABCMeta):
         @return True if the datasource did successfully update cached metadata
             due to source_event_type.
         """
-        supported_events = {}
+        supported_events = {}  # type: Dict[EventScope, set]
         for event in source_event_types:
-            for update_scope, update_events in self.update_events.items():
+            for update_scope, update_events in self.supported_update_events.items():  # noqa: E501
                 if event in update_events:
                     if not supported_events.get(update_scope):
                         supported_events[update_scope] = set()
@@ -671,7 +685,8 @@ class DataSource(CloudInitPickleMixin, metaclass=abc.ABCMeta):
         for scope, matched_events in supported_events.items():
             LOG.debug(
                 "Update datasource metadata and %s config due to events: %s",
-                scope, ', '.join(matched_events))
+                scope.value,
+                ', '.join([event.value for event in matched_events]))
             # Each datasource has a cached config property which needs clearing
             # Once cleared that config property will be regenerated from
             # current metadata.
@@ -682,7 +697,7 @@ class DataSource(CloudInitPickleMixin, metaclass=abc.ABCMeta):
             if result:
                 return True
         LOG.debug("Datasource %s not updated for events: %s", self,
-                  ', '.join(source_event_types))
+                  ', '.join([event.value for event in source_event_types]))
         return False
 
     def check_instance_id(self, sys_cfg):
@@ -789,7 +804,9 @@ def find_source(sys_cfg, distro, paths, ds_deps, cfg_list, pkg_list, reporter):
             with myrep:
                 LOG.debug("Seeing if we can get any data from %s", cls)
                 s = cls(sys_cfg, distro, paths)
-                if s.update_metadata([EventType.BOOT_NEW_INSTANCE]):
+                if s.update_metadata_if_supported(
+                    [EventType.BOOT_NEW_INSTANCE]
+                ):
                     myrep.message = "found %s data from %s" % (mode, name)
                     return (s, type_utils.obj_name(cls))
         except Exception:
diff --git a/cloudinit/sources/tests/test_init.py b/cloudinit/sources/tests/test_init.py
index 1420a988..a2b052a6 100644
--- a/cloudinit/sources/tests/test_init.py
+++ b/cloudinit/sources/tests/test_init.py
@@ -5,7 +5,7 @@ import inspect
 import os
 import stat
 
-from cloudinit.event import EventType
+from cloudinit.event import EventScope, EventType
 from cloudinit.helpers import Paths
 from cloudinit import importer
 from cloudinit.sources import (
@@ -618,24 +618,29 @@ class TestDataSource(CiTestCase):
         self.assertEqual('himom', getattr(self.datasource, cached_attr_name))
         self.assertEqual('updated', self.datasource.myattr)
 
+    @mock.patch.dict(DataSource.default_update_events, {
+        EventScope.NETWORK: {EventType.BOOT_NEW_INSTANCE}})
+    @mock.patch.dict(DataSource.supported_update_events, {
+        EventScope.NETWORK: {EventType.BOOT_NEW_INSTANCE}})
     def test_update_metadata_only_acts_on_supported_update_events(self):
-        """update_metadata won't get_data on unsupported update events."""
-        self.datasource.update_events['network'].discard(EventType.BOOT)
+        """update_metadata_if_supported wont get_data on unsupported events."""
         self.assertEqual(
-            {'network': set([EventType.BOOT_NEW_INSTANCE])},
-            self.datasource.update_events)
+            {EventScope.NETWORK: set([EventType.BOOT_NEW_INSTANCE])},
+            self.datasource.default_update_events
+        )
 
         def fake_get_data():
             raise Exception('get_data should not be called')
 
         self.datasource.get_data = fake_get_data
         self.assertFalse(
-            self.datasource.update_metadata(
+            self.datasource.update_metadata_if_supported(
                 source_event_types=[EventType.BOOT]))
 
+    @mock.patch.dict(DataSource.supported_update_events, {
+        EventScope.NETWORK: {EventType.BOOT_NEW_INSTANCE}})
     def test_update_metadata_returns_true_on_supported_update_event(self):
-        """update_metadata returns get_data response on supported events."""
-
+        """update_metadata_if_supported returns get_data on supported events"""
         def fake_get_data():
             return True
 
@@ -643,14 +648,16 @@ class TestDataSource(CiTestCase):
         self.datasource._network_config = 'something'
         self.datasource._dirty_cache = True
         self.assertTrue(
-            self.datasource.update_metadata(
+            self.datasource.update_metadata_if_supported(
                 source_event_types=[
                     EventType.BOOT, EventType.BOOT_NEW_INSTANCE]))
         self.assertEqual(UNSET, self.datasource._network_config)
+
         self.assertIn(
             "DEBUG: Update datasource metadata and network config due to"
-            " events: New instance first boot",
-            self.logs.getvalue())
+            " events: boot-new-instance",
+            self.logs.getvalue()
+        )
 
 
 class TestRedactSensitiveData(CiTestCase):
diff --git a/cloudinit/stages.py b/cloudinit/stages.py
index 5bacc85d..bbded1e9 100644
--- a/cloudinit/stages.py
+++ b/cloudinit/stages.py
@@ -8,9 +8,11 @@ import copy
 import os
 import pickle
 import sys
+from collections import namedtuple
+from typing import Dict, Set
 
 from cloudinit.settings import (
-    FREQUENCIES, CLOUD_CONFIG, PER_INSTANCE, RUN_CLOUD_CONFIG)
+    FREQUENCIES, CLOUD_CONFIG, PER_INSTANCE, PER_ONCE, RUN_CLOUD_CONFIG)
 
 from cloudinit import handlers
 
@@ -21,7 +23,11 @@ from cloudinit.handlers.jinja_template import JinjaTemplatePartHandler
 from cloudinit.handlers.shell_script import ShellScriptPartHandler
 from cloudinit.handlers.upstart_job import UpstartJobPartHandler
 
-from cloudinit.event import EventType
+from cloudinit.event import (
+    EventScope,
+    EventType,
+    userdata_to_events,
+)
 from cloudinit.sources import NetworkConfigSource
 
 from cloudinit import cloud
@@ -118,6 +124,7 @@ class Init(object):
 
     def _initial_subdirs(self):
         c_dir = self.paths.cloud_dir
+        run_dir = self.paths.run_dir
         initial_dirs = [
             c_dir,
             os.path.join(c_dir, 'scripts'),
@@ -130,6 +137,7 @@ class Init(object):
             os.path.join(c_dir, 'handlers'),
             os.path.join(c_dir, 'sem'),
             os.path.join(c_dir, 'data'),
+            os.path.join(run_dir, 'sem'),
         ]
         return initial_dirs
 
@@ -341,6 +349,11 @@ class Init(object):
         return self._previous_iid
 
     def is_new_instance(self):
+        """Return true if this is a new instance.
+
+        If datasource has already been initialized, this will return False,
+        even on first boot.
+        """
         previous = self.previous_iid()
         ret = (previous == NO_PREVIOUS_INSTANCE_ID or
                previous != self.datasource.get_instance_id())
@@ -702,6 +715,46 @@ class Init(object):
         return (self.distro.generate_fallback_config(),
                 NetworkConfigSource.fallback)
 
+    def update_event_enabled(
+        self, event_source_type: EventType, scope: EventScope = None
+    ) -> bool:
+        """Determine if a particular EventType is enabled.
+
+        For the `event_source_type` passed in, check whether this EventType
+        is enabled in the `updates` section of the userdata. If `updates`
+        is not enabled in userdata, check if defined as one of the
+        `default_events` on the datasource. `scope` may be used to
+        narrow the check to a particular `EventScope`.
+
+        Note that on first boot, userdata may NOT be available yet. In this
+        case, we only have the data source's `default_update_events`,
+        so an event that should be enabled in userdata may be denied.
+        """
+        default_events = self.datasource.default_update_events  # type: Dict[EventScope, Set[EventType]]    # noqa: E501
+        user_events = userdata_to_events(self.cfg.get('updates', {}))  # type: Dict[EventScope, Set[EventType]]  # noqa: E501
+        # A value in the first will override a value in the second
+        allowed = util.mergemanydict([
+            copy.deepcopy(user_events),
+            copy.deepcopy(default_events),
+        ])
+        LOG.debug('Allowed events: %s', allowed)
+
+        if not scope:
+            scopes = allowed.keys()
+        else:
+            scopes = [scope]
+        scope_values = [s.value for s in scopes]
+
+        for evt_scope in scopes:
+            if event_source_type in allowed.get(evt_scope, []):
+                LOG.debug('Event Allowed: scope=%s EventType=%s',
+                          evt_scope.value, event_source_type)
+                return True
+
+        LOG.debug('Event Denied: scopes=%s EventType=%s',
+                  scope_values, event_source_type)
+        return False
+
     def _apply_netcfg_names(self, netcfg):
         try:
             LOG.debug("applying net config names for %s", netcfg)
@@ -709,27 +762,51 @@ class Init(object):
         except Exception as e:
             LOG.warning("Failed to rename devices: %s", e)
 
+    def _get_per_boot_network_semaphore(self):
+        return namedtuple('Semaphore', 'semaphore args')(
+            helpers.FileSemaphores(self.paths.get_runpath('sem')),
+            ('apply_network_config', PER_ONCE)
+        )
+
+    def _network_already_configured(self) -> bool:
+        sem = self._get_per_boot_network_semaphore()
+        return sem.semaphore.has_run(*sem.args)
+
     def apply_network_config(self, bring_up):
-        # get a network config
+        """Apply the network config.
+
+        Find the config, determine whether to apply it, apply it via
+        the distro, and optionally bring it up
+        """
         netcfg, src = self._find_networking_config()
         if netcfg is None:
             LOG.info("network config is disabled by %s", src)
             return
 
-        # request an update if needed/available
-        if self.datasource is not NULL_DATA_SOURCE:
-            if not self.is_new_instance():
-                if not self.datasource.update_metadata([EventType.BOOT]):
-                    LOG.debug(
-                        "No network config applied. Neither a new instance"
-                        " nor datasource network update on '%s' event",
-                        EventType.BOOT)
-                    # nothing new, but ensure proper names
-                    self._apply_netcfg_names(netcfg)
-                    return
-                else:
-                    # refresh netcfg after update
-                    netcfg, src = self._find_networking_config()
+        def event_enabled_and_metadata_updated(event_type):
+            return self.update_event_enabled(
+                event_type, scope=EventScope.NETWORK
+            ) and self.datasource.update_metadata_if_supported([event_type])
+
+        def should_run_on_boot_event():
+            return (not self._network_already_configured() and
+                    event_enabled_and_metadata_updated(EventType.BOOT))
+
+        if (
+            self.datasource is not NULL_DATA_SOURCE and
+            not self.is_new_instance() and
+            not should_run_on_boot_event() and
+            not event_enabled_and_metadata_updated(EventType.BOOT_LEGACY)
+        ):
+            LOG.debug(
+                "No network config applied. Neither a new instance"
+                " nor datasource network update allowed")
+            # nothing new, but ensure proper names
+            self._apply_netcfg_names(netcfg)
+            return
+
+        # refresh netcfg after update
+        netcfg, src = self._find_networking_config()
 
         # ensure all physical devices in config are present
         self.distro.networking.wait_for_physdevs(netcfg)
@@ -740,8 +817,12 @@ class Init(object):
         # rendering config
         LOG.info("Applying network configuration from %s bringup=%s: %s",
                  src, bring_up, netcfg)
+
+        sem = self._get_per_boot_network_semaphore()
         try:
-            return self.distro.apply_network_config(netcfg, bring_up=bring_up)
+            with sem.semaphore.lock(*sem.args):
+                return self.distro.apply_network_config(
+                    netcfg, bring_up=bring_up)
         except net.RendererNotFoundError as e:
             LOG.error("Unable to render networking. Network config is "
                       "likely broken: %s", e)
diff --git a/cloudinit/tests/test_event.py b/cloudinit/tests/test_event.py
new file mode 100644
index 00000000..3da4c70c
--- /dev/null
+++ b/cloudinit/tests/test_event.py
@@ -0,0 +1,26 @@
+# This file is part of cloud-init. See LICENSE file for license information.
+"""Tests related to cloudinit.event module."""
+from cloudinit.event import EventType, EventScope, userdata_to_events
+
+
+class TestEvent:
+    def test_userdata_to_events(self):
+        userdata = {'network': {'when': ['boot']}}
+        expected = {EventScope.NETWORK: {EventType.BOOT}}
+        assert expected == userdata_to_events(userdata)
+
+    def test_invalid_scope(self, caplog):
+        userdata = {'networkasdfasdf': {'when': ['boot']}}
+        userdata_to_events(userdata)
+        assert (
+            "'networkasdfasdf' is not a valid EventScope! Update data "
+            "will be ignored for 'networkasdfasdf' scope"
+        ) in caplog.text
+
+    def test_invalid_event(self, caplog):
+        userdata = {'network': {'when': ['bootasdfasdf']}}
+        userdata_to_events(userdata)
+        assert (
+            "'bootasdfasdf' is not a valid EventType! Update data "
+            "will be ignored for 'network' scope"
+        ) in caplog.text
diff --git a/cloudinit/tests/test_stages.py b/cloudinit/tests/test_stages.py
index d2d1b37f..a06a2bde 100644
--- a/cloudinit/tests/test_stages.py
+++ b/cloudinit/tests/test_stages.py
@@ -1,7 +1,6 @@
 # This file is part of cloud-init. See LICENSE file for license information.
 
 """Tests related to cloudinit.stages module."""
-
 import os
 import stat
 
@@ -11,7 +10,7 @@ from cloudinit import stages
 from cloudinit import sources
 from cloudinit.sources import NetworkConfigSource
 
-from cloudinit.event import EventType
+from cloudinit.event import EventScope, EventType
 from cloudinit.util import write_file
 
 from cloudinit.tests.helpers import CiTestCase, mock
@@ -52,6 +51,8 @@ class TestInit(CiTestCase):
             'distro': 'ubuntu', 'paths': {'cloud_dir': self.tmpdir,
                                           'run_dir': self.tmpdir}}}
         self.init.datasource = FakeDataSource(paths=self.init.paths)
+        self._real_is_new_instance = self.init.is_new_instance
+        self.init.is_new_instance = mock.Mock(return_value=True)
 
     def test_wb__find_networking_config_disabled(self):
         """find_networking_config returns no config when disabled."""
@@ -291,6 +292,7 @@ class TestInit(CiTestCase):
         m_macs.return_value = {'42:42:42:42:42:42': 'eth9'}
 
         self.init._find_networking_config = fake_network_config
+
         self.init.apply_network_config(True)
         self.init.distro.apply_network_config_names.assert_called_with(net_cfg)
         self.init.distro.apply_network_config.assert_called_with(
@@ -299,6 +301,7 @@ class TestInit(CiTestCase):
     @mock.patch('cloudinit.distros.ubuntu.Distro')
     def test_apply_network_on_same_instance_id(self, m_ubuntu):
         """Only call distro.apply_network_config_names on same instance id."""
+        self.init.is_new_instance = self._real_is_new_instance
         old_instance_id = os.path.join(
             self.init.paths.get_cpath('data'), 'instance-id')
         write_file(old_instance_id, TEST_INSTANCE_ID)
@@ -311,18 +314,19 @@ class TestInit(CiTestCase):
             return net_cfg, NetworkConfigSource.fallback
 
         self.init._find_networking_config = fake_network_config
+
         self.init.apply_network_config(True)
         self.init.distro.apply_network_config_names.assert_called_with(net_cfg)
         self.init.distro.apply_network_config.assert_not_called()
-        self.assertIn(
-            'No network config applied. Neither a new instance'
-            " nor datasource network update on '%s' event" % EventType.BOOT,
-            self.logs.getvalue())
-
-    @mock.patch('cloudinit.net.get_interfaces_by_mac')
-    @mock.patch('cloudinit.distros.ubuntu.Distro')
-    def test_apply_network_on_datasource_allowed_event(self, m_ubuntu, m_macs):
-        """Apply network if datasource.update_metadata permits BOOT event."""
+        assert (
+            "No network config applied. Neither a new instance nor datasource "
+            "network update allowed"
+        ) in self.logs.getvalue()
+
+    # CiTestCase doesn't work with pytest.mark.parametrize, and moving this
+    # functionality to a separate class is more cumbersome than it'd be worth
+    # at the moment, so use this as a simple setup
+    def _apply_network_setup(self, m_macs):
         old_instance_id = os.path.join(
             self.init.paths.get_cpath('data'), 'instance-id')
         write_file(old_instance_id, TEST_INSTANCE_ID)
@@ -338,12 +342,80 @@ class TestInit(CiTestCase):
 
         self.init._find_networking_config = fake_network_config
         self.init.datasource = FakeDataSource(paths=self.init.paths)
-        self.init.datasource.update_events = {'network': [EventType.BOOT]}
+        self.init.is_new_instance = mock.Mock(return_value=False)
+        return net_cfg
+
+    @mock.patch('cloudinit.net.get_interfaces_by_mac')
+    @mock.patch('cloudinit.distros.ubuntu.Distro')
+    @mock.patch.dict(sources.DataSource.default_update_events, {
+        EventScope.NETWORK: {EventType.BOOT_NEW_INSTANCE, EventType.BOOT}})
+    def test_apply_network_allowed_when_default_boot(
+        self, m_ubuntu, m_macs
+    ):
+        """Apply network if datasource permits BOOT event."""
+        net_cfg = self._apply_network_setup(m_macs)
+
         self.init.apply_network_config(True)
-        self.init.distro.apply_network_config_names.assert_called_with(net_cfg)
+        assert mock.call(
+            net_cfg
+        ) == self.init.distro.apply_network_config_names.call_args_list[-1]
+        assert mock.call(
+            net_cfg, bring_up=True
+        ) == self.init.distro.apply_network_config.call_args_list[-1]
+
+    @mock.patch('cloudinit.net.get_interfaces_by_mac')
+    @mock.patch('cloudinit.distros.ubuntu.Distro')
+    @mock.patch.dict(sources.DataSource.default_update_events, {
+        EventScope.NETWORK: {EventType.BOOT_NEW_INSTANCE}})
+    def test_apply_network_disabled_when_no_default_boot(
+        self, m_ubuntu, m_macs
+    ):
+        """Don't apply network if datasource has no BOOT event."""
+        self._apply_network_setup(m_macs)
+        self.init.apply_network_config(True)
+        self.init.distro.apply_network_config.assert_not_called()
+        assert (
+            "No network config applied. Neither a new instance nor datasource "
+            "network update allowed"
+        ) in self.logs.getvalue()
+
+    @mock.patch('cloudinit.net.get_interfaces_by_mac')
+    @mock.patch('cloudinit.distros.ubuntu.Distro')
+    @mock.patch.dict(sources.DataSource.default_update_events, {
+        EventScope.NETWORK: {EventType.BOOT_NEW_INSTANCE}})
+    def test_apply_network_allowed_with_userdata_overrides(
+        self, m_ubuntu, m_macs
+    ):
+        """Apply network if userdata overrides default config"""
+        net_cfg = self._apply_network_setup(m_macs)
+        self.init._cfg = {'updates': {'network': {'when': ['boot']}}}
+        self.init.apply_network_config(True)
+        self.init.distro.apply_network_config_names.assert_called_with(
+            net_cfg)
         self.init.distro.apply_network_config.assert_called_with(
             net_cfg, bring_up=True)
 
+    @mock.patch('cloudinit.net.get_interfaces_by_mac')
+    @mock.patch('cloudinit.distros.ubuntu.Distro')
+    @mock.patch.dict(sources.DataSource.supported_update_events, {
+        EventScope.NETWORK: {EventType.BOOT_NEW_INSTANCE}})
+    def test_apply_network_disabled_when_unsupported(
+        self, m_ubuntu, m_macs
+    ):
+        """Don't apply network config if unsupported.
+
+        Shouldn't work even when specified as userdata
+        """
+        self._apply_network_setup(m_macs)
+
+        self.init._cfg = {'updates': {'network': {'when': ['boot']}}}
+        self.init.apply_network_config(True)
+        self.init.distro.apply_network_config.assert_not_called()
+        assert (
+            "No network config applied. Neither a new instance nor datasource "
+            "network update allowed"
+        ) in self.logs.getvalue()
+
 
 class TestInit_InitializeFilesystem:
     """Tests for cloudinit.stages.Init._initialize_filesystem.
diff --git a/doc/rtd/index.rst b/doc/rtd/index.rst
index 10e8228f..33c6b56a 100644
--- a/doc/rtd/index.rst
+++ b/doc/rtd/index.rst
@@ -49,6 +49,7 @@ Having trouble? We would like to help!
 
    topics/format.rst
    topics/examples.rst
+   topics/events.rst
    topics/modules.rst
    topics/merging.rst
 
diff --git a/doc/rtd/topics/events.rst b/doc/rtd/topics/events.rst
new file mode 100644
index 00000000..463208cc
--- /dev/null
+++ b/doc/rtd/topics/events.rst
@@ -0,0 +1,83 @@
+.. _events:
+
+******************
+Events and Updates
+******************
+
+Events
+======
+
+`Cloud-init`_ will fetch and apply cloud and user data configuration
+upon several event types. The two most common events for cloud-init
+are when an instance first boots and any subsequent boot thereafter (reboot).
+In addition to boot events, cloud-init users and vendors are interested
+in when devices are added. cloud-init currently supports the following
+event types:
+
+- **BOOT_NEW_INSTANCE**: New instance first boot
+- **BOOT**: Any system boot other than 'BOOT_NEW_INSTANCE'
+- **BOOT_LEGACY**: Similar to 'BOOT', but applies networking config twice each
+  boot: once during Local stage, then again in Network stage. As this behavior
+  was previously the default behavior, this option exists to prevent regressing
+  such behavior.
+
+Future work will likely include infrastructure and support for the following
+events:
+
+- **HOTPLUG**: Dynamic add of a system device
+- **METADATA_CHANGE**: An instance's metadata has change
+- **USER_REQUEST**: Directed request to update
+
+Datasource Event Support
+========================
+
+All :ref:`datasources` by default support the ``BOOT_NEW_INSTANCE`` event.
+Each Datasource will declare a set of these events that it is capable of
+handling. Datasources may not support all event types. In some cases a system
+may be configured to allow a particular event but may be running on
+a platform whose datasource cannot support the event.
+
+Configuring Event Updates
+=========================
+
+Update configuration may be specified via user data,
+which can be used to enable or disable handling of specific events.
+This configuration will be honored as long as the events are supported by
+the datasource. However, configuration will always be applied at first
+boot, regardless of the user data specified.
+
+Updates
+~~~~~~~
+Update policy configuration defines which
+events are allowed to be handled. This is separate from whether a
+particular platform or datasource has the capability for such events.
+
+**scope**: *<name of the scope for event policy>*
+
+The ``scope`` value is a string which defines under which domain does the
+event occur. Currently the only one known scope is ``network``, though more
+scopes may be added in the future. Scopes are defined by convention but
+arbitrary values can be used.
+
+**when**: *<list of events to handle for a particular scope>*
+
+Each ``scope`` requires a ``when`` element to specify which events
+are to allowed to be handled.
+
+
+Examples
+========
+
+apply network config every boot
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+On every boot, apply network configuration found in the datasource.
+
+.. code-block:: shell-session
+
+ # apply network config on every boot
+ updates:
+   network:
+     when: ['boot']
+
+.. _Cloud-init: https://launchpad.net/cloud-init
+.. vi: textwidth=78
diff --git a/tests/integration_tests/modules/test_user_events.py b/tests/integration_tests/modules/test_user_events.py
new file mode 100644
index 00000000..a45cad72
--- /dev/null
+++ b/tests/integration_tests/modules/test_user_events.py
@@ -0,0 +1,95 @@
+"""Test user-overridable events.
+
+This is currently limited to applying network config on BOOT events.
+"""
+
+import pytest
+import re
+import yaml
+
+from tests.integration_tests.instances import IntegrationInstance
+
+
+def _add_dummy_bridge_to_netplan(client: IntegrationInstance):
+    # Update netplan configuration to ensure it doesn't change on reboot
+    netplan = yaml.safe_load(
+        client.execute('cat /etc/netplan/50-cloud-init.yaml')
+    )
+    # Just a dummy bridge to do nothing
+    try:
+        netplan['network']['bridges']['dummy0'] = {'dhcp4': False}
+    except KeyError:
+        netplan['network']['bridges'] = {'dummy0': {'dhcp4': False}}
+
+    dumped_netplan = yaml.dump(netplan)
+    client.write_to_file('/etc/netplan/50-cloud-init.yaml', dumped_netplan)
+
+
+@pytest.mark.lxd_container
+@pytest.mark.lxd_vm
+@pytest.mark.ec2
+@pytest.mark.gce
+@pytest.mark.oci
+@pytest.mark.openstack
+@pytest.mark.not_xenial
+def test_boot_event_disabled_by_default(client: IntegrationInstance):
+    log = client.read_from_file('/var/log/cloud-init.log')
+    assert 'Applying network configuration' in log
+    assert 'dummy0' not in client.execute('ls /sys/class/net')
+
+    _add_dummy_bridge_to_netplan(client)
+    client.execute('rm /var/log/cloud-init.log')
+
+    client.restart()
+    log2 = client.read_from_file('/var/log/cloud-init.log')
+
+    # We attempt to apply network config twice on every boot.
+    # Ensure neither time works.
+    assert 2 == len(
+        re.findall(r"Event Denied: scopes=\['network'\] EventType=boot[^-]",
+                   log2)
+    )
+    assert 2 == log2.count(
+        "Event Denied: scopes=['network'] EventType=boot-legacy"
+    )
+    assert 2 == log2.count(
+        "No network config applied. Neither a new instance"
+        " nor datasource network update allowed"
+    )
+
+    assert 'dummy0' in client.execute('ls /sys/class/net')
+
+
+def _test_network_config_applied_on_reboot(client: IntegrationInstance):
+    log = client.read_from_file('/var/log/cloud-init.log')
+    assert 'Applying network configuration' in log
+    assert 'dummy0' not in client.execute('ls /sys/class/net')
+
+    _add_dummy_bridge_to_netplan(client)
+    client.execute('rm /var/log/cloud-init.log')
+    client.restart()
+    log = client.read_from_file('/var/log/cloud-init.log')
+
+    assert 'Event Allowed: scope=network EventType=boot' in log
+    assert 'Applying network configuration' in log
+    assert 'dummy0' not in client.execute('ls /sys/class/net')
+
+
+@pytest.mark.azure
+@pytest.mark.not_xenial
+def test_boot_event_enabled_by_default(client: IntegrationInstance):
+    _test_network_config_applied_on_reboot(client)
+
+
+USER_DATA = """\
+#cloud-config
+updates:
+  network:
+    when: [boot]
+"""
+
+
+@pytest.mark.not_xenial
+@pytest.mark.user_data(USER_DATA)
+def test_boot_event_enabled(client: IntegrationInstance):
+    _test_network_config_applied_on_reboot(client)
diff --git a/tests/unittests/test_datasource/test_azure.py b/tests/unittests/test_datasource/test_azure.py
index 742d1faa..54e06119 100644
--- a/tests/unittests/test_datasource/test_azure.py
+++ b/tests/unittests/test_datasource/test_azure.py
@@ -3163,8 +3163,8 @@ class TestRemoveUbuntuNetworkConfigScripts(CiTestCase):
 
         expected_logs = [
             'INFO: Removing Ubuntu extended network scripts because cloud-init'
-            ' updates Azure network configuration on the following event:'
-            ' System boot.',
+            ' updates Azure network configuration on the following events:'
+            " ['boot', 'boot-legacy']",
             'Recursively deleting %s' % subdir,
             'Attempting to remove %s' % file1]
         for log in expected_logs:
diff --git a/tests/unittests/test_datasource/test_smartos.py b/tests/unittests/test_datasource/test_smartos.py
index 5847a384..9c499672 100644
--- a/tests/unittests/test_datasource/test_smartos.py
+++ b/tests/unittests/test_datasource/test_smartos.py
@@ -29,7 +29,7 @@ from cloudinit.sources.DataSourceSmartOS import (
     convert_smartos_network_data as convert_net,
     SMARTOS_ENV_KVM, SERIAL_DEVICE, get_smartos_environ,
     identify_file)
-from cloudinit.event import EventType
+from cloudinit.event import EventScope, EventType
 
 from cloudinit import helpers as c_helpers
 from cloudinit.util import (b64e, write_file)
@@ -653,8 +653,12 @@ class TestSmartOSDataSource(FilesystemMockingTestCase):
     def test_reconfig_network_on_boot(self):
         # Test to ensure that network is configured from metadata on each boot
         dsrc = self._get_ds(mockdata=MOCK_RETURNS)
-        self.assertSetEqual(set([EventType.BOOT_NEW_INSTANCE, EventType.BOOT]),
-                            dsrc.update_events['network'])
+        self.assertSetEqual(
+            {EventType.BOOT_NEW_INSTANCE,
+             EventType.BOOT,
+             EventType.BOOT_LEGACY},
+            dsrc.default_update_events[EventScope.NETWORK]
+        )
 
 
 class TestIdentifyFile(CiTestCase):
diff --git a/tox.ini b/tox.ini
index bf8cb78b..a2981b98 100644
--- a/tox.ini
+++ b/tox.ini
@@ -174,7 +174,7 @@ markers =
     gce: test will only run on GCE platform
     azure: test will only run on Azure platform
     oci: test will only run on OCI platform
-    openstack: test will only run on openstack
+    openstack: test will only run on openstack platform
     lxd_config_dict: set the config_dict passed on LXD instance creation
     lxd_container: test will only run in LXD container
     lxd_use_exec: `execute` will use `lxc exec` instead of SSH