diff options
| -rw-r--r-- | cloudinit/event.py | 69 | ||||
| -rwxr-xr-x | cloudinit/sources/DataSourceAzure.py | 15 | ||||
| -rw-r--r-- | cloudinit/sources/DataSourceEc2.py | 10 | ||||
| -rw-r--r-- | cloudinit/sources/DataSourceRbxCloud.py | 9 | ||||
| -rw-r--r-- | cloudinit/sources/DataSourceScaleway.py | 10 | ||||
| -rw-r--r-- | cloudinit/sources/DataSourceSmartOS.py | 8 | ||||
| -rw-r--r-- | cloudinit/sources/__init__.py | 41 | ||||
| -rw-r--r-- | cloudinit/sources/tests/test_init.py | 29 | ||||
| -rw-r--r-- | cloudinit/stages.py | 117 | ||||
| -rw-r--r-- | cloudinit/tests/test_event.py | 26 | ||||
| -rw-r--r-- | cloudinit/tests/test_stages.py | 98 | ||||
| -rw-r--r-- | doc/rtd/index.rst | 1 | ||||
| -rw-r--r-- | doc/rtd/topics/events.rst | 83 | ||||
| -rw-r--r-- | tests/integration_tests/modules/test_user_events.py | 95 | ||||
| -rw-r--r-- | tests/unittests/test_datasource/test_azure.py | 4 | ||||
| -rw-r--r-- | tests/unittests/test_datasource/test_smartos.py | 10 | ||||
| -rw-r--r-- | tox.ini | 2 |
17 files changed, 545 insertions, 82 deletions
diff --git a/cloudinit/event.py b/cloudinit/event.py index f7b311fb..76a0afc6 100644 --- a/cloudinit/event.py +++ b/cloudinit/event.py @@ -1,17 +1,72 @@ # This file is part of cloud-init. See LICENSE file for license information. - """Classes and functions related to event handling.""" +from enum import Enum +from typing import Dict, Set + +from cloudinit import log as logging + +LOG = logging.getLogger(__name__) + -# Event types which can generate maintenance requests for cloud-init. -class EventType(object): - BOOT = "System boot" - BOOT_NEW_INSTANCE = "New instance first boot" +class EventScope(Enum): + # NETWORK is currently the only scope, but we want to leave room to + # grow other scopes (e.g., STORAGE) without having to make breaking + # changes to the user config + NETWORK = 'network' - # TODO: Cloud-init will grow support for the follow event types: - # UDEV + def __str__(self): # pylint: disable=invalid-str-returned + return self.value + + +class EventType(Enum): + """Event types which can generate maintenance requests for cloud-init.""" + # Cloud-init should grow support for the follow event types: + # HOTPLUG # METADATA_CHANGE # USER_REQUEST + BOOT = "boot" + BOOT_NEW_INSTANCE = "boot-new-instance" + BOOT_LEGACY = "boot-legacy" + + def __str__(self): # pylint: disable=invalid-str-returned + return self.value + + +def userdata_to_events(user_config: dict) -> Dict[EventScope, Set[EventType]]: + """Convert userdata into update config format defined on datasource. + + Userdata is in the form of (e.g): + {'network': {'when': ['boot']}} + + DataSource config is in the form of: + {EventScope.Network: {EventType.BOOT}} + + Take the first and return the second + """ + update_config = {} + for scope, scope_list in user_config.items(): + try: + new_scope = EventScope(scope) + except ValueError as e: + LOG.warning( + "%s! Update data will be ignored for '%s' scope", + str(e), + scope, + ) + continue + try: + new_values = [EventType(x) for x in scope_list['when']] + except ValueError as e: + LOG.warning( + "%s! Update data will be ignored for '%s' scope", + str(e), + scope, + ) + new_values = [] + update_config[new_scope] = set(new_values) + + return update_config # vi: ts=4 expandtab diff --git a/cloudinit/sources/DataSourceAzure.py b/cloudinit/sources/DataSourceAzure.py index 2f3390c3..dcdf9f8f 100755 --- a/cloudinit/sources/DataSourceAzure.py +++ b/cloudinit/sources/DataSourceAzure.py @@ -22,7 +22,7 @@ import requests from cloudinit import dmi from cloudinit import log as logging from cloudinit import net -from cloudinit.event import EventType +from cloudinit.event import EventScope, EventType from cloudinit.net import device_driver from cloudinit.net.dhcp import EphemeralDHCPv4 from cloudinit import sources @@ -338,6 +338,13 @@ def temporary_hostname(temp_hostname, cfg, hostname_command='hostname'): class DataSourceAzure(sources.DataSource): dsname = 'Azure' + # Regenerate network config new_instance boot and every boot + default_update_events = {EventScope.NETWORK: { + EventType.BOOT_NEW_INSTANCE, + EventType.BOOT, + EventType.BOOT_LEGACY + }} + _negotiated = False _metadata_imds = sources.UNSET _ci_pkl_version = 1 @@ -352,8 +359,6 @@ class DataSourceAzure(sources.DataSource): BUILTIN_DS_CONFIG]) self.dhclient_lease_file = self.ds_cfg.get('dhclient_lease_file') self._network_config = None - # Regenerate network config new_instance boot and every boot - self.update_events['network'].add(EventType.BOOT) self._ephemeral_dhcp_ctx = None self.failed_desired_api_version = False self.iso_dev = None @@ -2309,8 +2314,8 @@ def maybe_remove_ubuntu_network_config_scripts(paths=None): LOG.info( 'Removing Ubuntu extended network scripts because' ' cloud-init updates Azure network configuration on the' - ' following event: %s.', - EventType.BOOT) + ' following events: %s.', + [EventType.BOOT.value, EventType.BOOT_LEGACY.value]) logged = True if os.path.isdir(path): util.del_dir(path) diff --git a/cloudinit/sources/DataSourceEc2.py b/cloudinit/sources/DataSourceEc2.py index a2105dc7..8a7f7c60 100644 --- a/cloudinit/sources/DataSourceEc2.py +++ b/cloudinit/sources/DataSourceEc2.py @@ -8,6 +8,7 @@ # # This file is part of cloud-init. See LICENSE file for license information. +import copy import os import time @@ -20,7 +21,7 @@ from cloudinit import sources from cloudinit import url_helper as uhelp from cloudinit import util from cloudinit import warnings -from cloudinit.event import EventType +from cloudinit.event import EventScope, EventType LOG = logging.getLogger(__name__) @@ -426,7 +427,12 @@ class DataSourceEc2(sources.DataSource): # Non-VPC (aka Classic) Ec2 instances need to rewrite the # network config file every boot due to MAC address change. if self.is_classic_instance(): - self.update_events['network'].add(EventType.BOOT) + self.default_update_events = copy.deepcopy( + self.default_update_events) + self.default_update_events[EventScope.NETWORK].add( + EventType.BOOT) + self.default_update_events[EventScope.NETWORK].add( + EventType.BOOT_LEGACY) else: LOG.warning("Metadata 'network' key not valid: %s.", net_md) self._network_config = result diff --git a/cloudinit/sources/DataSourceRbxCloud.py b/cloudinit/sources/DataSourceRbxCloud.py index 0b8994bf..bb69e998 100644 --- a/cloudinit/sources/DataSourceRbxCloud.py +++ b/cloudinit/sources/DataSourceRbxCloud.py @@ -17,7 +17,7 @@ from cloudinit import log as logging from cloudinit import sources from cloudinit import subp from cloudinit import util -from cloudinit.event import EventType +from cloudinit.event import EventScope, EventType LOG = logging.getLogger(__name__) ETC_HOSTS = '/etc/hosts' @@ -206,10 +206,11 @@ def read_user_data_callback(mount_dir): class DataSourceRbxCloud(sources.DataSource): dsname = "RbxCloud" - update_events = {'network': [ + default_update_events = {EventScope.NETWORK: { EventType.BOOT_NEW_INSTANCE, - EventType.BOOT - ]} + EventType.BOOT, + EventType.BOOT_LEGACY + }} def __init__(self, sys_cfg, distro, paths): sources.DataSource.__init__(self, sys_cfg, distro, paths) diff --git a/cloudinit/sources/DataSourceScaleway.py b/cloudinit/sources/DataSourceScaleway.py index 41be7665..7b8974a2 100644 --- a/cloudinit/sources/DataSourceScaleway.py +++ b/cloudinit/sources/DataSourceScaleway.py @@ -31,8 +31,8 @@ from cloudinit import sources from cloudinit import url_helper from cloudinit import util from cloudinit import net +from cloudinit.event import EventScope, EventType from cloudinit.net.dhcp import EphemeralDHCPv4, NoDHCPLeaseError -from cloudinit.event import EventType LOG = logging.getLogger(__name__) @@ -172,7 +172,13 @@ def query_data_api(api_type, api_address, retries, timeout): class DataSourceScaleway(sources.DataSource): dsname = "Scaleway" - update_events = {'network': [EventType.BOOT_NEW_INSTANCE, EventType.BOOT]} + default_update_events = { + EventScope.NETWORK: { + EventType.BOOT_NEW_INSTANCE, + EventType.BOOT, + EventType.BOOT_LEGACY + } + } def __init__(self, sys_cfg, distro, paths): super(DataSourceScaleway, self).__init__(sys_cfg, distro, paths) diff --git a/cloudinit/sources/DataSourceSmartOS.py b/cloudinit/sources/DataSourceSmartOS.py index fd292baa..9b16bf8d 100644 --- a/cloudinit/sources/DataSourceSmartOS.py +++ b/cloudinit/sources/DataSourceSmartOS.py @@ -36,7 +36,7 @@ from cloudinit import serial from cloudinit import sources from cloudinit import subp from cloudinit import util -from cloudinit.event import EventType +from cloudinit.event import EventScope, EventType LOG = logging.getLogger(__name__) @@ -170,6 +170,11 @@ class DataSourceSmartOS(sources.DataSource): smartos_type = sources.UNSET md_client = sources.UNSET + default_update_events = {EventScope.NETWORK: { + EventType.BOOT_NEW_INSTANCE, + EventType.BOOT, + EventType.BOOT_LEGACY + }} def __init__(self, sys_cfg, distro, paths): sources.DataSource.__init__(self, sys_cfg, distro, paths) @@ -181,7 +186,6 @@ class DataSourceSmartOS(sources.DataSource): self.metadata = {} self.network_data = None self._network_config = None - self.update_events['network'].add(EventType.BOOT) self.script_base_d = os.path.join(self.paths.get_cpath("scripts")) diff --git a/cloudinit/sources/__init__.py b/cloudinit/sources/__init__.py index 7d74f8d9..a07c4b4f 100644 --- a/cloudinit/sources/__init__.py +++ b/cloudinit/sources/__init__.py @@ -13,6 +13,7 @@ import copy import json import os from collections import namedtuple +from typing import Dict, List from cloudinit import dmi from cloudinit import importer @@ -22,7 +23,7 @@ from cloudinit import type_utils from cloudinit import user_data as ud from cloudinit import util from cloudinit.atomic_helper import write_json -from cloudinit.event import EventType +from cloudinit.event import EventScope, EventType from cloudinit.filters import launch_index from cloudinit.persistence import CloudInitPickleMixin from cloudinit.reporting import events @@ -175,12 +176,23 @@ class DataSource(CloudInitPickleMixin, metaclass=abc.ABCMeta): # The datasource defines a set of supported EventTypes during which # the datasource can react to changes in metadata and regenerate - # network configuration on metadata changes. - # A datasource which supports writing network config on each system boot - # would call update_events['network'].add(EventType.BOOT). + # network configuration on metadata changes. These are defined in + # `supported_network_events`. + # The datasource also defines a set of default EventTypes that the + # datasource can react to. These are the event types that will be used + # if not overridden by the user. + # A datasource requiring to write network config on each system boot + # would call default_update_events['network'].add(EventType.BOOT). # Default: generate network config on new instance id (first boot). - update_events = {'network': set([EventType.BOOT_NEW_INSTANCE])} + supported_update_events = {EventScope.NETWORK: { + EventType.BOOT_NEW_INSTANCE, + EventType.BOOT, + EventType.BOOT_LEGACY, + }} + default_update_events = {EventScope.NETWORK: { + EventType.BOOT_NEW_INSTANCE, + }} # N-tuple listing default values for any metadata-related class # attributes cached on an instance by a process_data runs. These attribute @@ -648,10 +660,12 @@ class DataSource(CloudInitPickleMixin, metaclass=abc.ABCMeta): def get_package_mirror_info(self): return self.distro.get_package_mirror_info(data_source=self) - def update_metadata(self, source_event_types): + def update_metadata_if_supported( + self, source_event_types: List[EventType] + ) -> bool: """Refresh cached metadata if the datasource supports this event. - The datasource has a list of update_events which + The datasource has a list of supported_update_events which trigger refreshing all cached metadata as well as refreshing the network configuration. @@ -661,9 +675,9 @@ class DataSource(CloudInitPickleMixin, metaclass=abc.ABCMeta): @return True if the datasource did successfully update cached metadata due to source_event_type. """ - supported_events = {} + supported_events = {} # type: Dict[EventScope, set] for event in source_event_types: - for update_scope, update_events in self.update_events.items(): + for update_scope, update_events in self.supported_update_events.items(): # noqa: E501 if event in update_events: if not supported_events.get(update_scope): supported_events[update_scope] = set() @@ -671,7 +685,8 @@ class DataSource(CloudInitPickleMixin, metaclass=abc.ABCMeta): for scope, matched_events in supported_events.items(): LOG.debug( "Update datasource metadata and %s config due to events: %s", - scope, ', '.join(matched_events)) + scope.value, + ', '.join([event.value for event in matched_events])) # Each datasource has a cached config property which needs clearing # Once cleared that config property will be regenerated from # current metadata. @@ -682,7 +697,7 @@ class DataSource(CloudInitPickleMixin, metaclass=abc.ABCMeta): if result: return True LOG.debug("Datasource %s not updated for events: %s", self, - ', '.join(source_event_types)) + ', '.join([event.value for event in source_event_types])) return False def check_instance_id(self, sys_cfg): @@ -789,7 +804,9 @@ def find_source(sys_cfg, distro, paths, ds_deps, cfg_list, pkg_list, reporter): with myrep: LOG.debug("Seeing if we can get any data from %s", cls) s = cls(sys_cfg, distro, paths) - if s.update_metadata([EventType.BOOT_NEW_INSTANCE]): + if s.update_metadata_if_supported( + [EventType.BOOT_NEW_INSTANCE] + ): myrep.message = "found %s data from %s" % (mode, name) return (s, type_utils.obj_name(cls)) except Exception: diff --git a/cloudinit/sources/tests/test_init.py b/cloudinit/sources/tests/test_init.py index 1420a988..a2b052a6 100644 --- a/cloudinit/sources/tests/test_init.py +++ b/cloudinit/sources/tests/test_init.py @@ -5,7 +5,7 @@ import inspect import os import stat -from cloudinit.event import EventType +from cloudinit.event import EventScope, EventType from cloudinit.helpers import Paths from cloudinit import importer from cloudinit.sources import ( @@ -618,24 +618,29 @@ class TestDataSource(CiTestCase): self.assertEqual('himom', getattr(self.datasource, cached_attr_name)) self.assertEqual('updated', self.datasource.myattr) + @mock.patch.dict(DataSource.default_update_events, { + EventScope.NETWORK: {EventType.BOOT_NEW_INSTANCE}}) + @mock.patch.dict(DataSource.supported_update_events, { + EventScope.NETWORK: {EventType.BOOT_NEW_INSTANCE}}) def test_update_metadata_only_acts_on_supported_update_events(self): - """update_metadata won't get_data on unsupported update events.""" - self.datasource.update_events['network'].discard(EventType.BOOT) + """update_metadata_if_supported wont get_data on unsupported events.""" self.assertEqual( - {'network': set([EventType.BOOT_NEW_INSTANCE])}, - self.datasource.update_events) + {EventScope.NETWORK: set([EventType.BOOT_NEW_INSTANCE])}, + self.datasource.default_update_events + ) def fake_get_data(): raise Exception('get_data should not be called') self.datasource.get_data = fake_get_data self.assertFalse( - self.datasource.update_metadata( + self.datasource.update_metadata_if_supported( source_event_types=[EventType.BOOT])) + @mock.patch.dict(DataSource.supported_update_events, { + EventScope.NETWORK: {EventType.BOOT_NEW_INSTANCE}}) def test_update_metadata_returns_true_on_supported_update_event(self): - """update_metadata returns get_data response on supported events.""" - + """update_metadata_if_supported returns get_data on supported events""" def fake_get_data(): return True @@ -643,14 +648,16 @@ class TestDataSource(CiTestCase): self.datasource._network_config = 'something' self.datasource._dirty_cache = True self.assertTrue( - self.datasource.update_metadata( + self.datasource.update_metadata_if_supported( source_event_types=[ EventType.BOOT, EventType.BOOT_NEW_INSTANCE])) self.assertEqual(UNSET, self.datasource._network_config) + self.assertIn( "DEBUG: Update datasource metadata and network config due to" - " events: New instance first boot", - self.logs.getvalue()) + " events: boot-new-instance", + self.logs.getvalue() + ) class TestRedactSensitiveData(CiTestCase): diff --git a/cloudinit/stages.py b/cloudinit/stages.py index 5bacc85d..bbded1e9 100644 --- a/cloudinit/stages.py +++ b/cloudinit/stages.py @@ -8,9 +8,11 @@ import copy import os import pickle import sys +from collections import namedtuple +from typing import Dict, Set from cloudinit.settings import ( - FREQUENCIES, CLOUD_CONFIG, PER_INSTANCE, RUN_CLOUD_CONFIG) + FREQUENCIES, CLOUD_CONFIG, PER_INSTANCE, PER_ONCE, RUN_CLOUD_CONFIG) from cloudinit import handlers @@ -21,7 +23,11 @@ from cloudinit.handlers.jinja_template import JinjaTemplatePartHandler from cloudinit.handlers.shell_script import ShellScriptPartHandler from cloudinit.handlers.upstart_job import UpstartJobPartHandler -from cloudinit.event import EventType +from cloudinit.event import ( + EventScope, + EventType, + userdata_to_events, +) from cloudinit.sources import NetworkConfigSource from cloudinit import cloud @@ -118,6 +124,7 @@ class Init(object): def _initial_subdirs(self): c_dir = self.paths.cloud_dir + run_dir = self.paths.run_dir initial_dirs = [ c_dir, os.path.join(c_dir, 'scripts'), @@ -130,6 +137,7 @@ class Init(object): os.path.join(c_dir, 'handlers'), os.path.join(c_dir, 'sem'), os.path.join(c_dir, 'data'), + os.path.join(run_dir, 'sem'), ] return initial_dirs @@ -341,6 +349,11 @@ class Init(object): return self._previous_iid def is_new_instance(self): + """Return true if this is a new instance. + + If datasource has already been initialized, this will return False, + even on first boot. + """ previous = self.previous_iid() ret = (previous == NO_PREVIOUS_INSTANCE_ID or previous != self.datasource.get_instance_id()) @@ -702,6 +715,46 @@ class Init(object): return (self.distro.generate_fallback_config(), NetworkConfigSource.fallback) + def update_event_enabled( + self, event_source_type: EventType, scope: EventScope = None + ) -> bool: + """Determine if a particular EventType is enabled. + + For the `event_source_type` passed in, check whether this EventType + is enabled in the `updates` section of the userdata. If `updates` + is not enabled in userdata, check if defined as one of the + `default_events` on the datasource. `scope` may be used to + narrow the check to a particular `EventScope`. + + Note that on first boot, userdata may NOT be available yet. In this + case, we only have the data source's `default_update_events`, + so an event that should be enabled in userdata may be denied. + """ + default_events = self.datasource.default_update_events # type: Dict[EventScope, Set[EventType]] # noqa: E501 + user_events = userdata_to_events(self.cfg.get('updates', {})) # type: Dict[EventScope, Set[EventType]] # noqa: E501 + # A value in the first will override a value in the second + allowed = util.mergemanydict([ + copy.deepcopy(user_events), + copy.deepcopy(default_events), + ]) + LOG.debug('Allowed events: %s', allowed) + + if not scope: + scopes = allowed.keys() + else: + scopes = [scope] + scope_values = [s.value for s in scopes] + + for evt_scope in scopes: + if event_source_type in allowed.get(evt_scope, []): + LOG.debug('Event Allowed: scope=%s EventType=%s', + evt_scope.value, event_source_type) + return True + + LOG.debug('Event Denied: scopes=%s EventType=%s', + scope_values, event_source_type) + return False + def _apply_netcfg_names(self, netcfg): try: LOG.debug("applying net config names for %s", netcfg) @@ -709,27 +762,51 @@ class Init(object): except Exception as e: LOG.warning("Failed to rename devices: %s", e) + def _get_per_boot_network_semaphore(self): + return namedtuple('Semaphore', 'semaphore args')( + helpers.FileSemaphores(self.paths.get_runpath('sem')), + ('apply_network_config', PER_ONCE) + ) + + def _network_already_configured(self) -> bool: + sem = self._get_per_boot_network_semaphore() + return sem.semaphore.has_run(*sem.args) + def apply_network_config(self, bring_up): - # get a network config + """Apply the network config. + + Find the config, determine whether to apply it, apply it via + the distro, and optionally bring it up + """ netcfg, src = self._find_networking_config() if netcfg is None: LOG.info("network config is disabled by %s", src) return - # request an update if needed/available - if self.datasource is not NULL_DATA_SOURCE: - if not self.is_new_instance(): - if not self.datasource.update_metadata([EventType.BOOT]): - LOG.debug( - "No network config applied. Neither a new instance" - " nor datasource network update on '%s' event", - EventType.BOOT) - # nothing new, but ensure proper names - self._apply_netcfg_names(netcfg) - return - else: - # refresh netcfg after update - netcfg, src = self._find_networking_config() + def event_enabled_and_metadata_updated(event_type): + return self.update_event_enabled( + event_type, scope=EventScope.NETWORK + ) and self.datasource.update_metadata_if_supported([event_type]) + + def should_run_on_boot_event(): + return (not self._network_already_configured() and + event_enabled_and_metadata_updated(EventType.BOOT)) + + if ( + self.datasource is not NULL_DATA_SOURCE and + not self.is_new_instance() and + not should_run_on_boot_event() and + not event_enabled_and_metadata_updated(EventType.BOOT_LEGACY) + ): + LOG.debug( + "No network config applied. Neither a new instance" + " nor datasource network update allowed") + # nothing new, but ensure proper names + self._apply_netcfg_names(netcfg) + return + + # refresh netcfg after update + netcfg, src = self._find_networking_config() # ensure all physical devices in config are present self.distro.networking.wait_for_physdevs(netcfg) @@ -740,8 +817,12 @@ class Init(object): # rendering config LOG.info("Applying network configuration from %s bringup=%s: %s", src, bring_up, netcfg) + + sem = self._get_per_boot_network_semaphore() try: - return self.distro.apply_network_config(netcfg, bring_up=bring_up) + with sem.semaphore.lock(*sem.args): + return self.distro.apply_network_config( + netcfg, bring_up=bring_up) except net.RendererNotFoundError as e: LOG.error("Unable to render networking. Network config is " "likely broken: %s", e) diff --git a/cloudinit/tests/test_event.py b/cloudinit/tests/test_event.py new file mode 100644 index 00000000..3da4c70c --- /dev/null +++ b/cloudinit/tests/test_event.py @@ -0,0 +1,26 @@ +# This file is part of cloud-init. See LICENSE file for license information. +"""Tests related to cloudinit.event module.""" +from cloudinit.event import EventType, EventScope, userdata_to_events + + +class TestEvent: + def test_userdata_to_events(self): + userdata = {'network': {'when': ['boot']}} + expected = {EventScope.NETWORK: {EventType.BOOT}} + assert expected == userdata_to_events(userdata) + + def test_invalid_scope(self, caplog): + userdata = {'networkasdfasdf': {'when': ['boot']}} + userdata_to_events(userdata) + assert ( + "'networkasdfasdf' is not a valid EventScope! Update data " + "will be ignored for 'networkasdfasdf' scope" + ) in caplog.text + + def test_invalid_event(self, caplog): + userdata = {'network': {'when': ['bootasdfasdf']}} + userdata_to_events(userdata) + assert ( + "'bootasdfasdf' is not a valid EventType! Update data " + "will be ignored for 'network' scope" + ) in caplog.text diff --git a/cloudinit/tests/test_stages.py b/cloudinit/tests/test_stages.py index d2d1b37f..a06a2bde 100644 --- a/cloudinit/tests/test_stages.py +++ b/cloudinit/tests/test_stages.py @@ -1,7 +1,6 @@ # This file is part of cloud-init. See LICENSE file for license information. """Tests related to cloudinit.stages module.""" - import os import stat @@ -11,7 +10,7 @@ from cloudinit import stages from cloudinit import sources from cloudinit.sources import NetworkConfigSource -from cloudinit.event import EventType +from cloudinit.event import EventScope, EventType from cloudinit.util import write_file from cloudinit.tests.helpers import CiTestCase, mock @@ -52,6 +51,8 @@ class TestInit(CiTestCase): 'distro': 'ubuntu', 'paths': {'cloud_dir': self.tmpdir, 'run_dir': self.tmpdir}}} self.init.datasource = FakeDataSource(paths=self.init.paths) + self._real_is_new_instance = self.init.is_new_instance + self.init.is_new_instance = mock.Mock(return_value=True) def test_wb__find_networking_config_disabled(self): """find_networking_config returns no config when disabled.""" @@ -291,6 +292,7 @@ class TestInit(CiTestCase): m_macs.return_value = {'42:42:42:42:42:42': 'eth9'} self.init._find_networking_config = fake_network_config + self.init.apply_network_config(True) self.init.distro.apply_network_config_names.assert_called_with(net_cfg) self.init.distro.apply_network_config.assert_called_with( @@ -299,6 +301,7 @@ class TestInit(CiTestCase): @mock.patch('cloudinit.distros.ubuntu.Distro') def test_apply_network_on_same_instance_id(self, m_ubuntu): """Only call distro.apply_network_config_names on same instance id.""" + self.init.is_new_instance = self._real_is_new_instance old_instance_id = os.path.join( self.init.paths.get_cpath('data'), 'instance-id') write_file(old_instance_id, TEST_INSTANCE_ID) @@ -311,18 +314,19 @@ class TestInit(CiTestCase): return net_cfg, NetworkConfigSource.fallback self.init._find_networking_config = fake_network_config + self.init.apply_network_config(True) self.init.distro.apply_network_config_names.assert_called_with(net_cfg) self.init.distro.apply_network_config.assert_not_called() - self.assertIn( - 'No network config applied. Neither a new instance' - " nor datasource network update on '%s' event" % EventType.BOOT, - self.logs.getvalue()) - - @mock.patch('cloudinit.net.get_interfaces_by_mac') - @mock.patch('cloudinit.distros.ubuntu.Distro') - def test_apply_network_on_datasource_allowed_event(self, m_ubuntu, m_macs): - """Apply network if datasource.update_metadata permits BOOT event.""" + assert ( + "No network config applied. Neither a new instance nor datasource " + "network update allowed" + ) in self.logs.getvalue() + + # CiTestCase doesn't work with pytest.mark.parametrize, and moving this + # functionality to a separate class is more cumbersome than it'd be worth + # at the moment, so use this as a simple setup + def _apply_network_setup(self, m_macs): old_instance_id = os.path.join( self.init.paths.get_cpath('data'), 'instance-id') write_file(old_instance_id, TEST_INSTANCE_ID) @@ -338,12 +342,80 @@ class TestInit(CiTestCase): self.init._find_networking_config = fake_network_config self.init.datasource = FakeDataSource(paths=self.init.paths) - self.init.datasource.update_events = {'network': [EventType.BOOT]} + self.init.is_new_instance = mock.Mock(return_value=False) + return net_cfg + + @mock.patch('cloudinit.net.get_interfaces_by_mac') + @mock.patch('cloudinit.distros.ubuntu.Distro') + @mock.patch.dict(sources.DataSource.default_update_events, { + EventScope.NETWORK: {EventType.BOOT_NEW_INSTANCE, EventType.BOOT}}) + def test_apply_network_allowed_when_default_boot( + self, m_ubuntu, m_macs + ): + """Apply network if datasource permits BOOT event.""" + net_cfg = self._apply_network_setup(m_macs) + self.init.apply_network_config(True) - self.init.distro.apply_network_config_names.assert_called_with(net_cfg) + assert mock.call( + net_cfg + ) == self.init.distro.apply_network_config_names.call_args_list[-1] + assert mock.call( + net_cfg, bring_up=True + ) == self.init.distro.apply_network_config.call_args_list[-1] + + @mock.patch('cloudinit.net.get_interfaces_by_mac') + @mock.patch('cloudinit.distros.ubuntu.Distro') + @mock.patch.dict(sources.DataSource.default_update_events, { + EventScope.NETWORK: {EventType.BOOT_NEW_INSTANCE}}) + def test_apply_network_disabled_when_no_default_boot( + self, m_ubuntu, m_macs + ): + """Don't apply network if datasource has no BOOT event.""" + self._apply_network_setup(m_macs) + self.init.apply_network_config(True) + self.init.distro.apply_network_config.assert_not_called() + assert ( + "No network config applied. Neither a new instance nor datasource " + "network update allowed" + ) in self.logs.getvalue() + + @mock.patch('cloudinit.net.get_interfaces_by_mac') + @mock.patch('cloudinit.distros.ubuntu.Distro') + @mock.patch.dict(sources.DataSource.default_update_events, { + EventScope.NETWORK: {EventType.BOOT_NEW_INSTANCE}}) + def test_apply_network_allowed_with_userdata_overrides( + self, m_ubuntu, m_macs + ): + """Apply network if userdata overrides default config""" + net_cfg = self._apply_network_setup(m_macs) + self.init._cfg = {'updates': {'network': {'when': ['boot']}}} + self.init.apply_network_config(True) + self.init.distro.apply_network_config_names.assert_called_with( + net_cfg) self.init.distro.apply_network_config.assert_called_with( net_cfg, bring_up=True) + @mock.patch('cloudinit.net.get_interfaces_by_mac') + @mock.patch('cloudinit.distros.ubuntu.Distro') + @mock.patch.dict(sources.DataSource.supported_update_events, { + EventScope.NETWORK: {EventType.BOOT_NEW_INSTANCE}}) + def test_apply_network_disabled_when_unsupported( + self, m_ubuntu, m_macs + ): + """Don't apply network config if unsupported. + + Shouldn't work even when specified as userdata + """ + self._apply_network_setup(m_macs) + + self.init._cfg = {'updates': {'network': {'when': ['boot']}}} + self.init.apply_network_config(True) + self.init.distro.apply_network_config.assert_not_called() + assert ( + "No network config applied. Neither a new instance nor datasource " + "network update allowed" + ) in self.logs.getvalue() + class TestInit_InitializeFilesystem: """Tests for cloudinit.stages.Init._initialize_filesystem. diff --git a/doc/rtd/index.rst b/doc/rtd/index.rst index 10e8228f..33c6b56a 100644 --- a/doc/rtd/index.rst +++ b/doc/rtd/index.rst @@ -49,6 +49,7 @@ Having trouble? We would like to help! topics/format.rst topics/examples.rst + topics/events.rst topics/modules.rst topics/merging.rst diff --git a/doc/rtd/topics/events.rst b/doc/rtd/topics/events.rst new file mode 100644 index 00000000..463208cc --- /dev/null +++ b/doc/rtd/topics/events.rst @@ -0,0 +1,83 @@ +.. _events: + +****************** +Events and Updates +****************** + +Events +====== + +`Cloud-init`_ will fetch and apply cloud and user data configuration +upon several event types. The two most common events for cloud-init +are when an instance first boots and any subsequent boot thereafter (reboot). +In addition to boot events, cloud-init users and vendors are interested +in when devices are added. cloud-init currently supports the following +event types: + +- **BOOT_NEW_INSTANCE**: New instance first boot +- **BOOT**: Any system boot other than 'BOOT_NEW_INSTANCE' +- **BOOT_LEGACY**: Similar to 'BOOT', but applies networking config twice each + boot: once during Local stage, then again in Network stage. As this behavior + was previously the default behavior, this option exists to prevent regressing + such behavior. + +Future work will likely include infrastructure and support for the following +events: + +- **HOTPLUG**: Dynamic add of a system device +- **METADATA_CHANGE**: An instance's metadata has change +- **USER_REQUEST**: Directed request to update + +Datasource Event Support +======================== + +All :ref:`datasources` by default support the ``BOOT_NEW_INSTANCE`` event. +Each Datasource will declare a set of these events that it is capable of +handling. Datasources may not support all event types. In some cases a system +may be configured to allow a particular event but may be running on +a platform whose datasource cannot support the event. + +Configuring Event Updates +========================= + +Update configuration may be specified via user data, +which can be used to enable or disable handling of specific events. +This configuration will be honored as long as the events are supported by +the datasource. However, configuration will always be applied at first +boot, regardless of the user data specified. + +Updates +~~~~~~~ +Update policy configuration defines which +events are allowed to be handled. This is separate from whether a +particular platform or datasource has the capability for such events. + +**scope**: *<name of the scope for event policy>* + +The ``scope`` value is a string which defines under which domain does the +event occur. Currently the only one known scope is ``network``, though more +scopes may be added in the future. Scopes are defined by convention but +arbitrary values can be used. + +**when**: *<list of events to handle for a particular scope>* + +Each ``scope`` requires a ``when`` element to specify which events +are to allowed to be handled. + + +Examples +======== + +apply network config every boot +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +On every boot, apply network configuration found in the datasource. + +.. code-block:: shell-session + + # apply network config on every boot + updates: + network: + when: ['boot'] + +.. _Cloud-init: https://launchpad.net/cloud-init +.. vi: textwidth=78 diff --git a/tests/integration_tests/modules/test_user_events.py b/tests/integration_tests/modules/test_user_events.py new file mode 100644 index 00000000..a45cad72 --- /dev/null +++ b/tests/integration_tests/modules/test_user_events.py @@ -0,0 +1,95 @@ +"""Test user-overridable events. + +This is currently limited to applying network config on BOOT events. +""" + +import pytest +import re +import yaml + +from tests.integration_tests.instances import IntegrationInstance + + +def _add_dummy_bridge_to_netplan(client: IntegrationInstance): + # Update netplan configuration to ensure it doesn't change on reboot + netplan = yaml.safe_load( + client.execute('cat /etc/netplan/50-cloud-init.yaml') + ) + # Just a dummy bridge to do nothing + try: + netplan['network']['bridges']['dummy0'] = {'dhcp4': False} + except KeyError: + netplan['network']['bridges'] = {'dummy0': {'dhcp4': False}} + + dumped_netplan = yaml.dump(netplan) + client.write_to_file('/etc/netplan/50-cloud-init.yaml', dumped_netplan) + + +@pytest.mark.lxd_container +@pytest.mark.lxd_vm +@pytest.mark.ec2 +@pytest.mark.gce +@pytest.mark.oci +@pytest.mark.openstack +@pytest.mark.not_xenial +def test_boot_event_disabled_by_default(client: IntegrationInstance): + log = client.read_from_file('/var/log/cloud-init.log') + assert 'Applying network configuration' in log + assert 'dummy0' not in client.execute('ls /sys/class/net') + + _add_dummy_bridge_to_netplan(client) + client.execute('rm /var/log/cloud-init.log') + + client.restart() + log2 = client.read_from_file('/var/log/cloud-init.log') + + # We attempt to apply network config twice on every boot. + # Ensure neither time works. + assert 2 == len( + re.findall(r"Event Denied: scopes=\['network'\] EventType=boot[^-]", + log2) + ) + assert 2 == log2.count( + "Event Denied: scopes=['network'] EventType=boot-legacy" + ) + assert 2 == log2.count( + "No network config applied. Neither a new instance" + " nor datasource network update allowed" + ) + + assert 'dummy0' in client.execute('ls /sys/class/net') + + +def _test_network_config_applied_on_reboot(client: IntegrationInstance): + log = client.read_from_file('/var/log/cloud-init.log') + assert 'Applying network configuration' in log + assert 'dummy0' not in client.execute('ls /sys/class/net') + + _add_dummy_bridge_to_netplan(client) + client.execute('rm /var/log/cloud-init.log') + client.restart() + log = client.read_from_file('/var/log/cloud-init.log') + + assert 'Event Allowed: scope=network EventType=boot' in log + assert 'Applying network configuration' in log + assert 'dummy0' not in client.execute('ls /sys/class/net') + + +@pytest.mark.azure +@pytest.mark.not_xenial +def test_boot_event_enabled_by_default(client: IntegrationInstance): + _test_network_config_applied_on_reboot(client) + + +USER_DATA = """\ +#cloud-config +updates: + network: + when: [boot] +""" + + +@pytest.mark.not_xenial +@pytest.mark.user_data(USER_DATA) +def test_boot_event_enabled(client: IntegrationInstance): + _test_network_config_applied_on_reboot(client) diff --git a/tests/unittests/test_datasource/test_azure.py b/tests/unittests/test_datasource/test_azure.py index 742d1faa..54e06119 100644 --- a/tests/unittests/test_datasource/test_azure.py +++ b/tests/unittests/test_datasource/test_azure.py @@ -3163,8 +3163,8 @@ class TestRemoveUbuntuNetworkConfigScripts(CiTestCase): expected_logs = [ 'INFO: Removing Ubuntu extended network scripts because cloud-init' - ' updates Azure network configuration on the following event:' - ' System boot.', + ' updates Azure network configuration on the following events:' + " ['boot', 'boot-legacy']", 'Recursively deleting %s' % subdir, 'Attempting to remove %s' % file1] for log in expected_logs: diff --git a/tests/unittests/test_datasource/test_smartos.py b/tests/unittests/test_datasource/test_smartos.py index 5847a384..9c499672 100644 --- a/tests/unittests/test_datasource/test_smartos.py +++ b/tests/unittests/test_datasource/test_smartos.py @@ -29,7 +29,7 @@ from cloudinit.sources.DataSourceSmartOS import ( convert_smartos_network_data as convert_net, SMARTOS_ENV_KVM, SERIAL_DEVICE, get_smartos_environ, identify_file) -from cloudinit.event import EventType +from cloudinit.event import EventScope, EventType from cloudinit import helpers as c_helpers from cloudinit.util import (b64e, write_file) @@ -653,8 +653,12 @@ class TestSmartOSDataSource(FilesystemMockingTestCase): def test_reconfig_network_on_boot(self): # Test to ensure that network is configured from metadata on each boot dsrc = self._get_ds(mockdata=MOCK_RETURNS) - self.assertSetEqual(set([EventType.BOOT_NEW_INSTANCE, EventType.BOOT]), - dsrc.update_events['network']) + self.assertSetEqual( + {EventType.BOOT_NEW_INSTANCE, + EventType.BOOT, + EventType.BOOT_LEGACY}, + dsrc.default_update_events[EventScope.NETWORK] + ) class TestIdentifyFile(CiTestCase): @@ -174,7 +174,7 @@ markers = gce: test will only run on GCE platform azure: test will only run on Azure platform oci: test will only run on OCI platform - openstack: test will only run on openstack + openstack: test will only run on openstack platform lxd_config_dict: set the config_dict passed on LXD instance creation lxd_container: test will only run in LXD container lxd_use_exec: `execute` will use `lxc exec` instead of SSH |