diff options
Diffstat (limited to 'cloudinit/sources/__init__.py')
| -rw-r--r-- | cloudinit/sources/__init__.py | 471 |
1 files changed, 312 insertions, 159 deletions
diff --git a/cloudinit/sources/__init__.py b/cloudinit/sources/__init__.py index 9dccc687..88028cfa 100644 --- a/cloudinit/sources/__init__.py +++ b/cloudinit/sources/__init__.py @@ -13,17 +13,18 @@ import copy import json import os from collections import namedtuple +from typing import Dict, List # noqa: F401 -from cloudinit import dmi -from cloudinit import importer +from cloudinit import dmi, importer from cloudinit import log as logging -from cloudinit import net -from cloudinit import type_utils +from cloudinit import net, type_utils from cloudinit import user_data as ud from cloudinit import util from cloudinit.atomic_helper import write_json -from cloudinit.event import EventType +from cloudinit.distros import Distro +from cloudinit.event import EventScope, EventType from cloudinit.filters import launch_index +from cloudinit.persistence import CloudInitPickleMixin from cloudinit.reporting import events DSMODE_DISABLED = "disabled" @@ -35,42 +36,48 @@ VALID_DSMODES = [DSMODE_DISABLED, DSMODE_LOCAL, DSMODE_NETWORK] DEP_FILESYSTEM = "FILESYSTEM" DEP_NETWORK = "NETWORK" -DS_PREFIX = 'DataSource' +DS_PREFIX = "DataSource" EXPERIMENTAL_TEXT = ( "EXPERIMENTAL: The structure and format of content scoped under the 'ds'" - " key may change in subsequent releases of cloud-init.") + " key may change in subsequent releases of cloud-init." +) # File in which public available instance meta-data is written # security-sensitive key values are redacted from this world-readable file -INSTANCE_JSON_FILE = 'instance-data.json' +INSTANCE_JSON_FILE = "instance-data.json" # security-sensitive key values are present in this root-readable file -INSTANCE_JSON_SENSITIVE_FILE = 'instance-data-sensitive.json' -REDACT_SENSITIVE_VALUE = 'redacted for non-root user' +INSTANCE_JSON_SENSITIVE_FILE = "instance-data-sensitive.json" +REDACT_SENSITIVE_VALUE = "redacted for non-root user" # Key which can be provide a cloud's official product name to cloud-init -METADATA_CLOUD_NAME_KEY = 'cloud-name' +METADATA_CLOUD_NAME_KEY = "cloud-name" UNSET = "_unset" -METADATA_UNKNOWN = 'unknown' +METADATA_UNKNOWN = "unknown" LOG = logging.getLogger(__name__) # CLOUD_ID_REGION_PREFIX_MAP format is: # <region-match-prefix>: (<new-cloud-id>: <test_allowed_cloud_callable>) CLOUD_ID_REGION_PREFIX_MAP = { - 'cn-': ('aws-china', lambda c: c == 'aws'), # only change aws regions - 'us-gov-': ('aws-gov', lambda c: c == 'aws'), # only change aws regions - 'china': ('azure-china', lambda c: c == 'azure'), # only change azure + "cn-": ("aws-china", lambda c: c == "aws"), # only change aws regions + "us-gov-": ("aws-gov", lambda c: c == "aws"), # only change aws regions + "china": ("azure-china", lambda c: c == "azure"), # only change azure } # NetworkConfigSource represents the canonical list of network config sources # that cloud-init knows about. (Python 2.7 lacks PEP 435, so use a singleton # namedtuple as an enum; see https://stackoverflow.com/a/6971002) -_NETCFG_SOURCE_NAMES = ('cmdline', 'ds', 'system_cfg', 'fallback', 'initramfs') -NetworkConfigSource = namedtuple('NetworkConfigSource', - _NETCFG_SOURCE_NAMES)(*_NETCFG_SOURCE_NAMES) +_NETCFG_SOURCE_NAMES = ("cmdline", "ds", "system_cfg", "fallback", "initramfs") +NetworkConfigSource = namedtuple("NetworkConfigSource", _NETCFG_SOURCE_NAMES)( + *_NETCFG_SOURCE_NAMES +) + + +class DatasourceUnpickleUserDataError(Exception): + """Raised when userdata is unable to be unpickled due to python upgrades""" class DataSourceNotFoundException(Exception): @@ -81,7 +88,7 @@ class InvalidMetaDataException(Exception): """Raised when metadata is broken, unavailable or disabled.""" -def process_instance_metadata(metadata, key_path='', sensitive_keys=()): +def process_instance_metadata(metadata, key_path="", sensitive_keys=()): """Process all instance metadata cleaning it up for persisting as json. Strip ci-b64 prefix and catalog any 'base64_encoded_keys' as a list @@ -93,22 +100,23 @@ def process_instance_metadata(metadata, key_path='', sensitive_keys=()): sens_keys = [] for key, val in metadata.items(): if key_path: - sub_key_path = key_path + '/' + key + sub_key_path = key_path + "/" + key else: sub_key_path = key if key in sensitive_keys or sub_key_path in sensitive_keys: sens_keys.append(sub_key_path) - if isinstance(val, str) and val.startswith('ci-b64:'): + if isinstance(val, str) and val.startswith("ci-b64:"): base64_encoded_keys.append(sub_key_path) - md_copy[key] = val.replace('ci-b64:', '') + md_copy[key] = val.replace("ci-b64:", "") if isinstance(val, dict): return_val = process_instance_metadata( - val, sub_key_path, sensitive_keys) - base64_encoded_keys.extend(return_val.pop('base64_encoded_keys')) - sens_keys.extend(return_val.pop('sensitive_keys')) + val, sub_key_path, sensitive_keys + ) + base64_encoded_keys.extend(return_val.pop("base64_encoded_keys")) + sens_keys.extend(return_val.pop("sensitive_keys")) md_copy[key] = return_val - md_copy['base64_encoded_keys'] = sorted(base64_encoded_keys) - md_copy['sensitive_keys'] = sorted(sens_keys) + md_copy["base64_encoded_keys"] = sorted(base64_encoded_keys) + md_copy["sensitive_keys"] = sorted(sens_keys) return md_copy @@ -117,11 +125,11 @@ def redact_sensitive_keys(metadata, redact_value=REDACT_SENSITIVE_VALUE): Replace any keys values listed in 'sensitive_keys' with redact_value. """ - if not metadata.get('sensitive_keys', []): + if not metadata.get("sensitive_keys", []): return metadata md_copy = copy.deepcopy(metadata) - for key_path in metadata.get('sensitive_keys'): - path_parts = key_path.split('/') + for key_path in metadata.get("sensitive_keys"): + path_parts = key_path.split("/") obj = md_copy for path in path_parts: if isinstance(obj[path], dict) and path != path_parts[-1]: @@ -131,17 +139,24 @@ def redact_sensitive_keys(metadata, redact_value=REDACT_SENSITIVE_VALUE): URLParams = namedtuple( - 'URLParms', ['max_wait_seconds', 'timeout_seconds', 'num_retries']) + "URLParms", + [ + "max_wait_seconds", + "timeout_seconds", + "num_retries", + "sec_between_retries", + ], +) -class DataSource(metaclass=abc.ABCMeta): +class DataSource(CloudInitPickleMixin, metaclass=abc.ABCMeta): dsmode = DSMODE_NETWORK - default_locale = 'en_US.UTF-8' + default_locale = "en_US.UTF-8" # Datasource name needs to be set by subclasses to determine which # cloud-config datasource key is loaded - dsname = '_undef' + dsname = "_undef" # Cached cloud_name as determined by _get_cloud_name _cloud_name = None @@ -162,40 +177,71 @@ class DataSource(metaclass=abc.ABCMeta): # configuration will be used without considering any that follow.) This # should always be a subset of the members of NetworkConfigSource with no # duplicate entries. - network_config_sources = (NetworkConfigSource.cmdline, - NetworkConfigSource.initramfs, - NetworkConfigSource.system_cfg, - NetworkConfigSource.ds) + network_config_sources = ( + NetworkConfigSource.cmdline, + NetworkConfigSource.initramfs, + NetworkConfigSource.system_cfg, + NetworkConfigSource.ds, + ) # read_url_params - url_max_wait = -1 # max_wait < 0 means do not wait - url_timeout = 10 # timeout for each metadata url read attempt - url_retries = 5 # number of times to retry url upon 404 + url_max_wait = -1 # max_wait < 0 means do not wait + url_timeout = 10 # timeout for each metadata url read attempt + url_retries = 5 # number of times to retry url upon 404 + url_sec_between_retries = 1 # amount of seconds to wait between retries # The datasource defines a set of supported EventTypes during which # the datasource can react to changes in metadata and regenerate - # network configuration on metadata changes. - # A datasource which supports writing network config on each system boot - # would call update_events['network'].add(EventType.BOOT). + # network configuration on metadata changes. These are defined in + # `supported_network_events`. + # The datasource also defines a set of default EventTypes that the + # datasource can react to. These are the event types that will be used + # if not overridden by the user. + # A datasource requiring to write network config on each system boot + # would call default_update_events['network'].add(EventType.BOOT). # Default: generate network config on new instance id (first boot). - update_events = {'network': set([EventType.BOOT_NEW_INSTANCE])} + supported_update_events = { + EventScope.NETWORK: { + EventType.BOOT_NEW_INSTANCE, + EventType.BOOT, + EventType.BOOT_LEGACY, + EventType.HOTPLUG, + } + } + default_update_events = { + EventScope.NETWORK: { + EventType.BOOT_NEW_INSTANCE, + } + } # N-tuple listing default values for any metadata-related class # attributes cached on an instance by a process_data runs. These attribute # values are reset via clear_cached_attrs during any update_metadata call. cached_attr_defaults = ( - ('ec2_metadata', UNSET), ('network_json', UNSET), - ('metadata', {}), ('userdata', None), ('userdata_raw', None), - ('vendordata', None), ('vendordata_raw', None)) + ("ec2_metadata", UNSET), + ("network_json", UNSET), + ("metadata", {}), + ("userdata", None), + ("userdata_raw", None), + ("vendordata", None), + ("vendordata_raw", None), + ("vendordata2", None), + ("vendordata2_raw", None), + ) _dirty_cache = False # N-tuple of keypaths or keynames redact from instance-data.json for # non-root users - sensitive_metadata_keys = ('merged_cfg', 'security-credentials',) + sensitive_metadata_keys = ( + "merged_cfg", + "security-credentials", + ) + + _ci_pkl_version = 1 - def __init__(self, sys_cfg, distro, paths, ud_proc=None): + def __init__(self, sys_cfg, distro: Distro, paths, ud_proc=None): self.sys_cfg = sys_cfg self.distro = distro self.paths = paths @@ -203,10 +249,13 @@ class DataSource(metaclass=abc.ABCMeta): self.metadata = {} self.userdata_raw = None self.vendordata = None + self.vendordata2 = None self.vendordata_raw = None + self.vendordata2_raw = None self.ds_cfg = util.get_cfg_by_path( - self.sys_cfg, ("datasource", self.dsname), {}) + self.sys_cfg, ("datasource", self.dsname), {} + ) if not self.ds_cfg: self.ds_cfg = {} @@ -215,6 +264,28 @@ class DataSource(metaclass=abc.ABCMeta): else: self.ud_proc = ud_proc + def _unpickle(self, ci_pkl_version: int) -> None: + """Perform deserialization fixes for Paths.""" + if not hasattr(self, "vendordata2"): + self.vendordata2 = None + if not hasattr(self, "vendordata2_raw"): + self.vendordata2_raw = None + if hasattr(self, "userdata") and self.userdata is not None: + # If userdata stores MIME data, on < python3.6 it will be + # missing the 'policy' attribute that exists on >=python3.6. + # Calling str() on the userdata will attempt to access this + # policy attribute. This will raise an exception, causing + # the pickle load to fail, so cloud-init will discard the cache + try: + str(self.userdata) + except AttributeError as e: + LOG.debug( + "Unable to unpickle datasource: %s." + " Ignoring current cache.", + e, + ) + raise DatasourceUnpickleUserDataError() from e + def __str__(self): return type_utils.obj_name(self) @@ -228,28 +299,33 @@ class DataSource(metaclass=abc.ABCMeta): # metadata to discover that content sysinfo = instance_data["sys_info"] return { - 'v1': { - '_beta_keys': ['subplatform'], - 'availability-zone': availability_zone, - 'availability_zone': availability_zone, - 'cloud-name': self.cloud_name, - 'cloud_name': self.cloud_name, - 'distro': sysinfo["dist"][0], - 'distro_version': sysinfo["dist"][1], - 'distro_release': sysinfo["dist"][2], - 'platform': self.platform_type, - 'public_ssh_keys': self.get_public_ssh_keys(), - 'python_version': sysinfo["python"], - 'instance-id': instance_id, - 'instance_id': instance_id, - 'kernel_release': sysinfo["uname"][2], - 'local-hostname': local_hostname, - 'local_hostname': local_hostname, - 'machine': sysinfo["uname"][4], - 'region': self.region, - 'subplatform': self.subplatform, - 'system_platform': sysinfo["platform"], - 'variant': sysinfo["variant"]}} + "v1": { + "_beta_keys": ["subplatform"], + "availability-zone": availability_zone, + "availability_zone": availability_zone, + "cloud_id": canonical_cloud_id( + self.cloud_name, self.region, self.platform_type + ), + "cloud-name": self.cloud_name, + "cloud_name": self.cloud_name, + "distro": sysinfo["dist"][0], + "distro_version": sysinfo["dist"][1], + "distro_release": sysinfo["dist"][2], + "platform": self.platform_type, + "public_ssh_keys": self.get_public_ssh_keys(), + "python_version": sysinfo["python"], + "instance-id": instance_id, + "instance_id": instance_id, + "kernel_release": sysinfo["uname"][2], + "local-hostname": local_hostname, + "local_hostname": local_hostname, + "machine": sysinfo["uname"][4], + "region": self.region, + "subplatform": self.subplatform, + "system_platform": sysinfo["platform"], + "variant": sysinfo["variant"], + } + } def clear_cached_attrs(self, attr_defaults=()): """Reset any cached metadata attributes to datasource defaults. @@ -290,48 +366,61 @@ class DataSource(metaclass=abc.ABCMeta): @return True on successful write, False otherwise. """ - if hasattr(self, '_crawled_metadata'): + if hasattr(self, "_crawled_metadata"): # Any datasource with _crawled_metadata will best represent # most recent, 'raw' metadata crawled_metadata = copy.deepcopy( - getattr(self, '_crawled_metadata')) - crawled_metadata.pop('user-data', None) - crawled_metadata.pop('vendor-data', None) - instance_data = {'ds': crawled_metadata} + getattr(self, "_crawled_metadata") + ) + crawled_metadata.pop("user-data", None) + crawled_metadata.pop("vendor-data", None) + instance_data = {"ds": crawled_metadata} else: - instance_data = {'ds': {'meta_data': self.metadata}} - if hasattr(self, 'network_json'): - network_json = getattr(self, 'network_json') + instance_data = {"ds": {"meta_data": self.metadata}} + if hasattr(self, "network_json"): + network_json = getattr(self, "network_json") if network_json != UNSET: - instance_data['ds']['network_json'] = network_json - if hasattr(self, 'ec2_metadata'): - ec2_metadata = getattr(self, 'ec2_metadata') + instance_data["ds"]["network_json"] = network_json + if hasattr(self, "ec2_metadata"): + ec2_metadata = getattr(self, "ec2_metadata") if ec2_metadata != UNSET: - instance_data['ds']['ec2_metadata'] = ec2_metadata - instance_data['ds']['_doc'] = EXPERIMENTAL_TEXT + instance_data["ds"]["ec2_metadata"] = ec2_metadata + instance_data["ds"]["_doc"] = EXPERIMENTAL_TEXT # Add merged cloud.cfg and sys info for jinja templates and cli query - instance_data['merged_cfg'] = copy.deepcopy(self.sys_cfg) - instance_data['merged_cfg']['_doc'] = ( - 'Merged cloud-init system config from /etc/cloud/cloud.cfg and' - ' /etc/cloud/cloud.cfg.d/') - instance_data['sys_info'] = util.system_info() - instance_data.update( - self._get_standardized_metadata(instance_data)) + instance_data["merged_cfg"] = copy.deepcopy(self.sys_cfg) + instance_data["merged_cfg"]["_doc"] = ( + "Merged cloud-init system config from /etc/cloud/cloud.cfg and" + " /etc/cloud/cloud.cfg.d/" + ) + instance_data["sys_info"] = util.system_info() + instance_data.update(self._get_standardized_metadata(instance_data)) try: # Process content base64encoding unserializable values content = util.json_dumps(instance_data) # Strip base64: prefix and set base64_encoded_keys list. processed_data = process_instance_metadata( json.loads(content), - sensitive_keys=self.sensitive_metadata_keys) + sensitive_keys=self.sensitive_metadata_keys, + ) except TypeError as e: - LOG.warning('Error persisting instance-data.json: %s', str(e)) + LOG.warning("Error persisting instance-data.json: %s", str(e)) return False except UnicodeDecodeError as e: - LOG.warning('Error persisting instance-data.json: %s', str(e)) + LOG.warning("Error persisting instance-data.json: %s", str(e)) return False - json_sensitive_file = os.path.join(self.paths.run_dir, - INSTANCE_JSON_SENSITIVE_FILE) + json_sensitive_file = os.path.join( + self.paths.run_dir, INSTANCE_JSON_SENSITIVE_FILE + ) + cloud_id = instance_data["v1"].get("cloud_id", "none") + cloud_id_file = os.path.join(self.paths.run_dir, "cloud-id") + util.write_file(f"{cloud_id_file}-{cloud_id}", f"{cloud_id}\n") + if os.path.exists(cloud_id_file): + prev_cloud_id_file = os.path.realpath(cloud_id_file) + else: + prev_cloud_id_file = cloud_id_file + util.sym_link(f"{cloud_id_file}-{cloud_id}", cloud_id_file, force=True) + if prev_cloud_id_file != cloud_id_file: + util.del_file(prev_cloud_id_file) write_json(json_sensitive_file, processed_data, mode=0o600) json_file = os.path.join(self.paths.run_dir, INSTANCE_JSON_FILE) # World readable @@ -341,8 +430,9 @@ class DataSource(metaclass=abc.ABCMeta): def _get_data(self): """Walk metadata sources, process crawled data and save attributes.""" raise NotImplementedError( - 'Subclasses of DataSource must implement _get_data which' - ' sets self.metadata, vendordata_raw and userdata_raw.') + "Subclasses of DataSource must implement _get_data which" + " sets self.metadata, vendordata_raw and userdata_raw." + ) def get_url_params(self): """Return the Datasource's prefered url_read parameters. @@ -357,28 +447,52 @@ class DataSource(metaclass=abc.ABCMeta): max_wait = int(self.ds_cfg.get("max_wait", self.url_max_wait)) except ValueError: util.logexc( - LOG, "Config max_wait '%s' is not an int, using default '%s'", - self.ds_cfg.get("max_wait"), max_wait) + LOG, + "Config max_wait '%s' is not an int, using default '%s'", + self.ds_cfg.get("max_wait"), + max_wait, + ) timeout = self.url_timeout try: - timeout = max( - 0, int(self.ds_cfg.get("timeout", self.url_timeout))) + timeout = max(0, int(self.ds_cfg.get("timeout", self.url_timeout))) except ValueError: timeout = self.url_timeout util.logexc( - LOG, "Config timeout '%s' is not an int, using default '%s'", - self.ds_cfg.get('timeout'), timeout) + LOG, + "Config timeout '%s' is not an int, using default '%s'", + self.ds_cfg.get("timeout"), + timeout, + ) retries = self.url_retries try: retries = int(self.ds_cfg.get("retries", self.url_retries)) except Exception: util.logexc( - LOG, "Config retries '%s' is not an int, using default '%s'", - self.ds_cfg.get('retries'), retries) + LOG, + "Config retries '%s' is not an int, using default '%s'", + self.ds_cfg.get("retries"), + retries, + ) - return URLParams(max_wait, timeout, retries) + sec_between_retries = self.url_sec_between_retries + try: + sec_between_retries = int( + self.ds_cfg.get( + "sec_between_retries", self.url_sec_between_retries + ) + ) + except Exception: + util.logexc( + LOG, + "Config sec_between_retries '%s' is not an int," + " using default '%s'", + self.ds_cfg.get("sec_between_retries"), + sec_between_retries, + ) + + return URLParams(max_wait, timeout, retries, sec_between_retries) def get_userdata(self, apply_filter=False): if self.userdata is None: @@ -392,6 +506,11 @@ class DataSource(metaclass=abc.ABCMeta): self.vendordata = self.ud_proc.process(self.get_vendordata_raw()) return self.vendordata + def get_vendordata2(self): + if self.vendordata2 is None: + self.vendordata2 = self.ud_proc.process(self.get_vendordata2_raw()) + return self.vendordata2 + @property def fallback_interface(self): """Determine the network interface used during local network config.""" @@ -399,13 +518,13 @@ class DataSource(metaclass=abc.ABCMeta): self._fallback_interface = net.find_fallback_nic() if self._fallback_interface is None: LOG.warning( - "Did not find a fallback interface on %s.", - self.cloud_name) + "Did not find a fallback interface on %s.", self.cloud_name + ) return self._fallback_interface @property def platform_type(self): - if not hasattr(self, '_platform_type'): + if not hasattr(self, "_platform_type"): # Handle upgrade path where pickled datasource has no _platform. self._platform_type = self.dsname.lower() if not self._platform_type: @@ -424,7 +543,7 @@ class DataSource(metaclass=abc.ABCMeta): nocloud: seed-dir (/seed/dir/path) lxd: nocloud (/seed/dir/path) """ - if not hasattr(self, '_subplatform'): + if not hasattr(self, "_subplatform"): # Handle upgrade path where pickled datasource has no _platform. self._subplatform = self._get_subplatform() if not self._subplatform: @@ -433,8 +552,8 @@ class DataSource(metaclass=abc.ABCMeta): def _get_subplatform(self): """Subclasses should implement to return a "slug (detail)" string.""" - if hasattr(self, 'metadata_address'): - return 'metadata (%s)' % getattr(self, 'metadata_address') + if hasattr(self, "metadata_address"): + return "metadata (%s)" % getattr(self, "metadata_address") return METADATA_UNKNOWN @property @@ -453,8 +572,10 @@ class DataSource(metaclass=abc.ABCMeta): else: self._cloud_name = self._get_cloud_name().lower() LOG.debug( - 'Ignoring metadata provided key %s: non-string type %s', - METADATA_CLOUD_NAME_KEY, type(cloud_name)) + "Ignoring metadata provided key %s: non-string type %s", + METADATA_CLOUD_NAME_KEY, + type(cloud_name), + ) else: self._cloud_name = self._get_cloud_name().lower() return self._cloud_name @@ -471,8 +592,8 @@ class DataSource(metaclass=abc.ABCMeta): def launch_index(self): if not self.metadata: return None - if 'launch-index' in self.metadata: - return self.metadata['launch-index'] + if "launch-index" in self.metadata: + return self.metadata["launch-index"] return None def _filter_xdata(self, processed_ud): @@ -494,6 +615,9 @@ class DataSource(metaclass=abc.ABCMeta): def get_vendordata_raw(self): return self.vendordata_raw + def get_vendordata2_raw(self): + return self.vendordata2_raw + # the data sources' config_obj is a cloud-config formated # object that came to it from ways other than cloud-config # because cloud-config content would be handled elsewhere @@ -501,7 +625,7 @@ class DataSource(metaclass=abc.ABCMeta): return {} def get_public_ssh_keys(self): - return normalize_pubkey_data(self.metadata.get('public-keys')) + return normalize_pubkey_data(self.metadata.get("public-keys")) def publish_host_keys(self, hostkeys): """Publish the public SSH host keys (found in /etc/ssh/*.pub). @@ -523,7 +647,7 @@ class DataSource(metaclass=abc.ABCMeta): if not short_name.startswith(nfrom): continue for nto in tlist: - cand = "/dev/%s%s" % (nto, short_name[len(nfrom):]) + cand = "/dev/%s%s" % (nto, short_name[len(nfrom) :]) if os.path.exists(cand): return cand return None @@ -548,20 +672,21 @@ class DataSource(metaclass=abc.ABCMeta): @property def availability_zone(self): top_level_az = self.metadata.get( - 'availability-zone', self.metadata.get('availability_zone')) + "availability-zone", self.metadata.get("availability_zone") + ) if top_level_az: return top_level_az - return self.metadata.get('placement', {}).get('availability-zone') + return self.metadata.get("placement", {}).get("availability-zone") @property def region(self): - return self.metadata.get('region') + return self.metadata.get("region") def get_instance_id(self): - if not self.metadata or 'instance-id' not in self.metadata: + if not self.metadata or "instance-id" not in self.metadata: # Return a magic not really instance id string return "iid-datasource" - return str(self.metadata['instance-id']) + return str(self.metadata["instance-id"]) def get_hostname(self, fqdn=False, resolve_ip=False, metadata_only=False): """Get hostname or fqdn from the datasource. Look it up if desired. @@ -579,7 +704,7 @@ class DataSource(metaclass=abc.ABCMeta): defhost = "localhost" domain = defdomain - if not self.metadata or not self.metadata.get('local-hostname'): + if not self.metadata or not self.metadata.get("local-hostname"): if metadata_only: return None # this is somewhat questionable really. @@ -600,14 +725,14 @@ class DataSource(metaclass=abc.ABCMeta): else: # if there is an ipv4 address in 'local-hostname', then # make up a hostname (LP: #475354) in format ip-xx.xx.xx.xx - lhost = self.metadata['local-hostname'] + lhost = self.metadata["local-hostname"] if net.is_ipv4_address(lhost): toks = [] if resolve_ip: toks = util.gethostbyaddr(lhost) if toks: - toks = str(toks).split('.') + toks = str(toks).split(".") else: toks = ["ip-%s" % lhost.replace(".", "-")] else: @@ -615,7 +740,7 @@ class DataSource(metaclass=abc.ABCMeta): if len(toks) > 1: hostname = toks[0] - domain = '.'.join(toks[1:]) + domain = ".".join(toks[1:]) else: hostname = toks[0] @@ -627,10 +752,25 @@ class DataSource(metaclass=abc.ABCMeta): def get_package_mirror_info(self): return self.distro.get_package_mirror_info(data_source=self) - def update_metadata(self, source_event_types): + def get_supported_events(self, source_event_types: List[EventType]): + supported_events = {} # type: Dict[EventScope, set] + for event in source_event_types: + for ( + update_scope, + update_events, + ) in self.supported_update_events.items(): + if event in update_events: + if not supported_events.get(update_scope): + supported_events[update_scope] = set() + supported_events[update_scope].add(event) + return supported_events + + def update_metadata_if_supported( + self, source_event_types: List[EventType] + ) -> bool: """Refresh cached metadata if the datasource supports this event. - The datasource has a list of update_events which + The datasource has a list of supported_update_events which trigger refreshing all cached metadata as well as refreshing the network configuration. @@ -640,28 +780,27 @@ class DataSource(metaclass=abc.ABCMeta): @return True if the datasource did successfully update cached metadata due to source_event_type. """ - supported_events = {} - for event in source_event_types: - for update_scope, update_events in self.update_events.items(): - if event in update_events: - if not supported_events.get(update_scope): - supported_events[update_scope] = set() - supported_events[update_scope].add(event) + supported_events = self.get_supported_events(source_event_types) for scope, matched_events in supported_events.items(): LOG.debug( "Update datasource metadata and %s config due to events: %s", - scope, ', '.join(matched_events)) + scope.value, + ", ".join([event.value for event in matched_events]), + ) # Each datasource has a cached config property which needs clearing # Once cleared that config property will be regenerated from # current metadata. - self.clear_cached_attrs((('_%s_config' % scope, UNSET),)) + self.clear_cached_attrs((("_%s_config" % scope, UNSET),)) if supported_events: self.clear_cached_attrs() result = self.get_data() if result: return True - LOG.debug("Datasource %s not updated for events: %s", self, - ', '.join(source_event_types)) + LOG.debug( + "Datasource %s not updated for events: %s", + self, + ", ".join([event.value for event in source_event_types]), + ) return False def check_instance_id(self, sys_cfg): @@ -683,8 +822,9 @@ class DataSource(metaclass=abc.ABCMeta): if candidate in valid: return candidate else: - LOG.warning("invalid dsmode '%s', using default=%s", - candidate, default) + LOG.warning( + "invalid dsmode '%s', using default=%s", candidate, default + ) return default return default @@ -763,19 +903,23 @@ def find_source(sys_cfg, distro, paths, ds_deps, cfg_list, pkg_list, reporter): name="search-%s" % name.replace("DataSource", ""), description="searching for %s data from %s" % (mode, name), message="no %s data found from %s" % (mode, name), - parent=reporter) + parent=reporter, + ) try: with myrep: LOG.debug("Seeing if we can get any data from %s", cls) s = cls(sys_cfg, distro, paths) - if s.update_metadata([EventType.BOOT_NEW_INSTANCE]): + if s.update_metadata_if_supported( + [EventType.BOOT_NEW_INSTANCE] + ): myrep.message = "found %s data from %s" % (mode, name) return (s, type_utils.obj_name(cls)) except Exception: util.logexc(LOG, "Getting data from %s failed", cls) - msg = ("Did not find any data source," - " searched classes: (%s)") % (", ".join(ds_names)) + msg = "Did not find any data source, searched classes: (%s)" % ", ".join( + ds_names + ) raise DataSourceNotFoundException(msg) @@ -785,15 +929,25 @@ def find_source(sys_cfg, distro, paths, ds_deps, cfg_list, pkg_list, reporter): # Return an ordered list of classes that match (if any) def list_sources(cfg_list, depends, pkg_list): src_list = [] - LOG.debug(("Looking for data source in: %s," - " via packages %s that matches dependencies %s"), - cfg_list, pkg_list, depends) + LOG.debug( + "Looking for data source in: %s," + " via packages %s that matches dependencies %s", + cfg_list, + pkg_list, + depends, + ) for ds_name in cfg_list: if not ds_name.startswith(DS_PREFIX): - ds_name = '%s%s' % (DS_PREFIX, ds_name) - m_locs, _looked_locs = importer.find_module(ds_name, - pkg_list, - ['get_datasource_list']) + ds_name = "%s%s" % (DS_PREFIX, ds_name) + m_locs, _looked_locs = importer.find_module( + ds_name, pkg_list, ["get_datasource_list"] + ) + if not m_locs: + LOG.error( + "Could not import %s. Does the DataSource exist and " + "is it importable?", + ds_name, + ) for m_loc in m_locs: mod = importer.import_module(m_loc) lister = getattr(mod, "get_datasource_list") @@ -804,7 +958,7 @@ def list_sources(cfg_list, depends, pkg_list): return src_list -def instance_id_matches_system_uuid(instance_id, field='system-uuid'): +def instance_id_matches_system_uuid(instance_id, field="system-uuid"): # quickly (local check only) if self.instance_id is still valid # we check kernel command line or files. if not instance_id: @@ -854,8 +1008,7 @@ def convert_vendordata(data, recurse=True): return copy.deepcopy(data) if isinstance(data, dict): if recurse is True: - return convert_vendordata(data.get('cloud-init'), - recurse=False) + return convert_vendordata(data.get("cloud-init"), recurse=False) raise ValueError("vendordata['cloud-init'] cannot be dict") raise ValueError("Unknown data type for vendordata: %s" % type(data)) |