summaryrefslogtreecommitdiff
path: root/cloudinit/sources/__init__.py
diff options
context:
space:
mode:
Diffstat (limited to 'cloudinit/sources/__init__.py')
-rw-r--r--cloudinit/sources/__init__.py471
1 files changed, 312 insertions, 159 deletions
diff --git a/cloudinit/sources/__init__.py b/cloudinit/sources/__init__.py
index 9dccc687..88028cfa 100644
--- a/cloudinit/sources/__init__.py
+++ b/cloudinit/sources/__init__.py
@@ -13,17 +13,18 @@ import copy
import json
import os
from collections import namedtuple
+from typing import Dict, List # noqa: F401
-from cloudinit import dmi
-from cloudinit import importer
+from cloudinit import dmi, importer
from cloudinit import log as logging
-from cloudinit import net
-from cloudinit import type_utils
+from cloudinit import net, type_utils
from cloudinit import user_data as ud
from cloudinit import util
from cloudinit.atomic_helper import write_json
-from cloudinit.event import EventType
+from cloudinit.distros import Distro
+from cloudinit.event import EventScope, EventType
from cloudinit.filters import launch_index
+from cloudinit.persistence import CloudInitPickleMixin
from cloudinit.reporting import events
DSMODE_DISABLED = "disabled"
@@ -35,42 +36,48 @@ VALID_DSMODES = [DSMODE_DISABLED, DSMODE_LOCAL, DSMODE_NETWORK]
DEP_FILESYSTEM = "FILESYSTEM"
DEP_NETWORK = "NETWORK"
-DS_PREFIX = 'DataSource'
+DS_PREFIX = "DataSource"
EXPERIMENTAL_TEXT = (
"EXPERIMENTAL: The structure and format of content scoped under the 'ds'"
- " key may change in subsequent releases of cloud-init.")
+ " key may change in subsequent releases of cloud-init."
+)
# File in which public available instance meta-data is written
# security-sensitive key values are redacted from this world-readable file
-INSTANCE_JSON_FILE = 'instance-data.json'
+INSTANCE_JSON_FILE = "instance-data.json"
# security-sensitive key values are present in this root-readable file
-INSTANCE_JSON_SENSITIVE_FILE = 'instance-data-sensitive.json'
-REDACT_SENSITIVE_VALUE = 'redacted for non-root user'
+INSTANCE_JSON_SENSITIVE_FILE = "instance-data-sensitive.json"
+REDACT_SENSITIVE_VALUE = "redacted for non-root user"
# Key which can be provide a cloud's official product name to cloud-init
-METADATA_CLOUD_NAME_KEY = 'cloud-name'
+METADATA_CLOUD_NAME_KEY = "cloud-name"
UNSET = "_unset"
-METADATA_UNKNOWN = 'unknown'
+METADATA_UNKNOWN = "unknown"
LOG = logging.getLogger(__name__)
# CLOUD_ID_REGION_PREFIX_MAP format is:
# <region-match-prefix>: (<new-cloud-id>: <test_allowed_cloud_callable>)
CLOUD_ID_REGION_PREFIX_MAP = {
- 'cn-': ('aws-china', lambda c: c == 'aws'), # only change aws regions
- 'us-gov-': ('aws-gov', lambda c: c == 'aws'), # only change aws regions
- 'china': ('azure-china', lambda c: c == 'azure'), # only change azure
+ "cn-": ("aws-china", lambda c: c == "aws"), # only change aws regions
+ "us-gov-": ("aws-gov", lambda c: c == "aws"), # only change aws regions
+ "china": ("azure-china", lambda c: c == "azure"), # only change azure
}
# NetworkConfigSource represents the canonical list of network config sources
# that cloud-init knows about. (Python 2.7 lacks PEP 435, so use a singleton
# namedtuple as an enum; see https://stackoverflow.com/a/6971002)
-_NETCFG_SOURCE_NAMES = ('cmdline', 'ds', 'system_cfg', 'fallback', 'initramfs')
-NetworkConfigSource = namedtuple('NetworkConfigSource',
- _NETCFG_SOURCE_NAMES)(*_NETCFG_SOURCE_NAMES)
+_NETCFG_SOURCE_NAMES = ("cmdline", "ds", "system_cfg", "fallback", "initramfs")
+NetworkConfigSource = namedtuple("NetworkConfigSource", _NETCFG_SOURCE_NAMES)(
+ *_NETCFG_SOURCE_NAMES
+)
+
+
+class DatasourceUnpickleUserDataError(Exception):
+ """Raised when userdata is unable to be unpickled due to python upgrades"""
class DataSourceNotFoundException(Exception):
@@ -81,7 +88,7 @@ class InvalidMetaDataException(Exception):
"""Raised when metadata is broken, unavailable or disabled."""
-def process_instance_metadata(metadata, key_path='', sensitive_keys=()):
+def process_instance_metadata(metadata, key_path="", sensitive_keys=()):
"""Process all instance metadata cleaning it up for persisting as json.
Strip ci-b64 prefix and catalog any 'base64_encoded_keys' as a list
@@ -93,22 +100,23 @@ def process_instance_metadata(metadata, key_path='', sensitive_keys=()):
sens_keys = []
for key, val in metadata.items():
if key_path:
- sub_key_path = key_path + '/' + key
+ sub_key_path = key_path + "/" + key
else:
sub_key_path = key
if key in sensitive_keys or sub_key_path in sensitive_keys:
sens_keys.append(sub_key_path)
- if isinstance(val, str) and val.startswith('ci-b64:'):
+ if isinstance(val, str) and val.startswith("ci-b64:"):
base64_encoded_keys.append(sub_key_path)
- md_copy[key] = val.replace('ci-b64:', '')
+ md_copy[key] = val.replace("ci-b64:", "")
if isinstance(val, dict):
return_val = process_instance_metadata(
- val, sub_key_path, sensitive_keys)
- base64_encoded_keys.extend(return_val.pop('base64_encoded_keys'))
- sens_keys.extend(return_val.pop('sensitive_keys'))
+ val, sub_key_path, sensitive_keys
+ )
+ base64_encoded_keys.extend(return_val.pop("base64_encoded_keys"))
+ sens_keys.extend(return_val.pop("sensitive_keys"))
md_copy[key] = return_val
- md_copy['base64_encoded_keys'] = sorted(base64_encoded_keys)
- md_copy['sensitive_keys'] = sorted(sens_keys)
+ md_copy["base64_encoded_keys"] = sorted(base64_encoded_keys)
+ md_copy["sensitive_keys"] = sorted(sens_keys)
return md_copy
@@ -117,11 +125,11 @@ def redact_sensitive_keys(metadata, redact_value=REDACT_SENSITIVE_VALUE):
Replace any keys values listed in 'sensitive_keys' with redact_value.
"""
- if not metadata.get('sensitive_keys', []):
+ if not metadata.get("sensitive_keys", []):
return metadata
md_copy = copy.deepcopy(metadata)
- for key_path in metadata.get('sensitive_keys'):
- path_parts = key_path.split('/')
+ for key_path in metadata.get("sensitive_keys"):
+ path_parts = key_path.split("/")
obj = md_copy
for path in path_parts:
if isinstance(obj[path], dict) and path != path_parts[-1]:
@@ -131,17 +139,24 @@ def redact_sensitive_keys(metadata, redact_value=REDACT_SENSITIVE_VALUE):
URLParams = namedtuple(
- 'URLParms', ['max_wait_seconds', 'timeout_seconds', 'num_retries'])
+ "URLParms",
+ [
+ "max_wait_seconds",
+ "timeout_seconds",
+ "num_retries",
+ "sec_between_retries",
+ ],
+)
-class DataSource(metaclass=abc.ABCMeta):
+class DataSource(CloudInitPickleMixin, metaclass=abc.ABCMeta):
dsmode = DSMODE_NETWORK
- default_locale = 'en_US.UTF-8'
+ default_locale = "en_US.UTF-8"
# Datasource name needs to be set by subclasses to determine which
# cloud-config datasource key is loaded
- dsname = '_undef'
+ dsname = "_undef"
# Cached cloud_name as determined by _get_cloud_name
_cloud_name = None
@@ -162,40 +177,71 @@ class DataSource(metaclass=abc.ABCMeta):
# configuration will be used without considering any that follow.) This
# should always be a subset of the members of NetworkConfigSource with no
# duplicate entries.
- network_config_sources = (NetworkConfigSource.cmdline,
- NetworkConfigSource.initramfs,
- NetworkConfigSource.system_cfg,
- NetworkConfigSource.ds)
+ network_config_sources = (
+ NetworkConfigSource.cmdline,
+ NetworkConfigSource.initramfs,
+ NetworkConfigSource.system_cfg,
+ NetworkConfigSource.ds,
+ )
# read_url_params
- url_max_wait = -1 # max_wait < 0 means do not wait
- url_timeout = 10 # timeout for each metadata url read attempt
- url_retries = 5 # number of times to retry url upon 404
+ url_max_wait = -1 # max_wait < 0 means do not wait
+ url_timeout = 10 # timeout for each metadata url read attempt
+ url_retries = 5 # number of times to retry url upon 404
+ url_sec_between_retries = 1 # amount of seconds to wait between retries
# The datasource defines a set of supported EventTypes during which
# the datasource can react to changes in metadata and regenerate
- # network configuration on metadata changes.
- # A datasource which supports writing network config on each system boot
- # would call update_events['network'].add(EventType.BOOT).
+ # network configuration on metadata changes. These are defined in
+ # `supported_network_events`.
+ # The datasource also defines a set of default EventTypes that the
+ # datasource can react to. These are the event types that will be used
+ # if not overridden by the user.
+ # A datasource requiring to write network config on each system boot
+ # would call default_update_events['network'].add(EventType.BOOT).
# Default: generate network config on new instance id (first boot).
- update_events = {'network': set([EventType.BOOT_NEW_INSTANCE])}
+ supported_update_events = {
+ EventScope.NETWORK: {
+ EventType.BOOT_NEW_INSTANCE,
+ EventType.BOOT,
+ EventType.BOOT_LEGACY,
+ EventType.HOTPLUG,
+ }
+ }
+ default_update_events = {
+ EventScope.NETWORK: {
+ EventType.BOOT_NEW_INSTANCE,
+ }
+ }
# N-tuple listing default values for any metadata-related class
# attributes cached on an instance by a process_data runs. These attribute
# values are reset via clear_cached_attrs during any update_metadata call.
cached_attr_defaults = (
- ('ec2_metadata', UNSET), ('network_json', UNSET),
- ('metadata', {}), ('userdata', None), ('userdata_raw', None),
- ('vendordata', None), ('vendordata_raw', None))
+ ("ec2_metadata", UNSET),
+ ("network_json", UNSET),
+ ("metadata", {}),
+ ("userdata", None),
+ ("userdata_raw", None),
+ ("vendordata", None),
+ ("vendordata_raw", None),
+ ("vendordata2", None),
+ ("vendordata2_raw", None),
+ )
_dirty_cache = False
# N-tuple of keypaths or keynames redact from instance-data.json for
# non-root users
- sensitive_metadata_keys = ('merged_cfg', 'security-credentials',)
+ sensitive_metadata_keys = (
+ "merged_cfg",
+ "security-credentials",
+ )
+
+ _ci_pkl_version = 1
- def __init__(self, sys_cfg, distro, paths, ud_proc=None):
+ def __init__(self, sys_cfg, distro: Distro, paths, ud_proc=None):
self.sys_cfg = sys_cfg
self.distro = distro
self.paths = paths
@@ -203,10 +249,13 @@ class DataSource(metaclass=abc.ABCMeta):
self.metadata = {}
self.userdata_raw = None
self.vendordata = None
+ self.vendordata2 = None
self.vendordata_raw = None
+ self.vendordata2_raw = None
self.ds_cfg = util.get_cfg_by_path(
- self.sys_cfg, ("datasource", self.dsname), {})
+ self.sys_cfg, ("datasource", self.dsname), {}
+ )
if not self.ds_cfg:
self.ds_cfg = {}
@@ -215,6 +264,28 @@ class DataSource(metaclass=abc.ABCMeta):
else:
self.ud_proc = ud_proc
+ def _unpickle(self, ci_pkl_version: int) -> None:
+ """Perform deserialization fixes for Paths."""
+ if not hasattr(self, "vendordata2"):
+ self.vendordata2 = None
+ if not hasattr(self, "vendordata2_raw"):
+ self.vendordata2_raw = None
+ if hasattr(self, "userdata") and self.userdata is not None:
+ # If userdata stores MIME data, on < python3.6 it will be
+ # missing the 'policy' attribute that exists on >=python3.6.
+ # Calling str() on the userdata will attempt to access this
+ # policy attribute. This will raise an exception, causing
+ # the pickle load to fail, so cloud-init will discard the cache
+ try:
+ str(self.userdata)
+ except AttributeError as e:
+ LOG.debug(
+ "Unable to unpickle datasource: %s."
+ " Ignoring current cache.",
+ e,
+ )
+ raise DatasourceUnpickleUserDataError() from e
+
def __str__(self):
return type_utils.obj_name(self)
@@ -228,28 +299,33 @@ class DataSource(metaclass=abc.ABCMeta):
# metadata to discover that content
sysinfo = instance_data["sys_info"]
return {
- 'v1': {
- '_beta_keys': ['subplatform'],
- 'availability-zone': availability_zone,
- 'availability_zone': availability_zone,
- 'cloud-name': self.cloud_name,
- 'cloud_name': self.cloud_name,
- 'distro': sysinfo["dist"][0],
- 'distro_version': sysinfo["dist"][1],
- 'distro_release': sysinfo["dist"][2],
- 'platform': self.platform_type,
- 'public_ssh_keys': self.get_public_ssh_keys(),
- 'python_version': sysinfo["python"],
- 'instance-id': instance_id,
- 'instance_id': instance_id,
- 'kernel_release': sysinfo["uname"][2],
- 'local-hostname': local_hostname,
- 'local_hostname': local_hostname,
- 'machine': sysinfo["uname"][4],
- 'region': self.region,
- 'subplatform': self.subplatform,
- 'system_platform': sysinfo["platform"],
- 'variant': sysinfo["variant"]}}
+ "v1": {
+ "_beta_keys": ["subplatform"],
+ "availability-zone": availability_zone,
+ "availability_zone": availability_zone,
+ "cloud_id": canonical_cloud_id(
+ self.cloud_name, self.region, self.platform_type
+ ),
+ "cloud-name": self.cloud_name,
+ "cloud_name": self.cloud_name,
+ "distro": sysinfo["dist"][0],
+ "distro_version": sysinfo["dist"][1],
+ "distro_release": sysinfo["dist"][2],
+ "platform": self.platform_type,
+ "public_ssh_keys": self.get_public_ssh_keys(),
+ "python_version": sysinfo["python"],
+ "instance-id": instance_id,
+ "instance_id": instance_id,
+ "kernel_release": sysinfo["uname"][2],
+ "local-hostname": local_hostname,
+ "local_hostname": local_hostname,
+ "machine": sysinfo["uname"][4],
+ "region": self.region,
+ "subplatform": self.subplatform,
+ "system_platform": sysinfo["platform"],
+ "variant": sysinfo["variant"],
+ }
+ }
def clear_cached_attrs(self, attr_defaults=()):
"""Reset any cached metadata attributes to datasource defaults.
@@ -290,48 +366,61 @@ class DataSource(metaclass=abc.ABCMeta):
@return True on successful write, False otherwise.
"""
- if hasattr(self, '_crawled_metadata'):
+ if hasattr(self, "_crawled_metadata"):
# Any datasource with _crawled_metadata will best represent
# most recent, 'raw' metadata
crawled_metadata = copy.deepcopy(
- getattr(self, '_crawled_metadata'))
- crawled_metadata.pop('user-data', None)
- crawled_metadata.pop('vendor-data', None)
- instance_data = {'ds': crawled_metadata}
+ getattr(self, "_crawled_metadata")
+ )
+ crawled_metadata.pop("user-data", None)
+ crawled_metadata.pop("vendor-data", None)
+ instance_data = {"ds": crawled_metadata}
else:
- instance_data = {'ds': {'meta_data': self.metadata}}
- if hasattr(self, 'network_json'):
- network_json = getattr(self, 'network_json')
+ instance_data = {"ds": {"meta_data": self.metadata}}
+ if hasattr(self, "network_json"):
+ network_json = getattr(self, "network_json")
if network_json != UNSET:
- instance_data['ds']['network_json'] = network_json
- if hasattr(self, 'ec2_metadata'):
- ec2_metadata = getattr(self, 'ec2_metadata')
+ instance_data["ds"]["network_json"] = network_json
+ if hasattr(self, "ec2_metadata"):
+ ec2_metadata = getattr(self, "ec2_metadata")
if ec2_metadata != UNSET:
- instance_data['ds']['ec2_metadata'] = ec2_metadata
- instance_data['ds']['_doc'] = EXPERIMENTAL_TEXT
+ instance_data["ds"]["ec2_metadata"] = ec2_metadata
+ instance_data["ds"]["_doc"] = EXPERIMENTAL_TEXT
# Add merged cloud.cfg and sys info for jinja templates and cli query
- instance_data['merged_cfg'] = copy.deepcopy(self.sys_cfg)
- instance_data['merged_cfg']['_doc'] = (
- 'Merged cloud-init system config from /etc/cloud/cloud.cfg and'
- ' /etc/cloud/cloud.cfg.d/')
- instance_data['sys_info'] = util.system_info()
- instance_data.update(
- self._get_standardized_metadata(instance_data))
+ instance_data["merged_cfg"] = copy.deepcopy(self.sys_cfg)
+ instance_data["merged_cfg"]["_doc"] = (
+ "Merged cloud-init system config from /etc/cloud/cloud.cfg and"
+ " /etc/cloud/cloud.cfg.d/"
+ )
+ instance_data["sys_info"] = util.system_info()
+ instance_data.update(self._get_standardized_metadata(instance_data))
try:
# Process content base64encoding unserializable values
content = util.json_dumps(instance_data)
# Strip base64: prefix and set base64_encoded_keys list.
processed_data = process_instance_metadata(
json.loads(content),
- sensitive_keys=self.sensitive_metadata_keys)
+ sensitive_keys=self.sensitive_metadata_keys,
+ )
except TypeError as e:
- LOG.warning('Error persisting instance-data.json: %s', str(e))
+ LOG.warning("Error persisting instance-data.json: %s", str(e))
return False
except UnicodeDecodeError as e:
- LOG.warning('Error persisting instance-data.json: %s', str(e))
+ LOG.warning("Error persisting instance-data.json: %s", str(e))
return False
- json_sensitive_file = os.path.join(self.paths.run_dir,
- INSTANCE_JSON_SENSITIVE_FILE)
+ json_sensitive_file = os.path.join(
+ self.paths.run_dir, INSTANCE_JSON_SENSITIVE_FILE
+ )
+ cloud_id = instance_data["v1"].get("cloud_id", "none")
+ cloud_id_file = os.path.join(self.paths.run_dir, "cloud-id")
+ util.write_file(f"{cloud_id_file}-{cloud_id}", f"{cloud_id}\n")
+ if os.path.exists(cloud_id_file):
+ prev_cloud_id_file = os.path.realpath(cloud_id_file)
+ else:
+ prev_cloud_id_file = cloud_id_file
+ util.sym_link(f"{cloud_id_file}-{cloud_id}", cloud_id_file, force=True)
+ if prev_cloud_id_file != cloud_id_file:
+ util.del_file(prev_cloud_id_file)
write_json(json_sensitive_file, processed_data, mode=0o600)
json_file = os.path.join(self.paths.run_dir, INSTANCE_JSON_FILE)
# World readable
@@ -341,8 +430,9 @@ class DataSource(metaclass=abc.ABCMeta):
def _get_data(self):
"""Walk metadata sources, process crawled data and save attributes."""
raise NotImplementedError(
- 'Subclasses of DataSource must implement _get_data which'
- ' sets self.metadata, vendordata_raw and userdata_raw.')
+ "Subclasses of DataSource must implement _get_data which"
+ " sets self.metadata, vendordata_raw and userdata_raw."
+ )
def get_url_params(self):
"""Return the Datasource's prefered url_read parameters.
@@ -357,28 +447,52 @@ class DataSource(metaclass=abc.ABCMeta):
max_wait = int(self.ds_cfg.get("max_wait", self.url_max_wait))
except ValueError:
util.logexc(
- LOG, "Config max_wait '%s' is not an int, using default '%s'",
- self.ds_cfg.get("max_wait"), max_wait)
+ LOG,
+ "Config max_wait '%s' is not an int, using default '%s'",
+ self.ds_cfg.get("max_wait"),
+ max_wait,
+ )
timeout = self.url_timeout
try:
- timeout = max(
- 0, int(self.ds_cfg.get("timeout", self.url_timeout)))
+ timeout = max(0, int(self.ds_cfg.get("timeout", self.url_timeout)))
except ValueError:
timeout = self.url_timeout
util.logexc(
- LOG, "Config timeout '%s' is not an int, using default '%s'",
- self.ds_cfg.get('timeout'), timeout)
+ LOG,
+ "Config timeout '%s' is not an int, using default '%s'",
+ self.ds_cfg.get("timeout"),
+ timeout,
+ )
retries = self.url_retries
try:
retries = int(self.ds_cfg.get("retries", self.url_retries))
except Exception:
util.logexc(
- LOG, "Config retries '%s' is not an int, using default '%s'",
- self.ds_cfg.get('retries'), retries)
+ LOG,
+ "Config retries '%s' is not an int, using default '%s'",
+ self.ds_cfg.get("retries"),
+ retries,
+ )
- return URLParams(max_wait, timeout, retries)
+ sec_between_retries = self.url_sec_between_retries
+ try:
+ sec_between_retries = int(
+ self.ds_cfg.get(
+ "sec_between_retries", self.url_sec_between_retries
+ )
+ )
+ except Exception:
+ util.logexc(
+ LOG,
+ "Config sec_between_retries '%s' is not an int,"
+ " using default '%s'",
+ self.ds_cfg.get("sec_between_retries"),
+ sec_between_retries,
+ )
+
+ return URLParams(max_wait, timeout, retries, sec_between_retries)
def get_userdata(self, apply_filter=False):
if self.userdata is None:
@@ -392,6 +506,11 @@ class DataSource(metaclass=abc.ABCMeta):
self.vendordata = self.ud_proc.process(self.get_vendordata_raw())
return self.vendordata
+ def get_vendordata2(self):
+ if self.vendordata2 is None:
+ self.vendordata2 = self.ud_proc.process(self.get_vendordata2_raw())
+ return self.vendordata2
+
@property
def fallback_interface(self):
"""Determine the network interface used during local network config."""
@@ -399,13 +518,13 @@ class DataSource(metaclass=abc.ABCMeta):
self._fallback_interface = net.find_fallback_nic()
if self._fallback_interface is None:
LOG.warning(
- "Did not find a fallback interface on %s.",
- self.cloud_name)
+ "Did not find a fallback interface on %s.", self.cloud_name
+ )
return self._fallback_interface
@property
def platform_type(self):
- if not hasattr(self, '_platform_type'):
+ if not hasattr(self, "_platform_type"):
# Handle upgrade path where pickled datasource has no _platform.
self._platform_type = self.dsname.lower()
if not self._platform_type:
@@ -424,7 +543,7 @@ class DataSource(metaclass=abc.ABCMeta):
nocloud: seed-dir (/seed/dir/path)
lxd: nocloud (/seed/dir/path)
"""
- if not hasattr(self, '_subplatform'):
+ if not hasattr(self, "_subplatform"):
# Handle upgrade path where pickled datasource has no _platform.
self._subplatform = self._get_subplatform()
if not self._subplatform:
@@ -433,8 +552,8 @@ class DataSource(metaclass=abc.ABCMeta):
def _get_subplatform(self):
"""Subclasses should implement to return a "slug (detail)" string."""
- if hasattr(self, 'metadata_address'):
- return 'metadata (%s)' % getattr(self, 'metadata_address')
+ if hasattr(self, "metadata_address"):
+ return "metadata (%s)" % getattr(self, "metadata_address")
return METADATA_UNKNOWN
@property
@@ -453,8 +572,10 @@ class DataSource(metaclass=abc.ABCMeta):
else:
self._cloud_name = self._get_cloud_name().lower()
LOG.debug(
- 'Ignoring metadata provided key %s: non-string type %s',
- METADATA_CLOUD_NAME_KEY, type(cloud_name))
+ "Ignoring metadata provided key %s: non-string type %s",
+ METADATA_CLOUD_NAME_KEY,
+ type(cloud_name),
+ )
else:
self._cloud_name = self._get_cloud_name().lower()
return self._cloud_name
@@ -471,8 +592,8 @@ class DataSource(metaclass=abc.ABCMeta):
def launch_index(self):
if not self.metadata:
return None
- if 'launch-index' in self.metadata:
- return self.metadata['launch-index']
+ if "launch-index" in self.metadata:
+ return self.metadata["launch-index"]
return None
def _filter_xdata(self, processed_ud):
@@ -494,6 +615,9 @@ class DataSource(metaclass=abc.ABCMeta):
def get_vendordata_raw(self):
return self.vendordata_raw
+ def get_vendordata2_raw(self):
+ return self.vendordata2_raw
+
# the data sources' config_obj is a cloud-config formated
# object that came to it from ways other than cloud-config
# because cloud-config content would be handled elsewhere
@@ -501,7 +625,7 @@ class DataSource(metaclass=abc.ABCMeta):
return {}
def get_public_ssh_keys(self):
- return normalize_pubkey_data(self.metadata.get('public-keys'))
+ return normalize_pubkey_data(self.metadata.get("public-keys"))
def publish_host_keys(self, hostkeys):
"""Publish the public SSH host keys (found in /etc/ssh/*.pub).
@@ -523,7 +647,7 @@ class DataSource(metaclass=abc.ABCMeta):
if not short_name.startswith(nfrom):
continue
for nto in tlist:
- cand = "/dev/%s%s" % (nto, short_name[len(nfrom):])
+ cand = "/dev/%s%s" % (nto, short_name[len(nfrom) :])
if os.path.exists(cand):
return cand
return None
@@ -548,20 +672,21 @@ class DataSource(metaclass=abc.ABCMeta):
@property
def availability_zone(self):
top_level_az = self.metadata.get(
- 'availability-zone', self.metadata.get('availability_zone'))
+ "availability-zone", self.metadata.get("availability_zone")
+ )
if top_level_az:
return top_level_az
- return self.metadata.get('placement', {}).get('availability-zone')
+ return self.metadata.get("placement", {}).get("availability-zone")
@property
def region(self):
- return self.metadata.get('region')
+ return self.metadata.get("region")
def get_instance_id(self):
- if not self.metadata or 'instance-id' not in self.metadata:
+ if not self.metadata or "instance-id" not in self.metadata:
# Return a magic not really instance id string
return "iid-datasource"
- return str(self.metadata['instance-id'])
+ return str(self.metadata["instance-id"])
def get_hostname(self, fqdn=False, resolve_ip=False, metadata_only=False):
"""Get hostname or fqdn from the datasource. Look it up if desired.
@@ -579,7 +704,7 @@ class DataSource(metaclass=abc.ABCMeta):
defhost = "localhost"
domain = defdomain
- if not self.metadata or not self.metadata.get('local-hostname'):
+ if not self.metadata or not self.metadata.get("local-hostname"):
if metadata_only:
return None
# this is somewhat questionable really.
@@ -600,14 +725,14 @@ class DataSource(metaclass=abc.ABCMeta):
else:
# if there is an ipv4 address in 'local-hostname', then
# make up a hostname (LP: #475354) in format ip-xx.xx.xx.xx
- lhost = self.metadata['local-hostname']
+ lhost = self.metadata["local-hostname"]
if net.is_ipv4_address(lhost):
toks = []
if resolve_ip:
toks = util.gethostbyaddr(lhost)
if toks:
- toks = str(toks).split('.')
+ toks = str(toks).split(".")
else:
toks = ["ip-%s" % lhost.replace(".", "-")]
else:
@@ -615,7 +740,7 @@ class DataSource(metaclass=abc.ABCMeta):
if len(toks) > 1:
hostname = toks[0]
- domain = '.'.join(toks[1:])
+ domain = ".".join(toks[1:])
else:
hostname = toks[0]
@@ -627,10 +752,25 @@ class DataSource(metaclass=abc.ABCMeta):
def get_package_mirror_info(self):
return self.distro.get_package_mirror_info(data_source=self)
- def update_metadata(self, source_event_types):
+ def get_supported_events(self, source_event_types: List[EventType]):
+ supported_events = {} # type: Dict[EventScope, set]
+ for event in source_event_types:
+ for (
+ update_scope,
+ update_events,
+ ) in self.supported_update_events.items():
+ if event in update_events:
+ if not supported_events.get(update_scope):
+ supported_events[update_scope] = set()
+ supported_events[update_scope].add(event)
+ return supported_events
+
+ def update_metadata_if_supported(
+ self, source_event_types: List[EventType]
+ ) -> bool:
"""Refresh cached metadata if the datasource supports this event.
- The datasource has a list of update_events which
+ The datasource has a list of supported_update_events which
trigger refreshing all cached metadata as well as refreshing the
network configuration.
@@ -640,28 +780,27 @@ class DataSource(metaclass=abc.ABCMeta):
@return True if the datasource did successfully update cached metadata
due to source_event_type.
"""
- supported_events = {}
- for event in source_event_types:
- for update_scope, update_events in self.update_events.items():
- if event in update_events:
- if not supported_events.get(update_scope):
- supported_events[update_scope] = set()
- supported_events[update_scope].add(event)
+ supported_events = self.get_supported_events(source_event_types)
for scope, matched_events in supported_events.items():
LOG.debug(
"Update datasource metadata and %s config due to events: %s",
- scope, ', '.join(matched_events))
+ scope.value,
+ ", ".join([event.value for event in matched_events]),
+ )
# Each datasource has a cached config property which needs clearing
# Once cleared that config property will be regenerated from
# current metadata.
- self.clear_cached_attrs((('_%s_config' % scope, UNSET),))
+ self.clear_cached_attrs((("_%s_config" % scope, UNSET),))
if supported_events:
self.clear_cached_attrs()
result = self.get_data()
if result:
return True
- LOG.debug("Datasource %s not updated for events: %s", self,
- ', '.join(source_event_types))
+ LOG.debug(
+ "Datasource %s not updated for events: %s",
+ self,
+ ", ".join([event.value for event in source_event_types]),
+ )
return False
def check_instance_id(self, sys_cfg):
@@ -683,8 +822,9 @@ class DataSource(metaclass=abc.ABCMeta):
if candidate in valid:
return candidate
else:
- LOG.warning("invalid dsmode '%s', using default=%s",
- candidate, default)
+ LOG.warning(
+ "invalid dsmode '%s', using default=%s", candidate, default
+ )
return default
return default
@@ -763,19 +903,23 @@ def find_source(sys_cfg, distro, paths, ds_deps, cfg_list, pkg_list, reporter):
name="search-%s" % name.replace("DataSource", ""),
description="searching for %s data from %s" % (mode, name),
message="no %s data found from %s" % (mode, name),
- parent=reporter)
+ parent=reporter,
+ )
try:
with myrep:
LOG.debug("Seeing if we can get any data from %s", cls)
s = cls(sys_cfg, distro, paths)
- if s.update_metadata([EventType.BOOT_NEW_INSTANCE]):
+ if s.update_metadata_if_supported(
+ [EventType.BOOT_NEW_INSTANCE]
+ ):
myrep.message = "found %s data from %s" % (mode, name)
return (s, type_utils.obj_name(cls))
except Exception:
util.logexc(LOG, "Getting data from %s failed", cls)
- msg = ("Did not find any data source,"
- " searched classes: (%s)") % (", ".join(ds_names))
+ msg = "Did not find any data source, searched classes: (%s)" % ", ".join(
+ ds_names
+ )
raise DataSourceNotFoundException(msg)
@@ -785,15 +929,25 @@ def find_source(sys_cfg, distro, paths, ds_deps, cfg_list, pkg_list, reporter):
# Return an ordered list of classes that match (if any)
def list_sources(cfg_list, depends, pkg_list):
src_list = []
- LOG.debug(("Looking for data source in: %s,"
- " via packages %s that matches dependencies %s"),
- cfg_list, pkg_list, depends)
+ LOG.debug(
+ "Looking for data source in: %s,"
+ " via packages %s that matches dependencies %s",
+ cfg_list,
+ pkg_list,
+ depends,
+ )
for ds_name in cfg_list:
if not ds_name.startswith(DS_PREFIX):
- ds_name = '%s%s' % (DS_PREFIX, ds_name)
- m_locs, _looked_locs = importer.find_module(ds_name,
- pkg_list,
- ['get_datasource_list'])
+ ds_name = "%s%s" % (DS_PREFIX, ds_name)
+ m_locs, _looked_locs = importer.find_module(
+ ds_name, pkg_list, ["get_datasource_list"]
+ )
+ if not m_locs:
+ LOG.error(
+ "Could not import %s. Does the DataSource exist and "
+ "is it importable?",
+ ds_name,
+ )
for m_loc in m_locs:
mod = importer.import_module(m_loc)
lister = getattr(mod, "get_datasource_list")
@@ -804,7 +958,7 @@ def list_sources(cfg_list, depends, pkg_list):
return src_list
-def instance_id_matches_system_uuid(instance_id, field='system-uuid'):
+def instance_id_matches_system_uuid(instance_id, field="system-uuid"):
# quickly (local check only) if self.instance_id is still valid
# we check kernel command line or files.
if not instance_id:
@@ -854,8 +1008,7 @@ def convert_vendordata(data, recurse=True):
return copy.deepcopy(data)
if isinstance(data, dict):
if recurse is True:
- return convert_vendordata(data.get('cloud-init'),
- recurse=False)
+ return convert_vendordata(data.get("cloud-init"), recurse=False)
raise ValueError("vendordata['cloud-init'] cannot be dict")
raise ValueError("Unknown data type for vendordata: %s" % type(data))