# Copyright (C) 2012 Canonical Ltd. # Copyright (C) 2012 Yahoo! Inc. # # Author: Scott Moser # Author: Joshua Harlow # # This file is part of cloud-init. See LICENSE file for license information. import abc import base64 import copy import functools import os from cloudinit import ec2_utils from cloudinit import log as logging from cloudinit import net from cloudinit import sources from cloudinit import subp from cloudinit import url_helper from cloudinit import util from cloudinit.sources import BrokenMetadata # See https://docs.openstack.org/user-guide/cli-config-drive.html LOG = logging.getLogger(__name__) FILES_V1 = { # Path <-> (metadata key name, translator function, default value) 'etc/network/interfaces': ('network_config', lambda x: x, ''), 'meta.js': ('meta_js', util.load_json, {}), "root/.ssh/authorized_keys": ('authorized_keys', lambda x: x, ''), } KEY_COPIES = ( # Cloud-init metadata names <-> (metadata key, is required) ('local-hostname', 'hostname', False), ('instance-id', 'uuid', True), ) # Versions and names taken from nova source nova/api/metadata/base.py OS_LATEST = 'latest' OS_FOLSOM = '2012-08-10' OS_GRIZZLY = '2013-04-04' OS_HAVANA = '2013-10-17' OS_LIBERTY = '2015-10-15' # NEWTON_ONE adds 'devices' to md (sriov-pf-passthrough-neutron-port-vlan) OS_NEWTON_ONE = '2016-06-30' # NEWTON_TWO adds vendor_data2.json (vendordata-reboot) OS_NEWTON_TWO = '2016-10-06' # OS_OCATA adds 'vif' field to devices (sriov-pf-passthrough-neutron-port-vlan) OS_OCATA = '2017-02-22' # OS_ROCKY adds a vf_trusted field to devices (sriov-trusted-vfs) OS_ROCKY = '2018-08-27' # keep this in chronological order. new supported versions go at the end. OS_VERSIONS = ( OS_FOLSOM, OS_GRIZZLY, OS_HAVANA, OS_LIBERTY, OS_NEWTON_ONE, OS_NEWTON_TWO, OS_OCATA, OS_ROCKY, ) KNOWN_PHYSICAL_TYPES = ( None, 'bgpovs', # not present in OpenStack upstream but used on OVH cloud. 'bridge', 'cascading', # not present in OpenStack upstream, used on OpenTelekomCloud 'dvs', 'ethernet', 'hw_veb', 'hyperv', 'ovs', 'phy', 'tap', 'vhostuser', 'vif', ) class NonReadable(IOError): pass class SourceMixin(object): def _ec2_name_to_device(self, name): if not self.ec2_metadata: return None bdm = self.ec2_metadata.get('block-device-mapping', {}) for (ent_name, device) in bdm.items(): if name == ent_name: return device return None def get_public_ssh_keys(self): name = "public_keys" if self.version == 1: name = "public-keys" return sources.normalize_pubkey_data(self.metadata.get(name)) def _os_name_to_device(self, name): device = None try: criteria = 'LABEL=%s' % (name) if name == 'swap': criteria = 'TYPE=%s' % (name) dev_entries = util.find_devs_with(criteria) if dev_entries: device = dev_entries[0] except subp.ProcessExecutionError: pass return device def _validate_device_name(self, device): if not device: return None if not device.startswith("/"): device = "/dev/%s" % device if os.path.exists(device): return device # Durn, try adjusting the mapping remapped = self._remap_device(os.path.basename(device)) if remapped: LOG.debug("Remapped device name %s => %s", device, remapped) return remapped return None def device_name_to_device(self, name): # Translate a 'name' to a 'physical' device if not name: return None # Try the ec2 mapping first names = [name] if name == 'root': names.insert(0, 'ami') if name == 'ami': names.append('root') device = None LOG.debug("Using ec2 style lookup to find device %s", names) for n in names: device = self._ec2_name_to_device(n) device = self._validate_device_name(device) if device: break # Try the openstack way second if not device: LOG.debug("Using openstack style lookup to find device %s", names) for n in names: device = self._os_name_to_device(n) device = self._validate_device_name(device) if device: break # Ok give up... if not device: return None else: LOG.debug("Mapped %s to device %s", name, device) return device class BaseReader(metaclass=abc.ABCMeta): def __init__(self, base_path): self.base_path = base_path @abc.abstractmethod def _path_join(self, base, *add_ons): pass @abc.abstractmethod def _path_read(self, path, decode=False): pass @abc.abstractmethod def _fetch_available_versions(self): pass @abc.abstractmethod def _read_ec2_metadata(self): pass def _find_working_version(self): try: versions_available = self._fetch_available_versions() except Exception as e: LOG.debug("Unable to read openstack versions from %s due to: %s", self.base_path, e) versions_available = [] # openstack.OS_VERSIONS is stored in chronological order, so # reverse it to check newest first. supported = [v for v in reversed(list(OS_VERSIONS))] selected_version = OS_LATEST for potential_version in supported: if potential_version not in versions_available: continue selected_version = potential_version break LOG.debug("Selected version '%s' from %s", selected_version, versions_available) return selected_version def _read_content_path(self, item, decode=False): path = item.get('content_path', '').lstrip("/") path_pieces = path.split("/") valid_pieces = [p for p in path_pieces if len(p)] if not valid_pieces: raise BrokenMetadata("Item %s has no valid content path" % (item)) path = self._path_join(self.base_path, "openstack", *path_pieces) return self._path_read(path, decode=decode) def read_v2(self): """Reads a version 2 formatted location. Return a dict with metadata, userdata, ec2-metadata, dsmode, network_config, files and version (2). If not a valid location, raise a NonReadable exception. """ load_json_anytype = functools.partial( util.load_json, root_types=(dict, list, str)) def datafiles(version): files = {} files['metadata'] = ( # File path to read self._path_join("openstack", version, 'meta_data.json'), # Is it required? True, # Translator function (applied after loading) util.load_json, ) files['userdata'] = ( self._path_join("openstack", version, 'user_data'), False, lambda x: x, ) files['vendordata'] = ( self._path_join("openstack", version, 'vendor_data.json'), False, load_json_anytype, ) files['networkdata'] = ( self._path_join("openstack", version, 'network_data.json'), False, load_json_anytype, ) return files results = { 'userdata': '', 'version': 2, } data = datafiles(self._find_working_version()) for (name, (path, required, translator)) in data.items(): path = self._path_join(self.base_path, path) data = None found = False try: data = self._path_read(path) except IOError as e: if not required: LOG.debug("Failed reading optional path %s due" " to: %s", path, e) else: LOG.debug("Failed reading mandatory path %s due" " to: %s", path, e) else: found = True if required and not found: raise NonReadable("Missing mandatory path: %s" % path) if found and translator: try: data = translator(data) except Exception as e: raise BrokenMetadata("Failed to process " "path %s: %s" % (path, e)) if found: results[name] = data metadata = results['metadata'] if 'random_seed' in metadata: random_seed = metadata['random_seed'] try: metadata['random_seed'] = base64.b64decode(random_seed) except (ValueError, TypeError) as e: raise BrokenMetadata("Badly formatted metadata" " random_seed entry: %s" % e) # load any files that were provided files = {} metadata_files = metadata.get('files', []) for item in metadata_files: if 'path' not in item: continue path = item['path'] try: files[path] = self._read_content_path(item) except Exception as e: raise BrokenMetadata("Failed to read provided " "file %s: %s" % (path, e)) results['files'] = files # The 'network_config' item in metadata is a content pointer # to the network config that should be applied. It is just a # ubuntu/debian '/etc/network/interfaces' file. net_item = metadata.get("network_config", None) if net_item: try: content = self._read_content_path(net_item, decode=True) results['network_config'] = content except IOError as e: raise BrokenMetadata("Failed to read network" " configuration: %s" % (e)) # To openstack, user can specify meta ('nova boot --meta=key=value') # and those will appear under metadata['meta']. # if they specify 'dsmode' they're indicating the mode that they intend # for this datasource to operate in. try: results['dsmode'] = metadata['meta']['dsmode'] except KeyError: pass # Read any ec2-metadata (if applicable) results['ec2-metadata'] = self._read_ec2_metadata() # Perform some misc. metadata key renames... for (target_key, source_key, is_required) in KEY_COPIES: if is_required and source_key not in metadata: raise BrokenMetadata("No '%s' entry in metadata" % source_key) if source_key in metadata: metadata[target_key] = metadata.get(source_key) return results class ConfigDriveReader(BaseReader): def __init__(self, base_path): super(ConfigDriveReader, self).__init__(base_path) self._versions = None def _path_join(self, base, *add_ons): components = [base] + list(add_ons) return os.path.join(*components) def _path_read(self, path, decode=False): return util.load_file(path, decode=decode) def _fetch_available_versions(self): if self._versions is None: path = self._path_join(self.base_path, 'openstack') found = [d for d in os.listdir(path) if os.path.isdir(os.path.join(path))] self._versions = sorted(found) return self._versions def _read_ec2_metadata(self): path = self._path_join(self.base_path, 'ec2', 'latest', 'meta-data.json') if not os.path.exists(path): return {} else: try: return util.load_json(self._path_read(path)) except Exception as e: raise BrokenMetadata("Failed to process " "path %s: %s" % (path, e)) def read_v1(self): """Reads a version 1 formatted location. Return a dict with metadata, userdata, dsmode, files and version (1). If not a valid path, raise a NonReadable exception. """ found = {} for name in FILES_V1.keys(): path = self._path_join(self.base_path, name) if os.path.exists(path): found[name] = path if len(found) == 0: raise NonReadable("%s: no files found" % (self.base_path)) md = {} for (name, (key, translator, default)) in FILES_V1.items(): if name in found: path = found[name] try: contents = self._path_read(path) except IOError: raise BrokenMetadata("Failed to read: %s" % path) try: # Disable not-callable pylint check; pylint isn't able to # determine that every member of FILES_V1 has a callable in # the appropriate position md[key] = translator(contents) # pylint: disable=E1102 except Exception as e: raise BrokenMetadata("Failed to process " "path %s: %s" % (path, e)) else: md[key] = copy.deepcopy(default) keydata = md['authorized_keys'] meta_js = md['meta_js'] # keydata in meta_js is preferred over "injected" keydata = meta_js.get('public-keys', keydata) if keydata: lines = keydata.splitlines() md['public-keys'] = [ line for line in lines if len(line) and not line.startswith("#") ] # config-drive-v1 has no way for openstack to provide the instance-id # so we copy that into metadata from the user input if 'instance-id' in meta_js: md['instance-id'] = meta_js['instance-id'] results = { 'version': 1, 'metadata': md, } # allow the user to specify 'dsmode' in a meta tag if 'dsmode' in meta_js: results['dsmode'] = meta_js['dsmode'] # config-drive-v1 has no way of specifying user-data, so the user has # to cheat and stuff it in a meta tag also. results['userdata'] = meta_js.get('user-data', '') # this implementation does not support files other than # network/interfaces and authorized_keys... results['files'] = {} return results class MetadataReader(BaseReader): def __init__(self, base_url, ssl_details=None, timeout=5, retries=5): super(MetadataReader, self).__init__(base_url) self.ssl_details = ssl_details self.timeout = float(timeout) self.retries = int(retries) self._versions = None def _fetch_available_versions(self): # /openstack/ returns a newline separated list of versions if self._versions is not None: return self._versions found = [] version_path = self._path_join(self.base_path, "openstack") content = self._path_read(version_path, decode=True) for line in content.splitlines(): line = line.strip() if not line: continue found.append(line) self._versions = found return self._versions def _path_read(self, path, decode=False): def should_retry_cb(_request_args, cause): try: code = int(cause.code) if code >= 400: return False except (TypeError, ValueError): # Older versions of requests didn't have a code. pass return True response = url_helper.readurl(path, retries=self.retries, ssl_details=self.ssl_details, timeout=self.timeout, exception_cb=should_retry_cb) if decode: return response.contents.decode() else: return response.contents def _path_join(self, base, *add_ons): return url_helper.combine_url(base, *add_ons) def _read_ec2_metadata(self): return ec2_utils.get_instance_metadata(ssl_details=self.ssl_details, timeout=self.timeout, retries=self.retries) # Convert OpenStack ConfigDrive NetworkData json to network_config yaml def convert_net_json(network_json=None, known_macs=None): """Return a dictionary of network_config by parsing provided OpenStack ConfigDrive NetworkData json format OpenStack network_data.json provides a 3 element dictionary - "links" (links are network devices, physical or virtual) - "networks" (networks are ip network configurations for one or more links) - services (non-ip services, like dns) networks and links are combined via network items referencing specific links via a 'link_id' which maps to a links 'id' field. To convert this format to network_config yaml, we first iterate over the links and then walk the network list to determine if any of the networks utilize the current link; if so we generate a subnet entry for the device We also need to map network_data.json fields to network_config fields. For example, the network_data links 'id' field is equivalent to network_config 'name' field for devices. We apply more of this mapping to the various link types that we encounter. There are additional fields that are populated in the network_data.json from OpenStack that are not relevant to network_config yaml, so we enumerate a dictionary of valid keys for network_yaml and apply filtering to drop these superflous keys from the network_config yaml. """ if network_json is None: return None # dict of network_config key for filtering network_json valid_keys = { 'physical': [ 'name', 'type', 'mac_address', 'subnets', 'params', 'mtu', ], 'subnet': [ 'type', 'address', 'netmask', 'broadcast', 'metric', 'gateway', 'pointopoint', 'scope', 'dns_nameservers', 'dns_search', 'routes', ], } links = network_json.get('links', []) networks = network_json.get('networks', []) services = network_json.get('services', []) link_updates = [] link_id_info = {} bond_name_fmt = "bond%d" bond_number = 0 config = [] for link in links: subnets = [] cfg = dict((k, v) for k, v in link.items() if k in valid_keys['physical']) # 'name' is not in openstack spec yet, but we will support it if it is # present. The 'id' in the spec is currently implemented as the host # nic's name, meaning something like 'tap-adfasdffd'. We do not want # to name guest devices with such ugly names. if 'name' in link: cfg['name'] = link['name'] link_mac_addr = None if link.get('ethernet_mac_address'): link_mac_addr = link.get('ethernet_mac_address').lower() link_id_info[link['id']] = link_mac_addr curinfo = {'name': cfg.get('name'), 'mac': link_mac_addr, 'id': link['id'], 'type': link['type']} for network in [n for n in networks if n['link'] == link['id']]: subnet = dict((k, v) for k, v in network.items() if k in valid_keys['subnet']) if network['type'] == 'ipv4_dhcp': subnet.update({'type': 'dhcp4'}) elif network['type'] == 'ipv6_dhcp': subnet.update({'type': 'dhcp6'}) elif network['type'] in ['ipv6_slaac', 'ipv6_dhcpv6-stateless', 'ipv6_dhcpv6-stateful']: subnet.update({'type': network['type']}) elif network['type'] in ['ipv4', 'ipv6']: subnet.update({ 'type': 'static', 'address': network.get('ip_address'), }) # Enable accept_ra for stateful and legacy ipv6_dhcp types if network['type'] in ['ipv6_dhcpv6-stateful', 'ipv6_dhcp']: cfg.update({'accept-ra': True}) if network['type'] == 'ipv4': subnet['ipv4'] = True if network['type'] == 'ipv6': subnet['ipv6'] = True subnets.append(subnet) cfg.update({'subnets': subnets}) if link['type'] in ['bond']: params = {} if link_mac_addr: params['mac_address'] = link_mac_addr for k, v in link.items(): if k == 'bond_links': continue elif k.startswith('bond'): params.update({k: v}) # openstack does not provide a name for the bond. # they do provide an 'id', but that is possibly non-sensical. # so we just create our own name. link_name = bond_name_fmt % bond_number bond_number += 1 # bond_links reference links by their id, but we need to add # to the network config by their nic name. # store that in bond_links_needed, and update these later. link_updates.append( (cfg, 'bond_interfaces', '%s', copy.deepcopy(link['bond_links'])) ) cfg.update({'params': params, 'name': link_name}) curinfo['name'] = link_name elif link['type'] in ['vlan']: name = "%s.%s" % (link['vlan_link'], link['vlan_id']) cfg.update({ 'name': name, 'vlan_id': link['vlan_id'], 'mac_address': link['vlan_mac_address'], }) link_updates.append((cfg, 'vlan_link', '%s', link['vlan_link'])) link_updates.append((cfg, 'name', "%%s.%s" % link['vlan_id'], link['vlan_link'])) curinfo.update({'mac': link['vlan_mac_address'], 'name': name}) else: if link['type'] not in KNOWN_PHYSICAL_TYPES: LOG.warning('Unknown network_data link type (%s); treating as' ' physical', link['type']) cfg.update({'type': 'physical', 'mac_address': link_mac_addr}) config.append(cfg) link_id_info[curinfo['id']] = curinfo need_names = [d for d in config if d.get('type') == 'physical' and 'name' not in d] if need_names or link_updates: if known_macs is None: known_macs = net.get_interfaces_by_mac() # go through and fill out the link_id_info with names for _link_id, info in link_id_info.items(): if info.get('name'): continue if info.get('mac') in known_macs: info['name'] = known_macs[info['mac']] for d in need_names: mac = d.get('mac_address') if not mac: raise ValueError("No mac_address or name entry for %s" % d) if mac not in known_macs: raise ValueError("Unable to find a system nic for %s" % d) d['name'] = known_macs[mac] for cfg, key, fmt, targets in link_updates: if isinstance(targets, (list, tuple)): cfg[key] = [ fmt % link_id_info[target]['name'] for target in targets ] else: cfg[key] = fmt % link_id_info[targets]['name'] # Infiniband interfaces may be referenced in network_data.json by a 6 byte # Ethernet MAC-style address, and we use that address to look up the # interface name above. Now ensure that the hardware address is set to the # full 20 byte address. ib_known_hwaddrs = net.get_ib_hwaddrs_by_interface() if ib_known_hwaddrs: for cfg in config: if cfg['name'] in ib_known_hwaddrs: cfg['mac_address'] = ib_known_hwaddrs[cfg['name']] cfg['type'] = 'infiniband' for service in services: cfg = service cfg.update({'type': 'nameserver'}) config.append(cfg) return {'version': 1, 'config': config} # vi: ts=4 expandtab