From 2b85dabb802766e0b3b1949d744c8860c0cb838a Mon Sep 17 00:00:00 2001 From: Ryan Harper Date: Wed, 23 Mar 2016 11:05:22 -0500 Subject: configdata: parse and convert openstack network_data json to network_config --- cloudinit/sources/DataSourceConfigDrive.py | 137 +++++++++++++++++++++++++++++ 1 file changed, 137 insertions(+) (limited to 'cloudinit/sources/DataSourceConfigDrive.py') diff --git a/cloudinit/sources/DataSourceConfigDrive.py b/cloudinit/sources/DataSourceConfigDrive.py index 6fc9e05b..d84fab54 100644 --- a/cloudinit/sources/DataSourceConfigDrive.py +++ b/cloudinit/sources/DataSourceConfigDrive.py @@ -18,6 +18,7 @@ # You should have received a copy of the GNU General Public License # along with this program. If not, see . +import copy import os from cloudinit import log as logging @@ -50,6 +51,8 @@ class DataSourceConfigDrive(openstack.SourceMixin, sources.DataSource): self.seed_dir = os.path.join(paths.seed_dir, 'config_drive') self.version = None self.ec2_metadata = None + self._network_config = None + self.network_json = None self.files = {} def __str__(self): @@ -144,12 +147,27 @@ class DataSourceConfigDrive(openstack.SourceMixin, sources.DataSource): LOG.warn("Invalid content in vendor-data: %s", e) self.vendordata_raw = None + nd = results.get('networkdata') + self.networkdata_pure = nd + try: + self.network_json = openstack.convert_networkdata_json(nd) + except ValueError as e: + LOG.warn("Invalid content in network-data: %s", e) + self.network_json = None + + if self.network_json: + self._network_config = convert_network_data(self.network_json) + return True def check_instance_id(self): # quickly (local check only) if self.instance_id is still valid return sources.instance_id_matches_system_uuid(self.get_instance_id()) + @property + def network_config(self): + return self._network_config + class DataSourceConfigDriveNet(DataSourceConfigDrive): def __init__(self, sys_cfg, distro, paths): @@ -287,3 +305,122 @@ datasources = [ # Return a list of data sources that match this set of dependencies def get_datasource_list(depends): return sources.list_from_depends(depends, datasources) + + +# Convert OpenStack ConfigDrive NetworkData json to network_config yaml +def convert_network_data(network_json=None): + """Return a dictionary of network_config by parsing provided + OpenStack ConfigDrive NetworkData json format + + OpenStack network_data.json provides a 3 element dictionary + - "links" (links are network devices, physical or virtual) + - "networks" (networks are ip network configurations for one or more + links) + - services (non-ip services, like dns) + + networks and links are combined via network items referencing specific + links via a 'link_id' which maps to a links 'id' field. + + To convert this format to network_config yaml, we first iterate over the + links and then walk the network list to determine if any of the networks + utilize the current link; if so we generate a subnet entry for the device + + We also need to map network_data.json fields to network_config fields. For + example, the network_data links 'id' field is equivalent to network_config + 'name' field for devices. We apply more of this mapping to the various + link types that we encounter. + + There are additional fields that are populated in the network_data.json + from OpenStack that are not relevant to network_config yaml, so we + enumerate a dictionary of valid keys for network_yaml and apply filtering + to drop these superflous keys from the network_config yaml. + """ + if network_json is None: + return None + + # dict of network_config key for filtering network_json + valid_keys = { + 'physical': [ + 'name', + 'type', + 'mac_address', + 'subnets', + 'params', + ], + 'subnet': [ + 'type', + 'address', + 'netmask', + 'broadcast', + 'metric', + 'gateway', + 'pointopoint', + 'mtu', + 'scope', + 'dns_nameservers', + 'dns_search', + 'routes', + ], + } + + links = network_json.get('links', []) + networks = network_json.get('networks', []) + services = network_json.get('services', []) + + config = [] + for link in links: + subnets = [] + cfg = {k: v for k, v in link.items() + if k in valid_keys['physical']} + cfg.update({'name': link['id']}) + for network in [net for net in networks + if net['link'] == link['id']]: + subnet = {k: v for k, v in network.items() + if k in valid_keys['subnet']} + if 'dhcp' in network['type']: + t = 'dhcp6' if network['type'].startswith('ipv6') else 'dhcp4' + subnet.update({ + 'type': t, + }) + else: + subnet.update({ + 'type': 'static', + 'address': network.get('ip_address'), + }) + subnets.append(subnet) + cfg.update({'subnets': subnets}) + if link['type'] in ['ethernet', 'vif', 'ovs']: + cfg.update({ + 'type': 'physical', + 'mac_address': link['ethernet_mac_address']}) + elif link['type'] in ['bond']: + params = {} + for k, v in link.items(): + if k == 'bond_links': + continue + elif k.startswith('bond'): + params.update({k: v}) + cfg.update({ + 'bond_interfaces': copy.deepcopy(link['bond_links']), + 'params': params, + }) + elif link['type'] in ['vlan']: + cfg.update({ + 'name': "%s.%s" % (link['vlan_link'], + link['vlan_id']), + 'vlan_link': link['vlan_link'], + 'vlan_id': link['vlan_id'], + 'mac_address': link['vlan_mac_address'], + }) + else: + raise ValueError( + 'Unknown network_data link type: %s' % link['type']) + + config.append(cfg) + + for service in services: + cfg = service + cfg.update({'type': 'nameserver'}) + config.append(cfg) + + return {'version': 1, 'config': config} -- cgit v1.2.3 From 6b79e2c6f9a7342163691be9e785cef1aa642541 Mon Sep 17 00:00:00 2001 From: Ryan Harper Date: Wed, 23 Mar 2016 14:17:10 -0500 Subject: fix openstack versions s/KILO/LIBERY drop networkdata read helper --- cloudinit/sources/DataSourceConfigDrive.py | 2 +- cloudinit/sources/helpers/openstack.py | 29 ++--------------------------- 2 files changed, 3 insertions(+), 28 deletions(-) (limited to 'cloudinit/sources/DataSourceConfigDrive.py') diff --git a/cloudinit/sources/DataSourceConfigDrive.py b/cloudinit/sources/DataSourceConfigDrive.py index d84fab54..15dddefe 100644 --- a/cloudinit/sources/DataSourceConfigDrive.py +++ b/cloudinit/sources/DataSourceConfigDrive.py @@ -150,7 +150,7 @@ class DataSourceConfigDrive(openstack.SourceMixin, sources.DataSource): nd = results.get('networkdata') self.networkdata_pure = nd try: - self.network_json = openstack.convert_networkdata_json(nd) + self.network_json = util.load_json(nd) except ValueError as e: LOG.warn("Invalid content in network-data: %s", e) self.network_json = None diff --git a/cloudinit/sources/helpers/openstack.py b/cloudinit/sources/helpers/openstack.py index eb50a7be..1aa6bbae 100644 --- a/cloudinit/sources/helpers/openstack.py +++ b/cloudinit/sources/helpers/openstack.py @@ -51,13 +51,13 @@ OS_LATEST = 'latest' OS_FOLSOM = '2012-08-10' OS_GRIZZLY = '2013-04-04' OS_HAVANA = '2013-10-17' -OS_KILO = '2015-10-15' +OS_LIBERTY = '2015-10-15' # keep this in chronological order. new supported versions go at the end. OS_VERSIONS = ( OS_FOLSOM, OS_GRIZZLY, OS_HAVANA, - OS_KILO, + OS_LIBERTY, ) @@ -497,28 +497,3 @@ def convert_vendordata_json(data, recurse=True): recurse=False) raise ValueError("vendordata['cloud-init'] cannot be dict") raise ValueError("Unknown data type for vendordata: %s" % type(data)) - - -def convert_networkdata_json(data, recurse=True): - """ data: a loaded json *object* (strings, arrays, dicts). - return something suitable for cloudinit networkdata_raw. - - if data is: - None: return None - string: return string - list: return data - the list is then processed in UserDataProcessor - dict: return convert_networkdata_json(data.get('cloud-init')) - """ - if not data: - return None - if isinstance(data, six.string_types): - return data - if isinstance(data, list): - return copy.deepcopy(data) - if isinstance(data, dict): - if recurse is True: - return convert_networkdata_json(data.get('cloud-init'), - recurse=False) - raise ValueError("networkdata['cloud-init'] cannot be dict") - raise ValueError("Unknown data type for networkdata: %s" % type(data)) -- cgit v1.2.3 From 32e81553907eaba84345252527f208d29151620f Mon Sep 17 00:00:00 2001 From: Ryan Harper Date: Wed, 23 Mar 2016 16:56:02 -0500 Subject: network_data: add link type 'phys', no need to reload json data --- cloudinit/sources/DataSourceConfigDrive.py | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) (limited to 'cloudinit/sources/DataSourceConfigDrive.py') diff --git a/cloudinit/sources/DataSourceConfigDrive.py b/cloudinit/sources/DataSourceConfigDrive.py index 15dddefe..db813f6e 100644 --- a/cloudinit/sources/DataSourceConfigDrive.py +++ b/cloudinit/sources/DataSourceConfigDrive.py @@ -147,10 +147,8 @@ class DataSourceConfigDrive(openstack.SourceMixin, sources.DataSource): LOG.warn("Invalid content in vendor-data: %s", e) self.vendordata_raw = None - nd = results.get('networkdata') - self.networkdata_pure = nd try: - self.network_json = util.load_json(nd) + self.network_json = results.get('networkdata') except ValueError as e: LOG.warn("Invalid content in network-data: %s", e) self.network_json = None @@ -389,7 +387,7 @@ def convert_network_data(network_json=None): }) subnets.append(subnet) cfg.update({'subnets': subnets}) - if link['type'] in ['ethernet', 'vif', 'ovs']: + if link['type'] in ['ethernet', 'vif', 'ovs', 'phy']: cfg.update({ 'type': 'physical', 'mac_address': link['ethernet_mac_address']}) -- cgit v1.2.3 From 9c0b3fc96fc33107dde8e89b02a63dbfb04e207c Mon Sep 17 00:00:00 2001 From: Ryan Harper Date: Thu, 24 Mar 2016 13:41:25 -0500 Subject: fix review comments net: add render_route comment to document why we added || true to route statements DataSourceConfigDrive: Only convert network_json to network_config when caller reads network_config attr. Cache the conversion. --- cloudinit/net/__init__.py | 14 ++++++++++++++ cloudinit/sources/DataSourceConfigDrive.py | 6 +++--- 2 files changed, 17 insertions(+), 3 deletions(-) (limited to 'cloudinit/sources/DataSourceConfigDrive.py') diff --git a/cloudinit/net/__init__.py b/cloudinit/net/__init__.py index 76cd4e8b..2435055b 100644 --- a/cloudinit/net/__init__.py +++ b/cloudinit/net/__init__.py @@ -349,6 +349,20 @@ def iface_add_attrs(iface): def render_route(route, indent=""): + """ When rendering routes for an iface, in some cases applying a route + may result in the route command returning non-zero which produces + some confusing output for users manually using ifup/ifdown[1]. To + that end, we will optionally include an '|| true' postfix to each + route line allowing users to work with ifup/ifdown without using + --force option. + + We may at somepoint not want to emit this additional postfix, and + add a 'strict' flag to this function. When called with strict=True, + then we will not append the postfix. + + 1. http://askubuntu.com/questions/168033/ + how-to-set-static-routes-in-ubuntu-server + """ content = "" up = indent + "post-up route add" down = indent + "pre-down route del" diff --git a/cloudinit/sources/DataSourceConfigDrive.py b/cloudinit/sources/DataSourceConfigDrive.py index db813f6e..14676f97 100644 --- a/cloudinit/sources/DataSourceConfigDrive.py +++ b/cloudinit/sources/DataSourceConfigDrive.py @@ -153,9 +153,6 @@ class DataSourceConfigDrive(openstack.SourceMixin, sources.DataSource): LOG.warn("Invalid content in network-data: %s", e) self.network_json = None - if self.network_json: - self._network_config = convert_network_data(self.network_json) - return True def check_instance_id(self): @@ -164,6 +161,9 @@ class DataSourceConfigDrive(openstack.SourceMixin, sources.DataSource): @property def network_config(self): + if self._network_config is None: + if self.network_json is not None: + self._network_config = convert_network_data(self.network_json) return self._network_config -- cgit v1.2.3