From ef7368ef61c47fbb0bc03e6e7a5bc4571d492baf Mon Sep 17 00:00:00 2001 From: Sankar Tanguturi Date: Tue, 8 Mar 2016 12:41:08 -0800 Subject: - Ignored return code 1 for 'pkill' command in config_nic.py - Added few utility functions to report events to the underlying VMware Virtualization platform - Re-factored code little bit. - Executed ./tools/run-pep8 and no pep8 errors were reported. --- cloudinit/sources/helpers/vmware/imc/config_nic.py | 14 ++-- .../sources/helpers/vmware/imc/guestcust_error.py | 24 +++++++ .../sources/helpers/vmware/imc/guestcust_event.py | 27 ++++++++ .../sources/helpers/vmware/imc/guestcust_state.py | 25 +++++++ .../sources/helpers/vmware/imc/guestcust_util.py | 79 ++++++++++++++++++++++ 5 files changed, 163 insertions(+), 6 deletions(-) create mode 100644 cloudinit/sources/helpers/vmware/imc/guestcust_error.py create mode 100644 cloudinit/sources/helpers/vmware/imc/guestcust_event.py create mode 100644 cloudinit/sources/helpers/vmware/imc/guestcust_state.py create mode 100644 cloudinit/sources/helpers/vmware/imc/guestcust_util.py (limited to 'cloudinit/sources/helpers') diff --git a/cloudinit/sources/helpers/vmware/imc/config_nic.py b/cloudinit/sources/helpers/vmware/imc/config_nic.py index 172a1649..42fbcc7e 100644 --- a/cloudinit/sources/helpers/vmware/imc/config_nic.py +++ b/cloudinit/sources/helpers/vmware/imc/config_nic.py @@ -47,12 +47,12 @@ class NicConfigurator: """ primary_nics = [nic for nic in self.nics if nic.primary] if not primary_nics: - return None + return None elif len(primary_nics) > 1: - raise Exception('There can only be one primary nic', + raise Exception('There can only be one primary nic', [nic.mac for nic in primary_nics]) else: - return primary_nics[0] + return primary_nics[0] def find_devices(self): """ @@ -186,8 +186,9 @@ class NicConfigurator: lines = [] for addr in addrs: - lines.append(' up route -A inet6 add default gw %s metric 10000' % - addr.gateway) + lines.append( + ' up route -A inet6 add default gw %s metric 10000' % + addr.gateway) return lines @@ -206,7 +207,8 @@ class NicConfigurator: def clear_dhcp(self): logger.info('Clearing DHCP leases') - util.subp(["pkill", "dhclient"]) + # Ignore the return code 1. + util.subp(["pkill", "dhclient"], rcs=[0, 1]) util.subp(["rm", "-f", "/var/lib/dhcp/*"]) def if_down_up(self): diff --git a/cloudinit/sources/helpers/vmware/imc/guestcust_error.py b/cloudinit/sources/helpers/vmware/imc/guestcust_error.py new file mode 100644 index 00000000..1b04161f --- /dev/null +++ b/cloudinit/sources/helpers/vmware/imc/guestcust_error.py @@ -0,0 +1,24 @@ +# vi: ts=4 expandtab +# +# Copyright (C) 2016 Canonical Ltd. +# Copyright (C) 2016 VMware Inc. +# +# Author: Sankar Tanguturi +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 3, as +# published by the Free Software Foundation. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + + +class GuestCustErrorEnum: + """Specifies different errors of Guest Customization engine""" + + GUESTCUST_ERROR_SUCCESS = 0 diff --git a/cloudinit/sources/helpers/vmware/imc/guestcust_event.py b/cloudinit/sources/helpers/vmware/imc/guestcust_event.py new file mode 100644 index 00000000..fc22568f --- /dev/null +++ b/cloudinit/sources/helpers/vmware/imc/guestcust_event.py @@ -0,0 +1,27 @@ +# vi: ts=4 expandtab +# +# Copyright (C) 2016 Canonical Ltd. +# Copyright (C) 2016 VMware Inc. +# +# Author: Sankar Tanguturi +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 3, as +# published by the Free Software Foundation. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + + +class GuestCustEventEnum: + """Specifies different types of Guest Customization Events""" + + GUESTCUST_EVENT_CUSTOMIZE_FAILED = 100 + GUESTCUST_EVENT_NETWORK_SETUP_FAILED = 101 + GUESTCUST_EVENT_ENABLE_NICS = 103 + GUESTCUST_EVENT_QUERY_NICS = 104 diff --git a/cloudinit/sources/helpers/vmware/imc/guestcust_state.py b/cloudinit/sources/helpers/vmware/imc/guestcust_state.py new file mode 100644 index 00000000..f255be5f --- /dev/null +++ b/cloudinit/sources/helpers/vmware/imc/guestcust_state.py @@ -0,0 +1,25 @@ +# vi: ts=4 expandtab +# +# Copyright (C) 2016 Canonical Ltd. +# Copyright (C) 2016 VMware Inc. +# +# Author: Sankar Tanguturi +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 3, as +# published by the Free Software Foundation. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + + +class GuestCustStateEnum: + """Specifies different states of Guest Customization engine""" + + GUESTCUST_STATE_RUNNING = 4 + GUESTCUST_STATE_DONE = 5 diff --git a/cloudinit/sources/helpers/vmware/imc/guestcust_util.py b/cloudinit/sources/helpers/vmware/imc/guestcust_util.py new file mode 100644 index 00000000..2466a47e --- /dev/null +++ b/cloudinit/sources/helpers/vmware/imc/guestcust_util.py @@ -0,0 +1,79 @@ +# vi: ts=4 expandtab +# +# Copyright (C) 2016 Canonical Ltd. +# Copyright (C) 2016 VMware Inc. +# +# Author: Sankar Tanguturi +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 3, as +# published by the Free Software Foundation. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +import logging +import os + +from cloudinit import util + + +logger = logging.getLogger(__name__) + + +CLOUDINIT_LOG_FILE = "/var/log/cloud-init.log" + + +# This will send a RPC command to the underlying +# VMware Virtualization Platform. +def send_rpc(rpc): + if not rpc: + return None + + rc = 1 + output = "Error sending the RPC command" + + try: + logger.debug("Sending RPC command: %s", rpc) + (rc, output) = util.subp(["vmware-rpctool", rpc], rcs=[0]) + except Exception as e: + logger.debug("Failed to send RPC command") + logger.exception(e) + + return (rc, output) + + +# This will send the customization status to the +# underlying VMware Virtualization Platform. +def set_customization_status(custstate, custerror, errormessage=None): + message = "" + + if errormessage: + message = CLOUDINIT_LOG_FILE + "@" + errormessage + else: + message = CLOUDINIT_LOG_FILE + + rpc = "deployPkg.update.state %d %d %s" % (custstate, custerror, message) + (rc, output) = send_rpc(rpc) + + +# This will read the file nics.txt in the specified directory +# and return the content +def get_nics_to_enable(dirpath): + if not dirpath: + return None + + NICS_SIZE = 1024 + nicsfilepath = os.path.join(dirpath, "nics.txt") + if not os.path.exists(nicsfilepath): + return None + + with open(nicsfilepath, 'r') as fp: + nics = fp.read(NICS_SIZE) + + return nics -- cgit v1.2.3 From a6e0922a4d34ede6df000dd8fc4bb3531218d69f Mon Sep 17 00:00:00 2001 From: Sankar Tanguturi Date: Wed, 9 Mar 2016 16:02:34 -0800 Subject: - Fixed few issues with return values form util.subp() - Added a new utility method to send a RPC for enabling NICS - Modified DataSourceOVF.py to enable nics. - Executed ./tools/run-pep8 and no issues were reported. --- cloudinit/sources/DataSourceOVF.py | 11 +++- .../sources/helpers/vmware/imc/guestcust_util.py | 60 ++++++++++++++++++++-- 2 files changed, 64 insertions(+), 7 deletions(-) (limited to 'cloudinit/sources/helpers') diff --git a/cloudinit/sources/DataSourceOVF.py b/cloudinit/sources/DataSourceOVF.py index 0fbdf0b8..bc13b71a 100644 --- a/cloudinit/sources/DataSourceOVF.py +++ b/cloudinit/sources/DataSourceOVF.py @@ -40,8 +40,11 @@ from cloudinit.sources.helpers.vmware.imc.guestcust_state import \ GuestCustStateEnum from cloudinit.sources.helpers.vmware.imc.guestcust_error import \ GuestCustErrorEnum -from cloudinit.sources.helpers.vmware.imc.guestcust_util import \ - set_customization_status +from cloudinit.sources.helpers.vmware.imc.guestcust_util import ( + set_customization_status, + get_nics_to_enable, + enable_nics +) LOG = logging.getLogger(__name__) @@ -100,10 +103,13 @@ class DataSourceOVF(sources.DataSource): LOG.debug("Customization for VMware platform is disabled.") if vmwareImcConfigFilePath: + nics = "" try: cf = ConfigFile(vmwareImcConfigFilePath) conf = Config(cf) (md, ud, cfg) = read_vmware_imc(conf) + dirpath = os.path.dirname(vmwareImcConfigFilePath) + nics = get_nics_to_enable(dirpath) except Exception as e: LOG.debug("Error parsing the customization Config File") LOG.exception(e) @@ -128,6 +134,7 @@ class DataSourceOVF(sources.DataSource): return False vmwarePlatformFound = True + enable_nics(nics) set_customization_status( GuestCustStateEnum.GUESTCUST_STATE_DONE, GuestCustErrorEnum.GUESTCUST_ERROR_SUCCESS) diff --git a/cloudinit/sources/helpers/vmware/imc/guestcust_util.py b/cloudinit/sources/helpers/vmware/imc/guestcust_util.py index 2466a47e..b8c58f1e 100644 --- a/cloudinit/sources/helpers/vmware/imc/guestcust_util.py +++ b/cloudinit/sources/helpers/vmware/imc/guestcust_util.py @@ -19,14 +19,20 @@ import logging import os +import time from cloudinit import util +from .guestcust_state import GuestCustStateEnum +from .guestcust_error import GuestCustErrorEnum +from .guestcust_event import GuestCustEventEnum logger = logging.getLogger(__name__) CLOUDINIT_LOG_FILE = "/var/log/cloud-init.log" +QUERY_NICS_SUPPORTED = "queryNicsSupported" +NICS_STATUS_CONNECTED = "connected" # This will send a RPC command to the underlying @@ -35,17 +41,20 @@ def send_rpc(rpc): if not rpc: return None - rc = 1 - output = "Error sending the RPC command" + out = "" + err = "Error sending the RPC command" try: logger.debug("Sending RPC command: %s", rpc) - (rc, output) = util.subp(["vmware-rpctool", rpc], rcs=[0]) + (out, err) = util.subp(["vmware-rpctool", rpc], rcs=[0]) + # Remove the trailing newline in the output. + if out: + out = out.rstrip() except Exception as e: logger.debug("Failed to send RPC command") logger.exception(e) - return (rc, output) + return (out, err) # This will send the customization status to the @@ -59,7 +68,8 @@ def set_customization_status(custstate, custerror, errormessage=None): message = CLOUDINIT_LOG_FILE rpc = "deployPkg.update.state %d %d %s" % (custstate, custerror, message) - (rc, output) = send_rpc(rpc) + (out, err) = send_rpc(rpc) + return (out, err) # This will read the file nics.txt in the specified directory @@ -77,3 +87,43 @@ def get_nics_to_enable(dirpath): nics = fp.read(NICS_SIZE) return nics + + +# This will send a RPC command to the underlying VMware Virtualization platform +# and enable nics. +def enable_nics(nics): + if not nics: + logger.warning("No Nics found") + return + + enableNicsWaitRetries = 5 + enableNicsWaitCount = 5 + enableNicsWaitSeconds = 1 + + for attempt in range(0, enableNicsWaitRetries): + logger.debug("Trying to connect interfaces, attempt %d", attempt) + (out, err) = set_customization_status( + GuestCustStateEnum.GUESTCUST_STATE_RUNNING, + GuestCustEventEnum.GUESTCUST_EVENT_ENABLE_NICS, + nics) + if not out: + time.sleep(enableNicsWaitCount * enableNicsWaitSeconds) + continue + + if out != QUERY_NICS_SUPPORTED: + logger.warning("NICS connection status query is not supported") + return + + for count in range(0, enableNicsWaitCount): + (out, err) = set_customization_status( + GuestCustStateEnum.GUESTCUST_STATE_RUNNING, + GuestCustEventEnum.GUESTCUST_EVENT_QUERY_NICS, + nics) + if out and out == NICS_STATUS_CONNECTED: + logger.info("NICS are connected on %d second", count) + return + + time.sleep(enableNicsWaitSeconds) + + logger.warning("Can't connect network interfaces after %d attempts", + enableNicsWaitRetries) -- cgit v1.2.3 From 03998cd336b3906dc1eb675fff1ddeb1272668d3 Mon Sep 17 00:00:00 2001 From: Sankar Tanguturi Date: Fri, 11 Mar 2016 13:29:28 -0800 Subject: - Fixed few pep8 and flake8 issues. - Changed the really long 'from ... import ...' statements. --- cloudinit/sources/DataSourceOVF.py | 21 +++++++++------------ .../sources/helpers/vmware/imc/guestcust_util.py | 13 ++++++------- 2 files changed, 15 insertions(+), 19 deletions(-) (limited to 'cloudinit/sources/helpers') diff --git a/cloudinit/sources/DataSourceOVF.py b/cloudinit/sources/DataSourceOVF.py index fec13b93..65cefc48 100644 --- a/cloudinit/sources/DataSourceOVF.py +++ b/cloudinit/sources/DataSourceOVF.py @@ -31,16 +31,13 @@ import time from cloudinit import log as logging from cloudinit import sources from cloudinit import util -from cloudinit.sources.helpers.vmware.imc.config import Config -from cloudinit.sources.helpers.vmware.imc.config_file import ConfigFile -from cloudinit.sources.helpers.vmware.imc.config_nic import NicConfigurator -from cloudinit.sources.helpers.vmware.imc.guestcust_event import \ - GuestCustEventEnum -from cloudinit.sources.helpers.vmware.imc.guestcust_state import \ - GuestCustStateEnum -from cloudinit.sources.helpers.vmware.imc.guestcust_error import \ - GuestCustErrorEnum -from cloudinit.sources.helpers.vmware.imc.guestcust_util import ( +from .helpers.vmware.imc.config import Config +from .helpers.vmware.imc.config_file import ConfigFile +from .helpers.vmware.imc.config_nic import NicConfigurator +from .helpers.vmware.imc.guestcust_event import GuestCustEventEnum +from .helpers.vmware.imc.guestcust_state import GuestCustStateEnum +from .helpers.vmware.imc.guestcust_error import GuestCustErrorEnum +from .helpers.vmware.imc.guestcust_util import ( set_customization_status, get_nics_to_enable, enable_nics @@ -135,8 +132,8 @@ class DataSourceOVF(sources.DataSource): vmwarePlatformFound = True enable_nics(nics) set_customization_status( - GuestCustStateEnum.GUESTCUST_STATE_DONE, - GuestCustErrorEnum.GUESTCUST_ERROR_SUCCESS) + GuestCustStateEnum.GUESTCUST_STATE_DONE, + GuestCustErrorEnum.GUESTCUST_ERROR_SUCCESS) elif seedfile: # Found a seed dir seed = os.path.join(self.paths.seed_dir, seedfile) diff --git a/cloudinit/sources/helpers/vmware/imc/guestcust_util.py b/cloudinit/sources/helpers/vmware/imc/guestcust_util.py index b8c58f1e..d39f0a65 100644 --- a/cloudinit/sources/helpers/vmware/imc/guestcust_util.py +++ b/cloudinit/sources/helpers/vmware/imc/guestcust_util.py @@ -24,7 +24,6 @@ import time from cloudinit import util from .guestcust_state import GuestCustStateEnum -from .guestcust_error import GuestCustErrorEnum from .guestcust_event import GuestCustEventEnum logger = logging.getLogger(__name__) @@ -103,9 +102,9 @@ def enable_nics(nics): for attempt in range(0, enableNicsWaitRetries): logger.debug("Trying to connect interfaces, attempt %d", attempt) (out, err) = set_customization_status( - GuestCustStateEnum.GUESTCUST_STATE_RUNNING, - GuestCustEventEnum.GUESTCUST_EVENT_ENABLE_NICS, - nics) + GuestCustStateEnum.GUESTCUST_STATE_RUNNING, + GuestCustEventEnum.GUESTCUST_EVENT_ENABLE_NICS, + nics) if not out: time.sleep(enableNicsWaitCount * enableNicsWaitSeconds) continue @@ -116,9 +115,9 @@ def enable_nics(nics): for count in range(0, enableNicsWaitCount): (out, err) = set_customization_status( - GuestCustStateEnum.GUESTCUST_STATE_RUNNING, - GuestCustEventEnum.GUESTCUST_EVENT_QUERY_NICS, - nics) + GuestCustStateEnum.GUESTCUST_STATE_RUNNING, + GuestCustEventEnum.GUESTCUST_EVENT_QUERY_NICS, + nics) if out and out == NICS_STATUS_CONNECTED: logger.info("NICS are connected on %d second", count) return -- cgit v1.2.3 From 2b85dabb802766e0b3b1949d744c8860c0cb838a Mon Sep 17 00:00:00 2001 From: Ryan Harper Date: Wed, 23 Mar 2016 11:05:22 -0500 Subject: configdata: parse and convert openstack network_data json to network_config --- cloudinit/net/__init__.py | 34 +++-- cloudinit/net/network_state.py | 45 ++++++- cloudinit/sources/DataSourceConfigDrive.py | 137 +++++++++++++++++++++ cloudinit/sources/helpers/openstack.py | 34 ++++- .../unittests/test_datasource/test_configdrive.py | 50 +++++++- 5 files changed, 289 insertions(+), 11 deletions(-) (limited to 'cloudinit/sources/helpers') diff --git a/cloudinit/net/__init__.py b/cloudinit/net/__init__.py index ae7b1c04..76cd4e8b 100644 --- a/cloudinit/net/__init__.py +++ b/cloudinit/net/__init__.py @@ -336,7 +336,7 @@ def iface_add_attrs(iface): 'index', 'subnets', ] - if iface['type'] not in ['bond', 'bridge']: + if iface['type'] not in ['bond', 'bridge', 'vlan']: ignore_map.append('mac_address') for key, value in iface.items(): @@ -348,19 +348,34 @@ def iface_add_attrs(iface): return content -def render_route(route): - content = "up route add" +def render_route(route, indent=""): + content = "" + up = indent + "post-up route add" + down = indent + "pre-down route del" + eol = " || true\n" mapping = { 'network': '-net', 'netmask': 'netmask', 'gateway': 'gw', 'metric': 'metric', } - for k in ['network', 'netmask', 'gateway', 'metric']: - if k in route: - content += " %s %s" % (mapping[k], route[k]) + if route['network'] == '0.0.0.0' and route['netmask'] == '0.0.0.0': + default_gw = " default gw %s" % route['gateway'] + content += up + default_gw + eol + content += down + default_gw + eol + elif route['network'] == '::' and route['netmask'] == 0: + # ipv6! + default_gw = " -A inet6 default gw %s" % route['gateway'] + content += up + default_gw + eol + content += down + default_gw + eol + else: + route_line = "" + for k in ['network', 'netmask', 'gateway', 'metric']: + if k in route: + route_line += " %s %s" % (mapping[k], route[k]) + content += up + route_line + eol + content += down + route_line + eol - content += '\n' return content @@ -384,6 +399,7 @@ def render_interfaces(network_state): if len(value): content += " dns-{} {}\n".format(dnskey, " ".join(value)) + content += "\n" for iface in sorted(interfaces.values(), key=lambda k: (order[k['type']], k['name'])): content += "auto {name}\n".format(**iface) @@ -409,6 +425,8 @@ def render_interfaces(network_state): content += iface_add_subnet(iface, subnet) content += iface_add_attrs(iface) + for route in subnet.get('routes', []): + content += render_route(route, indent=" ") content += "\n" else: content += "iface {name} {inet} {mode}\n".format(**iface) @@ -419,7 +437,7 @@ def render_interfaces(network_state): content += render_route(route) # global replacements until v2 format - content = content.replace('mac_address', 'hwaddress') + content = content.replace('mac_address', 'hwaddress ether') return content diff --git a/cloudinit/net/network_state.py b/cloudinit/net/network_state.py index df04c526..e32d2cdf 100644 --- a/cloudinit/net/network_state.py +++ b/cloudinit/net/network_state.py @@ -124,6 +124,17 @@ class NetworkState: iface = interfaces.get(command['name'], {}) for param, val in command.get('params', {}).items(): iface.update({param: val}) + + # convert subnet ipv6 netmask to cidr as needed + subnets = command.get('subnets') + if subnets: + for subnet in subnets: + if subnet['type'] == 'static': + if 'netmask' in subnet and ':' in subnet['address']: + subnet['netmask'] = mask2cidr(subnet['netmask']) + for route in subnet.get('routes', []): + if 'netmask' in route: + route['netmask'] = mask2cidr(route['netmask']) iface.update({ 'name': command.get('name'), 'type': command.get('type'), @@ -133,7 +144,7 @@ class NetworkState: 'mtu': command.get('mtu'), 'address': None, 'gateway': None, - 'subnets': command.get('subnets'), + 'subnets': subnets, }) self.network_state['interfaces'].update({command.get('name'): iface}) self.dump_network_state() @@ -144,6 +155,7 @@ class NetworkState: iface eth0.222 inet static address 10.10.10.1 netmask 255.255.255.0 + hwaddress ether BC:76:4E:06:96:B3 vlan-raw-device eth0 ''' required_keys = [ @@ -335,6 +347,37 @@ def cidr2mask(cidr): return ".".join([str(x) for x in mask]) +def ipv4mask2cidr(mask): + if '.' not in mask: + return mask + return sum([bin(int(x)).count('1') for x in mask.split('.')]) + + +def ipv6mask2cidr(mask): + if ':' not in mask: + return mask + + bitCount = [0, 0x8000, 0xc000, 0xe000, 0xf000, 0xf800, 0xfc00, 0xfe00, + 0xff00, 0xff80, 0xffc0, 0xffe0, 0xfff0, 0xfff8, 0xfffc, + 0xfffe, 0xffff] + cidr = 0 + for word in mask.split(':'): + if not word or int(word, 16) == 0: + break + cidr += bitCount.index(int(word, 16)) + + return cidr + + +def mask2cidr(mask): + if ':' in mask: + return ipv6mask2cidr(mask) + elif '.' in mask: + return ipv4mask2cidr(mask) + else: + return mask + + if __name__ == '__main__': import sys import random diff --git a/cloudinit/sources/DataSourceConfigDrive.py b/cloudinit/sources/DataSourceConfigDrive.py index 6fc9e05b..d84fab54 100644 --- a/cloudinit/sources/DataSourceConfigDrive.py +++ b/cloudinit/sources/DataSourceConfigDrive.py @@ -18,6 +18,7 @@ # You should have received a copy of the GNU General Public License # along with this program. If not, see . +import copy import os from cloudinit import log as logging @@ -50,6 +51,8 @@ class DataSourceConfigDrive(openstack.SourceMixin, sources.DataSource): self.seed_dir = os.path.join(paths.seed_dir, 'config_drive') self.version = None self.ec2_metadata = None + self._network_config = None + self.network_json = None self.files = {} def __str__(self): @@ -144,12 +147,27 @@ class DataSourceConfigDrive(openstack.SourceMixin, sources.DataSource): LOG.warn("Invalid content in vendor-data: %s", e) self.vendordata_raw = None + nd = results.get('networkdata') + self.networkdata_pure = nd + try: + self.network_json = openstack.convert_networkdata_json(nd) + except ValueError as e: + LOG.warn("Invalid content in network-data: %s", e) + self.network_json = None + + if self.network_json: + self._network_config = convert_network_data(self.network_json) + return True def check_instance_id(self): # quickly (local check only) if self.instance_id is still valid return sources.instance_id_matches_system_uuid(self.get_instance_id()) + @property + def network_config(self): + return self._network_config + class DataSourceConfigDriveNet(DataSourceConfigDrive): def __init__(self, sys_cfg, distro, paths): @@ -287,3 +305,122 @@ datasources = [ # Return a list of data sources that match this set of dependencies def get_datasource_list(depends): return sources.list_from_depends(depends, datasources) + + +# Convert OpenStack ConfigDrive NetworkData json to network_config yaml +def convert_network_data(network_json=None): + """Return a dictionary of network_config by parsing provided + OpenStack ConfigDrive NetworkData json format + + OpenStack network_data.json provides a 3 element dictionary + - "links" (links are network devices, physical or virtual) + - "networks" (networks are ip network configurations for one or more + links) + - services (non-ip services, like dns) + + networks and links are combined via network items referencing specific + links via a 'link_id' which maps to a links 'id' field. + + To convert this format to network_config yaml, we first iterate over the + links and then walk the network list to determine if any of the networks + utilize the current link; if so we generate a subnet entry for the device + + We also need to map network_data.json fields to network_config fields. For + example, the network_data links 'id' field is equivalent to network_config + 'name' field for devices. We apply more of this mapping to the various + link types that we encounter. + + There are additional fields that are populated in the network_data.json + from OpenStack that are not relevant to network_config yaml, so we + enumerate a dictionary of valid keys for network_yaml and apply filtering + to drop these superflous keys from the network_config yaml. + """ + if network_json is None: + return None + + # dict of network_config key for filtering network_json + valid_keys = { + 'physical': [ + 'name', + 'type', + 'mac_address', + 'subnets', + 'params', + ], + 'subnet': [ + 'type', + 'address', + 'netmask', + 'broadcast', + 'metric', + 'gateway', + 'pointopoint', + 'mtu', + 'scope', + 'dns_nameservers', + 'dns_search', + 'routes', + ], + } + + links = network_json.get('links', []) + networks = network_json.get('networks', []) + services = network_json.get('services', []) + + config = [] + for link in links: + subnets = [] + cfg = {k: v for k, v in link.items() + if k in valid_keys['physical']} + cfg.update({'name': link['id']}) + for network in [net for net in networks + if net['link'] == link['id']]: + subnet = {k: v for k, v in network.items() + if k in valid_keys['subnet']} + if 'dhcp' in network['type']: + t = 'dhcp6' if network['type'].startswith('ipv6') else 'dhcp4' + subnet.update({ + 'type': t, + }) + else: + subnet.update({ + 'type': 'static', + 'address': network.get('ip_address'), + }) + subnets.append(subnet) + cfg.update({'subnets': subnets}) + if link['type'] in ['ethernet', 'vif', 'ovs']: + cfg.update({ + 'type': 'physical', + 'mac_address': link['ethernet_mac_address']}) + elif link['type'] in ['bond']: + params = {} + for k, v in link.items(): + if k == 'bond_links': + continue + elif k.startswith('bond'): + params.update({k: v}) + cfg.update({ + 'bond_interfaces': copy.deepcopy(link['bond_links']), + 'params': params, + }) + elif link['type'] in ['vlan']: + cfg.update({ + 'name': "%s.%s" % (link['vlan_link'], + link['vlan_id']), + 'vlan_link': link['vlan_link'], + 'vlan_id': link['vlan_id'], + 'mac_address': link['vlan_mac_address'], + }) + else: + raise ValueError( + 'Unknown network_data link type: %s' % link['type']) + + config.append(cfg) + + for service in services: + cfg = service + cfg.update({'type': 'nameserver'}) + config.append(cfg) + + return {'version': 1, 'config': config} diff --git a/cloudinit/sources/helpers/openstack.py b/cloudinit/sources/helpers/openstack.py index bd93d22f..eb50a7be 100644 --- a/cloudinit/sources/helpers/openstack.py +++ b/cloudinit/sources/helpers/openstack.py @@ -51,11 +51,13 @@ OS_LATEST = 'latest' OS_FOLSOM = '2012-08-10' OS_GRIZZLY = '2013-04-04' OS_HAVANA = '2013-10-17' +OS_KILO = '2015-10-15' # keep this in chronological order. new supported versions go at the end. OS_VERSIONS = ( OS_FOLSOM, OS_GRIZZLY, OS_HAVANA, + OS_KILO, ) @@ -229,6 +231,11 @@ class BaseReader(object): False, load_json_anytype, ) + files['networkdata'] = ( + self._path_join("openstack", version, 'network_data.json'), + False, + load_json_anytype, + ) return files results = { @@ -334,7 +341,7 @@ class ConfigDriveReader(BaseReader): path = self._path_join(self.base_path, 'openstack') found = [d for d in os.listdir(path) if os.path.isdir(os.path.join(path))] - self._versions = found + self._versions = sorted(found) return self._versions def _read_ec2_metadata(self): @@ -490,3 +497,28 @@ def convert_vendordata_json(data, recurse=True): recurse=False) raise ValueError("vendordata['cloud-init'] cannot be dict") raise ValueError("Unknown data type for vendordata: %s" % type(data)) + + +def convert_networkdata_json(data, recurse=True): + """ data: a loaded json *object* (strings, arrays, dicts). + return something suitable for cloudinit networkdata_raw. + + if data is: + None: return None + string: return string + list: return data + the list is then processed in UserDataProcessor + dict: return convert_networkdata_json(data.get('cloud-init')) + """ + if not data: + return None + if isinstance(data, six.string_types): + return data + if isinstance(data, list): + return copy.deepcopy(data) + if isinstance(data, dict): + if recurse is True: + return convert_networkdata_json(data.get('cloud-init'), + recurse=False) + raise ValueError("networkdata['cloud-init'] cannot be dict") + raise ValueError("Unknown data type for networkdata: %s" % type(data)) diff --git a/tests/unittests/test_datasource/test_configdrive.py b/tests/unittests/test_datasource/test_configdrive.py index bfd787d1..01f8c5ce 100644 --- a/tests/unittests/test_datasource/test_configdrive.py +++ b/tests/unittests/test_datasource/test_configdrive.py @@ -59,6 +59,34 @@ OSTACK_META = { CONTENT_0 = b'This is contents of /etc/foo.cfg\n' CONTENT_1 = b'# this is /etc/bar/bar.cfg\n' +NETWORK_DATA = { + 'services': [ + {'type': 'dns', 'address': '199.204.44.24'}, + {'type': 'dns', 'address': '199.204.47.54'} + ], + 'links': [ + {'vif_id': '2ecc7709-b3f7-4448-9580-e1ec32d75bbd', + 'ethernet_mac_address': 'fa:16:3e:69:b0:58', + 'type': 'ovs', 'mtu': None, 'id': 'tap2ecc7709-b3'}, + {'vif_id': '2f88d109-5b57-40e6-af32-2472df09dc33', + 'ethernet_mac_address': 'fa:16:3e:d4:57:ad', + 'type': 'ovs', 'mtu': None, 'id': 'tap2f88d109-5b'}, + {'vif_id': '1a5382f8-04c5-4d75-ab98-d666c1ef52cc', + 'ethernet_mac_address': 'fa:16:3e:05:30:fe', + 'type': 'ovs', 'mtu': None, 'id': 'tap1a5382f8-04'} + ], + 'networks': [ + {'link': 'tap2ecc7709-b3', 'type': 'ipv4_dhcp', + 'network_id': '6d6357ac-0f70-4afa-8bd7-c274cc4ea235', + 'id': 'network0'}, + {'link': 'tap2f88d109-5b', 'type': 'ipv4_dhcp', + 'network_id': 'd227a9b3-6960-4d94-8976-ee5788b44f54', + 'id': 'network1'}, + {'link': 'tap1a5382f8-04', 'type': 'ipv4_dhcp', + 'network_id': 'dab2ba57-cae2-4311-a5ed-010b263891f5', + 'id': 'network2'} + ] +} CFG_DRIVE_FILES_V2 = { 'ec2/2009-04-04/meta-data.json': json.dumps(EC2_META), @@ -70,7 +98,11 @@ CFG_DRIVE_FILES_V2 = { 'openstack/content/0000': CONTENT_0, 'openstack/content/0001': CONTENT_1, 'openstack/latest/meta_data.json': json.dumps(OSTACK_META), - 'openstack/latest/user_data': USER_DATA} + 'openstack/latest/user_data': USER_DATA, + 'openstack/latest/network_data.json': json.dumps(NETWORK_DATA), + 'openstack/2015-10-15/meta_data.json': json.dumps(OSTACK_META), + 'openstack/2015-10-15/user_data': USER_DATA, + 'openstack/2015-10-15/network_data.json': json.dumps(NETWORK_DATA)} class TestConfigDriveDataSource(TestCase): @@ -225,6 +257,7 @@ class TestConfigDriveDataSource(TestCase): self.assertEqual(USER_DATA, found['userdata']) self.assertEqual(expected_md, found['metadata']) + self.assertEqual(NETWORK_DATA, found['networkdata']) self.assertEqual(found['files']['/etc/foo.cfg'], CONTENT_0) self.assertEqual(found['files']['/etc/bar/bar.cfg'], CONTENT_1) @@ -321,6 +354,19 @@ class TestConfigDriveDataSource(TestCase): self.assertEqual(myds.get_public_ssh_keys(), [OSTACK_META['public_keys']['mykey']]) + def test_network_data_is_found(self): + """Verify that network_data is present in ds in config-drive-v2.""" + populate_dir(self.tmp, CFG_DRIVE_FILES_V2) + myds = cfg_ds_from_dir(self.tmp) + self.assertEqual(myds.network_json, NETWORK_DATA) + + def test_network_config_is_converted(self): + """Verify that network_data is converted and present on ds object.""" + populate_dir(self.tmp, CFG_DRIVE_FILES_V2) + myds = cfg_ds_from_dir(self.tmp) + network_config = ds.convert_network_data(NETWORK_DATA) + self.assertEqual(myds.network_config, network_config) + def cfg_ds_from_dir(seed_d): found = ds.read_config_drive(seed_d) @@ -339,6 +385,8 @@ def populate_ds_from_read_config(cfg_ds, source, results): cfg_ds.ec2_metadata = results.get('ec2-metadata') cfg_ds.userdata_raw = results.get('userdata') cfg_ds.version = results.get('version') + cfg_ds.network_json = results.get('networkdata') + cfg_ds._network_config = ds.convert_network_data(cfg_ds.network_json) def populate_dir(seed_dir, files): -- cgit v1.2.3 From 6b79e2c6f9a7342163691be9e785cef1aa642541 Mon Sep 17 00:00:00 2001 From: Ryan Harper Date: Wed, 23 Mar 2016 14:17:10 -0500 Subject: fix openstack versions s/KILO/LIBERY drop networkdata read helper --- cloudinit/sources/DataSourceConfigDrive.py | 2 +- cloudinit/sources/helpers/openstack.py | 29 ++--------------------------- 2 files changed, 3 insertions(+), 28 deletions(-) (limited to 'cloudinit/sources/helpers') diff --git a/cloudinit/sources/DataSourceConfigDrive.py b/cloudinit/sources/DataSourceConfigDrive.py index d84fab54..15dddefe 100644 --- a/cloudinit/sources/DataSourceConfigDrive.py +++ b/cloudinit/sources/DataSourceConfigDrive.py @@ -150,7 +150,7 @@ class DataSourceConfigDrive(openstack.SourceMixin, sources.DataSource): nd = results.get('networkdata') self.networkdata_pure = nd try: - self.network_json = openstack.convert_networkdata_json(nd) + self.network_json = util.load_json(nd) except ValueError as e: LOG.warn("Invalid content in network-data: %s", e) self.network_json = None diff --git a/cloudinit/sources/helpers/openstack.py b/cloudinit/sources/helpers/openstack.py index eb50a7be..1aa6bbae 100644 --- a/cloudinit/sources/helpers/openstack.py +++ b/cloudinit/sources/helpers/openstack.py @@ -51,13 +51,13 @@ OS_LATEST = 'latest' OS_FOLSOM = '2012-08-10' OS_GRIZZLY = '2013-04-04' OS_HAVANA = '2013-10-17' -OS_KILO = '2015-10-15' +OS_LIBERTY = '2015-10-15' # keep this in chronological order. new supported versions go at the end. OS_VERSIONS = ( OS_FOLSOM, OS_GRIZZLY, OS_HAVANA, - OS_KILO, + OS_LIBERTY, ) @@ -497,28 +497,3 @@ def convert_vendordata_json(data, recurse=True): recurse=False) raise ValueError("vendordata['cloud-init'] cannot be dict") raise ValueError("Unknown data type for vendordata: %s" % type(data)) - - -def convert_networkdata_json(data, recurse=True): - """ data: a loaded json *object* (strings, arrays, dicts). - return something suitable for cloudinit networkdata_raw. - - if data is: - None: return None - string: return string - list: return data - the list is then processed in UserDataProcessor - dict: return convert_networkdata_json(data.get('cloud-init')) - """ - if not data: - return None - if isinstance(data, six.string_types): - return data - if isinstance(data, list): - return copy.deepcopy(data) - if isinstance(data, dict): - if recurse is True: - return convert_networkdata_json(data.get('cloud-init'), - recurse=False) - raise ValueError("networkdata['cloud-init'] cannot be dict") - raise ValueError("Unknown data type for networkdata: %s" % type(data)) -- cgit v1.2.3