From 1f8183ff4750cc7f8798749987ef10912719544d Mon Sep 17 00:00:00 2001 From: Maitreyee Saikia Date: Tue, 15 Aug 2017 09:33:50 -0600 Subject: vcloud directory: Guest Customization support for passwords This feature enables the following VMware VCloud Director functionality: 1. Setting admin password 2. Expire password. 3. Set admin password and expire. Password configuration is triggered only as part of a full recustomization, that happens either on first power on or when "poweron and full recustomization" is selected. Full customization flow is determined by marker files. Unique marker ids are generated when full recustomization is requested. And marker file based on these marker ids help to determine if we need to execute the above configuration. --- cloudinit/sources/DataSourceOVF.py | 63 +++++++++++++++++++++++++++++++++++++- 1 file changed, 62 insertions(+), 1 deletion(-) (limited to 'cloudinit/sources/DataSourceOVF.py') diff --git a/cloudinit/sources/DataSourceOVF.py b/cloudinit/sources/DataSourceOVF.py index f20c9a65..73d38771 100644 --- a/cloudinit/sources/DataSourceOVF.py +++ b/cloudinit/sources/DataSourceOVF.py @@ -25,6 +25,8 @@ from cloudinit.sources.helpers.vmware.imc.config_file \ import ConfigFile from cloudinit.sources.helpers.vmware.imc.config_nic \ import NicConfigurator +from cloudinit.sources.helpers.vmware.imc.config_passwd \ + import PasswordConfigurator from cloudinit.sources.helpers.vmware.imc.guestcust_error \ import GuestCustErrorEnum from cloudinit.sources.helpers.vmware.imc.guestcust_event \ @@ -117,6 +119,8 @@ class DataSourceOVF(sources.DataSource): (md, ud, cfg) = read_vmware_imc(conf) dirpath = os.path.dirname(vmwareImcConfigFilePath) nics = get_nics_to_enable(dirpath) + markerid = conf.marker_id + markerexists = check_marker_exists(markerid) except Exception as e: LOG.debug("Error parsing the customization Config File") LOG.exception(e) @@ -127,7 +131,6 @@ class DataSourceOVF(sources.DataSource): return False finally: util.del_dir(os.path.dirname(vmwareImcConfigFilePath)) - try: LOG.debug("Applying the Network customization") nicConfigurator = NicConfigurator(conf.nics) @@ -140,6 +143,35 @@ class DataSourceOVF(sources.DataSource): GuestCustEventEnum.GUESTCUST_EVENT_NETWORK_SETUP_FAILED) enable_nics(nics) return False + if markerid and not markerexists: + LOG.debug("Applying password customization") + pwdConfigurator = PasswordConfigurator() + adminpwd = conf.admin_password + try: + resetpwd = conf.reset_password + if adminpwd or resetpwd: + pwdConfigurator.configure(adminpwd, resetpwd, + self.distro) + else: + LOG.debug("Changing password is not needed") + except Exception as e: + LOG.debug("Error applying Password Configuration: %s", e) + set_customization_status( + GuestCustStateEnum.GUESTCUST_STATE_RUNNING, + GuestCustEventEnum.GUESTCUST_EVENT_CUSTOMIZE_FAILED) + enable_nics(nics) + return False + if markerid: + LOG.debug("Handle marker creation") + try: + setup_marker_files(markerid) + except Exception as e: + LOG.debug("Error creating marker files: %s", e) + set_customization_status( + GuestCustStateEnum.GUESTCUST_STATE_RUNNING, + GuestCustEventEnum.GUESTCUST_EVENT_CUSTOMIZE_FAILED) + enable_nics(nics) + return False vmwarePlatformFound = True set_customization_status( @@ -445,4 +477,33 @@ datasources = ( def get_datasource_list(depends): return sources.list_from_depends(depends, datasources) + +# To check if marker file exists +def check_marker_exists(markerid): + """ + Check the existence of a marker file. + Presence of marker file determines whether a certain code path is to be + executed. It is needed for partial guest customization in VMware. + """ + if not markerid: + return False + markerfile = "/.markerfile-" + markerid + if os.path.exists(markerfile): + return True + return False + + +# Create a marker file +def setup_marker_files(markerid): + """ + Create a new marker file. + Marker files are unique to a full customization workflow in VMware + environment. + """ + if not markerid: + return + markerfile = "/.markerfile-" + markerid + util.del_file("/.markerfile-*.txt") + open(markerfile, 'w').close() + # vi: ts=4 expandtab -- cgit v1.2.3 From a1dfdda2a2ae20fe026881980ddf7d16110f06e2 Mon Sep 17 00:00:00 2001 From: Sankar Tanguturi Date: Thu, 7 Sep 2017 22:16:16 -0600 Subject: vmware customization: return network config format For customizing the machines hosted on 'VMWare' hypervisor, the datasource should return the 'network config' data in 'curtin' format. This branch also fixes /etc/network/interfaces replacing the line "source /etc/network/interfaces.d/*.cfg" which is incorrectly removed when VMWare's Perl Customization Engine writes /etc/network/interfaces. Modify the code to read the customization configuration and return the converted data. Added few tests. LP: #1675063 --- cloudinit/sources/DataSourceOVF.py | 91 ++++++--- cloudinit/sources/helpers/vmware/imc/config_nic.py | 201 ++++++++++++------- .../sources/helpers/vmware/imc/guestcust_util.py | 12 +- tests/unittests/test_vmware_config_file.py | 217 +++++++++++++++++++++ 4 files changed, 418 insertions(+), 103 deletions(-) (limited to 'cloudinit/sources/DataSourceOVF.py') diff --git a/cloudinit/sources/DataSourceOVF.py b/cloudinit/sources/DataSourceOVF.py index 73d38771..aa5f798d 100644 --- a/cloudinit/sources/DataSourceOVF.py +++ b/cloudinit/sources/DataSourceOVF.py @@ -51,6 +51,10 @@ class DataSourceOVF(sources.DataSource): self.cfg = {} self.supported_seed_starts = ("/", "file://") self.vmware_customization_supported = True + self._network_config = None + self._vmware_nics_to_enable = None + self._vmware_cust_conf = None + self._vmware_cust_found = False def __str__(self): root = sources.DataSource.__str__(self) @@ -60,8 +64,8 @@ class DataSourceOVF(sources.DataSource): found = [] md = {} ud = "" - vmwarePlatformFound = False - vmwareImcConfigFilePath = '' + vmwareImcConfigFilePath = None + nicspath = None defaults = { "instance-id": "iid-dsovf", @@ -101,25 +105,26 @@ class DataSourceOVF(sources.DataSource): logfunc=LOG.debug, msg="waiting for configuration file", func=wait_for_imc_cfg_file, - args=("/var/run/vmware-imc", "cust.cfg", max_wait)) + args=("cust.cfg", max_wait)) if vmwareImcConfigFilePath: LOG.debug("Found VMware Customization Config File at %s", vmwareImcConfigFilePath) + nicspath = wait_for_imc_cfg_file( + filename="nics.txt", maxwait=10, naplen=5) else: LOG.debug("Did not find VMware Customization Config File") else: LOG.debug("Customization for VMware platform is disabled.") if vmwareImcConfigFilePath: - nics = "" + self._vmware_nics_to_enable = "" try: cf = ConfigFile(vmwareImcConfigFilePath) - conf = Config(cf) - (md, ud, cfg) = read_vmware_imc(conf) - dirpath = os.path.dirname(vmwareImcConfigFilePath) - nics = get_nics_to_enable(dirpath) - markerid = conf.marker_id + self._vmware_cust_conf = Config(cf) + (md, ud, cfg) = read_vmware_imc(self._vmware_cust_conf) + self._vmware_nics_to_enable = get_nics_to_enable(nicspath) + markerid = self._vmware_cust_conf.marker_id markerexists = check_marker_exists(markerid) except Exception as e: LOG.debug("Error parsing the customization Config File") @@ -127,28 +132,29 @@ class DataSourceOVF(sources.DataSource): set_customization_status( GuestCustStateEnum.GUESTCUST_STATE_RUNNING, GuestCustEventEnum.GUESTCUST_EVENT_CUSTOMIZE_FAILED) - enable_nics(nics) - return False + raise e finally: util.del_dir(os.path.dirname(vmwareImcConfigFilePath)) try: - LOG.debug("Applying the Network customization") - nicConfigurator = NicConfigurator(conf.nics) - nicConfigurator.configure() + LOG.debug("Preparing the Network configuration") + self._network_config = get_network_config_from_conf( + self._vmware_cust_conf, + True, + True, + self.distro.osfamily) except Exception as e: - LOG.debug("Error applying the Network Configuration") LOG.exception(e) set_customization_status( GuestCustStateEnum.GUESTCUST_STATE_RUNNING, GuestCustEventEnum.GUESTCUST_EVENT_NETWORK_SETUP_FAILED) - enable_nics(nics) - return False + raise e + if markerid and not markerexists: LOG.debug("Applying password customization") pwdConfigurator = PasswordConfigurator() - adminpwd = conf.admin_password + adminpwd = self._vmware_cust_conf.admin_password try: - resetpwd = conf.reset_password + resetpwd = self._vmware_cust_conf.reset_password if adminpwd or resetpwd: pwdConfigurator.configure(adminpwd, resetpwd, self.distro) @@ -159,7 +165,6 @@ class DataSourceOVF(sources.DataSource): set_customization_status( GuestCustStateEnum.GUESTCUST_STATE_RUNNING, GuestCustEventEnum.GUESTCUST_EVENT_CUSTOMIZE_FAILED) - enable_nics(nics) return False if markerid: LOG.debug("Handle marker creation") @@ -170,14 +175,18 @@ class DataSourceOVF(sources.DataSource): set_customization_status( GuestCustStateEnum.GUESTCUST_STATE_RUNNING, GuestCustEventEnum.GUESTCUST_EVENT_CUSTOMIZE_FAILED) - enable_nics(nics) return False - vmwarePlatformFound = True + self._vmware_cust_found = True + found.append('vmware-tools') + + # TODO: Need to set the status to DONE only when the + # customization is done successfully. set_customization_status( GuestCustStateEnum.GUESTCUST_STATE_DONE, GuestCustErrorEnum.GUESTCUST_ERROR_SUCCESS) - enable_nics(nics) + enable_nics(self._vmware_nics_to_enable) + else: np = {'iso': transport_iso9660, 'vmware-guestd': transport_vmware_guestd, } @@ -192,7 +201,7 @@ class DataSourceOVF(sources.DataSource): found.append(name) # There was no OVF transports found - if len(found) == 0 and not vmwarePlatformFound: + if len(found) == 0: return False if 'seedfrom' in md and md['seedfrom']: @@ -237,6 +246,10 @@ class DataSourceOVF(sources.DataSource): def get_config_obj(self): return self.cfg + @property + def network_config(self): + return self._network_config + class DataSourceOVFNet(DataSourceOVF): def __init__(self, sys_cfg, distro, paths): @@ -268,12 +281,13 @@ def get_max_wait_from_cfg(cfg): return max_wait -def wait_for_imc_cfg_file(dirpath, filename, maxwait=180, naplen=5): +def wait_for_imc_cfg_file(filename, maxwait=180, naplen=5, + dirpath="/var/run/vmware-imc"): waited = 0 while waited < maxwait: - fileFullPath = search_file(dirpath, filename) - if fileFullPath: + fileFullPath = os.path.join(dirpath, filename) + if os.path.isfile(fileFullPath): return fileFullPath LOG.debug("Waiting for VMware Customization Config File") time.sleep(naplen) @@ -281,6 +295,26 @@ def wait_for_imc_cfg_file(dirpath, filename, maxwait=180, naplen=5): return None +def get_network_config_from_conf(config, use_system_devices=True, + configure=False, osfamily=None): + nicConfigurator = NicConfigurator(config.nics, use_system_devices) + nics_cfg_list = nicConfigurator.generate(configure, osfamily) + + return get_network_config(nics_cfg_list, + config.name_servers, + config.dns_suffixes) + + +def get_network_config(nics=None, nameservers=None, search=None): + config_list = nics + + if nameservers or search: + config_list.append({'type': 'nameserver', 'address': nameservers, + 'search': search}) + + return {'version': 1, 'config': config_list} + + # This will return a dict with some content # meta-data, user-data, some config def read_vmware_imc(config): @@ -296,6 +330,9 @@ def read_vmware_imc(config): if config.timezone: cfg['timezone'] = config.timezone + # Generate a unique instance-id so that re-customization will + # happen in cloud-init + md['instance-id'] = "iid-vmware-" + util.rand_str(strlen=8) return (md, ud, cfg) diff --git a/cloudinit/sources/helpers/vmware/imc/config_nic.py b/cloudinit/sources/helpers/vmware/imc/config_nic.py index 67ac21db..2fb07c59 100644 --- a/cloudinit/sources/helpers/vmware/imc/config_nic.py +++ b/cloudinit/sources/helpers/vmware/imc/config_nic.py @@ -9,22 +9,48 @@ import logging import os import re +from cloudinit.net.network_state import mask_to_net_prefix from cloudinit import util logger = logging.getLogger(__name__) +def gen_subnet(ip, netmask): + """ + Return the subnet for a given ip address and a netmask + @return (str): the subnet + @param ip: ip address + @param netmask: netmask + """ + ip_array = ip.split(".") + mask_array = netmask.split(".") + result = [] + for index in list(range(4)): + result.append(int(ip_array[index]) & int(mask_array[index])) + + return ".".join([str(x) for x in result]) + + class NicConfigurator(object): - def __init__(self, nics): + def __init__(self, nics, use_system_devices=True): """ Initialize the Nic Configurator @param nics (list) an array of nics to configure + @param use_system_devices (Bool) Get the MAC names from the system + if this is True. If False, then mac names will be retrieved from + the specified nics. """ self.nics = nics self.mac2Name = {} self.ipv4PrimaryGateway = None self.ipv6PrimaryGateway = None - self.find_devices() + + if use_system_devices: + self.find_devices() + else: + for nic in self.nics: + self.mac2Name[nic.mac.lower()] = nic.name + self._primaryNic = self.get_primary_nic() def get_primary_nic(self): @@ -61,138 +87,163 @@ class NicConfigurator(object): def gen_one_nic(self, nic): """ - Return the lines needed to configure a nic - @return (str list): the string list to configure the nic + Return the config list needed to configure a nic + @return (list): the subnets and routes list to configure the nic @param nic (NicBase): the nic to configure """ - lines = [] - name = self.mac2Name.get(nic.mac.lower()) + mac = nic.mac.lower() + name = self.mac2Name.get(mac) if not name: raise ValueError('No known device has MACADDR: %s' % nic.mac) - if nic.onboot: - lines.append('auto %s' % name) + nics_cfg_list = [] + + cfg = {'type': 'physical', 'name': name, 'mac_address': mac} + + subnet_list = [] + route_list = [] # Customize IPv4 - lines.extend(self.gen_ipv4(name, nic)) + (subnets, routes) = self.gen_ipv4(name, nic) + subnet_list.extend(subnets) + route_list.extend(routes) # Customize IPv6 - lines.extend(self.gen_ipv6(name, nic)) + (subnets, routes) = self.gen_ipv6(name, nic) + subnet_list.extend(subnets) + route_list.extend(routes) + + cfg.update({'subnets': subnet_list}) - lines.append('') + nics_cfg_list.append(cfg) + if route_list: + nics_cfg_list.extend(route_list) - return lines + return nics_cfg_list def gen_ipv4(self, name, nic): """ - Return the lines needed to configure the IPv4 setting of a nic - @return (str list): the string list to configure the gateways - @param name (str): name of the nic + Return the set of subnets and routes needed to configure the + IPv4 settings of a nic + @return (set): the set of subnet and routes to configure the gateways + @param name (str): subnet and route list for the nic @param nic (NicBase): the nic to configure """ - lines = [] + + subnet = {} + route_list = [] + + if nic.onboot: + subnet.update({'control': 'auto'}) bootproto = nic.bootProto.lower() if nic.ipv4_mode.lower() == 'disabled': bootproto = 'manual' - lines.append('iface %s inet %s' % (name, bootproto)) if bootproto != 'static': - return lines + subnet.update({'type': 'dhcp'}) + return ([subnet], route_list) + else: + subnet.update({'type': 'static'}) # Static Ipv4 addrs = nic.staticIpv4 if not addrs: - return lines + return ([subnet], route_list) v4 = addrs[0] if v4.ip: - lines.append(' address %s' % v4.ip) + subnet.update({'address': v4.ip}) if v4.netmask: - lines.append(' netmask %s' % v4.netmask) + subnet.update({'netmask': v4.netmask}) # Add the primary gateway if nic.primary and v4.gateways: self.ipv4PrimaryGateway = v4.gateways[0] - lines.append(' gateway %s metric 0' % self.ipv4PrimaryGateway) - return lines + subnet.update({'gateway': self.ipv4PrimaryGateway}) + return [subnet] # Add routes if there is no primary nic if not self._primaryNic: - lines.extend(self.gen_ipv4_route(nic, v4.gateways)) + route_list.extend(self.gen_ipv4_route(nic, + v4.gateways, + v4.netmask)) - return lines + return ([subnet], route_list) - def gen_ipv4_route(self, nic, gateways): + def gen_ipv4_route(self, nic, gateways, netmask): """ - Return the lines needed to configure additional Ipv4 route - @return (str list): the string list to configure the gateways + Return the routes list needed to configure additional Ipv4 route + @return (list): the route list to configure the gateways @param nic (NicBase): the nic to configure @param gateways (str list): the list of gateways """ - lines = [] + route_list = [] + + cidr = mask_to_net_prefix(netmask) for gateway in gateways: - lines.append(' up route add default gw %s metric 10000' % - gateway) + destination = "%s/%d" % (gen_subnet(gateway, netmask), cidr) + route_list.append({'destination': destination, + 'type': 'route', + 'gateway': gateway, + 'metric': 10000}) - return lines + return route_list def gen_ipv6(self, name, nic): """ - Return the lines needed to configure the gateways for a nic - @return (str list): the string list to configure the gateways + Return the set of subnets and routes needed to configure the + gateways for a nic + @return (set): the set of subnets and routes to configure the gateways @param name (str): name of the nic @param nic (NicBase): the nic to configure """ - lines = [] if not nic.staticIpv6: - return lines + return ([], []) + subnet_list = [] # Static Ipv6 addrs = nic.staticIpv6 - lines.append('iface %s inet6 static' % name) - lines.append(' address %s' % addrs[0].ip) - lines.append(' netmask %s' % addrs[0].netmask) - for addr in addrs[1:]: - lines.append(' up ifconfig %s inet6 add %s/%s' % (name, addr.ip, - addr.netmask)) - # Add the primary gateway - if nic.primary: - for addr in addrs: - if addr.gateway: - self.ipv6PrimaryGateway = addr.gateway - lines.append(' gateway %s' % self.ipv6PrimaryGateway) - return lines + for addr in addrs: + subnet = {'type': 'static6', + 'address': addr.ip, + 'netmask': addr.netmask} + subnet_list.append(subnet) - # Add routes if there is no primary nic - if not self._primaryNic: - lines.extend(self._genIpv6Route(name, nic, addrs)) + # TODO: Add the primary gateway + + route_list = [] + # TODO: Add routes if there is no primary nic + # if not self._primaryNic: + # route_list.extend(self._genIpv6Route(name, nic, addrs)) - return lines + return (subnet_list, route_list) def _genIpv6Route(self, name, nic, addrs): - lines = [] + route_list = [] for addr in addrs: - lines.append(' up route -A inet6 add default gw ' - '%s metric 10000' % addr.gateway) + route_list.append({'type': 'route', + 'gateway': addr.gateway, + 'metric': 10000}) + + return route_list - return lines + def generate(self, configure=False, osfamily=None): + """Return the config elements that are needed to configure the nics""" + if configure: + logger.info("Configuring the interfaces file") + self.configure(osfamily) - def generate(self): - """Return the lines that is needed to configure the nics""" - lines = [] - lines.append('iface lo inet loopback') - lines.append('auto lo') - lines.append('') + nics_cfg_list = [] for nic in self.nics: - lines.extend(self.gen_one_nic(nic)) + nics_cfg_list.extend(self.gen_one_nic(nic)) - return lines + return nics_cfg_list def clear_dhcp(self): logger.info('Clearing DHCP leases') @@ -201,11 +252,16 @@ class NicConfigurator(object): util.subp(["pkill", "dhclient"], rcs=[0, 1]) util.subp(["rm", "-f", "/var/lib/dhcp/*"]) - def configure(self): + def configure(self, osfamily=None): """ - Configure the /etc/network/intefaces + Configure the /etc/network/interfaces Make a back up of the original """ + + if not osfamily or osfamily != "debian": + logger.info("Debian OS not detected. Skipping the configure step") + return + containingDir = '/etc/network' interfaceFile = os.path.join(containingDir, 'interfaces') @@ -215,10 +271,13 @@ class NicConfigurator(object): if not os.path.exists(originalFile) and os.path.exists(interfaceFile): os.rename(interfaceFile, originalFile) - lines = self.generate() - with open(interfaceFile, 'w') as fp: - for line in lines: - fp.write('%s\n' % line) + lines = [ + "# DO NOT EDIT THIS FILE BY HAND --" + " AUTOMATICALLY GENERATED BY cloud-init", + "source /etc/network/interfaces.d/*.cfg", + ] + + util.write_file(interfaceFile, content='\n'.join(lines)) self.clear_dhcp() diff --git a/cloudinit/sources/helpers/vmware/imc/guestcust_util.py b/cloudinit/sources/helpers/vmware/imc/guestcust_util.py index 1ab6bd41..44075255 100644 --- a/cloudinit/sources/helpers/vmware/imc/guestcust_util.py +++ b/cloudinit/sources/helpers/vmware/imc/guestcust_util.py @@ -59,14 +59,16 @@ def set_customization_status(custstate, custerror, errormessage=None): return (out, err) -# This will read the file nics.txt in the specified directory -# and return the content -def get_nics_to_enable(dirpath): - if not dirpath: +def get_nics_to_enable(nicsfilepath): + """Reads the NICS from the specified file path and returns the content + + @param nicsfilepath: Absolute file path to the NICS.txt file. + """ + + if not nicsfilepath: return None NICS_SIZE = 1024 - nicsfilepath = os.path.join(dirpath, "nics.txt") if not os.path.exists(nicsfilepath): return None diff --git a/tests/unittests/test_vmware_config_file.py b/tests/unittests/test_vmware_config_file.py index d8651077..808d303a 100644 --- a/tests/unittests/test_vmware_config_file.py +++ b/tests/unittests/test_vmware_config_file.py @@ -8,9 +8,13 @@ import logging import sys +from cloudinit.sources.DataSourceOVF import get_network_config_from_conf +from cloudinit.sources.DataSourceOVF import read_vmware_imc from cloudinit.sources.helpers.vmware.imc.boot_proto import BootProtoEnum from cloudinit.sources.helpers.vmware.imc.config import Config from cloudinit.sources.helpers.vmware.imc.config_file import ConfigFile +from cloudinit.sources.helpers.vmware.imc.config_nic import gen_subnet +from cloudinit.sources.helpers.vmware.imc.config_nic import NicConfigurator from cloudinit.tests.helpers import CiTestCase logging.basicConfig(level=logging.DEBUG, stream=sys.stdout) @@ -20,6 +24,7 @@ logger = logging.getLogger(__name__) class TestVmwareConfigFile(CiTestCase): def test_utility_methods(self): + """Tests basic utility methods of ConfigFile class""" cf = ConfigFile("tests/data/vmware/cust-dhcp-2nic.cfg") cf.clear() @@ -43,7 +48,26 @@ class TestVmwareConfigFile(CiTestCase): self.assertFalse(cf.should_keep_current_value("BAR"), "keepBar") self.assertTrue(cf.should_remove_current_value("BAR"), "removeBar") + def test_datasource_instance_id(self): + """Tests instance id for the DatasourceOVF""" + cf = ConfigFile("tests/data/vmware/cust-dhcp-2nic.cfg") + + instance_id_prefix = 'iid-vmware-' + + conf = Config(cf) + + (md1, _, _) = read_vmware_imc(conf) + self.assertIn(instance_id_prefix, md1["instance-id"]) + self.assertEqual(len(md1["instance-id"]), len(instance_id_prefix) + 8) + + (md2, _, _) = read_vmware_imc(conf) + self.assertIn(instance_id_prefix, md2["instance-id"]) + self.assertEqual(len(md2["instance-id"]), len(instance_id_prefix) + 8) + + self.assertNotEqual(md1["instance-id"], md2["instance-id"]) + def test_configfile_static_2nics(self): + """Tests Config class for a configuration with two static NICs.""" cf = ConfigFile("tests/data/vmware/cust-static-2nic.cfg") conf = Config(cf) @@ -81,6 +105,7 @@ class TestVmwareConfigFile(CiTestCase): self.assertTrue(not nics[1].staticIpv6, "ipv61 dhcp") def test_config_file_dhcp_2nics(self): + """Tests Config class for a configuration with two DHCP NICs.""" cf = ConfigFile("tests/data/vmware/cust-dhcp-2nic.cfg") conf = Config(cf) @@ -117,5 +142,197 @@ class TestVmwareConfigFile(CiTestCase): conf = Config(cf) self.assertTrue(conf.reset_password, "reset password") + def test_get_config_nameservers(self): + """Tests DNS and nameserver settings in a configuration.""" + cf = ConfigFile("tests/data/vmware/cust-static-2nic.cfg") + + config = Config(cf) + + network_config = get_network_config_from_conf(config, False) + + self.assertEqual(1, network_config.get('version')) + + config_types = network_config.get('config') + name_servers = None + dns_suffixes = None + + for type in config_types: + if type.get('type') == 'nameserver': + name_servers = type.get('address') + dns_suffixes = type.get('search') + break + + self.assertEqual(['10.20.145.1', '10.20.145.2'], + name_servers, + "dns") + self.assertEqual(['eng.vmware.com', 'proxy.vmware.com'], + dns_suffixes, + "suffixes") + + def test_gen_subnet(self): + """Tests if gen_subnet properly calculates network subnet from + IPv4 address and netmask""" + ip_subnet_list = [['10.20.87.253', '255.255.252.0', '10.20.84.0'], + ['10.20.92.105', '255.255.252.0', '10.20.92.0'], + ['192.168.0.10', '255.255.0.0', '192.168.0.0']] + for entry in ip_subnet_list: + self.assertEqual(entry[2], gen_subnet(entry[0], entry[1]), + "Subnet for a specified ip and netmask") + + def test_get_config_dns_suffixes(self): + """Tests if get_network_config_from_conf properly + generates nameservers and dns settings from a + specified configuration""" + cf = ConfigFile("tests/data/vmware/cust-dhcp-2nic.cfg") + + config = Config(cf) + + network_config = get_network_config_from_conf(config, False) + + self.assertEqual(1, network_config.get('version')) + + config_types = network_config.get('config') + name_servers = None + dns_suffixes = None + + for type in config_types: + if type.get('type') == 'nameserver': + name_servers = type.get('address') + dns_suffixes = type.get('search') + break + + self.assertEqual([], + name_servers, + "dns") + self.assertEqual(['eng.vmware.com'], + dns_suffixes, + "suffixes") + + def test_get_nics_list_dhcp(self): + """Tests if NicConfigurator properly calculates network subnets + for a configuration with a list of DHCP NICs""" + cf = ConfigFile("tests/data/vmware/cust-dhcp-2nic.cfg") + + config = Config(cf) + + nicConfigurator = NicConfigurator(config.nics, False) + nics_cfg_list = nicConfigurator.generate() + + self.assertEqual(2, len(nics_cfg_list), "number of config elements") + + nic1 = {'name': 'NIC1'} + nic2 = {'name': 'NIC2'} + for cfg in nics_cfg_list: + if cfg.get('name') == nic1.get('name'): + nic1.update(cfg) + elif cfg.get('name') == nic2.get('name'): + nic2.update(cfg) + + self.assertEqual('physical', nic1.get('type'), 'type of NIC1') + self.assertEqual('NIC1', nic1.get('name'), 'name of NIC1') + self.assertEqual('00:50:56:a6:8c:08', nic1.get('mac_address'), + 'mac address of NIC1') + subnets = nic1.get('subnets') + self.assertEqual(1, len(subnets), 'number of subnets for NIC1') + subnet = subnets[0] + self.assertEqual('dhcp', subnet.get('type'), 'DHCP type for NIC1') + self.assertEqual('auto', subnet.get('control'), 'NIC1 Control type') + + self.assertEqual('physical', nic2.get('type'), 'type of NIC2') + self.assertEqual('NIC2', nic2.get('name'), 'name of NIC2') + self.assertEqual('00:50:56:a6:5a:de', nic2.get('mac_address'), + 'mac address of NIC2') + subnets = nic2.get('subnets') + self.assertEqual(1, len(subnets), 'number of subnets for NIC2') + subnet = subnets[0] + self.assertEqual('dhcp', subnet.get('type'), 'DHCP type for NIC2') + self.assertEqual('auto', subnet.get('control'), 'NIC2 Control type') + + def test_get_nics_list_static(self): + """Tests if NicConfigurator properly calculates network subnets + for a configuration with 2 static NICs""" + cf = ConfigFile("tests/data/vmware/cust-static-2nic.cfg") + + config = Config(cf) + + nicConfigurator = NicConfigurator(config.nics, False) + nics_cfg_list = nicConfigurator.generate() + + self.assertEqual(5, len(nics_cfg_list), "number of elements") + + nic1 = {'name': 'NIC1'} + nic2 = {'name': 'NIC2'} + route_list = [] + for cfg in nics_cfg_list: + cfg_type = cfg.get('type') + if cfg_type == 'physical': + if cfg.get('name') == nic1.get('name'): + nic1.update(cfg) + elif cfg.get('name') == nic2.get('name'): + nic2.update(cfg) + elif cfg_type == 'route': + route_list.append(cfg) + + self.assertEqual('physical', nic1.get('type'), 'type of NIC1') + self.assertEqual('NIC1', nic1.get('name'), 'name of NIC1') + self.assertEqual('00:50:56:a6:8c:08', nic1.get('mac_address'), + 'mac address of NIC1') + + subnets = nic1.get('subnets') + self.assertEqual(2, len(subnets), 'Number of subnets') + + static_subnet = [] + static6_subnet = [] + + for subnet in subnets: + subnet_type = subnet.get('type') + if subnet_type == 'static': + static_subnet.append(subnet) + elif subnet_type == 'static6': + static6_subnet.append(subnet) + else: + self.assertEqual(True, False, 'Unknown type') + + self.assertEqual(1, len(static_subnet), 'Number of static subnet') + self.assertEqual(1, len(static6_subnet), 'Number of static6 subnet') + + subnet = static_subnet[0] + self.assertEqual('10.20.87.154', subnet.get('address'), + 'IPv4 address of static subnet') + self.assertEqual('255.255.252.0', subnet.get('netmask'), + 'NetMask of static subnet') + self.assertEqual('auto', subnet.get('control'), + 'control for static subnet') + + subnet = static6_subnet[0] + self.assertEqual('fc00:10:20:87::154', subnet.get('address'), + 'IPv6 address of static subnet') + self.assertEqual('64', subnet.get('netmask'), + 'NetMask of static6 subnet') + + route_set = set(['10.20.87.253', '10.20.87.105', '192.168.0.10']) + for route in route_list: + self.assertEqual(10000, route.get('metric'), 'metric of route') + gateway = route.get('gateway') + if gateway in route_set: + route_set.discard(gateway) + else: + self.assertEqual(True, False, 'invalid gateway %s' % (gateway)) + + self.assertEqual('physical', nic2.get('type'), 'type of NIC2') + self.assertEqual('NIC2', nic2.get('name'), 'name of NIC2') + self.assertEqual('00:50:56:a6:ef:7d', nic2.get('mac_address'), + 'mac address of NIC2') + + subnets = nic2.get('subnets') + self.assertEqual(1, len(subnets), 'Number of subnets for NIC2') + + subnet = subnets[0] + self.assertEqual('static', subnet.get('type'), 'Subnet type') + self.assertEqual('192.168.6.102', subnet.get('address'), + 'Subnet address') + self.assertEqual('255.255.0.0', subnet.get('netmask'), + 'Subnet netmask') + # vi: ts=4 expandtab -- cgit v1.2.3 From 7629702ca0956fb26d27ee19ed99306f73421c66 Mon Sep 17 00:00:00 2001 From: Sankar Tanguturi Date: Mon, 11 Sep 2017 07:50:01 -0700 Subject: vmware: Enable nics before sending the SUCCESS event. The network devices should be enabled before sending the 'SUCCESS' event to the underlying hypervisor. --- cloudinit/sources/DataSourceOVF.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'cloudinit/sources/DataSourceOVF.py') diff --git a/cloudinit/sources/DataSourceOVF.py b/cloudinit/sources/DataSourceOVF.py index aa5f798d..24b45d55 100644 --- a/cloudinit/sources/DataSourceOVF.py +++ b/cloudinit/sources/DataSourceOVF.py @@ -182,10 +182,10 @@ class DataSourceOVF(sources.DataSource): # TODO: Need to set the status to DONE only when the # customization is done successfully. + enable_nics(self._vmware_nics_to_enable) set_customization_status( GuestCustStateEnum.GUESTCUST_STATE_DONE, GuestCustErrorEnum.GUESTCUST_ERROR_SUCCESS) - enable_nics(self._vmware_nics_to_enable) else: np = {'iso': transport_iso9660, -- cgit v1.2.3 From da6562e21d0b17a0957adc0c5a2c9da076e0d219 Mon Sep 17 00:00:00 2001 From: Ryan Harper Date: Tue, 19 Sep 2017 11:10:09 -0500 Subject: DataSourceOVF: use util.find_devs_with(TYPE=iso9660) DataSourceOVF attempts to find iso files via walking os.listdir('/dev/') which is far too wide. This approach is too invasive and can sometimes race with systemd attempting to fsck and mount devices. Instead, utilize cloudinit.util.find_devs_with to filter devices by criteria (which uses blkid under the covers). This results in fewer attempts to mount block devices which do not contain iso filesystems. Unittest changes include: - cloudinit.tests.helpers; introduce add_patch() helper - Add unittest coverage for DataSourceOVF use of transport_iso9660 LP: #1718287 --- cloudinit/sources/DataSourceOVF.py | 74 ++++++++----- cloudinit/tests/helpers.py | 10 ++ tests/unittests/test_datasource/test_ovf.py | 164 ++++++++++++++++++++++++++++ 3 files changed, 221 insertions(+), 27 deletions(-) (limited to 'cloudinit/sources/DataSourceOVF.py') diff --git a/cloudinit/sources/DataSourceOVF.py b/cloudinit/sources/DataSourceOVF.py index 24b45d55..ccebf11a 100644 --- a/cloudinit/sources/DataSourceOVF.py +++ b/cloudinit/sources/DataSourceOVF.py @@ -375,26 +375,56 @@ def get_ovf_env(dirname): return (None, False) -# Transport functions take no input and return -# a 3 tuple of content, path, filename -def transport_iso9660(require_iso=True): +def maybe_cdrom_device(devname): + """Test if devname matches known list of devices which may contain iso9660 + filesystems. - # default_regex matches values in - # /lib/udev/rules.d/60-cdrom_id.rules - # KERNEL!="sr[0-9]*|hd[a-z]|xvd*", GOTO="cdrom_end" - envname = "CLOUD_INIT_CDROM_DEV_REGEX" - default_regex = "^(sr[0-9]+|hd[a-z]|xvd.*)" + Be helpful in accepting either knames (with no leading /dev/) or full path + names, but do not allow paths outside of /dev/, like /dev/foo/bar/xxx. + """ + if not devname: + return False + elif not isinstance(devname, util.string_types): + raise ValueError("Unexpected input for devname: %s" % devname) + + # resolve '..' and multi '/' elements + devname = os.path.normpath(devname) - devname_regex = os.environ.get(envname, default_regex) + # drop leading '/dev/' + if devname.startswith("/dev/"): + # partition returns tuple (before, partition, after) + devname = devname.partition("/dev/")[-1] + + # ignore leading slash (/sr0), else fail on / in name (foo/bar/xvdc) + if devname.startswith("/"): + devname = devname.split("/")[-1] + elif devname.count("/") > 0: + return False + + # if empty string + if not devname: + return False + + # default_regex matches values in /lib/udev/rules.d/60-cdrom_id.rules + # KERNEL!="sr[0-9]*|hd[a-z]|xvd*", GOTO="cdrom_end" + default_regex = r"^(sr[0-9]+|hd[a-z]|xvd.*)" + devname_regex = os.environ.get("CLOUD_INIT_CDROM_DEV_REGEX", default_regex) cdmatch = re.compile(devname_regex) + return cdmatch.match(devname) is not None + + +# Transport functions take no input and return +# a 3 tuple of content, path, filename +def transport_iso9660(require_iso=True): + # Go through mounts to see if it was already mounted mounts = util.mounts() for (dev, info) in mounts.items(): fstype = info['fstype'] if fstype != "iso9660" and require_iso: continue - if cdmatch.match(dev[5:]) is None: # take off '/dev/' + if not maybe_cdrom_device(dev): continue mp = info['mountpoint'] (fname, contents) = get_ovf_env(mp) @@ -406,29 +436,19 @@ def transport_iso9660(require_iso=True): else: mtype = None - devs = os.listdir("/dev/") - devs.sort() + # generate a list of devices with mtype filesystem, filter by regex + devs = [dev for dev in + util.find_devs_with("TYPE=%s" % mtype if mtype else None) + if maybe_cdrom_device(dev)] for dev in devs: - fullp = os.path.join("/dev/", dev) - - if (fullp in mounts or - not cdmatch.match(dev) or os.path.isdir(fullp)): - continue - - try: - # See if we can read anything at all...?? - util.peek_file(fullp, 512) - except IOError: - continue - try: - (fname, contents) = util.mount_cb(fullp, get_ovf_env, mtype=mtype) + (fname, contents) = util.mount_cb(dev, get_ovf_env, mtype=mtype) except util.MountFailedError: - LOG.debug("%s not mountable as iso9660", fullp) + LOG.debug("%s not mountable as iso9660", dev) continue if contents is not False: - return (contents, fullp, fname) + return (contents, dev, fname) return (False, None, None) diff --git a/cloudinit/tests/helpers.py b/cloudinit/tests/helpers.py index 28e26622..6f88a5b7 100644 --- a/cloudinit/tests/helpers.py +++ b/cloudinit/tests/helpers.py @@ -104,6 +104,16 @@ class TestCase(unittest2.TestCase): super(TestCase, self).setUp() self.reset_global_state() + def add_patch(self, target, attr, **kwargs): + """Patches specified target object and sets it as attr on test + instance also schedules cleanup""" + if 'autospec' not in kwargs: + kwargs['autospec'] = True + m = mock.patch(target, **kwargs) + p = m.start() + self.addCleanup(m.stop) + setattr(self, attr, p) + class CiTestCase(TestCase): """This is the preferred test case base class unless user diff --git a/tests/unittests/test_datasource/test_ovf.py b/tests/unittests/test_datasource/test_ovf.py index 9dbf4dd9..700da86c 100644 --- a/tests/unittests/test_datasource/test_ovf.py +++ b/tests/unittests/test_datasource/test_ovf.py @@ -5,6 +5,7 @@ # This file is part of cloud-init. See LICENSE file for license information. import base64 +from collections import OrderedDict from cloudinit.tests import helpers as test_helpers @@ -70,4 +71,167 @@ class TestReadOvfEnv(test_helpers.TestCase): self.assertEqual({'password': "passw0rd"}, cfg) self.assertIsNone(ud) + +class TestTransportIso9660(test_helpers.CiTestCase): + + def setUp(self): + super(TestTransportIso9660, self).setUp() + self.add_patch('cloudinit.util.find_devs_with', + 'm_find_devs_with') + self.add_patch('cloudinit.util.mounts', 'm_mounts') + self.add_patch('cloudinit.util.mount_cb', 'm_mount_cb') + self.add_patch('cloudinit.sources.DataSourceOVF.get_ovf_env', + 'm_get_ovf_env') + self.m_get_ovf_env.return_value = ('myfile', 'mycontent') + + def test_find_already_mounted(self): + """Check we call get_ovf_env from on matching mounted devices""" + mounts = { + '/dev/sr9': { + 'fstype': 'iso9660', + 'mountpoint': 'wark/media/sr9', + 'opts': 'ro', + } + } + self.m_mounts.return_value = mounts + + (contents, fullp, fname) = dsovf.transport_iso9660() + self.assertEqual("mycontent", contents) + self.assertEqual("/dev/sr9", fullp) + self.assertEqual("myfile", fname) + + def test_find_already_mounted_skips_non_iso9660(self): + """Check we call get_ovf_env ignoring non iso9660""" + mounts = { + '/dev/xvdb': { + 'fstype': 'vfat', + 'mountpoint': 'wark/foobar', + 'opts': 'defaults,noatime', + }, + '/dev/xvdc': { + 'fstype': 'iso9660', + 'mountpoint': 'wark/media/sr9', + 'opts': 'ro', + } + } + # We use an OrderedDict here to ensure we check xvdb before xvdc + # as we're not mocking the regex matching, however, if we place + # an entry in the results then we can be reasonably sure that + # we're skipping an entry which fails to match. + self.m_mounts.return_value = ( + OrderedDict(sorted(mounts.items(), key=lambda t: t[0]))) + + (contents, fullp, fname) = dsovf.transport_iso9660() + self.assertEqual("mycontent", contents) + self.assertEqual("/dev/xvdc", fullp) + self.assertEqual("myfile", fname) + + def test_find_already_mounted_matches_kname(self): + """Check we dont regex match on basename of the device""" + mounts = { + '/dev/foo/bar/xvdc': { + 'fstype': 'iso9660', + 'mountpoint': 'wark/media/sr9', + 'opts': 'ro', + } + } + # we're skipping an entry which fails to match. + self.m_mounts.return_value = mounts + + (contents, fullp, fname) = dsovf.transport_iso9660() + self.assertEqual(False, contents) + self.assertIsNone(fullp) + self.assertIsNone(fname) + + def test_mount_cb_called_on_blkdevs_with_iso9660(self): + """Check we call mount_cb on blockdevs with iso9660 only""" + self.m_mounts.return_value = {} + self.m_find_devs_with.return_value = ['/dev/sr0'] + self.m_mount_cb.return_value = ("myfile", "mycontent") + + (contents, fullp, fname) = dsovf.transport_iso9660() + + self.m_mount_cb.assert_called_with( + "/dev/sr0", dsovf.get_ovf_env, mtype="iso9660") + self.assertEqual("mycontent", contents) + self.assertEqual("/dev/sr0", fullp) + self.assertEqual("myfile", fname) + + def test_mount_cb_called_on_blkdevs_with_iso9660_check_regex(self): + """Check we call mount_cb on blockdevs with iso9660 and match regex""" + self.m_mounts.return_value = {} + self.m_find_devs_with.return_value = [ + '/dev/abc', '/dev/my-cdrom', '/dev/sr0'] + self.m_mount_cb.return_value = ("myfile", "mycontent") + + (contents, fullp, fname) = dsovf.transport_iso9660() + + self.m_mount_cb.assert_called_with( + "/dev/sr0", dsovf.get_ovf_env, mtype="iso9660") + self.assertEqual("mycontent", contents) + self.assertEqual("/dev/sr0", fullp) + self.assertEqual("myfile", fname) + + def test_mount_cb_not_called_no_matches(self): + """Check we don't call mount_cb if nothing matches""" + self.m_mounts.return_value = {} + self.m_find_devs_with.return_value = ['/dev/vg/myovf'] + + (contents, fullp, fname) = dsovf.transport_iso9660() + + self.assertEqual(0, self.m_mount_cb.call_count) + self.assertEqual(False, contents) + self.assertIsNone(fullp) + self.assertIsNone(fname) + + def test_mount_cb_called_require_iso_false(self): + """Check we call mount_cb on blockdevs with require_iso=False""" + self.m_mounts.return_value = {} + self.m_find_devs_with.return_value = ['/dev/xvdz'] + self.m_mount_cb.return_value = ("myfile", "mycontent") + + (contents, fullp, fname) = dsovf.transport_iso9660(require_iso=False) + + self.m_mount_cb.assert_called_with( + "/dev/xvdz", dsovf.get_ovf_env, mtype=None) + self.assertEqual("mycontent", contents) + self.assertEqual("/dev/xvdz", fullp) + self.assertEqual("myfile", fname) + + def test_maybe_cdrom_device_none(self): + """Test maybe_cdrom_device returns False for none/empty input""" + self.assertFalse(dsovf.maybe_cdrom_device(None)) + self.assertFalse(dsovf.maybe_cdrom_device('')) + + def test_maybe_cdrom_device_non_string_exception(self): + """Test maybe_cdrom_device raises ValueError on non-string types""" + with self.assertRaises(ValueError): + dsovf.maybe_cdrom_device({'a': 'eleven'}) + + def test_maybe_cdrom_device_false_on_multi_dir_paths(self): + """Test maybe_cdrom_device is false on /dev[/.*]/* paths""" + self.assertFalse(dsovf.maybe_cdrom_device('/dev/foo/sr0')) + self.assertFalse(dsovf.maybe_cdrom_device('foo/sr0')) + self.assertFalse(dsovf.maybe_cdrom_device('../foo/sr0')) + self.assertFalse(dsovf.maybe_cdrom_device('../foo/sr0')) + + def test_maybe_cdrom_device_true_on_hd_partitions(self): + """Test maybe_cdrom_device is false on /dev/hd[a-z][0-9]+ paths""" + self.assertTrue(dsovf.maybe_cdrom_device('/dev/hda1')) + self.assertTrue(dsovf.maybe_cdrom_device('hdz9')) + + def test_maybe_cdrom_device_true_on_valid_relative_paths(self): + """Test maybe_cdrom_device normalizes paths""" + self.assertTrue(dsovf.maybe_cdrom_device('/dev/wark/../sr9')) + self.assertTrue(dsovf.maybe_cdrom_device('///sr0')) + self.assertTrue(dsovf.maybe_cdrom_device('/sr0')) + self.assertTrue(dsovf.maybe_cdrom_device('//dev//hda')) + + def test_maybe_cdrom_device_true_on_xvd_partitions(self): + """Test maybe_cdrom_device returns true on xvd*""" + self.assertTrue(dsovf.maybe_cdrom_device('/dev/xvda')) + self.assertTrue(dsovf.maybe_cdrom_device('/dev/xvda1')) + self.assertTrue(dsovf.maybe_cdrom_device('xvdza1')) + +# # vi: ts=4 expandtab -- cgit v1.2.3