diff options
| -rw-r--r-- | cloudinit/sources/DataSourceOVF.py | 162 | ||||
| -rw-r--r-- | cloudinit/sources/helpers/vmware/imc/config.py | 12 | ||||
| -rw-r--r-- | cloudinit/sources/helpers/vmware/imc/guestcust_error.py | 1 | ||||
| -rw-r--r-- | tests/unittests/test_datasource/test_ovf.py | 291 | ||||
| -rw-r--r-- | tests/unittests/test_vmware_config_file.py | 16 |
5 files changed, 468 insertions, 14 deletions
diff --git a/cloudinit/sources/DataSourceOVF.py b/cloudinit/sources/DataSourceOVF.py index 741c140a..94d9f1b9 100644 --- a/cloudinit/sources/DataSourceOVF.py +++ b/cloudinit/sources/DataSourceOVF.py @@ -16,6 +16,7 @@ from xml.dom import minidom from cloudinit import dmi from cloudinit import log as logging +from cloudinit import safeyaml from cloudinit import sources from cloudinit import subp from cloudinit import util @@ -47,6 +48,7 @@ LOG = logging.getLogger(__name__) CONFGROUPNAME_GUESTCUSTOMIZATION = "deployPkg" GUESTCUSTOMIZATION_ENABLE_CUST_SCRIPTS = "enable-custom-scripts" +VMWARE_IMC_DIR = "/var/run/vmware-imc" class DataSourceOVF(sources.DataSource): @@ -99,9 +101,7 @@ class DataSourceOVF(sources.DataSource): if not self.vmware_customization_supported: LOG.debug("Skipping the check for " "VMware Customization support") - elif not util.get_cfg_option_bool( - self.sys_cfg, "disable_vmware_customization", True): - + else: search_paths = ( "/usr/lib/vmware-tools", "/usr/lib64/vmware-tools", "/usr/lib/open-vm-tools", "/usr/lib64/open-vm-tools") @@ -119,7 +119,9 @@ class DataSourceOVF(sources.DataSource): # When the VM is powered on, the "VMware Tools" daemon # copies the customization specification file to # /var/run/vmware-imc directory. cloud-init code needs - # to search for the file in that directory. + # to search for the file in that directory which indicates + # that required metadata and userdata files are now + # present. max_wait = get_max_wait_from_cfg(self.ds_cfg) vmwareImcConfigFilePath = util.log_time( logfunc=LOG.debug, @@ -129,26 +131,83 @@ class DataSourceOVF(sources.DataSource): else: LOG.debug("Did not find the customization plugin.") + md_path = None if vmwareImcConfigFilePath: + imcdirpath = os.path.dirname(vmwareImcConfigFilePath) + cf = ConfigFile(vmwareImcConfigFilePath) + self._vmware_cust_conf = Config(cf) LOG.debug("Found VMware Customization Config File at %s", vmwareImcConfigFilePath) - nicspath = wait_for_imc_cfg_file( - filename="nics.txt", maxwait=10, naplen=5) + try: + (md_path, ud_path, nicspath) = collect_imc_file_paths( + self._vmware_cust_conf) + except FileNotFoundError as e: + _raise_error_status( + "File(s) missing in directory", + e, + GuestCustEvent.GUESTCUST_EVENT_CUSTOMIZE_FAILED, + vmwareImcConfigFilePath, + self._vmware_cust_conf) else: LOG.debug("Did not find VMware Customization Config File") - else: - LOG.debug("Customization for VMware platform is disabled.") - if vmwareImcConfigFilePath: + # Honor disable_vmware_customization setting on metadata absent + if not md_path: + if util.get_cfg_option_bool(self.sys_cfg, + "disable_vmware_customization", + True): + LOG.debug( + "Customization for VMware platform is disabled.") + # reset vmwareImcConfigFilePath to None to avoid + # customization for VMware platform + vmwareImcConfigFilePath = None + + use_raw_data = bool(vmwareImcConfigFilePath and md_path) + if use_raw_data: + set_gc_status(self._vmware_cust_conf, "Started") + LOG.debug("Start to load cloud-init meta data and user data") + try: + (md, ud, cfg, network) = load_cloudinit_data(md_path, ud_path) + + if network: + self._network_config = network + else: + self._network_config = ( + self.distro.generate_fallback_config() + ) + + except safeyaml.YAMLError as e: + _raise_error_status( + "Error parsing the cloud-init meta data", + e, + GuestCustErrorEnum.GUESTCUST_ERROR_WRONG_META_FORMAT, + vmwareImcConfigFilePath, + self._vmware_cust_conf) + except Exception as e: + _raise_error_status( + "Error loading cloud-init configuration", + e, + GuestCustEvent.GUESTCUST_EVENT_CUSTOMIZE_FAILED, + vmwareImcConfigFilePath, + self._vmware_cust_conf) + + self._vmware_cust_found = True + found.append('vmware-tools') + + util.del_dir(imcdirpath) + set_customization_status( + GuestCustStateEnum.GUESTCUST_STATE_DONE, + GuestCustErrorEnum.GUESTCUST_ERROR_SUCCESS) + set_gc_status(self._vmware_cust_conf, "Successful") + + elif vmwareImcConfigFilePath: + # Load configuration from vmware_imc self._vmware_nics_to_enable = "" try: - cf = ConfigFile(vmwareImcConfigFilePath) - self._vmware_cust_conf = Config(cf) set_gc_status(self._vmware_cust_conf, "Started") (md, ud, cfg) = read_vmware_imc(self._vmware_cust_conf) self._vmware_nics_to_enable = get_nics_to_enable(nicspath) - imcdirpath = os.path.dirname(vmwareImcConfigFilePath) product_marker = self._vmware_cust_conf.marker_id hasmarkerfile = check_marker_exists( product_marker, os.path.join(self.paths.cloud_dir, 'data')) @@ -684,4 +743,83 @@ def _raise_error_status(prefix, error, event, config_file, conf): util.del_dir(os.path.dirname(config_file)) raise error + +def load_cloudinit_data(md_path, ud_path): + """ + Load the cloud-init meta data, user data, cfg and network from the + given files + + @return: 4-tuple of configuration + metadata, userdata, cfg={}, network + + @raises: FileNotFoundError if md_path or ud_path are absent + """ + LOG.debug('load meta data from: %s: user data from: %s', + md_path, ud_path) + md = {} + ud = None + network = None + + md = safeload_yaml_or_dict(util.load_file(md_path)) + + if 'network' in md: + network = md['network'] + + if ud_path: + ud = util.load_file(ud_path).replace("\r", "") + return md, ud, {}, network + + +def safeload_yaml_or_dict(data): + ''' + The meta data could be JSON or YAML. Since YAML is a strict superset of + JSON, we will unmarshal the data as YAML. If data is None then a new + dictionary is returned. + ''' + if not data: + return {} + return safeyaml.load(data) + + +def collect_imc_file_paths(cust_conf): + ''' + collect all the other imc files. + + metadata is preferred to nics.txt configuration data. + + If metadata file exists because it is specified in customization + configuration, then metadata is required and userdata is optional. + + @return a 3-tuple containing desired configuration file paths if present + Expected returns: + 1. user provided metadata and userdata (md_path, ud_path, None) + 2. user provided metadata (md_path, None, None) + 3. user-provided network config (None, None, nics_path) + 4. No config found (None, None, None) + ''' + md_path = None + ud_path = None + nics_path = None + md_file = cust_conf.meta_data_name + if md_file: + md_path = os.path.join(VMWARE_IMC_DIR, md_file) + if not os.path.exists(md_path): + raise FileNotFoundError("meta data file is not found: %s" + % md_path) + + ud_file = cust_conf.user_data_name + if ud_file: + ud_path = os.path.join(VMWARE_IMC_DIR, ud_file) + if not os.path.exists(ud_path): + raise FileNotFoundError("user data file is not found: %s" + % ud_path) + else: + nics_path = os.path.join(VMWARE_IMC_DIR, "nics.txt") + if not os.path.exists(nics_path): + LOG.debug('%s does not exist.', nics_path) + nics_path = None + + return md_path, ud_path, nics_path + + # vi: ts=4 expandtab diff --git a/cloudinit/sources/helpers/vmware/imc/config.py b/cloudinit/sources/helpers/vmware/imc/config.py index 7109aef3..bdfab5a0 100644 --- a/cloudinit/sources/helpers/vmware/imc/config.py +++ b/cloudinit/sources/helpers/vmware/imc/config.py @@ -27,6 +27,8 @@ class Config(object): UTC = 'DATETIME|UTC' POST_GC_STATUS = 'MISC|POST-GC-STATUS' DEFAULT_RUN_POST_SCRIPT = 'MISC|DEFAULT-RUN-POST-CUST-SCRIPT' + CLOUDINIT_META_DATA = 'CLOUDINIT|METADATA' + CLOUDINIT_USER_DATA = 'CLOUDINIT|USERDATA' def __init__(self, configFile): self._configFile = configFile @@ -130,4 +132,14 @@ class Config(object): raise ValueError('defaultRunPostScript value should be yes/no') return defaultRunPostScript == 'yes' + @property + def meta_data_name(self): + """Return the name of cloud-init meta data.""" + return self._configFile.get(Config.CLOUDINIT_META_DATA, None) + + @property + def user_data_name(self): + """Return the name of cloud-init user data.""" + return self._configFile.get(Config.CLOUDINIT_USER_DATA, None) + # vi: ts=4 expandtab diff --git a/cloudinit/sources/helpers/vmware/imc/guestcust_error.py b/cloudinit/sources/helpers/vmware/imc/guestcust_error.py index 65ae7390..96d839b8 100644 --- a/cloudinit/sources/helpers/vmware/imc/guestcust_error.py +++ b/cloudinit/sources/helpers/vmware/imc/guestcust_error.py @@ -11,5 +11,6 @@ class GuestCustErrorEnum(object): GUESTCUST_ERROR_SUCCESS = 0 GUESTCUST_ERROR_SCRIPT_DISABLED = 6 + GUESTCUST_ERROR_WRONG_META_FORMAT = 9 # vi: ts=4 expandtab diff --git a/tests/unittests/test_datasource/test_ovf.py b/tests/unittests/test_datasource/test_ovf.py index 16773de5..dce01f5d 100644 --- a/tests/unittests/test_datasource/test_ovf.py +++ b/tests/unittests/test_datasource/test_ovf.py @@ -17,6 +17,7 @@ from cloudinit.helpers import Paths from cloudinit.sources import DataSourceOVF as dsovf from cloudinit.sources.helpers.vmware.imc.config_custom_script import ( CustomScriptNotFound) +from cloudinit.safeyaml import YAMLError MPATH = 'cloudinit.sources.DataSourceOVF.' @@ -138,16 +139,29 @@ class TestDatasourceOVF(CiTestCase): 'DEBUG: No system-product-name found', self.logs.getvalue()) def test_get_data_no_vmware_customization_disabled(self): - """When vmware customization is disabled via sys_cfg log a message.""" + """When cloud-init workflow for vmware is disabled via sys_cfg and + no meta data provided, log a message. + """ paths = Paths({'cloud_dir': self.tdir}) ds = self.datasource( sys_cfg={'disable_vmware_customization': True}, distro={}, paths=paths) + conf_file = self.tmp_path('test-cust', self.tdir) + conf_content = dedent("""\ + [CUSTOM-SCRIPT] + SCRIPT-NAME = test-script + [MISC] + MARKER-ID = 12345345 + """) + util.write_file(conf_file, conf_content) retcode = wrap_and_call( 'cloudinit.sources.DataSourceOVF', {'dmi.read_dmi_data': 'vmware', 'transport_iso9660': NOT_FOUND, - 'transport_vmware_guestinfo': NOT_FOUND}, + 'transport_vmware_guestinfo': NOT_FOUND, + 'util.del_dir': True, + 'search_file': self.tdir, + 'wait_for_imc_cfg_file': conf_file}, ds.get_data) self.assertFalse(retcode, 'Expected False return from ds.get_data') self.assertIn( @@ -344,6 +358,279 @@ class TestDatasourceOVF(CiTestCase): 'vmware (%s/seed/ovf-env.xml)' % self.tdir, ds.subplatform) + def test_get_data_cloudinit_metadata_json(self): + """Test metadata can be loaded to cloud-init metadata and network. + The metadata format is json. + """ + paths = Paths({'cloud_dir': self.tdir}) + ds = self.datasource( + sys_cfg={'disable_vmware_customization': True}, distro={}, + paths=paths) + # Prepare the conf file + conf_file = self.tmp_path('test-cust', self.tdir) + conf_content = dedent("""\ + [CLOUDINIT] + METADATA = test-meta + """) + util.write_file(conf_file, conf_content) + # Prepare the meta data file + metadata_file = self.tmp_path('test-meta', self.tdir) + metadata_content = dedent("""\ + { + "instance-id": "cloud-vm", + "local-hostname": "my-host.domain.com", + "network": { + "version": 2, + "ethernets": { + "eths": { + "match": { + "name": "ens*" + }, + "dhcp4": true + } + } + } + } + """) + util.write_file(metadata_file, metadata_content) + + with mock.patch(MPATH + 'set_customization_status', + return_value=('msg', b'')): + result = wrap_and_call( + 'cloudinit.sources.DataSourceOVF', + {'dmi.read_dmi_data': 'vmware', + 'util.del_dir': True, + 'search_file': self.tdir, + 'wait_for_imc_cfg_file': conf_file, + 'collect_imc_file_paths': [self.tdir + '/test-meta', '', ''], + 'get_nics_to_enable': ''}, + ds._get_data) + + self.assertTrue(result) + self.assertEqual("cloud-vm", ds.metadata['instance-id']) + self.assertEqual("my-host.domain.com", ds.metadata['local-hostname']) + self.assertEqual(2, ds.network_config['version']) + self.assertTrue(ds.network_config['ethernets']['eths']['dhcp4']) + + def test_get_data_cloudinit_metadata_yaml(self): + """Test metadata can be loaded to cloud-init metadata and network. + The metadata format is yaml. + """ + paths = Paths({'cloud_dir': self.tdir}) + ds = self.datasource( + sys_cfg={'disable_vmware_customization': True}, distro={}, + paths=paths) + # Prepare the conf file + conf_file = self.tmp_path('test-cust', self.tdir) + conf_content = dedent("""\ + [CLOUDINIT] + METADATA = test-meta + """) + util.write_file(conf_file, conf_content) + # Prepare the meta data file + metadata_file = self.tmp_path('test-meta', self.tdir) + metadata_content = dedent("""\ + instance-id: cloud-vm + local-hostname: my-host.domain.com + network: + version: 2 + ethernets: + nics: + match: + name: ens* + dhcp4: yes + """) + util.write_file(metadata_file, metadata_content) + + with mock.patch(MPATH + 'set_customization_status', + return_value=('msg', b'')): + result = wrap_and_call( + 'cloudinit.sources.DataSourceOVF', + {'dmi.read_dmi_data': 'vmware', + 'util.del_dir': True, + 'search_file': self.tdir, + 'wait_for_imc_cfg_file': conf_file, + 'collect_imc_file_paths': [self.tdir + '/test-meta', '', ''], + 'get_nics_to_enable': ''}, + ds._get_data) + + self.assertTrue(result) + self.assertEqual("cloud-vm", ds.metadata['instance-id']) + self.assertEqual("my-host.domain.com", ds.metadata['local-hostname']) + self.assertEqual(2, ds.network_config['version']) + self.assertTrue(ds.network_config['ethernets']['nics']['dhcp4']) + + def test_get_data_cloudinit_metadata_not_valid(self): + """Test metadata is not JSON or YAML format. + """ + paths = Paths({'cloud_dir': self.tdir}) + ds = self.datasource( + sys_cfg={'disable_vmware_customization': True}, distro={}, + paths=paths) + + # Prepare the conf file + conf_file = self.tmp_path('test-cust', self.tdir) + conf_content = dedent("""\ + [CLOUDINIT] + METADATA = test-meta + """) + util.write_file(conf_file, conf_content) + + # Prepare the meta data file + metadata_file = self.tmp_path('test-meta', self.tdir) + metadata_content = "[This is not json or yaml format]a=b" + util.write_file(metadata_file, metadata_content) + + with mock.patch(MPATH + 'set_customization_status', + return_value=('msg', b'')): + with self.assertRaises(YAMLError) as context: + wrap_and_call( + 'cloudinit.sources.DataSourceOVF', + {'dmi.read_dmi_data': 'vmware', + 'util.del_dir': True, + 'search_file': self.tdir, + 'wait_for_imc_cfg_file': conf_file, + 'collect_imc_file_paths': [ + self.tdir + '/test-meta', '', '' + ], + 'get_nics_to_enable': ''}, + ds.get_data) + + self.assertIn("expected '<document start>', but found '<scalar>'", + str(context.exception)) + + def test_get_data_cloudinit_metadata_not_found(self): + """Test metadata file can't be found. + """ + paths = Paths({'cloud_dir': self.tdir}) + ds = self.datasource( + sys_cfg={'disable_vmware_customization': True}, distro={}, + paths=paths) + # Prepare the conf file + conf_file = self.tmp_path('test-cust', self.tdir) + conf_content = dedent("""\ + [CLOUDINIT] + METADATA = test-meta + """) + util.write_file(conf_file, conf_content) + # Don't prepare the meta data file + + with mock.patch(MPATH + 'set_customization_status', + return_value=('msg', b'')): + with self.assertRaises(FileNotFoundError) as context: + wrap_and_call( + 'cloudinit.sources.DataSourceOVF', + {'dmi.read_dmi_data': 'vmware', + 'util.del_dir': True, + 'search_file': self.tdir, + 'wait_for_imc_cfg_file': conf_file, + 'get_nics_to_enable': ''}, + ds.get_data) + + self.assertIn('is not found', str(context.exception)) + + def test_get_data_cloudinit_userdata(self): + """Test user data can be loaded to cloud-init user data. + """ + paths = Paths({'cloud_dir': self.tdir}) + ds = self.datasource( + sys_cfg={'disable_vmware_customization': False}, distro={}, + paths=paths) + + # Prepare the conf file + conf_file = self.tmp_path('test-cust', self.tdir) + conf_content = dedent("""\ + [CLOUDINIT] + METADATA = test-meta + USERDATA = test-user + """) + util.write_file(conf_file, conf_content) + + # Prepare the meta data file + metadata_file = self.tmp_path('test-meta', self.tdir) + metadata_content = dedent("""\ + instance-id: cloud-vm + local-hostname: my-host.domain.com + network: + version: 2 + ethernets: + nics: + match: + name: ens* + dhcp4: yes + """) + util.write_file(metadata_file, metadata_content) + + # Prepare the user data file + userdata_file = self.tmp_path('test-user', self.tdir) + userdata_content = "This is the user data" + util.write_file(userdata_file, userdata_content) + + with mock.patch(MPATH + 'set_customization_status', + return_value=('msg', b'')): + result = wrap_and_call( + 'cloudinit.sources.DataSourceOVF', + {'dmi.read_dmi_data': 'vmware', + 'util.del_dir': True, + 'search_file': self.tdir, + 'wait_for_imc_cfg_file': conf_file, + 'collect_imc_file_paths': [self.tdir + '/test-meta', + self.tdir + '/test-user', ''], + 'get_nics_to_enable': ''}, + ds._get_data) + + self.assertTrue(result) + self.assertEqual("cloud-vm", ds.metadata['instance-id']) + self.assertEqual(userdata_content, ds.userdata_raw) + + def test_get_data_cloudinit_userdata_not_found(self): + """Test userdata file can't be found. + """ + paths = Paths({'cloud_dir': self.tdir}) + ds = self.datasource( + sys_cfg={'disable_vmware_customization': True}, distro={}, + paths=paths) + + # Prepare the conf file + conf_file = self.tmp_path('test-cust', self.tdir) + conf_content = dedent("""\ + [CLOUDINIT] + METADATA = test-meta + USERDATA = test-user + """) + util.write_file(conf_file, conf_content) + + # Prepare the meta data file + metadata_file = self.tmp_path('test-meta', self.tdir) + metadata_content = dedent("""\ + instance-id: cloud-vm + local-hostname: my-host.domain.com + network: + version: 2 + ethernets: + nics: + match: + name: ens* + dhcp4: yes + """) + util.write_file(metadata_file, metadata_content) + + # Don't prepare the user data file + + with mock.patch(MPATH + 'set_customization_status', + return_value=('msg', b'')): + with self.assertRaises(FileNotFoundError) as context: + wrap_and_call( + 'cloudinit.sources.DataSourceOVF', + {'dmi.read_dmi_data': 'vmware', + 'util.del_dir': True, + 'search_file': self.tdir, + 'wait_for_imc_cfg_file': conf_file, + 'get_nics_to_enable': ''}, + ds.get_data) + + self.assertIn('is not found', str(context.exception)) + class TestTransportIso9660(CiTestCase): diff --git a/tests/unittests/test_vmware_config_file.py b/tests/unittests/test_vmware_config_file.py index 9c7d25fa..430cc69f 100644 --- a/tests/unittests/test_vmware_config_file.py +++ b/tests/unittests/test_vmware_config_file.py @@ -525,5 +525,21 @@ class TestVmwareNetConfig(CiTestCase): 'gateway': '10.20.87.253'}]}], nc.generate()) + def test_meta_data(self): + cf = ConfigFile("tests/data/vmware/cust-dhcp-2nic.cfg") + conf = Config(cf) + self.assertIsNone(conf.meta_data_name) + cf._insertKey("CLOUDINIT|METADATA", "test-metadata") + conf = Config(cf) + self.assertEqual("test-metadata", conf.meta_data_name) + + def test_user_data(self): + cf = ConfigFile("tests/data/vmware/cust-dhcp-2nic.cfg") + conf = Config(cf) + self.assertIsNone(conf.user_data_name) + cf._insertKey("CLOUDINIT|USERDATA", "test-userdata") + conf = Config(cf) + self.assertEqual("test-userdata", conf.user_data_name) + # vi: ts=4 expandtab |