[VMware] Support cloudinit raw data feature (#691)

This feature will modify VMware datasource to read from meta data and user data which are specified by VMware vSphere user. If meta data/user data are found in cloud-init configuration directory, datasource will parse the meta data/network and user data from the configuration file, otherwise it will continue to parse them from traditional customization configuration file as before. The supported meta data file is in json or yaml format.

3 files changed, 163 insertions, 12 deletions

diff --git a/cloudinit/sources/DataSourceOVF.py b/cloudinit/sources/DataSourceOVF.py
index 741c140a..94d9f1b9 100644
--- a/cloudinit/sources/DataSourceOVF.py
+++ b/cloudinit/sources/DataSourceOVF.py
@@ -16,6 +16,7 @@ from xml.dom import minidom
 
 from cloudinit import dmi
 from cloudinit import log as logging
+from cloudinit import safeyaml
 from cloudinit import sources
 from cloudinit import subp
 from cloudinit import util
@@ -47,6 +48,7 @@ LOG = logging.getLogger(__name__)
 
 CONFGROUPNAME_GUESTCUSTOMIZATION = "deployPkg"
 GUESTCUSTOMIZATION_ENABLE_CUST_SCRIPTS = "enable-custom-scripts"
+VMWARE_IMC_DIR = "/var/run/vmware-imc"
 
 
 class DataSourceOVF(sources.DataSource):
@@ -99,9 +101,7 @@ class DataSourceOVF(sources.DataSource):
             if not self.vmware_customization_supported:
                 LOG.debug("Skipping the check for "
                           "VMware Customization support")
-            elif not util.get_cfg_option_bool(
-                    self.sys_cfg, "disable_vmware_customization", True):
-
+            else:
                 search_paths = (
                     "/usr/lib/vmware-tools", "/usr/lib64/vmware-tools",
                     "/usr/lib/open-vm-tools", "/usr/lib64/open-vm-tools")
@@ -119,7 +119,9 @@ class DataSourceOVF(sources.DataSource):
                     # When the VM is powered on, the "VMware Tools" daemon
                     # copies the customization specification file to
                     # /var/run/vmware-imc directory. cloud-init code needs
-                    # to search for the file in that directory.
+                    # to search for the file in that directory which indicates
+                    # that required metadata and userdata files are now
+                    # present.
                     max_wait = get_max_wait_from_cfg(self.ds_cfg)
                     vmwareImcConfigFilePath = util.log_time(
                         logfunc=LOG.debug,
@@ -129,26 +131,83 @@ class DataSourceOVF(sources.DataSource):
                 else:
                     LOG.debug("Did not find the customization plugin.")
 
+                md_path = None
                 if vmwareImcConfigFilePath:
+                    imcdirpath = os.path.dirname(vmwareImcConfigFilePath)
+                    cf = ConfigFile(vmwareImcConfigFilePath)
+                    self._vmware_cust_conf = Config(cf)
                     LOG.debug("Found VMware Customization Config File at %s",
                               vmwareImcConfigFilePath)
-                    nicspath = wait_for_imc_cfg_file(
-                        filename="nics.txt", maxwait=10, naplen=5)
+                    try:
+                        (md_path, ud_path, nicspath) = collect_imc_file_paths(
+                            self._vmware_cust_conf)
+                    except FileNotFoundError as e:
+                        _raise_error_status(
+                            "File(s) missing in directory",
+                            e,
+                            GuestCustEvent.GUESTCUST_EVENT_CUSTOMIZE_FAILED,
+                            vmwareImcConfigFilePath,
+                            self._vmware_cust_conf)
                 else:
                     LOG.debug("Did not find VMware Customization Config File")
-            else:
-                LOG.debug("Customization for VMware platform is disabled.")
 
-        if vmwareImcConfigFilePath:
+                # Honor disable_vmware_customization setting on metadata absent
+                if not md_path:
+                    if util.get_cfg_option_bool(self.sys_cfg,
+                                                "disable_vmware_customization",
+                                                True):
+                        LOG.debug(
+                            "Customization for VMware platform is disabled.")
+                        # reset vmwareImcConfigFilePath to None to avoid
+                        # customization for VMware platform
+                        vmwareImcConfigFilePath = None
+
+        use_raw_data = bool(vmwareImcConfigFilePath and md_path)
+        if use_raw_data:
+            set_gc_status(self._vmware_cust_conf, "Started")
+            LOG.debug("Start to load cloud-init meta data and user data")
+            try:
+                (md, ud, cfg, network) = load_cloudinit_data(md_path, ud_path)
+
+                if network:
+                    self._network_config = network
+                else:
+                    self._network_config = (
+                        self.distro.generate_fallback_config()
+                    )
+
+            except safeyaml.YAMLError as e:
+                _raise_error_status(
+                    "Error parsing the cloud-init meta data",
+                    e,
+                    GuestCustErrorEnum.GUESTCUST_ERROR_WRONG_META_FORMAT,
+                    vmwareImcConfigFilePath,
+                    self._vmware_cust_conf)
+            except Exception as e:
+                _raise_error_status(
+                    "Error loading cloud-init configuration",
+                    e,
+                    GuestCustEvent.GUESTCUST_EVENT_CUSTOMIZE_FAILED,
+                    vmwareImcConfigFilePath,
+                    self._vmware_cust_conf)
+
+            self._vmware_cust_found = True
+            found.append('vmware-tools')
+
+            util.del_dir(imcdirpath)
+            set_customization_status(
+                GuestCustStateEnum.GUESTCUST_STATE_DONE,
+                GuestCustErrorEnum.GUESTCUST_ERROR_SUCCESS)
+            set_gc_status(self._vmware_cust_conf, "Successful")
+
+        elif vmwareImcConfigFilePath:
+            # Load configuration from vmware_imc
             self._vmware_nics_to_enable = ""
             try:
-                cf = ConfigFile(vmwareImcConfigFilePath)
-                self._vmware_cust_conf = Config(cf)
                 set_gc_status(self._vmware_cust_conf, "Started")
 
                 (md, ud, cfg) = read_vmware_imc(self._vmware_cust_conf)
                 self._vmware_nics_to_enable = get_nics_to_enable(nicspath)
-                imcdirpath = os.path.dirname(vmwareImcConfigFilePath)
                 product_marker = self._vmware_cust_conf.marker_id
                 hasmarkerfile = check_marker_exists(
                     product_marker, os.path.join(self.paths.cloud_dir, 'data'))
@@ -684,4 +743,83 @@ def _raise_error_status(prefix, error, event, config_file, conf):
     util.del_dir(os.path.dirname(config_file))
     raise error
 
+
+def load_cloudinit_data(md_path, ud_path):
+    """
+    Load the cloud-init meta data, user data, cfg and network from the
+    given files
+
+    @return: 4-tuple of configuration
+        metadata, userdata, cfg={}, network
+
+    @raises: FileNotFoundError if md_path or ud_path are absent
+    """
+    LOG.debug('load meta data from: %s: user data from: %s',
+              md_path, ud_path)
+    md = {}
+    ud = None
+    network = None
+
+    md = safeload_yaml_or_dict(util.load_file(md_path))
+
+    if 'network' in md:
+        network = md['network']
+
+    if ud_path:
+        ud = util.load_file(ud_path).replace("\r", "")
+    return md, ud, {}, network
+
+
+def safeload_yaml_or_dict(data):
+    '''
+    The meta data could be JSON or YAML. Since YAML is a strict superset of
+    JSON, we will unmarshal the data as YAML. If data is None then a new
+    dictionary is returned.
+    '''
+    if not data:
+        return {}
+    return safeyaml.load(data)
+
+
+def collect_imc_file_paths(cust_conf):
+    '''
+    collect all the other imc files.
+
+    metadata is preferred to nics.txt configuration data.
+
+    If metadata file exists because it is specified in customization
+    configuration, then metadata is required and userdata is optional.
+
+    @return a 3-tuple containing desired configuration file paths if present
+        Expected returns:
+             1. user provided metadata and userdata (md_path, ud_path, None)
+             2. user provided metadata (md_path, None, None)
+             3. user-provided network config (None, None, nics_path)
+             4. No config found (None, None, None)
+    '''
+    md_path = None
+    ud_path = None
+    nics_path = None
+    md_file = cust_conf.meta_data_name
+    if md_file:
+        md_path = os.path.join(VMWARE_IMC_DIR, md_file)
+        if not os.path.exists(md_path):
+            raise FileNotFoundError("meta data file is not found: %s"
+                                    % md_path)
+
+        ud_file = cust_conf.user_data_name
+        if ud_file:
+            ud_path = os.path.join(VMWARE_IMC_DIR, ud_file)
+            if not os.path.exists(ud_path):
+                raise FileNotFoundError("user data file is not found: %s"
+                                        % ud_path)
+    else:
+        nics_path = os.path.join(VMWARE_IMC_DIR, "nics.txt")
+        if not os.path.exists(nics_path):
+            LOG.debug('%s does not exist.', nics_path)
+            nics_path = None
+
+    return md_path, ud_path, nics_path
+
+
 # vi: ts=4 expandtab
diff --git a/cloudinit/sources/helpers/vmware/imc/config.py b/cloudinit/sources/helpers/vmware/imc/config.py
index 7109aef3..bdfab5a0 100644
--- a/cloudinit/sources/helpers/vmware/imc/config.py
+++ b/cloudinit/sources/helpers/vmware/imc/config.py
@@ -27,6 +27,8 @@ class Config(object):
     UTC = 'DATETIME|UTC'
     POST_GC_STATUS = 'MISC|POST-GC-STATUS'
     DEFAULT_RUN_POST_SCRIPT = 'MISC|DEFAULT-RUN-POST-CUST-SCRIPT'
+    CLOUDINIT_META_DATA = 'CLOUDINIT|METADATA'
+    CLOUDINIT_USER_DATA = 'CLOUDINIT|USERDATA'
 
     def __init__(self, configFile):
         self._configFile = configFile
@@ -130,4 +132,14 @@ class Config(object):
             raise ValueError('defaultRunPostScript value should be yes/no')
         return defaultRunPostScript == 'yes'
 
+    @property
+    def meta_data_name(self):
+        """Return the name of cloud-init meta data."""
+        return self._configFile.get(Config.CLOUDINIT_META_DATA, None)
+
+    @property
+    def user_data_name(self):
+        """Return the name of cloud-init user data."""
+        return self._configFile.get(Config.CLOUDINIT_USER_DATA, None)
+
 # vi: ts=4 expandtab
diff --git a/cloudinit/sources/helpers/vmware/imc/guestcust_error.py b/cloudinit/sources/helpers/vmware/imc/guestcust_error.py
index 65ae7390..96d839b8 100644
--- a/cloudinit/sources/helpers/vmware/imc/guestcust_error.py
+++ b/cloudinit/sources/helpers/vmware/imc/guestcust_error.py
@@ -11,5 +11,6 @@ class GuestCustErrorEnum(object):
 
     GUESTCUST_ERROR_SUCCESS = 0
     GUESTCUST_ERROR_SCRIPT_DISABLED = 6
+    GUESTCUST_ERROR_WRONG_META_FORMAT = 9
 
 # vi: ts=4 expandtab