merge from trunk

13 files changed, 345 insertions, 22 deletions

diff --git a/cloudinit/config/cc_set_passwords.py b/cloudinit/config/cc_set_passwords.py
index 0c315361..58e1b713 100644
--- a/cloudinit/config/cc_set_passwords.py
+++ b/cloudinit/config/cc_set_passwords.py
@@ -45,8 +45,6 @@ def handle(_name, cfg, cloud, log, args):
         password = util.get_cfg_option_str(cfg, "password", None)
 
     expire = True
-    pw_auth = "no"
-    change_pwauth = False
     plist = None
 
     if 'chpasswd' in cfg:
@@ -104,11 +102,24 @@ def handle(_name, cfg, cloud, log, args):
     change_pwauth = False
     pw_auth = None
     if 'ssh_pwauth' in cfg:
-        change_pwauth = True
         if util.is_true(cfg['ssh_pwauth']):
+            change_pwauth = True
             pw_auth = 'yes'
-        if util.is_false(cfg['ssh_pwauth']):
+        elif util.is_false(cfg['ssh_pwauth']):
+            change_pwauth = True
             pw_auth = 'no'
+        elif str(cfg['ssh_pwauth']).lower() == 'unchanged':
+            log.debug('Leaving auth line unchanged')
+            change_pwauth = False
+        elif not str(cfg['ssh_pwauth']).strip():
+            log.debug('Leaving auth line unchanged')
+            change_pwauth = False
+        elif not cfg['ssh_pwauth']:
+            log.debug('Leaving auth line unchanged')
+            change_pwauth = False
+        else:
+            msg = 'Unrecognized value %s for ssh_pwauth' % cfg['ssh_pwauth']
+            util.logexc(log, msg)
 
     if change_pwauth:
         replaced_auth = False
diff --git a/cloudinit/sources/DataSourceAzure.py b/cloudinit/sources/DataSourceAzure.py
index 2af0ad9b..832b3063 100644
--- a/cloudinit/sources/DataSourceAzure.py
+++ b/cloudinit/sources/DataSourceAzure.py
@@ -254,6 +254,10 @@ class DataSourceAzureNet(sources.DataSource):
     def get_config_obj(self):
         return self.cfg
 
+    def check_instance_id(self):
+        # quickly (local check only) if self.instance_id is still valid
+        return sources.instance_id_matches_system_uuid(self.get_instance_id())
+
 
 def count_files(mp):
     return len(fnmatch.filter(os.listdir(mp), '*[!cdrom]*'))
diff --git a/cloudinit/sources/DataSourceConfigDrive.py b/cloudinit/sources/DataSourceConfigDrive.py
index e3916208..6fc9e05b 100644
--- a/cloudinit/sources/DataSourceConfigDrive.py
+++ b/cloudinit/sources/DataSourceConfigDrive.py
@@ -146,6 +146,10 @@ class DataSourceConfigDrive(openstack.SourceMixin, sources.DataSource):
 
         return True
 
+    def check_instance_id(self):
+        # quickly (local check only) if self.instance_id is still valid
+        return sources.instance_id_matches_system_uuid(self.get_instance_id())
+
 
 class DataSourceConfigDriveNet(DataSourceConfigDrive):
     def __init__(self, sys_cfg, distro, paths):
diff --git a/cloudinit/sources/DataSourceNoCloud.py b/cloudinit/sources/DataSourceNoCloud.py
index 64853385..538df7d9 100644
--- a/cloudinit/sources/DataSourceNoCloud.py
+++ b/cloudinit/sources/DataSourceNoCloud.py
@@ -209,6 +209,41 @@ class DataSourceNoCloud(sources.DataSource):
                   mydata['meta-data']['dsmode'])
         return False
 
+    def check_instance_id(self):
+        # quickly (local check only) if self.instance_id is still valid
+        # we check kernel command line or files.
+        current = self.get_instance_id()
+        if not current:
+            return None
+
+        quick_id = _quick_read_instance_id(cmdline_id=self.cmdline_id,
+                                           dirs=[self.seed_dir])
+        if not quick_id:
+            return None
+        return quick_id == current
+
+
+def _quick_read_instance_id(cmdline_id, dirs=None):
+    if dirs is None:
+        dirs = []
+
+    iid_key = 'instance-id'
+    if cmdline_id is None:
+        fill = {}
+        if parse_cmdline_data(cmdline_id, fill) and iid_key in fill:
+            return fill[iid_key]
+
+    for d in dirs:
+        try:
+            data = util.pathprefix2dict(d, required=['meta-data'])
+            md = util.load_yaml(data['meta-data'])
+            if iid_key in md:
+                return md[iid_key]
+        except ValueError:
+            pass
+
+    return None
+
 
 # Returns true or false indicating if cmdline indicated
 # that this module should be used
diff --git a/cloudinit/sources/DataSourceOVF.py b/cloudinit/sources/DataSourceOVF.py
index 23996b4a..5734d233 100644
--- a/cloudinit/sources/DataSourceOVF.py
+++ b/cloudinit/sources/DataSourceOVF.py
@@ -31,9 +31,17 @@ import time
 from cloudinit import log as logging
 from cloudinit import sources
 from cloudinit import util
-from cloudinit.sources.helpers.vmware.imc.config import Config
-from cloudinit.sources.helpers.vmware.imc.config_file import ConfigFile
-from cloudinit.sources.helpers.vmware.imc.config_nic import NicConfigurator
+from .helpers.vmware.imc.config import Config
+from .helpers.vmware.imc.config_file import ConfigFile
+from .helpers.vmware.imc.config_nic import NicConfigurator
+from .helpers.vmware.imc.guestcust_event import GuestCustEventEnum
+from .helpers.vmware.imc.guestcust_state import GuestCustStateEnum
+from .helpers.vmware.imc.guestcust_error import GuestCustErrorEnum
+from .helpers.vmware.imc.guestcust_util import (
+    set_customization_status,
+    get_nics_to_enable,
+    enable_nics
+)
 
 LOG = logging.getLogger(__name__)
 
@@ -73,6 +81,9 @@ class DataSourceOVF(sources.DataSource):
                     self.sys_cfg, "disable_vmware_customization", True):
                 deployPkgPluginPath = search_file("/usr/lib/vmware-tools",
                                                   "libdeployPkgPlugin.so")
+                if not deployPkgPluginPath:
+                    deployPkgPluginPath = search_file("/usr/lib/open-vm-tools",
+                                                      "libdeployPkgPlugin.so")
                 if deployPkgPluginPath:
                     vmwareImcConfigFilePath = util.log_time(
                         logfunc=LOG.debug,
@@ -88,19 +99,41 @@ class DataSourceOVF(sources.DataSource):
                 LOG.debug("Customization for VMware platform is disabled.")
 
         if vmwareImcConfigFilePath:
+            nics = ""
             try:
                 cf = ConfigFile(vmwareImcConfigFilePath)
                 conf = Config(cf)
                 (md, ud, cfg) = read_vmware_imc(conf)
-                nicConfigurator = NicConfigurator(conf.nics)
-                nicConfigurator.configure()
-                vmwarePlatformFound = True
-            except Exception as inst:
-                LOG.debug("Error while parsing the Customization "
-                          "Config File: %s", inst)
+                dirpath = os.path.dirname(vmwareImcConfigFilePath)
+                nics = get_nics_to_enable(dirpath)
+            except Exception as e:
+                LOG.debug("Error parsing the customization Config File")
+                LOG.exception(e)
+                set_customization_status(
+                    GuestCustStateEnum.GUESTCUST_STATE_RUNNING,
+                    GuestCustEventEnum.GUESTCUST_EVENT_CUSTOMIZE_FAILED)
+                return False
             finally:
                 dirPath = os.path.dirname(vmwareImcConfigFilePath)
                 shutil.rmtree(dirPath)
+
+            try:
+                LOG.debug("Applying the Network customization")
+                nicConfigurator = NicConfigurator(conf.nics)
+                nicConfigurator.configure()
+            except Exception as e:
+                LOG.debug("Error applying the Network Configuration")
+                LOG.exception(e)
+                set_customization_status(
+                    GuestCustStateEnum.GUESTCUST_STATE_RUNNING,
+                    GuestCustEventEnum.GUESTCUST_EVENT_NETWORK_SETUP_FAILED)
+                return False
+
+            vmwarePlatformFound = True
+            enable_nics(nics)
+            set_customization_status(
+                GuestCustStateEnum.GUESTCUST_STATE_DONE,
+                GuestCustErrorEnum.GUESTCUST_ERROR_SUCCESS)
         elif seedfile:
             # Found a seed dir
             seed = os.path.join(self.paths.seed_dir, seedfile)
@@ -198,6 +231,9 @@ def read_vmware_imc(config):
         else:
             md['local-hostname'] = config.host_name
 
+    if config.timezone:
+        cfg['timezone'] = config.timezone
+
     return (md, ud, cfg)
 
 
diff --git a/cloudinit/sources/DataSourceOpenStack.py b/cloudinit/sources/DataSourceOpenStack.py
index 469c2e2a..79bb9d63 100644
--- a/cloudinit/sources/DataSourceOpenStack.py
+++ b/cloudinit/sources/DataSourceOpenStack.py
@@ -150,6 +150,10 @@ class DataSourceOpenStack(openstack.SourceMixin, sources.DataSource):
 
         return True
 
+    def check_instance_id(self):
+        # quickly (local check only) if self.instance_id is still valid
+        return sources.instance_id_matches_system_uuid(self.get_instance_id())
+
 
 def read_metadata_service(base_url, ssl_details=None):
     reader = openstack.MetadataReader(base_url, ssl_details=ssl_details)
diff --git a/cloudinit/sources/__init__.py b/cloudinit/sources/__init__.py
index d3cfa560..28540a7b 100644
--- a/cloudinit/sources/__init__.py
+++ b/cloudinit/sources/__init__.py
@@ -217,6 +217,10 @@ class DataSource(object):
     def get_package_mirror_info(self):
         return self.distro.get_package_mirror_info(data_source=self)
 
+    def check_instance_id(self):
+        # quickly (local check only) if self.instance_id is still
+        return False
+
 
 def normalize_pubkey_data(pubkey_data):
     keys = []
@@ -299,6 +303,18 @@ def list_sources(cfg_list, depends, pkg_list):
     return src_list
 
 
+def instance_id_matches_system_uuid(instance_id, field='system-uuid'):
+    # quickly (local check only) if self.instance_id is still valid
+    # we check kernel command line or files.
+    if not instance_id:
+        return False
+
+    dmi_value = util.read_dmi_data(field)
+    if not dmi_value:
+        return False
+    return instance_id.lower() == dmi_value.lower()
+
+
 # 'depends' is a list of dependencies (DEP_FILESYSTEM)
 # ds_list is a list of 2 item lists
 # ds_list = [
diff --git a/cloudinit/sources/helpers/vmware/imc/config_nic.py b/cloudinit/sources/helpers/vmware/imc/config_nic.py
index 8c5c08cf..77098a05 100644
--- a/cloudinit/sources/helpers/vmware/imc/config_nic.py
+++ b/cloudinit/sources/helpers/vmware/imc/config_nic.py
@@ -205,7 +205,8 @@ class NicConfigurator:
     def clear_dhcp(self):
         logger.info('Clearing DHCP leases')
 
-        util.subp(["pkill", "dhclient"])
+        # Ignore the return code 1.
+        util.subp(["pkill", "dhclient"], rcs=[0, 1])
         util.subp(["rm", "-f", "/var/lib/dhcp/*"])
 
     def if_down_up(self):
diff --git a/cloudinit/sources/helpers/vmware/imc/guestcust_error.py b/cloudinit/sources/helpers/vmware/imc/guestcust_error.py
new file mode 100644
index 00000000..1b04161f
--- /dev/null
+++ b/cloudinit/sources/helpers/vmware/imc/guestcust_error.py
@@ -0,0 +1,24 @@
+# vi: ts=4 expandtab

+#

+#    Copyright (C) 2016 Canonical Ltd.

+#    Copyright (C) 2016 VMware Inc.

+#

+#    Author: Sankar Tanguturi <stanguturi@vmware.com>

+#

+#    This program is free software: you can redistribute it and/or modify

+#    it under the terms of the GNU General Public License version 3, as

+#    published by the Free Software Foundation.

+#

+#    This program is distributed in the hope that it will be useful,

+#    but WITHOUT ANY WARRANTY; without even the implied warranty of

+#    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the

+#    GNU General Public License for more details.

+#

+#    You should have received a copy of the GNU General Public License

+#    along with this program.  If not, see <http://www.gnu.org/licenses/>.

+

+

+class GuestCustErrorEnum:

+    """Specifies different errors of Guest Customization engine"""

+

+    GUESTCUST_ERROR_SUCCESS = 0

diff --git a/cloudinit/sources/helpers/vmware/imc/guestcust_event.py b/cloudinit/sources/helpers/vmware/imc/guestcust_event.py
new file mode 100644
index 00000000..fc22568f
--- /dev/null
+++ b/cloudinit/sources/helpers/vmware/imc/guestcust_event.py
@@ -0,0 +1,27 @@
+# vi: ts=4 expandtab

+#

+#    Copyright (C) 2016 Canonical Ltd.

+#    Copyright (C) 2016 VMware Inc.

+#

+#    Author: Sankar Tanguturi <stanguturi@vmware.com>

+#

+#    This program is free software: you can redistribute it and/or modify

+#    it under the terms of the GNU General Public License version 3, as

+#    published by the Free Software Foundation.

+#

+#    This program is distributed in the hope that it will be useful,

+#    but WITHOUT ANY WARRANTY; without even the implied warranty of

+#    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the

+#    GNU General Public License for more details.

+#

+#    You should have received a copy of the GNU General Public License

+#    along with this program.  If not, see <http://www.gnu.org/licenses/>.

+

+

+class GuestCustEventEnum:

+    """Specifies different types of Guest Customization Events"""

+

+    GUESTCUST_EVENT_CUSTOMIZE_FAILED = 100

+    GUESTCUST_EVENT_NETWORK_SETUP_FAILED = 101

+    GUESTCUST_EVENT_ENABLE_NICS = 103

+    GUESTCUST_EVENT_QUERY_NICS = 104

diff --git a/cloudinit/sources/helpers/vmware/imc/guestcust_state.py b/cloudinit/sources/helpers/vmware/imc/guestcust_state.py
new file mode 100644
index 00000000..f255be5f
--- /dev/null
+++ b/cloudinit/sources/helpers/vmware/imc/guestcust_state.py
@@ -0,0 +1,25 @@
+# vi: ts=4 expandtab

+#

+#    Copyright (C) 2016 Canonical Ltd.

+#    Copyright (C) 2016 VMware Inc.

+#

+#    Author: Sankar Tanguturi <stanguturi@vmware.com>

+#

+#    This program is free software: you can redistribute it and/or modify

+#    it under the terms of the GNU General Public License version 3, as

+#    published by the Free Software Foundation.

+#

+#    This program is distributed in the hope that it will be useful,

+#    but WITHOUT ANY WARRANTY; without even the implied warranty of

+#    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the

+#    GNU General Public License for more details.

+#

+#    You should have received a copy of the GNU General Public License

+#    along with this program.  If not, see <http://www.gnu.org/licenses/>.

+

+

+class GuestCustStateEnum:

+    """Specifies different states of Guest Customization engine"""

+

+    GUESTCUST_STATE_RUNNING = 4

+    GUESTCUST_STATE_DONE = 5

diff --git a/cloudinit/sources/helpers/vmware/imc/guestcust_util.py b/cloudinit/sources/helpers/vmware/imc/guestcust_util.py
new file mode 100644
index 00000000..d39f0a65
--- /dev/null
+++ b/cloudinit/sources/helpers/vmware/imc/guestcust_util.py
@@ -0,0 +1,128 @@
+# vi: ts=4 expandtab

+#

+#    Copyright (C) 2016 Canonical Ltd.

+#    Copyright (C) 2016 VMware Inc.

+#

+#    Author: Sankar Tanguturi <stanguturi@vmware.com>

+#

+#    This program is free software: you can redistribute it and/or modify

+#    it under the terms of the GNU General Public License version 3, as

+#    published by the Free Software Foundation.

+#

+#    This program is distributed in the hope that it will be useful,

+#    but WITHOUT ANY WARRANTY; without even the implied warranty of

+#    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the

+#    GNU General Public License for more details.

+#

+#    You should have received a copy of the GNU General Public License

+#    along with this program.  If not, see <http://www.gnu.org/licenses/>.

+

+import logging

+import os

+import time

+

+from cloudinit import util

+

+from .guestcust_state import GuestCustStateEnum

+from .guestcust_event import GuestCustEventEnum

+

+logger = logging.getLogger(__name__)

+

+

+CLOUDINIT_LOG_FILE = "/var/log/cloud-init.log"

+QUERY_NICS_SUPPORTED = "queryNicsSupported"

+NICS_STATUS_CONNECTED = "connected"

+

+

+# This will send a RPC command to the underlying

+# VMware Virtualization Platform.

+def send_rpc(rpc):

+    if not rpc:

+        return None

+

+    out = ""

+    err = "Error sending the RPC command"

+

+    try:

+        logger.debug("Sending RPC command: %s", rpc)

+        (out, err) = util.subp(["vmware-rpctool", rpc], rcs=[0])

+        # Remove the trailing newline in the output.

+        if out:

+            out = out.rstrip()

+    except Exception as e:

+        logger.debug("Failed to send RPC command")

+        logger.exception(e)

+

+    return (out, err)

+

+

+# This will send the customization status to the

+# underlying VMware Virtualization Platform.

+def set_customization_status(custstate, custerror, errormessage=None):

+    message = ""

+

+    if errormessage:

+        message = CLOUDINIT_LOG_FILE + "@" + errormessage

+    else:

+        message = CLOUDINIT_LOG_FILE

+

+    rpc = "deployPkg.update.state %d %d %s" % (custstate, custerror, message)

+    (out, err) = send_rpc(rpc)

+    return (out, err)

+

+

+# This will read the file nics.txt in the specified directory

+# and return the content

+def get_nics_to_enable(dirpath):

+    if not dirpath:

+        return None

+

+    NICS_SIZE = 1024

+    nicsfilepath = os.path.join(dirpath, "nics.txt")

+    if not os.path.exists(nicsfilepath):

+        return None

+

+    with open(nicsfilepath, 'r') as fp:

+        nics = fp.read(NICS_SIZE)

+

+    return nics

+

+

+# This will send a RPC command to the underlying VMware Virtualization platform

+# and enable nics.

+def enable_nics(nics):

+    if not nics:

+        logger.warning("No Nics found")

+        return

+

+    enableNicsWaitRetries = 5

+    enableNicsWaitCount = 5

+    enableNicsWaitSeconds = 1

+

+    for attempt in range(0, enableNicsWaitRetries):

+        logger.debug("Trying to connect interfaces, attempt %d", attempt)

+        (out, err) = set_customization_status(

+            GuestCustStateEnum.GUESTCUST_STATE_RUNNING,

+            GuestCustEventEnum.GUESTCUST_EVENT_ENABLE_NICS,

+            nics)

+        if not out:

+            time.sleep(enableNicsWaitCount * enableNicsWaitSeconds)

+            continue

+

+        if out != QUERY_NICS_SUPPORTED:

+            logger.warning("NICS connection status query is not supported")

+            return

+

+        for count in range(0, enableNicsWaitCount):

+            (out, err) = set_customization_status(

+                GuestCustStateEnum.GUESTCUST_STATE_RUNNING,

+                GuestCustEventEnum.GUESTCUST_EVENT_QUERY_NICS,

+                nics)

+            if out and out == NICS_STATUS_CONNECTED:

+                logger.info("NICS are connected on %d second", count)

+                return

+

+            time.sleep(enableNicsWaitSeconds)

+

+    logger.warning("Can't connect network interfaces after %d attempts",

+                   enableNicsWaitRetries)

diff --git a/cloudinit/stages.py b/cloudinit/stages.py
index dbcf3d55..edad6450 100644
--- a/cloudinit/stages.py
+++ b/cloudinit/stages.py
@@ -140,7 +140,7 @@ class Init(object):
         ]
         return initial_dirs
 
-    def purge_cache(self, rm_instance_lnk=True):
+    def purge_cache(self, rm_instance_lnk=False):
         rm_list = []
         rm_list.append(self.paths.boot_finished)
         if rm_instance_lnk:
@@ -238,21 +238,29 @@ class Init(object):
         cfg_list = self.cfg.get('datasource_list') or []
         return (cfg_list, pkg_list)
 
-    def _get_data_source(self):
+    def _get_data_source(self, existing):
         if self.datasource is not NULL_DATA_SOURCE:
             return self.datasource
 
         with events.ReportEventStack(
                 name="check-cache",
-                description="attempting to read from cache",
+                description="attempting to read from cache [%s]" % existing,
                 parent=self.reporter) as myrep:
             ds = self._restore_from_cache()
-            if ds:
-                LOG.debug("Restored from cache, datasource: %s", ds)
-                myrep.description = "restored from cache"
+            if ds and existing == "trust":
+                myrep.description = "restored from cache: %s" % ds
+            elif ds and existing == "check":
+                if hasattr(ds, 'check_instance_id') and ds.check_instance_id():
+                    myrep.description = "restored from checked cache: %s" % ds
+                else:
+                    myrep.description = "cache invalid in datasource: %s" % ds
+                    ds = None
             else:
                 myrep.description = "no cache found"
+            LOG.debug(myrep.description)
+
         if not ds:
+            util.del_file(self.paths.instance_link)
             (cfg_list, pkg_list) = self._get_datasources()
             # Deep copy so that user-data handlers can not modify
             # (which will affect user-data handlers down the line...)
@@ -332,8 +340,8 @@ class Init(object):
         self._reset()
         return iid
 
-    def fetch(self):
-        return self._get_data_source()
+    def fetch(self, existing="check"):
+        return self._get_data_source(existing=existing)
 
     def instancify(self):
         return self._reflect_cur_instance()