quickly check to see if the previous instance id is still valid

This adds a check in cloud-init to see if the existing (cached)
datasource is still valid.  It relies on support from the Datasource
to implement 'check_instance_id'.  That method should quickly determine
(if possible) if the instance id found in the datasource is still valid.

This means that we can still notice new instance ids without
depending on a network datasource on every boot.

I've also implemented check_instance_id for the superclass and for
sources:
  DataSourceAzure (check dmi data)
  DataSourceConfigDrive (check dmi data)
  DataSourceNocloud (check the seeded data or kernel command line)
  DataSourceOpenstack (check dmi data)

LP: #1553815

8 files changed, 89 insertions, 19 deletions

diff --git a/ChangeLog b/ChangeLog
index 0ec4f49e..b08665b0 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -92,6 +92,8 @@
  - doc: mention label for nocloud datasource must be 'cidata' [Peter Hurley]
  - ssh_pwauth: fix module to support 'unchanged' and match behavior
    described in documentation [Chris Cosby]
+ - quickly check to see if the previous instance id is still valid to
+   avoid dependency on network metadata service on every boot (LP: #1553815)
 
 0.7.6:
  - open 0.7.6
diff --git a/bin/cloud-init b/bin/cloud-init
index 7f665e7e..11cc0237 100755
--- a/bin/cloud-init
+++ b/bin/cloud-init
@@ -212,6 +212,7 @@ def main_init(name, args):
     # Stage 4
     path_helper = init.paths
     if not args.local:
+        existing = "trust"
         sys.stderr.write("%s\n" % (netinfo.debug_info()))
         LOG.debug(("Checking to see if files that we need already"
                    " exist from a previous run that would allow us"
@@ -236,21 +237,17 @@ def main_init(name, args):
             LOG.debug("Execution continuing, no previous run detected that"
                       " would allow us to stop early.")
     else:
-        # The cache is not instance specific, so it has to be purged
-        # but we want 'start' to benefit from a cache if
-        # a previous start-local populated one...
-        manual_clean = util.get_cfg_option_bool(init.cfg,
-                                                'manual_cache_clean', False)
-        if manual_clean:
-            LOG.debug("Not purging instance link, manual cleaning enabled")
-            init.purge_cache(False)
-        else:
-            init.purge_cache()
+        existing = "check"
+        if util.get_cfg_option_bool(init.cfg, 'manual_cache_clean', False):
+            existing = "trust"
+
+        init.purge_cache()
         # Delete the non-net file as well
         util.del_file(os.path.join(path_helper.get_cpath("data"), "no-net"))
+
     # Stage 5
     try:
-        init.fetch()
+        init.fetch(existing=existing)
     except sources.DataSourceNotFoundException:
         # In the case of 'cloud-init init' without '--local' it is a bit
         # more likely that the user would consider it failure if nothing was
diff --git a/cloudinit/sources/DataSourceAzure.py b/cloudinit/sources/DataSourceAzure.py
index 2af0ad9b..832b3063 100644
--- a/cloudinit/sources/DataSourceAzure.py
+++ b/cloudinit/sources/DataSourceAzure.py
@@ -254,6 +254,10 @@ class DataSourceAzureNet(sources.DataSource):
     def get_config_obj(self):
         return self.cfg
 
+    def check_instance_id(self):
+        # quickly (local check only) if self.instance_id is still valid
+        return sources.instance_id_matches_system_uuid(self.get_instance_id())
+
 
 def count_files(mp):
     return len(fnmatch.filter(os.listdir(mp), '*[!cdrom]*'))
diff --git a/cloudinit/sources/DataSourceConfigDrive.py b/cloudinit/sources/DataSourceConfigDrive.py
index e3916208..6fc9e05b 100644
--- a/cloudinit/sources/DataSourceConfigDrive.py
+++ b/cloudinit/sources/DataSourceConfigDrive.py
@@ -146,6 +146,10 @@ class DataSourceConfigDrive(openstack.SourceMixin, sources.DataSource):
 
         return True
 
+    def check_instance_id(self):
+        # quickly (local check only) if self.instance_id is still valid
+        return sources.instance_id_matches_system_uuid(self.get_instance_id())
+
 
 class DataSourceConfigDriveNet(DataSourceConfigDrive):
     def __init__(self, sys_cfg, distro, paths):
diff --git a/cloudinit/sources/DataSourceNoCloud.py b/cloudinit/sources/DataSourceNoCloud.py
index 4cad6877..d07e6f84 100644
--- a/cloudinit/sources/DataSourceNoCloud.py
+++ b/cloudinit/sources/DataSourceNoCloud.py
@@ -197,6 +197,41 @@ class DataSourceNoCloud(sources.DataSource):
                   mydata['meta-data']['dsmode'])
         return False
 
+    def check_instance_id(self):
+        # quickly (local check only) if self.instance_id is still valid
+        # we check kernel command line or files.
+        current = self.get_instance_id()
+        if not current:
+            return None
+
+        quick_id = _quick_read_instance_id(cmdline_id=self.cmdline_id,
+                                           dirs=[self.seed_dir])
+        if not quick_id:
+            return None
+        return quick_id == current
+
+
+def _quick_read_instance_id(cmdline_id, dirs=None):
+    if dirs is None:
+        dirs = []
+
+    iid_key = 'instance-id'
+    if cmdline_id is None:
+        fill = {}
+        if parse_cmdline_data(cmdline_id, fill) and iid_key in fill:
+            return fill[iid_key]
+
+    for d in dirs:
+        try:
+            data = util.pathprefix2dict(d, required=['meta-data'])
+            md = util.load_yaml(data['meta-data'])
+            if iid_key in md:
+                return md[iid_key]
+        except ValueError:
+            pass
+
+    return None
+
 
 # Returns true or false indicating if cmdline indicated
 # that this module should be used
diff --git a/cloudinit/sources/DataSourceOpenStack.py b/cloudinit/sources/DataSourceOpenStack.py
index 469c2e2a..79bb9d63 100644
--- a/cloudinit/sources/DataSourceOpenStack.py
+++ b/cloudinit/sources/DataSourceOpenStack.py
@@ -150,6 +150,10 @@ class DataSourceOpenStack(openstack.SourceMixin, sources.DataSource):
 
         return True
 
+    def check_instance_id(self):
+        # quickly (local check only) if self.instance_id is still valid
+        return sources.instance_id_matches_system_uuid(self.get_instance_id())
+
 
 def read_metadata_service(base_url, ssl_details=None):
     reader = openstack.MetadataReader(base_url, ssl_details=ssl_details)
diff --git a/cloudinit/sources/__init__.py b/cloudinit/sources/__init__.py
index d3cfa560..28540a7b 100644
--- a/cloudinit/sources/__init__.py
+++ b/cloudinit/sources/__init__.py
@@ -217,6 +217,10 @@ class DataSource(object):
     def get_package_mirror_info(self):
         return self.distro.get_package_mirror_info(data_source=self)
 
+    def check_instance_id(self):
+        # quickly (local check only) if self.instance_id is still
+        return False
+
 
 def normalize_pubkey_data(pubkey_data):
     keys = []
@@ -299,6 +303,18 @@ def list_sources(cfg_list, depends, pkg_list):
     return src_list
 
 
+def instance_id_matches_system_uuid(instance_id, field='system-uuid'):
+    # quickly (local check only) if self.instance_id is still valid
+    # we check kernel command line or files.
+    if not instance_id:
+        return False
+
+    dmi_value = util.read_dmi_data(field)
+    if not dmi_value:
+        return False
+    return instance_id.lower() == dmi_value.lower()
+
+
 # 'depends' is a list of dependencies (DEP_FILESYSTEM)
 # ds_list is a list of 2 item lists
 # ds_list = [
diff --git a/cloudinit/stages.py b/cloudinit/stages.py
index dbcf3d55..edad6450 100644
--- a/cloudinit/stages.py
+++ b/cloudinit/stages.py
@@ -140,7 +140,7 @@ class Init(object):
         ]
         return initial_dirs
 
-    def purge_cache(self, rm_instance_lnk=True):
+    def purge_cache(self, rm_instance_lnk=False):
         rm_list = []
         rm_list.append(self.paths.boot_finished)
         if rm_instance_lnk:
@@ -238,21 +238,29 @@ class Init(object):
         cfg_list = self.cfg.get('datasource_list') or []
         return (cfg_list, pkg_list)
 
-    def _get_data_source(self):
+    def _get_data_source(self, existing):
         if self.datasource is not NULL_DATA_SOURCE:
             return self.datasource
 
         with events.ReportEventStack(
                 name="check-cache",
-                description="attempting to read from cache",
+                description="attempting to read from cache [%s]" % existing,
                 parent=self.reporter) as myrep:
             ds = self._restore_from_cache()
-            if ds:
-                LOG.debug("Restored from cache, datasource: %s", ds)
-                myrep.description = "restored from cache"
+            if ds and existing == "trust":
+                myrep.description = "restored from cache: %s" % ds
+            elif ds and existing == "check":
+                if hasattr(ds, 'check_instance_id') and ds.check_instance_id():
+                    myrep.description = "restored from checked cache: %s" % ds
+                else:
+                    myrep.description = "cache invalid in datasource: %s" % ds
+                    ds = None
             else:
                 myrep.description = "no cache found"
+            LOG.debug(myrep.description)
+
         if not ds:
+            util.del_file(self.paths.instance_link)
             (cfg_list, pkg_list) = self._get_datasources()
             # Deep copy so that user-data handlers can not modify
             # (which will affect user-data handlers down the line...)
@@ -332,8 +340,8 @@ class Init(object):
         self._reset()
         return iid
 
-    def fetch(self):
-        return self._get_data_source()
+    def fetch(self, existing="check"):
+        return self._get_data_source(existing=existing)
 
     def instancify(self):
         return self._reflect_cur_instance()