quickly check to see if the previous instance id is still valid

This adds a check in cloud-init to see if the existing (cached)
datasource is still valid.  It relies on support from the Datasource
to implement 'check_instance_id'.  That method should quickly determine
(if possible) if the instance id found in the datasource is still valid.

This means that we can still notice new instance ids without
depending on a network datasource on every boot.

I've also implemented check_instance_id for the superclass and for
3 classes:
  DataSourceAzure (check dmi data)
  DataSourceOpenstack (check dmi data)
  DataSourceNocloud (check the seeded data or kernel command line)


LP: #1553815

5 files changed, 75 insertions, 8 deletions

diff --git a/cloudinit/sources/DataSourceAzure.py b/cloudinit/sources/DataSourceAzure.py
index 2af0ad9b..832b3063 100644
--- a/cloudinit/sources/DataSourceAzure.py
+++ b/cloudinit/sources/DataSourceAzure.py
@@ -254,6 +254,10 @@ class DataSourceAzureNet(sources.DataSource):
     def get_config_obj(self):
         return self.cfg
 
+    def check_instance_id(self):
+        # quickly (local check only) if self.instance_id is still valid
+        return sources.instance_id_matches_system_uuid(self.get_instance_id())
+
 
 def count_files(mp):
     return len(fnmatch.filter(os.listdir(mp), '*[!cdrom]*'))
diff --git a/cloudinit/sources/DataSourceNoCloud.py b/cloudinit/sources/DataSourceNoCloud.py
index 4cad6877..d07e6f84 100644
--- a/cloudinit/sources/DataSourceNoCloud.py
+++ b/cloudinit/sources/DataSourceNoCloud.py
@@ -197,6 +197,41 @@ class DataSourceNoCloud(sources.DataSource):
                   mydata['meta-data']['dsmode'])
         return False
 
+    def check_instance_id(self):
+        # quickly (local check only) if self.instance_id is still valid
+        # we check kernel command line or files.
+        current = self.get_instance_id()
+        if not current:
+            return None
+
+        quick_id = _quick_read_instance_id(cmdline_id=self.cmdline_id,
+                                           dirs=[self.seed_dir])
+        if not quick_id:
+            return None
+        return quick_id == current
+
+
+def _quick_read_instance_id(cmdline_id, dirs=None):
+    if dirs is None:
+        dirs = []
+
+    iid_key = 'instance-id'
+    if cmdline_id is None:
+        fill = {}
+        if parse_cmdline_data(cmdline_id, fill) and iid_key in fill:
+            return fill[iid_key]
+
+    for d in dirs:
+        try:
+            data = util.pathprefix2dict(d, required=['meta-data'])
+            md = util.load_yaml(data['meta-data'])
+            if iid_key in md:
+                return md[iid_key]
+        except ValueError:
+            pass
+
+    return None
+
 
 # Returns true or false indicating if cmdline indicated
 # that this module should be used
diff --git a/cloudinit/sources/DataSourceOpenStack.py b/cloudinit/sources/DataSourceOpenStack.py
index 469c2e2a..79bb9d63 100644
--- a/cloudinit/sources/DataSourceOpenStack.py
+++ b/cloudinit/sources/DataSourceOpenStack.py
@@ -150,6 +150,10 @@ class DataSourceOpenStack(openstack.SourceMixin, sources.DataSource):
 
         return True
 
+    def check_instance_id(self):
+        # quickly (local check only) if self.instance_id is still valid
+        return sources.instance_id_matches_system_uuid(self.get_instance_id())
+
 
 def read_metadata_service(base_url, ssl_details=None):
     reader = openstack.MetadataReader(base_url, ssl_details=ssl_details)
diff --git a/cloudinit/sources/__init__.py b/cloudinit/sources/__init__.py
index d3cfa560..28540a7b 100644
--- a/cloudinit/sources/__init__.py
+++ b/cloudinit/sources/__init__.py
@@ -217,6 +217,10 @@ class DataSource(object):
     def get_package_mirror_info(self):
         return self.distro.get_package_mirror_info(data_source=self)
 
+    def check_instance_id(self):
+        # quickly (local check only) if self.instance_id is still
+        return False
+
 
 def normalize_pubkey_data(pubkey_data):
     keys = []
@@ -299,6 +303,18 @@ def list_sources(cfg_list, depends, pkg_list):
     return src_list
 
 
+def instance_id_matches_system_uuid(instance_id, field='system-uuid'):
+    # quickly (local check only) if self.instance_id is still valid
+    # we check kernel command line or files.
+    if not instance_id:
+        return False
+
+    dmi_value = util.read_dmi_data(field)
+    if not dmi_value:
+        return False
+    return instance_id.lower() == dmi_value.lower()
+
+
 # 'depends' is a list of dependencies (DEP_FILESYSTEM)
 # ds_list is a list of 2 item lists
 # ds_list = [
diff --git a/cloudinit/stages.py b/cloudinit/stages.py
index dbcf3d55..edad6450 100644
--- a/cloudinit/stages.py
+++ b/cloudinit/stages.py
@@ -140,7 +140,7 @@ class Init(object):
         ]
         return initial_dirs
 
-    def purge_cache(self, rm_instance_lnk=True):
+    def purge_cache(self, rm_instance_lnk=False):
         rm_list = []
         rm_list.append(self.paths.boot_finished)
         if rm_instance_lnk:
@@ -238,21 +238,29 @@ class Init(object):
         cfg_list = self.cfg.get('datasource_list') or []
         return (cfg_list, pkg_list)
 
-    def _get_data_source(self):
+    def _get_data_source(self, existing):
         if self.datasource is not NULL_DATA_SOURCE:
             return self.datasource
 
         with events.ReportEventStack(
                 name="check-cache",
-                description="attempting to read from cache",
+                description="attempting to read from cache [%s]" % existing,
                 parent=self.reporter) as myrep:
             ds = self._restore_from_cache()
-            if ds:
-                LOG.debug("Restored from cache, datasource: %s", ds)
-                myrep.description = "restored from cache"
+            if ds and existing == "trust":
+                myrep.description = "restored from cache: %s" % ds
+            elif ds and existing == "check":
+                if hasattr(ds, 'check_instance_id') and ds.check_instance_id():
+                    myrep.description = "restored from checked cache: %s" % ds
+                else:
+                    myrep.description = "cache invalid in datasource: %s" % ds
+                    ds = None
             else:
                 myrep.description = "no cache found"
+            LOG.debug(myrep.description)
+
         if not ds:
+            util.del_file(self.paths.instance_link)
             (cfg_list, pkg_list) = self._get_datasources()
             # Deep copy so that user-data handlers can not modify
             # (which will affect user-data handlers down the line...)
@@ -332,8 +340,8 @@ class Init(object):
         self._reset()
         return iid
 
-    def fetch(self):
-        return self._get_data_source()
+    def fetch(self, existing="check"):
+        return self._get_data_source(existing=existing)
 
     def instancify(self):
         return self._reflect_cur_instance()