improve network configuration

This branch accomplishes several things:
 - centrally handle 'dsmode' to be 'local' or 'net.
   This allows local data sources to run before networking
   but still have user-data read by default when networking is available.

 - support networking information being read on dreamcompute
   dreamcompute's openstack declares networking via the
   /etc/network/interfaces style 'network_config' format.
 
 - support reading and applying networking information on SmartOS

 - improve reading networking from openstack network_data.json (LP: #1577982)
   add support for mtu and routes and many miscellaneous fixes.

 - support for renaming devices in a container (LP: #1579130).
   Also rename network devices as instructed by the host on
   every boot where cloud-init networking is enabled.  This is required
   because a.) containers do not get systemd.link files applied
   as they do not have udev. b.) if the initramfs is out of date
   then we need to apply them.

 - remove blocking of udev rules (LP: #1577844, LP: #1571761)

LP: #1577982, #1579130, #1577844, #1571761

1 files changed, 69 insertions, 24 deletions

diff --git a/cloudinit/stages.py b/cloudinit/stages.py
index 002e5832..5756e74d 100644
--- a/cloudinit/stages.py
+++ b/cloudinit/stages.py
@@ -52,6 +52,7 @@ from cloudinit import util
 LOG = logging.getLogger(__name__)
 
 NULL_DATA_SOURCE = None
+NO_PREVIOUS_INSTANCE_ID = "NO_PREVIOUS_INSTANCE_ID"
 
 
 class Init(object):
@@ -67,6 +68,7 @@ class Init(object):
         # Changed only when a fetch occurs
         self.datasource = NULL_DATA_SOURCE
         self.ds_restored = False
+        self._previous_iid = None
 
         if reporter is None:
             reporter = events.ReportEventStack(
@@ -213,6 +215,31 @@ class Init(object):
         cfg_list = self.cfg.get('datasource_list') or []
         return (cfg_list, pkg_list)
 
+    def _restore_from_checked_cache(self, existing):
+        if existing not in ("check", "trust"):
+            raise ValueError("Unexpected value for existing: %s" % existing)
+
+        ds = self._restore_from_cache()
+        if not ds:
+            return (None, "no cache found")
+
+        run_iid_fn = self.paths.get_runpath('instance_id')
+        if os.path.exists(run_iid_fn):
+            run_iid = util.load_file(run_iid_fn).strip()
+        else:
+            run_iid = None
+
+        if run_iid == ds.get_instance_id():
+            return (ds, "restored from cache with run check: %s" % ds)
+        elif existing == "trust":
+            return (ds, "restored from cache: %s" % ds)
+        else:
+            if (hasattr(ds, 'check_instance_id') and
+                    ds.check_instance_id(self.cfg)):
+                return (ds, "restored from checked cache: %s" % ds)
+            else:
+                return (None, "cache invalid in datasource: %s" % ds)
+
     def _get_data_source(self, existing):
         if self.datasource is not NULL_DATA_SOURCE:
             return self.datasource
@@ -221,19 +248,9 @@ class Init(object):
                 name="check-cache",
                 description="attempting to read from cache [%s]" % existing,
                 parent=self.reporter) as myrep:
-            ds = self._restore_from_cache()
-            if ds and existing == "trust":
-                myrep.description = "restored from cache: %s" % ds
-            elif ds and existing == "check":
-                if (hasattr(ds, 'check_instance_id') and
-                        ds.check_instance_id(self.cfg)):
-                    myrep.description = "restored from checked cache: %s" % ds
-                else:
-                    myrep.description = "cache invalid in datasource: %s" % ds
-                    ds = None
-            else:
-                myrep.description = "no cache found"
 
+            ds, desc = self._restore_from_checked_cache(existing)
+            myrep.description = desc
             self.ds_restored = bool(ds)
             LOG.debug(myrep.description)
 
@@ -301,23 +318,41 @@ class Init(object):
 
         # What the instance id was and is...
         iid = self.datasource.get_instance_id()
-        previous_iid = None
         iid_fn = os.path.join(dp, 'instance-id')
-        try:
-            previous_iid = util.load_file(iid_fn).strip()
-        except Exception:
-            pass
-        if not previous_iid:
-            previous_iid = iid
+
+        previous_iid = self.previous_iid()
         util.write_file(iid_fn, "%s\n" % iid)
+        util.write_file(self.paths.get_runpath('instance_id'), "%s\n" % iid)
         util.write_file(os.path.join(dp, 'previous-instance-id'),
                         "%s\n" % (previous_iid))
+
+        self._write_to_cache()
         # Ensure needed components are regenerated
         # after change of instance which may cause
         # change of configuration
         self._reset()
         return iid
 
+    def previous_iid(self):
+        if self._previous_iid is not None:
+            return self._previous_iid
+
+        dp = self.paths.get_cpath('data')
+        iid_fn = os.path.join(dp, 'instance-id')
+        try:
+            self._previous_iid = util.load_file(iid_fn).strip()
+        except Exception:
+            self._previous_iid = NO_PREVIOUS_INSTANCE_ID
+
+        LOG.debug("previous iid found to be %s", self._previous_iid)
+        return self._previous_iid
+
+    def is_new_instance(self):
+        previous = self.previous_iid()
+        ret = (previous == NO_PREVIOUS_INSTANCE_ID or
+               previous != self.datasource.get_instance_id())
+        return ret
+
     def fetch(self, existing="check"):
         return self._get_data_source(existing=existing)
 
@@ -332,8 +367,6 @@ class Init(object):
                            reporter=self.reporter)
 
     def update(self):
-        if not self._write_to_cache():
-            return
         self._store_userdata()
         self._store_vendordata()
 
@@ -593,15 +626,27 @@ class Init(object):
                 return (ncfg, loc)
         return (net.generate_fallback_config(), "fallback")
 
-    def apply_network_config(self):
+    def apply_network_config(self, bring_up):
         netcfg, src = self._find_networking_config()
         if netcfg is None:
             LOG.info("network config is disabled by %s", src)
             return
 
-        LOG.info("Applying network configuration from %s: %s", src, netcfg)
         try:
-            return self.distro.apply_network_config(netcfg)
+            LOG.debug("applying net config names for %s" % netcfg)
+            self.distro.apply_network_config_names(netcfg)
+        except Exception as e:
+            LOG.warn("Failed to rename devices: %s", e)
+
+        if (self.datasource is not NULL_DATA_SOURCE and
+                not self.is_new_instance()):
+            LOG.debug("not a new instance. network config is not applied.")
+            return
+
+        LOG.info("Applying network configuration from %s bringup=%s: %s",
+                 src, bring_up, netcfg)
+        try:
+            return self.distro.apply_network_config(netcfg, bring_up=bring_up)
         except NotImplementedError:
             LOG.warn("distro '%s' does not implement apply_network_config. "
                      "networking may not be configured properly." %