improve network configuration

This branch accomplishes several things:
 - centrally handle 'dsmode' to be 'local' or 'net.
   This allows local data sources to run before networking
   but still have user-data read by default when networking is available.

 - support networking information being read on dreamcompute
   dreamcompute's openstack declares networking via the
   /etc/network/interfaces style 'network_config' format.
 
 - support reading and applying networking information on SmartOS

 - improve reading networking from openstack network_data.json (LP: #1577982)
   add support for mtu and routes and many miscellaneous fixes.

 - support for renaming devices in a container (LP: #1579130).
   Also rename network devices as instructed by the host on
   every boot where cloud-init networking is enabled.  This is required
   because a.) containers do not get systemd.link files applied
   as they do not have udev. b.) if the initramfs is out of date
   then we need to apply them.

 - remove blocking of udev rules (LP: #1577844, LP: #1571761)

LP: #1577982, #1579130, #1577844, #1571761

1 files changed, 38 insertions, 22 deletions

diff --git a/bin/cloud-init b/bin/cloud-init
index 5857af32..21c3a684 100755
--- a/bin/cloud-init
+++ b/bin/cloud-init
@@ -211,27 +211,27 @@ def main_init(name, args):
         util.logexc(LOG, "Failed to initialize, likely bad things to come!")
     # Stage 4
     path_helper = init.paths
-    if not args.local:
+    mode = sources.DSMODE_LOCAL if args.local else sources.DSMODE_NETWORK
+
+    if mode == sources.DSMODE_NETWORK:
         existing = "trust"
         sys.stderr.write("%s\n" % (netinfo.debug_info()))
         LOG.debug(("Checking to see if files that we need already"
                    " exist from a previous run that would allow us"
                    " to stop early."))
+        # no-net is written by upstart cloud-init-nonet when network failed
+        # to come up
         stop_files = [
             os.path.join(path_helper.get_cpath("data"), "no-net"),
-            path_helper.get_ipath_cur("obj_pkl"),
         ]
         existing_files = []
         for fn in stop_files:
-            try:
-                c = util.load_file(fn)
-                if len(c):
-                    existing_files.append((fn, len(c)))
-            except Exception:
-                pass
+            if os.path.isfile(fn):
+                existing_files.append(fn)
+
         if existing_files:
-            LOG.debug("Exiting early due to the existence of %s files",
-                      existing_files)
+            LOG.debug("[%s] Exiting. stop file %s existed",
+                      mode, existing_files)
             return (None, [])
         else:
             LOG.debug("Execution continuing, no previous run detected that"
@@ -248,34 +248,50 @@ def main_init(name, args):
     # Stage 5
     try:
         init.fetch(existing=existing)
+        # if in network mode, and the datasource is local
+        # then work was done at that stage.
+        if mode == sources.DSMODE_NETWORK and init.datasource.dsmode != mode:
+            LOG.debug("[%s] Exiting. datasource %s in local mode",
+                      mode, init.datasource)
+            return (None, [])
     except sources.DataSourceNotFoundException:
         # In the case of 'cloud-init init' without '--local' it is a bit
         # more likely that the user would consider it failure if nothing was
         # found. When using upstart it will also mentions job failure
         # in console log if exit code is != 0.
-        if args.local:
+        if mode == sources.DSMODE_LOCAL:
             LOG.debug("No local datasource found")
         else:
             util.logexc(LOG, ("No instance datasource found!"
                               " Likely bad things to come!"))
         if not args.force:
-            init.apply_network_config()
-            if args.local:
+            init.apply_network_config(bring_up=not args.local)
+            LOG.debug("[%s] Exiting without datasource in local mode", mode)
+            if mode == sources.DSMODE_LOCAL:
                 return (None, [])
             else:
                 return (None, ["No instance datasource found."])
-
-    if args.local:
-        if not init.ds_restored:
-            # if local mode and the datasource was not restored from cache
-            # (this is not first boot) then apply networking.
-            init.apply_network_config()
         else:
-            LOG.debug("skipping networking config from restored datasource.")
+            LOG.debug("[%s] barreling on in force mode without datasource",
+                      mode)
 
     # Stage 6
     iid = init.instancify()
-    LOG.debug("%s will now be targeting instance id: %s", name, iid)
+    LOG.debug("[%s] %s will now be targeting instance id: %s. new=%s",
+              mode, name, iid, init.is_new_instance())
+
+    init.apply_network_config(bring_up=bool(mode != sources.DSMODE_LOCAL))
+
+    if mode == sources.DSMODE_LOCAL:
+        if init.datasource.dsmode != mode:
+            LOG.debug("[%s] Exiting. datasource %s not in local mode.",
+                      mode, init.datasource)
+            return (init.datasource, [])
+        else:
+            LOG.debug("[%s] %s is in local mode, will apply init modules now.",
+                      mode, init.datasource)
+
+    # update fully realizes user-data (pulling in #include if necessary)
     init.update()
     # Stage 7
     try:
@@ -528,7 +544,7 @@ def status_wrapper(name, args, data_d=None, link_d=None):
         v1[mode]['errors'] = [str(e) for e in errors]
 
     except Exception as e:
-        util.logexc(LOG, "failed of stage %s", mode)
+        util.logexc(LOG, "failed stage %s", mode)
         print_exc("failed run of stage %s" % mode)
         v1[mode]['errors'] = [str(e)]