merge trunk at 0.7.7~bzr1245

82 files changed, 3784 insertions, 1877 deletions

diff --git a/cloudinit/cmd/__init__.py b/cloudinit/cmd/__init__.py
new file mode 100644
index 00000000..da124641
--- /dev/null
+++ b/cloudinit/cmd/__init__.py
@@ -0,0 +1,21 @@
+# vi: ts=4 expandtab
+#
+#    Copyright (C) 2012 Canonical Ltd.
+#    Copyright (C) 2012 Hewlett-Packard Development Company, L.P.
+#    Copyright (C) 2012 Yahoo! Inc.
+#
+#    Author: Scott Moser <scott.moser@canonical.com>
+#    Author: Juerg Haefliger <juerg.haefliger@hp.com>
+#    Author: Joshua Harlow <harlowja@yahoo-inc.com>
+#
+#    This program is free software: you can redistribute it and/or modify
+#    it under the terms of the GNU General Public License version 3, as
+#    published by the Free Software Foundation.
+#
+#    This program is distributed in the hope that it will be useful,
+#    but WITHOUT ANY WARRANTY; without even the implied warranty of
+#    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+#    GNU General Public License for more details.
+#
+#    You should have received a copy of the GNU General Public License
+#    along with this program.  If not, see <http://www.gnu.org/licenses/>.
diff --git a/cloudinit/cmd/main.py b/cloudinit/cmd/main.py
new file mode 100644
index 00000000..63621c1d
--- /dev/null
+++ b/cloudinit/cmd/main.py
@@ -0,0 +1,685 @@
+#!/usr/bin/python
+# vi: ts=4 expandtab
+#
+#    Copyright (C) 2012 Canonical Ltd.
+#    Copyright (C) 2012 Hewlett-Packard Development Company, L.P.
+#    Copyright (C) 2012 Yahoo! Inc.
+#
+#    Author: Scott Moser <scott.moser@canonical.com>
+#    Author: Juerg Haefliger <juerg.haefliger@hp.com>
+#    Author: Joshua Harlow <harlowja@yahoo-inc.com>
+#
+#    This program is free software: you can redistribute it and/or modify
+#    it under the terms of the GNU General Public License version 3, as
+#    published by the Free Software Foundation.
+#
+#    This program is distributed in the hope that it will be useful,
+#    but WITHOUT ANY WARRANTY; without even the implied warranty of
+#    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+#    GNU General Public License for more details.
+#
+#    You should have received a copy of the GNU General Public License
+#    along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+import argparse
+import json
+import os
+import sys
+import tempfile
+import time
+import traceback
+
+from cloudinit import patcher
+patcher.patch()  # noqa
+
+from cloudinit import log as logging
+from cloudinit import netinfo
+from cloudinit import signal_handler
+from cloudinit import sources
+from cloudinit import stages
+from cloudinit import templater
+from cloudinit import util
+from cloudinit import version
+
+from cloudinit import reporting
+from cloudinit.reporting import events
+
+from cloudinit.settings import (PER_INSTANCE, PER_ALWAYS, PER_ONCE,
+                                CLOUD_CONFIG)
+
+
+# Pretty little cheetah formatted welcome message template
+WELCOME_MSG_TPL = ("Cloud-init v. ${version} running '${action}' at "
+                   "${timestamp}. Up ${uptime} seconds.")
+
+# Module section template
+MOD_SECTION_TPL = "cloud_%s_modules"
+
+# Things u can query on
+QUERY_DATA_TYPES = [
+    'data',
+    'data_raw',
+    'instance_id',
+]
+
+# Frequency shortname to full name
+# (so users don't have to remember the full name...)
+FREQ_SHORT_NAMES = {
+    'instance': PER_INSTANCE,
+    'always': PER_ALWAYS,
+    'once': PER_ONCE,
+}
+
+LOG = logging.getLogger()
+
+
+# Used for when a logger may not be active
+# and we still want to print exceptions...
+def print_exc(msg=''):
+    if msg:
+        sys.stderr.write("%s\n" % (msg))
+    sys.stderr.write('-' * 60)
+    sys.stderr.write("\n")
+    traceback.print_exc(file=sys.stderr)
+    sys.stderr.write('-' * 60)
+    sys.stderr.write("\n")
+
+
+def welcome(action, msg=None):
+    if not msg:
+        msg = welcome_format(action)
+    util.multi_log("%s\n" % (msg),
+                   console=False, stderr=True, log=LOG)
+    return msg
+
+
+def welcome_format(action):
+    tpl_params = {
+        'version': version.version_string(),
+        'uptime': util.uptime(),
+        'timestamp': util.time_rfc2822(),
+        'action': action,
+    }
+    return templater.render_string(WELCOME_MSG_TPL, tpl_params)
+
+
+def extract_fns(args):
+    # Files are already opened so lets just pass that along
+    # since it would of broke if it couldn't have
+    # read that file already...
+    fn_cfgs = []
+    if args.files:
+        for fh in args.files:
+            # The realpath is more useful in logging
+            # so lets resolve to that...
+            fn_cfgs.append(os.path.realpath(fh.name))
+    return fn_cfgs
+
+
+def run_module_section(mods, action_name, section):
+    full_section_name = MOD_SECTION_TPL % (section)
+    (which_ran, failures) = mods.run_section(full_section_name)
+    total_attempted = len(which_ran) + len(failures)
+    if total_attempted == 0:
+        msg = ("No '%s' modules to run"
+               " under section '%s'") % (action_name, full_section_name)
+        sys.stderr.write("%s\n" % (msg))
+        LOG.debug(msg)
+        return []
+    else:
+        LOG.debug("Ran %s modules with %s failures",
+                  len(which_ran), len(failures))
+        return failures
+
+
+def apply_reporting_cfg(cfg):
+    if cfg.get('reporting'):
+        reporting.update_configuration(cfg.get('reporting'))
+
+
+def main_init(name, args):
+    deps = [sources.DEP_FILESYSTEM, sources.DEP_NETWORK]
+    if args.local:
+        deps = [sources.DEP_FILESYSTEM]
+
+    if not args.local:
+        # See doc/kernel-cmdline.txt
+        #
+        # This is used in maas datasource, in "ephemeral" (read-only root)
+        # environment where the instance netboots to iscsi ro root.
+        # and the entity that controls the pxe config has to configure
+        # the maas datasource.
+        #
+        # Could be used elsewhere, only works on network based (not local).
+        root_name = "%s.d" % (CLOUD_CONFIG)
+        target_fn = os.path.join(root_name, "91_kernel_cmdline_url.cfg")
+        util.read_write_cmdline_url(target_fn)
+
+    # Cloud-init 'init' stage is broken up into the following sub-stages
+    # 1. Ensure that the init object fetches its config without errors
+    # 2. Setup logging/output redirections with resultant config (if any)
+    # 3. Initialize the cloud-init filesystem
+    # 4. Check if we can stop early by looking for various files
+    # 5. Fetch the datasource
+    # 6. Connect to the current instance location + update the cache
+    # 7. Consume the userdata (handlers get activated here)
+    # 8. Construct the modules object
+    # 9. Adjust any subsequent logging/output redirections using the modules
+    #    objects config as it may be different from init object
+    # 10. Run the modules for the 'init' stage
+    # 11. Done!
+    if not args.local:
+        w_msg = welcome_format(name)
+    else:
+        w_msg = welcome_format("%s-local" % (name))
+    init = stages.Init(ds_deps=deps, reporter=args.reporter)
+    # Stage 1
+    init.read_cfg(extract_fns(args))
+    # Stage 2
+    outfmt = None
+    errfmt = None
+    try:
+        LOG.debug("Closing stdin")
+        util.close_stdin()
+        (outfmt, errfmt) = util.fixup_output(init.cfg, name)
+    except Exception:
+        util.logexc(LOG, "Failed to setup output redirection!")
+        print_exc("Failed to setup output redirection!")
+    if args.debug:
+        # Reset so that all the debug handlers are closed out
+        LOG.debug(("Logging being reset, this logger may no"
+                   " longer be active shortly"))
+        logging.resetLogging()
+    logging.setupLogging(init.cfg)
+    apply_reporting_cfg(init.cfg)
+
+    # Any log usage prior to setupLogging above did not have local user log
+    # config applied.  We send the welcome message now, as stderr/out have
+    # been redirected and log now configured.
+    welcome(name, msg=w_msg)
+
+    # Stage 3
+    try:
+        init.initialize()
+    except Exception:
+        util.logexc(LOG, "Failed to initialize, likely bad things to come!")
+    # Stage 4
+    path_helper = init.paths
+    mode = sources.DSMODE_LOCAL if args.local else sources.DSMODE_NETWORK
+
+    if mode == sources.DSMODE_NETWORK:
+        existing = "trust"
+        sys.stderr.write("%s\n" % (netinfo.debug_info()))
+        LOG.debug(("Checking to see if files that we need already"
+                   " exist from a previous run that would allow us"
+                   " to stop early."))
+        # no-net is written by upstart cloud-init-nonet when network failed
+        # to come up
+        stop_files = [
+            os.path.join(path_helper.get_cpath("data"), "no-net"),
+        ]
+        existing_files = []
+        for fn in stop_files:
+            if os.path.isfile(fn):
+                existing_files.append(fn)
+
+        if existing_files:
+            LOG.debug("[%s] Exiting. stop file %s existed",
+                      mode, existing_files)
+            return (None, [])
+        else:
+            LOG.debug("Execution continuing, no previous run detected that"
+                      " would allow us to stop early.")
+    else:
+        existing = "check"
+        if util.get_cfg_option_bool(init.cfg, 'manual_cache_clean', False):
+            existing = "trust"
+
+        init.purge_cache()
+        # Delete the non-net file as well
+        util.del_file(os.path.join(path_helper.get_cpath("data"), "no-net"))
+
+    # Stage 5
+    try:
+        init.fetch(existing=existing)
+        # if in network mode, and the datasource is local
+        # then work was done at that stage.
+        if mode == sources.DSMODE_NETWORK and init.datasource.dsmode != mode:
+            LOG.debug("[%s] Exiting. datasource %s in local mode",
+                      mode, init.datasource)
+            return (None, [])
+    except sources.DataSourceNotFoundException:
+        # In the case of 'cloud-init init' without '--local' it is a bit
+        # more likely that the user would consider it failure if nothing was
+        # found. When using upstart it will also mentions job failure
+        # in console log if exit code is != 0.
+        if mode == sources.DSMODE_LOCAL:
+            LOG.debug("No local datasource found")
+        else:
+            util.logexc(LOG, ("No instance datasource found!"
+                              " Likely bad things to come!"))
+        if not args.force:
+            init.apply_network_config(bring_up=not args.local)
+            LOG.debug("[%s] Exiting without datasource in local mode", mode)
+            if mode == sources.DSMODE_LOCAL:
+                return (None, [])
+            else:
+                return (None, ["No instance datasource found."])
+        else:
+            LOG.debug("[%s] barreling on in force mode without datasource",
+                      mode)
+
+    # Stage 6
+    iid = init.instancify()
+    LOG.debug("[%s] %s will now be targeting instance id: %s. new=%s",
+              mode, name, iid, init.is_new_instance())
+
+    init.apply_network_config(bring_up=bool(mode != sources.DSMODE_LOCAL))
+
+    if mode == sources.DSMODE_LOCAL:
+        if init.datasource.dsmode != mode:
+            LOG.debug("[%s] Exiting. datasource %s not in local mode.",
+                      mode, init.datasource)
+            return (init.datasource, [])
+        else:
+            LOG.debug("[%s] %s is in local mode, will apply init modules now.",
+                      mode, init.datasource)
+
+    # update fully realizes user-data (pulling in #include if necessary)
+    init.update()
+    # Stage 7
+    try:
+        # Attempt to consume the data per instance.
+        # This may run user-data handlers and/or perform
+        # url downloads and such as needed.
+        (ran, _results) = init.cloudify().run('consume_data',
+                                              init.consume_data,
+                                              args=[PER_INSTANCE],
+                                              freq=PER_INSTANCE)
+        if not ran:
+            # Just consume anything that is set to run per-always
+            # if nothing ran in the per-instance code
+            #
+            # See: https://bugs.launchpad.net/bugs/819507 for a little
+            # reason behind this...
+            init.consume_data(PER_ALWAYS)
+    except Exception:
+        util.logexc(LOG, "Consuming user data failed!")
+        return (init.datasource, ["Consuming user data failed!"])
+
+    apply_reporting_cfg(init.cfg)
+
+    # Stage 8 - re-read and apply relevant cloud-config to include user-data
+    mods = stages.Modules(init, extract_fns(args), reporter=args.reporter)
+    # Stage 9
+    try:
+        outfmt_orig = outfmt
+        errfmt_orig = errfmt
+        (outfmt, errfmt) = util.get_output_cfg(mods.cfg, name)
+        if outfmt_orig != outfmt or errfmt_orig != errfmt:
+            LOG.warn("Stdout, stderr changing to (%s, %s)", outfmt, errfmt)
+            (outfmt, errfmt) = util.fixup_output(mods.cfg, name)
+    except Exception:
+        util.logexc(LOG, "Failed to re-adjust output redirection!")
+    logging.setupLogging(mods.cfg)
+
+    # Stage 10
+    return (init.datasource, run_module_section(mods, name, name))
+
+
+def main_modules(action_name, args):
+    name = args.mode
+    # Cloud-init 'modules' stages are broken up into the following sub-stages
+    # 1. Ensure that the init object fetches its config without errors
+    # 2. Get the datasource from the init object, if it does
+    #    not exist then that means the main_init stage never
+    #    worked, and thus this stage can not run.
+    # 3. Construct the modules object
+    # 4. Adjust any subsequent logging/output redirections using
+    #    the modules objects configuration
+    # 5. Run the modules for the given stage name
+    # 6. Done!
+    w_msg = welcome_format("%s:%s" % (action_name, name))
+    init = stages.Init(ds_deps=[], reporter=args.reporter)
+    # Stage 1
+    init.read_cfg(extract_fns(args))
+    # Stage 2
+    try:
+        init.fetch(existing="trust")
+    except sources.DataSourceNotFoundException:
+        # There was no datasource found, theres nothing to do
+        msg = ('Can not apply stage %s, no datasource found! Likely bad '
+               'things to come!' % name)
+        util.logexc(LOG, msg)
+        print_exc(msg)
+        if not args.force:
+            return [(msg)]
+    # Stage 3
+    mods = stages.Modules(init, extract_fns(args), reporter=args.reporter)
+    # Stage 4
+    try:
+        LOG.debug("Closing stdin")
+        util.close_stdin()
+        util.fixup_output(mods.cfg, name)
+    except Exception:
+        util.logexc(LOG, "Failed to setup output redirection!")
+    if args.debug:
+        # Reset so that all the debug handlers are closed out
+        LOG.debug(("Logging being reset, this logger may no"
+                   " longer be active shortly"))
+        logging.resetLogging()
+    logging.setupLogging(mods.cfg)
+    apply_reporting_cfg(init.cfg)
+
+    # now that logging is setup and stdout redirected, send welcome
+    welcome(name, msg=w_msg)
+
+    # Stage 5
+    return run_module_section(mods, name, name)
+
+
+def main_query(name, _args):
+    raise NotImplementedError(("Action '%s' is not"
+                               " currently implemented") % (name))
+
+
+def main_single(name, args):
+    # Cloud-init single stage is broken up into the following sub-stages
+    # 1. Ensure that the init object fetches its config without errors
+    # 2. Attempt to fetch the datasource (warn if it doesn't work)
+    # 3. Construct the modules object
+    # 4. Adjust any subsequent logging/output redirections using
+    #    the modules objects configuration
+    # 5. Run the single module
+    # 6. Done!
+    mod_name = args.name
+    w_msg = welcome_format(name)
+    init = stages.Init(ds_deps=[], reporter=args.reporter)
+    # Stage 1
+    init.read_cfg(extract_fns(args))
+    # Stage 2
+    try:
+        init.fetch(existing="trust")
+    except sources.DataSourceNotFoundException:
+        # There was no datasource found,
+        # that might be bad (or ok) depending on
+        # the module being ran (so continue on)
+        util.logexc(LOG, ("Failed to fetch your datasource,"
+                          " likely bad things to come!"))
+        print_exc(("Failed to fetch your datasource,"
+                   " likely bad things to come!"))
+        if not args.force:
+            return 1
+    # Stage 3
+    mods = stages.Modules(init, extract_fns(args), reporter=args.reporter)
+    mod_args = args.module_args
+    if mod_args:
+        LOG.debug("Using passed in arguments %s", mod_args)
+    mod_freq = args.frequency
+    if mod_freq:
+        LOG.debug("Using passed in frequency %s", mod_freq)
+        mod_freq = FREQ_SHORT_NAMES.get(mod_freq)
+    # Stage 4
+    try:
+        LOG.debug("Closing stdin")
+        util.close_stdin()
+        util.fixup_output(mods.cfg, None)
+    except Exception:
+        util.logexc(LOG, "Failed to setup output redirection!")
+    if args.debug:
+        # Reset so that all the debug handlers are closed out
+        LOG.debug(("Logging being reset, this logger may no"
+                   " longer be active shortly"))
+        logging.resetLogging()
+    logging.setupLogging(mods.cfg)
+    apply_reporting_cfg(init.cfg)
+
+    # now that logging is setup and stdout redirected, send welcome
+    welcome(name, msg=w_msg)
+
+    # Stage 5
+    (which_ran, failures) = mods.run_single(mod_name,
+                                            mod_args,
+                                            mod_freq)
+    if failures:
+        LOG.warn("Ran %s but it failed!", mod_name)
+        return 1
+    elif not which_ran:
+        LOG.warn("Did not run %s, does it exist?", mod_name)
+        return 1
+    else:
+        # Guess it worked
+        return 0
+
+
+def atomic_write_file(path, content, mode='w'):
+    tf = None
+    try:
+        tf = tempfile.NamedTemporaryFile(dir=os.path.dirname(path),
+                                         delete=False, mode=mode)
+        tf.write(content)
+        tf.close()
+        os.rename(tf.name, path)
+    except Exception as e:
+        if tf is not None:
+            os.unlink(tf.name)
+        raise e
+
+
+def atomic_write_json(path, data):
+    return atomic_write_file(path, json.dumps(data, indent=1) + "\n")
+
+
+def status_wrapper(name, args, data_d=None, link_d=None):
+    if data_d is None:
+        data_d = os.path.normpath("/var/lib/cloud/data")
+    if link_d is None:
+        link_d = os.path.normpath("/run/cloud-init")
+
+    status_path = os.path.join(data_d, "status.json")
+    status_link = os.path.join(link_d, "status.json")
+    result_path = os.path.join(data_d, "result.json")
+    result_link = os.path.join(link_d, "result.json")
+
+    util.ensure_dirs((data_d, link_d,))
+
+    (_name, functor) = args.action
+
+    if name == "init":
+        if args.local:
+            mode = "init-local"
+        else:
+            mode = "init"
+    elif name == "modules":
+        mode = "modules-%s" % args.mode
+    else:
+        raise ValueError("unknown name: %s" % name)
+
+    modes = ('init', 'init-local', 'modules-config', 'modules-final')
+
+    status = None
+    if mode == 'init-local':
+        for f in (status_link, result_link, status_path, result_path):
+            util.del_file(f)
+    else:
+        try:
+            status = json.loads(util.load_file(status_path))
+        except Exception:
+            pass
+
+    if status is None:
+        nullstatus = {
+            'errors': [],
+            'start': None,
+            'finished': None,
+        }
+        status = {'v1': {}}
+        for m in modes:
+            status['v1'][m] = nullstatus.copy()
+        status['v1']['datasource'] = None
+
+    v1 = status['v1']
+    v1['stage'] = mode
+    v1[mode]['start'] = time.time()
+
+    atomic_write_json(status_path, status)
+    util.sym_link(os.path.relpath(status_path, link_d), status_link,
+                  force=True)
+
+    try:
+        ret = functor(name, args)
+        if mode in ('init', 'init-local'):
+            (datasource, errors) = ret
+            if datasource is not None:
+                v1['datasource'] = str(datasource)
+        else:
+            errors = ret
+
+        v1[mode]['errors'] = [str(e) for e in errors]
+
+    except Exception as e:
+        util.logexc(LOG, "failed stage %s", mode)
+        print_exc("failed run of stage %s" % mode)
+        v1[mode]['errors'] = [str(e)]
+
+    v1[mode]['finished'] = time.time()
+    v1['stage'] = None
+
+    atomic_write_json(status_path, status)
+
+    if mode == "modules-final":
+        # write the 'finished' file
+        errors = []
+        for m in modes:
+            if v1[m]['errors']:
+                errors.extend(v1[m].get('errors', []))
+
+        atomic_write_json(result_path,
+                          {'v1': {'datasource': v1['datasource'],
+                                  'errors': errors}})
+        util.sym_link(os.path.relpath(result_path, link_d), result_link,
+                      force=True)
+
+    return len(v1[mode]['errors'])
+
+
+def main(sysv_args=None):
+    if sysv_args is not None:
+        parser = argparse.ArgumentParser(prog=sysv_args[0])
+        sysv_args = sysv_args[1:]
+    else:
+        parser = argparse.ArgumentParser()
+
+    # Top level args
+    parser.add_argument('--version', '-v', action='version',
+                        version='%(prog)s ' + (version.version_string()))
+    parser.add_argument('--file', '-f', action='append',
+                        dest='files',
+                        help=('additional yaml configuration'
+                              ' files to use'),
+                        type=argparse.FileType('rb'))
+    parser.add_argument('--debug', '-d', action='store_true',
+                        help=('show additional pre-action'
+                              ' logging (default: %(default)s)'),
+                        default=False)
+    parser.add_argument('--force', action='store_true',
+                        help=('force running even if no datasource is'
+                              ' found (use at your own risk)'),
+                        dest='force',
+                        default=False)
+
+    parser.set_defaults(reporter=None)
+    subparsers = parser.add_subparsers()
+
+    # Each action and its sub-options (if any)
+    parser_init = subparsers.add_parser('init',
+                                        help=('initializes cloud-init and'
+                                              ' performs initial modules'))
+    parser_init.add_argument("--local", '-l', action='store_true',
+                             help="start in local mode (default: %(default)s)",
+                             default=False)
+    # This is used so that we can know which action is selected +
+    # the functor to use to run this subcommand
+    parser_init.set_defaults(action=('init', main_init))
+
+    # These settings are used for the 'config' and 'final' stages
+    parser_mod = subparsers.add_parser('modules',
+                                       help=('activates modules using '
+                                             'a given configuration key'))
+    parser_mod.add_argument("--mode", '-m', action='store',
+                            help=("module configuration name "
+                                  "to use (default: %(default)s)"),
+                            default='config',
+                            choices=('init', 'config', 'final'))
+    parser_mod.set_defaults(action=('modules', main_modules))
+
+    # These settings are used when you want to query information
+    # stored in the cloud-init data objects/directories/files
+    parser_query = subparsers.add_parser('query',
+                                         help=('query information stored '
+                                               'in cloud-init'))
+    parser_query.add_argument("--name", '-n', action="store",
+                              help="item name to query on",
+                              required=True,
+                              choices=QUERY_DATA_TYPES)
+    parser_query.set_defaults(action=('query', main_query))
+
+    # This subcommand allows you to run a single module
+    parser_single = subparsers.add_parser('single',
+                                          help=('run a single module '))
+    parser_single.set_defaults(action=('single', main_single))
+    parser_single.add_argument("--name", '-n', action="store",
+                               help="module name to run",
+                               required=True)
+    parser_single.add_argument("--frequency", action="store",
+                               help=("frequency of the module"),
+                               required=False,
+                               choices=list(FREQ_SHORT_NAMES.keys()))
+    parser_single.add_argument("--report", action="store_true",
+                               help="enable reporting",
+                               required=False)
+    parser_single.add_argument("module_args", nargs="*",
+                               metavar='argument',
+                               help=('any additional arguments to'
+                                     ' pass to this module'))
+    parser_single.set_defaults(action=('single', main_single))
+
+    args = parser.parse_args(args=sysv_args)
+
+    try:
+        (name, functor) = args.action
+    except AttributeError:
+        parser.error('too few arguments')
+
+    # Setup basic logging to start (until reinitialized)
+    # iff in debug mode...
+    if args.debug:
+        logging.setupBasicLogging()
+
+    # Setup signal handlers before running
+    signal_handler.attach_handlers()
+
+    if name in ("modules", "init"):
+        functor = status_wrapper
+
+    report_on = True
+    if name == "init":
+        if args.local:
+            rname, rdesc = ("init-local", "searching for local datasources")
+        else:
+            rname, rdesc = ("init-network",
+                            "searching for network datasources")
+    elif name == "modules":
+        rname, rdesc = ("modules-%s" % args.mode,
+                        "running modules for %s" % args.mode)
+    elif name == "single":
+        rname, rdesc = ("single/%s" % args.name,
+                        "running single module %s" % args.name)
+        report_on = args.report
+
+    args.reporter = events.ReportEventStack(
+        rname, rdesc, reporting_enabled=report_on)
+    with args.reporter:
+        return util.log_time(
+            logfunc=LOG.debug, msg="cloud-init mode '%s'" % name,
+            get_uptime=True, func=functor, args=(name, args))
diff --git a/cloudinit/config/cc_apt_configure.py b/cloudinit/config/cc_apt_configure.py
index 702977cb..05ad4b03 100644
--- a/cloudinit/config/cc_apt_configure.py
+++ b/cloudinit/config/cc_apt_configure.py
@@ -22,6 +22,7 @@ import glob
 import os
 import re
 
+from cloudinit import gpg
 from cloudinit import templater
 from cloudinit import util
 
@@ -34,21 +35,6 @@ APT_PROXY_FN = "/etc/apt/apt.conf.d/95cloud-init-proxy"
 # this will match 'XXX:YYY' (ie, 'cloud-archive:foo' or 'ppa:bar')
 ADD_APT_REPO_MATCH = r"^[\w-]+:\w"
 
-# A temporary shell program to get a given gpg key
-# from a given keyserver
-EXPORT_GPG_KEYID = """
-    k=${1} ks=${2};
-    exec 2>/dev/null
-    [ -n "$k" ] || exit 1;
-    armour=$(gpg --list-keys --armour "${k}")
-    if [ -z "${armour}" ]; then
-       gpg --keyserver ${ks} --recv $k >/dev/null &&
-          armour=$(gpg --export --armour "${k}") &&
-          gpg --batch --yes --delete-keys "${k}"
-    fi
-    [ -n "${armour}" ] && echo "${armour}"
-"""
-
 
 def handle(name, cfg, cloud, log, _args):
     if util.is_false(cfg.get('apt_configure_enabled', True)):
@@ -70,7 +56,7 @@ def handle(name, cfg, cloud, log, _args):
 
     if not util.get_cfg_option_bool(cfg,
                                     'apt_preserve_sources_list', False):
-        generate_sources_list(release, mirrors, cloud, log)
+        generate_sources_list(cfg, release, mirrors, cloud, log)
         old_mirrors = cfg.get('apt_old_mirrors',
                               {"primary": "archive.ubuntu.com/ubuntu",
                                "security": "security.ubuntu.com/ubuntu"})
@@ -94,8 +80,8 @@ def handle(name, cfg, cloud, log, _args):
             def matcher(x):
                 return False
 
-        errors = add_sources(cfg['apt_sources'], params,
-                             aa_repo_match=matcher)
+        errors = add_apt_sources(cfg['apt_sources'], params,
+                                 aa_repo_match=matcher)
         for e in errors:
             log.warn("Add source error: %s", ':'.join(e))
 
@@ -108,17 +94,7 @@ def handle(name, cfg, cloud, log, _args):
             util.logexc(log, "Failed to run debconf-set-selections")
 
 
-# get gpg keyid from keyserver
-def getkeybyid(keyid, keyserver):
-    with util.ExtendedTemporaryFile(suffix='.sh', mode="w+", ) as fh:
-        fh.write(EXPORT_GPG_KEYID)
-        fh.flush()
-        cmd = ['/bin/sh', fh.name, keyid, keyserver]
-        (stdout, _stderr) = util.subp(cmd)
-        return stdout.strip()
-
-
-def mirror2lists_fileprefix(mirror):
+def mirrorurl_to_apt_fileprefix(mirror):
     string = mirror
     # take off http:// or ftp://
     if string.endswith("/"):
@@ -135,8 +111,8 @@ def rename_apt_lists(old_mirrors, new_mirrors, lists_d="/var/lib/apt/lists"):
         nmirror = new_mirrors.get(name)
         if not nmirror:
             continue
-        oprefix = os.path.join(lists_d, mirror2lists_fileprefix(omirror))
-        nprefix = os.path.join(lists_d, mirror2lists_fileprefix(nmirror))
+        oprefix = os.path.join(lists_d, mirrorurl_to_apt_fileprefix(omirror))
+        nprefix = os.path.join(lists_d, mirrorurl_to_apt_fileprefix(nmirror))
         if oprefix == nprefix:
             continue
         olen = len(oprefix)
@@ -149,7 +125,17 @@ def get_release():
     return stdout.strip()
 
 
-def generate_sources_list(codename, mirrors, cloud, log):
+def generate_sources_list(cfg, codename, mirrors, cloud, log):
+    params = {'codename': codename}
+    for k in mirrors:
+        params[k] = mirrors[k]
+
+    custtmpl = cfg.get('apt_custom_sources_list', None)
+    if custtmpl is not None:
+        templater.render_string_to_file(custtmpl,
+                                        '/etc/apt/sources.list', params)
+        return
+
     template_fn = cloud.get_template_filename('sources.list.%s' %
                                               (cloud.distro.name))
     if not template_fn:
@@ -158,13 +144,61 @@ def generate_sources_list(codename, mirrors, cloud, log):
             log.warn("No template found, not rendering /etc/apt/sources.list")
             return
 
-    params = {'codename': codename}
-    for k in mirrors:
-        params[k] = mirrors[k]
     templater.render_to_file(template_fn, '/etc/apt/sources.list', params)
 
 
-def add_sources(srclist, template_params=None, aa_repo_match=None):
+def add_apt_key_raw(key):
+    """
+    actual adding of a key as defined in key argument
+    to the system
+    """
+    try:
+        util.subp(('apt-key', 'add', '-'), key)
+    except util.ProcessExecutionError:
+        raise ValueError('failed to add apt GPG Key to apt keyring')
+
+
+def add_apt_key(ent):
+    """
+    add key to the system as defined in ent (if any)
+    supports raw keys or keyid's
+    The latter will as a first step fetch the raw key from a keyserver
+    """
+    if 'keyid' in ent and 'key' not in ent:
+        keyserver = "keyserver.ubuntu.com"
+        if 'keyserver' in ent:
+            keyserver = ent['keyserver']
+        ent['key'] = gpg.get_key_by_id(ent['keyid'], keyserver)
+
+    if 'key' in ent:
+        add_apt_key_raw(ent['key'])
+
+
+def convert_to_new_format(srclist):
+    """convert_to_new_format
+       convert the old list based format to the new dict based one
+    """
+    srcdict = {}
+    if isinstance(srclist, list):
+        for srcent in srclist:
+            if 'filename' not in srcent:
+                # file collides for multiple !filename cases for compatibility
+                # yet we need them all processed, so not same dictionary key
+                srcent['filename'] = "cloud_config_sources.list"
+                key = util.rand_dict_key(srcdict, "cloud_config_sources.list")
+            else:
+                # all with filename use that as key (matching new format)
+                key = srcent['filename']
+            srcdict[key] = srcent
+    elif isinstance(srclist, dict):
+        srcdict = srclist
+    else:
+        raise ValueError("unknown apt_sources format")
+
+    return srcdict
+
+
+def add_apt_sources(srclist, template_params=None, aa_repo_match=None):
     """
     add entries in /etc/apt/sources.list.d for each abbreviated
     sources.list entry in 'srclist'.  When rendering template, also
@@ -174,18 +208,34 @@ def add_sources(srclist, template_params=None, aa_repo_match=None):
         template_params = {}
 
     if aa_repo_match is None:
-        def aa_repo_match(x):
+        def _aa_repo_match(x):
             return False
+        aa_repo_match = _aa_repo_match
 
     errorlist = []
-    for ent in srclist:
+    srcdict = convert_to_new_format(srclist)
+
+    for filename in srcdict:
+        ent = srcdict[filename]
+        if 'filename' not in ent:
+            ent['filename'] = filename
+
+        # keys can be added without specifying a source
+        try:
+            add_apt_key(ent)
+        except ValueError as detail:
+            errorlist.append([ent, detail])
+
         if 'source' not in ent:
             errorlist.append(["", "missing source"])
             continue
-
         source = ent['source']
         source = templater.render_string(source, template_params)
 
+        if not ent['filename'].startswith(os.path.sep):
+            ent['filename'] = os.path.join("/etc/apt/sources.list.d/",
+                                           ent['filename'])
+
         if aa_repo_match(source):
             try:
                 util.subp(["add-apt-repository", source])
@@ -194,33 +244,10 @@ def add_sources(srclist, template_params=None, aa_repo_match=None):
                                   ("add-apt-repository failed. " + str(e))])
             continue
 
-        if 'filename' not in ent:
-            ent['filename'] = 'cloud_config_sources.list'
-
-        if not ent['filename'].startswith("/"):
-            ent['filename'] = os.path.join("/etc/apt/sources.list.d/",
-                                           ent['filename'])
-
-        if ('keyid' in ent and 'key' not in ent):
-            ks = "keyserver.ubuntu.com"
-            if 'keyserver' in ent:
-                ks = ent['keyserver']
-            try:
-                ent['key'] = getkeybyid(ent['keyid'], ks)
-            except:
-                errorlist.append([source, "failed to get key from %s" % ks])
-                continue
-
-        if 'key' in ent:
-            try:
-                util.subp(('apt-key', 'add', '-'), ent['key'])
-            except:
-                errorlist.append([source, "failed add key"])
-
         try:
             contents = "%s\n" % (source)
             util.write_file(ent['filename'], contents, omode="ab")
-        except:
+        except Exception:
             errorlist.append([source,
                              "failed write to file %s" % ent['filename']])
 
diff --git a/cloudinit/config/cc_bootcmd.py b/cloudinit/config/cc_bootcmd.py
index a295cc4e..b763a3c3 100644
--- a/cloudinit/config/cc_bootcmd.py
+++ b/cloudinit/config/cc_bootcmd.py
@@ -38,7 +38,7 @@ def handle(name, cfg, cloud, log, _args):
             content = util.shellify(cfg["bootcmd"])
             tmpf.write(util.encode_text(content))
             tmpf.flush()
-        except:
+        except Exception:
             util.logexc(log, "Failed to shellify bootcmd")
             raise
 
@@ -49,6 +49,6 @@ def handle(name, cfg, cloud, log, _args):
                 env['INSTANCE_ID'] = str(iid)
             cmd = ['/bin/sh', tmpf.name]
             util.subp(cmd, env=env, capture=False)
-        except:
+        except Exception:
             util.logexc(log, "Failed to run bootcmd module %s", name)
             raise
diff --git a/cloudinit/config/cc_disk_setup.py b/cloudinit/config/cc_disk_setup.py
index bbaf9646..b642f1f8 100644
--- a/cloudinit/config/cc_disk_setup.py
+++ b/cloudinit/config/cc_disk_setup.py
@@ -198,7 +198,7 @@ def is_device_valid(name, partition=False):
     d_type = ""
     try:
         d_type = device_type(name)
-    except:
+    except Exception:
         LOG.warn("Query against device %s failed" % name)
         return False
 
diff --git a/cloudinit/config/cc_emit_upstart.py b/cloudinit/config/cc_emit_upstart.py
index 86ae97ab..98828b9e 100644
--- a/cloudinit/config/cc_emit_upstart.py
+++ b/cloudinit/config/cc_emit_upstart.py
@@ -20,8 +20,8 @@
 
 import os
 
-from cloudinit.settings import PER_ALWAYS
 from cloudinit import log as logging
+from cloudinit.settings import PER_ALWAYS
 from cloudinit import util
 
 frequency = PER_ALWAYS
@@ -56,7 +56,7 @@ def handle(name, _cfg, cloud, log, args):
         event_names = ['cloud-config']
 
     if not is_upstart_system():
-        log.debug("not upstart system, '%s' disabled")
+        log.debug("not upstart system, '%s' disabled", name)
         return
 
     cfgpath = cloud.paths.get_ipath_cur("cloud_config")
diff --git a/cloudinit/config/cc_fan.py b/cloudinit/config/cc_fan.py
index 39e3850e..545fee22 100644
--- a/cloudinit/config/cc_fan.py
+++ b/cloudinit/config/cc_fan.py
@@ -37,8 +37,8 @@ If cloud-init sees a 'fan' entry in cloud-config it will
 """
 
 from cloudinit import log as logging
-from cloudinit import util
 from cloudinit.settings import PER_INSTANCE
+from cloudinit import util
 
 LOG = logging.getLogger(__name__)
 
diff --git a/cloudinit/config/cc_final_message.py b/cloudinit/config/cc_final_message.py
index 4a51476f..c9021eb1 100644
--- a/cloudinit/config/cc_final_message.py
+++ b/cloudinit/config/cc_final_message.py
@@ -66,7 +66,7 @@ def handle(_name, cfg, cloud, log, args):
     try:
         contents = "%s - %s - v. %s\n" % (uptime, ts, cver)
         util.write_file(boot_fin_fn, contents)
-    except:
+    except Exception:
         util.logexc(log, "Failed to write boot finished file %s", boot_fin_fn)
 
     if cloud.datasource.is_disconnected:
diff --git a/cloudinit/config/cc_growpart.py b/cloudinit/config/cc_growpart.py
index 859d69f1..40560f11 100644
--- a/cloudinit/config/cc_growpart.py
+++ b/cloudinit/config/cc_growpart.py
@@ -36,13 +36,13 @@ DEFAULT_CONFIG = {
 }
 
 
-def enum(**enums):
-    return type('Enum', (), enums)
+class RESIZE(object):
+    SKIPPED = "SKIPPED"
+    CHANGED = "CHANGED"
+    NOCHANGE = "NOCHANGE"
+    FAILED = "FAILED"
 
 
-RESIZE = enum(SKIPPED="SKIPPED", CHANGED="CHANGED", NOCHANGE="NOCHANGE",
-              FAILED="FAILED")
-
 LOG = logging.getLogger(__name__)
 
 
diff --git a/cloudinit/config/cc_grub_dpkg.py b/cloudinit/config/cc_grub_dpkg.py
index 3c2d9985..156722d9 100644
--- a/cloudinit/config/cc_grub_dpkg.py
+++ b/cloudinit/config/cc_grub_dpkg.py
@@ -69,5 +69,5 @@ def handle(name, cfg, _cloud, log, _args):
 
     try:
         util.subp(['debconf-set-selections'], dconf_sel)
-    except:
+    except Exception:
         util.logexc(log, "Failed to run debconf-set-selections for grub-dpkg")
diff --git a/cloudinit/config/cc_keys_to_console.py b/cloudinit/config/cc_keys_to_console.py
index aa844ee9..9a02f056 100644
--- a/cloudinit/config/cc_keys_to_console.py
+++ b/cloudinit/config/cc_keys_to_console.py
@@ -57,6 +57,6 @@ def handle(name, cfg, cloud, log, _args):
         (stdout, _stderr) = util.subp(cmd)
         util.multi_log("%s\n" % (stdout.strip()),
                        stderr=False, console=True)
-    except:
+    except Exception:
         log.warn("Writing keys to the system console failed!")
         raise
diff --git a/cloudinit/config/cc_lxd.py b/cloudinit/config/cc_lxd.py
index bf735648..70d4e7c3 100644
--- a/cloudinit/config/cc_lxd.py
+++ b/cloudinit/config/cc_lxd.py
@@ -52,7 +52,8 @@ def handle(name, cfg, cloud, log, args):
     # Get config
     lxd_cfg = cfg.get('lxd')
     if not lxd_cfg:
-        log.debug("Skipping module named %s, not present or disabled by cfg")
+        log.debug("Skipping module named %s, not present or disabled by cfg",
+                  name)
         return
     if not isinstance(lxd_cfg, dict):
         log.warn("lxd config must be a dictionary. found a '%s'",
@@ -111,7 +112,7 @@ def handle(name, cfg, cloud, log, args):
             data = "\n".join(["set %s %s" % (k, v)
                               for k, v in debconf.items()]) + "\n"
             util.subp(['debconf-communicate'], data)
-        except:
+        except Exception:
             util.logexc(log, "Failed to run '%s' for lxd with" % dconf_comm)
 
         # Remove the existing configuration file (forces re-generation)
diff --git a/cloudinit/config/cc_mounts.py b/cloudinit/config/cc_mounts.py
index 4fe3ee21..2b981935 100644
--- a/cloudinit/config/cc_mounts.py
+++ b/cloudinit/config/cc_mounts.py
@@ -244,7 +244,7 @@ def handle_swapcfg(swapcfg):
                     LOG.debug("swap file %s already in use.", fname)
                     return fname
             LOG.debug("swap file %s existed, but not in /proc/swaps", fname)
-        except:
+        except Exception:
             LOG.warn("swap file %s existed. Error reading /proc/swaps", fname)
             return fname
 
@@ -379,7 +379,7 @@ def handle(_name, cfg, cloud, log, _args):
             toks = WS.split(line)
             if toks[3].find(comment) != -1:
                 continue
-        except:
+        except Exception:
             pass
         fstab_lines.append(line)
 
@@ -390,16 +390,16 @@ def handle(_name, cfg, cloud, log, _args):
     if needswap:
         try:
             util.subp(("swapon", "-a"))
-        except:
+        except Exception:
             util.logexc(log, "Activating swap via 'swapon -a' failed")
 
     for d in dirs:
         try:
             util.ensure_dir(d)
-        except:
+        except Exception:
             util.logexc(log, "Failed to make '%s' config-mount", d)
 
     try:
         util.subp(("mount", "-a"))
-    except:
+    except Exception:
         util.logexc(log, "Activating mounts via 'mount -a' failed")
diff --git a/cloudinit/config/cc_phone_home.py b/cloudinit/config/cc_phone_home.py
index 3dcc9459..72176d42 100644
--- a/cloudinit/config/cc_phone_home.py
+++ b/cloudinit/config/cc_phone_home.py
@@ -65,7 +65,7 @@ def handle(name, cfg, cloud, log, args):
     tries = ph_cfg.get('tries')
     try:
         tries = int(tries)
-    except:
+    except Exception:
         tries = 10
         util.logexc(log, "Configuration entry 'tries' is not an integer, "
                     "using %s instead", tries)
@@ -87,7 +87,7 @@ def handle(name, cfg, cloud, log, args):
     for (n, path) in pubkeys.items():
         try:
             all_keys[n] = util.load_file(path)
-        except:
+        except Exception:
             util.logexc(log, "%s: failed to open, can not phone home that "
                         "data!", path)
 
@@ -117,6 +117,6 @@ def handle(name, cfg, cloud, log, args):
         util.read_file_or_url(url, data=real_submit_keys,
                               retries=tries, sec_between=3,
                               ssl_details=util.fetch_ssl_details(cloud.paths))
-    except:
+    except Exception:
         util.logexc(log, "Failed to post phone home data to %s in %s tries",
                     url, tries)
diff --git a/cloudinit/config/cc_rightscale_userdata.py b/cloudinit/config/cc_rightscale_userdata.py
index 0ecf3a4d..8118fac4 100644
--- a/cloudinit/config/cc_rightscale_userdata.py
+++ b/cloudinit/config/cc_rightscale_userdata.py
@@ -52,7 +52,7 @@ MY_HOOKNAME = 'CLOUD_INIT_REMOTE_HOOK'
 def handle(name, _cfg, cloud, log, _args):
     try:
         ud = cloud.get_userdata_raw()
-    except:
+    except Exception:
         log.debug("Failed to get raw userdata in module %s", name)
         return
 
@@ -63,7 +63,7 @@ def handle(name, _cfg, cloud, log, _args):
                        "did not find %s in parsed"
                        " raw userdata"), name, MY_HOOKNAME)
             return
-    except:
+    except Exception:
         util.logexc(log, "Failed to parse query string %s into a dictionary",
                     ud)
         raise
diff --git a/cloudinit/config/cc_runcmd.py b/cloudinit/config/cc_runcmd.py
index 66dc3363..bc09d38c 100644
--- a/cloudinit/config/cc_runcmd.py
+++ b/cloudinit/config/cc_runcmd.py
@@ -34,5 +34,5 @@ def handle(name, cfg, cloud, log, _args):
     try:
         content = util.shellify(cmd)
         util.write_file(out_fn, content, 0o700)
-    except:
+    except Exception:
         util.logexc(log, "Failed to shellify %s into file %s", cmd, out_fn)
diff --git a/cloudinit/config/cc_scripts_per_boot.py b/cloudinit/config/cc_scripts_per_boot.py
index 42b987eb..ee3b6c9f 100644
--- a/cloudinit/config/cc_scripts_per_boot.py
+++ b/cloudinit/config/cc_scripts_per_boot.py
@@ -35,7 +35,7 @@ def handle(name, _cfg, cloud, log, _args):
     runparts_path = os.path.join(cloud.get_cpath(), 'scripts', SCRIPT_SUBDIR)
     try:
         util.runparts(runparts_path)
-    except:
+    except Exception:
         log.warn("Failed to run module %s (%s in %s)",
                  name, SCRIPT_SUBDIR, runparts_path)
         raise
diff --git a/cloudinit/config/cc_scripts_per_instance.py b/cloudinit/config/cc_scripts_per_instance.py
index b5d71c13..c0d62b12 100644
--- a/cloudinit/config/cc_scripts_per_instance.py
+++ b/cloudinit/config/cc_scripts_per_instance.py
@@ -35,7 +35,7 @@ def handle(name, _cfg, cloud, log, _args):
     runparts_path = os.path.join(cloud.get_cpath(), 'scripts', SCRIPT_SUBDIR)
     try:
         util.runparts(runparts_path)
-    except:
+    except Exception:
         log.warn("Failed to run module %s (%s in %s)",
                  name, SCRIPT_SUBDIR, runparts_path)
         raise
diff --git a/cloudinit/config/cc_scripts_per_once.py b/cloudinit/config/cc_scripts_per_once.py
index d77d36d5..ecb527f6 100644
--- a/cloudinit/config/cc_scripts_per_once.py
+++ b/cloudinit/config/cc_scripts_per_once.py
@@ -35,7 +35,7 @@ def handle(name, _cfg, cloud, log, _args):
     runparts_path = os.path.join(cloud.get_cpath(), 'scripts', SCRIPT_SUBDIR)
     try:
         util.runparts(runparts_path)
-    except:
+    except Exception:
         log.warn("Failed to run module %s (%s in %s)",
                  name, SCRIPT_SUBDIR, runparts_path)
         raise
diff --git a/cloudinit/config/cc_scripts_user.py b/cloudinit/config/cc_scripts_user.py
index 5c53014f..699857d1 100644
--- a/cloudinit/config/cc_scripts_user.py
+++ b/cloudinit/config/cc_scripts_user.py
@@ -36,7 +36,7 @@ def handle(name, _cfg, cloud, log, _args):
     runparts_path = os.path.join(cloud.get_ipath_cur(), SCRIPT_SUBDIR)
     try:
         util.runparts(runparts_path)
-    except:
+    except Exception:
         log.warn("Failed to run module %s (%s in %s)",
                  name, SCRIPT_SUBDIR, runparts_path)
         raise
diff --git a/cloudinit/config/cc_scripts_vendor.py b/cloudinit/config/cc_scripts_vendor.py
index 0c9e504e..80bf10ff 100644
--- a/cloudinit/config/cc_scripts_vendor.py
+++ b/cloudinit/config/cc_scripts_vendor.py
@@ -37,7 +37,7 @@ def handle(name, cfg, cloud, log, _args):
 
     try:
         util.runparts(runparts_path, exe_prefix=prefix)
-    except:
+    except Exception:
         log.warn("Failed to run module %s (%s in %s)",
                  name, SCRIPT_SUBDIR, runparts_path)
         raise
diff --git a/cloudinit/config/cc_seed_random.py b/cloudinit/config/cc_seed_random.py
index 1b011216..5085c23a 100644
--- a/cloudinit/config/cc_seed_random.py
+++ b/cloudinit/config/cc_seed_random.py
@@ -24,8 +24,8 @@ import os
 
 from six import BytesIO
 
-from cloudinit.settings import PER_INSTANCE
 from cloudinit import log as logging
+from cloudinit.settings import PER_INSTANCE
 from cloudinit import util
 
 frequency = PER_INSTANCE
diff --git a/cloudinit/config/cc_set_passwords.py b/cloudinit/config/cc_set_passwords.py
index 58e1b713..5c8c23b8 100644
--- a/cloudinit/config/cc_set_passwords.py
+++ b/cloudinit/config/cc_set_passwords.py
@@ -155,7 +155,7 @@ def handle(_name, cfg, cloud, log, args):
             cmd = filter(None, cmd)  # Remove empty arguments
             util.subp(cmd)
             log.debug("Restarted the ssh daemon")
-        except:
+        except Exception:
             util.logexc(log, "Restarting of the ssh daemon failed")
 
     if len(errors):
diff --git a/cloudinit/config/cc_snappy.py b/cloudinit/config/cc_snappy.py
index fa9d54a0..1a485ee6 100644
--- a/cloudinit/config/cc_snappy.py
+++ b/cloudinit/config/cc_snappy.py
@@ -47,12 +47,12 @@ Example config:
 """
 
 from cloudinit import log as logging
-from cloudinit import util
 from cloudinit.settings import PER_INSTANCE
+from cloudinit import util
 
 import glob
-import tempfile
 import os
+import tempfile
 
 LOG = logging.getLogger(__name__)
 
diff --git a/cloudinit/config/cc_ssh.py b/cloudinit/config/cc_ssh.py
index d24e43c0..cb9b70aa 100644
--- a/cloudinit/config/cc_ssh.py
+++ b/cloudinit/config/cc_ssh.py
@@ -57,7 +57,7 @@ def handle(_name, cfg, cloud, log, _args):
         for f in glob.glob(key_pth):
             try:
                 util.del_file(f)
-            except:
+            except Exception:
                 util.logexc(log, "Failed deleting key file %s", f)
 
     if "ssh_keys" in cfg:
@@ -78,7 +78,7 @@ def handle(_name, cfg, cloud, log, _args):
                 with util.SeLinuxGuard("/etc/ssh", recursive=True):
                     util.subp(cmd, capture=False)
                 log.debug("Generated a key for %s from %s", pair[0], pair[1])
-            except:
+            except Exception:
                 util.logexc(log, "Failed generated a key for %s from %s",
                             pair[0], pair[1])
     else:
@@ -122,7 +122,7 @@ def handle(_name, cfg, cloud, log, _args):
             keys.extend(cfgkeys)
 
         apply_credentials(keys, user, disable_root, disable_root_opts)
-    except:
+    except Exception:
         util.logexc(log, "Applying ssh credentials failed!")
 
 
diff --git a/cloudinit/config/cc_ssh_import_id.py b/cloudinit/config/cc_ssh_import_id.py
index 2d480d7e..28c4585b 100644
--- a/cloudinit/config/cc_ssh_import_id.py
+++ b/cloudinit/config/cc_ssh_import_id.py
@@ -52,14 +52,14 @@ def handle(_name, cfg, cloud, log, args):
         else:
             try:
                 import_ids = user_cfg['ssh_import_id']
-            except:
+            except Exception:
                 log.debug("User %s is not configured for ssh_import_id", user)
                 continue
 
         try:
             import_ids = util.uniq_merge(import_ids)
             import_ids = [str(i) for i in import_ids]
-        except:
+        except Exception:
             log.debug("User %s is not correctly configured for ssh_import_id",
                       user)
             continue
diff --git a/cloudinit/config/cc_ubuntu_init_switch.py b/cloudinit/config/cc_ubuntu_init_switch.py
index 7e88ed85..884d79f1 100644
--- a/cloudinit/config/cc_ubuntu_init_switch.py
+++ b/cloudinit/config/cc_ubuntu_init_switch.py
@@ -40,10 +40,10 @@ It can be configured with the following option structure::
     mechanism you've used to switch the init system.
 """
 
-from cloudinit.settings import PER_INSTANCE
+from cloudinit.distros import ubuntu
 from cloudinit import log as logging
+from cloudinit.settings import PER_INSTANCE
 from cloudinit import util
-from cloudinit.distros import ubuntu
 
 import os
 import time
diff --git a/cloudinit/config/cc_write_files.py b/cloudinit/config/cc_write_files.py
index 351cfc8c..b1096b9b 100644
--- a/cloudinit/config/cc_write_files.py
+++ b/cloudinit/config/cc_write_files.py
@@ -79,6 +79,8 @@ def write_files(name, files, log):
 
 
 def decode_perms(perm, default, log):
+    if perm is None:
+        return default
     try:
         if isinstance(perm, six.integer_types + (float,)):
             # Just 'downcast' it (if a float)
diff --git a/cloudinit/cs_utils.py b/cloudinit/cs_utils.py
index 83ac1a0e..412431f2 100644
--- a/cloudinit/cs_utils.py
+++ b/cloudinit/cs_utils.py
@@ -33,7 +33,8 @@ API Docs: http://cloudsigma-docs.readthedocs.org/en/latest/server_context.html
 import json
 import platform
 
-import serial
+from cloudinit import serial
+
 
 # these high timeouts are necessary as read may read a lot of data.
 READ_TIMEOUT = 60
diff --git a/cloudinit/distros/__init__.py b/cloudinit/distros/__init__.py
index 5879dabf..14b500f8 100644
--- a/cloudinit/distros/__init__.py
+++ b/cloudinit/distros/__init__.py
@@ -31,6 +31,7 @@ import stat
 
 from cloudinit import importer
 from cloudinit import log as logging
+from cloudinit import net
 from cloudinit import ssh_util
 from cloudinit import type_utils
 from cloudinit import util
@@ -50,8 +51,8 @@ OSFAMILIES = {
 LOG = logging.getLogger(__name__)
 
 
+@six.add_metaclass(abc.ABCMeta)
 class Distro(object):
-    __metaclass__ = abc.ABCMeta
 
     usr_lib_exec = "/usr/lib"
     hosts_fn = "/etc/hosts"
@@ -97,7 +98,7 @@ class Distro(object):
         try:
             res = os.lstat('/run/systemd/system')
             return stat.S_ISDIR(res.st_mode)
-        except:
+        except Exception:
             return False
 
     @abc.abstractmethod
@@ -128,6 +129,8 @@ class Distro(object):
                                         mirror_info=arch_info)
 
     def apply_network(self, settings, bring_up=True):
+        # this applies network where 'settings' is interfaces(5) style
+        # it is obsolete compared to apply_network_config
         # Write it out
         dev_names = self._write_network(settings)
         # Now try to bring them up
@@ -143,6 +146,9 @@ class Distro(object):
             return self._bring_up_interfaces(dev_names)
         return False
 
+    def apply_network_config_names(self, netconfig):
+        net.apply_network_config_names(netconfig)
+
     @abc.abstractmethod
     def apply_locale(self, locale, out_fn=None):
         raise NotImplementedError()
@@ -448,7 +454,7 @@ class Distro(object):
             keys = kwargs['ssh_authorized_keys']
             if isinstance(keys, six.string_types):
                 keys = [keys]
-            if isinstance(keys, dict):
+            elif isinstance(keys, dict):
                 keys = list(keys.values())
             if keys is not None:
                 if not isinstance(keys, (tuple, list, set)):
diff --git a/cloudinit/distros/arch.py b/cloudinit/distros/arch.py
index 93a2e008..66209f22 100644
--- a/cloudinit/distros/arch.py
+++ b/cloudinit/distros/arch.py
@@ -196,6 +196,6 @@ def convert_resolv_conf(settings):
     """Returns a settings string formatted for resolv.conf."""
     result = ''
     if isinstance(settings, list):
-        for ns in list:
+        for ns in settings:
             result = result + 'nameserver %s\n' % ns
-        return result
+    return result
diff --git a/cloudinit/distros/debian.py b/cloudinit/distros/debian.py
index 75ab340f..5ae9a509 100644
--- a/cloudinit/distros/debian.py
+++ b/cloudinit/distros/debian.py
@@ -25,8 +25,9 @@ import os
 from cloudinit import distros
 from cloudinit import helpers
 from cloudinit import log as logging
+from cloudinit.net import eni
+from cloudinit.net.network_state import parse_net_config_data
 from cloudinit import util
-from cloudinit import net
 
 from cloudinit.distros.parsers.hostname import HostnameConf
 
@@ -42,6 +43,13 @@ APT_GET_WRAPPER = {
     'enabled': 'auto',
 }
 
+ENI_HEADER = """# This file is generated from information provided by
+# the datasource.  Changes to it will not persist across an instance.
+# To disable cloud-init's network configuration capabilities, write a file
+# /etc/cloud/cloud.cfg.d/99-disable-network-config.cfg with the following:
+# network: {config: disabled}
+"""
+
 
 class Distro(distros.Distro):
     hostname_conf_fn = "/etc/hostname"
@@ -56,6 +64,12 @@ class Distro(distros.Distro):
         # should only happen say once per instance...)
         self._runner = helpers.Runners(paths)
         self.osfamily = 'debian'
+        self._net_renderer = eni.Renderer({
+            'eni_path': self.network_conf_fn,
+            'eni_header': ENI_HEADER,
+            'links_prefix_path': None,
+            'netrules_path': None,
+        })
 
     def apply_locale(self, locale, out_fn=None):
         if not out_fn:
@@ -79,13 +93,9 @@ class Distro(distros.Distro):
         return ['all']
 
     def _write_network_config(self, netconfig):
-        ns = net.parse_net_config_data(netconfig)
-        net.render_network_state(target="/", network_state=ns,
-                                 eni=self.network_conf_fn,
-                                 links_prefix=self.links_prefix,
-                                 netrules=None)
+        ns = parse_net_config_data(netconfig)
+        self._net_renderer.render_network_state("/", ns)
         _maybe_remove_legacy_eth0()
-
         return []
 
     def _bring_up_interfaces(self, device_names):
@@ -221,7 +231,7 @@ def _maybe_remove_legacy_eth0(path="/etc/network/interfaces.d/eth0.cfg"):
             msg = "removed %s with known contents" % path
         else:
             msg = (bmsg + " '%s' exists with user configured content." % path)
-    except:
+    except Exception:
         msg = bmsg + " %s exists, but could not be read." % path
 
     LOG.warn(msg)
diff --git a/cloudinit/distros/rhel.py b/cloudinit/distros/rhel.py
index 812e7002..1aa42d75 100644
--- a/cloudinit/distros/rhel.py
+++ b/cloudinit/distros/rhel.py
@@ -23,6 +23,8 @@
 from cloudinit import distros
 from cloudinit import helpers
 from cloudinit import log as logging
+from cloudinit.net.network_state import parse_net_config_data
+from cloudinit.net import sysconfig
 from cloudinit import util
 
 from cloudinit.distros import net_util
@@ -59,10 +61,16 @@ class Distro(distros.Distro):
         # should only happen say once per instance...)
         self._runner = helpers.Runners(paths)
         self.osfamily = 'redhat'
+        self._net_renderer = sysconfig.Renderer()
 
     def install_packages(self, pkglist):
         self.package_command('install', pkgs=pkglist)
 
+    def _write_network_config(self, netconfig):
+        ns = parse_net_config_data(netconfig)
+        self._net_renderer.render_network_state("/", ns)
+        return []
+
     def _write_network(self, settings):
         # TODO(harlowja) fix this... since this is the ubuntu format
         entries = net_util.translate_network(settings)
diff --git a/cloudinit/ec2_utils.py b/cloudinit/ec2_utils.py
index 37b92a83..76dda042 100644
--- a/cloudinit/ec2_utils.py
+++ b/cloudinit/ec2_utils.py
@@ -144,9 +144,7 @@ def _skip_retry_on_codes(status_codes, _request_args, cause):
     """Returns if a request should retry based on a given set of codes that
     case retrying to be stopped/skipped.
     """
-    if cause.code in status_codes:
-        return False
-    return True
+    return cause.code in status_codes
 
 
 def get_instance_userdata(api_version='latest',
diff --git a/cloudinit/gpg.py b/cloudinit/gpg.py
new file mode 100644
index 00000000..6a76d785
--- /dev/null
+++ b/cloudinit/gpg.py
@@ -0,0 +1,74 @@
+"""gpg.py - Collection of gpg key related functions"""
+# vi: ts=4 expandtab
+#
+#    Copyright (C) 2016 Canonical Ltd.
+#
+#    Author: Scott Moser <scott.moser@canonical.com>
+#    Author: Christian Ehrhardt <christian.ehrhardt@canonical.com>
+#
+#    This program is free software: you can redistribute it and/or modify
+#    it under the terms of the GNU General Public License version 3, as
+#    published by the Free Software Foundation.
+#
+#    This program is distributed in the hope that it will be useful,
+#    but WITHOUT ANY WARRANTY; without even the implied warranty of
+#    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+#    GNU General Public License for more details.
+#
+#    You should have received a copy of the GNU General Public License
+#    along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+from cloudinit import log as logging
+from cloudinit import util
+
+LOG = logging.getLogger(__name__)
+
+
+def export_armour(key):
+    """Export gpg key, armoured key gets returned"""
+    try:
+        (armour, _) = util.subp(["gpg", "--export", "--armour", key],
+                                capture=True)
+    except util.ProcessExecutionError as error:
+        # debug, since it happens for any key not on the system initially
+        LOG.debug('Failed to export armoured key "%s": %s', key, error)
+        armour = None
+    return armour
+
+
+def receive_key(key, keyserver):
+    """Receive gpg key from the specified keyserver"""
+    LOG.debug('Receive gpg key "%s"', key)
+    try:
+        util.subp(["gpg", "--keyserver", keyserver, "--recv-keys", key],
+                  capture=True)
+    except util.ProcessExecutionError as error:
+        raise ValueError(('Failed to import key "%s" '
+                          'from server "%s" - error %s') %
+                         (key, keyserver, error))
+
+
+def delete_key(key):
+    """Delete the specified key from the local gpg ring"""
+    try:
+        util.subp(["gpg", "--batch", "--yes", "--delete-keys", key],
+                  capture=True)
+    except util.ProcessExecutionError as error:
+        LOG.warn('Failed delete key "%s": %s', key, error)
+
+
+def get_key_by_id(keyid, keyserver="keyserver.ubuntu.com"):
+    """get gpg keyid from keyserver"""
+    armour = export_armour(keyid)
+    if not armour:
+        try:
+            receive_key(keyid, keyserver=keyserver)
+            armour = export_armour(keyid)
+        except ValueError:
+            LOG.exception('Failed to obtain gpg key %s', keyid)
+            raise
+        finally:
+            # delete just imported key to leave environment as it was before
+            delete_key(keyid)
+
+    return armour
diff --git a/cloudinit/handlers/__init__.py b/cloudinit/handlers/__init__.py
index 53d5604a..b6c43ce8 100644
--- a/cloudinit/handlers/__init__.py
+++ b/cloudinit/handlers/__init__.py
@@ -71,8 +71,8 @@ INCLUSION_SRCH = sorted(list(INCLUSION_TYPES_MAP.keys()),
                         key=(lambda e: 0 - len(e)))
 
 
+@six.add_metaclass(abc.ABCMeta)
 class Handler(object):
-    __metaclass__ = abc.ABCMeta
 
     def __init__(self, frequency, version=2):
         self.handler_version = version
@@ -118,7 +118,7 @@ def run_part(mod, data, filename, payload, frequency, headers):
             mod.handle_part(data, content_type, filename, payload)
         else:
             raise ValueError("Unknown module version %s" % (mod_ver))
-    except:
+    except Exception:
         util.logexc(LOG, "Failed calling handler %s (%s, %s, %s) with "
                     "frequency %s", mod, content_type, filename, mod_ver,
                     frequency)
@@ -157,7 +157,7 @@ def walker_handle_handler(pdata, _ctype, _filename, payload):
         # register if it fails starting.
         handlers.register(mod, initialized=True)
         pdata['handlercount'] = curcount + 1
-    except:
+    except Exception:
         util.logexc(LOG, "Failed at registering python file: %s (part "
                     "handler %s)", modfname, curcount)
 
diff --git a/cloudinit/handlers/cloud_config.py b/cloudinit/handlers/cloud_config.py
index 07b6d0e0..cad4dc0f 100644
--- a/cloudinit/handlers/cloud_config.py
+++ b/cloudinit/handlers/cloud_config.py
@@ -158,6 +158,6 @@ class CloudConfigPartHandler(handlers.Handler):
             for i in ("\n", "\r", "\t"):
                 filename = filename.replace(i, " ")
             self.file_names.append(filename.strip())
-        except:
+        except Exception:
             util.logexc(LOG, "Failed at merging in cloud config part from %s",
                         filename)
diff --git a/cloudinit/handlers/upstart_job.py b/cloudinit/handlers/upstart_job.py
index c5bea711..ab381e00 100644
--- a/cloudinit/handlers/upstart_job.py
+++ b/cloudinit/handlers/upstart_job.py
@@ -80,7 +80,7 @@ def _has_suitable_upstart():
         return False
     try:
         (version_out, _err) = util.subp(["initctl", "version"])
-    except:
+    except Exception:
         util.logexc(LOG, "initctl version failed")
         return False
 
diff --git a/cloudinit/helpers.py b/cloudinit/helpers.py
index 0cf982f3..fb95babc 100644
--- a/cloudinit/helpers.py
+++ b/cloudinit/helpers.py
@@ -86,7 +86,7 @@ class FileSemaphores(object):
         name = canon_sem_name(name)
         try:
             yield self._acquire(name, freq)
-        except:
+        except Exception:
             if clear_on_fail:
                 self.clear(name, freq)
             raise
@@ -219,7 +219,7 @@ class ConfigMerger(object):
                 ds_cfg = self._ds.get_config_obj()
                 if ds_cfg and isinstance(ds_cfg, (dict)):
                     d_cfgs.append(ds_cfg)
-            except:
+            except Exception:
                 util.logexc(LOG, "Failed loading of datasource config object "
                             "from %s", self._ds)
         return d_cfgs
@@ -230,7 +230,7 @@ class ConfigMerger(object):
             e_fn = os.environ[CFG_ENV_NAME]
             try:
                 e_cfgs.append(util.read_conf(e_fn))
-            except:
+            except Exception:
                 util.logexc(LOG, 'Failed loading of env. config from %s',
                             e_fn)
         return e_cfgs
@@ -251,7 +251,7 @@ class ConfigMerger(object):
             if cc_fn and os.path.isfile(cc_fn):
                 try:
                     i_cfgs.append(util.read_conf(cc_fn))
-                except:
+                except Exception:
                     util.logexc(LOG, 'Failed loading of cloud-config from %s',
                                 cc_fn)
         return i_cfgs
@@ -268,7 +268,7 @@ class ConfigMerger(object):
             for c_fn in self._fns:
                 try:
                     cfgs.append(util.read_conf(c_fn))
-                except:
+                except Exception:
                     util.logexc(LOG, "Failed loading of configuration from %s",
                                 c_fn)
 
@@ -328,6 +328,7 @@ class Paths(object):
         self.cfgs = path_cfgs
         # Populate all the initial paths
         self.cloud_dir = path_cfgs.get('cloud_dir', '/var/lib/cloud')
+        self.run_dir = path_cfgs.get('run_dir', '/run/cloud-init')
         self.instance_link = os.path.join(self.cloud_dir, 'instance')
         self.boot_finished = os.path.join(self.instance_link, "boot-finished")
         self.upstart_conf_d = path_cfgs.get('upstart_dir')
@@ -349,26 +350,19 @@ class Paths(object):
             "data": "data",
             "vendordata_raw": "vendor-data.txt",
             "vendordata": "vendor-data.txt.i",
+            "instance_id": ".instance-id",
         }
         # Set when a datasource becomes active
         self.datasource = ds
 
     # get_ipath_cur: get the current instance path for an item
     def get_ipath_cur(self, name=None):
-        ipath = self.instance_link
-        add_on = self.lookups.get(name)
-        if add_on:
-            ipath = os.path.join(ipath, add_on)
-        return ipath
+        return self._get_path(self.instance_link, name)
 
     # get_cpath : get the "clouddir" (/var/lib/cloud/<name>)
     # for a name in dirmap
     def get_cpath(self, name=None):
-        cpath = self.cloud_dir
-        add_on = self.lookups.get(name)
-        if add_on:
-            cpath = os.path.join(cpath, add_on)
-        return cpath
+        return self._get_path(self.cloud_dir, name)
 
     # _get_ipath : get the instance path for a name in pathmap
     # (/var/lib/cloud/instances/<instance>/<name>)
@@ -378,7 +372,8 @@ class Paths(object):
         iid = self.datasource.get_instance_id()
         if iid is None:
             return None
-        ipath = os.path.join(self.cloud_dir, 'instances', str(iid))
+        path_safe_iid = str(iid).replace(os.sep, '_')
+        ipath = os.path.join(self.cloud_dir, 'instances', path_safe_iid)
         add_on = self.lookups.get(name)
         if add_on:
             ipath = os.path.join(ipath, add_on)
@@ -396,6 +391,14 @@ class Paths(object):
         else:
             return ipath
 
+    def _get_path(self, base, name=None):
+        if name is None:
+            return base
+        return os.path.join(base, self.lookups[name])
+
+    def get_runpath(self, name=None):
+        return self._get_path(self.run_dir, name)
+
 
 # This config parser will not throw when sections don't exist
 # and you are setting values on those sections which is useful
diff --git a/cloudinit/net/__init__.py b/cloudinit/net/__init__.py
index 31544fd8..63e54f91 100644
--- a/cloudinit/net/__init__.py
+++ b/cloudinit/net/__init__.py
@@ -16,41 +16,15 @@
 #   You should have received a copy of the GNU Affero General Public License
 #   along with Curtin.  If not, see <http://www.gnu.org/licenses/>.
 
-import base64
 import errno
-import glob
-import gzip
-import io
+import logging
 import os
 import re
-import shlex
 
-from cloudinit import log as logging
 from cloudinit import util
-from .udev import generate_udev_rule
-from . import network_state
 
 LOG = logging.getLogger(__name__)
-
 SYS_CLASS_NET = "/sys/class/net/"
-LINKS_FNAME_PREFIX = "etc/systemd/network/50-cloud-init-"
-
-NET_CONFIG_OPTIONS = [
-    "address", "netmask", "broadcast", "network", "metric", "gateway",
-    "pointtopoint", "media", "mtu", "hostname", "leasehours", "leasetime",
-    "vendor", "client", "bootfile", "server", "hwaddr", "provider", "frame",
-    "netnum", "endpoint", "local", "ttl",
-    ]
-
-NET_CONFIG_COMMANDS = [
-    "pre-up", "up", "post-up", "down", "pre-down", "post-down",
-    ]
-
-NET_CONFIG_BRIDGE_OPTIONS = [
-    "bridge_ageing", "bridge_bridgeprio", "bridge_fd", "bridge_gcinit",
-    "bridge_hello", "bridge_maxage", "bridge_maxwait", "bridge_stp",
-    ]
-
 DEFAULT_PRIMARY_INTERFACE = 'eth0'
 
 
@@ -60,23 +34,22 @@ def sys_dev_path(devname, path=""):
 
 def read_sys_net(devname, path, translate=None, enoent=None, keyerror=None):
     try:
-        contents = ""
-        with open(sys_dev_path(devname, path), "r") as fp:
-            contents = fp.read().strip()
-        if translate is None:
-            return contents
-
-        try:
-            return translate.get(contents)
-        except KeyError:
-            LOG.debug("found unexpected value '%s' in '%s/%s'", contents,
-                      devname, path)
-            if keyerror is not None:
-                return keyerror
-            raise
-    except OSError as e:
-        if e.errno == errno.ENOENT and enoent is not None:
-            return enoent
+        contents = util.load_file(sys_dev_path(devname, path))
+    except (OSError, IOError) as e:
+        if getattr(e, 'errno', None) == errno.ENOENT:
+            if enoent is not None:
+                return enoent
+        raise
+    contents = contents.strip()
+    if translate is None:
+        return contents
+    try:
+        return translate.get(contents)
+    except KeyError:
+        LOG.debug("found unexpected value '%s' in '%s/%s'", contents,
+                  devname, path)
+        if keyerror is not None:
+            return keyerror
         raise
 
 
@@ -127,505 +100,7 @@ def get_devicelist():
 
 
 class ParserError(Exception):
-    """Raised when parser has issue parsing the interfaces file."""
-
-
-def parse_deb_config_data(ifaces, contents, src_dir, src_path):
-    """Parses the file contents, placing result into ifaces.
-
-    '_source_path' is added to every dictionary entry to define which file
-    the configration information came from.
-
-    :param ifaces: interface dictionary
-    :param contents: contents of interfaces file
-    :param src_dir: directory interfaces file was located
-    :param src_path: file path the `contents` was read
-    """
-    currif = None
-    for line in contents.splitlines():
-        line = line.strip()
-        if line.startswith('#'):
-            continue
-        split = line.split(' ')
-        option = split[0]
-        if option == "source-directory":
-            parsed_src_dir = split[1]
-            if not parsed_src_dir.startswith("/"):
-                parsed_src_dir = os.path.join(src_dir, parsed_src_dir)
-            for expanded_path in glob.glob(parsed_src_dir):
-                dir_contents = os.listdir(expanded_path)
-                dir_contents = [
-                    os.path.join(expanded_path, path)
-                    for path in dir_contents
-                    if (os.path.isfile(os.path.join(expanded_path, path)) and
-                        re.match("^[a-zA-Z0-9_-]+$", path) is not None)
-                ]
-                for entry in dir_contents:
-                    with open(entry, "r") as fp:
-                        src_data = fp.read().strip()
-                    abs_entry = os.path.abspath(entry)
-                    parse_deb_config_data(
-                        ifaces, src_data,
-                        os.path.dirname(abs_entry), abs_entry)
-        elif option == "source":
-            new_src_path = split[1]
-            if not new_src_path.startswith("/"):
-                new_src_path = os.path.join(src_dir, new_src_path)
-            for expanded_path in glob.glob(new_src_path):
-                with open(expanded_path, "r") as fp:
-                    src_data = fp.read().strip()
-                abs_path = os.path.abspath(expanded_path)
-                parse_deb_config_data(
-                    ifaces, src_data,
-                    os.path.dirname(abs_path), abs_path)
-        elif option == "auto":
-            for iface in split[1:]:
-                if iface not in ifaces:
-                    ifaces[iface] = {
-                        # Include the source path this interface was found in.
-                        "_source_path": src_path
-                    }
-                ifaces[iface]['auto'] = True
-        elif option == "iface":
-            iface, family, method = split[1:4]
-            if iface not in ifaces:
-                ifaces[iface] = {
-                    # Include the source path this interface was found in.
-                    "_source_path": src_path
-                }
-            elif 'family' in ifaces[iface]:
-                raise ParserError(
-                    "Interface %s can only be defined once. "
-                    "Re-defined in '%s'." % (iface, src_path))
-            ifaces[iface]['family'] = family
-            ifaces[iface]['method'] = method
-            currif = iface
-        elif option == "hwaddress":
-            ifaces[currif]['hwaddress'] = split[1]
-        elif option in NET_CONFIG_OPTIONS:
-            ifaces[currif][option] = split[1]
-        elif option in NET_CONFIG_COMMANDS:
-            if option not in ifaces[currif]:
-                ifaces[currif][option] = []
-            ifaces[currif][option].append(' '.join(split[1:]))
-        elif option.startswith('dns-'):
-            if 'dns' not in ifaces[currif]:
-                ifaces[currif]['dns'] = {}
-            if option == 'dns-search':
-                ifaces[currif]['dns']['search'] = []
-                for domain in split[1:]:
-                    ifaces[currif]['dns']['search'].append(domain)
-            elif option == 'dns-nameservers':
-                ifaces[currif]['dns']['nameservers'] = []
-                for server in split[1:]:
-                    ifaces[currif]['dns']['nameservers'].append(server)
-        elif option.startswith('bridge_'):
-            if 'bridge' not in ifaces[currif]:
-                ifaces[currif]['bridge'] = {}
-            if option in NET_CONFIG_BRIDGE_OPTIONS:
-                bridge_option = option.replace('bridge_', '', 1)
-                ifaces[currif]['bridge'][bridge_option] = split[1]
-            elif option == "bridge_ports":
-                ifaces[currif]['bridge']['ports'] = []
-                for iface in split[1:]:
-                    ifaces[currif]['bridge']['ports'].append(iface)
-            elif option == "bridge_hw" and split[1].lower() == "mac":
-                ifaces[currif]['bridge']['mac'] = split[2]
-            elif option == "bridge_pathcost":
-                if 'pathcost' not in ifaces[currif]['bridge']:
-                    ifaces[currif]['bridge']['pathcost'] = {}
-                ifaces[currif]['bridge']['pathcost'][split[1]] = split[2]
-            elif option == "bridge_portprio":
-                if 'portprio' not in ifaces[currif]['bridge']:
-                    ifaces[currif]['bridge']['portprio'] = {}
-                ifaces[currif]['bridge']['portprio'][split[1]] = split[2]
-        elif option.startswith('bond-'):
-            if 'bond' not in ifaces[currif]:
-                ifaces[currif]['bond'] = {}
-            bond_option = option.replace('bond-', '', 1)
-            ifaces[currif]['bond'][bond_option] = split[1]
-    for iface in ifaces.keys():
-        if 'auto' not in ifaces[iface]:
-            ifaces[iface]['auto'] = False
-
-
-def parse_deb_config(path):
-    """Parses a debian network configuration file."""
-    ifaces = {}
-    with open(path, "r") as fp:
-        contents = fp.read().strip()
-    abs_path = os.path.abspath(path)
-    parse_deb_config_data(
-        ifaces, contents,
-        os.path.dirname(abs_path), abs_path)
-    return ifaces
-
-
-def parse_net_config_data(net_config):
-    """Parses the config, returns NetworkState dictionary
-
-    :param net_config: curtin network config dict
-    """
-    state = None
-    if 'version' in net_config and 'config' in net_config:
-        ns = network_state.NetworkState(version=net_config.get('version'),
-                                        config=net_config.get('config'))
-        ns.parse_config()
-        state = ns.network_state
-
-    return state
-
-
-def parse_net_config(path):
-    """Parses a curtin network configuration file and
-       return network state"""
-    ns = None
-    net_config = util.read_conf(path)
-    if 'network' in net_config:
-        ns = parse_net_config_data(net_config.get('network'))
-
-    return ns
-
-
-def _load_shell_content(content, add_empty=False, empty_val=None):
-    """Given shell like syntax (key=value\nkey2=value2\n) in content
-       return the data in dictionary form.  If 'add_empty' is True
-       then add entries in to the returned dictionary for 'VAR='
-       variables.  Set their value to empty_val."""
-    data = {}
-    for line in shlex.split(content):
-        key, value = line.split("=", 1)
-        if not value:
-            value = empty_val
-        if add_empty or value:
-            data[key] = value
-
-    return data
-
-
-def _klibc_to_config_entry(content, mac_addrs=None):
-    """Convert a klibc writtent shell content file to a 'config' entry
-    When ip= is seen on the kernel command line in debian initramfs
-    and networking is brought up, ipconfig will populate
-    /run/net-<name>.cfg.
-
-    The files are shell style syntax, and examples are in the tests
-    provided here.  There is no good documentation on this unfortunately.
-
-    DEVICE=<name> is expected/required and PROTO should indicate if
-    this is 'static' or 'dhcp'.
-    """
-
-    if mac_addrs is None:
-        mac_addrs = {}
-
-    data = _load_shell_content(content)
-    try:
-        name = data['DEVICE']
-    except KeyError:
-        raise ValueError("no 'DEVICE' entry in data")
-
-    # ipconfig on precise does not write PROTO
-    proto = data.get('PROTO')
-    if not proto:
-        if data.get('filename'):
-            proto = 'dhcp'
-        else:
-            proto = 'static'
-
-    if proto not in ('static', 'dhcp'):
-        raise ValueError("Unexpected value for PROTO: %s" % proto)
-
-    iface = {
-        'type': 'physical',
-        'name': name,
-        'subnets': [],
-    }
-
-    if name in mac_addrs:
-        iface['mac_address'] = mac_addrs[name]
-
-    # originally believed there might be IPV6* values
-    for v, pre in (('ipv4', 'IPV4'),):
-        # if no IPV4ADDR or IPV6ADDR, then go on.
-        if pre + "ADDR" not in data:
-            continue
-        subnet = {'type': proto, 'control': 'manual'}
-
-        # these fields go right on the subnet
-        for key in ('NETMASK', 'BROADCAST', 'GATEWAY'):
-            if pre + key in data:
-                subnet[key.lower()] = data[pre + key]
-
-        dns = []
-        # handle IPV4DNS0 or IPV6DNS0
-        for nskey in ('DNS0', 'DNS1'):
-            ns = data.get(pre + nskey)
-            # verify it has something other than 0.0.0.0 (or ipv6)
-            if ns and len(ns.strip(":.0")):
-                dns.append(data[pre + nskey])
-        if dns:
-            subnet['dns_nameservers'] = dns
-            # add search to both ipv4 and ipv6, as it has no namespace
-            search = data.get('DOMAINSEARCH')
-            if search:
-                if ',' in search:
-                    subnet['dns_search'] = search.split(",")
-                else:
-                    subnet['dns_search'] = search.split()
-
-        iface['subnets'].append(subnet)
-
-    return name, iface
-
-
-def config_from_klibc_net_cfg(files=None, mac_addrs=None):
-    if files is None:
-        files = glob.glob('/run/net*.conf')
-
-    entries = []
-    names = {}
-    for cfg_file in files:
-        name, entry = _klibc_to_config_entry(util.load_file(cfg_file),
-                                             mac_addrs=mac_addrs)
-        if name in names:
-            raise ValueError(
-                "device '%s' defined multiple times: %s and %s" % (
-                    name, names[name], cfg_file))
-
-        names[name] = cfg_file
-        entries.append(entry)
-    return {'config': entries, 'version': 1}
-
-
-def render_persistent_net(network_state):
-    ''' Given state, emit udev rules to map
-        mac to ifname
-    '''
-    content = ""
-    interfaces = network_state.get('interfaces')
-    for iface in interfaces.values():
-        # for physical interfaces write out a persist net udev rule
-        if iface['type'] == 'physical' and \
-           'name' in iface and iface.get('mac_address'):
-            content += generate_udev_rule(iface['name'],
-                                          iface['mac_address'])
-
-    return content
-
-
-# TODO: switch valid_map based on mode inet/inet6
-def iface_add_subnet(iface, subnet):
-    content = ""
-    valid_map = [
-        'address',
-        'netmask',
-        'broadcast',
-        'metric',
-        'gateway',
-        'pointopoint',
-        'mtu',
-        'scope',
-        'dns_search',
-        'dns_nameservers',
-    ]
-    for key, value in subnet.items():
-        if value and key in valid_map:
-            if type(value) == list:
-                value = " ".join(value)
-            if '_' in key:
-                key = key.replace('_', '-')
-            content += "    {} {}\n".format(key, value)
-
-    return content
-
-
-# TODO: switch to valid_map for attrs
-def iface_add_attrs(iface):
-    content = ""
-    ignore_map = [
-        'control',
-        'index',
-        'inet',
-        'mode',
-        'name',
-        'subnets',
-        'type',
-    ]
-    if iface['type'] not in ['bond', 'bridge', 'vlan']:
-        ignore_map.append('mac_address')
-
-    for key, value in iface.items():
-        if value and key not in ignore_map:
-            if type(value) == list:
-                value = " ".join(value)
-            content += "    {} {}\n".format(key, value)
-
-    return content
-
-
-def render_route(route, indent=""):
-    """ When rendering routes for an iface, in some cases applying a route
-    may result in the route command returning non-zero which produces
-    some confusing output for users manually using ifup/ifdown[1].  To
-    that end, we will optionally include an '|| true' postfix to each
-    route line allowing users to work with ifup/ifdown without using
-    --force option.
-
-    We may at somepoint not want to emit this additional postfix, and
-    add a 'strict' flag to this function.  When called with strict=True,
-    then we will not append the postfix.
-
-    1. http://askubuntu.com/questions/168033/
-             how-to-set-static-routes-in-ubuntu-server
-    """
-    content = ""
-    up = indent + "post-up route add"
-    down = indent + "pre-down route del"
-    eol = " || true\n"
-    mapping = {
-        'network': '-net',
-        'netmask': 'netmask',
-        'gateway': 'gw',
-        'metric': 'metric',
-    }
-    if route['network'] == '0.0.0.0' and route['netmask'] == '0.0.0.0':
-        default_gw = " default gw %s" % route['gateway']
-        content += up + default_gw + eol
-        content += down + default_gw + eol
-    elif route['network'] == '::' and route['netmask'] == 0:
-        # ipv6!
-        default_gw = " -A inet6 default gw %s" % route['gateway']
-        content += up + default_gw + eol
-        content += down + default_gw + eol
-    else:
-        route_line = ""
-        for k in ['network', 'netmask', 'gateway', 'metric']:
-            if k in route:
-                route_line += " %s %s" % (mapping[k], route[k])
-        content += up + route_line + eol
-        content += down + route_line + eol
-
-    return content
-
-
-def iface_start_entry(iface, index):
-    fullname = iface['name']
-    if index != 0:
-        fullname += ":%s" % index
-
-    control = iface['control']
-    if control == "auto":
-        cverb = "auto"
-    elif control in ("hotplug",):
-        cverb = "allow-" + control
-    else:
-        cverb = "# control-" + control
-
-    subst = iface.copy()
-    subst.update({'fullname': fullname, 'cverb': cverb})
-
-    return ("{cverb} {fullname}\n"
-            "iface {fullname} {inet} {mode}\n").format(**subst)
-
-
-def render_interfaces(network_state):
-    ''' Given state, emit etc/network/interfaces content '''
-
-    content = ""
-    interfaces = network_state.get('interfaces')
-    ''' Apply a sort order to ensure that we write out
-        the physical interfaces first; this is critical for
-        bonding
-    '''
-    order = {
-        'physical': 0,
-        'bond': 1,
-        'bridge': 2,
-        'vlan': 3,
-    }
-    content += "auto lo\niface lo inet loopback\n"
-    for dnskey, value in network_state.get('dns', {}).items():
-        if len(value):
-            content += "    dns-{} {}\n".format(dnskey, " ".join(value))
-
-    for iface in sorted(interfaces.values(),
-                        key=lambda k: (order[k['type']], k['name'])):
-
-        if content[-2:] != "\n\n":
-            content += "\n"
-        subnets = iface.get('subnets', {})
-        if subnets:
-            for index, subnet in zip(range(0, len(subnets)), subnets):
-                if content[-2:] != "\n\n":
-                    content += "\n"
-                iface['index'] = index
-                iface['mode'] = subnet['type']
-                iface['control'] = subnet.get('control', 'auto')
-                if iface['mode'].endswith('6'):
-                    iface['inet'] += '6'
-                elif iface['mode'] == 'static' and ":" in subnet['address']:
-                    iface['inet'] += '6'
-                if iface['mode'].startswith('dhcp'):
-                    iface['mode'] = 'dhcp'
-
-                content += iface_start_entry(iface, index)
-                content += iface_add_subnet(iface, subnet)
-                content += iface_add_attrs(iface)
-        else:
-            # ifenslave docs say to auto the slave devices
-            if 'bond-master' in iface:
-                content += "auto {name}\n".format(**iface)
-            content += "iface {name} {inet} {mode}\n".format(**iface)
-            content += iface_add_attrs(iface)
-
-    for route in network_state.get('routes'):
-        content += render_route(route)
-
-    # global replacements until v2 format
-    content = content.replace('mac_address', 'hwaddress')
-    return content
-
-
-def render_network_state(target, network_state, eni="etc/network/interfaces",
-                         links_prefix=LINKS_FNAME_PREFIX,
-                         netrules='etc/udev/rules.d/70-persistent-net.rules'):
-
-    fpeni = os.path.sep.join((target, eni,))
-    util.ensure_dir(os.path.dirname(fpeni))
-    with open(fpeni, 'w+') as f:
-        f.write(render_interfaces(network_state))
-
-    if netrules:
-        netrules = os.path.sep.join((target, netrules,))
-        util.ensure_dir(os.path.dirname(netrules))
-        with open(netrules, 'w+') as f:
-            f.write(render_persistent_net(network_state))
-
-    if links_prefix:
-        render_systemd_links(target, network_state, links_prefix)
-
-
-def render_systemd_links(target, network_state,
-                         links_prefix=LINKS_FNAME_PREFIX):
-    fp_prefix = os.path.sep.join((target, links_prefix))
-    for f in glob.glob(fp_prefix + "*"):
-        os.unlink(f)
-
-    interfaces = network_state.get('interfaces')
-    for iface in interfaces.values():
-        if (iface['type'] == 'physical' and 'name' in iface and
-                iface.get('mac_address')):
-            fname = fp_prefix + iface['name'] + ".link"
-            with open(fname, "w") as fp:
-                fp.write("\n".join([
-                    "[Match]",
-                    "MACAddress=" + iface['mac_address'],
-                    "",
-                    "[Link]",
-                    "Name=" + iface['name'],
-                    ""
-                ]))
+    """Raised when a parser has issue parsing a file/content."""
 
 
 def is_disabled_cfg(cfg):
@@ -638,7 +113,6 @@ def sys_netdev_info(name, field):
     if not os.path.exists(os.path.join(SYS_CLASS_NET, name)):
         raise OSError("%s: interface does not exist in %s" %
                       (name, SYS_CLASS_NET))
-
     fname = os.path.join(SYS_CLASS_NET, name, field)
     if not os.path.exists(fname):
         raise OSError("%s: could not find sysfs entry: %s" % (name, fname))
@@ -718,56 +192,179 @@ def generate_fallback_config():
     return nconf
 
 
-def _decomp_gzip(blob, strict=True):
-    # decompress blob. raise exception if not compressed unless strict=False.
-    with io.BytesIO(blob) as iobuf:
-        gzfp = None
-        try:
-            gzfp = gzip.GzipFile(mode="rb", fileobj=iobuf)
-            return gzfp.read()
-        except IOError:
-            if strict:
-                raise
-            return blob
-        finally:
-            if gzfp:
-                gzfp.close()
+def apply_network_config_names(netcfg, strict_present=True, strict_busy=True):
+    """read the network config and rename devices accordingly.
+    if strict_present is false, then do not raise exception if no devices
+    match.  if strict_busy is false, then do not raise exception if the
+    device cannot be renamed because it is currently configured."""
+    renames = []
+    for ent in netcfg.get('config', {}):
+        if ent.get('type') != 'physical':
+            continue
+        mac = ent.get('mac_address')
+        name = ent.get('name')
+        if not mac:
+            continue
+        renames.append([mac, name])
 
+    return _rename_interfaces(renames)
 
-def _b64dgz(b64str, gzipped="try"):
-    # decode a base64 string.  If gzipped is true, transparently uncompresss
-    # if gzipped is 'try', then try gunzip, returning the original on fail.
-    try:
-        blob = base64.b64decode(b64str)
-    except TypeError:
-        raise ValueError("Invalid base64 text: %s" % b64str)
 
-    if not gzipped:
-        return blob
+def _get_current_rename_info(check_downable=True):
+    """Collect information necessary for rename_interfaces."""
+    names = get_devicelist()
+    bymac = {}
+    for n in names:
+        bymac[get_interface_mac(n)] = {
+            'name': n, 'up': is_up(n), 'downable': None}
+
+    if check_downable:
+        nmatch = re.compile(r"[0-9]+:\s+(\w+)[@:]")
+        ipv6, _err = util.subp(['ip', '-6', 'addr', 'show', 'permanent',
+                                'scope', 'global'], capture=True)
+        ipv4, _err = util.subp(['ip', '-4', 'addr', 'show'], capture=True)
+
+        nics_with_addresses = set()
+        for bytes_out in (ipv6, ipv4):
+            nics_with_addresses.update(nmatch.findall(bytes_out))
+
+        for d in bymac.values():
+            d['downable'] = (d['up'] is False or
+                             d['name'] not in nics_with_addresses)
 
-    return _decomp_gzip(blob, strict=gzipped != "try")
+    return bymac
 
 
-def read_kernel_cmdline_config(files=None, mac_addrs=None, cmdline=None):
-    if cmdline is None:
-        cmdline = util.get_cmdline()
+def _rename_interfaces(renames, strict_present=True, strict_busy=True,
+                       current_info=None):
 
-    if 'network-config=' in cmdline:
-        data64 = None
-        for tok in cmdline.split():
-            if tok.startswith("network-config="):
-                data64 = tok.split("=", 1)[1]
-        if data64:
-            return util.load_yaml(_b64dgz(data64))
+    if not len(renames):
+        LOG.debug("no interfaces to rename")
+        return
+
+    if current_info is None:
+        current_info = _get_current_rename_info()
+
+    cur_bymac = {}
+    for mac, data in current_info.items():
+        cur = data.copy()
+        cur['mac'] = mac
+        cur_bymac[mac] = cur
+
+    def update_byname(bymac):
+        return {data['name']: data for data in bymac.values()}
+
+    def rename(cur, new):
+        util.subp(["ip", "link", "set", cur, "name", new], capture=True)
 
-    if 'ip=' not in cmdline:
-        return None
+    def down(name):
+        util.subp(["ip", "link", "set", name, "down"], capture=True)
 
-    if mac_addrs is None:
-        mac_addrs = {k: sys_netdev_info(k, 'address')
-                     for k in get_devicelist()}
+    def up(name):
+        util.subp(["ip", "link", "set", name, "up"], capture=True)
 
-    return config_from_klibc_net_cfg(files=files, mac_addrs=mac_addrs)
+    ops = []
+    errors = []
+    ups = []
+    cur_byname = update_byname(cur_bymac)
+    tmpname_fmt = "cirename%d"
+    tmpi = -1
+
+    for mac, new_name in renames:
+        cur = cur_bymac.get(mac, {})
+        cur_name = cur.get('name')
+        cur_ops = []
+        if cur_name == new_name:
+            # nothing to do
+            continue
+
+        if not cur_name:
+            if strict_present:
+                errors.append(
+                    "[nic not present] Cannot rename mac=%s to %s"
+                    ", not available." % (mac, new_name))
+            continue
 
+        if cur['up']:
+            msg = "[busy] Error renaming mac=%s from %s to %s"
+            if not cur['downable']:
+                if strict_busy:
+                    errors.append(msg % (mac, cur_name, new_name))
+                continue
+            cur['up'] = False
+            cur_ops.append(("down", mac, new_name, (cur_name,)))
+            ups.append(("up", mac, new_name, (new_name,)))
+
+        if new_name in cur_byname:
+            target = cur_byname[new_name]
+            if target['up']:
+                msg = "[busy-target] Error renaming mac=%s from %s to %s."
+                if not target['downable']:
+                    if strict_busy:
+                        errors.append(msg % (mac, cur_name, new_name))
+                    continue
+                else:
+                    cur_ops.append(("down", mac, new_name, (new_name,)))
+
+            tmp_name = None
+            while tmp_name is None or tmp_name in cur_byname:
+                tmpi += 1
+                tmp_name = tmpname_fmt % tmpi
+
+            cur_ops.append(("rename", mac, new_name, (new_name, tmp_name)))
+            target['name'] = tmp_name
+            cur_byname = update_byname(cur_bymac)
+            if target['up']:
+                ups.append(("up", mac, new_name, (tmp_name,)))
+
+        cur_ops.append(("rename", mac, new_name, (cur['name'], new_name)))
+        cur['name'] = new_name
+        cur_byname = update_byname(cur_bymac)
+        ops += cur_ops
+
+    opmap = {'rename': rename, 'down': down, 'up': up}
+
+    if len(ops) + len(ups) == 0:
+        if len(errors):
+            LOG.debug("unable to do any work for renaming of %s", renames)
+        else:
+            LOG.debug("no work necessary for renaming of %s", renames)
+    else:
+        LOG.debug("achieving renaming of %s with ops %s", renames, ops + ups)
+
+        for op, mac, new_name, params in ops + ups:
+            try:
+                opmap.get(op)(*params)
+            except Exception as e:
+                errors.append(
+                    "[unknown] Error performing %s%s for %s, %s: %s" %
+                    (op, params, mac, new_name, e))
+
+    if len(errors):
+        raise Exception('\n'.join(errors))
+
+
+def get_interface_mac(ifname):
+    """Returns the string value of an interface's MAC Address"""
+    return read_sys_net(ifname, "address", enoent=False)
+
+
+def get_interfaces_by_mac(devs=None):
+    """Build a dictionary of tuples {mac: name}"""
+    if devs is None:
+        try:
+            devs = get_devicelist()
+        except OSError as e:
+            if e.errno == errno.ENOENT:
+                devs = []
+            else:
+                raise
+    ret = {}
+    for name in devs:
+        mac = get_interface_mac(name)
+        # some devices may not have a mac (tun0)
+        if mac:
+            ret[mac] = name
+    return ret
 
 # vi: ts=4 expandtab syntax=python
diff --git a/cloudinit/net/cmdline.py b/cloudinit/net/cmdline.py
new file mode 100644
index 00000000..822a020b
--- /dev/null
+++ b/cloudinit/net/cmdline.py
@@ -0,0 +1,203 @@
+#   Copyright (C) 2013-2014 Canonical Ltd.
+#
+#   Author: Scott Moser <scott.moser@canonical.com>
+#   Author: Blake Rouse <blake.rouse@canonical.com>
+#
+#   Curtin is free software: you can redistribute it and/or modify it under
+#   the terms of the GNU Affero General Public License as published by the
+#   Free Software Foundation, either version 3 of the License, or (at your
+#   option) any later version.
+#
+#   Curtin is distributed in the hope that it will be useful, but WITHOUT ANY
+#   WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
+#   FOR A PARTICULAR PURPOSE.  See the GNU Affero General Public License for
+#   more details.
+#
+#   You should have received a copy of the GNU Affero General Public License
+#   along with Curtin.  If not, see <http://www.gnu.org/licenses/>.
+
+import base64
+import glob
+import gzip
+import io
+import shlex
+import sys
+
+import six
+
+from . import get_devicelist
+from . import sys_netdev_info
+
+from cloudinit import util
+
+PY26 = sys.version_info[0:2] == (2, 6)
+
+
+def _shlex_split(blob):
+    if PY26 and isinstance(blob, six.text_type):
+        # Older versions don't support unicode input
+        blob = blob.encode("utf8")
+    return shlex.split(blob)
+
+
+def _load_shell_content(content, add_empty=False, empty_val=None):
+    """Given shell like syntax (key=value\nkey2=value2\n) in content
+       return the data in dictionary form.  If 'add_empty' is True
+       then add entries in to the returned dictionary for 'VAR='
+       variables.  Set their value to empty_val."""
+    data = {}
+    for line in _shlex_split(content):
+        key, value = line.split("=", 1)
+        if not value:
+            value = empty_val
+        if add_empty or value:
+            data[key] = value
+
+    return data
+
+
+def _klibc_to_config_entry(content, mac_addrs=None):
+    """Convert a klibc writtent shell content file to a 'config' entry
+    When ip= is seen on the kernel command line in debian initramfs
+    and networking is brought up, ipconfig will populate
+    /run/net-<name>.cfg.
+
+    The files are shell style syntax, and examples are in the tests
+    provided here.  There is no good documentation on this unfortunately.
+
+    DEVICE=<name> is expected/required and PROTO should indicate if
+    this is 'static' or 'dhcp'.
+    """
+
+    if mac_addrs is None:
+        mac_addrs = {}
+
+    data = _load_shell_content(content)
+    try:
+        name = data['DEVICE']
+    except KeyError:
+        raise ValueError("no 'DEVICE' entry in data")
+
+    # ipconfig on precise does not write PROTO
+    proto = data.get('PROTO')
+    if not proto:
+        if data.get('filename'):
+            proto = 'dhcp'
+        else:
+            proto = 'static'
+
+    if proto not in ('static', 'dhcp'):
+        raise ValueError("Unexpected value for PROTO: %s" % proto)
+
+    iface = {
+        'type': 'physical',
+        'name': name,
+        'subnets': [],
+    }
+
+    if name in mac_addrs:
+        iface['mac_address'] = mac_addrs[name]
+
+    # originally believed there might be IPV6* values
+    for v, pre in (('ipv4', 'IPV4'),):
+        # if no IPV4ADDR or IPV6ADDR, then go on.
+        if pre + "ADDR" not in data:
+            continue
+        subnet = {'type': proto, 'control': 'manual'}
+
+        # these fields go right on the subnet
+        for key in ('NETMASK', 'BROADCAST', 'GATEWAY'):
+            if pre + key in data:
+                subnet[key.lower()] = data[pre + key]
+
+        dns = []
+        # handle IPV4DNS0 or IPV6DNS0
+        for nskey in ('DNS0', 'DNS1'):
+            ns = data.get(pre + nskey)
+            # verify it has something other than 0.0.0.0 (or ipv6)
+            if ns and len(ns.strip(":.0")):
+                dns.append(data[pre + nskey])
+        if dns:
+            subnet['dns_nameservers'] = dns
+            # add search to both ipv4 and ipv6, as it has no namespace
+            search = data.get('DOMAINSEARCH')
+            if search:
+                if ',' in search:
+                    subnet['dns_search'] = search.split(",")
+                else:
+                    subnet['dns_search'] = search.split()
+
+        iface['subnets'].append(subnet)
+
+    return name, iface
+
+
+def config_from_klibc_net_cfg(files=None, mac_addrs=None):
+    if files is None:
+        files = glob.glob('/run/net*.conf')
+
+    entries = []
+    names = {}
+    for cfg_file in files:
+        name, entry = _klibc_to_config_entry(util.load_file(cfg_file),
+                                             mac_addrs=mac_addrs)
+        if name in names:
+            raise ValueError(
+                "device '%s' defined multiple times: %s and %s" % (
+                    name, names[name], cfg_file))
+
+        names[name] = cfg_file
+        entries.append(entry)
+    return {'config': entries, 'version': 1}
+
+
+def _decomp_gzip(blob, strict=True):
+    # decompress blob. raise exception if not compressed unless strict=False.
+    with io.BytesIO(blob) as iobuf:
+        gzfp = None
+        try:
+            gzfp = gzip.GzipFile(mode="rb", fileobj=iobuf)
+            return gzfp.read()
+        except IOError:
+            if strict:
+                raise
+            return blob
+        finally:
+            if gzfp:
+                gzfp.close()
+
+
+def _b64dgz(b64str, gzipped="try"):
+    # decode a base64 string.  If gzipped is true, transparently uncompresss
+    # if gzipped is 'try', then try gunzip, returning the original on fail.
+    try:
+        blob = base64.b64decode(b64str)
+    except TypeError:
+        raise ValueError("Invalid base64 text: %s" % b64str)
+
+    if not gzipped:
+        return blob
+
+    return _decomp_gzip(blob, strict=gzipped != "try")
+
+
+def read_kernel_cmdline_config(files=None, mac_addrs=None, cmdline=None):
+    if cmdline is None:
+        cmdline = util.get_cmdline()
+
+    if 'network-config=' in cmdline:
+        data64 = None
+        for tok in cmdline.split():
+            if tok.startswith("network-config="):
+                data64 = tok.split("=", 1)[1]
+        if data64:
+            return util.load_yaml(_b64dgz(data64))
+
+    if 'ip=' not in cmdline:
+        return None
+
+    if mac_addrs is None:
+        mac_addrs = dict((k, sys_netdev_info(k, 'address'))
+                         for k in get_devicelist())
+
+    return config_from_klibc_net_cfg(files=files, mac_addrs=mac_addrs)
diff --git a/cloudinit/net/eni.py b/cloudinit/net/eni.py
new file mode 100644
index 00000000..e5ed10fd
--- /dev/null
+++ b/cloudinit/net/eni.py
@@ -0,0 +1,450 @@
+# vi: ts=4 expandtab
+#
+#    This program is free software: you can redistribute it and/or modify
+#    it under the terms of the GNU General Public License version 3, as
+#    published by the Free Software Foundation.
+#
+#    This program is distributed in the hope that it will be useful,
+#    but WITHOUT ANY WARRANTY; without even the implied warranty of
+#    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+#    GNU General Public License for more details.
+#
+#    You should have received a copy of the GNU General Public License
+#    along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+import glob
+import os
+import re
+
+from . import ParserError
+
+from . import renderer
+
+from cloudinit import util
+
+
+NET_CONFIG_COMMANDS = [
+    "pre-up", "up", "post-up", "down", "pre-down", "post-down",
+]
+
+NET_CONFIG_BRIDGE_OPTIONS = [
+    "bridge_ageing", "bridge_bridgeprio", "bridge_fd", "bridge_gcinit",
+    "bridge_hello", "bridge_maxage", "bridge_maxwait", "bridge_stp",
+]
+
+NET_CONFIG_OPTIONS = [
+    "address", "netmask", "broadcast", "network", "metric", "gateway",
+    "pointtopoint", "media", "mtu", "hostname", "leasehours", "leasetime",
+    "vendor", "client", "bootfile", "server", "hwaddr", "provider", "frame",
+    "netnum", "endpoint", "local", "ttl",
+]
+
+
+# TODO: switch valid_map based on mode inet/inet6
+def _iface_add_subnet(iface, subnet):
+    content = ""
+    valid_map = [
+        'address',
+        'netmask',
+        'broadcast',
+        'metric',
+        'gateway',
+        'pointopoint',
+        'mtu',
+        'scope',
+        'dns_search',
+        'dns_nameservers',
+    ]
+    for key, value in subnet.items():
+        if value and key in valid_map:
+            if type(value) == list:
+                value = " ".join(value)
+            if '_' in key:
+                key = key.replace('_', '-')
+            content += "    {} {}\n".format(key, value)
+
+    return content
+
+
+# TODO: switch to valid_map for attrs
+
+def _iface_add_attrs(iface):
+    content = ""
+    ignore_map = [
+        'control',
+        'index',
+        'inet',
+        'mode',
+        'name',
+        'subnets',
+        'type',
+    ]
+    if iface['type'] not in ['bond', 'bridge', 'vlan']:
+        ignore_map.append('mac_address')
+
+    for key, value in iface.items():
+        if value and key not in ignore_map:
+            if type(value) == list:
+                value = " ".join(value)
+            content += "    {} {}\n".format(key, value)
+
+    return content
+
+
+def _iface_start_entry(iface, index):
+    fullname = iface['name']
+    if index != 0:
+        fullname += ":%s" % index
+
+    control = iface['control']
+    if control == "auto":
+        cverb = "auto"
+    elif control in ("hotplug",):
+        cverb = "allow-" + control
+    else:
+        cverb = "# control-" + control
+
+    subst = iface.copy()
+    subst.update({'fullname': fullname, 'cverb': cverb})
+
+    return ("{cverb} {fullname}\n"
+            "iface {fullname} {inet} {mode}\n").format(**subst)
+
+
+def _parse_deb_config_data(ifaces, contents, src_dir, src_path):
+    """Parses the file contents, placing result into ifaces.
+
+    '_source_path' is added to every dictionary entry to define which file
+    the configration information came from.
+
+    :param ifaces: interface dictionary
+    :param contents: contents of interfaces file
+    :param src_dir: directory interfaces file was located
+    :param src_path: file path the `contents` was read
+    """
+    currif = None
+    for line in contents.splitlines():
+        line = line.strip()
+        if line.startswith('#'):
+            continue
+        split = line.split(' ')
+        option = split[0]
+        if option == "source-directory":
+            parsed_src_dir = split[1]
+            if not parsed_src_dir.startswith("/"):
+                parsed_src_dir = os.path.join(src_dir, parsed_src_dir)
+            for expanded_path in glob.glob(parsed_src_dir):
+                dir_contents = os.listdir(expanded_path)
+                dir_contents = [
+                    os.path.join(expanded_path, path)
+                    for path in dir_contents
+                    if (os.path.isfile(os.path.join(expanded_path, path)) and
+                        re.match("^[a-zA-Z0-9_-]+$", path) is not None)
+                ]
+                for entry in dir_contents:
+                    with open(entry, "r") as fp:
+                        src_data = fp.read().strip()
+                    abs_entry = os.path.abspath(entry)
+                    _parse_deb_config_data(
+                        ifaces, src_data,
+                        os.path.dirname(abs_entry), abs_entry)
+        elif option == "source":
+            new_src_path = split[1]
+            if not new_src_path.startswith("/"):
+                new_src_path = os.path.join(src_dir, new_src_path)
+            for expanded_path in glob.glob(new_src_path):
+                with open(expanded_path, "r") as fp:
+                    src_data = fp.read().strip()
+                abs_path = os.path.abspath(expanded_path)
+                _parse_deb_config_data(
+                    ifaces, src_data,
+                    os.path.dirname(abs_path), abs_path)
+        elif option == "auto":
+            for iface in split[1:]:
+                if iface not in ifaces:
+                    ifaces[iface] = {
+                        # Include the source path this interface was found in.
+                        "_source_path": src_path
+                    }
+                ifaces[iface]['auto'] = True
+        elif option == "iface":
+            iface, family, method = split[1:4]
+            if iface not in ifaces:
+                ifaces[iface] = {
+                    # Include the source path this interface was found in.
+                    "_source_path": src_path
+                }
+            elif 'family' in ifaces[iface]:
+                raise ParserError(
+                    "Interface %s can only be defined once. "
+                    "Re-defined in '%s'." % (iface, src_path))
+            ifaces[iface]['family'] = family
+            ifaces[iface]['method'] = method
+            currif = iface
+        elif option == "hwaddress":
+            if split[1] == "ether":
+                val = split[2]
+            else:
+                val = split[1]
+            ifaces[currif]['hwaddress'] = val
+        elif option in NET_CONFIG_OPTIONS:
+            ifaces[currif][option] = split[1]
+        elif option in NET_CONFIG_COMMANDS:
+            if option not in ifaces[currif]:
+                ifaces[currif][option] = []
+            ifaces[currif][option].append(' '.join(split[1:]))
+        elif option.startswith('dns-'):
+            if 'dns' not in ifaces[currif]:
+                ifaces[currif]['dns'] = {}
+            if option == 'dns-search':
+                ifaces[currif]['dns']['search'] = []
+                for domain in split[1:]:
+                    ifaces[currif]['dns']['search'].append(domain)
+            elif option == 'dns-nameservers':
+                ifaces[currif]['dns']['nameservers'] = []
+                for server in split[1:]:
+                    ifaces[currif]['dns']['nameservers'].append(server)
+        elif option.startswith('bridge_'):
+            if 'bridge' not in ifaces[currif]:
+                ifaces[currif]['bridge'] = {}
+            if option in NET_CONFIG_BRIDGE_OPTIONS:
+                bridge_option = option.replace('bridge_', '', 1)
+                ifaces[currif]['bridge'][bridge_option] = split[1]
+            elif option == "bridge_ports":
+                ifaces[currif]['bridge']['ports'] = []
+                for iface in split[1:]:
+                    ifaces[currif]['bridge']['ports'].append(iface)
+            elif option == "bridge_hw" and split[1].lower() == "mac":
+                ifaces[currif]['bridge']['mac'] = split[2]
+            elif option == "bridge_pathcost":
+                if 'pathcost' not in ifaces[currif]['bridge']:
+                    ifaces[currif]['bridge']['pathcost'] = {}
+                ifaces[currif]['bridge']['pathcost'][split[1]] = split[2]
+            elif option == "bridge_portprio":
+                if 'portprio' not in ifaces[currif]['bridge']:
+                    ifaces[currif]['bridge']['portprio'] = {}
+                ifaces[currif]['bridge']['portprio'][split[1]] = split[2]
+        elif option.startswith('bond-'):
+            if 'bond' not in ifaces[currif]:
+                ifaces[currif]['bond'] = {}
+            bond_option = option.replace('bond-', '', 1)
+            ifaces[currif]['bond'][bond_option] = split[1]
+    for iface in ifaces.keys():
+        if 'auto' not in ifaces[iface]:
+            ifaces[iface]['auto'] = False
+
+
+def parse_deb_config(path):
+    """Parses a debian network configuration file."""
+    ifaces = {}
+    with open(path, "r") as fp:
+        contents = fp.read().strip()
+    abs_path = os.path.abspath(path)
+    _parse_deb_config_data(
+        ifaces, contents,
+        os.path.dirname(abs_path), abs_path)
+    return ifaces
+
+
+def convert_eni_data(eni_data):
+    # return a network config representation of what is in eni_data
+    ifaces = {}
+    _parse_deb_config_data(ifaces, eni_data, src_dir=None, src_path=None)
+    return _ifaces_to_net_config_data(ifaces)
+
+
+def _ifaces_to_net_config_data(ifaces):
+    """Return network config that represents the ifaces data provided.
+    ifaces = parse_deb_config("/etc/network/interfaces")
+    config = ifaces_to_net_config_data(ifaces)
+    state = parse_net_config_data(config)."""
+    devs = {}
+    for name, data in ifaces.items():
+        # devname is 'eth0' for name='eth0:1'
+        devname = name.partition(":")[0]
+        if devname == "lo":
+            # currently provding 'lo' in network config results in duplicate
+            # entries. in rendered interfaces file. so skip it.
+            continue
+        if devname not in devs:
+            devs[devname] = {'type': 'physical', 'name': devname,
+                             'subnets': []}
+            # this isnt strictly correct, but some might specify
+            # hwaddress on a nic for matching / declaring name.
+            if 'hwaddress' in data:
+                devs[devname]['mac_address'] = data['hwaddress']
+        subnet = {'_orig_eni_name': name, 'type': data['method']}
+        if data.get('auto'):
+            subnet['control'] = 'auto'
+        else:
+            subnet['control'] = 'manual'
+
+        if data.get('method') == 'static':
+            subnet['address'] = data['address']
+
+        for copy_key in ('netmask', 'gateway', 'broadcast'):
+            if copy_key in data:
+                subnet[copy_key] = data[copy_key]
+
+        if 'dns' in data:
+            for n in ('nameservers', 'search'):
+                if n in data['dns'] and data['dns'][n]:
+                    subnet['dns_' + n] = data['dns'][n]
+        devs[devname]['subnets'].append(subnet)
+
+    return {'version': 1,
+            'config': [devs[d] for d in sorted(devs)]}
+
+
+class Renderer(renderer.Renderer):
+    """Renders network information in a /etc/network/interfaces format."""
+
+    def __init__(self, config=None):
+        if not config:
+            config = {}
+        self.eni_path = config.get('eni_path', 'etc/network/interfaces')
+        self.eni_header = config.get('eni_header', None)
+        self.links_path_prefix = config.get(
+            'links_path_prefix', 'etc/systemd/network/50-cloud-init-')
+        self.netrules_path = config.get(
+            'netrules_path', 'etc/udev/rules.d/70-persistent-net.rules')
+
+    def _render_route(self, route, indent=""):
+        """When rendering routes for an iface, in some cases applying a route
+        may result in the route command returning non-zero which produces
+        some confusing output for users manually using ifup/ifdown[1].  To
+        that end, we will optionally include an '|| true' postfix to each
+        route line allowing users to work with ifup/ifdown without using
+        --force option.
+
+        We may at somepoint not want to emit this additional postfix, and
+        add a 'strict' flag to this function.  When called with strict=True,
+        then we will not append the postfix.
+
+        1. http://askubuntu.com/questions/168033/
+                 how-to-set-static-routes-in-ubuntu-server
+        """
+        content = ""
+        up = indent + "post-up route add"
+        down = indent + "pre-down route del"
+        eol = " || true\n"
+        mapping = {
+            'network': '-net',
+            'netmask': 'netmask',
+            'gateway': 'gw',
+            'metric': 'metric',
+        }
+        if route['network'] == '0.0.0.0' and route['netmask'] == '0.0.0.0':
+            default_gw = " default gw %s" % route['gateway']
+            content += up + default_gw + eol
+            content += down + default_gw + eol
+        elif route['network'] == '::' and route['netmask'] == 0:
+            # ipv6!
+            default_gw = " -A inet6 default gw %s" % route['gateway']
+            content += up + default_gw + eol
+            content += down + default_gw + eol
+        else:
+            route_line = ""
+            for k in ['network', 'netmask', 'gateway', 'metric']:
+                if k in route:
+                    route_line += " %s %s" % (mapping[k], route[k])
+            content += up + route_line + eol
+            content += down + route_line + eol
+        return content
+
+    def _render_interfaces(self, network_state):
+        '''Given state, emit etc/network/interfaces content.'''
+
+        content = ""
+        content += "auto lo\niface lo inet loopback\n"
+
+        nameservers = network_state.dns_nameservers
+        if nameservers:
+            content += "    dns-nameservers %s\n" % (" ".join(nameservers))
+        searchdomains = network_state.dns_searchdomains
+        if searchdomains:
+            content += "    dns-search %s\n" % (" ".join(searchdomains))
+
+        ''' Apply a sort order to ensure that we write out
+            the physical interfaces first; this is critical for
+            bonding
+        '''
+        order = {
+            'physical': 0,
+            'bond': 1,
+            'bridge': 2,
+            'vlan': 3,
+        }
+        for iface in sorted(network_state.iter_interfaces(),
+                            key=lambda k: (order[k['type']], k['name'])):
+
+            if content[-2:] != "\n\n":
+                content += "\n"
+            subnets = iface.get('subnets', {})
+            if subnets:
+                for index, subnet in zip(range(0, len(subnets)), subnets):
+                    if content[-2:] != "\n\n":
+                        content += "\n"
+                    iface['index'] = index
+                    iface['mode'] = subnet['type']
+                    iface['control'] = subnet.get('control', 'auto')
+                    if iface['mode'].endswith('6'):
+                        iface['inet'] += '6'
+                    elif (iface['mode'] == 'static' and
+                          ":" in subnet['address']):
+                        iface['inet'] += '6'
+                    if iface['mode'].startswith('dhcp'):
+                        iface['mode'] = 'dhcp'
+
+                    content += _iface_start_entry(iface, index)
+                    content += _iface_add_subnet(iface, subnet)
+                    content += _iface_add_attrs(iface)
+                    for route in subnet.get('routes', []):
+                        content += self._render_route(route, indent="    ")
+            else:
+                # ifenslave docs say to auto the slave devices
+                if 'bond-master' in iface:
+                    content += "auto {name}\n".format(**iface)
+                content += "iface {name} {inet} {mode}\n".format(**iface)
+                content += _iface_add_attrs(iface)
+
+        for route in network_state.iter_routes():
+            content += self._render_route(route)
+
+        # global replacements until v2 format
+        content = content.replace('mac_address', 'hwaddress')
+        return content
+
+    def render_network_state(self, target, network_state):
+        fpeni = os.path.join(target, self.eni_path)
+        util.ensure_dir(os.path.dirname(fpeni))
+        header = self.eni_header if self.eni_header else ""
+        util.write_file(fpeni, header + self._render_interfaces(network_state))
+
+        if self.netrules_path:
+            netrules = os.path.join(target, self.netrules_path)
+            util.ensure_dir(os.path.dirname(netrules))
+            util.write_file(netrules,
+                            self._render_persistent_net(network_state))
+
+        if self.links_path_prefix:
+            self._render_systemd_links(target, network_state,
+                                       links_prefix=self.links_path_prefix)
+
+    def _render_systemd_links(self, target, network_state, links_prefix):
+        fp_prefix = os.path.join(target, links_prefix)
+        for f in glob.glob(fp_prefix + "*"):
+            os.unlink(f)
+        for iface in network_state.iter_interfaces():
+            if (iface['type'] == 'physical' and 'name' in iface and
+                    iface.get('mac_address')):
+                fname = fp_prefix + iface['name'] + ".link"
+                content = "\n".join([
+                    "[Match]",
+                    "MACAddress=" + iface['mac_address'],
+                    "",
+                    "[Link]",
+                    "Name=" + iface['name'],
+                    ""
+                ])
+                util.write_file(fname, content)
diff --git a/cloudinit/net/network_state.py b/cloudinit/net/network_state.py
index e32d2cdf..8ca5106f 100644
--- a/cloudinit/net/network_state.py
+++ b/cloudinit/net/network_state.py
@@ -15,9 +15,13 @@
 #   You should have received a copy of the GNU Affero General Public License
 #   along with Curtin.  If not, see <http://www.gnu.org/licenses/>.
 
-from cloudinit import log as logging
+import copy
+import functools
+import logging
+
+import six
+
 from cloudinit import util
-from cloudinit.util import yaml_dumps as dump_config
 
 LOG = logging.getLogger(__name__)
 
@@ -27,80 +31,198 @@ NETWORK_STATE_REQUIRED_KEYS = {
 }
 
 
-def from_state_file(state_file):
-    network_state = None
-    state = util.read_conf(state_file)
-    network_state = NetworkState()
-    network_state.load(state)
+def parse_net_config_data(net_config, skip_broken=True):
+    """Parses the config, returns NetworkState object
 
-    return network_state
+    :param net_config: curtin network config dict
+    """
+    state = None
+    if 'version' in net_config and 'config' in net_config:
+        nsi = NetworkStateInterpreter(version=net_config.get('version'),
+                                      config=net_config.get('config'))
+        nsi.parse_config(skip_broken=skip_broken)
+        state = nsi.network_state
+    return state
 
 
-class NetworkState:
-    def __init__(self, version=NETWORK_STATE_VERSION, config=None):
-        self.version = version
-        self.config = config
-        self.network_state = {
-            'interfaces': {},
-            'routes': [],
-            'dns': {
-                'nameservers': [],
-                'search': [],
-            }
+def parse_net_config(path, skip_broken=True):
+    """Parses a curtin network configuration file and
+       return network state"""
+    ns = None
+    net_config = util.read_conf(path)
+    if 'network' in net_config:
+        ns = parse_net_config_data(net_config.get('network'),
+                                   skip_broken=skip_broken)
+    return ns
+
+
+def from_state_file(state_file):
+    state = util.read_conf(state_file)
+    nsi = NetworkStateInterpreter()
+    nsi.load(state)
+    return nsi
+
+
+def diff_keys(expected, actual):
+    missing = set(expected)
+    for key in actual:
+        missing.discard(key)
+    return missing
+
+
+class InvalidCommand(Exception):
+    pass
+
+
+def ensure_command_keys(required_keys):
+
+    def wrapper(func):
+
+        @functools.wraps(func)
+        def decorator(self, command, *args, **kwargs):
+            if required_keys:
+                missing_keys = diff_keys(required_keys, command)
+                if missing_keys:
+                    raise InvalidCommand("Command missing %s of required"
+                                         " keys %s" % (missing_keys,
+                                                       required_keys))
+            return func(self, command, *args, **kwargs)
+
+        return decorator
+
+    return wrapper
+
+
+class CommandHandlerMeta(type):
+    """Metaclass that dynamically creates a 'command_handlers' attribute.
+
+    This will scan the to-be-created class for methods that start with
+    'handle_' and on finding those will populate a class attribute mapping
+    so that those methods can be quickly located and called.
+    """
+    def __new__(cls, name, parents, dct):
+        command_handlers = {}
+        for attr_name, attr in dct.items():
+            if callable(attr) and attr_name.startswith('handle_'):
+                handles_what = attr_name[len('handle_'):]
+                if handles_what:
+                    command_handlers[handles_what] = attr
+        dct['command_handlers'] = command_handlers
+        return super(CommandHandlerMeta, cls).__new__(cls, name,
+                                                      parents, dct)
+
+
+class NetworkState(object):
+
+    def __init__(self, network_state, version=NETWORK_STATE_VERSION):
+        self._network_state = copy.deepcopy(network_state)
+        self._version = version
+
+    @property
+    def version(self):
+        return self._version
+
+    def iter_routes(self, filter_func=None):
+        for route in self._network_state.get('routes', []):
+            if filter_func is not None:
+                if filter_func(route):
+                    yield route
+            else:
+                yield route
+
+    @property
+    def dns_nameservers(self):
+        try:
+            return self._network_state['dns']['nameservers']
+        except KeyError:
+            return []
+
+    @property
+    def dns_searchdomains(self):
+        try:
+            return self._network_state['dns']['search']
+        except KeyError:
+            return []
+
+    def iter_interfaces(self, filter_func=None):
+        ifaces = self._network_state.get('interfaces', {})
+        for iface in six.itervalues(ifaces):
+            if filter_func is None:
+                yield iface
+            else:
+                if filter_func(iface):
+                    yield iface
+
+
+@six.add_metaclass(CommandHandlerMeta)
+class NetworkStateInterpreter(object):
+
+    initial_network_state = {
+        'interfaces': {},
+        'routes': [],
+        'dns': {
+            'nameservers': [],
+            'search': [],
         }
-        self.command_handlers = self.get_command_handlers()
+    }
 
-    def get_command_handlers(self):
-        METHOD_PREFIX = 'handle_'
-        methods = filter(lambda x: callable(getattr(self, x)) and
-                         x.startswith(METHOD_PREFIX),  dir(self))
-        handlers = {}
-        for m in methods:
-            key = m.replace(METHOD_PREFIX, '')
-            handlers[key] = getattr(self, m)
+    def __init__(self, version=NETWORK_STATE_VERSION, config=None):
+        self._version = version
+        self._config = config
+        self._network_state = copy.deepcopy(self.initial_network_state)
+        self._parsed = False
 
-        return handlers
+    @property
+    def network_state(self):
+        return NetworkState(self._network_state, version=self._version)
 
     def dump(self):
         state = {
-            'version': self.version,
-            'config': self.config,
-            'network_state': self.network_state,
+            'version': self._version,
+            'config': self._config,
+            'network_state': self._network_state,
         }
-        return dump_config(state)
+        return util.yaml_dumps(state)
 
     def load(self, state):
         if 'version' not in state:
             LOG.error('Invalid state, missing version field')
-            raise Exception('Invalid state, missing version field')
+            raise ValueError('Invalid state, missing version field')
 
         required_keys = NETWORK_STATE_REQUIRED_KEYS[state['version']]
-        if not self.valid_command(state, required_keys):
-            msg = 'Invalid state, missing keys: {}'.format(required_keys)
+        missing_keys = diff_keys(required_keys, state)
+        if missing_keys:
+            msg = 'Invalid state, missing keys: %s' % (missing_keys)
             LOG.error(msg)
-            raise Exception(msg)
+            raise ValueError(msg)
 
         # v1 - direct attr mapping, except version
         for key in [k for k in required_keys if k not in ['version']]:
             setattr(self, key, state[key])
-        self.command_handlers = self.get_command_handlers()
 
     def dump_network_state(self):
-        return dump_config(self.network_state)
+        return util.yaml_dumps(self._network_state)
 
-    def parse_config(self):
+    def parse_config(self, skip_broken=True):
         # rebuild network state
-        for command in self.config:
-            handler = self.command_handlers.get(command['type'])
-            handler(command)
-
-    def valid_command(self, command, required_keys):
-        if not required_keys:
-            return False
-
-        found_keys = [key for key in command.keys() if key in required_keys]
-        return len(found_keys) == len(required_keys)
-
+        for command in self._config:
+            command_type = command['type']
+            try:
+                handler = self.command_handlers[command_type]
+            except KeyError:
+                raise RuntimeError("No handler found for"
+                                   " command '%s'" % command_type)
+            try:
+                handler(self, command)
+            except InvalidCommand:
+                if not skip_broken:
+                    raise
+                else:
+                    LOG.warn("Skipping invalid command: %s", command,
+                             exc_info=True)
+                    LOG.debug(self.dump_network_state())
+
+    @ensure_command_keys(['name'])
     def handle_physical(self, command):
         '''
         command = {
@@ -112,15 +234,8 @@ class NetworkState:
              ]
         }
         '''
-        required_keys = [
-            'name',
-        ]
-        if not self.valid_command(command, required_keys):
-            LOG.warn('Skipping Invalid command: {}'.format(command))
-            LOG.debug(self.dump_network_state())
-            return
 
-        interfaces = self.network_state.get('interfaces')
+        interfaces = self._network_state.get('interfaces', {})
         iface = interfaces.get(command['name'], {})
         for param, val in command.get('params', {}).items():
             iface.update({param: val})
@@ -146,9 +261,10 @@ class NetworkState:
             'gateway': None,
             'subnets': subnets,
         })
-        self.network_state['interfaces'].update({command.get('name'): iface})
+        self._network_state['interfaces'].update({command.get('name'): iface})
         self.dump_network_state()
 
+    @ensure_command_keys(['name', 'vlan_id', 'vlan_link'])
     def handle_vlan(self, command):
         '''
             auto eth0.222
@@ -158,23 +274,14 @@ class NetworkState:
                     hwaddress ether BC:76:4E:06:96:B3
                     vlan-raw-device eth0
         '''
-        required_keys = [
-            'name',
-            'vlan_link',
-            'vlan_id',
-        ]
-        if not self.valid_command(command, required_keys):
-            print('Skipping Invalid command: {}'.format(command))
-            print(self.dump_network_state())
-            return
-
-        interfaces = self.network_state.get('interfaces')
+        interfaces = self._network_state.get('interfaces', {})
         self.handle_physical(command)
         iface = interfaces.get(command.get('name'), {})
         iface['vlan-raw-device'] = command.get('vlan_link')
         iface['vlan_id'] = command.get('vlan_id')
         interfaces.update({iface['name']: iface})
 
+    @ensure_command_keys(['name', 'bond_interfaces', 'params'])
     def handle_bond(self, command):
         '''
     #/etc/network/interfaces
@@ -200,23 +307,14 @@ class NetworkState:
          bond-updelay 200
          bond-lacp-rate 4
         '''
-        required_keys = [
-            'name',
-            'bond_interfaces',
-            'params',
-        ]
-        if not self.valid_command(command, required_keys):
-            print('Skipping Invalid command: {}'.format(command))
-            print(self.dump_network_state())
-            return
 
         self.handle_physical(command)
-        interfaces = self.network_state.get('interfaces')
+        interfaces = self._network_state.get('interfaces')
         iface = interfaces.get(command.get('name'), {})
         for param, val in command.get('params').items():
             iface.update({param: val})
         iface.update({'bond-slaves': 'none'})
-        self.network_state['interfaces'].update({iface['name']: iface})
+        self._network_state['interfaces'].update({iface['name']: iface})
 
         # handle bond slaves
         for ifname in command.get('bond_interfaces'):
@@ -228,14 +326,15 @@ class NetworkState:
                 # inject placeholder
                 self.handle_physical(cmd)
 
-            interfaces = self.network_state.get('interfaces')
+            interfaces = self._network_state.get('interfaces', {})
             bond_if = interfaces.get(ifname)
             bond_if['bond-master'] = command.get('name')
             # copy in bond config into slave
             for param, val in command.get('params').items():
                 bond_if.update({param: val})
-            self.network_state['interfaces'].update({ifname: bond_if})
+            self._network_state['interfaces'].update({ifname: bond_if})
 
+    @ensure_command_keys(['name', 'bridge_interfaces', 'params'])
     def handle_bridge(self, command):
         '''
             auto br0
@@ -263,19 +362,10 @@ class NetworkState:
             "bridge_waitport",
         ]
         '''
-        required_keys = [
-            'name',
-            'bridge_interfaces',
-            'params',
-        ]
-        if not self.valid_command(command, required_keys):
-            print('Skipping Invalid command: {}'.format(command))
-            print(self.dump_network_state())
-            return
 
         # find one of the bridge port ifaces to get mac_addr
         # handle bridge_slaves
-        interfaces = self.network_state.get('interfaces')
+        interfaces = self._network_state.get('interfaces', {})
         for ifname in command.get('bridge_interfaces'):
             if ifname in interfaces:
                 continue
@@ -286,7 +376,7 @@ class NetworkState:
             # inject placeholder
             self.handle_physical(cmd)
 
-        interfaces = self.network_state.get('interfaces')
+        interfaces = self._network_state.get('interfaces', {})
         self.handle_physical(command)
         iface = interfaces.get(command.get('name'), {})
         iface['bridge_ports'] = command['bridge_interfaces']
@@ -295,16 +385,9 @@ class NetworkState:
 
         interfaces.update({iface['name']: iface})
 
+    @ensure_command_keys(['address'])
     def handle_nameserver(self, command):
-        required_keys = [
-            'address',
-        ]
-        if not self.valid_command(command, required_keys):
-            print('Skipping Invalid command: {}'.format(command))
-            print(self.dump_network_state())
-            return
-
-        dns = self.network_state.get('dns')
+        dns = self._network_state.get('dns')
         if 'address' in command:
             addrs = command['address']
             if not type(addrs) == list:
@@ -318,16 +401,9 @@ class NetworkState:
             for path in paths:
                 dns['search'].append(path)
 
+    @ensure_command_keys(['destination'])
     def handle_route(self, command):
-        required_keys = [
-            'destination',
-        ]
-        if not self.valid_command(command, required_keys):
-            print('Skipping Invalid command: {}'.format(command))
-            print(self.dump_network_state())
-            return
-
-        routes = self.network_state.get('routes')
+        routes = self._network_state.get('routes', [])
         network, cidr = command['destination'].split("/")
         netmask = cidr2mask(int(cidr))
         route = {
@@ -376,71 +452,3 @@ def mask2cidr(mask):
         return ipv4mask2cidr(mask)
     else:
         return mask
-
-
-if __name__ == '__main__':
-    import sys
-    import random
-    from cloudinit import net
-
-    def load_config(nc):
-        version = nc.get('version')
-        config = nc.get('config')
-        return (version, config)
-
-    def test_parse(network_config):
-        (version, config) = load_config(network_config)
-        ns1 = NetworkState(version=version, config=config)
-        ns1.parse_config()
-        random.shuffle(config)
-        ns2 = NetworkState(version=version, config=config)
-        ns2.parse_config()
-        print("----NS1-----")
-        print(ns1.dump_network_state())
-        print()
-        print("----NS2-----")
-        print(ns2.dump_network_state())
-        print("NS1 == NS2 ?=> {}".format(
-            ns1.network_state == ns2.network_state))
-        eni = net.render_interfaces(ns2.network_state)
-        print(eni)
-        udev_rules = net.render_persistent_net(ns2.network_state)
-        print(udev_rules)
-
-    def test_dump_and_load(network_config):
-        print("Loading network_config into NetworkState")
-        (version, config) = load_config(network_config)
-        ns1 = NetworkState(version=version, config=config)
-        ns1.parse_config()
-        print("Dumping state to file")
-        ns1_dump = ns1.dump()
-        ns1_state = "/tmp/ns1.state"
-        with open(ns1_state, "w+") as f:
-            f.write(ns1_dump)
-
-        print("Loading state from file")
-        ns2 = from_state_file(ns1_state)
-        print("NS1 == NS2 ?=> {}".format(
-            ns1.network_state == ns2.network_state))
-
-    def test_output(network_config):
-        (version, config) = load_config(network_config)
-        ns1 = NetworkState(version=version, config=config)
-        ns1.parse_config()
-        random.shuffle(config)
-        ns2 = NetworkState(version=version, config=config)
-        ns2.parse_config()
-        print("NS1 == NS2 ?=> {}".format(
-            ns1.network_state == ns2.network_state))
-        eni_1 = net.render_interfaces(ns1.network_state)
-        eni_2 = net.render_interfaces(ns2.network_state)
-        print(eni_1)
-        print(eni_2)
-        print("eni_1 == eni_2 ?=> {}".format(
-            eni_1 == eni_2))
-
-    y = util.read_conf(sys.argv[1])
-    network_config = y.get('network')
-    test_parse(network_config)
-    test_dump_and_load(network_config)
-    test_output(network_config)
diff --git a/cloudinit/net/renderer.py b/cloudinit/net/renderer.py
new file mode 100644
index 00000000..310cbe0d
--- /dev/null
+++ b/cloudinit/net/renderer.py
@@ -0,0 +1,48 @@
+#   Copyright (C) 2013-2014 Canonical Ltd.
+#
+#   Author: Scott Moser <scott.moser@canonical.com>
+#   Author: Blake Rouse <blake.rouse@canonical.com>
+#
+#   Curtin is free software: you can redistribute it and/or modify it under
+#   the terms of the GNU Affero General Public License as published by the
+#   Free Software Foundation, either version 3 of the License, or (at your
+#   option) any later version.
+#
+#   Curtin is distributed in the hope that it will be useful, but WITHOUT ANY
+#   WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
+#   FOR A PARTICULAR PURPOSE.  See the GNU Affero General Public License for
+#   more details.
+#
+#   You should have received a copy of the GNU Affero General Public License
+#   along with Curtin.  If not, see <http://www.gnu.org/licenses/>.
+
+import six
+
+from .udev import generate_udev_rule
+
+
+def filter_by_type(match_type):
+    return lambda iface: match_type == iface['type']
+
+
+def filter_by_name(match_name):
+    return lambda iface: match_name == iface['name']
+
+
+filter_by_physical = filter_by_type('physical')
+
+
+class Renderer(object):
+
+    @staticmethod
+    def _render_persistent_net(network_state):
+        """Given state, emit udev rules to map mac to ifname."""
+        # TODO(harlowja): this seems shared between eni renderer and
+        # this, so move it to a shared location.
+        content = six.StringIO()
+        for iface in network_state.iter_interfaces(filter_by_physical):
+            # for physical interfaces write out a persist net udev rule
+            if 'name' in iface and iface.get('mac_address'):
+                content.write(generate_udev_rule(iface['name'],
+                                                 iface['mac_address']))
+        return content.getvalue()
diff --git a/cloudinit/net/sysconfig.py b/cloudinit/net/sysconfig.py
new file mode 100644
index 00000000..c53acf71
--- /dev/null
+++ b/cloudinit/net/sysconfig.py
@@ -0,0 +1,400 @@
+# vi: ts=4 expandtab
+#
+#    This program is free software: you can redistribute it and/or modify
+#    it under the terms of the GNU General Public License version 3, as
+#    published by the Free Software Foundation.
+#
+#    This program is distributed in the hope that it will be useful,
+#    but WITHOUT ANY WARRANTY; without even the implied warranty of
+#    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+#    GNU General Public License for more details.
+#
+#    You should have received a copy of the GNU General Public License
+#    along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+import os
+import re
+
+import six
+
+from cloudinit.distros.parsers import resolv_conf
+from cloudinit import util
+
+from . import renderer
+
+
+def _make_header(sep='#'):
+    lines = [
+        "Created by cloud-init on instance boot automatically, do not edit.",
+        "",
+    ]
+    for i in range(0, len(lines)):
+        if lines[i]:
+            lines[i] = sep + " " + lines[i]
+        else:
+            lines[i] = sep
+    return "\n".join(lines)
+
+
+def _is_default_route(route):
+    if route['network'] == '::' and route['netmask'] == 0:
+        return True
+    if route['network'] == '0.0.0.0' and route['netmask'] == '0.0.0.0':
+        return True
+    return False
+
+
+def _quote_value(value):
+    if re.search(r"\s", value):
+        # This doesn't handle complex cases...
+        if value.startswith('"') and value.endswith('"'):
+            return value
+        else:
+            return '"%s"' % value
+    else:
+        return value
+
+
+class ConfigMap(object):
+    """Sysconfig like dictionary object."""
+
+    # Why does redhat prefer yes/no to true/false??
+    _bool_map = {
+        True: 'yes',
+        False: 'no',
+    }
+
+    def __init__(self):
+        self._conf = {}
+
+    def __setitem__(self, key, value):
+        self._conf[key] = value
+
+    def drop(self, key):
+        self._conf.pop(key, None)
+
+    def __len__(self):
+        return len(self._conf)
+
+    def to_string(self):
+        buf = six.StringIO()
+        buf.write(_make_header())
+        if self._conf:
+            buf.write("\n")
+        for key in sorted(self._conf.keys()):
+            value = self._conf[key]
+            if isinstance(value, bool):
+                value = self._bool_map[value]
+            if not isinstance(value, six.string_types):
+                value = str(value)
+            buf.write("%s=%s\n" % (key, _quote_value(value)))
+        return buf.getvalue()
+
+
+class Route(ConfigMap):
+    """Represents a route configuration."""
+
+    route_fn_tpl = '%(base)s/network-scripts/route-%(name)s'
+
+    def __init__(self, route_name, base_sysconf_dir):
+        super(Route, self).__init__()
+        self.last_idx = 1
+        self.has_set_default = False
+        self._route_name = route_name
+        self._base_sysconf_dir = base_sysconf_dir
+
+    def copy(self):
+        r = Route(self._route_name, self._base_sysconf_dir)
+        r._conf = self._conf.copy()
+        r.last_idx = self.last_idx
+        r.has_set_default = self.has_set_default
+        return r
+
+    @property
+    def path(self):
+        return self.route_fn_tpl % ({'base': self._base_sysconf_dir,
+                                     'name': self._route_name})
+
+
+class NetInterface(ConfigMap):
+    """Represents a sysconfig/networking-script (and its config + children)."""
+
+    iface_fn_tpl = '%(base)s/network-scripts/ifcfg-%(name)s'
+
+    iface_types = {
+        'ethernet': 'Ethernet',
+        'bond': 'Bond',
+        'bridge': 'Bridge',
+    }
+
+    def __init__(self, iface_name, base_sysconf_dir, kind='ethernet'):
+        super(NetInterface, self).__init__()
+        self.children = []
+        self.routes = Route(iface_name, base_sysconf_dir)
+        self._kind = kind
+        self._iface_name = iface_name
+        self._conf['DEVICE'] = iface_name
+        self._conf['TYPE'] = self.iface_types[kind]
+        self._base_sysconf_dir = base_sysconf_dir
+
+    @property
+    def name(self):
+        return self._iface_name
+
+    @name.setter
+    def name(self, iface_name):
+        self._iface_name = iface_name
+        self._conf['DEVICE'] = iface_name
+
+    @property
+    def kind(self):
+        return self._kind
+
+    @kind.setter
+    def kind(self, kind):
+        self._kind = kind
+        self._conf['TYPE'] = self.iface_types[kind]
+
+    @property
+    def path(self):
+        return self.iface_fn_tpl % ({'base': self._base_sysconf_dir,
+                                     'name': self.name})
+
+    def copy(self, copy_children=False, copy_routes=False):
+        c = NetInterface(self.name, self._base_sysconf_dir, kind=self._kind)
+        c._conf = self._conf.copy()
+        if copy_children:
+            c.children = list(self.children)
+        if copy_routes:
+            c.routes = self.routes.copy()
+        return c
+
+
+class Renderer(renderer.Renderer):
+    """Renders network information in a /etc/sysconfig format."""
+
+    # See: https://access.redhat.com/documentation/en-US/\
+    #      Red_Hat_Enterprise_Linux/6/html/Deployment_Guide/\
+    #      s1-networkscripts-interfaces.html (or other docs for
+    #                                         details about this)
+
+    iface_defaults = tuple([
+        ('ONBOOT', True),
+        ('USERCTL', False),
+        ('NM_CONTROLLED', False),
+        ('BOOTPROTO', 'none'),
+    ])
+
+    # If these keys exist, then there values will be used to form
+    # a BONDING_OPTS grouping; otherwise no grouping will be set.
+    bond_tpl_opts = tuple([
+        ('bond_mode', "mode=%s"),
+        ('bond_xmit_hash_policy', "xmit_hash_policy=%s"),
+        ('bond_miimon', "miimon=%s"),
+    ])
+
+    bridge_opts_keys = tuple([
+        ('bridge_stp', 'STP'),
+        ('bridge_ageing', 'AGEING'),
+        ('bridge_bridgeprio', 'PRIO'),
+    ])
+
+    def __init__(self, config=None):
+        if not config:
+            config = {}
+        self.sysconf_dir = config.get('sysconf_dir', 'etc/sysconfig/')
+        self.netrules_path = config.get(
+            'netrules_path', 'etc/udev/rules.d/70-persistent-net.rules')
+        self.dns_path = config.get('dns_path', 'etc/resolv.conf')
+
+    @classmethod
+    def _render_iface_shared(cls, iface, iface_cfg):
+        for k, v in cls.iface_defaults:
+            iface_cfg[k] = v
+        for (old_key, new_key) in [('mac_address', 'HWADDR'), ('mtu', 'MTU')]:
+            old_value = iface.get(old_key)
+            if old_value is not None:
+                iface_cfg[new_key] = old_value
+
+    @classmethod
+    def _render_subnet(cls, iface_cfg, route_cfg, subnet):
+        subnet_type = subnet.get('type')
+        if subnet_type == 'dhcp6':
+            iface_cfg['DHCPV6C'] = True
+            iface_cfg['IPV6INIT'] = True
+            iface_cfg['BOOTPROTO'] = 'dhcp'
+        elif subnet_type in ['dhcp4', 'dhcp']:
+            iface_cfg['BOOTPROTO'] = 'dhcp'
+        elif subnet_type == 'static':
+            iface_cfg['BOOTPROTO'] = 'static'
+            if subnet.get('ipv6'):
+                iface_cfg['IPV6ADDR'] = subnet['address']
+                iface_cfg['IPV6INIT'] = True
+            else:
+                iface_cfg['IPADDR'] = subnet['address']
+        else:
+            raise ValueError("Unknown subnet type '%s' found"
+                             " for interface '%s'" % (subnet_type,
+                                                      iface_cfg.name))
+        if 'netmask' in subnet:
+            iface_cfg['NETMASK'] = subnet['netmask']
+        for route in subnet.get('routes', []):
+            if _is_default_route(route):
+                if route_cfg.has_set_default:
+                    raise ValueError("Duplicate declaration of default"
+                                     " route found for interface '%s'"
+                                     % (iface_cfg.name))
+                # NOTE(harlowja): ipv6 and ipv4 default gateways
+                gw_key = 'GATEWAY0'
+                nm_key = 'NETMASK0'
+                addr_key = 'ADDRESS0'
+                # The owning interface provides the default route.
+                #
+                # TODO(harlowja): add validation that no other iface has
+                # also provided the default route?
+                iface_cfg['DEFROUTE'] = True
+                if 'gateway' in route:
+                    iface_cfg['GATEWAY'] = route['gateway']
+                route_cfg.has_set_default = True
+            else:
+                gw_key = 'GATEWAY%s' % route_cfg.last_idx
+                nm_key = 'NETMASK%s' % route_cfg.last_idx
+                addr_key = 'ADDRESS%s' % route_cfg.last_idx
+                route_cfg.last_idx += 1
+            for (old_key, new_key) in [('gateway', gw_key),
+                                       ('netmask', nm_key),
+                                       ('network', addr_key)]:
+                if old_key in route:
+                    route_cfg[new_key] = route[old_key]
+
+    @classmethod
+    def _render_bonding_opts(cls, iface_cfg, iface):
+        bond_opts = []
+        for (bond_key, value_tpl) in cls.bond_tpl_opts:
+            # Seems like either dash or underscore is possible?
+            bond_keys = [bond_key, bond_key.replace("_", "-")]
+            for bond_key in bond_keys:
+                if bond_key in iface:
+                    bond_value = iface[bond_key]
+                    if isinstance(bond_value, (tuple, list)):
+                        bond_value = " ".join(bond_value)
+                    bond_opts.append(value_tpl % (bond_value))
+                    break
+        if bond_opts:
+            iface_cfg['BONDING_OPTS'] = " ".join(bond_opts)
+
+    @classmethod
+    def _render_physical_interfaces(cls, network_state, iface_contents):
+        physical_filter = renderer.filter_by_physical
+        for iface in network_state.iter_interfaces(physical_filter):
+            iface_name = iface['name']
+            iface_subnets = iface.get("subnets", [])
+            iface_cfg = iface_contents[iface_name]
+            route_cfg = iface_cfg.routes
+            if len(iface_subnets) == 1:
+                cls._render_subnet(iface_cfg, route_cfg, iface_subnets[0])
+            elif len(iface_subnets) > 1:
+                for i, iface_subnet in enumerate(iface_subnets,
+                                                 start=len(iface.children)):
+                    iface_sub_cfg = iface_cfg.copy()
+                    iface_sub_cfg.name = "%s:%s" % (iface_name, i)
+                    iface.children.append(iface_sub_cfg)
+                    cls._render_subnet(iface_sub_cfg, route_cfg, iface_subnet)
+
+    @classmethod
+    def _render_bond_interfaces(cls, network_state, iface_contents):
+        bond_filter = renderer.filter_by_type('bond')
+        for iface in network_state.iter_interfaces(bond_filter):
+            iface_name = iface['name']
+            iface_cfg = iface_contents[iface_name]
+            cls._render_bonding_opts(iface_cfg, iface)
+            iface_master_name = iface['bond-master']
+            iface_cfg['MASTER'] = iface_master_name
+            iface_cfg['SLAVE'] = True
+            # Ensure that the master interface (and any of its children)
+            # are actually marked as being bond types...
+            master_cfg = iface_contents[iface_master_name]
+            master_cfgs = [master_cfg]
+            master_cfgs.extend(master_cfg.children)
+            for master_cfg in master_cfgs:
+                master_cfg['BONDING_MASTER'] = True
+                master_cfg.kind = 'bond'
+
+    @staticmethod
+    def _render_vlan_interfaces(network_state, iface_contents):
+        vlan_filter = renderer.filter_by_type('vlan')
+        for iface in network_state.iter_interfaces(vlan_filter):
+            iface_name = iface['name']
+            iface_cfg = iface_contents[iface_name]
+            iface_cfg['VLAN'] = True
+            iface_cfg['PHYSDEV'] = iface_name[:iface_name.rfind('.')]
+
+    @staticmethod
+    def _render_dns(network_state, existing_dns_path=None):
+        content = resolv_conf.ResolvConf("")
+        if existing_dns_path and os.path.isfile(existing_dns_path):
+            content = resolv_conf.ResolvConf(util.load_file(existing_dns_path))
+        for nameserver in network_state.dns_nameservers:
+            content.add_nameserver(nameserver)
+        for searchdomain in network_state.dns_searchdomains:
+            content.add_search_domain(searchdomain)
+        return "\n".join([_make_header(';'), str(content)])
+
+    @classmethod
+    def _render_bridge_interfaces(cls, network_state, iface_contents):
+        bridge_filter = renderer.filter_by_type('bridge')
+        for iface in network_state.iter_interfaces(bridge_filter):
+            iface_name = iface['name']
+            iface_cfg = iface_contents[iface_name]
+            iface_cfg.kind = 'bridge'
+            for old_key, new_key in cls.bridge_opts_keys:
+                if old_key in iface:
+                    iface_cfg[new_key] = iface[old_key]
+            # Is this the right key to get all the connected interfaces?
+            for bridged_iface_name in iface.get('bridge_ports', []):
+                # Ensure all bridged interfaces are correctly tagged
+                # as being bridged to this interface.
+                bridged_cfg = iface_contents[bridged_iface_name]
+                bridged_cfgs = [bridged_cfg]
+                bridged_cfgs.extend(bridged_cfg.children)
+                for bridge_cfg in bridged_cfgs:
+                    bridge_cfg['BRIDGE'] = iface_name
+
+    @classmethod
+    def _render_sysconfig(cls, base_sysconf_dir, network_state):
+        '''Given state, return /etc/sysconfig files + contents'''
+        iface_contents = {}
+        for iface in network_state.iter_interfaces():
+            iface_name = iface['name']
+            iface_cfg = NetInterface(iface_name, base_sysconf_dir)
+            cls._render_iface_shared(iface, iface_cfg)
+            iface_contents[iface_name] = iface_cfg
+        cls._render_physical_interfaces(network_state, iface_contents)
+        cls._render_bond_interfaces(network_state, iface_contents)
+        cls._render_vlan_interfaces(network_state, iface_contents)
+        cls._render_bridge_interfaces(network_state, iface_contents)
+        contents = {}
+        for iface_name, iface_cfg in iface_contents.items():
+            if iface_cfg or iface_cfg.children:
+                contents[iface_cfg.path] = iface_cfg.to_string()
+                for iface_cfg in iface_cfg.children:
+                    if iface_cfg:
+                        contents[iface_cfg.path] = iface_cfg.to_string()
+            if iface_cfg.routes:
+                contents[iface_cfg.routes.path] = iface_cfg.routes.to_string()
+        return contents
+
+    def render_network_state(self, target, network_state):
+        base_sysconf_dir = os.path.join(target, self.sysconf_dir)
+        for path, data in self._render_sysconfig(base_sysconf_dir,
+                                                 network_state).items():
+            util.write_file(path, data)
+        if self.dns_path:
+            dns_path = os.path.join(target, self.dns_path)
+            resolv_content = self._render_dns(network_state,
+                                              existing_dns_path=dns_path)
+            util.write_file(dns_path, resolv_content)
+        if self.netrules_path:
+            netrules_content = self._render_persistent_net(network_state)
+            netrules_path = os.path.join(target, self.netrules_path)
+            util.write_file(netrules_path, netrules_content)
diff --git a/cloudinit/net/udev.py b/cloudinit/net/udev.py
index 6435ace0..09188295 100644
--- a/cloudinit/net/udev.py
+++ b/cloudinit/net/udev.py
@@ -48,7 +48,7 @@ def generate_udev_rule(interface, mac):
         compose_udev_equality('DRIVERS', '?*'),
         compose_udev_attr_equality('address', mac),
         compose_udev_setting('NAME', interface),
-        ])
+    ])
     return '%s\n' % rule
 
 # vi: ts=4 expandtab syntax=python
diff --git a/cloudinit/netinfo.py b/cloudinit/netinfo.py
index e30d6fb5..d8698a5d 100644
--- a/cloudinit/netinfo.py
+++ b/cloudinit/netinfo.py
@@ -20,10 +20,11 @@
 #    You should have received a copy of the GNU General Public License
 #    along with this program.  If not, see <http://www.gnu.org/licenses/>.
 
-import cloudinit.util as util
-from cloudinit.log import logging
 import re
 
+from cloudinit import log as logging
+from cloudinit import util
+
 from prettytable import PrettyTable
 
 LOG = logging.getLogger()
@@ -163,7 +164,7 @@ def route_info():
 def getgateway():
     try:
         routes = route_info()
-    except:
+    except Exception:
         pass
     else:
         for r in routes.get('ipv4', []):
diff --git a/cloudinit/reporting/events.py b/cloudinit/reporting/events.py
index 2f767f64..df2b9b4a 100644
--- a/cloudinit/reporting/events.py
+++ b/cloudinit/reporting/events.py
@@ -33,11 +33,13 @@ class ReportingEvent(object):
     """Encapsulation of event formatting."""
 
     def __init__(self, event_type, name, description,
-                 origin=DEFAULT_EVENT_ORIGIN, timestamp=time.time()):
+                 origin=DEFAULT_EVENT_ORIGIN, timestamp=None):
         self.event_type = event_type
         self.name = name
         self.description = description
         self.origin = origin
+        if timestamp is None:
+            timestamp = time.time()
         self.timestamp = timestamp
 
     def as_string(self):
diff --git a/cloudinit/reporting/handlers.py b/cloudinit/reporting/handlers.py
index 3212d173..dff20ecb 100644
--- a/cloudinit/reporting/handlers.py
+++ b/cloudinit/reporting/handlers.py
@@ -4,9 +4,9 @@ import abc
 import json
 import six
 
-from ..registry import DictRegistry
-from .. import (url_helper, util)
-from .. import log as logging
+from cloudinit import log as logging
+from cloudinit.registry import DictRegistry
+from cloudinit import (url_helper, util)
 
 
 LOG = logging.getLogger(__name__)
@@ -36,7 +36,7 @@ class LogHandler(ReportingHandler):
             input_level = level
             try:
                 level = getattr(logging, level.upper())
-            except:
+            except Exception:
                 LOG.warn("invalid level '%s', using WARN", input_level)
                 level = logging.WARN
         self.level = level
@@ -81,7 +81,7 @@ class WebHookHandler(ReportingHandler):
                 self.endpoint, data=json.dumps(event.as_dict()),
                 timeout=self.timeout,
                 retries=self.retries, ssl_details=self.ssl_details)
-        except:
+        except Exception:
             LOG.warn("failed posting event: %s" % event.as_string())
 
 
diff --git a/cloudinit/serial.py b/cloudinit/serial.py
new file mode 100644
index 00000000..af45c13e
--- /dev/null
+++ b/cloudinit/serial.py
@@ -0,0 +1,50 @@
+# vi: ts=4 expandtab
+#
+#    This program is free software: you can redistribute it and/or modify
+#    it under the terms of the GNU General Public License version 3, as
+#    published by the Free Software Foundation.
+#
+#    This program is distributed in the hope that it will be useful,
+#    but WITHOUT ANY WARRANTY; without even the implied warranty of
+#    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+#    GNU General Public License for more details.
+#
+#    You should have received a copy of the GNU General Public License
+#    along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+
+from __future__ import absolute_import
+
+try:
+    from serial import Serial
+except ImportError:
+    # For older versions of python (ie 2.6) pyserial may not exist and/or
+    # work and/or be installed, so make a dummy/fake serial that blows up
+    # when used...
+    class Serial(object):
+        def __init__(self, *args, **kwargs):
+            pass
+
+        @staticmethod
+        def isOpen():
+            return False
+
+        @staticmethod
+        def write(data):
+            raise IOError("Unable to perform serial `write` operation,"
+                          " pyserial not installed.")
+
+        @staticmethod
+        def readline():
+            raise IOError("Unable to perform serial `readline` operation,"
+                          " pyserial not installed.")
+
+        @staticmethod
+        def flush():
+            raise IOError("Unable to perform serial `flush` operation,"
+                          " pyserial not installed.")
+
+        @staticmethod
+        def read(size=1):
+            raise IOError("Unable to perform serial `read` operation,"
+                          " pyserial not installed.")
diff --git a/cloudinit/sources/DataSourceAltCloud.py b/cloudinit/sources/DataSourceAltCloud.py
index cd61df31..a3529609 100644
--- a/cloudinit/sources/DataSourceAltCloud.py
+++ b/cloudinit/sources/DataSourceAltCloud.py
@@ -205,8 +205,7 @@ class DataSourceAltCloud(sources.DataSource):
                         _err.message)
             return False
         except OSError as _err:
-            util.logexc(LOG, 'Failed command: %s\n%s', ' '.join(cmd),
-                        _err.message)
+            util.logexc(LOG, 'Failed command: %s\n%s', ' '.join(cmd), _err)
             return False
 
         floppy_dev = '/dev/fd0'
diff --git a/cloudinit/sources/DataSourceAzure.py b/cloudinit/sources/DataSourceAzure.py
index 698f4cac..8c7e8673 100644
--- a/cloudinit/sources/DataSourceAzure.py
+++ b/cloudinit/sources/DataSourceAzure.py
@@ -27,11 +27,12 @@ import xml.etree.ElementTree as ET
 
 from xml.dom import minidom
 
+from cloudinit.sources.helpers.azure import get_metadata_from_fabric
+
 from cloudinit import log as logging
 from cloudinit.settings import PER_ALWAYS
 from cloudinit import sources
 from cloudinit import util
-from cloudinit.sources.helpers.azure import get_metadata_from_fabric
 
 LOG = logging.getLogger(__name__)
 
@@ -40,7 +41,8 @@ DEFAULT_METADATA = {"instance-id": "iid-AZURE-NODE"}
 AGENT_START = ['service', 'walinuxagent', 'start']
 BOUNCE_COMMAND = [
     'sh', '-xc',
-    "i=$interface; x=0; ifdown $i || x=$?; ifup $i || x=$?; exit $x"]
+    "i=$interface; x=0; ifdown $i || x=$?; ifup $i || x=$?; exit $x"
+]
 
 BUILTIN_DS_CONFIG = {
     'agent_command': AGENT_START,
@@ -51,7 +53,7 @@ BUILTIN_DS_CONFIG = {
         'policy': True,
         'command': BOUNCE_COMMAND,
         'hostname_command': 'hostname',
-        },
+    },
     'disk_aliases': {'ephemeral0': '/dev/sdb'},
 }
 
@@ -60,7 +62,7 @@ BUILTIN_CLOUD_CONFIG = {
         'ephemeral0': {'table_type': 'gpt',
                        'layout': [100],
                        'overwrite': True},
-        },
+    },
     'fs_setup': [{'filesystem': 'ext4',
                   'device': 'ephemeral0.1',
                   'replace_fs': 'ntfs'}],
@@ -312,7 +314,7 @@ def support_new_ephemeral(cfg):
     file_count = 0
     try:
         file_count = util.mount_cb(device, count_files)
-    except:
+    except Exception:
         return None
     LOG.debug("fabric prepared ephmeral0.1 has %s files on it", file_count)
 
@@ -421,7 +423,7 @@ def write_files(datadir, files, dirmode=None):
                     elem.text = DEF_PASSWD_REDACTION
             return ET.tostring(root)
         except Exception:
-            LOG.critical("failed to redact userpassword in {}".format(fname))
+            LOG.critical("failed to redact userpassword in %s", fname)
             return cnt
 
     if not datadir:
diff --git a/cloudinit/sources/DataSourceBigstep.py b/cloudinit/sources/DataSourceBigstep.py
index b5ee4129..f80956a5 100644
--- a/cloudinit/sources/DataSourceBigstep.py
+++ b/cloudinit/sources/DataSourceBigstep.py
@@ -4,13 +4,13 @@
 #    Author: Alexandru Sirbu <alexandru.sirbu@bigstep.com>
 #
 
-import json
 import errno
+import json
 
 from cloudinit import log as logging
 from cloudinit import sources
-from cloudinit import util
 from cloudinit import url_helper
+from cloudinit import util
 
 LOG = logging.getLogger(__name__)
 
diff --git a/cloudinit/sources/DataSourceCloudSigma.py b/cloudinit/sources/DataSourceCloudSigma.py
index d7d4e844..d1f806d6 100644
--- a/cloudinit/sources/DataSourceCloudSigma.py
+++ b/cloudinit/sources/DataSourceCloudSigma.py
@@ -19,15 +19,14 @@ from base64 import b64decode
 import os
 import re
 
+from cloudinit.cs_utils import Cepko
+
 from cloudinit import log as logging
 from cloudinit import sources
 from cloudinit import util
-from cloudinit.cs_utils import Cepko
 
 LOG = logging.getLogger(__name__)
 
-VALID_DSMODES = ("local", "net", "disabled")
-
 
 class DataSourceCloudSigma(sources.DataSource):
     """
@@ -37,7 +36,6 @@ class DataSourceCloudSigma(sources.DataSource):
     http://cloudsigma-docs.readthedocs.org/en/latest/server_context.html
     """
     def __init__(self, sys_cfg, distro, paths):
-        self.dsmode = 'local'
         self.cepko = Cepko()
         self.ssh_public_key = ''
         sources.DataSource.__init__(self, sys_cfg, distro, paths)
@@ -77,17 +75,15 @@ class DataSourceCloudSigma(sources.DataSource):
         try:
             server_context = self.cepko.all().result
             server_meta = server_context['meta']
-        except:
+        except Exception:
             # TODO: check for explicit "config on", and then warn
             # but since no explicit config is available now, just debug.
             LOG.debug("CloudSigma: Unable to read from serial port")
             return False
 
-        dsmode = server_meta.get('cloudinit-dsmode', self.dsmode)
-        if dsmode not in VALID_DSMODES:
-            LOG.warn("Invalid dsmode %s, assuming default of 'net'", dsmode)
-            dsmode = 'net'
-        if dsmode == "disabled" or dsmode != self.dsmode:
+        self.dsmode = self._determine_dsmode(
+            [server_meta.get('cloudinit-dsmode')])
+        if dsmode == sources.DSMODE_DISABLED:
             return False
 
         base64_fields = server_meta.get('base64_fields', '').split(',')
@@ -119,17 +115,13 @@ class DataSourceCloudSigma(sources.DataSource):
         return self.metadata['uuid']
 
 
-class DataSourceCloudSigmaNet(DataSourceCloudSigma):
-    def __init__(self, sys_cfg, distro, paths):
-        DataSourceCloudSigma.__init__(self, sys_cfg, distro, paths)
-        self.dsmode = 'net'
-
+# Legacy: Must be present in case we load an old pkl object
+DataSourceCloudSigmaNet = DataSourceCloudSigma
 
 # Used to match classes to dependencies. Since this datasource uses the serial
 # port network is not really required, so it's okay to load without it, too.
 datasources = [
     (DataSourceCloudSigma, (sources.DEP_FILESYSTEM)),
-    (DataSourceCloudSigmaNet, (sources.DEP_FILESYSTEM, sources.DEP_NETWORK)),
 ]
 
 
diff --git a/cloudinit/sources/DataSourceCloudStack.py b/cloudinit/sources/DataSourceCloudStack.py
index 455a4652..4de1f563 100644
--- a/cloudinit/sources/DataSourceCloudStack.py
+++ b/cloudinit/sources/DataSourceCloudStack.py
@@ -25,14 +25,15 @@
 #    along with this program.  If not, see <http://www.gnu.org/licenses/>.
 
 import os
-import time
 from socket import inet_ntoa
 from struct import pack
+import time
 
 from cloudinit import ec2_utils as ec2
 from cloudinit import log as logging
+from cloudinit import sources
 from cloudinit import url_helper as uhelp
-from cloudinit import sources, util
+from cloudinit import util
 
 LOG = logging.getLogger(__name__)
 
@@ -206,7 +207,8 @@ def get_latest_lease():
     latest_mtime = -1
     latest_file = None
     for file_name in lease_files:
-        if file_name.endswith(".lease") or file_name.endswith(".leases"):
+        if file_name.startswith("dhclient.") and \
+           (file_name.endswith(".lease") or file_name.endswith(".leases")):
             abs_path = os.path.join(lease_d, file_name)
             mtime = os.path.getmtime(abs_path)
             if mtime > latest_mtime:
diff --git a/cloudinit/sources/DataSourceConfigDrive.py b/cloudinit/sources/DataSourceConfigDrive.py
index 3fa62ef3..3130e618 100644
--- a/cloudinit/sources/DataSourceConfigDrive.py
+++ b/cloudinit/sources/DataSourceConfigDrive.py
@@ -18,13 +18,14 @@
 #    You should have received a copy of the GNU General Public License
 #    along with this program.  If not, see <http://www.gnu.org/licenses/>.
 
-import copy
 import os
 
 from cloudinit import log as logging
 from cloudinit import sources
 from cloudinit import util
 
+from cloudinit.net import eni
+
 from cloudinit.sources.helpers import openstack
 
 LOG = logging.getLogger(__name__)
@@ -35,7 +36,6 @@ DEFAULT_MODE = 'pass'
 DEFAULT_METADATA = {
     "instance-id": DEFAULT_IID,
 }
-VALID_DSMODES = ("local", "net", "pass", "disabled")
 FS_TYPES = ('vfat', 'iso9660')
 LABEL_TYPES = ('config-2',)
 POSSIBLE_MOUNTS = ('sr', 'cd')
@@ -47,12 +47,13 @@ class DataSourceConfigDrive(openstack.SourceMixin, sources.DataSource):
     def __init__(self, sys_cfg, distro, paths):
         super(DataSourceConfigDrive, self).__init__(sys_cfg, distro, paths)
         self.source = None
-        self.dsmode = 'local'
         self.seed_dir = os.path.join(paths.seed_dir, 'config_drive')
         self.version = None
         self.ec2_metadata = None
         self._network_config = None
         self.network_json = None
+        self.network_eni = None
+        self.known_macs = None
         self.files = {}
 
     def __str__(self):
@@ -98,38 +99,22 @@ class DataSourceConfigDrive(openstack.SourceMixin, sources.DataSource):
 
         md = results.get('metadata', {})
         md = util.mergemanydict([md, DEFAULT_METADATA])
-        user_dsmode = results.get('dsmode', None)
-        if user_dsmode not in VALID_DSMODES + (None,):
-            LOG.warn("User specified invalid mode: %s", user_dsmode)
-            user_dsmode = None
 
-        dsmode = get_ds_mode(cfgdrv_ver=results['version'],
-                             ds_cfg=self.ds_cfg.get('dsmode'),
-                             user=user_dsmode)
+        self.dsmode = self._determine_dsmode(
+            [results.get('dsmode'), self.ds_cfg.get('dsmode'),
+             sources.DSMODE_PASS if results['version'] == 1 else None])
 
-        if dsmode == "disabled":
-            # most likely user specified
+        if self.dsmode == sources.DSMODE_DISABLED:
             return False
 
-        # TODO(smoser): fix this, its dirty.
-        # we want to do some things (writing files and network config)
-        # only on first boot, and even then, we want to do so in the
-        # local datasource (so they happen earlier) even if the configured
-        # dsmode is 'net' or 'pass'. To do this, we check the previous
-        # instance-id
+        # This is legacy and sneaky.  If dsmode is 'pass' then write
+        # 'injected files' and apply legacy ENI network format.
         prev_iid = get_previous_iid(self.paths)
         cur_iid = md['instance-id']
-        if prev_iid != cur_iid and self.dsmode == "local":
+        if prev_iid != cur_iid and self.dsmode == sources.DSMODE_PASS:
             on_first_boot(results, distro=self.distro)
-
-        # dsmode != self.dsmode here if:
-        #  * dsmode = "pass",  pass means it should only copy files and then
-        #    pass to another datasource
-        #  * dsmode = "net" and self.dsmode = "local"
-        #    so that user boothooks would be applied with network, the
-        #    local datasource just gets out of the way, and lets the net claim
-        if dsmode != self.dsmode:
-            LOG.debug("%s: not claiming datasource, dsmode=%s", self, dsmode)
+            LOG.debug("%s: not claiming datasource, dsmode=%s", self,
+                      self.dsmode)
             return False
 
         self.source = found
@@ -147,15 +132,14 @@ class DataSourceConfigDrive(openstack.SourceMixin, sources.DataSource):
             LOG.warn("Invalid content in vendor-data: %s", e)
             self.vendordata_raw = None
 
-        try:
-            self.network_json = results.get('networkdata')
-        except ValueError as e:
-            LOG.warn("Invalid content in network-data: %s", e)
-            self.network_json = None
-
+        # network_config is an /etc/network/interfaces formated file and is
+        # obsolete compared to networkdata (from network_data.json) but both
+        # might be present.
+        self.network_eni = results.get("network_config")
+        self.network_json = results.get('networkdata')
         return True
 
-    def check_instance_id(self):
+    def check_instance_id(self, sys_cfg):
         # quickly (local check only) if self.instance_id is still valid
         return sources.instance_id_matches_system_uuid(self.get_instance_id())
 
@@ -163,41 +147,17 @@ class DataSourceConfigDrive(openstack.SourceMixin, sources.DataSource):
     def network_config(self):
         if self._network_config is None:
             if self.network_json is not None:
-                self._network_config = convert_network_data(self.network_json)
+                LOG.debug("network config provided via network_json")
+                self._network_config = openstack.convert_net_json(
+                    self.network_json, known_macs=self.known_macs)
+            elif self.network_eni is not None:
+                self._network_config = eni.convert_eni_data(self.network_eni)
+                LOG.debug("network config provided via converted eni data")
+            else:
+                LOG.debug("no network configuration available")
         return self._network_config
 
 
-class DataSourceConfigDriveNet(DataSourceConfigDrive):
-    def __init__(self, sys_cfg, distro, paths):
-        DataSourceConfigDrive.__init__(self, sys_cfg, distro, paths)
-        self.dsmode = 'net'
-
-
-def get_ds_mode(cfgdrv_ver, ds_cfg=None, user=None):
-    """Determine what mode should be used.
-    valid values are 'pass', 'disabled', 'local', 'net'
-    """
-    # user passed data trumps everything
-    if user is not None:
-        return user
-
-    if ds_cfg is not None:
-        return ds_cfg
-
-    # at config-drive version 1, the default behavior was pass.  That
-    # meant to not use use it as primary data source, but expect a ec2 metadata
-    # source. for version 2, we default to 'net', which means
-    # the DataSourceConfigDriveNet, would be used.
-    #
-    # this could change in the future.  If there was definitive metadata
-    # that indicated presense of an openstack metadata service, then
-    # we could change to 'pass' by default also. The motivation for that
-    # would be 'cloud-init query' as the web service could be more dynamic
-    if cfgdrv_ver == 1:
-        return "pass"
-    return "net"
-
-
 def read_config_drive(source_dir):
     reader = openstack.ConfigDriveReader(source_dir)
     finders = [
@@ -231,9 +191,12 @@ def on_first_boot(data, distro=None):
                         % (type(data)))
     net_conf = data.get("network_config", '')
     if net_conf and distro:
-        LOG.debug("Updating network interfaces from config drive")
+        LOG.warn("Updating network interfaces from config drive")
         distro.apply_network(net_conf)
-    files = data.get('files', {})
+    write_injected_files(data.get('files'))
+
+
+def write_injected_files(files):
     if files:
         LOG.debug("Writing %s injected files", len(files))
         for (filename, content) in files.items():
@@ -293,132 +256,15 @@ def find_candidate_devs(probe_optical=True):
     return devices
 
 
+# Legacy: Must be present in case we load an old pkl object
+DataSourceConfigDriveNet = DataSourceConfigDrive
+
 # Used to match classes to dependencies
 datasources = [
-    (DataSourceConfigDrive, (sources.DEP_FILESYSTEM, )),
-    (DataSourceConfigDriveNet, (sources.DEP_FILESYSTEM, sources.DEP_NETWORK)),
+    (DataSourceConfigDrive, (sources.DEP_FILESYSTEM,)),
 ]
 
 
 # Return a list of data sources that match this set of dependencies
 def get_datasource_list(depends):
     return sources.list_from_depends(depends, datasources)
-
-
-# Convert OpenStack ConfigDrive NetworkData json to network_config yaml
-def convert_network_data(network_json=None):
-    """Return a dictionary of network_config by parsing provided
-       OpenStack ConfigDrive NetworkData json format
-
-    OpenStack network_data.json provides a 3 element dictionary
-      - "links" (links are network devices, physical or virtual)
-      - "networks" (networks are ip network configurations for one or more
-                    links)
-      -  services (non-ip services, like dns)
-
-    networks and links are combined via network items referencing specific
-    links via a 'link_id' which maps to a links 'id' field.
-
-    To convert this format to network_config yaml, we first iterate over the
-    links and then walk the network list to determine if any of the networks
-    utilize the current link; if so we generate a subnet entry for the device
-
-    We also need to map network_data.json fields to network_config fields. For
-    example, the network_data links 'id' field is equivalent to network_config
-    'name' field for devices.  We apply more of this mapping to the various
-    link types that we encounter.
-
-    There are additional fields that are populated in the network_data.json
-    from OpenStack that are not relevant to network_config yaml, so we
-    enumerate a dictionary of valid keys for network_yaml and apply filtering
-    to drop these superflous keys from the network_config yaml.
-    """
-    if network_json is None:
-        return None
-
-    # dict of network_config key for filtering network_json
-    valid_keys = {
-        'physical': [
-            'name',
-            'type',
-            'mac_address',
-            'subnets',
-            'params',
-        ],
-        'subnet': [
-            'type',
-            'address',
-            'netmask',
-            'broadcast',
-            'metric',
-            'gateway',
-            'pointopoint',
-            'mtu',
-            'scope',
-            'dns_nameservers',
-            'dns_search',
-            'routes',
-        ],
-    }
-
-    links = network_json.get('links', [])
-    networks = network_json.get('networks', [])
-    services = network_json.get('services', [])
-
-    config = []
-    for link in links:
-        subnets = []
-        cfg = {k: v for k, v in link.items()
-               if k in valid_keys['physical']}
-        cfg.update({'name': link['id']})
-        for network in [net for net in networks
-                        if net['link'] == link['id']]:
-            subnet = {k: v for k, v in network.items()
-                      if k in valid_keys['subnet']}
-            if 'dhcp' in network['type']:
-                t = 'dhcp6' if network['type'].startswith('ipv6') else 'dhcp4'
-                subnet.update({
-                    'type': t,
-                })
-            else:
-                subnet.update({
-                    'type': 'static',
-                    'address': network.get('ip_address'),
-                })
-            subnets.append(subnet)
-        cfg.update({'subnets': subnets})
-        if link['type'] in ['ethernet', 'vif', 'ovs', 'phy']:
-            cfg.update({
-                'type': 'physical',
-                'mac_address': link['ethernet_mac_address']})
-        elif link['type'] in ['bond']:
-            params = {}
-            for k, v in link.items():
-                if k == 'bond_links':
-                    continue
-                elif k.startswith('bond'):
-                    params.update({k: v})
-            cfg.update({
-                'bond_interfaces': copy.deepcopy(link['bond_links']),
-                'params': params,
-            })
-        elif link['type'] in ['vlan']:
-            cfg.update({
-                'name': "%s.%s" % (link['vlan_link'],
-                                   link['vlan_id']),
-                'vlan_link': link['vlan_link'],
-                'vlan_id': link['vlan_id'],
-                'mac_address': link['vlan_mac_address'],
-            })
-        else:
-            raise ValueError(
-                'Unknown network_data link type: %s' % link['type'])
-
-        config.append(cfg)
-
-    for service in services:
-        cfg = service
-        cfg.update({'type': 'nameserver'})
-        config.append(cfg)
-
-    return {'version': 1, 'config': config}
diff --git a/cloudinit/sources/DataSourceDigitalOcean.py b/cloudinit/sources/DataSourceDigitalOcean.py
index 12e863d2..44a17a00 100644
--- a/cloudinit/sources/DataSourceDigitalOcean.py
+++ b/cloudinit/sources/DataSourceDigitalOcean.py
@@ -14,10 +14,10 @@
 #    You should have received a copy of the GNU General Public License
 #    along with this program.  If not, see <http://www.gnu.org/licenses/>.
 
+from cloudinit import ec2_utils
 from cloudinit import log as logging
-from cloudinit import util
 from cloudinit import sources
-from cloudinit import ec2_utils
+from cloudinit import util
 
 import functools
 
diff --git a/cloudinit/sources/DataSourceGCE.py b/cloudinit/sources/DataSourceGCE.py
index 7e7fc033..c660a350 100644
--- a/cloudinit/sources/DataSourceGCE.py
+++ b/cloudinit/sources/DataSourceGCE.py
@@ -18,14 +18,14 @@
 from base64 import b64decode
 
 from cloudinit import log as logging
-from cloudinit import util
 from cloudinit import sources
 from cloudinit import url_helper
+from cloudinit import util
 
 LOG = logging.getLogger(__name__)
 
 BUILTIN_DS_CONFIG = {
-    'metadata_url': 'http://metadata.google.internal./computeMetadata/v1/'
+    'metadata_url': 'http://metadata.google.internal/computeMetadata/v1/'
 }
 REQUIRED_FIELDS = ('instance-id', 'availability-zone', 'local-hostname')
 
@@ -71,7 +71,7 @@ class DataSourceGCE(sources.DataSource):
             index = public_key.index(':')
             if index > 0:
                 return public_key[(index + 1):]
-        except:
+        except Exception:
             return public_key
 
     def get_data(self):
diff --git a/cloudinit/sources/DataSourceNoCloud.py b/cloudinit/sources/DataSourceNoCloud.py
index 74d0e5ec..cdc9eef5 100644
--- a/cloudinit/sources/DataSourceNoCloud.py
+++ b/cloudinit/sources/DataSourceNoCloud.py
@@ -24,6 +24,7 @@ import errno
 import os
 
 from cloudinit import log as logging
+from cloudinit.net import eni
 from cloudinit import sources
 from cloudinit import util
 
@@ -33,9 +34,7 @@ LOG = logging.getLogger(__name__)
 class DataSourceNoCloud(sources.DataSource):
     def __init__(self, sys_cfg, distro, paths):
         sources.DataSource.__init__(self, sys_cfg, distro, paths)
-        self.dsmode = 'local'
         self.seed = None
-        self.cmdline_id = "ds=nocloud"
         self.seed_dirs = [os.path.join(paths.seed_dir, 'nocloud'),
                           os.path.join(paths.seed_dir, 'nocloud-net')]
         self.seed_dir = None
@@ -58,10 +57,10 @@ class DataSourceNoCloud(sources.DataSource):
         try:
             # Parse the kernel command line, getting data passed in
             md = {}
-            if parse_cmdline_data(self.cmdline_id, md):
+            if load_cmdline_data(md):
                 found.append("cmdline")
                 mydata = _merge_new_seed(mydata, {'meta-data': md})
-        except:
+        except Exception:
             util.logexc(LOG, "Unable to parse command line data")
             return False
 
@@ -123,12 +122,6 @@ class DataSourceNoCloud(sources.DataSource):
 
                     mydata = _merge_new_seed(mydata, seeded)
 
-                    # For seed from a device, the default mode is 'net'.
-                    # that is more likely to be what is desired.  If they want
-                    # dsmode of local, then they must specify that.
-                    if 'dsmode' not in mydata['meta-data']:
-                        mydata['meta-data']['dsmode'] = "net"
-
                     LOG.debug("Using data from %s", dev)
                     found.append(dev)
                     break
@@ -144,7 +137,6 @@ class DataSourceNoCloud(sources.DataSource):
         if len(found) == 0:
             return False
 
-        seeded_network = None
         # The special argument "seedfrom" indicates we should
         # attempt to seed the userdata / metadata from its value
         # its primarily value is in allowing the user to type less
@@ -160,10 +152,6 @@ class DataSourceNoCloud(sources.DataSource):
                 LOG.debug("Seed from %s not supported by %s", seedfrom, self)
                 return False
 
-            if (mydata['meta-data'].get('network-interfaces') or
-                    mydata.get('network-config')):
-                seeded_network = self.dsmode
-
             # This could throw errors, but the user told us to do it
             # so if errors are raised, let them raise
             (md_seed, ud) = util.read_seeded(seedfrom, timeout=None)
@@ -179,35 +167,21 @@ class DataSourceNoCloud(sources.DataSource):
         mydata['meta-data'] = util.mergemanydict([mydata['meta-data'],
                                                   defaults])
 
-        netdata = {'format': None, 'data': None}
-        if mydata['meta-data'].get('network-interfaces'):
-            netdata['format'] = 'interfaces'
-            netdata['data'] = mydata['meta-data']['network-interfaces']
-        elif mydata.get('network-config'):
-            netdata['format'] = 'network-config'
-            netdata['data'] = mydata['network-config']
-
-        # if this is the local datasource or 'seedfrom' was used
-        # and the source of the seed was self.dsmode.
-        # Then see if there is network config to apply.
-        # note this is obsolete network-interfaces style seeding.
-        if self.dsmode in ("local", seeded_network):
-            if mydata['meta-data'].get('network-interfaces'):
-                LOG.debug("Updating network interfaces from %s", self)
-                self.distro.apply_network(
-                    mydata['meta-data']['network-interfaces'])
-
-        if mydata['meta-data']['dsmode'] == self.dsmode:
-            self.seed = ",".join(found)
-            self.metadata = mydata['meta-data']
-            self.userdata_raw = mydata['user-data']
-            self.vendordata_raw = mydata['vendor-data']
-            self._network_config = mydata['network-config']
-            return True
+        self.dsmode = self._determine_dsmode(
+            [mydata['meta-data'].get('dsmode')])
 
-        LOG.debug("%s: not claiming datasource, dsmode=%s", self,
-                  mydata['meta-data']['dsmode'])
-        return False
+        if self.dsmode == sources.DSMODE_DISABLED:
+            LOG.debug("%s: not claiming datasource, dsmode=%s", self,
+                      self.dsmode)
+            return False
+
+        self.seed = ",".join(found)
+        self.metadata = mydata['meta-data']
+        self.userdata_raw = mydata['user-data']
+        self.vendordata_raw = mydata['vendor-data']
+        self._network_config = mydata['network-config']
+        self._network_eni = mydata['meta-data'].get('network-interfaces')
+        return True
 
     def check_instance_id(self, sys_cfg):
         # quickly (local check only) if self.instance_id is still valid
@@ -219,26 +193,27 @@ class DataSourceNoCloud(sources.DataSource):
         # LP: #1568150 need getattr in the case that an old class object
         # has been loaded from a pickled file and now executing new source.
         dirs = getattr(self, 'seed_dirs', [self.seed_dir])
-        quick_id = _quick_read_instance_id(cmdline_id=self.cmdline_id,
-                                           dirs=dirs)
+        quick_id = _quick_read_instance_id(dirs=dirs)
         if not quick_id:
             return None
         return quick_id == current
 
     @property
     def network_config(self):
+        if self._network_config is None:
+            if self._network_eni is not None:
+                self._network_config = eni.convert_eni_data(self._network_eni)
         return self._network_config
 
 
-def _quick_read_instance_id(cmdline_id, dirs=None):
+def _quick_read_instance_id(dirs=None):
     if dirs is None:
         dirs = []
 
     iid_key = 'instance-id'
-    if cmdline_id is None:
-        fill = {}
-        if parse_cmdline_data(cmdline_id, fill) and iid_key in fill:
-            return fill[iid_key]
+    fill = {}
+    if load_cmdline_data(fill) and iid_key in fill:
+        return fill[iid_key]
 
     for d in dirs:
         if d is None:
@@ -254,8 +229,22 @@ def _quick_read_instance_id(cmdline_id, dirs=None):
     return None
 
 
+def load_cmdline_data(fill, cmdline=None):
+    pairs = [("ds=nocloud", sources.DSMODE_LOCAL),
+             ("ds=nocloud-net", sources.DSMODE_NETWORK)]
+    for idstr, dsmode in pairs:
+        if parse_cmdline_data(idstr, fill, cmdline):
+            # if dsmode was explicitly in the commanad line, then
+            # prefer it to the dsmode based on the command line id
+            if 'dsmode' not in fill:
+                fill['dsmode'] = dsmode
+            return True
+    return False
+
+
 # Returns true or false indicating if cmdline indicated
-# that this module should be used
+# that this module should be used.  Updates dictionary 'fill'
+# with data that was found.
 # Example cmdline:
 #  root=LABEL=uec-rootfs ro ds=nocloud
 def parse_cmdline_data(ds_id, fill, cmdline=None):
@@ -288,7 +277,7 @@ def parse_cmdline_data(ds_id, fill, cmdline=None):
             continue
         try:
             (k, v) = item.split("=", 1)
-        except:
+        except Exception:
             k = item
             v = None
         if k in s2l:
@@ -319,9 +308,7 @@ def _merge_new_seed(cur, seeded):
 class DataSourceNoCloudNet(DataSourceNoCloud):
     def __init__(self, sys_cfg, distro, paths):
         DataSourceNoCloud.__init__(self, sys_cfg, distro, paths)
-        self.cmdline_id = "ds=nocloud-net"
         self.supported_seed_starts = ("http://", "https://", "ftp://")
-        self.dsmode = "net"
 
 
 # Used to match classes to dependencies
diff --git a/cloudinit/sources/DataSourceOVF.py b/cloudinit/sources/DataSourceOVF.py
index 2a6cd050..43347cfb 100644
--- a/cloudinit/sources/DataSourceOVF.py
+++ b/cloudinit/sources/DataSourceOVF.py
@@ -30,16 +30,23 @@ import time
 from cloudinit import log as logging
 from cloudinit import sources
 from cloudinit import util
-from .helpers.vmware.imc.config import Config
-from .helpers.vmware.imc.config_file import ConfigFile
-from .helpers.vmware.imc.config_nic import NicConfigurator
-from .helpers.vmware.imc.guestcust_event import GuestCustEventEnum
-from .helpers.vmware.imc.guestcust_state import GuestCustStateEnum
-from .helpers.vmware.imc.guestcust_error import GuestCustErrorEnum
-from .helpers.vmware.imc.guestcust_util import (
-    set_customization_status,
+
+from cloudinit.sources.helpers.vmware.imc.config \
+    import Config
+from cloudinit.sources.helpers.vmware.imc.config_file \
+    import ConfigFile
+from cloudinit.sources.helpers.vmware.imc.config_nic \
+    import NicConfigurator
+from cloudinit.sources.helpers.vmware.imc.guestcust_error \
+    import GuestCustErrorEnum
+from cloudinit.sources.helpers.vmware.imc.guestcust_event \
+    import GuestCustEventEnum
+from cloudinit.sources.helpers.vmware.imc.guestcust_state \
+    import GuestCustStateEnum
+from cloudinit.sources.helpers.vmware.imc.guestcust_util import (
+    enable_nics,
     get_nics_to_enable,
-    enable_nics
+    set_customization_status
 )
 
 LOG = logging.getLogger(__name__)
@@ -262,7 +269,7 @@ def read_ovf_environment(contents):
         elif prop == "user-data":
             try:
                 ud = base64.decodestring(val)
-            except:
+            except Exception:
                 ud = val
     return (md, ud, cfg)
 
@@ -277,7 +284,7 @@ def get_ovf_env(dirname):
             try:
                 contents = util.load_file(full_fn)
                 return (fname, contents)
-            except:
+            except Exception:
                 util.logexc(LOG, "Failed loading ovf file %s", full_fn)
     return (None, False)
 
diff --git a/cloudinit/sources/DataSourceOpenNebula.py b/cloudinit/sources/DataSourceOpenNebula.py
index 681f3a96..7b3a76b9 100644
--- a/cloudinit/sources/DataSourceOpenNebula.py
+++ b/cloudinit/sources/DataSourceOpenNebula.py
@@ -37,16 +37,13 @@ from cloudinit import util
 LOG = logging.getLogger(__name__)
 
 DEFAULT_IID = "iid-dsopennebula"
-DEFAULT_MODE = 'net'
 DEFAULT_PARSEUSER = 'nobody'
 CONTEXT_DISK_FILES = ["context.sh"]
-VALID_DSMODES = ("local", "net", "disabled")
 
 
 class DataSourceOpenNebula(sources.DataSource):
     def __init__(self, sys_cfg, distro, paths):
         sources.DataSource.__init__(self, sys_cfg, distro, paths)
-        self.dsmode = 'local'
         self.seed = None
         self.seed_dir = os.path.join(paths.seed_dir, 'opennebula')
 
@@ -93,52 +90,27 @@ class DataSourceOpenNebula(sources.DataSource):
         md = util.mergemanydict([md, defaults])
 
         # check for valid user specified dsmode
-        user_dsmode = results['metadata'].get('DSMODE', None)
-        if user_dsmode not in VALID_DSMODES + (None,):
-            LOG.warn("user specified invalid mode: %s", user_dsmode)
-            user_dsmode = None
-
-        # decide dsmode
-        if user_dsmode:
-            dsmode = user_dsmode
-        elif self.ds_cfg.get('dsmode'):
-            dsmode = self.ds_cfg.get('dsmode')
-        else:
-            dsmode = DEFAULT_MODE
-
-        if dsmode == "disabled":
-            # most likely user specified
-            return False
-
-        # apply static network configuration only in 'local' dsmode
-        if ('network-interfaces' in results and self.dsmode == "local"):
-            LOG.debug("Updating network interfaces from %s", self)
-            self.distro.apply_network(results['network-interfaces'])
+        self.dsmode = self._determine_dsmode(
+            [results.get('DSMODE'), self.ds_cfg.get('dsmode')])
 
-        if dsmode != self.dsmode:
-            LOG.debug("%s: not claiming datasource, dsmode=%s", self, dsmode)
+        if self.dsmode == sources.DSMODE_DISABLED:
             return False
 
         self.seed = seed
+        self.network_eni = results.get("network_config")
         self.metadata = md
         self.userdata_raw = results.get('userdata')
         return True
 
     def get_hostname(self, fqdn=False, resolve_ip=None):
         if resolve_ip is None:
-            if self.dsmode == 'net':
+            if self.dsmode == sources.DSMODE_NETWORK:
                 resolve_ip = True
             else:
                 resolve_ip = False
         return sources.DataSource.get_hostname(self, fqdn, resolve_ip)
 
 
-class DataSourceOpenNebulaNet(DataSourceOpenNebula):
-    def __init__(self, sys_cfg, distro, paths):
-        DataSourceOpenNebula.__init__(self, sys_cfg, distro, paths)
-        self.dsmode = 'net'
-
-
 class NonContextDiskDir(Exception):
     pass
 
@@ -443,10 +415,12 @@ def read_context_disk_dir(source_dir, asuser=None):
     return results
 
 
+# Legacy: Must be present in case we load an old pkl object
+DataSourceOpenNebulaNet = DataSourceOpenNebula
+
 # Used to match classes to dependencies
 datasources = [
     (DataSourceOpenNebula, (sources.DEP_FILESYSTEM, )),
-    (DataSourceOpenNebulaNet, (sources.DEP_FILESYSTEM, sources.DEP_NETWORK)),
 ]
 
 
diff --git a/cloudinit/sources/DataSourceOpenStack.py b/cloudinit/sources/DataSourceOpenStack.py
index 3af17b10..c06d17f3 100644
--- a/cloudinit/sources/DataSourceOpenStack.py
+++ b/cloudinit/sources/DataSourceOpenStack.py
@@ -33,13 +33,11 @@ DEFAULT_IID = "iid-dsopenstack"
 DEFAULT_METADATA = {
     "instance-id": DEFAULT_IID,
 }
-VALID_DSMODES = ("net", "disabled")
 
 
 class DataSourceOpenStack(openstack.SourceMixin, sources.DataSource):
     def __init__(self, sys_cfg, distro, paths):
         super(DataSourceOpenStack, self).__init__(sys_cfg, distro, paths)
-        self.dsmode = 'net'
         self.metadata_address = None
         self.ssl_details = util.fetch_ssl_details(self.paths)
         self.version = None
@@ -103,7 +101,7 @@ class DataSourceOpenStack(openstack.SourceMixin, sources.DataSource):
         self.metadata_address = url2base.get(avail_url)
         return bool(avail_url)
 
-    def get_data(self):
+    def get_data(self, retries=5, timeout=5):
         try:
             if not self.wait_for_metadata_service():
                 return False
@@ -115,7 +113,9 @@ class DataSourceOpenStack(openstack.SourceMixin, sources.DataSource):
                                     'Crawl of openstack metadata service',
                                     read_metadata_service,
                                     args=[self.metadata_address],
-                                    kwargs={'ssl_details': self.ssl_details})
+                                    kwargs={'ssl_details': self.ssl_details,
+                                            'retries': retries,
+                                            'timeout': timeout})
         except openstack.NonReadable:
             return False
         except (openstack.BrokenMetadata, IOError):
@@ -123,11 +123,8 @@ class DataSourceOpenStack(openstack.SourceMixin, sources.DataSource):
                         self.metadata_address)
             return False
 
-        user_dsmode = results.get('dsmode', None)
-        if user_dsmode not in VALID_DSMODES + (None,):
-            LOG.warn("User specified invalid mode: %s", user_dsmode)
-            user_dsmode = None
-        if user_dsmode == 'disabled':
+        self.dsmode = self._determine_dsmode([results.get('dsmode')])
+        if self.dsmode == sources.DSMODE_DISABLED:
             return False
 
         md = results.get('metadata', {})
@@ -153,8 +150,10 @@ class DataSourceOpenStack(openstack.SourceMixin, sources.DataSource):
         return sources.instance_id_matches_system_uuid(self.get_instance_id())
 
 
-def read_metadata_service(base_url, ssl_details=None):
-    reader = openstack.MetadataReader(base_url, ssl_details=ssl_details)
+def read_metadata_service(base_url, ssl_details=None,
+                          timeout=5, retries=5):
+    reader = openstack.MetadataReader(base_url, ssl_details=ssl_details,
+                                      timeout=timeout, retries=retries)
     return reader.read_v2()
 
 
diff --git a/cloudinit/sources/DataSourceSmartOS.py b/cloudinit/sources/DataSourceSmartOS.py
index 6cbd8dfa..08bc132b 100644
--- a/cloudinit/sources/DataSourceSmartOS.py
+++ b/cloudinit/sources/DataSourceSmartOS.py
@@ -32,21 +32,19 @@
 #       http://us-east.manta.joyent.com/jmc/public/mdata/datadict.html
 #       Comments with "@datadictionary" are snippets of the definition
 
+import base64
 import binascii
-import contextlib
+import json
 import os
 import random
 import re
 import socket
-import stat
-
-import serial
 
 from cloudinit import log as logging
+from cloudinit import serial
 from cloudinit import sources
 from cloudinit import util
 
-
 LOG = logging.getLogger(__name__)
 
 SMARTOS_ATTRIB_MAP = {
@@ -64,14 +62,36 @@ SMARTOS_ATTRIB_MAP = {
     'operator-script': ('sdc:operator-script', False),
 }
 
+SMARTOS_ATTRIB_JSON = {
+    # Cloud-init Key : (SmartOS Key known JSON)
+    'network-data': 'sdc:nics',
+}
+
+SMARTOS_ENV_LX_BRAND = "lx-brand"
+SMARTOS_ENV_KVM = "kvm"
+
 DS_NAME = 'SmartOS'
 DS_CFG_PATH = ['datasource', DS_NAME]
+NO_BASE64_DECODE = [
+    'iptables_disable',
+    'motd_sys_info',
+    'root_authorized_keys',
+    'sdc:datacenter_name',
+    'sdc:uuid'
+    'user-data',
+    'user-script',
+]
+
+METADATA_SOCKFILE = '/native/.zonecontrol/metadata.sock'
+SERIAL_DEVICE = '/dev/ttyS1'
+SERIAL_TIMEOUT = 60
+
 # BUILT-IN DATASOURCE CONFIGURATION
 #  The following is the built-in configuration. If the values
 #  are not set via the system configuration, then these default
 #  will be used:
 #    serial_device: which serial device to use for the meta-data
-#    seed_timeout: how long to wait on the device
+#    serial_timeout: how long to wait on the device
 #    no_base64_decode: values which are not base64 encoded and
 #            are fetched directly from SmartOS, not meta-data values
 #    base64_keys: meta-data keys that are delivered in base64
@@ -81,16 +101,10 @@ DS_CFG_PATH = ['datasource', DS_NAME]
 #    fs_setup: describes how to format the ephemeral drive
 #
 BUILTIN_DS_CONFIG = {
-    'serial_device': '/dev/ttyS1',
-    'metadata_sockfile': '/native/.zonecontrol/metadata.sock',
-    'seed_timeout': 60,
-    'no_base64_decode': ['root_authorized_keys',
-                         'motd_sys_info',
-                         'iptables_disable',
-                         'user-data',
-                         'user-script',
-                         'sdc:datacenter_name',
-                         'sdc:uuid'],
+    'serial_device': SERIAL_DEVICE,
+    'serial_timeout': SERIAL_TIMEOUT,
+    'metadata_sockfile': METADATA_SOCKFILE,
+    'no_base64_decode': NO_BASE64_DECODE,
     'base64_keys': [],
     'base64_all': False,
     'disk_aliases': {'ephemeral0': '/dev/vdb'},
@@ -154,59 +168,41 @@ LEGACY_USER_D = "/var/db"
 
 
 class DataSourceSmartOS(sources.DataSource):
+    _unset = "_unset"
+    smartos_type = _unset
+    md_client = _unset
+
     def __init__(self, sys_cfg, distro, paths):
         sources.DataSource.__init__(self, sys_cfg, distro, paths)
-        self.is_smartdc = None
         self.ds_cfg = util.mergemanydict([
             self.ds_cfg,
             util.get_cfg_by_path(sys_cfg, DS_CFG_PATH, {}),
             BUILTIN_DS_CONFIG])
 
         self.metadata = {}
+        self.network_data = None
+        self._network_config = None
 
-        # SDC LX-Brand Zones lack dmidecode (no /dev/mem) but
-        # report 'BrandZ virtual linux' as the kernel version
-        if os.uname()[3].lower() == 'brandz virtual linux':
-            LOG.debug("Host is SmartOS, guest in Zone")
-            self.is_smartdc = True
-            self.smartos_type = 'lx-brand'
-            self.cfg = {}
-            self.seed = self.ds_cfg.get("metadata_sockfile")
-        else:
-            self.is_smartdc = True
-            self.smartos_type = 'kvm'
-            self.seed = self.ds_cfg.get("serial_device")
-            self.cfg = BUILTIN_CLOUD_CONFIG
-            self.seed_timeout = self.ds_cfg.get("serial_timeout")
-        self.smartos_no_base64 = self.ds_cfg.get('no_base64_decode')
-        self.b64_keys = self.ds_cfg.get('base64_keys')
-        self.b64_all = self.ds_cfg.get('base64_all')
         self.script_base_d = os.path.join(self.paths.get_cpath("scripts"))
 
+        self._init()
+
     def __str__(self):
         root = sources.DataSource.__str__(self)
-        return "%s [seed=%s]" % (root, self.seed)
-
-    def _get_seed_file_object(self):
-        if not self.seed:
-            raise AttributeError("seed device is not set")
-
-        if self.smartos_type == 'lx-brand':
-            if not stat.S_ISSOCK(os.stat(self.seed).st_mode):
-                LOG.debug("Seed %s is not a socket", self.seed)
-                return None
-            sock = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM)
-            sock.connect(self.seed)
-            return sock.makefile('rwb')
-        else:
-            if not stat.S_ISCHR(os.stat(self.seed).st_mode):
-                LOG.debug("Seed %s is not a character device")
-                return None
-            ser = serial.Serial(self.seed, timeout=self.seed_timeout)
-            if not ser.isOpen():
-                raise SystemError("Unable to open %s" % self.seed)
-            return ser
-        return None
+        return "%s [client=%s]" % (root, self.md_client)
+
+    def _init(self):
+        if self.smartos_type == self._unset:
+            self.smartos_type = get_smartos_environ()
+            if self.smartos_type is None:
+                self.md_client = None
+
+        if self.md_client == self._unset:
+            self.md_client = jmc_client_factory(
+                smartos_type=self.smartos_type,
+                metadata_sockfile=self.ds_cfg['metadata_sockfile'],
+                serial_device=self.ds_cfg['serial_device'],
+                serial_timeout=self.ds_cfg['serial_timeout'])
 
     def _set_provisioned(self):
         '''Mark the instance provisioning state as successful.
@@ -225,50 +221,26 @@ class DataSourceSmartOS(sources.DataSource):
                       '/'.join([svc_path, 'provision_success']))
 
     def get_data(self):
+        self._init()
+
         md = {}
         ud = ""
 
-        if not device_exists(self.seed):
-            LOG.debug("No metadata device '%s' found for SmartOS datasource",
-                      self.seed)
-            return False
-
-        uname_arch = os.uname()[4]
-        if uname_arch.startswith("arm") or uname_arch == "aarch64":
-            # Disabling because dmidcode in dmi_data() crashes kvm process
-            LOG.debug("Disabling SmartOS datasource on arm (LP: #1243287)")
+        if not self.smartos_type:
+            LOG.debug("Not running on smartos")
             return False
 
-        # SDC KVM instances will provide dmi data, LX-brand does not
-        if self.smartos_type == 'kvm':
-            dmi_info = dmi_data()
-            if dmi_info is None:
-                LOG.debug("No dmidata utility found")
-                return False
-
-            system_type = dmi_info
-            if 'smartdc' not in system_type.lower():
-                LOG.debug("Host is not on SmartOS. system_type=%s",
-                          system_type)
-                return False
-            LOG.debug("Host is SmartOS, guest in KVM")
-
-        seed_obj = self._get_seed_file_object()
-        if seed_obj is None:
-            LOG.debug('Seed file object not found.')
+        if not self.md_client.exists():
+            LOG.debug("No metadata device '%r' found for SmartOS datasource",
+                      self.md_client)
             return False
-        with contextlib.closing(seed_obj) as seed:
-            b64_keys = self.query('base64_keys', seed, strip=True, b64=False)
-            if b64_keys is not None:
-                self.b64_keys = [k.strip() for k in str(b64_keys).split(',')]
 
-            b64_all = self.query('base64_all', seed, strip=True, b64=False)
-            if b64_all is not None:
-                self.b64_all = util.is_true(b64_all)
+        for ci_noun, attribute in SMARTOS_ATTRIB_MAP.items():
+            smartos_noun, strip = attribute
+            md[ci_noun] = self.md_client.get(smartos_noun, strip=strip)
 
-            for ci_noun, attribute in SMARTOS_ATTRIB_MAP.items():
-                smartos_noun, strip = attribute
-                md[ci_noun] = self.query(smartos_noun, seed, strip=strip)
+        for ci_noun, smartos_noun in SMARTOS_ATTRIB_JSON.items():
+            md[ci_noun] = self.md_client.get_json(smartos_noun)
 
         # @datadictionary: This key may contain a program that is written
         # to a file in the filesystem of the guest on each boot and then
@@ -318,6 +290,7 @@ class DataSourceSmartOS(sources.DataSource):
         self.metadata = util.mergemanydict([md, self.metadata])
         self.userdata_raw = ud
         self.vendordata_raw = md['vendor-data']
+        self.network_data = md['network-data']
 
         self._set_provisioned()
         return True
@@ -326,69 +299,20 @@ class DataSourceSmartOS(sources.DataSource):
         return self.ds_cfg['disk_aliases'].get(name)
 
     def get_config_obj(self):
-        return self.cfg
+        if self.smartos_type == SMARTOS_ENV_KVM:
+            return BUILTIN_CLOUD_CONFIG
+        return {}
 
     def get_instance_id(self):
         return self.metadata['instance-id']
 
-    def query(self, noun, seed_file, strip=False, default=None, b64=None):
-        if b64 is None:
-            if noun in self.smartos_no_base64:
-                b64 = False
-            elif self.b64_all or noun in self.b64_keys:
-                b64 = True
-
-        return self._query_data(noun, seed_file, strip=strip,
-                                default=default, b64=b64)
-
-    def _query_data(self, noun, seed_file, strip=False,
-                    default=None, b64=None):
-        """Makes a request via "GET <NOUN>"
-
-           In the response, the first line is the status, while subsequent
-           lines are is the value. A blank line with a "." is used to
-           indicate end of response.
-
-           If the response is expected to be base64 encoded, then set
-           b64encoded to true. Unfortantely, there is no way to know if
-           something is 100% encoded, so this method relies on being told
-           if the data is base64 or not.
-        """
-
-        if not noun:
-            return False
-
-        response = JoyentMetadataClient(seed_file).get_metadata(noun)
-
-        if response is None:
-            return default
-
-        if b64 is None:
-            b64 = self._query_data('b64-%s' % noun, seed_file, b64=False,
-                                   default=False, strip=True)
-            b64 = util.is_true(b64)
-
-        resp = None
-        if b64 or strip:
-            resp = "".join(response).rstrip()
-        else:
-            resp = "".join(response)
-
-        if b64:
-            try:
-                return util.b64d(resp)
-            # Bogus input produces different errors in Python 2 and 3;
-            # catch both.
-            except (TypeError, binascii.Error):
-                LOG.warn("Failed base64 decoding key '%s'", noun)
-                return resp
-
-        return resp
-
-
-def device_exists(device):
-    """Symplistic method to determine if the device exists or not"""
-    return os.path.exists(device)
+    @property
+    def network_config(self):
+        if self._network_config is None:
+            if self.network_data is not None:
+                self._network_config = (
+                    convert_smartos_network_data(self.network_data))
+        return self._network_config
 
 
 class JoyentMetadataFetchException(Exception):
@@ -407,8 +331,11 @@ class JoyentMetadataClient(object):
         r' (?P<body>(?P<request_id>[0-9a-f]+) (?P<status>SUCCESS|NOTFOUND)'
         r'( (?P<payload>.+))?)')
 
-    def __init__(self, metasource):
-        self.metasource = metasource
+    def __init__(self, smartos_type=None, fp=None):
+        if smartos_type is None:
+            smartos_type = get_smartos_environ()
+        self.smartos_type = smartos_type
+        self.fp = fp
 
     def _checksum(self, body):
         return '{0:08x}'.format(
@@ -436,37 +363,229 @@ class JoyentMetadataClient(object):
         LOG.debug('Value "%s" found.', value)
         return value
 
-    def get_metadata(self, metadata_key):
-        LOG.debug('Fetching metadata key "%s"...', metadata_key)
+    def request(self, rtype, param=None):
         request_id = '{0:08x}'.format(random.randint(0, 0xffffffff))
-        message_body = '{0} GET {1}'.format(request_id,
-                                            util.b64e(metadata_key))
+        message_body = ' '.join((request_id, rtype,))
+        if param:
+            message_body += ' ' + base64.b64encode(param.encode()).decode()
         msg = 'V2 {0} {1} {2}\n'.format(
             len(message_body), self._checksum(message_body), message_body)
         LOG.debug('Writing "%s" to metadata transport.', msg)
-        self.metasource.write(msg.encode('ascii'))
-        self.metasource.flush()
+
+        need_close = False
+        if not self.fp:
+            self.open_transport()
+            need_close = True
+
+        self.fp.write(msg.encode('ascii'))
+        self.fp.flush()
 
         response = bytearray()
-        response.extend(self.metasource.read(1))
+        response.extend(self.fp.read(1))
         while response[-1:] != b'\n':
-            response.extend(self.metasource.read(1))
+            response.extend(self.fp.read(1))
+
+        if need_close:
+            self.close_transport()
+
         response = response.rstrip().decode('ascii')
         LOG.debug('Read "%s" from metadata transport.', response)
 
         if 'SUCCESS' not in response:
             return None
 
-        return self._get_value_from_frame(request_id, response)
+        value = self._get_value_from_frame(request_id, response)
+        return value
+
+    def get(self, key, default=None, strip=False):
+        result = self.request(rtype='GET', param=key)
+        if result is None:
+            return default
+        if result and strip:
+            result = result.strip()
+        return result
+
+    def get_json(self, key, default=None):
+        result = self.get(key, default=default)
+        if result is None:
+            return default
+        return json.loads(result)
+
+    def list(self):
+        result = self.request(rtype='KEYS')
+        if result:
+            result = result.split('\n')
+        return result
+
+    def put(self, key, val):
+        param = b' '.join([base64.b64encode(i.encode())
+                           for i in (key, val)]).decode()
+        return self.request(rtype='PUT', param=param)
+
+    def delete(self, key):
+        return self.request(rtype='DELETE', param=key)
+
+    def close_transport(self):
+        if self.fp:
+            self.fp.close()
+            self.fp = None
+
+    def __enter__(self):
+        if self.fp:
+            return self
+        self.open_transport()
+        return self
+
+    def __exit__(self, exc_type, exc_value, traceback):
+        self.close_transport()
+        return
+
+    def open_transport(self):
+        raise NotImplementedError
+
+
+class JoyentMetadataSocketClient(JoyentMetadataClient):
+    def __init__(self, socketpath):
+        self.socketpath = socketpath
+
+    def open_transport(self):
+        sock = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM)
+        sock.connect(self.socketpath)
+        self.fp = sock.makefile('rwb')
+
+    def exists(self):
+        return os.path.exists(self.socketpath)
+
+    def __repr__(self):
+        return "%s(socketpath=%s)" % (self.__class__.__name__, self.socketpath)
+
+
+class JoyentMetadataSerialClient(JoyentMetadataClient):
+    def __init__(self, device, timeout=10, smartos_type=None):
+        super(JoyentMetadataSerialClient, self).__init__(smartos_type)
+        self.device = device
+        self.timeout = timeout
+
+    def exists(self):
+        return os.path.exists(self.device)
+
+    def open_transport(self):
+        ser = serial.Serial(self.device, timeout=self.timeout)
+        if not ser.isOpen():
+            raise SystemError("Unable to open %s" % self.device)
+        self.fp = ser
+
+    def __repr__(self):
+        return "%s(device=%s, timeout=%s)" % (
+            self.__class__.__name__, self.device, self.timeout)
+
+
+class JoyentMetadataLegacySerialClient(JoyentMetadataSerialClient):
+    """V1 of the protocol was not safe for all values.
+    Thus, we allowed the user to pass values in as base64 encoded.
+    Users may still reasonably expect to be able to send base64 data
+    and have it transparently decoded.  So even though the V2 format is
+    now used, and is safe (using base64 itself), we keep legacy support.
+
+    The way for a user to do this was:
+      a.) specify 'base64_keys' key whose value is a comma delimited
+          list of keys that were base64 encoded.
+      b.) base64_all: string interpreted as a boolean that indicates
+          if all keys are base64 encoded.
+      c.) set a key named b64-<keyname> with a boolean indicating that
+          <keyname> is base64 encoded."""
+
+    def __init__(self, device, timeout=10, smartos_type=None):
+        s = super(JoyentMetadataLegacySerialClient, self)
+        s.__init__(device, timeout, smartos_type)
+        self.base64_keys = None
+        self.base64_all = None
+
+    def _init_base64_keys(self, reset=False):
+        if reset:
+            self.base64_keys = None
+            self.base64_all = None
+
+        keys = None
+        if self.base64_all is None:
+            keys = self.list()
+            if 'base64_all' in keys:
+                self.base64_all = util.is_true(self._get("base64_all"))
+            else:
+                self.base64_all = False
+
+        if self.base64_all:
+            # short circuit if base64_all is true
+            return
+
+        if self.base64_keys is None:
+            if keys is None:
+                keys = self.list()
+            b64_keys = set()
+            if 'base64_keys' in keys:
+                b64_keys = set(self._get("base64_keys").split(","))
+
+            # now add any b64-<keyname> that has a true value
+            for key in [k[3:] for k in keys if k.startswith("b64-")]:
+                if util.is_true(self._get(key)):
+                    b64_keys.add(key)
+                else:
+                    if key in b64_keys:
+                        b64_keys.remove(key)
+
+            self.base64_keys = b64_keys
+
+    def _get(self, key, default=None, strip=False):
+        return (super(JoyentMetadataLegacySerialClient, self).
+                get(key, default=default, strip=strip))
+
+    def is_b64_encoded(self, key, reset=False):
+        if key in NO_BASE64_DECODE:
+            return False
+
+        self._init_base64_keys(reset=reset)
+        if self.base64_all:
+            return True
+
+        return key in self.base64_keys
+
+    def get(self, key, default=None, strip=False):
+        mdefault = object()
+        val = self._get(key, strip=False, default=mdefault)
+        if val is mdefault:
+            return default
+
+        if self.is_b64_encoded(key):
+            try:
+                val = base64.b64decode(val.encode()).decode()
+            # Bogus input produces different errors in Python 2 and 3
+            except (TypeError, binascii.Error):
+                LOG.warn("Failed base64 decoding key '%s': %s", key, val)
+
+        if strip:
+            val = val.strip()
+
+        return val
 
 
-def dmi_data():
-    sys_type = util.read_dmi_data("system-product-name")
+def jmc_client_factory(
+        smartos_type=None, metadata_sockfile=METADATA_SOCKFILE,
+        serial_device=SERIAL_DEVICE, serial_timeout=SERIAL_TIMEOUT,
+        uname_version=None):
 
-    if not sys_type:
+    if smartos_type is None:
+        smartos_type = get_smartos_environ(uname_version)
+
+    if smartos_type is None:
         return None
+    elif smartos_type == SMARTOS_ENV_KVM:
+        return JoyentMetadataLegacySerialClient(
+            device=serial_device, timeout=serial_timeout,
+            smartos_type=smartos_type)
+    elif smartos_type == SMARTOS_ENV_LX_BRAND:
+        return JoyentMetadataSocketClient(socketpath=metadata_sockfile)
 
-    return sys_type
+    raise ValueError("Unknown value for smartos_type: %s" % smartos_type)
 
 
 def write_boot_content(content, content_f, link=None, shebang=False,
@@ -522,15 +641,141 @@ def write_boot_content(content, content_f, link=None, shebang=False,
                 util.ensure_dir(os.path.dirname(link))
                 os.symlink(content_f, link)
         except IOError as e:
-            util.logexc(LOG, "failed establishing content link", e)
+            util.logexc(LOG, "failed establishing content link: %s", e)
+
+
+def get_smartos_environ(uname_version=None, product_name=None,
+                        uname_arch=None):
+    uname = os.uname()
+    if uname_arch is None:
+        uname_arch = uname[4]
+
+    if uname_arch.startswith("arm") or uname_arch == "aarch64":
+        return None
+
+    # SDC LX-Brand Zones lack dmidecode (no /dev/mem) but
+    # report 'BrandZ virtual linux' as the kernel version
+    if uname_version is None:
+        uname_version = uname[3]
+    if uname_version.lower() == 'brandz virtual linux':
+        return SMARTOS_ENV_LX_BRAND
+
+    if product_name is None:
+        system_type = util.read_dmi_data("system-product-name")
+    else:
+        system_type = product_name
+
+    if system_type and 'smartdc' in system_type.lower():
+        return SMARTOS_ENV_KVM
+
+    return None
+
+
+# Covert SMARTOS 'sdc:nics' data to network_config yaml
+def convert_smartos_network_data(network_data=None):
+    """Return a dictionary of network_config by parsing provided
+       SMARTOS sdc:nics configuration data
+
+    sdc:nics data is a dictionary of properties of a nic and the ip
+    configuration desired.  Additional nic dictionaries are appended
+    to the list.
+
+    Converting the format is straightforward though it does include
+    duplicate information as well as data which appears to be relevant
+    to the hostOS rather than the guest.
+
+    For each entry in the nics list returned from query sdc:nics, we
+    create a type: physical entry, and extract the interface properties:
+    'mac' -> 'mac_address', 'mtu', 'interface' -> 'name'.  The remaining
+    keys are related to ip configuration.  For each ip in the 'ips' list
+    we create a subnet entry under 'subnets' pairing the ip to a one in
+    the 'gateways' list.
+    """
+
+    valid_keys = {
+        'physical': [
+            'mac_address',
+            'mtu',
+            'name',
+            'params',
+            'subnets',
+            'type',
+        ],
+        'subnet': [
+            'address',
+            'broadcast',
+            'dns_nameservers',
+            'dns_search',
+            'gateway',
+            'metric',
+            'netmask',
+            'pointopoint',
+            'routes',
+            'scope',
+            'type',
+        ],
+    }
+
+    config = []
+    for nic in network_data:
+        cfg = {k: v for k, v in nic.items()
+               if k in valid_keys['physical']}
+        cfg.update({
+            'type': 'physical',
+            'name': nic['interface']})
+        if 'mac' in nic:
+            cfg.update({'mac_address': nic['mac']})
+
+        subnets = []
+        for ip, gw in zip(nic['ips'], nic['gateways']):
+            subnet = {k: v for k, v in nic.items()
+                      if k in valid_keys['subnet']}
+            subnet.update({
+                'type': 'static',
+                'address': ip,
+                'gateway': gw,
+            })
+            subnets.append(subnet)
+        cfg.update({'subnets': subnets})
+        config.append(cfg)
+
+    return {'version': 1, 'config': config}
 
 
 # Used to match classes to dependencies
 datasources = [
-    (DataSourceSmartOS, (sources.DEP_FILESYSTEM, sources.DEP_NETWORK)),
+    (DataSourceSmartOS, (sources.DEP_FILESYSTEM, )),
 ]
 
 
 # Return a list of data sources that match this set of dependencies
 def get_datasource_list(depends):
     return sources.list_from_depends(depends, datasources)
+
+
+if __name__ == "__main__":
+    import sys
+    jmc = jmc_client_factory()
+    if jmc is None:
+        print("Do not appear to be on smartos.")
+        sys.exit(1)
+    if len(sys.argv) == 1:
+        keys = (list(SMARTOS_ATTRIB_JSON.keys()) +
+                list(SMARTOS_ATTRIB_MAP.keys()))
+    else:
+        keys = sys.argv[1:]
+
+    data = {}
+    for key in keys:
+        if key in SMARTOS_ATTRIB_JSON:
+            keyname = SMARTOS_ATTRIB_JSON[key]
+            data[key] = jmc.get_json(keyname)
+        else:
+            if key in SMARTOS_ATTRIB_MAP:
+                keyname, strip = SMARTOS_ATTRIB_MAP[key]
+            else:
+                keyname, strip = (key, False)
+            val = jmc.get(keyname, strip=strip)
+            data[key] = jmc.get(keyname, strip=strip)
+
+    print(json.dumps(data, indent=1))
diff --git a/cloudinit/sources/__init__.py b/cloudinit/sources/__init__.py
index 6bf2c33b..2a6b8d90 100644
--- a/cloudinit/sources/__init__.py
+++ b/cloudinit/sources/__init__.py
@@ -34,6 +34,13 @@ from cloudinit import util
 from cloudinit.filters import launch_index
 from cloudinit.reporting import events
 
+DSMODE_DISABLED = "disabled"
+DSMODE_LOCAL = "local"
+DSMODE_NETWORK = "net"
+DSMODE_PASS = "pass"
+
+VALID_DSMODES = [DSMODE_DISABLED, DSMODE_LOCAL, DSMODE_NETWORK]
+
 DEP_FILESYSTEM = "FILESYSTEM"
 DEP_NETWORK = "NETWORK"
 DS_PREFIX = 'DataSource'
@@ -45,10 +52,9 @@ class DataSourceNotFoundException(Exception):
     pass
 
 
+@six.add_metaclass(abc.ABCMeta)
 class DataSource(object):
 
-    __metaclass__ = abc.ABCMeta
-
     def __init__(self, sys_cfg, distro, paths, ud_proc=None):
         self.sys_cfg = sys_cfg
         self.distro = distro
@@ -58,6 +64,7 @@ class DataSource(object):
         self.userdata_raw = None
         self.vendordata = None
         self.vendordata_raw = None
+        self.dsmode = DSMODE_NETWORK
 
         # find the datasource config name.
         # remove 'DataSource' from classname on front, and remove 'Net' on end.
@@ -224,10 +231,35 @@ class DataSource(object):
         # quickly (local check only) if self.instance_id is still
         return False
 
+    @staticmethod
+    def _determine_dsmode(candidates, default=None, valid=None):
+        # return the first candidate that is non None, warn if not valid
+        if default is None:
+            default = DSMODE_NETWORK
+
+        if valid is None:
+            valid = VALID_DSMODES
+
+        for candidate in candidates:
+            if candidate is None:
+                continue
+            if candidate in valid:
+                return candidate
+            else:
+                LOG.warn("invalid dsmode '%s', using default=%s",
+                         candidate, default)
+                return default
+
+        return default
+
     @property
     def network_config(self):
         return None
 
+    @property
+    def first_instance_boot(self):
+        return
+
 
 def normalize_pubkey_data(pubkey_data):
     keys = []
diff --git a/cloudinit/sources/helpers/azure.py b/cloudinit/sources/helpers/azure.py
index 018cac6d..63ccf10e 100644
--- a/cloudinit/sources/helpers/azure.py
+++ b/cloudinit/sources/helpers/azure.py
@@ -5,6 +5,7 @@ import socket
 import struct
 import tempfile
 import time
+
 from contextlib import contextmanager
 from xml.etree import ElementTree
 
@@ -220,7 +221,7 @@ class WALinuxAgentShim(object):
             if 'unknown-245' in line:
                 value = line.strip(' ').split(' ', 2)[-1].strip(';\n"')
         if value is None:
-            raise Exception('No endpoint found in DHCP config.')
+            raise ValueError('No endpoint found in DHCP config.')
         endpoint_ip_address = WALinuxAgentShim.get_ip_from_lease_value(value)
         LOG.debug('Azure endpoint found at %s', endpoint_ip_address)
         return endpoint_ip_address
diff --git a/cloudinit/sources/helpers/openstack.py b/cloudinit/sources/helpers/openstack.py
index 1aa6bbae..d52cb56a 100644
--- a/cloudinit/sources/helpers/openstack.py
+++ b/cloudinit/sources/helpers/openstack.py
@@ -28,6 +28,7 @@ import six
 
 from cloudinit import ec2_utils
 from cloudinit import log as logging
+from cloudinit import net
 from cloudinit import sources
 from cloudinit import url_helper
 from cloudinit import util
@@ -145,8 +146,8 @@ class SourceMixin(object):
             return device
 
 
+@six.add_metaclass(abc.ABCMeta)
 class BaseReader(object):
-    __metaclass__ = abc.ABCMeta
 
     def __init__(self, base_path):
         self.base_path = base_path
@@ -156,7 +157,7 @@ class BaseReader(object):
         pass
 
     @abc.abstractmethod
-    def _path_read(self, path):
+    def _path_read(self, path, decode=False):
         pass
 
     @abc.abstractmethod
@@ -190,14 +191,14 @@ class BaseReader(object):
                   versions_available)
         return selected_version
 
-    def _read_content_path(self, item):
+    def _read_content_path(self, item, decode=False):
         path = item.get('content_path', '').lstrip("/")
         path_pieces = path.split("/")
         valid_pieces = [p for p in path_pieces if len(p)]
         if not valid_pieces:
             raise BrokenMetadata("Item %s has no valid content path" % (item))
         path = self._path_join(self.base_path, "openstack", *path_pieces)
-        return self._path_read(path)
+        return self._path_read(path, decode=decode)
 
     def read_v2(self):
         """Reads a version 2 formatted location.
@@ -298,7 +299,8 @@ class BaseReader(object):
         net_item = metadata.get("network_config", None)
         if net_item:
             try:
-                results['network_config'] = self._read_content_path(net_item)
+                content = self._read_content_path(net_item, decode=True)
+                results['network_config'] = content
             except IOError as e:
                 raise BrokenMetadata("Failed to read network"
                                      " configuration: %s" % (e))
@@ -333,8 +335,8 @@ class ConfigDriveReader(BaseReader):
         components = [base] + list(add_ons)
         return os.path.join(*components)
 
-    def _path_read(self, path):
-        return util.load_file(path, decode=False)
+    def _path_read(self, path, decode=False):
+        return util.load_file(path, decode=decode)
 
     def _fetch_available_versions(self):
         if self._versions is None:
@@ -446,7 +448,7 @@ class MetadataReader(BaseReader):
         self._versions = found
         return self._versions
 
-    def _path_read(self, path):
+    def _path_read(self, path, decode=False):
 
         def should_retry_cb(_request_args, cause):
             try:
@@ -463,7 +465,10 @@ class MetadataReader(BaseReader):
                                       ssl_details=self.ssl_details,
                                       timeout=self.timeout,
                                       exception_cb=should_retry_cb)
-        return response.contents
+        if decode:
+            return response.contents.decode()
+        else:
+            return response.contents
 
     def _path_join(self, base, *add_ons):
         return url_helper.combine_url(base, *add_ons)
@@ -474,8 +479,152 @@ class MetadataReader(BaseReader):
                                                retries=self.retries)
 
 
+# Convert OpenStack ConfigDrive NetworkData json to network_config yaml
+def convert_net_json(network_json=None, known_macs=None):
+    """Return a dictionary of network_config by parsing provided
+       OpenStack ConfigDrive NetworkData json format
+
+    OpenStack network_data.json provides a 3 element dictionary
+      - "links" (links are network devices, physical or virtual)
+      - "networks" (networks are ip network configurations for one or more
+                    links)
+      -  services (non-ip services, like dns)
+
+    networks and links are combined via network items referencing specific
+    links via a 'link_id' which maps to a links 'id' field.
+
+    To convert this format to network_config yaml, we first iterate over the
+    links and then walk the network list to determine if any of the networks
+    utilize the current link; if so we generate a subnet entry for the device
+
+    We also need to map network_data.json fields to network_config fields. For
+    example, the network_data links 'id' field is equivalent to network_config
+    'name' field for devices.  We apply more of this mapping to the various
+    link types that we encounter.
+
+    There are additional fields that are populated in the network_data.json
+    from OpenStack that are not relevant to network_config yaml, so we
+    enumerate a dictionary of valid keys for network_yaml and apply filtering
+    to drop these superflous keys from the network_config yaml.
+    """
+    if network_json is None:
+        return None
+
+    # dict of network_config key for filtering network_json
+    valid_keys = {
+        'physical': [
+            'name',
+            'type',
+            'mac_address',
+            'subnets',
+            'params',
+            'mtu',
+        ],
+        'subnet': [
+            'type',
+            'address',
+            'netmask',
+            'broadcast',
+            'metric',
+            'gateway',
+            'pointopoint',
+            'scope',
+            'dns_nameservers',
+            'dns_search',
+            'routes',
+        ],
+    }
+
+    links = network_json.get('links', [])
+    networks = network_json.get('networks', [])
+    services = network_json.get('services', [])
+
+    config = []
+    for link in links:
+        subnets = []
+        cfg = {k: v for k, v in link.items()
+               if k in valid_keys['physical']}
+        # 'name' is not in openstack spec yet, but we will support it if it is
+        # present.  The 'id' in the spec is currently implemented as the host
+        # nic's name, meaning something like 'tap-adfasdffd'.  We do not want
+        # to name guest devices with such ugly names.
+        if 'name' in link:
+            cfg['name'] = link['name']
+
+        for network in [n for n in networks
+                        if n['link'] == link['id']]:
+            subnet = {k: v for k, v in network.items()
+                      if k in valid_keys['subnet']}
+            if 'dhcp' in network['type']:
+                t = 'dhcp6' if network['type'].startswith('ipv6') else 'dhcp4'
+                subnet.update({
+                    'type': t,
+                })
+            else:
+                subnet.update({
+                    'type': 'static',
+                    'address': network.get('ip_address'),
+                })
+            if network['type'] == 'ipv4':
+                subnet['ipv4'] = True
+            if network['type'] == 'ipv6':
+                subnet['ipv6'] = True
+            subnets.append(subnet)
+        cfg.update({'subnets': subnets})
+        if link['type'] in ['ethernet', 'vif', 'ovs', 'phy', 'bridge']:
+            cfg.update({
+                'type': 'physical',
+                'mac_address': link['ethernet_mac_address']})
+        elif link['type'] in ['bond']:
+            params = {}
+            for k, v in link.items():
+                if k == 'bond_links':
+                    continue
+                elif k.startswith('bond'):
+                    params.update({k: v})
+            cfg.update({
+                'bond_interfaces': copy.deepcopy(link['bond_links']),
+                'params': params,
+            })
+        elif link['type'] in ['vlan']:
+            cfg.update({
+                'name': "%s.%s" % (link['vlan_link'],
+                                   link['vlan_id']),
+                'vlan_link': link['vlan_link'],
+                'vlan_id': link['vlan_id'],
+                'mac_address': link['vlan_mac_address'],
+            })
+        else:
+            raise ValueError(
+                'Unknown network_data link type: %s' % link['type'])
+
+        config.append(cfg)
+
+    need_names = [d for d in config
+                  if d.get('type') == 'physical' and 'name' not in d]
+
+    if need_names:
+        if known_macs is None:
+            known_macs = net.get_interfaces_by_mac()
+
+        for d in need_names:
+            mac = d.get('mac_address')
+            if not mac:
+                raise ValueError("No mac_address or name entry for %s" % d)
+            if mac not in known_macs:
+                raise ValueError("Unable to find a system nic for %s" % d)
+            d['name'] = known_macs[mac]
+
+    for service in services:
+        cfg = service
+        cfg.update({'type': 'nameserver'})
+        config.append(cfg)
+
+    return {'version': 1, 'config': config}
+
+
 def convert_vendordata_json(data, recurse=True):
-    """ data: a loaded json *object* (strings, arrays, dicts).
+    """data: a loaded json *object* (strings, arrays, dicts).
     return something suitable for cloudinit vendordata_raw.
 
     if data is:
diff --git a/cloudinit/sources/helpers/vmware/imc/boot_proto.py b/cloudinit/sources/helpers/vmware/imc/boot_proto.py
index faba5887..fb53ec1d 100644
--- a/cloudinit/sources/helpers/vmware/imc/boot_proto.py
+++ b/cloudinit/sources/helpers/vmware/imc/boot_proto.py
@@ -1,25 +1,25 @@
-# vi: ts=4 expandtab

-#

-#    Copyright (C) 2015 Canonical Ltd.

-#    Copyright (C) 2015 VMware Inc.

-#

-#    Author: Sankar Tanguturi <stanguturi@vmware.com>

-#

-#    This program is free software: you can redistribute it and/or modify

-#    it under the terms of the GNU General Public License version 3, as

-#    published by the Free Software Foundation.

-#

-#    This program is distributed in the hope that it will be useful,

-#    but WITHOUT ANY WARRANTY; without even the implied warranty of

-#    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the

-#    GNU General Public License for more details.

-#

-#    You should have received a copy of the GNU General Public License

-#    along with this program.  If not, see <http://www.gnu.org/licenses/>.

-

-

-class BootProtoEnum:

-    """Specifies the NIC Boot Settings."""

-

-    DHCP = 'dhcp'

-    STATIC = 'static'

+# vi: ts=4 expandtab
+#
+#    Copyright (C) 2015 Canonical Ltd.
+#    Copyright (C) 2015 VMware Inc.
+#
+#    Author: Sankar Tanguturi <stanguturi@vmware.com>
+#
+#    This program is free software: you can redistribute it and/or modify
+#    it under the terms of the GNU General Public License version 3, as
+#    published by the Free Software Foundation.
+#
+#    This program is distributed in the hope that it will be useful,
+#    but WITHOUT ANY WARRANTY; without even the implied warranty of
+#    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+#    GNU General Public License for more details.
+#
+#    You should have received a copy of the GNU General Public License
+#    along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+
+class BootProtoEnum(object):
+    """Specifies the NIC Boot Settings."""
+
+    DHCP = 'dhcp'
+    STATIC = 'static'
diff --git a/cloudinit/sources/helpers/vmware/imc/config.py b/cloudinit/sources/helpers/vmware/imc/config.py
index aebc12a0..d645c497 100644
--- a/cloudinit/sources/helpers/vmware/imc/config.py
+++ b/cloudinit/sources/helpers/vmware/imc/config.py
@@ -1,95 +1,95 @@
-# vi: ts=4 expandtab

-#

-#    Copyright (C) 2015 Canonical Ltd.

-#    Copyright (C) 2015 VMware Inc.

-#

-#    Author: Sankar Tanguturi <stanguturi@vmware.com>

-#

-#    This program is free software: you can redistribute it and/or modify

-#    it under the terms of the GNU General Public License version 3, as

-#    published by the Free Software Foundation.

-#

-#    This program is distributed in the hope that it will be useful,

-#    but WITHOUT ANY WARRANTY; without even the implied warranty of

-#    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the

-#    GNU General Public License for more details.

-#

-#    You should have received a copy of the GNU General Public License

-#    along with this program.  If not, see <http://www.gnu.org/licenses/>.

-

-from .nic import Nic

-

-

-class Config:

-    """

-    Stores the Contents specified in the Customization

-    Specification file.

-    """

-

-    DNS = 'DNS|NAMESERVER|'

-    SUFFIX = 'DNS|SUFFIX|'

-    PASS = 'PASSWORD|-PASS'

-    TIMEZONE = 'DATETIME|TIMEZONE'

-    UTC = 'DATETIME|UTC'

-    HOSTNAME = 'NETWORK|HOSTNAME'

-    DOMAINNAME = 'NETWORK|DOMAINNAME'

-

-    def __init__(self, configFile):

-        self._configFile = configFile

-

-    @property

-    def host_name(self):

-        """Return the hostname."""

-        return self._configFile.get(Config.HOSTNAME, None)

-

-    @property

-    def domain_name(self):

-        """Return the domain name."""

-        return self._configFile.get(Config.DOMAINNAME, None)

-

-    @property

-    def timezone(self):

-        """Return the timezone."""

-        return self._configFile.get(Config.TIMEZONE, None)

-

-    @property

-    def utc(self):

-        """Retrieves whether to set time to UTC or Local."""

-        return self._configFile.get(Config.UTC, None)

-

-    @property

-    def admin_password(self):

-        """Return the root password to be set."""

-        return self._configFile.get(Config.PASS, None)

-

-    @property

-    def name_servers(self):

-        """Return the list of DNS servers."""

-        res = []

-        cnt = self._configFile.get_count_with_prefix(Config.DNS)

-        for i in range(1, cnt + 1):

-            key = Config.DNS + str(i)

-            res.append(self._configFile[key])

-

-        return res

-

-    @property

-    def dns_suffixes(self):

-        """Return the list of DNS Suffixes."""

-        res = []

-        cnt = self._configFile.get_count_with_prefix(Config.SUFFIX)

-        for i in range(1, cnt + 1):

-            key = Config.SUFFIX + str(i)

-            res.append(self._configFile[key])

-

-        return res

-

-    @property

-    def nics(self):

-        """Return the list of associated NICs."""

-        res = []

-        nics = self._configFile['NIC-CONFIG|NICS']

-        for nic in nics.split(','):

-            res.append(Nic(nic, self._configFile))

-

-        return res

+# vi: ts=4 expandtab
+#
+#    Copyright (C) 2015 Canonical Ltd.
+#    Copyright (C) 2015 VMware Inc.
+#
+#    Author: Sankar Tanguturi <stanguturi@vmware.com>
+#
+#    This program is free software: you can redistribute it and/or modify
+#    it under the terms of the GNU General Public License version 3, as
+#    published by the Free Software Foundation.
+#
+#    This program is distributed in the hope that it will be useful,
+#    but WITHOUT ANY WARRANTY; without even the implied warranty of
+#    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+#    GNU General Public License for more details.
+#
+#    You should have received a copy of the GNU General Public License
+#    along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+from .nic import Nic
+
+
+class Config(object):
+    """
+    Stores the Contents specified in the Customization
+    Specification file.
+    """
+
+    DNS = 'DNS|NAMESERVER|'
+    SUFFIX = 'DNS|SUFFIX|'
+    PASS = 'PASSWORD|-PASS'
+    TIMEZONE = 'DATETIME|TIMEZONE'
+    UTC = 'DATETIME|UTC'
+    HOSTNAME = 'NETWORK|HOSTNAME'
+    DOMAINNAME = 'NETWORK|DOMAINNAME'
+
+    def __init__(self, configFile):
+        self._configFile = configFile
+
+    @property
+    def host_name(self):
+        """Return the hostname."""
+        return self._configFile.get(Config.HOSTNAME, None)
+
+    @property
+    def domain_name(self):
+        """Return the domain name."""
+        return self._configFile.get(Config.DOMAINNAME, None)
+
+    @property
+    def timezone(self):
+        """Return the timezone."""
+        return self._configFile.get(Config.TIMEZONE, None)
+
+    @property
+    def utc(self):
+        """Retrieves whether to set time to UTC or Local."""
+        return self._configFile.get(Config.UTC, None)
+
+    @property
+    def admin_password(self):
+        """Return the root password to be set."""
+        return self._configFile.get(Config.PASS, None)
+
+    @property
+    def name_servers(self):
+        """Return the list of DNS servers."""
+        res = []
+        cnt = self._configFile.get_count_with_prefix(Config.DNS)
+        for i in range(1, cnt + 1):
+            key = Config.DNS + str(i)
+            res.append(self._configFile[key])
+
+        return res
+
+    @property
+    def dns_suffixes(self):
+        """Return the list of DNS Suffixes."""
+        res = []
+        cnt = self._configFile.get_count_with_prefix(Config.SUFFIX)
+        for i in range(1, cnt + 1):
+            key = Config.SUFFIX + str(i)
+            res.append(self._configFile[key])
+
+        return res
+
+    @property
+    def nics(self):
+        """Return the list of associated NICs."""
+        res = []
+        nics = self._configFile['NIC-CONFIG|NICS']
+        for nic in nics.split(','):
+            res.append(Nic(nic, self._configFile))
+
+        return res
diff --git a/cloudinit/sources/helpers/vmware/imc/config_namespace.py b/cloudinit/sources/helpers/vmware/imc/config_namespace.py
index 7266b699..b28830f5 100644
--- a/cloudinit/sources/helpers/vmware/imc/config_namespace.py
+++ b/cloudinit/sources/helpers/vmware/imc/config_namespace.py
@@ -1,25 +1,25 @@
-# vi: ts=4 expandtab

-#

-#    Copyright (C) 2015 Canonical Ltd.

-#    Copyright (C) 2015 VMware Inc.

-#

-#    Author: Sankar Tanguturi <stanguturi@vmware.com>

-#

-#    This program is free software: you can redistribute it and/or modify

-#    it under the terms of the GNU General Public License version 3, as

-#    published by the Free Software Foundation.

-#

-#    This program is distributed in the hope that it will be useful,

-#    but WITHOUT ANY WARRANTY; without even the implied warranty of

-#    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the

-#    GNU General Public License for more details.

-#

-#    You should have received a copy of the GNU General Public License

-#    along with this program.  If not, see <http://www.gnu.org/licenses/>.

-

-from .config_source import ConfigSource

-

-

-class ConfigNamespace(ConfigSource):

-    """Specifies the Config Namespace."""

-    pass

+# vi: ts=4 expandtab
+#
+#    Copyright (C) 2015 Canonical Ltd.
+#    Copyright (C) 2015 VMware Inc.
+#
+#    Author: Sankar Tanguturi <stanguturi@vmware.com>
+#
+#    This program is free software: you can redistribute it and/or modify
+#    it under the terms of the GNU General Public License version 3, as
+#    published by the Free Software Foundation.
+#
+#    This program is distributed in the hope that it will be useful,
+#    but WITHOUT ANY WARRANTY; without even the implied warranty of
+#    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+#    GNU General Public License for more details.
+#
+#    You should have received a copy of the GNU General Public License
+#    along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+from .config_source import ConfigSource
+
+
+class ConfigNamespace(ConfigSource):
+    """Specifies the Config Namespace."""
+    pass
diff --git a/cloudinit/sources/helpers/vmware/imc/config_nic.py b/cloudinit/sources/helpers/vmware/imc/config_nic.py
index 77098a05..511cc918 100644
--- a/cloudinit/sources/helpers/vmware/imc/config_nic.py
+++ b/cloudinit/sources/helpers/vmware/imc/config_nic.py
@@ -26,7 +26,7 @@ from cloudinit import util
 logger = logging.getLogger(__name__)
 
 
-class NicConfigurator:
+class NicConfigurator(object):
     def __init__(self, nics):
         """
         Initialize the Nic Configurator
diff --git a/cloudinit/sources/helpers/vmware/imc/config_source.py b/cloudinit/sources/helpers/vmware/imc/config_source.py
index a367e476..28ef306a 100644
--- a/cloudinit/sources/helpers/vmware/imc/config_source.py
+++ b/cloudinit/sources/helpers/vmware/imc/config_source.py
@@ -1,23 +1,23 @@
-# vi: ts=4 expandtab

-#

-#    Copyright (C) 2015 Canonical Ltd.

-#    Copyright (C) 2015 VMware Inc.

-#

-#    Author: Sankar Tanguturi <stanguturi@vmware.com>

-#

-#    This program is free software: you can redistribute it and/or modify

-#    it under the terms of the GNU General Public License version 3, as

-#    published by the Free Software Foundation.

-#

-#    This program is distributed in the hope that it will be useful,

-#    but WITHOUT ANY WARRANTY; without even the implied warranty of

-#    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the

-#    GNU General Public License for more details.

-#

-#    You should have received a copy of the GNU General Public License

-#    along with this program.  If not, see <http://www.gnu.org/licenses/>.

-

-

-class ConfigSource:

-    """Specifies a source for the Config Content."""

-    pass

+# vi: ts=4 expandtab
+#
+#    Copyright (C) 2015 Canonical Ltd.
+#    Copyright (C) 2015 VMware Inc.
+#
+#    Author: Sankar Tanguturi <stanguturi@vmware.com>
+#
+#    This program is free software: you can redistribute it and/or modify
+#    it under the terms of the GNU General Public License version 3, as
+#    published by the Free Software Foundation.
+#
+#    This program is distributed in the hope that it will be useful,
+#    but WITHOUT ANY WARRANTY; without even the implied warranty of
+#    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+#    GNU General Public License for more details.
+#
+#    You should have received a copy of the GNU General Public License
+#    along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+
+class ConfigSource(object):
+    """Specifies a source for the Config Content."""
+    pass
diff --git a/cloudinit/sources/helpers/vmware/imc/guestcust_error.py b/cloudinit/sources/helpers/vmware/imc/guestcust_error.py
index 1b04161f..d1546852 100644
--- a/cloudinit/sources/helpers/vmware/imc/guestcust_error.py
+++ b/cloudinit/sources/helpers/vmware/imc/guestcust_error.py
@@ -1,24 +1,24 @@
-# vi: ts=4 expandtab

-#

-#    Copyright (C) 2016 Canonical Ltd.

-#    Copyright (C) 2016 VMware Inc.

-#

-#    Author: Sankar Tanguturi <stanguturi@vmware.com>

-#

-#    This program is free software: you can redistribute it and/or modify

-#    it under the terms of the GNU General Public License version 3, as

-#    published by the Free Software Foundation.

-#

-#    This program is distributed in the hope that it will be useful,

-#    but WITHOUT ANY WARRANTY; without even the implied warranty of

-#    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the

-#    GNU General Public License for more details.

-#

-#    You should have received a copy of the GNU General Public License

-#    along with this program.  If not, see <http://www.gnu.org/licenses/>.

-

-

-class GuestCustErrorEnum:

-    """Specifies different errors of Guest Customization engine"""

-

-    GUESTCUST_ERROR_SUCCESS = 0

+# vi: ts=4 expandtab
+#
+#    Copyright (C) 2016 Canonical Ltd.
+#    Copyright (C) 2016 VMware Inc.
+#
+#    Author: Sankar Tanguturi <stanguturi@vmware.com>
+#
+#    This program is free software: you can redistribute it and/or modify
+#    it under the terms of the GNU General Public License version 3, as
+#    published by the Free Software Foundation.
+#
+#    This program is distributed in the hope that it will be useful,
+#    but WITHOUT ANY WARRANTY; without even the implied warranty of
+#    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+#    GNU General Public License for more details.
+#
+#    You should have received a copy of the GNU General Public License
+#    along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+
+class GuestCustErrorEnum(object):
+    """Specifies different errors of Guest Customization engine"""
+
+    GUESTCUST_ERROR_SUCCESS = 0
diff --git a/cloudinit/sources/helpers/vmware/imc/guestcust_event.py b/cloudinit/sources/helpers/vmware/imc/guestcust_event.py
index fc22568f..ce90c898 100644
--- a/cloudinit/sources/helpers/vmware/imc/guestcust_event.py
+++ b/cloudinit/sources/helpers/vmware/imc/guestcust_event.py
@@ -1,27 +1,27 @@
-# vi: ts=4 expandtab

-#

-#    Copyright (C) 2016 Canonical Ltd.

-#    Copyright (C) 2016 VMware Inc.

-#

-#    Author: Sankar Tanguturi <stanguturi@vmware.com>

-#

-#    This program is free software: you can redistribute it and/or modify

-#    it under the terms of the GNU General Public License version 3, as

-#    published by the Free Software Foundation.

-#

-#    This program is distributed in the hope that it will be useful,

-#    but WITHOUT ANY WARRANTY; without even the implied warranty of

-#    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the

-#    GNU General Public License for more details.

-#

-#    You should have received a copy of the GNU General Public License

-#    along with this program.  If not, see <http://www.gnu.org/licenses/>.

-

-

-class GuestCustEventEnum:

-    """Specifies different types of Guest Customization Events"""

-

-    GUESTCUST_EVENT_CUSTOMIZE_FAILED = 100

-    GUESTCUST_EVENT_NETWORK_SETUP_FAILED = 101

-    GUESTCUST_EVENT_ENABLE_NICS = 103

-    GUESTCUST_EVENT_QUERY_NICS = 104

+# vi: ts=4 expandtab
+#
+#    Copyright (C) 2016 Canonical Ltd.
+#    Copyright (C) 2016 VMware Inc.
+#
+#    Author: Sankar Tanguturi <stanguturi@vmware.com>
+#
+#    This program is free software: you can redistribute it and/or modify
+#    it under the terms of the GNU General Public License version 3, as
+#    published by the Free Software Foundation.
+#
+#    This program is distributed in the hope that it will be useful,
+#    but WITHOUT ANY WARRANTY; without even the implied warranty of
+#    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+#    GNU General Public License for more details.
+#
+#    You should have received a copy of the GNU General Public License
+#    along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+
+class GuestCustEventEnum(object):
+    """Specifies different types of Guest Customization Events"""
+
+    GUESTCUST_EVENT_CUSTOMIZE_FAILED = 100
+    GUESTCUST_EVENT_NETWORK_SETUP_FAILED = 101
+    GUESTCUST_EVENT_ENABLE_NICS = 103
+    GUESTCUST_EVENT_QUERY_NICS = 104
diff --git a/cloudinit/sources/helpers/vmware/imc/guestcust_state.py b/cloudinit/sources/helpers/vmware/imc/guestcust_state.py
index f255be5f..422a096d 100644
--- a/cloudinit/sources/helpers/vmware/imc/guestcust_state.py
+++ b/cloudinit/sources/helpers/vmware/imc/guestcust_state.py
@@ -1,25 +1,25 @@
-# vi: ts=4 expandtab

-#

-#    Copyright (C) 2016 Canonical Ltd.

-#    Copyright (C) 2016 VMware Inc.

-#

-#    Author: Sankar Tanguturi <stanguturi@vmware.com>

-#

-#    This program is free software: you can redistribute it and/or modify

-#    it under the terms of the GNU General Public License version 3, as

-#    published by the Free Software Foundation.

-#

-#    This program is distributed in the hope that it will be useful,

-#    but WITHOUT ANY WARRANTY; without even the implied warranty of

-#    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the

-#    GNU General Public License for more details.

-#

-#    You should have received a copy of the GNU General Public License

-#    along with this program.  If not, see <http://www.gnu.org/licenses/>.

-

-

-class GuestCustStateEnum:

-    """Specifies different states of Guest Customization engine"""

-

-    GUESTCUST_STATE_RUNNING = 4

-    GUESTCUST_STATE_DONE = 5

+# vi: ts=4 expandtab
+#
+#    Copyright (C) 2016 Canonical Ltd.
+#    Copyright (C) 2016 VMware Inc.
+#
+#    Author: Sankar Tanguturi <stanguturi@vmware.com>
+#
+#    This program is free software: you can redistribute it and/or modify
+#    it under the terms of the GNU General Public License version 3, as
+#    published by the Free Software Foundation.
+#
+#    This program is distributed in the hope that it will be useful,
+#    but WITHOUT ANY WARRANTY; without even the implied warranty of
+#    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+#    GNU General Public License for more details.
+#
+#    You should have received a copy of the GNU General Public License
+#    along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+
+class GuestCustStateEnum(object):
+    """Specifies different states of Guest Customization engine"""
+
+    GUESTCUST_STATE_RUNNING = 4
+    GUESTCUST_STATE_DONE = 5
diff --git a/cloudinit/sources/helpers/vmware/imc/guestcust_util.py b/cloudinit/sources/helpers/vmware/imc/guestcust_util.py
index d39f0a65..c07c5949 100644
--- a/cloudinit/sources/helpers/vmware/imc/guestcust_util.py
+++ b/cloudinit/sources/helpers/vmware/imc/guestcust_util.py
@@ -1,128 +1,128 @@
-# vi: ts=4 expandtab

-#

-#    Copyright (C) 2016 Canonical Ltd.

-#    Copyright (C) 2016 VMware Inc.

-#

-#    Author: Sankar Tanguturi <stanguturi@vmware.com>

-#

-#    This program is free software: you can redistribute it and/or modify

-#    it under the terms of the GNU General Public License version 3, as

-#    published by the Free Software Foundation.

-#

-#    This program is distributed in the hope that it will be useful,

-#    but WITHOUT ANY WARRANTY; without even the implied warranty of

-#    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the

-#    GNU General Public License for more details.

-#

-#    You should have received a copy of the GNU General Public License

-#    along with this program.  If not, see <http://www.gnu.org/licenses/>.

-

-import logging

-import os

-import time

-

-from cloudinit import util

-

-from .guestcust_state import GuestCustStateEnum

-from .guestcust_event import GuestCustEventEnum

-

-logger = logging.getLogger(__name__)

-

-

-CLOUDINIT_LOG_FILE = "/var/log/cloud-init.log"

-QUERY_NICS_SUPPORTED = "queryNicsSupported"

-NICS_STATUS_CONNECTED = "connected"

-

-

-# This will send a RPC command to the underlying

-# VMware Virtualization Platform.

-def send_rpc(rpc):

-    if not rpc:

-        return None

-

-    out = ""

-    err = "Error sending the RPC command"

-

-    try:

-        logger.debug("Sending RPC command: %s", rpc)

-        (out, err) = util.subp(["vmware-rpctool", rpc], rcs=[0])

-        # Remove the trailing newline in the output.

-        if out:

-            out = out.rstrip()

-    except Exception as e:

-        logger.debug("Failed to send RPC command")

-        logger.exception(e)

-

-    return (out, err)

-

-

-# This will send the customization status to the

-# underlying VMware Virtualization Platform.

-def set_customization_status(custstate, custerror, errormessage=None):

-    message = ""

-

-    if errormessage:

-        message = CLOUDINIT_LOG_FILE + "@" + errormessage

-    else:

-        message = CLOUDINIT_LOG_FILE

-

-    rpc = "deployPkg.update.state %d %d %s" % (custstate, custerror, message)

-    (out, err) = send_rpc(rpc)

-    return (out, err)

-

-

-# This will read the file nics.txt in the specified directory

-# and return the content

-def get_nics_to_enable(dirpath):

-    if not dirpath:

-        return None

-

-    NICS_SIZE = 1024

-    nicsfilepath = os.path.join(dirpath, "nics.txt")

-    if not os.path.exists(nicsfilepath):

-        return None

-

-    with open(nicsfilepath, 'r') as fp:

-        nics = fp.read(NICS_SIZE)

-

-    return nics

-

-

-# This will send a RPC command to the underlying VMware Virtualization platform

-# and enable nics.

-def enable_nics(nics):

-    if not nics:

-        logger.warning("No Nics found")

-        return

-

-    enableNicsWaitRetries = 5

-    enableNicsWaitCount = 5

-    enableNicsWaitSeconds = 1

-

-    for attempt in range(0, enableNicsWaitRetries):

-        logger.debug("Trying to connect interfaces, attempt %d", attempt)

-        (out, err) = set_customization_status(

-            GuestCustStateEnum.GUESTCUST_STATE_RUNNING,

-            GuestCustEventEnum.GUESTCUST_EVENT_ENABLE_NICS,

-            nics)

-        if not out:

-            time.sleep(enableNicsWaitCount * enableNicsWaitSeconds)

-            continue

-

-        if out != QUERY_NICS_SUPPORTED:

-            logger.warning("NICS connection status query is not supported")

-            return

-

-        for count in range(0, enableNicsWaitCount):

-            (out, err) = set_customization_status(

-                GuestCustStateEnum.GUESTCUST_STATE_RUNNING,

-                GuestCustEventEnum.GUESTCUST_EVENT_QUERY_NICS,

-                nics)

-            if out and out == NICS_STATUS_CONNECTED:

-                logger.info("NICS are connected on %d second", count)

-                return

-

-            time.sleep(enableNicsWaitSeconds)

-

-    logger.warning("Can't connect network interfaces after %d attempts",

-                   enableNicsWaitRetries)

+# vi: ts=4 expandtab
+#
+#    Copyright (C) 2016 Canonical Ltd.
+#    Copyright (C) 2016 VMware Inc.
+#
+#    Author: Sankar Tanguturi <stanguturi@vmware.com>
+#
+#    This program is free software: you can redistribute it and/or modify
+#    it under the terms of the GNU General Public License version 3, as
+#    published by the Free Software Foundation.
+#
+#    This program is distributed in the hope that it will be useful,
+#    but WITHOUT ANY WARRANTY; without even the implied warranty of
+#    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+#    GNU General Public License for more details.
+#
+#    You should have received a copy of the GNU General Public License
+#    along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+import logging
+import os
+import time
+
+from cloudinit import util
+
+from .guestcust_event import GuestCustEventEnum
+from .guestcust_state import GuestCustStateEnum
+
+logger = logging.getLogger(__name__)
+
+
+CLOUDINIT_LOG_FILE = "/var/log/cloud-init.log"
+QUERY_NICS_SUPPORTED = "queryNicsSupported"
+NICS_STATUS_CONNECTED = "connected"
+
+
+# This will send a RPC command to the underlying
+# VMware Virtualization Platform.
+def send_rpc(rpc):
+    if not rpc:
+        return None
+
+    out = ""
+    err = "Error sending the RPC command"
+
+    try:
+        logger.debug("Sending RPC command: %s", rpc)
+        (out, err) = util.subp(["vmware-rpctool", rpc], rcs=[0])
+        # Remove the trailing newline in the output.
+        if out:
+            out = out.rstrip()
+    except Exception as e:
+        logger.debug("Failed to send RPC command")
+        logger.exception(e)
+
+    return (out, err)
+
+
+# This will send the customization status to the
+# underlying VMware Virtualization Platform.
+def set_customization_status(custstate, custerror, errormessage=None):
+    message = ""
+
+    if errormessage:
+        message = CLOUDINIT_LOG_FILE + "@" + errormessage
+    else:
+        message = CLOUDINIT_LOG_FILE
+
+    rpc = "deployPkg.update.state %d %d %s" % (custstate, custerror, message)
+    (out, err) = send_rpc(rpc)
+    return (out, err)
+
+
+# This will read the file nics.txt in the specified directory
+# and return the content
+def get_nics_to_enable(dirpath):
+    if not dirpath:
+        return None
+
+    NICS_SIZE = 1024
+    nicsfilepath = os.path.join(dirpath, "nics.txt")
+    if not os.path.exists(nicsfilepath):
+        return None
+
+    with open(nicsfilepath, 'r') as fp:
+        nics = fp.read(NICS_SIZE)
+
+    return nics
+
+
+# This will send a RPC command to the underlying VMware Virtualization platform
+# and enable nics.
+def enable_nics(nics):
+    if not nics:
+        logger.warning("No Nics found")
+        return
+
+    enableNicsWaitRetries = 5
+    enableNicsWaitCount = 5
+    enableNicsWaitSeconds = 1
+
+    for attempt in range(0, enableNicsWaitRetries):
+        logger.debug("Trying to connect interfaces, attempt %d", attempt)
+        (out, err) = set_customization_status(
+            GuestCustStateEnum.GUESTCUST_STATE_RUNNING,
+            GuestCustEventEnum.GUESTCUST_EVENT_ENABLE_NICS,
+            nics)
+        if not out:
+            time.sleep(enableNicsWaitCount * enableNicsWaitSeconds)
+            continue
+
+        if out != QUERY_NICS_SUPPORTED:
+            logger.warning("NICS connection status query is not supported")
+            return
+
+        for count in range(0, enableNicsWaitCount):
+            (out, err) = set_customization_status(
+                GuestCustStateEnum.GUESTCUST_STATE_RUNNING,
+                GuestCustEventEnum.GUESTCUST_EVENT_QUERY_NICS,
+                nics)
+            if out and out == NICS_STATUS_CONNECTED:
+                logger.info("NICS are connected on %d second", count)
+                return
+
+            time.sleep(enableNicsWaitSeconds)
+
+    logger.warning("Can't connect network interfaces after %d attempts",
+                   enableNicsWaitRetries)
diff --git a/cloudinit/sources/helpers/vmware/imc/ipv4_mode.py b/cloudinit/sources/helpers/vmware/imc/ipv4_mode.py
index 33f88726..873ddc3b 100644
--- a/cloudinit/sources/helpers/vmware/imc/ipv4_mode.py
+++ b/cloudinit/sources/helpers/vmware/imc/ipv4_mode.py
@@ -18,7 +18,7 @@
 #    along with this program.  If not, see <http://www.gnu.org/licenses/>.
 
 
-class Ipv4ModeEnum:
+class Ipv4ModeEnum(object):
     """
     The IPv4 configuration mode which directly represents the user's goal.
 
diff --git a/cloudinit/sources/helpers/vmware/imc/nic_base.py b/cloudinit/sources/helpers/vmware/imc/nic_base.py
index 030ba311..3c892db0 100644
--- a/cloudinit/sources/helpers/vmware/imc/nic_base.py
+++ b/cloudinit/sources/helpers/vmware/imc/nic_base.py
@@ -18,7 +18,7 @@
 #    along with this program.  If not, see <http://www.gnu.org/licenses/>.
 
 
-class NicBase:
+class NicBase(object):
     """
     Define what are expected of each nic.
     The following properties should be provided in an implementation class.
@@ -93,7 +93,7 @@ class NicBase:
         raise NotImplementedError('Check constraints on properties')
 
 
-class StaticIpv4Base:
+class StaticIpv4Base(object):
     """
     Define what are expected of a static IPv4 setting
     The following properties should be provided in an implementation class.
@@ -124,7 +124,7 @@ class StaticIpv4Base:
         raise NotImplementedError('Ipv4 GATEWAY')
 
 
-class StaticIpv6Base:
+class StaticIpv6Base(object):
     """Define what are expected of a static IPv6 setting
     The following properties should be provided in an implementation class.
     """
diff --git a/cloudinit/stages.py b/cloudinit/stages.py
index ffb15165..47deac6e 100644
--- a/cloudinit/stages.py
+++ b/cloudinit/stages.py
@@ -44,14 +44,16 @@ from cloudinit import helpers
 from cloudinit import importer
 from cloudinit import log as logging
 from cloudinit import net
+from cloudinit.net import cmdline
+from cloudinit.reporting import events
 from cloudinit import sources
 from cloudinit import type_utils
 from cloudinit import util
-from cloudinit.reporting import events
 
 LOG = logging.getLogger(__name__)
 
 NULL_DATA_SOURCE = None
+NO_PREVIOUS_INSTANCE_ID = "NO_PREVIOUS_INSTANCE_ID"
 
 
 class Init(object):
@@ -67,6 +69,7 @@ class Init(object):
         # Changed only when a fetch occurs
         self.datasource = NULL_DATA_SOURCE
         self.ds_restored = False
+        self._previous_iid = None
 
         if reporter is None:
             reporter = events.ReportEventStack(
@@ -213,6 +216,31 @@ class Init(object):
         cfg_list = self.cfg.get('datasource_list') or []
         return (cfg_list, pkg_list)
 
+    def _restore_from_checked_cache(self, existing):
+        if existing not in ("check", "trust"):
+            raise ValueError("Unexpected value for existing: %s" % existing)
+
+        ds = self._restore_from_cache()
+        if not ds:
+            return (None, "no cache found")
+
+        run_iid_fn = self.paths.get_runpath('instance_id')
+        if os.path.exists(run_iid_fn):
+            run_iid = util.load_file(run_iid_fn).strip()
+        else:
+            run_iid = None
+
+        if run_iid == ds.get_instance_id():
+            return (ds, "restored from cache with run check: %s" % ds)
+        elif existing == "trust":
+            return (ds, "restored from cache: %s" % ds)
+        else:
+            if (hasattr(ds, 'check_instance_id') and
+                    ds.check_instance_id(self.cfg)):
+                return (ds, "restored from checked cache: %s" % ds)
+            else:
+                return (None, "cache invalid in datasource: %s" % ds)
+
     def _get_data_source(self, existing):
         if self.datasource is not NULL_DATA_SOURCE:
             return self.datasource
@@ -221,19 +249,9 @@ class Init(object):
                 name="check-cache",
                 description="attempting to read from cache [%s]" % existing,
                 parent=self.reporter) as myrep:
-            ds = self._restore_from_cache()
-            if ds and existing == "trust":
-                myrep.description = "restored from cache: %s" % ds
-            elif ds and existing == "check":
-                if (hasattr(ds, 'check_instance_id') and
-                        ds.check_instance_id(self.cfg)):
-                    myrep.description = "restored from checked cache: %s" % ds
-                else:
-                    myrep.description = "cache invalid in datasource: %s" % ds
-                    ds = None
-            else:
-                myrep.description = "no cache found"
 
+            ds, desc = self._restore_from_checked_cache(existing)
+            myrep.description = desc
             self.ds_restored = bool(ds)
             LOG.debug(myrep.description)
 
@@ -301,23 +319,41 @@ class Init(object):
 
         # What the instance id was and is...
         iid = self.datasource.get_instance_id()
-        previous_iid = None
         iid_fn = os.path.join(dp, 'instance-id')
-        try:
-            previous_iid = util.load_file(iid_fn).strip()
-        except Exception:
-            pass
-        if not previous_iid:
-            previous_iid = iid
+
+        previous_iid = self.previous_iid()
         util.write_file(iid_fn, "%s\n" % iid)
+        util.write_file(self.paths.get_runpath('instance_id'), "%s\n" % iid)
         util.write_file(os.path.join(dp, 'previous-instance-id'),
                         "%s\n" % (previous_iid))
+
+        self._write_to_cache()
         # Ensure needed components are regenerated
         # after change of instance which may cause
         # change of configuration
         self._reset()
         return iid
 
+    def previous_iid(self):
+        if self._previous_iid is not None:
+            return self._previous_iid
+
+        dp = self.paths.get_cpath('data')
+        iid_fn = os.path.join(dp, 'instance-id')
+        try:
+            self._previous_iid = util.load_file(iid_fn).strip()
+        except Exception:
+            self._previous_iid = NO_PREVIOUS_INSTANCE_ID
+
+        LOG.debug("previous iid found to be %s", self._previous_iid)
+        return self._previous_iid
+
+    def is_new_instance(self):
+        previous = self.previous_iid()
+        ret = (previous == NO_PREVIOUS_INSTANCE_ID or
+               previous != self.datasource.get_instance_id())
+        return ret
+
     def fetch(self, existing="check"):
         return self._get_data_source(existing=existing)
 
@@ -332,8 +368,6 @@ class Init(object):
                            reporter=self.reporter)
 
     def update(self):
-        if not self._write_to_cache():
-            return
         self._store_userdata()
         self._store_vendordata()
 
@@ -483,7 +517,7 @@ class Init(object):
                 c_handlers.initialized.remove(mod)
                 try:
                     handlers.call_end(mod, data, frequency)
-                except:
+                except Exception:
                     util.logexc(LOG, "Failed to finalize handler: %s", mod)
 
         try:
@@ -579,13 +613,13 @@ class Init(object):
         if os.path.exists(disable_file):
             return (None, disable_file)
 
-        cmdline_cfg = ('cmdline', net.read_kernel_cmdline_config())
+        cmdline_cfg = ('cmdline', cmdline.read_kernel_cmdline_config())
         dscfg = ('ds', None)
         if self.datasource and hasattr(self.datasource, 'network_config'):
             dscfg = ('ds', self.datasource.network_config)
         sys_cfg = ('system_cfg', self.cfg.get('network'))
 
-        for loc, ncfg in (cmdline_cfg, dscfg, sys_cfg):
+        for loc, ncfg in (cmdline_cfg, sys_cfg, dscfg):
             if net.is_disabled_cfg(ncfg):
                 LOG.debug("network config disabled by %s", loc)
                 return (None, loc)
@@ -593,15 +627,27 @@ class Init(object):
                 return (ncfg, loc)
         return (net.generate_fallback_config(), "fallback")
 
-    def apply_network_config(self):
+    def apply_network_config(self, bring_up):
         netcfg, src = self._find_networking_config()
         if netcfg is None:
             LOG.info("network config is disabled by %s", src)
             return
 
-        LOG.info("Applying network configuration from %s: %s", src, netcfg)
         try:
-            return self.distro.apply_network_config(netcfg)
+            LOG.debug("applying net config names for %s" % netcfg)
+            self.distro.apply_network_config_names(netcfg)
+        except Exception as e:
+            LOG.warn("Failed to rename devices: %s", e)
+
+        if (self.datasource is not NULL_DATA_SOURCE and
+                not self.is_new_instance()):
+            LOG.debug("not a new instance. network config is not applied.")
+            return
+
+        LOG.info("Applying network configuration from %s bringup=%s: %s",
+                 src, bring_up, netcfg)
+        try:
+            return self.distro.apply_network_config(netcfg, bring_up=bring_up)
         except NotImplementedError:
             LOG.warn("distro '%s' does not implement apply_network_config. "
                      "networking may not be configured properly." %
@@ -794,16 +840,16 @@ class Modules(object):
 def fetch_base_config():
     base_cfgs = []
     default_cfg = util.get_builtin_cfg()
-    kern_contents = util.read_cc_from_cmdline()
-
-    # Kernel/cmdline parameters override system config
-    if kern_contents:
-        base_cfgs.append(util.load_yaml(kern_contents, default={}))
 
     # Anything in your conf.d location??
     # or the 'default' cloud.cfg location???
     base_cfgs.append(util.read_conf_with_confd(CLOUD_CONFIG))
 
+    # Kernel/cmdline parameters override system config
+    kern_contents = util.read_cc_from_cmdline()
+    if kern_contents:
+        base_cfgs.append(util.load_yaml(kern_contents, default={}))
+
     # And finally the default gets to play
     if default_cfg:
         base_cfgs.append(default_cfg)
diff --git a/cloudinit/templater.py b/cloudinit/templater.py
index a9231482..41ef27e3 100644
--- a/cloudinit/templater.py
+++ b/cloudinit/templater.py
@@ -3,10 +3,12 @@
 #    Copyright (C) 2012 Canonical Ltd.
 #    Copyright (C) 2012 Hewlett-Packard Development Company, L.P.
 #    Copyright (C) 2012 Yahoo! Inc.
+#    Copyright (C) 2016 Amazon.com, Inc. or its affiliates.
 #
 #    Author: Scott Moser <scott.moser@canonical.com>
 #    Author: Juerg Haefliger <juerg.haefliger@hp.com>
 #    Author: Joshua Harlow <harlowja@yahoo-inc.com>
+#    Author: Andrew Jorgensen <ajorgens@amazon.com>
 #
 #    This program is free software: you can redistribute it and/or modify
 #    it under the terms of the GNU General Public License version 3, as
@@ -102,12 +104,11 @@ def detect_template(text):
         rest = ''
     type_match = TYPE_MATCHER.match(ident)
     if not type_match:
-        if not CHEETAH_AVAILABLE:
-            LOG.warn("Cheetah not available as the default renderer for"
-                     " unknown template, reverting to the basic renderer.")
-            return ('basic', basic_render, text)
-        else:
+        if CHEETAH_AVAILABLE:
+            LOG.debug("Using Cheetah as the renderer for unknown template.")
             return ('cheetah', cheetah_render, text)
+        else:
+            return ('basic', basic_render, text)
     else:
         template_type = type_match.group(1).lower().strip()
         if template_type not in ('jinja', 'cheetah', 'basic'):
@@ -142,6 +143,11 @@ def render_to_file(fn, outfn, params, mode=0o644):
     util.write_file(outfn, contents, mode=mode)
 
 
+def render_string_to_file(content, outfn, params, mode=0o644):
+    contents = render_string(content, params)
+    util.write_file(outfn, contents, mode=mode)
+
+
 def render_string(content, params):
     if not params:
         params = {}
diff --git a/cloudinit/url_helper.py b/cloudinit/url_helper.py
index 936f7da5..c05e9d90 100644
--- a/cloudinit/url_helper.py
+++ b/cloudinit/url_helper.py
@@ -28,8 +28,9 @@ import time
 
 from email.utils import parsedate
 from functools import partial
-from requests import exceptions
+
 import oauthlib.oauth1 as oauth1
+from requests import exceptions
 
 from six.moves.urllib.parse import (
     urlparse, urlunparse,
@@ -61,7 +62,7 @@ try:
         SSL_ENABLED = True
     if _REQ_VER >= LooseVersion('0.7.0') and _REQ_VER < LooseVersion('1.0.0'):
         CONFIG_ENABLED = True
-except:
+except ImportError:
     pass
 
 
diff --git a/cloudinit/user_data.py b/cloudinit/user_data.py
index f7c5787c..393bf0bb 100644
--- a/cloudinit/user_data.py
+++ b/cloudinit/user_data.py
@@ -178,7 +178,7 @@ class UserDataProcessor(object):
                 payload = util.load_yaml(msg.get_payload(decode=True))
                 if payload:
                     payload_idx = payload.get('launch-index')
-            except:
+            except Exception:
                 pass
         # Header overrides contents, for now (?) or the other way around?
         if header_idx is not None:
@@ -334,19 +334,23 @@ def is_skippable(part):
 
 
 # Coverts a raw string into a mime message
-def convert_string(raw_data, headers=None):
+def convert_string(raw_data, content_type=NOT_MULTIPART_TYPE):
     if not raw_data:
         raw_data = ''
-    if not headers:
-        headers = {}
-    data = util.decode_binary(util.decomp_gzip(raw_data))
-    if "mime-version:" in data[0:4096].lower():
-        msg = util.message_from_string(data)
-        for (key, val) in headers.items():
-            _replace_header(msg, key, val)
-    else:
-        mtype = headers.get(CONTENT_TYPE, NOT_MULTIPART_TYPE)
-        maintype, subtype = mtype.split("/", 1)
-        msg = MIMEBase(maintype, subtype, *headers)
+
+    def create_binmsg(data, content_type):
+        maintype, subtype = content_type.split("/", 1)
+        msg = MIMEBase(maintype, subtype)
         msg.set_payload(data)
+        return msg
+
+    try:
+        data = util.decode_binary(util.decomp_gzip(raw_data))
+        if "mime-version:" in data[0:4096].lower():
+            msg = util.message_from_string(data)
+        else:
+            msg = create_binmsg(data, content_type)
+    except UnicodeDecodeError:
+        msg = create_binmsg(raw_data, content_type)
+
     return msg
diff --git a/cloudinit/util.py b/cloudinit/util.py
index 0d21e11b..e5dd61a0 100644
--- a/cloudinit/util.py
+++ b/cloudinit/util.py
@@ -171,7 +171,8 @@ class ProcessExecutionError(IOError):
 
     def __init__(self, stdout=None, stderr=None,
                  exit_code=None, cmd=None,
-                 description=None, reason=None):
+                 description=None, reason=None,
+                 errno=None):
         if not cmd:
             self.cmd = '-'
         else:
@@ -202,6 +203,7 @@ class ProcessExecutionError(IOError):
         else:
             self.reason = '-'
 
+        self.errno = errno
         message = self.MESSAGE_TMPL % {
             'description': self.description,
             'cmd': self.cmd,
@@ -288,7 +290,7 @@ def fork_cb(child_cb, *args, **kwargs):
         try:
             child_cb(*args, **kwargs)
             os._exit(0)
-        except:
+        except Exception:
             logexc(LOG, "Failed forking and calling callback %s",
                    type_utils.obj_name(child_cb))
             os._exit(1)
@@ -336,6 +338,16 @@ def rand_str(strlen=32, select_from=None):
     return "".join([random.choice(select_from) for _x in range(0, strlen)])
 
 
+def rand_dict_key(dictionary, postfix=None):
+    if not postfix:
+        postfix = ""
+    while True:
+        newkey = rand_str(strlen=8) + "_" + postfix
+        if newkey not in dictionary:
+            break
+    return newkey
+
+
 def read_conf(fname):
     try:
         return load_yaml(load_file(fname), default={})
@@ -472,7 +484,7 @@ def is_ipv4(instr):
 
     try:
         toks = [x for x in toks if int(x) < 256 and int(x) >= 0]
-    except:
+    except Exception:
         return False
 
     return len(toks) == 4
@@ -1147,7 +1159,14 @@ def find_devs_with(criteria=None, oformat='device',
         options.append(path)
     cmd = blk_id_cmd + options
     # See man blkid for why 2 is added
-    (out, _err) = subp(cmd, rcs=[0, 2])
+    try:
+        (out, _err) = subp(cmd, rcs=[0, 2])
+    except ProcessExecutionError as e:
+        if e.errno == errno.ENOENT:
+            # blkid not found...
+            out = ""
+        else:
+            raise
     entries = []
     for line in out.splitlines():
         line = line.strip()
@@ -1210,7 +1229,7 @@ def get_cmdline():
     else:
         try:
             cmdline = load_file("/proc/cmdline").strip()
-        except:
+        except Exception:
             cmdline = ""
 
     PROC_CMDLINE = cmdline
@@ -1380,7 +1399,7 @@ def read_write_cmdline_url(target_fn):
     if not os.path.exists(target_fn):
         try:
             (key, url, content) = get_cmdline_url()
-        except:
+        except Exception:
             logexc(LOG, "Failed fetching command line url")
             return
         try:
@@ -1391,7 +1410,7 @@ def read_write_cmdline_url(target_fn):
             elif key and not content:
                 LOG.debug(("Command line key %s with url"
                           " %s had no contents"), key, url)
-        except:
+        except Exception:
             logexc(LOG, "Failed writing url content to %s", target_fn)
 
 
@@ -1449,7 +1468,7 @@ def mounts():
                     mp = m.group(2)
                     fstype = m.group(3)
                     opts = m.group(4)
-            except:
+            except Exception:
                 continue
             # If the name of the mount point contains spaces these
             # can be escaped as '\040', so undo that..
@@ -1575,7 +1594,7 @@ def copy(src, dest):
 def time_rfc2822():
     try:
         ts = time.strftime("%a, %d %b %Y %H:%M:%S %z", time.gmtime())
-    except:
+    except Exception:
         ts = "??"
     return ts
 
@@ -1601,7 +1620,7 @@ def uptime():
             bootup = buf.value
             uptime_str = now - bootup
 
-    except:
+    except Exception:
         logexc(LOG, "Unable to read uptime using method: %s" % method)
     return uptime_str
 
@@ -1696,7 +1715,8 @@ def subp(args, data=None, rcs=None, env=None, capture=True, shell=False,
         sp = subprocess.Popen(args, **kws)
         (out, err) = sp.communicate(data)
     except OSError as e:
-        raise ProcessExecutionError(cmd=args, reason=e)
+        raise ProcessExecutionError(cmd=args, reason=e,
+                                    errno=e.errno)
     rc = sp.returncode
     if rc not in rcs:
         raise ProcessExecutionError(stdout=out, stderr=err,
@@ -2055,7 +2075,7 @@ def log_time(logfunc, msg, func, args=None, kwargs=None, get_uptime=False):
                 tmsg += " (N/A)"
         try:
             logfunc(msg + tmsg)
-        except:
+        except Exception:
             pass
     return ret
 
@@ -2190,7 +2210,7 @@ def _call_dmidecode(key, dmidecode_path):
             return ""
         return result
     except (IOError, OSError) as _err:
-        LOG.debug('failed dmidecode cmd: %s\n%s', cmd, _err.message)
+        LOG.debug('failed dmidecode cmd: %s\n%s', cmd, _err)
         return None