Rebase against master

8 files changed, 589 insertions, 366 deletions

diff --git a/cloudinit/sources/DataSourceCloudSigma.py b/cloudinit/sources/DataSourceCloudSigma.py
index 33fe78b9..d1f806d6 100644
--- a/cloudinit/sources/DataSourceCloudSigma.py
+++ b/cloudinit/sources/DataSourceCloudSigma.py
@@ -27,8 +27,6 @@ from cloudinit import util
 
 LOG = logging.getLogger(__name__)
 
-VALID_DSMODES = ("local", "net", "disabled")
-
 
 class DataSourceCloudSigma(sources.DataSource):
     """
@@ -38,7 +36,6 @@ class DataSourceCloudSigma(sources.DataSource):
     http://cloudsigma-docs.readthedocs.org/en/latest/server_context.html
     """
     def __init__(self, sys_cfg, distro, paths):
-        self.dsmode = 'local'
         self.cepko = Cepko()
         self.ssh_public_key = ''
         sources.DataSource.__init__(self, sys_cfg, distro, paths)
@@ -84,11 +81,9 @@ class DataSourceCloudSigma(sources.DataSource):
             LOG.debug("CloudSigma: Unable to read from serial port")
             return False
 
-        dsmode = server_meta.get('cloudinit-dsmode', self.dsmode)
-        if dsmode not in VALID_DSMODES:
-            LOG.warn("Invalid dsmode %s, assuming default of 'net'", dsmode)
-            dsmode = 'net'
-        if dsmode == "disabled" or dsmode != self.dsmode:
+        self.dsmode = self._determine_dsmode(
+            [server_meta.get('cloudinit-dsmode')])
+        if dsmode == sources.DSMODE_DISABLED:
             return False
 
         base64_fields = server_meta.get('base64_fields', '').split(',')
@@ -120,17 +115,13 @@ class DataSourceCloudSigma(sources.DataSource):
         return self.metadata['uuid']
 
 
-class DataSourceCloudSigmaNet(DataSourceCloudSigma):
-    def __init__(self, sys_cfg, distro, paths):
-        DataSourceCloudSigma.__init__(self, sys_cfg, distro, paths)
-        self.dsmode = 'net'
-
+# Legacy: Must be present in case we load an old pkl object
+DataSourceCloudSigmaNet = DataSourceCloudSigma
 
 # Used to match classes to dependencies. Since this datasource uses the serial
 # port network is not really required, so it's okay to load without it, too.
 datasources = [
     (DataSourceCloudSigma, (sources.DEP_FILESYSTEM)),
-    (DataSourceCloudSigmaNet, (sources.DEP_FILESYSTEM, sources.DEP_NETWORK)),
 ]
 
 
diff --git a/cloudinit/sources/DataSourceConfigDrive.py b/cloudinit/sources/DataSourceConfigDrive.py
index 4478c4e2..cea08de2 100644
--- a/cloudinit/sources/DataSourceConfigDrive.py
+++ b/cloudinit/sources/DataSourceConfigDrive.py
@@ -24,6 +24,8 @@ from cloudinit import log as logging
 from cloudinit import sources
 from cloudinit import util
 
+from cloudinit.net import eni
+
 from cloudinit.sources.helpers import openstack
 
 LOG = logging.getLogger(__name__)
@@ -34,7 +36,6 @@ DEFAULT_MODE = 'pass'
 DEFAULT_METADATA = {
     "instance-id": DEFAULT_IID,
 }
-VALID_DSMODES = ("local", "net", "pass", "disabled")
 FS_TYPES = ('vfat', 'iso9660')
 LABEL_TYPES = ('config-2',)
 POSSIBLE_MOUNTS = ('sr', 'cd')
@@ -46,12 +47,13 @@ class DataSourceConfigDrive(openstack.SourceMixin, sources.DataSource):
     def __init__(self, sys_cfg, distro, paths):
         super(DataSourceConfigDrive, self).__init__(sys_cfg, distro, paths)
         self.source = None
-        self.dsmode = 'local'
         self.seed_dir = os.path.join(paths.seed_dir, 'config_drive')
         self.version = None
         self.ec2_metadata = None
         self._network_config = None
         self.network_json = None
+        self.network_eni = None
+        self.known_macs = None
         self.files = {}
 
     def __str__(self):
@@ -97,39 +99,23 @@ class DataSourceConfigDrive(openstack.SourceMixin, sources.DataSource):
 
         md = results.get('metadata', {})
         md = util.mergemanydict([md, DEFAULT_METADATA])
-        user_dsmode = results.get('dsmode', None)
-        if user_dsmode not in VALID_DSMODES + (None,):
-            LOG.warn("User specified invalid mode: %s", user_dsmode)
-            user_dsmode = None
 
-        dsmode = get_ds_mode(cfgdrv_ver=results['version'],
-                             ds_cfg=self.ds_cfg.get('dsmode'),
-                             user=user_dsmode)
+        self.dsmode = self._determine_dsmode(
+            [results.get('dsmode'), self.ds_cfg.get('dsmode'),
+             sources.DSMODE_PASS if results['version'] == 1 else None])
 
-        if dsmode == "disabled":
-            # most likely user specified
+        if self.dsmode == sources.DSMODE_DISABLED:
             return False
 
-        # TODO(smoser): fix this, its dirty.
-        # we want to do some things (writing files and network config)
-        # only on first boot, and even then, we want to do so in the
-        # local datasource (so they happen earlier) even if the configured
-        # dsmode is 'net' or 'pass'. To do this, we check the previous
-        # instance-id
+        # This is legacy and sneaky.  If dsmode is 'pass' then write
+        # 'injected files' and apply legacy ENI network format.
         prev_iid = get_previous_iid(self.paths)
         cur_iid = md['instance-id']
-        if prev_iid != cur_iid and \
-           self.dsmode == "local" and not skip_first_boot:
+        if (prev_iid != cur_iid and self.dsmode == sources.DSMODE_PASS
+                and not skip_first_boot):
             on_first_boot(results, distro=self.distro)
-
-        # dsmode != self.dsmode here if:
-        #  * dsmode = "pass",  pass means it should only copy files and then
-        #    pass to another datasource
-        #  * dsmode = "net" and self.dsmode = "local"
-        #    so that user boothooks would be applied with network, the
-        #    local datasource just gets out of the way, and lets the net claim
-        if dsmode != self.dsmode:
-            LOG.debug("%s: not claiming datasource, dsmode=%s", self, dsmode)
+            LOG.debug("%s: not claiming datasource, dsmode=%s", self,
+                      self.dsmode)
             return False
 
         self.source = found
@@ -147,12 +133,11 @@ class DataSourceConfigDrive(openstack.SourceMixin, sources.DataSource):
             LOG.warn("Invalid content in vendor-data: %s", e)
             self.vendordata_raw = None
 
-        try:
-            self.network_json = results.get('networkdata')
-        except ValueError as e:
-            LOG.warn("Invalid content in network-data: %s", e)
-            self.network_json = None
-
+        # network_config is an /etc/network/interfaces formated file and is
+        # obsolete compared to networkdata (from network_data.json) but both
+        # might be present.
+        self.network_eni = results.get("network_config")
+        self.network_json = results.get('networkdata')
         return True
 
     def check_instance_id(self, sys_cfg):
@@ -163,42 +148,17 @@ class DataSourceConfigDrive(openstack.SourceMixin, sources.DataSource):
     def network_config(self):
         if self._network_config is None:
             if self.network_json is not None:
+                LOG.debug("network config provided via network_json")
                 self._network_config = openstack.convert_net_json(
-                    self.network_json)
+                    self.network_json, known_macs=self.known_macs)
+            elif self.network_eni is not None:
+                self._network_config = eni.convert_eni_data(self.network_eni)
+                LOG.debug("network config provided via converted eni data")
+            else:
+                LOG.debug("no network configuration available")
         return self._network_config
 
 
-class DataSourceConfigDriveNet(DataSourceConfigDrive):
-    def __init__(self, sys_cfg, distro, paths):
-        DataSourceConfigDrive.__init__(self, sys_cfg, distro, paths)
-        self.dsmode = 'net'
-
-
-def get_ds_mode(cfgdrv_ver, ds_cfg=None, user=None):
-    """Determine what mode should be used.
-    valid values are 'pass', 'disabled', 'local', 'net'
-    """
-    # user passed data trumps everything
-    if user is not None:
-        return user
-
-    if ds_cfg is not None:
-        return ds_cfg
-
-    # at config-drive version 1, the default behavior was pass.  That
-    # meant to not use use it as primary data source, but expect a ec2 metadata
-    # source. for version 2, we default to 'net', which means
-    # the DataSourceConfigDriveNet, would be used.
-    #
-    # this could change in the future.  If there was definitive metadata
-    # that indicated presense of an openstack metadata service, then
-    # we could change to 'pass' by default also. The motivation for that
-    # would be 'cloud-init query' as the web service could be more dynamic
-    if cfgdrv_ver == 1:
-        return "pass"
-    return "net"
-
-
 def read_config_drive(source_dir):
     reader = openstack.ConfigDriveReader(source_dir)
     finders = [
@@ -232,9 +192,12 @@ def on_first_boot(data, distro=None):
                         % (type(data)))
     net_conf = data.get("network_config", '')
     if net_conf and distro:
-        LOG.debug("Updating network interfaces from config drive")
+        LOG.warn("Updating network interfaces from config drive")
         distro.apply_network(net_conf)
-    files = data.get('files', {})
+    write_injected_files(data.get('files'))
+
+
+def write_injected_files(files):
     if files:
         LOG.debug("Writing %s injected files", len(files))
         for (filename, content) in files.items():
@@ -294,10 +257,12 @@ def find_candidate_devs(probe_optical=True):
     return devices
 
 
+# Legacy: Must be present in case we load an old pkl object
+DataSourceConfigDriveNet = DataSourceConfigDrive
+
 # Used to match classes to dependencies
 datasources = [
-    (DataSourceConfigDrive, (sources.DEP_FILESYSTEM, )),
-    (DataSourceConfigDriveNet, (sources.DEP_FILESYSTEM, sources.DEP_NETWORK)),
+    (DataSourceConfigDrive, (sources.DEP_FILESYSTEM,)),
 ]
 
 
diff --git a/cloudinit/sources/DataSourceNoCloud.py b/cloudinit/sources/DataSourceNoCloud.py
index 48c61a90..7e30118c 100644
--- a/cloudinit/sources/DataSourceNoCloud.py
+++ b/cloudinit/sources/DataSourceNoCloud.py
@@ -24,6 +24,7 @@ import errno
 import os
 
 from cloudinit import log as logging
+from cloudinit import net
 from cloudinit import sources
 from cloudinit import util
 
@@ -35,7 +36,6 @@ class DataSourceNoCloud(sources.DataSource):
         sources.DataSource.__init__(self, sys_cfg, distro, paths)
         self.dsmode = 'local'
         self.seed = None
-        self.cmdline_id = "ds=nocloud"
         self.seed_dirs = [os.path.join(paths.seed_dir, 'nocloud'),
                           os.path.join(paths.seed_dir, 'nocloud-net')]
         self.seed_dir = None
@@ -58,7 +58,7 @@ class DataSourceNoCloud(sources.DataSource):
         try:
             # Parse the kernel command line, getting data passed in
             md = {}
-            if parse_cmdline_data(self.cmdline_id, md):
+            if load_cmdline_data(md):
                 found.append("cmdline")
                 mydata = _merge_new_seed(mydata, {'meta-data': md})
         except Exception:
@@ -123,12 +123,6 @@ class DataSourceNoCloud(sources.DataSource):
 
                     mydata = _merge_new_seed(mydata, seeded)
 
-                    # For seed from a device, the default mode is 'net'.
-                    # that is more likely to be what is desired.  If they want
-                    # dsmode of local, then they must specify that.
-                    if 'dsmode' not in mydata['meta-data']:
-                        mydata['meta-data']['dsmode'] = "net"
-
                     LOG.debug("Using data from %s", dev)
                     found.append(dev)
                     break
@@ -144,7 +138,6 @@ class DataSourceNoCloud(sources.DataSource):
         if len(found) == 0:
             return False
 
-        seeded_network = None
         # The special argument "seedfrom" indicates we should
         # attempt to seed the userdata / metadata from its value
         # its primarily value is in allowing the user to type less
@@ -160,10 +153,6 @@ class DataSourceNoCloud(sources.DataSource):
                 LOG.debug("Seed from %s not supported by %s", seedfrom, self)
                 return False
 
-            if (mydata['meta-data'].get('network-interfaces') or
-                    mydata.get('network-config')):
-                seeded_network = self.dsmode
-
             # This could throw errors, but the user told us to do it
             # so if errors are raised, let them raise
             (md_seed, ud) = util.read_seeded(seedfrom, timeout=None)
@@ -179,35 +168,21 @@ class DataSourceNoCloud(sources.DataSource):
         mydata['meta-data'] = util.mergemanydict([mydata['meta-data'],
                                                   defaults])
 
-        netdata = {'format': None, 'data': None}
-        if mydata['meta-data'].get('network-interfaces'):
-            netdata['format'] = 'interfaces'
-            netdata['data'] = mydata['meta-data']['network-interfaces']
-        elif mydata.get('network-config'):
-            netdata['format'] = 'network-config'
-            netdata['data'] = mydata['network-config']
-
-        # if this is the local datasource or 'seedfrom' was used
-        # and the source of the seed was self.dsmode.
-        # Then see if there is network config to apply.
-        # note this is obsolete network-interfaces style seeding.
-        if self.dsmode in ("local", seeded_network):
-            if mydata['meta-data'].get('network-interfaces'):
-                LOG.debug("Updating network interfaces from %s", self)
-                self.distro.apply_network(
-                    mydata['meta-data']['network-interfaces'])
-
-        if mydata['meta-data']['dsmode'] == self.dsmode:
-            self.seed = ",".join(found)
-            self.metadata = mydata['meta-data']
-            self.userdata_raw = mydata['user-data']
-            self.vendordata_raw = mydata['vendor-data']
-            self._network_config = mydata['network-config']
-            return True
+        self.dsmode = self._determine_dsmode(
+            [mydata['meta-data'].get('dsmode')])
 
-        LOG.debug("%s: not claiming datasource, dsmode=%s", self,
-                  mydata['meta-data']['dsmode'])
-        return False
+        if self.dsmode == sources.DSMODE_DISABLED:
+            LOG.debug("%s: not claiming datasource, dsmode=%s", self,
+                      self.dsmode)
+            return False
+
+        self.seed = ",".join(found)
+        self.metadata = mydata['meta-data']
+        self.userdata_raw = mydata['user-data']
+        self.vendordata_raw = mydata['vendor-data']
+        self._network_config = mydata['network-config']
+        self._network_eni = mydata['meta-data'].get('network-interfaces')
+        return True
 
     def check_instance_id(self, sys_cfg):
         # quickly (local check only) if self.instance_id is still valid
@@ -227,6 +202,9 @@ class DataSourceNoCloud(sources.DataSource):
 
     @property
     def network_config(self):
+        if self._network_config is None:
+            if self.network_eni is not None:
+                self._network_config = net.convert_eni_data(self.network_eni)
         return self._network_config
 
 
@@ -254,8 +232,22 @@ def _quick_read_instance_id(cmdline_id, dirs=None):
     return None
 
 
+def load_cmdline_data(fill, cmdline=None):
+    pairs = [("ds=nocloud", sources.DSMODE_LOCAL),
+             ("ds=nocloud-net", sources.DSMODE_NETWORK)]
+    for idstr, dsmode in pairs:
+        if parse_cmdline_data(idstr, fill, cmdline):
+            # if dsmode was explicitly in the commanad line, then
+            # prefer it to the dsmode based on the command line id
+            if 'dsmode' not in fill:
+                fill['dsmode'] = dsmode
+            return True
+    return False
+
+
 # Returns true or false indicating if cmdline indicated
-# that this module should be used
+# that this module should be used.  Updates dictionary 'fill'
+# with data that was found.
 # Example cmdline:
 #  root=LABEL=uec-rootfs ro ds=nocloud
 def parse_cmdline_data(ds_id, fill, cmdline=None):
@@ -319,9 +311,7 @@ def _merge_new_seed(cur, seeded):
 class DataSourceNoCloudNet(DataSourceNoCloud):
     def __init__(self, sys_cfg, distro, paths):
         DataSourceNoCloud.__init__(self, sys_cfg, distro, paths)
-        self.cmdline_id = "ds=nocloud-net"
         self.supported_seed_starts = ("http://", "https://", "ftp://")
-        self.dsmode = "net"
 
 
 # Used to match classes to dependencies
diff --git a/cloudinit/sources/DataSourceOpenNebula.py b/cloudinit/sources/DataSourceOpenNebula.py
index 681f3a96..8f85b115 100644
--- a/cloudinit/sources/DataSourceOpenNebula.py
+++ b/cloudinit/sources/DataSourceOpenNebula.py
@@ -37,16 +37,13 @@ from cloudinit import util
 LOG = logging.getLogger(__name__)
 
 DEFAULT_IID = "iid-dsopennebula"
-DEFAULT_MODE = 'net'
 DEFAULT_PARSEUSER = 'nobody'
 CONTEXT_DISK_FILES = ["context.sh"]
-VALID_DSMODES = ("local", "net", "disabled")
 
 
 class DataSourceOpenNebula(sources.DataSource):
     def __init__(self, sys_cfg, distro, paths):
         sources.DataSource.__init__(self, sys_cfg, distro, paths)
-        self.dsmode = 'local'
         self.seed = None
         self.seed_dir = os.path.join(paths.seed_dir, 'opennebula')
 
@@ -93,52 +90,27 @@ class DataSourceOpenNebula(sources.DataSource):
         md = util.mergemanydict([md, defaults])
 
         # check for valid user specified dsmode
-        user_dsmode = results['metadata'].get('DSMODE', None)
-        if user_dsmode not in VALID_DSMODES + (None,):
-            LOG.warn("user specified invalid mode: %s", user_dsmode)
-            user_dsmode = None
-
-        # decide dsmode
-        if user_dsmode:
-            dsmode = user_dsmode
-        elif self.ds_cfg.get('dsmode'):
-            dsmode = self.ds_cfg.get('dsmode')
-        else:
-            dsmode = DEFAULT_MODE
-
-        if dsmode == "disabled":
-            # most likely user specified
-            return False
-
-        # apply static network configuration only in 'local' dsmode
-        if ('network-interfaces' in results and self.dsmode == "local"):
-            LOG.debug("Updating network interfaces from %s", self)
-            self.distro.apply_network(results['network-interfaces'])
+        self.dsmode = self._determine_dsmode(
+            [results.get('DSMODE'), self.ds_cfg.get('dsmode')])
 
-        if dsmode != self.dsmode:
-            LOG.debug("%s: not claiming datasource, dsmode=%s", self, dsmode)
+        if self.dsmode == sources.DSMODE_DISABLED:
             return False
 
         self.seed = seed
+        self.network_eni = results.get("network_config")
         self.metadata = md
         self.userdata_raw = results.get('userdata')
         return True
 
     def get_hostname(self, fqdn=False, resolve_ip=None):
         if resolve_ip is None:
-            if self.dsmode == 'net':
+            if self.dsmode == sources.DSMODE_NET:
                 resolve_ip = True
             else:
                 resolve_ip = False
         return sources.DataSource.get_hostname(self, fqdn, resolve_ip)
 
 
-class DataSourceOpenNebulaNet(DataSourceOpenNebula):
-    def __init__(self, sys_cfg, distro, paths):
-        DataSourceOpenNebula.__init__(self, sys_cfg, distro, paths)
-        self.dsmode = 'net'
-
-
 class NonContextDiskDir(Exception):
     pass
 
@@ -443,10 +415,12 @@ def read_context_disk_dir(source_dir, asuser=None):
     return results
 
 
+# Legacy: Must be present in case we load an old pkl object
+DataSourceOpenNebulaNet = DataSourceOpenNebula
+
 # Used to match classes to dependencies
 datasources = [
     (DataSourceOpenNebula, (sources.DEP_FILESYSTEM, )),
-    (DataSourceOpenNebulaNet, (sources.DEP_FILESYSTEM, sources.DEP_NETWORK)),
 ]
 
 
diff --git a/cloudinit/sources/DataSourceOpenStack.py b/cloudinit/sources/DataSourceOpenStack.py
index 3af17b10..c06d17f3 100644
--- a/cloudinit/sources/DataSourceOpenStack.py
+++ b/cloudinit/sources/DataSourceOpenStack.py
@@ -33,13 +33,11 @@ DEFAULT_IID = "iid-dsopenstack"
 DEFAULT_METADATA = {
     "instance-id": DEFAULT_IID,
 }
-VALID_DSMODES = ("net", "disabled")
 
 
 class DataSourceOpenStack(openstack.SourceMixin, sources.DataSource):
     def __init__(self, sys_cfg, distro, paths):
         super(DataSourceOpenStack, self).__init__(sys_cfg, distro, paths)
-        self.dsmode = 'net'
         self.metadata_address = None
         self.ssl_details = util.fetch_ssl_details(self.paths)
         self.version = None
@@ -103,7 +101,7 @@ class DataSourceOpenStack(openstack.SourceMixin, sources.DataSource):
         self.metadata_address = url2base.get(avail_url)
         return bool(avail_url)
 
-    def get_data(self):
+    def get_data(self, retries=5, timeout=5):
         try:
             if not self.wait_for_metadata_service():
                 return False
@@ -115,7 +113,9 @@ class DataSourceOpenStack(openstack.SourceMixin, sources.DataSource):
                                     'Crawl of openstack metadata service',
                                     read_metadata_service,
                                     args=[self.metadata_address],
-                                    kwargs={'ssl_details': self.ssl_details})
+                                    kwargs={'ssl_details': self.ssl_details,
+                                            'retries': retries,
+                                            'timeout': timeout})
         except openstack.NonReadable:
             return False
         except (openstack.BrokenMetadata, IOError):
@@ -123,11 +123,8 @@ class DataSourceOpenStack(openstack.SourceMixin, sources.DataSource):
                         self.metadata_address)
             return False
 
-        user_dsmode = results.get('dsmode', None)
-        if user_dsmode not in VALID_DSMODES + (None,):
-            LOG.warn("User specified invalid mode: %s", user_dsmode)
-            user_dsmode = None
-        if user_dsmode == 'disabled':
+        self.dsmode = self._determine_dsmode([results.get('dsmode')])
+        if self.dsmode == sources.DSMODE_DISABLED:
             return False
 
         md = results.get('metadata', {})
@@ -153,8 +150,10 @@ class DataSourceOpenStack(openstack.SourceMixin, sources.DataSource):
         return sources.instance_id_matches_system_uuid(self.get_instance_id())
 
 
-def read_metadata_service(base_url, ssl_details=None):
-    reader = openstack.MetadataReader(base_url, ssl_details=ssl_details)
+def read_metadata_service(base_url, ssl_details=None,
+                          timeout=5, retries=5):
+    reader = openstack.MetadataReader(base_url, ssl_details=ssl_details,
+                                      timeout=timeout, retries=retries)
     return reader.read_v2()
 
 
diff --git a/cloudinit/sources/DataSourceSmartOS.py b/cloudinit/sources/DataSourceSmartOS.py
index c7641eb3..08bc132b 100644
--- a/cloudinit/sources/DataSourceSmartOS.py
+++ b/cloudinit/sources/DataSourceSmartOS.py
@@ -32,13 +32,13 @@
 #       http://us-east.manta.joyent.com/jmc/public/mdata/datadict.html
 #       Comments with "@datadictionary" are snippets of the definition
 
+import base64
 import binascii
-import contextlib
+import json
 import os
 import random
 import re
 import socket
-import stat
 
 from cloudinit import log as logging
 from cloudinit import serial
@@ -62,14 +62,36 @@ SMARTOS_ATTRIB_MAP = {
     'operator-script': ('sdc:operator-script', False),
 }
 
+SMARTOS_ATTRIB_JSON = {
+    # Cloud-init Key : (SmartOS Key known JSON)
+    'network-data': 'sdc:nics',
+}
+
+SMARTOS_ENV_LX_BRAND = "lx-brand"
+SMARTOS_ENV_KVM = "kvm"
+
 DS_NAME = 'SmartOS'
 DS_CFG_PATH = ['datasource', DS_NAME]
+NO_BASE64_DECODE = [
+    'iptables_disable',
+    'motd_sys_info',
+    'root_authorized_keys',
+    'sdc:datacenter_name',
+    'sdc:uuid'
+    'user-data',
+    'user-script',
+]
+
+METADATA_SOCKFILE = '/native/.zonecontrol/metadata.sock'
+SERIAL_DEVICE = '/dev/ttyS1'
+SERIAL_TIMEOUT = 60
+
 # BUILT-IN DATASOURCE CONFIGURATION
 #  The following is the built-in configuration. If the values
 #  are not set via the system configuration, then these default
 #  will be used:
 #    serial_device: which serial device to use for the meta-data
-#    seed_timeout: how long to wait on the device
+#    serial_timeout: how long to wait on the device
 #    no_base64_decode: values which are not base64 encoded and
 #            are fetched directly from SmartOS, not meta-data values
 #    base64_keys: meta-data keys that are delivered in base64
@@ -79,16 +101,10 @@ DS_CFG_PATH = ['datasource', DS_NAME]
 #    fs_setup: describes how to format the ephemeral drive
 #
 BUILTIN_DS_CONFIG = {
-    'serial_device': '/dev/ttyS1',
-    'metadata_sockfile': '/native/.zonecontrol/metadata.sock',
-    'seed_timeout': 60,
-    'no_base64_decode': ['root_authorized_keys',
-                         'motd_sys_info',
-                         'iptables_disable',
-                         'user-data',
-                         'user-script',
-                         'sdc:datacenter_name',
-                         'sdc:uuid'],
+    'serial_device': SERIAL_DEVICE,
+    'serial_timeout': SERIAL_TIMEOUT,
+    'metadata_sockfile': METADATA_SOCKFILE,
+    'no_base64_decode': NO_BASE64_DECODE,
     'base64_keys': [],
     'base64_all': False,
     'disk_aliases': {'ephemeral0': '/dev/vdb'},
@@ -152,59 +168,41 @@ LEGACY_USER_D = "/var/db"
 
 
 class DataSourceSmartOS(sources.DataSource):
+    _unset = "_unset"
+    smartos_type = _unset
+    md_client = _unset
+
     def __init__(self, sys_cfg, distro, paths):
         sources.DataSource.__init__(self, sys_cfg, distro, paths)
-        self.is_smartdc = None
         self.ds_cfg = util.mergemanydict([
             self.ds_cfg,
             util.get_cfg_by_path(sys_cfg, DS_CFG_PATH, {}),
             BUILTIN_DS_CONFIG])
 
         self.metadata = {}
+        self.network_data = None
+        self._network_config = None
 
-        # SDC LX-Brand Zones lack dmidecode (no /dev/mem) but
-        # report 'BrandZ virtual linux' as the kernel version
-        if os.uname()[3].lower() == 'brandz virtual linux':
-            LOG.debug("Host is SmartOS, guest in Zone")
-            self.is_smartdc = True
-            self.smartos_type = 'lx-brand'
-            self.cfg = {}
-            self.seed = self.ds_cfg.get("metadata_sockfile")
-        else:
-            self.is_smartdc = True
-            self.smartos_type = 'kvm'
-            self.seed = self.ds_cfg.get("serial_device")
-            self.cfg = BUILTIN_CLOUD_CONFIG
-            self.seed_timeout = self.ds_cfg.get("serial_timeout")
-        self.smartos_no_base64 = self.ds_cfg.get('no_base64_decode')
-        self.b64_keys = self.ds_cfg.get('base64_keys')
-        self.b64_all = self.ds_cfg.get('base64_all')
         self.script_base_d = os.path.join(self.paths.get_cpath("scripts"))
 
+        self._init()
+
     def __str__(self):
         root = sources.DataSource.__str__(self)
-        return "%s [seed=%s]" % (root, self.seed)
-
-    def _get_seed_file_object(self):
-        if not self.seed:
-            raise AttributeError("seed device is not set")
-
-        if self.smartos_type == 'lx-brand':
-            if not stat.S_ISSOCK(os.stat(self.seed).st_mode):
-                LOG.debug("Seed %s is not a socket", self.seed)
-                return None
-            sock = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM)
-            sock.connect(self.seed)
-            return sock.makefile('rwb')
-        else:
-            if not stat.S_ISCHR(os.stat(self.seed).st_mode):
-                LOG.debug("Seed %s is not a character device")
-                return None
-            ser = serial.Serial(self.seed, timeout=self.seed_timeout)
-            if not ser.isOpen():
-                raise SystemError("Unable to open %s" % self.seed)
-            return ser
-        return None
+        return "%s [client=%s]" % (root, self.md_client)
+
+    def _init(self):
+        if self.smartos_type == self._unset:
+            self.smartos_type = get_smartos_environ()
+            if self.smartos_type is None:
+                self.md_client = None
+
+        if self.md_client == self._unset:
+            self.md_client = jmc_client_factory(
+                smartos_type=self.smartos_type,
+                metadata_sockfile=self.ds_cfg['metadata_sockfile'],
+                serial_device=self.ds_cfg['serial_device'],
+                serial_timeout=self.ds_cfg['serial_timeout'])
 
     def _set_provisioned(self):
         '''Mark the instance provisioning state as successful.
@@ -223,50 +221,26 @@ class DataSourceSmartOS(sources.DataSource):
                       '/'.join([svc_path, 'provision_success']))
 
     def get_data(self):
+        self._init()
+
         md = {}
         ud = ""
 
-        if not device_exists(self.seed):
-            LOG.debug("No metadata device '%s' found for SmartOS datasource",
-                      self.seed)
+        if not self.smartos_type:
+            LOG.debug("Not running on smartos")
             return False
 
-        uname_arch = os.uname()[4]
-        if uname_arch.startswith("arm") or uname_arch == "aarch64":
-            # Disabling because dmidcode in dmi_data() crashes kvm process
-            LOG.debug("Disabling SmartOS datasource on arm (LP: #1243287)")
+        if not self.md_client.exists():
+            LOG.debug("No metadata device '%r' found for SmartOS datasource",
+                      self.md_client)
             return False
 
-        # SDC KVM instances will provide dmi data, LX-brand does not
-        if self.smartos_type == 'kvm':
-            dmi_info = dmi_data()
-            if dmi_info is None:
-                LOG.debug("No dmidata utility found")
-                return False
-
-            system_type = dmi_info
-            if 'smartdc' not in system_type.lower():
-                LOG.debug("Host is not on SmartOS. system_type=%s",
-                          system_type)
-                return False
-            LOG.debug("Host is SmartOS, guest in KVM")
-
-        seed_obj = self._get_seed_file_object()
-        if seed_obj is None:
-            LOG.debug('Seed file object not found.')
-            return False
-        with contextlib.closing(seed_obj) as seed:
-            b64_keys = self.query('base64_keys', seed, strip=True, b64=False)
-            if b64_keys is not None:
-                self.b64_keys = [k.strip() for k in str(b64_keys).split(',')]
+        for ci_noun, attribute in SMARTOS_ATTRIB_MAP.items():
+            smartos_noun, strip = attribute
+            md[ci_noun] = self.md_client.get(smartos_noun, strip=strip)
 
-            b64_all = self.query('base64_all', seed, strip=True, b64=False)
-            if b64_all is not None:
-                self.b64_all = util.is_true(b64_all)
-
-            for ci_noun, attribute in SMARTOS_ATTRIB_MAP.items():
-                smartos_noun, strip = attribute
-                md[ci_noun] = self.query(smartos_noun, seed, strip=strip)
+        for ci_noun, smartos_noun in SMARTOS_ATTRIB_JSON.items():
+            md[ci_noun] = self.md_client.get_json(smartos_noun)
 
         # @datadictionary: This key may contain a program that is written
         # to a file in the filesystem of the guest on each boot and then
@@ -316,6 +290,7 @@ class DataSourceSmartOS(sources.DataSource):
         self.metadata = util.mergemanydict([md, self.metadata])
         self.userdata_raw = ud
         self.vendordata_raw = md['vendor-data']
+        self.network_data = md['network-data']
 
         self._set_provisioned()
         return True
@@ -324,69 +299,20 @@ class DataSourceSmartOS(sources.DataSource):
         return self.ds_cfg['disk_aliases'].get(name)
 
     def get_config_obj(self):
-        return self.cfg
+        if self.smartos_type == SMARTOS_ENV_KVM:
+            return BUILTIN_CLOUD_CONFIG
+        return {}
 
     def get_instance_id(self):
         return self.metadata['instance-id']
 
-    def query(self, noun, seed_file, strip=False, default=None, b64=None):
-        if b64 is None:
-            if noun in self.smartos_no_base64:
-                b64 = False
-            elif self.b64_all or noun in self.b64_keys:
-                b64 = True
-
-        return self._query_data(noun, seed_file, strip=strip,
-                                default=default, b64=b64)
-
-    def _query_data(self, noun, seed_file, strip=False,
-                    default=None, b64=None):
-        """Makes a request via "GET <NOUN>"
-
-           In the response, the first line is the status, while subsequent
-           lines are is the value. A blank line with a "." is used to
-           indicate end of response.
-
-           If the response is expected to be base64 encoded, then set
-           b64encoded to true. Unfortantely, there is no way to know if
-           something is 100% encoded, so this method relies on being told
-           if the data is base64 or not.
-        """
-
-        if not noun:
-            return False
-
-        response = JoyentMetadataClient(seed_file).get_metadata(noun)
-
-        if response is None:
-            return default
-
-        if b64 is None:
-            b64 = self._query_data('b64-%s' % noun, seed_file, b64=False,
-                                   default=False, strip=True)
-            b64 = util.is_true(b64)
-
-        resp = None
-        if b64 or strip:
-            resp = "".join(response).rstrip()
-        else:
-            resp = "".join(response)
-
-        if b64:
-            try:
-                return util.b64d(resp)
-            # Bogus input produces different errors in Python 2 and 3;
-            # catch both.
-            except (TypeError, binascii.Error):
-                LOG.warn("Failed base64 decoding key '%s'", noun)
-                return resp
-
-        return resp
-
-
-def device_exists(device):
-    """Symplistic method to determine if the device exists or not"""
-    return os.path.exists(device)
+    @property
+    def network_config(self):
+        if self._network_config is None:
+            if self.network_data is not None:
+                self._network_config = (
+                    convert_smartos_network_data(self.network_data))
+        return self._network_config
 
 
 class JoyentMetadataFetchException(Exception):
@@ -405,8 +331,11 @@ class JoyentMetadataClient(object):
         r' (?P<body>(?P<request_id>[0-9a-f]+) (?P<status>SUCCESS|NOTFOUND)'
         r'( (?P<payload>.+))?)')
 
-    def __init__(self, metasource):
-        self.metasource = metasource
+    def __init__(self, smartos_type=None, fp=None):
+        if smartos_type is None:
+            smartos_type = get_smartos_environ()
+        self.smartos_type = smartos_type
+        self.fp = fp
 
     def _checksum(self, body):
         return '{0:08x}'.format(
@@ -434,37 +363,229 @@ class JoyentMetadataClient(object):
         LOG.debug('Value "%s" found.', value)
         return value
 
-    def get_metadata(self, metadata_key):
-        LOG.debug('Fetching metadata key "%s"...', metadata_key)
+    def request(self, rtype, param=None):
         request_id = '{0:08x}'.format(random.randint(0, 0xffffffff))
-        message_body = '{0} GET {1}'.format(request_id,
-                                            util.b64e(metadata_key))
+        message_body = ' '.join((request_id, rtype,))
+        if param:
+            message_body += ' ' + base64.b64encode(param.encode()).decode()
         msg = 'V2 {0} {1} {2}\n'.format(
             len(message_body), self._checksum(message_body), message_body)
         LOG.debug('Writing "%s" to metadata transport.', msg)
-        self.metasource.write(msg.encode('ascii'))
-        self.metasource.flush()
+
+        need_close = False
+        if not self.fp:
+            self.open_transport()
+            need_close = True
+
+        self.fp.write(msg.encode('ascii'))
+        self.fp.flush()
 
         response = bytearray()
-        response.extend(self.metasource.read(1))
+        response.extend(self.fp.read(1))
         while response[-1:] != b'\n':
-            response.extend(self.metasource.read(1))
+            response.extend(self.fp.read(1))
+
+        if need_close:
+            self.close_transport()
+
         response = response.rstrip().decode('ascii')
         LOG.debug('Read "%s" from metadata transport.', response)
 
         if 'SUCCESS' not in response:
             return None
 
-        return self._get_value_from_frame(request_id, response)
+        value = self._get_value_from_frame(request_id, response)
+        return value
+
+    def get(self, key, default=None, strip=False):
+        result = self.request(rtype='GET', param=key)
+        if result is None:
+            return default
+        if result and strip:
+            result = result.strip()
+        return result
+
+    def get_json(self, key, default=None):
+        result = self.get(key, default=default)
+        if result is None:
+            return default
+        return json.loads(result)
+
+    def list(self):
+        result = self.request(rtype='KEYS')
+        if result:
+            result = result.split('\n')
+        return result
+
+    def put(self, key, val):
+        param = b' '.join([base64.b64encode(i.encode())
+                           for i in (key, val)]).decode()
+        return self.request(rtype='PUT', param=param)
+
+    def delete(self, key):
+        return self.request(rtype='DELETE', param=key)
+
+    def close_transport(self):
+        if self.fp:
+            self.fp.close()
+            self.fp = None
+
+    def __enter__(self):
+        if self.fp:
+            return self
+        self.open_transport()
+        return self
+
+    def __exit__(self, exc_type, exc_value, traceback):
+        self.close_transport()
+        return
+
+    def open_transport(self):
+        raise NotImplementedError
+
+
+class JoyentMetadataSocketClient(JoyentMetadataClient):
+    def __init__(self, socketpath):
+        self.socketpath = socketpath
+
+    def open_transport(self):
+        sock = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM)
+        sock.connect(self.socketpath)
+        self.fp = sock.makefile('rwb')
+
+    def exists(self):
+        return os.path.exists(self.socketpath)
+
+    def __repr__(self):
+        return "%s(socketpath=%s)" % (self.__class__.__name__, self.socketpath)
+
+
+class JoyentMetadataSerialClient(JoyentMetadataClient):
+    def __init__(self, device, timeout=10, smartos_type=None):
+        super(JoyentMetadataSerialClient, self).__init__(smartos_type)
+        self.device = device
+        self.timeout = timeout
+
+    def exists(self):
+        return os.path.exists(self.device)
+
+    def open_transport(self):
+        ser = serial.Serial(self.device, timeout=self.timeout)
+        if not ser.isOpen():
+            raise SystemError("Unable to open %s" % self.device)
+        self.fp = ser
+
+    def __repr__(self):
+        return "%s(device=%s, timeout=%s)" % (
+            self.__class__.__name__, self.device, self.timeout)
+
+
+class JoyentMetadataLegacySerialClient(JoyentMetadataSerialClient):
+    """V1 of the protocol was not safe for all values.
+    Thus, we allowed the user to pass values in as base64 encoded.
+    Users may still reasonably expect to be able to send base64 data
+    and have it transparently decoded.  So even though the V2 format is
+    now used, and is safe (using base64 itself), we keep legacy support.
+
+    The way for a user to do this was:
+      a.) specify 'base64_keys' key whose value is a comma delimited
+          list of keys that were base64 encoded.
+      b.) base64_all: string interpreted as a boolean that indicates
+          if all keys are base64 encoded.
+      c.) set a key named b64-<keyname> with a boolean indicating that
+          <keyname> is base64 encoded."""
+
+    def __init__(self, device, timeout=10, smartos_type=None):
+        s = super(JoyentMetadataLegacySerialClient, self)
+        s.__init__(device, timeout, smartos_type)
+        self.base64_keys = None
+        self.base64_all = None
+
+    def _init_base64_keys(self, reset=False):
+        if reset:
+            self.base64_keys = None
+            self.base64_all = None
+
+        keys = None
+        if self.base64_all is None:
+            keys = self.list()
+            if 'base64_all' in keys:
+                self.base64_all = util.is_true(self._get("base64_all"))
+            else:
+                self.base64_all = False
+
+        if self.base64_all:
+            # short circuit if base64_all is true
+            return
+
+        if self.base64_keys is None:
+            if keys is None:
+                keys = self.list()
+            b64_keys = set()
+            if 'base64_keys' in keys:
+                b64_keys = set(self._get("base64_keys").split(","))
+
+            # now add any b64-<keyname> that has a true value
+            for key in [k[3:] for k in keys if k.startswith("b64-")]:
+                if util.is_true(self._get(key)):
+                    b64_keys.add(key)
+                else:
+                    if key in b64_keys:
+                        b64_keys.remove(key)
+
+            self.base64_keys = b64_keys
+
+    def _get(self, key, default=None, strip=False):
+        return (super(JoyentMetadataLegacySerialClient, self).
+                get(key, default=default, strip=strip))
+
+    def is_b64_encoded(self, key, reset=False):
+        if key in NO_BASE64_DECODE:
+            return False
+
+        self._init_base64_keys(reset=reset)
+        if self.base64_all:
+            return True
+
+        return key in self.base64_keys
+
+    def get(self, key, default=None, strip=False):
+        mdefault = object()
+        val = self._get(key, strip=False, default=mdefault)
+        if val is mdefault:
+            return default
+
+        if self.is_b64_encoded(key):
+            try:
+                val = base64.b64decode(val.encode()).decode()
+            # Bogus input produces different errors in Python 2 and 3
+            except (TypeError, binascii.Error):
+                LOG.warn("Failed base64 decoding key '%s': %s", key, val)
+
+        if strip:
+            val = val.strip()
+
+        return val
+
 
+def jmc_client_factory(
+        smartos_type=None, metadata_sockfile=METADATA_SOCKFILE,
+        serial_device=SERIAL_DEVICE, serial_timeout=SERIAL_TIMEOUT,
+        uname_version=None):
 
-def dmi_data():
-    sys_type = util.read_dmi_data("system-product-name")
+    if smartos_type is None:
+        smartos_type = get_smartos_environ(uname_version)
 
-    if not sys_type:
+    if smartos_type is None:
         return None
+    elif smartos_type == SMARTOS_ENV_KVM:
+        return JoyentMetadataLegacySerialClient(
+            device=serial_device, timeout=serial_timeout,
+            smartos_type=smartos_type)
+    elif smartos_type == SMARTOS_ENV_LX_BRAND:
+        return JoyentMetadataSocketClient(socketpath=metadata_sockfile)
 
-    return sys_type
+    raise ValueError("Unknown value for smartos_type: %s" % smartos_type)
 
 
 def write_boot_content(content, content_f, link=None, shebang=False,
@@ -520,15 +641,141 @@ def write_boot_content(content, content_f, link=None, shebang=False,
                 util.ensure_dir(os.path.dirname(link))
                 os.symlink(content_f, link)
         except IOError as e:
-            util.logexc(LOG, "failed establishing content link", e)
+            util.logexc(LOG, "failed establishing content link: %s", e)
+
+
+def get_smartos_environ(uname_version=None, product_name=None,
+                        uname_arch=None):
+    uname = os.uname()
+    if uname_arch is None:
+        uname_arch = uname[4]
+
+    if uname_arch.startswith("arm") or uname_arch == "aarch64":
+        return None
+
+    # SDC LX-Brand Zones lack dmidecode (no /dev/mem) but
+    # report 'BrandZ virtual linux' as the kernel version
+    if uname_version is None:
+        uname_version = uname[3]
+    if uname_version.lower() == 'brandz virtual linux':
+        return SMARTOS_ENV_LX_BRAND
+
+    if product_name is None:
+        system_type = util.read_dmi_data("system-product-name")
+    else:
+        system_type = product_name
+
+    if system_type and 'smartdc' in system_type.lower():
+        return SMARTOS_ENV_KVM
+
+    return None
+
+
+# Covert SMARTOS 'sdc:nics' data to network_config yaml
+def convert_smartos_network_data(network_data=None):
+    """Return a dictionary of network_config by parsing provided
+       SMARTOS sdc:nics configuration data
+
+    sdc:nics data is a dictionary of properties of a nic and the ip
+    configuration desired.  Additional nic dictionaries are appended
+    to the list.
+
+    Converting the format is straightforward though it does include
+    duplicate information as well as data which appears to be relevant
+    to the hostOS rather than the guest.
+
+    For each entry in the nics list returned from query sdc:nics, we
+    create a type: physical entry, and extract the interface properties:
+    'mac' -> 'mac_address', 'mtu', 'interface' -> 'name'.  The remaining
+    keys are related to ip configuration.  For each ip in the 'ips' list
+    we create a subnet entry under 'subnets' pairing the ip to a one in
+    the 'gateways' list.
+    """
+
+    valid_keys = {
+        'physical': [
+            'mac_address',
+            'mtu',
+            'name',
+            'params',
+            'subnets',
+            'type',
+        ],
+        'subnet': [
+            'address',
+            'broadcast',
+            'dns_nameservers',
+            'dns_search',
+            'gateway',
+            'metric',
+            'netmask',
+            'pointopoint',
+            'routes',
+            'scope',
+            'type',
+        ],
+    }
+
+    config = []
+    for nic in network_data:
+        cfg = {k: v for k, v in nic.items()
+               if k in valid_keys['physical']}
+        cfg.update({
+            'type': 'physical',
+            'name': nic['interface']})
+        if 'mac' in nic:
+            cfg.update({'mac_address': nic['mac']})
+
+        subnets = []
+        for ip, gw in zip(nic['ips'], nic['gateways']):
+            subnet = {k: v for k, v in nic.items()
+                      if k in valid_keys['subnet']}
+            subnet.update({
+                'type': 'static',
+                'address': ip,
+                'gateway': gw,
+            })
+            subnets.append(subnet)
+        cfg.update({'subnets': subnets})
+        config.append(cfg)
+
+    return {'version': 1, 'config': config}
 
 
 # Used to match classes to dependencies
 datasources = [
-    (DataSourceSmartOS, (sources.DEP_FILESYSTEM, sources.DEP_NETWORK)),
+    (DataSourceSmartOS, (sources.DEP_FILESYSTEM, )),
 ]
 
 
 # Return a list of data sources that match this set of dependencies
 def get_datasource_list(depends):
     return sources.list_from_depends(depends, datasources)
+
+
+if __name__ == "__main__":
+    import sys
+    jmc = jmc_client_factory()
+    if jmc is None:
+        print("Do not appear to be on smartos.")
+        sys.exit(1)
+    if len(sys.argv) == 1:
+        keys = (list(SMARTOS_ATTRIB_JSON.keys()) +
+                list(SMARTOS_ATTRIB_MAP.keys()))
+    else:
+        keys = sys.argv[1:]
+
+    data = {}
+    for key in keys:
+        if key in SMARTOS_ATTRIB_JSON:
+            keyname = SMARTOS_ATTRIB_JSON[key]
+            data[key] = jmc.get_json(keyname)
+        else:
+            if key in SMARTOS_ATTRIB_MAP:
+                keyname, strip = SMARTOS_ATTRIB_MAP[key]
+            else:
+                keyname, strip = (key, False)
+            val = jmc.get(keyname, strip=strip)
+            data[key] = jmc.get(keyname, strip=strip)
+
+    print(json.dumps(data, indent=1))
diff --git a/cloudinit/sources/__init__.py b/cloudinit/sources/__init__.py
index 43e4fd57..2a6b8d90 100644
--- a/cloudinit/sources/__init__.py
+++ b/cloudinit/sources/__init__.py
@@ -34,6 +34,13 @@ from cloudinit import util
 from cloudinit.filters import launch_index
 from cloudinit.reporting import events
 
+DSMODE_DISABLED = "disabled"
+DSMODE_LOCAL = "local"
+DSMODE_NETWORK = "net"
+DSMODE_PASS = "pass"
+
+VALID_DSMODES = [DSMODE_DISABLED, DSMODE_LOCAL, DSMODE_NETWORK]
+
 DEP_FILESYSTEM = "FILESYSTEM"
 DEP_NETWORK = "NETWORK"
 DS_PREFIX = 'DataSource'
@@ -57,6 +64,7 @@ class DataSource(object):
         self.userdata_raw = None
         self.vendordata = None
         self.vendordata_raw = None
+        self.dsmode = DSMODE_NETWORK
 
         # find the datasource config name.
         # remove 'DataSource' from classname on front, and remove 'Net' on end.
@@ -223,10 +231,35 @@ class DataSource(object):
         # quickly (local check only) if self.instance_id is still
         return False
 
+    @staticmethod
+    def _determine_dsmode(candidates, default=None, valid=None):
+        # return the first candidate that is non None, warn if not valid
+        if default is None:
+            default = DSMODE_NETWORK
+
+        if valid is None:
+            valid = VALID_DSMODES
+
+        for candidate in candidates:
+            if candidate is None:
+                continue
+            if candidate in valid:
+                return candidate
+            else:
+                LOG.warn("invalid dsmode '%s', using default=%s",
+                         candidate, default)
+                return default
+
+        return default
+
     @property
     def network_config(self):
         return None
 
+    @property
+    def first_instance_boot(self):
+        return
+
 
 def normalize_pubkey_data(pubkey_data):
     keys = []
diff --git a/cloudinit/sources/helpers/openstack.py b/cloudinit/sources/helpers/openstack.py
index 1af99118..494335b3 100644
--- a/cloudinit/sources/helpers/openstack.py
+++ b/cloudinit/sources/helpers/openstack.py
@@ -28,6 +28,7 @@ import six
 
 from cloudinit import ec2_utils
 from cloudinit import log as logging
+from cloudinit import net
 from cloudinit import sources
 from cloudinit import url_helper
 from cloudinit import util
@@ -190,14 +191,14 @@ class BaseReader(object):
                   versions_available)
         return selected_version
 
-    def _read_content_path(self, item):
+    def _read_content_path(self, item, decode=False):
         path = item.get('content_path', '').lstrip("/")
         path_pieces = path.split("/")
         valid_pieces = [p for p in path_pieces if len(p)]
         if not valid_pieces:
             raise BrokenMetadata("Item %s has no valid content path" % (item))
         path = self._path_join(self.base_path, "openstack", *path_pieces)
-        return self._path_read(path)
+        return self._path_read(path, decode=decode)
 
     def read_v2(self):
         """Reads a version 2 formatted location.
@@ -298,7 +299,8 @@ class BaseReader(object):
         net_item = metadata.get("network_config", None)
         if net_item:
             try:
-                results['network_config'] = self._read_content_path(net_item)
+                content = self._read_content_path(net_item, decode=True)
+                results['network_config'] = content
             except IOError as e:
                 raise BrokenMetadata("Failed to read network"
                                      " configuration: %s" % (e))
@@ -333,8 +335,8 @@ class ConfigDriveReader(BaseReader):
         components = [base] + list(add_ons)
         return os.path.join(*components)
 
-    def _path_read(self, path):
-        return util.load_file(path, decode=False)
+    def _path_read(self, path, decode=False):
+        return util.load_file(path, decode=decode)
 
     def _fetch_available_versions(self):
         if self._versions is None:
@@ -446,7 +448,7 @@ class MetadataReader(BaseReader):
         self._versions = found
         return self._versions
 
-    def _path_read(self, path):
+    def _path_read(self, path, decode=False):
 
         def should_retry_cb(_request_args, cause):
             try:
@@ -463,7 +465,10 @@ class MetadataReader(BaseReader):
                                       ssl_details=self.ssl_details,
                                       timeout=self.timeout,
                                       exception_cb=should_retry_cb)
-        return response.contents
+        if decode:
+            return response.contents.decode()
+        else:
+            return response.contents
 
     def _path_join(self, base, *add_ons):
         return url_helper.combine_url(base, *add_ons)
@@ -474,7 +479,8 @@ class MetadataReader(BaseReader):
                                                retries=self.retries)
 
 
-def convert_net_json(network_json):
+# Convert OpenStack ConfigDrive NetworkData json to network_config yaml
+def convert_net_json(network_json=None, known_macs=None):
     """Return a dictionary of network_config by parsing provided
        OpenStack ConfigDrive NetworkData json format
 
@@ -501,8 +507,10 @@ def convert_net_json(network_json):
     enumerate a dictionary of valid keys for network_yaml and apply filtering
     to drop these superflous keys from the network_config yaml.
     """
+    if network_json is None:
+        return None
 
-    # Dict of network_config key for filtering network_json
+    # dict of network_config key for filtering network_json
     valid_keys = {
         'physical': [
             'name',
@@ -510,6 +518,7 @@ def convert_net_json(network_json):
             'mac_address',
             'subnets',
             'params',
+            'mtu',
         ],
         'subnet': [
             'type',
@@ -519,7 +528,6 @@ def convert_net_json(network_json):
             'metric',
             'gateway',
             'pointopoint',
-            'mtu',
             'scope',
             'dns_nameservers',
             'dns_search',
@@ -534,13 +542,19 @@ def convert_net_json(network_json):
     config = []
     for link in links:
         subnets = []
-        cfg = dict((k, v) for k, v in link.items()
-                   if k in valid_keys['physical'])
-        cfg.update({'name': link['id']})
-        for network in [net for net in networks
-                        if net['link'] == link['id']]:
-            subnet = dict((k, v) for k, v in network.items()
-                          if k in valid_keys['subnet'])
+        cfg = {k: v for k, v in link.items()
+               if k in valid_keys['physical']}
+        # 'name' is not in openstack spec yet, but we will support it if it is
+        # present.  The 'id' in the spec is currently implemented as the host
+        # nic's name, meaning something like 'tap-adfasdffd'.  We do not want
+        # to name guest devices with such ugly names.
+        if 'name' in link:
+            cfg['name'] = link['name']
+
+        for network in [n for n in networks
+                        if n['link'] == link['id']]:
+            subnet = {k: v for k, v in network.items()
+                      if k in valid_keys['subnet']}
             if 'dhcp' in network['type']:
                 t = 'dhcp6' if network['type'].startswith('ipv6') else 'dhcp4'
                 subnet.update({
@@ -551,13 +565,13 @@ def convert_net_json(network_json):
                     'type': 'static',
                     'address': network.get('ip_address'),
                 })
-                if network['type'] == 'ipv6':
-                    subnet['ipv6'] = True
-                else:
-                    subnet['ipv4'] = True
+            if network['type'] == 'ipv4':
+                subnet['ipv4'] = True
+            if network['type'] == 'ipv6':
+                subnet['ipv6'] = True
             subnets.append(subnet)
         cfg.update({'subnets': subnets})
-        if link['type'] in ['ethernet', 'vif', 'ovs', 'phy']:
+        if link['type'] in ['ethernet', 'vif', 'ovs', 'phy', 'bridge']:
             cfg.update({
                 'type': 'physical',
                 'mac_address': link['ethernet_mac_address']})
@@ -580,19 +594,29 @@ def convert_net_json(network_json):
                 'vlan_id': link['vlan_id'],
                 'mac_address': link['vlan_mac_address'],
             })
-        elif link['type'] in ['bridge']:
-            cfg.update({
-                'type': 'bridge',
-                'mac_address': link['ethernet_mac_address'],
-                'mtu': link['mtu']})
         else:
             raise ValueError(
                 'Unknown network_data link type: %s' % link['type'])
 
         config.append(cfg)
 
+    need_names = [d for d in config
+                  if d.get('type') == 'physical' and 'name' not in d]
+
+    if need_names:
+        if known_macs is None:
+            known_macs = net.get_interfaces_by_mac()
+
+        for d in need_names:
+            mac = d.get('mac_address')
+            if not mac:
+                raise ValueError("No mac_address or name entry for %s" % d)
+            if mac not in known_macs:
+                raise ValueError("Unable to find a system nic for %s" % d)
+            d['name'] = known_macs[mac]
+
     for service in services:
-        cfg = copy.deepcopy(service)
+        cfg = service
         cfg.update({'type': 'nameserver'})
         config.append(cfg)