Merged trunk lp:cloud-init

11 files changed, 849 insertions, 75 deletions

diff --git a/cloudinit/sources/DataSourceAltCloud.py b/cloudinit/sources/DataSourceAltCloud.py
index 9812bdcb..a834f8eb 100644
--- a/cloudinit/sources/DataSourceAltCloud.py
+++ b/cloudinit/sources/DataSourceAltCloud.py
@@ -1,10 +1,11 @@
 # vi: ts=4 expandtab
 #
 #    Copyright (C) 2009-2010 Canonical Ltd.
-#    Copyright (C) 2012 Hewlett-Packard Development Company, L.P.
+#    Copyright (C) 2012, 2013 Hewlett-Packard Development Company, L.P.
 #    Copyright (C) 2012 Yahoo! Inc.
 #
 #    Author: Joe VLcek <JVLcek@RedHat.com>
+#    Author: Juerg Haefliger <juerg.haefliger@hp.com>
 #
 #    This program is free software: you can redistribute it and/or modify
 #    it under the terms of the GNU General Public License version 3, as
@@ -30,6 +31,7 @@ import os.path
 from cloudinit import log as logging
 from cloudinit import sources
 from cloudinit import util
+
 from cloudinit.util import ProcessExecutionError
 
 LOG = logging.getLogger(__name__)
@@ -78,7 +80,7 @@ def read_user_data_callback(mount_dir):
         try:
             user_data = util.load_file(user_data_file).strip()
         except IOError:
-            util.logexc(LOG, ('Failed accessing user data file.'))
+            util.logexc(LOG, 'Failed accessing user data file.')
             return None
 
     return user_data
@@ -91,8 +93,8 @@ class DataSourceAltCloud(sources.DataSource):
         self.supported_seed_starts = ("/", "file://")
 
     def __str__(self):
-        mstr = "%s [seed=%s]" % (util.obj_name(self), self.seed)
-        return mstr
+        root = sources.DataSource.__str__(self)
+        return "%s [seed=%s]" % (root, self.seed)
 
     def get_cloud_type(self):
         '''
@@ -177,7 +179,7 @@ class DataSourceAltCloud(sources.DataSource):
             return False
 
         # No user data found
-        util.logexc(LOG, ('Failed accessing user data.'))
+        util.logexc(LOG, 'Failed accessing user data.')
         return False
 
     def user_data_rhevm(self):
@@ -204,12 +206,12 @@ class DataSourceAltCloud(sources.DataSource):
             (cmd_out, _err) = util.subp(cmd)
             LOG.debug(('Command: %s\nOutput%s') % (' '.join(cmd), cmd_out))
         except ProcessExecutionError, _err:
-            util.logexc(LOG, (('Failed command: %s\n%s') % \
-                (' '.join(cmd), _err.message)))
+            util.logexc(LOG, 'Failed command: %s\n%s', ' '.join(cmd),
+                        _err.message)
             return False
         except OSError, _err:
-            util.logexc(LOG, (('Failed command: %s\n%s') % \
-                (' '.join(cmd), _err.message)))
+            util.logexc(LOG, 'Failed command: %s\n%s', ' '.join(cmd),
+                        _err.message)
             return False
 
         floppy_dev = '/dev/fd0'
@@ -221,12 +223,12 @@ class DataSourceAltCloud(sources.DataSource):
             (cmd_out, _err) = util.subp(cmd)
             LOG.debug(('Command: %s\nOutput%s') % (' '.join(cmd), cmd_out))
         except ProcessExecutionError, _err:
-            util.logexc(LOG, (('Failed command: %s\n%s') % \
-                (' '.join(cmd), _err.message)))
+            util.logexc(LOG, 'Failed command: %s\n%s', ' '.join(cmd),
+                        _err.message)
             return False
         except OSError, _err:
-            util.logexc(LOG, (('Failed command: %s\n%s') % \
-                (' '.join(cmd), _err.message)))
+            util.logexc(LOG, 'Failed command: %s\n%s', ' '.join(cmd),
+                        _err.message)
             return False
 
         try:
@@ -235,8 +237,8 @@ class DataSourceAltCloud(sources.DataSource):
             if err.errno != errno.ENOENT:
                 raise
         except util.MountFailedError:
-            util.logexc(LOG, ("Failed to mount %s"
-                              " when looking for user data"), floppy_dev)
+            util.logexc(LOG, "Failed to mount %s when looking for user data",
+                        floppy_dev)
 
         self.userdata_raw = return_str
         self.metadata = META_DATA_NOT_SUPPORTED
@@ -271,8 +273,8 @@ class DataSourceAltCloud(sources.DataSource):
                 if err.errno != errno.ENOENT:
                     raise
             except util.MountFailedError:
-                util.logexc(LOG, ("Failed to mount %s"
-                                  " when looking for user data"), cdrom_dev)
+                util.logexc(LOG, "Failed to mount %s when looking for user "
+                            "data", cdrom_dev)
 
         self.userdata_raw = return_str
         self.metadata = META_DATA_NOT_SUPPORTED
diff --git a/cloudinit/sources/DataSourceAzure.py b/cloudinit/sources/DataSourceAzure.py
new file mode 100644
index 00000000..66d7728b
--- /dev/null
+++ b/cloudinit/sources/DataSourceAzure.py
@@ -0,0 +1,502 @@
+# vi: ts=4 expandtab
+#
+#    Copyright (C) 2013 Canonical Ltd.
+#
+#    Author: Scott Moser <scott.moser@canonical.com>
+#
+#    This program is free software: you can redistribute it and/or modify
+#    it under the terms of the GNU General Public License version 3, as
+#    published by the Free Software Foundation.
+#
+#    This program is distributed in the hope that it will be useful,
+#    but WITHOUT ANY WARRANTY; without even the implied warranty of
+#    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+#    GNU General Public License for more details.
+#
+#    You should have received a copy of the GNU General Public License
+#    along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+import base64
+import crypt
+import os
+import os.path
+import time
+from xml.dom import minidom
+
+from cloudinit import log as logging
+from cloudinit import sources
+from cloudinit import util
+
+LOG = logging.getLogger(__name__)
+
+DS_NAME = 'Azure'
+DEFAULT_METADATA = {"instance-id": "iid-AZURE-NODE"}
+AGENT_START = ['service', 'walinuxagent', 'start']
+BOUNCE_COMMAND = ['sh', '-xc',
+    "i=$interface; x=0; ifdown $i || x=$?; ifup $i || x=$?; exit $x"]
+
+BUILTIN_DS_CONFIG = {
+    'agent_command': AGENT_START,
+    'data_dir': "/var/lib/waagent",
+    'set_hostname': True,
+    'hostname_bounce': {
+        'interface': 'eth0',
+        'policy': True,
+        'command': BOUNCE_COMMAND,
+        'hostname_command': 'hostname',
+    }
+}
+DS_CFG_PATH = ['datasource', DS_NAME]
+
+
+class DataSourceAzureNet(sources.DataSource):
+    def __init__(self, sys_cfg, distro, paths):
+        sources.DataSource.__init__(self, sys_cfg, distro, paths)
+        self.seed_dir = os.path.join(paths.seed_dir, 'azure')
+        self.cfg = {}
+        self.seed = None
+        self.ds_cfg = util.mergemanydict([
+            util.get_cfg_by_path(sys_cfg, DS_CFG_PATH, {}),
+            BUILTIN_DS_CONFIG])
+
+    def __str__(self):
+        root = sources.DataSource.__str__(self)
+        return "%s [seed=%s]" % (root, self.seed)
+
+    def get_data(self):
+        # azure removes/ejects the cdrom containing the ovf-env.xml
+        # file on reboot.  So, in order to successfully reboot we
+        # need to look in the datadir and consider that valid
+        ddir = self.ds_cfg['data_dir']
+
+        candidates = [self.seed_dir]
+        candidates.extend(list_possible_azure_ds_devs())
+        if ddir:
+            candidates.append(ddir)
+
+        found = None
+
+        for cdev in candidates:
+            try:
+                if cdev.startswith("/dev/"):
+                    ret = util.mount_cb(cdev, load_azure_ds_dir)
+                else:
+                    ret = load_azure_ds_dir(cdev)
+
+            except NonAzureDataSource:
+                continue
+            except BrokenAzureDataSource as exc:
+                raise exc
+            except util.MountFailedError:
+                LOG.warn("%s was not mountable" % cdev)
+                continue
+
+            (md, self.userdata_raw, cfg, files) = ret
+            self.seed = cdev
+            self.metadata = util.mergemanydict([md, DEFAULT_METADATA])
+            self.cfg = cfg
+            found = cdev
+
+            LOG.debug("found datasource in %s", cdev)
+            break
+
+        if not found:
+            return False
+
+        if found == ddir:
+            LOG.debug("using files cached in %s", ddir)
+
+        # now update ds_cfg to reflect contents pass in config
+        usercfg = util.get_cfg_by_path(self.cfg, DS_CFG_PATH, {})
+        self.ds_cfg = util.mergemanydict([usercfg, self.ds_cfg])
+        mycfg = self.ds_cfg
+
+        # walinux agent writes files world readable, but expects
+        # the directory to be protected.
+        write_files(mycfg['data_dir'], files, dirmode=0700)
+
+        # handle the hostname 'publishing'
+        try:
+            handle_set_hostname(mycfg.get('set_hostname'),
+                                self.metadata.get('local-hostname'),
+                                mycfg['hostname_bounce'])
+        except Exception as e:
+            LOG.warn("Failed publishing hostname: %s" % e)
+            util.logexc(LOG, "handling set_hostname failed")
+
+        try:
+            invoke_agent(mycfg['agent_command'])
+        except util.ProcessExecutionError:
+            # claim the datasource even if the command failed
+            util.logexc(LOG, "agent command '%s' failed.",
+                        mycfg['agent_command'])
+
+        shcfgxml = os.path.join(mycfg['data_dir'], "SharedConfig.xml")
+        wait_for = [shcfgxml]
+
+        fp_files = []
+        for pk in self.cfg.get('_pubkeys', []):
+            bname = pk['fingerprint'] + ".crt"
+            fp_files += [os.path.join(mycfg['data_dir'], bname)]
+
+        missing = util.log_time(logfunc=LOG.debug, msg="waiting for files",
+                                func=wait_for_files,
+                                args=(wait_for + fp_files,))
+        if len(missing):
+            LOG.warn("Did not find files, but going on: %s", missing)
+
+        if shcfgxml in missing:
+            LOG.warn("SharedConfig.xml missing, using static instance-id")
+        else:
+            try:
+                self.metadata['instance-id'] = iid_from_shared_config(shcfgxml)
+            except ValueError as e:
+                LOG.warn("failed to get instance id in %s: %s" % (shcfgxml, e))
+
+        pubkeys = pubkeys_from_crt_files(fp_files)
+
+        self.metadata['public-keys'] = pubkeys
+
+        return True
+
+    def get_config_obj(self):
+        return self.cfg
+
+
+def handle_set_hostname(enabled, hostname, cfg):
+    if not util.is_true(enabled):
+        return
+
+    if not hostname:
+        LOG.warn("set_hostname was true but no local-hostname")
+        return
+
+    apply_hostname_bounce(hostname=hostname, policy=cfg['policy'],
+                          interface=cfg['interface'],
+                          command=cfg['command'],
+                          hostname_command=cfg['hostname_command'])
+
+
+def apply_hostname_bounce(hostname, policy, interface, command,
+                          hostname_command="hostname"):
+    # set the hostname to 'hostname' if it is not already set to that.
+    # then, if policy is not off, bounce the interface using command
+    prev_hostname = util.subp(hostname_command, capture=True)[0].strip()
+
+    util.subp([hostname_command, hostname])
+
+    msg = ("phostname=%s hostname=%s policy=%s interface=%s" %
+           (prev_hostname, hostname, policy, interface))
+
+    if util.is_false(policy):
+        LOG.debug("pubhname: policy false, skipping [%s]", msg)
+        return
+
+    if prev_hostname == hostname and policy != "force":
+        LOG.debug("pubhname: no change, policy != force. skipping. [%s]", msg)
+        return
+
+    env = os.environ.copy()
+    env['interface'] = interface
+    env['hostname'] = hostname
+    env['old_hostname'] = prev_hostname
+
+    if command == "builtin":
+        command = BOUNCE_COMMAND
+
+    LOG.debug("pubhname: publishing hostname [%s]", msg)
+    shell = not isinstance(command, (list, tuple))
+    # capture=False, see comments in bug 1202758 and bug 1206164.
+    util.log_time(logfunc=LOG.debug, msg="publishing hostname",
+        get_uptime=True, func=util.subp,
+        kwargs={'args': command, 'shell': shell, 'capture': False,
+                'env': env})
+
+
+def crtfile_to_pubkey(fname):
+    pipeline = ('openssl x509 -noout -pubkey < "$0" |'
+                'ssh-keygen -i -m PKCS8 -f /dev/stdin')
+    (out, _err) = util.subp(['sh', '-c', pipeline, fname], capture=True)
+    return out.rstrip()
+
+
+def pubkeys_from_crt_files(flist):
+    pubkeys = []
+    errors = []
+    for fname in flist:
+        try:
+            pubkeys.append(crtfile_to_pubkey(fname))
+        except util.ProcessExecutionError:
+            errors.extend(fname)
+
+    if errors:
+        LOG.warn("failed to convert the crt files to pubkey: %s" % errors)
+
+    return pubkeys
+
+
+def wait_for_files(flist, maxwait=60, naplen=.5):
+    need = set(flist)
+    waited = 0
+    while waited < maxwait:
+        need -= set([f for f in need if os.path.exists(f)])
+        if len(need) == 0:
+            return []
+        time.sleep(naplen)
+        waited += naplen
+    return need
+
+
+def write_files(datadir, files, dirmode=None):
+    if not datadir:
+        return
+    if not files:
+        files = {}
+    util.ensure_dir(datadir, dirmode)
+    for (name, content) in files.items():
+        util.write_file(filename=os.path.join(datadir, name),
+                        content=content, mode=0600)
+
+
+def invoke_agent(cmd):
+    # this is a function itself to simplify patching it for test
+    if cmd:
+        LOG.debug("invoking agent: %s" % cmd)
+        util.subp(cmd, shell=(not isinstance(cmd, list)))
+    else:
+        LOG.debug("not invoking agent")
+
+
+def find_child(node, filter_func):
+    ret = []
+    if not node.hasChildNodes():
+        return ret
+    for child in node.childNodes:
+        if filter_func(child):
+            ret.append(child)
+    return ret
+
+
+def load_azure_ovf_pubkeys(sshnode):
+    # This parses a 'SSH' node formatted like below, and returns
+    # an array of dicts.
+    #  [{'fp': '6BE7A7C3C8A8F4B123CCA5D0C2F1BE4CA7B63ED7',
+    #    'path': 'where/to/go'}]
+    #
+    # <SSH><PublicKeys>
+    #   <PublicKey><Fingerprint>ABC</FingerPrint><Path>/ABC</Path>
+    #   ...
+    # </PublicKeys></SSH>
+    results = find_child(sshnode, lambda n: n.localName == "PublicKeys")
+    if len(results) == 0:
+        return []
+    if len(results) > 1:
+        raise BrokenAzureDataSource("Multiple 'PublicKeys'(%s) in SSH node" %
+                                    len(results))
+
+    pubkeys_node = results[0]
+    pubkeys = find_child(pubkeys_node, lambda n: n.localName == "PublicKey")
+
+    if len(pubkeys) == 0:
+        return []
+
+    found = []
+    text_node = minidom.Document.TEXT_NODE
+
+    for pk_node in pubkeys:
+        if not pk_node.hasChildNodes():
+            continue
+        cur = {'fingerprint': "", 'path': ""}
+        for child in pk_node.childNodes:
+            if (child.nodeType == text_node or not child.localName):
+                continue
+
+            name = child.localName.lower()
+
+            if name not in cur.keys():
+                continue
+
+            if (len(child.childNodes) != 1 or
+                child.childNodes[0].nodeType != text_node):
+                continue
+
+            cur[name] = child.childNodes[0].wholeText.strip()
+        found.append(cur)
+
+    return found
+
+
+def single_node_at_path(node, pathlist):
+    curnode = node
+    for tok in pathlist:
+        results = find_child(curnode, lambda n: n.localName == tok)
+        if len(results) == 0:
+            raise ValueError("missing %s token in %s" % (tok, str(pathlist)))
+        if len(results) > 1:
+            raise ValueError("found %s nodes of type %s looking for %s" %
+                             (len(results), tok, str(pathlist)))
+        curnode = results[0]
+
+    return curnode
+
+
+def read_azure_ovf(contents):
+    try:
+        dom = minidom.parseString(contents)
+    except Exception as e:
+        raise NonAzureDataSource("invalid xml: %s" % e)
+
+    results = find_child(dom.documentElement,
+        lambda n: n.localName == "ProvisioningSection")
+
+    if len(results) == 0:
+        raise NonAzureDataSource("No ProvisioningSection")
+    if len(results) > 1:
+        raise BrokenAzureDataSource("found '%d' ProvisioningSection items" %
+                                    len(results))
+    provSection = results[0]
+
+    lpcs_nodes = find_child(provSection,
+        lambda n: n.localName == "LinuxProvisioningConfigurationSet")
+
+    if len(results) == 0:
+        raise NonAzureDataSource("No LinuxProvisioningConfigurationSet")
+    if len(results) > 1:
+        raise BrokenAzureDataSource("found '%d' %ss" %
+                                    ("LinuxProvisioningConfigurationSet",
+                                     len(results)))
+    lpcs = lpcs_nodes[0]
+
+    if not lpcs.hasChildNodes():
+        raise BrokenAzureDataSource("no child nodes of configuration set")
+
+    md_props = 'seedfrom'
+    md = {'azure_data': {}}
+    cfg = {}
+    ud = ""
+    password = None
+    username = None
+
+    for child in lpcs.childNodes:
+        if child.nodeType == dom.TEXT_NODE or not child.localName:
+            continue
+
+        name = child.localName.lower()
+
+        simple = False
+        value = ""
+        if (len(child.childNodes) == 1 and
+            child.childNodes[0].nodeType == dom.TEXT_NODE):
+            simple = True
+            value = child.childNodes[0].wholeText
+
+        attrs = {k: v for k, v in child.attributes.items()}
+
+        # we accept either UserData or CustomData.  If both are present
+        # then behavior is undefined.
+        if (name == "userdata" or name == "customdata"):
+            if attrs.get('encoding') in (None, "base64"):
+                ud = base64.b64decode(''.join(value.split()))
+            else:
+                ud = value
+        elif name == "username":
+            username = value
+        elif name == "userpassword":
+            password = value
+        elif name == "hostname":
+            md['local-hostname'] = value
+        elif name == "dscfg":
+            if attrs.get('encoding') in (None, "base64"):
+                dscfg = base64.b64decode(''.join(value.split()))
+            else:
+                dscfg = value
+            cfg['datasource'] = {DS_NAME: util.load_yaml(dscfg, default={})}
+        elif name == "ssh":
+            cfg['_pubkeys'] = load_azure_ovf_pubkeys(child)
+        elif name == "disablesshpasswordauthentication":
+            cfg['ssh_pwauth'] = util.is_false(value)
+        elif simple:
+            if name in md_props:
+                md[name] = value
+            else:
+                md['azure_data'][name] = value
+
+    defuser = {}
+    if username:
+        defuser['name'] = username
+    if password:
+        defuser['passwd'] = encrypt_pass(password)
+        defuser['lock_passwd'] = False
+
+    if defuser:
+        cfg['system_info'] = {'default_user': defuser}
+
+    if 'ssh_pwauth' not in cfg and password:
+        cfg['ssh_pwauth'] = True
+
+    return (md, ud, cfg)
+
+
+def encrypt_pass(password, salt_id="$6$"):
+    return crypt.crypt(password, salt_id + util.rand_str(strlen=16))
+
+
+def list_possible_azure_ds_devs():
+    # return a sorted list of devices that might have a azure datasource
+    devlist = []
+    for fstype in ("iso9660", "udf"):
+        devlist.extend(util.find_devs_with("TYPE=%s" % fstype))
+
+    devlist.sort(reverse=True)
+    return devlist
+
+
+def load_azure_ds_dir(source_dir):
+    ovf_file = os.path.join(source_dir, "ovf-env.xml")
+
+    if not os.path.isfile(ovf_file):
+        raise NonAzureDataSource("No ovf-env file found")
+
+    with open(ovf_file, "r") as fp:
+        contents = fp.read()
+
+    md, ud, cfg = read_azure_ovf(contents)
+    return (md, ud, cfg, {'ovf-env.xml': contents})
+
+
+def iid_from_shared_config(path):
+    with open(path, "rb") as fp:
+        content = fp.read()
+    return iid_from_shared_config_content(content)
+
+
+def iid_from_shared_config_content(content):
+    """
+    find INSTANCE_ID in:
+    <?xml version="1.0" encoding="utf-8"?>
+    <SharedConfig version="1.0.0.0" goalStateIncarnation="1">
+      <Deployment name="INSTANCE_ID" guid="{...}" incarnation="0">
+        <Service name="..." guid="{00000000-0000-0000-0000-000000000000}" />
+    """
+    dom = minidom.parseString(content)
+    depnode = single_node_at_path(dom, ["SharedConfig", "Deployment"])
+    return depnode.attributes.get('name').value
+
+
+class BrokenAzureDataSource(Exception):
+    pass
+
+
+class NonAzureDataSource(Exception):
+    pass
+
+
+# Used to match classes to dependencies
+datasources = [
+  (DataSourceAzureNet, (sources.DEP_FILESYSTEM, sources.DEP_NETWORK)),
+]
+
+
+# Return a list of data sources that match this set of dependencies
+def get_datasource_list(depends):
+    return sources.list_from_depends(depends, datasources)
diff --git a/cloudinit/sources/DataSourceCloudStack.py b/cloudinit/sources/DataSourceCloudStack.py
index 275caf0d..08f661e4 100644
--- a/cloudinit/sources/DataSourceCloudStack.py
+++ b/cloudinit/sources/DataSourceCloudStack.py
@@ -4,11 +4,13 @@
 #    Copyright (C) 2012 Cosmin Luta
 #    Copyright (C) 2012 Yahoo! Inc.
 #    Copyright (C) 2012 Gerard Dethier
+#    Copyright (C) 2013 Hewlett-Packard Development Company, L.P.
 #
 #    Author: Cosmin Luta <q4break@gmail.com>
 #    Author: Scott Moser <scott.moser@canonical.com>
 #    Author: Joshua Harlow <harlowja@yahoo-inc.com>
 #    Author: Gerard Dethier <g.dethier@gmail.com>
+#    Author: Juerg Haefliger <juerg.haefliger@hp.com>
 #
 #    This program is free software: you can redistribute it and/or modify
 #    it under the terms of the GNU General Public License version 3, as
@@ -48,9 +50,6 @@ class DataSourceCloudStack(sources.DataSource):
             raise RuntimeError("No virtual router found!")
         self.metadata_address = "http://%s/" % (vr_addr)
 
-    def __str__(self):
-        return util.obj_name(self)
-
     def _get_url_settings(self):
         mcfg = self.ds_cfg
         if not mcfg:
@@ -112,8 +111,8 @@ class DataSourceCloudStack(sources.DataSource):
                       int(time.time() - start_time))
             return True
         except Exception:
-            util.logexc(LOG, ('Failed fetching from metadata '
-                              'service %s'), self.metadata_address)
+            util.logexc(LOG, 'Failed fetching from metadata service %s',
+                        self.metadata_address)
             return False
 
     def get_instance_id(self):
diff --git a/cloudinit/sources/DataSourceConfigDrive.py b/cloudinit/sources/DataSourceConfigDrive.py
index ec016a1d..835f2a9a 100644
--- a/cloudinit/sources/DataSourceConfigDrive.py
+++ b/cloudinit/sources/DataSourceConfigDrive.py
@@ -51,7 +51,9 @@ class DataSourceConfigDrive(sources.DataSource):
         self.ec2_metadata = None
 
     def __str__(self):
-        mstr = "%s [%s,ver=%s]" % (util.obj_name(self), self.dsmode,
+        root = sources.DataSource.__str__(self)
+        mstr = "%s [%s,ver=%s]" % (root,
+                                   self.dsmode,
                                    self.version)
         mstr += "[source=%s]" % (self.source)
         return mstr
@@ -152,7 +154,7 @@ class DataSourceConfigDrive(sources.DataSource):
             return False
 
         md = results['metadata']
-        md = util.mergedict(md, DEFAULT_METADATA)
+        md = util.mergemanydict([md, DEFAULT_METADATA])
 
         # Perform some metadata 'fixups'
         #
@@ -256,6 +258,10 @@ def find_candidate_devs():
         * labeled with 'config-2'
     """
 
+    # Query optical drive to get it in blkid cache for 2.6 kernels
+    util.find_devs_with(path="/dev/sr0")
+    util.find_devs_with(path="/dev/sr1")
+
     by_fstype = (util.find_devs_with("TYPE=vfat") +
                  util.find_devs_with("TYPE=iso9660"))
     by_label = util.find_devs_with("LABEL=config-2")
diff --git a/cloudinit/sources/DataSourceEc2.py b/cloudinit/sources/DataSourceEc2.py
index 2db53446..f010e640 100644
--- a/cloudinit/sources/DataSourceEc2.py
+++ b/cloudinit/sources/DataSourceEc2.py
@@ -49,9 +49,6 @@ class DataSourceEc2(sources.DataSource):
         self.seed_dir = os.path.join(paths.seed_dir, "ec2")
         self.api_ver = DEF_MD_VERSION
 
-    def __str__(self):
-        return util.obj_name(self)
-
     def get_data(self):
         seed_ret = {}
         if util.read_optional_seed(seed_ret, base=(self.seed_dir + "/")):
diff --git a/cloudinit/sources/DataSourceMAAS.py b/cloudinit/sources/DataSourceMAAS.py
index b55d8a21..dfe90bc6 100644
--- a/cloudinit/sources/DataSourceMAAS.py
+++ b/cloudinit/sources/DataSourceMAAS.py
@@ -27,7 +27,7 @@ import urllib2
 
 from cloudinit import log as logging
 from cloudinit import sources
-from cloudinit import url_helper as uhelp
+from cloudinit import url_helper
 from cloudinit import util
 
 LOG = logging.getLogger(__name__)
@@ -50,7 +50,8 @@ class DataSourceMAAS(sources.DataSource):
         self.oauth_clockskew = None
 
     def __str__(self):
-        return "%s [%s]" % (util.obj_name(self), self.base_url)
+        root = sources.DataSource.__str__(self)
+        return "%s [%s]" % (root, self.base_url)
 
     def get_data(self):
         mcfg = self.ds_cfg
@@ -79,7 +80,8 @@ class DataSourceMAAS(sources.DataSource):
             self.base_url = url
 
             (userdata, metadata) = read_maas_seed_url(self.base_url,
-                                                      self.md_headers)
+                                                      self._md_headers,
+                                                      paths=self.paths)
             self.userdata_raw = userdata
             self.metadata = metadata
             return True
@@ -87,7 +89,7 @@ class DataSourceMAAS(sources.DataSource):
             util.logexc(LOG, "Failed fetching metadata from url %s", url)
             return False
 
-    def md_headers(self, url):
+    def _md_headers(self, url):
         mcfg = self.ds_cfg
 
         # If we are missing token_key, token_secret or consumer_key
@@ -131,36 +133,37 @@ class DataSourceMAAS(sources.DataSource):
         starttime = time.time()
         check_url = "%s/%s/meta-data/instance-id" % (url, MD_VERSION)
         urls = [check_url]
-        url = uhelp.wait_for_url(urls=urls, max_wait=max_wait,
-                                 timeout=timeout, exception_cb=self._except_cb,
-                                 headers_cb=self.md_headers)
+        url = url_helper.wait_for_url(urls=urls, max_wait=max_wait,
+                                      timeout=timeout,
+                                      exception_cb=self._except_cb,
+                                      headers_cb=self._md_headers)
 
         if url:
             LOG.debug("Using metadata source: '%s'", url)
         else:
             LOG.critical("Giving up on md from %s after %i seconds",
-                            urls, int(time.time() - starttime))
+                         urls, int(time.time() - starttime))
 
         return bool(url)
 
     def _except_cb(self, msg, exception):
-        if not (isinstance(exception, urllib2.HTTPError) and
+        if not (isinstance(exception, url_helper.UrlError) and
                 (exception.code == 403 or exception.code == 401)):
             return
+
         if 'date' not in exception.headers:
-            LOG.warn("date field not in %d headers" % exception.code)
+            LOG.warn("Missing header 'date' in %s response", exception.code)
             return
 
         date = exception.headers['date']
-
         try:
             ret_time = time.mktime(parsedate(date))
-        except:
-            LOG.warn("failed to convert datetime '%s'")
+        except Exception as e:
+            LOG.warn("Failed to convert datetime '%s': %s", date, e)
             return
 
         self.oauth_clockskew = int(ret_time - time.time())
-        LOG.warn("set oauth clockskew to %d" % self.oauth_clockskew)
+        LOG.warn("Setting oauth clockskew to %d", self.oauth_clockskew)
         return
 
 
@@ -188,11 +191,11 @@ def read_maas_seed_dir(seed_d):
 
 
 def read_maas_seed_url(seed_url, header_cb=None, timeout=None,
-    version=MD_VERSION):
+                       version=MD_VERSION, paths=None):
     """
     Read the maas datasource at seed_url.
-    header_cb is a method that should return a headers dictionary that will
-    be given to urllib2.Request()
+      - header_cb is a method that should return a headers dictionary for
+        a given url
 
     Expected format of seed_url is are the following files:
       * <seed_url>/<version>/meta-data/instance-id
@@ -215,18 +218,28 @@ def read_maas_seed_url(seed_url, header_cb=None, timeout=None,
     md = {}
     for name in file_order:
         url = files.get(name)
-        if header_cb:
-            headers = header_cb(url)
+        if not header_cb:
+            def _cb(url):
+                return {}
+            header_cb = _cb
+
+        if name == 'user-data':
+            retries = 0
         else:
-            headers = {}
+            retries = None
+
         try:
-            resp = uhelp.readurl(url, headers=headers, timeout=timeout)
+            ssl_details = util.fetch_ssl_details(paths)
+            resp = util.read_file_or_url(url, retries=retries,
+                                         headers_cb=header_cb,
+                                         timeout=timeout,
+                                         ssl_details=ssl_details)
             if resp.ok():
                 md[name] = str(resp)
             else:
                 LOG.warn(("Fetching from %s resulted in"
                           " an invalid http code %s"), url, resp.code)
-        except urllib2.HTTPError as e:
+        except url_helper.UrlError as e:
             if e.code != 404:
                 raise
     return check_seed_contents(md, seed_url)
@@ -369,7 +382,8 @@ if __name__ == "__main__":
         if args.subcmd == "check-seed":
             if args.url.startswith("http"):
                 (userdata, metadata) = read_maas_seed_url(args.url,
-                    header_cb=my_headers, version=args.apiver)
+                                                          header_cb=my_headers,
+                                                          version=args.apiver)
             else:
                 (userdata, metadata) = read_maas_seed_url(args.url)
             print "=== userdata ==="
diff --git a/cloudinit/sources/DataSourceNoCloud.py b/cloudinit/sources/DataSourceNoCloud.py
index 097bbc52..4ef92a56 100644
--- a/cloudinit/sources/DataSourceNoCloud.py
+++ b/cloudinit/sources/DataSourceNoCloud.py
@@ -1,7 +1,7 @@
 # vi: ts=4 expandtab
 #
 #    Copyright (C) 2009-2010 Canonical Ltd.
-#    Copyright (C) 2012 Hewlett-Packard Development Company, L.P.
+#    Copyright (C) 2012, 2013 Hewlett-Packard Development Company, L.P.
 #    Copyright (C) 2012 Yahoo! Inc.
 #
 #    Author: Scott Moser <scott.moser@canonical.com>
@@ -40,9 +40,8 @@ class DataSourceNoCloud(sources.DataSource):
         self.supported_seed_starts = ("/", "file://")
 
     def __str__(self):
-        mstr = "%s [seed=%s][dsmode=%s]" % (util.obj_name(self),
-                                            self.seed, self.dsmode)
-        return mstr
+        root = sources.DataSource.__str__(self)
+        return "%s [seed=%s][dsmode=%s]" % (root, self.seed, self.dsmode)
 
     def get_data(self):
         defaults = {
@@ -65,7 +64,7 @@ class DataSourceNoCloud(sources.DataSource):
         # Check to see if the seed dir has data.
         seedret = {}
         if util.read_optional_seed(seedret, base=self.seed_dir + "/"):
-            md = util.mergedict(md, seedret['meta-data'])
+            md = util.mergemanydict([md, seedret['meta-data']])
             ud = seedret['user-data']
             found.append(self.seed_dir)
             LOG.debug("Using seeded cache data from %s", self.seed_dir)
@@ -82,15 +81,19 @@ class DataSourceNoCloud(sources.DataSource):
             if self.ds_cfg['user-data']:
                 ud = self.ds_cfg['user-data']
             if self.ds_cfg['meta-data'] is not False:
-                md = util.mergedict(md, self.ds_cfg['meta-data'])
+                md = util.mergemanydict([md, self.ds_cfg['meta-data']])
             if 'ds_config' not in found:
                 found.append("ds_config")
 
-        if self.ds_cfg.get('fs_label', "cidata"):
+        label = self.ds_cfg.get('fs_label', "cidata")
+        if label is not None:
+            # Query optical drive to get it in blkid cache for 2.6 kernels
+            util.find_devs_with(path="/dev/sr0")
+            util.find_devs_with(path="/dev/sr1")
+
             fslist = util.find_devs_with("TYPE=vfat")
             fslist.extend(util.find_devs_with("TYPE=iso9660"))
 
-            label = self.ds_cfg.get('fs_label')
             label_list = util.find_devs_with("LABEL=%s" % label)
             devlist = list(set(fslist) & set(label_list))
             devlist.sort(reverse=True)
@@ -100,7 +103,7 @@ class DataSourceNoCloud(sources.DataSource):
                     LOG.debug("Attempting to use data from %s", dev)
 
                     (newmd, newud) = util.mount_cb(dev, util.read_seeded)
-                    md = util.mergedict(newmd, md)
+                    md = util.mergemanydict([newmd, md])
                     ud = newud
 
                     # For seed from a device, the default mode is 'net'.
@@ -116,8 +119,8 @@ class DataSourceNoCloud(sources.DataSource):
                     if e.errno != errno.ENOENT:
                         raise
                 except util.MountFailedError:
-                    util.logexc(LOG, ("Failed to mount %s"
-                                      " when looking for data"), dev)
+                    util.logexc(LOG, "Failed to mount %s when looking for "
+                                "data", dev)
 
         # There was no indication on kernel cmdline or data
         # in the seeddir suggesting this handler should be used.
@@ -150,11 +153,11 @@ class DataSourceNoCloud(sources.DataSource):
             LOG.debug("Using seeded cache data from %s", seedfrom)
 
             # Values in the command line override those from the seed
-            md = util.mergedict(md, md_seed)
+            md = util.mergemanydict([md, md_seed])
             found.append(seedfrom)
 
         # Now that we have exhausted any other places merge in the defaults
-        md = util.mergedict(md, defaults)
+        md = util.mergemanydict([md, defaults])
 
         # Update the network-interfaces if metadata had 'network-interfaces'
         # entry and this is the local datasource, or 'seedfrom' was used
diff --git a/cloudinit/sources/DataSourceNone.py b/cloudinit/sources/DataSourceNone.py
index c2125bee..12a8a992 100644
--- a/cloudinit/sources/DataSourceNone.py
+++ b/cloudinit/sources/DataSourceNone.py
@@ -18,7 +18,6 @@
 
 from cloudinit import log as logging
 from cloudinit import sources
-from cloudinit import util
 
 LOG = logging.getLogger(__name__)
 
@@ -41,9 +40,6 @@ class DataSourceNone(sources.DataSource):
     def get_instance_id(self):
         return 'iid-datasource-none'
 
-    def __str__(self):
-        return util.obj_name(self)
-
     @property
     def is_disconnected(self):
         return True
diff --git a/cloudinit/sources/DataSourceOVF.py b/cloudinit/sources/DataSourceOVF.py
index e90150c6..77b43e17 100644
--- a/cloudinit/sources/DataSourceOVF.py
+++ b/cloudinit/sources/DataSourceOVF.py
@@ -43,7 +43,8 @@ class DataSourceOVF(sources.DataSource):
         self.supported_seed_starts = ("/", "file://")
 
     def __str__(self):
-        return "%s [seed=%s]" % (util.obj_name(self), self.seed)
+        root = sources.DataSource.__str__(self)
+        return "%s [seed=%s]" % (root, self.seed)
 
     def get_data(self):
         found = []
@@ -93,11 +94,11 @@ class DataSourceOVF(sources.DataSource):
             (md_seed, ud) = util.read_seeded(seedfrom, timeout=None)
             LOG.debug("Using seeded cache data from %s", seedfrom)
 
-            md = util.mergedict(md, md_seed)
+            md = util.mergemanydict([md, md_seed])
             found.append(seedfrom)
 
         # Now that we have exhausted any other places merge in the defaults
-        md = util.mergedict(md, defaults)
+        md = util.mergemanydict([md, defaults])
 
         self.seed = ",".join(found)
         self.metadata = md
@@ -193,6 +194,11 @@ def transport_iso9660(require_iso=True):
         if contents is not False:
             return (contents, dev, fname)
 
+    if require_iso:
+        mtype = "iso9660"
+    else:
+        mtype = None
+
     devs = os.listdir("/dev/")
     devs.sort()
     for dev in devs:
@@ -210,7 +216,7 @@ def transport_iso9660(require_iso=True):
 
         try:
             (fname, contents) = util.mount_cb(fullp,
-                                               get_ovf_env, mtype="iso9660")
+                                               get_ovf_env, mtype=mtype)
         except util.MountFailedError:
             LOG.debug("%s not mountable as iso9660" % fullp)
             continue
diff --git a/cloudinit/sources/DataSourceSmartOS.py b/cloudinit/sources/DataSourceSmartOS.py
new file mode 100644
index 00000000..d348d20b
--- /dev/null
+++ b/cloudinit/sources/DataSourceSmartOS.py
@@ -0,0 +1,244 @@
+# vi: ts=4 expandtab
+#
+#    Copyright (C) 2013 Canonical Ltd.
+#
+#    Author: Ben Howard <ben.howard@canonical.com>
+#
+#    This program is free software: you can redistribute it and/or modify
+#    it under the terms of the GNU General Public License version 3, as
+#    published by the Free Software Foundation.
+#
+#    This program is distributed in the hope that it will be useful,
+#    but WITHOUT ANY WARRANTY; without even the implied warranty of
+#    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+#    GNU General Public License for more details.
+#
+#    You should have received a copy of the GNU General Public License
+#    along with this program.  If not, see <http://www.gnu.org/licenses/>.
+#
+#
+#    Datasource for provisioning on SmartOS. This works on Joyent
+#        and public/private Clouds using SmartOS.
+#
+#    SmartOS hosts use a serial console (/dev/ttyS1) on Linux Guests.
+#        The meta-data is transmitted via key/value pairs made by
+#        requests on the console. For example, to get the hostname, you
+#        would send "GET hostname" on /dev/ttyS1.
+#
+
+
+import base64
+from cloudinit import log as logging
+from cloudinit import sources
+from cloudinit import util
+import os
+import os.path
+import serial
+
+DEF_TTY_LOC = '/dev/ttyS1'
+DEF_TTY_TIMEOUT = 60
+LOG = logging.getLogger(__name__)
+
+SMARTOS_ATTRIB_MAP = {
+    #Cloud-init Key : (SmartOS Key, Strip line endings)
+    'local-hostname': ('hostname', True),
+    'public-keys': ('root_authorized_keys', True),
+    'user-script': ('user-script', False),
+    'user-data': ('user-data', False),
+    'iptables_disable': ('iptables_disable', True),
+    'motd_sys_info': ('motd_sys_info', True),
+}
+
+# These are values which will never be base64 encoded.
+# They come from the cloud platform, not user
+SMARTOS_NO_BASE64 = ['root_authorized_keys', 'motd_sys_info',
+                     'iptables_disable']
+
+
+class DataSourceSmartOS(sources.DataSource):
+    def __init__(self, sys_cfg, distro, paths):
+        sources.DataSource.__init__(self, sys_cfg, distro, paths)
+        self.seed_dir = os.path.join(paths.seed_dir, 'sdc')
+        self.is_smartdc = None
+
+        self.seed = self.ds_cfg.get("serial_device", DEF_TTY_LOC)
+        self.seed_timeout = self.ds_cfg.get("serial_timeout", DEF_TTY_TIMEOUT)
+        self.smartos_no_base64 = self.ds_cfg.get('no_base64_decode',
+                                                 SMARTOS_NO_BASE64)
+        self.b64_keys = self.ds_cfg.get('base64_keys', [])
+        self.b64_all = self.ds_cfg.get('base64_all', False)
+
+    def __str__(self):
+        root = sources.DataSource.__str__(self)
+        return "%s [seed=%s]" % (root, self.seed)
+
+    def get_data(self):
+        md = {}
+        ud = ""
+
+        if not os.path.exists(self.seed):
+            LOG.debug("Host does not appear to be on SmartOS")
+            return False
+        self.seed = self.seed
+
+        dmi_info = dmi_data()
+        if dmi_info is False:
+            LOG.debug("No dmidata utility found")
+            return False
+
+        system_uuid, system_type = dmi_info
+        if 'smartdc' not in system_type.lower():
+            LOG.debug("Host is not on SmartOS. system_type=%s", system_type)
+            return False
+        self.is_smartdc = True
+        md['instance-id'] = system_uuid
+
+        b64_keys = self.query('base64_keys', strip=True, b64=False)
+        if b64_keys is not None:
+            self.b64_keys = [k.strip() for k in str(b64_keys).split(',')]
+
+        b64_all = self.query('base64_all', strip=True, b64=False)
+        if b64_all is not None:
+            self.b64_all = util.is_true(b64_all)
+
+        for ci_noun, attribute in SMARTOS_ATTRIB_MAP.iteritems():
+            smartos_noun, strip = attribute
+            md[ci_noun] = self.query(smartos_noun, strip=strip)
+
+        if not md['local-hostname']:
+            md['local-hostname'] = system_uuid
+
+        ud = None
+        if md['user-data']:
+            ud = md['user-data']
+        elif md['user-script']:
+            ud = md['user-script']
+
+        self.metadata = md
+        self.userdata_raw = ud
+        return True
+
+    def get_instance_id(self):
+        return self.metadata['instance-id']
+
+    def query(self, noun, strip=False, default=None, b64=None):
+        if b64 is None:
+            if noun in self.smartos_no_base64:
+                b64 = False
+            elif self.b64_all or noun in self.b64_keys:
+                b64 = True
+
+        return query_data(noun=noun, strip=strip, seed_device=self.seed,
+                          seed_timeout=self.seed_timeout, default=default,
+                          b64=b64)
+
+
+def get_serial(seed_device, seed_timeout):
+    """This is replaced in unit testing, allowing us to replace
+        serial.Serial with a mocked class.
+
+        The timeout value of 60 seconds should never be hit. The value
+        is taken from SmartOS own provisioning tools. Since we are reading
+        each line individually up until the single ".", the transfer is
+        usually very fast (i.e. microseconds) to get the response.
+    """
+    if not seed_device:
+        raise AttributeError("seed_device value is not set")
+
+    ser = serial.Serial(seed_device, timeout=seed_timeout)
+    if not ser.isOpen():
+        raise SystemError("Unable to open %s" % seed_device)
+
+    return ser
+
+
+def query_data(noun, seed_device, seed_timeout, strip=False, default=None,
+               b64=None):
+    """Makes a request to via the serial console via "GET <NOUN>"
+
+        In the response, the first line is the status, while subsequent lines
+        are is the value. A blank line with a "." is used to indicate end of
+        response.
+
+        If the response is expected to be base64 encoded, then set b64encoded
+        to true. Unfortantely, there is no way to know if something is 100%
+        encoded, so this method relies on being told if the data is base64 or
+        not.
+    """
+
+    if not noun:
+        return False
+
+    ser = get_serial(seed_device, seed_timeout)
+    ser.write("GET %s\n" % noun.rstrip())
+    status = str(ser.readline()).rstrip()
+    response = []
+    eom_found = False
+
+    if 'SUCCESS' not in status:
+        ser.close()
+        return default
+
+    while not eom_found:
+        m = ser.readline()
+        if m.rstrip() == ".":
+            eom_found = True
+        else:
+            response.append(m)
+
+    ser.close()
+
+    if b64 is None:
+        b64 = query_data('b64-%s' % noun, seed_device=seed_device,
+                            seed_timeout=seed_timeout, b64=False,
+                            default=False, strip=True)
+        b64 = util.is_true(b64)
+
+    resp = None
+    if b64 or strip:
+        resp = "".join(response).rstrip()
+    else:
+        resp = "".join(response)
+
+    if b64:
+        try:
+            return base64.b64decode(resp)
+        except TypeError:
+            LOG.warn("Failed base64 decoding key '%s'", noun)
+            return resp
+
+    return resp
+
+
+def dmi_data():
+    sys_uuid, sys_type = None, None
+    dmidecode_path = util.which('dmidecode')
+    if not dmidecode_path:
+        return False
+
+    sys_uuid_cmd = [dmidecode_path, "-s", "system-uuid"]
+    try:
+        LOG.debug("Getting hostname from dmidecode")
+        (sys_uuid, _err) = util.subp(sys_uuid_cmd)
+    except Exception as e:
+        util.logexc(LOG, "Failed to get system UUID", e)
+
+    sys_type_cmd = [dmidecode_path, "-s", "system-product-name"]
+    try:
+        LOG.debug("Determining hypervisor product name via dmidecode")
+        (sys_type, _err) = util.subp(sys_type_cmd)
+    except Exception as e:
+        util.logexc(LOG, "Failed to get system UUID", e)
+
+    return sys_uuid.lower(), sys_type
+
+
+# Used to match classes to dependencies
+datasources = [
+    (DataSourceSmartOS, (sources.DEP_FILESYSTEM, sources.DEP_NETWORK)),
+]
+
+
+# Return a list of data sources that match this set of dependencies
+def get_datasource_list(depends):
+    return sources.list_from_depends(depends, datasources)
diff --git a/cloudinit/sources/__init__.py b/cloudinit/sources/__init__.py
index 0bad4c8b..b0e43954 100644
--- a/cloudinit/sources/__init__.py
+++ b/cloudinit/sources/__init__.py
@@ -25,6 +25,7 @@ import os
 
 from cloudinit import importer
 from cloudinit import log as logging
+from cloudinit import type_utils
 from cloudinit import user_data as ud
 from cloudinit import util
 
@@ -52,7 +53,7 @@ class DataSource(object):
         self.userdata = None
         self.metadata = None
         self.userdata_raw = None
-        name = util.obj_name(self)
+        name = type_utils.obj_name(self)
         if name.startswith(DS_PREFIX):
             name = name[len(DS_PREFIX):]
         self.ds_cfg = util.get_cfg_by_path(self.sys_cfg,
@@ -62,6 +63,9 @@ class DataSource(object):
         else:
             self.ud_proc = ud_proc
 
+    def __str__(self):
+        return type_utils.obj_name(self)
+
     def get_userdata(self, apply_filter=False):
         if self.userdata is None:
             self.userdata = self.ud_proc.process(self.get_userdata_raw())
@@ -131,7 +135,8 @@ class DataSource(object):
 
     @property
     def availability_zone(self):
-        return self.metadata.get('availability-zone')
+        return self.metadata.get('availability-zone',
+                                 self.metadata.get('availability_zone'))
 
     def get_instance_id(self):
         if not self.metadata or 'instance-id' not in self.metadata:
@@ -221,7 +226,7 @@ def normalize_pubkey_data(pubkey_data):
 
 def find_source(sys_cfg, distro, paths, ds_deps, cfg_list, pkg_list):
     ds_list = list_sources(cfg_list, ds_deps, pkg_list)
-    ds_names = [util.obj_name(f) for f in ds_list]
+    ds_names = [type_utils.obj_name(f) for f in ds_list]
     LOG.debug("Searching for data source in: %s", ds_names)
 
     for cls in ds_list:
@@ -229,7 +234,7 @@ def find_source(sys_cfg, distro, paths, ds_deps, cfg_list, pkg_list):
             LOG.debug("Seeing if we can get any data from %s", cls)
             s = cls(sys_cfg, distro, paths)
             if s.get_data():
-                return (s, util.obj_name(cls))
+                return (s, type_utils.obj_name(cls))
         except Exception:
             util.logexc(LOG, "Getting data from %s failed", cls)