Rebase on upstream

27 files changed, 1257 insertions, 308 deletions

diff --git a/cloudinit/config/cc_debug.py b/cloudinit/config/cc_debug.py
index cfd31fa1..7219b0f8 100644
--- a/cloudinit/config/cc_debug.py
+++ b/cloudinit/config/cc_debug.py
@@ -14,10 +14,10 @@
 #    You should have received a copy of the GNU General Public License
 #    along with this program.  If not, see <http://www.gnu.org/licenses/>.
 
-from StringIO import StringIO
-from cloudinit import util
 from cloudinit import type_utils
+from cloudinit import util
 import copy
+from StringIO import StringIO
 
 
 def _make_header(text):
diff --git a/cloudinit/config/cc_growpart.py b/cloudinit/config/cc_growpart.py
index 0dd92a46..f52c41f0 100644
--- a/cloudinit/config/cc_growpart.py
+++ b/cloudinit/config/cc_growpart.py
@@ -80,30 +80,6 @@ class ResizeFailedException(Exception):
     pass
 
 
-class ResizeParted(object):
-    def available(self):
-        myenv = os.environ.copy()
-        myenv['LANG'] = 'C'
-
-        try:
-            (out, _err) = util.subp(["parted", "--help"], env=myenv)
-            if re.search(r"COMMAND.*resizepart\s+", out, re.DOTALL):
-                return True
-
-        except util.ProcessExecutionError:
-            pass
-        return False
-
-    def resize(self, diskdev, partnum, partdev):
-        before = get_size(partdev)
-        try:
-            util.subp(["parted", diskdev, "resizepart", partnum])
-        except util.ProcessExecutionError as e:
-            raise ResizeFailedException(e)
-
-        return (before, get_size(partdev))
-
-
 class ResizeGrowPart(object):
     def available(self):
         myenv = os.environ.copy()
@@ -138,6 +114,41 @@ class ResizeGrowPart(object):
         return (before, get_size(partdev))
 
 
+class ResizeGpart(object):
+    def available(self):
+        if not util.which('gpart'):
+            return False
+        return True
+
+    def resize(self, diskdev, partnum, partdev):
+        """
+        GPT disks store metadata at the beginning (primary) and at the
+        end (secondary) of the disk. When launching an image with a
+        larger disk compared to the original image, the secondary copy
+        is lost. Thus, the metadata will be marked CORRUPT, and need to
+        be recovered.
+        """
+        try:
+            util.subp(["gpart", "recover", diskdev])
+        except util.ProcessExecutionError as e:
+            if e.exit_code != 0:
+                util.logexc(LOG, "Failed: gpart recover %s", diskdev)
+                raise ResizeFailedException(e)
+
+        before = get_size(partdev)
+        try:
+            util.subp(["gpart", "resize", "-i", partnum, diskdev])
+        except util.ProcessExecutionError as e:
+            util.logexc(LOG, "Failed: gpart resize -i %s %s", partnum, diskdev)
+            raise ResizeFailedException(e)
+
+        # Since growing the FS requires a reboot, make sure we reboot
+        # first when this module has finished.
+        open('/var/run/reboot-required', 'a').close()
+
+        return (before, get_size(partdev))
+
+
 def get_size(filename):
     fd = os.open(filename, os.O_RDONLY)
     try:
@@ -156,6 +167,12 @@ def device_part_info(devpath):
     bname = os.path.basename(rpath)
     syspath = "/sys/class/block/%s" % bname
 
+    # FreeBSD doesn't know of sysfs so just get everything we need from
+    # the device, like /dev/vtbd0p2.
+    if util.system_info()["platform"].startswith('FreeBSD'):
+        m = re.search('^(/dev/.+)p([0-9])$', devpath)
+        return (m.group(1), m.group(2))
+
     if not os.path.exists(syspath):
         raise ValueError("%s had no syspath (%s)" % (devpath, syspath))
 
@@ -206,7 +223,8 @@ def resize_devices(resizer, devices):
                          "stat of '%s' failed: %s" % (blockdev, e),))
             continue
 
-        if not stat.S_ISBLK(statret.st_mode):
+        if (not stat.S_ISBLK(statret.st_mode) and
+                not stat.S_ISCHR(statret.st_mode)):
             info.append((devent, RESIZE.SKIPPED,
                          "device '%s' not a block device" % blockdev,))
             continue
@@ -279,6 +297,4 @@ def handle(_name, cfg, _cloud, log, _args):
         else:
             log.debug("'%s' %s: %s" % (entry, action, msg))
 
-# LP: 1212444 FIXME re-order and favor ResizeParted
-#RESIZERS = (('growpart', ResizeGrowPart),)
-RESIZERS = (('growpart', ResizeGrowPart), ('parted', ResizeParted))
+RESIZERS = (('growpart', ResizeGrowPart), ('gpart', ResizeGpart))
diff --git a/cloudinit/config/cc_power_state_change.py b/cloudinit/config/cc_power_state_change.py
index e3150808..561c5abd 100644
--- a/cloudinit/config/cc_power_state_change.py
+++ b/cloudinit/config/cc_power_state_change.py
@@ -22,6 +22,7 @@ from cloudinit import util
 import errno
 import os
 import re
+import signal
 import subprocess
 import time
 
@@ -30,6 +31,24 @@ frequency = PER_INSTANCE
 EXIT_FAIL = 254
 
 
+def givecmdline(pid):
+    # Returns the cmdline for the given process id. In Linux we can use procfs
+    # for this but on BSD there is /usr/bin/procstat.
+    try:
+        # Example output from procstat -c 1
+        #   PID COMM             ARGS
+        #     1 init             /bin/init --
+        if util.system_info()["platform"].startswith('FreeBSD'):
+            (output, _err) = util.subp(['procstat', '-c', str(pid)])
+            line = output.splitlines()[1]
+            m = re.search('\d+ (\w|\.|-)+\s+(/\w.+)', line)
+            return m.group(2)
+        else:
+            return util.load_file("/proc/%s/cmdline" % pid)
+    except IOError:
+        return None
+
+
 def handle(_name, cfg, _cloud, log, _args):
 
     try:
@@ -42,8 +61,8 @@ def handle(_name, cfg, _cloud, log, _args):
         return
 
     mypid = os.getpid()
-    cmdline = util.load_file("/proc/%s/cmdline" % mypid)
 
+    cmdline = givecmdline(mypid)
     if not cmdline:
         log.warn("power_state: failed to get cmdline of current process")
         return
@@ -119,8 +138,6 @@ def run_after_pid_gone(pid, pidcmdline, timeout, log, func, args):
     msg = None
     end_time = time.time() + timeout
 
-    cmdline_f = "/proc/%s/cmdline" % pid
-
     def fatal(msg):
         if log:
             log.warn(msg)
@@ -134,16 +151,14 @@ def run_after_pid_gone(pid, pidcmdline, timeout, log, func, args):
             break
 
         try:
-            cmdline = ""
-            with open(cmdline_f) as fp:
-                cmdline = fp.read()
+            cmdline = givecmdline(pid)
             if cmdline != pidcmdline:
                 msg = "cmdline changed for %s [now: %s]" % (pid, cmdline)
                 break
 
         except IOError as ioerr:
             if ioerr.errno in known_errnos:
-                msg = "pidfile '%s' gone [%d]" % (cmdline_f, ioerr.errno)
+                msg = "pidfile gone [%d]" % ioerr.errno
             else:
                 fatal("IOError during wait: %s" % ioerr)
             break
diff --git a/cloudinit/config/cc_resizefs.py b/cloudinit/config/cc_resizefs.py
index 56040fdd..be406034 100644
--- a/cloudinit/config/cc_resizefs.py
+++ b/cloudinit/config/cc_resizefs.py
@@ -39,6 +39,10 @@ def _resize_ext(mount_point, devpth):  # pylint: disable=W0613
 def _resize_xfs(mount_point, devpth):  # pylint: disable=W0613
     return ('xfs_growfs', devpth)
 
+
+def _resize_ufs(mount_point, devpth):  # pylint: disable=W0613
+    return ('growfs', devpth)
+
 # Do not use a dictionary as these commands should be able to be used
 # for multiple filesystem types if possible, e.g. one command for
 # ext2, ext3 and ext4.
@@ -46,11 +50,31 @@ RESIZE_FS_PREFIXES_CMDS = [
     ('btrfs', _resize_btrfs),
     ('ext', _resize_ext),
     ('xfs', _resize_xfs),
+    ('ufs', _resize_ufs),
 ]
 
 NOBLOCK = "noblock"
 
 
+def rootdev_from_cmdline(cmdline):
+    found = None
+    for tok in cmdline.split():
+        if tok.startswith("root="):
+            found = tok[5:]
+            break
+    if found is None:
+        return None
+
+    if found.startswith("/dev/"):
+        return found
+    if found.startswith("LABEL="):
+        return "/dev/disk/by-label/" + found[len("LABEL="):]
+    if found.startswith("UUID="):
+        return "/dev/disk/by-uuid/" + found[len("UUID="):]
+
+    return "/dev/" + found
+
+
 def handle(name, cfg, _cloud, log, args):
     if len(args) != 0:
         resize_root = args[0]
@@ -78,10 +102,20 @@ def handle(name, cfg, _cloud, log, args):
     info = "dev=%s mnt_point=%s path=%s" % (devpth, mount_point, resize_what)
     log.debug("resize_info: %s" % info)
 
+    container = util.is_container()
+
+    if (devpth == "/dev/root" and not os.path.exists(devpth) and
+        not container):
+        devpth = rootdev_from_cmdline(util.get_cmdline())
+        if devpth is None:
+            log.warn("Unable to find device '/dev/root'")
+            return
+        log.debug("Converted /dev/root to '%s' per kernel cmdline", devpth)
+
     try:
         statret = os.stat(devpth)
     except OSError as exc:
-        if util.is_container() and exc.errno == errno.ENOENT:
+        if container and exc.errno == errno.ENOENT:
             log.debug("Device '%s' did not exist in container. "
                       "cannot resize: %s" % (devpth, info))
         elif exc.errno == errno.ENOENT:
@@ -91,8 +125,8 @@ def handle(name, cfg, _cloud, log, args):
             raise exc
         return
 
-    if not stat.S_ISBLK(statret.st_mode):
-        if util.is_container():
+    if not stat.S_ISBLK(statret.st_mode) and not stat.S_ISCHR(statret.st_mode):
+        if container:
             log.debug("device '%s' not a block device in container."
                       " cannot resize: %s" % (devpth, info))
         else:
diff --git a/cloudinit/config/cc_scripts_vendor.py b/cloudinit/config/cc_scripts_vendor.py
new file mode 100644
index 00000000..0c9e504e
--- /dev/null
+++ b/cloudinit/config/cc_scripts_vendor.py
@@ -0,0 +1,43 @@
+# vi: ts=4 expandtab
+#
+#    Copyright (C) 2014 Canonical Ltd.
+#
+#    Author: Ben Howard <ben.howard@canonical.com>
+#
+#    This program is free software: you can redistribute it and/or modify
+#    it under the terms of the GNU General Public License version 3, as
+#    published by the Free Software Foundation.
+#
+#    This program is distributed in the hope that it will be useful,
+#    but WITHOUT ANY WARRANTY; without even the implied warranty of
+#    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+#    GNU General Public License for more details.
+#
+#    You should have received a copy of the GNU General Public License
+#    along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+import os
+
+from cloudinit import util
+
+from cloudinit.settings import PER_INSTANCE
+
+frequency = PER_INSTANCE
+
+SCRIPT_SUBDIR = 'vendor'
+
+
+def handle(name, cfg, cloud, log, _args):
+    # This is written to by the vendor data handlers
+    # any vendor data shell scripts get placed in runparts_path
+    runparts_path = os.path.join(cloud.get_ipath_cur(), 'scripts',
+                                 SCRIPT_SUBDIR)
+
+    prefix = util.get_cfg_by_path(cfg, ('vendor_data', 'prefix'), [])
+
+    try:
+        util.runparts(runparts_path, exe_prefix=prefix)
+    except:
+        log.warn("Failed to run module %s (%s in %s)",
+                 name, SCRIPT_SUBDIR, runparts_path)
+        raise
diff --git a/cloudinit/distros/__init__.py b/cloudinit/distros/__init__.py
index 74e95797..46b67fa3 100644
--- a/cloudinit/distros/__init__.py
+++ b/cloudinit/distros/__init__.py
@@ -39,6 +39,7 @@ from cloudinit.distros.parsers import hosts
 OSFAMILIES = {
     'debian': ['debian', 'ubuntu'],
     'redhat': ['fedora', 'rhel'],
+    'freebsd': ['freebsd'],
     'suse': ['sles']
 }
 
diff --git a/cloudinit/distros/freebsd.py b/cloudinit/distros/freebsd.py
new file mode 100644
index 00000000..afb502c9
--- /dev/null
+++ b/cloudinit/distros/freebsd.py
@@ -0,0 +1,259 @@
+# vi: ts=4 expandtab
+#
+#    Copyright (C) 2014 Harm Weites
+#
+#    Author: Harm Weites <harm@weites.com>
+#
+#    This program is free software: you can redistribute it and/or modify
+#    it under the terms of the GNU General Public License version 3, as
+#    published by the Free Software Foundation.
+#
+#    This program is distributed in the hope that it will be useful,
+#    but WITHOUT ANY WARRANTY; without even the implied warranty of
+#    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+#    GNU General Public License for more details.
+#
+#    You should have received a copy of the GNU General Public License
+#    along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+from StringIO import StringIO
+
+import re
+
+from cloudinit import distros
+from cloudinit import helpers
+from cloudinit import log as logging
+from cloudinit import ssh_util
+from cloudinit import util
+
+LOG = logging.getLogger(__name__)
+
+
+class Distro(distros.Distro):
+    rc_conf_fn = "/etc/rc.conf"
+    login_conf_fn = '/etc/login.conf'
+    login_conf_fn_bak = '/etc/login.conf.orig'
+
+    def __init__(self, name, cfg, paths):
+        distros.Distro.__init__(self, name, cfg, paths)
+        # This will be used to restrict certain
+        # calls from repeatly happening (when they
+        # should only happen say once per instance...)
+        self._runner = helpers.Runners(paths)
+        self.osfamily = 'freebsd'
+
+    # Updates a key in /etc/rc.conf.
+    def updatercconf(self, key, value):
+        LOG.debug("updatercconf: %s => %s", key, value)
+        conf = self.loadrcconf()
+        config_changed = False
+        for item in conf:
+            if item == key and conf[item] != value:
+                conf[item] = value
+                LOG.debug("[rc.conf]: Value %s for key %s needs to be changed",
+                          value, key)
+                config_changed = True
+
+        if config_changed:
+            LOG.debug("Writing new %s file", self.rc_conf_fn)
+            buf = StringIO()
+            for keyval in conf.items():
+                buf.write("%s=%s\n" % keyval)
+            util.write_file(self.rc_conf_fn, buf.getvalue())
+
+    # Load the contents of /etc/rc.conf and store all keys in a dict.
+    def loadrcconf(self):
+        conf = {}
+        lines = util.load_file(self.rc_conf_fn).splitlines()
+        for line in lines:
+            tok = line.split('=')
+            conf[tok[0]] = tok[1].rstrip()
+        return conf
+
+    def readrcconf(self, key):
+        conf = self.loadrcconf()
+        try:
+            val = conf[key]
+        except KeyError:
+            val = None
+        return val
+
+    def _read_system_hostname(self):
+        sys_hostname = self._read_hostname()
+        return ('rc.conf', sys_hostname)
+
+    def _read_hostname(self, filename, default=None):
+        hostname = None
+        try:
+            hostname = self.readrcconf('hostname')
+        except IOError:
+            pass
+        if not hostname:
+            return default
+        return hostname
+
+    def _select_hostname(self, hostname, fqdn):
+        if not hostname:
+            return fqdn
+        return hostname
+
+    def _write_hostname(self, hostname, filename):
+        self.updatercconf('hostname', hostname)
+
+    def create_group(self, name, members):
+        group_add_cmd = ['pw', '-n', name]
+        if util.is_group(name):
+            LOG.warn("Skipping creation of existing group '%s'", name)
+        else:
+            try:
+                util.subp(group_add_cmd)
+                LOG.info("Created new group %s", name)
+            except Exception as e:
+                util.logexc(LOG, "Failed to create group %s", name)
+                raise e
+
+        if len(members) > 0:
+            for member in members:
+                if not util.is_user(member):
+                    LOG.warn("Unable to add group member '%s' to group '%s'"
+                             "; user does not exist.", member, name)
+                    continue
+                try:
+                    util.subp(['pw', 'usermod', '-n', name, '-G', member])
+                    LOG.info("Added user '%s' to group '%s'", member, name)
+                except Exception:
+                    util.logexc(LOG, "Failed to add user '%s' to group '%s'",
+                                member, name)
+
+    def add_user(self, name, **kwargs):
+        if util.is_user(name):
+            LOG.info("User %s already exists, skipping.", name)
+            return False
+
+        adduser_cmd = ['pw', 'useradd', '-n', name]
+        log_adduser_cmd = ['pw', 'useradd', '-n', name]
+
+        adduser_opts = {
+                "homedir": '-d',
+                "gecos": '-c',
+                "primary_group": '-g',
+                "groups": '-G',
+                "passwd": '-h',
+                "shell": '-s',
+                "inactive": '-E',
+        }
+        adduser_flags = {
+                "no_user_group": '--no-user-group',
+                "system": '--system',
+                "no_log_init": '--no-log-init',
+        }
+
+        redact_opts = ['passwd']
+
+        for key, val in kwargs.iteritems():
+            if key in adduser_opts and val and isinstance(val, basestring):
+                adduser_cmd.extend([adduser_opts[key], val])
+
+                # Redact certain fields from the logs
+                if key in redact_opts:
+                    log_adduser_cmd.extend([adduser_opts[key], 'REDACTED'])
+                else:
+                    log_adduser_cmd.extend([adduser_opts[key], val])
+
+            elif key in adduser_flags and val:
+                adduser_cmd.append(adduser_flags[key])
+                log_adduser_cmd.append(adduser_flags[key])
+
+        if 'no_create_home' in kwargs or 'system' in kwargs:
+            adduser_cmd.append('-d/nonexistent')
+            log_adduser_cmd.append('-d/nonexistent')
+        else:
+            adduser_cmd.append('-d/usr/home/%s' % name)
+            adduser_cmd.append('-m')
+            log_adduser_cmd.append('-d/usr/home/%s' % name)
+            log_adduser_cmd.append('-m')
+
+        # Run the command
+        LOG.info("Adding user %s", name)
+        try:
+            util.subp(adduser_cmd, logstring=log_adduser_cmd)
+        except Exception as e:
+            util.logexc(LOG, "Failed to create user %s", name)
+            raise e
+
+    # TODO:
+    def set_passwd(self, user, passwd, hashed=False):
+        return False
+
+    def lock_passwd(self, name):
+        try:
+            util.subp(['pw', 'usermod', name, '-h', '-'])
+        except Exception as e:
+            util.logexc(LOG, "Failed to lock user %s", name)
+            raise e
+
+    # TODO:
+    def write_sudo_rules(self, name, rules, sudo_file=None):
+        LOG.debug("[write_sudo_rules] Name: %s", name)
+
+    def create_user(self, name, **kwargs):
+        self.add_user(name, **kwargs)
+
+        # Set password if plain-text password provided and non-empty
+        if 'plain_text_passwd' in kwargs and kwargs['plain_text_passwd']:
+            self.set_passwd(name, kwargs['plain_text_passwd'])
+
+        # Default locking down the account. 'lock_passwd' defaults to True.
+        # lock account unless lock_password is False.
+        if kwargs.get('lock_passwd', True):
+            self.lock_passwd(name)
+
+        # Configure sudo access
+        if 'sudo' in kwargs:
+            self.write_sudo_rules(name, kwargs['sudo'])
+
+        # Import SSH keys
+        if 'ssh_authorized_keys' in kwargs:
+            keys = set(kwargs['ssh_authorized_keys']) or []
+            ssh_util.setup_user_keys(keys, name, options=None)
+
+    def _write_network(self, settings):
+        return
+
+    def apply_locale(self, locale, out_fn=None):
+        # Adjust the locals value to the new value
+        newconf = StringIO()
+        for line in util.load_file(self.login_conf_fn).splitlines():
+            newconf.write(re.sub(r'^default:',
+                                 r'default:lang=%s:' % locale, line))
+            newconf.write("\n")
+
+        # Make a backup of login.conf.
+        util.copy(self.login_conf_fn, self.login_conf_fn_bak)
+
+        # And write the new login.conf.
+        util.write_file(self.login_conf_fn, newconf.getvalue())
+
+        try:
+            LOG.debug("Running cap_mkdb for %s", locale)
+            util.subp(['cap_mkdb', self.login_conf_fn])
+        except util.ProcessExecutionError:
+            # cap_mkdb failed, so restore the backup.
+            util.logexc(LOG, "Failed to apply locale %s", locale)
+            try:
+                util.copy(self.login_conf_fn_bak, self.login_conf_fn)
+            except IOError:
+                util.logexc(LOG, "Failed to restore %s backup",
+                            self.login_conf_fn)
+
+    def install_packages(self, pkglist):
+        return
+
+    def package_command(self, cmd, args=None, pkgs=None):
+        return
+
+    def set_timezone(self, tz):
+        return
+
+    def update_package_sources(self):
+        return
diff --git a/cloudinit/distros/net_util.py b/cloudinit/distros/net_util.py
new file mode 100644
index 00000000..b9bcfd8b
--- /dev/null
+++ b/cloudinit/distros/net_util.py
@@ -0,0 +1,163 @@
+# vi: ts=4 expandtab
+#
+#    Copyright (C) 2012 Canonical Ltd.
+#    Copyright (C) 2012, 2013 Hewlett-Packard Development Company, L.P.
+#    Copyright (C) 2012 Yahoo! Inc.
+#
+#    Author: Scott Moser <scott.moser@canonical.com>
+#    Author: Juerg Haefliger <juerg.haefliger@hp.com>
+#    Author: Joshua Harlow <harlowja@yahoo-inc.com>
+#
+#    This program is free software: you can redistribute it and/or modify
+#    it under the terms of the GNU General Public License version 3, as
+#    published by the Free Software Foundation.
+#
+#    This program is distributed in the hope that it will be useful,
+#    but WITHOUT ANY WARRANTY; without even the implied warranty of
+#    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+#    GNU General Public License for more details.
+#
+#    You should have received a copy of the GNU General Public License
+#    along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+
+# This is a util function to translate debian based distro interface blobs as
+# given in /etc/network/interfaces to an *somewhat* agnostic format for
+# distributions that use other formats.
+#
+# TODO(harlowja) remove when we have python-netcf active...
+#
+# The format is the following:
+# {
+#    <device-name>: {
+#        # All optional (if not existent in original format)
+#        "netmask": <ip>,
+#        "broadcast": <ip>,
+#        "gateway": <ip>,
+#        "address": <ip>,
+#        "bootproto": "static"|"dhcp",
+#        "dns-search": <hostname>,
+#        "hwaddress": <mac-address>,
+#        "auto": True (or non-existent),
+#        "dns-nameservers": [<ip/hostname>, ...],
+#    }
+# }
+#
+# Things to note, comments are removed, if a ubuntu/debian interface is
+# marked as auto then only then first segment (?) is retained, ie
+# 'auto eth0 eth0:1' just marks eth0 as auto (not eth0:1).
+#
+# Example input:
+#
+# auto lo
+# iface lo inet loopback
+#
+# auto eth0
+# iface eth0 inet static
+#         address 10.0.0.1
+#         netmask 255.255.252.0
+#         broadcast 10.0.0.255
+#         gateway 10.0.0.2
+#         dns-nameservers 98.0.0.1 98.0.0.2
+#
+# Example output:
+# {
+#     "lo": {
+#         "auto": true
+#     },
+#     "eth0": {
+#         "auto": true,
+#         "dns-nameservers": [
+#             "98.0.0.1",
+#             "98.0.0.2"
+#         ],
+#         "broadcast": "10.0.0.255",
+#         "netmask": "255.255.252.0",
+#         "bootproto": "static",
+#         "address": "10.0.0.1",
+#         "gateway": "10.0.0.2"
+#     }
+# }
+
+def translate_network(settings):
+    # Get the standard cmd, args from the ubuntu format
+    entries = []
+    for line in settings.splitlines():
+        line = line.strip()
+        if not line or line.startswith("#"):
+            continue
+        split_up = line.split(None, 1)
+        if len(split_up) <= 1:
+            continue
+        entries.append(split_up)
+    # Figure out where each iface section is
+    ifaces = []
+    consume = {}
+    for (cmd, args) in entries:
+        if cmd == 'iface':
+            if consume:
+                ifaces.append(consume)
+                consume = {}
+            consume[cmd] = args
+        else:
+            consume[cmd] = args
+    # Check if anything left over to consume
+    absorb = False
+    for (cmd, args) in consume.iteritems():
+        if cmd == 'iface':
+            absorb = True
+    if absorb:
+        ifaces.append(consume)
+    # Now translate
+    real_ifaces = {}
+    for info in ifaces:
+        if 'iface' not in info:
+            continue
+        iface_details = info['iface'].split(None)
+        dev_name = None
+        if len(iface_details) >= 1:
+            dev = iface_details[0].strip().lower()
+            if dev:
+                dev_name = dev
+        if not dev_name:
+            continue
+        iface_info = {}
+        if len(iface_details) >= 3:
+            proto_type = iface_details[2].strip().lower()
+            # Seems like this can be 'loopback' which we don't
+            # really care about
+            if proto_type in ['dhcp', 'static']:
+                iface_info['bootproto'] = proto_type
+        # These can just be copied over
+        for k in ['netmask', 'address', 'gateway', 'broadcast']:
+            if k in info:
+                val = info[k].strip().lower()
+                if val:
+                    iface_info[k] = val
+        # Name server info provided??
+        if 'dns-nameservers' in info:
+            iface_info['dns-nameservers'] = info['dns-nameservers'].split()
+        # Name server search info provided??
+        if 'dns-search' in info:
+            iface_info['dns-search'] = info['dns-search'].split()
+        # Is any mac address spoofing going on??
+        if 'hwaddress' in info:
+            hw_info = info['hwaddress'].lower().strip()
+            hw_split = hw_info.split(None, 1)
+            if len(hw_split) == 2 and hw_split[0].startswith('ether'):
+                hw_addr = hw_split[1]
+                if hw_addr:
+                    iface_info['hwaddress'] = hw_addr
+        real_ifaces[dev_name] = iface_info
+    # Check for those that should be started on boot via 'auto'
+    for (cmd, args) in entries:
+        if cmd == 'auto':
+            # Seems like auto can be like 'auto eth0 eth0:1' so just get the
+            # first part out as the device name
+            args = args.split(None)
+            if not args:
+                continue
+            dev_name = args[0].strip().lower()
+            if dev_name in real_ifaces:
+                real_ifaces[dev_name]['auto'] = True
+    return real_ifaces
diff --git a/cloudinit/distros/rhel.py b/cloudinit/distros/rhel.py
index 30195384..e8abf111 100644
--- a/cloudinit/distros/rhel.py
+++ b/cloudinit/distros/rhel.py
@@ -25,6 +25,7 @@ from cloudinit import helpers
 from cloudinit import log as logging
 from cloudinit import util
 
+from cloudinit.distros import net_util
 from cloudinit.distros import rhel_util
 from cloudinit.settings import PER_INSTANCE
 
@@ -63,7 +64,7 @@ class Distro(distros.Distro):
 
     def _write_network(self, settings):
         # TODO(harlowja) fix this... since this is the ubuntu format
-        entries = rhel_util.translate_network(settings)
+        entries = net_util.translate_network(settings)
         LOG.debug("Translated ubuntu style network settings %s into %s",
                   settings, entries)
         # Make the intermediate format as the rhel format...
diff --git a/cloudinit/distros/rhel_util.py b/cloudinit/distros/rhel_util.py
index 1aba58b8..063d536e 100644
--- a/cloudinit/distros/rhel_util.py
+++ b/cloudinit/distros/rhel_util.py
@@ -30,94 +30,6 @@ from cloudinit import util
 LOG = logging.getLogger(__name__)
 
 
-# This is a util function to translate Debian based distro interface blobs as
-# given in /etc/network/interfaces to an equivalent format for distributions
-# that use ifcfg-* style (Red Hat and SUSE).
-# TODO(harlowja) remove when we have python-netcf active...
-def translate_network(settings):
-    # Get the standard cmd, args from the ubuntu format
-    entries = []
-    for line in settings.splitlines():
-        line = line.strip()
-        if not line or line.startswith("#"):
-            continue
-        split_up = line.split(None, 1)
-        if len(split_up) <= 1:
-            continue
-        entries.append(split_up)
-    # Figure out where each iface section is
-    ifaces = []
-    consume = {}
-    for (cmd, args) in entries:
-        if cmd == 'iface':
-            if consume:
-                ifaces.append(consume)
-                consume = {}
-            consume[cmd] = args
-        else:
-            consume[cmd] = args
-    # Check if anything left over to consume
-    absorb = False
-    for (cmd, args) in consume.iteritems():
-        if cmd == 'iface':
-            absorb = True
-    if absorb:
-        ifaces.append(consume)
-    # Now translate
-    real_ifaces = {}
-    for info in ifaces:
-        if 'iface' not in info:
-            continue
-        iface_details = info['iface'].split(None)
-        dev_name = None
-        if len(iface_details) >= 1:
-            dev = iface_details[0].strip().lower()
-            if dev:
-                dev_name = dev
-        if not dev_name:
-            continue
-        iface_info = {}
-        if len(iface_details) >= 3:
-            proto_type = iface_details[2].strip().lower()
-            # Seems like this can be 'loopback' which we don't
-            # really care about
-            if proto_type in ['dhcp', 'static']:
-                iface_info['bootproto'] = proto_type
-        # These can just be copied over
-        for k in ['netmask', 'address', 'gateway', 'broadcast']:
-            if k in info:
-                val = info[k].strip().lower()
-                if val:
-                    iface_info[k] = val
-        # Name server info provided??
-        if 'dns-nameservers' in info:
-            iface_info['dns-nameservers'] = info['dns-nameservers'].split()
-        # Name server search info provided??
-        if 'dns-search' in info:
-            iface_info['dns-search'] = info['dns-search'].split()
-        # Is any mac address spoofing going on??
-        if 'hwaddress' in info:
-            hw_info = info['hwaddress'].lower().strip()
-            hw_split = hw_info.split(None, 1)
-            if len(hw_split) == 2 and hw_split[0].startswith('ether'):
-                hw_addr = hw_split[1]
-                if hw_addr:
-                    iface_info['hwaddress'] = hw_addr
-        real_ifaces[dev_name] = iface_info
-    # Check for those that should be started on boot via 'auto'
-    for (cmd, args) in entries:
-        if cmd == 'auto':
-            # Seems like auto can be like 'auto eth0 eth0:1' so just get the
-            # first part out as the device name
-            args = args.split(None)
-            if not args:
-                continue
-            dev_name = args[0].strip().lower()
-            if dev_name in real_ifaces:
-                real_ifaces[dev_name]['auto'] = True
-    return real_ifaces
-
-
 # Helper function to update a RHEL/SUSE /etc/sysconfig/* file
 def update_sysconfig_file(fn, adjustments, allow_empty=False):
     if not adjustments:
diff --git a/cloudinit/distros/sles.py b/cloudinit/distros/sles.py
index f2ac4efc..9788a1ba 100644
--- a/cloudinit/distros/sles.py
+++ b/cloudinit/distros/sles.py
@@ -26,6 +26,7 @@ from cloudinit import helpers
 from cloudinit import log as logging
 from cloudinit import util
 
+from cloudinit.distros import net_util
 from cloudinit.distros import rhel_util
 from cloudinit.settings import PER_INSTANCE
 
@@ -54,7 +55,7 @@ class Distro(distros.Distro):
 
     def _write_network(self, settings):
         # Convert debian settings to ifcfg format
-        entries = rhel_util.translate_network(settings)
+        entries = net_util.translate_network(settings)
         LOG.debug("Translated ubuntu style network settings %s into %s",
                   settings, entries)
         # Make the intermediate format as the suse format...
diff --git a/cloudinit/ec2_utils.py b/cloudinit/ec2_utils.py
index fcd511c5..7f4c0443 100644
--- a/cloudinit/ec2_utils.py
+++ b/cloudinit/ec2_utils.py
@@ -16,48 +16,181 @@
 #    You should have received a copy of the GNU General Public License
 #    along with this program.  If not, see <http://www.gnu.org/licenses/>.
 
-import boto.utils as boto_utils
-
-# Versions of boto >= 2.6.0 (and possibly 2.5.2)
-# try to lazily load the metadata backing, which
-# doesn't work so well in cloud-init especially
-# since the metadata is serialized and actions are
-# performed where the metadata server may be blocked
-# (thus the datasource will start failing) resulting
-# in url exceptions when fields that do exist (or
-# would have existed) do not exist due to the blocking
-# that occurred.
-
-# TODO(harlowja): https://github.com/boto/boto/issues/1401
-# When boto finally moves to using requests, we should be able
-# to provide it ssl details, it does not yet, so we can't provide them...
-
-
-def _unlazy_dict(mp):
-    if not isinstance(mp, (dict)):
-        return mp
-    # Walk over the keys/values which
-    # forces boto to unlazy itself and
-    # has no effect on dictionaries that
-    # already have there items.
-    for (_k, v) in mp.items():
-        _unlazy_dict(v)
-    return mp
-
-
-def get_instance_userdata(api_version, metadata_address):
-    # Note: boto.utils.get_instance_metadata returns '' for empty string
-    # so the change from non-true to '' is not specifically necessary, but
-    # this way cloud-init will get consistent behavior even if boto changed
-    # in the future to return a None on "no user-data provided".
-    ud = boto_utils.get_instance_userdata(api_version, None, metadata_address)
-    if not ud:
-        ud = ''
-    return ud
-
-
-def get_instance_metadata(api_version, metadata_address):
-    metadata = boto_utils.get_instance_metadata(api_version, metadata_address)
-    if not isinstance(metadata, (dict)):
-        metadata = {}
-    return _unlazy_dict(metadata)
+import httplib
+from urlparse import (urlparse, urlunparse)
+
+import functools
+import json
+import urllib
+
+from cloudinit import log as logging
+from cloudinit import url_helper
+from cloudinit import util
+
+LOG = logging.getLogger(__name__)
+SKIP_USERDATA_CODES = frozenset([httplib.NOT_FOUND])
+
+
+def maybe_json_object(text):
+    if not text:
+        return False
+    text = text.strip()
+    if text.startswith("{") and text.endswith("}"):
+        return True
+    return False
+
+
+def combine_url(base, add_on):
+    base_parsed = list(urlparse(base))
+    path = base_parsed[2]
+    if path and not path.endswith("/"):
+        path += "/"
+    path += urllib.quote(str(add_on), safe="/:")
+    base_parsed[2] = path
+    return urlunparse(base_parsed)
+
+
+# See: http://bit.ly/TyoUQs
+#
+class MetadataMaterializer(object):
+    def __init__(self, blob, base_url, caller):
+        self._blob = blob
+        self._md = None
+        self._base_url = base_url
+        self._caller = caller
+
+    def _parse(self, blob):
+        leaves = {}
+        children = []
+        if not blob:
+            return (leaves, children)
+
+        def has_children(item):
+            if item.endswith("/"):
+                return True
+            else:
+                return False
+
+        def get_name(item):
+            if item.endswith("/"):
+                return item.rstrip("/")
+            return item
+
+        for field in blob.splitlines():
+            field = field.strip()
+            field_name = get_name(field)
+            if not field or not field_name:
+                continue
+            if has_children(field):
+                if field_name not in children:
+                    children.append(field_name)
+            else:
+                contents = field.split("=", 1)
+                resource = field_name
+                if len(contents) > 1:
+                    # What a PITA...
+                    (ident, sub_contents) = contents
+                    ident = util.safe_int(ident)
+                    if ident is not None:
+                        resource = "%s/openssh-key" % (ident)
+                        field_name = sub_contents
+                leaves[field_name] = resource
+        return (leaves, children)
+
+    def materialize(self):
+        if self._md is not None:
+            return self._md
+        self._md = self._materialize(self._blob, self._base_url)
+        return self._md
+
+    def _decode_leaf_blob(self, field, blob):
+        if not blob:
+            return blob
+        if maybe_json_object(blob):
+            try:
+                # Assume it's json, unless it fails parsing...
+                return json.loads(blob)
+            except (ValueError, TypeError) as e:
+                LOG.warn("Field %s looked like a json object, but it was"
+                         " not: %s", field, e)
+        if blob.find("\n") != -1:
+            return blob.splitlines()
+        return blob
+
+    def _materialize(self, blob, base_url):
+        (leaves, children) = self._parse(blob)
+        child_contents = {}
+        for c in children:
+            child_url = combine_url(base_url, c)
+            if not child_url.endswith("/"):
+                child_url += "/"
+            child_blob = str(self._caller(child_url))
+            child_contents[c] = self._materialize(child_blob, child_url)
+        leaf_contents = {}
+        for (field, resource) in leaves.items():
+            leaf_url = combine_url(base_url, resource)
+            leaf_blob = str(self._caller(leaf_url))
+            leaf_contents[field] = self._decode_leaf_blob(field, leaf_blob)
+        joined = {}
+        joined.update(child_contents)
+        for field in leaf_contents.keys():
+            if field in joined:
+                LOG.warn("Duplicate key found in results from %s", base_url)
+            else:
+                joined[field] = leaf_contents[field]
+        return joined
+
+
+def _skip_retry_on_codes(status_codes, _request_args, cause):
+    """Returns if a request should retry based on a given set of codes that
+    case retrying to be stopped/skipped.
+    """
+    if cause.code in status_codes:
+        return False
+    return True
+
+
+def get_instance_userdata(api_version='latest',
+                          metadata_address='http://169.254.169.254',
+                          ssl_details=None, timeout=5, retries=5):
+    ud_url = combine_url(metadata_address, api_version)
+    ud_url = combine_url(ud_url, 'user-data')
+    user_data = ''
+    try:
+        # It is ok for userdata to not exist (thats why we are stopping if
+        # NOT_FOUND occurs) and just in that case returning an empty string.
+        exception_cb = functools.partial(_skip_retry_on_codes,
+                                         SKIP_USERDATA_CODES)
+        response = util.read_file_or_url(ud_url,
+                                         ssl_details=ssl_details,
+                                         timeout=timeout,
+                                         retries=retries,
+                                         exception_cb=exception_cb)
+        user_data = str(response)
+    except url_helper.UrlError as e:
+        if e.code not in SKIP_USERDATA_CODES:
+            util.logexc(LOG, "Failed fetching userdata from url %s", ud_url)
+    except Exception:
+        util.logexc(LOG, "Failed fetching userdata from url %s", ud_url)
+    return user_data
+
+
+def get_instance_metadata(api_version='latest',
+                          metadata_address='http://169.254.169.254',
+                          ssl_details=None, timeout=5, retries=5):
+    md_url = combine_url(metadata_address, api_version)
+    md_url = combine_url(md_url, 'meta-data')
+    caller = functools.partial(util.read_file_or_url,
+                               ssl_details=ssl_details, timeout=timeout,
+                               retries=retries)
+
+    try:
+        response = caller(md_url)
+        materializer = MetadataMaterializer(str(response), md_url, caller)
+        md = materializer.materialize()
+        if not isinstance(md, (dict)):
+            md = {}
+        return md
+    except Exception:
+        util.logexc(LOG, "Failed fetching metadata from url %s", md_url)
+        return {}
diff --git a/cloudinit/handlers/__init__.py b/cloudinit/handlers/__init__.py
index 2ddc75f4..059d7495 100644
--- a/cloudinit/handlers/__init__.py
+++ b/cloudinit/handlers/__init__.py
@@ -187,6 +187,10 @@ def _escape_string(text):
 
 def walker_callback(data, filename, payload, headers):
     content_type = headers['Content-Type']
+    if content_type in data.get('excluded'):
+        LOG.debug('content_type "%s" is excluded', content_type)
+        return
+
     if content_type in PART_CONTENT_TYPES:
         walker_handle_handler(data, content_type, filename, payload)
         return
diff --git a/cloudinit/handlers/cloud_config.py b/cloudinit/handlers/cloud_config.py
index 34a73115..4232700f 100644
--- a/cloudinit/handlers/cloud_config.py
+++ b/cloudinit/handlers/cloud_config.py
@@ -66,6 +66,8 @@ class CloudConfigPartHandler(handlers.Handler):
         handlers.Handler.__init__(self, PER_ALWAYS, version=3)
         self.cloud_buf = None
         self.cloud_fn = paths.get_ipath("cloud_config")
+        if 'cloud_config_path' in _kwargs:
+            self.cloud_fn = paths.get_ipath(_kwargs["cloud_config_path"])
         self.file_names = []
 
     def list_types(self):
diff --git a/cloudinit/handlers/shell_script.py b/cloudinit/handlers/shell_script.py
index 62289d98..30c1ed89 100644
--- a/cloudinit/handlers/shell_script.py
+++ b/cloudinit/handlers/shell_script.py
@@ -36,6 +36,8 @@ class ShellScriptPartHandler(handlers.Handler):
     def __init__(self, paths, **_kwargs):
         handlers.Handler.__init__(self, PER_ALWAYS)
         self.script_dir = paths.get_ipath_cur('scripts')
+        if 'script_path' in _kwargs:
+            self.script_dir = paths.get_ipath_cur(_kwargs['script_path'])
 
     def list_types(self):
         return [
diff --git a/cloudinit/helpers.py b/cloudinit/helpers.py
index e5eac6a7..e701126e 100644
--- a/cloudinit/helpers.py
+++ b/cloudinit/helpers.py
@@ -200,11 +200,13 @@ class Runners(object):
 
 class ConfigMerger(object):
     def __init__(self, paths=None, datasource=None,
-                 additional_fns=None, base_cfg=None):
+                 additional_fns=None, base_cfg=None,
+                 include_vendor=True):
         self._paths = paths
         self._ds = datasource
         self._fns = additional_fns
         self._base_cfg = base_cfg
+        self._include_vendor = include_vendor
         # Created on first use
         self._cfg = None
 
@@ -237,13 +239,19 @@ class ConfigMerger(object):
         # a configuration file to use when running...
         if not self._paths:
             return i_cfgs
-        cc_fn = self._paths.get_ipath_cur('cloud_config')
-        if cc_fn and os.path.isfile(cc_fn):
-            try:
-                i_cfgs.append(util.read_conf(cc_fn))
-            except:
-                util.logexc(LOG, 'Failed loading of cloud-config from %s',
-                            cc_fn)
+
+        cc_paths = ['cloud_config']
+        if self._include_vendor:
+            cc_paths.append('vendor_cloud_config')
+
+        for cc_p in cc_paths:
+            cc_fn = self._paths.get_ipath_cur(cc_p)
+            if cc_fn and os.path.isfile(cc_fn):
+                try:
+                    i_cfgs.append(util.read_conf(cc_fn))
+                except:
+                    util.logexc(LOG, 'Failed loading of cloud-config from %s',
+                                cc_fn)
         return i_cfgs
 
     def _read_cfg(self):
@@ -331,13 +339,17 @@ class Paths(object):
         self.lookups = {
            "handlers": "handlers",
            "scripts": "scripts",
+           "vendor_scripts": "scripts/vendor",
            "sem": "sem",
            "boothooks": "boothooks",
            "userdata_raw": "user-data.txt",
            "userdata": "user-data.txt.i",
            "obj_pkl": "obj.pkl",
            "cloud_config": "cloud-config.txt",
+           "vendor_cloud_config": "vendor-cloud-config.txt",
            "data": "data",
+           "vendordata_raw": "vendor-data.txt",
+           "vendordata": "vendor-data.txt.i",
         }
         # Set when a datasource becomes active
         self.datasource = ds
diff --git a/cloudinit/netinfo.py b/cloudinit/netinfo.py
index feba5a62..ac3c011f 100644
--- a/cloudinit/netinfo.py
+++ b/cloudinit/netinfo.py
@@ -21,6 +21,7 @@
 #    along with this program.  If not, see <http://www.gnu.org/licenses/>.
 
 import cloudinit.util as util
+import re
 
 from prettytable import PrettyTable
 
@@ -40,27 +41,40 @@ def netdev_info(empty=""):
         toks = line.lower().strip().split()
         if toks[0] == "up":
             devs[curdev]['up'] = True
+        # If the output of ifconfig doesn't contain the required info in the
+        # obvious place, use a regex filter to be sure.
+        elif len(toks) > 1:
+            if re.search(r"flags=\d+<up,", toks[1]):
+                devs[curdev]['up'] = True
 
         fieldpost = ""
         if toks[0] == "inet6":
             fieldpost = "6"
 
         for i in range(len(toks)):
-            if toks[i] == "hwaddr":
+            if toks[i] == "hwaddr" or toks[i] == "ether":
                 try:
                     devs[curdev]["hwaddr"] = toks[i + 1]
                 except IndexError:
                     pass
-            for field in ("addr", "bcast", "mask"):
+
+            # Couple the different items we're interested in with the correct
+            # field since FreeBSD/CentOS/Fedora differ in the output.
+            ifconfigfields = {
+                "addr:": "addr", "inet": "addr",
+                "bcast:": "bcast", "broadcast": "bcast",
+                "mask:": "mask", "netmask": "mask"
+            }
+            for origfield, field in ifconfigfields.items():
                 target = "%s%s" % (field, fieldpost)
                 if devs[curdev].get(target, ""):
                     continue
-                if toks[i] == "%s:" % field:
+                if toks[i] == "%s" % origfield:
                     try:
                         devs[curdev][target] = toks[i + 1]
                     except IndexError:
                         pass
-                elif toks[i].startswith("%s:" % field):
+                elif toks[i].startswith("%s" % origfield):
                     devs[curdev][target] = toks[i][len(field) + 1:]
 
     if empty != "":
@@ -73,15 +87,32 @@ def netdev_info(empty=""):
 
 
 def route_info():
-    (route_out, _err) = util.subp(["route", "-n"])
+    (route_out, _err) = util.subp(["netstat", "-rn"])
     routes = []
     entries = route_out.splitlines()[1:]
     for line in entries:
         if not line:
             continue
         toks = line.split()
-        if len(toks) < 8 or toks[0] == "Kernel" or toks[0] == "Destination":
+
+        # FreeBSD shows 6 items in the routing table:
+        #  Destination  Gateway    Flags Refs    Use  Netif Expire
+        #  default      10.65.0.1  UGS      0  34920 vtnet0
+        #
+        # Linux netstat shows 2 more:
+        #  Destination  Gateway    Genmask  Flags MSS Window irtt Iface
+        #  0.0.0.0      10.65.0.1  0.0.0.0  UG      0 0         0 eth0
+        if (len(toks) < 6 or toks[0] == "Kernel" or
+                toks[0] == "Destination" or toks[0] == "Internet" or
+                toks[0] == "Internet6" or toks[0] == "Routing"):
             continue
+
+        if len(toks) < 8:
+            toks.append("-")
+            toks.append("-")
+            toks[7] = toks[5]
+            toks[5] = "-"
+
         entry = {
             'destination': toks[0],
             'gateway': toks[1],
@@ -92,6 +123,7 @@ def route_info():
             'use': toks[6],
             'iface': toks[7],
         }
+
         routes.append(entry)
     return routes
 
diff --git a/cloudinit/settings.py b/cloudinit/settings.py
index 5df7f557..7be2199a 100644
--- a/cloudinit/settings.py
+++ b/cloudinit/settings.py
@@ -52,6 +52,7 @@ CFG_BUILTIN = {
         },
         'distro': 'ubuntu',
     },
+    'vendor_data': {'enabled': True, 'prefix': []},
 }
 
 # Valid frequencies of handlers/modules
diff --git a/cloudinit/sources/DataSourceConfigDrive.py b/cloudinit/sources/DataSourceConfigDrive.py
index 4f437244..2a244496 100644
--- a/cloudinit/sources/DataSourceConfigDrive.py
+++ b/cloudinit/sources/DataSourceConfigDrive.py
@@ -284,8 +284,10 @@ def find_candidate_devs():
     # followed by fstype items, but with dupes removed
     combined = (by_label + [d for d in by_fstype if d not in by_label])
 
-    # We are looking for block device (sda, not sda1), ignore partitions
-    combined = [d for d in combined if not util.is_partition(d)]
+    # We are looking for a block device or partition with necessary label or
+    # an unpartitioned block device.
+    combined = [d for d in combined
+                    if d in by_label or not util.is_partition(d)]
 
     return combined
 
diff --git a/cloudinit/sources/DataSourceNoCloud.py b/cloudinit/sources/DataSourceNoCloud.py
index 4ef92a56..cbaac29f 100644
--- a/cloudinit/sources/DataSourceNoCloud.py
+++ b/cloudinit/sources/DataSourceNoCloud.py
@@ -50,40 +50,47 @@ class DataSourceNoCloud(sources.DataSource):
         }
 
         found = []
-        md = {}
-        ud = ""
+        mydata = {'meta-data': {}, 'user-data': "", 'vendor-data': ""}
 
         try:
             # Parse the kernel command line, getting data passed in
+            md = {}
             if parse_cmdline_data(self.cmdline_id, md):
                 found.append("cmdline")
+            mydata.update(md)
         except:
             util.logexc(LOG, "Unable to parse command line data")
             return False
 
         # Check to see if the seed dir has data.
-        seedret = {}
-        if util.read_optional_seed(seedret, base=self.seed_dir + "/"):
-            md = util.mergemanydict([md, seedret['meta-data']])
-            ud = seedret['user-data']
+        pp2d_kwargs = {'required': ['user-data', 'meta-data'],
+                       'optional': ['vendor-data']}
+
+        try:
+            seeded = util.pathprefix2dict(self.seed_dir, **pp2d_kwargs)
             found.append(self.seed_dir)
-            LOG.debug("Using seeded cache data from %s", self.seed_dir)
+            LOG.debug("Using seeded data from %s", self.seed_dir)
+        except ValueError as e:
+            pass
+
+        if self.seed_dir in found:
+            mydata = _merge_new_seed(mydata, seeded)
 
         # If the datasource config had a 'seedfrom' entry, then that takes
         # precedence over a 'seedfrom' that was found in a filesystem
         # but not over external media
-        if 'seedfrom' in self.ds_cfg and self.ds_cfg['seedfrom']:
-            found.append("ds_config")
-            md["seedfrom"] = self.ds_cfg['seedfrom']
+        if self.ds_cfg.get('seedfrom'):
+            found.append("ds_config_seedfrom")
+            mydata['meta-data']["seedfrom"] = self.ds_cfg['seedfrom']
 
-        # if ds_cfg has 'user-data' and 'meta-data'
+        # fields appropriately named can also just come from the datasource
+        # config (ie, 'user-data', 'meta-data', 'vendor-data' there)
         if 'user-data' in self.ds_cfg and 'meta-data' in self.ds_cfg:
-            if self.ds_cfg['user-data']:
-                ud = self.ds_cfg['user-data']
-            if self.ds_cfg['meta-data'] is not False:
-                md = util.mergemanydict([md, self.ds_cfg['meta-data']])
-            if 'ds_config' not in found:
-                found.append("ds_config")
+            mydata = _merge_new_seed(mydata, self.ds_cfg)
+            found.append("ds_config")
+
+        def _pp2d_callback(mp, data):
+            util.pathprefix2dict(mp, **data)
 
         label = self.ds_cfg.get('fs_label', "cidata")
         if label is not None:
@@ -102,15 +109,21 @@ class DataSourceNoCloud(sources.DataSource):
                 try:
                     LOG.debug("Attempting to use data from %s", dev)
 
-                    (newmd, newud) = util.mount_cb(dev, util.read_seeded)
-                    md = util.mergemanydict([newmd, md])
-                    ud = newud
+                    try:
+                        seeded = util.mount_cb(dev, _pp2d_callback)
+                    except ValueError as e:
+                        if dev in label_list:
+                            LOG.warn("device %s with label=%s not a"
+                                     "valid seed.", dev, label)
+                        continue
+
+                    mydata = _merge_new_seed(mydata, seeded)
 
                     # For seed from a device, the default mode is 'net'.
                     # that is more likely to be what is desired.  If they want
                     # dsmode of local, then they must specify that.
-                    if 'dsmode' not in md:
-                        md['dsmode'] = "net"
+                    if 'dsmode' not in mydata['meta-data']:
+                        mydata['meta-data'] = "net"
 
                     LOG.debug("Using data from %s", dev)
                     found.append(dev)
@@ -133,8 +146,8 @@ class DataSourceNoCloud(sources.DataSource):
         # attempt to seed the userdata / metadata from its value
         # its primarily value is in allowing the user to type less
         # on the command line, ie: ds=nocloud;s=http://bit.ly/abcdefg
-        if "seedfrom" in md:
-            seedfrom = md["seedfrom"]
+        if "seedfrom" in mydata['meta-data']:
+            seedfrom = mydata['meta-data']["seedfrom"]
             seedfound = False
             for proto in self.supported_seed_starts:
                 if seedfrom.startswith(proto):
@@ -144,7 +157,7 @@ class DataSourceNoCloud(sources.DataSource):
                 LOG.debug("Seed from %s not supported by %s", seedfrom, self)
                 return False
 
-            if 'network-interfaces' in md:
+            if 'network-interfaces' in mydata['meta-data']:
                 seeded_interfaces = self.dsmode
 
             # This could throw errors, but the user told us to do it
@@ -153,25 +166,30 @@ class DataSourceNoCloud(sources.DataSource):
             LOG.debug("Using seeded cache data from %s", seedfrom)
 
             # Values in the command line override those from the seed
-            md = util.mergemanydict([md, md_seed])
+            mydata['meta-data'] = util.mergemanydict([mydata['meta-data'],
+                                                      md_seed])
+            mydata['user-data'] = ud
             found.append(seedfrom)
 
         # Now that we have exhausted any other places merge in the defaults
-        md = util.mergemanydict([md, defaults])
+        mydata['meta-data'] = util.mergemanydict([mydata['meta-data'],
+                                                  defaults])
 
         # Update the network-interfaces if metadata had 'network-interfaces'
         # entry and this is the local datasource, or 'seedfrom' was used
         # and the source of the seed was self.dsmode
         # ('local' for NoCloud, 'net' for NoCloudNet')
-        if ('network-interfaces' in md and
+        if ('network-interfaces' in mydata['meta-data'] and
             (self.dsmode in ("local", seeded_interfaces))):
             LOG.debug("Updating network interfaces from %s", self)
-            self.distro.apply_network(md['network-interfaces'])
+            self.distro.apply_network(
+                mydata['meta-data']['network-interfaces'])
 
-        if md['dsmode'] == self.dsmode:
+        if mydata['meta-data']['dsmode'] == self.dsmode:
             self.seed = ",".join(found)
-            self.metadata = md
-            self.userdata_raw = ud
+            self.metadata = mydata['meta-data']
+            self.userdata_raw = mydata['user-data']
+            self.vendordata = mydata['vendor-data']
             return True
 
         LOG.debug("%s: not claiming datasource, dsmode=%s", self, md['dsmode'])
@@ -222,6 +240,16 @@ def parse_cmdline_data(ds_id, fill, cmdline=None):
     return True
 
 
+def _merge_new_seed(cur, seeded):
+    ret = cur.copy()
+    ret['meta-data'] = util.mergemanydict([cur['meta-data'],
+                                          util.load_yaml(seeded['meta-data'])])
+    ret['user-data'] = seeded['user-data']
+    if 'vendor-data' in seeded:
+        ret['vendor-data'] = seeded['vendor-data']
+    return ret
+
+
 class DataSourceNoCloudNet(DataSourceNoCloud):
     def __init__(self, sys_cfg, distro, paths):
         DataSourceNoCloud.__init__(self, sys_cfg, distro, paths)
diff --git a/cloudinit/sources/DataSourceOpenNebula.py b/cloudinit/sources/DataSourceOpenNebula.py
index 07dc25ff..b0464cbb 100644
--- a/cloudinit/sources/DataSourceOpenNebula.py
+++ b/cloudinit/sources/DataSourceOpenNebula.py
@@ -323,7 +323,7 @@ def parse_shell_config(content, keylist=None, bash=None, asuser=None,
     (output, _error) = util.subp(cmd, data=bcmd)
 
     # exclude vars in bash that change on their own or that we used
-    excluded = ("RANDOM", "LINENO", "_", "__v")
+    excluded = ("RANDOM", "LINENO", "SECONDS", "_", "__v")
     preset = {}
     ret = {}
     target = None
diff --git a/cloudinit/sources/DataSourceSmartOS.py b/cloudinit/sources/DataSourceSmartOS.py
index 551b20c4..140c7814 100644
--- a/cloudinit/sources/DataSourceSmartOS.py
+++ b/cloudinit/sources/DataSourceSmartOS.py
@@ -25,7 +25,9 @@
 #        requests on the console. For example, to get the hostname, you
 #        would send "GET hostname" on /dev/ttyS1.
 #
-
+#   Certain behavior is defined by the DataDictionary
+#       http://us-east.manta.joyent.com/jmc/public/mdata/datadict.html
+#       Comments with "@datadictionary" are snippets of the definition
 
 import base64
 from cloudinit import log as logging
@@ -43,10 +45,12 @@ SMARTOS_ATTRIB_MAP = {
     'local-hostname': ('hostname', True),
     'public-keys': ('root_authorized_keys', True),
     'user-script': ('user-script', False),
-    'user-data': ('user-data', False),
+    'legacy-user-data': ('user-data', False),
+    'user-data': ('cloud-init:user-data', False),
     'iptables_disable': ('iptables_disable', True),
     'motd_sys_info': ('motd_sys_info', True),
-    'availability_zone': ('datacenter_name', True),
+    'availability_zone': ('sdc:datacenter_name', True),
+    'vendordata': ('sdc:operator-script', False),
 }
 
 DS_NAME = 'SmartOS'
@@ -70,7 +74,11 @@ BUILTIN_DS_CONFIG = {
     'seed_timeout': 60,
     'no_base64_decode': ['root_authorized_keys',
                          'motd_sys_info',
-                         'iptables_disable'],
+                         'iptables_disable',
+                         'user-data',
+                         'user-script',
+                         'sdc:datacenter_name',
+                        ],
     'base64_keys': [],
     'base64_all': False,
     'disk_aliases': {'ephemeral0': '/dev/vdb'},
@@ -87,6 +95,11 @@ BUILTIN_CLOUD_CONFIG = {
                   'device': 'ephemeral0'}],
 }
 
+# @datadictionary: this is legacy path for placing files from metadata
+#   per the SmartOS location. It is not preferable, but is done for
+#   legacy reasons
+LEGACY_USER_D = "/var/db"
+
 
 class DataSourceSmartOS(sources.DataSource):
     def __init__(self, sys_cfg, distro, paths):
@@ -106,6 +119,9 @@ class DataSourceSmartOS(sources.DataSource):
         self.smartos_no_base64 = self.ds_cfg.get('no_base64_decode')
         self.b64_keys = self.ds_cfg.get('base64_keys')
         self.b64_all = self.ds_cfg.get('base64_all')
+        self.script_base_d = os.path.join(self.paths.get_cpath("scripts"))
+        self.user_script_d = os.path.join(self.paths.get_cpath("scripts"),
+                                          'per-boot')
 
     def __str__(self):
         root = sources.DataSource.__str__(self)
@@ -143,17 +159,36 @@ class DataSourceSmartOS(sources.DataSource):
             smartos_noun, strip = attribute
             md[ci_noun] = self.query(smartos_noun, strip=strip)
 
+        # @datadictionary: This key may contain a program that is written
+        # to a file in the filesystem of the guest on each boot and then
+        # executed. It may be of any format that would be considered
+        # executable in the guest instance.
+        u_script = md.get('user-script')
+        u_script_f = "%s/99_user_script" % self.user_script_d
+        u_script_l = "%s/user-script" % LEGACY_USER_D
+        write_boot_content(u_script, u_script_f, link=u_script_l, shebang=True,
+                           mode=0700)
+
+        # @datadictionary:  This key has no defined format, but its value
+        # is written to the file /var/db/mdata-user-data on each boot prior
+        # to the phase that runs user-script. This file is not to be executed.
+        # This allows a configuration file of some kind to be injected into
+        # the machine to be consumed by the user-script when it runs.
+        u_data = md.get('legacy-user-data')
+        u_data_f = "%s/mdata-user-data" % LEGACY_USER_D
+        write_boot_content(u_data, u_data_f)
+
+        # Handle the cloud-init regular meta
         if not md['local-hostname']:
             md['local-hostname'] = system_uuid
 
         ud = None
         if md['user-data']:
             ud = md['user-data']
-        elif md['user-script']:
-            ud = md['user-script']
 
         self.metadata = util.mergemanydict([md, self.metadata])
         self.userdata_raw = ud
+        self.vendordata_raw = md['vendordata']
         return True
 
     def device_name_to_device(self, name):
@@ -277,6 +312,62 @@ def dmi_data():
     return (sys_uuid.lower().strip(), sys_type.strip())
 
 
+def write_boot_content(content, content_f, link=None, shebang=False,
+                       mode=0400):
+    """
+    Write the content to content_f. Under the following rules:
+        1. If no content, remove the file
+        2. Write the content
+        3. If executable and no file magic, add it
+        4. If there is a link, create it
+
+    @param content: what to write
+    @param content_f: the file name
+    @param backup_d: the directory to save the backup at
+    @param link: if defined, location to create a symlink to
+    @param shebang: if no file magic, set shebang
+    @param mode: file mode
+
+    Becuase of the way that Cloud-init executes scripts (no shell),
+    a script will fail to execute if does not have a magic bit (shebang) set
+    for the file. If shebang=True, then the script will be checked for a magic
+    bit and to the SmartOS default of assuming that bash.
+    """
+
+    if not content and os.path.exists(content_f):
+        os.unlink(content_f)
+    if link and os.path.islink(link):
+        os.unlink(link)
+    if not content:
+        return
+
+    util.write_file(content_f, content, mode=mode)
+
+    if shebang and not content.startswith("#!"):
+        try:
+            cmd = ["file", "--brief", "--mime-type", content_f]
+            (f_type, _err) = util.subp(cmd)
+            LOG.debug("script %s mime type is %s", content_f, f_type)
+            if f_type.strip() == "text/plain":
+                new_content = "\n".join(["#!/bin/bash", content])
+                util.write_file(content_f, new_content, mode=mode)
+                LOG.debug("added shebang to file %s", content_f)
+
+        except Exception as e:
+            util.logexc(LOG, ("Failed to identify script type for %s" %
+                             content_f, e))
+
+    if link:
+        try:
+            if os.path.islink(link):
+                os.unlink(link)
+            if content and os.path.exists(content_f):
+                util.ensure_dir(os.path.dirname(link))
+                os.symlink(content_f, link)
+        except IOError as e:
+            util.logexc(LOG, "failed establishing content link", e)
+
+
 # Used to match classes to dependencies
 datasources = [
     (DataSourceSmartOS, (sources.DEP_FILESYSTEM, sources.DEP_NETWORK)),
diff --git a/cloudinit/sources/__init__.py b/cloudinit/sources/__init__.py
index 7dc1fbde..fef4d460 100644
--- a/cloudinit/sources/__init__.py
+++ b/cloudinit/sources/__init__.py
@@ -53,6 +53,8 @@ class DataSource(object):
         self.userdata = None
         self.metadata = None
         self.userdata_raw = None
+        self.vendordata = None
+        self.vendordata_raw = None
 
         # find the datasource config name.
         # remove 'DataSource' from classname on front, and remove 'Net' on end.
@@ -77,9 +79,14 @@ class DataSource(object):
         if self.userdata is None:
             self.userdata = self.ud_proc.process(self.get_userdata_raw())
         if apply_filter:
-            return self._filter_userdata(self.userdata)
+            return self._filter_xdata(self.userdata)
         return self.userdata
 
+    def get_vendordata(self):
+        if self.vendordata is None:
+            self.vendordata = self.ud_proc.process(self.get_vendordata_raw())
+        return self.vendordata
+
     @property
     def launch_index(self):
         if not self.metadata:
@@ -88,7 +95,7 @@ class DataSource(object):
             return self.metadata['launch-index']
         return None
 
-    def _filter_userdata(self, processed_ud):
+    def _filter_xdata(self, processed_ud):
         filters = [
             launch_index.Filter(util.safe_int(self.launch_index)),
         ]
@@ -104,6 +111,9 @@ class DataSource(object):
     def get_userdata_raw(self):
         return self.userdata_raw
 
+    def get_vendordata_raw(self):
+        return self.vendordata_raw
+
     # the data sources' config_obj is a cloud-config formated
     # object that came to it from ways other than cloud-config
     # because cloud-config content would be handled elsewhere
@@ -119,7 +129,7 @@ class DataSource(object):
         # when the kernel named them 'vda' or 'xvda'
         # we want to return the correct value for what will actually
         # exist in this instance
-        mappings = {"sd": ("vd", "xvd")}
+        mappings = {"sd": ("vd", "xvd", "vtb")}
         for (nfrom, tlist) in mappings.iteritems():
             if not short_name.startswith(nfrom):
                 continue
diff --git a/cloudinit/stages.py b/cloudinit/stages.py
index 07c55802..593b72a2 100644
--- a/cloudinit/stages.py
+++ b/cloudinit/stages.py
@@ -123,6 +123,7 @@ class Init(object):
             os.path.join(c_dir, 'scripts', 'per-instance'),
             os.path.join(c_dir, 'scripts', 'per-once'),
             os.path.join(c_dir, 'scripts', 'per-boot'),
+            os.path.join(c_dir, 'scripts', 'vendor'),
             os.path.join(c_dir, 'seed'),
             os.path.join(c_dir, 'instances'),
             os.path.join(c_dir, 'handlers'),
@@ -319,6 +320,7 @@ class Init(object):
         if not self._write_to_cache():
             return
         self._store_userdata()
+        self._store_vendordata()
 
     def _store_userdata(self):
         raw_ud = "%s" % (self.datasource.get_userdata_raw())
@@ -326,11 +328,20 @@ class Init(object):
         processed_ud = "%s" % (self.datasource.get_userdata())
         util.write_file(self._get_ipath('userdata'), processed_ud, 0600)
 
-    def _default_userdata_handlers(self):
-        opts = {
+    def _store_vendordata(self):
+        raw_vd = "%s" % (self.datasource.get_vendordata_raw())
+        util.write_file(self._get_ipath('vendordata_raw'), raw_vd, 0600)
+        processed_vd = "%s" % (self.datasource.get_vendordata())
+        util.write_file(self._get_ipath('vendordata'), processed_vd, 0600)
+
+    def _default_handlers(self, opts=None):
+        if opts is None:
+            opts = {}
+
+        opts.update({
             'paths': self.paths,
             'datasource': self.datasource,
-        }
+        })
         # TODO(harlowja) Hmmm, should we dynamically import these??
         def_handlers = [
             cc_part.CloudConfigPartHandler(**opts),
@@ -340,7 +351,23 @@ class Init(object):
         ]
         return def_handlers
 
-    def consume_userdata(self, frequency=PER_INSTANCE):
+    def _default_userdata_handlers(self):
+        return self._default_handlers()
+
+    def _default_vendordata_handlers(self):
+        return self._default_handlers(
+            opts={'script_path': 'vendor_scripts',
+                  'cloud_config_path': 'vendor_cloud_config'})
+
+    def _do_handlers(self, data_msg, c_handlers_list, frequency,
+                     excluded=None):
+        """
+        Generalized handlers suitable for use with either vendordata
+        or userdata
+        """
+        if excluded is None:
+            excluded = []
+
         cdir = self.paths.get_cpath("handlers")
         idir = self._get_ipath("handlers")
 
@@ -352,12 +379,6 @@ class Init(object):
             if d and d not in sys.path:
                 sys.path.insert(0, d)
 
-        # Ensure datasource fetched before activation (just incase)
-        user_data_msg = self.datasource.get_userdata(True)
-
-        # This keeps track of all the active handlers
-        c_handlers = helpers.ContentHandlers()
-
         def register_handlers_in_dir(path):
             # Attempts to register any handler modules under the given path.
             if not path or not os.path.isdir(path):
@@ -382,13 +403,16 @@ class Init(object):
                     util.logexc(LOG, "Failed to register handler from %s",
                                 fname)
 
+        # This keeps track of all the active handlers
+        c_handlers = helpers.ContentHandlers()
+
         # Add any handlers in the cloud-dir
         register_handlers_in_dir(cdir)
 
         # Register any other handlers that come from the default set. This
         # is done after the cloud-dir handlers so that the cdir modules can
         # take over the default user-data handler content-types.
-        for mod in self._default_userdata_handlers():
+        for mod in c_handlers_list:
             types = c_handlers.register(mod, overwrite=False)
             if types:
                 LOG.debug("Added default handler for %s from %s", types, mod)
@@ -406,7 +430,7 @@ class Init(object):
                 handlers.call_begin(mod, data, frequency)
                 c_handlers.initialized.append(mod)
 
-        def walk_handlers():
+        def walk_handlers(excluded):
             # Walk the user data
             part_data = {
                 'handlers': c_handlers,
@@ -419,9 +443,9 @@ class Init(object):
                 # to help write there contents to files with numbered
                 # names...
                 'handlercount': 0,
+                'excluded': excluded,
             }
-            handlers.walk(user_data_msg, handlers.walker_callback,
-                          data=part_data)
+            handlers.walk(data_msg, handlers.walker_callback, data=part_data)
 
         def finalize_handlers():
             # Give callbacks opportunity to finalize
@@ -438,10 +462,16 @@ class Init(object):
 
         try:
             init_handlers()
-            walk_handlers()
+            walk_handlers(excluded)
         finally:
             finalize_handlers()
 
+    def consume_data(self, frequency=PER_INSTANCE):
+        # Consume the userdata first, because we need want to let the part
+        # handlers run first (for merging stuff)
+        self._consume_userdata(frequency)
+        self._consume_vendordata(frequency)
+
         # Perform post-consumption adjustments so that
         # modules that run during the init stage reflect
         # this consumed set.
@@ -453,6 +483,64 @@ class Init(object):
         # objects before the load of the userdata happened,
         # this is expected.
 
+    def _consume_vendordata(self, frequency=PER_INSTANCE):
+        """
+        Consume the vendordata and run the part handlers on it
+        """
+        # User-data should have been consumed first.
+        # So we merge the other available cloud-configs (everything except
+        # vendor provided), and check whether or not we should consume
+        # vendor data at all. That gives user or system a chance to override.
+        if not self.datasource.get_vendordata_raw():
+            LOG.debug("no vendordata from datasource")
+            return
+
+        _cc_merger = helpers.ConfigMerger(paths=self._paths,
+                                          datasource=self.datasource,
+                                          additional_fns=[],
+                                          base_cfg=self.cfg,
+                                          include_vendor=False)
+        vdcfg = _cc_merger.cfg.get('vendor_data', {})
+
+        if not isinstance(vdcfg, dict):
+            vdcfg = {'enabled': False}
+            LOG.warn("invalid 'vendor_data' setting. resetting to: %s", vdcfg)
+
+        enabled = vdcfg.get('enabled')
+        no_handlers = vdcfg.get('disabled_handlers', None)
+
+        if not util.is_true(enabled):
+            LOG.debug("vendordata consumption is disabled.")
+            return
+
+        LOG.debug("vendor data will be consumed. disabled_handlers=%s",
+                  no_handlers)
+
+        # Ensure vendordata source fetched before activation (just incase)
+        vendor_data_msg = self.datasource.get_vendordata()
+
+        # This keeps track of all the active handlers, while excluding what the
+        # users doesn't want run, i.e. boot_hook, cloud_config, shell_script
+        c_handlers_list = self._default_vendordata_handlers()
+
+        # Run the handlers
+        self._do_handlers(vendor_data_msg, c_handlers_list, frequency,
+                          excluded=no_handlers)
+
+    def _consume_userdata(self, frequency=PER_INSTANCE):
+        """
+        Consume the userdata and run the part handlers
+        """
+
+        # Ensure datasource fetched before activation (just incase)
+        user_data_msg = self.datasource.get_userdata(True)
+
+        # This keeps track of all the active handlers
+        c_handlers_list = self._default_handlers()
+
+        # Run the handlers
+        self._do_handlers(user_data_msg, c_handlers_list, frequency)
+
 
 class Modules(object):
     def __init__(self, init, cfg_files=None):
diff --git a/cloudinit/url_helper.py b/cloudinit/url_helper.py
index 19a30409..97ed75ad 100644
--- a/cloudinit/url_helper.py
+++ b/cloudinit/url_helper.py
@@ -20,6 +20,7 @@
 #    You should have received a copy of the GNU General Public License
 #    along with this program.  If not, see <http://www.gnu.org/licenses/>.
 
+import httplib
 import time
 
 import requests
@@ -32,6 +33,8 @@ from cloudinit import version
 
 LOG = logging.getLogger(__name__)
 
+NOT_FOUND = httplib.NOT_FOUND
+
 # Check if requests has ssl support (added in requests >= 0.8.8)
 SSL_ENABLED = False
 CONFIG_ENABLED = False  # This was added in 0.7 (but taken out in >=1.0)
@@ -58,6 +61,31 @@ def _cleanurl(url):
     return urlunparse(parsed_url)
 
 
+# Made to have same accessors as UrlResponse so that the
+# read_file_or_url can return this or that object and the
+# 'user' of those objects will not need to know the difference.
+class StringResponse(object):
+    def __init__(self, contents, code=200):
+        self.code = code
+        self.headers = {}
+        self.contents = contents
+        self.url = None
+
+    def ok(self, *args, **kwargs):  # pylint: disable=W0613
+        if self.code != 200:
+            return False
+        return True
+
+    def __str__(self):
+        return self.contents
+
+
+class FileResponse(StringResponse):
+    def __init__(self, path, contents, code=200):
+        StringResponse.__init__(self, contents, code=code)
+        self.url = path
+
+
 class UrlResponse(object):
     def __init__(self, response):
         self._response = response
@@ -103,7 +131,7 @@ class UrlError(IOError):
 
 def readurl(url, data=None, timeout=None, retries=0, sec_between=1,
             headers=None, headers_cb=None, ssl_details=None,
-            check_status=True, allow_redirects=True):
+            check_status=True, allow_redirects=True, exception_cb=None):
     url = _cleanurl(url)
     req_args = {
         'url': url,
@@ -163,14 +191,13 @@ def readurl(url, data=None, timeout=None, retries=0, sec_between=1,
     # Handle retrying ourselves since the built-in support
     # doesn't handle sleeping between tries...
     for i in range(0, manual_tries):
+        req_args['headers'] = headers_cb(url)
+        filtered_req_args = {}
+        for (k, v) in req_args.items():
+            if k == 'data':
+                continue
+            filtered_req_args[k] = v
         try:
-            req_args['headers'] = headers_cb(url)
-            filtered_req_args = {}
-            for (k, v) in req_args.items():
-                if k == 'data':
-                    continue
-                filtered_req_args[k] = v
-
             LOG.debug("[%s/%s] open '%s' with %s configuration", i,
                       manual_tries, url, filtered_req_args)
 
@@ -196,6 +223,8 @@ def readurl(url, data=None, timeout=None, retries=0, sec_between=1,
                     # ssl exceptions are not going to get fixed by waiting a
                     # few seconds
                     break
+            if exception_cb and not exception_cb(filtered_req_args, excps[-1]):
+                break
             if i + 1 < manual_tries and sec_between > 0:
                 LOG.debug("Please wait %s seconds while we wait to try again",
                           sec_between)
diff --git a/cloudinit/user_data.py b/cloudinit/user_data.py
index d49ea094..de6487d8 100644
--- a/cloudinit/user_data.py
+++ b/cloudinit/user_data.py
@@ -88,7 +88,11 @@ class UserDataProcessor(object):
 
     def process(self, blob):
         accumulating_msg = MIMEMultipart()
-        self._process_msg(convert_string(blob), accumulating_msg)
+        if isinstance(blob, list):
+            for b in blob:
+                self._process_msg(convert_string(b), accumulating_msg)
+        else:
+            self._process_msg(convert_string(blob), accumulating_msg)
         return accumulating_msg
 
     def _process_msg(self, base_msg, append_msg):
@@ -307,7 +311,8 @@ class UserDataProcessor(object):
     def _attach_part(self, outer_msg, part):
         """
         Attach a message to an outer message. outermsg must be a MIMEMultipart.
-        Modifies a header in the outer message to keep track of number of attachments.
+        Modifies a header in the outer message to keep track of number of
+        attachments.
         """
         part_count = self._multi_part_count(outer_msg)
         self._process_before_attach(part, part_count + 1)
diff --git a/cloudinit/util.py b/cloudinit/util.py
index a37172dc..08cdd8c8 100644
--- a/cloudinit/util.py
+++ b/cloudinit/util.py
@@ -26,6 +26,7 @@ from StringIO import StringIO
 
 import contextlib
 import copy as obj_copy
+import ctypes
 import errno
 import glob
 import grp
@@ -36,6 +37,7 @@ import os.path
 import platform
 import pwd
 import random
+import re
 import shutil
 import socket
 import stat
@@ -72,31 +74,6 @@ FN_ALLOWED = ('_-.()' + string.digits + string.ascii_letters)
 CONTAINER_TESTS = ['running-in-container', 'lxc-is-container']
 
 
-# Made to have same accessors as UrlResponse so that the
-# read_file_or_url can return this or that object and the
-# 'user' of those objects will not need to know the difference.
-class StringResponse(object):
-    def __init__(self, contents, code=200):
-        self.code = code
-        self.headers = {}
-        self.contents = contents
-        self.url = None
-
-    def ok(self, *args, **kwargs):  # pylint: disable=W0613
-        if self.code != 200:
-            return False
-        return True
-
-    def __str__(self):
-        return self.contents
-
-
-class FileResponse(StringResponse):
-    def __init__(self, path, contents, code=200):
-        StringResponse.__init__(self, contents, code=code)
-        self.url = path
-
-
 class ProcessExecutionError(IOError):
 
     MESSAGE_TMPL = ('%(description)s\n'
@@ -392,11 +369,11 @@ def is_ipv4(instr):
         return False
 
     try:
-        toks = [x for x in toks if (int(x) < 256 and int(x) > 0)]
+        toks = [x for x in toks if int(x) < 256 and int(x) >= 0]
     except:
         return False
 
-    return (len(toks) == 4)
+    return len(toks) == 4
 
 
 def get_cfg_option_bool(yobj, key, default=False):
@@ -608,18 +585,28 @@ def del_dir(path):
     shutil.rmtree(path)
 
 
-def runparts(dirp, skip_no_exist=True):
+def runparts(dirp, skip_no_exist=True, exe_prefix=None):
     if skip_no_exist and not os.path.isdir(dirp):
         return
 
     failed = []
     attempted = []
+
+    if exe_prefix is None:
+        prefix = []
+    elif isinstance(exe_prefix, str):
+        prefix = [str(exe_prefix)]
+    elif isinstance(exe_prefix, list):
+        prefix = exe_prefix
+    else:
+        raise TypeError("exe_prefix must be None, str, or list")
+
     for exe_name in sorted(os.listdir(dirp)):
         exe_path = os.path.join(dirp, exe_name)
         if os.path.isfile(exe_path) and os.access(exe_path, os.X_OK):
             attempted.append(exe_path)
             try:
-                subp([exe_path], capture=False)
+                subp(prefix + [exe_path], capture=False)
             except ProcessExecutionError as e:
                 logexc(LOG, "Failed running %s [%s]", exe_path, e.exit_code)
                 failed.append(e)
@@ -639,8 +626,8 @@ def read_optional_seed(fill, base="", ext="", timeout=5):
         fill['user-data'] = ud
         fill['meta-data'] = md
         return True
-    except IOError as e:
-        if e.errno == errno.ENOENT:
+    except url_helper.UrlError as e:
+        if e.code == url_helper.NOT_FOUND:
             return False
         raise
 
@@ -679,7 +666,7 @@ def fetch_ssl_details(paths=None):
 
 def read_file_or_url(url, timeout=5, retries=10,
                      headers=None, data=None, sec_between=1, ssl_details=None,
-                     headers_cb=None):
+                     headers_cb=None, exception_cb=None):
     url = url.lstrip()
     if url.startswith("/"):
         url = "file://%s" % url
@@ -687,7 +674,14 @@ def read_file_or_url(url, timeout=5, retries=10,
         if data:
             LOG.warn("Unable to post data to file resource %s", url)
         file_path = url[len("file://"):]
-        return FileResponse(file_path, contents=load_file(file_path))
+        try:
+            contents = load_file(file_path)
+        except IOError as e:
+            code = e.errno
+            if e.errno == errno.ENOENT:
+                code = url_helper.NOT_FOUND
+            raise url_helper.UrlError(cause=e, code=code, headers=None)
+        return url_helper.FileResponse(file_path, contents=contents)
     else:
         return url_helper.readurl(url,
                                   timeout=timeout,
@@ -696,7 +690,8 @@ def read_file_or_url(url, timeout=5, retries=10,
                                   headers_cb=headers_cb,
                                   data=data,
                                   sec_between=sec_between,
-                                  ssl_details=ssl_details)
+                                  ssl_details=ssl_details,
+                                  exception_cb=exception_cb)
 
 
 def load_yaml(blob, default=None, allowed=(dict,)):
@@ -865,8 +860,8 @@ def get_fqdn_from_hosts(hostname, filename="/etc/hosts"):
         IP_address canonical_hostname [aliases...]
 
       Fields of the entry are separated by any number of  blanks  and/or  tab
-      characters.  Text  from	a "#" character until the end of the line is a
-      comment, and is ignored.	 Host  names  may  contain  only  alphanumeric
+      characters.  Text  from a "#" character until the end of the line is a
+      comment, and is ignored. Host  names  may  contain  only  alphanumeric
       characters, minus signs ("-"), and periods (".").  They must begin with
       an  alphabetic  character  and  end  with  an  alphanumeric  character.
       Optional aliases provide for name changes, alternate spellings, shorter
@@ -950,7 +945,7 @@ def is_resolvable(name):
                 pass
         _DNS_REDIRECT_IP = badips
         if badresults:
-            LOG.debug("detected dns redirection: %s" % badresults)
+            LOG.debug("detected dns redirection: %s", badresults)
 
     try:
         result = socket.getaddrinfo(name, None)
@@ -977,7 +972,7 @@ def gethostbyaddr(ip):
 
 def is_resolvable_url(url):
     """determine if this url is resolvable (existing or ip)."""
-    return (is_resolvable(urlparse.urlparse(url).hostname))
+    return is_resolvable(urlparse.urlparse(url).hostname)
 
 
 def search_for_mirror(candidates):
@@ -1302,11 +1297,26 @@ def mounts():
     mounted = {}
     try:
         # Go through mounts to see what is already mounted
-        mount_locs = load_file("/proc/mounts").splitlines()
+        if os.path.exists("/proc/mounts"):
+            mount_locs = load_file("/proc/mounts").splitlines()
+            method = 'proc'
+        else:
+            (mountoutput, _err) = subp("mount")
+            mount_locs = mountoutput.splitlines()
+            method = 'mount'
+        mountre = r'^(/dev/[\S]+) on (/.*) \((.+), .+, (.+)\)$'
         for mpline in mount_locs:
-            # Format at: man fstab
+            # Linux: /dev/sda1 on /boot type ext4 (rw,relatime,data=ordered)
+            # FreeBSD: /dev/vtbd0p2 on / (ufs, local, journaled soft-updates)
             try:
-                (dev, mp, fstype, opts, _freq, _passno) = mpline.split()
+                if method == 'proc':
+                    (dev, mp, fstype, opts, _freq, _passno) = mpline.split()
+                else:
+                    m = re.search(mountre, mpline)
+                    dev = m.group(1)
+                    mp = m.group(2)
+                    fstype = m.group(3)
+                    opts = m.group(4)
             except:
                 continue
             # If the name of the mount point contains spaces these
@@ -1317,9 +1327,9 @@ def mounts():
                 'mountpoint': mp,
                 'opts': opts,
             }
-        LOG.debug("Fetched %s mounts from %s", mounted, "/proc/mounts")
+        LOG.debug("Fetched %s mounts from %s", mounted, method)
     except (IOError, OSError):
-        logexc(LOG, "Failed fetching mount points from /proc/mounts")
+        logexc(LOG, "Failed fetching mount points")
     return mounted
 
 
@@ -1376,7 +1386,7 @@ def get_builtin_cfg():
 
 
 def sym_link(source, link):
-    LOG.debug("Creating symbolic link from %r => %r" % (link, source))
+    LOG.debug("Creating symbolic link from %r => %r", link, source)
     os.symlink(source, link)
 
 
@@ -1404,12 +1414,27 @@ def time_rfc2822():
 
 def uptime():
     uptime_str = '??'
+    method = 'unknown'
     try:
-        contents = load_file("/proc/uptime").strip()
-        if contents:
-            uptime_str = contents.split()[0]
+        if os.path.exists("/proc/uptime"):
+            method = '/proc/uptime'
+            contents = load_file("/proc/uptime").strip()
+            if contents:
+                uptime_str = contents.split()[0]
+        else:
+            method = 'ctypes'
+            libc = ctypes.CDLL('/lib/libc.so.7')
+            size = ctypes.c_size_t()
+            buf = ctypes.c_int()
+            size.value = ctypes.sizeof(buf)
+            libc.sysctlbyname("kern.boottime", ctypes.byref(buf),
+                              ctypes.byref(size), None, 0)
+            now = time.time()
+            bootup = buf.value
+            uptime_str = now - bootup
+
     except:
-        logexc(LOG, "Unable to read uptime from /proc/uptime")
+        logexc(LOG, "Unable to read uptime using method: %s" % method)
     return uptime_str
 
 
@@ -1748,6 +1773,19 @@ def parse_mtab(path):
     return None
 
 
+def parse_mount(path):
+    (mountoutput, _err) = subp("mount")
+    mount_locs = mountoutput.splitlines()
+    for line in mount_locs:
+        m = re.search(r'^(/dev/[\S]+) on (/.*) \((.+), .+, (.+)\)$', line)
+        devpth = m.group(1)
+        mount_point = m.group(2)
+        fs_type = m.group(3)
+        if mount_point == path:
+            return devpth, fs_type, mount_point
+    return None
+
+
 def get_mount_info(path, log=LOG):
     # Use /proc/$$/mountinfo to find the device where path is mounted.
     # This is done because with a btrfs filesystem using os.stat(path)
@@ -1781,8 +1819,10 @@ def get_mount_info(path, log=LOG):
     if os.path.exists(mountinfo_path):
         lines = load_file(mountinfo_path).splitlines()
         return parse_mount_info(path, lines, log)
-    else:
+    elif os.path.exists("/etc/mtab"):
         return parse_mtab(path)
+    else:
+        return parse_mount(path)
 
 
 def which(program):
@@ -1795,7 +1835,7 @@ def which(program):
         if is_exe(program):
             return program
     else:
-        for path in os.environ["PATH"].split(os.pathsep):
+        for path in os.environ.get("PATH", "").split(os.pathsep):
             path = path.strip('"')
             exe_file = os.path.join(path, program)
             if is_exe(exe_file):
@@ -1849,3 +1889,28 @@ def expand_dotted_devname(dotted):
         return toks
     else:
         return (dotted, None)
+
+
+def pathprefix2dict(base, required=None, optional=None, delim=os.path.sep):
+    # return a dictionary populated with keys in 'required' and 'optional'
+    # by reading files in prefix + delim + entry
+    if required is None:
+        required = []
+    if optional is None:
+        optional = []
+
+    missing = []
+    ret = {}
+    for f in required + optional:
+        try:
+            ret[f] = load_file(base + delim + f, quiet=False)
+        except IOError as e:
+            if e.errno != errno.ENOENT:
+                raise
+            if f in required:
+                missing.append(f)
+
+    if len(missing):
+        raise ValueError("Missing required files: %s", ','.join(missing))
+
+    return ret